@oar-sh/oar 0.9.12 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/extension.mjs +9 -0
- package/.github/extensions/web-relay/server-lifecycle/managed-server.mjs +15 -1
- package/.github/extensions/web-relay/skills/question-bridge.mjs +4 -1
- package/CHANGELOG.md +508 -0
- package/README.md +12 -12
- package/bin/oar.js +155 -94
- package/package.json +5 -5
- package/server/README.md +479 -157
- package/server/claude-cloud-worker/claude-cloud-session-process.mjs +10 -4
- package/server/claude-worker/claude-session-process.mjs +92 -15
- package/server/claude-worker/claude-turn-publisher.mjs +20 -1
- package/server/claude-worker/subagent-timeline-recorder.mjs +75 -8
- package/server/claude-worker/task-output-discovery.mjs +28 -2
- package/server/copilot-worker/copilot-ask-user-tool.mjs +1 -1
- package/server/copilot-worker/copilot-continuation-signals.mjs +11 -5
- package/server/copilot-worker/copilot-prompt-context.mjs +3 -1
- package/server/copilot-worker/copilot-question-bridge.mjs +11 -6
- package/server/copilot-worker/copilot-sdk-adapter.mjs +11 -0
- package/server/copilot-worker/copilot-sdk-session-process.mjs +24 -1
- package/server/cursor-worker/cursor-ask-user-tool.mjs +1 -1
- package/server/cursor-worker/cursor-sdk-adapter.mjs +22 -7
- package/server/cursor-worker/cursor-session-worker.mjs +9 -2
- package/server/cursor-worker/cursor-turn-runner.mjs +18 -3
- package/server/db-schema.mjs +10 -0
- package/server/grok-worker/acp-host-services.mjs +6 -3
- package/server/grok-worker/grok-background-tasks.mjs +7 -0
- package/server/grok-worker/grok-sdk-adapter.mjs +14 -0
- package/server/grok-worker/grok-turn-runner.mjs +13 -3
- package/server/migrations/0018-background-task-rows.mjs +101 -0
- package/server/public/app/action-confirmations.js +12 -15
- package/server/public/app/api-client.js +60 -4
- package/server/public/app/ask-user-view.js +29 -4
- package/server/public/app/background-tasks-view.mjs +56 -7
- package/server/public/app/bootstrap.js +7 -0
- package/server/public/app/conversation-preferences.mjs +10 -4
- package/server/public/app/conversation-provider-indicator.mjs +18 -8
- package/server/public/app/conversation-view.js +33 -9
- package/server/public/app/host-suspend-ui.js +45 -44
- package/server/public/app/journal-view.js +9 -3
- package/server/public/app/push-settings.js +1 -0
- package/server/public/app/question-answer-rule.mjs +15 -0
- package/server/public/app/question-multi-select.mjs +9 -2
- package/server/public/app/relay-event-view.mjs +5 -1
- package/server/public/app/relay-prompt-sanitizer.mjs +9 -2
- package/server/public/app/relay-restart-note-view.mjs +36 -0
- package/server/public/app/relay-restart-ui.js +568 -0
- package/server/public/app/settings-modal.js +52 -0
- package/server/public/app/socket-handlers.js +1 -1
- package/server/public/app/task-console-render.mjs +171 -7
- package/server/public/app/task-console.mjs +111 -11
- package/server/public/app/task-output-text.mjs +15 -6
- package/server/public/app/transcript-jump-bottom-view.mjs +5 -1
- package/server/public/app/transcript-jump-bottom.mjs +20 -0
- package/server/public/index.html +76 -5
- package/server/relay-exit-codes.mjs +6 -0
- package/server/relay-self-restart.mjs +6 -1
- package/server/relay-tools.md +17 -9
- package/server/repositories/message-repository.mjs +32 -28
- package/server/repositories/remote-relay-repository.mjs +4 -2
- package/server/repositories/subagent-timeline-statements.mjs +2 -0
- package/server/routes/ask-user-routes.mjs +41 -7
- package/server/routes/messages-routes.mjs +220 -37
- package/server/routes/preview-routes.mjs +73 -7
- package/server/routes/relay-board-routes.mjs +19 -2
- package/server/routes/relay-restart-routes.mjs +73 -0
- package/server/routes/sessions-routes.mjs +79 -5
- package/server/server-runtime.mjs +335 -218
- package/server/services/about-service.mjs +13 -12
- package/server/services/background-task-persistence.mjs +133 -0
- package/server/services/background-task-store.mjs +240 -13
- package/server/services/conversation-model-fallback.mjs +57 -0
- package/server/services/cursor-session-root-service.mjs +70 -6
- package/server/services/data-dir-resolution.mjs +208 -19
- package/server/services/earlier-database-service.mjs +631 -0
- package/server/services/host-suspend-command.mjs +2 -1
- package/server/services/oar-cli-helpers.mjs +6 -1
- package/server/services/oar-state-layout.mjs +107 -0
- package/server/services/oar-state-migration-service.mjs +387 -96
- package/server/services/package-data-rescue-preinstall.mjs +22 -0
- package/server/services/package-data-rescue.mjs +137 -0
- package/server/services/preview-proxy-server.mjs +14 -1
- package/server/services/preview-registry-service.mjs +3 -2
- package/server/services/process-list-service.mjs +2 -1
- package/server/services/provider-model-selection.mjs +195 -0
- package/server/services/push-dispatch-service.mjs +42 -8
- package/server/services/relay-event-service.mjs +210 -14
- package/server/services/relay-prompt-sanitizer.mjs +7 -0
- package/server/services/relay-restart-picture.mjs +209 -0
- package/server/services/relay-restart-queue.mjs +387 -0
- package/server/services/relay-shutdown-gate.mjs +4 -1
- package/server/services/relay-singleton-guard.mjs +300 -36
- package/server/services/remote-relay-caller-context.mjs +7 -2
- package/server/services/remote-relay-dispatcher.mjs +217 -28
- package/server/services/restart-resume-service.mjs +303 -0
- package/server/services/session-activity-tracker.mjs +52 -15
- package/server/services/session-worker-launch-service.mjs +31 -1
- package/server/services/session-worker-process-service.mjs +6 -4
- package/server/services/stale-turn-recovery.mjs +99 -0
- package/server/services/subagent-timeline-service.mjs +4 -1
- package/server/services/task-output-service.mjs +217 -18
- package/server/services/windows-autostart-service.mjs +329 -31
- package/server/services/windows-boot-autostart-service.mjs +188 -60
- package/shared/preview-instructions.mjs +17 -12
- package/shared/preview-tool-core.mjs +32 -14
- package/shared/question-answer-rule.mjs +63 -0
- package/shared/relay-events.mjs +106 -3
- package/shared/remote-relay-contract.mjs +32 -15
- package/shared/subagent-timeline.mjs +4 -0
- package/shared/task-output-text.mjs +196 -83
- package/shared/windows-system-tools.mjs +25 -0
- package/shared/worker-runtime/config-loader.mjs +54 -8
- package/shared/worker-runtime/prompt-context.mjs +3 -1
- package/shared/worker-runtime/worker-websocket-link.mjs +96 -20
- package/shared/worker-runtime/working-directory.mjs +65 -0
- package/server/uploads/image_input.md +0 -540
|
@@ -42,7 +42,7 @@ import { registerRelayBoardRoutes } from './routes/relay-board-routes.mjs';
|
|
|
42
42
|
import { registerCacheRoutes } from './routes/cache-routes.mjs';
|
|
43
43
|
import { registerGitRoutes } from './routes/git-routes.mjs';
|
|
44
44
|
import { registerClaudeCloudRoutes } from './routes/claude-cloud-routes.mjs';
|
|
45
|
-
import { registerPreviewRoutes } from './routes/preview-routes.mjs';
|
|
45
|
+
import { createPreviewScopeResolver, registerPreviewRoutes } from './routes/preview-routes.mjs';
|
|
46
46
|
import { registerRemoteRelayRoutes } from './routes/remote-relay-routes.mjs';
|
|
47
47
|
import { registerRelayEventRoutes } from './routes/relay-event-routes.mjs';
|
|
48
48
|
import { createRemoteRelayRepository } from './repositories/remote-relay-repository.mjs';
|
|
@@ -75,6 +75,13 @@ import { createHostSuspendService } from './services/host-suspend-service.mjs';
|
|
|
75
75
|
import { USAGE_LIMIT_ACCOUNT_SOCKET_EVENT, createUsageLimitPauseService } from './services/usage-limit-pause-service.mjs';
|
|
76
76
|
import { createSessionActivityTracker, resolveBootReportingWindowMs } from './services/session-activity-tracker.mjs';
|
|
77
77
|
import { createBackgroundTaskStore } from './services/background-task-store.mjs';
|
|
78
|
+
import { createSqliteBackgroundTaskPersistence } from './services/background-task-persistence.mjs';
|
|
79
|
+
import {
|
|
80
|
+
RESTART_RESUME_SETTING_KEY,
|
|
81
|
+
createRestartResumeService,
|
|
82
|
+
parseRestartResumeUpdate,
|
|
83
|
+
readRestartResumeSetting,
|
|
84
|
+
} from './services/restart-resume-service.mjs';
|
|
78
85
|
import { createBackgroundWorkView } from './services/background-work-view.mjs';
|
|
79
86
|
import { createRelayEventService } from './services/relay-event-service.mjs';
|
|
80
87
|
import { createRelayEventLinkService } from './services/relay-event-link-service.mjs';
|
|
@@ -90,6 +97,7 @@ import { runHostSuspendToRam } from './services/host-suspend-command.mjs';
|
|
|
90
97
|
import { createActiveDeviceTracker } from './services/push-active-device-service.mjs';
|
|
91
98
|
import { registerPushRoutes } from './routes/push-routes.mjs';
|
|
92
99
|
import { registerAboutRoutes } from './routes/about-routes.mjs';
|
|
100
|
+
import { registerRelayRestartRoutes } from './routes/relay-restart-routes.mjs';
|
|
93
101
|
import { createAboutService, resolveBuildCommit } from './services/about-service.mjs';
|
|
94
102
|
import { createImageOperationService } from './services/image-operation-service.mjs';
|
|
95
103
|
import {
|
|
@@ -116,7 +124,9 @@ import { createClaudeSessionRootResolver } from './services/claude-session-root-
|
|
|
116
124
|
import { createCursorSessionRootResolver } from './services/cursor-session-root-service.mjs';
|
|
117
125
|
import { createGitChangesService } from './services/git-changes-service.mjs';
|
|
118
126
|
import { createRelaySingletonGuard } from './services/relay-singleton-guard.mjs';
|
|
119
|
-
import {
|
|
127
|
+
import { adoptEarlierDatabase, lostPackageDatabaseWarning, packageDatabaseEvidence } from './services/earlier-database-service.mjs';
|
|
128
|
+
import { ensureDataDirs, resolveDataDir, resolveUploadDir, resolveUploadReadDirs, uploadPathInDirs } from './services/data-dir-resolution.mjs';
|
|
129
|
+
import { mergeStrayMigratedUploads } from './services/oar-state-migration-service.mjs';
|
|
120
130
|
import { createRelayRestartOrchestrator } from './services/relay-restart-orchestrator-service.mjs';
|
|
121
131
|
import { createRelayBridgeOwnerService } from './services/relay-bridge-owner-service.mjs';
|
|
122
132
|
import { createRelayCliLauncherService } from './services/relay-cli-launcher-service.mjs';
|
|
@@ -165,7 +175,7 @@ import {
|
|
|
165
175
|
migrateLegacyConfigFeatures,
|
|
166
176
|
resolveBootFeatureFlags,
|
|
167
177
|
} from './features.mjs';
|
|
168
|
-
import { RELAY_RESTART_EXIT_CODE } from './relay-exit-codes.mjs';
|
|
178
|
+
import { RELAY_RESTART_EXIT_CODE, RELAY_STOP_EXIT_CODE } from './relay-exit-codes.mjs';
|
|
169
179
|
import { DEFAULT_QUESTION_TIMEOUT_MS, questionExpiresAt } from '../shared/question-timeout.mjs';
|
|
170
180
|
import {
|
|
171
181
|
parseTurnCeilingUpdate,
|
|
@@ -183,8 +193,17 @@ import {
|
|
|
183
193
|
parseSharePathPrivacyUpdate,
|
|
184
194
|
readSharePathPrivacySetting,
|
|
185
195
|
} from '../shared/share-path-privacy.mjs';
|
|
186
|
-
import { isRelayShutdownQueued,
|
|
196
|
+
import { isRelayShutdownQueued, relayShutdownGateReply } from './services/relay-shutdown-gate.mjs';
|
|
197
|
+
import {
|
|
198
|
+
createRelayRestartQueue,
|
|
199
|
+
parseRelayRestartMaxWaitUpdate,
|
|
200
|
+
readRelayRestartMaxWaitSetting,
|
|
201
|
+
resolveRelayRestartMaxWaitOverrideMs,
|
|
202
|
+
RELAY_RESTART_MAX_WAIT_SETTING_KEY,
|
|
203
|
+
} from './services/relay-restart-queue.mjs';
|
|
204
|
+
import { createRestartPictureCollector } from './services/relay-restart-picture.mjs';
|
|
187
205
|
import { createStaleRecoveryClockGuard } from './services/stale-recovery-clock-guard.mjs';
|
|
206
|
+
import { createStaleTurnRecovery } from './services/stale-turn-recovery.mjs';
|
|
188
207
|
import {
|
|
189
208
|
STALE_SUBAGENT_RUN_BOOT_DELAY_MS,
|
|
190
209
|
STALE_SUBAGENT_RUN_SWEEP_INTERVAL_MS,
|
|
@@ -232,7 +251,7 @@ import {
|
|
|
232
251
|
import { createModelVariantCatalogService } from './services/model-variant-catalog-service.mjs';
|
|
233
252
|
import { applyPwaManifestTheme, normalizePwaTheme } from './services/pwa-manifest-theme.mjs';
|
|
234
253
|
import { mapSubagentRunHeader } from './services/subagent-timeline-service.mjs';
|
|
235
|
-
import { describeExecError, describeSpawnEnvironment, windowsSystemToolPath } from '../shared/windows-system-tools.mjs';
|
|
254
|
+
import { describeExecError, describeSpawnEnvironment, normalizeTempEnv, windowsSystemToolPath } from '../shared/windows-system-tools.mjs';
|
|
236
255
|
|
|
237
256
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
238
257
|
const REPO_ROOT = path.resolve(__dirname, '..');
|
|
@@ -263,9 +282,8 @@ const APP_CONFIG_PLACEHOLDER = /window\.__COPILOT_APP_CONFIG = \{ basePath: '[^'
|
|
|
263
282
|
const PWA_VERSION_PLACEHOLDER = /const __PWA_VERSION = '[^']*';/;
|
|
264
283
|
// The console goes to a terminal, a tmux pane or nowhere: keep a copy to read
|
|
265
284
|
// after a restart or an incident.
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
});
|
|
285
|
+
const relayConsoleLogPath = resolveRelayConsoleLogPath({ env: process.env, serverDir: __dirname });
|
|
286
|
+
const relayConsoleLog = installRelayConsoleLog({ logPath: relayConsoleLogPath });
|
|
269
287
|
const ttyConsoleRuntime = await maybeStartTtyConsole({
|
|
270
288
|
serverDir: __dirname,
|
|
271
289
|
logsDir: path.join(__dirname, 'logs'),
|
|
@@ -273,6 +291,8 @@ const ttyConsoleRuntime = await maybeStartTtyConsole({
|
|
|
273
291
|
});
|
|
274
292
|
// The terminal console takes the console's methods over when it starts.
|
|
275
293
|
relayConsoleLog?.rewrap();
|
|
294
|
+
// TEMP and TMP for this relay and everything it starts (normalizeTempEnv).
|
|
295
|
+
for (const change of normalizeTempEnv()) console.log(`[startup] ${change} (quotes taken off, or the folder is not there)`);
|
|
276
296
|
|
|
277
297
|
// ─── Config ────────────────────────────────────────────────────────────────────
|
|
278
298
|
const CONFIG_PATH = process.env.COPILOT_WEB_RELAY_CONFIG
|
|
@@ -280,26 +300,32 @@ const CONFIG_PATH = process.env.COPILOT_WEB_RELAY_CONFIG
|
|
|
280
300
|
: path.join(__dirname, 'config.json');
|
|
281
301
|
// env var > config.json `dataDir` > <server>/data — see data-dir-resolution.mjs
|
|
282
302
|
// for why the config can carry it (a dev checkout pointing at a release
|
|
283
|
-
// install's data
|
|
303
|
+
// install's data).
|
|
284
304
|
const DATA_DIR = resolveDataDir({ env: process.env, configPath: CONFIG_PATH, serverDir: __dirname });
|
|
285
305
|
const DB_PATH = path.join(DATA_DIR, 'copilot.db');
|
|
286
306
|
const RELAY_LOCK_PATH = path.join(DATA_DIR, 'relay-server.lock');
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
307
|
+
// Uploads follow the data dir whenever the env var or config.json chose it.
|
|
308
|
+
const UPLOAD_DIR = resolveUploadDir({ dataDir: DATA_DIR, env: process.env, serverDir: __dirname });
|
|
309
|
+
// Uploads earlier versions left in <server>/uploads or <state root>/uploads
|
|
310
|
+
// are read in place; nothing there is written, moved or deleted.
|
|
311
|
+
const UPLOAD_READ_DIRS = resolveUploadReadDirs({ uploadDir: UPLOAD_DIR, dataDir: DATA_DIR, serverDir: __dirname });
|
|
290
312
|
const INITIAL_WORKSPACE_ROOT = resolveStartupWorkspaceRoot(__dirname);
|
|
291
313
|
const WORKSPACE_ROOT_LOCKED = true;
|
|
292
314
|
const SESSION_COOKIE = 'copilot_session';
|
|
293
315
|
const AUTH_COOKIE = 'copilot_auth';
|
|
294
316
|
|
|
295
|
-
|
|
296
|
-
fs.mkdirSync(DATA_DIR, { recursive: true, mode: 0o700 });
|
|
297
|
-
}
|
|
317
|
+
ensureDataDirs({ dataDir: DATA_DIR, uploadDir: UPLOAD_DIR });
|
|
298
318
|
if (process.platform !== 'win32') {
|
|
299
319
|
fs.chmodSync(DATA_DIR, 0o700);
|
|
300
320
|
}
|
|
301
|
-
|
|
302
|
-
|
|
321
|
+
// Earlier migrations put a checkout's uploads in <state root>/uploads, which
|
|
322
|
+
// nothing read: merge them in (never overwriting, never deleting the source).
|
|
323
|
+
mergeStrayMigratedUploads({ dataDir: DATA_DIR, uploadDir: UPLOAD_DIR, logger: console });
|
|
324
|
+
for (const dir of UPLOAD_READ_DIRS.slice(1)) {
|
|
325
|
+
try {
|
|
326
|
+
const count = fs.readdirSync(dir).filter((name) => isSha256(name)).length;
|
|
327
|
+
if (count) console.log(`[uploads] New uploads go to ${UPLOAD_DIR}; ${count} earlier upload${count === 1 ? '' : 's'} in ${dir} are read from there.`);
|
|
328
|
+
} catch {}
|
|
303
329
|
}
|
|
304
330
|
|
|
305
331
|
async function refreshOpenAIProviderModels() {
|
|
@@ -1081,18 +1107,20 @@ function setCursorProviderSettings(update = {}) {
|
|
|
1081
1107
|
const CURSOR_MODEL_DISCOVERY_TIMEOUT_MS = 20_000;
|
|
1082
1108
|
|
|
1083
1109
|
// Reasoning tiers for one discovered Cursor model, in composer vocabulary.
|
|
1084
|
-
// Cursor exposes effort as a per-model parameter named 'effort'
|
|
1085
|
-
// whose value list varies by model; 'extra-high' is
|
|
1086
|
-
// composer's 'xhigh'. Values with no composer
|
|
1087
|
-
// dropped rather than remapped onto a tier the
|
|
1110
|
+
// Cursor exposes effort as a per-model parameter named 'effort', 'reasoning'
|
|
1111
|
+
// or 'reasoning_effort' whose value list varies by model; 'extra-high' is
|
|
1112
|
+
// Cursor's spelling of the composer's 'xhigh'. Values with no composer
|
|
1113
|
+
// equivalent (e.g. 'minimal') are dropped rather than remapped onto a tier the
|
|
1114
|
+
// model does not actually have.
|
|
1088
1115
|
// Mirrors resolveCursorReasoningParams exactly, including its parameter
|
|
1089
|
-
// precedence: it consults 'effort' first
|
|
1090
|
-
// so scanning
|
|
1116
|
+
// precedence: it consults 'effort' first, then 'reasoning', then
|
|
1117
|
+
// 'reasoning_effort', so scanning several would advertise an off switch the
|
|
1118
|
+
// worker never sends.
|
|
1091
1119
|
function cursorEntrySupportsReasoningOff(entry) {
|
|
1092
1120
|
const parameters = Array.isArray(entry?.parameters) ? entry.parameters : [];
|
|
1093
1121
|
const byId = new Map(parameters.map((param) => [String(param?.id || '').trim().toLowerCase(), param]));
|
|
1094
1122
|
if (byId.has('thinking')) return true;
|
|
1095
|
-
const effortParam = byId.get('effort') || byId.get('reasoning');
|
|
1123
|
+
const effortParam = byId.get('effort') || byId.get('reasoning') || byId.get('reasoning_effort');
|
|
1096
1124
|
const values = Array.isArray(effortParam?.values) ? effortParam.values : [];
|
|
1097
1125
|
return values.some((value) => (
|
|
1098
1126
|
String((value && typeof value === 'object' ? value.value : value) || '').trim().toLowerCase() === 'none'
|
|
@@ -1105,7 +1133,7 @@ function cursorEntryEffortLevels(entry) {
|
|
|
1105
1133
|
// listing both parameters must not have its tiers read from one and its off
|
|
1106
1134
|
// switch from the other.
|
|
1107
1135
|
const byId = new Map(parameters.map((param) => [String(param?.id || '').trim().toLowerCase(), param]));
|
|
1108
|
-
const effortParam = byId.get('effort') || byId.get('reasoning');
|
|
1136
|
+
const effortParam = byId.get('effort') || byId.get('reasoning') || byId.get('reasoning_effort');
|
|
1109
1137
|
if (!effortParam) return [];
|
|
1110
1138
|
const levels = [];
|
|
1111
1139
|
for (const value of (Array.isArray(effortParam.values) ? effortParam.values : [])) {
|
|
@@ -1808,18 +1836,43 @@ function isProcessAlive(pid) {
|
|
|
1808
1836
|
}
|
|
1809
1837
|
}
|
|
1810
1838
|
|
|
1839
|
+
// The guard's own stale-lock rules (lockHolderMayRun; an EPERM is alive).
|
|
1811
1840
|
const relaySingletonGuard = createRelaySingletonGuard({
|
|
1812
1841
|
lockPath: RELAY_LOCK_PATH,
|
|
1813
1842
|
pid: process.pid,
|
|
1814
1843
|
token: config.authToken,
|
|
1815
|
-
isProcessAlive,
|
|
1816
1844
|
logger: console,
|
|
1817
1845
|
});
|
|
1818
1846
|
try {
|
|
1819
1847
|
relaySingletonGuard.acquire();
|
|
1820
1848
|
} catch (error) {
|
|
1849
|
+
// A held lock is not a crash: the supervisors do not retry 78.
|
|
1821
1850
|
console.error(`[server] ${error?.message || String(error)}`);
|
|
1822
|
-
process.exit(1);
|
|
1851
|
+
process.exit(error?.code === 'ERELAYLOCKED' ? RELAY_STOP_EXIT_CODE : 1);
|
|
1852
|
+
}
|
|
1853
|
+
// Before the database opens, under the lock (adoptEarlierDatabase); the
|
|
1854
|
+
// result also feeds lostPackageDatabaseWarning.
|
|
1855
|
+
let earlierDatabase;
|
|
1856
|
+
{
|
|
1857
|
+
try {
|
|
1858
|
+
earlierDatabase = adoptEarlierDatabase({
|
|
1859
|
+
dataDir: DATA_DIR,
|
|
1860
|
+
uploadDir: UPLOAD_DIR,
|
|
1861
|
+
serverDir: __dirname,
|
|
1862
|
+
configPath: CONFIG_PATH,
|
|
1863
|
+
packageRoot: REPO_ROOT,
|
|
1864
|
+
Database,
|
|
1865
|
+
authToken: config.authToken,
|
|
1866
|
+
logger: console,
|
|
1867
|
+
});
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
earlierDatabase = { status: 'refused', error: `[data] Not starting: the database this relay ran on until now could not be looked at (${error?.message || error}).` };
|
|
1870
|
+
}
|
|
1871
|
+
if (earlierDatabase.status === 'refused') {
|
|
1872
|
+
console.error(earlierDatabase.error);
|
|
1873
|
+
try { relaySingletonGuard.release(); } catch {}
|
|
1874
|
+
process.exit(RELAY_STOP_EXIT_CODE);
|
|
1875
|
+
}
|
|
1823
1876
|
}
|
|
1824
1877
|
|
|
1825
1878
|
const MAX_REQUEUE_RETRIES = Number.isFinite(Number(config.maxRequeueRetries))
|
|
@@ -1877,7 +1930,7 @@ const runtimeTimers = {
|
|
|
1877
1930
|
questionExpiry: null,
|
|
1878
1931
|
pendingWorkerPrime: null,
|
|
1879
1932
|
sharedViewerPrune: null,
|
|
1880
|
-
|
|
1933
|
+
restoredTaskSettle: null,
|
|
1881
1934
|
};
|
|
1882
1935
|
|
|
1883
1936
|
function normalizeRemotePath(value) {
|
|
@@ -2112,7 +2165,7 @@ function setSharePathPrivacyMode(value) {
|
|
|
2112
2165
|
// checkout, data and upload dirs, the workspace roots it knows, and the OS
|
|
2113
2166
|
// user name. The share routes add the conversation's own roots.
|
|
2114
2167
|
function sharePathKnownRoots() {
|
|
2115
|
-
const roots = [REPO_ROOT, DATA_DIR,
|
|
2168
|
+
const roots = [REPO_ROOT, DATA_DIR, ...UPLOAD_READ_DIRS, currentWorkspaceRootPath(), INITIAL_WORKSPACE_ROOT];
|
|
2116
2169
|
const userNames = [];
|
|
2117
2170
|
try { roots.push(os.homedir()); } catch {}
|
|
2118
2171
|
try { userNames.push(os.userInfo().username); } catch {}
|
|
@@ -2202,6 +2255,21 @@ function setTaskStallMinutes(value) {
|
|
|
2202
2255
|
return { ok: true, minutes: parsed.minutes };
|
|
2203
2256
|
}
|
|
2204
2257
|
|
|
2258
|
+
// "After a relay restart, tell interrupted sessions to carry on" (off by default).
|
|
2259
|
+
function getRestartResumeEnabled() {
|
|
2260
|
+
return readRestartResumeSetting(readAppSettingValue(RESTART_RESUME_SETTING_KEY));
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
function setRestartResumeEnabled(value) {
|
|
2264
|
+
if (typeof stmts?.upsertAppSetting?.run !== 'function') {
|
|
2265
|
+
return { ok: false, error: 'Restart settings are unavailable' };
|
|
2266
|
+
}
|
|
2267
|
+
const parsed = parseRestartResumeUpdate(value);
|
|
2268
|
+
if (!parsed.ok) return parsed;
|
|
2269
|
+
stmts.upsertAppSetting.run(RESTART_RESUME_SETTING_KEY, parsed.enabled ? '1' : '0', new Date().toISOString());
|
|
2270
|
+
return { ok: true, enabled: parsed.enabled };
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2205
2273
|
function resolveDefaultSessionWorkspaceRootState() {
|
|
2206
2274
|
return resolveDefaultSessionWorkspaceRootStateFromService({
|
|
2207
2275
|
storedPath: readAppSettingValue(DEFAULT_SESSION_WORKSPACE_ROOT_KEY),
|
|
@@ -2648,6 +2716,7 @@ const configuredConversationSessionMode =
|
|
|
2648
2716
|
updateModelCatalog({ models: [DEFAULT_MODEL], currentModel: DEFAULT_MODEL, defaultModel: DEFAULT_MODEL, source: 'bootstrap' });
|
|
2649
2717
|
|
|
2650
2718
|
// ─── SQLite Setup ─────────────────────────────────────────────────────────────
|
|
2719
|
+
const databaseCreatedAtStart = !fs.existsSync(DB_PATH);
|
|
2651
2720
|
const db = new Database(DB_PATH);
|
|
2652
2721
|
if (process.platform !== 'win32') {
|
|
2653
2722
|
fs.chmodSync(DB_PATH, 0o600);
|
|
@@ -3011,6 +3080,7 @@ function buildSessionWorkerLaunchEnvForSession(targetSessionId) {
|
|
|
3011
3080
|
enabled: true,
|
|
3012
3081
|
model: cursorModel,
|
|
3013
3082
|
apiKey: cursorSettings.apiKey,
|
|
3083
|
+
dataDir: DATA_DIR,
|
|
3014
3084
|
});
|
|
3015
3085
|
}
|
|
3016
3086
|
if (providerType === 'grok') {
|
|
@@ -3451,140 +3521,98 @@ function attachTtyConsoleRuntimeTitle(runtime) {
|
|
|
3451
3521
|
}
|
|
3452
3522
|
attachTtyConsoleRuntimeTitle(ttyConsoleRuntime);
|
|
3453
3523
|
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3472
|
-
|
|
3524
|
+
// ─── Queued relay restart / shutdown ─────────────────────────────────────────
|
|
3525
|
+
// A request waits until no turn is processing and holds new turns meanwhile
|
|
3526
|
+
// (services/relay-shutdown-gate.mjs); past the maximum wait it asks instead
|
|
3527
|
+
// of acting (services/relay-restart-queue.mjs). Every client shows a banner
|
|
3528
|
+
// with what it waits for, from `relay_shutdown_state` (sent on each change
|
|
3529
|
+
// and on connect).
|
|
3530
|
+
const relayRestartMaxWaitOverrideMs = resolveRelayRestartMaxWaitOverrideMs(process.env);
|
|
3531
|
+
if (relayRestartMaxWaitOverrideMs !== null) {
|
|
3532
|
+
console.log(`${runtimeLogPrefix()}Relay restart max wait pinned to ${relayRestartMaxWaitOverrideMs} ms (OAR_RELAY_RESTART_MAX_WAIT_MS)`);
|
|
3533
|
+
}
|
|
3534
|
+
function getRelayRestartMaxWaitMinutes() {
|
|
3535
|
+
return readRelayRestartMaxWaitSetting(readAppSettingValue(RELAY_RESTART_MAX_WAIT_SETTING_KEY));
|
|
3536
|
+
}
|
|
3537
|
+
function setRelayRestartMaxWaitMinutes(value) {
|
|
3538
|
+
if (!stmts.upsertAppSetting) return { ok: false, error: 'Relay restart settings are unavailable' };
|
|
3539
|
+
const parsed = parseRelayRestartMaxWaitUpdate(value);
|
|
3540
|
+
if (!parsed.ok) return parsed;
|
|
3541
|
+
stmts.upsertAppSetting.run(RELAY_RESTART_MAX_WAIT_SETTING_KEY, String(parsed.minutes), new Date().toISOString());
|
|
3542
|
+
// A queued request measures its current period against the new value.
|
|
3543
|
+
relayRestartQueue.refresh();
|
|
3544
|
+
return { ok: true, maxWaitMinutes: parsed.minutes };
|
|
3545
|
+
}
|
|
3546
|
+
let restartPictureCollector = null;
|
|
3547
|
+
// What a restart would wait for, hold and leave running (the confirmation
|
|
3548
|
+
// asks before anything is queued). Built on first use: the background-work
|
|
3549
|
+
// view is set up further down.
|
|
3550
|
+
function collectRestartPicture({ action = 'restart' } = {}) {
|
|
3551
|
+
if (!restartPictureCollector) {
|
|
3552
|
+
restartPictureCollector = createRestartPictureCollector({
|
|
3553
|
+
db,
|
|
3554
|
+
backgroundWorkView,
|
|
3555
|
+
listHeldTasks: () => backgroundTaskStore.listRestored().map(({ key, task }) => ({
|
|
3556
|
+
conversationId: resolveConversationRecord({ conversationId: key, sdkSessionId: key })?.id || key,
|
|
3557
|
+
description: task?.description || task?.taskType || '',
|
|
3558
|
+
})),
|
|
3559
|
+
providerOf: (conversationId) => stmts.getRuntimeSessionByConversation?.get?.(conversationId)?.provider_type,
|
|
3560
|
+
titleOf: (conversationId) => hostActivityTitle(conversationId),
|
|
3561
|
+
});
|
|
3473
3562
|
}
|
|
3474
|
-
return {
|
|
3475
|
-
status: runtimeShutdownStarted ? 'shutting_down' : 'queued',
|
|
3476
|
-
action: pending.action || 'shutdown',
|
|
3477
|
-
restart: pending.action === 'restart',
|
|
3478
|
-
reason: pending.reason || null,
|
|
3479
|
-
requestedBy: pending.requestedBy || null,
|
|
3480
|
-
requestedAt: pending.requestedAt || null,
|
|
3481
|
-
};
|
|
3563
|
+
return restartPictureCollector({ action });
|
|
3482
3564
|
}
|
|
3565
|
+
const relayRestartQueue = createRelayRestartQueue({
|
|
3566
|
+
getCounts: () => queueCounts(),
|
|
3567
|
+
collectPicture: collectRestartPicture,
|
|
3568
|
+
startShutdown: ({ action, reason, requestedBy, requestedAt }) => {
|
|
3569
|
+
const exitCode = action === 'restart' ? RELAY_RESTART_EXIT_CODE : 0;
|
|
3570
|
+
// The next start tells the conversations that asked for relay_restarted
|
|
3571
|
+
// how this one ended (services/relay-event-service.mjs).
|
|
3572
|
+
try { relayEventService?.noteRelayExit({ action, reason, requestedBy, requestedAt }); } catch {}
|
|
3573
|
+
sessionWorkerWebSocketService.emitDraining(`api-${action}:${reason}`);
|
|
3574
|
+
void shutdownRuntime(`api-${action}:${reason}`, { exitCode });
|
|
3575
|
+
},
|
|
3576
|
+
getMaxWaitMs: () => (relayRestartMaxWaitOverrideMs !== null
|
|
3577
|
+
? relayRestartMaxWaitOverrideMs
|
|
3578
|
+
: getRelayRestartMaxWaitMinutes() * 60_000),
|
|
3579
|
+
isShuttingDown: () => runtimeShutdownStarted,
|
|
3580
|
+
onStateChange: (state) => {
|
|
3581
|
+
try { io.emit('relay_shutdown_state', state); } catch {}
|
|
3582
|
+
},
|
|
3583
|
+
onMaxWaitReached: (state) => {
|
|
3584
|
+
const sessions = Array.isArray(state.waitingFor) ? state.waitingFor.length : 0;
|
|
3585
|
+
const minutes = Math.max(1, Math.round(Number(state.maxWaitMs || 0) / 60_000));
|
|
3586
|
+
void pushDispatchService.notifyRelayRestart({
|
|
3587
|
+
body: `The queued relay ${state.action} has waited ${minutes} min for ${sessions} session${sessions === 1 ? '' : 's'}. Restart now, keep waiting or cancel it.`,
|
|
3588
|
+
});
|
|
3589
|
+
},
|
|
3590
|
+
logger: console,
|
|
3591
|
+
logPrefix: () => runtimeLogPrefix(),
|
|
3592
|
+
});
|
|
3483
3593
|
|
|
3484
|
-
// Every client shows a banner while a shutdown/restart is queued, so the
|
|
3485
|
-
// state goes out on each transition (request, cancel, drain) and on connect.
|
|
3486
3594
|
function relayShutdownStatePayload() {
|
|
3487
|
-
return
|
|
3488
|
-
}
|
|
3489
|
-
function emitRelayShutdownState() {
|
|
3490
|
-
try { io.emit('relay_shutdown_state', relayShutdownStatePayload()); } catch {}
|
|
3595
|
+
return relayRestartQueue.getState();
|
|
3491
3596
|
}
|
|
3492
3597
|
|
|
3493
3598
|
function cancelRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
if (!pendingRelayShutdownRequest) {
|
|
3498
|
-
return { cancelled: false, ...relayShutdownStatePayload() };
|
|
3499
|
-
}
|
|
3500
|
-
const dropped = pendingRelayShutdownRequest;
|
|
3501
|
-
pendingRelayShutdownRequest = null;
|
|
3502
|
-
if (runtimeTimers.shutdownDrain) {
|
|
3503
|
-
try { clearInterval(runtimeTimers.shutdownDrain); } catch {}
|
|
3504
|
-
runtimeTimers.shutdownDrain = null;
|
|
3505
|
-
}
|
|
3506
|
-
console.log(
|
|
3507
|
-
`${runtimeLogPrefix()}Relay ${dropped.action || 'shutdown'} request cancelled by `
|
|
3508
|
-
+ `${String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api'}`,
|
|
3509
|
-
);
|
|
3510
|
-
emitRelayShutdownState();
|
|
3511
|
-
return { cancelled: true, dropped, ...relayShutdownStatePayload() };
|
|
3599
|
+
const result = relayRestartQueue.cancel({ requestedBy });
|
|
3600
|
+
if (result?.cancelled) relayEventService?.noteRestartCancelled();
|
|
3601
|
+
return result;
|
|
3512
3602
|
}
|
|
3513
3603
|
|
|
3514
|
-
function
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
const restartRequested = normalizeRestartRequestFlag(restart);
|
|
3518
|
-
const requestedAction = restartRequested ? 'restart' : 'shutdown';
|
|
3519
|
-
if (runtimeShutdownStarted) {
|
|
3520
|
-
return {
|
|
3521
|
-
accepted: false,
|
|
3522
|
-
status: 'shutting_down',
|
|
3523
|
-
action: requestedAction,
|
|
3524
|
-
restart: restartRequested,
|
|
3525
|
-
reason: safeReason,
|
|
3526
|
-
};
|
|
3527
|
-
}
|
|
3528
|
-
|
|
3529
|
-
// Shutdown is deferred until no turn is processing; it never interrupts an
|
|
3530
|
-
// active turn. From here on no pending row is handed out (the dequeue paths
|
|
3531
|
-
// read relayShutdownQueued): a turn started meanwhile would postpone the
|
|
3532
|
-
// restart again. Pending and parked rows wait in the database and run
|
|
3533
|
-
// after the restart.
|
|
3534
|
-
if (!pendingRelayShutdownRequest) {
|
|
3535
|
-
pendingRelayShutdownRequest = {
|
|
3536
|
-
action: requestedAction,
|
|
3537
|
-
reason: safeReason,
|
|
3538
|
-
requestedBy: safeRequestedBy,
|
|
3539
|
-
requestedAt: new Date().toISOString(),
|
|
3540
|
-
};
|
|
3541
|
-
} else if (restartRequested && pendingRelayShutdownRequest.action !== 'restart') {
|
|
3542
|
-
pendingRelayShutdownRequest = {
|
|
3543
|
-
...pendingRelayShutdownRequest,
|
|
3544
|
-
action: 'restart',
|
|
3545
|
-
reason: safeReason,
|
|
3546
|
-
requestedBy: safeRequestedBy,
|
|
3547
|
-
requestedAt: new Date().toISOString(),
|
|
3548
|
-
};
|
|
3549
|
-
}
|
|
3550
|
-
|
|
3551
|
-
const tryShutdownWhenIdle = () => {
|
|
3552
|
-
if (runtimeShutdownStarted) return;
|
|
3553
|
-
const counts = queueCounts();
|
|
3554
|
-
if (!queuedShutdownCanProceed(counts)) return;
|
|
3555
|
-
const requestInfo = pendingRelayShutdownRequest;
|
|
3556
|
-
pendingRelayShutdownRequest = null;
|
|
3557
|
-
if (runtimeTimers.shutdownDrain) {
|
|
3558
|
-
try { clearInterval(runtimeTimers.shutdownDrain); } catch {}
|
|
3559
|
-
runtimeTimers.shutdownDrain = null;
|
|
3560
|
-
}
|
|
3561
|
-
const action = requestInfo?.action === 'restart' ? 'restart' : 'shutdown';
|
|
3562
|
-
const exitCode = action === 'restart' ? RELAY_RESTART_EXIT_CODE : 0;
|
|
3563
|
-
console.log(
|
|
3564
|
-
`${runtimeLogPrefix()}Relay ${action} request accepted by ${requestInfo?.requestedBy || 'unknown'}`
|
|
3565
|
-
+ `; no turn is processing (${counts.pendingCount} pending, ${counts.parkedCount} parked wait), exiting now (reason=${requestInfo?.reason || 'manual-request'})`,
|
|
3566
|
-
);
|
|
3567
|
-
sessionWorkerWebSocketService.emitDraining(`api-${action}:${requestInfo?.reason || 'manual-request'}`);
|
|
3568
|
-
void shutdownRuntime(`api-${action}:${requestInfo?.reason || 'manual-request'}`, { exitCode });
|
|
3569
|
-
emitRelayShutdownState();
|
|
3570
|
-
};
|
|
3604
|
+
function keepWaitingForRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
|
|
3605
|
+
return relayRestartQueue.keepWaiting({ requestedBy });
|
|
3606
|
+
}
|
|
3571
3607
|
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
if (
|
|
3577
|
-
const
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
status: 'queued',
|
|
3581
|
-
action: pendingRelayShutdownRequest?.action || requestedAction,
|
|
3582
|
-
restart: (pendingRelayShutdownRequest?.action || requestedAction) === 'restart',
|
|
3583
|
-
requestedAt: pendingRelayShutdownRequest?.requestedAt || new Date().toISOString(),
|
|
3584
|
-
reason: pendingRelayShutdownRequest?.reason || safeReason,
|
|
3585
|
-
requestedBy: pendingRelayShutdownRequest?.requestedBy || safeRequestedBy,
|
|
3586
|
-
queue: counts,
|
|
3587
|
-
};
|
|
3608
|
+
function requestRelayShutdown(options = {}) {
|
|
3609
|
+
// The loop guard of relay_restarted: a conversation whose turn answers the
|
|
3610
|
+
// last restart is not woken by this one. Noted first, since an immediate
|
|
3611
|
+
// request exits from inside the call.
|
|
3612
|
+
if (!runtimeShutdownStarted) relayEventService?.noteRestartRequested();
|
|
3613
|
+
const result = relayRestartQueue.request(options);
|
|
3614
|
+
if (!relayRestartQueue.isQueued() && !runtimeShutdownStarted) relayEventService?.noteRestartCancelled();
|
|
3615
|
+
return result;
|
|
3588
3616
|
}
|
|
3589
3617
|
|
|
3590
3618
|
function toNullableInt(value) {
|
|
@@ -4227,7 +4255,9 @@ function isSha256(value) {
|
|
|
4227
4255
|
}
|
|
4228
4256
|
|
|
4229
4257
|
function uploadPathForSha(sha256) {
|
|
4230
|
-
|
|
4258
|
+
let expectedSize = null;
|
|
4259
|
+
try { expectedSize = stmts.getUploadFile.get(String(sha256 || '').trim().toLowerCase())?.size_bytes ?? null; } catch {}
|
|
4260
|
+
return uploadPathInDirs(UPLOAD_READ_DIRS, sha256, { expectedSize });
|
|
4231
4261
|
}
|
|
4232
4262
|
|
|
4233
4263
|
function uploadContentUrlForSha(sha256) {
|
|
@@ -4908,7 +4938,10 @@ function persistUploadBuffer(buffer, options = {}) {
|
|
|
4908
4938
|
if (!Buffer.isBuffer(buffer) || !buffer.length) throw new Error('Empty upload');
|
|
4909
4939
|
if (buffer.length > MAX_UPLOAD_BYTES) throw new Error('Uploaded file too large');
|
|
4910
4940
|
const sha256 = crypto.createHash('sha256').update(buffer).digest('hex');
|
|
4911
|
-
|
|
4941
|
+
// Always kept in the upload dir: the folders of earlier versions can go
|
|
4942
|
+
// (a package update, a clean checkout), so a new reference never relies on
|
|
4943
|
+
// them. An existing file is never overwritten.
|
|
4944
|
+
const filePath = path.join(UPLOAD_DIR, sha256);
|
|
4912
4945
|
if (!fs.existsSync(filePath)) {
|
|
4913
4946
|
const tmpPath = path.join(UPLOAD_DIR, `${sha256}.${process.pid}.${Date.now()}.tmp`);
|
|
4914
4947
|
fs.writeFileSync(tmpPath, buffer);
|
|
@@ -4946,13 +4979,14 @@ function buildStoredAttachment({ sha256, name, type, size }) {
|
|
|
4946
4979
|
if (!isSha256(normalizedSha)) return null;
|
|
4947
4980
|
const normalizedName = String(name || '').trim().slice(0, 255) || `upload-${normalizedSha.slice(0, 12)}`;
|
|
4948
4981
|
const normalizedType = String(type || '').trim().toLowerCase().slice(0, 127) || 'application/octet-stream';
|
|
4982
|
+
const filePath = uploadPathForSha(normalizedSha);
|
|
4949
4983
|
return {
|
|
4950
4984
|
name: normalizedName,
|
|
4951
4985
|
type: normalizedType,
|
|
4952
4986
|
size: Number(size || 0),
|
|
4953
4987
|
sha256: normalizedSha,
|
|
4954
|
-
path:
|
|
4955
|
-
reference: `@${
|
|
4988
|
+
path: filePath,
|
|
4989
|
+
reference: `@${filePath}`,
|
|
4956
4990
|
contentUrl: uploadContentUrlForSha(normalizedSha),
|
|
4957
4991
|
};
|
|
4958
4992
|
}
|
|
@@ -5025,7 +5059,9 @@ function deleteOrphanedUploads(hashes) {
|
|
|
5025
5059
|
const refs = Number(stmts.countUploadRefsBySha.get(sha256)?.cnt || 0);
|
|
5026
5060
|
if (refs > 0) continue;
|
|
5027
5061
|
stmts.deleteUploadFile.run(sha256);
|
|
5028
|
-
|
|
5062
|
+
// Only the upload dir is swept; the folders of earlier versions are
|
|
5063
|
+
// read-only to the relay.
|
|
5064
|
+
const filePath = path.join(UPLOAD_DIR, sha256);
|
|
5029
5065
|
if (fs.existsSync(filePath)) {
|
|
5030
5066
|
try { fs.unlinkSync(filePath); } catch {}
|
|
5031
5067
|
}
|
|
@@ -5248,10 +5284,14 @@ function previewCardsForResponse(responseMessageId) {
|
|
|
5248
5284
|
.slice(0, 5);
|
|
5249
5285
|
}
|
|
5250
5286
|
|
|
5251
|
-
// Live background-task sets, as the session workers publish them.
|
|
5252
|
-
// only: after a runtime start the heartbeat's digest brings each
|
|
5253
|
-
// (
|
|
5254
|
-
|
|
5287
|
+
// Live background-task sets, as the session workers publish them. The sets
|
|
5288
|
+
// are memory only: after a runtime start the heartbeat's digest brings each
|
|
5289
|
+
// set back. The console rows (the output file of a running shell, the ended
|
|
5290
|
+
// rows of the last hour) are stored (migration 0018) and restored here; a
|
|
5291
|
+
// running one is held until its worker reports (services/background-task-store.mjs).
|
|
5292
|
+
const backgroundTaskStore = createBackgroundTaskStore({
|
|
5293
|
+
persistence: createSqliteBackgroundTaskPersistence(db, { logger: console }),
|
|
5294
|
+
});
|
|
5255
5295
|
// The output a worker posts for a task with no output file (the console's
|
|
5256
5296
|
// fallback); memory only, an hour.
|
|
5257
5297
|
const taskOutputSnapshots = createTaskOutputSnapshots();
|
|
@@ -5366,6 +5406,29 @@ const usageLimitPauseService = createUsageLimitPauseService({
|
|
|
5366
5406
|
logger: console,
|
|
5367
5407
|
});
|
|
5368
5408
|
|
|
5409
|
+
// Turns a restart cut off for good (their worker went with the relay, and a
|
|
5410
|
+
// second delivery would run the prompt twice): with the setting on, their
|
|
5411
|
+
// session gets one note to carry on. The rows processing right now are the
|
|
5412
|
+
// ones the runtime before left; the recovery paths below report what became
|
|
5413
|
+
// of each (services/restart-resume-service.mjs).
|
|
5414
|
+
const restartResumeService = createRestartResumeService({
|
|
5415
|
+
db,
|
|
5416
|
+
stmts,
|
|
5417
|
+
isEnabled: getRestartResumeEnabled,
|
|
5418
|
+
emit: (event, payload) => io.emit(event, payload),
|
|
5419
|
+
// The note is an ordinary pending message: the primer starts a stopped
|
|
5420
|
+
// worker, the socket service hands it to a running one.
|
|
5421
|
+
wake: () => {
|
|
5422
|
+
void primePendingSessionWorkers('restart-resume');
|
|
5423
|
+
sessionWorkerWebSocketService?.emitQueueChanged?.('restart-resume');
|
|
5424
|
+
},
|
|
5425
|
+
logger: console,
|
|
5426
|
+
});
|
|
5427
|
+
restartResumeService.captureBoot();
|
|
5428
|
+
function noteRestartTurnOutcome(row, outcome) {
|
|
5429
|
+
try { restartResumeService.noteOutcome(row, outcome); } catch {}
|
|
5430
|
+
}
|
|
5431
|
+
|
|
5369
5432
|
// ─── Relay events ─────────────────────────────────────────────────────────────
|
|
5370
5433
|
// What each session is doing (state, tasks, quiet time) and the subscriptions
|
|
5371
5434
|
// through which a watching session is woken with the events of the sessions
|
|
@@ -5531,56 +5594,21 @@ function failConsumedRowTerminally(row, reason) {
|
|
|
5531
5594
|
});
|
|
5532
5595
|
}
|
|
5533
5596
|
|
|
5534
|
-
|
|
5535
|
-
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
|
|
5539
|
-
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
|
|
5544
|
-
|
|
5545
|
-
|
|
5546
|
-
|
|
5547
|
-
|
|
5548
|
-
|
|
5549
|
-
const listedRows = stmts.listRecoverableProcessing.all(params);
|
|
5550
|
-
for (const row of listedRows.filter((entry) => entry?.consumed_at)) failConsumedRowTerminally(row, 'stale-recovery');
|
|
5551
|
-
const allRows = listedRows.filter((entry) => !entry?.consumed_at);
|
|
5552
|
-
if (!allRows.length) return [];
|
|
5553
|
-
// Enforce the retry budget before requeueing: failing these rows first
|
|
5554
|
-
// flips them out of 'processing' so the recovery UPDATE below skips them.
|
|
5555
|
-
const overBudget = allRows.filter((row) => Number(row?.retry_count || 0) + 1 >= MAX_REQUEUE_RETRIES);
|
|
5556
|
-
for (const row of overBudget) failRecoveredRowTerminally(row, 'stale-recovery');
|
|
5557
|
-
const rows = allRows.filter((row) => Number(row?.retry_count || 0) + 1 < MAX_REQUEUE_RETRIES);
|
|
5558
|
-
if (!rows.length) return [];
|
|
5559
|
-
|
|
5560
|
-
const tx = db.transaction(() => {
|
|
5561
|
-
stmts.recoverProcessingBefore.run({ ...params, requeueAt: requeueAtIso });
|
|
5562
|
-
const settleRecoveredAbortControls = db.prepare(`
|
|
5563
|
-
UPDATE relay_control_requests
|
|
5564
|
-
SET status = 'failed',
|
|
5565
|
-
error = ?,
|
|
5566
|
-
updated_at = ?,
|
|
5567
|
-
completed_at = ?
|
|
5568
|
-
WHERE queue_message_id = ?
|
|
5569
|
-
AND type IN ('abort_turn', 'abort_subagent')
|
|
5570
|
-
AND status IN ('pending', 'processing')
|
|
5571
|
-
`);
|
|
5572
|
-
const now = new Date().toISOString();
|
|
5573
|
-
for (const row of rows) {
|
|
5574
|
-
settleRecoveredAbortControls.run('queue-recovered', now, now, row.id);
|
|
5575
|
-
}
|
|
5576
|
-
});
|
|
5577
|
-
tx();
|
|
5578
|
-
|
|
5579
|
-
for (const row of rows) {
|
|
5580
|
-
io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'pending' });
|
|
5581
|
-
}
|
|
5582
|
-
io.emit('queue_updated', { recovered: rows.length });
|
|
5583
|
-
return rows;
|
|
5597
|
+
// The stale-turn sweep itself lives in services/stale-turn-recovery.mjs so it
|
|
5598
|
+
// can be tested against a real database; io is created further down, hence
|
|
5599
|
+
// the late-bound emit.
|
|
5600
|
+
const staleTurnRecovery = createStaleTurnRecovery({
|
|
5601
|
+
db,
|
|
5602
|
+
stmts,
|
|
5603
|
+
emit: (event, payload) => io.emit(event, payload),
|
|
5604
|
+
maxRequeueRetries: MAX_REQUEUE_RETRIES,
|
|
5605
|
+
failRecoveredRowTerminally,
|
|
5606
|
+
failConsumedRowTerminally,
|
|
5607
|
+
noteRestartTurnOutcome,
|
|
5608
|
+
warn: (line) => console.warn(`${runtimeLogPrefix()}${line}`),
|
|
5609
|
+
});
|
|
5610
|
+
function recoverProcessingOlderThan(cutoffIso, requeueAtIso, options) {
|
|
5611
|
+
return staleTurnRecovery.recoverProcessingOlderThan(cutoffIso, requeueAtIso, options);
|
|
5584
5612
|
}
|
|
5585
5613
|
|
|
5586
5614
|
// One `/api/usage` needs the same token for the quota read and the billing
|
|
@@ -5915,7 +5943,7 @@ const readContextFromSessionEvents = contextSnapshotService.readContextFromSessi
|
|
|
5915
5943
|
// Claude-provider sessions live under the Agent SDK's project layout rather than
|
|
5916
5944
|
// ~/.copilot/session-state, so their browsable session folder resolves separately.
|
|
5917
5945
|
const claudeSessionRootResolver = createClaudeSessionRootResolver({ fs, path });
|
|
5918
|
-
const cursorSessionRootResolver = createCursorSessionRootResolver({ fs, path, serverDir: __dirname });
|
|
5946
|
+
const cursorSessionRootResolver = createCursorSessionRootResolver({ fs, path, serverDir: __dirname, dataDir: DATA_DIR });
|
|
5919
5947
|
|
|
5920
5948
|
// ─── Express + Socket.io Setup ────────────────────────────────────────────────
|
|
5921
5949
|
const app = express();
|
|
@@ -6442,14 +6470,17 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
|
|
|
6442
6470
|
// No user prompt to replay: a dead worker's continuation is torn down.
|
|
6443
6471
|
stmts.dropStaleContinuation?.run?.(row.id);
|
|
6444
6472
|
io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
|
|
6473
|
+
noteRestartTurnOutcome(row, 'cut');
|
|
6445
6474
|
continue;
|
|
6446
6475
|
}
|
|
6447
6476
|
if (row.consumed_at) {
|
|
6448
6477
|
failConsumedRowTerminally(row, `worker-died-${reason}`);
|
|
6478
|
+
noteRestartTurnOutcome(row, 'cut');
|
|
6449
6479
|
continue;
|
|
6450
6480
|
}
|
|
6451
6481
|
if (Number(row.retry_count || 0) + 1 >= MAX_REQUEUE_RETRIES) {
|
|
6452
6482
|
failRecoveredRowTerminally(row, `worker-died-${reason}`);
|
|
6483
|
+
noteRestartTurnOutcome(row, 'failed');
|
|
6453
6484
|
continue;
|
|
6454
6485
|
}
|
|
6455
6486
|
// The dead worker's subagent runs can never be terminated by the replay
|
|
@@ -6457,6 +6488,7 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
|
|
|
6457
6488
|
closeStrandedSubagentRuns(stmts.listRunningSubagentRunsByQueueMessage?.all?.(row.id) || [], row.id);
|
|
6458
6489
|
stmts.recoverProcessingRowKeepOwner?.run?.(requeueAt, row.id);
|
|
6459
6490
|
io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'pending' });
|
|
6491
|
+
noteRestartTurnOutcome(row, 'requeued');
|
|
6460
6492
|
recovered += 1;
|
|
6461
6493
|
}
|
|
6462
6494
|
if (recovered > 0) {
|
|
@@ -6569,7 +6601,7 @@ function sessionHasBackgroundWork(sid) {
|
|
|
6569
6601
|
const detachedWorkerWatch = createDetachedWorkerWatch({
|
|
6570
6602
|
isEnabled: () => featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true
|
|
6571
6603
|
&& !runtimeShutdownStarted
|
|
6572
|
-
&& !
|
|
6604
|
+
&& !relayRestartQueue.isQueued(),
|
|
6573
6605
|
listWaitingSessionIds: () => (stmts.listPendingWorkerOwnerSessionIds?.all?.(25) || [])
|
|
6574
6606
|
.map((row) => row?.sdk_session_id),
|
|
6575
6607
|
hasWorkerSocket: (sid) => sessionWorkerWebSocketService.hasWorkerSocket(sid) === true,
|
|
@@ -6766,10 +6798,10 @@ const runtimeState = {
|
|
|
6766
6798
|
get relayPaused() { return relayPaused; },
|
|
6767
6799
|
set relayPaused(value) { relayPaused = value; },
|
|
6768
6800
|
get ttyConsoleActive() { return Boolean(ttyConsoleRuntime?.tty); },
|
|
6769
|
-
get relayShutdown() { return
|
|
6801
|
+
get relayShutdown() { return relayRestartQueue.getState(); },
|
|
6770
6802
|
// A shutdown or restart waits for the processing turns: no pending row
|
|
6771
|
-
// goes out meanwhile (
|
|
6772
|
-
get relayShutdownQueued() { return
|
|
6803
|
+
// goes out meanwhile (services/relay-restart-queue.mjs).
|
|
6804
|
+
get relayShutdownQueued() { return relayRestartQueue.isQueued(); },
|
|
6773
6805
|
get hostSuspend() { return hostSuspendService.getState(); },
|
|
6774
6806
|
get usageLimit() {
|
|
6775
6807
|
return { account: usageLimitPauseService.getAccountState(), pauses: usageLimitPauseService.listPauses() };
|
|
@@ -6788,6 +6820,34 @@ const windowsBootAutostartService = createWindowsBootAutostartService({
|
|
|
6788
6820
|
packageRoot: REPO_ROOT,
|
|
6789
6821
|
configPath: CONFIG_PATH,
|
|
6790
6822
|
});
|
|
6823
|
+
// Autostart entries written before they pinned the data and log folders start
|
|
6824
|
+
// the relay on a database inside the package directory, or before TEMP and
|
|
6825
|
+
// the data drive are there. Brought up to date once this relay holds the
|
|
6826
|
+
// lock and its port; edited ones stay.
|
|
6827
|
+
function refreshWindowsAutostartEntries() {
|
|
6828
|
+
// Entries that ran the relay on the database inside the package.
|
|
6829
|
+
const unpinnedLaunchers = [];
|
|
6830
|
+
const onEarlierForm = ({ path: launcher, pinnedData }) => { if (!pinnedData) unpinnedLaunchers.push(launcher); };
|
|
6831
|
+
for (const [name, refresh] of [
|
|
6832
|
+
['sign-in entry', () => windowsAutostartService.refreshInstalledEntry({ onEarlierForm })],
|
|
6833
|
+
['boot launcher', () => windowsBootAutostartService.refreshInstalledLauncher({ onEarlierForm })],
|
|
6834
|
+
]) {
|
|
6835
|
+
try {
|
|
6836
|
+
if (refresh()) console.log(`[autostart] brought the ${name} up to date: it pins the data and log folders and waits for TEMP and the data drive`);
|
|
6837
|
+
} catch (error) {
|
|
6838
|
+
console.warn(`[autostart] could not bring the ${name} up to date: ${error?.message || error}`);
|
|
6839
|
+
}
|
|
6840
|
+
}
|
|
6841
|
+
const warning = lostPackageDatabaseWarning({
|
|
6842
|
+
unpinnedLaunchers,
|
|
6843
|
+
earlierDatabase,
|
|
6844
|
+
databaseCreated: databaseCreatedAtStart,
|
|
6845
|
+
evidence: unpinnedLaunchers.length > 0 && packageDatabaseEvidence({ serverDir: __dirname, ownDirs: [DATA_DIR, path.dirname(relayConsoleLogPath)] }),
|
|
6846
|
+
serverDir: __dirname,
|
|
6847
|
+
dbPath: DB_PATH,
|
|
6848
|
+
});
|
|
6849
|
+
if (warning) console.error(warning);
|
|
6850
|
+
}
|
|
6791
6851
|
|
|
6792
6852
|
const gitChangesService = createGitChangesService();
|
|
6793
6853
|
// GET /api/git/remote: where a folder lives on GitHub (New Chat, Claude Cloud).
|
|
@@ -7017,6 +7077,9 @@ const sharedRouteDeps = {
|
|
|
7017
7077
|
pokeTaskHealth: () => taskHealthPoller.poke(),
|
|
7018
7078
|
getTaskStallMinutes,
|
|
7019
7079
|
setTaskStallMinutes,
|
|
7080
|
+
getRestartResumeEnabled,
|
|
7081
|
+
setRestartResumeEnabled,
|
|
7082
|
+
noteRestartTurnOutcome,
|
|
7020
7083
|
// A closure, not the registry itself: this deps object is built before the
|
|
7021
7084
|
// preview lane is constructed further down the file.
|
|
7022
7085
|
listConversationPreviews: (conversationId) => previewRegistry.listForConversation(conversationId),
|
|
@@ -7049,6 +7112,7 @@ const sharedRouteDeps = {
|
|
|
7049
7112
|
fs,
|
|
7050
7113
|
path,
|
|
7051
7114
|
uploadsDir: UPLOAD_DIR,
|
|
7115
|
+
uploadReadDirs: UPLOAD_READ_DIRS,
|
|
7052
7116
|
fetchUsageSummary,
|
|
7053
7117
|
fetchCopilotBillingUsage,
|
|
7054
7118
|
fetchGrokBillingUsage: createGrokBillingUsageFetcher(),
|
|
@@ -7146,6 +7210,16 @@ registerRelayEventRoutes(app, {
|
|
|
7146
7210
|
relayEventService,
|
|
7147
7211
|
remoteRelayInbound,
|
|
7148
7212
|
});
|
|
7213
|
+
registerRelayRestartRoutes(app, {
|
|
7214
|
+
auth,
|
|
7215
|
+
// Read only: a GET neither fires the restart nor asks the question.
|
|
7216
|
+
getState: () => relayRestartQueue.getState(),
|
|
7217
|
+
collectRestartPicture,
|
|
7218
|
+
keepWaiting: keepWaitingForRelayShutdown,
|
|
7219
|
+
getMaxWaitMinutes: getRelayRestartMaxWaitMinutes,
|
|
7220
|
+
setMaxWaitMinutes: setRelayRestartMaxWaitMinutes,
|
|
7221
|
+
maxWaitOverrideMs: relayRestartMaxWaitOverrideMs,
|
|
7222
|
+
});
|
|
7149
7223
|
registerAboutRoutes(app, {
|
|
7150
7224
|
auth,
|
|
7151
7225
|
aboutService: createAboutService({
|
|
@@ -7236,6 +7310,8 @@ function recoverStaleMessages() {
|
|
|
7236
7310
|
const ceilingBeforeIso = ceilingMs > 0 ? new Date(Date.now() - ceilingMs).toISOString() : null;
|
|
7237
7311
|
const requeueAt = addMsIso(cliOnline ? 30_000 : 2_000);
|
|
7238
7312
|
const recoveredRows = recoverProcessingOlderThan(cutoff, requeueAt, { ceilingBeforeIso });
|
|
7313
|
+
// Turns of the start that ended on their own decide their session's note.
|
|
7314
|
+
try { restartResumeService.tick(); } catch {}
|
|
7239
7315
|
if (recoveredRows.length > 0) {
|
|
7240
7316
|
const ceilingNote = ceilingBeforeIso ? `, ceiling ${Math.round(ceilingMs / 60_000)}min` : '';
|
|
7241
7317
|
console.log(
|
|
@@ -7561,6 +7637,27 @@ const taskOutputRooms = createTaskOutputRooms({
|
|
|
7561
7637
|
// the live tasks with an output file.
|
|
7562
7638
|
const taskHealthPoller = createTaskHealthPoller({ store: backgroundTaskStore, io });
|
|
7563
7639
|
taskHealthPoller.start();
|
|
7640
|
+
// Console tasks held since the restart whose worker has not reported within
|
|
7641
|
+
// twice the boot window (at least 15 s: a test relay has none): that worker
|
|
7642
|
+
// did not survive the restart. They are marked ended (their output stays
|
|
7643
|
+
// readable for the hour), never running.
|
|
7644
|
+
runtimeTimers.restoredTaskSettle = setTimeout(() => {
|
|
7645
|
+
runtimeTimers.restoredTaskSettle = null;
|
|
7646
|
+
const keys = backgroundTaskStore.endUnconfirmed();
|
|
7647
|
+
if (!keys.length) return;
|
|
7648
|
+
console.log(`${runtimeLogPrefix()}BG-TASKS ${keys.length} session(s) did not report after the restart; their console tasks are marked ended`);
|
|
7649
|
+
for (const key of keys) {
|
|
7650
|
+
try {
|
|
7651
|
+
io.emit('background_tasks', {
|
|
7652
|
+
conversationId: key,
|
|
7653
|
+
tasks: backgroundTaskStore.withHealth(backgroundTaskStore.get(key), key),
|
|
7654
|
+
endedTasks: backgroundTaskStore.ended(key),
|
|
7655
|
+
});
|
|
7656
|
+
} catch {}
|
|
7657
|
+
}
|
|
7658
|
+
taskHealthPoller.poke();
|
|
7659
|
+
}, Math.max(15_000, 2 * resolveBootReportingWindowMs(process.env)));
|
|
7660
|
+
runtimeTimers.restoredTaskSettle?.unref?.();
|
|
7564
7661
|
// The handshake presents the token as `auth.token` or the cookie; a query
|
|
7565
7662
|
// `token` is not read (see services/relay-auth.mjs).
|
|
7566
7663
|
io.use(relayAuth.socketMiddleware);
|
|
@@ -7723,12 +7820,21 @@ registerPreviewRoutes(app, {
|
|
|
7723
7820
|
new Date().toISOString(),
|
|
7724
7821
|
);
|
|
7725
7822
|
},
|
|
7823
|
+
// A scope named by a conversation's id or its session id covers both.
|
|
7824
|
+
resolveConversationScope: createPreviewScopeResolver({
|
|
7825
|
+
getConversation: (id) => {
|
|
7826
|
+
const conv = stmts.getConvAnyStatus.get(id) || null;
|
|
7827
|
+
return conv && String(conv.status || '').trim().toLowerCase() !== 'deleted' ? conv : null;
|
|
7828
|
+
},
|
|
7829
|
+
getConversationBySdkSessionId: (id) => stmts.getConvBySdkSessionId?.get?.(id) || null,
|
|
7830
|
+
}),
|
|
7726
7831
|
});
|
|
7727
7832
|
|
|
7728
7833
|
function clearRuntimeTimers() {
|
|
7729
7834
|
for (const timer of Object.values(runtimeTimers)) {
|
|
7730
7835
|
if (!timer) continue;
|
|
7731
7836
|
try { clearInterval(timer); } catch {}
|
|
7837
|
+
try { clearTimeout(timer); } catch {}
|
|
7732
7838
|
}
|
|
7733
7839
|
subagentRecordRetention?.stop();
|
|
7734
7840
|
try { taskHealthPoller.stop(); } catch {}
|
|
@@ -7739,7 +7845,8 @@ function clearRuntimeTimers() {
|
|
|
7739
7845
|
runtimeTimers.questionExpiry = null;
|
|
7740
7846
|
runtimeTimers.pendingWorkerPrime = null;
|
|
7741
7847
|
runtimeTimers.sharedViewerPrune = null;
|
|
7742
|
-
runtimeTimers.
|
|
7848
|
+
runtimeTimers.restoredTaskSettle = null;
|
|
7849
|
+
try { relayRestartQueue.dispose(); } catch {}
|
|
7743
7850
|
}
|
|
7744
7851
|
|
|
7745
7852
|
let runtimeShutdownPromise = null;
|
|
@@ -7780,6 +7887,9 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
|
|
|
7780
7887
|
console.warn(`${runtimeLogPrefix()}Grok auth service shutdown failed: ${error?.message || error}`);
|
|
7781
7888
|
}
|
|
7782
7889
|
sessionWorkerWebSocketService.stop();
|
|
7890
|
+
// A queued request wrote its own end already; this covers the rest
|
|
7891
|
+
// (signals, the owner gone).
|
|
7892
|
+
try { relayEventService.noteRelayExit({ action: runtimeShutdownExitCode === RELAY_RESTART_EXIT_CODE ? 'restart' : 'shutdown', reason }); } catch {}
|
|
7783
7893
|
try { relayEventService.stop(); } catch {}
|
|
7784
7894
|
try { relayEventLinkService.stop(); } catch {}
|
|
7785
7895
|
tmuxInspectorSocketService.stop();
|
|
@@ -7910,12 +8020,19 @@ httpServer.on('error', (error) => {
|
|
|
7910
8020
|
process.exit(1);
|
|
7911
8021
|
});
|
|
7912
8022
|
httpServer.listen(config.port, listenHost, () => {
|
|
8023
|
+
// Conversations that asked for relay_restarted hear that the relay is back:
|
|
8024
|
+
// once it listens, so a start that fails to bind does not take the record
|
|
8025
|
+
// of how the run before ended.
|
|
8026
|
+
try { relayEventService.noteRelayStarted(); } catch (error) {
|
|
8027
|
+
console.warn(`${runtimeLogPrefix()}relay_restarted not recorded: ${error?.message || error}`);
|
|
8028
|
+
}
|
|
7913
8029
|
const readyBanner = buildRelayReadyBannerData();
|
|
7914
8030
|
console.log('');
|
|
7915
8031
|
for (const line of (readyBanner.lines || [])) {
|
|
7916
8032
|
console.log(line);
|
|
7917
8033
|
}
|
|
7918
8034
|
console.log('\nCLI status: waiting for first heartbeat...\n');
|
|
8035
|
+
refreshWindowsAutostartEntries();
|
|
7919
8036
|
|
|
7920
8037
|
// Start SSH tunnel after server is listening
|
|
7921
8038
|
sshTunnelManager.start();
|