@oar-sh/oar 0.9.19 → 0.9.21

package/server/server-runtime.mjs

@@ -28,6 +28,8 @@ import { createDetachedWorkerWatch } from './services/detached-worker-watch-serv
 import { applyCdnNoStoreHeaders } from './services/file-serving.mjs';
 import { pickLiveTurnRowId } from './services/live-turn-picker.mjs';
 import { buildSteerSettleFailure, steerAgentLabelForProvider } from '../shared/steer-settle-failure.mjs';
+import { PLAN_BOARD_APPROVE_ONLY, planBoardShownActions } from '../shared/plan-board-actions.mjs';
+import { carryPlanOnlyToCompacted, readPlanOnly } from './services/conversation-preferences-service.mjs';
 import { sanitizeRelayQuestionContext as sanitizeRelayQuestionContextWith } from './services/relay-question-context.mjs';
 import { isRealPathWithinRoot } from './services/workspace-symlink-guard.mjs';
 import { applySchema } from './db-schema.mjs';
@@ -73,7 +75,11 @@ import { createDeepSeekUsageImportService } from './services/deepseek-usage-impo
 import { createClaudeApiSettingsService } from './services/claude-api-settings-service.mjs';
 import { createClaudeBillingSwitchService, normalizeClaudeBilling } from './services/claude-billing-switch-service.mjs';
 import { createClaudeBillingSwapService } from './services/claude-billing-swap-service.mjs';
+import { createWorkerReplacement, createWorkerReplacementFence } from './services/worker-replacement.mjs';
+import { sessionLaunchMutex } from './services/session-launch-mutex-service.mjs';
+import { createWorkerRecycleService, sweepArchivedWorkers } from './services/worker-recycle-service.mjs';
 import { createClaudeApiUsageService } from './services/claude-api-usage-service.mjs';
+import { createTokenUsageService } from './services/token-usage-service.mjs';
 import { createClaudeApiCapWatch, createClaudeApiPlanUsage, createClaudeApiStopNoteLedger } from './services/plan-usage-claude-api.mjs';
 import { DEEPSEEK_DEFAULT_AUTO_COMPACT_WINDOW, DEFAULT_DEEPSEEK_MODEL, deepseekModelLabel } from '../shared/deepseek-models.mjs';
 import { createClaudeCloudSessionService } from './services/claude-cloud-session-service.mjs';
@@ -101,6 +107,7 @@ import { createRelayEventService } from './services/relay-event-service.mjs';
 import { createRelayScheduleService } from './services/relay-schedule-service.mjs';
 import { createRelayAskService } from './services/relay-ask-service.mjs';
 import { createRelayAskLinkService } from './services/relay-ask-link-service.mjs';
+import { createRemoteInboxWatchService } from './services/remote-inbox-watch-service.mjs';
 import { createRelayEventLinkService } from './services/relay-event-link-service.mjs';
 import { buildClaudeAttributionSettings, normalizeClaudeAttributionMode, resolveClaudeAttributionMode, DEFAULT_CLAUDE_ATTRIBUTION_MODE } from '../shared/claude-attribution.mjs';
 import {
@@ -115,7 +122,8 @@ import { createActiveDeviceTracker } from './services/push-active-device-service
 import { registerPushRoutes } from './routes/push-routes.mjs';
 import { registerAboutRoutes } from './routes/about-routes.mjs';
 import { registerRelayRestartRoutes } from './routes/relay-restart-routes.mjs';
-import { createAboutService, resolveBuildCommit } from './services/about-service.mjs';
+import { createAboutService } from './services/about-service.mjs';
+import { resolveBuildId } from '../shared/build-id.mjs';
 import { createImageOperationService } from './services/image-operation-service.mjs';
 import {
   normalizeDriveAbsolutePath as _normalizeDriveAbsolutePath,
@@ -142,7 +150,10 @@ import { createCursorSessionRootResolver } from './services/cursor-session-root-
 import { createGitChangesService } from './services/git-changes-service.mjs';
 import { createRelaySingletonGuard } from './services/relay-singleton-guard.mjs';
 import { startRelayMemoryLog } from './relay-heap-guard.mjs';
-import { adoptEarlierDatabase, lostPackageDatabaseWarning, packageDatabaseEvidence } from './services/earlier-database-service.mjs';
+import { TAKEOVER_PROGRESS_FILE, adoptEarlierDatabase, lostPackageDatabaseWarning, packageDatabaseEvidence } from './services/earlier-database-service.mjs';
+import { checkDatabaseFile, resolveDatabaseFile, settleDatabaseFile } from './services/database-file.mjs';
+import { describeBackups, describeNextBackup, maybeBackupBeforeStart, writeDbLastRun } from './services/db-backup-service.mjs';
+import { resolveStateLayout } from './services/oar-state-layout.mjs';
 import { ensureDataDirs, resolveDataDir, resolveUploadDir, resolveUploadReadDirs, uploadPathInDirs } from './services/data-dir-resolution.mjs';
 import { mergeStrayMigratedUploads } from './services/oar-state-migration-service.mjs';
 import { createRelayRestartOrchestrator } from './services/relay-restart-orchestrator-service.mjs';
@@ -219,7 +230,8 @@ import {
   resolveRelayRestartMaxWaitOverrideMs,
   RELAY_RESTART_MAX_WAIT_SETTING_KEY,
 } from './services/relay-restart-queue.mjs';
-import { createRestartPictureCollector } from './services/relay-restart-picture.mjs';
+import { createRestartPictureCollector, detectRelayHosting } from './services/relay-restart-picture.mjs';
+import { createWorkerScope } from './services/worker-scope.mjs';
 import { createStaleRecoveryClockGuard } from './services/stale-recovery-clock-guard.mjs';
 import { createStaleTurnRecovery } from './services/stale-turn-recovery.mjs';
 import {
@@ -290,7 +302,12 @@ const INSTALL_METHOD = fs.existsSync(path.join(PACKAGE_ROOT, '.git')) ? 'git-che
 // When this relay process started, and the commit its code came from (read
 // once: a checkout moved on later still runs the code it started with).
 const RELAY_STARTED_AT = new Date(Date.now() - process.uptime() * 1000).toISOString();
-const BUILD_COMMIT = resolveBuildCommit(PACKAGE_ROOT);
+// The build id (version plus commit) names that code for the session workers:
+// a worker that states another one started on older code. Read in the same
+// pass as the commit so the two cannot disagree.
+const RELAY_BUILD = resolveBuildId(PACKAGE_ROOT);
+const BUILD_COMMIT = RELAY_BUILD.commit;
+const BUILD_ID = RELAY_BUILD.id;
 const PUBLIC_DIR = path.join(__dirname, 'public');
 const INDEX_HTML_PATH = path.join(PUBLIC_DIR, 'index.html');
 const PWA_MANIFEST_PATH = path.join(PUBLIC_DIR, 'manifest.webmanifest');
@@ -320,7 +337,9 @@ const CONFIG_PATH    = process.env.COPILOT_WEB_RELAY_CONFIG
 // for why the config can carry it (a dev checkout pointing at a release
 // install's data).
 const DATA_DIR = resolveDataDir({ env: process.env, configPath: CONFIG_PATH, serverDir: __dirname });
-const DB_PATH        = path.join(DATA_DIR, 'copilot.db');
+// DB_PATH (oar.db, or the name of installs from before 0.9.20) is
+// resolved under the lock, once an earlier database may have been brought
+// over: see resolveDatabaseFile below.
 const RELAY_LOCK_PATH = path.join(DATA_DIR, 'relay-server.lock');
 // Uploads follow the data dir whenever the env var or config.json chose it.
 const UPLOAD_DIR     = resolveUploadDir({ dataDir: DATA_DIR, env: process.env, serverDir: __dirname });
@@ -1441,7 +1460,16 @@ async function refreshGrokProviderModels() {
   }
 }
 
-const DEFAULT_CONFIG = { authToken: '', port: 3333, pollIntervalMs: 3000, conversationSessionMode: 'isolated', localhostOnly: true };
+const DEFAULT_CONFIG = {
+  authToken: '',
+  port: 3333,
+  pollIntervalMs: 3000,
+  conversationSessionMode: 'isolated',
+  localhostOnly: true,
+  // A copy of the database at the first start of each new version, the last
+  // `keep` kept (db-backup-service.mjs); `dir` (absolute) moves the folder.
+  dbBackups: { enabled: true, keep: 2 },
+};
 // How long a turn may go without any sign of life from its worker before the
 // relay assumes the worker died. This is an inactivity window, not a cap on turn
 // length — see recoverStaleMessages.
@@ -1868,8 +1896,28 @@ try {
   console.error(`[server] ${error?.message || String(error)}`);
   process.exit(error?.code === 'ERELAYLOCKED' ? RELAY_STOP_EXIT_CODE : 1);
 }
-// Before the database opens, under the lock (adoptEarlierDatabase); the
-// result also feeds lostPackageDatabaseWarning.
+// Before the database opens, under the lock: both names there are settled
+// first (the earlier name's database, not newer, is set aside, else the
+// relay stops), then adoptEarlierDatabase, whose result also feeds
+// lostPackageDatabaseWarning.
+{
+  let settled;
+  try {
+    settled = settleDatabaseFile({
+      dataDir: DATA_DIR,
+      Database,
+      progressFile: path.join(DATA_DIR, TAKEOVER_PROGRESS_FILE),
+      logger: console,
+    });
+  } catch (error) {
+    settled = { status: 'refused', error: `[data] Not starting: the database files in ${DATA_DIR} could not be looked at (${error?.message || error}).` };
+  }
+  if (settled.status === 'refused') {
+    console.error(settled.error);
+    try { relaySingletonGuard.release(); } catch {}
+    process.exit(RELAY_STOP_EXIT_CODE);
+  }
+}
 let earlierDatabase;
 {
   try {
@@ -1892,6 +1940,21 @@ let earlierDatabase;
     process.exit(RELAY_STOP_EXIT_CODE);
   }
 }
+// Resolved after the adoption, which may have placed the database: `created`
+// only when neither name is there now.
+const databaseFile = resolveDatabaseFile(DATA_DIR);
+const DB_PATH = databaseFile.path;
+{
+  // A dangling link, or an oar.db-wal/-shm without oar.db (a later rename to
+  // oar.db would take its pages in), stops the start; other strays are said.
+  const check = checkDatabaseFile(databaseFile);
+  for (const warning of check.warnings) console.warn(warning);
+  if (check.refusal) {
+    console.error(check.refusal);
+    try { relaySingletonGuard.release(); } catch {}
+    process.exit(RELAY_STOP_EXIT_CODE);
+  }
+}
 
 const MAX_REQUEUE_RETRIES = Number.isFinite(Number(config.maxRequeueRetries))
   ? Math.max(1, Number(config.maxRequeueRetries))
@@ -1952,6 +2015,8 @@ const runtimeTimers = {
   claudeApiCapClock: null,
   memoryLog: null,
   memoryWatch: null,
+  // Once, 30 s after the relay listens: archived sessions' workers stopped.
+  archivedWorkerSweep: null,
 };
 
 function normalizeRemotePath(value) {
@@ -2737,7 +2802,7 @@ const configuredConversationSessionMode =
 updateModelCatalog({ models: [DEFAULT_MODEL], currentModel: DEFAULT_MODEL, defaultModel: DEFAULT_MODEL, source: 'bootstrap' });
 
 // ─── SQLite Setup ─────────────────────────────────────────────────────────────
-const databaseCreatedAtStart = !fs.existsSync(DB_PATH);
+const databaseCreatedAtStart = databaseFile.created;
 const db = new Database(DB_PATH);
 if (process.platform !== 'win32') {
   fs.chmodSync(DB_PATH, 0o600);
@@ -2750,11 +2815,39 @@ if (process.platform !== 'win32') {
   }
 }
 
+// Where the state root keeps backups (npm), or a checkout (server/backups).
+const STATE_LAYOUT = resolveStateLayout({ packageRoot: REPO_ROOT });
+// A copy of the database before this version's migrations, when another
+// version (on a checkout: another commit) ran on it last; before applySchema
+// and before reconcileUpdateAttempt, which removes the in-app update's
+// record that names the version before. Never stops the start.
+let dbBackupAtStart = null;
+try {
+  dbBackupAtStart = await maybeBackupBeforeStart({
+    db,
+    Database,
+    dbPath: DB_PATH,
+    dataDir: DATA_DIR,
+    databaseCreated: databaseCreatedAtStart,
+    layout: STATE_LAYOUT,
+    serverDir: __dirname,
+    config,
+    runningVersion: RUNNING_VERSION,
+    runningCommit: BUILD_COMMIT,
+    logger: console,
+  });
+} catch (error) {
+  console.warn(`[backup] failed: ${error?.message || error}; this start goes on without a backup of the version before.`);
+}
+
 // Base DDL plus the column-backfill/upgrade migrations, extracted verbatim to
 // db-schema.mjs so tests can build the exact production schema. Runs the
 // CREATE TABLE IF NOT EXISTS block, the FTS setup, and every ALTER TABLE
 // upgrade in the original order.
 applySchema(db);
+// A database that had no app_settings table before applySchema (a new one):
+// the version that runs on it now.
+if (dbBackupAtStart?.deferredLastRun) writeDbLastRun(db, dbBackupAtStart.deferredLastRun);
 
 // ─── Prepared Statements ──────────────────────────────────────────────────────
 // Owns its own schema (plan-usage tables) and the derived Cursor spend
@@ -3245,11 +3338,16 @@ function buildSessionWorkerLaunchEnvForSession(targetSessionId) {
   if (!copilotSdkEngine) return openAiEnv;
   return applyCopilotSdkProviderEnvironment(openAiEnv, { enabled: true, model });
 }
+// Under a systemd user service, session workers start in scopes of their
+// own (worker-scope.mjs); asked once, at the first worker start.
+const workerScope = createWorkerScope({ hosting: detectRelayHosting(), log: (message) => console.log(`${runtimeLogPrefix()}${message}`) });
 const relayCliLauncherService = createRelayCliLauncherService({
   cwd: (targetSessionId) => resolveLaunchWorkspaceRootForSession(targetSessionId),
   env: sessionWorkerLaunchEnv,
   log: (message) => console.log(`${runtimeLogPrefix()}${message}`),
+  workerScope,
 });
+
 async function spawnSessionWorkerCli(targetSessionId, { allowProcessReuse = true } = {}) {
   const normalizedTargetSessionId = String(targetSessionId || '').trim();
   if (!normalizedTargetSessionId) {
@@ -3281,6 +3379,7 @@ async function spawnSessionWorkerCli(targetSessionId, { allowProcessReuse = true
   const resolvedWorkspaceRoot = resolveLaunchWorkspaceRootForSession(normalizedTargetSessionId);
   const workerLaunchEnv = buildSessionWorkerLaunchEnvForSession(normalizedTargetSessionId);
   const launched = await launchSessionCli({
+    workerScope,
     targetSessionId: normalizedTargetSessionId,
     cwd: resolvedWorkspaceRoot,
     env: workerLaunchEnv,
@@ -3359,6 +3458,7 @@ const updateCheckService = updateCheckKilled ? null : createUpdateCheckService({
 });
 const updateInstallService = createUpdateInstallService({
   runningVersion: RUNNING_VERSION,
+  describeNextBackup: () => describeNextBackup({ layout: STATE_LAYOUT, serverDir: __dirname, dataDir: DATA_DIR, dbPath: DB_PATH, config }),
   installMethod: INSTALL_METHOD,
   packageRoot: PACKAGE_ROOT,
   readSetting: readAppSettingValue,
@@ -3676,6 +3776,7 @@ function collectRestartPicture({ action = 'restart' } = {}) {
     restartPictureCollector = createRestartPictureCollector({
       db,
       backgroundWorkView,
+      workerScopes: () => workerScope.inUse(),
       listHeldTasks: () => backgroundTaskStore.listRestored().map(({ key, task }) => ({
         conversationId: resolveConversationRecord({ conversationId: key, sdkSessionId: key })?.id || key,
         description: task?.description || task?.taskType || '',
@@ -4055,19 +4156,28 @@ function parseBoardContext(rawContext) {
 
 function formatRelayBoardRow(row) {
   if (!row) return null;
+  const boardType = String(row.board_type || '').trim().toLowerCase() || 'generic';
+  const status = String(row.status || 'pending').trim().toLowerCase() || 'pending';
+  const planOnly = readPlanOnly(stmts.getConvAnyStatus.get(row.conversation_id));
   return {
     id: row.id,
     queueId: row.queue_id,
     conversationId: row.conversation_id,
     messageId: row.message_id,
-    boardType: String(row.board_type || '').trim().toLowerCase() || 'generic',
+    boardType,
     mode: normalizeRelayMode(row.relay_mode) || DEFAULT_RELAY_MODE,
     title: String(row.title || '').trim(),
     body: String(row.body || '').trim(),
-    actions: parseBoardActions(row.actions_json),
-    recommendedAction: String(row.recommended_action || '').trim() || null,
+    // A pending plan card stored before "Approve plan" existed shows it too.
+    actions: planBoardShownActions({ boardType, status }, parseBoardActions(row.actions_json)),
+    // On a plan-only session a pending plan card recommends "Approve plan".
+    recommendedAction: (planOnly && status === 'pending' && boardType === 'plan_ready')
+      ? PLAN_BOARD_APPROVE_ONLY
+      : (String(row.recommended_action || '').trim() || null),
+    // The UI asks before an action that leaves plan mode on such a session.
+    planOnly,
     context: parseBoardContext(row.context_json),
-    status: String(row.status || 'pending').trim().toLowerCase() || 'pending',
+    status,
     selectedAction: String(row.selected_action || '').trim() || null,
     actedAt: row.acted_at || null,
     createdAt: row.created_at,
@@ -5249,6 +5359,8 @@ function createCompactedConversation(sourceConversationId) {
     // The new conversation stays in the old one's session group, and heads
     // it if the old one did; the archived source stays a member.
     conversationGroups.carryToCompacted(sourceConversationId, targetConversationId, now);
+    // A plan-only conversation's lock goes along, with its saved mode.
+    carryPlanOnlyToCompacted({ db, stmts, source, targetConversationId, updatedAt: now });
     // A Claude chat keeps its billing (the subscription or the API key, a
     // pending swap's included): the new chat's first worker launches on it,
     // and it is never guessed again from a Claude model id. Other providers
@@ -5550,6 +5662,9 @@ const usageLimitPauseService = createUsageLimitPauseService({
 // workers' reports (services/claude-api-usage-service.mjs); the Claude API
 // card and its cap read it.
 const claudeApiUsageService = createClaudeApiUsageService({ db, logger: console });
+// Tokens per turn and per day for every provider, booked from the
+// workers' reports; the Tokens sections in Check Usage and About read it.
+const tokenUsageService = createTokenUsageService({ db, logger: console });
 // This month's spend on the key against the cap, and the Check Usage card
 // (services/plan-usage-claude-api.mjs), over the same local days.
 const claudeApiPlanUsage = createClaudeApiPlanUsage({
@@ -6446,6 +6561,23 @@ async function requestSessionWorkerSocketDelivery({ sessionId, pid, reason = 'wo
       restartOrchestrator: relayRestartOrchestrator?.getState?.() || null,
     };
   }
+  // The process asking is being replaced (the worker recycle passed its holds
+  // for it): no work, as for a killed session; the worker stays ready, and
+  // what waits goes to the new one. Unlike the killed marker above, this
+  // survives the marker's grace and every clearRestartSchedule.
+  if (workerReplacementFence.isHeld(requesterSessionId, pid)) {
+    return {
+      message: null,
+      routing: {
+        enabled: true,
+        requesterSessionId,
+        blockedReason: 'worker-replacement',
+        lifecycle: sessionWorkerSupervisor?.getLifecycleState?.(requesterSessionId) || null,
+        fallbackRestart: null,
+      },
+      restartOrchestrator: relayRestartOrchestrator?.getState?.() || null,
+    };
+  }
   sessionWorkerSupervisor?.noteSessionHeartbeat?.(requesterSessionId);
   if (runtimeState.relayPaused) {
     return { message: null, paused: true, reason: 'relay_paused' };
@@ -6525,6 +6657,9 @@ async function requestSessionWorkerSocketDelivery({ sessionId, pid, reason = 'wo
     workerKind: workerKind || sessionWorkerRegistry?.getWorker?.(requesterSessionId)?.kind || null,
     claudeApiClaimCheck,
     onBillingFence: handleClaudeBillingFence,
+    // The fence check above, asked again before every claim: ensureWorker
+    // and a transient error's backoff both wait.
+    isReplacementFenced: () => workerReplacementFence.isHeld(requesterSessionId, pid),
   });
   if (!dequeueResult?.message) {
     return {
@@ -6838,48 +6973,38 @@ async function sweepDeadWorkerProcessingRowsOnce() {
 // claude-billing-swap-service.mjs): a Claude chat's next row of the other
 // billing is refused to the worker it has, which is replaced, once idle, by
 // one of the row's kind. The old one is stopped like the kill route does, and
-// the new one launched only once every process of the session is gone.
+// the new one launched only once every process of the session is gone
+// (services/worker-replacement.mjs, which the worker recycle below shares).
+// The two get one instance each so a failed stop's warning names the caller.
+// They share one replacement fence, the one the deliveries check (the socket
+// delivery above and /api/pending): a process being replaced is handed no
+// work. The worker recycle raises it; the billing swap does not, since its
+// own delivery fence already keeps the other billing's rows from the old
+// worker.
+const workerReplacementFence = createWorkerReplacementFence();
+function createRuntimeWorkerReplacement(tag) {
+  return createWorkerReplacement({
+    registry: sessionWorkerRegistry,
+    supervisor: sessionWorkerSupervisor,
+    processInspector: sessionWorkerProcessInspector,
+    stopSessionWorkerProcesses,
+    isProcessAlive,
+    killTmuxSession,
+    logger: console,
+    logPrefix: () => `${runtimeLogPrefix()}${tag} `,
+    fence: workerReplacementFence,
+  });
+}
+// A stop that fails backs off (the swap service retries it) and clears the
+// killed marker, so the session's worker, alive or relaunched, keeps asking
+// for work meanwhile. The fence still keeps the other billing's rows from it.
+const billingSwapWorkerReplacement = createRuntimeWorkerReplacement('BILLING SWAP');
 const claudeBillingSwapService = createClaudeBillingSwapService({
   getWorker: (sid) => sessionWorkerRegistry?.getWorker?.(sid) || null,
   // A row the session claimed and has not finished: never stopped under it.
   hasRunningTurn: (sid) => (stmts.listProcessingRowsForOwner?.all?.(sid) || []).length > 0,
-  stopWorker: async (sid) => {
-    const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
-    // Nothing may start the session while it is being stopped: a launch then
-    // would adopt the old process, kind and all.
-    sessionWorkerSupervisor?.markKilled?.(sid);
-    // A stop that fails backs off (the swap service retries it): the killed
-    // marker goes with it, so the session's worker, alive or relaunched,
-    // keeps asking for work meanwhile. The fence still keeps the other
-    // billing's rows from it.
-    const backOff = (error, remainingPids = []) => {
-      sessionWorkerSupervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
-      console.warn(`${runtimeLogPrefix()}BILLING SWAP STOP FAILED session=${sid.slice(0, 8)} ${error}${remainingPids.length ? ` remaining=${remainingPids.join(',')}` : ''} — kill marker cleared, retrying with a backoff`);
-      return { ok: false, error, remainingPids };
-    };
-    let stopped = null;
-    try {
-      await sessionWorkerSupervisor?.cancelPendingStart?.(sid, { wait: true });
-      stopped = await stopSessionWorkerProcesses({
-        sdkSessionId: sid,
-        worker,
-        processInspector: sessionWorkerProcessInspector,
-        isPidAliveImpl: isProcessAlive,
-        killTmuxSessionImpl: killTmuxSession,
-      });
-    } catch (error) {
-      return backOff(String(error?.message || error || 'worker-stop-failed'));
-    }
-    if (!stopped.ok) return backOff(stopped.error || 'worker-survived-stop', stopped.remainingPids || []);
-    sessionWorkerRegistry?.removeWorker?.(sid);
-    sessionWorkerSupervisor?.resetHealth?.(sid, { clearFailureCount: false });
-    return { ok: true };
-  },
-  launchWorker: async (sid) => {
-    sessionWorkerSupervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
-    const launched = await sessionWorkerSupervisor?.ensureWorker?.(sid, { allowProcessReuse: false });
-    return { ok: launched?.ok === true, error: launched?.error || null };
-  },
+  stopWorker: billingSwapWorkerReplacement.stopWorker,
+  launchWorker: billingSwapWorkerReplacement.launchWorker,
   onStuck: ({ conversationId, rowBilling, attempts, reason }) => {
     if (!conversationId) return;
     const at = new Date().toISOString();
@@ -6911,6 +7036,95 @@ const claudeBillingSwapService = createClaudeBillingSwapService({
   logger: console,
 });
 
+// Whether a worker's session is one the relay keeps no worker for: its
+// conversation is archived (or deleted), or there is no conversation behind
+// the session at all. A conversation in use found either way (the runtime
+// session's, or the one naming the session id) keeps its worker: a session
+// is never stopped for good on a missing link alone.
+function isWorkerSessionArchived(sid) {
+  const runtime = stmts.getRuntimeSessionBySdkSessionId?.get?.(sid) || null;
+  const viaRuntime = runtime?.conversation_id ? (stmts.getConvAnyStatus?.get?.(runtime.conversation_id) || null) : null;
+  const viaSessionId = stmts.getConvBySdkSessionId?.get?.(sid) || null;
+  const inUse = (conversation) => Boolean(conversation)
+    && Number(conversation.archived || 0) !== 1
+    && String(conversation.status || '').trim().toLowerCase() !== 'deleted';
+  return !inUse(viaRuntime) && !inUse(viaSessionId);
+}
+
+// The worker recycle (services/worker-recycle-service.mjs): a worker that
+// runs older code than this relay (another build id, or none stated) is
+// replaced between turns by one on this relay's code; an archived session's
+// is stopped for good. Its own replacement instance, so a failed stop says
+// WORKER RECYCLE rather than BILLING SWAP.
+const recycleWorkerReplacement = createRuntimeWorkerReplacement('WORKER RECYCLE');
+const workerRecycleService = createWorkerRecycleService({
+  relayBuildId: BUILD_ID,
+  listWorkers: () => sessionWorkerRegistry.listWorkers(),
+  getWorker: (sid) => sessionWorkerRegistry.getWorker(sid),
+  // A row the session claimed and has not finished: never stopped under it.
+  hasRunningTurn: (sid) => (stmts.listProcessingRowsForOwner?.all?.(sid) || []).length > 0,
+  // From the supervisor's lifecycle, not its firstObservedHeartbeatAt: that
+  // is set only for a worker it launched itself, and the workers that
+  // reconnect after a relay restart (the ones most likely stale) never get it.
+  hasHeartbeated: (sid) => {
+    const lc = sessionWorkerSupervisor.getLifecycleState(sid);
+    return !!lc?.lastHeartbeatAt && !lc?.awaitingHeartbeat;
+  },
+  isArchived: (sid) => isWorkerSessionArchived(sid),
+  // The session's last delivery or turn end, for a worker that does not say
+  // whether its CLI child runs (message-repository getOwnerLastActivityAt).
+  quietSince: (sid) => {
+    const at = Date.parse(stmts.getOwnerLastActivityAt?.get?.(sid)?.last_activity_at || '');
+    return Number.isFinite(at) ? at : null;
+  },
+  stopWorker: recycleWorkerReplacement.stopWorker,
+  launchWorker: recycleWorkerReplacement.launchWorker,
+  fence: recycleWorkerReplacement.fence,
+  // The stop and the launch under the per-session launch mutex the Change
+  // CWD relaunch and the launch routes take (sessions-routes.mjs).
+  runExclusive: (sid, fn) => sessionLaunchMutex.runExclusive(sid, fn),
+  logger: console,
+});
+
+// Once at start, after the restart's own worker reconnects settled: what
+// still runs for an archived conversation is stopped and stays stopped.
+// Through the recycle's replacement, one session at a time; never throws.
+async function sweepArchivedSessionWorkersAtStart() {
+  if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED !== true) return null;
+  if (runtimeShutdownStarted) return null;
+  // The first lookup reads the process list afresh; the rest share it for the
+  // inspector's cache moment, so a long archive costs one read, not one each.
+  let freshRead = true;
+  return sweepArchivedWorkers({
+    listArchivedSessionIds: () => (stmts.listArchivedRuntimeSessionIds?.all?.() || []).map((row) => row?.sdk_session_id),
+    hasLiveProcess: async (sid) => {
+      if (runtimeShutdownStarted) return false;
+      const pid = Number(sessionWorkerRegistry?.getWorker?.(sid)?.pid);
+      if (Number.isInteger(pid) && pid > 0 && isProcessAlive(pid)) return true;
+      const fresh = freshRead;
+      freshRead = false;
+      return Boolean(await sessionWorkerProcessInspector?.findProcessForSessionAsync?.(sid, { fresh }));
+    },
+    isArchived: (sid) => isWorkerSessionArchived(sid),
+    // The process the fence keeps deliveries from during the stop: the
+    // registry's live pid, else the one the process list shows, else none
+    // (every process of the session is fenced then).
+    workerPid: async (sid) => {
+      const pid = Number(sessionWorkerRegistry?.getWorker?.(sid)?.pid);
+      if (Number.isInteger(pid) && pid > 0 && isProcessAlive(pid)) return pid;
+      const found = await sessionWorkerProcessInspector?.findProcessForSessionAsync?.(sid);
+      return Number(found?.processId) || null;
+    },
+    stopWorker: recycleWorkerReplacement.stopWorker,
+    // Guarded as the recycle's archived stop is: a message that unarchives
+    // the session meanwhile must not start a turn on the process being
+    // stopped.
+    fence: workerReplacementFence,
+    runExclusive: (sid, fn) => sessionLaunchMutex.runExclusive(sid, fn),
+    logger: console,
+  });
+}
+
 // The Claude API monthly cap (plan decision 12): `{ reached, error? }`. The
 // one seam the claim check below, the billing switch, an agent's send and a
 // new Claude API chat ask: this month's priced estimate in
@@ -7083,6 +7297,16 @@ const sessionWorkerWebSocketService = createSessionWorkerWebSocketService({
     const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
     if (worker && worker.kind !== kind) sessionWorkerRegistry?.upsertWorker?.({ ...worker, kind });
   },
+  // The build id a worker says it started on: the registry keeps it so a
+  // worker on older code can be told from one on the relay's.
+  onWorkerBuildId: (sessionId, buildId) => {
+    const sid = String(sessionId || '').trim();
+    if (!sid) return;
+    const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
+    if (worker && worker.buildId !== buildId) sessionWorkerRegistry?.upsertWorker?.({ ...worker, buildId });
+  },
+  // Sent in server.hello, so a worker on other code can say so in its log.
+  buildId: BUILD_ID,
   isWorkerProcessAlive: (pid) => isProcessAlive(Number(pid)),
   pathPrefix: remotePath,
   pollIntervalMs: 500,
@@ -7094,6 +7318,10 @@ runtimeTimers.deadWorkerSweep = setInterval(() => {
   });
 }, 30_000);
 if (typeof runtimeTimers.deadWorkerSweep.unref === 'function') runtimeTimers.deadWorkerSweep.unref();
+// Workers on older code are recycled only where the relay launches them;
+// with routing off the service never starts. Its timer is its own (unref'd)
+// and stops with the runtime timers.
+if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true) workerRecycleService.start();
 const tmuxInspectorAccessPolicy = createTmuxInspectorAccessPolicy({
   sessionWorkerRegistry,
   sessionWorkerSupervisor,
@@ -7253,6 +7481,13 @@ let relayPaused = false;
 const relayBridgeOwnerService = createRelayBridgeOwnerService();
 
 const runtimeState = {
+  // The relay's own build id, read once at start; null when package.json
+  // names no version.
+  buildId: BUILD_ID,
+  // The worker recycle's status() (the relay's build id, the stale count,
+  // the recycles in flight and done, who waits on what): a function, so a
+  // route reads it when asked and needs no wiring of its own.
+  workerRecycle: () => workerRecycleService.status(),
   get cliOnline() { return cliOnline; },
   set cliOnline(value) { cliOnline = value; },
   get relayPaused() { return relayPaused; },
@@ -7362,6 +7597,16 @@ const relayAskLinkService = createRelayAskLinkService({
   client: remoteRelayClient,
   logger: console,
 });
+// Paired relays' cards a turn waits on: the badge here stays current, and
+// one relay pushes a card its own relay reached no device for.
+const remoteInboxWatchService = createRemoteInboxWatchService({
+  registry: remoteRelayRegistry,
+  client: remoteRelayClient,
+  push: pushDispatchService,
+  hasSockets: () => io.of('/').sockets.size > 0,
+  emit: (event) => io.emit(event),
+  logger: console,
+});
 const remoteRelayPairing = createRemoteRelayPairing({
   registry: remoteRelayRegistry,
   client: remoteRelayClient,
@@ -7579,6 +7824,8 @@ const sharedRouteDeps = {
   sessionWorkerSupervisor,
   sessionWorkerRegistry,
   sessionWorkerProcessInspector,
+  // /api/pending refuses work to a process the worker recycle is replacing.
+  workerReplacementFence,
   buildContextResponseText,
   readContextFromSessionEvents,
   sessionHistoryRefreshService,
@@ -7647,6 +7894,7 @@ const sharedRouteDeps = {
   usageLimitPauseService,
   claudeBillingSwitchService,
   claudeApiUsageService,
+  tokenUsageService,
   claudeApiClaimCheck,
   onClaudeBillingFence: handleClaudeBillingFence,
   claudeApiPlanUsage,
@@ -7719,11 +7967,13 @@ registerRelayAskRoutes(app, {
   db,
   stmts,
   formatQuestionRow,
+  formatRelayBoardRow,
   relayAskService,
   relayAskLinkService,
   remoteRelayInbound,
   remoteRelayRegistry,
   remoteRelayClient,
+  questionNotice: (questionId) => pushDispatchService.questionNotice(questionId),
 });
 registerRelayRestartRoutes(app, {
   auth,
@@ -7742,6 +7992,9 @@ registerAboutRoutes(app, {
     dbPath: DB_PATH,
     version: RUNNING_VERSION,
     commit: BUILD_COMMIT,
+    // The workers' build ids are counted against this one (the About modal's
+    // "Workers: 14, 3 on an older build").
+    buildId: BUILD_ID,
     installMethod: INSTALL_METHOD,
     startedAt: RELAY_STARTED_AT,
     packageRoot: PACKAGE_ROOT,
@@ -7758,6 +8011,7 @@ registerAboutRoutes(app, {
     isRelayCliOnline: () => runtimeState.cliOnline === true,
     listWorkers: () => sessionWorkerRegistry.listWorkers(),
     getUpdateState: () => buildUpdateStatePayload(),
+    describeBackups: () => describeBackups({ layout: STATE_LAYOUT, serverDir: __dirname, dataDir: DATA_DIR, config, sizes: false }),
   }),
 });
 function markCliOffline(reason = 'offline', { clearOwner = true } = {}) {
@@ -8109,6 +8363,8 @@ relayEventLinkService.start();
 relayScheduleService.start();
 // Asks on paired relays still open there: look for their answers again.
 relayAskLinkService.start();
+// Paired relays' Inboxes, every ~20 s.
+remoteInboxWatchService.start();
 runtimeTimers.pendingWorkerPrime = setInterval(() => {
   void primePendingSessionWorkers('maintenance');
   detachedWorkerWatch.sweep().catch((error) => {
@@ -8374,6 +8630,7 @@ function clearRuntimeTimers() {
     try { clearTimeout(timer); } catch {}
   }
   subagentRecordRetention?.stop();
+  try { workerRecycleService.stop(); } catch {}
   try { taskHealthPoller.stop(); } catch {}
   try { taskOutputRooms.stop(); } catch {}
   runtimeTimers.cliStatus = null;
@@ -8383,6 +8640,7 @@ function clearRuntimeTimers() {
   runtimeTimers.pendingWorkerPrime = null;
   runtimeTimers.sharedViewerPrune = null;
   runtimeTimers.restoredTaskSettle = null;
+  runtimeTimers.archivedWorkerSweep = null;
   try { relayRestartQueue.dispose(); } catch {}
 }
 
@@ -8431,6 +8689,7 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
   try { relayEventLinkService.stop(); } catch {}
   try { relayScheduleService.stop(); } catch {}
   try { relayAskLinkService.stop(); } catch {}
+  try { remoteInboxWatchService.stop(); } catch {}
   tmuxInspectorSocketService.stop();
   stopWorkspaceFileWatcher();
   sshTunnelManager.stop();
@@ -8593,4 +8852,16 @@ httpServer.listen(config.port, listenHost, () => {
   // Deferred and unref'd (a few seconds behind the startup import's own CLI
   // spawn); a relay without an installed Copilot CLI skips it entirely.
   copilotModelDiscoveryService.scheduleStartupRefresh();
+  // Archived conversations' workers: stopped once, 30 s in, so the workers
+  // that reconnect after the restart and the first prime have settled. With
+  // routing off the relay launches no workers, so there is nothing to sweep.
+  if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true) {
+    runtimeTimers.archivedWorkerSweep = setTimeout(() => {
+      runtimeTimers.archivedWorkerSweep = null;
+      sweepArchivedSessionWorkersAtStart().catch((error) => {
+        console.warn(`${runtimeLogPrefix()}WORKER RECYCLE archived sweep failed: ${error?.message || error}`);
+      });
+    }, 30_000);
+    runtimeTimers.archivedWorkerSweep.unref?.();
+  }
 });