@oar-sh/oar 0.9.20 → 0.9.22

package/server/server-runtime.mjs

@@ -36,6 +36,7 @@ import { applySchema } from './db-schema.mjs';
 import { createSessionRepository } from './repositories/session-repository.mjs';
 import { createMessageRepository } from './repositories/message-repository.mjs';
 import { createQuestionRepository } from './repositories/question-repository.mjs';
+import { mapRelayActivityRow, mapRelayThoughtRow, sanitizeActivityText } from './repositories/relay-turn-rows.mjs';
 import { createImageConversationRepository } from './repositories/image-conversation-repository.mjs';
 import { registerSessionsRoutes } from './routes/sessions-routes.mjs';
 import { buildDequeuedRelayMessage, claudeApiRemedyText, dequeuePendingMessageForWorkerLoop, registerMessagesRoutes } from './routes/messages-routes.mjs';
@@ -75,6 +76,9 @@ import { createDeepSeekUsageImportService } from './services/deepseek-usage-impo
 import { createClaudeApiSettingsService } from './services/claude-api-settings-service.mjs';
 import { createClaudeBillingSwitchService, normalizeClaudeBilling } from './services/claude-billing-switch-service.mjs';
 import { createClaudeBillingSwapService } from './services/claude-billing-swap-service.mjs';
+import { createWorkerReplacement, createWorkerReplacementFence } from './services/worker-replacement.mjs';
+import { sessionLaunchMutex } from './services/session-launch-mutex-service.mjs';
+import { createWorkerRecycleService, sweepArchivedWorkers } from './services/worker-recycle-service.mjs';
 import { createClaudeApiUsageService } from './services/claude-api-usage-service.mjs';
 import { createTokenUsageService } from './services/token-usage-service.mjs';
 import { createClaudeApiCapWatch, createClaudeApiPlanUsage, createClaudeApiStopNoteLedger } from './services/plan-usage-claude-api.mjs';
@@ -119,7 +123,8 @@ import { createActiveDeviceTracker } from './services/push-active-device-service
 import { registerPushRoutes } from './routes/push-routes.mjs';
 import { registerAboutRoutes } from './routes/about-routes.mjs';
 import { registerRelayRestartRoutes } from './routes/relay-restart-routes.mjs';
-import { createAboutService, resolveBuildCommit } from './services/about-service.mjs';
+import { createAboutService } from './services/about-service.mjs';
+import { resolveBuildId } from '../shared/build-id.mjs';
 import { createImageOperationService } from './services/image-operation-service.mjs';
 import {
   normalizeDriveAbsolutePath as _normalizeDriveAbsolutePath,
@@ -229,6 +234,9 @@ import {
 import { createRestartPictureCollector, detectRelayHosting } from './services/relay-restart-picture.mjs';
 import { createWorkerScope } from './services/worker-scope.mjs';
 import { createStaleRecoveryClockGuard } from './services/stale-recovery-clock-guard.mjs';
+import { createArmableInterval, createCliPresence } from './services/idle-timers.mjs';
+import { installQueueWakeHook } from './services/queue-wake-hook.mjs';
+import { createPendingWorkerPrimer, createQueueWaker } from './services/queue-waker.mjs';
 import { createStaleTurnRecovery } from './services/stale-turn-recovery.mjs';
 import {
   STALE_SUBAGENT_RUN_BOOT_DELAY_MS,
@@ -266,7 +274,11 @@ import {
   readTaskStallMinutesSetting,
 } from '../shared/task-output-text.mjs';
 import { normalizeRelayThoughtList } from './public/app/relay-thoughts.mjs';
-import { capRelayActivityEntries } from './public/app/activity-replay-state.mjs';
+import {
+  capFinishedActivityEntries,
+  capRelayActivityEntries,
+  inFlightActivityCap,
+} from './public/app/activity-replay-state.mjs';
 import {
   isOpenAIModelId,
   isSafeClaudeModelId,
@@ -298,7 +310,12 @@ const INSTALL_METHOD = fs.existsSync(path.join(PACKAGE_ROOT, '.git')) ? 'git-che
 // When this relay process started, and the commit its code came from (read
 // once: a checkout moved on later still runs the code it started with).
 const RELAY_STARTED_AT = new Date(Date.now() - process.uptime() * 1000).toISOString();
-const BUILD_COMMIT = resolveBuildCommit(PACKAGE_ROOT);
+// The build id (version plus commit) names that code for the session workers:
+// a worker that states another one started on older code. Read in the same
+// pass as the commit so the two cannot disagree.
+const RELAY_BUILD = resolveBuildId(PACKAGE_ROOT);
+const BUILD_COMMIT = RELAY_BUILD.commit;
+const BUILD_ID = RELAY_BUILD.id;
 const PUBLIC_DIR = path.join(__dirname, 'public');
 const INDEX_HTML_PATH = path.join(PUBLIC_DIR, 'index.html');
 const PWA_MANIFEST_PATH = path.join(PUBLIC_DIR, 'manifest.webmanifest');
@@ -1996,18 +2013,36 @@ let runtimeShutdownStarted = false;
 // Set once the database is open (see the retention sweep near the other sweeps).
 let subagentRecordRetention = null;
 const runtimeTimers = {
-  cliStatus: null,
   ownerWatchdog: null,
   staleRecovery: null,
   questionExpiry: null,
-  pendingWorkerPrime: null,
-  sharedViewerPrune: null,
   restoredTaskSettle: null,
   claudeApiCapClock: null,
   memoryLog: null,
   memoryWatch: null,
+  // Once, 30 s after the relay listens: archived sessions' workers stopped.
+  archivedWorkerSweep: null,
 };
 
+// The worker prime and the detached-worker sweep, every 5 s while a pending
+// row waits for its worker (rows in retry backoff included, so the worker is
+// warm when the backoff matures). Armed at startup, by every prime that finds
+// a row and by every queue snapshot with a row pending; a pass that leaves no
+// such row disarms it. Declared up here so a prime early in the boot can arm
+// it; its tick runs only after the boot.
+const pendingWorkerPrimeTimer = createArmableInterval({
+  intervalMs: 5_000,
+  tick: async () => {
+    await Promise.allSettled([
+      primePendingSessionWorkers('maintenance'),
+      detachedWorkerWatch.sweep().catch((error) => {
+        console.warn(`${runtimeLogPrefix()}WORKER DETACHED sweep failed: ${error?.message || error}`);
+      }),
+    ]);
+    return hasPendingWorkerOwnerRows();
+  },
+});
+
 function normalizeRemotePath(value) {
   if (typeof value !== 'string') return '';
   const trimmed = value.trim();
@@ -2850,6 +2885,16 @@ const stmts = {
   ...createQuestionRepository(db),
   ...createImageConversationRepository(db),
 };
+// Every queue write that can make a row deliverable wakes the delivery
+// (services/queue-wake-hook.mjs): installed before any route or startup
+// write. The waker exists once the worker socket service does; a write
+// before that is covered by the startup prime and the service's start.
+// Without the hook (it could not be installed) the queue is polled.
+let queueWaker = null;
+const queueWakeHook = installQueueWakeHook(db, {
+  onWake: (reason) => { queueWaker?.wake(reason, { prime: false }); },
+  log: (line) => console.warn(`[queue-wake] ${line}`),
+});
 const statusEventService = createStatusEventService(db);
 // Public shares and their access log (migration 0016).
 const shareRepository = createShareRepository(db);
@@ -3411,6 +3456,8 @@ const sessionWorkerSupervisor = createSessionWorkerSupervisor({
   diagnosticPlanReference: () => path.join(currentWorkspaceRootPath(), '.cursor', 'plans', 'worker-startup-monitoring-plan.md'),
   log: (message) => console.warn(`${runtimeLogPrefix()}${message}`),
   onLaunchState: (event) => workerLaunchNoteService.handle(event),
+  // A kill's grace ended: the session's rows may go to its next worker.
+  onKillBlockLifted: () => { wakeSessionWorkerQueue('kill-block-lifted'); },
 });
 // Feature flags moved from config.json into app_settings; adopt any legacy
 // `features` key once, then resolve the boot snapshot (defaults < stored rows
@@ -3811,7 +3858,11 @@ function relayShutdownStatePayload() {
 
 function cancelRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
   const result = relayRestartQueue.cancel({ requestedBy });
-  if (result?.cancelled) relayEventService?.noteRestartCancelled();
+  if (result?.cancelled) {
+    relayEventService?.noteRestartCancelled();
+    // The queued restart held deliveries back: they go out now.
+    wakeSessionWorkerQueue('restart-cancelled');
+  }
   return result;
 }
 
@@ -5380,52 +5431,28 @@ function createCompactedConversation(sourceConversationId) {
   };
 }
 
-function sanitizeActivityText(value) {
-  const text = String(value || '').trim();
-  if (!text) return '';
-  return text.slice(0, 200);
-}
-
-function mapRelayActivityRow(row) {
-  const text = sanitizeActivityText(row?.text);
-  if (!text) return null;
-  const subagentRunId = row?.subagent_run_id ? String(row.subagent_run_id).trim() : null;
-  // Structured rows (compaction boundaries) carry their payload alongside the
-  // prose so the transcript can promote them to a break row; junk JSON is
-  // simply dropped back to a prose-only entry.
-  let metadata = null;
-  if (row?.metadata_json) {
-    try {
-      const parsed = JSON.parse(String(row.metadata_json));
-      if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) metadata = parsed;
-    } catch { metadata = null; }
-  }
-  return {
-    text,
-    subagentRunId: subagentRunId || null,
-    ...(metadata ? { metadata } : {}),
-  };
-}
-
-const RELAY_ACTIVITY_ROW_LIMIT = 48;
-
+// Per thread (the main agent, each sub-agent), the newest rows; structured
+// rows are always kept (activity-replay-state.mjs). A finished turn: the main
+// agent's 200, each sub-agent's 50, and a turn whose main agent ran more says
+// how many it left out.
 function relayActivityForResponse(responseMessageId) {
-  return capRelayActivityEntries(
+  return capFinishedActivityEntries(
     stmts.listActivityByResponse
       .all(responseMessageId)
       .map(mapRelayActivityRow)
       .filter(Boolean),
-    RELAY_ACTIVITY_ROW_LIMIT,
   );
 }
 
-function relayActivityForQueueMessage(queueMessageId) {
+// A running turn: as many per thread as the browser shows (`activityCap`, the
+// request's param; 50 when it sends none, 500 at most).
+function relayActivityForQueueMessage(queueMessageId, activityCap = undefined) {
   return capRelayActivityEntries(
     stmts.listActivityByQueueMessage
       .all(queueMessageId)
       .map(mapRelayActivityRow)
       .filter(Boolean),
-    RELAY_ACTIVITY_ROW_LIMIT,
+    inFlightActivityCap(activityCap),
   );
 }
 
@@ -5447,19 +5474,6 @@ function relayStreamEventsForQueueMessage(queueMessageId) {
     .slice(0, 5000);
 }
 
-function mapRelayThoughtRow(row) {
-  const seq = Math.max(0, Math.trunc(Number(row?.seq || 0)));
-  const subagentRunId = row?.subagent_run_id ? String(row.subagent_run_id).trim() : null;
-  return {
-    reasoningId: row?.reasoning_id ? String(row.reasoning_id) : null,
-    seq,
-    text: String(row?.text || ''),
-    done: Number(row?.done || 0) === 1,
-    timestamp: row?.created_at || null,
-    subagentRunId: subagentRunId || null,
-  };
-}
-
 function relayThoughtsForResponse(responseMessageId) {
   const rows = stmts.listThoughtsByResponse?.all(responseMessageId) || [];
   return normalizeRelayThoughtList(rows.map(mapRelayThoughtRow)).slice(0, 5000);
@@ -5643,7 +5657,7 @@ const usageLimitPauseService = createUsageLimitPauseService({
   stmts,
   emit: (event, payload) => io.emit(event, payload),
   // A follow-up resumed by hand is due at once: wake its worker.
-  onQueueChanged: (reason) => { void primePendingSessionWorkers(reason); },
+  onQueueChanged: (reason) => { wakeSessionWorkerQueue(reason); },
   logger: console,
 });
 
@@ -5675,8 +5689,7 @@ const claudeBillingSwitchService = createClaudeBillingSwitchService({
   // What the swap waits for: a turn, a question, a compaction, background tasks.
   getPending: (runtime) => claudeBillingSwapService.pendingFor(runtime?.sdk_session_id || runtime?.conversation_id),
   onQueueChanged: (reason) => {
-    void primePendingSessionWorkers(reason);
-    sessionWorkerWebSocketService?.emitQueueChanged?.(reason);
+    wakeSessionWorkerQueue(reason);
   },
   logger: console,
 });
@@ -5724,8 +5737,7 @@ const restartResumeService = createRestartResumeService({
   // The note is an ordinary pending message: the primer starts a stopped
   // worker, the socket service hands it to a running one.
   wake: () => {
-    void primePendingSessionWorkers('restart-resume');
-    sessionWorkerWebSocketService?.emitQueueChanged?.('restart-resume');
+    wakeSessionWorkerQueue('restart-resume');
   },
   logger: console,
 });
@@ -5755,8 +5767,7 @@ relayEventService = createRelayEventService({
   // The event row is an ordinary pending message: the primer starts a
   // stopped worker, the socket service hands it to a running one.
   wakeWatcher: () => {
-    void primePendingSessionWorkers('relay-event');
-    sessionWorkerWebSocketService?.emitQueueChanged?.('relay-event');
+    wakeSessionWorkerQueue('relay-event');
   },
   relayName: () => remoteRelayRegistry?.selfName?.() || getPwaAppName?.() || '',
   onRemoteChange: (relayId) => { void relayEventLinkService?.fetchNow(relayId); },
@@ -5772,8 +5783,7 @@ const relayScheduleService = createRelayScheduleService({
   emit: (event, payload) => io.emit(event, payload),
   // The scheduled message is an ordinary pending one, like a relay event's.
   wake: () => {
-    void primePendingSessionWorkers('relay-schedule');
-    sessionWorkerWebSocketService?.emitQueueChanged?.('relay-schedule');
+    wakeSessionWorkerQueue('relay-schedule');
   },
   // A firing into another session is a prompt of the conversation that set
   // it: the agents' loop guard counts it (and holds it once it wants the user).
@@ -5836,7 +5846,9 @@ function resolveLiveTurnQueueRow(conversationId) {
   return byId.get(liveId) || rows[0];
 }
 
-function inFlightStateForConversation(conversationId) {
+// `activityCap`: the tool lines per thread the asking page shows (the
+// conversation route passes its `activityCap` param; others get the default).
+function inFlightStateForConversation(conversationId, { activityCap = undefined } = {}) {
   const row = resolveLiveTurnQueueRow(conversationId);
   if (!row) return null;
   const streamEvents = relayStreamEventsForQueueMessage(row.id);
@@ -5848,7 +5860,7 @@ function inFlightStateForConversation(conversationId) {
     mode: normalizeRelayMode(row.relay_mode) || DEFAULT_RELAY_MODE,
     timestamp: row.timestamp || null,
     processingAt: row.processing_at || null,
-    activities: relayActivityForQueueMessage(row.id),
+    activities: relayActivityForQueueMessage(row.id, activityCap),
     thoughts: relayThoughtsForQueueMessage(row.id),
     streamEvents,
     streamDone: !!lastStreamEvent?.done,
@@ -6381,6 +6393,15 @@ const SHARED_VIEWER_MAX_GLOBAL = Number.isFinite(Number(config.sharedPresenceMax
   ? Math.max(SHARED_VIEWER_MAX_PER_CONVERSATION, Math.trunc(Number(config.sharedPresenceMaxGlobal)))
   : 5_000;
 const sharedViewersByConversation = new Map();
+// Armed when the first share viewer arrives, disarmed by the prune that
+// leaves none: a relay nobody watches through a share link has no timer.
+const sharedViewerPruneTimer = createArmableInterval({
+  intervalMs: 5_000,
+  tick: () => {
+    pruneSharedViewerPresence();
+    return sharedViewersByConversation.size > 0;
+  },
+});
 
 function sanitizeSharedViewerId(value) {
   const id = String(value || '').trim().replace(/[^a-zA-Z0-9:_-]+/g, '');
@@ -6521,6 +6542,7 @@ function markSharedViewerPresence({ conversationId, token, viewerId } = {}) {
   }
   const before = viewers.size;
   viewers.set(key, { lastSeenAt: now });
+  sharedViewerPruneTimer.arm();
   const watcherCount = viewers.size;
   if (before !== watcherCount) {
     emitConversationWatcherCount(convId);
@@ -6550,6 +6572,23 @@ async function requestSessionWorkerSocketDelivery({ sessionId, pid, reason = 'wo
       restartOrchestrator: relayRestartOrchestrator?.getState?.() || null,
     };
   }
+  // The process asking is being replaced (the worker recycle passed its holds
+  // for it): no work, as for a killed session; the worker stays ready, and
+  // what waits goes to the new one. Unlike the killed marker above, this
+  // survives the marker's grace and every clearRestartSchedule.
+  if (workerReplacementFence.isHeld(requesterSessionId, pid)) {
+    return {
+      message: null,
+      routing: {
+        enabled: true,
+        requesterSessionId,
+        blockedReason: 'worker-replacement',
+        lifecycle: sessionWorkerSupervisor?.getLifecycleState?.(requesterSessionId) || null,
+        fallbackRestart: null,
+      },
+      restartOrchestrator: relayRestartOrchestrator?.getState?.() || null,
+    };
+  }
   sessionWorkerSupervisor?.noteSessionHeartbeat?.(requesterSessionId);
   if (runtimeState.relayPaused) {
     return { message: null, paused: true, reason: 'relay_paused' };
@@ -6629,6 +6668,9 @@ async function requestSessionWorkerSocketDelivery({ sessionId, pid, reason = 'wo
     workerKind: workerKind || sessionWorkerRegistry?.getWorker?.(requesterSessionId)?.kind || null,
     claudeApiClaimCheck,
     onBillingFence: handleClaudeBillingFence,
+    // The fence check above, asked again before every claim: ensureWorker
+    // and a transient error's backoff both wait.
+    isReplacementFenced: () => workerReplacementFence.isHeld(requesterSessionId, pid),
   });
   if (!dequeueResult?.message) {
     return {
@@ -6777,7 +6819,7 @@ async function recoverUndeliveredSessionWorkerMessage({ pending = null, sessionI
   console.warn(`${runtimeLogPrefix()}WS REQUEUE ${messageId.slice(0, 8)} session=${requesterSessionId ? requesterSessionId.slice(0, 8) : 'unknown'} retry=${retryCount} reason=${String(reason || 'ws-send-failed')}`);
   io.emit('message_status', { messageId, conversationId: row.conversation_id, status: 'pending' });
   io.emit('queue_updated', { recovered: 1, reason: 'worker-ws-send-failed' });
-  sessionWorkerWebSocketService?.emitQueueChanged?.('worker-ws-send-failed');
+  wakeSessionWorkerQueue('worker-ws-send-failed');
   return true;
 }
 
@@ -6867,11 +6909,10 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
   }
   if (recovered > 0) {
     io.emit('queue_updated', { recovered, reason: `worker-died:${reason}` });
-    sessionWorkerWebSocketService?.emitQueueChanged?.(`worker-died:${reason}`);
   }
   sessionWorkerSupervisor?.markError?.(sid, `worker-died:${reason}`);
   // The rows stay owned, so the primer is what respawns their worker.
-  void primePendingSessionWorkers(`worker-died:${reason}`);
+  wakeSessionWorkerQueue(`worker-died:${reason}`);
   return recovered;
 }
 
@@ -6942,48 +6983,50 @@ async function sweepDeadWorkerProcessingRowsOnce() {
 // claude-billing-swap-service.mjs): a Claude chat's next row of the other
 // billing is refused to the worker it has, which is replaced, once idle, by
 // one of the row's kind. The old one is stopped like the kill route does, and
-// the new one launched only once every process of the session is gone.
+// the new one launched only once every process of the session is gone
+// (services/worker-replacement.mjs, which the worker recycle below shares).
+// The two get one instance each so a failed stop's warning names the caller.
+// They share one replacement fence, the one the deliveries check (the socket
+// delivery above and /api/pending): a process being replaced is handed no
+// work. The worker recycle raises it; the billing swap does not, since its
+// own delivery fence already keeps the other billing's rows from the old
+// worker.
+// A lifted fence (the replacement launched, or its stop failed and the old
+// worker serves again) hands out what waited behind it.
+const workerReplacementFence = (() => {
+  const fence = createWorkerReplacementFence();
+  return {
+    ...fence,
+    release(sid, pid) {
+      const released = fence.release(sid, pid);
+      if (released) wakeSessionWorkerQueue('replacement-fence-lifted');
+      return released;
+    },
+  };
+})();
+function createRuntimeWorkerReplacement(tag) {
+  return createWorkerReplacement({
+    registry: sessionWorkerRegistry,
+    supervisor: sessionWorkerSupervisor,
+    processInspector: sessionWorkerProcessInspector,
+    stopSessionWorkerProcesses,
+    isProcessAlive,
+    killTmuxSession,
+    logger: console,
+    logPrefix: () => `${runtimeLogPrefix()}${tag} `,
+    fence: workerReplacementFence,
+  });
+}
+// A stop that fails backs off (the swap service retries it) and clears the
+// killed marker, so the session's worker, alive or relaunched, keeps asking
+// for work meanwhile. The fence still keeps the other billing's rows from it.
+const billingSwapWorkerReplacement = createRuntimeWorkerReplacement('BILLING SWAP');
 const claudeBillingSwapService = createClaudeBillingSwapService({
   getWorker: (sid) => sessionWorkerRegistry?.getWorker?.(sid) || null,
   // A row the session claimed and has not finished: never stopped under it.
   hasRunningTurn: (sid) => (stmts.listProcessingRowsForOwner?.all?.(sid) || []).length > 0,
-  stopWorker: async (sid) => {
-    const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
-    // Nothing may start the session while it is being stopped: a launch then
-    // would adopt the old process, kind and all.
-    sessionWorkerSupervisor?.markKilled?.(sid);
-    // A stop that fails backs off (the swap service retries it): the killed
-    // marker goes with it, so the session's worker, alive or relaunched,
-    // keeps asking for work meanwhile. The fence still keeps the other
-    // billing's rows from it.
-    const backOff = (error, remainingPids = []) => {
-      sessionWorkerSupervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
-      console.warn(`${runtimeLogPrefix()}BILLING SWAP STOP FAILED session=${sid.slice(0, 8)} ${error}${remainingPids.length ? ` remaining=${remainingPids.join(',')}` : ''} — kill marker cleared, retrying with a backoff`);
-      return { ok: false, error, remainingPids };
-    };
-    let stopped = null;
-    try {
-      await sessionWorkerSupervisor?.cancelPendingStart?.(sid, { wait: true });
-      stopped = await stopSessionWorkerProcesses({
-        sdkSessionId: sid,
-        worker,
-        processInspector: sessionWorkerProcessInspector,
-        isPidAliveImpl: isProcessAlive,
-        killTmuxSessionImpl: killTmuxSession,
-      });
-    } catch (error) {
-      return backOff(String(error?.message || error || 'worker-stop-failed'));
-    }
-    if (!stopped.ok) return backOff(stopped.error || 'worker-survived-stop', stopped.remainingPids || []);
-    sessionWorkerRegistry?.removeWorker?.(sid);
-    sessionWorkerSupervisor?.resetHealth?.(sid, { clearFailureCount: false });
-    return { ok: true };
-  },
-  launchWorker: async (sid) => {
-    sessionWorkerSupervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
-    const launched = await sessionWorkerSupervisor?.ensureWorker?.(sid, { allowProcessReuse: false });
-    return { ok: launched?.ok === true, error: launched?.error || null };
-  },
+  stopWorker: billingSwapWorkerReplacement.stopWorker,
+  launchWorker: billingSwapWorkerReplacement.launchWorker,
   onStuck: ({ conversationId, rowBilling, attempts, reason }) => {
     if (!conversationId) return;
     const at = new Date().toISOString();
@@ -7015,6 +7058,95 @@ const claudeBillingSwapService = createClaudeBillingSwapService({
   logger: console,
 });
 
+// Whether a worker's session is one the relay keeps no worker for: its
+// conversation is archived (or deleted), or there is no conversation behind
+// the session at all. A conversation in use found either way (the runtime
+// session's, or the one naming the session id) keeps its worker: a session
+// is never stopped for good on a missing link alone.
+function isWorkerSessionArchived(sid) {
+  const runtime = stmts.getRuntimeSessionBySdkSessionId?.get?.(sid) || null;
+  const viaRuntime = runtime?.conversation_id ? (stmts.getConvAnyStatus?.get?.(runtime.conversation_id) || null) : null;
+  const viaSessionId = stmts.getConvBySdkSessionId?.get?.(sid) || null;
+  const inUse = (conversation) => Boolean(conversation)
+    && Number(conversation.archived || 0) !== 1
+    && String(conversation.status || '').trim().toLowerCase() !== 'deleted';
+  return !inUse(viaRuntime) && !inUse(viaSessionId);
+}
+
+// The worker recycle (services/worker-recycle-service.mjs): a worker that
+// runs older code than this relay (another build id, or none stated) is
+// replaced between turns by one on this relay's code; an archived session's
+// is stopped for good. Its own replacement instance, so a failed stop says
+// WORKER RECYCLE rather than BILLING SWAP.
+const recycleWorkerReplacement = createRuntimeWorkerReplacement('WORKER RECYCLE');
+const workerRecycleService = createWorkerRecycleService({
+  relayBuildId: BUILD_ID,
+  listWorkers: () => sessionWorkerRegistry.listWorkers(),
+  getWorker: (sid) => sessionWorkerRegistry.getWorker(sid),
+  // A row the session claimed and has not finished: never stopped under it.
+  hasRunningTurn: (sid) => (stmts.listProcessingRowsForOwner?.all?.(sid) || []).length > 0,
+  // From the supervisor's lifecycle, not its firstObservedHeartbeatAt: that
+  // is set only for a worker it launched itself, and the workers that
+  // reconnect after a relay restart (the ones most likely stale) never get it.
+  hasHeartbeated: (sid) => {
+    const lc = sessionWorkerSupervisor.getLifecycleState(sid);
+    return !!lc?.lastHeartbeatAt && !lc?.awaitingHeartbeat;
+  },
+  isArchived: (sid) => isWorkerSessionArchived(sid),
+  // The session's last delivery or turn end, for a worker that does not say
+  // whether its CLI child runs (message-repository getOwnerLastActivityAt).
+  quietSince: (sid) => {
+    const at = Date.parse(stmts.getOwnerLastActivityAt?.get?.(sid)?.last_activity_at || '');
+    return Number.isFinite(at) ? at : null;
+  },
+  stopWorker: recycleWorkerReplacement.stopWorker,
+  launchWorker: recycleWorkerReplacement.launchWorker,
+  fence: recycleWorkerReplacement.fence,
+  // The stop and the launch under the per-session launch mutex the Change
+  // CWD relaunch and the launch routes take (sessions-routes.mjs).
+  runExclusive: (sid, fn) => sessionLaunchMutex.runExclusive(sid, fn),
+  logger: console,
+});
+
+// Once at start, after the restart's own worker reconnects settled: what
+// still runs for an archived conversation is stopped and stays stopped.
+// Through the recycle's replacement, one session at a time; never throws.
+async function sweepArchivedSessionWorkersAtStart() {
+  if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED !== true) return null;
+  if (runtimeShutdownStarted) return null;
+  // The first lookup reads the process list afresh; the rest share it for the
+  // inspector's cache moment, so a long archive costs one read, not one each.
+  let freshRead = true;
+  return sweepArchivedWorkers({
+    listArchivedSessionIds: () => (stmts.listArchivedRuntimeSessionIds?.all?.() || []).map((row) => row?.sdk_session_id),
+    hasLiveProcess: async (sid) => {
+      if (runtimeShutdownStarted) return false;
+      const pid = Number(sessionWorkerRegistry?.getWorker?.(sid)?.pid);
+      if (Number.isInteger(pid) && pid > 0 && isProcessAlive(pid)) return true;
+      const fresh = freshRead;
+      freshRead = false;
+      return Boolean(await sessionWorkerProcessInspector?.findProcessForSessionAsync?.(sid, { fresh }));
+    },
+    isArchived: (sid) => isWorkerSessionArchived(sid),
+    // The process the fence keeps deliveries from during the stop: the
+    // registry's live pid, else the one the process list shows, else none
+    // (every process of the session is fenced then).
+    workerPid: async (sid) => {
+      const pid = Number(sessionWorkerRegistry?.getWorker?.(sid)?.pid);
+      if (Number.isInteger(pid) && pid > 0 && isProcessAlive(pid)) return pid;
+      const found = await sessionWorkerProcessInspector?.findProcessForSessionAsync?.(sid);
+      return Number(found?.processId) || null;
+    },
+    stopWorker: recycleWorkerReplacement.stopWorker,
+    // Guarded as the recycle's archived stop is: a message that unarchives
+    // the session meanwhile must not start a turn on the process being
+    // stopped.
+    fence: workerReplacementFence,
+    runExclusive: (sid, fn) => sessionLaunchMutex.runExclusive(sid, fn),
+    logger: console,
+  });
+}
+
 // The Claude API monthly cap (plan decision 12): `{ reached, error? }`. The
 // one seam the claim check below, the billing switch, an agent's send and a
 // new Claude API chat ask: this month's priced estimate in
@@ -7178,6 +7310,21 @@ const sessionWorkerWebSocketService = createSessionWorkerWebSocketService({
   },
   onDeliverySendFailed: recoverUndeliveredSessionWorkerMessage,
   onWorkerSocketClosed: handleWorkerSocketClosed,
+  // The CLI presence window is wider while a worker socket is open.
+  onSocketCountChange: (count) => cliPresence.noteWorkerSocketCount(count),
+  // A row pending (in backoff too) keeps the worker prime armed, and the
+  // poll primes now and then while rows wait.
+  onQueueSnapshot: (snapshot) => {
+    if (snapshot.pendingCount > 0) queueWaker?.primeFromBackstop('backstop');
+  },
+  // A retry time or a usage-limit hold coming due wakes the queue on time.
+  nextDueAtMs: () => {
+    if (runtimeShutdownStarted) return null;
+    const due = stmts.nextPendingAttemptAt?.get?.(new Date().toISOString())?.due;
+    const dueMs = due ? Date.parse(due) : NaN;
+    return Number.isFinite(dueMs) ? dueMs : null;
+  },
+  onDue: (reason) => { wakeSessionWorkerQueue(reason); },
   requestWork: requestSessionWorkerSocketDelivery,
   // What a worker says it is ("claude", "claude-api", ...): the registry
   // keeps it for the billing fence and for what a response ran on.
@@ -7187,17 +7334,45 @@ const sessionWorkerWebSocketService = createSessionWorkerWebSocketService({
     const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
     if (worker && worker.kind !== kind) sessionWorkerRegistry?.upsertWorker?.({ ...worker, kind });
   },
+  // The build id a worker says it started on: the registry keeps it so a
+  // worker on older code can be told from one on the relay's.
+  onWorkerBuildId: (sessionId, buildId) => {
+    const sid = String(sessionId || '').trim();
+    if (!sid) return;
+    const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
+    if (worker && worker.buildId !== buildId) sessionWorkerRegistry?.upsertWorker?.({ ...worker, buildId });
+  },
+  // Sent in server.hello, so a worker on other code can say so in its log.
+  buildId: BUILD_ID,
   isWorkerProcessAlive: (pid) => isProcessAlive(Number(pid)),
   pathPrefix: remotePath,
-  pollIntervalMs: 500,
+  // A backstop under the queue's own wakes; the poll every 500 ms it was
+  // when those cannot be installed.
+  pollIntervalMs: queueWakeHook.installed ? 10_000 : 500,
   logger: console,
 });
+let queueWakeStarted = false;
+queueWaker = createQueueWaker({
+  prime: (reason) => primePendingSessionWorkers(reason),
+  notify: (reason) => {
+    sessionWorkerWebSocketService.notifyQueueChanged(reason).catch((error) => {
+      console.warn(`${runtimeLogPrefix()}QUEUE WAKE failed reason=${reason} err=${error?.message || error}`);
+    });
+  },
+  armPrime: () => pendingWorkerPrimeTimer.arm(),
+  isReady: () => queueWakeStarted && !runtimeShutdownStarted,
+  warn: (line) => console.warn(`${runtimeLogPrefix()}${line}`),
+});
 runtimeTimers.deadWorkerSweep = setInterval(() => {
   sweepDeadWorkerProcessingRows().catch((error) => {
     console.warn(`${runtimeLogPrefix()}WORKER SWEEP failed: ${error?.message || error}`);
   });
 }, 30_000);
 if (typeof runtimeTimers.deadWorkerSweep.unref === 'function') runtimeTimers.deadWorkerSweep.unref();
+// Workers on older code are recycled only where the relay launches them;
+// with routing off the service never starts. Its timer is its own (unref'd)
+// and stops with the runtime timers.
+if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true) workerRecycleService.start();
 const tmuxInspectorAccessPolicy = createTmuxInspectorAccessPolicy({
   sessionWorkerRegistry,
   sessionWorkerSupervisor,
@@ -7214,40 +7389,39 @@ const tmuxInspectorSocketService = createTmuxInspectorSocketService({
   logger: console,
 });
 
-async function primePendingSessionWorkers(reason = 'pending-worker-prime') {
-  if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED !== true) return 0;
-  if (runtimeShutdownStarted) return 0;
-  if (typeof sessionWorkerSupervisor?.ensureWorker !== 'function') return 0;
-  if (typeof stmts.listPendingWorkerOwnerSessionIds?.all !== 'function') return 0;
-
-  // No next_attempt_at filter here: rows sitting in retry backoff still need
-  // their worker warm, otherwise the 2s relay poll wins the race the moment
-  // the backoff matures and the turn executes on the wrong provider.
-  const rows = stmts.listPendingWorkerOwnerSessionIds.all(25);
-  let requested = 0;
-  for (const row of rows) {
-    const sdkSessionId = String(row?.sdk_session_id || '').trim();
-    if (!sdkSessionId) continue;
-    if (sessionWorkerSupervisor?.isKillBlocked?.(sdkSessionId) === true) continue;
-    try {
-      // Per session: a throw here costs this session its launch, never the
-      // rest of the pass.
-      if (failClaudeApiRowsWithoutLaunch(sdkSessionId)) continue;
-    } catch (error) {
-      console.warn(`${runtimeLogPrefix()}WORKER PRIME skipped session=${sdkSessionId.slice(0, 8)} err=${error?.message || error}`);
-      continue;
-    }
-    requested += 1;
-    try {
-      const result = await sessionWorkerSupervisor.ensureWorker(sdkSessionId);
-      if (!result?.ok) {
-        console.warn(`${runtimeLogPrefix()}WORKER PRIME blocked session=${sdkSessionId.slice(0, 8)} reason=${String(result?.error || reason || 'pending-worker-prime')}`);
-      }
-    } catch (error) {
-      console.warn(`${runtimeLogPrefix()}WORKER PRIME failed session=${sdkSessionId.slice(0, 8)} reason=${String(reason || 'pending-worker-prime')} err=${error?.message || error}`);
-    }
-  }
-  return requested;
+// No next_attempt_at filter in the listing: rows sitting in retry backoff
+// still need their worker warm, otherwise the poll wins the race the moment
+// the backoff matures and the turn executes on the wrong provider.
+const pendingWorkerPrimer = createPendingWorkerPrimer({
+  isEnabled: () => featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true
+    && !runtimeShutdownStarted
+    && typeof sessionWorkerSupervisor?.ensureWorker === 'function'
+    && typeof stmts.listPendingWorkerOwnerSessionIds?.all === 'function',
+  listPendingOwnerSessionIds: (limit) => stmts.listPendingWorkerOwnerSessionIds.all(limit),
+  isKillBlocked: (sid) => sessionWorkerSupervisor?.isKillBlocked?.(sid) === true,
+  failRowsWithoutLaunch: (sid) => failClaudeApiRowsWithoutLaunch(sid),
+  ensureWorker: (sid) => sessionWorkerSupervisor.ensureWorker(sid),
+  // Rows waiting for their worker: the maintenance pass retries until none is.
+  onRowsWaiting: () => pendingWorkerPrimeTimer.arm(),
+  warn: (line) => console.warn(`${runtimeLogPrefix()}${line}`),
+});
+
+function primePendingSessionWorkers(reason = 'pending-worker-prime') {
+  return pendingWorkerPrimer(reason);
+}
+
+/**
+ * The queue may hold work for a worker now (a row written, a gate lifted, a
+ * retry come due): hand rows to the ready workers, prime the sessions with
+ * pending rows, keep the maintenance pass armed.
+ */
+function wakeSessionWorkerQueue(reason = 'queue-wake') {
+  return queueWaker?.wake(reason) === true;
+}
+
+function hasPendingWorkerOwnerRows() {
+  if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED !== true || runtimeShutdownStarted) return false;
+  return (stmts.listPendingWorkerOwnerSessionIds?.all?.(1) || []).length > 0;
 }
 
 // Guard first, so a tunnelled request to a worker path is rejected before any
@@ -7351,16 +7525,35 @@ app.get(['/share-preview/:conversationId', '/share-preview/:conversationId/'], (
 app.use(express.static(PUBLIC_DIR, { etag: false, setHeaders: (res) => res.setHeader('Cache-Control', 'no-store') }));
 
 // ─── CLI Status Tracking ──────────────────────────────────────────────────────
-let cliLastSeen = null;
+// When the workers were last heard from, without a polling timer: the window
+// is 45 s while a worker socket is open (an idle worker beats every 30 s) and
+// 10 s once none is (services/idle-timers.mjs).
+const cliPresence = createCliPresence({
+  onExpired: () => {
+    if (cliOnline) markCliOffline('heartbeat-timeout');
+  },
+});
 let cliOnline   = false;
 let relayPaused = false;
 const relayBridgeOwnerService = createRelayBridgeOwnerService();
 
 const runtimeState = {
+  // The relay's own build id, read once at start; null when package.json
+  // names no version.
+  buildId: BUILD_ID,
+  // The worker recycle's status() (the relay's build id, the stale count,
+  // the recycles in flight and done, who waits on what): a function, so a
+  // route reads it when asked and needs no wiring of its own.
+  workerRecycle: () => workerRecycleService.status(),
   get cliOnline() { return cliOnline; },
   set cliOnline(value) { cliOnline = value; },
   get relayPaused() { return relayPaused; },
-  set relayPaused(value) { relayPaused = value; },
+  set relayPaused(value) {
+    const wasPaused = relayPaused;
+    relayPaused = value;
+    // Resumed: what was queued while paused goes out now.
+    if (wasPaused && !relayPaused) wakeSessionWorkerQueue('relay-resumed');
+  },
   get ttyConsoleActive() { return Boolean(ttyConsoleRuntime?.tty); },
   get relayShutdown() { return relayRestartQueue.getState(); },
   // A shutdown or restart waits for the processing turns: no pending row
@@ -7693,6 +7886,8 @@ const sharedRouteDeps = {
   sessionWorkerSupervisor,
   sessionWorkerRegistry,
   sessionWorkerProcessInspector,
+  // /api/pending refuses work to a process the worker recycle is replacing.
+  workerReplacementFence,
   buildContextResponseText,
   readContextFromSessionEvents,
   sessionHistoryRefreshService,
@@ -7859,6 +8054,9 @@ registerAboutRoutes(app, {
     dbPath: DB_PATH,
     version: RUNNING_VERSION,
     commit: BUILD_COMMIT,
+    // The workers' build ids are counted against this one (the About modal's
+    // "Workers: 14, 3 on an older build").
+    buildId: BUILD_ID,
     installMethod: INSTALL_METHOD,
     startedAt: RELAY_STARTED_AT,
     packageRoot: PACKAGE_ROOT,
@@ -7879,7 +8077,7 @@ registerAboutRoutes(app, {
   }),
 });
 function markCliOffline(reason = 'offline', { clearOwner = true } = {}) {
-  cliLastSeen = null;
+  cliPresence.clear();
   const wasOnline = cliOnline;
   cliOnline = false;
   if (clearOwner) {
@@ -7893,20 +8091,6 @@ function markCliOffline(reason = 'offline', { clearOwner = true } = {}) {
   }
 }
 
-function checkCliStatus() {
-  const wasOnline = cliOnline;
-  cliOnline = cliLastSeen !== null && (Date.now() - cliLastSeen) < 10_000;
-  if (wasOnline !== cliOnline) {
-    if (!cliOnline) {
-      markCliOffline('heartbeat-timeout');
-      return;
-    }
-    console.log(`${runtimeLogPrefix()}CLI ONLINE`);
-    io.emit('cli_status', { online: true });
-  }
-}
-runtimeTimers.cliStatus = setInterval(checkCliStatus, 2000);
-
 if (managedOwnerPid) {
   runtimeTimers.ownerWatchdog = setInterval(() => {
     if (isProcessAlive(managedOwnerPid)) return;
@@ -7936,6 +8120,10 @@ const staleRecoveryClockGuard = createStaleRecoveryClockGuard({
 });
 function recoverStaleMessages() {
   if (staleRecoveryClockGuard.tick().skip) return;
+  // The tick itself stays: the guard reads the gaps between ticks, and a timer
+  // that slept while nothing ran would look like a clock jump. Only the work
+  // is skipped while no row is processing and no restart outcome is open.
+  if (!stmts.hasProcessingRow.get() && !restartResumeService.pending().length) return;
   const staleWindowMs = cliOnline
     ? processingTimeoutMs
     : Math.min(processingTimeoutMs, OFFLINE_STALE_RECOVER_MS);
@@ -7975,8 +8163,6 @@ function expirePendingQuestions() {
 }
 runtimeTimers.questionExpiry = setInterval(expirePendingQuestions, 10_000);
 expirePendingQuestions();
-runtimeTimers.sharedViewerPrune = setInterval(pruneSharedViewerPresence, 5_000);
-if (typeof runtimeTimers.sharedViewerPrune.unref === 'function') runtimeTimers.sharedViewerPrune.unref();
 // Memory in the status log: now, every 10 minutes, and once per crossing of
 // 80 % of the heap limit (relay-heap-guard.mjs).
 {
@@ -8208,7 +8394,7 @@ function emitToClientsExceptSessionId(event, payload, sessionId) {
 // ─── CLI Routes ───────────────────────────────────────────────────────────────
 
 function touchCli() {
-  cliLastSeen = Date.now();
+  cliPresence.touch();
   relayRestartOrchestrator.noteCliOnline();
   if (!cliOnline) {
     cliOnline = true;
@@ -8217,6 +8403,7 @@ function touchCli() {
   }
 }
 sessionWorkerWebSocketService.start();
+queueWakeStarted = true;
 void primePendingSessionWorkers('startup');
 // Undelivered events get their timers back; stalled, all_idle and the
 // heartbeat are swept from here on.
@@ -8229,13 +8416,7 @@ relayScheduleService.start();
 relayAskLinkService.start();
 // Paired relays' Inboxes, every ~20 s.
 remoteInboxWatchService.start();
-runtimeTimers.pendingWorkerPrime = setInterval(() => {
-  void primePendingSessionWorkers('maintenance');
-  detachedWorkerWatch.sweep().catch((error) => {
-    console.warn(`${runtimeLogPrefix()}WORKER DETACHED sweep failed: ${error?.message || error}`);
-  });
-}, 5_000);
-if (typeof runtimeTimers.pendingWorkerPrime.unref === 'function') runtimeTimers.pendingWorkerPrime.unref();
+pendingWorkerPrimeTimer.arm();
 
 // POST /api/heartbeat — CLI sends a ping every poll interval
 
@@ -8364,6 +8545,7 @@ const sshTunnelManager = createSshTunnelManager({
   logger: console,
   runtimeShutdownRef: () => runtimeShutdownStarted,
   configBaseDir: __dirname,
+  onBlockingLifted: () => { wakeSessionWorkerQueue('ssh-tunnel-connected'); },
 });
 const tunnelState = sshTunnelManager.state;
 runtimeState.tunnelState = tunnelState;
@@ -8377,6 +8559,7 @@ const cloudflaredTunnelManager = createCloudflaredTunnelManager({
   logger: console,
   runtimeShutdownRef: () => runtimeShutdownStarted,
   configBaseDir: __dirname,
+  onBlockingLifted: () => { wakeSessionWorkerQueue('cloudflared-tunnel-connected'); },
 });
 runtimeState.cloudflaredTunnelState = cloudflaredTunnelManager.state;
 
@@ -8494,16 +8677,18 @@ function clearRuntimeTimers() {
     try { clearTimeout(timer); } catch {}
   }
   subagentRecordRetention?.stop();
+  try { workerRecycleService.stop(); } catch {}
   try { taskHealthPoller.stop(); } catch {}
   try { taskOutputRooms.stop(); } catch {}
-  runtimeTimers.cliStatus = null;
   runtimeTimers.ownerWatchdog = null;
   runtimeTimers.staleRecovery = null;
   runtimeTimers.questionExpiry = null;
-  runtimeTimers.pendingWorkerPrime = null;
-  runtimeTimers.sharedViewerPrune = null;
+  pendingWorkerPrimeTimer.stop();
+  sharedViewerPruneTimer.stop();
   runtimeTimers.restoredTaskSettle = null;
+  runtimeTimers.archivedWorkerSweep = null;
   try { relayRestartQueue.dispose(); } catch {}
+  try { cliPresence.clear(); } catch {}
 }
 
 let runtimeShutdownPromise = null;
@@ -8714,4 +8899,16 @@ httpServer.listen(config.port, listenHost, () => {
   // Deferred and unref'd (a few seconds behind the startup import's own CLI
   // spawn); a relay without an installed Copilot CLI skips it entirely.
   copilotModelDiscoveryService.scheduleStartupRefresh();
+  // Archived conversations' workers: stopped once, 30 s in, so the workers
+  // that reconnect after the restart and the first prime have settled. With
+  // routing off the relay launches no workers, so there is nothing to sweep.
+  if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true) {
+    runtimeTimers.archivedWorkerSweep = setTimeout(() => {
+      runtimeTimers.archivedWorkerSweep = null;
+      sweepArchivedSessionWorkersAtStart().catch((error) => {
+        console.warn(`${runtimeLogPrefix()}WORKER RECYCLE archived sweep failed: ${error?.message || error}`);
+      });
+    }, 30_000);
+    runtimeTimers.archivedWorkerSweep.unref?.();
+  }
 });