@oar-sh/oar 0.9.18 → 0.9.20

package/server/relay-heap-guard.mjs

@@ -0,0 +1,351 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import v8 from 'v8';
+
+import { resolveDataDir } from './services/data-dir-resolution.mjs';
+
+// The config.json key an operator sets to pin the runtime's heap (MB). A key
+// and not an env var: a shell's NODE_OPTIONS is lost on the next relaunch by
+// systemd, an autostart launcher or the CLI extension; config.json is read by
+// every one of them.
+export const HEAP_LIMIT_CONFIG_KEY = 'heapLimitMb';
+export const MIN_HEAP_LIMIT_MB = 512;
+const DEFAULT_FLOOR_MB = 2048;
+const DEFAULT_CAP_MB = 8192;
+const DEFAULT_RAM_SHARE = 0.6;
+// v8's heap_size_limit is old space plus the young generation, while
+// --max-old-space-size sets old space only: compare like with like. Node 24
+// reserves 3 × 64 MB semi-spaces (old space 2048 → heap_size_limit 2240);
+// older Nodes reserve less, which only sets this floor a little low.
+const YOUNG_GENERATION_ALLOWANCE_MB = 192;
+const MB = 1024 * 1024;
+
+const HEAP_FLAG = /^--max[-_]old[-_]space[-_]size(?:=(.*))?$/;
+// Node 24.11+: the old space as a percentage of the memory Node sees.
+const HEAP_PERCENTAGE_FLAG = /^--max[-_]old[-_]space[-_]size[-_]percentage(?:=(.*))?$/;
+const SNAPSHOT_FLAG = /^--heapsnapshot[-_]near[-_]heap[-_]limit(?:=|$)/;
+const DIAGNOSTIC_DIR_FLAG = /^--diagnostic[-_]dir(?:=(.*))?$/;
+
+/**
+ * NODE_OPTIONS as Node splits it (ParseNodeOptionsEnvVar): on spaces, except
+ * inside double quotes, which are taken off; inside them a backslash escapes
+ * the next character. A single quote is an ordinary character.
+ */
+export function splitNodeOptions(value) {
+  const args = [];
+  let current = '';
+  let inString = false;
+  const text = String(value || '');
+  for (let index = 0; index < text.length; index += 1) {
+    let char = text[index];
+    if (char === '\\' && inString && index + 1 < text.length) {
+      index += 1;
+      char = text[index];
+    } else if (char === ' ' && !inString) {
+      if (current) args.push(current);
+      current = '';
+      continue;
+    } else if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+    current += char;
+  }
+  if (current) args.push(current);
+  return args;
+}
+
+// The value the last `flag` in an arg list sets (`=V` or V as the next arg),
+// as in Node; undefined when none sets it.
+function lastFlagValue(args, flag) {
+  let found;
+  for (let index = 0; index < args.length; index += 1) {
+    const match = flag.exec(String(args[index]));
+    if (match) found = match[1] !== undefined ? match[1] : String(args[index + 1] ?? '');
+  }
+  return found;
+}
+
+function positiveNumber(value) {
+  const number = Number.parseFloat(String(value ?? ''));
+  return Number.isFinite(number) && number > 0 ? number : null;
+}
+
+/**
+ * The heap limit the operator set in execArgv or NODE_OPTIONS: `{ mb }` for
+ * --max-old-space-size, `{ percentage }` for --max-old-space-size-percentage,
+ * with its `source`; null when neither sets one. As in Node, a percentage
+ * wins over a size wherever each is set, and execArgv over NODE_OPTIONS.
+ */
+export function operatorHeapLimit({ execArgv = [], nodeOptions = '' } = {}) {
+  const sources = [['execArgv', Array.from(execArgv || [])], ['NODE_OPTIONS', splitNodeOptions(nodeOptions)]];
+  for (const [source, args] of sources) {
+    const percentage = positiveNumber(lastFlagValue(args, HEAP_PERCENTAGE_FLAG));
+    if (percentage) return { percentage, source };
+  }
+  for (const [source, args] of sources) {
+    const mb = positiveNumber(lastFlagValue(args, HEAP_FLAG));
+    if (mb) return { mb: Math.floor(mb), source };
+  }
+  return null;
+}
+
+/**
+ * The memory this host gives the relay: a container's or cgroup's limit
+ * (process.constrainedMemory) when it is set and under the RAM, else the RAM.
+ * Without a limit it reports 0, nothing, or a huge number: ignored.
+ */
+export function availableMemory({ totalMem = os.totalmem(), constrainedMem = process.constrainedMemory?.() } = {}) {
+  const total = Number(totalMem) || 0;
+  const constrained = Number(constrainedMem);
+  if (Number.isFinite(constrained) && constrained > 0 && constrained < total) {
+    return { bytes: constrained, label: 'memory limit' };
+  }
+  return { bytes: total, label: 'RAM' };
+}
+
+/**
+ * 60 % of RAM in whole 64 MB steps, between 2048 and 8192 MB, and never under
+ * Node's own default (`nodeDefaultMb`, old space only). 60 % leaves room for
+ * the CLI workers and for writing a heap snapshot: a heap that fills RAM ends
+ * in the kernel's OOM killer, which leaves neither a snapshot nor a log line.
+ */
+export function defaultHeapLimitMb({ totalMem = os.totalmem(), nodeDefaultMb = 0 } = {}) {
+  const share = Math.floor((Number(totalMem) || 0) * DEFAULT_RAM_SHARE / MB / 64) * 64;
+  const clamped = Math.min(DEFAULT_CAP_MB, Math.max(DEFAULT_FLOOR_MB, share));
+  return Math.max(clamped, Math.floor(Number(nodeDefaultMb) || 0));
+}
+
+/** Node's default old-space limit, from this process's v8 heap limit. */
+export function nodeDefaultOldSpaceMb(heapSizeLimit = v8.getHeapStatistics().heap_size_limit) {
+  return Math.max(0, Math.floor(Number(heapSizeLimit) / MB) - YOUNG_GENERATION_ALLOWANCE_MB);
+}
+
+function resolveConfigPath(env, serverDir) {
+  const fromEnv = String(env?.COPILOT_WEB_RELAY_CONFIG || '').trim();
+  return fromEnv ? path.resolve(fromEnv) : path.join(String(serverDir || ''), 'config.json');
+}
+
+/**
+ * config.json's heapLimitMb: `{ mb }`, `{ invalid }` for a value that is not a
+ * whole number of at least MIN_HEAP_LIMIT_MB, or null when it sets none. An
+ * unreadable config sets none: the runtime reports config problems itself.
+ */
+export function configuredHeapLimit({ configPath, fsImpl = fs } = {}) {
+  let parsed = null;
+  try { parsed = JSON.parse(fsImpl.readFileSync(configPath, 'utf8')); } catch { return null; }
+  const value = parsed?.[HEAP_LIMIT_CONFIG_KEY];
+  if (value === undefined || value === null || value === '') return null;
+  const mb = Number(value);
+  return Number.isInteger(mb) && mb >= MIN_HEAP_LIMIT_MB ? { mb } : { invalid: value };
+}
+
+// Free bytes on the disk of `dir`, or null when unknown. Also used by the
+// database backup's space check (db-backup-service.mjs), with diskBytes.
+export function freeBytes(dir, fsImpl) {
+  try {
+    if (typeof fsImpl.statfsSync !== 'function') return null;
+    const stats = fsImpl.statfsSync(dir);
+    return Number(stats.bavail) * Number(stats.bsize);
+  } catch {
+    return null;
+  }
+}
+
+// The size of the disk of `dir`, or null when unknown.
+export function diskBytes(dir, fsImpl) {
+  try {
+    if (typeof fsImpl.statfsSync !== 'function') return null;
+    const stats = fsImpl.statfsSync(dir);
+    return Number(stats.blocks) * Number(stats.bsize);
+  } catch {
+    return null;
+  }
+}
+
+/** A path for the log: `~` for the home folder, so a log line names no user. */
+export function displayPath(target, homeDir = os.homedir()) {
+  const resolved = path.resolve(String(target || ''));
+  const home = String(homeDir || '').trim() ? path.resolve(String(homeDir)) : '';
+  if (!home) return resolved;
+  const relative = path.relative(home, resolved);
+  if (relative === '') return '~';
+  if (relative.startsWith('..') || path.isAbsolute(relative)) return resolved;
+  return `~${path.sep}${relative}`;
+}
+
+function countSnapshots(dir, fsImpl) {
+  try {
+    return fsImpl.readdirSync(dir).filter((name) => name.endsWith('.heapsnapshot')).length;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * The Node flags every relay runtime starts with: a known heap limit, and one
+ * heap snapshot near it, written to `<data dir>/diagnostics`.
+ *
+ * The heap limit comes from execArgv or NODE_OPTIONS when the operator set
+ * one there (nothing added), else config.json's heapLimitMb, else
+ * defaultHeapLimitMb; neither of the last two goes above the memory the host
+ * gives the relay (availableMemory). The diagnostics folder is made 0700 (a
+ * snapshot holds the auth token and provider keys); a --diagnostic-dir or
+ * --heapsnapshot-near-heap-limit the operator set is kept as it is. When the
+ * folder cannot be made, or its drive has less than twice the heap limit
+ * free (the database lives there too), no snapshot is asked for. Says what
+ * it chose on `logger`, with the home folder as `~`.
+ */
+export function relayRuntimeNodeFlags({
+  env = process.env,
+  execArgv = process.execArgv,
+  serverDir,
+  totalMem = os.totalmem(),
+  constrainedMem = process.constrainedMemory?.(),
+  heapSizeLimit = v8.getHeapStatistics().heap_size_limit,
+  fsImpl = fs,
+  platform = process.platform,
+  homeDir = os.homedir(),
+  logger = console,
+} = {}) {
+  const log = (line) => logger?.log?.(line);
+  const operatorArgs = [...Array.from(execArgv || []), ...splitNodeOptions(env?.NODE_OPTIONS)];
+  const memory = availableMemory({ totalMem, constrainedMem });
+  const memoryMb = Math.floor(memory.bytes / MB / 64) * 64;
+  const memoryText = `${(memory.bytes / 1024 ** 3).toFixed(1)} GB ${memory.label}`;
+  const flags = [];
+
+  let heapLimitMb;
+  let source;
+  const operator = operatorHeapLimit({ execArgv, nodeOptions: env?.NODE_OPTIONS });
+  if (operator?.mb) {
+    heapLimitMb = operator.mb;
+    source = `from ${operator.source}`;
+  } else if (operator?.percentage) {
+    heapLimitMb = Math.floor(memory.bytes * operator.percentage / 100 / MB);
+    source = `${operator.percentage}% of ${memoryText}, from ${operator.source}`;
+  } else {
+    const configured = configuredHeapLimit({ configPath: resolveConfigPath(env, serverDir), fsImpl });
+    if (configured?.invalid !== undefined) {
+      log(`[relay] config ${HEAP_LIMIT_CONFIG_KEY}=${JSON.stringify(configured.invalid)} ignored: expected a whole number of MB, at least ${MIN_HEAP_LIMIT_MB}`);
+    }
+    if (configured?.mb) {
+      heapLimitMb = configured.mb;
+      source = `from config ${HEAP_LIMIT_CONFIG_KEY}`;
+      if (memoryMb >= MIN_HEAP_LIMIT_MB && heapLimitMb > memoryMb) {
+        log(`[relay] config ${HEAP_LIMIT_CONFIG_KEY}=${heapLimitMb} is more than the ${memoryText}; using ${memoryMb} MB`);
+        heapLimitMb = memoryMb;
+      }
+    } else {
+      heapLimitMb = defaultHeapLimitMb({ totalMem: memory.bytes, nodeDefaultMb: nodeDefaultOldSpaceMb(heapSizeLimit) });
+      if (memoryMb >= MIN_HEAP_LIMIT_MB) heapLimitMb = Math.min(heapLimitMb, memoryMb);
+      source = `default: ${Math.round(DEFAULT_RAM_SHARE * 100)}% of ${memoryText}`;
+    }
+    flags.push(`--max-old-space-size=${heapLimitMb}`);
+  }
+  log(`[relay] heap limit ${heapLimitMb} MB (${source})`);
+
+  // The operator's --diagnostic-dir is theirs: checked, never made or changed.
+  const operatorDir = lastFlagValue(operatorArgs, DIAGNOSTIC_DIR_FLAG);
+  const diagnosticDir = operatorDir
+    ? path.resolve(operatorDir)
+    : path.join(resolveDataDir({ env, configPath: resolveConfigPath(env, serverDir), serverDir, fsImpl }), 'diagnostics');
+  const shownDir = displayPath(diagnosticDir, homeDir);
+  let snapshot = false;
+  try {
+    if (!operatorDir) {
+      fsImpl.mkdirSync(diagnosticDir, { recursive: true, mode: 0o700 });
+      // mkdir leaves a folder that was there as it was.
+      if (platform !== 'win32') fsImpl.chmodSync?.(diagnosticDir, 0o700);
+    }
+    const free = freeBytes(diagnosticDir, fsImpl);
+    if (operatorArgs.some((arg) => SNAPSHOT_FLAG.test(arg))) {
+      snapshot = true;
+    } else if (free !== null && free < 2 * heapLimitMb * MB) {
+      log(`[relay] no heap snapshot near the limit: ${shownDir} has ${Math.floor(free / MB)} MB free, under twice the heap limit`);
+    } else {
+      flags.push('--heapsnapshot-near-heap-limit=1');
+      snapshot = true;
+    }
+    if (!operatorDir) flags.push(`--diagnostic-dir=${diagnosticDir}`);
+    if (snapshot) {
+      log(`[relay] heap snapshots near the limit go to ${shownDir} (${countSnapshots(diagnosticDir, fsImpl)} already there)`);
+    }
+  } catch (error) {
+    log(`[relay] no heap snapshot near the limit: cannot prepare ${shownDir} (${error?.code || error?.message || error})`);
+  }
+
+  return { flags, heapLimitMb, source, diagnosticDir, snapshot };
+}
+
+/**
+ * relayRuntimeNodeFlags' flags, or none when it fails: a runtime must start
+ * even when its heap guard cannot be worked out.
+ */
+export function relayRuntimeNodeFlagsOrNone(nodeFlags, options = {}) {
+  try {
+    return nodeFlags(options);
+  } catch (error) {
+    options.logger?.log?.(`[relay] starting the runtime without heap flags: ${error?.message || error}`);
+    return { flags: [] };
+  }
+}
+
+function toMb(bytes) {
+  return Math.round((Number(bytes) || 0) / MB);
+}
+
+/** `[relay] memory rss=<MB> heapUsed=<MB> heapLimit=<MB>` */
+export function formatMemoryLine({ rss, heapUsed, heapLimit }) {
+  return `[relay] memory rss=${toMb(rss)} heapUsed=${toMb(heapUsed)} heapLimit=${toMb(heapLimit)}`;
+}
+
+/**
+ * Says once when heapUsed crosses `threshold` of the heap limit, and again
+ * only after it has been back under it. `check` returns whether it said so.
+ */
+export function createHeapWatch({ threshold = 0.8, logger = console } = {}) {
+  let above = false;
+  return {
+    check({ heapUsed, heapLimit }) {
+      const over = Number(heapLimit) > 0 && Number(heapUsed) >= threshold * Number(heapLimit);
+      const crossed = over && !above;
+      above = over;
+      if (crossed) {
+        logger?.log?.(`[relay] memory high: heapUsed=${toMb(heapUsed)} is over ${Math.round(threshold * 100)}% of heapLimit=${toMb(heapLimit)}`);
+      }
+      return crossed;
+    },
+  };
+}
+
+/**
+ * The runtime's memory in the status log: a line now and every 10 minutes,
+ * and a silent check every minute that says when heapUsed crosses 80 % of
+ * the limit. Both timers are unref'd; returns them for runtimeTimers.
+ */
+export function startRelayMemoryLog({
+  logger = console,
+  memoryUsage = () => process.memoryUsage(),
+  heapLimit = () => v8.getHeapStatistics().heap_size_limit,
+  logIntervalMs = 10 * 60 * 1000,
+  watchIntervalMs = 60 * 1000,
+} = {}) {
+  const watch = createHeapWatch({ logger });
+  const sample = () => {
+    const { rss, heapUsed } = memoryUsage();
+    return { rss, heapUsed, heapLimit: heapLimit() };
+  };
+  const logLine = () => {
+    const now = sample();
+    logger?.log?.(formatMemoryLine(now));
+    watch.check(now);
+  };
+  logLine();
+  const log = setInterval(logLine, logIntervalMs);
+  log.unref?.();
+  const check = setInterval(() => watch.check(sample()), watchIntervalMs);
+  check.unref?.();
+  return { log, check };
+}

package/server/relay-self-restart.mjs

@@ -1,6 +1,8 @@
-import { spawn } from 'child_process';
+import { spawn, spawnSync } from 'child_process';
 import path from 'path';
 import { RELAY_RESTART_EXIT_CODE, RELAY_STOP_EXIT_CODE } from './relay-exit-codes.mjs';
+import { relayRuntimeNodeFlags, relayRuntimeNodeFlagsOrNone } from './relay-heap-guard.mjs';
+import { detectRelayHosting } from './services/relay-restart-picture.mjs';
 
 // Role selection travels on argv, never in the environment. The runtime's env is
 // inherited by tmux worker sessions and by the Copilot CLI it launches, and the
@@ -34,8 +36,11 @@ export function spawnRelayRuntime({
   detached = false,
   windowsHide = true,
   logger = console,
+  // The heap limit and heap snapshot flags (relay-heap-guard.mjs).
+  nodeFlags = relayRuntimeNodeFlags,
 } = {}) {
-  const child = spawnImpl(process.execPath, [...execArgv, scriptPath, RELAY_RUNTIME_FLAG, ...args], {
+  const { flags } = relayRuntimeNodeFlagsOrNone(nodeFlags, { env, execArgv, serverDir: path.dirname(String(scriptPath || '')), logger });
+  const child = spawnImpl(process.execPath, [...flags, ...execArgv, scriptPath, RELAY_RUNTIME_FLAG, ...args], {
     cwd,
     env,
     detached,
@@ -72,6 +77,64 @@ function waitForChildExit(child) {
   });
 }
 
+// A systemctl --user call of the handover: its result, never a throw.
+function runSystemctl(spawnSyncImpl, args) {
+  try {
+    return spawnSyncImpl('systemctl', ['--user', ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 15_000 }) || { status: null };
+  } catch (error) {
+    return { status: null, error };
+  }
+}
+
+function systemctlFailure(result) {
+  if (result.error) return result.error.message || String(result.error);
+  const said = String(result.stderr || '').trim().split('\n')[0];
+  return `status ${result.status ?? 'none'}${said ? `: ${said}` : ''}`;
+}
+
+/**
+ * The runtime asked for a restart (exit 75). Under a user service whose main
+ * process this supervisor is, systemd restarts the whole service, so the new
+ * supervisor and runtime run the code on disk with fresh Node flags: true
+ * once systemd's stop (SIGTERM) has come. False, with one log line, wherever
+ * that cannot be done (not the unit's main process: a relay started inside a
+ * worker of the service), systemctl fails, or no stop comes within
+ * `handoverWaitMs`; the caller relaunches the runtime itself then.
+ */
+async function handRestartToSystemd({ hosting, spawnSyncImpl, pid, handoverWaitMs, shutdownRequested, isShuttingDown, delay, logger }) {
+  if (!hosting?.systemd || !hosting.userManager || !hosting.unit) return false;
+  const { unit } = hosting;
+  const main = runSystemctl(spawnSyncImpl, ['show', '-p', 'MainPID', '--value', unit]);
+  const mainPid = Number(String(main.stdout ?? '').trim());
+  if (main.status !== 0 || mainPid !== pid) {
+    const whose = main.status === 0 ? `its main process is ${mainPid || 'none'}` : `its main process could not be read (${systemctlFailure(main)})`;
+    logger?.log?.(`[relay] restart: not handed to systemd, ${unit} is not this supervisor's (pid ${pid}; ${whose}).`);
+    return false;
+  }
+  // Clears a failed state and the restart backoff; restart's answer alone decides.
+  runSystemctl(spawnSyncImpl, ['reset-failed', unit]);
+  // A stop that came in meanwhile wins: a queued restart could replace its job.
+  // The calls above block; a signal that arrived during them waits on the
+  // signal pipe, which only the loop's poll phase reads. This runs from the
+  // runtime's 'exit' callback, itself in a poll phase, so one setImmediate
+  // would run in the check phase right after it, before the pipe is read;
+  // the second runs after the next poll phase has delivered the signal.
+  await new Promise((resolve) => setImmediate(() => setImmediate(resolve)));
+  if (isShuttingDown()) {
+    logger?.log?.(`[relay] restart: a stop came in; not restarting ${unit}.`);
+    return true;
+  }
+  const restart = runSystemctl(spawnSyncImpl, ['restart', '--no-block', unit]);
+  if (restart.status !== 0) {
+    logger?.log?.(`[relay] restart: systemctl --user restart --no-block ${unit} failed (${systemctlFailure(restart)}).`);
+    return false;
+  }
+  logger?.log?.(`[relay] restart handed to systemd: ${unit} restarts with the code on disk.`);
+  const stopped = await Promise.race([shutdownRequested.then(() => true), Promise.resolve(delay(handoverWaitMs)).then(isShuttingDown)]);
+  if (!stopped) logger?.log?.(`[relay] restart: systemd did not stop ${unit} within ${Math.round(handoverWaitMs / 1000)} s.`);
+  return stopped;
+}
+
 export async function runDirectRelaySupervisor({
   scriptPath,
   args = [],
@@ -84,31 +147,54 @@ export async function runDirectRelaySupervisor({
   exitImpl = (code) => process.exit(code),
   logger = console,
   installSignalHandlers = true,
+  signals = process,
+  nodeFlags = relayRuntimeNodeFlags,
   restartDelayMs = 500,
   maxCrashRestarts = 3,
+  // A runtime that ran this long before it crashed starts the count afresh:
+  // crashes days apart are not a crash loop.
+  crashCountResetMs = 10 * 60 * 1000,
+  now = Date.now,
+  // How this process is run (detectRelayHosting); read at the first restart
+  // request when not given.
+  hosting = null,
+  spawnSyncImpl = spawnSync,
+  pid = process.pid,
+  // How long a restart handed to systemd waits for its SIGTERM.
+  handoverWaitMs = 30_000,
 } = {}) {
   let runtimeProc = null;
   let shuttingDown = false;
   let shutdownExitCode = 0;
+  let shutdownArrived;
+  const shutdownRequested = new Promise((resolve) => { shutdownArrived = resolve; });
 
   const requestShutdown = (signal = 'SIGTERM', exitCode = 0) => {
     if (shuttingDown) return;
     shuttingDown = true;
     shutdownExitCode = exitCode;
     stopChildProcess(runtimeProc, { signal });
+    shutdownArrived();
   };
 
   const handleSigInt = () => requestShutdown('SIGINT', 0);
   const handleSigTerm = () => requestShutdown('SIGTERM', 0);
 
   if (installSignalHandlers) {
-    process.on('SIGINT', handleSigInt);
-    process.on('SIGTERM', handleSigTerm);
+    signals.on('SIGINT', handleSigInt);
+    signals.on('SIGTERM', handleSigTerm);
   }
 
   try {
     let crashCount = 0;
     while (true) {
+      // A Ctrl-C reaches the runtime (same process group) before this
+      // supervisor's handler can run: the runtime's signal exit looks like a
+      // crash, and the shutdown arrives during the delay. Never relaunch then.
+      if (shuttingDown) {
+        return await exitImpl(shutdownExitCode);
+      }
+      const startedAt = now();
       runtimeProc = spawnRelayRuntime({
         env,
         cwd,
@@ -119,6 +205,7 @@ export async function runDirectRelaySupervisor({
         stdio,
         detached: false,
         logger,
+        nodeFlags,
       });
       if (runtimeProc.stdout?.on) {
         runtimeProc.stdout.on('data', (chunk) => process.stdout.write(chunk));
@@ -139,10 +226,28 @@ export async function runDirectRelaySupervisor({
         return await exitImpl(1);
       }
 
-      const exitCode = Number.isInteger(Number(result.code)) ? Number(result.code) : null;
+      // A runtime killed by a signal (an out-of-memory abort is SIGABRT, the
+      // kernel's OOM killer SIGKILL) exits with code null: a crash. Number(null)
+      // is 0, which once ended the supervisor as if the runtime stopped cleanly.
+      const exitCode = result.code !== null && result.code !== undefined && Number.isInteger(Number(result.code))
+        ? Number(result.code)
+        : null;
+      const signal = exitCode === null ? (result.signal || null) : null;
       if (exitCode === RELAY_RESTART_EXIT_CODE) {
-        logger?.log?.(`[relay] runtime requested restart; relaunching ${path.basename(String(scriptPath || ''))}...`);
         crashCount = 0;
+        const handedOver = await handRestartToSystemd({
+          hosting: hosting || detectRelayHosting({ env }),
+          spawnSyncImpl,
+          pid,
+          handoverWaitMs,
+          shutdownRequested,
+          isShuttingDown: () => shuttingDown,
+          delay,
+          logger,
+        });
+        // systemd's stop: this supervisor ends, systemd starts the next one.
+        if (handedOver || shuttingDown) return await exitImpl(shutdownExitCode);
+        logger?.log?.(`[relay] runtime requested restart; relaunching ${path.basename(String(scriptPath || ''))}...`);
         await delay(restartDelayMs);
         continue;
       }
@@ -156,8 +261,10 @@ export async function runDirectRelaySupervisor({
         return await exitImpl(RELAY_STOP_EXIT_CODE);
       }
 
+      if (now() - startedAt >= crashCountResetMs) crashCount = 0;
       const attempt = crashCount + 1;
-      logger?.error?.(`[relay] runtime crashed with exit code ${exitCode ?? 'null'} (crash ${attempt}/${maxCrashRestarts}).`);
+      const how = signal ? `signal ${signal}` : `exit code ${exitCode ?? 'null'}`;
+      logger?.error?.(`[relay] runtime crashed with ${how} (crash ${attempt}/${maxCrashRestarts}).`);
       if (attempt >= maxCrashRestarts) {
         logger?.error?.('[relay] crash limit reached; supervisor stopping.');
         return await exitImpl(1);
@@ -168,8 +275,8 @@ export async function runDirectRelaySupervisor({
 
   } finally {
     if (installSignalHandlers) {
-      process.off('SIGINT', handleSigInt);
-      process.off('SIGTERM', handleSigTerm);
+      signals.off('SIGINT', handleSigInt);
+      signals.off('SIGTERM', handleSigTerm);
     }
   }
 }

package/server/relay-tools.md

@@ -39,19 +39,20 @@ A paired relay stays locked until the user mentions it in this conversation (`@n
 
 This relay itself is listed with "self": true when its owner switched agent sessions on, and is addressed by its name or as "this"; it needs no mention. Use it to hand parts of your task to other sessions:
 
-- `create_session` starts a session with any provider that `relay_info` lists: a local one in a folder ("cwd"), or provider "claude-cloud" with "repo" (a GitHub URL or owner/repo) and an optional "branch". At most 4 sessions you started may work at the same time; a fifth is refused until one finishes.
+- `create_session` starts a session with any provider that `relay_info` lists: a local one in a folder ("cwd"), or provider "claude-cloud" with "repo" (a GitHub URL or owner/repo) and an optional "branch". The provider ids are the target relay's, as `relay_info` shows them; an id it does not know is refused with the list it has. A session started before a relay update can use providers the update added, without a restart. Without "mode" the new session gets your session's relay mode. At most 4 sessions you started may work at the same time; a fifth is refused until one finishes.
 - The first `create_session` on this relay shows the user an approval card, once per conversation. A session that an agent created cannot create sessions itself (one level only), unless the owner allowed it for that session: `list_sessions` and `read_session` then show "mayCreateSessions": true. Only the owner sets that.
 - Sessions you start here are grouped under this session in the user's sidebar. `list_sessions` lists in the sidebar's order (a group sorts by its newest session, head first) and shows the group as a "group" field (id, name, role); `read_session` adds the group's live members with their state. `create_session` takes "group" (a group id, or its name in any case, from `relay_info`'s "groups") to put the new session into an existing group instead, here or on a paired relay; an unknown group is refused with the groups listed and nothing is created, and the result carries the "group". Renaming and dissolving groups is the user's.
-- `update_session` ("session" plus any of "model", "effort", "mode", "auto_compact_window", "title", "group", "thinking_enabled", "thinking_display", "cwd") changes another session's settings for its next turns without prompting it, here or on a paired relay, and answers the settings and "changed"; a group name that does not exist makes one, "none" takes the session out (a group the user made stays, even empty; one update_session made goes once empty), a new "cwd" restarts its CLI there (refused while it works), "auto_compact_window" 0 is the model default. The model is checked as for `send`. A failed call still lists in "changed" what it wrote before the failing step. A mode set this way lasts until the next `send` that names another mode or comes from a caller in another mode: a `send` without "mode" passes the caller's own mode.
+- `update_session` ("session" plus any of "model", "effort", "mode", "auto_compact_window", "title", "group", "thinking_enabled", "thinking_display", "cwd", "planOnly") changes another session's settings for its next turns without prompting it, here or on a paired relay, and answers the settings and "changed"; a group name that does not exist makes one, "none" takes the session out (a group the user made stays, even empty; one update_session made goes once empty), a new "cwd" restarts its CLI there (refused while it works), "auto_compact_window" 0 is the model default. The model is checked as for `send`. A failed call still lists in "changed" what it wrote before the failing step. A mode set this way lasts until the next `update_session` that names another mode; a `send` that names a mode runs only that turn in it, and one without "mode" leaves the session in its own mode.
+- "planOnly": true makes a plan-only session: agents can't take it out of plan mode. Set it only when the user asks; plan mode alone never sets it. `create_session` with "planOnly": true and no "mode" starts in plan; with another mode named it is refused and nothing is created. A target relay too old for planOnly creates the session without the lock and says so in "planOnlyWarning". `update_session` with "planOnly": true sets it on an existing session (with "mode": "plan" unless it is in plan already). While it is set, a `send` or `update_session` that names another mode is refused with 409, code "plan_only" ("This session is plan-only; agents can't change its mode. Clear planOnly first, or ask the user."); a `send` without "mode" runs in the session's saved mode. To clear it, pass "planOnly": false in the same `update_session` that names the new mode (`{"planOnly":false,"mode":"agent"}`); alone, or with "mode": "plan", it is ignored, and on a plan-only session the answer notes "planOnly unchanged: clear it in the same call as the new mode." Use `update_session` for all this, not the preferences route (it refuses an agent's mode change with 409 and any planOnly change with 403). `list_sessions` and `read_session` show "planOnly": true. In a plan-only conversation you count as a plan-mode caller, also in the one agent turn after "Approve plan": your `create_session` defaults to plan and your write actions ask the user first.
 - Only a session your conversation started may be updated freely; any other (the user's, another agent's, the one that started you) needs the user's approval card once per conversation and relay (refused: `REMOTE_RELAY_PERMISSION`); never your own conversation.
-- A "model" for `create_session` or `send` must be an id `relay_info` lists for the provider (Claude also takes "opus", "sonnet", "haiku"). An unknown id is refused with the close matches and the valid ids (`Unknown model 'gpt-5-4' for Copilot on relay "…"; did you mean 'gpt-5.4'? Valid ids: … Nothing was sent.`), and nothing is created or sent: pick from the list instead of guessing. If the answer has a "modelNote" saying the model could not be checked, the relay had no model list yet.
+- A "model" for `create_session` or `send` must be an id `relay_info` lists for the provider (Claude also takes "opus", "sonnet", "haiku"). An unknown id is refused with the close matches and the valid ids (`Unknown model 'gpt-5-4' for Copilot on relay "…"; did you mean 'gpt-5.4'? Valid ids: … Nothing was sent.`), and nothing is created or sent: pick from the list instead of guessing. Copilot ids use dots (`claude-opus-5.5`, not the Claude form `claude-opus-5-5`); a hyphen form is refused with the close match named, and a Copilot session never moves to another provider because of its model name. If the answer has a "modelNote" saying the model could not be checked, the relay had no model list yet.
 - A new session knows nothing of this conversation: write a self-contained prompt. A Claude Cloud session works on a fresh clone in a sandbox at Anthropic, so name the branch it should work on and push.
 - Follow each session with `wait` (the message id comes back from `create_session` and `send`) and `read_session`; "wait_seconds" is limited to the maximum set on the relay (default 600).
 - Sessions run on the relay owner's accounts and cost their usage: start only what the task needs.
 - Pass only the arguments an action takes. `create_session` and `update_session` refuse any other one, list their valid arguments, and change nothing; every other action still runs and names the arguments it did not take in "ignored".
 - `read_task_output` ("session", "task_id" from `read_session`'s tasks, optional "lines") reads the end of a background shell or Monitor task's output, also for an hour after it ended: the last lines, minutes since its last output, and pass/fail counts for a test run. Credentials are scrubbed out.
 
-On this relay and on a paired one, `subscribe` ("sessions": ids, or ["all"]) wakes you when something happens in the sessions you watch, instead of polling them. Leave "events" out unless you need more: the default (question_asked, error, turn_end, subagent_finished, task_failed, stalled, all_idle) wakes you for questions, errors, replies, finished sub-agents, failed tasks (a non-zero exit included), stalls and all_idle, but not for each shell command that ended well. To get those too, pass that list plus "task_finished" (and "task_started" for the starts); each one costs you a turn, so do it only if you must follow every command. "relay_restarted" (this relay only, never in the default) wakes you when the relay runs again after a restart, with who requested it and why; named alone it needs no sessions.
+On this relay and on a paired one, `subscribe` ("sessions": ids, or ["all"]) wakes you when something happens in the sessions you watch, instead of polling them. Leave "events" out unless you need more: the default (question_asked, error, turn_end, subagent_finished, task_failed, stalled, all_idle, board_settled) wakes you for questions, errors, replies, finished sub-agents, failed tasks (a non-zero exit included), stalls, all_idle and a plan card the user settled (board_settled: the button chosen and the message it queued, if any), but not for each shell command that ended well. To get those too, pass that list plus "task_finished" (and "task_started" for the starts); each one costs you a turn, so do it only if you must follow every command. "relay_restarted" (this relay only, never in the default) wakes you when the relay runs again after a restart, with who requested it and why; named alone it needs no sessions.
 
 Each wake-up costs you a turn; these `subscribe` options cut them (questions and errors always come at once):
 - "digest_minutes": N gathers the subscription's other events into one message every N minutes (1 to 1440).

package/server/repositories/message-repository.mjs

@@ -5,6 +5,33 @@ import { createMessagePinStatements } from './message-pin-statements.mjs';
 import { createConversationHistoryPageStatements } from '../services/conversation-history-page.mjs';
 import { sessionWorkerProviderSqlList } from '../../shared/provider-routing.mjs';
 
+/**
+ * The claim's plan-boundary hold, part of every findPending*
+ * statement: a pending row waits while its conversation has a live turn on
+ * the other side of plan mode, because handing it to the worker would steer
+ * it into that turn and switch what the turn may do. The conversation's later
+ * rows wait behind it whatever their mode, so one conversation keeps its
+ * order, while the claim goes on to other conversations' rows.
+ * `PLAN_BOUNDARY_HOLD_MARK` lets a test find any claim without it.
+ */
+export const PLAN_BOUNDARY_HOLD_MARK = '/* plan-boundary-hold */';
+
+export function planBoundaryHoldSql(alias) {
+  return `${PLAN_BOUNDARY_HOLD_MARK} NOT EXISTS (
+    SELECT 1
+    FROM queue held
+    JOIN queue live
+      ON live.conversation_id = held.conversation_id
+     AND live.status = 'processing'
+     AND live.id != held.id
+    WHERE held.conversation_id = ${alias}.conversation_id
+      AND held.status = 'pending'
+      AND COALESCE(held.kind, '') != 'continuation'
+      AND (held.timestamp < ${alias}.timestamp OR (held.timestamp = ${alias}.timestamp AND held.rowid <= ${alias}.rowid))
+      AND (LOWER(TRIM(COALESCE(live.relay_mode, ''))) = 'plan') != (LOWER(TRIM(COALESCE(held.relay_mode, ''))) = 'plan')
+  )`;
+}
+
 export function createMessageRepository(db) {
     const shareVisibility = createShareVisibilityStatements(db);
     const messagePins = createMessagePinStatements(db);
@@ -187,7 +214,7 @@ export function createMessageRepository(db) {
         // steer-recovered row waited 5 minutes while newer messages jumped
         // it). Head-of-line protection is the WHERE clause's next_attempt_at
         // backoff plus MAX_REQUEUE_RETRIES, which fails a row for good.
-        findPending:    db.prepare(`SELECT * FROM queue WHERE status = 'pending' AND COALESCE(kind, '') != 'continuation' AND (next_attempt_at IS NULL OR next_attempt_at <= ?) ORDER BY timestamp ASC, rowid ASC LIMIT 1`),
+        findPending:    db.prepare(`SELECT q.* FROM queue q WHERE q.status = 'pending' AND COALESCE(q.kind, '') != 'continuation' AND (q.next_attempt_at IS NULL OR q.next_attempt_at <= ?) AND ${planBoundaryHoldSql('q')} ORDER BY q.timestamp ASC, q.rowid ASC LIMIT 1`),
         // Global fallback for requesters without a session identity (the legacy
         // Copilot relay CLI). Conversations bound to a session-worker provider
         // (claude/cursor/grok) are excluded outright: handing one of their turns
@@ -218,6 +245,7 @@ export function createMessageRepository(db) {
               NULLIF(rsc.provider_type, ''),
               'github'
             )) NOT IN (${sessionWorkerProviderSqlList()})
+            AND ${planBoundaryHoldSql('q')}
           ORDER BY
             q.timestamp ASC,
             q.rowid ASC
@@ -254,6 +282,7 @@ export function createMessageRepository(db) {
                 NULLIF(c.sdk_session_id, '')
               ) = ?
             )
+            AND ${planBoundaryHoldSql('q')}
           ORDER BY
             CASE
               WHEN COALESCE(
@@ -282,6 +311,7 @@ export function createMessageRepository(db) {
               NULLIF(rs.sdk_session_id, ''),
               NULLIF(c.sdk_session_id, '')
             ) = ?
+            AND ${planBoundaryHoldSql('q')}
           ORDER BY
             q.timestamp ASC,
             q.rowid ASC