@oar-sh/oar 0.9.20 → 0.9.21

package/server/services/worker-replacement.mjs

@@ -0,0 +1,197 @@
+/**
+ * Replacing a session's worker: the stop and the launch that the Claude
+ * billing swap (claude-billing-swap-service.mjs) and the worker recycle
+ * (worker-recycle-service.mjs) share, and the predicate both use to tell
+ * whether a worker may be replaced now.
+ *
+ * The stop is the kill route's: nothing may start the session while it is
+ * stopped, every process of the session must be gone before it counts as
+ * stopped, and only then is the registry entry dropped. The launch never
+ * reuses a live process, so two CLIs never share a session's transcript.
+ * The fence keeps the deliveries away from the process being replaced for the
+ * whole of it (createWorkerReplacementFence).
+ */
+
+import { stopSessionWorkerProcesses as defaultStopSessionWorkerProcesses } from './session-worker-stop-service.mjs';
+
+/**
+ * What keeps a worker from being replaced now: null when nothing does, else
+ * 'turn', 'question', 'compaction' or 'background'. From the steering
+ * snapshot its heartbeat sends; a worker too old to say `releasable` is read
+ * from the rest of it.
+ */
+export function workerReleaseHold(worker) {
+  if (!worker) return null;
+  const steering = worker.steering || null;
+  if (steering && typeof steering.releasable === 'boolean') {
+    return steering.releasable ? null : (steering.releaseHold || 'turn');
+  }
+  if (steering) {
+    if (steering.holdReason === 'question') return 'question';
+    if (steering.holdReason === 'compaction') return 'compaction';
+    if (steering.turnActive) return 'turn';
+    if (steering.autoCompactWindow?.backgroundWork) return 'background';
+    return null;
+  }
+  return String(worker.status || '') === 'processing' ? 'turn' : null;
+}
+
+function toFencePid(value) {
+  const pid = Number(value);
+  return Number.isInteger(pid) && pid > 0 ? pid : null;
+}
+
+/**
+ * The replacement fence: while a worker process is being replaced, no work
+ * is handed to it.
+ *
+ * The killed marker the stop sets does not do this. It lapses after the
+ * supervisor's short kill grace (a stop that verifies every process gone can
+ * take longer), and any path that relaunches a session lifts it at once
+ * (`clearRestartSchedule({ resetKilledMarker: true })`, the delivery's own
+ * fallback restart among them), after which a delivery would start a turn on
+ * the very process being killed. The fence is its own structure, so nothing
+ * that resets the supervisor's lifecycle state touches it.
+ *
+ * Keyed by session and the pid being replaced: the new worker has another
+ * pid and is never fenced, so it takes the work that waited meanwhile. A
+ * request that names no pid cannot be told from the old process and is
+ * refused while the session's fence is up.
+ *
+ * - `hold(sid, pid)`: raise the fence for that process of the session.
+ * - `isHeld(sid, pid)`: true while the fence is up for that pid (or for any
+ *   pid of the session, when `pid` is unknown).
+ * - `release(sid, pid?)`: lower it; with a pid, only if that is the fenced
+ *   one, so a late release from an older replacement never lifts a newer one.
+ */
+export function createWorkerReplacementFence() {
+  const held = new Map();
+
+  function key(sid) {
+    return String(sid || '').trim();
+  }
+
+  function hold(sid, pid) {
+    const sessionId = key(sid);
+    if (!sessionId) return false;
+    held.set(sessionId, toFencePid(pid));
+    return true;
+  }
+
+  function isHeld(sid, pid = null) {
+    const sessionId = key(sid);
+    if (!sessionId || !held.has(sessionId)) return false;
+    const fencedPid = held.get(sessionId);
+    const askingPid = toFencePid(pid);
+    // An unknown pid on either side: the old process cannot be ruled out.
+    if (fencedPid === null || askingPid === null) return true;
+    return fencedPid === askingPid;
+  }
+
+  function release(sid, pid = undefined) {
+    const sessionId = key(sid);
+    if (!sessionId || !held.has(sessionId)) return false;
+    if (pid !== undefined && toFencePid(pid) !== held.get(sessionId)) return false;
+    held.delete(sessionId);
+    return true;
+  }
+
+  function list() {
+    return Array.from(held, ([sdkSessionId, pid]) => ({ sdkSessionId, pid }));
+  }
+
+  return { hold, isHeld, release, list };
+}
+
+export function createWorkerReplacement({
+  // The session worker registry: getWorker(sid), removeWorker(sid).
+  registry,
+  // The session worker supervisor: markKilled, cancelPendingStart,
+  // clearRestartSchedule, resetHealth, ensureWorker.
+  supervisor,
+  // Handed to stopSessionWorkerProcesses to find every process of a session.
+  processInspector = null,
+  // session-worker-stop-service.mjs by default; tests inject a fake so they
+  // need no processes: async ({ sdkSessionId, worker, processInspector,
+  // isPidAliveImpl, killTmuxSessionImpl }) → { ok, error?, remainingPids? }.
+  stopSessionWorkerProcesses = defaultStopSessionWorkerProcesses,
+  isProcessAlive = undefined,
+  killTmuxSession = undefined,
+  logger = console,
+  // The caller's tag for its warnings ('BILLING SWAP', 'WORKER RECYCLE'), and
+  // a function for the runtime's own log prefix, or a fixed string.
+  logPrefix = '',
+  // The replacement fence the caller raises before the stop and lowers once
+  // the launch is done (createWorkerReplacementFence). Passed in so the
+  // runtime's replacements and its delivery paths share one; this module
+  // only hands it back.
+  fence = createWorkerReplacementFence(),
+} = {}) {
+  if (typeof stopSessionWorkerProcesses !== 'function') {
+    throw new Error('createWorkerReplacement requires stopSessionWorkerProcesses');
+  }
+
+  function prefix() {
+    try {
+      return typeof logPrefix === 'function' ? String(logPrefix() || '') : String(logPrefix || '');
+    } catch {
+      return '';
+    }
+  }
+
+  /**
+   * Stop every process of the session. `{ ok: true }` once they are all gone
+   * and the registry and the supervisor's health state are cleared; else
+   * `{ ok: false, error, remainingPids }`, and the caller retries with a
+   * backoff.
+   *
+   * `keepKilled: true` is for a session that must stay stopped (an archived
+   * conversation): the killed marker set here is left in place on either
+   * outcome, as the archive route leaves it, so nothing relaunches the
+   * session meanwhile.
+   */
+  async function stopWorker(sid, { keepKilled = false } = {}) {
+    const worker = registry?.getWorker?.(sid) || null;
+    // Nothing may start the session while it is being stopped: a launch then
+    // would adopt the old process, kind and build and all.
+    supervisor?.markKilled?.(sid);
+    // A stop that fails backs off (the caller retries it). Unless the session
+    // is to stay stopped, the killed marker goes with it, so the session's
+    // worker, alive or relaunched, keeps asking for work meanwhile.
+    const backOff = (error, remainingPids = []) => {
+      if (!keepKilled) supervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
+      const remaining = remainingPids.length ? ` remaining=${remainingPids.join(',')}` : '';
+      const marker = keepKilled ? 'kill marker kept' : 'kill marker cleared';
+      try {
+        logger.warn?.(`${prefix()}STOP FAILED session=${String(sid).slice(0, 8)} ${error}${remaining} — ${marker}, retrying with a backoff`);
+      } catch {}
+      return { ok: false, error, remainingPids };
+    };
+    let stopped = null;
+    try {
+      await supervisor?.cancelPendingStart?.(sid, { wait: true });
+      stopped = await stopSessionWorkerProcesses({
+        sdkSessionId: sid,
+        worker,
+        processInspector,
+        isPidAliveImpl: isProcessAlive,
+        killTmuxSessionImpl: killTmuxSession,
+      });
+    } catch (error) {
+      return backOff(String(error?.message || error || 'worker-stop-failed'));
+    }
+    if (!stopped?.ok) return backOff(stopped?.error || 'worker-survived-stop', stopped?.remainingPids || []);
+    registry?.removeWorker?.(sid);
+    supervisor?.resetHealth?.(sid, { clearFailureCount: false });
+    return { ok: true };
+  }
+
+  /** Launch the session's worker, never reusing a live process: `{ ok, error }`. */
+  async function launchWorker(sid) {
+    supervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
+    const launched = await supervisor?.ensureWorker?.(sid, { allowProcessReuse: false });
+    return { ok: launched?.ok === true, error: launched?.error || null };
+  }
+
+  return { stopWorker, launchWorker, fence };
+}

package/shared/build-id.mjs

@@ -0,0 +1,128 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+// The build id names the code a process runs, so the relay can tell a session
+// worker that started on older code from one that runs what the relay runs.
+//
+// It is `${version}+${commit.slice(0, 12)}` when both are known, the version
+// alone when there is no commit, and null when there is no version either. The
+// commit is part of it because on a git checkout a deploy is a new
+// commit at the same version; on an npm install without a `gitHead` stamp the
+// version alone decides.
+//
+// The relay and every worker compute it once at process start and never read
+// it again, so a tree that moved on under a running process still describes
+// the code that process started with.
+//
+// This module lives in shared/ because both the relay and the workers import
+// it; it depends on nothing beyond node:fs and node:path for the same reason.
+
+const COMMIT_PATTERN = /^[0-9a-f]{40}$/i;
+const COMMIT_ID_LENGTH = 12;
+
+// The relay stores the build id a worker states, so what it accepts is bounded
+// in length and limited to the characters a version and a commit use.
+const BUILD_ID_MAX_LENGTH = 80;
+const BUILD_ID_PATTERN = /^[A-Za-z0-9+._-]+$/;
+
+function readFileText(fsImpl, filePath) {
+  try {
+    return String(fsImpl.readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function readPackageManifest(packageRoot, { fsImpl, pathImpl }) {
+  if (!packageRoot) return null;
+  try {
+    const manifest = JSON.parse(readFileText(fsImpl, pathImpl.join(packageRoot, 'package.json')) || '');
+    return manifest && typeof manifest === 'object' ? manifest : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The commit a git checkout has checked out, read from its .git files rather
+ * than from a `git` process: HEAD, then the loose ref, then packed-refs. A
+ * linked worktree's .git is a file naming its git dir, whose refs live in the
+ * common dir. Anything unexpected reads as null. `pathImpl` is the host's path
+ * module; tests pass path.posix and path.win32.
+ */
+export function readGitCommit(root, { fsImpl = fs, pathImpl = path } = {}) {
+  if (!root) return null;
+  const dotGit = pathImpl.join(root, '.git');
+  let gitDir = dotGit;
+  const pointer = readFileText(fsImpl, dotGit);
+  if (pointer !== null) {
+    const match = pointer.match(/^gitdir:\s*(.+)$/m);
+    if (!match) return null;
+    gitDir = pathImpl.resolve(root, match[1].trim());
+  }
+  const head = readFileText(fsImpl, pathImpl.join(gitDir, 'HEAD'));
+  if (head === null) return null;
+  const headText = head.trim();
+  if (COMMIT_PATTERN.test(headText)) return headText.toLowerCase();
+  const refMatch = headText.match(/^ref:\s*(\S+)$/);
+  if (!refMatch) return null;
+  const ref = refMatch[1];
+  const commonDirText = readFileText(fsImpl, pathImpl.join(gitDir, 'commondir'));
+  const commonDir = commonDirText ? pathImpl.resolve(gitDir, commonDirText.trim()) : gitDir;
+  for (const dir of new Set([gitDir, commonDir])) {
+    const loose = readFileText(fsImpl, pathImpl.join(dir, ...ref.split('/')));
+    if (loose && COMMIT_PATTERN.test(loose.trim())) return loose.trim().toLowerCase();
+  }
+  const packed = readFileText(fsImpl, pathImpl.join(commonDir, 'packed-refs')) || '';
+  for (const line of packed.split(/\r?\n/)) {
+    const [hash, name] = line.trim().split(/\s+/);
+    if (name === ref && COMMIT_PATTERN.test(hash || '')) return hash.toLowerCase();
+  }
+  return null;
+}
+
+/** The commit this build came from: the checkout's HEAD, else a `gitHead` stamp in package.json. */
+export function resolveBuildCommit(packageRoot, { fsImpl = fs, pathImpl = path } = {}) {
+  const fromGit = readGitCommit(packageRoot, { fsImpl, pathImpl });
+  if (fromGit) return fromGit;
+  const stamp = String(readPackageManifest(packageRoot, { fsImpl, pathImpl })?.gitHead || '').trim();
+  return COMMIT_PATTERN.test(stamp) ? stamp.toLowerCase() : null;
+}
+
+/** The package.json `version`, or null when the manifest cannot be read or names none. */
+export function readPackageVersion(packageRoot, { fsImpl = fs, pathImpl = path } = {}) {
+  const version = readPackageManifest(packageRoot, { fsImpl, pathImpl })?.version;
+  return typeof version === 'string' && version.trim() ? version.trim() : null;
+}
+
+/**
+ * The build id of the tree at `packageRoot`: `{ version, commit, id }`. Call it
+ * once at process start and keep the result; see the header for why.
+ */
+export function resolveBuildId(packageRoot, { fsImpl = fs, pathImpl = path } = {}) {
+  const version = readPackageVersion(packageRoot, { fsImpl, pathImpl });
+  const commit = resolveBuildCommit(packageRoot, { fsImpl, pathImpl });
+  let id = null;
+  if (version && commit) id = `${version}+${commit.slice(0, COMMIT_ID_LENGTH)}`;
+  else if (version) id = version;
+  return { version, commit, id };
+}
+
+/**
+ * True only when both ids are known and differ. An unknown id on either side
+ * is not evidence of a change, so it never reads as one.
+ */
+export function buildIdsDiffer(a, b) {
+  if (typeof a !== 'string' || typeof b !== 'string') return false;
+  const left = a.trim();
+  const right = b.trim();
+  return !!left && !!right && left !== right;
+}
+
+/** A stated build id, trimmed, or null when it is too long or carries other characters. */
+export function normalizeBuildId(value) {
+  if (typeof value !== 'string') return null;
+  const text = value.trim();
+  if (!text || text.length > BUILD_ID_MAX_LENGTH || !BUILD_ID_PATTERN.test(text)) return null;
+  return text;
+}

package/shared/worker-runtime/heartbeat.mjs

@@ -37,6 +37,10 @@ export function createHeartbeatController({
   // relay hands a Claude chat's row only to a worker of the row's billing,
   // and knows a worker's kind from what it says here and in its hello.
   workerKind = null,
+  // Optional: the build id of the code this worker started on
+  // (shared/build-id.mjs). A worker whose websocket is down still beats, so
+  // the relay learns the build here as well as from the hello.
+  buildId = null,
   // requestPulse's window: one turn transition can flip the steering
   // snapshot several times in quick succession, and each flip only needs to
   // reach the relay once.
@@ -71,6 +75,7 @@ export function createHeartbeatController({
         ...(settleFailed.length ? { settleFailed } : {}),
         ...(Array.isArray(backgroundTaskIds) ? { backgroundTasks: backgroundTaskDigest(backgroundTaskIds) } : {}),
         ...(workerKind ? { workerKind: String(workerKind) } : {}),
+        ...(typeof buildId === "string" && buildId.trim() ? { buildId: buildId.trim() } : {}),
       };
       const response = await api("POST", "/api/heartbeat", body);
       // Failed by the relay, or skipped because this worker has no claim on

package/shared/worker-runtime/simple-steering-state.mjs

@@ -0,0 +1,38 @@
+/**
+ * The heartbeat steering snapshot of a worker that runs one delivered message
+ * at a time and never steers into a running turn: the Cursor, Grok and Claude
+ * cloud workers. They send it for the release fields, which tell the relay
+ * whether it may stop this worker and launch a fresh one now (its recycle
+ * after a deploy), the same fields the Claude and Copilot SDK workers send.
+ *
+ * `supported: false` keeps the composer as it was for these kinds: it offers
+ * mid-turn steering only to a worker that says `supported: true`, and reads
+ * neither the hold nor the cancellable ids of one that does not.
+ *
+ * - `activeAttempt`: the runner's `getActiveQueueAttempt()` (`{ id, attemptId }`
+ *   or null), the row the running turn owns.
+ * - `turnActive`: the runner's own word that a turn runs (`isTurnActive()`),
+ *   for a turn whose row has no id to show.
+ * - `backgroundTaskCount`: live background work that ends with the worker
+ *   (Grok's terminals).
+ *
+ * A turn holds the release ('turn'), then live background work ('background').
+ */
+export function buildSimpleSteeringState({ activeAttempt = null, turnActive = false, backgroundTaskCount = 0 } = {}) {
+  const messageId = String(activeAttempt?.id || '').trim() || null;
+  const turn = turnActive === true || Boolean(messageId);
+  const tasks = Number(backgroundTaskCount);
+  const liveTasks = Number.isInteger(tasks) && tasks > 0 ? tasks : 0;
+  const releaseHold = turn ? 'turn' : (liveTasks > 0 ? 'background' : null);
+  return {
+    turnActive: turn,
+    canSteer: false,
+    holdReason: null,
+    messageId,
+    supported: false,
+    cancellableIds: [],
+    releasable: !releaseHold,
+    releaseHold,
+    backgroundTaskCount: liveTasks,
+  };
+}

package/shared/worker-runtime/worker-websocket-link.mjs

@@ -1,3 +1,5 @@
+import { buildIdsDiffer } from "../build-id.mjs";
+
 function normalizeText(value) {
   const text = String(value || "").trim();
   return text || null;
@@ -70,6 +72,10 @@ export function createWorkerWebSocketLink({
   // every hello and ready: the relay hands a Claude chat's row only to a
   // worker of the row's billing.
   workerKind = null,
+  // The build id of the code this worker started on (shared/build-id.mjs),
+  // stated in every hello and ready next to the kind: the relay recycles a
+  // worker whose build differs from its own once the worker is idle.
+  buildId = null,
   minBackoffMs = 1000,
   maxBackoffMs = 8_000,
   jitterMs = 250,
@@ -121,6 +127,12 @@ export function createWorkerWebSocketLink({
   let lastPongAt = null;
   let lastQueueChangedAt = null;
   let lastDeliveryAt = null;
+  // The build id the relay named in its latest server.hello (null when it
+  // named none), and whether this connection has already logged that it
+  // differs from ours: the relay acks every worker.hello with another
+  // server.hello, and the log needs the news only once per connection.
+  let relayBuildId = null;
+  let buildMismatchLogged = false;
 
   function getNowMs() {
     const value = Number(now());
@@ -175,6 +187,7 @@ export function createWorkerWebSocketLink({
       sessionId: normalizeText(getSessionId()),
       pid: normalizePositiveInt(getPid()),
       ...(normalizeText(workerKind) ? { kind: normalizeText(workerKind) } : {}),
+      ...(normalizeText(buildId) ? { buildId: normalizeText(buildId) } : {}),
       ...(held ? { ready: false } : {}),
     });
     if (sent) lastHelloSentAt = getNowMs();
@@ -203,6 +216,7 @@ export function createWorkerWebSocketLink({
       sessionId: normalizeText(getSessionId()),
       pid: normalizePositiveInt(getPid()),
       ...(normalizeText(workerKind) ? { kind: normalizeText(workerKind) } : {}),
+      ...(normalizeText(buildId) ? { buildId: normalizeText(buildId) } : {}),
     });
     if (sent) lastReadySentAt = getNowMs();
     return sent;
@@ -393,6 +407,7 @@ export function createWorkerWebSocketLink({
         clearReadyRefreshTimer();
         // The next connection may reach a different (restarted) relay.
         serverCapabilities = new Set();
+        buildMismatchLogged = false;
       }
       if (stopped || !reconnect) return true;
       // A newer socket is already under way and owns the reconnect.
@@ -440,6 +455,13 @@ export function createWorkerWebSocketLink({
       }
       if (payload?.type === "server.hello") {
         serverCapabilities = new Set(Array.isArray(payload.capabilities) ? payload.capabilities.map(String) : []);
+        relayBuildId = normalizeText(payload.buildId);
+        // Only a line in the log: the relay decides when to replace this
+        // worker, and the worker takes no action of its own on it.
+        if (!buildMismatchLogged && buildIdsDiffer(relayBuildId, normalizeText(buildId))) {
+          buildMismatchLogged = true;
+          dbg(`relay build ${relayBuildId}, mine ${normalizeText(buildId)}: the relay will recycle this worker when idle`);
+        }
         if (!serverHelloSeen) {
           serverHelloSeen = true;
           // The handshake is complete: only now does the backoff start over.
@@ -536,6 +558,7 @@ export function createWorkerWebSocketLink({
       lastPongAt,
       lastQueueChangedAt,
       lastDeliveryAt,
+      relayBuildId,
     };
   }
 

package/shared/worker-runtime/working-directory.mjs

@@ -16,7 +16,7 @@ import fs from 'node:fs';
 export const WORKING_DIRECTORY_MISSING_CODE = 'working-directory-missing';
 
 export const WORKING_DIRECTORY_GUIDANCE =
-  'Restore the folder, or choose another working directory for this conversation (⋯ → Change CWD), '
+  'Restore the folder, or choose another working directory for this conversation (︙ → Change CWD), '
   + 'then send the message again.';
 
 // What a failed launch says, in the words of the SDKs: Node's spawn error, the