@oar-sh/oar 0.9.20 → 0.9.21

package/server/services/remote-relay-dispatcher.mjs

@@ -147,6 +147,9 @@ const CATALOG_PROVIDER_LABELS = Object.freeze({
 });
 // Worker statuses that mean "working on something right now" (scope active).
 const WORKING_WORKER_STATUSES = new Set(['starting', 'processing']);
+// A worker process that is up; the one a session's `worker` field names when
+// the registry holds more than one entry for its conversation.
+const LIVE_WORKER_STATUSES = new Set(['starting', 'ready', 'processing']);
 // Terminal turn failures are stored as ordinary assistant rows; these are the
 // texts the relay writes for them (buildTerminalFailureTextForChat and the two
 // recovery-limit paths in messages-routes / server-runtime). Each is the whole
@@ -194,6 +197,11 @@ function shortId(value) {
   return toText(value).slice(0, 8) || '-';
 }
 
+/** A session item's `worker` field, left out when the relay has no worker registered for it. */
+function workerField(worker) {
+  return worker ? { worker: { ...worker } } : {};
+}
+
 function nonNegativeInt(value) {
   const number = Number(value);
   return Number.isInteger(number) && number >= 0 ? number : 0;
@@ -1494,17 +1502,42 @@ export function createRemoteRelayDispatcher({
     };
   }
 
-  async function liveWorkerConversationIds(ctx) {
+  /**
+   * The target relay's session workers, from its /api/status: `working`, the
+   * conversations whose worker is starting or running a turn, and
+   * `workerFor(conversationId)`, that conversation's `worker` field (kind,
+   * build id, and whether it runs other code than that relay, which the relay
+   * judges against its own build id and recycles once idle), or null. A relay
+   * that predates build ids gives null build ids and never `stale`; one that
+   * cannot be read gives nothing.
+   */
+  async function sessionWorkers(ctx) {
+    const working = new Set();
+    const byConversation = new Map();
     try {
       const status = await call(ctx, 'GET', '/api/status');
       const workers = Array.isArray(status?.sessionWorker?.workers) ? status.sessionWorker.workers : [];
-      return new Set(workers
-        .filter((worker) => WORKING_WORKER_STATUSES.has(toText(worker?.status).toLowerCase()))
-        .map((worker) => toText(worker?.conversationId))
-        .filter(Boolean));
-    } catch {
-      return new Set();
-    }
+      for (const worker of workers) {
+        const conversationId = toText(worker?.conversationId);
+        if (!conversationId) continue;
+        const workerStatus = toText(worker?.status).toLowerCase();
+        if (WORKING_WORKER_STATUSES.has(workerStatus)) working.add(conversationId);
+        const known = byConversation.get(conversationId);
+        if (known && (known.live || !LIVE_WORKER_STATUSES.has(workerStatus))) continue;
+        byConversation.set(conversationId, {
+          live: LIVE_WORKER_STATUSES.has(workerStatus),
+          field: {
+            kind: toText(worker?.kind) || null,
+            buildId: toText(worker?.buildId) || null,
+            stale: worker?.stale === true,
+          },
+        });
+      }
+    } catch {}
+    return {
+      working,
+      workerFor: (conversationId) => byConversation.get(toText(conversationId))?.field || null,
+    };
   }
 
   /** The session group of a listed row: id, display name and the row's role. */
@@ -1521,7 +1554,7 @@ export function createRemoteRelayDispatcher({
     };
   }
 
-  function mapSession(row, workerIds, ctx = null, groups = null) {
+  function mapSession(row, workers, ctx = null, groups = null) {
     const id = toText(row?.id);
     const via = toText(row?.origin?.relayName);
     // A session an agent of that relay created there, and from which session.
@@ -1531,7 +1564,7 @@ export function createRemoteRelayDispatcher({
       title: toText(row?.title),
       provider: toText(row?.runtimeProviderType) || 'github',
       model: toText(row?.runtimeProviderModel) || toText(row?.runtimeModel) || toText(row?.preferredModel) || null,
-      active: row?.activeTurn === true || !!workerIds?.has(id) || isBusyState(row?.sessionState),
+      active: row?.activeTurn === true || !!workers?.working.has(id) || isBusyState(row?.sessionState),
       ...sessionActivityFields(row),
       updatedAt: row?.updatedAt || null,
       messageCount: nonNegativeInt(row?.messageCount),
@@ -1623,10 +1656,12 @@ export function createRemoteRelayDispatcher({
       cursor = decodeCursor(args.cursor, ['beforeConversationId', 'beforeUpdatedAt', 'beforeGroupId']);
       if (!cursor) throw invalidInput('cursor is not a nextCursor from list_sessions');
     }
-    const workerIds = scope === 'active' ? await liveWorkerConversationIds(ctx) : null;
+    // The active scope filters on the workers, so it reads them first; the
+    // others read them once the list is known, for the `worker` fields only.
+    const workers = scope === 'active' ? await sessionWorkers(ctx) : null;
     const filtering = !!query || scope === 'active';
     const keep = (row) => (!query || toText(row?.title).toLowerCase().includes(query))
-      && (scope !== 'active' || row?.activeTurn === true || workerIds.has(toText(row?.id)) || isBusyState(row?.sessionState));
+      && (scope !== 'active' || row?.activeTurn === true || workers.working.has(toText(row?.id)) || isBusyState(row?.sessionState));
 
     const sessions = [];
     const seen = new Set();
@@ -1649,7 +1684,7 @@ export function createRemoteRelayDispatcher({
         // A group served again whole (its cursor row vanished) repeats rows.
         if (seen.has(toText(row?.id))) continue;
         seen.add(toText(row?.id));
-        if (keep(row)) sessions.push(mapSession(row, workerIds, ctx, groups));
+        if (keep(row)) sessions.push(mapSession(row, workers, ctx, groups));
         if (sessions.length >= limit) break;
       }
       scanned += consumed;
@@ -1667,6 +1702,12 @@ export function createRemoteRelayDispatcher({
       hasMore = consumed < rows.length || response?.pageInfo?.hasMore === true;
       if (sessions.length >= limit || !hasMore || !rows.length) break;
     }
+    // Each session's worker process, while the relay has one registered for it.
+    const workerInfo = workers || (sessions.length ? await sessionWorkers(ctx) : null);
+    for (let index = 0; index < sessions.length; index += 1) {
+      const worker = workerInfo?.workerFor(sessions[index].id);
+      if (worker) sessions[index] = { ...sessions[index], ...workerField(worker) };
+    }
 
     const nextCursor = scope === 'all' && hasMore && cursor ? encodeCursor(cursor) : null;
     const note = filtering && hasMore && sessions.length < limit && scope !== 'all'
@@ -1749,7 +1790,12 @@ export function createRemoteRelayDispatcher({
       query: { limit: args.last, beforeMessageId: before?.beforeMessageId, beforeTimestamp: before?.beforeTimestamp, groupMembers: 1 },
     });
     const conversationId = toText(conversation?.id) || args.session;
-    const pendingQuestions = await loadPendingQuestions(ctx, conversationId, { includeAsks: true });
+    // Neither read rejects: a relay that cannot say leaves the fields out.
+    const [pendingQuestions, workers] = await Promise.all([
+      loadPendingQuestions(ctx, conversationId, { includeAsks: true }),
+      sessionWorkers(ctx),
+    ]);
+    const worker = workers.workerFor(conversationId);
     const { messages, omitted } = shapeMessages(conversation?.messages, {
       maxChars: args.max_chars,
       includeActivity: args.include_activity,
@@ -1798,6 +1844,7 @@ export function createRemoteRelayDispatcher({
         ...(via ? { via } : {}),
         ...(conversation?.mayCreateSessions === true ? { mayCreateSessions: true } : {}),
         ...(conversation?.planOnly === true ? { planOnly: true } : {}),
+        ...workerField(worker),
         ...(group ? { group } : {}),
         openUrl: remoteConversationUrl(ctx.relay.url, conversationId) || null,
       },

package/server/services/session-activity-tracker.mjs

@@ -83,7 +83,7 @@ export function resolveBootReportingWindowMs(env = process.env, log = () => {})
 export const UNREPORTED_BACKGROUND_TASK_ID = 'unreported-background-work';
 // It stays until the worker reports, and a worker on older code never does
 // for a task that prints nothing: the way out is the session's Kill session.
-export const UNREPORTED_BACKGROUND_TASK_DESCRIPTION = 'background work, details not reported since the relay restarted (Kill session in its ⋯ menu ends it)';
+export const UNREPORTED_BACKGROUND_TASK_DESCRIPTION = 'background work, details not reported since the relay restarted (Kill session in its ︙ menu ends it)';
 // A `running` background run row that its worker's task set does not list
 // and nothing has updated for this long is not running work (see
 // `runListedByWorker`).

package/server/services/session-launch-mutex-service.mjs

@@ -0,0 +1,23 @@
+/**
+ * The per-session launch mutex, one for the whole relay process, keyed by
+ * sdkSessionId.
+ *
+ * What runs under it: the Change CWD relaunch, the launch route and the
+ * delete and archive routes (sessions-routes.mjs), the worker recycle's
+ * recycle, archived stop and launch retry (worker-recycle-service.mjs), and
+ * the archived sweep at relay start (sweepArchivedWorkers there). What does
+ * not: the Claude billing swap, which keeps the old worker away from the
+ * rows by its own delivery fence; the send path's ensureWorker; and the kill
+ * route (messages-routes.mjs), so a Kill that lands in a recycle's window
+ * between its stop and its launch (about 3 s) can look undone, because the
+ * launch starts a fresh idle worker. No turn is cut by that. Taking the
+ * mutex in those places is a follow-up.
+ *
+ * It used to be built inside the sessions routes' registration, where nothing
+ * outside the routes could take it, so a recycle could stop a worker while a
+ * relaunch was half done. A module singleton is the one place both import.
+ */
+
+import { createKeyedMutex } from './keyed-mutex-service.mjs';
+
+export const sessionLaunchMutex = createKeyedMutex();

package/server/services/session-worker-registry-service.mjs

@@ -1,5 +1,7 @@
 'use strict';
 
+import { normalizeBuildId } from '../../shared/build-id.mjs';
+
 function normalizeSessionId(value) {
   const text = String(value || '').trim();
   return text || null;
@@ -141,6 +143,12 @@ export function normalizeWorkerSteeringSnapshot(raw) {
     const tasks = Number(raw.backgroundTaskCount);
     snapshot.backgroundTaskCount = Number.isInteger(tasks) && tasks > 0 ? Math.min(tasks, 999) : 0;
   }
+  // Whether the worker's CLI child still runs. The recycle replaces a worker
+  // only once that child is gone, because a live CLI may hold scheduled jobs
+  // of its own (CronCreate, ScheduleWakeup) that a stop would lose. Only a
+  // worker that reports it carries the key: an absent key means "cannot say",
+  // which the recycle treats differently from false.
+  if (typeof raw.cliAlive === 'boolean') snapshot.cliAlive = raw.cliAlive;
   return snapshot;
 }
 
@@ -160,6 +168,12 @@ function sanitizeState(raw = null) {
     lastError: String(base.lastError || '').trim() || null,
     steering: normalizeWorkerSteeringSnapshot(base.steering),
     kind: normalizeWorkerKind(base.kind),
+    // The build id of the code the worker process started on, as it stated
+    // it; null when it did not say or said something malformed.
+    buildId: normalizeBuildId(base.buildId),
+    // When the relay first heard of the current pid: the process's age as far
+    // as the relay can tell, without asking the OS. Null without a pid.
+    startedAt: normalizeTimestamp(base.startedAt, null),
     updatedAt: normalizeTimestamp(base.updatedAt, nowIso),
     createdAt,
   };
@@ -266,11 +280,33 @@ export function createSessionWorkerRegistry() {
     // heartbeated — indefinitely, if none ever did.
     // The kind is the same: a process's own word. An upsert that does not
     // state it keeps the known one, unless the process it described is gone.
+    // The build id and the start time follow the same rule.
     const statesKind = Object.prototype.hasOwnProperty.call(rawState, 'kind');
+    const statesBuildId = Object.prototype.hasOwnProperty.call(rawState, 'buildId');
+    const statesStartedAt = Object.prototype.hasOwnProperty.call(rawState, 'startedAt');
     if (!statesKind) next.kind = existing?.kind || null;
+    if (!statesBuildId) next.buildId = existing?.buildId || null;
+    if (!statesStartedAt) next.startedAt = existing?.startedAt || null;
     if (existing && isWorkerReplacedOrGone(existing, next)) {
       next.steering = null;
       if (!statesKind) next.kind = null;
+      if (!statesBuildId) next.buildId = null;
+      if (!statesStartedAt) next.startedAt = null;
+    }
+    // A pid the entry did not have before is the first the relay hears of
+    // that process, so its start time is now. Callers spread the old entry
+    // into the upsert that brings the new pid, so a stated startedAt counts
+    // only when it is not the old process's own.
+    // The build id is that process's own word as well, and it arrives with its
+    // hello or heartbeat, not with the upsert that brings the pid. A build id
+    // equal to the old entry's is the old process's, spread in, and would make
+    // a recycled worker read as the old code (or an old worker as current)
+    // until the new one spoke; it is dropped. A different one was stated for
+    // this process and stays.
+    if (next.pid && next.pid !== existing?.pid) {
+      const statedOwnStart = statesStartedAt && next.startedAt && next.startedAt !== existing?.startedAt;
+      if (!statedOwnStart) next.startedAt = new Date().toISOString();
+      if (next.buildId && next.buildId === existing?.buildId) next.buildId = null;
     }
     clearIndexesForSession(existing);
     bySession.set(state.sdkSessionId, next);

package/server/services/session-worker-websocket-service.mjs

@@ -1,6 +1,7 @@
 'use strict';
 
 import { URL } from 'url';
+import { normalizeBuildId } from '../../shared/build-id.mjs';
 
 function normalizeText(value) {
   const text = String(value || '').trim();
@@ -111,6 +112,13 @@ export function createSessionWorkerWebSocketService({
   // A worker stated its kind ("claude", "claude-api", ...) in a hello or a
   // ready: (sessionId, kind, pid). The registry keeps it.
   onWorkerKind = () => {},
+  // A worker stated the build id of the code it started on, in a hello or a
+  // ready: (sessionId, buildId, pid). The registry keeps it, so the relay can
+  // tell a worker on older code from one that runs what the relay runs.
+  onWorkerBuildId = () => {},
+  // The relay's own build id, sent in every server.hello so a worker can log
+  // that it runs other code; null leaves the key out.
+  buildId = null,
   // Liveness probe for a socket's worker process. A killed worker's socket
   // survives its process by an event-loop turn (the kill path blocks the loop,
   // so this service's timer runs before the socket close lands), and asking for
@@ -136,6 +144,8 @@ export function createSessionWorkerWebSocketService({
   const logDebug = typeof logger?.debug === 'function' ? logger.debug.bind(logger) : noop;
   const logWarn = typeof logger?.warn === 'function' ? logger.warn.bind(logger) : noop;
 
+  const relayBuildId = normalizeBuildId(buildId);
+
   let interval = null;
   let lastSnapshot = null;
   let attached = false;
@@ -245,6 +255,32 @@ export function createSessionWorkerWebSocketService({
     try { onWorkerKind(meta.sessionId, kind, meta.pid); } catch {}
   }
 
+  // The build id a hello or ready states, kept and reported like the kind. A
+  // value that is malformed or too long is ignored rather than stored, since
+  // the worker side is not trusted to bound what it sends.
+  function noteWorkerBuildId(meta, payload) {
+    const workerBuildId = normalizeBuildId(payload?.buildId);
+    if (!workerBuildId || meta.workerBuildId === workerBuildId) return;
+    meta.workerBuildId = workerBuildId;
+    try { onWorkerBuildId(meta.sessionId, workerBuildId, meta.pid); } catch {}
+  }
+
+  // Both server.hello frames (on connection and the ack of worker.hello) carry
+  // the same fields; an unknown relay build id leaves its key out rather than
+  // sending null.
+  function serverHello(reason, sessionId) {
+    const event = {
+      type: 'server.hello',
+      reason,
+      capabilities: WORKER_PROTOCOL_CAPABILITIES,
+      queue: normalizeQueueSnapshot(queueCounts()),
+      sessionId,
+      timestamp: nowIso(),
+    };
+    if (relayBuildId) event.buildId = relayBuildId;
+    return event;
+  }
+
   function noteHeartbeat(meta, reason = 'worker-message') {
     if (!meta?.sessionId) return;
     try {
@@ -397,14 +433,7 @@ export function createSessionWorkerWebSocketService({
     });
     if (identity.sessionId) supersedePriorSockets(ws, identity.sessionId);
     touchCli();
-    emitEvent(ws, {
-      type: 'server.hello',
-      reason: 'connected',
-      capabilities: WORKER_PROTOCOL_CAPABILITIES,
-      queue: normalizeQueueSnapshot(queueCounts()),
-      sessionId: identity.sessionId,
-      timestamp: nowIso(),
-    });
+    emitEvent(ws, serverHello('connected', identity.sessionId));
 
     ws.on('message', (raw) => {
       touchCli();
@@ -420,19 +449,13 @@ export function createSessionWorkerWebSocketService({
         if (!bindSocketIdentity(ws, meta, payload)) return;
         meta.lastSeenAtMs = nowMsSafe();
         noteWorkerKind(meta, payload);
+        noteWorkerBuildId(meta, payload);
         // A worker holding deliveries (open question card, compaction) binds
         // its identity with hello but must not be handed work.
         meta.ready = payload.ready !== false;
         meta.lastHelloAt = nowIso();
         noteHeartbeat(meta, 'worker-hello');
-        emitEvent(ws, {
-          type: 'server.hello',
-          reason: 'ack',
-          capabilities: WORKER_PROTOCOL_CAPABILITIES,
-          queue: normalizeQueueSnapshot(queueCounts()),
-          sessionId: meta.sessionId,
-          timestamp: nowIso(),
-        });
+        emitEvent(ws, serverHello('ack', meta.sessionId));
         void maybeDeliverToSocket(ws, 'worker-hello');
         return;
       }
@@ -440,6 +463,7 @@ export function createSessionWorkerWebSocketService({
         if (!bindSocketIdentity(ws, meta, payload)) return;
         meta.lastSeenAtMs = nowMsSafe();
         noteWorkerKind(meta, payload);
+        noteWorkerBuildId(meta, payload);
         meta.ready = true;
         meta.lastReadyAt = nowIso();
         noteHeartbeat(meta, String(payload.reason || 'worker-ready'));