@oar-sh/oar 0.9.1 → 0.9.3

package/server/services/session-worker-registry-service.mjs

@@ -60,6 +60,54 @@ function normalizeStatus(value) {
   return ALLOWED_STATUSES.has(mapped) ? mapped : 'new';
 }
 
+const STEERING_HOLD_REASONS = new Set(['question', 'compaction', 'adoption', 'delivery', 'other']);
+const STEERING_MESSAGE_ID_MAX_LENGTH = 64;
+const STEERING_CANCELLABLE_IDS_MAX = 50;
+
+// Queue message ids the worker still holds pushed-but-unconsumed (the Copilot
+// SDK's queued lane): trimmed strings only, deduped, capped — a worker cannot
+// grow the status payload without bound.
+function normalizeSteeringCancellableIds(raw) {
+  if (!Array.isArray(raw)) return [];
+  const seen = new Set();
+  const ids = [];
+  for (const value of raw) {
+    if (typeof value !== 'string') continue;
+    const id = value.trim();
+    if (!id || id.length > STEERING_MESSAGE_ID_MAX_LENGTH || seen.has(id)) continue;
+    seen.add(id);
+    ids.push(id);
+    if (ids.length >= STEERING_CANCELLABLE_IDS_MAX) break;
+  }
+  return ids;
+}
+
+// The worker's composer-facing steering snapshot, published on its heartbeat.
+// Callers follow the registry convention of spreading the existing entry into
+// upserts, so an update without `steering` keeps the last known snapshot.
+// Exported so the heartbeat route compares apples to apples: normalizing there
+// with different rules would make an unknown holdReason never converge and
+// re-upsert on every heartbeat.
+//
+// `supported` is the worker's opt-in to mid-turn steering and the client
+// composer's only steering gate: a worker that never sets it — the extension
+// path, or a Claude worker old enough to send the 4-field snapshot — keeps
+// "Queue". `cancellableIds` lists the pushed rows the worker can still pull
+// back out of its runtime (un-steer).
+export function normalizeWorkerSteeringSnapshot(raw) {
+  if (!raw || typeof raw !== 'object') return null;
+  const holdReason = String(raw.holdReason || '').trim().toLowerCase();
+  const messageId = String(raw.messageId || '').trim().slice(0, STEERING_MESSAGE_ID_MAX_LENGTH);
+  return {
+    turnActive: raw.turnActive === true,
+    canSteer: raw.canSteer === true,
+    holdReason: STEERING_HOLD_REASONS.has(holdReason) ? holdReason : null,
+    messageId: messageId || null,
+    supported: raw.supported === true,
+    cancellableIds: normalizeSteeringCancellableIds(raw.cancellableIds),
+  };
+}
+
 function sanitizeState(raw = null) {
   const nowIso = new Date().toISOString();
   const base = raw && typeof raw === 'object' ? raw : {};
@@ -74,11 +122,23 @@ function sanitizeState(raw = null) {
     queueDepth: toNonNegativeInt(base.queueDepth, 0),
     retryCount: toNonNegativeInt(base.retryCount, 0),
     lastError: String(base.lastError || '').trim() || null,
+    steering: normalizeWorkerSteeringSnapshot(base.steering),
     updatedAt: normalizeTimestamp(base.updatedAt, nowIso),
     createdAt,
   };
 }
 
+// Statuses that mean the process a snapshot described is gone or not yet
+// replaced by one that has reported. 'error' counts only on the way IN: a
+// live worker marked errored (a requeue retry) keeps heartbeating, and its
+// next snapshot must stick rather than be cleared on every upsert.
+const WORKER_GONE_STATUSES = new Set(['new', 'starting', 'stopped', 'error']);
+
+function isWorkerReplacedOrGone(existing, next) {
+  if (existing.pid && next.pid !== existing.pid) return true;
+  return next.status !== existing.status && WORKER_GONE_STATUSES.has(next.status);
+}
+
 function toSnapshot(entry) {
   return Object.freeze({ ...entry });
 }
@@ -91,9 +151,18 @@ export function createSessionWorkerRegistry() {
 
   function clearIndexesForSession(entry) {
     if (!entry) return;
-    if (entry.workerId) sessionByWorkerId.delete(entry.workerId);
-    if (entry.conversationId) sessionByConversationId.delete(entry.conversationId);
-    if (entry.runtimeSessionId) sessionByRuntimeSessionId.delete(entry.runtimeSessionId);
+    // Only drop an index that still points at THIS entry's session: a newer
+    // entry may have legitimately claimed the same workerId/conversationId/
+    // runtimeSessionId since, and deleting blindly would orphan it.
+    if (entry.workerId && sessionByWorkerId.get(entry.workerId) === entry.sdkSessionId) {
+      sessionByWorkerId.delete(entry.workerId);
+    }
+    if (entry.conversationId && sessionByConversationId.get(entry.conversationId) === entry.sdkSessionId) {
+      sessionByConversationId.delete(entry.conversationId);
+    }
+    if (entry.runtimeSessionId && sessionByRuntimeSessionId.get(entry.runtimeSessionId) === entry.sdkSessionId) {
+      sessionByRuntimeSessionId.delete(entry.runtimeSessionId);
+    }
   }
 
   function updateIndexesForSession(entry) {
@@ -154,12 +223,26 @@ export function createSessionWorkerRegistry() {
       createdAt: existing?.createdAt || state.createdAt,
       updatedAt: new Date().toISOString(),
     });
+    // A steering snapshot describes one live worker process. Callers spread
+    // the old entry into every upsert, so without this a crashed worker's
+    // hold outlived it and the composer showed it until a replacement
+    // heartbeated — indefinitely, if none ever did.
+    if (existing && isWorkerReplacedOrGone(existing, next)) next.steering = null;
     clearIndexesForSession(existing);
     bySession.set(state.sdkSessionId, next);
     updateIndexesForSession(next);
     return toSnapshot(next);
   }
 
+  /** Drop a session's steering snapshot (its worker process is known dead). */
+  function clearSteering(sdkSessionId) {
+    const sessionId = normalizeSessionId(sdkSessionId);
+    const existing = sessionId ? bySession.get(sessionId) : null;
+    if (!existing?.steering) return false;
+    bySession.set(sessionId, { ...existing, steering: null, updatedAt: new Date().toISOString() });
+    return true;
+  }
+
   function removeWorker(sdkSessionId) {
     const sessionId = normalizeSessionId(sdkSessionId);
     if (!sessionId) return false;
@@ -169,6 +252,34 @@ export function createSessionWorkerRegistry() {
     return bySession.delete(sessionId);
   }
 
+  // Placeholder→real session rekey (audit #22). Session sync migrates the DB
+  // side (conversation binding, runtime session, queue owners) in one
+  // transaction; this is the in-memory half, moving the registry entry and its
+  // secondary indexes so the placeholder never lingers as a phantom worker.
+  function rekeyWorker(fromSdkSessionId, toSdkSessionId) {
+    const from = normalizeSessionId(fromSdkSessionId);
+    const to = normalizeSessionId(toSdkSessionId);
+    if (!from || !to || from === to) return null;
+    const placeholder = bySession.get(from);
+    if (!placeholder) return null;
+    const existingTarget = bySession.get(to) || null;
+    // An entry already registered under the real id is newer than the
+    // placeholder — its fields win, the placeholder only backfills gaps.
+    const next = sanitizeState({
+      ...placeholder,
+      ...(existingTarget || {}),
+      sdkSessionId: to,
+      createdAt: placeholder.createdAt,
+      updatedAt: new Date().toISOString(),
+    });
+    clearIndexesForSession(placeholder);
+    if (existingTarget) clearIndexesForSession(existingTarget);
+    bySession.delete(from);
+    bySession.set(to, next);
+    updateIndexesForSession(next);
+    return toSnapshot(next);
+  }
+
   function clearAll() {
     bySession.clear();
     sessionByWorkerId.clear();
@@ -187,7 +298,9 @@ export function createSessionWorkerRegistry() {
     getWorkerByRuntimeSessionId,
     listWorkers,
     upsertWorker,
+    clearSteering,
     removeWorker,
+    rekeyWorker,
     clearAll,
   };
 }

package/server/services/session-worker-supervisor-service.mjs

@@ -482,10 +482,16 @@ export function createSessionWorkerSupervisor({
     if (current && !isTransitionAllowed(current.status, requestedStatus)) {
       return current;
     }
+    // The steering snapshot is the worker's to set (its heartbeat) and the
+    // registry's to clear (process gone or replaced). Callers build nextState
+    // from an older copy of the entry — ensureWorker from the 'starting' one,
+    // taken before the new process heartbeated — so passing theirs through
+    // would overwrite the snapshot the worker sent in the meantime.
+    const { steering: _staleSteering, ...patch } = nextState;
     const next = registry.upsertWorker({
       ...(current || {}),
       sdkSessionId: sessionId,
-      ...nextState,
+      ...patch,
       status: requestedStatus,
       updatedAt: nowIso(),
     });

package/server/services/session-worker-websocket-service.mjs

@@ -85,6 +85,19 @@ function queueSnapshotChanged(left, right) {
     || left.parkedCount !== right.parkedCount;
 }
 
+// The worker link heartbeats every ~10s (readyRefreshMs in
+// worker-websocket-link.mjs) and itself abandons a connection after 30s of
+// server silence (staleConnectionMs). Mirror that on this side: three missed
+// heartbeats and a retained socket no longer proves the worker is alive.
+const WORKER_SOCKET_LIVENESS_MAX_SILENCE_MS = 30_000;
+
+// Protocol features this relay understands, advertised in server.hello. A
+// worker that updated on disk before the relay restarted must not rely on
+// them otherwise: an older relay ignores worker.unready / hello ready:false
+// and treats a `steering-held` requeue as a failure (retry, backoff,
+// worker marked errored).
+export const WORKER_PROTOCOL_CAPABILITIES = Object.freeze(['worker-unready', 'steering-held']);
+
 export function createSessionWorkerWebSocketService({
   WebSocketServerImpl,
   httpServer,
@@ -104,6 +117,7 @@ export function createSessionWorkerWebSocketService({
   pathPrefix = '',
   pollIntervalMs = 1000,
   nowIso = () => new Date().toISOString(),
+  nowMs = () => Date.now(),
   logger = console,
 } = {}) {
   if (!WebSocketServerImpl) throw new Error('createSessionWorkerWebSocketService requires WebSocketServerImpl');
@@ -123,6 +137,66 @@ export function createSessionWorkerWebSocketService({
   let lastSnapshot = null;
   let attached = false;
 
+  function nowMsSafe() {
+    const value = Number(nowMs());
+    return Number.isFinite(value) ? value : Date.now();
+  }
+
+  function removeSocket(socket, reason) {
+    const meta = clientState.get(socket);
+    clients.delete(socket);
+    clientState.delete(socket);
+    // Death-detection hook: a worker socket closing while its session owns
+    // an in-flight processing row used to be silently forgotten, leaving
+    // the row to the 600s stale sweep. The handler PID-probes and recovers.
+    if (meta?.sessionId) {
+      Promise.resolve(onWorkerSocketClosed({
+        sessionId: meta.sessionId,
+        pid: meta.pid || null,
+        reason,
+      })).catch(() => {});
+    }
+  }
+
+  function supersedePriorSockets(ws, sessionId) {
+    for (const socket of Array.from(clients)) {
+      if (socket === ws) continue;
+      const meta = clientState.get(socket);
+      if (meta?.sessionId !== sessionId) continue;
+      // One authoritative socket generation per session: the newest connection
+      // wins, and the prior socket is dropped synchronously so it can neither
+      // be handed work nor count as liveness while its close event is in
+      // flight. Before this, multiple ready sockets could claim one session
+      // and each pull rows for it.
+      logWarn(`[worker-ws] superseding prior socket for ${sessionId.slice(0, 8)}`);
+      removeSocket(socket, 'superseded');
+      try { socket.close(); } catch {}
+    }
+  }
+
+  function bindSocketIdentity(ws, meta, payload) {
+    const claimedSessionId = normalizeText(payload.sessionId);
+    const claimedPid = normalizePositiveInt(payload.pid);
+    // A socket's identity binds once (upgrade URL or first identifying frame).
+    // A later frame claiming a different session or PID is a misidentified or
+    // rekeyed worker; closing is safer than ignoring the frame — the worker
+    // reconnects and re-binds atomically instead of keeping a socket that
+    // receives one session's deliveries while claiming to be another.
+    if ((meta.sessionId && claimedSessionId && claimedSessionId !== meta.sessionId)
+      || (meta.pid && claimedPid && claimedPid !== meta.pid)) {
+      logWarn(`[worker-ws] identity rebind rejected for ${String(meta.sessionId || 'unknown').slice(0, 8)} claimed=${String(claimedSessionId || 'none').slice(0, 8)}`);
+      removeSocket(ws, 'identity-rebind');
+      try { ws.close(); } catch {}
+      return false;
+    }
+    if (!meta.sessionId && claimedSessionId) {
+      meta.sessionId = claimedSessionId;
+      supersedePriorSockets(ws, claimedSessionId);
+    }
+    if (!meta.pid && claimedPid) meta.pid = claimedPid;
+    return true;
+  }
+
   function toQueueChangedEvent(reason = 'queue-update') {
     const snapshot = normalizeQueueSnapshot(queueCounts());
     return {
@@ -302,13 +376,16 @@ export function createSessionWorkerWebSocketService({
       ready: false,
       delivering: false,
       connectedAt: nowIso(),
+      lastSeenAtMs: nowMsSafe(),
       lastDeliveredAt: null,
       deliveryFailures: 0,
     });
+    if (identity.sessionId) supersedePriorSockets(ws, identity.sessionId);
     touchCli();
     emitEvent(ws, {
       type: 'server.hello',
       reason: 'connected',
+      capabilities: WORKER_PROTOCOL_CAPABILITIES,
       queue: normalizeQueueSnapshot(queueCounts()),
       sessionId: identity.sessionId,
       timestamp: nowIso(),
@@ -325,14 +402,17 @@ export function createSessionWorkerWebSocketService({
       const meta = clientState.get(ws);
       if (!meta || !payload || typeof payload !== 'object') return;
       if (payload.type === 'worker.hello') {
-        meta.sessionId = normalizeText(payload.sessionId) || meta.sessionId;
-        meta.pid = normalizePositiveInt(payload.pid) || meta.pid;
-        meta.ready = true;
+        if (!bindSocketIdentity(ws, meta, payload)) return;
+        meta.lastSeenAtMs = nowMsSafe();
+        // A worker holding deliveries (open question card, compaction) binds
+        // its identity with hello but must not be handed work.
+        meta.ready = payload.ready !== false;
         meta.lastHelloAt = nowIso();
         noteHeartbeat(meta, 'worker-hello');
         emitEvent(ws, {
           type: 'server.hello',
           reason: 'ack',
+          capabilities: WORKER_PROTOCOL_CAPABILITIES,
           queue: normalizeQueueSnapshot(queueCounts()),
           sessionId: meta.sessionId,
           timestamp: nowIso(),
@@ -341,17 +421,28 @@ export function createSessionWorkerWebSocketService({
         return;
       }
       if (payload.type === 'worker.ready') {
-        meta.sessionId = normalizeText(payload.sessionId) || meta.sessionId;
-        meta.pid = normalizePositiveInt(payload.pid) || meta.pid;
+        if (!bindSocketIdentity(ws, meta, payload)) return;
+        meta.lastSeenAtMs = nowMsSafe();
         meta.ready = true;
         meta.lastReadyAt = nowIso();
         noteHeartbeat(meta, String(payload.reason || 'worker-ready'));
         void maybeDeliverToSocket(ws, String(payload.reason || 'worker-ready'));
         return;
       }
+      if (payload.type === 'worker.unready') {
+        // Readiness is otherwise sticky until the next delivery: a ready sent
+        // while the worker could steer would still stand after a hold began,
+        // and the delivery it drew would only be handed back.
+        if (!bindSocketIdentity(ws, meta, payload)) return;
+        meta.lastSeenAtMs = nowMsSafe();
+        meta.ready = false;
+        meta.lastUnreadyAt = nowIso();
+        noteHeartbeat(meta, String(payload.reason || 'worker-unready'));
+        return;
+      }
       if (payload.type === 'worker.ping') {
-        meta.sessionId = normalizeText(payload.sessionId) || meta.sessionId;
-        meta.pid = normalizePositiveInt(payload.pid) || meta.pid;
+        if (!bindSocketIdentity(ws, meta, payload)) return;
+        meta.lastSeenAtMs = nowMsSafe();
         meta.lastPingAt = nowIso();
         noteHeartbeat(meta, String(payload.reason || 'worker-ping'));
         emitEvent(ws, {
@@ -362,23 +453,8 @@ export function createSessionWorkerWebSocketService({
         });
       }
     });
-    const dropSocket = (reason) => {
-      const meta = clientState.get(ws);
-      clients.delete(ws);
-      clientState.delete(ws);
-      // Death-detection hook: a worker socket closing while its session owns
-      // an in-flight processing row used to be silently forgotten, leaving
-      // the row to the 600s stale sweep. The handler PID-probes and recovers.
-      if (meta?.sessionId) {
-        Promise.resolve(onWorkerSocketClosed({
-          sessionId: meta.sessionId,
-          pid: meta.pid || null,
-          reason,
-        })).catch(() => {});
-      }
-    };
-    ws.on('close', () => dropSocket('close'));
-    ws.on('error', () => dropSocket('error'));
+    ws.on('close', () => removeSocket(ws, 'close'));
+    ws.on('error', () => removeSocket(ws, 'error'));
   }
 
   function hasWorkerSocket(sessionId) {
@@ -390,6 +466,24 @@ export function createSessionWorkerWebSocketService({
     return false;
   }
 
+  /**
+   * Socket-backed liveness for dead-worker recovery. Mere socket retention is
+   * not proof of life — a dead worker's socket lingers until its close event
+   * lands — so the socket must have been heard from within the liveness
+   * window, or its claimed PID must probe alive.
+   */
+  function hasLiveWorkerSocket(sessionId) {
+    const wanted = normalizeText(sessionId);
+    if (!wanted) return false;
+    const cutoffMs = nowMsSafe() - WORKER_SOCKET_LIVENESS_MAX_SILENCE_MS;
+    for (const meta of clientState.values()) {
+      if (meta?.sessionId !== wanted) continue;
+      if (Number(meta.lastSeenAtMs || 0) >= cutoffMs) return true;
+      if (meta.pid && isWorkerProcessAlive(meta.pid) === true) return true;
+    }
+    return false;
+  }
+
   function start() {
     if (!attached) {
       httpServer.on('upgrade', onUpgrade);
@@ -479,5 +573,6 @@ export function createSessionWorkerWebSocketService({
     handleUpgrade,
     sendControlToSession,
     hasWorkerSocket,
+    hasLiveWorkerSocket,
   };
 }

package/server/services/status-event-service.mjs

@@ -91,9 +91,10 @@ export function createStatusEventService(db, {
     const normalizedConversationId = String(conversationId || '').trim();
     const dedupeKey = `${token}:${ip}`;
     const previousSeenAt = sharedAccessDedupe.get(dedupeKey);
-    sharedAccessDedupe.set(dedupeKey, now);
-    pruneSharedAccessDedupe(now);
     if (Number.isFinite(previousSeenAt) && (now - previousSeenAt) <= normalizedDedupeTtlMs) {
+      // A dedupe hit slides the window — a persisted event already backs it.
+      sharedAccessDedupe.set(dedupeKey, now);
+      pruneSharedAccessDedupe(now);
       return { event: null, deduped: true };
     }
 
@@ -113,6 +114,11 @@ export function createStatusEventService(db, {
       deleteExpiredEvents.run(normalizedMaxEvents);
     });
     persist();
+    // Committed only after the insert lands: marking the key first made a
+    // failed insert report the immediate retry as deduplicated with nothing
+    // persisted.
+    sharedAccessDedupe.set(dedupeKey, now);
+    pruneSharedAccessDedupe(now);
     return { event, deduped: false };
   }
 

package/server/services/steer-resend.mjs

@@ -0,0 +1,45 @@
+// Resend of a steer a Stop cut off unanswered (kind='stopped'): shared by the
+// POST /api/message guard and the transcript payload, so "Resent" on screen
+// and the relay's refusal of a second Resend agree.
+
+/**
+ * Whether an earlier Resend still stands. A cancelled queued message loses
+ * its queue row; a failed one keeps it as 'failed'; a finished one may have
+ * been pruned from the queue, which its answer (linked by source) proves.
+ */
+export function isLiveResend({ hasQueueRow = false, queueStatus = null, answered = false } = {}) {
+  if (hasQueueRow) {
+    return !['failed', 'cancelled'].includes(String(queueStatus || '').trim().toLowerCase());
+  }
+  return !!answered;
+}
+
+/**
+ * Original message id → the id of a Resend of it that still stands, from one
+ * conversation's messages (DB rows) and queue rows ({ id, status }).
+ */
+export function liveResendsByOriginal(dbMessages = [], queueRows = []) {
+  const queueById = new Map((Array.isArray(queueRows) ? queueRows : [])
+    .map((row) => [String(row?.id || '').trim(), row]));
+  const answeredSources = new Set();
+  for (const message of Array.isArray(dbMessages) ? dbMessages : []) {
+    if (message?.role !== 'assistant') continue;
+    const source = String(message?.source_message_id || '').trim();
+    if (source) answeredSources.add(source);
+  }
+  for (const row of queueById.values()) {
+    if (row?.response_message_id) answeredSources.add(String(row.id || '').trim());
+  }
+  const result = new Map();
+  for (const message of Array.isArray(dbMessages) ? dbMessages : []) {
+    if (message?.role !== 'user') continue;
+    const original = String(message?.resend_of_message_id || '').trim();
+    if (!original || result.has(original)) continue;
+    const id = String(message?.id || '').trim();
+    const queueRow = queueById.get(id);
+    if (isLiveResend({ hasQueueRow: !!queueRow, queueStatus: queueRow?.status, answered: answeredSources.has(id) })) {
+      result.set(original, id);
+    }
+  }
+  return result;
+}