@oar-sh/oar 0.9.20 → 0.9.22

package/server/routes/sessions-routes.mjs

@@ -43,6 +43,7 @@ import { readStoredClaudeContextUsage, resolveAutoCompactWindowStatus } from '..
 import { buildContextUsageView } from '../services/context-usage-view.mjs';
 import { cleanupGeneratedImagesForConversation as cleanupGeneratedImagesForConversationDefault } from '../services/generated-image-cleanup-service.mjs';
 import { stopSessionWorkerProcesses } from '../services/session-worker-stop-service.mjs';
+import { isWorkerOnOtherBuild } from '../services/worker-recycle-service.mjs';
 import { createClaudeAuthService } from '../services/claude-auth-service.mjs';
 import { createCliInstallService } from '../services/cli-install-service.mjs';
 import { createGrokAuthService } from '../services/grok-auth-service.mjs';
@@ -69,7 +70,7 @@ import { redactPreviewTokens } from '../services/preview-registry-service.mjs';
 import { createSharePathRedactor } from '../services/share-path-redaction.mjs';
 import { createSubagentTimelineService, TIMELINE_READ_DEFAULT_LIMIT } from '../services/subagent-timeline-service.mjs';
 import { sanitizeSubagentRunId } from '../../shared/subagent-run-id.mjs';
-import { createKeyedMutex } from '../services/keyed-mutex-service.mjs';
+import { sessionLaunchMutex } from '../services/session-launch-mutex-service.mjs';
 import { createFixedWindowRateLimiter } from '../services/rate-limit-service.mjs';
 import { requestHostOf } from '../services/request-boundary.mjs';
 import { parseRemoteRelayOriginJson } from '../repositories/remote-relay-repository.mjs';
@@ -77,6 +78,7 @@ import { createShareRepository } from '../repositories/share-repository.mjs';
 import { createShareAccessLog, readShareRequestViewer } from '../services/share-access-log-service.mjs';
 import { GROUP_NAME_MAX, resolveGroupDisplayName } from '../repositories/conversation-group-repository.mjs';
 import { conversationGroupPosition, conversationGroupRole, groupActivityTimes, pageConversationRows } from '../public/app/conversation-group-order.mjs';
+import { detailsVersion, parseHeldDetailsParam, thoughtFingerprint } from '../public/app/held-details.mjs';
 import { admitRemoteRelayRequest, publicRemoteRelayOrigin, readRemoteRelayRequest } from '../services/remote-relay-inbound.mjs';
 import {
   RELAUNCH_COALESCE_WINDOW_MS,
@@ -1452,11 +1454,27 @@ export function buildSessionWorkerStatusPayload({
   // What the last read of the process list did ({ at, durationMs, processes,
   // error }); a slow box shows here before it shows as 502s.
   processList = null,
+  // The relay's own build id (runtimeState.buildId): each worker's is shown
+  // next to it, so a worker still on the code before a deploy stands out.
+  relayBuildId = null,
+  // The worker recycle's status() (stale count, recycles in flight and done,
+  // who waits on what), or null while the recycle is not wired.
+  recycleStatus = null,
 } = {}) {
   const snapshot = supervisorSnapshot && typeof supervisorSnapshot === 'object'
     ? supervisorSnapshot
     : {};
-  const workers = Array.isArray(snapshot.workers) ? snapshot.workers : [];
+  const relayBuild = typeof relayBuildId === 'string' && relayBuildId.trim() ? relayBuildId.trim() : null;
+  const workers = (Array.isArray(snapshot.workers) ? snapshot.workers : []).map((worker) => (
+    worker && typeof worker === 'object'
+      ? {
+        ...worker,
+        buildId: typeof worker.buildId === 'string' && worker.buildId ? worker.buildId : null,
+        startedAt: worker.startedAt || null,
+        stale: isWorkerOnOtherBuild(worker, relayBuild),
+      }
+      : worker
+  ));
   const onlineWorkerStatuses = new Set(SESSION_WORKER_ACTIVE_STATUSES);
   const onlineCount = workers.reduce((count, worker) => {
     const status = normalizeWorkerStatusText(worker?.status, 'new');
@@ -1561,6 +1579,8 @@ export function buildSessionWorkerStatusPayload({
     onlineUnassignedProcessCount,
     counts: snapshot.counts && typeof snapshot.counts === 'object' ? snapshot.counts : {},
     workers,
+    buildId: relayBuild,
+    recycle: recycleStatus && typeof recycleStatus === 'object' ? recycleStatus : null,
     pendingStarts: toSafeNonNegativeInt(snapshot.pendingStarts, 0),
     lifecycle: Array.isArray(snapshot.lifecycle) ? snapshot.lifecycle : [],
     integrity,
@@ -2608,9 +2628,10 @@ export function registerSessionsRoutes(app, deps) {
     bucketTtlMs: 60_000,
     maxBuckets: 4_096,
   });
-  // Serializes everything that reaches the session-worker launch path, keyed by
-  // sdkSessionId, plus a coalescer so a duplicated request replays one result.
-  const sessionLaunchMutex = createKeyedMutex();
+  // Everything that reaches the session-worker launch path is serialized by
+  // the relay-wide sessionLaunchMutex (services/session-launch-mutex-service.mjs),
+  // keyed by sdkSessionId, which the worker recycle takes too; plus a
+  // coalescer so a duplicated request replays one result.
   const relaunchCoalescer = createRelaunchCoalescer({ windowMs: RELAUNCH_COALESCE_WINDOW_MS });
 
   function getDeepSeekProviderSettings() {
@@ -4686,6 +4707,59 @@ export function registerSessionsRoutes(app, deps) {
   });
 
   // GET /api/conversation/:id — get paginated conversation history
+  // The details version of each finished reply (held-details.mjs), one query
+  // per table for the whole window (the ids go in as one JSON parameter).
+  // Tool rows are only inserted: their count and highest row id. Thought rows
+  // are updated in place as they stream and when done: their rows are read
+  // (id, done, text) and fingerprinted, so any change shows.
+  let detailsVersionStatements = null;
+  function relayDetailsVersions(messageIds) {
+    const ids = [...new Set(messageIds.filter(Boolean))];
+    const versions = new Map();
+    if (!ids.length || typeof db?.prepare !== 'function') return versions;
+    try {
+      detailsVersionStatements ||= {
+        activity: db.prepare(`
+          SELECT response_message_id AS id, COUNT(*) AS n, MAX(rowid) AS max_rowid
+          FROM relay_activity
+          WHERE response_message_id IN (SELECT value FROM json_each(?))
+          GROUP BY response_message_id
+        `),
+        thought: db.prepare(`
+          SELECT response_message_id AS response_id, rowid AS row_id, done, text
+          FROM relay_thought
+          WHERE response_message_id IN (SELECT value FROM json_each(?))
+        `),
+      };
+      const idsJson = JSON.stringify(ids);
+      const activity = new Map();
+      for (const row of detailsVersionStatements.activity.all(idsJson) || []) {
+        activity.set(String(row?.id || ''), { n: Number(row.n) || 0, max: Number(row.max_rowid) || 0 });
+      }
+      const thoughtRows = new Map();
+      for (const row of detailsVersionStatements.thought.all(idsJson) || []) {
+        const id = String(row?.response_id || '');
+        if (!thoughtRows.has(id)) thoughtRows.set(id, []);
+        thoughtRows.get(id).push({ rowid: row.row_id, done: row.done, text: row.text });
+      }
+      for (const id of ids) {
+        const tools = activity.get(id) || { n: 0, max: 0 };
+        const thoughts = thoughtRows.get(id) || [];
+        versions.set(id, detailsVersion({
+          activityCount: tools.n,
+          activityMaxRowId: tools.max,
+          thoughtCount: thoughts.length,
+          thoughtMaxRowId: thoughts.reduce((max, row) => Math.max(max, Number(row.rowid) || 0), 0),
+          thoughtHash: thoughtFingerprint(thoughts),
+        }));
+      }
+    } catch {
+      // No versions: nothing is held, everything goes out in full.
+      return new Map();
+    }
+    return versions;
+  }
+
   app.get('/api/conversation/:id', auth, (req, res) => {
     const requestedId = String(req.params.id || '').trim();
     const limit = normalizeConversationHistoryLimit(req.query.limit, 20);
@@ -4709,7 +4783,9 @@ export function registerSessionsRoutes(app, deps) {
       supportedRelayModes: SUPPORTED_RELAY_MODES,
       defaultRelayMode: DEFAULT_RELAY_MODE,
     });
-    const inFlight = inFlightStateForConversation(resolvedConversationId);
+    // The running turn's tool lines: as many per thread as this page shows
+    // (its "Tool lines shown" setting); none given, the relay's default.
+    const inFlight = inFlightStateForConversation(resolvedConversationId, { activityCap: req.query.activityCap });
     const sdkSessionId = String(conv.sdk_session_id || resolvedConversationId || '').trim();
     const sessionUsageSummary = resolveSessionUsageSummaryForSdkSession({
       sdkSessionId,
@@ -4748,14 +4824,23 @@ export function registerSessionsRoutes(app, deps) {
         .map((row) => [String(row?.response_message_id || '').trim(), String(row?.id || '').trim()])
         .filter(([responseMessageId, sourceMessageId]) => !!responseMessageId && !!sourceMessageId),
     );
+    // The live poll names the finished replies whose tool lines and thoughts
+    // it holds (`heldDetails`, held-details.mjs); those whose details have
+    // not changed since (the same `detailsVersion`) go out without them.
+    const assistantIds = dbMessages.filter((m) => m.role === 'assistant').map((m) => String(m.id));
+    const detailsVersions = relayDetailsVersions(assistantIds);
+    const heldParam = parseHeldDetailsParam(req.query.heldDetails);
+    const heldNow = new Set(heldParam
+      ? assistantIds.filter((id) => heldParam.get(id) === detailsVersions.get(id))
+      : []);
     const relayActivitiesByMessageId = new Map(
       dbMessages
-        .filter((m) => m.role === 'assistant')
+        .filter((m) => m.role === 'assistant' && !heldNow.has(String(m.id)))
         .map((m) => [m.id, relayActivityForResponse(m.id)]),
     );
     const relayThoughtsByMessageId = new Map(
       dbMessages
-        .filter((m) => m.role === 'assistant')
+        .filter((m) => m.role === 'assistant' && !heldNow.has(String(m.id)))
         .map((m) => [m.id, relayThoughtsForResponse ? relayThoughtsForResponse(m.id) : []]),
     );
     const subagentRunsByMessageId = new Map(
@@ -4796,8 +4881,14 @@ export function registerSessionsRoutes(app, deps) {
     });
     messages = messages.map((message) => {
       if (message.role !== 'assistant') return message;
-      const sourceMessageId = responseMessageToSourceId.get(String(message.id || '').trim()) || message.sourceMessageId || undefined;
-      return sourceMessageId ? { ...message, sourceMessageId } : message;
+      const id = String(message.id || '').trim();
+      const sourceMessageId = responseMessageToSourceId.get(id) || message.sourceMessageId || undefined;
+      // Held: the page fills these from what it has. Otherwise the version
+      // the page keeps with them.
+      const details = heldNow.has(id)
+        ? { activities: [], thoughts: [], detailsHeld: true }
+        : (detailsVersions.has(id) ? { detailsVersion: detailsVersions.get(id) } : {});
+      return { ...message, ...(sourceMessageId ? { sourceMessageId } : {}), ...details };
     });
     const history = sqlPage
       ? { messages, pageInfo: sqlPage.pageInfo }
@@ -6099,7 +6190,7 @@ export function registerSessionsRoutes(app, deps) {
           // Kill session stops the session's worker, and the delete counts no
           // tasks for a conversation whose worker is gone.
           message: activity.unstoppableTaskCount > 0
-            ? 'Background tasks are still running in this conversation, and some have no Stop button. Use Kill session in its ⋯ menu, then delete it.'
+            ? 'Background tasks are still running in this conversation, and some have no Stop button. Use Kill session in its ︙ menu, then delete it.'
             : 'Background tasks are still running in this conversation. Stop them in the task panel, then delete it.',
         },
       };
@@ -6258,7 +6349,7 @@ export function registerSessionsRoutes(app, deps) {
               error: 'conversation-active',
               reason: 'background-tasks',
               message: activity.unstoppableTaskCount > 0
-                ? 'Background tasks are still running in this conversation, and some have no Stop button. Use Kill session in its ⋯ menu, then archive it.'
+                ? 'Background tasks are still running in this conversation, and some have no Stop button. Use Kill session in its ︙ menu, then archive it.'
                 : 'Background tasks are still running in this conversation. Stop them in the task panel, then archive it.',
             },
           };
@@ -7129,6 +7220,18 @@ export function registerSessionsRoutes(app, deps) {
     }
   });
 
+  // The worker recycle's status, read at request time: the recycle is set on
+  // runtimeState after the routes are registered, and a status that throws
+  // must not take /api/status down with it.
+  function readWorkerRecycleStatus() {
+    if (typeof runtimeState.workerRecycle !== 'function') return null;
+    try {
+      return runtimeState.workerRecycle() || null;
+    } catch {
+      return null;
+    }
+  }
+
   // GET /api/status — overall status
   app.get('/api/status', auth, (req, res) => {
     ensureSessionId(req, res);
@@ -7146,6 +7249,8 @@ export function registerSessionsRoutes(app, deps) {
       supervisorSnapshot: sessionWorkerSupervisor?.snapshot?.({ pendingQuestionSessionIds }) || null,
       queueRows: listSessionWorkerQueueRows.all(...SESSION_WORKER_STATUS_QUEUE_STATES),
       processList: sessionWorkerProcessInspector?.getLastRead?.() || null,
+      relayBuildId: runtimeState.buildId ?? null,
+      recycleStatus: readWorkerRecycleStatus(),
     });
     const activeRuntimeSessionCount = featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true
       ? Number(sessionWorkerStatus?.onlineBoundProcessCount || 0)