@oar-sh/oar 0.9.9 → 0.9.11

package/server/claude-worker/claude-session-process.mjs

@@ -38,10 +38,12 @@ import {
   createClaudeTurnPublisher,
   isStaleAttemptError,
 } from './claude-turn-publisher.mjs';
+import { createSubagentTimelineRecorder } from './subagent-timeline-recorder.mjs';
 import { createAskUserBridge } from '../../shared/ask-user-bridge.mjs';
 import { createRemoteRelayToolGate } from '../../shared/remote-relay-tool-core.mjs';
 import { buildRelayStopFailure } from '../../shared/relay-stop-failure.mjs';
 import { createTurnLiveness, resolveTurnStallWindows } from '../../shared/worker-runtime/turn-liveness.mjs';
+import { createKeptPosts } from '../../shared/worker-runtime/kept-posts.mjs';
 import {
   ANSWERED_ELSEWHERE_KIND,
   DELIVERY_ANSWERED_ELSEWHERE_TEXT,
@@ -49,6 +51,7 @@ import {
   STEER_STOPPED_TEXT,
 } from '../../shared/steer-settle-markers.mjs';
 import { stripSteerNote, withSteerNoteContent } from '../../shared/steer-note.mjs';
+import { isRelayEventDelivery } from '../../shared/relay-events.mjs';
 import {
   stripSlashCommandGuard,
   stripSlashCommandGuardContent,
@@ -544,11 +547,19 @@ export function createClaudeSessionRunner({
   // the socket link: a hold withdraws the relay's readiness, and its end
   // re-arms it at once so a held message steers into the resumed turn.
   onDeliveryReadinessChange = () => {},
+  // The relay turned this worker's newest task set away (`stale`): the
+  // worker asks for a heartbeat, on which the relay learns which process
+  // holds the session and asks for the set again.
+  onBackgroundTaskSetStale = () => {},
   // Whether CLI processes carry the relay's `remote_relay` tool
   // (shared/remote-relay-tool-core.mjs). Default: ask the relay.
   remoteRelayToolGate = null,
+  // Records every sub-agent's frames as its timeline, run-scoped (see
+  // subagent-timeline-recorder.mjs). Injectable for tests.
+  subagentTimelineRecorder = null,
   dbg = () => {},
 } = {}) {
+  const timelineRecorder = subagentTimelineRecorder || createSubagentTimelineRecorder({ api, conversationId: sdkSessionId, dbg });
   const publisher = createClaudeTurnPublisher({
     api,
     dbg,
@@ -598,6 +609,34 @@ export function createClaudeSessionRunner({
   // id → terminalError for settling rows the worker gave up posting; reported
   // in the heartbeat's `settleFailed` until the relay confirms it failed them.
   const settleFailedTerminals = new Map();
+  // Settle notices of background tasks whose post failed (the relay was
+  // away): sent again after the next heartbeat it answers.
+  // Every post of the task set carries its place in this worker's order:
+  // two posts can overtake each other on the way, and the relay applies the
+  // newer one only. `epoch` tells this process from one before it.
+  const taskSetEpoch = Date.now();
+  let taskSetSeq = 0;
+  const nextTaskSetOrder = () => ({ epoch: taskSetEpoch, seq: ++taskSetSeq });
+  let taskSetStaleNotedAt = 0;
+  /**
+   * The relay's answer to a set post. `stale` for the newest one means the
+   * relay holds no current set of this worker; an older one was simply
+   * overtaken. At most one extra heartbeat per ten seconds: the periodic
+   * beat repeats the request anyway.
+   */
+  function noteTaskSetAnswer(answer, order) {
+    if (answer?.stale !== true || order.seq !== taskSetSeq) return;
+    dbg('the relay dropped the newest background task set', answer.reason || '');
+    if (Date.now() - taskSetStaleNotedAt < 10_000) return;
+    taskSetStaleNotedAt = Date.now();
+    try { onBackgroundTaskSetStale(); } catch {}
+  }
+  const postTaskSet = (body) => {
+    const order = nextTaskSetOrder();
+    return api('POST', '/api/background-tasks', { conversationId: sdkSessionId, ...body, order })
+      .then((answer) => noteTaskSetAnswer(answer, order));
+  };
+  const keptTaskSettles = createKeptPosts({ api, dbg: (...args) => dbg('background task settle:', ...args) });
   // Row id → the latest in-flight /api/queue-consumed POST touching it.
   const consumedMarkChains = new Map();
 
@@ -1688,10 +1727,19 @@ export function createClaudeSessionRunner({
    * which renders it as the composer's background-tasks panel. Membership
    * changes post immediately; chatty progress updates are trailing-edge
    * throttled. Advisory: failures must never disturb the stream consumer.
+   *
+   * `resync`: the relay asked for the set again on a heartbeat, because its
+   * own set for this worker differs (it restarted, or a post was lost). The
+   * same set, flagged, and "no tasks" is an answer too. Returns the post.
    */
-  function publishBackgroundTasks({ throttled = false } = {}) {
+  function publishBackgroundTasks({ throttled = false, resync = false } = {}) {
     const processRef = proc;
-    if (!processRef) return;
+    if (!processRef) {
+      if (!resync) return Promise.resolve();
+      return postTaskSet({ tasks: [], resync: true }).catch((error) => {
+        dbg('background task resync failed', error?.message || String(error));
+      });
+    }
     const send = () => {
       processRef.taskPublishTimer = null;
       const tasks = [...processRef.liveTasks.entries()].map(([taskId, task]) => {
@@ -1723,7 +1771,7 @@ export function createClaudeSessionRunner({
           ...(task.workflowProgress ? { workflowProgress: task.workflowProgress } : {}),
         };
       });
-      api('POST', '/api/background-tasks', { conversationId: sdkSessionId, tasks }).catch((error) => {
+      return postTaskSet({ tasks, ...(resync ? { resync: true } : {}) }).catch((error) => {
         dbg('background task publish failed', error?.message || String(error));
       });
     };
@@ -1732,12 +1780,12 @@ export function createClaudeSessionRunner({
         clearTimeout(processRef.taskPublishTimer);
         processRef.taskPublishTimer = null;
       }
-      send();
-      return;
+      return send();
     }
-    if (processRef.taskPublishTimer) return;
+    if (processRef.taskPublishTimer) return Promise.resolve();
     processRef.taskPublishTimer = setTimeout(send, 2_000);
     processRef.taskPublishTimer.unref?.();
+    return Promise.resolve();
   }
 
   // ---------------------------------------------------------------------------
@@ -2435,7 +2483,12 @@ export function createClaudeSessionRunner({
       // capture a settling workflow's final digest for the transcript card
       // (the notification then refines the status when it lands).
       for (const [taskId, task] of previous) {
-        if (!proc.liveTasks.has(taskId)) bufferSettledWorkflowRun(taskId, task);
+        if (proc.liveTasks.has(taskId)) continue;
+        bufferSettledWorkflowRun(taskId, task);
+        // What the task was, for its settle notice: the row is gone by the
+        // time the notification says how it ended.
+        proc.departedTasks.set(taskId, task);
+        if (proc.departedTasks.size > 50) proc.departedTasks.delete(proc.departedTasks.keys().next().value);
       }
       syncWorkflowPoller(proc);
       publishBackgroundTasks();
@@ -2481,6 +2534,29 @@ export function createClaudeSessionRunner({
       // tree publishes before the row clears (best-effort: the CLI may have
       // already removed the task via background_tasks_changed).
       if (taskId) publishFinalWorkflowDigest(taskId, String(sdkMessage.status || '').trim());
+      // How the task ended, for the relay's task status and the watchers of
+      // this session (the live set only says it is gone). Ambient tasks are
+      // not activity. Advisory: a failed post never disturbs the stream.
+      if (taskId && sdkMessage.ambient !== true) {
+        // A task that was in the published set is described too, so that a
+        // relay which no longer knows it (it restarted meanwhile) can still
+        // tell its watchers the end. A failed post is kept and sent again.
+        const published = proc.liveTasks.get(taskId) || proc.departedTasks.get(taskId) || null;
+        proc.departedTasks.delete(taskId);
+        void keptTaskSettles.post('/api/background-task-settled', {
+          conversationId: sdkSessionId,
+          taskId,
+          toolUseId: String(sdkMessage.tool_use_id || published?.toolUseId || '').trim() || null,
+          status: String(sdkMessage.status || '').trim() || 'completed',
+          summary: String(sdkMessage.summary || '').trim().slice(0, 500) || null,
+          ...(sdkMessage.reason ? { reason: String(sdkMessage.reason).slice(0, 40) } : {}),
+          ...(published?.description ? {
+            description: String(published.description).slice(0, 500),
+            taskType: String(published.taskType || ''),
+            startedAt: published.startedAt || null,
+          } : {}),
+        });
+      }
       // A settled session-level task means the CLI is about to dequeue a
       // continuation turn — the process must not idle out under it. The SDK
       // flags silent notifications explicitly: skip_transcript means no
@@ -2681,6 +2757,9 @@ export function createClaudeSessionRunner({
   }
 
   async function handleSdkMessage(sdkMessage) {
+    // Before any turn takes the frame: a background agent's frames arrive
+    // between turns too, and belong to the run, not to whichever turn is open.
+    timelineRecorder.observe(sdkMessage);
     const usageLimitReport = usageLimit.observe(sdkMessage);
     if (usageLimitReport) publisher.publishUsageLimitReport({ conversationId: sdkSessionId, sdkSessionId, report: usageLimitReport });
     const suppressed = maybeSuppressStopFollowUp(sdkMessage);
@@ -2702,6 +2781,12 @@ export function createClaudeSessionRunner({
       await settling;
     }
     if (!ctx) return;
+    timelineRecorder.claimSpawns(sdkMessage, ctx);
+    // A background agent spawned by an earlier turn: its frames are not this
+    // turn's, and published here they would be filed under it. Its record
+    // goes to its own run through the recorder above.
+    const subagentParent = String(sdkMessage?.parent_tool_use_id || '').trim();
+    if (subagentParent && timelineRecorder.isForeignRun(subagentParent, ctx)) return;
     const actions = ctx.normalizer.normalize(sdkMessage);
     for (const action of actions) {
       // Decided here, while the reports the turn ran under are the latest.
@@ -3201,6 +3286,10 @@ export function createClaudeSessionRunner({
     const wasCurrent = proc === processRef;
     // The next spawn reads the registration decision; refresh it now.
     if (wasCurrent) void remoteRelayGate.refresh();
+    // Agents die with their process: their runs end, what streamed closes.
+    if (wasCurrent) {
+      void timelineRecorder.processEnded().catch((error) => dbg('subagent timeline flush failed', error?.message || String(error)));
+    }
     stopLifecycleTimer(processRef);
     if (processRef.taskPublishTimer) {
       clearTimeout(processRef.taskPublishTimer);
@@ -3217,7 +3306,7 @@ export function createClaudeSessionRunner({
     if (processRef.liveTasks.size) {
       processRef.liveTasks = new Map();
       if (wasCurrent) {
-        api('POST', '/api/background-tasks', { conversationId: sdkSessionId, tasks: [] }).catch(() => {});
+        postTaskSet({ tasks: [] }).catch(() => {});
       }
     }
     // Steers pushed into a turn the user stopped: if the Stop fell back to
@@ -3299,6 +3388,9 @@ export function createClaudeSessionRunner({
       permissionMode: permissionModeForRelayMode(relayMode),
       liveTasks: new Map(),
       knownTasks: new Set(),
+      // taskId → the row of a task that left the live set, until its
+      // notification arrives.
+      departedTasks: new Map(),
       // taskId → { runId, labels, digestJson } for live local_workflow tasks;
       // pruned by syncWorkflowPoller when a task leaves the live set.
       workflowStates: new Map(),
@@ -3913,7 +4005,9 @@ export function createClaudeSessionRunner({
         // noted steer no longer starts with "/" and is pushed as noted. The
         // replay signatures strip the guard too, so expectedText stays the
         // user's own text here as well.
-        const pushedContent = withSlashCommandGuardContent(steered && procRef.activeCtx?.kind !== 'continuation'
+        // Nor into an event message the relay wrote: the note speaks as the
+        // user, the event must not (shared/relay-events.mjs).
+        const pushedContent = withSlashCommandGuardContent(steered && procRef.activeCtx?.kind !== 'continuation' && !isRelayEventDelivery(message)
           ? withSteerNoteContent(content)
           : content);
         procRef.pendingDelivered.push({ ctx, expectedText: contentText(content), pushedAt: Date.now(), steered });
@@ -4000,6 +4094,21 @@ export function createClaudeSessionRunner({
     return false;
   }
 
+  /** The ids of the live background tasks, for the heartbeat's digest. */
+  function backgroundTaskIds() {
+    return proc ? [...proc.liveTasks.keys()] : [];
+  }
+
+  /** The relay asked for the task set again (heartbeat answer). */
+  function resyncBackgroundTasks() {
+    return publishBackgroundTasks({ resync: true });
+  }
+
+  /** The relay answered a heartbeat: settle notices kept for it go out. */
+  function flushKeptTaskSettles() {
+    return keptTaskSettles.flush();
+  }
+
   return {
     handlePendingPayload,
     canAcceptSteering,
@@ -4012,6 +4121,9 @@ export function createClaudeSessionRunner({
     isTurnActive,
     stopBackgroundTask,
     stopBackgroundTaskByToolUseId,
+    backgroundTaskIds,
+    resyncBackgroundTasks,
+    flushKeptTaskSettles,
     shutdown,
     // Test seams / observability.
     _getProcess: () => proc,

package/server/claude-worker/claude-session-worker.mjs

@@ -119,6 +119,7 @@ async function main() {
     // A readiness flip is a hold starting or ending (question card,
     // compaction, a turn-opening delivery): the steering snapshot changed
     // with it, so it goes out now rather than on the next 10 s heartbeat.
+    onBackgroundTaskSetStale: () => heartbeat?.requestPulse(),
     onDeliveryReadinessChange: (ready) => {
       linkBridge.onDeliveryReadinessChange(ready);
       heartbeat?.requestPulse();
@@ -147,6 +148,12 @@ async function main() {
     // Consumed rows the runner could not fail itself; the relay fails them.
     getSettleFailed: () => turnRunner.getSettleFailed(),
     onSettleFailedHandled: (ids) => turnRunner.acknowledgeSettleFailed(ids),
+    // The relay holds the task set in memory only: the digest of the live
+    // task ids rides every beat, and a relay that is out of step (it
+    // restarted, a post was lost) asks for the set again.
+    getBackgroundTaskIds: () => turnRunner.backgroundTaskIds(),
+    onResyncBackgroundTasks: () => turnRunner.resyncBackgroundTasks(),
+    onAnswered: () => turnRunner.flushKeptTaskSettles(),
   });
 
   const wsLink = createWorkerWebSocketLink({
@@ -164,7 +171,8 @@ async function main() {
     // even between deliveries — a delivery drawn into it is only handed back.
     getDeliveryHeld: () => turnRunner.isDeliveryHeld(),
     // A relay that just (re)started knows nothing of this worker until it
-    // heartbeats; the composer's steering snapshot rides that beat.
+    // heartbeats; the composer's steering snapshot rides that beat, and so
+    // does the digest that makes the relay ask for the background tasks.
     onServerHello: () => heartbeat.requestPulse(),
     getSessionId: () => sdkSessionId,
     getPid: () => process.pid,

package/server/claude-worker/claude-turn-publisher.mjs

@@ -127,6 +127,9 @@ export function createClaudeTurnPublisher({ api, dbg = () => {}, takeWorkflowRun
         parentSubagentId: payload.parentSubagentId || undefined,
         displayName: payload.displayName || undefined,
         status: payload.status,
+        ...(payload.subagentType ? { subagentType: payload.subagentType } : {}),
+        ...(payload.background ? { background: true } : {}),
+        ...(payload.timelineOwner ? { timelineOwner: payload.timelineOwner } : {}),
         ...attemptFields(message),
       }).catch(() => {});
       return;

package/server/claude-worker/sdk-message-normalizer.mjs

@@ -116,6 +116,23 @@ export function isSubagentToolName(name) {
   return SUBAGENT_TOOL_NAMES.has(String(name || '').trim().toLowerCase());
 }
 
+const BACKGROUND_LAUNCH_STATUSES = new Set(['async_launched', 'remote_launched']);
+
+/**
+ * The tool result of an Agent/Task spawn that went to the background: the
+ * structured result says so (`tool_use_result.status`), and the receipt text
+ * the model reads is the fallback for producers without it.
+ */
+export function isBackgroundLaunch(block, toolUseResult = null) {
+  const status = String(toolUseResult?.status || '').trim();
+  if (BACKGROUND_LAUNCH_STATUSES.has(status)) return true;
+  const content = block?.content;
+  const text = typeof content === 'string'
+    ? content
+    : (Array.isArray(content) ? content.map((part) => String(part?.text || '')).join(' ') : '');
+  return /^\s*Async agent launched/i.test(text);
+}
+
 export function summarizeToolInput(toolName, input) {
   if (!input || typeof input !== 'object') return '';
   // `mcp__relay__remote_relay` reads as what it does on which relay
@@ -287,7 +304,8 @@ export function createSdkMessageNormalizer() {
         || 'Subagent',
       ).trim() || 'Subagent';
       const parentSubagentId = subagentRunIdFor(parentToolUseId);
-      knownSubagentRuns.set(toolUseId, { displayName, parentSubagentId });
+      const subagentType = String(input.subagent_type || '').trim();
+      knownSubagentRuns.set(toolUseId, { displayName, parentSubagentId, subagentType });
       actions.push({
         channel: 'subagent',
         payload: {
@@ -295,6 +313,10 @@ export function createSdkMessageNormalizer() {
           parentSubagentId,
           displayName,
           status: 'running',
+          ...(subagentType ? { subagentType } : {}),
+          // The worker's timeline recorder writes this run's record itself;
+          // the relay must not copy the lane frames into it as well.
+          timelineOwner: 'worker',
         },
       });
     }
@@ -308,19 +330,24 @@ export function createSdkMessageNormalizer() {
     return actions;
   }
 
-  function actionsForToolResultBlock(block, parentToolUseId) {
+  function actionsForToolResultBlock(block, parentToolUseId, toolUseResult = null) {
     const actions = [];
     const toolUseId = String(block?.tool_use_id || '').trim();
     const isError = block?.is_error === true;
     if (toolUseId && knownSubagentRuns.has(toolUseId)) {
       const run = knownSubagentRuns.get(toolUseId);
+      // A background launch answers its spawn at once with a receipt; the
+      // agent is only starting. Its end arrives later as a task
+      // notification, which the worker's timeline recorder reports.
+      const launchedInBackground = !isError && isBackgroundLaunch(block, toolUseResult);
       actions.push({
         channel: 'subagent',
         payload: {
           subagentRunId: toolUseId,
           parentSubagentId: run.parentSubagentId,
           displayName: run.displayName,
-          status: isError ? 'failed' : 'completed',
+          status: launchedInBackground ? 'running' : (isError ? 'failed' : 'completed'),
+          ...(launchedInBackground ? { background: true } : {}),
         },
       });
     }
@@ -574,7 +601,7 @@ export function createSdkMessageNormalizer() {
       const content = Array.isArray(sdkMessage?.message?.content) ? sdkMessage.message.content : [];
       for (const block of content) {
         if (block?.type === 'tool_result') {
-          actions.push(...actionsForToolResultBlock(block, parentToolUseId));
+          actions.push(...actionsForToolResultBlock(block, parentToolUseId, sdkMessage?.tool_use_result || null));
         }
       }
       return actions;