@oar-sh/oar 0.9.13 → 0.9.15

package/server/services/relay-singleton-guard.mjs

@@ -1,6 +1,128 @@
 import fs from "fs";
+import os from "os";
 import path from "path";
 import crypto from "crypto";
+import { spawnSync } from "child_process";
+import { createRequire } from "module";
+
+// An EPERM is a process this user may not signal: alive.
+export function defaultIsProcessAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+let processTimes;
+// When a process started, in ms, or null when that cannot be told. Windows
+// asks the kernel (GetProcessTimes through koffi, which the Copilot SDK
+// brings): PowerShell fails in ConstrainedLanguage mode and with a broken
+// TEMP. Elsewhere `ps`.
+export function processStartedAt(pid, { platform = process.platform } = {}) {
+  try {
+    if (platform !== "win32") {
+      // `lstart`, the full date; LC_ALL=C to parse; unparsable: cannot tell.
+      const result = spawnSync("ps", ["-o", "lstart=", "-p", String(Number(pid))], { encoding: "utf8", timeout: 5000, env: { ...process.env, LC_ALL: "C" } });
+      const ms = Date.parse(String(result.stdout || "").trim());
+      return Number.isFinite(ms) ? ms : null;
+    }
+    if (!processTimes) {
+      const koffi = createRequire(import.meta.url)("koffi");
+      const kernel32 = koffi.load("kernel32.dll");
+      processTimes = {
+        open: kernel32.func("void * __stdcall OpenProcess(uint32_t access, bool inherit, uint32_t pid)"),
+        times: kernel32.func("bool __stdcall GetProcessTimes(void *h, _Out_ uint64_t *a, _Out_ uint64_t *b, _Out_ uint64_t *c, _Out_ uint64_t *d)"),
+        close: kernel32.func("bool __stdcall CloseHandle(void *h)"),
+      };
+    }
+    const handle = processTimes.open(0x1000, false, Number(pid)); // PROCESS_QUERY_LIMITED_INFORMATION
+    if (!handle) return null;
+    try {
+      const created = [0n];
+      if (!processTimes.times(handle, created, [0n], [0n], [0n])) return null;
+      return Number(BigInt(created[0]) / 10000n) - 11644473600000; // FILETIME: 100 ns since 1601
+    } finally {
+      processTimes.close(handle);
+    }
+  } catch {
+    return null;
+  }
+}
+
+let bootId;
+// Linux's id of the running boot, '' elsewhere or when unreadable.
+export function currentBootId({ platform = process.platform, readFile = fs.readFileSync } = {}) {
+  if (platform !== "linux") return "";
+  try {
+    bootId ??= String(readFile("/proc/sys/kernel/random/boot_id", "utf8")).trim();
+    return bootId;
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * A process's start as the system keeps it, which no later clock step moves:
+ * Windows' creation time (exact), Linux's start in ticks since boot with the
+ * boot id, elsewhere `ps` (to the second). Null when it cannot be told.
+ */
+export function processIdentity(pid, { platform = process.platform, readFile = fs.readFileSync } = {}) {
+  try {
+    if (platform === "linux") {
+      const stat = String(readFile(`/proc/${Number(pid)}/stat`, "utf8"));
+      const ticks = stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19]; // field 22, starttime
+      return /^\d+$/.test(ticks) ? { start: `ticks:${ticks}`, bootId: currentBootId({ platform, readFile }) } : null;
+    }
+    const ms = processStartedAt(pid, { platform });
+    if (ms === null) return null;
+    return { start: platform === "win32" ? `win:${ms}` : ms };
+  } catch {
+    return null;
+  }
+}
+
+// Exact for the kernel's own values, to the second for `ps`.
+function sameStart(a, b) {
+  return typeof a === "number" && typeof b === "number" ? Math.abs(a - b) <= 1000 : a === b;
+}
+
+/**
+ * Whether the relay a lock names may still run; true when that cannot be
+ * told: better a refused start than two relays. A lock this version wrote
+ * (format 2) never goes by the clock: a live pid (EPERM counts) holds it
+ * when its start matches the recorded holder, or when either start is
+ * unknown; it is stale when the pid's start differs, or when the boot id it
+ * recorded (Linux) is not the running one. Only a lock without the format
+ * (0.9.13's) goes by the clock, the one case a clock step can mislead:
+ * written before the last boot, or older than the pid's process by more
+ * than 5 s (a reused pid), it is stale.
+ */
+export function lockHolderMayRun({
+  lock,
+  isProcessAlive = defaultIsProcessAlive,
+  identityOf = processIdentity,
+  bootIdOf = currentBootId,
+  startedAtOf = processStartedAt,
+  bootTime = Date.now() - os.uptime() * 1000,
+}) {
+  const pid = Number.parseInt(String(lock?.pid ?? ""), 10);
+  if (!Number.isInteger(pid) || pid <= 0 || !isProcessAlive(pid)) return false;
+  if (lock?.format >= 2 || lock?.holder) {
+    const current = lock.holder ? identityOf(pid) : null;
+    const recordedBoot = lock.holder?.bootId || lock.bootId;
+    const runningBoot = current?.bootId || bootIdOf();
+    if (recordedBoot && runningBoot && recordedBoot !== runningBoot) return false;
+    return !current || sameStart(lock.holder.start, current.start);
+  }
+  const lockedAt = Date.parse(String(lock?.startedAt || ""));
+  if (Number.isFinite(lockedAt) && lockedAt < bootTime) return false;
+  const startedAt = startedAtOf(pid);
+  if (Number.isFinite(lockedAt) && startedAt !== null && startedAt > lockedAt + 5000) return false;
+  return true;
+}
 
 function hashToken(value) {
   const token = String(value || "").trim();
@@ -21,6 +143,9 @@ function safeReadLock(lockPath) {
       pid: Number.parseInt(String(parsed?.pid ?? ""), 10),
       tokenHash,
       startedAt: typeof parsed?.startedAt === "string" ? parsed.startedAt : null,
+      holder: parsed?.holder && typeof parsed.holder === "object" ? parsed.holder : null,
+      format: Number(parsed?.format) || 0,
+      bootId: typeof parsed?.bootId === "string" ? parsed.bootId : "",
     };
   } catch {
     return null;
@@ -33,23 +158,27 @@ export function createRelaySingletonGuard({
   token,
   now = () => new Date().toISOString(),
   isProcessAlive,
+  // When a pid's process started, and when the machine booted (lockHolderMayRun).
+  identityOf = processIdentity,
+  bootIdOf = currentBootId,
+  startedAtOf = processStartedAt,
+  bootTime = Date.now() - os.uptime() * 1000,
+  // An unreadable lock is read again for `readRetryMs`; still unreadable,
+  // it is stale only when its time is more than `unreadableStaleMs` off.
+  readRetryMs = 2000,
+  unreadableStaleMs = 10_000,
+  clock = Date.now,
+  sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms),
   logger = console,
 }) {
   const lockFilePath = path.resolve(String(lockPath || ""));
   const currentPid = Number.parseInt(String(pid || process.pid), 10);
   const currentTokenHash = hashToken(token);
-  const processAlive = typeof isProcessAlive === "function"
-    ? isProcessAlive
-    : (candidatePid) => {
-      if (!Number.isInteger(candidatePid) || candidatePid <= 0) return false;
-      try {
-        process.kill(candidatePid, 0);
-        return true;
-      } catch {
-        return false;
-      }
-    };
+  const processAlive = typeof isProcessAlive === "function" ? isProcessAlive : defaultIsProcessAlive;
 
+  // The whole lock goes into a file of its own, linked into place (fails
+  // when a lock is there): no start ever reads one half written. Where the
+  // folder takes no hard links (FAT, exFAT), created and written in place.
   function writeLock() {
     fs.mkdirSync(path.dirname(lockFilePath), { recursive: true });
     const payload = {
@@ -59,47 +188,182 @@ export function createRelaySingletonGuard({
       // credential out of a file that has historically been world-readable.
       tokenHash: currentTokenHash,
       startedAt: now(),
+      // The holder, as lockHolderMayRun tells it without the clock (null:
+      // unknown, which then holds the lock while its pid lives).
+      format: 2,
+      holder: identityOf(currentPid) || null,
+      ...(bootIdOf() ? { bootId: bootIdOf() } : {}),
     };
-    const fd = fs.openSync(lockFilePath, "wx", 0o600);
+    const text = `${JSON.stringify(payload, null, 2)}\n`;
+    const staged = `${lockFilePath}.${currentPid}-${crypto.randomBytes(4).toString("hex")}.tmp`;
+    let linked = false;
+    let written = false;
     try {
-      fs.writeFileSync(fd, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
+      fs.writeFileSync(staged, text, { encoding: "utf8", mode: 0o600, flag: "wx" });
+      written = true;
+      fs.linkSync(staged, lockFilePath);
+      linked = true;
+    } catch (error) {
+      if (written && error?.code === "EEXIST") throw error;
     } finally {
-      try { fs.closeSync(fd); } catch {}
+      if (written) try { fs.unlinkSync(staged); } catch {}
+    }
+    if (!linked) {
+      const fd = fs.openSync(lockFilePath, "wx", 0o600);
+      try {
+        fs.writeFileSync(fd, text, "utf8");
+      } finally {
+        try { fs.closeSync(fd); } catch {}
+      }
     }
     return payload;
   }
 
-  function acquire() {
+  // Read again for a while (an antivirus scan, a lock written in place);
+  // still unreadable: stale when its time is far off, else held.
+  function readLockWithRetry() {
+    const deadline = clock() + readRetryMs;
+    for (;;) {
+      const lock = safeReadLock(lockFilePath);
+      if (lock || clock() >= deadline) return lock;
+      sleep(100);
+    }
+  }
+  function unreadableLockIsStale() {
     try {
-      return writeLock();
-    } catch (error) {
-      if (error?.code !== "EEXIST") {
-        throw error;
-      }
+      return Math.abs(clock() - fs.statSync(lockFilePath).mtimeMs) > unreadableStaleMs;
+    } catch {
+      return true;
     }
+  }
 
-    const existingLock = safeReadLock(lockFilePath);
-    const existingPid = Number.isInteger(existingLock?.pid) ? existingLock.pid : null;
-    const existingTokenHash = typeof existingLock?.tokenHash === "string" ? existingLock.tokenHash : "";
-    const existingStartedAt = existingLock?.startedAt || "unknown";
+  const held = (message) => Object.assign(new Error(message), { code: "ERELAYLOCKED" });
+  const RENAME_RETRY_MS = [50, 100, 200, 400, 800];
 
-    if (existingPid && processAlive(existingPid)) {
-      if (existingTokenHash && existingTokenHash === currentTokenHash) {
-        throw new Error(`Relay already running (pid=${existingPid}, startedAt=${existingStartedAt}).`);
-      }
-      throw new Error(
-        `Relay appears to be owned by another live process (pid=${existingPid}, startedAt=${existingStartedAt}).`
-      );
+  // The lock as judged: its fields, or for one that cannot be read its size
+  // and time, which a moved-aside file must still have to be the same lock.
+  function sameLock(file, judged) {
+    try {
+      const lock = safeReadLock(file);
+      if (judged.lock) return !!lock && ["pid", "tokenHash", "startedAt"].every((key) => Object.is(lock[key], judged.lock[key]));
+      const stat = fs.statSync(file);
+      return !lock && stat.size === judged.size && stat.mtimeMs === judged.mtimeMs;
+    } catch {
+      return false;
     }
+  }
 
+  // A lock put back after it turned out fresh: linked, else (no hard links)
+  // created with its content; a lock there by then wins, the file stays.
+  function putBack(moved) {
     try {
-      fs.unlinkSync(lockFilePath);
-      logger?.warn?.(`[server] Recovered stale singleton lock: ${lockFilePath}`);
-    } catch (unlinkError) {
-      if (unlinkError?.code !== "ENOENT") throw unlinkError;
+      try {
+        fs.linkSync(moved, lockFilePath);
+      } catch (error) {
+        if (error?.code === "EEXIST") throw error;
+        fs.writeFileSync(lockFilePath, fs.readFileSync(moved), { flag: "wx", mode: 0o600 });
+      }
+      fs.unlinkSync(moved);
+    } catch (error) {
+      logger?.warn?.(`[server] A relay lock moved aside could not be put back (${error?.code || error}); it is in ${moved}.`);
     }
+  }
 
-    return writeLock();
+  // Files a crashed recovery or write left beside the lock (`.stale-<pid>-`,
+  // `.<pid>-….tmp`), never read: removed once their process is gone.
+  function removeLeftovers() {
+    const base = path.basename(lockFilePath);
+    let names = [];
+    try { names = fs.readdirSync(path.dirname(lockFilePath)); } catch {}
+    for (const name of names) {
+      if (!name.startsWith(`${base}.`)) continue;
+      const leftPid = Number(/^\.(?:stale-)?(\d+)-[0-9a-f]+(?:\.tmp)?$/.exec(name.slice(base.length))?.[1]);
+      if (leftPid && leftPid !== currentPid && !processAlive(leftPid)) try { fs.unlinkSync(path.join(path.dirname(lockFilePath), name)); } catch {}
+    }
+  }
+
+  function acquire() {
+    const payload = takeLock();
+    removeLeftovers();
+    return payload;
+  }
+
+  // A stale lock is moved aside, never unlinked unseen: the file moved must
+  // be the one judged, else another start took the lock meanwhile and it
+  // goes back. Gone already (another start recovered it): from the top, at
+  // most three times.
+  function takeLock() {
+    for (let attempt = 1; ; attempt += 1) {
+      try {
+        return writeLock();
+      } catch (error) {
+        if (error?.code !== "EEXIST") throw error;
+      }
+
+      const existingLock = readLockWithRetry();
+      let judged = { lock: existingLock };
+      if (!existingLock) {
+        if (!unreadableLockIsStale()) {
+          throw held(`A relay lock that cannot be read yet is there, written in the last ${unreadableStaleMs / 1000} s (lock ${lockFilePath}). If no relay is starting, delete it.`);
+        }
+        try {
+          const stat = fs.statSync(lockFilePath);
+          judged = { lock: null, size: stat.size, mtimeMs: stat.mtimeMs };
+        } catch {
+          if (attempt < 3) continue;
+          throw held(`The relay lock ${lockFilePath} keeps changing.`);
+        }
+      }
+      const existingPid = Number.isInteger(existingLock?.pid) ? existingLock.pid : null;
+      const existingTokenHash = typeof existingLock?.tokenHash === "string" ? existingLock.tokenHash : "";
+      const existingStartedAt = existingLock?.startedAt || "unknown";
+
+      // A lock from before the last boot, or whose pid now belongs to a
+      // process started after it, is stale (a crash, a power loss, a forced
+      // reboot): recovered below.
+      if (existingPid && lockHolderMayRun({ lock: existingLock, isProcessAlive: processAlive, identityOf, bootIdOf, startedAtOf, bootTime })) {
+        const owner = existingTokenHash && existingTokenHash === currentTokenHash
+          ? "Relay already running"
+          : "Relay appears to be owned by another live process";
+        // Delete the lock only when its holder cannot be told to be a relay.
+        const known = existingLock.holder && identityOf(existingPid);
+        throw held(`${owner} (pid=${existingPid}, startedAt=${existingStartedAt}; lock ${lockFilePath}). ${known
+          ? `It is held by relay process ${existingPid}: oar stop, or end process ${existingPid}.`
+          : `If no relay is running, end process ${existingPid} or delete the lock.`}`);
+      }
+
+      const moved = `${lockFilePath}.stale-${currentPid}-${crypto.randomBytes(4).toString("hex")}`;
+      for (let retry = 0; ; retry += 1) {
+        try {
+          fs.renameSync(lockFilePath, moved);
+          break;
+        } catch (error) {
+          if (error?.code === "ENOENT") break;
+          // Held open (an antivirus scan): briefly, then counted as held.
+          if (retry < RENAME_RETRY_MS.length && ["EBUSY", "EPERM", "EACCES"].includes(error?.code)) {
+            sleep(RENAME_RETRY_MS[retry]);
+            continue;
+          }
+          throw held(`The stale relay lock ${lockFilePath} could not be moved aside (${error?.code || error}); delete it if no relay is running.`);
+        }
+      }
+      if (!fs.existsSync(moved)) {
+        if (attempt < 3) continue;
+        throw held(`The relay lock ${lockFilePath} keeps changing.`);
+      }
+      if (!sameLock(moved, judged)) {
+        putBack(moved);
+        throw held(`Another relay took the lock ${lockFilePath} while this one recovered it.`);
+      }
+      try { fs.unlinkSync(moved); } catch {}
+      logger?.warn?.(`[server] Recovered stale singleton lock: ${lockFilePath}`);
+      try {
+        return writeLock();
+      } catch (error) {
+        if (error?.code !== "EEXIST") throw error;
+        if (attempt >= 3) throw held(`The relay lock ${lockFilePath} keeps changing.`);
+      }
+    }
   }
 
   function release() {

package/server/services/remote-relay-caller-context.mjs

@@ -8,6 +8,7 @@ import {
   REMOTE_RELAY_PROVIDERS,
 } from '../../shared/remote-relay-contract.mjs';
 import { DEFAULT_QUESTION_TIMEOUT_MS, questionExpiresAt } from '../../shared/question-timeout.mjs';
+import { readApprovalAnswer } from '../../shared/question-answer-rule.mjs';
 import { createQuestionRepository } from '../repositories/question-repository.mjs';
 import { pickLiveTurnRowId } from './live-turn-picker.mjs';
 import { sanitizeRelayQuestionContext } from './relay-question-context.mjs';
@@ -427,8 +428,12 @@ export function createRemoteRelayCallerContext({
       const current = questionStatements.getQuestion.get(questionId);
       if (!current) return { approved: false, error: 'The approval question was removed.' };
       if (current.status === 'answered') {
-        const approved = toText(current.answer).toLowerCase() === APPROVAL_ALLOW.toLowerCase();
-        return approved ? { approved: true } : { approved: false };
+        const decision = readApprovalAnswer(current.answer, { approve: APPROVAL_ALLOW, deny: APPROVAL_DENY });
+        if (decision === 'approve') return { approved: true };
+        if (decision === 'deny') return { approved: false };
+        // The answer route takes only the choices; anything else in the row
+        // is no decision of the user's, so it is not passed off as their Deny.
+        return { approved: false, error: 'The approval question was answered with neither Allow nor Deny. Go on without it, or ask the user first.' };
       }
       if (current.status !== 'pending') {
         return {

package/server/services/remote-relay-dispatcher.mjs

@@ -25,8 +25,9 @@ import {
   withRemotePromptHeader,
 } from '../../shared/remote-relay-contract.mjs';
 import { schemaFields } from '../../shared/question-schema.mjs';
+import { matchQuestionChoice } from '../../shared/question-answer-rule.mjs';
 import { ANSWERED_ELSEWHERE_KIND } from '../../shared/steer-settle-markers.mjs';
-import { RELAY_EVENT_DEFAULT_TYPES } from '../../shared/relay-events.mjs';
+import { RELAY_EVENT_DEFAULT_TYPES, RELAY_EVENT_RELAY_LEVEL_TYPES } from '../../shared/relay-events.mjs';
 import { checkRequestedProviderModel, isAutoModelId, providerModelLabel, unknownProviderModelText } from './provider-model-selection.mjs';
 import { cleanTaskOutput, parseRunnerSummary, stallMinutes, stripSgr } from '../../shared/task-output-text.mjs';
 import { TASK_OUTPUT_NONE_MESSAGES } from '../public/app/task-console-render.mjs';
@@ -2132,7 +2133,14 @@ export function createRemoteRelayDispatcher({
       if (!labels.includes(label)) labels.push(label);
     }
     // The card's own format: the picked labels, then any typed text, ", "-joined.
-    const extra = toText(args.answer);
+    // A card that takes no typed text gets an option or nothing.
+    let extra = toText(args.answer);
+    if (extra && question.allowFreeform === false) {
+      extra = matchQuestionChoice(extra, options);
+      if (!extra) {
+        throw invalidInput(`This question takes only its options (${options.join(', ')}), no typed answer; pass one of them in choices.`);
+      }
+    }
     if (extra && !labels.includes(extra)) labels.push(extra);
     const text = labels.join(', ');
     return { answer: text, text };
@@ -2332,7 +2340,7 @@ export function createRemoteRelayDispatcher({
     if (!ctx.relay.local) return subscribeRemoteAction(ctx, args);
     const service = eventService(ctx);
     const all = args.scope === 'all';
-    const ids = all ? [] : args.sessions;
+    const ids = all ? [] : (args.sessions || []);
     // Refused before anyone is asked: the conversation itself, unknown ids.
     if (!all && ids.includes(ctx.conversationId)) {
       throw new DispatchFailure(400, REMOTE_RELAY_ERROR_CODES.ownSession,
@@ -2349,11 +2357,15 @@ export function createRemoteRelayDispatcher({
       titles.push({ id, title: toText(page.title) || null });
     }
     const events = Array.isArray(args.events) && args.events.length ? args.events : [...RELAY_EVENT_DEFAULT_TYPES];
-    const decision = await ensureWatchApproval(ctx, {
-      all,
-      watch: { kind: 'subscribe', all, sessions: titles, events, once: args.once === true, heartbeatMinutes: args.heartbeat_minutes ?? null },
-    });
-    if (!decision.approved) throw new DispatchFailure(403, decision.code, decision.error);
+    // Only relay_restarted, no session: nothing of another conversation is
+    // watched, so there is nothing to ask the user about.
+    if (all || ids.length) {
+      const decision = await ensureWatchApproval(ctx, {
+        all,
+        watch: { kind: 'subscribe', all, sessions: titles, events, once: args.once === true, heartbeatMinutes: args.heartbeat_minutes ?? null },
+      });
+      if (!decision.approved) throw new DispatchFailure(403, decision.code, decision.error);
+    }
     const subscription = relayEventCall(() => service.subscribe({
       watcherConversationId: ctx.conversationId,
       scope: all ? 'all' : 'sessions',
@@ -2364,8 +2376,9 @@ export function createRemoteRelayDispatcher({
       stalledMinutes: args.stalled_minutes ?? null,
       heartbeatMinutes: args.heartbeat_minutes ?? null,
     }));
+    const what = all ? 'all sessions' : (ids.length ? `${ids.length} session${ids.length === 1 ? '' : 's'}` : 'relay_restarted');
     return {
-      summary: `subscribe → ${ctx.relay.name}: ${all ? 'all sessions' : `${ids.length} session${ids.length === 1 ? '' : 's'}`} (${subscription.id})`,
+      summary: `subscribe → ${ctx.relay.name}: ${what} (${subscription.id})`,
       subscription,
       note: EVENT_NOTE,
     };
@@ -2378,6 +2391,14 @@ export function createRemoteRelayDispatcher({
    * mirror; the link fetches the events from then on.
    */
   async function subscribeRemoteAction(ctx, args) {
+    const relayLevel = (args.events || []).filter((type) => RELAY_EVENT_RELAY_LEVEL_TYPES.includes(type));
+    if (relayLevel.length) {
+      throw new DispatchFailure(400, REMOTE_RELAY_ERROR_CODES.invalidInput,
+        `${relayLevel.join(', ')} is about the relay you run on: subscribe to it on "${REMOTE_RELAY_LOCAL_ALIAS}". An outage of relay "${ctx.relay.name}" shows as link_lost and link_restored.`);
+    }
+    if (!args.sessions?.length && args.scope !== 'all') {
+      throw new DispatchFailure(400, REMOTE_RELAY_ERROR_CODES.invalidInput, 'subscribe on a paired relay needs sessions (ids from list_sessions) or ["all"].');
+    }
     const service = eventService(ctx);
     const all = args.scope === 'all';
     relayEventCall(() => service.checkWatcherCapacity(ctx.conversationId));

package/server/services/session-activity-tracker.mjs

@@ -25,7 +25,9 @@
  *  - `noteTaskSet` — every POST /api/background-tasks (REPLACE semantics):
  *    a new task id is a start, a task gone from the set has ended, a changed
  *    task (tool call, summary, tokens) is activity. A task that started
- *    before this runtime did was announced by the runtime before it.
+ *    before this runtime did was announced by the runtime before it. A task
+ *    that ended and is listed under another run (`taskRunId`: a resumed
+ *    Claude sub-agent keeps its task id) is a start again.
  *  - `noteTaskSettled` — how a task ended, where the worker says so (the
  *    Claude worker's task_notification). A task this runtime never knew
  *    still gets its end event when the worker describes it.
@@ -168,6 +170,11 @@ export function createSessionActivityTracker({
   const sessions = new Map();
   // Edges already reported for an end, by `${conversationId}:${key}`.
   const endedKeys = new Map();
+  // `${conversationId}:${taskId}` → the run of the task that ended (the
+  // SDK's run id, `taskRunId`), where the worker names one. A resumed Claude
+  // sub-agent keeps its task id and runs again under a new run id: a set
+  // that lists it with another run is a new start, not a late set.
+  const endedRuns = new Map();
   // `${conversationId}:${taskId}` of the ends nobody reported: the task was
   // only missing from a set for the grace. A later set that lists it again
   // says it never ended.
@@ -216,16 +223,25 @@ export function createSessionActivityTracker({
     for (const key of [task.taskId, task.toolUseId ? `run:${task.toolUseId}` : '']) {
       if (key) endedKeys.set(`${conversationId}:${key}`, at);
     }
+    if (task.taskRunId) endedRuns.set(`${conversationId}:${task.taskId}`, task.taskRunId);
+    else endedRuns.delete(`${conversationId}:${task.taskId}`);
     if (endedKeys.size > 2000) {
       for (const [key, when] of endedKeys) {
         if (at - when > RECENT_TASK_TTL_MS) {
           endedKeys.delete(key);
           endedByAbsence.delete(key);
+          endedRuns.delete(key);
         }
       }
     }
   }
 
+  /** The set lists an ended task under another run: it was resumed. */
+  function isNewRun(conversationId, taskId, taskRunId) {
+    const ended = endedRuns.get(`${conversationId}:${taskId}`);
+    return !!taskRunId && !!ended && ended !== taskRunId;
+  }
+
   function wasEnded(conversationId, task) {
     return [task.taskId, task.toolUseId ? `run:${task.toolUseId}` : '']
       .some((key) => key && endedKeys.has(`${conversationId}:${key}`));
@@ -308,6 +324,7 @@ export function createSessionActivityTracker({
   /** Takes back an end that was only an absence; the task's old row goes from the recent ones. */
   function unEnd(conversationId, entry, taskId, toolUseId) {
     endedByAbsence.delete(`${conversationId}:${taskId}`);
+    endedRuns.delete(`${conversationId}:${taskId}`);
     const index = entry.recent.findIndex((task) => task.taskId === taskId);
     const [old] = index >= 0 ? entry.recent.splice(index, 1) : [null];
     for (const key of [taskId, toolUseId ? `run:${toolUseId}` : '', old?.toolUseId ? `run:${old.toolUseId}` : '']) {
@@ -332,6 +349,7 @@ export function createSessionActivityTracker({
       seen.add(taskId);
       const signature = taskSignature(raw);
       const known = entry.live.get(taskId);
+      const taskRunId = toText(raw.taskRunId);
       if (known && !known.leftAtMs) {
         if (known.signature !== signature) {
           known.signature = signature;
@@ -340,32 +358,45 @@ export function createSessionActivityTracker({
         }
         known.description = toText(raw.description) || known.description;
         known.toolUseId = toText(raw.toolUseId) || known.toolUseId;
+        known.taskRunId = taskRunId || known.taskRunId;
         known.summary = toText(raw.summary) || known.summary;
         known.model = toText(raw.model) || known.model;
         continue;
       }
       if (known?.leftAtMs) {
-        // Back in the set after it left: it did not end after all.
-        if (known.graceTimer) clearTimer(known.graceTimer);
-        known.graceTimer = null;
-        known.leftAtMs = 0;
-        continue;
+        if (!taskRunId || !known.taskRunId || known.taskRunId === taskRunId) {
+          // Back in the set after it left: it did not end after all.
+          if (known.graceTimer) clearTimer(known.graceTimer);
+          known.graceTimer = null;
+          known.leftAtMs = 0;
+          continue;
+        }
+        // Back under another run (resumed within the grace): the run that
+        // left ended, how is not known; the new one starts below.
+        finishTask(id, known, { status: 'completed', reported: false });
       }
       let revived = null;
       if (wasEnded(id, { taskId, toolUseId: toText(raw.toolUseId) })) {
-        // Settled already (its notice came before the set dropped it).
-        if (!endedByAbsence.has(`${id}:${taskId}`)) continue;
-        // Ended only because a set did not list it (a set that arrived out
-        // of turn, a worker that left it out for a moment) and listed
-        // again: it never ended. It comes back as the task it was, with no
-        // second start. The end that was reported for it stays reported;
-        // its real end will be another.
-        revived = unEnd(id, entry, taskId, toText(raw.toolUseId)) || {};
+        if (isNewRun(id, taskId, taskRunId)) {
+          // Resumed: the same task, a new run. Its first run's end stays
+          // reported; this run starts (and will end) on its own.
+          unEnd(id, entry, taskId, toText(raw.toolUseId));
+        } else {
+          // Settled already (its notice came before the set dropped it).
+          if (!endedByAbsence.has(`${id}:${taskId}`)) continue;
+          // Ended only because a set did not list it (a set that arrived out
+          // of turn, a worker that left it out for a moment) and listed
+          // again: it never ended. It comes back as the task it was, with no
+          // second start. The end that was reported for it stays reported;
+          // its real end will be another.
+          revived = unEnd(id, entry, taskId, toText(raw.toolUseId)) || {};
+        }
       }
       const taskStartedAtMs = msOf(raw.startedAt);
       const task = {
         taskId,
         toolUseId: toText(raw.toolUseId) || null,
+        taskRunId: taskRunId || null,
         taskType: toText(raw.taskType),
         kind: taskKindOf(raw.taskType),
         description: toText(raw.description).slice(0, 500),
@@ -395,16 +426,21 @@ export function createSessionActivityTracker({
   }
 
   function noteTaskSettled(conversationId, {
-    taskId, toolUseId, status, summary, exitCode, description, taskType, startedAt,
+    taskId, toolUseId, taskRunId, status, summary, exitCode, description, taskType, startedAt,
   } = {}) {
     const id = toText(conversationId);
     if (!id) return;
     const entry = session(id);
     const key = toText(taskId);
     const run = toText(toolUseId);
+    const settledRun = toText(taskRunId);
     const task = (key && entry.live.get(key))
       || [...entry.live.values()].find((candidate) => run && candidate.toolUseId === run)
       || null;
+    // The end of an earlier run of a task that was resumed since (a notice
+    // the worker sent again late): the run that is live now goes on.
+    if (task && settledRun && task.taskRunId && task.taskRunId !== settledRun) return;
+    if (task && settledRun) task.taskRunId = settledRun;
     const settled = {
       status: normalizeTaskStatus(status || 'completed'),
       summary: toText(summary) || null,
@@ -439,6 +475,7 @@ export function createSessionActivityTracker({
       const unknown = {
         taskId: key,
         toolUseId: runId || null,
+        taskRunId: settledRun || null,
         taskType: toText(taskType),
         kind: taskKindOf(taskType),
         description: toText(description).slice(0, 500),