@ctrl-spc/cs 0.7.15 → 0.7.16

package/dist/orchestrator.js

@@ -1,6 +1,8 @@
+import { executionIdentity } from './agents.js';
+import { CodexHomeFailure } from './codex-home.js';
 import { execFile, spawn as spawnChild } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { beginOwnedClaim, ownedWorkAllowed, hasOwnedWorkContext, ownedWorkerClaimedAt, ownedWorkerExecutionHeld, reserveOwnedWork, legacyHeldTodoIds, interruptionReceipts, acknowledgeInterruption, legacySnapshotNeeded, saveLegacySnapshot, legacyTodoHolds, releaseLegacyTodoHold, recordExitedInterruption, beginRpcClaim, deferRpcClaim, pendingRpcClaims, recordRpcClaims, finishRpcClaims } from './daemon-processes.js';
+import { beginOwnedClaim, ownedWorkAllowed, hasOwnedWorkContext, pendingTerminalOutcomes, acknowledgeTerminalOutcome, updateTerminalOutcome, ownedWorkerClaimedAt, ownedWorkerExecutionHeld, reserveOwnedWork, legacyHeldTodoIds, interruptionReceipts, acknowledgeInterruption, legacySnapshotNeeded, saveLegacySnapshot, legacyTodoHolds, releaseLegacyTodoHold, recordExitedInterruption, beginRpcClaim, deferRpcClaim, pendingRpcClaims, recordRpcClaims, finishRpcClaims, recoveredRpcClaims, pendingRpcReference, retireRpcReference } from './daemon-processes.js';
 import { promisify } from 'node:util';
 import { agentPath, detectAgents } from './agents.js';
 /* 18a Slice 7: the two Windows spawn rules, in one module so a caller cannot
@@ -9,7 +11,7 @@ import { agentPath, detectAgents } from './agents.js';
 import { killTree, needsShell, processIsAlive, quoteForCmd, spawnOwnedProcess, releaseOwnedProcess } from './win-shell.js';
 export { quoteForCmd } from './win-shell.js';
 import { ORCHESTRATOR_MAX_CONCURRENT, ORCHESTRATOR_SPAWN_TIMEOUT_MS } from './env.js';
-import { agentDisplayName, plainFailureReason } from './failure-reason.js';
+import { agentDisplayName, nativeFailureKind, failureMessage } from './failure-reason.js';
 /* 18c SLICE 9 (GAP 15): the two request-scoped releases. They live in mcp.ts
    beside the session-end routes and the reservations table's other writers, so
    there is ONE place that knows how a lease is let go, rather than two that can
@@ -200,7 +202,7 @@ export async function claimNextTodo(client, machineId, agent,
    because the claim must stay ONE statement (see below), and because a
    blocked unit is the OLDEST claimable row: released and re-claimed it would
    be picked first every tick, starving everything behind it. */
-skipIds = []) {
+skipIds = [], recoveryOnly = false) {
     /* THROUGH AN RPC, NOT PostgREST — and this is not a style preference, it is
        the fix for a bug that made the whole slice inert.
   
@@ -229,7 +231,7 @@ skipIds = []) {
         p_machine_id: machineId,
         p_agent: agent,
         p_skip_ids: skipIds,
-    });
+    }, recoveryOnly);
     if (error)
         throw error;
     /* 18b Slice 2: the claim also carries the request's OWN project (id and name),
@@ -493,7 +495,7 @@ claimedAt) {
     if (error)
         throw error;
     if (!written?.length)
-        return;
+        return false;
     /* 18c SLICE 1 — CLOSE OUT THE LAST ACTIVITY LINE.
      *
      * report_activity only ever flips the PREVIOUS 'doing' row to 'done' when a
@@ -517,6 +519,7 @@ claimedAt) {
     catch (err) {
         console.warn(`answerTodo: closing out the last activity line failed: ${err.message}`);
     }
+    return true;
 }
 /**
  * !Cleanup PHASE 6 (I15) — DID THIS WORKER ASK SOMETHING AND EXIT?
@@ -542,7 +545,7 @@ claimedAt) {
  * a card asking the user for something that does not exist, unclearable except
  * by Stop. A wrong `done` is recoverable by Retry; a phantom question is not.
  */
-export async function todoHasOpenAsk(client, todoId, warn) {
+export async function todoHasOpenAsk(client, todoId, warn, strict = false) {
     try {
         const { data, error } = await client
             .from('cliv2_todo_asks')
@@ -554,11 +557,13 @@ export async function todoHasOpenAsk(client, todoId, warn) {
             throw new Error(error.message);
         if ((data ?? []).length > 0)
             return true;
-        if (await todoHasOpenDecision(client, todoId, warn))
+        if (await todoHasOpenDecision(client, todoId, warn, strict))
             return true;
-        return await todoHasOpenBlockedNotice(client, todoId, warn);
+        return await todoHasOpenBlockedNotice(client, todoId, warn, strict);
     }
     catch (err) {
+        if (strict)
+            throw err;
         warn(`[orchestrator] could not check for open questions: ${err.message}`);
         return false;
     }
@@ -582,7 +587,7 @@ export async function todoHasOpenAsk(client, todoId, warn) {
  * Best-effort and false on error, like its two siblings, and for the same
  * reason: a wrong `done` is recoverable by Retry, a phantom park is not.
  */
-export async function todoHasOpenBlockedNotice(client, todoId, warn) {
+export async function todoHasOpenBlockedNotice(client, todoId, warn, strict = false) {
     try {
         const { data, error } = await client
             .from('cliv2_blocked_notices')
@@ -595,6 +600,8 @@ export async function todoHasOpenBlockedNotice(client, todoId, warn) {
         return (data ?? []).length > 0;
     }
     catch (err) {
+        if (strict)
+            throw err;
         warn(`[orchestrator] could not check for an open stop: ${err.message}`);
         return false;
     }
@@ -635,7 +642,7 @@ const PERMISSION_CATEGORY = 'cliv2_permission';
  * BEST-EFFORT, like every read in the tick. A failure returns false, which costs
  * a wrongly-finished card rather than a lost run.
  */
-export async function todoHasOpenDecision(client, todoId, warn) {
+export async function todoHasOpenDecision(client, todoId, warn, strict = false) {
     try {
         const { data: links, error: linkError } = await client
             .from('cliv2_loose_todo_links')
@@ -667,6 +674,8 @@ export async function todoHasOpenDecision(client, todoId, warn) {
         return (data ?? []).length > 0;
     }
     catch (err) {
+        if (strict)
+            throw err;
         warn(`[orchestrator] could not check the work item's questions: ${err.message}`);
         return false;
     }
@@ -779,7 +788,7 @@ export async function todoRunLeftATrace(client, todoId, warn,
  *  same answer for the scope gate and resolving it twice would let the two
  *  disagree about which item this run is for. Null for a panel request with
  *  no work item, which is the greenfield case. */
-workItemId = null) {
+workItemId = null, strict = false) {
     try {
         /* ANY ONE IS ENOUGH, so the cheapest and most likely is asked first and the
            rest are skipped once one answers yes. A session is what the guidance
@@ -828,6 +837,8 @@ workItemId = null) {
         return (dropped.data?.dropped_activity ?? 0) > 0;
     }
     catch (err) {
+        if (strict)
+            throw err;
         /* See the block comment: towards `done`, so a transient read failure never
            discards a run that really did the work. */
         warn(`[orchestrator] could not check whether the run recorded anything: ${err.message}`);
@@ -866,9 +877,10 @@ claimedAt) {
         .update({ answer: trimmed || null, state: 'needs-input' })
         .eq('id', todoId)
         .in('state', ['working', 'needs-input']);
-    const { error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write);
+    const { data, error } = await (claimedAt ? write.eq('claimed_at', claimedAt) : write).select('id');
     if (error)
         throw error;
+    return !!data?.length;
 }
 /**
  * !Cleanup PHASE 6 (I17) — HAS THE USER STOPPED THIS?
@@ -950,6 +962,41 @@ export const MAX_ATTEMPTS = 3;
  *  so there is one count: a second read could land after another process had
  *  touched the row, and a wait computed from a stale count is a wait the card's
  *  own countdown would disagree with. */
+/** The existing terminal write, also used for a nonretryable native refusal. */
+async function failClaimedTodo(client, todoId, reason, expectedClaim, count) {
+    let attempts = count ?? 0;
+    if (count === undefined) {
+        const { data, error } = await client.from('cliv2_loose_todos').select('attempts,claimed_at').eq('id', todoId).maybeSingle();
+        if (error)
+            throw error;
+        if (!data || data.claimed_at !== expectedClaim)
+            return { fate: 'stale', attempts: data?.attempts ?? 0 };
+        attempts = Number(data.attempts ?? 0) + 1;
+    }
+    let failure = client
+        .from('cliv2_loose_todos')
+        .update({
+        state: 'failed',
+        attempts,
+        stopped_reason: reason.trim() || 'the work failed repeatedly',
+        claimed_machine_id: null,
+        claimed_agent: null,
+        claimed_at: null,
+        /* 18a SLICE 6: cleared at the bound. A wait on a terminal row would be
+           read by nothing and would outlive the row's own meaning; the only
+           route back is Retry, which means now. */
+        retry_after: null,
+    })
+        .eq('id', todoId)
+        .in('state', ['working', 'needs-input']);
+    failure = expectedClaim ? failure.eq('claimed_at', expectedClaim) : failure.is('claimed_at', null);
+    const { data: failed, error: failError } = await failure.select('id');
+    if (failError)
+        throw failError;
+    if (!failed?.length)
+        return { fate: 'stale', attempts: attempts - 1 };
+    return { fate: 'failed', attempts };
+}
 export async function releaseOrFail(client, todoId, reason, 
 /** 18a Slice 6: injectable so a test can place the wait exactly. */
 nowMs = Date.now(), 
@@ -974,31 +1021,8 @@ claimedAt) {
     }
     const attempts = (current?.attempts ?? 0) + 1;
     const expectedClaim = claimedAt ?? current?.claimed_at ?? null;
-    if (attempts >= MAX_ATTEMPTS) {
-        let failure = client
-            .from('cliv2_loose_todos')
-            .update({
-            state: 'failed',
-            attempts,
-            stopped_reason: reason.trim() || 'the work failed repeatedly',
-            claimed_machine_id: null,
-            claimed_agent: null,
-            claimed_at: null,
-            /* 18a SLICE 6: cleared at the bound. A wait on a terminal row would be
-               read by nothing and would outlive the row's own meaning; the only
-               route back is Retry, which means now. */
-            retry_after: null,
-        })
-            .eq('id', todoId)
-            .in('state', ['working', 'needs-input']);
-        failure = expectedClaim ? failure.eq('claimed_at', expectedClaim) : failure.is('claimed_at', null);
-        const { data: failed, error: failError } = await failure.select('id');
-        if (failError)
-            throw failError;
-        if (!failed?.length)
-            return { fate: 'stale', attempts: current?.attempts ?? 0 };
-        return { fate: 'failed', attempts };
-    }
+    if (attempts >= MAX_ATTEMPTS)
+        return failClaimedTodo(client, todoId, reason, expectedClaim, attempts);
     /* 18a SLICE 6: THE WAIT IS WRITTEN ON THE SAME STATEMENT AS THE COUNT, for
        the reason the count is on the same statement as the release: a wait that
        can fail on its own is a wait the daemon might not honour, and the failure
@@ -1270,7 +1294,7 @@ injectedSpawn,
  *  worker's, and injected by the same convention (`deps.resolveContext`) so a
  *  test can point it somewhere without `resolveClaimContext` growing
  *  parameters for the suite's sake. */
-resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
+resolveContext = (claim) => resolveClaimContext(client, claim, warn), observeHarness) {
     /* ═══ 18k SLICE 10b — NO TOOLS MEANS NO ANSWER, the tick's own rule. ═══
        An answerer now holds the worker's toolset, so one started while the tools
        server is down would have NONE of it: it could not read the work item, could
@@ -1315,14 +1339,14 @@ resolveContext = (claim) => resolveClaimContext(client, claim, warn)) {
     const endClaim = beginOwnedClaim();
     state.takeInFlight = true;
     try {
-        await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext);
+        await takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext, observeHarness);
     }
     finally {
         state.takeInFlight = false;
         endClaim();
     }
 }
-async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext) {
+async function takeRunMessagesInner(client, userId, machineId, agents, state, log, warn, spawn, resolveContext, observeHarness) {
     /* ═══ 18k SLICE 10 — A MACHINE ANSWERS ITS OWN RUNS, DESIGNATED OR NOT. ═══
        Slice 7 gated this whole function on being the designated orchestrator, and
        that is wrong for work already claimed: the user may switch the toolbar to
@@ -1394,6 +1418,13 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
        scoped take with an empty id list, which correctly returns nothing — so this
        is a saved round trip every ten seconds on every machine in the fleet that is
        not participating, not a safety property. The safety is the scope itself. */
+    if (hasOwnedWorkContext()) {
+        for (const claim of recoveredRpcClaims('cli').filter(row => row.action !== 'cliv2_claim_next_loose_todo')) {
+            for (const ref of cliClaimReferences(claim.action, claim.result))
+                if (ref.surface === 'reply' && pendingRpcReference(ref) && !state.answering.has(ref.todoId) && !ownTodoIds.includes(ref.todoId))
+                    ownTodoIds.push(ref.todoId);
+        }
+    }
     if (ownTodoIds.length === 0 && !designated)
         return;
     try {
@@ -1561,7 +1592,7 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
             void answerRunMessages(client, todoId, group.messageIds, { agent: plan.agent, cwd: context.checkout, workingCopyElsewhere: plan.workingCopyElsewhere }, state, spawn, warn, { surface: 'reply', todoId,
                 messageLeases: taken.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, taken_at: row.taken_at })),
                 grantLeases: granted.filter((row) => row.todo_id === todoId).map((row) => ({ id: row.id, resume_taken_at: row.resume_taken_at })),
-                observedPendingMessageIds: null }, machineId).catch(err => {
+                observedPendingMessageIds: null }, machineId, undefined, observeHarness).catch(err => {
                 warn(`[orchestrator] could not answer ${shortId(todoId)}: ${err.message}`);
             });
         }
@@ -1571,7 +1602,7 @@ async function takeRunMessagesInner(client, userId, machineId, agents, state, lo
     }
     finally {
         if (hasOwnedWorkContext())
-            finishRpcClaims(claimOperations);
+            finishRpcClaims(claimOperations, true);
     }
 }
 /**
@@ -2207,7 +2238,7 @@ plan, state,
 /** 18k Slice 10b — THE WORKER'S OWN SPAWN SHAPE, not a responder's. There is no
  *  second adapter any more: this is what `defaultSpawn(mcpServer)` returns, and
  *  the answerer differs from a dispatch only in passing no `alsoRunningIn`. */
-spawn, warn, attempt, machineId, owned) {
+spawn, warn, attempt, machineId, owned, observeHarness) {
     /* ═══ 18k SLICE 10b — ONE ANSWER PER RUN AT A TIME. ═══
        DEPTH, NOT THE PRIMARY DEFENCE. `takeRunMessagesInner` subtracts
        `state.answering` from the scoped take's id list, so a run this machine is
@@ -2237,8 +2268,9 @@ spawn, warn, attempt, machineId, owned) {
     owned = hasOwnedWorkContext() ? reserveOwnedWork(attempt, plan.agent) : undefined;
     state.answering.add(todoId);
     try {
-        const answered = await answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned);
-        owned?.complete();
+        const answered = await answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned, observeHarness);
+        if (answered)
+            owned?.complete();
         return answered;
     }
     finally {
@@ -2246,18 +2278,49 @@ spawn, warn, attempt, machineId, owned) {
         owned?.finishPreparation();
     }
 }
-async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned) {
+async function settleReplyOutcome(client, machineId, attempt, outcome, onAccepted) {
+    const todoId = attempt.todoId;
+    const finished = await client.rpc('cliv2_finish_reply_attempt', {
+        p_todo_id: todoId, p_machine_id: machineId, p_message_leases: attempt.messageLeases,
+        p_grant_leases: attempt.grantLeases, p_body: outcome.text,
+        ...(!outcome.ok && !outcome.stopped ? { p_failure_kind: outcome.failureKind ?? 'unknown' } : {}),
+    });
+    if (finished.error)
+        throw finished.error;
+    const receipt = finished.data?.[0]?.outcome;
+    if (receipt === 'stale')
+        return false;
+    if (receipt === 'failed' || receipt === 'already_failed') {
+        onAccepted?.();
+        return false;
+    }
+    if (receipt !== 'accepted' && receipt !== 'already_settled')
+        throw new Error('The reply acknowledgement was incomplete.');
+    onAccepted?.();
+    if (outcome.stopped && outcome.replyingToClaim)
+        await releaseStoppedTodo(client, todoId, outcome.replyingToClaim);
+    else if (outcome.workerIsLive && outcome.replyingToClaim) {
+        const scheduled = await client.from('cliv2_loose_todos')
+            .update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
+            .eq('id', todoId).eq('claimed_at', outcome.replyingToClaim).in('state', ['working', 'needs-input']);
+        if (scheduled.error)
+            throw scheduled.error;
+    }
+    return !outcome.stopped;
+}
+async function answerRunMessagesInner(client, todoId, messageIds, plan, state, spawn, warn, attempt, machineId, owned, observeHarness) {
     /* ALREADY GIVEN UP ON. Said ONCE, at the bound, and then silently skipped —
        the `reportTakeFailed` rule: a line every two minutes forever would bury
        every useful one. */
-    const attempts = Math.max(...messageIds.map(id => state.replyAttempts.get(id) ?? 0));
+    const sourceIds = [...messageIds, ...(attempt?.grantLeases ?? []).map(grant => `grant:${grant.id}`)];
+    const attempts = Math.max(0, ...sourceIds.map(id => state.replyAttempts.get(id) ?? 0));
     if (attempts >= MAX_REPLY_ATTEMPTS)
         return false;
     const fail = (reason) => {
-        for (const id of messageIds) {
+        for (const id of sourceIds) {
             state.replyAttempts.set(id, (state.replyAttempts.get(id) ?? 0) + 1);
         }
-        const spent = Math.max(...messageIds.map(id => state.replyAttempts.get(id) ?? 0));
+        const spent = Math.max(0, ...sourceIds.map(id => state.replyAttempts.get(id) ?? 0));
         if (spent >= MAX_REPLY_ATTEMPTS) {
             warn(`[orchestrator] gave up answering ${shortId(todoId)} after ${MAX_REPLY_ATTEMPTS} attempts: ${reason}`);
         }
@@ -2335,6 +2398,7 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
         attempt.observedPendingMessageIds = (pending.data ?? []).map((row) => row.id);
         owned?.setReference(attempt);
     }
+    const harnessIdentity = executionIdentity(plan.agent);
     let spawnedPid = null;
     const result = await spawn(plan.agent, buildReplyPrompt(record, bodies, { hasCheckout: plan.cwd !== null, workingCopyElsewhere: plan.workingCopyElsewhere }, workerIsLive), undefined, plan.cwd, () => todoWasStopped(client, todoId), (pid) => {
         state.livePids.add(pid);
@@ -2346,40 +2410,34 @@ async function answerRunMessagesInner(client, todoId, messageIds, plan, state, s
         state.livePids.delete(spawnedPid);
     if (result.interrupted)
         return false;
-    if (!result.ok && !result.stopped) {
-        fail(result.error);
-        return false;
-    }
-    const body = result.stopped ? 'Stopped.' : result.output.trim();
-    if (!body) {
-        fail('it said nothing');
+    const failureKind = result.failureKind ?? nativeFailureKind(result.saidTail ?? '');
+    if (!result.ok && !result.stopped && failureKind === 'transient' && attempts + 1 < MAX_REPLY_ATTEMPTS) {
+        fail(failureMessage(plan.agent, failureKind));
+        owned?.complete();
         return false;
     }
+    if (!result.ok && !result.stopped && failureKind === 'transient')
+        fail(failureMessage(plan.agent, failureKind));
+    const succeeded = result.ok && !!result.output.trim();
+    const body = result.stopped ? 'Stopped.' : succeeded ? result.output.trim() : null;
     if (!attempt || !machineId)
         throw new Error('The reply has no exact source lease identity.');
-    const finished = await client.rpc('cliv2_finish_reply_attempt', {
-        p_todo_id: todoId, p_machine_id: machineId, p_message_leases: attempt.messageLeases,
-        p_grant_leases: attempt.grantLeases, p_body: body,
+    const terminal = { surface: 'reply', completedAt: Date.now(), harnessIdentity, ok: succeeded, text: body, reason: succeeded ? '' : failureMessage(plan.agent, failureKind), failureKind: succeeded ? undefined : failureKind, stopped: !!result.stopped, replyingToClaim, workerIsLive: result.ok && workerIsLive };
+    owned?.deferOutcome(terminal);
+    // A durable terminal result is awaiting acknowledgement, never another native attempt.
+    for (const id of sourceIds)
+        state.replyAttempts.delete(id);
+    const answered = await settleReplyOutcome(client, machineId, attempt, terminal, () => {
+        if (succeeded && !result.stopped)
+            observeHarness?.(plan.agent, 'authenticated', 'dispatch-success', terminal.completedAt, terminal.harnessIdentity);
+        else if (failureKind === 'authentication')
+            observeHarness?.(plan.agent, 'sign-in-required', 'provider-rejected', terminal.completedAt, terminal.harnessIdentity);
     });
-    if (finished.error)
-        throw finished.error;
-    const outcome = finished.data?.[0]?.outcome;
-    if (outcome === 'stale')
-        return false;
-    if (outcome !== 'accepted' && outcome !== 'already_settled')
-        throw new Error('The reply acknowledgement was incomplete.');
-    for (const id of messageIds)
+    owned?.acknowledgeOutcome();
+    owned?.complete();
+    for (const id of sourceIds)
         state.replyAttempts.delete(id);
-    if (result.stopped && replyingToClaim)
-        await releaseStoppedTodo(client, todoId, replyingToClaim);
-    else if (workerIsLive && replyingToClaim) {
-        const scheduled = await client.from('cliv2_loose_todos')
-            .update({ claimed_machine_id: null, claimed_agent: null, claimed_at: null })
-            .eq('id', todoId).eq('claimed_at', replyingToClaim).in('state', ['working', 'needs-input']);
-        if (scheduled.error)
-            throw scheduled.error;
-    }
-    return !result.stopped;
+    return answered;
 }
 /** 18k Slice 7 — say ONCE that the take is failing, not every 10 seconds.
  *
@@ -3708,10 +3766,8 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
     const codexHome = agent === 'claude'
         ? null
         : ensureCodexRunHome(mcpServer, runTodoId, runTodoId ? `${runTodoId}-${randomUUID().slice(0, 8)}` : null);
-    if (agent !== 'claude' && !codexHome) {
-        console.warn('[orchestrator] codex is running WITHOUT an isolated configuration: no signed-in codex ' +
-            'credential to seed a per-run home. This worker inherits the MCP servers, instruction ' +
-            'files and permission posture on this machine.');
+    if (codexHome instanceof CodexHomeFailure) {
+        return { ok: false, output: '', error: codexHome.message, failureKind: codexHome.kind };
     }
     /* Spread the parent's env rather than replacing it: codex needs PATH, HOME,
        the proxy variables and the user's shell environment to run at all. Only the
@@ -4126,7 +4182,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
                    401 that means signed out. Each tailed separately so a chatty stdout
                    cannot push stderr out of the window. */
                 const saidTail = `${stdout.slice(-STDERR_TAIL_BYTES)}\n${stderrTail}`;
-                return settle({ ok: false, output: '', error: `${agent} ${how}`, saidTail });
+                return settle({ ok: false, output: '', error: `${agent} ${how}`, saidTail, failureKind: nativeFailureKind(stdout, stderrTail) });
             }
             /* 18c SLICE 8 — CODEX CAN EXIT 0 ON A TURN THAT FAILED, so the exit code
                above is not the last word. Checked AFTER the code, because a non-zero
@@ -4136,7 +4192,7 @@ export async function spawnAgent(agent, prompt, timeoutMs = ORCHESTRATOR_SPAWN_T
                release path, like every other failure, so the unit is retried. */
             if (codexTurnFailed !== null) {
                 const saidTail = `${stdout.slice(-STDERR_TAIL_BYTES)}\n${stderrTail}`;
-                return settle({ ok: false, output: '', error: `${agent} ${codexTurnFailed}`, saidTail });
+                return settle({ ok: false, output: '', error: `${agent} ${codexTurnFailed}`, saidTail, failureKind: nativeFailureKind(stdout, stderrTail) });
             }
             /* PREFER WHAT WAS CAPTURED LIVE, FOR BOTH HARNESSES (18c Slice 8 — it used
                to be claude only). `stdout` is now the fallback only for a run that
@@ -4353,7 +4409,7 @@ export async function startWorker(client, todoId, machineId, agent, warn = conso
 /** Move a worker through its lifecycle. `died` REQUIRES a reason — the table's
  *  check constraint refuses the row otherwise, deliberately: a failure the panel
  *  cannot explain gives the user nothing to act on. */
-export async function updateWorker(client, workerId, state, error = null, warn = console.warn) {
+export async function updateWorker(client, workerId, state, error = null, warn = console.warn, strict = false) {
     if (!workerId)
         return;
     try {
@@ -4370,6 +4426,8 @@ export async function updateWorker(client, workerId, state, error = null, warn =
             throw writeError;
     }
     catch (err) {
+        if (strict)
+            throw err;
         warn(`[orchestrator] could not update the worker: ${err.message}`);
     }
 }
@@ -5270,9 +5328,14 @@ export async function orchestratorTick(deps) {
         return 'busy';
     state.inFlight += 1;
     try {
-        await reconcileInterruptedWork(deps.client, deps.machineId);
+        await reconcileInterruptedWork(deps.client, deps.machineId, deps.observeHarness);
         return await runTick(deps, state, now, log, warn);
     }
+    catch (error) {
+        // This supervising tick must survive an outage and retry its saved recovery.
+        warn(`orchestrator poll failed, will retry: ${error instanceof Error ? error.message : JSON.stringify(error)}`);
+        return 'failed';
+    }
     finally {
         state.inFlight -= 1;
     }
@@ -5295,6 +5358,7 @@ async function runTick(deps, state, now, log, warn) {
            line is precisely what was wrong with it: a failing agent stopped the whole
            daemon rather than stopping itself. The wait is now per request, on the row,
            and the claim itself skips a request whose moment has not come. */
+        const recovered = hasOwnedWorkContext() ? recoveredRpcClaims('cli').find(row => row.action === 'cliv2_claim_next_loose_todo' && agents.includes(row.args?.p_agent)) : undefined;
         let role;
         try {
             const { data, error } = await client
@@ -5311,8 +5375,12 @@ async function runTick(deps, state, now, log, warn) {
             // is not a designation, and guessing would be exactly the fleet-wide race
             // the machine check exists to prevent.
             warn(`orchestrator designation read failed, will retry: ${err.message}`);
-            return 'failed';
+            if (!recovered)
+                return 'failed';
+            role = { kind: 'none' };
         }
+        if (recovered)
+            role = { kind: 'this-machine', agent: recovered.args.p_agent };
         reportRole(state, role, machineName, log);
         if (role.kind !== 'this-machine')
             return 'not-designated';
@@ -5383,7 +5451,7 @@ async function runTick(deps, state, now, log, warn) {
         for (;;) {
             let claim;
             try {
-                claim = await claimNextTodo(client, machineId, role.agent, [...skip, ...legacyHeldTodoIds()]);
+                claim = await claimNextTodo(client, machineId, role.agent, [...skip, ...legacyHeldTodoIds()], !!recovered);
                 if (claim?.claimOperationId)
                     claimOperations.push(claim.claimOperationId);
             }
@@ -5465,7 +5533,7 @@ async function runTick(deps, state, now, log, warn) {
                 }
                 const dispatched = dispatchClaim(deps, state, now, log, warn, role, runClaim, context, chosen);
                 if (hasOwnedWorkContext())
-                    finishRpcClaims(claimOperations);
+                    finishRpcClaims(claimOperations, true);
                 endClaim();
                 return await dispatched;
             }
@@ -5497,7 +5565,7 @@ async function runTick(deps, state, now, log, warn) {
     }
     finally {
         if (hasOwnedWorkContext())
-            finishRpcClaims(claimOperations);
+            finishRpcClaims(claimOperations, true);
         endClaim();
     }
 }
@@ -5554,6 +5622,57 @@ async function dispatchClaim(deps, state, now, log, warn, role, claim, context,
         forgetSecrets(claim.id);
     }
 }
+async function settleWorkerOutcome(client, workerId, claim, role, result, targeted, now, startedAt, log, warn, saved, persist, onAccepted) {
+    let disposition = saved?.disposition;
+    if (!disposition) {
+        if (!result.ok) {
+            const kind = result.failureKind ?? nativeFailureKind(result.saidTail ?? '');
+            const reason = failureMessage(role.agent, kind);
+            disposition = { worker: 'died', reason, todo: kind === 'transient' ? 'retry' : 'failed' };
+        }
+        else if (await todoHasOpenAsk(client, claim.id, warn, true)) {
+            disposition = { worker: 'done', reason: '', todo: 'needs-input' };
+        }
+        else if (!await todoRunLeftATrace(client, claim.id, warn, targeted, true)) {
+            disposition = { worker: 'died', reason: NOTHING_RECORDED_REASON, todo: 'retry' };
+        }
+        else
+            disposition = { worker: 'done', reason: '', todo: 'done' };
+        if (saved) {
+            saved.disposition = disposition;
+            persist?.(saved);
+        }
+    }
+    if (disposition.todo === 'needs-input') {
+        const accepted = await suspendTodoForAsk(client, claim.id, result.output, claim.claimedAt);
+        if (accepted)
+            onAccepted?.();
+        await updateWorker(client, workerId, 'done', null, warn, true);
+        log(`[orchestrator] ${shortId(claim.id)} is waiting on the user`);
+        return 'answered';
+    }
+    if (disposition.todo === 'done') {
+        const accepted = await answerTodo(client, claim.id, result.output, claim.claimedAt);
+        if (accepted)
+            onAccepted?.();
+        await updateWorker(client, workerId, 'done', null, warn, true);
+        log(`[orchestrator] answered ${shortId(claim.id)} in ${((now().getTime() - startedAt) / 1000).toFixed(1)}s`);
+        return 'answered';
+    }
+    await updateWorker(client, workerId, 'died', disposition.reason, warn, true);
+    const { fate, attempts } = disposition.todo === 'retry'
+        ? await releaseOrFail(client, claim.id, disposition.reason, now().getTime(), claim.claimedAt)
+        : await failClaimedTodo(client, claim.id, disposition.reason, claim.claimedAt ?? null);
+    if (fate === 'stale')
+        return 'nothing-to-do';
+    onAccepted?.();
+    if (fate === 'failed') {
+        warn(`[orchestrator] ${shortId(claim.id)} failed: ${disposition.reason}`);
+        return 'failed';
+    }
+    warn(`[orchestrator] retrying ${shortId(claim.id)} in ${Math.round(retryDelayMs(attempts) / 1000)}s`);
+    return 'released';
+}
 async function dispatchClaimInner(deps, state, now, log, warn, role, claim, 
 /* !Cleanup Phase 7 Slice 3 — RESOLVED BY THE CALLER, and passed in rather than
    resolved again here. `runTick` needs the same answer first, to know which
@@ -5669,6 +5788,7 @@ chosenCodebases = [], owned) {
     let workerId = null;
     /* !Cleanup Phase 6b (I43) — outside the try for the same reason: the `finally`
        has to remove it from `livePids` however this run ended. */
+    let terminalDeferred = false;
     let spawnedPid = null;
     try {
         log(`[orchestrator] picked up "${claim.instruction}" (${shortId(claim.id)})`);
@@ -5855,115 +5975,35 @@ chosenCodebases = [], owned) {
         if (result.interrupted)
             return 'nothing-to-do';
         if (result.stopped) {
-            await updateWorker(client, workerId, 'died', 'stopped by the user', warn);
+            terminalDeferred = !!owned;
+            owned?.deferOutcome({ surface: 'worker', ok: false, text: null, reason: 'stopped by the user', stopped: true });
+            await updateWorker(client, workerId, 'died', 'stopped by the user', warn, true);
             await releaseStoppedTodo(client, claim.id, claim.claimedAt);
+            owned?.acknowledgeOutcome();
+            terminalDeferred = false;
             log(`[orchestrator] ${shortId(claim.id)} was stopped by the user`);
             return 'released';
         }
-        if (!result.ok) {
-            /* THE WORKER DIED. Its row says so, WITH the reason, before the unit is
-               released — so a panel watching this moment sees "worker stopped" rather
-               than a unit that silently went back to unclaimed. Order matters: the
-               release is what makes the todo claimable again, so anything that must
-               be true about the dead worker has to be written first. */
-            /* 18a SLICE 5: THE WORKER ROW CARRIES THE PLAIN REASON, because that row
-               is what the card reads BETWEEN attempts. `result.error` stays the log's
-               string; the stored one is the sentence. */
-            const plain = plainFailureReason(role.agent, result.error, result.saidTail ?? '');
-            await updateWorker(client, workerId, 'died', plain, warn);
-            /* ux.md state D — released, not left claimed and silent. !Cleanup Phase 6
-               (I18): released ONLY while attempts remain. At the bound this writes
-               `failed` with the reason instead, which is what stops the unit being
-               re-claimed forever while reading as Working. */
-            const { fate, attempts } = await releaseOrFail(client, claim.id, plain, now().getTime(), claim.claimedAt);
-            if (fate === 'stale')
-                return 'nothing-to-do';
-            if (fate === 'failed') {
-                warn(`[orchestrator] ${result.error} — ${shortId(claim.id)} failed after ${MAX_ATTEMPTS} attempts`);
-                return 'failed';
-            }
-            /* 18a SLICE 6: THIS REQUEST WAITS, AND NOTHING ELSE DOES. The wait went
-               onto the row with the attempt count; what is left here is saying so, in
-               the line that used to say only "releasing". */
-            const seconds = Math.round(retryDelayMs(attempts) / 1000);
-            warn(`[orchestrator] ${result.error}: retrying ${shortId(claim.id)} in ${seconds}s`);
-            return 'released';
-        }
-        /* !Cleanup PHASE 6 (I15) — A QUESTION IS NOT A FINISHED JOB.
-           The child exited cleanly, which until now meant `done` unconditionally.
-           But a one-shot headless run ASKS AND EXITS — that is the only way it can
-           handle a question — so the clean exit says nothing about whether the work
-           is over. If an ask is open on this unit, the truthful state is that it
-           needs the user, and the Inbox already has the question. */
-        if (await todoHasOpenAsk(client, claim.id, warn)) {
-            await suspendTodoForAsk(client, claim.id, result.output, claim.claimedAt);
-            await updateWorker(client, workerId, 'done', null, warn);
-            log(`[orchestrator] ${shortId(claim.id)} is waiting on the user`);
-            return 'answered';
-        }
-        /* A CONFIDENT ANSWER WITH NOTHING BEHIND IT IS NOT A FINISHED JOB.
-           See `todoRunLeftATrace` for the measurement: a codex worker that was
-           correctly configured, connected, and fetched the tool list still reported
-           that it had no tools, twice in three identical runs, and one reported
-           ATTACHING a screenshot it never attached. Every one of them exited 0 with
-           prose on stdout, which is all the two checks above look at, so every one
-           of them landed here and was written `done`.
-    
-           PLACED AFTER THE ASK CHECK, and that order is load-bearing. A run that
-           asked a question and exited has legitimately recorded something and is
-           already handled as `needs-input`; reaching this check first would relabel
-           the product's most important state as a failure.
-    
-           RELEASED FOR RETRY RATHER THAN FAILED OUTRIGHT, through the SAME
-           `releaseOrFail` the dead-worker path uses. This is intermittent by nature
-           (the third run of the identical prompt worked perfectly), so a retry is
-           very likely to succeed, and the attempt bound is what stops it looping
-           forever. At the bound the card says why, in words about what is KNOWN
-           (nothing was recorded) rather than repeating the agent's own account of a
-           run that has just been shown to be an unreliable narrator.
-    
-           `targeted` IS PASSED, and without it this check is blind to the primary
-           flow. A session opened on a work item has `todo_id` NULL by the table's
-           own one-anchor constraint, and the prompt tells the agent to open it that
-           way whenever there is an item, so a `todo_id`-only lookup could never
-           match a targeted run. See `todoRunLeftATrace`. It is the same value the
-           scope gate above already resolved, so the two cannot disagree about which
-           item this request is for. */
-        if (!await todoRunLeftATrace(client, claim.id, warn, targeted)) {
-            /* THE WORKER DIED WITH NO STEP, and that blank is load-bearing rather
-               than incidental. `interruptedRunStep` looks up the most recent `died`
-               worker on this request and `describeInterruptedRun` then tells the
-               REPLACEMENT agent that a previous attempt "stopped partway" and that
-               anything it did "IS STILL THERE". For a genuinely toolless run that is
-               true and useful. Clearing the step keeps the claim honest about the one
-               thing that is known: this run recorded nothing, so there is no last
-               action to hand on and nothing for the next agent to resume from. */
-            await updateWorker(client, workerId, 'died', NOTHING_RECORDED_REASON, warn);
-            const { fate, attempts } = await releaseOrFail(client, claim.id, NOTHING_RECORDED_REASON, now().getTime(), claim.claimedAt);
-            if (fate === 'stale')
-                return 'nothing-to-do';
-            if (fate === 'failed') {
-                warn(`[orchestrator] ${shortId(claim.id)} recorded nothing and failed after ${MAX_ATTEMPTS} attempts`);
-                return 'failed';
-            }
-            const seconds = Math.round(retryDelayMs(attempts) / 1000);
-            warn(`[orchestrator] ${shortId(claim.id)} recorded nothing: retrying in ${seconds}s`);
-            return 'released';
-        }
-        /* FENCED ON THE CLAIM THIS DISPATCH TOOK (18d Slice 4). If this run handed
-           its stage on, the claim is already cleared and a fresh worker is on the
-           row, so this write must do nothing rather than mark the request finished
-           over work that is still running. See answerTodo's own comment. */
-        await answerTodo(client, claim.id, result.output, claim.claimedAt);
-        // The worker finished and its answer is committed. Marked done AFTER the
-        // answer write, so a worker is never reported finished for a unit that has
-        // no answer on it.
-        await updateWorker(client, workerId, 'done', null, warn);
-        const seconds = ((now().getTime() - startedAt) / 1000).toFixed(1);
-        log(`[orchestrator] answered ${shortId(claim.id)} in ${seconds}s`);
-        return 'answered';
+        terminalDeferred = !!owned;
+        const terminal = { surface: 'worker', ok: result.ok, text: result.ok ? result.output : null,
+            reason: result.ok ? '' : failureMessage(role.agent, result.failureKind ?? nativeFailureKind(result.saidTail ?? '')),
+            failureKind: result.failureKind ?? nativeFailureKind(result.saidTail ?? ''), workItemId: targeted };
+        owned?.deferOutcome(terminal);
+        const settled = await settleWorkerOutcome(client, workerId, claim, role, result, targeted, now, startedAt, log, warn, terminal, owned?.deferOutcome, () => {
+            if (result.ok)
+                deps.observeHarness?.(role.agent, 'authenticated', 'dispatch-success');
+            else if (result.failureKind === 'authentication')
+                deps.observeHarness?.(role.agent, 'sign-in-required', 'provider-rejected');
+        });
+        owned?.acknowledgeOutcome();
+        terminalDeferred = false;
+        return settled;
     }
     catch (err) {
+        if (terminalDeferred) {
+            warn('The completed execution is waiting for its original account to acknowledge the result.');
+            return 'failed';
+        }
         // The answer write itself failed (network, RLS, a reverted migration). The
         // todo is CLAIMED and has no answer — the worst state — so release it here
         // too. A release that also fails is warned and the row is left claimed;
@@ -6036,10 +6076,59 @@ function cliClaimReferences(action, result) {
         throw new Error('The claim journal returned an unrecognized lease identity.');
     });
 }
-async function cliClaim(client, action, args) {
+async function recoveredCliClaim(client, action, args) {
+    const skip = (args.p_skip_ids ?? args.p_skip_todo_ids ?? []);
+    for (const claim of recoveredRpcClaims('cli').filter(row => row.action === action)) {
+        if (action === 'cliv2_claim_next_loose_todo' && claim.args?.p_agent !== args.p_agent)
+            continue;
+        const rows = [];
+        for (const row of claim.result ?? []) {
+            const ref = cliClaimReferences(action, [row])[0];
+            if (ref.surface === 'panel' || !pendingRpcReference(ref))
+                continue;
+            if (skip.includes(ref.todoId) || Array.isArray(args.p_todo_ids) && !args.p_todo_ids.includes(ref.todoId))
+                continue;
+            let current = false;
+            if (ref.surface === 'worker') {
+                const read = await client.from('cliv2_loose_todos').select('state,claimed_at,claimed_machine_id').eq('id', ref.todoId).maybeSingle();
+                if (read.error)
+                    throw read.error;
+                current = !!read.data && read.data.claimed_at === ref.claimedAt && read.data.claimed_machine_id === args.p_machine_id && ['working', 'needs-input'].includes(read.data.state);
+            }
+            else if (action === 'cliv2_take_run_messages') {
+                const source = ref.messageLeases[0];
+                const read = await client.from('cliv2_run_messages').select('taken_at,taken_machine_id,answered_by,interrupted_at').eq('id', source.id).maybeSingle();
+                if (read.error)
+                    throw read.error;
+                current = !!read.data && read.data.taken_at === source.taken_at && read.data.taken_machine_id === args.p_machine_id && !read.data.answered_by && !read.data.interrupted_at;
+            }
+            else {
+                const source = ref.grantLeases[0];
+                const read = await client.from('cliv2_permission_requests').select('resume_taken_at,resume_machine_id,resume_answered_by,resume_interrupted_at,state').eq('id', source.id).maybeSingle();
+                if (read.error)
+                    throw read.error;
+                current = !!read.data && read.data.resume_taken_at === source.resume_taken_at && read.data.resume_machine_id === args.p_machine_id && !read.data.resume_answered_by && !read.data.resume_interrupted_at && read.data.state === 'granted';
+            }
+            if (current)
+                rows.push(row);
+            else
+                retireRpcReference(ref);
+        }
+        if (rows.length)
+            return { data: rows, error: null, operationId: claim.id };
+        finishRpcClaims([claim.id], true);
+    }
+    return null;
+}
+async function cliClaim(client, action, args, recoveryOnly = false) {
     if (!hasOwnedWorkContext())
         return { ...await client.rpc(action, args), operationId: undefined };
-    const operationId = beginRpcClaim('cli', action);
+    const recovered = await recoveredCliClaim(client, action, args);
+    if (recovered)
+        return recovered;
+    if (recoveryOnly)
+        return { data: [], error: null, operationId: undefined };
+    const operationId = beginRpcClaim('cli', action, args);
     try {
         const response = await client.rpc(action, { ...args, p_operation_id: operationId });
         if (!response.error) {
@@ -6049,7 +6138,10 @@ async function cliClaim(client, action, args) {
             const row = journal.data?.[0];
             if (row?.outcome !== 'completed' || row.action !== action)
                 throw new Error('The claim was cancelled before it could start.');
-            recordRpcClaims(operationId, cliClaimReferences(action, row.result));
+            recordRpcClaims(operationId, cliClaimReferences(action, row.result), false, row.result);
+            if (row.result.length === 0)
+                finishRpcClaims([operationId]);
+            return { ...response, data: row.result, operationId: row.result.length ? operationId : undefined };
         }
         if (response.error)
             deferRpcClaim(operationId);
@@ -6083,13 +6175,46 @@ async function legacyRows(client, table, columns, todoId) {
             throw new Error('The old work snapshot has no continuation identity.');
     }
 }
-export async function reconcileInterruptedWork(client, machineId) {
+export async function reconcileInterruptedWork(client, machineId, observeHarness) {
     if (!hasOwnedWorkContext())
         return;
     const capability = await client.rpc('cliv2_interrupt_machine_work', { p_machine_id: machineId,
         p_operation_id: randomUUID(), p_interrupted_at: new Date().toISOString(), p_workers: [], p_replies: [] });
     if (capability.error)
         throw capability.error;
+    for (const pending of pendingTerminalOutcomes()) {
+        if (pending.reference.surface === 'reply') {
+            await settleReplyOutcome(client, machineId, pending.reference, pending.outcome, () => {
+                const outcome = pending.outcome;
+                if (!outcome.completedAt || !outcome.harnessIdentity || !['claude', 'codex'].includes(pending.harness ?? '') || outcome.stopped)
+                    return;
+                if (outcome.ok || outcome.failureKind === 'authentication')
+                    observeHarness?.(pending.harness, outcome.ok ? 'authenticated' : 'sign-in-required', outcome.ok ? 'dispatch-success' : 'provider-rejected', outcome.completedAt, outcome.harnessIdentity);
+            });
+            acknowledgeTerminalOutcome(pending.id);
+            continue;
+        }
+        if (pending.reference.surface !== 'worker')
+            continue;
+        const ref = pending.reference;
+        if (pending.outcome.stopped) {
+            await updateWorker(client, ref.workerId, 'died', 'stopped by the user', message => console.warn(message), true);
+            await releaseStoppedTodo(client, ref.todoId, ref.claimedAt);
+            acknowledgeTerminalOutcome(pending.id);
+            continue;
+        }
+        const { data, error } = await client.from('cliv2_loose_todos').select('claimed_at,state').eq('id', ref.todoId).maybeSingle();
+        if (error)
+            throw error;
+        if (data?.claimed_at === ref.claimedAt && ['working', 'needs-input'].includes(data.state)) {
+            await settleWorkerOutcome(client, ref.workerId, { id: ref.todoId, claimedAt: ref.claimedAt }, { agent: pending.harness ?? 'Agent' }, { ok: pending.outcome.ok, output: pending.outcome.text ?? '', error: pending.outcome.reason, failureKind: pending.outcome.failureKind }, pending.outcome.workItemId ?? null, () => new Date(), Date.now(), () => { }, message => console.warn(message), pending.outcome, value => updateTerminalOutcome(pending.id, value));
+        }
+        else {
+            const disposition = pending.outcome.disposition;
+            await updateWorker(client, ref.workerId, disposition?.worker ?? (pending.outcome.ok ? 'done' : 'died'), disposition?.reason || pending.outcome.reason || null, message => console.warn(message), true);
+        }
+        acknowledgeTerminalOutcome(pending.id);
+    }
     for (const claim of pendingRpcClaims('cli')) {
         const journal = await client.rpc('cliv2_reconcile_claim', { p_machine_id: machineId, p_operation_id: claim.id });
         if (journal.error)
@@ -6098,7 +6223,7 @@ export async function reconcileInterruptedWork(client, machineId) {
         if (row?.outcome === 'cancelled')
             recordRpcClaims(claim.id, [], true);
         else if (row?.outcome === 'completed' && row.action === claim.action)
-            recordRpcClaims(claim.id, cliClaimReferences(claim.action, row.result), true);
+            recordRpcClaims(claim.id, cliClaimReferences(claim.action, row.result), false, row.result);
         else
             throw new Error('The pending claim could not be reconciled.');
     }