@cmgmyr/hive 1.6.0 → 1.7.0

package/dist/scheduler.js

@@ -12,7 +12,7 @@ import { getProjectByPath, listProjects, queenHomeDir } from "./context.js";
 import { collectPortfolio } from "./portfolio.js";
 import { QUEEN_GENERATED_MARKER, readQueenBrief, renderQueenDashboard } from "./queenDashboard.js";
 import { COMMAND_KIND, stoppingMarkerLive } from "./processes.js";
-import { closeAgentRow, isLeadActorId, LEAD_ACTOR_PREFIX, LEAD_KIND, reapCodexHomeForClosedAgent } from "./spawn.js";
+import { closeAgentRow, isLeadActorId, LEAD_ACTOR_PREFIX, LEAD_KIND, PaneOwnershipLost, reapCodexHomeForClosedAgent, requireStillOwned, } from "./spawn.js";
 import { awaitingFirstPrompt, awaitingFirstPromptSql } from "./firstPrompt.js";
 import { evaluateLeadWatch, leadSubscription } from "./leadWatch.js";
 import { describeLiveTasks, describeOneTask, liveBackgroundTasks, STALL_BOUND_SECONDS, } from "./backgroundTasks.js";
@@ -20,7 +20,7 @@ import { MESSAGE_MAX_ROWS, MESSAGE_RETENTION } from "./leadMessage.js";
 import { harnessFor, hasTranscriptSignal, paneClassifierFor, screenClassifiable, transcriptDirFor } from "./harnesses.js";
 import { transcriptDir, readContextFill } from "./transcript.js";
 import { ageSecondsSince, describeLastLogEvent, humanizeAge, lastLogEvent, reportsAgentStateLog, } from "./stateProvenance.js";
-import { capturePane, foreignSocket, holdsHumanInput, liveTargets, maskChoiceMarker, paneInCopyMode, paneReissued, rowAlive, rowAliveProbe, rowLiveProbe, renderTrailerView, sanitizeTail, sendText, tailCaptureLines, sleep, tmuxSocketPath, } from "./tmux.js";
+import { capturePane, foreignSocket, holdsHumanInput, liveTargets, maskChoiceMarker, observationFailed, paneInCopyMode, paneReissued, rowAlive, rowAliveProbe, rowLiveProbe, rowOwnership, renderTrailerView, sanitizeTail, sendText, tailCaptureLines, sleep, tmuxSocketPath, } from "./tmux.js";
 import { appendTeardown, NOT_ATTRIBUTED, sqlNow, teardownWindow, TEARDOWN_TRIGGER, } from "./teardown.js";
 export const WATCH_SCOPE_PROJECT = "project";
 const isStandingWatch = (timer) => timer.watch_scope === WATCH_SCOPE_PROJECT;
@@ -28,6 +28,24 @@ export const DELIVER_SOCKET_JOIN = `LEFT JOIN agents ON agents.id = (
   SELECT a.id FROM agents a WHERE a.actor_id = wakes.deliver_actor
    ORDER BY (a.status = 'running') DESC, a.id DESC LIMIT 1
 )`;
+const DELIVER_ROW_COLUMNS = `COALESCE(agents.tmux_socket, '') AS deliver_socket,
+  COALESCE(agents.pane_pid, '') AS deliver_pane_pid,
+  COALESCE(agents.command, '') AS deliver_command,
+  agents.id AS deliver_row_id, agents.status AS deliver_row_status,
+  COALESCE(agents.tmux_target, '') AS deliver_row_target`;
+// A row's pid certifies only the pane that row records, never a different deliver_pane.
+const deliveryIdentity = (row) => ({
+    tmux_target: row.deliver_pane,
+    tmux_socket: row.deliver_socket,
+    pane_pid: row.deliver_row_target === row.deliver_pane ? row.deliver_pane_pid : "",
+});
+function deliveryOwnership(row, snapshot) {
+    if (row.deliver_row_id === null)
+        return "raw";
+    if (row.deliver_row_status !== "running")
+        return "closed";
+    return rowOwnership(deliveryIdentity(row), snapshot);
+}
 export const ACTIVE_TIMER_WHERE = "cancelled_at IS NULL AND (fired_at IS NULL OR repeat_every_ms IS NOT NULL)";
 const SETTLE_WINDOW = "-15 seconds";
 const prepared = new Map();
@@ -92,7 +110,7 @@ const teardownMember = (row, swept) => ({
 // Leads are READ for the roster and never swept: janitor's own query excludes them deliberately, and
 // a crew death that took a lead's pane is exactly the casualty the roster would otherwise omit.
 function goneLeadRows(snapshot) {
-    return stmt(`SELECT ${CREW_ROW_COLUMNS} FROM agents WHERE status = 'running' AND kind = ? AND tmux_target != ''`).all(LEAD_KIND).filter((row) => rowAliveProbe(row.tmux_socket, row.tmux_target, snapshot).live === false);
+    return stmt(`SELECT ${CREW_ROW_COLUMNS} FROM agents WHERE status = 'running' AND kind = ? AND tmux_target != ''`).all(LEAD_KIND).filter((row) => ["gone", "reissued"].includes(rowOwnership(row, snapshot)));
 }
 function recordTeardown(snapshot, swept) {
     try {
@@ -156,11 +174,11 @@ function reportDeadProcess(agent, snapshot) {
         // interrupted between the two is not reported as a crash.
         if (stoppingMarkerLive(agent.project_id, agent.id))
             return;
-        const lead = stmt(`SELECT actor_id, tmux_target, tmux_socket FROM agents
+        const lead = stmt(`SELECT actor_id, tmux_target, tmux_socket, pane_pid FROM agents
         WHERE project_id = ? AND kind = ? AND status = 'running' ORDER BY id DESC LIMIT 1`).get(agent.project_id, LEAD_KIND);
         // Never mint a notice hive could not deliver: with no live lead pane there is nobody this could
         // ever reach, and the row would sit in wakes forever.
-        if (!lead?.tmux_target || rowAlive(lead.tmux_socket, lead.tmux_target, snapshot) !== true)
+        if (!lead?.tmux_target || rowOwnership(lead, snapshot) !== "live")
             return;
         insertNotice({ project_id: agent.project_id, owner: lead.actor_id }, lead.actor_id, lead.tmux_target, deadProcessBody(agent.name), null);
     }
@@ -176,9 +194,7 @@ export function reportRunningBuildChange(snapshot) {
             return;
         const lead = stmt(`SELECT project_id, actor_id, tmux_target, tmux_socket, pane_pid, command FROM agents
        WHERE actor_id = ? AND kind = ? AND status = 'running'`).get(actor, LEAD_KIND);
-        if (!lead?.tmux_target || !screenClassifiable(lead.command) ||
-            rowAlive(lead.tmux_socket, lead.tmux_target, snapshot) !== true ||
-            paneReissued(lead.pane_pid, rowAliveProbe(lead.tmux_socket, lead.tmux_target, snapshot)))
+        if (!lead?.tmux_target || !screenClassifiable(lead.command) || rowOwnership(lead, snapshot) !== "live")
             return;
         insertNotice({ project_id: lead.project_id, owner: lead.actor_id }, lead.actor_id, lead.tmux_target, runningBuildNotice(change), null);
         reportedBuilds.add(change.disk.build_id);
@@ -220,15 +236,19 @@ export function janitor(snapshot = liveTargets()) {
     // recording that as a teardown manufactures an incident out of an unset PATH.
     const sawARealServer = snapshot.serverAnswered !== false;
     const teardown = sawARealServer && snapshot.panes.size === 0 && swept.length > 0 ? recordTeardown(snapshot, swept) : undefined;
-    const wakes = stmt(`SELECT wakes.id, wakes.due_at, wakes.held_reason, wakes.deliver_pane,
-            COALESCE(agents.tmux_socket, '') AS deliver_socket
+    const wakes = stmt(`SELECT wakes.id, wakes.due_at, wakes.held_reason, wakes.deliver_pane, wakes.deliver_actor, ${DELIVER_ROW_COLUMNS}
        FROM wakes ${DELIVER_SOCKET_JOIN}
       WHERE ${ACTIVE_TIMER_WHERE} AND wakes.deliver_actor NOT LIKE ?
         AND wakes.created_at < datetime('now', ?)`).all(`${LEAD_ACTOR_PREFIX}%`, SETTLE_WINDOW);
     for (const timer of wakes) {
+        if (deliveryOwnership(timer, snapshot) === "closed") {
+            cancelWithReason(timer.id, HELD_REASON_ACTOR_CLOSED);
+            cancelledTimers += 1;
+            continue;
+        }
         if (rowAlive(timer.deliver_socket, timer.deliver_pane, snapshot) === false) {
-            if (wasHeldForPaneReissue(timer.held_reason)) {
-                holdTimer(timer, HELD_REASON_PANE_REISSUED_THEN_DEAD);
+            if (wasHeldForPaneIdentity(timer.held_reason)) {
+                holdTimer(timer, identityThenDead(timer.held_reason));
             }
             else {
                 cancelTimer(timer.id);
@@ -247,6 +267,9 @@ export function janitor(snapshot = liveTargets()) {
 function cancelTimer(timerId) {
     stmt("UPDATE wakes SET cancelled_at = datetime('now') WHERE id = ?").run(timerId);
 }
+function cancelWithReason(timerId, reason) {
+    bestEffortRun("UPDATE wakes SET cancelled_at = datetime('now'), held_reason = ? WHERE id = ? AND cancelled_at IS NULL", reason, timerId);
+}
 export const LOG_RETENTION = "-7 days";
 const LOG_MAX_ROWS = 20_000;
 function pruneStateLog() {
@@ -469,9 +492,7 @@ export async function tick(snapshot) {
         maybeBackupHourly(db, dataDir);
         maybeGenerateDashboards();
         const now = stmt("SELECT datetime('now') AS now").get().now;
-        const candidates = stmt(`SELECT wakes.*, COALESCE(agents.tmux_socket, '') AS deliver_socket,
-              COALESCE(agents.pane_pid, '') AS deliver_pane_pid,
-              COALESCE(agents.command, '') AS deliver_command
+        const candidates = stmt(`SELECT wakes.*, ${DELIVER_ROW_COLUMNS}
          FROM wakes ${DELIVER_SOCKET_JOIN}
         WHERE wakes.cancelled_at IS NULL AND (
          (wakes.kind = 'delay' AND wakes.due_at <= datetime('now')
@@ -567,6 +588,28 @@ const HELD_REASON_PANE_REISSUED_THEN_DEAD = `${HELD_REASON_PANE_REISSUED_PREFIX}
 export function wasHeldForPaneReissue(heldReason) {
     return heldReason != null && heldReason.startsWith(HELD_REASON_PANE_REISSUED_PREFIX);
 }
+const HELD_REASON_PANE_UNKNOWN_PREFIX = "pane ownership for this wake's row cannot be verified (no pane pid was recorded for it, its row now " +
+    "records a different pane, or the pane is on another tmux socket); held rather than typed into a pane " +
+    "that may not be its own - ";
+const HELD_REASON_PANE_UNKNOWN_LEAD = `${HELD_REASON_PANE_UNKNOWN_PREFIX}run \`hive lead\` to start or adopt a lead with a recorded pane and re-point it`;
+const HELD_REASON_PANE_UNKNOWN_WORKER = `${HELD_REASON_PANE_UNKNOWN_PREFIX}nothing re-points a worker's wake automatically, so cancel it with ` +
+    "wake_cancel (any running lead may, even though the wake is not theirs), or leave it: it keeps holding " +
+    "rather than deliver wrongly";
+const HELD_REASON_PANE_UNKNOWN_THEN_DEAD = `${HELD_REASON_PANE_UNKNOWN_PREFIX}that pane has since gone dead too; nothing re-points a worker's wake ` +
+    "automatically, so cancel it with wake_cancel (any running lead may) if it is no longer needed";
+// One family for both identity holds, so a later dead pane re-holds them instead of cancelling.
+export function wasHeldForPaneIdentity(heldReason) {
+    return wasHeldForPaneReissue(heldReason) || (heldReason?.startsWith(HELD_REASON_PANE_UNKNOWN_PREFIX) ?? false);
+}
+const identityThenDead = (heldReason) => wasHeldForPaneReissue(heldReason) ? HELD_REASON_PANE_REISSUED_THEN_DEAD : HELD_REASON_PANE_UNKNOWN_THEN_DEAD;
+export const HELD_REASON_ACTOR_CLOSED = "delivery actor closed: the row this wake was bound to is closed, so nothing was typed and the wake was " +
+    "cancelled rather than delivered into a pane that row no longer owns";
+const HELD_REASON_LEAD_ROW_CLOSED = "the lead's row is closed, so no pane is this wake's to type into; held until `hive lead` starts or adopts a " +
+    "lead and re-points it";
+export function isLeadRowClosedHold(heldReason) {
+    return heldReason === HELD_REASON_LEAD_ROW_CLOSED;
+}
+const HELD_REASON_OWNERSHIP_LOST_AFTER_CLAIM = "delivery stopped after claim: ";
 export const HELD_REASON_UNCLASSIFIABLE_PANE_PREFIX = "the pane's harness is not one hive can classify; ";
 export const HELD_REASON_UNCLASSIFIABLE_PANE = `${HELD_REASON_UNCLASSIFIABLE_PANE_PREFIX}neither the dialog guard nor the unsubmitted-text ` +
     "guard can read this screen and delivery would type blind. wake_set and wake_when_idle refuse such a " +
@@ -599,13 +642,14 @@ export const HELD_REASON_CONVERSATION = "a human talked to this lead more recent
 const CONVERSATION_HOLD_TTL_SECONDS = 5 * 60;
 const CONVERSATION_HOLD_TTL = `-${CONVERSATION_HOLD_TTL_SECONDS} seconds`;
 // Must stay well under NOTICE_MAX_AGE, past which noticeDisposition cancels a held notice rather than
-// typing stale news (it says so in a replacement wake; it is not silent). Measured against due_at, not first_held_at: cli.ts's hive-lead
+// typing stale news (it says so in a replacement wake; it is not silent). Measured against due_at (idleCeilingStart for an idle wake), not first_held_at: cli.ts's hive-lead
 // re-point clears held_at (and so first_held_at) on every ordinary reattach, which would otherwise
 // launder this ceiling indefinitely.
 const CONVERSATION_HOLD_MAX = "-15 minutes";
-function conversationHoldsWake(timer) {
-    if (timer.due_at !== null) {
-        const withinCeiling = stmt(`SELECT ? >= datetime('now', ?) AS within`).get(timer.due_at, CONVERSATION_HOLD_MAX);
+function conversationHoldsWake(timer, snapshot) {
+    const clockStart = timer.due_at ?? idleCeilingStart(timer, snapshot);
+    if (clockStart !== null) {
+        const withinCeiling = stmt(`SELECT ? >= datetime('now', ?) AS within`).get(clockStart, CONVERSATION_HOLD_MAX);
         if (!withinCeiling.within)
             return false;
     }
@@ -671,11 +715,11 @@ function standingBlockNoticeBody(timer, names, observedAt) {
     return lines.join("\n");
 }
 function ownerPaneIfLive(timer, snapshot) {
-    const row = stmt(`SELECT tmux_target, tmux_socket FROM agents WHERE actor_id = ? AND status = 'running'
+    const row = stmt(`SELECT tmux_target, tmux_socket, pane_pid FROM agents WHERE actor_id = ? AND status = 'running'
       ORDER BY id DESC LIMIT 1`).get(timer.owner);
     if (!row?.tmux_target || snapshot === null)
         return null;
-    return rowAlive(row.tmux_socket, row.tmux_target, snapshot) === true ? row.tmux_target : null;
+    return rowOwnership(row, snapshot) === "live" ? row.tmux_target : null;
 }
 function blockNoticeTarget(timer, snapshot) {
     const pane = ownerPaneIfLive(timer, snapshot);
@@ -813,7 +857,7 @@ const standingBlockedRows = (timer, tellActor) => stmt(`SELECT ${CREW_COLUMNS},
 function blockedWatchedAgents(timer, snapshot, tellActor) {
     const rows = isStandingWatch(timer) ? standingBlockedRows(timer, tellActor) : oneShotBlockedRows(timer);
     return rows
-        .filter((r) => rowAlive(r.tmux_socket, r.tmux_target, snapshot) === true)
+        .filter((r) => rowOwnership(r, snapshot) === "live")
         .map((r) => ({
         id: r.id,
         name: r.name,
@@ -889,7 +933,7 @@ const unreported = (condition, episode) => `NOT EXISTS (
      WHERE n.wake_id = ? AND n.agent_id = a.id AND n.condition = '${condition}' AND n.episode = ${episode}
        AND NOT (nt.fired_at IS NOT NULL AND nt.typed_at IS NULL AND nt.cancelled_at IS NULL
                 AND nt.fired_at < datetime('now', '${NOTICE_RETRY_AFTER}')))`;
-const CREW_COLUMNS = `a.id, a.name, a.actor_id, a.tmux_target, a.tmux_socket, a.agent_state,
+const CREW_COLUMNS = `a.id, a.name, a.actor_id, a.tmux_target, a.tmux_socket, a.pane_pid, a.agent_state,
    a.state_changed_at, a.status, a.command, a.kind, a.resumed_at, a.closed_at,
    a.cwd, a.session_id, a.transcript_path`;
 // Scopes a standing watch to the OWNER's own crew (bound to owner, not deliver_actor - a watch can
@@ -1201,7 +1245,7 @@ function noteStandingTransitions(timer, snapshot) {
             for (const row of standingIdleRows(timer)) {
                 if (row.tmux_target === pane)
                     continue;
-                if (rowAlive(row.tmux_socket, row.tmux_target, snapshot) !== true)
+                if (rowOwnership(row, snapshot) !== "live")
                     continue;
                 if (!idleIsAFreshTransition(timer.id, row))
                     continue;
@@ -1321,7 +1365,7 @@ function noteStalledCrew(timer, snapshot, choices) {
             if (row.agent_state === "waiting") {
                 if (snapshot === null)
                     continue;
-                if (rowAlive(row.tmux_socket, row.tmux_target, snapshot) !== true)
+                if (rowOwnership(row, snapshot) !== "live")
                     continue;
                 if (awaitingChoice(row.tmux_target, row.command, choices) !== false)
                     continue;
@@ -1414,33 +1458,63 @@ const ageOutNotice = db.transaction((timer) => {
         return;
     insertNotice(timer, timer.deliver_actor, timer.deliver_pane, agedOutBody(timer), null);
 });
+function freshDeliveryRow(timerId) {
+    return stmt(`SELECT wakes.deliver_actor, wakes.deliver_pane, ${DELIVER_ROW_COLUMNS}
+       FROM wakes ${DELIVER_SOCKET_JOIN} WHERE wakes.id = ? AND wakes.cancelled_at IS NULL`).get(timerId);
+}
+// Applies the hold, cancel or wait for a row-bound wake whose row does not own deliver_pane.
+function refuseUnowned(timer, ownership, snapshot) {
+    const lead = isLeadActorId(timer.deliver_actor);
+    switch (ownership) {
+        case "closed":
+            if (lead)
+                holdTimer(timer, HELD_REASON_LEAD_ROW_CLOSED);
+            else
+                cancelWithReason(timer.id, HELD_REASON_ACTOR_CLOSED);
+            return;
+        case "unknown":
+            // A failed observation re-evaluates next tick; it is never read as gone.
+            if (!foreignSocket(timer.deliver_socket) && observationFailed(snapshot))
+                return;
+            holdTimer(timer, lead ? HELD_REASON_PANE_UNKNOWN_LEAD : HELD_REASON_PANE_UNKNOWN_WORKER);
+            return;
+        case "reissued":
+            holdTimer(timer, lead ? HELD_REASON_PANE_REISSUED_LEAD : HELD_REASON_PANE_REISSUED_WORKER);
+            return;
+        case "gone":
+            if (lead)
+                holdTimer(timer, HELD_REASON_LEAD_PANE_DEAD);
+            else if (wasHeldForPaneIdentity(timer.held_reason))
+                holdTimer(timer, identityThenDead(timer.held_reason));
+            else
+                cancelTimer(timer.id);
+            return;
+    }
+}
 function deliverable(timer, snapshot, choices) {
-    const probe = snapshot
-        ? rowAliveProbe(timer.deliver_socket, timer.deliver_pane, snapshot)
-        : rowLiveProbe(timer.deliver_socket, timer.deliver_pane);
-    const live = probe.live;
-    if (live === null)
+    const row = freshDeliveryRow(timer.id);
+    if (!row || row.deliver_pane !== timer.deliver_pane)
         return { ok: false };
-    if (!live) {
-        if (isLeadActorId(timer.deliver_actor)) {
-            holdTimer(timer, HELD_REASON_LEAD_PANE_DEAD);
-        }
-        else {
-            if (wasHeldForPaneReissue(timer.held_reason)) {
-                holdTimer(timer, HELD_REASON_PANE_REISSUED_THEN_DEAD);
-            }
-            else {
-                cancelTimer(timer.id);
-            }
+    let owner = null;
+    const ownership = deliveryOwnership(row, snapshot);
+    if (ownership === "raw") {
+        const probe = snapshot
+            ? rowAliveProbe(row.deliver_socket, row.deliver_pane, snapshot)
+            : rowLiveProbe(row.deliver_socket, row.deliver_pane);
+        if (probe.live === null)
+            return { ok: false };
+        if (!probe.live) {
+            refuseUnowned(timer, "gone", snapshot);
+            return { ok: false };
         }
-        return { ok: false };
     }
-    if (paneReissued(timer.deliver_pane_pid, probe)) {
-        holdTimer(timer, isLeadActorId(timer.deliver_actor)
-            ? HELD_REASON_PANE_REISSUED_LEAD
-            : HELD_REASON_PANE_REISSUED_WORKER);
+    else if (ownership !== "live") {
+        refuseUnowned(timer, ownership, snapshot);
         return { ok: false };
     }
+    else {
+        owner = { rowId: row.deliver_row_id, identity: deliveryIdentity(row) };
+    }
     // Above both pane reads: this population must not be captured at all, and the verdicts would be
     // wrong in opposite directions if they were.
     if (!screenClassifiable(timer.deliver_command)) {
@@ -1469,15 +1543,11 @@ function deliverable(timer, snapshot, choices) {
     // so it sits after them. No notice-claim of its own: the human being held for is the human at the
     // keyboard, and the statusline is what tells them (see the two dead-ends on notice-claim latching
     // and on splitting a hold reason in two).
-    if (isLeadActorId(timer.deliver_actor) && conversationHoldsWake(timer)) {
+    if (isLeadActorId(timer.deliver_actor) && conversationHoldsWake(timer, snapshot)) {
         holdTimer(timer, HELD_REASON_CONVERSATION);
         return { ok: false };
     }
-    const pid = timer.deliver_pane_pid === "" || probe.pid === null
-        ? "no-fact"
-        : probe.pid === timer.deliver_pane_pid
-            ? "ok"
-            : "reissued";
+    const pid = owner === null ? "no-fact" : "ok";
     const dialogSeen = dialog === false ? "no" : "unknown";
     const boxState = cacheEntry(timer.deliver_pane, choices).box;
     const box = boxState === undefined ? "unknown" : boxState === null ? "absent" : boxState.state;
@@ -1485,6 +1555,7 @@ function deliverable(timer, snapshot, choices) {
         ok: true,
         typedSeen: `live=yes pid=${pid} dialog=${dialogSeen} box=${box}`,
         firstHeldAt: timer.held_at != null ? timer.first_held_at : null,
+        owner,
     };
 }
 const PANE_CLAIM_TTL_SECONDS = 6;
@@ -1557,7 +1628,7 @@ async function fireDelayClaimed(timer, snapshot, choices) {
     }
     if (!claimed)
         return false;
-    await deliver(timer, "", choices, decision.typedSeen, decision.firstHeldAt);
+    await deliver(timer, "", choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
     return true;
 }
 function claimOneShot(timer) {
@@ -1573,10 +1644,10 @@ function watchedStates(timer, snapshot) {
         const agent = stmt(`SELECT *, created_at < datetime('now', ?) AS settled FROM agents WHERE id = ?`).get(SETTLE_WINDOW, id);
         if (!agent || agent.status !== "running")
             return GONE;
-        const alive = rowAlive(agent.tmux_socket, agent.tmux_target, snapshot);
-        if (alive === null)
+        const ownership = rowOwnership(agent, snapshot);
+        if (ownership === "unknown")
             return UNKNOWN;
-        if (!alive) {
+        if (ownership !== "live") {
             if (!agent.settled)
                 return UNKNOWN;
             return GONE;
@@ -1594,6 +1665,28 @@ function idleConditionMet(timer, snapshot) {
         ? states.some((s) => s.gone || (s.idle && s.since != null && s.since >= timer.created_at))
         : states.length > 0 && states.every((s) => s.idle);
 }
+// An idle wake has no due_at, so its ceiling clock is the moment its condition was met (the watched
+// worker's state_changed_at, latest for idle_all, earliest qualifying for idle_any), or max_wait_at
+// once it has timed out unmet. A met condition with no such moment (a gone worker) falls back to
+// first_held_at.
+function idleCeilingStart(timer, snapshot) {
+    if (timer.kind !== "idle_any" && timer.kind !== "idle_all")
+        return null;
+    if (snapshot === null || !idleConditionMet(timer, snapshot)) {
+        if (timer.max_wait_at === null)
+            return null;
+        const timedOut = stmt(`SELECT ? <= datetime('now') AS out`).get(timer.max_wait_at);
+        return timedOut.out ? timer.max_wait_at : null;
+    }
+    const states = watchedStates(timer, snapshot);
+    const sinces = states
+        .filter((s) => (timer.kind === "idle_any" ? s.idle && !s.gone && s.since != null && s.since >= timer.created_at : true))
+        .map((s) => s.since);
+    const met = sinces.length > 0 && sinces.every((v) => v != null) ? sinces : null;
+    if (met !== null)
+        return timer.kind === "idle_any" ? met.reduce((a, b) => (a < b ? a : b)) : met.reduce((a, b) => (a > b ? a : b));
+    return timer.held_at != null ? timer.first_held_at : null;
+}
 async function maybeFireIdle(timer, snapshot, now, choices) {
     const timedOut = timer.max_wait_at != null && timer.max_wait_at <= now;
     if (isStandingWatch(timer)) {
@@ -1606,7 +1699,7 @@ async function maybeFireIdle(timer, snapshot, now, choices) {
                 const decision = deliverable(timer, snapshot, choices);
                 if (!decision.ok || !claimOneShot(timer))
                     return false;
-                await deliver(timer, STANDING_EXPIRED_NOTE, choices, decision.typedSeen, decision.firstHeldAt);
+                await deliver(timer, STANDING_EXPIRED_NOTE, choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
                 return true;
             });
         }
@@ -1632,7 +1725,7 @@ async function maybeFireIdle(timer, snapshot, now, choices) {
             const decision = deliverable(timer, snapshot, choices);
             if (!decision.ok || !claimOneShot(timer))
                 return false;
-            await deliver(timer, timedOut && !idleMet ? "max wait reached" : "", choices, decision.typedSeen, decision.firstHeldAt);
+            await deliver(timer, timedOut && !idleMet ? "max wait reached" : "", choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
             return true;
         });
     }
@@ -1662,7 +1755,7 @@ async function maybeFireLeadWatch(timer, sub, snapshot, now, choices) {
         if (result.kind === "invalid") {
             bestEffortRun("UPDATE lead_idle_subscriptions SET terminal_reason = ? WHERE wake_id = ?", result.reason, timer.id);
         }
-        await deliver(timer, note, choices, decision.typedSeen, decision.firstHeldAt);
+        await deliver(timer, note, choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
         return true;
     });
 }
@@ -1700,7 +1793,7 @@ function watchedTail(timer) {
             return "";
         const shown = [];
         for (const id of ids.slice(0, TAIL_AGENTS)) {
-            const agent = stmt("SELECT name, tmux_target, tmux_socket, agent_state, state_changed_at, status, actor_id, command, kind, cwd, session_id, transcript_path FROM agents WHERE id = ?").get(id);
+            const agent = stmt("SELECT name, tmux_target, tmux_socket, pane_pid, agent_state, state_changed_at, status, actor_id, command, kind, cwd, session_id, transcript_path FROM agents WHERE id = ?").get(id);
             if (!agent)
                 continue;
             const context = contextClause(agent);
@@ -1713,6 +1806,12 @@ function watchedTail(timer) {
                     "socket than this process, so it cannot honestly be read from here.");
                 continue;
             }
+            const ownership = rowOwnership(agent);
+            if (ownership !== "live") {
+                shown.push(`${agent.name} (hive state now: ${stateNowClause(agent)}${context}): its pane ownership reads ${ownership}, ` +
+                    "so its terminal was not read.");
+                continue;
+            }
             let tail = "";
             try {
                 const view = paneClassifierFor(agent.command)?.trailerView(agent.tmux_target) ?? null;
@@ -1786,7 +1885,14 @@ export function shortRenderForLeadDelivery(timer) {
         return null;
     return standingNoticeBodyShort(timer, candidates, timer.id);
 }
-async function deliver(timer, note, choices, typedSeen, firstHeldAt) {
+// The claim already happened, so this records why and never re-arms or replays the wake.
+function recordLostAfterClaim(timer, pasted, why) {
+    bestEffortRun(`UPDATE wakes SET held_reason = ?,
+       typed_at = CASE WHEN ? = 1 THEN strftime('%Y-%m-%d %H:%M:%f', 'now') ELSE typed_at END,
+       cancelled_at = CASE WHEN repeat_every_ms IS NULL THEN datetime('now') ELSE cancelled_at END
+     WHERE id = ?`, HELD_REASON_OWNERSHIP_LOST_AFTER_CLAIM + why + (pasted ? " The body was pasted; no Enter was sent." : ""), pasted ? 1 : 0, timer.id);
+}
+async function deliver(timer, note, choices, typedSeen, firstHeldAt, owner) {
     const tail = watchedTail(timer);
     const prefix = `[hive wake #${timer.id}${note ? `, ${note}` : ""}] `;
     const body = (isLeadActorId(timer.deliver_actor) ? shortRenderForLeadDelivery(timer) : null) ?? timer.body;
@@ -1803,8 +1909,19 @@ async function deliver(timer, note, choices, typedSeen, firstHeldAt) {
        WHERE id = ?`, typedBusy, typedSeen, firstHeldAt, timer.id);
     const box = cacheEntry(timer.deliver_pane, choices).box;
     const strandedTextWouldHold = box !== undefined && box !== null && box.state !== "unknown";
+    let pasted = false;
     try {
-        await sendText(timer.deliver_pane, prefix + body + noticeStalenessNote(timer) + tail, true, strandedTextWouldHold ? recordTyped : undefined);
+        await sendText(timer.deliver_pane, prefix + body + noticeStalenessNote(timer) + tail, true, () => {
+            pasted = true;
+            if (strandedTextWouldHold)
+                recordTyped();
+        }, undefined, owner ? () => requireStillOwned(owner.rowId, owner.identity, `wake #${timer.id} delivery`) : undefined);
+    }
+    catch (err) {
+        if (!(err instanceof PaneOwnershipLost))
+            throw err;
+        recordLostAfterClaim(timer, pasted, err.message);
+        return;
     }
     finally {
         forgetPaneAnswers(timer.deliver_pane, choices);

package/dist/spawn.js

@@ -4,7 +4,7 @@ import { preservedRolloutPath, reapCodexHome } from "./codexHome.js";
 import { getProject } from "./context.js";
 import { dataDir, db } from "./db.js";
 import { loadProjectYml } from "./projectYml.js";
-import { applyLayout, claimInitialWindow, createWindow, DEFAULT_LAYOUT, ensureSession, crossServerRefusal, findProcessesWindow, findProjectWindow, isPaneTarget, panePid, paneWindow, PROCESSES_LAYOUT, processesPaneTitle, processesWindowName, retainOnExitArgs, rowLive, sessionName, setPaneTitle, makeProcessesWindow, tmux, tmuxSocketPath, untrustedTmuxServer, waitForPaneEstablished, windowLayout, windowOwner, windowTitle, } from "./tmux.js";
+import { applyLayout, claimInitialWindow, createWindow, DEFAULT_LAYOUT, ensureSession, crossServerRefusal, findProcessesWindow, findProjectWindow, isPaneTarget, panePidForRecord, paneWindow, PROCESSES_LAYOUT, processesPaneTitle, processesWindowName, retainOnExitArgs, rowLive, sessionName, setPaneTitle, makeProcessesWindow, tmux, tmuxSocketPath, untrustedTmuxServer, waitForPaneEstablished, windowLayout, windowOwner, windowTitle, rowOwnership, } from "./tmux.js";
 export function withWindowClaim(claim) {
     if (db.inTransaction) {
         throw new Error("withWindowClaim was called from inside an already-open transaction. better-sqlite3 nests a " +
@@ -51,10 +51,18 @@ export function agentIdentityEnv(actorId, name, projectPath) {
         HIVE_LEAD: "",
     };
 }
-function recordPane(agentId, target, socket) {
+function recordPane(agentId, target, socket, pid) {
     return (db
         .prepare("UPDATE agents SET tmux_target = ?, tmux_socket = ?, pane_pid = ? WHERE id = ? AND status = 'running'")
-        .run(target, socket, panePid(target), agentId).changes > 0);
+        .run(target, socket, pid, agentId).changes > 0);
+}
+function readPidOrDiscard(target) {
+    const pid = panePidForRecord(target);
+    if (pid !== "")
+        return pid;
+    discardOrphanedPane(target);
+    throw new Error(`tmux did not report a process id for the new pane ${target} (asked twice), so it was discarded rather than ` +
+        "recorded: a row with no pane pid can never be verified as owning its pane. Retry.");
 }
 // Two callers, one meaning: whatever is left in the window re-tiles to the layout it was arranged
 // with. agent_close kills a worker's pane through this; stopProcess (src/processes.ts) is the only
@@ -186,8 +194,9 @@ export function launchAgent(spec) {
         const envFlags = buildEnvFlags(env);
         const title = windowTitle(spec.projectName, spec.name);
         const { target, landedInProjectId, layoutApplied, inProcessesWindow } = placeAgentPane(session, spec, envFlags, commandString, title);
+        const pid = readPidOrDiscard(target);
         paneUp = true;
-        if (!recordPane(agentId, target, socket)) {
+        if (!recordPane(agentId, target, socket, pid)) {
             discardOrphanedPane(target);
             throw paneRacedRetirement(agentId);
         }
@@ -261,8 +270,9 @@ export function resumeAgent(spec) {
         const session = sessionName();
         const title = windowTitle(spec.projectName, spec.name);
         const { target, landedInProjectId } = placeAgentPane(session, spec, envFlags, spec.commandString, title);
+        const pid = readPidOrDiscard(target);
         paneUp = true;
-        if (!recordPane(spec.agentId, target, socket)) {
+        if (!recordPane(spec.agentId, target, socket, pid)) {
             discardOrphanedPane(target);
             throw paneRacedRetirement(spec.agentId);
         }
@@ -328,11 +338,51 @@ export function branchAt(cwd) {
         return branch;
     return git(["rev-parse", "--short", "HEAD"]) ?? "";
 }
-export function parkAgentRow(agentId, expectedTmuxTarget, branch) {
-    return db
-        .prepare("UPDATE agents SET status = 'closed', closed_at = datetime('now'), parked_at = datetime('now'), " +
-        "parked_branch = ? WHERE id = ? AND status = 'running' AND tmux_target = ? RETURNING parked_at")
-        .get(branch, agentId, expectedTmuxTarget)?.parked_at;
+export function parkAgentRow(agentId, expected, branch) {
+    const park = "UPDATE agents SET status = 'closed', closed_at = datetime('now'), parked_at = datetime('now'), " +
+        "parked_branch = ? WHERE id = ? AND status = 'running' AND tmux_target = ?";
+    const row = typeof expected === "string"
+        ? db.prepare(`${park} RETURNING parked_at`).get(branch, agentId, expected)
+        : db
+            .prepare(`${park} AND tmux_socket = ? AND pane_pid = ? RETURNING parked_at`)
+            .get(branch, agentId, expected.tmux_target, expected.tmux_socket, expected.pane_pid);
+    return row?.parked_at;
+}
+export const paneIdentity = (row) => ({
+    tmux_target: row.tmux_target,
+    tmux_socket: row.tmux_socket,
+    pane_pid: row.pane_pid,
+});
+// "changed" means the row is no longer running or no longer carries the identity the caller read.
+export function currentOwnership(agentId, expected, snapshot) {
+    const row = db
+        .prepare("SELECT status, tmux_target, tmux_socket, pane_pid FROM agents WHERE id = ?")
+        .get(agentId);
+    if (!row ||
+        row.status !== "running" ||
+        row.tmux_target !== expected.tmux_target ||
+        row.tmux_socket !== expected.tmux_socket ||
+        row.pane_pid !== expected.pane_pid) {
+        return "changed";
+    }
+    return rowOwnership(row, snapshot);
+}
+export class PaneOwnershipLost extends Error {
+    agentId;
+    ownership;
+    constructor(agentId, ownership, action) {
+        super(`[pane-ownership-lost] ${action} stopped before touching agent ${agentId}'s pane: ` +
+            (ownership === "changed"
+                ? "its row was closed or recorded a different pane since this call read it."
+                : `its pane ownership now reads ${ownership}, so the pane may belong to another process.`));
+        this.agentId = agentId;
+        this.ownership = ownership;
+    }
+}
+export function requireStillOwned(agentId, expected, action) {
+    const ownership = currentOwnership(agentId, expected);
+    if (ownership !== "live")
+        throw new PaneOwnershipLost(agentId, ownership, action);
 }
 export const PARK_STAMP_CLEARED = "parked_at = '', parked_branch = ''";
 export function releaseParkRow(agentId) {

package/dist/tmux.js

@@ -413,6 +413,10 @@ export function targetPid(target, snapshot) {
 export function panePid(target) {
     return targetLiveProbe(target).pid ?? "";
 }
+// A row recorded with an empty pid reads unknown for life, so a pane hive just created gets one retry.
+export function panePidForRecord(target) {
+    return panePid(target) || panePid(target);
+}
 const DESTROY_READINESS_BOUND_MS = 200;
 function paneHasEstablishedProcess(pid) {
     try {
@@ -492,6 +496,14 @@ export function rowOwnership(row, snapshot) {
         return "unknown";
     return pid === row.pane_pid ? "live" : "reissued";
 }
+// The one place a row's ownership is observed: a failed probe is its own answer, never unknown.
+export function observeRowOwnership(row) {
+    const observes = !foreignSocket(row.tmux_socket) && row.tmux_target !== "";
+    const snapshot = observes ? liveTargets() : null;
+    if (observes && observationFailed(snapshot))
+        return "probe-failed";
+    return rowOwnership(row, snapshot);
+}
 export function ownershipLiveness(ownership) {
     return ownership === "live" ? true : ownership === "unknown" ? null : false;
 }
@@ -1359,7 +1371,7 @@ export function findUnsafeControlChar(text, allowed) {
     }
     return null;
 }
-export async function sendText(target, text, submit = true, onPasted, onBuffered) {
+export async function sendText(target, text, submit = true, onPasted, onBuffered, beforePaneWrite) {
     // tmux hands a pane its input in 1022-byte writes, and only paste-buffer -p
     // brackets them, so send-keys -l loses everything before the last write.
     // An empty text skips the pair: set-buffer creates no buffer for it, so
@@ -1371,6 +1383,7 @@ export async function sendText(target, text, submit = true, onPasted, onBuffered
         tmux("set-buffer", "-b", buffer, "--", text);
         onBuffered?.();
         try {
+            beforePaneWrite?.();
             tmux("paste-buffer", "-d", "-p", "-b", buffer, "-t", target);
         }
         catch (err) {
@@ -1391,6 +1404,7 @@ export async function sendText(target, text, submit = true, onPasted, onBuffered
     }
     if (submit) {
         await sleep(ENTER_DELAY_MS);
+        beforePaneWrite?.();
         tmux("send-keys", "-t", target, "Enter");
     }
 }