@cmgmyr/hive 1.11.9 → 1.12.0

package/dist/scheduler.js

@@ -1,5 +1,5 @@
 import { QUEEN_AUDIT_RETENTION, QUEEN_AUDIT_MAX_ROWS, recentQueenAudit } from "./queenAudit.js";
-import { ACTIVE_TIMER_WHERE, DELIVERY_METHOD, FIRING_DROPPED_NOTE, HELD_REASON_ACTOR_CLOSED, HELD_REASON_CONVERSATION, HELD_REASON_COPY_MODE, HELD_REASON_HANDOFF, HELD_REASON_LEAD_BOOT_INPUT_BOX, HELD_REASON_LEAD_PANE_DEAD, HELD_REASON_LEAD_ROW_CLOSED, HELD_REASON_MODAL_CHOICE, HELD_REASON_OWNERSHIP_LOST_AFTER_CLAIM, HELD_REASON_PANE_REISSUED_LEAD, HELD_REASON_PANE_REISSUED_WORKER, HELD_REASON_PANE_UNKNOWN_LEAD, HELD_REASON_PANE_UNKNOWN_WORKER, HELD_REASON_UNCLASSIFIABLE_PANE, HELD_REASON_UNSUBMITTED_INPUT, HELD_REASON_UNSUBMITTED_INPUT_OWNER_UNRESOLVED, LOG_RETENTION, activeWakeWhere, holdableWakeWhere, identityThenDead, pendingOneShotWhere, repeatingFiringPendingWhere, repeatingSocketAwaitingWhere, repeatingSocketFiringWhere, schedulerCandidateWhere, sentWakeAtSql, socketAwaitingWhere, spentUntypedNotice, wasHeldForPaneIdentity, watchWatchingWhere, } from "./wakeSql.js";
+import { ACTIVE_TIMER_WHERE, DELIVERY_METHOD, FIRING_DROPPED_NOTE, HELD_REASON_ACTOR_CLOSED, HELD_REASON_CONVERSATION, HELD_REASON_COPY_MODE, HELD_REASON_HANDOFF, HELD_REASON_LEAD_BOOT_INPUT_BOX, HELD_REASON_LEAD_PANE_DEAD, HELD_REASON_LEAD_ROW_CLOSED, HELD_REASON_MODAL_CHOICE, HELD_REASON_OWNERSHIP_LOST_AFTER_CLAIM, HELD_REASON_PANE_CAPTURE_UNKNOWN, HELD_REASON_PANE_REISSUED_LEAD, HELD_REASON_PANE_REISSUED_WORKER, HELD_REASON_PANE_UNKNOWN_LEAD, HELD_REASON_PANE_UNKNOWN_WORKER, HELD_REASON_UNCLASSIFIABLE_PANE, HELD_REASON_UNSUBMITTED_INPUT, HELD_REASON_UNSUBMITTED_INPUT_OWNER_UNRESOLVED, HELD_REASON_USAGE_LIMIT, LOG_RETENTION, activeWakeWhere, holdableWakeWhere, identityThenDead, pendingOneShotWhere, repeatingFiringPendingWhere, repeatingSocketAwaitingWhere, repeatingSocketFiringWhere, schedulerCandidateWhere, sentWakeAtSql, socketAwaitingWhere, spentUntypedNotice, wasHeldForPaneIdentity, watchWatchingWhere, } from "./wakeSql.js";
 export { ACTIVE_TIMER_WHERE, DELIVERY_METHOD, HELD_REASON_ACTOR_CLOSED, HELD_REASON_CONVERSATION, HELD_REASON_COPY_MODE, HELD_REASON_LEAD_PANE_DEAD, HELD_REASON_UNCLASSIFIABLE_PANE, HELD_REASON_UNCLASSIFIABLE_PANE_PREFIX, HELD_REASON_UNSUBMITTED_INPUT, HELD_REASON_UNSUBMITTED_INPUT_PREFIX, isLeadRowClosedHold, isUnclassifiablePaneHold, isUnsubmittedInputHold, socketAwaitingWhere, wasHeldForPaneIdentity, wasHeldForPaneReissue, } from "./wakeSql.js";
 import { parseStoreTimestamp } from "./storeClock.js";
 import { runningBuildChange, runningBuildNotice, versionInfo } from "./version.js";
@@ -8,6 +8,7 @@ import { createHash } from "node:crypto";
 import { dirname, join, sep } from "node:path";
 import { dataDir, db, storeReplaced } from "./db.js";
 import { decodeKvValue } from "./kvValue.js";
+import { continueLimitedCodexWorkers, observeUsageLimit, usageLimitGate } from "./usageLimit.js";
 import { maybeBackupHourly } from "./backup.js";
 import { renderDashboardForWrite } from "./dashboard.js";
 import { snapshotProcesses } from "./processes.js";
@@ -28,6 +29,7 @@ import { driveLeadHandoff } from "./leadHandoff.js";
 import { handoffHoldsWake } from "./leadHandoffState.js";
 import { FIRST_MESSAGE_LOG_STATE, humanPromptSql, readLeadTurnState } from "./leadState.js";
 import { retryQuietLeadMessages } from "./leadMessageDelivery.js";
+import { markWorkerLost, WORKER_RECOVERY_RETENTION_SECONDS } from "./workerRecovery.js";
 import { harnessFor, hasTranscriptSignal, paneClassifierFor, screenClassifiable, transcriptDirFor } from "./harnesses.js";
 import { transcriptDir, readContextFill } from "./transcript.js";
 import { ageSecondsSince, describeLastLogEvent, humanizeAge, lastLogEvent, reportsAgentStateLog, } from "./stateProvenance.js";
@@ -86,7 +88,7 @@ export function startScheduler(intervalMs = DEFAULT_SCHEDULER_INTERVAL_MS) {
     }, intervalMs).unref();
 }
 const CREW_ROW_COLUMNS = "id, project_id, name, actor_id, kind, cwd, tmux_target, tmux_socket, pane_pid, session_id, agent_state, " +
-    "state_changed_at, created_at";
+    "state_changed_at, created_at, recovery_state";
 // Set only from a tick that SAW panes, so it separates "hive watched this socket die" from "hive
 // arrived after the fact" - the two bounds the record has to tell apart.
 let socketLastSeenAlive = null;
@@ -164,11 +166,13 @@ function recordTeardown(snapshot, swept) {
 function reapClosedCodexHomes() {
     let reaped = 0;
     try {
-        const rows = stmt("SELECT id, codex_home FROM agents WHERE codex_home != '' AND status = 'closed' AND parked_at = ''").all();
+        const rows = stmt(`SELECT id, codex_home FROM agents
+        WHERE codex_home != '' AND status = 'closed' AND parked_at = ''
+          AND (recovery_state != 'pending' OR closed_at <= datetime('now', ?))`).all(`-${WORKER_RECOVERY_RETENTION_SECONDS} seconds`);
         for (const row of rows) {
             try {
-                reapCodexHomeForClosedAgent(row.id, row.codex_home);
-                reaped += 1;
+                if (reapCodexHomeForClosedAgent(row.id, row.codex_home) !== null)
+                    reaped += 1;
             }
             catch {
             }
@@ -235,7 +239,19 @@ export function janitor(snapshot = liveTargets()) {
         if (probe.live === false || paneReissued(agent.pane_pid, probe)) {
             // The conditional UPDATE is the claim: two instances probing the same dead socket both reach
             // this branch, and only the one that actually closed the row may put it in a record.
-            if (closeAgentRow(agent.id)) {
+            const identity = {
+                tmux_target: agent.tmux_target,
+                tmux_socket: agent.tmux_socket,
+                pane_pid: agent.pane_pid,
+            };
+            const closed = agent.kind === "agent"
+                ? agent.recovery_state === "excluded"
+                    ? closeAgentRow(agent.id, identity)
+                    : snapshot?.serverAnswered === true
+                        ? markWorkerLost(agent.id, identity)
+                        : closeAgentRow(agent.id, identity)
+                : closeAgentRow(agent.id, identity);
+            if (closed) {
                 closedAgents += 1;
                 swept.push(agent);
             }
@@ -542,11 +558,18 @@ export async function tick(snapshot) {
         isolated("hourly backup", () => maybeBackupHourly(db, dataDir));
         isolated("dashboard generation", maybeGenerateDashboards);
         const now = stmt("SELECT datetime('now') AS now").get().now;
+        await isolated("Codex usage-limit continuation", () => continueLimitedCodexWorkers(Date.now()));
         const candidates = stmt(`SELECT wakes.*, ${DELIVER_ROW_COLUMNS}
          FROM wakes ${DELIVER_SOCKET_JOIN}
-        WHERE ${schedulerCandidateWhere("wakes.")}`).all();
+        WHERE ${schedulerCandidateWhere("wakes.")}
+        ORDER BY wakes.id`).all();
+        const orderedDelays = candidates
+            .filter((timer) => timer.kind === "delay")
+            .sort((a, b) => (a.due_at ?? "").localeCompare(b.due_at ?? "") || a.id - b.id);
+        let delayIndex = 0;
         const choices = new Map();
-        for (const timer of candidates) {
+        for (const candidate of candidates) {
+            const timer = candidate.kind === "delay" ? orderedDelays[delayIndex++] : candidate;
             await isolated(`timer ${timer.id}`, async () => {
                 if (timer.kind === "delay") {
                     await fireDelay(timer, tickSnapshot, choices);
@@ -622,6 +645,35 @@ function holdTimer(timer, reason) {
        first_held_at = COALESCE(CASE WHEN held_at IS NULL THEN NULL ELSE first_held_at END, datetime('now'))
      WHERE id = ? AND ${holdableWakeWhere()}`, reason, timer.id, timer.due_at, count, count);
 }
+function usageLimitHeld(timer) {
+    if (timer.deliver_row_id === null)
+        return false;
+    const now = Date.now();
+    observeUsageLimit(timer.deliver_row_id, now);
+    const gate = usageLimitGate(timer.deliver_row_id, now);
+    if (!gate.held)
+        return false;
+    holdTimer(timer, `${HELD_REASON_USAGE_LIMIT}${gate.until ?? "the recorded deadline"}`);
+    return true;
+}
+class UsageLimitRace extends Error {
+    until;
+    constructor(until) {
+        super(`${HELD_REASON_USAGE_LIMIT}${until}`);
+        this.until = until;
+    }
+}
+function unclaimUsageLimit(timer, until) {
+    try {
+        stmt(`UPDATE wakes SET due_at = ?, fired_at = NULL, fire_count = MAX(0, fire_count - 1),
+        typed_at = NULL, typed_busy = NULL, typed_seen = NULL, confirmed_at = NULL,
+        held_at = datetime('now'), held_reason = ?, first_held_at = COALESCE(first_held_at, datetime('now')),
+        socket_attempt_at = NULL, socket_delivery_note = NULL, delivery_method = NULL, delivered_by = NULL
+      WHERE id = ? AND fire_count = ? AND typed_at IS NULL AND cancelled_at IS NULL`)
+            .run(timer.due_at, `${HELD_REASON_USAGE_LIMIT}${until}`, timer.id, timer.fire_count);
+    }
+    catch { }
+}
 const LEAD_BOOT_BOX_WAIT_MS = 60_000;
 function leadBootSessionIsFresh(agentId, panePid) {
     const row = stmt("SELECT pane_started_at FROM agents WHERE id = ? AND pane_pid = ?").get(agentId, panePid);
@@ -927,6 +979,7 @@ function alreadyToldAbout(timerId, agentId, blockedSince) {
 const CONDITION_IDLE = "idle";
 const CONDITION_GONE = "gone";
 const NOTICE_MAX_AGE = "-1 hours";
+const PANE_CAPTURE_UNKNOWN_MAX_AGE = "-1 hours";
 const ROSTER_STILL_GOING = 8;
 const FINISHED_SHOWN_CAP = ROSTER_STILL_GOING;
 const STANDING_EXPIRED_NOTE = "this standing watch has expired and nothing is watching now; set a new one if the crew is still working";
@@ -1528,7 +1581,12 @@ function deliverable(timer, snapshot, choices) {
         holdTimer(timer, HELD_REASON_UNCLASSIFIABLE_PANE);
         return { ok: false };
     }
+    const releasedAfterCaptureHold = timer.held_reason === paneCaptureReleaseReason(timer.fire_count + 1);
     const dialog = awaitingChoice(timer.deliver_pane, timer.deliver_command, choices);
+    if (dialog === null && !releasedAfterCaptureHold && paneCaptureUnknownHoldApplies(timer)) {
+        holdTimer(timer, HELD_REASON_PANE_CAPTURE_UNKNOWN);
+        return { ok: false };
+    }
     if (dialog === true) {
         noteModalHold(timer, snapshot);
         return { ok: false };
@@ -1579,7 +1637,7 @@ function deliverable(timer, snapshot, choices) {
     const box = boxState === undefined ? "unknown" : boxState === null ? "absent" : boxState.state;
     return {
         ok: true,
-        typedSeen: `live=yes pid=${pid} dialog=${dialogSeen} box=${box}`,
+        typedSeen: `live=yes pid=${pid} dialog=${dialogSeen} box=${box}${releasedAfterCaptureHold ? "; released after unreadable-capture hold" : ""}`,
         firstHeldAt: timer.held_at != null ? timer.first_held_at : null,
         owner,
     };
@@ -1613,6 +1671,8 @@ async function fireDelay(timer, snapshot, choices) {
         holdTimer(timer, HELD_REASON_HANDOFF);
         return;
     }
+    if (usageLimitHeld(timer))
+        return;
     const disposition = noticeDisposition(timer);
     if (disposition === "aged") {
         try {
@@ -1628,7 +1688,37 @@ async function fireDelay(timer, snapshot, choices) {
     }
     if (timer.repeat_every_ms != null && repeatingFiringPending(timer.id))
         return;
-    await withPaneClaim(timer, () => fireDelayClaimed(timer, snapshot, choices));
+    const releaseReason = releaseExpiredPaneCaptureHold(timer);
+    const deliveryTimer = releaseReason === null ? timer : { ...timer, held_reason: releaseReason };
+    await withPaneClaim(deliveryTimer, () => fireDelayClaimed(deliveryTimer, snapshot, choices));
+}
+function paneCaptureReleaseReason(firing) {
+    return `released after unreadable-capture hold at the one-hour cap for firing #${firing}`;
+}
+function paneCaptureUnknownHoldApplies(timer) {
+    if (timer.deliver_row_id === null)
+        return false;
+    return stmt(`SELECT 1 AS hit FROM agent_usage_limits
+    WHERE agent_id = ? AND (
+      (released_at IS NULL AND expires_at >= datetime('now', '-1 hour'))
+      OR released_at >= datetime('now', '-1 hour')
+    )`).get(timer.deliver_row_id) !== undefined;
+}
+function releaseExpiredPaneCaptureHold(timer) {
+    if (timer.held_reason !== HELD_REASON_PANE_CAPTURE_UNKNOWN || timer.first_held_at === null)
+        return null;
+    try {
+        const age = stmt("SELECT ? <= datetime('now', ?) AS expired").get(timer.first_held_at, PANE_CAPTURE_UNKNOWN_MAX_AGE);
+        if (age.expired !== 1)
+            return null;
+        const reason = paneCaptureReleaseReason(timer.fire_count + 1);
+        const released = stmt(`UPDATE wakes SET held_reason = ?
+       WHERE id = ? AND held_reason = ? AND first_held_at IS ? AND ${holdableWakeWhere()}`).run(reason, timer.id, HELD_REASON_PANE_CAPTURE_UNKNOWN, timer.first_held_at, timer.due_at, timer.fire_count, timer.fire_count);
+        return released.changes === 1 ? reason : null;
+    }
+    catch {
+        return null;
+    }
 }
 async function fireDelayClaimed(timer, snapshot, choices) {
     const decision = deliverable(timer, snapshot, choices);
@@ -1644,21 +1734,34 @@ async function fireDelayClaimed(timer, snapshot, choices) {
          socket_attempt_at = NULL, socket_delivery_note = NULL, delivery_method = NULL, delivered_by = NULL
        WHERE id = ? AND due_at IS ? AND body IS ? AND repeat_every_ms IS ? AND cancelled_at IS NULL
          AND fire_count = ? AND NOT COALESCE((${REPEATING_FIRING_PENDING}), 0)
+         AND NOT EXISTS (SELECT 1 FROM agent_usage_limits u WHERE u.agent_id =
+           (SELECT a.id FROM agents a WHERE a.actor_id = wakes.deliver_actor
+             ORDER BY (a.status = 'running') DESC, a.id DESC LIMIT 1)
+           AND u.released_at IS NULL AND u.expires_at > datetime('now'))
        RETURNING fire_count, fired_at`).get(seconds, timer.id, timer.due_at, timer.body, timer.repeat_every_ms, timer.fire_count);
         if (claimed === undefined)
             return false;
         firing = { ...timer, fire_count: claimed.fire_count, fired_at: claimed.fired_at, socket_attempt_at: null, socket_delivery_note: null, delivery_method: null };
     }
-    else if (!claimOneShot(timer)) {
-        return false;
+    else {
+        const claimed = claimOneShot(timer);
+        if (!claimed)
+            return false;
+        firing = { ...timer, ...claimed };
     }
     await deliver(firing, "", choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
     return true;
 }
 function claimOneShot(timer) {
-    return (stmt(`UPDATE wakes SET fired_at = datetime('now'), fire_count = fire_count + 1
+    const claimed = stmt(`UPDATE wakes SET fired_at = datetime('now'), fire_count = fire_count + 1
        WHERE id = ? AND due_at IS ? AND body IS ? AND repeat_every_ms IS ?
-         AND ${pendingOneShotWhere()}`).run(timer.id, timer.due_at, timer.body, timer.repeat_every_ms).changes === 1);
+         AND ${pendingOneShotWhere()}
+         AND NOT EXISTS (SELECT 1 FROM agent_usage_limits u WHERE u.agent_id =
+           (SELECT a.id FROM agents a WHERE a.actor_id = wakes.deliver_actor
+             ORDER BY (a.status = 'running') DESC, a.id DESC LIMIT 1)
+           AND u.released_at IS NULL AND u.expires_at > datetime('now'))
+       RETURNING fire_count, fired_at`).get(timer.id, timer.due_at, timer.body, timer.repeat_every_ms);
+    return claimed ?? null;
 }
 const GONE = { idle: true, gone: true, since: null };
 const UNKNOWN = { idle: false, gone: false, since: null };
@@ -1712,6 +1815,8 @@ function idleCeilingStart(timer, snapshot) {
     return timer.held_at != null ? timer.first_held_at : null;
 }
 async function maybeFireIdle(timer, snapshot, now, choices) {
+    if (usageLimitHeld(timer))
+        return;
     const timedOut = timer.max_wait_at != null && timer.max_wait_at <= now;
     if (isStandingWatch(timer)) {
         noteStandingTransitions(timer, snapshot);
@@ -1721,9 +1826,12 @@ async function maybeFireIdle(timer, snapshot, now, choices) {
         if (timedOut) {
             await withPaneClaim(timer, async () => {
                 const decision = deliverable(timer, snapshot, choices);
-                if (!decision.ok || !claimOneShot(timer))
+                if (!decision.ok)
                     return false;
-                await deliver(timer, STANDING_EXPIRED_NOTE, choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
+                const claimed = claimOneShot(timer);
+                if (!claimed)
+                    return false;
+                await deliver({ ...timer, ...claimed }, STANDING_EXPIRED_NOTE, choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
                 return true;
             });
         }
@@ -1747,9 +1855,12 @@ async function maybeFireIdle(timer, snapshot, now, choices) {
     if (ready) {
         await withPaneClaim(timer, async () => {
             const decision = deliverable(timer, snapshot, choices);
-            if (!decision.ok || !claimOneShot(timer))
+            if (!decision.ok)
+                return false;
+            const claimed = claimOneShot(timer);
+            if (!claimed)
                 return false;
-            await deliver(timer, timedOut && !idleMet ? "max wait reached" : "", choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
+            await deliver({ ...timer, ...claimed }, timedOut && !idleMet ? "max wait reached" : "", choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
             return true;
         });
     }
@@ -1757,6 +1868,8 @@ async function maybeFireIdle(timer, snapshot, now, choices) {
 // A lead watch never enters the worker helpers below: it reads its own subscription row and the
 // watched lead's own turn record, and ends with a named reason rather than ever reading as idle.
 async function maybeFireLeadWatch(timer, sub, snapshot, now, choices) {
+    if (usageLimitHeld(timer))
+        return;
     const timedOut = timer.max_wait_at != null && timer.max_wait_at <= now;
     const readers = {
         awaitingChoice: (pane, command) => awaitingChoice(pane, command, choices),
@@ -1774,12 +1887,15 @@ async function maybeFireLeadWatch(timer, sub, snapshot, now, choices) {
         return;
     await withPaneClaim(timer, async () => {
         const decision = deliverable(timer, snapshot, choices);
-        if (!decision.ok || !claimOneShot(timer))
+        if (!decision.ok)
+            return false;
+        const claimed = claimOneShot(timer);
+        if (!claimed)
             return false;
         if (result.kind === "invalid") {
             bestEffortRun("UPDATE lead_idle_subscriptions SET terminal_reason = ? WHERE wake_id = ?", result.reason, timer.id);
         }
-        await deliver(timer, note, choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
+        await deliver({ ...timer, ...claimed }, note, choices, decision.typedSeen, decision.firstHeldAt, decision.owner);
         return true;
     });
 }
@@ -1966,11 +2082,11 @@ function socketStillRegistered(timer, owner) {
         row.pid === row.pane_pid);
 }
 // A failed stamp means the firing was cancelled or superseded, so nothing is posted.
-function stampRepeatingAttempt(timer, note, firstHeldAt) {
+function stampRepeatingAttempt(timer, note, typedSeen, firstHeldAt) {
     try {
         return (stmt(`UPDATE wakes SET socket_attempt_at = strftime('%Y-%m-%d %H:%M:%f', 'now'), socket_delivery_note = ?,
-           delivery_method = ?, delivered_by = ?, first_held_at = ?, held_at = NULL, held_reason = NULL, confirmed_at = NULL
-         WHERE id = ? AND fire_count = ? AND cancelled_at IS NULL AND typed_at IS NULL AND socket_attempt_at IS NULL`).run(note, DELIVERY_METHOD.socketRepeating, deliveredByForFiring(timer), firstHeldAt, timer.id, timer.fire_count).changes === 1);
+           delivery_method = ?, delivered_by = ?, first_held_at = ?, typed_seen = ?, held_at = NULL, held_reason = NULL, confirmed_at = NULL
+         WHERE id = ? AND fire_count = ? AND cancelled_at IS NULL AND typed_at IS NULL AND socket_attempt_at IS NULL`).run(note, DELIVERY_METHOD.socketRepeating, deliveredByForFiring(timer), firstHeldAt, typedSeen, timer.id, timer.fire_count).changes === 1);
     }
     catch {
         return false;
@@ -2013,25 +2129,31 @@ async function deliver(timer, note, choices, typedSeen, firstHeldAt, owner, fall
         const text = `${head}${note ? `, ${note}` : ""}] ${body + staleness + tail}\n\n${SOCKET_WAKE_FOOTER}`;
         if (!text.includes(CROSS_SESSION_CLOSE)) {
             if (repeating) {
-                if (!stampRepeatingAttempt(timer, note, firstHeldAt))
+                if (!stampRepeatingAttempt(timer, note, typedSeen, firstHeldAt))
                     return;
                 firing = timer.fire_count;
             }
             else {
                 bestEffortRun(`UPDATE wakes SET socket_attempt_at = strftime('%Y-%m-%d %H:%M:%f', 'now'), socket_delivery_note = ?,
-             delivery_method = ?, delivered_by = ?, first_held_at = ?, held_at = NULL, held_reason = NULL, confirmed_at = NULL
-           WHERE id = ?`, note, DELIVERY_METHOD.socket, deliveredByForFiring(timer), firstHeldAt, timer.id);
+             delivery_method = ?, delivered_by = ?, first_held_at = ?, typed_seen = ?, held_at = NULL, held_reason = NULL, confirmed_at = NULL
+           WHERE id = ?`, note, DELIVERY_METHOD.socket, deliveredByForFiring(timer), firstHeldAt, typedSeen, timer.id);
             }
             const posted = await postClaudeWake({
                 socketPath: timer.deliver_messaging_socket,
                 senderAddress: socketSenderAddress(dataDir),
                 text,
-                beforeWrite: () => socketStillRegistered(timer, owner),
+                beforeWrite: () => !usageLimitHeld(timer) && socketStillRegistered(timer, owner),
             });
             if (posted) {
                 forgetPaneAnswers(timer.deliver_pane, choices);
                 return;
             }
+            if (usageLimitHeld(timer)) {
+                const until = timer.deliver_row_id === null ? null : usageLimitGate(timer.deliver_row_id, Date.now()).until;
+                if (until)
+                    unclaimUsageLimit(timer, until);
+                return;
+            }
             if (firing !== null && !claimRepeatingFallback(timer))
                 return;
             note = withReDelivered(note);
@@ -2067,10 +2189,24 @@ async function deliver(timer, note, choices, typedSeen, firstHeldAt, owner, fall
                     requireStillOwned(owner.rowId, owner.identity, `wake #${timer.id} delivery`);
                 if (firing !== null)
                     requireCurrentFiring(timer.id, firing);
+                if (timer.deliver_row_id !== null) {
+                    observeUsageLimit(timer.deliver_row_id, Date.now());
+                    const gate = usageLimitGate(timer.deliver_row_id, Date.now());
+                    if (gate.held && gate.until)
+                        throw new UsageLimitRace(gate.until);
+                }
             }
             : undefined);
     }
     catch (err) {
+        if (err instanceof UsageLimitRace) {
+            if (!pasted)
+                unclaimUsageLimit(timer, err.until);
+            else
+                recordLostAfterClaim(timer, true, `${err.message} The body was pasted, and no Enter was sent.`, firing);
+            forgetPaneAnswers(timer.deliver_pane, choices);
+            return;
+        }
         if (fallback && !pasted && firing === null) {
             bestEffortRun("UPDATE wakes SET delivery_method = ?, delivered_by = ? WHERE id = ? AND typed_at IS NULL", DELIVERY_METHOD.socket, deliveredByForFiring(timer), timer.id);
         }
@@ -2125,6 +2261,8 @@ async function retryUnconfirmedSocketWakes(snapshot, choices) {
       WHERE ((${socketAwaitingWhere("wakes.")}) OR (${repeatingSocketAwaitingWhere("wakes.")}))
         AND wakes.socket_attempt_at >= datetime('now', ?)`).all(LOG_RETENTION);
     for (const timer of awaiting) {
+        if (usageLimitHeld(timer))
+            continue;
         if (!socketFallbackDue(timer))
             continue;
         await withPaneClaim(timer, () => fallbackClaimed(timer, snapshot, choices));

package/dist/spawn.js

@@ -1,10 +1,11 @@
 import { existsSync } from "node:fs";
-import { preservedRolloutPath, reapCodexHome } from "./codexHome.js";
+import { codexHomeDir, preservedRolloutPath, reapCodexHome } from "./codexHome.js";
 import { getProject } from "./context.js";
 import { LEAD_KIND, LEAD_NAME } from "./agentKinds.js";
 import { gitOut } from "./git.js";
 import { dataDir, db } from "./db.js";
 import { loadProjectYml } from "./projectYml.js";
+import { WORKER_RECOVERY_RETENTION_SECONDS } from "./workerRecovery.js";
 import { applyLayout, claimInitialWindow, createWindow, DEFAULT_LAYOUT, ensureSession, crossServerRefusal, findProcessesWindow, findProjectWindow, isPaneTarget, paneProcessExited, paneWindow, PROCESSES_LAYOUT, processesPaneTitle, processesWindowName, retainOnExitArgs, rowLive, sessionName, setPaneTitle, setRemainOnExit, makeProcessesWindow, tmux, tmuxSocketPath, untrustedTmuxServer, waitForPaneEstablished, windowLayout, windowOwner, windowTitle, rowOwnership, } from "./tmux.js";
 export function withWindowClaim(claim) {
     if (db.inTransaction) {
@@ -187,11 +188,11 @@ export function launchAgent(spec) {
     let info;
     try {
         info = db
-            .prepare("INSERT INTO agents (project_id, name, command, cwd, kind, parent_actor_id, tmux_socket, session_id, codex_home, model, extra_args, todo_id) " +
-            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+            .prepare("INSERT INTO agents (project_id, name, command, cwd, kind, parent_actor_id, tmux_socket, session_id, codex_home, model, extra_args, todo_id, forked_from_agent_id, forked_from_session_id, resumed_at) " +
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CASE WHEN ? IS NOT NULL THEN datetime('now') ELSE '' END)")
             .run(spec.projectId, spec.name, typeof spec.commandString === "string" ? spec.commandString : "", spec.cwd, spec.kind, spec.parentActor, socket, spec.sessionId ?? "", spec.codexHome ?? "", spec.model ?? null, spec.readOnly
             ? JSON.stringify({ args: spec.extraArgs ?? [], read_only: true })
-            : spec.extraArgs ? JSON.stringify(spec.extraArgs) : null, spec.todoId ?? null);
+            : spec.extraArgs ? JSON.stringify(spec.extraArgs) : null, spec.todoId ?? null, spec.forkedFromAgentId ?? null, spec.forkedFromSessionId ?? "", spec.forkedFromAgentId ?? null);
     }
     catch (e) {
         throw asNameClash(e, spec.name);
@@ -242,17 +243,21 @@ export const RESUME_FLIP_COLUMNS = [
     "parked_at",
     "parked_branch",
     "resumed_at",
+    "recovery_state",
     "tmux_socket",
     "command",
 ];
-export function resumeFlipSql() {
+export function resumeFlipSql(recoveryOnly = false) {
     return ("UPDATE agents SET status = 'running', closed_at = NULL, tmux_target = '', pane_pid = '', " +
-        `agent_state = 'unknown', state_changed_at = NULL, ${PARK_STAMP_CLEARED}, ` +
+        `agent_state = 'unknown', state_changed_at = NULL, ${PARK_STAMP_CLEARED}, recovery_state = '', ` +
         "resumed_at = datetime('now'), tmux_socket = ?, command = ? " +
-        "WHERE id = ? AND status = 'closed'");
+        `WHERE id = ? AND status = 'closed'${recoveryOnly ? " AND recovery_state = 'pending'" : ""}`);
 }
 function restoreFlippedRow(agentId, before) {
-    db.prepare(`UPDATE agents SET ${RESUME_FLIP_COLUMNS.map((c) => `${c} = ?`).join(", ")} ` +
+    const assignments = RESUME_FLIP_COLUMNS.map((column) => column === "recovery_state"
+        ? "recovery_state = CASE WHEN recovery_state = 'excluded' THEN 'excluded' ELSE ? END"
+        : `${column} = ?`).join(", ");
+    db.prepare(`UPDATE agents SET ${assignments} ` +
         "WHERE id = ? AND status = 'running' AND tmux_target = ''").run(...RESUME_FLIP_COLUMNS.map((c) => before[c] ?? null), agentId);
 }
 export function resumeAgent(spec) {
@@ -261,7 +266,7 @@ export function resumeAgent(spec) {
     const socket = tmuxSocketPath(process.env.TMUX, process.env.TMUX_TMPDIR);
     let flip;
     const captureBeforeFlip = db.prepare(`SELECT ${RESUME_FLIP_COLUMNS.join(", ")} FROM agents WHERE id = ?`);
-    const flipStatement = db.prepare(resumeFlipSql());
+    const flipStatement = db.prepare(resumeFlipSql(spec.recoveryOnly));
     try {
         flip = db
             .transaction(() => ({
@@ -352,7 +357,7 @@ export function branchAt(cwd) {
 }
 export function parkAgentRow(agentId, expected, branch) {
     const park = "UPDATE agents SET status = 'closed', closed_at = datetime('now'), parked_at = datetime('now'), " +
-        "parked_branch = ? WHERE id = ? AND status = 'running' AND tmux_target = ?";
+        "parked_branch = ?, recovery_state = 'excluded' WHERE id = ? AND status = 'running' AND tmux_target = ?";
     const row = db
         .prepare(`${park} AND tmux_socket = ? AND pane_pid = ? RETURNING parked_at`)
         .get(branch, agentId, expected.tmux_target, expected.tmux_socket, expected.pane_pid);
@@ -400,36 +405,27 @@ export function releaseParkRow(agentId) {
         .prepare(`UPDATE agents SET ${PARK_STAMP_CLEARED} WHERE id = ? AND status = 'closed' AND parked_at != ''`)
         .run(agentId).changes > 0);
 }
-// Row status alone is not the orphan test: a PARKED row is `status = 'closed'` too
-// (parkAgentRow above), and agent_park expects to resume it, cold-cache and all. `parked_at = ''`
-// is named explicitly rather than left to fall out of "closed" incidentally: codex can now be
-// parked too (harnessFor("codex").supportsResume, todo 563), exactly as this predicate already
-// assumed some non-claude harness eventually would - deriving the exclusion from harness capability
-// instead would have made this reap query's correctness depend on a table this file does not own.
-//
-// No conditional-UPDATE claim guards this, unlike closeAgentRow's "the conditional UPDATE is the
-// claim" - a deliberate difference, not a gap. That pattern earns its keep against an OBSERVABLE
-// side effect: a wake fired twice types into a pane twice. A reap has none - two instances racing
-// the same directory produce one deleted directory (rmSync's force:true swallows ENOENT even mid
-// recursive delete, not just on the top-level path), and the loser's `codex_home` update just
-// matches zero rows. Deleted twice is deleted once, so there is nothing here for a claim to arbitrate.
-//
-// rmSync runs before the clear, not after: if it throws (permissions, a mid-delete race), codex_home
-// stays set and this row is picked up again - by the next agent_close call, or by the janitor's own
-// backstop sweep - rather than the failure being silently swallowed by a claim that already fired.
-// The column IS the retry token; that is the guard here, it is just not shaped like a CAS.
-// Both callers wrap this, so a failure here costs a retry, never the close or the sweep it runs inside.
-// The target comes from the row's own transcript_path, not from reapCodexHome's returned list, so
-// a retry and a racing reap converge on the same file; the rewrite runs before the codex_home clear.
+// The selection can go stale while agent_resume claims the row, so revalidate and reap
+// under one SQLite writer claim before deleting the home.
 export function reapCodexHomeForClosedAgent(agentId, codexHome) {
-    const before = db.prepare("SELECT transcript_path FROM agents WHERE id = ?").get(agentId);
-    const oldTranscriptPath = before?.transcript_path ?? "";
-    const dest = oldTranscriptPath ? preservedRolloutPath(codexHome, oldTranscriptPath) : undefined;
-    reapCodexHome(codexHome);
-    if (dest && existsSync(dest)) {
-        db.prepare("UPDATE agents SET transcript_path = ? WHERE id = ? AND transcript_path = ?").run(dest, agentId, oldTranscriptPath);
-    }
-    db.prepare("UPDATE agents SET codex_home = '' WHERE id = ? AND codex_home = ?").run(agentId, codexHome);
+    return db.transaction(() => {
+        const row = db
+            .prepare(`SELECT transcript_path FROM agents
+         WHERE id = ? AND codex_home = ? AND status = 'closed' AND parked_at = ''
+           AND (recovery_state != 'pending' OR closed_at <= datetime('now', ?))`)
+            .get(agentId, codexHome, `-${WORKER_RECOVERY_RETENTION_SECONDS} seconds`);
+        if (!row)
+            return null;
+        const oldTranscriptPath = row.transcript_path ?? "";
+        const dest = oldTranscriptPath ? preservedRolloutPath(codexHome, oldTranscriptPath) : undefined;
+        const homeExisted = existsSync(codexHomeDir(codexHome));
+        reapCodexHome(codexHome);
+        if (dest && existsSync(dest)) {
+            db.prepare("UPDATE agents SET transcript_path = ? WHERE id = ? AND transcript_path = ?").run(dest, agentId, oldTranscriptPath);
+        }
+        db.prepare("UPDATE agents SET codex_home = '' WHERE id = ? AND codex_home = ?").run(agentId, codexHome);
+        return homeExisted;
+    }).immediate();
 }
 export function closeAgentRow(agentId, expected) {
     const info = expected === undefined

package/dist/todoCompletionGate.js

@@ -0,0 +1,124 @@
+import { spawn } from "node:child_process";
+import { isTrusted } from "./leadLaunch.js";
+import { loadProjectYml } from "./projectYml.js";
+const GATE_NAME = "hive:todo-completion-gate";
+const TIMEOUT_MS = 30_000;
+const OUTPUT_TAIL_BYTES = 4096;
+const GATE_APPROVAL_MESSAGE = "completion gate in hive.yml is not approved yet; run `hive` in a terminal in this project to review it.";
+export function completionGateTrust(argv) {
+    return {
+        name: GATE_NAME,
+        command: JSON.stringify(argv),
+        dir: null,
+        env: {},
+    };
+}
+export function todoCompletionGateTrusted(projectId, argv) {
+    const trust = completionGateTrust(argv);
+    return isTrusted(projectId, trust.name, trust.command, trust.dir, trust.env);
+}
+export function assertValidTodoCompletionGate(input, message) {
+    if (message)
+        throw failure("invalid", input, null, null, Buffer.alloc(0), Buffer.alloc(0), message);
+}
+export function assertTodoCompletionGateCurrent(input, originalArgv) {
+    const current = loadProjectYml(input.projectPath);
+    if (current.completionGateError || !current.config?.completion_gate) {
+        throw failure("changed", input, null, null, Buffer.alloc(0), Buffer.alloc(0));
+    }
+    if (JSON.stringify(current.config.completion_gate) !== JSON.stringify(originalArgv)) {
+        throw failure("changed", input, null, null, Buffer.alloc(0), Buffer.alloc(0), GATE_APPROVAL_MESSAGE);
+    }
+    if (!todoCompletionGateTrusted(input.projectId, originalArgv)) {
+        throw failure("untrusted", input, null, null, Buffer.alloc(0), Buffer.alloc(0));
+    }
+}
+function failure(reason, input, code, signal, stdout, stderr, detail) {
+    const output = [
+        stdout.length ? `stdout tail:\n${stdout.toString("utf8")}` : "",
+        stderr.length ? `stderr tail:\n${stderr.toString("utf8")}` : "",
+    ].filter(Boolean).join("\n");
+    return new Error(`[todo_completion_gate:${reason}] todo ${input.todoId}; exit ${code ?? "unavailable"}` +
+        `${signal ? `; signal ${signal}` : ""}${output ? `\n${output}` : ""}` +
+        (detail ? `\n${detail}` : "") +
+        (reason === "untrusted"
+            ? `\n${GATE_APPROVAL_MESSAGE}`
+            : ""));
+}
+export async function runTodoCompletionGate(input) {
+    if (!todoCompletionGateTrusted(input.projectId, input.argv)) {
+        throw failure("untrusted", input, null, null, Buffer.alloc(0), Buffer.alloc(0));
+    }
+    const [file, ...args] = input.argv;
+    await new Promise((resolve, reject) => {
+        const stdout = [];
+        const stderr = [];
+        let stdoutBytes = 0;
+        let stderrBytes = 0;
+        let settled = false;
+        let timedOut = false;
+        const child = spawn(file, args, {
+            cwd: input.projectPath,
+            env: {
+                ...process.env,
+                HIVE_TODO_ID: String(input.todoId),
+                HIVE_TODO_TITLE: input.title,
+                HIVE_PROJECT_ID: String(input.projectId),
+                HIVE_ACTOR_ID: input.actorId,
+            },
+            shell: false,
+            detached: process.platform !== "win32",
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        const appendTail = (chunks, chunk, current) => {
+            chunks.push(chunk);
+            let total = current + chunk.length;
+            while (total > OUTPUT_TAIL_BYTES && chunks.length) {
+                const excess = total - OUTPUT_TAIL_BYTES;
+                if (chunks[0].length <= excess) {
+                    total -= chunks.shift().length;
+                }
+                else {
+                    chunks[0] = chunks[0].subarray(excess);
+                    total -= excess;
+                }
+            }
+            return total;
+        };
+        child.stdout.on("data", (chunk) => { stdoutBytes = appendTail(stdout, chunk, stdoutBytes); });
+        child.stderr.on("data", (chunk) => { stderrBytes = appendTail(stderr, chunk, stderrBytes); });
+        const finish = (error) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            if (error)
+                reject(error);
+            else
+                resolve();
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            if (child.pid && process.platform !== "win32") {
+                try {
+                    process.kill(-child.pid, "SIGKILL");
+                }
+                catch { }
+            }
+            else
+                child.kill("SIGKILL");
+            child.stdout.destroy();
+            child.stderr.destroy();
+            finish(failure("timeout", input, null, null, Buffer.concat(stdout), Buffer.concat(stderr), `completion gate timed out after ${TIMEOUT_MS}ms`));
+        }, TIMEOUT_MS);
+        child.once("error", () => finish(failure("spawn", input, null, null, Buffer.concat(stdout), Buffer.concat(stderr))));
+        child.once("close", (code, signal) => {
+            if (timedOut)
+                return;
+            if (code === 0)
+                finish();
+            else
+                finish(failure(signal ? "signal" : "failed", input, code, signal, Buffer.concat(stdout), Buffer.concat(stderr)));
+        });
+    });
+}