@pinet/broker-core 0.2.4 → 0.2.7

package/dist/hibernation-orchestrator.js

@@ -0,0 +1,1096 @@
+import * as crypto from "node:crypto";
+import { evaluateHibernateEligibility } from "./lifecycle.js";
+import { sanitizeCheckpointReasonCode, sanitizeOperatorReason } from "./hibernation-status.js";
+export const DEFAULT_ORCHESTRATOR_CONFIG = {
+    handshakeTimeoutMs: 30_000,
+    wakeLeaseMs: 90_000,
+    hibernateLeaseMs: 120_000,
+    registrationTimeoutMs: 60_000,
+    maxWakeAttempts: 3,
+    maxConcurrentWakes: 2,
+    maxConcurrentWakesPerRepo: 1,
+};
+const HIBERNATABLE_PRIORITY = {
+    direct_a2a: 10,
+    slack_thread: 20,
+    scheduled: 30,
+    lane_assignment: 40,
+    manual: 15,
+};
+export function wakeTriggerPriority(kind) {
+    return HIBERNATABLE_PRIORITY[kind];
+}
+// agent-standards-ignore prefer-inline-single-use-helper: bounded fail-closed
+// timeout is a real async control-flow seam; inlining the race + timer cleanup
+// into hibernate() would obscure the checkpoint-safety path.
+async function withTimeout(work, timeoutMs, onTimeout) {
+    let timer = null;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(onTimeout()), timeoutMs);
+        timer.unref?.();
+    });
+    try {
+        return await Promise.race([work, timeout]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/**
+ * Broker-managed hibernation lifecycle orchestrator.
+ *
+ * Composes the durable primitives in {@link BrokerDB} (fenced leases, CAS
+ * lifecycle transitions, runtime specs, checkpoint receipts, generation
+ * reservations, wake queue, telemetry events) with injected process/tmux
+ * adapters. Every path fails closed: on any anomaly it releases its lease and
+ * quarantines the agent as `reap-candidate` with an actionable reason rather
+ * than guessing, rerouting affinity work, or killing a PID on PID alone.
+ */
+export class HibernationOrchestrator {
+    db;
+    process;
+    tmux;
+    brokerInstanceId;
+    config;
+    now;
+    newId;
+    awaitRuntimeRegistration;
+    constructor(deps) {
+        this.db = deps.db;
+        this.process = deps.process;
+        this.tmux = deps.tmux;
+        this.brokerInstanceId = deps.brokerInstanceId;
+        this.config = { ...DEFAULT_ORCHESTRATOR_CONFIG, ...(deps.config ?? {}) };
+        this.now = deps.now ?? (() => Date.now());
+        this.newId = deps.newId ?? (() => crypto.randomUUID());
+        this.awaitRuntimeRegistration =
+            deps.awaitRuntimeRegistration ?? ((ctx) => this.defaultAwaitRegistration(ctx));
+    }
+    // ─── Prepare (advance an eligible free agent toward `idle`) ─────
+    /**
+     * Advance an eligible, free, broker-managed agent through `grace` to `idle`
+     * so it becomes hibernation-ready. Idempotent and fenced by CAS. Used by both
+     * the manual `pinet hibernate` path and the auto scheduler. Never forces a
+     * working/unsafe/ineligible agent forward.
+     */
+    prepareHibernation(agentId, opts = {}) {
+        const reason = sanitizeOperatorReason(opts.reason) ?? "prepare_hibernation";
+        const actor = opts.actor ?? "broker";
+        const correlationId = opts.correlationId ?? this.newId();
+        let agent = this.db.getAgentById(agentId);
+        if (!agent)
+            return { ready: false, state: "live", reason: "unknown_agent" };
+        const runtimeSpec = this.db.getAgentRuntimeSpec(agentId);
+        if (runtimeSpec?.runtimeKind === "herdr") {
+            const unsupported = "hibernation unsupported on this runtime";
+            this.recordRefusal(agentId, "prepare_refused", unsupported, actor, correlationId);
+            return { ready: false, state: agent.lifecycleState ?? "live", reason: unsupported };
+        }
+        const eligibility = evaluateHibernateEligibility(agent);
+        if (!eligibility.eligible) {
+            this.recordRefusal(agentId, "prepare_refused", eligibility.reason, actor, correlationId);
+            return { ready: false, state: agent.lifecycleState ?? "live", reason: eligibility.reason };
+        }
+        // Legal path toward idle: live/active -> grace -> idle. Anything already at
+        // idle is ready; hibernating/hibernated/waking/terminated is not preparable.
+        const steps = [];
+        switch (agent.lifecycleState) {
+            case "idle":
+                return { ready: true, state: "idle", reason: "already_idle" };
+            case "live":
+            case "active":
+                steps.push("grace", "idle");
+                break;
+            case "grace":
+                steps.push("idle");
+                break;
+            default:
+                return {
+                    ready: false,
+                    state: agent.lifecycleState ?? "live",
+                    reason: `not_preparable:${agent.lifecycleState}`,
+                };
+        }
+        for (const toState of steps) {
+            agent = this.db.transitionAgentLifecycle({
+                agentId,
+                expectedVersion: agent.lifecycleVersion ?? 0,
+                toState,
+                reason,
+                actor,
+                correlationId,
+            });
+        }
+        return { ready: true, state: agent.lifecycleState ?? "idle", reason: "ready" };
+    }
+    // ─── Hibernate ──────────────────────────────────────────────────
+    async hibernate(agentId, opts = {}) {
+        const correlationId = opts.correlationId ?? this.newId();
+        const actor = opts.actor ?? "broker";
+        // Operator-authored reason: bound + path-redact at the orchestrator boundary
+        // so it is safe regardless of caller, before it can be persisted into any
+        // lifecycle row / event.
+        const reason = sanitizeOperatorReason(opts.reason) ?? "manual";
+        const startedAt = this.now();
+        const agent = this.db.getAgentById(agentId);
+        if (!agent)
+            return this.refuseHibernate(agentId, correlationId, "live", "unknown_agent", actor);
+        const runtimeSpec = this.db.getAgentRuntimeSpec(agentId);
+        if (runtimeSpec?.runtimeKind === "herdr") {
+            return this.refuseHibernate(agentId, correlationId, agent.lifecycleState ?? "live", "hibernation unsupported on this runtime", actor);
+        }
+        const eligibility = evaluateHibernateEligibility(agent);
+        if (!eligibility.eligible) {
+            return this.refuseHibernate(agentId, correlationId, agent.lifecycleState ?? "live", eligibility.reason, actor);
+        }
+        if (agent.lifecycleState !== "idle") {
+            return this.refuseHibernate(agentId, correlationId, agent.lifecycleState ?? "live", `not_idle:${agent.lifecycleState}`, actor);
+        }
+        const spec = runtimeSpec;
+        if (!spec) {
+            return this.refuseHibernate(agentId, correlationId, "idle", "missing_runtime_spec", actor);
+        }
+        const lease = this.db.acquireAgentLifecycleLease({
+            agentId,
+            operation: "hibernate",
+            ownerBrokerInstanceId: this.brokerInstanceId,
+            leaseId: this.newId(),
+            ttlMs: this.config.hibernateLeaseMs,
+            now: this.now(),
+        });
+        if (!lease) {
+            return this.refuseHibernate(agentId, correlationId, "idle", "lease_contended", actor);
+        }
+        // Fail-closed fault tracking: an unexpected adapter/DB rejection must never
+        // leave the agent stranded in `hibernating`. `enteredHibernating` records
+        // that we own the CAS transition; `teardownStarted` records that the
+        // process stop was attempted (so the runtime's liveness is now unknown).
+        let versionCursor = agent.lifecycleVersion ?? 0;
+        let enteredHibernating = false;
+        let teardownStarted = false;
+        try {
+            // idle -> hibernating (fenced CAS)
+            const hibernating = this.transitionFenced(lease, {
+                agentId,
+                expectedVersion: versionCursor,
+                toState: "hibernating",
+                reason,
+                actor,
+                correlationId,
+                triggerSource: opts.trigger,
+            });
+            versionCursor = hibernating.lifecycleVersion ?? versionCursor + 1;
+            enteredHibernating = true;
+            // Cooperative checkpoint handshake (fail closed on timeout).
+            const checkpoint = await withTimeout(this.process.requestCheckpoint(spec), this.config.handshakeTimeoutMs, () => ({
+                hibernateSafe: false,
+                reason: "checkpoint_timeout",
+                sessionResumeRef: null,
+                pendingInboxCount: 0,
+                rssBytes: null,
+            }));
+            this.db.recordAgentCheckpointReceipt({
+                agentId,
+                runtimeGeneration: agent.runtimeGeneration ?? 0,
+                correlationId,
+                hibernateSafe: checkpoint.hibernateSafe,
+                // Persist only an allowlisted machine code, never the runtime's raw
+                // reason prose — the receipt is durable and read on recovery.
+                reason: sanitizeCheckpointReasonCode(checkpoint.reason),
+                sessionResumeRef: checkpoint.sessionResumeRef,
+                pendingInboxCount: checkpoint.pendingInboxCount,
+                rssBytes: checkpoint.rssBytes,
+            });
+            // New work or an unsafe runtime must abort hibernation back to a safe
+            // live/active state — never exit the process.
+            const freshInbox = this.db.getUnreadInboxCount(agentId);
+            if (!checkpoint.hibernateSafe || freshInbox > 0 || checkpoint.pendingInboxCount > 0) {
+                // `checkpoint.reason` is runtime-authored; collapse it to an allowlisted
+                // machine code so no argv/env/path prose leaks through the abort surface.
+                const abortReason = !checkpoint.hibernateSafe
+                    ? `checkpoint_unsafe:${sanitizeCheckpointReasonCode(checkpoint.reason)}`
+                    : "work_arrived_during_checkpoint";
+                // Rollback-to-active before teardown is a fail-closed SAFETY transition:
+                // use an unfenced administrative CAS so a hibernate lease that expired
+                // during a slow checkpoint handshake cannot leave the row stranded in
+                // `hibernating`. The version CAS still blocks clobbering a concurrent
+                // legitimate writer.
+                const active = this.transitionAdministrative({
+                    agentId,
+                    expectedVersion: versionCursor,
+                    toState: "active",
+                    reason: abortReason,
+                    actor,
+                    correlationId,
+                });
+                return {
+                    ok: false,
+                    agentId,
+                    correlationId,
+                    state: active.lifecycleState ?? "active",
+                    reason: abortReason,
+                };
+            }
+            // Graceful teardown. Past this point the runtime has been asked to stop,
+            // so any subsequent fault leaves its liveness unknown → quarantine.
+            teardownStarted = true;
+            const stop = await this.process.stopRuntime(spec);
+            const stillAlive = await this.process.isRuntimeAlive(spec);
+            if (!stop.stopped || stillAlive) {
+                return this.quarantine(agentId, versionCursor, correlationId, actor, "runtime_survived_stop", lease);
+            }
+            const attachable = await this.tmux.isSessionAttachable(spec);
+            if (!attachable) {
+                return this.quarantine(agentId, versionCursor, correlationId, actor, "tmux_session_missing", lease);
+            }
+            // hibernating -> hibernated.
+            const durationMs = this.now() - startedAt;
+            const hibernated = this.transitionFenced(lease, {
+                agentId,
+                expectedVersion: versionCursor,
+                toState: "hibernated",
+                reason,
+                actor,
+                correlationId,
+                durationMs,
+                rssBytesBefore: checkpoint.rssBytes,
+                rssBytesAfter: stop.rssBytes,
+            });
+            return {
+                ok: true,
+                agentId,
+                correlationId,
+                state: hibernated.lifecycleState ?? "hibernated",
+                reason: "hibernated",
+                rssBytesBefore: checkpoint.rssBytes,
+                rssBytesAfter: stop.rssBytes,
+                durationMs,
+            };
+        }
+        catch {
+            // Redaction-by-construction: never surface the raw error (it can carry
+            // paths). Use a static fault code and fail closed based on how far we got.
+            const faultReason = "hibernate_fault";
+            if (!enteredHibernating) {
+                // State was never changed by us; surface a refusal without forcing a
+                // transition that might not be valid from the current state.
+                this.recordRefusal(agentId, "hibernate_refused", faultReason, actor, correlationId);
+                return {
+                    ok: false,
+                    agentId,
+                    correlationId,
+                    state: this.db.getAgentById(agentId)?.lifecycleState ?? "idle",
+                    reason: faultReason,
+                };
+            }
+            try {
+                if (!teardownStarted) {
+                    // The runtime was never asked to stop, so it is still alive: abort
+                    // back to active rather than quarantine. Unfenced administrative CAS
+                    // so an expired hibernate lease cannot strand the row in `hibernating`.
+                    const active = this.transitionAdministrative({
+                        agentId,
+                        expectedVersion: versionCursor,
+                        toState: "active",
+                        reason: faultReason,
+                        actor,
+                        correlationId,
+                    });
+                    return {
+                        ok: false,
+                        agentId,
+                        correlationId,
+                        state: active.lifecycleState ?? "active",
+                        reason: faultReason,
+                    };
+                }
+                // Teardown began; runtime liveness is unknown → quarantine for review.
+                return this.quarantine(agentId, versionCursor, correlationId, actor, faultReason, lease);
+            }
+            catch {
+                // Even the recovery transition failed (e.g. version raced). Report a
+                // safe failure; the lease still releases in `finally`.
+                return {
+                    ok: false,
+                    agentId,
+                    correlationId,
+                    state: this.db.getAgentById(agentId)?.lifecycleState ?? "hibernating",
+                    reason: faultReason,
+                };
+            }
+        }
+        finally {
+            this.db.releaseAgentLifecycleLease(agentId, lease.leaseId, lease.fenceToken);
+        }
+    }
+    // ─── Wake ───────────────────────────────────────────────────────
+    /**
+     * Cold-wake a hibernated agent as a single accepted runtime generation, then
+     * transition to `live` so the durable inbox drains in order. Concurrent
+     * triggers contend on the fenced wake lease; only one wins.
+     */
+    async wake(agentId, opts = {}) {
+        const correlationId = opts.correlationId ?? this.newId();
+        const actor = opts.actor ?? "broker";
+        // Bound + path-redact the operator-authored reason before it can be
+        // persisted into any lifecycle row / event.
+        const reason = sanitizeOperatorReason(opts.reason) ?? "manual";
+        const startedAt = this.now();
+        // Cross-cutting fault state, read by the outer catch:
+        //  - `acceptedGeneration`: the socket layer atomically accepted our generation,
+        //    so the runtime is live+connected. Acceptance is IRREVERSIBLE — a later
+        //    fault must NEVER quarantine it; leave `waking` for `recoverStrandedWakes`.
+        //  - `inFlightHandle`/`inFlightLaunched`: a launched-but-unaccepted attempt that
+        //    may still be running. Any escape must prove-stop it (via its attempt-bound
+        //    handle) or fail closed as `wake_ambiguous_launch` rather than leak it.
+        //  - `inFlightReservedGeneration`/`inFlightReservationNonce`: the in-flight
+        //    attempt's reservation identity, so the outer catch can settle it against
+        //    the acceptance boundary (`finalizeWakeAttempt`) before prove-stopping.
+        let acceptedGeneration = false;
+        let inFlightLaunched = false;
+        let inFlightHandle = null;
+        let inFlightReservedGeneration = null;
+        let inFlightReservationNonce = null;
+        const agent = this.db.getAgentById(agentId);
+        if (!agent)
+            return this.refuseWake(agentId, correlationId, "live", "unknown_agent", actor);
+        if (agent.lifecycleState !== "hibernated") {
+            return this.refuseWake(agentId, correlationId, agent.lifecycleState ?? "live", `not_hibernated:${agent.lifecycleState}`, actor);
+        }
+        const spec = this.db.getAgentRuntimeSpec(agentId);
+        if (!spec) {
+            return this.refuseWake(agentId, correlationId, "hibernated", "missing_runtime_spec", actor);
+        }
+        if (spec.runtimeKind !== "tmux") {
+            return this.refuseWake(agentId, correlationId, "hibernated", "hibernation unsupported on this runtime", actor);
+        }
+        const lease = this.db.acquireAgentLifecycleLease({
+            agentId,
+            operation: "wake",
+            ownerBrokerInstanceId: this.brokerInstanceId,
+            leaseId: this.newId(),
+            ttlMs: this.config.wakeLeaseMs,
+            now: this.now(),
+        });
+        if (!lease) {
+            // The fenced lifecycle lease is held by someone else. Only a *live wake*
+            // lease actually drains the inbox, so we must distinguish:
+            //   - a matching unexpired wake lease → a real in-flight wake is the single
+            //     winner and this trigger is satisfied by it (benign no-op);
+            //   - any other held lease (e.g. a lingering/expired hibernate lease around
+            //     a crash) → no wake is in flight, so this trigger must NOT be dropped.
+            //     Report distinct, retryable contention so the dispatcher requeues it.
+            const held = this.db.getAgentLifecycleLease(agentId);
+            const wakeInFlight = held !== null && held.operation === "wake" && Date.parse(held.expiresAt) > this.now();
+            const contentionReason = wakeInFlight ? "wake_in_progress" : "wake_lease_contended";
+            return this.refuseWake(agentId, correlationId, "hibernated", contentionReason, actor);
+        }
+        let versionCursor = agent.lifecycleVersion ?? 0;
+        try {
+            // hibernated -> waking (fenced CAS).
+            const waking = this.transitionFenced(lease, {
+                agentId,
+                expectedVersion: versionCursor,
+                toState: "waking",
+                reason,
+                actor,
+                correlationId,
+                triggerSource: opts.trigger,
+            });
+            versionCursor = waking.lifecycleVersion ?? versionCursor + 1;
+            const maxAttempts = Math.max(1, this.config.maxWakeAttempts);
+            for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+                // Keep the lease valid across this attempt's launch + registration waits
+                // so a legitimately long wake (whose cumulative waits can outrun a single
+                // lease TTL) still completes its fenced `live` transition instead of
+                // being quarantined on expiry. Renewal preserves the fence, so revival
+                // fencing is unchanged. A null result means ownership was lost to another
+                // broker → fail closed (quarantine) rather than double-drive the wake.
+                const renewed = this.db.renewAgentLifecycleLease({
+                    agentId,
+                    leaseId: lease.leaseId,
+                    fenceToken: lease.fenceToken,
+                    ttlMs: this.config.wakeLeaseMs,
+                    now: this.now(),
+                });
+                if (!renewed) {
+                    return this.quarantineWake(agentId, versionCursor, correlationId, actor, "wake_lease_lost", lease, attempt - 1);
+                }
+                const reservation = this.db.reserveWakeGeneration({
+                    agentId,
+                    wakeLeaseId: lease.leaseId,
+                    fenceToken: lease.fenceToken,
+                    correlationId,
+                    now: this.now(),
+                });
+                const launchCtx = {
+                    agentId,
+                    stableId: spec.stableId,
+                    wakeLeaseId: lease.leaseId,
+                    fenceToken: lease.fenceToken,
+                    reservedGeneration: reservation.reservedGeneration,
+                    reservationNonce: reservation.reservationNonce,
+                    correlationId,
+                    spec,
+                };
+                inFlightReservedGeneration = reservation.reservedGeneration;
+                inFlightReservationNonce = reservation.reservationNonce;
+                // Launch the replacement runtime. A throw MID-LAUNCH leaves an unknown
+                // process with no handle to address, so its liveness is unprovable. But
+                // the socket layer accepts a generation ATOMICALLY, so even a launch that
+                // threw may have registered+accepted in the race window — so we still
+                // settle TRANSACTIONALLY against the acceptance boundary before failing
+                // closed. `accepted` ⇒ the runtime is live, promote it. `unknown` ⇒ the
+                // settle did not commit, so an acceptance may still race — leave `waking`
+                // for recovery rather than quarantine. Only `fenced-unaccepted` (a
+                // committed consumption that makes acceptance impossible) quarantines
+                // (`wake_ambiguous_launch`, since the throwing launch left no handle).
+                inFlightLaunched = true;
+                inFlightHandle = null;
+                let launch;
+                try {
+                    launch = await this.tmux.respawnRuntime(launchCtx);
+                }
+                catch {
+                    const settled = this.settleWakeAttempt(agentId, reservation);
+                    if (settled === "accepted") {
+                        acceptedGeneration = true;
+                        inFlightLaunched = false;
+                        inFlightHandle = null;
+                        return this.promoteAcceptedWake({
+                            agentId,
+                            versionCursor,
+                            reason,
+                            actor,
+                            correlationId,
+                            reservedGeneration: reservation.reservedGeneration,
+                            attempt,
+                            startedAt,
+                        });
+                    }
+                    if (settled === "unknown") {
+                        return {
+                            ok: true,
+                            agentId,
+                            correlationId,
+                            state: this.db.getAgentById(agentId)?.lifecycleState ?? "waking",
+                            reason: "woken_recovery_pending",
+                        };
+                    }
+                    return this.quarantineWake(agentId, versionCursor, correlationId, actor, "wake_ambiguous_launch", lease, attempt);
+                }
+                inFlightLaunched = launch.launched;
+                inFlightHandle = launch.handle;
+                // Wait for the woken runtime to re-register and present its fence. The
+                // socket layer accepts the generation ATOMICALLY on a valid registration,
+                // so acceptance can happen even if the wait throws afterward — we swallow
+                // it and let the race-free settle below decide.
+                try {
+                    if (launch.launched)
+                        await this.awaitRuntimeRegistration(launchCtx);
+                }
+                catch {
+                    // Ignore: `settleWakeAttempt` is the authoritative, race-free classifier.
+                }
+                // AUTHORITATIVE, RACE-FREE acceptance classification. The socket accepts a
+                // generation atomically, so acceptance can land between our last waiter
+                // read and now. `finalizeWakeAttempt` runs in one transaction: if our
+                // generation was accepted it returns accepted; otherwise it consumes THIS
+                // attempt's exact-nonce reservation so the launched runtime can never be
+                // accepted afterwards, making the subsequent prove-stop safe.
+                const settled = this.settleWakeAttempt(agentId, reservation);
+                if (settled === "accepted") {
+                    // The socket layer already bound this runtime to our exact
+                    // lease/fence/reservation and atomically advanced+consumed the
+                    // generation, so from here acceptance is IRREVERSIBLE: the runtime is
+                    // live and connected and must NEVER be quarantined.
+                    acceptedGeneration = true;
+                    inFlightLaunched = false; // the attempt IS the live runtime; never stop it
+                    inFlightHandle = null;
+                    return this.promoteAcceptedWake({
+                        agentId,
+                        versionCursor,
+                        reason,
+                        actor,
+                        correlationId,
+                        reservedGeneration: reservation.reservedGeneration,
+                        attempt,
+                        startedAt,
+                    });
+                }
+                if (settled === "unknown") {
+                    // The atomic settle did NOT commit, so THIS attempt's reservation was
+                    // not provably consumed and an acceptance can still race any stop. We
+                    // must NOT prove-stop or retry the attempt (doing so could kill a
+                    // runtime that accepts a moment later). Leave the identity `waking`
+                    // (never quarantine) and hand it to `recoverStrandedWakes`, which
+                    // reconciles it once the transactional settle can commit.
+                    return {
+                        ok: true,
+                        agentId,
+                        correlationId,
+                        state: this.db.getAgentById(agentId)?.lifecycleState ?? "waking",
+                        reason: "woken_recovery_pending",
+                    };
+                }
+                // Failed attempt (`fenced-unaccepted`). The reservation is now
+                // transactionally consumed (settle above), so the
+                // launched runtime can never be accepted. But a launched-but-unaccepted
+                // process may still be RUNNING, and we must never leave one behind
+                // (whether we are about to relaunch on top of it OR about to quarantine
+                // and hand the durable row to a spec-addressed reaper that cannot see this
+                // PID), so we best-effort stop it and only proceed if we can PROVE it is
+                // gone; otherwise fail closed (`wake_ambiguous_launch`). Running on the
+                // final attempt too closes the same leak symmetrically.
+                //
+                // The stop/liveness proof is addressed by the attempt-bound handle from
+                // THIS launch (`launch.handle`), not the durable spec. Using the spec
+                // would target the pre-hibernation runtime's recorded PID generation —
+                // which is already dead — and so would always "confirm stopped" while the
+                // newly launched attempt kept running. A launch that produced no handle,
+                // or a stop/liveness probe that throws, is unprovable and must fail closed
+                // (`proveAttemptStopped` swallows throws and returns false).
+                if (launch.launched && !(await this.proveAttemptStopped(launch.handle))) {
+                    return this.quarantineWake(agentId, versionCursor, correlationId, actor, "wake_ambiguous_launch", lease, attempt);
+                }
+                inFlightLaunched = false; // this attempt is confirmed gone (or nothing launched)
+                inFlightHandle = null;
+                inFlightReservedGeneration = null;
+                inFlightReservationNonce = null;
+                this.db.recordAgentLifecycleEvent({
+                    agentId,
+                    fromState: "waking",
+                    toState: "waking",
+                    lifecycleVersion: versionCursor,
+                    reason,
+                    actor,
+                    correlationId,
+                    outcome: attempt < maxAttempts ? "wake_retry" : "wake_exhausted",
+                    errorCode: "launch_or_registration_not_accepted",
+                    fenceToken: lease.fenceToken,
+                });
+            }
+            // All attempts exhausted → quarantine.
+            return this.quarantineWake(agentId, versionCursor, correlationId, actor, "wake_attempts_exhausted", lease, maxAttempts);
+        }
+        catch {
+            // Fail-closed on an unexpected adapter/DB fault that escaped the per-attempt
+            // handling. Two invariants override the generic quarantine:
+            //
+            //  1. If our generation was already ACCEPTED, the runtime is live and
+            //     socket-bound. Acceptance is irreversible: never quarantine it — leave
+            //     `waking` for `recoverStrandedWakes` to promote to `live`.
+            if (acceptedGeneration) {
+                return {
+                    ok: true,
+                    agentId,
+                    correlationId,
+                    state: this.db.getAgentById(agentId)?.lifecycleState ?? "waking",
+                    reason: "woken_recovery_pending",
+                };
+            }
+            //  2. A launched-but-unaccepted attempt may still be running AND may have
+            //     raced to accept before this fault. Settle it against the acceptance
+            //     boundary TRANSACTIONALLY first: only `fenced-unaccepted` (a committed
+            //     reservation consumption) makes a prove-stop safe. `accepted` means it
+            //     is live — never quarantine it (leave `waking` for recovery). `unknown`
+            //     means the settle did not commit, so the reservation was not provably
+            //     consumed and an acceptance can still race a stop — we must NOT
+            //     prove-stop/quarantine either; leave `waking` for reconciliation.
+            //     Static reasons only; never surface raw errors.
+            if (inFlightLaunched &&
+                inFlightReservedGeneration !== null &&
+                inFlightReservationNonce !== null) {
+                const settled = this.settleWakeAttempt(agentId, {
+                    reservedGeneration: inFlightReservedGeneration,
+                    reservationNonce: inFlightReservationNonce,
+                });
+                if (settled === "accepted" || settled === "unknown") {
+                    return {
+                        ok: true,
+                        agentId,
+                        correlationId,
+                        state: this.db.getAgentById(agentId)?.lifecycleState ?? "waking",
+                        reason: "woken_recovery_pending",
+                    };
+                }
+                // settled === "fenced-unaccepted": reservation transactionally consumed →
+                // the launched runtime can no longer be accepted, so the prove-stop below
+                // is safe.
+            }
+            const faultReason = inFlightLaunched && !(await this.proveAttemptStopped(inFlightHandle))
+                ? "wake_ambiguous_launch"
+                : "wake_fault";
+            try {
+                return this.quarantineWake(agentId, versionCursor, correlationId, actor, faultReason, lease, 0);
+            }
+            catch {
+                return {
+                    ok: false,
+                    agentId,
+                    correlationId,
+                    state: this.db.getAgentById(agentId)?.lifecycleState ?? "waking",
+                    reason: faultReason,
+                };
+            }
+        }
+        finally {
+            this.db.releaseAgentLifecycleLease(agentId, lease.leaseId, lease.fenceToken);
+        }
+    }
+    /**
+     * Best-effort proof that a launched wake ATTEMPT's runtime is gone, addressed
+     * by its attempt-bound handle (never the durable spec). Fail-closed: a missing
+     * handle, an unconfirmed stop, a still-alive probe, OR any adapter throw all
+     * count as "not proven gone", so the caller quarantines (`wake_ambiguous_launch`)
+     * rather than relaunch on / strand a possibly-live runtime.
+     */
+    async proveAttemptStopped(handle) {
+        if (!handle)
+            return false;
+        try {
+            const stop = await this.process.stopLaunchedAttempt(handle);
+            if (!stop.stopped)
+                return false;
+            return !(await this.process.isLaunchedAttemptAlive(handle));
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Race-free, TRI-STATE settle of a wake attempt against the acceptance
+     * boundary. Delegates to the transactional {@link BrokerDB.finalizeWakeAttempt}:
+     *
+     *  - `"accepted"` — the socket already accepted our generation. The runtime is
+     *    live+bound; it must be promoted, NEVER stopped.
+     *  - `"fenced-unaccepted"` — the settle transaction COMMITTED a consumption of
+     *    THIS attempt's exact-nonce reservation, so the launched runtime can never
+     *    be accepted afterwards. Only now is a subsequent prove-stop safe.
+     *  - `"unknown"` — the settle transaction threw and did NOT commit, so the
+     *    reservation was NOT provably consumed. An acceptance may still race an
+     *    async prove-stop, so the caller must NOT stop or retry the attempt; it
+     *    leaves the identity `waking` for `recoverStrandedWakes` to reconcile once
+     *    the atomic settle can commit. A best-effort read here would be unsafe:
+     *    reading "not accepted" does not prevent a concurrent acceptance, so we
+     *    must never downgrade `unknown` to `fenced-unaccepted`.
+     */
+    // agent-standards-ignore prefer-inline-single-use-helper: shared by the
+    // per-attempt path and the outer fault backstop; a real acceptance-boundary seam.
+    settleWakeAttempt(agentId, reservation) {
+        try {
+            const result = this.db.finalizeWakeAttempt({
+                agentId,
+                reservedGeneration: reservation.reservedGeneration,
+                reservationNonce: reservation.reservationNonce,
+            });
+            return result.accepted ? "accepted" : "fenced-unaccepted";
+        }
+        catch {
+            return "unknown";
+        }
+    }
+    /**
+     * Finalize an ACCEPTED wake to `live`. Acceptance is irreversible: the socket
+     * bound this runtime to our exact lease/fence/reservation and atomically
+     * advanced+consumed the generation, so the runtime is live+connected. The
+     * `waking -> live` promotion is pure bookkeeping — driven with an unfenced
+     * administrative CAS (a lease that expired *after* acceptance must not throw a
+     * fenced transition and quarantine an already-live runtime) and GUARDED so any
+     * post-acceptance DB fault (transition, inbox count, wake completion) leaves the
+     * identity in `waking` (`woken_recovery_pending`) for `recoverStrandedWakes` to
+     * finish to `live`, rather than quarantining a live worker.
+     */
+    // agent-standards-ignore prefer-inline-single-use-helper: shared by the
+    // per-attempt accepted path and the respawn-throw accepted path.
+    promoteAcceptedWake(params) {
+        const durationMs = this.now() - params.startedAt;
+        try {
+            const live = this.transitionAdministrative({
+                agentId: params.agentId,
+                expectedVersion: params.versionCursor,
+                toState: "live",
+                reason: params.reason,
+                actor: params.actor,
+                correlationId: params.correlationId,
+                durationMs,
+                queueDepth: this.db.getUnreadInboxCount(params.agentId),
+            });
+            this.db.completeWakeForAgent(params.agentId);
+            return {
+                ok: true,
+                agentId: params.agentId,
+                correlationId: params.correlationId,
+                state: live.lifecycleState ?? "live",
+                reason: "woken",
+                runtimeGeneration: params.reservedGeneration,
+                attempts: params.attempt,
+                durationMs,
+            };
+        }
+        catch {
+            return {
+                ok: true,
+                agentId: params.agentId,
+                correlationId: params.correlationId,
+                state: this.db.getAgentById(params.agentId)?.lifecycleState ?? "waking",
+                reason: "woken_recovery_pending",
+                runtimeGeneration: params.reservedGeneration,
+                attempts: params.attempt,
+            };
+        }
+    }
+    /**
+     * Attempt to accept a launched runtime's generation on registration. Called
+     * by the socket server (or the injected registration waiter). Idempotent and
+     * fenced: only the reservation's exact lease/fence/generation is accepted.
+     */
+    acceptRuntimeRegistration(input) {
+        const acceptance = this.db.acceptRuntimeGeneration({ ...input, now: this.now() });
+        if (!acceptance.accepted) {
+            const agent = this.db.getAgentById(input.agentId);
+            this.db.recordAgentLifecycleEvent({
+                agentId: input.agentId,
+                fromState: agent?.lifecycleState ?? "waking",
+                toState: agent?.lifecycleState ?? "waking",
+                lifecycleVersion: agent?.lifecycleVersion ?? 0,
+                reason: "runtime_registration",
+                actor: "broker",
+                correlationId: input.wakeLeaseId,
+                outcome: "generation_rejected",
+                errorCode: acceptance.reason,
+                fenceToken: input.fenceToken,
+            });
+        }
+        return acceptance;
+    }
+    /**
+     * Reconcile lifecycle + wake-queue state left inconsistent by a broker crash.
+     * Intended to run once on broker startup (and is safe to re-run). DB-only and
+     * idempotent. Three classes of strand are repaired:
+     *
+     * 1. Agents in `waking` (crash between generation acceptance and the final
+     *    `waking -> live` transition):
+     *    - If a runtime already accepted its generation (reservation consumed and
+     *      runtime_generation advanced past the checkpoint's generation) only the
+     *      final live transition was lost → complete to `live` so the inbox drains.
+     *    - Otherwise the wake outcome is uncertain (a runtime may or may not have
+     *      launched) → fail closed to `reap-candidate` for manual review.
+     * 2. Agents in `hibernating` (crash mid-hibernate, before reaching the durable
+     *    `hibernated` state): the runtime may or may not have been torn down, so
+     *    completing to `hibernated` risks a double launch on the next wake → fail
+     *    closed to `reap-candidate` for manual review.
+     * 3. Wake-queue rows left in `dispatching` (crash mid-dispatch): the owning
+     *    dispatch loop is gone, so return them to `queued` (they also block the
+     *    unique active-agent index until reclaimed) so a fresh pass re-dispatches.
+     *
+     * Only a lease held by THIS live broker instance causes a skip (we are still
+     * actively driving that operation). A lease owned by a *different* instance is
+     * orphaned from a prior, now-dead broker — a crash normally leaves precisely
+     * such an unexpired-but-orphaned lease — so it is reconciled immediately rather
+     * than waiting out its TTL (during which the row would otherwise be stranded).
+     */
+    recoverStrandedWakes(opts = {}) {
+        const now = opts.now ?? this.now();
+        const recovered = [];
+        for (const agent of this.db.getAllAgents()) {
+            const strandedState = agent.lifecycleState;
+            if (strandedState !== "waking" && strandedState !== "hibernating")
+                continue;
+            const lease = this.db.getAgentLifecycleLease(agent.id);
+            const heldByThisBroker = lease !== null &&
+                Date.parse(lease.expiresAt) > now &&
+                lease.ownerBrokerInstanceId === this.brokerInstanceId;
+            if (heldByThisBroker)
+                continue;
+            const version = agent.lifecycleVersion ?? 0;
+            const correlationId = this.newId();
+            // A stranded `hibernating` agent never reached the durable `hibernated`
+            // state; its runtime liveness is unknown, so fail closed rather than risk
+            // a double launch by completing the hibernate.
+            if (strandedState === "hibernating") {
+                try {
+                    this.db.cancelWake(agent.id);
+                    this.db.clearAgentWakeReservation(agent.id);
+                    this.db.transitionAgentLifecycle({
+                        agentId: agent.id,
+                        expectedVersion: version,
+                        toState: "reap-candidate",
+                        reason: "hibernate_recovery_stranded",
+                        actor: "broker",
+                        correlationId,
+                    });
+                    if (lease)
+                        this.db.releaseAgentLifecycleLease(agent.id, lease.leaseId, lease.fenceToken);
+                    recovered.push({ agentId: agent.id, action: "quarantined" });
+                }
+                catch {
+                    // Raced with a live owner or a concurrent recovery pass; leave it be.
+                }
+                continue;
+            }
+            const reservation = this.db.getAgentWakeReservation(agent.id);
+            const checkpointGeneration = this.db.getLatestAgentCheckpointReceipt(agent.id)?.runtimeGeneration ?? null;
+            const currentGeneration = agent.runtimeGeneration ?? 0;
+            const generationAccepted = reservation === null &&
+                checkpointGeneration !== null &&
+                currentGeneration > checkpointGeneration;
+            try {
+                if (generationAccepted) {
+                    this.db.transitionAgentLifecycle({
+                        agentId: agent.id,
+                        expectedVersion: version,
+                        toState: "live",
+                        reason: "wake_recovery_complete",
+                        actor: "broker",
+                        correlationId,
+                    });
+                    this.db.completeWakeForAgent(agent.id);
+                    if (lease)
+                        this.db.releaseAgentLifecycleLease(agent.id, lease.leaseId, lease.fenceToken);
+                    recovered.push({ agentId: agent.id, action: "completed" });
+                }
+                else {
+                    this.db.clearAgentWakeReservation(agent.id);
+                    this.db.cancelWake(agent.id);
+                    this.db.transitionAgentLifecycle({
+                        agentId: agent.id,
+                        expectedVersion: version,
+                        toState: "reap-candidate",
+                        reason: "wake_recovery_stranded",
+                        actor: "broker",
+                        correlationId,
+                    });
+                    if (lease)
+                        this.db.releaseAgentLifecycleLease(agent.id, lease.leaseId, lease.fenceToken);
+                    recovered.push({ agentId: agent.id, action: "quarantined" });
+                }
+            }
+            catch {
+                // Raced with a live owner or a concurrent recovery pass; leave it be.
+            }
+        }
+        // Reclaim wake-queue rows orphaned mid-dispatch by the crash. On a fresh
+        // startup nothing is actively dispatching, so any `dispatching` row is
+        // stale; return it to `queued` for a fresh dispatch pass.
+        for (const row of this.db.listWakeQueue("dispatching")) {
+            const requeued = this.db.requeueWake(row.id);
+            if (requeued)
+                recovered.push({ agentId: row.agentId, action: "requeued" });
+        }
+        return recovered;
+    }
+    // ─── Wake queue dispatch ────────────────────────────────────────
+    /**
+     * Enqueue a wake trigger for a hibernated agent. Idempotent per agent and
+     * priority-ordered (targeted work first). Never fans out to broadcast.
+     */
+    enqueueWakeTrigger(input) {
+        const spec = this.db.getAgentRuntimeSpec(input.agentId);
+        this.db.enqueueWake({
+            agentId: input.agentId,
+            repoRoot: spec?.repoRoot ?? null,
+            triggerKind: input.triggerKind,
+            triggerMessageId: input.triggerMessageId ?? null,
+            priority: wakeTriggerPriority(input.triggerKind),
+            reason: input.reason,
+            correlationId: input.correlationId ?? this.newId(),
+        });
+    }
+    /**
+     * Dispatch queued wakes respecting global and per-repo concurrency limits,
+     * in priority then oldest-first order. Returns the results of wakes started
+     * this pass. Safe to call repeatedly (e.g. on a timer or after each trigger).
+     */
+    async dispatchWakeQueue() {
+        const results = [];
+        // Agents whose wake could not start this pass because a *non-wake* lease is
+        // transiently held. Their rows are requeued (not consumed); skip them for the
+        // remainder of this pass so a lingering lease cannot spin the loop.
+        const deferred = new Set();
+        for (;;) {
+            const globalInflight = this.db.countInflightWakes();
+            if (globalInflight >= this.config.maxConcurrentWakes)
+                break;
+            const next = this.selectNextDispatchableWake(deferred);
+            if (!next)
+                break;
+            const claimed = this.db.markWakeDispatching(next.id);
+            if (!claimed)
+                continue;
+            let result;
+            try {
+                result = await this.wake(claimed.agentId, {
+                    trigger: claimed.triggerKind,
+                    reason: claimed.reason,
+                    correlationId: claimed.correlationId,
+                });
+            }
+            catch {
+                // wake() is designed to fail closed without throwing, but a dispatching
+                // queue row must never strand the drain pass. Guard the finalization
+                // write ITSELF: if cancelling the row also throws (transient DB fault),
+                // leave the row `dispatching` — it carries no held lease and is requeued
+                // by `recoverStrandedWakes` on the next reconciliation pass — rather than
+                // letting the exception crash the loop and strand every other queued row.
+                let lifecycleState = "waking";
+                try {
+                    this.db.completeWakeQueueEntry(claimed.id, "cancelled");
+                    lifecycleState = this.db.getAgentById(claimed.agentId)?.lifecycleState ?? "waking";
+                }
+                catch {
+                    // Finalization write failed; the row stays reclaimable via reconciliation.
+                }
+                results.push({
+                    ok: false,
+                    agentId: claimed.agentId,
+                    correlationId: claimed.correlationId,
+                    state: lifecycleState,
+                    reason: "wake_fault",
+                });
+                continue;
+            }
+            try {
+                if (!result.ok && result.reason === "wake_in_progress") {
+                    // A live wake lease owner is already draining this agent; consume the row.
+                    this.db.completeWakeQueueEntry(claimed.id, "done");
+                }
+                else if (!result.ok && result.reason === "wake_lease_contended") {
+                    // A non-wake lifecycle lease is transiently holding the agent; no wake is
+                    // in flight, so the trigger must survive. Requeue and defer this agent so
+                    // the same lease cannot re-select it and spin this pass.
+                    this.db.requeueWake(claimed.id);
+                    deferred.add(claimed.agentId);
+                }
+                else {
+                    this.db.completeWakeQueueEntry(claimed.id, result.ok ? "done" : "cancelled");
+                }
+            }
+            catch {
+                // A transient finalization-write failure must not crash the drain pass or
+                // strand the other queued agents. The wake itself already resolved durably
+                // above; the row simply stays `dispatching` and is reclaimed by
+                // `recoverStrandedWakes` (which requeues every `dispatching` row) on the
+                // next startup/reconciliation pass. Fail-closed: never lose the trigger.
+            }
+            results.push(result);
+        }
+        return results;
+    }
+    selectNextDispatchableWake(deferred) {
+        const queued = this.db.listWakeQueue("queued");
+        for (const entry of queued) {
+            if (deferred?.has(entry.agentId))
+                continue;
+            const repoInflight = this.db.countInflightWakes(entry.repoRoot ?? null);
+            if (repoInflight >= this.config.maxConcurrentWakesPerRepo)
+                continue;
+            return entry;
+        }
+        return null;
+    }
+    // ─── Internal helpers ───────────────────────────────────────────
+    async defaultAwaitRegistration(ctx) {
+        const deadline = this.now() + this.config.registrationTimeoutMs;
+        for (;;) {
+            const agent = this.db.getAgentById(ctx.agentId);
+            if (agent?.runtimeGeneration === ctx.reservedGeneration)
+                return true;
+            if (this.now() >= deadline)
+                return false;
+            await new Promise((resolve) => setTimeout(resolve, 25));
+        }
+    }
+    refuseHibernate(agentId, correlationId, state, reason, actor) {
+        this.recordRefusal(agentId, "hibernate_refused", reason, actor, correlationId);
+        return { ok: false, agentId, correlationId, state, reason };
+    }
+    refuseWake(agentId, correlationId, state, reason, actor) {
+        this.recordRefusal(agentId, "wake_refused", reason, actor, correlationId);
+        return { ok: false, agentId, correlationId, state, reason };
+    }
+    recordRefusal(agentId, outcome, reason, actor, correlationId) {
+        const agent = this.db.getAgentById(agentId);
+        this.db.recordAgentLifecycleEvent({
+            agentId,
+            fromState: agent?.lifecycleState ?? "live",
+            toState: agent?.lifecycleState ?? "live",
+            lifecycleVersion: agent?.lifecycleVersion ?? 0,
+            reason,
+            actor,
+            correlationId,
+            outcome,
+        });
+    }
+    /**
+     * Drive a fenced lifecycle transition bound to the *live* held lease. Passing
+     * the full lease identity (fence + id + operation + current time) lets the DB
+     * reject an expired, superseded, or wrong-operation lease rather than trusting
+     * the fence token alone. `now` is read fresh per call so a lease that expires
+     * mid-operation cannot authorize a later transition.
+     */
+    transitionFenced(lease, input) {
+        return this.db.transitionAgentLifecycle({
+            ...input,
+            fenceToken: lease.fenceToken,
+            leaseId: lease.leaseId,
+            expectedOperation: lease.operation,
+            now: this.now(),
+        });
+    }
+    /**
+     * Fail-closed *safety* transition to a quarantine/abort state. Unlike a
+     * forward-progress transition this is deliberately UNFENCED: it must be able
+     * to fire even when our own lease has expired mid-operation (e.g. a wake whose
+     * cumulative adapter waits outran the lease TTL), otherwise the agent would be
+     * stranded in `waking`/`hibernating`. Safety is preserved by the version CAS
+     * inside `transitionAgentLifecycle`: if another broker legitimately advanced
+     * the agent (bumping the version) our recovery CAS fails and we do not clobber
+     * it; if nobody else touched it, we move it to the safe state.
+     */
+    transitionAdministrative(input) {
+        return this.db.transitionAgentLifecycle(input);
+    }
+    quarantine(agentId, expectedVersion, correlationId, actor, reason, _lease) {
+        const current = this.transitionAdministrative({
+            agentId,
+            expectedVersion,
+            toState: "reap-candidate",
+            reason,
+            actor,
+            correlationId,
+        });
+        return {
+            ok: false,
+            agentId,
+            correlationId,
+            state: current.lifecycleState ?? "reap-candidate",
+            reason,
+        };
+    }
+    quarantineWake(agentId, expectedVersion, correlationId, actor, reason, _lease, attempts) {
+        this.db.clearAgentWakeReservation(agentId);
+        const current = this.transitionAdministrative({
+            agentId,
+            expectedVersion,
+            toState: "reap-candidate",
+            reason,
+            actor,
+            correlationId,
+        });
+        return {
+            ok: false,
+            agentId,
+            correlationId,
+            state: current.lifecycleState ?? "reap-candidate",
+            reason,
+            attempts,
+        };
+    }
+}
+/** Convenience: derive whether an agent is a durable hibernation identity. */
+export function isDurableHibernationState(agent) {
+    const state = agent.lifecycleState;
+    return state === "hibernating" || state === "hibernated" || state === "waking";
+}