@pinet/broker-core 0.2.4 → 0.2.7

package/dist/hibernation-orchestrator.d.ts

@@ -0,0 +1,327 @@
+import type { BrokerDB } from "./schema.js";
+import type { AgentInfo, RuntimeGenerationAcceptance, TmuxAgentRuntimeSpec, WakeTriggerKind } from "./types.js";
+/**
+ * Result of asking a follower to cooperatively checkpoint before exit. When
+ * `hibernateSafe` is false the orchestrator aborts hibernation and keeps the
+ * runtime alive rather than exiting an unsafe process.
+ */
+export interface HibernationCheckpointOutcome {
+    hibernateSafe: boolean;
+    reason: string | null;
+    sessionResumeRef: string | null;
+    pendingInboxCount: number;
+    rssBytes: number | null;
+}
+/** Everything a launch adapter needs to bring back exactly the fenced runtime. */
+export interface RuntimeLaunchContext {
+    agentId: string;
+    stableId: string;
+    wakeLeaseId: string;
+    fenceToken: number;
+    reservedGeneration: number;
+    /** Per-attempt nonce the launched runtime must echo back on registration. */
+    reservationNonce: string;
+    correlationId: string;
+    spec: TmuxAgentRuntimeSpec;
+}
+/**
+ * Opaque, attempt-bound identity for a runtime a single wake attempt launched.
+ * Returned by {@link HibernationTmuxController.respawnRuntime} and required by
+ * the attempt-scoped stop/liveness checks so that retry cleanup proves the
+ * EXACT process THIS attempt spawned is gone — never the pre-hibernation runtime
+ * (whose recorded PID generation is already dead) nor a different attempt.
+ */
+export interface RuntimeAttemptHandle {
+    /** The reservation nonce of the launching attempt (binds the handle to it). */
+    readonly reservationNonce: string;
+    /** The tmux pane the attempt launched into (a real adapter locates the PID). */
+    readonly tmuxTarget: string;
+    /** OS pid of the launched runtime, if the adapter captured it. */
+    readonly pid: number | null;
+}
+/** Controls the dormant Pi runtime process (never tmux itself). */
+export interface HibernationProcessController {
+    /** Ask the follower to flush a checkpoint and confirm hibernate safety. */
+    requestCheckpoint(spec: TmuxAgentRuntimeSpec): Promise<HibernationCheckpointOutcome>;
+    /** Gracefully stop the live Pi runtime (TERM then bounded KILL) before hibernation. */
+    stopRuntime(spec: TmuxAgentRuntimeSpec): Promise<{
+        stopped: boolean;
+        rssBytes: number | null;
+    }>;
+    /** True while the recorded Pi PID/process generation is still alive. */
+    isRuntimeAlive(spec: TmuxAgentRuntimeSpec): Promise<boolean>;
+    /**
+     * Stop the runtime a SPECIFIC wake attempt launched, addressed by its
+     * attempt-bound handle (not the durable spec). Used by retry cleanup to prove
+     * the exact failed-attempt process is gone before relaunching.
+     */
+    stopLaunchedAttempt(handle: RuntimeAttemptHandle): Promise<{
+        stopped: boolean;
+    }>;
+    /** True while the runtime a SPECIFIC wake attempt launched is still alive. */
+    isLaunchedAttemptAlive(handle: RuntimeAttemptHandle): Promise<boolean>;
+}
+/** Reads/writes the attachable tmux shell that outlives the Pi runtime. */
+export interface HibernationTmuxController {
+    /** True if the recorded tmux session/pane still exists (operator-attachable). */
+    isSessionAttachable(spec: TmuxAgentRuntimeSpec): Promise<boolean>;
+    /**
+     * Launch exactly one replacement runtime into the recorded pane. The launched
+     * runtime is expected to register presenting the reservation fence. Resolves
+     * once the launch command has been issued (registration is confirmed
+     * separately via {@link BrokerDB.acceptRuntimeGeneration}). On a successful
+     * launch it returns an attempt-bound {@link RuntimeAttemptHandle} so retry
+     * cleanup can prove THIS attempt's runtime is stopped before relaunching; a
+     * launch that yields no handle is treated as unprovable (fail closed).
+     */
+    respawnRuntime(ctx: RuntimeLaunchContext): Promise<{
+        launched: boolean;
+        handle: RuntimeAttemptHandle | null;
+    }>;
+}
+export interface HibernationOrchestratorConfig {
+    /** Max time to wait for the cooperative checkpoint handshake. */
+    handshakeTimeoutMs: number;
+    /** Lease TTL for a wake operation. */
+    wakeLeaseMs: number;
+    /** Lease TTL for a hibernate operation. */
+    hibernateLeaseMs: number;
+    /** Time to wait for a launched runtime to register + accept its generation. */
+    registrationTimeoutMs: number;
+    /** Max wake attempts before quarantining as reap-candidate. */
+    maxWakeAttempts: number;
+    /** Global concurrent in-flight wakes. */
+    maxConcurrentWakes: number;
+    /** Per-repo concurrent in-flight wakes. */
+    maxConcurrentWakesPerRepo: number;
+}
+export declare const DEFAULT_ORCHESTRATOR_CONFIG: HibernationOrchestratorConfig;
+export interface HibernationOrchestratorDeps {
+    db: BrokerDB;
+    process: HibernationProcessController;
+    tmux: HibernationTmuxController;
+    brokerInstanceId: string;
+    config?: Partial<HibernationOrchestratorConfig>;
+    /** Injectable clock (ms). Defaults to Date.now. */
+    now?: () => number;
+    /** Injectable id generator. Defaults to crypto.randomUUID. */
+    newId?: () => string;
+    /**
+     * Confirm the launched runtime registered and its generation was accepted.
+     * Resolves true once accepted, false on timeout. Injected so tests can drive
+     * registration deterministically; the runtime wires this to the socket server.
+     */
+    awaitRuntimeRegistration?: (ctx: RuntimeLaunchContext) => Promise<boolean>;
+}
+export interface HibernateResult {
+    ok: boolean;
+    agentId: string;
+    correlationId: string;
+    state: string;
+    reason: string;
+    rssBytesBefore?: number | null;
+    rssBytesAfter?: number | null;
+    durationMs?: number;
+}
+export interface WakeResult {
+    ok: boolean;
+    agentId: string;
+    correlationId: string;
+    state: string;
+    reason: string;
+    runtimeGeneration?: number;
+    attempts?: number;
+    durationMs?: number;
+}
+/** Outcome of reconciling one agent/queue-row stranded by a broker crash. */
+export interface StrandedWakeRecovery {
+    /** Agent id, or (for `requeued`) the orphaned dispatch row's agent id. */
+    agentId: string;
+    /**
+     * - `completed`   — a stranded `waking` whose generation was already accepted; only the
+     *   final live transition was lost, so it was finished to `live`.
+     * - `quarantined` — a stranded `waking` or `hibernating` with an uncertain runtime; moved
+     *   to `reap-candidate` for manual review rather than risk a double launch.
+     * - `requeued`    — a `dispatching` wake-queue row orphaned mid-dispatch by a crash;
+     *   returned to `queued` so a fresh dispatch pass can pick it up.
+     */
+    action: "completed" | "quarantined" | "requeued";
+}
+export declare function wakeTriggerPriority(kind: WakeTriggerKind): number;
+/**
+ * Broker-managed hibernation lifecycle orchestrator.
+ *
+ * Composes the durable primitives in {@link BrokerDB} (fenced leases, CAS
+ * lifecycle transitions, runtime specs, checkpoint receipts, generation
+ * reservations, wake queue, telemetry events) with injected process/tmux
+ * adapters. Every path fails closed: on any anomaly it releases its lease and
+ * quarantines the agent as `reap-candidate` with an actionable reason rather
+ * than guessing, rerouting affinity work, or killing a PID on PID alone.
+ */
+export declare class HibernationOrchestrator {
+    private readonly db;
+    private readonly process;
+    private readonly tmux;
+    private readonly brokerInstanceId;
+    private readonly config;
+    private readonly now;
+    private readonly newId;
+    private readonly awaitRuntimeRegistration;
+    constructor(deps: HibernationOrchestratorDeps);
+    /**
+     * Advance an eligible, free, broker-managed agent through `grace` to `idle`
+     * so it becomes hibernation-ready. Idempotent and fenced by CAS. Used by both
+     * the manual `pinet hibernate` path and the auto scheduler. Never forces a
+     * working/unsafe/ineligible agent forward.
+     */
+    prepareHibernation(agentId: string, opts?: {
+        reason?: string;
+        actor?: string;
+        correlationId?: string;
+    }): {
+        ready: boolean;
+        state: string;
+        reason: string;
+    };
+    hibernate(agentId: string, opts?: {
+        reason?: string;
+        actor?: string;
+        trigger?: string;
+        correlationId?: string;
+    }): Promise<HibernateResult>;
+    /**
+     * Cold-wake a hibernated agent as a single accepted runtime generation, then
+     * transition to `live` so the durable inbox drains in order. Concurrent
+     * triggers contend on the fenced wake lease; only one wins.
+     */
+    wake(agentId: string, opts?: {
+        reason?: string;
+        actor?: string;
+        trigger?: WakeTriggerKind;
+        correlationId?: string;
+    }): Promise<WakeResult>;
+    /**
+     * Best-effort proof that a launched wake ATTEMPT's runtime is gone, addressed
+     * by its attempt-bound handle (never the durable spec). Fail-closed: a missing
+     * handle, an unconfirmed stop, a still-alive probe, OR any adapter throw all
+     * count as "not proven gone", so the caller quarantines (`wake_ambiguous_launch`)
+     * rather than relaunch on / strand a possibly-live runtime.
+     */
+    private proveAttemptStopped;
+    /**
+     * Race-free, TRI-STATE settle of a wake attempt against the acceptance
+     * boundary. Delegates to the transactional {@link BrokerDB.finalizeWakeAttempt}:
+     *
+     *  - `"accepted"` — the socket already accepted our generation. The runtime is
+     *    live+bound; it must be promoted, NEVER stopped.
+     *  - `"fenced-unaccepted"` — the settle transaction COMMITTED a consumption of
+     *    THIS attempt's exact-nonce reservation, so the launched runtime can never
+     *    be accepted afterwards. Only now is a subsequent prove-stop safe.
+     *  - `"unknown"` — the settle transaction threw and did NOT commit, so the
+     *    reservation was NOT provably consumed. An acceptance may still race an
+     *    async prove-stop, so the caller must NOT stop or retry the attempt; it
+     *    leaves the identity `waking` for `recoverStrandedWakes` to reconcile once
+     *    the atomic settle can commit. A best-effort read here would be unsafe:
+     *    reading "not accepted" does not prevent a concurrent acceptance, so we
+     *    must never downgrade `unknown` to `fenced-unaccepted`.
+     */
+    private settleWakeAttempt;
+    /**
+     * Finalize an ACCEPTED wake to `live`. Acceptance is irreversible: the socket
+     * bound this runtime to our exact lease/fence/reservation and atomically
+     * advanced+consumed the generation, so the runtime is live+connected. The
+     * `waking -> live` promotion is pure bookkeeping — driven with an unfenced
+     * administrative CAS (a lease that expired *after* acceptance must not throw a
+     * fenced transition and quarantine an already-live runtime) and GUARDED so any
+     * post-acceptance DB fault (transition, inbox count, wake completion) leaves the
+     * identity in `waking` (`woken_recovery_pending`) for `recoverStrandedWakes` to
+     * finish to `live`, rather than quarantining a live worker.
+     */
+    private promoteAcceptedWake;
+    /**
+     * Attempt to accept a launched runtime's generation on registration. Called
+     * by the socket server (or the injected registration waiter). Idempotent and
+     * fenced: only the reservation's exact lease/fence/generation is accepted.
+     */
+    acceptRuntimeRegistration(input: {
+        agentId: string;
+        wakeLeaseId: string;
+        fenceToken: number;
+        reservedGeneration: number;
+        reservationNonce: string;
+    }): RuntimeGenerationAcceptance;
+    /**
+     * Reconcile lifecycle + wake-queue state left inconsistent by a broker crash.
+     * Intended to run once on broker startup (and is safe to re-run). DB-only and
+     * idempotent. Three classes of strand are repaired:
+     *
+     * 1. Agents in `waking` (crash between generation acceptance and the final
+     *    `waking -> live` transition):
+     *    - If a runtime already accepted its generation (reservation consumed and
+     *      runtime_generation advanced past the checkpoint's generation) only the
+     *      final live transition was lost → complete to `live` so the inbox drains.
+     *    - Otherwise the wake outcome is uncertain (a runtime may or may not have
+     *      launched) → fail closed to `reap-candidate` for manual review.
+     * 2. Agents in `hibernating` (crash mid-hibernate, before reaching the durable
+     *    `hibernated` state): the runtime may or may not have been torn down, so
+     *    completing to `hibernated` risks a double launch on the next wake → fail
+     *    closed to `reap-candidate` for manual review.
+     * 3. Wake-queue rows left in `dispatching` (crash mid-dispatch): the owning
+     *    dispatch loop is gone, so return them to `queued` (they also block the
+     *    unique active-agent index until reclaimed) so a fresh pass re-dispatches.
+     *
+     * Only a lease held by THIS live broker instance causes a skip (we are still
+     * actively driving that operation). A lease owned by a *different* instance is
+     * orphaned from a prior, now-dead broker — a crash normally leaves precisely
+     * such an unexpired-but-orphaned lease — so it is reconciled immediately rather
+     * than waiting out its TTL (during which the row would otherwise be stranded).
+     */
+    recoverStrandedWakes(opts?: {
+        now?: number;
+    }): StrandedWakeRecovery[];
+    /**
+     * Enqueue a wake trigger for a hibernated agent. Idempotent per agent and
+     * priority-ordered (targeted work first). Never fans out to broadcast.
+     */
+    enqueueWakeTrigger(input: {
+        agentId: string;
+        triggerKind: WakeTriggerKind;
+        reason: string;
+        triggerMessageId?: number | null;
+        correlationId?: string;
+    }): void;
+    /**
+     * Dispatch queued wakes respecting global and per-repo concurrency limits,
+     * in priority then oldest-first order. Returns the results of wakes started
+     * this pass. Safe to call repeatedly (e.g. on a timer or after each trigger).
+     */
+    dispatchWakeQueue(): Promise<WakeResult[]>;
+    private selectNextDispatchableWake;
+    private defaultAwaitRegistration;
+    private refuseHibernate;
+    private refuseWake;
+    private recordRefusal;
+    /**
+     * Drive a fenced lifecycle transition bound to the *live* held lease. Passing
+     * the full lease identity (fence + id + operation + current time) lets the DB
+     * reject an expired, superseded, or wrong-operation lease rather than trusting
+     * the fence token alone. `now` is read fresh per call so a lease that expires
+     * mid-operation cannot authorize a later transition.
+     */
+    private transitionFenced;
+    /**
+     * Fail-closed *safety* transition to a quarantine/abort state. Unlike a
+     * forward-progress transition this is deliberately UNFENCED: it must be able
+     * to fire even when our own lease has expired mid-operation (e.g. a wake whose
+     * cumulative adapter waits outran the lease TTL), otherwise the agent would be
+     * stranded in `waking`/`hibernating`. Safety is preserved by the version CAS
+     * inside `transitionAgentLifecycle`: if another broker legitimately advanced
+     * the agent (bumping the version) our recovery CAS fails and we do not clobber
+     * it; if nobody else touched it, we move it to the safe state.
+     */
+    private transitionAdministrative;
+    private quarantine;
+    private quarantineWake;
+}
+/** Convenience: derive whether an agent is a durable hibernation identity. */
+export declare function isDurableHibernationState(agent: AgentInfo): boolean;