instar 1.3.499 → 1.3.501

package/dist/monitoring/ResumeQueue.d.ts

@@ -0,0 +1,223 @@
+/**
+ * ResumeQueue — the durable, ordered queue of mid-work reaped sessions
+ * awaiting revival (reap-notify spec R2.2/R2.3, parts of R2.9/R2.10).
+ *
+ * One queue per machine, JSON-file backed (`state/resume-queue.json`),
+ * in-memory authoritative with SYNCHRONOUS persist on every mutation using
+ * the crash-durable discipline: write temp → fsync temp → rename → fsync
+ * parent dir. A crash loses at most the latest mutation, which boot
+ * reconciliation absorbs (a lost enqueue is re-created from the reap-log; a
+ * lost transition replays as a failed attempt).
+ *
+ * Single-writer is ENFORCED, not assumed: a lockfile
+ * (`state/resume-queue.lock`, pid + hostname + heartbeat mtime) is taken at
+ * start. Stale-lock recovery is automatic on the SAME host (dead pid or old
+ * heartbeat ⇒ reclaim + log); a LIVE other process disables the queue loudly.
+ * HARD INVARIANT: a lock whose hostname differs from this host is NEVER
+ * liveness-probed or reclaimed — pid checks are meaningless cross-host — it
+ * disables the queue loudly instead, and the disable message documents the
+ * operational recovery (verify nothing else uses the state dir, delete the
+ * lock, restart).
+ *
+ * Eligibility (R2.2) is stricter than midWork: terminal + autonomous +
+ * (≥1 strong signal, OR topic-bound with ≥2 distinct weak signals); jobs
+ * must opt in (`resumeOnReap:true`, default false); sessions with neither a
+ * topic binding nor a jobSlug are excluded at enqueue; operator kills
+ * excluded by default; watchdog stuck-kills and topic-moved closeouts never
+ * queue. The resurrection LEDGER (keyed on stable identity
+ * `topicId ?? jobSlug ?? tmuxSession`, tombstones surviving dequeue, 24h
+ * reset window) caps kill-resume-kill loops at `maxResurrections`.
+ *
+ * Dry-run posture (the fleet's shipped default): entries ARE durably
+ * enqueued (observable in `GET /sessions/resume-queue` — the soak needs real
+ * midWork entries), but the drainer never spawns (it audits `would-resume`)
+ * and nothing user-facing claims a queued resume. Deliberate interpretation
+ * of "observe-only", recorded in the side-effects review.
+ */
+export type ResumePriorityClass = 'interactive' | 'job' | 'other';
+export type ResumeEntryStatus = 'queued' | 'starting' | 'respawned' | 'failed' | 'cancelled' | `invalidated:${string}` | `gave-up:${string}`;
+export interface ResumeQueueEntry {
+    id: string;
+    queuedAt: string;
+    /** TTL re-anchor on an operator requeue (R2.10); TTL keys on max(queuedAt, requeuedAt). */
+    requeuedAt?: string;
+    /** Stable identity: topicId ?? jobSlug ?? tmuxSession (R2.3). */
+    stableKey: string;
+    sessionName: string;
+    tmuxSession: string;
+    topicId?: number;
+    /** Conversation-resume UUID snapshot at enqueue time (R2.6 revalidates). */
+    resumeUuid?: string;
+    jobSlug?: string;
+    cwd: string;
+    worktreePath?: string;
+    priorityClass: ResumePriorityClass;
+    reason: string;
+    workEvidence: string[];
+    attempts: number;
+    status: ResumeEntryStatus;
+    lastAttemptAt?: string;
+    /** Backoff hold between attempts (R2.9). */
+    nextAttemptAt?: string;
+    /** Milliseconds of operator-pause time excluded from this entry's TTL clock. */
+    frozenMs?: number;
+    /** TTL expiry under sustained pressure carries this marker (R2.9). */
+    pressureStarved?: boolean;
+}
+/** Tombstone ledger row — survives dequeue (R2.9 resurrection cap). */
+export interface ResurrectionTombstone {
+    stableKey: string;
+    resurrections: number;
+    windowStartAt: string;
+    lastResumeAt?: string;
+}
+export interface ResumeQueueConfig {
+    enabled: boolean;
+    dryRun: boolean;
+    maxAttempts: number;
+    maxResurrections: number;
+    entryTtlHours: number;
+    maxQueueSize: number;
+    includeOperatorKills: boolean;
+}
+export declare const DEFAULT_RESUME_QUEUE_CONFIG: ResumeQueueConfig;
+export interface EnqueueDecision {
+    enqueued: boolean;
+    /** Why not (audit vocabulary). */
+    why?: string;
+    entry?: ResumeQueueEntry;
+}
+export interface ResumeCandidateInput {
+    sessionName: string;
+    tmuxSession: string;
+    topicId?: number | null;
+    jobSlug?: string;
+    jobResumeOptIn?: boolean;
+    resumeUuid?: string | null;
+    cwd: string;
+    worktreePath?: string;
+    reason: string;
+    disposition: 'terminal' | 'recovery-bounce';
+    origin: 'operator' | 'autonomous';
+    workEvidence: string[];
+}
+export interface ResumeQueueDeps {
+    /** `.instar` state dir — files live under `<stateDir>/state/`. */
+    stateDir: string;
+    /** Decision-transition audit sink (logs/resume-queue.jsonl). */
+    audit?: (event: Record<string, unknown>) => void;
+    /** Aggregated attention surface (ONE rolling item — the caller dedupes). */
+    raiseAggregated?: (kind: string, detail: string) => void;
+    now?: () => number;
+    hostname?: () => string;
+    /** pid liveness probe (tests override). */
+    pidAlive?: (pid: number) => boolean;
+}
+/** Pure eligibility classifier (R2.2) — exported for tests. */
+export declare function classifyEligibility(input: ResumeCandidateInput, cfg: Pick<ResumeQueueConfig, 'includeOperatorKills'>): {
+    eligible: boolean;
+    why?: string;
+};
+export declare class ResumeQueue {
+    private readonly deps;
+    private readonly cfg;
+    private readonly now;
+    private readonly statePath;
+    private readonly lockPath;
+    private state;
+    private seq;
+    /** Non-null ⇒ the queue is disabled (lock conflict / foreign lock). */
+    private disabledReason;
+    private lockHeld;
+    constructor(deps: ResumeQueueDeps, cfg?: Partial<ResumeQueueConfig>);
+    config(): ResumeQueueConfig;
+    isDisabled(): string | null;
+    isPaused(): boolean;
+    isDryRun(): boolean;
+    /**
+     * Acquire the single-writer lock + load the durable state. Returns false
+     * (queue disabled, reason in `isDisabled()`) on a live-other-process or
+     * foreign-host lock — never throws for those.
+     */
+    start(): boolean;
+    stop(): void;
+    /** Refresh the lock heartbeat (drainer calls this every tick). */
+    heartbeat(): void;
+    private acquireLock;
+    private load;
+    /** Crash-durable persist: temp → fsync temp → rename → fsync parent dir. */
+    private persist;
+    private audit;
+    /** Boot reconciliation half 1 (R2.4): `starting` found at load = failed attempt. */
+    private reconcileStartingEntries;
+    /**
+     * Boot reconciliation half 2 (R2.4): recent reap-log midWork terminal
+     * autonomous entries with no queue entry AND no tombstone are re-enqueued —
+     * closing the crash window where an enqueue's persist was lost. The caller
+     * supplies reconstruction (session records, resume map) — a candidate that
+     * cannot be reconstructed is skipped (the next re-reap recreates it).
+     */
+    reconcileFromReapLog(candidates: ResumeCandidateInput[]): number;
+    private stableKeyFor;
+    /**
+     * The enqueue decision (R2.2 + R2.3 + R2.9 resurrection cap). Audited on
+     * every outcome. In dry-run the entry is still durably enqueued (the soak's
+     * observable), but nothing downstream spawns or claims a resume.
+     */
+    considerEnqueue(input: ResumeCandidateInput, opts?: {
+        source?: string;
+    }): EnqueueDecision;
+    private tombstoneFor;
+    /** Record a successful resume (drainer, after spawn verification). */
+    recordResumeSuccess(stableKey: string): void;
+    /** Ordered open entries: interactive → job → other, FIFO inside each (R2.5). */
+    nextCandidates(): ResumeQueueEntry[];
+    list(): ResumeQueueEntry[];
+    get(id: string): ResumeQueueEntry | undefined;
+    /** State-transition helpers (each persists + audits the TRANSITION). */
+    transition(id: string, status: ResumeEntryStatus, extra?: Partial<ResumeQueueEntry>): boolean;
+    /** Operator/API cancel (an explicit per-topic stop cancels that topic's entries — R2.7). */
+    cancel(id: string): boolean;
+    cancelByTopic(topicId: number): number;
+    /**
+     * Requeue clamps (R2.10): eligible from gave-up:* ONLY — never `cancelled`
+     * (an operator stop) and refused while paused (callers enforce the 409).
+     * Resets attempts and RE-ANCHORS the TTL clock while preserving the original
+     * queuedAt for the R2.6 operator-stop / job-ran-since checks. Requeueing a
+     * gave-up:resurrection-cap entry grants exactly ONE audited override.
+     */
+    requeue(id: string): {
+        ok: boolean;
+        why?: string;
+    };
+    /** Queue-global pause (R2.7): entries keep their states; TTLs freeze. */
+    pause(reason: string): void;
+    /** Explicit unpause lever (R2.7) — accumulates the pause into frozenMs. */
+    unpause(): void;
+    pauseInfo(): {
+        paused: boolean;
+        pausedAt?: string;
+        reason?: string;
+    };
+    /**
+     * TTL sweep (R2.9): INCIDENT-AGE semantics — wall clock since
+     * max(queuedAt, requeuedAt), minus operator-pause freeze, NOT frozen by
+     * pressure. Expiries under sustained pressure carry the pressure-starved
+     * marker so a day-long overload reads differently from ordinary staleness.
+     * No sweep while paused (an operator pause must not silently expire the queue).
+     */
+    expireTtl(pressureCalm: boolean): ResumeQueueEntry[];
+    /** Snapshot for the API surface (R2.10). */
+    snapshot(): {
+        paused: boolean;
+        pauseReason?: string;
+        disabled: string | null;
+        dryRun: boolean;
+        entries: ResumeQueueEntry[];
+        tombstones: ResurrectionTombstone[];
+    };
+    /** True when a LIVE (non-dry-run) queued entry exists for this tmux session
+     *  — feeds the notifier's "restart is queued" line (R1.2). */
+    hasLiveQueuedEntryFor(tmuxSession: string): boolean;
+}
+//# sourceMappingURL=ResumeQueue.d.ts.map

package/dist/monitoring/ResumeQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ResumeQueue.d.ts","sourceRoot":"","sources":["../../src/monitoring/ResumeQueue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAQH,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG,KAAK,GAAG,OAAO,CAAC;AAElE,MAAM,MAAM,iBAAiB,GACzB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,QAAQ,GACR,WAAW,GACX,eAAe,MAAM,EAAE,GACvB,WAAW,MAAM,EAAE,CAAC;AAExB,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,mBAAmB,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,uEAAuE;AACvE,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,OAAO,CAAC;CAC/B;AAED,eAAO,MAAM,2BAA2B,EAAE,iBAQzC,CAAC;AAcF,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,gBAAgB,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,UAAU,GAAG,iBAAiB,CAAC;IAC5C,MAAM,EAAE,UAAU,GAAG,YAAY,CAAC;IAClC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjD,4EAA4E;IAC5E,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,MAAM,CAAC;IACxB,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACrC;AAED,+DAA+D;AAC/D,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,oBAAoB,EAC3B,GAAG,EAAE,IAAI,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,GACnD;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAmBrC;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;IACxC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,KAAK,CAA8E;IAC3F,OAAO,CAAC,GAAG,CAAK;IAChB,uEAAuE;IACvE,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,QAAQ,CAAS;gBAEb,IAAI,EAAE,eAAe,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC;IASnE,MAAM,IAAI,iBAAiB;IAI3B,UAAU,IAAI,MAAM,GAAG,IAAI;IAI3B,QAAQ,IAAI,OAAO;IAInB,QAAQ,IAAI,OAAO;IAInB;;;;OAIG;IACH,KAAK,IAAI,OAAO;IAOhB,IAAI,IAAI,IAAI;IAWZ,kEAAkE;IAClE,SAAS,IAAI,IAAI;IAYjB,OAAO,CAAC,WAAW;IAuEnB,OAAO,CAAC,IAAI;IA4CZ,4EAA4E;IAC5E,OAAO,CAAC,OAAO;IA4Bf,OAAO,CAAC,KAAK;IASb,oFAAoF;IACpF,OAAO,CAAC,wBAAwB;IAchC;;;;;;OAMG;IACH,oBAAoB,CAAC,UAAU,EAAE,oBAAoB,EAAE,GAAG,MAAM;IAchE,OAAO,CAAC,YAAY;IAMpB;;;;OAIG;IACH,eAAe,CAAC,KAAK,EAAE,oBAAoB,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe;IAoGzF,OAAO,CAAC,YAAY;IAIpB,sEAAsE;IACtE,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAc5C,gFAAgF;IAChF,cAAc,IAAI,gBAAgB,EAAE;IAUpC,IAAI,IAAI,gBAAgB,EAAE;IAI1B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAI7C,wEAAwE;IACxE,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,OAAO;IAW7F,4FAA4F;IAC5F,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAM3B,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAWtC;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE;IAiBlD,yEAAyE;IACzE,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAS3B,2EAA2E;IAC3E,OAAO,IAAI,IAAI;IAef,SAAS,IAAI;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;IAIpE;;;;;;OAMG;IACH,SAAS,CAAC,YAAY,EAAE,OAAO,GAAG,gBAAgB,EAAE;IAmBpD,4CAA4C;IAC5C,QAAQ,IAAI;QACV,MAAM,EAAE,OAAO,CAAC;QAChB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,MAAM,EAAE,OAAO,CAAC;QAChB,OAAO,EAAE,gBAAgB,EAAE,CAAC;QAC5B,UAAU,EAAE,qBAAqB,EAAE,CAAC;KACrC;IAWD;kEAC8D;IAC9D,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;CAMpD"}