instar 1.3.606 → 1.3.608

package/dist/monitoring/AutonomousLivenessReconciler.d.ts

@@ -0,0 +1,278 @@
+/**
+ * AutonomousLivenessReconciler — a level-triggered control loop that self-heals
+ * an autonomous run whose state file says ACTIVE (with time remaining) but has
+ * NO live session executing it.
+ *
+ * THE INCIDENT (2026-06-16): an autonomous run on topic 12476 was reaped under
+ * `age-limit` (the periodic per-session lifetime recycle). The recycle is meant
+ * to be invisible — the run should revive and continue — but the reap was tagged
+ * `midWork:false`, so it was never offered to the resume queue, and the run
+ * silently died. Its state file still said active with ~15h remaining; no tmux
+ * session was running it; nothing watched for the contradiction. When the user
+ * messaged 1h+ later there was no live session to answer.
+ *
+ * THE FIX (this module): don't perfect the edge — add a control loop that
+ * continuously compares desired state (run active+remaining) vs actual state
+ * (a live session exists) and converges. It self-heals regardless of HOW the
+ * run got orphaned. It is the liveness twin of the progress heartbeat: a
+ * heartbeat covers "alive but quiet"; this covers "dead but marked active".
+ *
+ * SIGNAL-ONLY / SAFE DIRECTION: in dryRun it only LOGS "would respawn" (plus a
+ * shadow "would-have-capped"). Live, its worst action is a bounded, capped,
+ * lease-gated, quota-gated, pressure-gated respawn of a run that genuinely
+ * should be alive (a false respawn wastes a few tokens; a missed one is the
+ * silent death above). It never blocks, delays, or rewrites anything. Ships dark
+ * on the fleet (dev-agent gate), dryRun-first on dev.
+ *
+ * THE RUN-STATE FILE IS UNTRUSTED. It is read ONLY for the binary
+ * "active + remaining" decision, the topicId, and the generation key
+ * (started_at). cwd / resumeUuid / spawn target are NEVER sourced from it — they
+ * come from authoritative registries (see §Action in the spec).
+ *
+ * Spec: docs/specs/autonomous-liveness-reconciler.md
+ */
+/** Per-topic reconcile-condition (explicit observed state, not reconstructed from logs). */
+export type ReconcileCondition = 'healthy' | 'orphaned-observing' | 'debouncing' | 'respawned' | 'capped' | 'blocked-quota' | 'blocked-pressure' | 'blocked-not-owner' | 'blocked-queue-owns' | 'mid-move';
+export interface ReconcilerActiveRun {
+    topicId: number;
+    remainingSeconds: number;
+    /** state-file `paused` flag */
+    paused: boolean;
+    /** mid-machine-move markers (a moving run is owned by neither machine) */
+    movedTo: string | null;
+    moveSuspended: boolean;
+    /**
+     * Generation key: the run's `started_at` epoch-ms (criterion 1). A candidate
+     * is only valid if no NEWER autonomous-run registration exists than this. Null
+     * when the file has no parseable started_at → fails toward NOT-a-candidate.
+     */
+    startedAtMs: number | null;
+}
+/** In-flight-spawn status for a topic (criterion 7). */
+export type InflightSpawnStatus = {
+    state: 'none';
+} | {
+    state: 'claimed';
+    sinceMs: number;
+} | {
+    state: 'spawning';
+    sinceMs: number;
+} | {
+    state: 'live-grace';
+    sinceMs: number;
+};
+export interface AutonomousLivenessReconcilerDeps {
+    now: () => number;
+    /**
+     * The active autonomous runs as the run-state files report them (already
+     * filtered to remaining > 0). Reuses activeAutonomousJobs +
+     * autonomousRunRemainingForTopic at the wiring layer.
+     */
+    listActiveRuns: () => ReconcilerActiveRun[];
+    /**
+     * Build the reconciler's OWN once-per-tick session→topic snapshot: the set of
+     * topic ids that currently have a live session. Called ONCE per tick (NOT the
+     * drainer's per-invocation liveSessionForTopic fan-out). Criterion 6 probes
+     * the returned set.
+     */
+    liveTopicSnapshot: () => Set<number>;
+    /** Is the resume queue globally paused (e.g. emergency stop)? */
+    queuePaused: () => boolean;
+    /** Is this topic already queued / in-flight in the resume queue? (criterion 7, queue arm) */
+    topicInResumeQueue: (topicId: number) => boolean;
+    /**
+     * Did the operator stop this topic since `sinceIso`? Reads ALL THREE arms
+     * (per-topic record, globalOperatorStopAt, the emergency-stop file mtime),
+     * bounded to the CURRENT run's start (NOT epoch-0). The SAME closure the
+     * drainer uses (shared-closure drift prevention).
+     */
+    operatorStoppedSince: (topicId: number, sinceIso: string) => boolean;
+    /** Multi-machine: does ANOTHER machine own this topic? (only the owner reconciles) */
+    topicOwnerElsewhere: (topicId: number) => boolean;
+    /**
+     * Does THIS machine hold the lease? DEFAULTS TO HELD (true) when syncStatus is
+     * null/single-machine — otherwise a single-machine dev agent self-blocks and
+     * the feature is inert exactly where it should act (spec criterion 5).
+     */
+    holdsLease: () => boolean;
+    /**
+     * Generation guard (criterion 1): the CURRENT registered started_at (epoch-ms)
+     * for this topic, or null if none. A candidate is obsolete (NOT current
+     * generation) if this is NEWER than the run's own startedAtMs.
+     */
+    currentGenerationMs: (topicId: number) => number | null;
+    /** Quota/budget headroom for a respawn right now (canSpawnSession().allowed). */
+    quotaOk: () => boolean;
+    /** Session-count cap headroom (the SAME cap the drainer respects). */
+    sessionCountOk: () => boolean;
+    /** Is a migration in flight (never spawn-storm past it)? */
+    migrationInFlight: () => boolean;
+    /** The reaper's live pressure tier (hoisted once per tick). */
+    pressureTier: () => 'normal' | 'moderate' | 'critical';
+    /** In-flight-spawn status for a topic (criterion 7 + atomic-claim coordination). */
+    inflightSpawnStatus: (topicId: number) => InflightSpawnStatus;
+    /**
+     * Resolve the resume UUID from the CANONICAL resume map (NOT the state file).
+     * Null when absent → raise attention (no silent fresh-conversation spawn)
+     * unless allowFreshFallback is true.
+     */
+    resolveResumeUuid: (topicId: number) => string | null;
+    /**
+     * Resolve the authoritative cwd from the topic-binding registry, realpath-
+     * resolved + validated inside the agent home. Returns null when it cannot be
+     * resolved to a SAFE path (missing binding, or a realpath escaping the agent
+     * home) → refuse the respawn (loud attention).
+     */
+    resolveCwd: (topicId: number) => string | null;
+    /**
+     * Is the topic↔session binding UNAMBIGUOUS (criterion 8)? false → routes to
+     * attention as "needs attention", NEVER an auto-respawn.
+     */
+    bindingUnambiguous: (topicId: number) => boolean;
+    /**
+     * Respawn the run (conversation-preserving) — the SAME primitive the resume
+     * drainer uses (spawnSessionForTopic). Receives the AUTHORITATIVE resolved
+     * inputs; tags the new session midWork so a later reaper kill is revived.
+     */
+    respawn: (input: {
+        topicId: number;
+        resumeUuid: string | null;
+        cwd: string;
+    }) => Promise<void>;
+    /**
+     * Atomic CAS claim of the in-flight-spawn key for a topic (process-local
+     * in-memory map). Returns true if THIS caller now owns the claim, false if
+     * someone else holds it. Released via releaseClaim.
+     */
+    claimInflight: (topicId: number) => boolean;
+    releaseClaim: (topicId: number) => void;
+    /**
+     * Post-spawn settle-kill: terminally abort the just-spawned session (clears
+     * its midWork tag first so the ResumeQueue does NOT revive an operator-stopped
+     * topic). Only called when a stop arrived during the async spawn or a
+     * duplicate live session appeared. NEVER routes through the revival path.
+     */
+    settleKill: (topicId: number) => Promise<void>;
+    /** Post one honest self-heal line to the topic. */
+    notifyTopic: (topicId: number, text: string) => Promise<void>;
+    /** Raise ONE coalesced attention item (the loop-brake "give up loudly" surface). */
+    raiseAggregated: (kind: string, detail: string) => void;
+    /** Append one audit entry (logs/autonomous-liveness.jsonl). */
+    audit: (entry: Record<string, unknown>) => void;
+    /**
+     * The ResumeQueue's resurrection count for the topic (the queue's own cap),
+     * counted toward the reconciler's redie cap so the two paths share ONE
+     * effective give-up bound. Optional — 0 when unavailable.
+     */
+    queueResurrectionCount?: (topicId: number) => number;
+    /** Optional durable cap-state load/save so the respawn cap survives a restart. */
+    loadCapState?: () => DurableCapState | null;
+    saveCapState?: (state: DurableCapState) => void;
+}
+/** Durable cap-state on disk: separate redie + spawn-failure counters per topic. */
+export interface DurableCapState {
+    /** topic → redie respawn timestamps (ms) within the rolling window. */
+    redie: Record<string, number[]>;
+    /** topic → spawn-failure timestamps (ms) within the rolling window. */
+    spawnFailure: Record<string, number[]>;
+}
+export interface AutonomousLivenessReconcilerConfig {
+    enabled?: boolean;
+    dryRun?: boolean;
+    tickIntervalSec?: number;
+    debounceTicks?: number;
+    debounceWindowSec?: number;
+    respawnTimeoutMs?: number;
+    respawnCapPerWindow?: number;
+    respawnCapWindowSec?: number;
+    spawnFailureRetryCeiling?: number;
+    maxPressureBlockedTicks?: number;
+    maxPressureBlockedSec?: number;
+    allowFreshFallback?: boolean;
+    inflightSpawnTtlMs?: number;
+    notifyUser?: boolean;
+}
+export declare class AutonomousLivenessReconciler {
+    private readonly deps;
+    private readonly cfg;
+    private timer;
+    /** Per-topic debounce observation state. */
+    private readonly observed;
+    /** Per-topic explicit condition record. */
+    private readonly conditions;
+    /** Redie counter: respawn-that-restarted-then-redied timestamps (ms). */
+    private redie;
+    /** Spawn-failure counter: respawn-that-threw-before-any-session timestamps (ms). */
+    private spawnFailure;
+    /** Topics that have hit the redie cap and been surfaced (so we don't re-raise every tick). */
+    private readonly cappedSurfaced;
+    /** Per-topic dedupe guards for the other raise-to-attention paths (P17). */
+    private readonly spawnFailSurfaced;
+    private readonly ambiguousSurfaced;
+    private readonly missingUuidSurfaced;
+    private readonly pressureSurfaced;
+    private lastTickAt;
+    private respawnTotal;
+    /** Effective inflight TTL (cfg.inflightSpawnTtlMs ?? respawnTimeoutMs + grace). */
+    private readonly inflightTtlMs;
+    constructor(deps: AutonomousLivenessReconcilerDeps, config?: AutonomousLivenessReconcilerConfig);
+    start(): void;
+    stop(): void;
+    /**
+     * One reconcile pass. All I/O goes through deps. Idempotent — a run already
+     * live / queued / not-owned / paused is a no-op. At most ONE respawn per tick.
+     */
+    tick(): Promise<void>;
+    /**
+     * Reconcile one candidate. Returns true if an actuation (respawn / would-respawn)
+     * happened this tick (counts toward the one-per-tick budget).
+     */
+    private actOn;
+    private isCurrentGeneration;
+    private inflightActive;
+    private freshWindow;
+    private withinRedieCap;
+    private recordRedie;
+    private recordSpawnFailure;
+    private persistCap;
+    private setCondition;
+    private clearPressureBlock;
+    private gc;
+    private withTimeout;
+    private scrub;
+    private safeBool;
+    private safe;
+    guardStatus(): {
+        enabled: boolean;
+        dryRun: boolean;
+        reason?: string;
+    };
+    status(): {
+        enabled: boolean;
+        dryRun: boolean;
+        tickIntervalSec: number;
+        lastTickAt: string | null;
+        respawnTotal: number;
+        capPerWindow: number;
+        capWindowSec: number;
+        observing: {
+            topicId: number;
+            count: number;
+            firstSeenAt: string;
+        }[];
+        conditions: {
+            topicId: number;
+            state: ReconcileCondition;
+            lastTransitionAt: string;
+        }[];
+        redie: {
+            topicId: number;
+            count: number;
+        }[];
+        spawnFailure: {
+            topicId: number;
+            count: number;
+        }[];
+    };
+}
+//# sourceMappingURL=AutonomousLivenessReconciler.d.ts.map

package/dist/monitoring/AutonomousLivenessReconciler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AutonomousLivenessReconciler.d.ts","sourceRoot":"","sources":["../../src/monitoring/AutonomousLivenessReconciler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,4FAA4F;AAC5F,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,oBAAoB,GACpB,YAAY,GACZ,WAAW,GACX,QAAQ,GACR,eAAe,GACf,kBAAkB,GAClB,mBAAmB,GACnB,oBAAoB,GACpB,UAAU,CAAC;AAEf,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,MAAM,EAAE,OAAO,CAAC;IAChB,0EAA0E;IAC1E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,wDAAwD;AACxD,MAAM,MAAM,mBAAmB,GAC3B;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GACjB;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,KAAK,EAAE,YAAY,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7C,MAAM,WAAW,gCAAgC;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB;;;;OAIG;IACH,cAAc,EAAE,MAAM,mBAAmB,EAAE,CAAC;IAC5C;;;;;OAKG;IACH,iBAAiB,EAAE,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,iEAAiE;IACjE,WAAW,EAAE,MAAM,OAAO,CAAC;IAC3B,6FAA6F;IAC7F,kBAAkB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IACjD;;;;;OAKG;IACH,oBAAoB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IACrE,sFAAsF;IACtF,mBAAmB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IAClD;;;;OAIG;IACH,UAAU,EAAE,MAAM,OAAO,CAAC;IAC1B;;;;OAIG;IACH,mBAAmB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACxD,iFAAiF;IACjF,OAAO,EAAE,MAAM,OAAO,CAAC;IACvB,sEAAsE;IACtE,cAAc,EAAE,MAAM,OAAO,CAAC;IAC9B,4DAA4D;IAC5D,iBAAiB,EAAE,MAAM,OAAO,CAAC;IACjC,+DAA+D;IAC/D,YAAY,EAAE,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;IACvD,oFAAoF;IACpF,mBAAmB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,mBAAmB,CAAC;IAC9D;;;;OAIG;IACH,iBAAiB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACtD;;;;;OAKG;IACH,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/C;;;OAGG;IACH,kBAAkB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IACjD;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/F;;;;OAIG;IACH,aAAa,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IAC5C,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC;;;;;OAKG;IACH,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,mDAAmD;IACnD,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9D,oFAAoF;IACpF,eAAe,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,+DAA+D;IAC/D,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAChD;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;IACrD,kFAAkF;IAClF,YAAY,CAAC,EAAE,MAAM,eAAe,GAAG,IAAI,CAAC;IAC5C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC;CACjD;AAED,oFAAoF;AACpF,MAAM,WAAW,eAAe;IAC9B,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAChC,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,kCAAkC;IACjD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAiCD,qBAAa,4BAA4B;IACvC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmC;IACxD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAElB;IACF,OAAO,CAAC,KAAK,CAA+C;IAC5D,4CAA4C;IAC5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkC;IAC3D,2CAA2C;IAC3C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsC;IACjE,yEAAyE;IACzE,OAAO,CAAC,KAAK,CAA+B;IAC5C,oFAAoF;IACpF,OAAO,CAAC,YAAY,CAA+B;IACnD,8FAA8F;IAC9F,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAqB;IACvD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAqB;IACvD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;IACzD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IACtD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,YAAY,CAAK;IACzB,mFAAmF;IACnF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;gBAE3B,IAAI,EAAE,gCAAgC,EAAE,MAAM,GAAE,kCAAuC;IA8BnG,KAAK,IAAI,IAAI;IAcb,IAAI,IAAI,IAAI;IAOZ;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiJ3B;;;OAGG;YACW,KAAK;IA2PnB,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,EAAE;YAmCI,WAAW;IAazB,OAAO,CAAC,KAAK;IASb,OAAO,CAAC,QAAQ;IAYhB,OAAO,CAAC,IAAI;IAWZ,WAAW,IAAI;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;IAOrE,MAAM,IAAI;QACR,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC;QAChB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QACrE,UAAU,EAAE;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,kBAAkB,CAAC;YAAC,gBAAgB,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QACvF,KAAK,EAAE;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC5C,YAAY,EAAE;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;KACpD;CA0BF"}