instar 1.3.710 → 1.3.712

package/dist/monitoring/RopeHealthMonitor.d.ts

@@ -0,0 +1,243 @@
+/**
+ * RopeHealthMonitor — U4.5 rope-health alerts
+ * (docs/specs/u4-5-rope-health-alerts.md).
+ *
+ * Mesh transport degradation was silent: a Tailscale key expiry dropped a rope
+ * with no warning; a persistently-down rope was visible only to someone who
+ * went looking; an all-transports-down partition — the precondition for silent
+ * message loss — had no prompt alert at all. This monitor is the productized,
+ * in-server detector: it runs its OWN bounded 30s evaluation loop (constructed
+ * and torn down by the real server boot — R-r2-2), reads the U4.3
+ * `PeerEndpointResolver.snapshot()` seam (the HARD data dependency; no interim
+ * fallback exists), and classifies each peer deterministically:
+ *
+ *   ok           — silence (no digest line, nothing).
+ *   degraded     — a rope down while ≥1 other rope is healthy, or a Tailscale
+ *                  key expiring within `keyExpiryWarnDays`. Digest-only.
+ *   peer-offline — ALL ropes down AND the peer's git-synced coarse heartbeat
+ *                  has STOPPED advancing (or the registry already marks it
+ *                  offline). Digest-only — a lid-close is NEVER urgent.
+ *   urgent       — ALL ropes down to a peer whose git-synced coarse heartbeat
+ *                  is still ADVANCING — advancement-since-onset semantics
+ *                  (R-r3-1): a beat NEWER than the all-down onset, observed
+ *                  after the onset. Freshness-window semantics are REJECTED
+ *                  (a just-lid-closed peer's last beat looks fresh for up to
+ *                  an hour). ONE HIGH attention item per episode.
+ *
+ * The urgent tier's honest detection latency is bounded by the heartbeat
+ * interval plus up to two git-sync cadences (~30-90 min); `urgentDebounceMs`
+ * is only the short-term flap filter on the all-down condition itself.
+ *
+ * Episode semantics: episodeKey = sorted machine pair + the onset quantized to
+ * 15 min — deterministically computable on BOTH sides without coordination;
+ * post-heal grouping matches ADJACENT quantization windows (R-r2-5). An episode
+ * ends only after `clearSustainMs` of continuous health (a blip cannot
+ * clear-then-re-fire). At most ONE item per side per episode; if a split-brain
+ * item is already open for the same peer, the monitor does not raise a second.
+ *
+ * Durable state (`state/rope-health.json`): transition-only writes with a short
+ * debounce (R-r2-4) — a steady-state evaluation never touches disk; counters
+ * lost to a restart are accepted (the safe direction: a restart re-debounces,
+ * never fabricates an episode).
+ *
+ * Content scrub (hard rule): alert/digest text carries rope KIND + machine
+ * NICKNAME + relative times ONLY — never raw IPs, URLs, tunnel hostnames,
+ * tailnet names, or account emails (the tailscale JSON's identifying fields
+ * never leave `parseTailscaleStatus`).
+ */
+import type { RopeHealthSnapshotRow } from '../core/PeerEndpointResolver.js';
+export type RopeHealthCondition = 'ok' | 'degraded' | 'peer-offline' | 'urgent' | 'unknown';
+export interface RopeHealthPeerInfo {
+    machineId: string;
+    /** User-facing handle — the ONLY peer identity that reaches alert/digest text. */
+    nickname: string;
+    /** Registry online flag (staleness-derived; WS4.2 offline-since semantics). */
+    registryOnline: boolean;
+}
+export type RopeHealthMetricEvent = 'evaluation' | 'transition-ok' | 'transition-degraded' | 'transition-peer-offline' | 'transition-urgent' | 'urgent-episode' | 'suppressed-by-sleep-gate' | 'suppressed-by-split-brain' | 'detected-not-notified-retry' | 'key-expiry-warning' | 'digest-emission';
+export interface RopeHealthMonitorDeps {
+    /** U4.3 read seam — the REAL resolver snapshot, never a copy. */
+    snapshot: () => RopeHealthSnapshotRow[];
+    /** This machine's stable id (episodeKey pair member). */
+    selfMachineId: string;
+    /** Known peers (excluding self) with nicknames + registry online flags. */
+    listPeers: () => RopeHealthPeerInfo[];
+    /**
+     * The mesh-INDEPENDENT liveness discriminator (R-r2-1): epoch-ms of the
+     * peer's last git-synced coarse heartbeat, or null when none/malformed.
+     */
+    readHeartbeatAtMs: (machineId: string) => number | null;
+    /** HIGH attention sink. May return a promise; a rejection = not-notified. */
+    raiseAttention: (item: {
+        id: string;
+        title: string;
+        body: string;
+    }) => unknown;
+    /** Episode-registry check: a split-brain item already open for this peer wins. */
+    splitBrainItemOpen?: (peerMachineId: string) => boolean;
+    /**
+     * Bounded exec seam for `tailscale status --json` (R-r2-3). Resolves to the
+     * raw stdout, or null when the CLI is absent/timed out (the expiry tier is
+     * then silently absent). Default impl provided; tests inject.
+     */
+    execTailscaleStatusJson?: () => Promise<string | null>;
+    /** Durable state file (state/rope-health.json). */
+    stateFilePath: string;
+    recordMetric?: (event: RopeHealthMetricEvent) => void;
+    now?: () => number;
+    logger?: (msg: string) => void;
+}
+export interface RopeHealthMonitorConfig {
+    /** The urgent tier master (rides the same dev gate; spec §5). Default true. */
+    urgentEnabled: boolean;
+    /** Time-pinned short-term flap filter on the all-down condition (R-r2-2). Default 60000. */
+    urgentDebounceMs: number;
+    /** Continuous health required before an episode ends. Default 600000. */
+    clearSustainMs: number;
+    /** Tailscale key expiry warning horizon. Default 14. */
+    keyExpiryWarnDays: number;
+    /** The monitor's OWN evaluation loop cadence. Default 30000. */
+    evaluateIntervalMs: number;
+    /** Key-expiry exec cadence. Default 3600000 (hourly). */
+    keyExpiryCheckIntervalMs: number;
+    /**
+     * HARD CAP on how long a self-wake grace window may suppress the urgent tier.
+     *
+     * HAZARD (docs/audits/multi-machine-seamless-ux-audit-2026-07.md finding
+     * P1-A7): SleepWakeDetector is known to emit FALSE wake events on this class
+     * of machine — event-loop stalls misread as sleeps (26 spurious "wake after
+     * ~12-24s sleep" events in one night while caffeinate held a
+     * PreventSystemSleep assertion). A false "recently slept" signal must never
+     * suppress a genuine partition alert for long, so the grace window is
+     * BOUNDED: suppression ends as soon as every (peer, kind) has been
+     * re-observed post-wake OR this cap elapses — whichever comes first. It is a
+     * short post-wake grace, never an open-ended veto. Default 300000 (5 min).
+     */
+    wakeGraceMaxMs: number;
+    /** Episode-key onset quantization. Default 900000 (15 min). */
+    episodeQuantumMs: number;
+    /** State-write debounce for transition bursts (R-r2-4). Default 2000. */
+    writeDebounceMs: number;
+}
+export declare const ROPE_HEALTH_DEFAULTS: RopeHealthMonitorConfig;
+/**
+ * Deterministic shared episode key: both sides compute the same value from the
+ * sorted machine pair + the onset's quantization window (spec §2).
+ */
+export declare function computeEpisodeKey(machineA: string, machineB: string, onsetMs: number, quantumMs?: number): string;
+/**
+ * Post-heal grouping (R-r2-5): two episode keys group when they name the same
+ * machine pair and their quantization windows are ADJACENT (±1 quantum) — the
+ * two sides detect at different instants, so an onset straddling a boundary
+ * yields adjacent window keys. Beyond one quantum the keys honestly show as
+ * two groups (declared best-effort display degradation).
+ */
+export declare function episodeKeysGroup(a: string, b: string, quantumMs?: number): boolean;
+/** One row of GET /mesh/rope-health (content-scrubbed: kind + nickname only). */
+export interface RopeHealthPeerView {
+    machineId: string;
+    nickname: string;
+    condition: RopeHealthCondition;
+    kinds: Array<{
+        kind: string;
+        dead: boolean;
+        consecutiveFailures: number;
+    }>;
+    allDownSince: number | null;
+    episodeKey: string | null;
+    urgentRaisedAt: number | null;
+    detectedNotNotified: boolean;
+}
+export interface RopeHealthStatus {
+    lastEvaluatedAt: number | null;
+    evaluations: number;
+    peers: RopeHealthPeerView[];
+    keyExpiry: {
+        /** false ⇒ CLI absent / never checked / unparseable — tier silently absent. */
+        available: boolean;
+        soonest?: {
+            role: 'self' | 'peer';
+            expiresAtIso: string;
+            inDays: number;
+        };
+        warn?: boolean;
+        lastCheckedAt?: number | null;
+    };
+    /** ≤3-sentence consolidated digest section, or null when everything is ok. */
+    digest: string | null;
+    counters: {
+        urgentEpisodes: number;
+        suppressedBySleepGate: number;
+        suppressedBySplitBrain: number;
+        notNotifiedRetries: number;
+    };
+}
+/** Default bounded exec: 5s hard timeout; absent CLI / any failure ⇒ null. */
+export declare function defaultExecTailscaleStatusJson(): Promise<string | null>;
+export declare class RopeHealthMonitor {
+    private readonly d;
+    private readonly cfg;
+    private readonly peers;
+    private timer;
+    private lastEvaluatedAt;
+    private evaluations;
+    private evaluating;
+    private ownWakeAt;
+    private keyExpiryLastCheckedAt;
+    private keyExpiryAvailable;
+    private keyExpirySoonest;
+    private keyExpiryAbsentLogged;
+    private keyExpiryInFlight;
+    private urgentEpisodes;
+    private suppressedBySleepGate;
+    private suppressedBySplitBrain;
+    private notNotifiedRetries;
+    private writePending;
+    private writeTimer;
+    constructor(deps: RopeHealthMonitorDeps, cfg?: Partial<RopeHealthMonitorConfig>);
+    private now;
+    private log;
+    private metric;
+    /** Start the monitor's OWN bounded evaluation loop (R-r2-2). */
+    start(): void;
+    /** Torn down at server shutdown (R-r2-2). Flushes any pending state write. */
+    stop(): void;
+    /** Is the evaluation loop currently armed? (e2e wiring-integrity read.) */
+    running(): boolean;
+    /**
+     * Own-machine wake note (SleepWakeDetector 'wake' — the ONE thing it can
+     * tell us). Post-wake, all snapshots are stale; urgent is suppressed until
+     * each (peer, kind) is re-observed post-wake — BOUNDED by wakeGraceMaxMs
+     * (P1-A7: false wake events must never become an open-ended veto).
+     */
+    noteOwnWake(atMs?: number): void;
+    /** One evaluation pass. Public for tests; the loop calls it on cadence. */
+    evaluate(): void;
+    private evaluateInner;
+    private settleHealthy;
+    private sleepGateSuppresses;
+    private raiseUrgent;
+    private settleNotified;
+    private markNotNotified;
+    private checkKeyExpiry;
+    private keyExpiryWarn;
+    /** The GET /mesh/rope-health read surface (content-scrubbed). */
+    status(): RopeHealthStatus;
+    /**
+     * The consolidated daily-digest section (≤3 sentences, clamped,
+     * machine-named), or null when everything is ok — the rope-health-digest
+     * job's content source. Deterministic; content-scrubbed by construction.
+     */
+    composeDigest(): string | null;
+    /** Record a digest emission (the digest job calls the route with ?digest=1). */
+    recordDigestEmission(): void;
+    private safeSnapshot;
+    private safePeers;
+    private safeHeartbeat;
+    private safeSplitBrainOpen;
+    private recordTransition;
+    private scheduleWrite;
+    private writeStateNow;
+    private loadState;
+}
+//# sourceMappingURL=RopeHealthMonitor.d.ts.map

package/dist/monitoring/RopeHealthMonitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RopeHealthMonitor.d.ts","sourceRoot":"","sources":["../../src/monitoring/RopeHealthMonitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAKH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,iCAAiC,CAAC;AAM7E,MAAM,MAAM,mBAAmB,GAAG,IAAI,GAAG,UAAU,GAAG,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE5F,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,kFAAkF;IAClF,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,MAAM,qBAAqB,GAC7B,YAAY,GACZ,eAAe,GACf,qBAAqB,GACrB,yBAAyB,GACzB,mBAAmB,GACnB,gBAAgB,GAChB,0BAA0B,GAC1B,2BAA2B,GAC3B,6BAA6B,GAC7B,oBAAoB,GACpB,iBAAiB,CAAC;AAEtB,MAAM,WAAW,qBAAqB;IACpC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,qBAAqB,EAAE,CAAC;IACxC,yDAAyD;IACzD,aAAa,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,SAAS,EAAE,MAAM,kBAAkB,EAAE,CAAC;IACtC;;;OAGG;IACH,iBAAiB,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACxD,6EAA6E;IAC7E,cAAc,EAAE,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;IAC/E,kFAAkF;IAClF,kBAAkB,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC;IACxD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACvD,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAC;IACtD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,+EAA+E;IAC/E,aAAa,EAAE,OAAO,CAAC;IACvB,4FAA4F;IAC5F,gBAAgB,EAAE,MAAM,CAAC;IACzB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gEAAgE;IAChE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;;;;;;;;;;OAYG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC;IACzB,yEAAyE;IACzE,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,oBAAoB,EAAE,uBAUlC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,SAAS,GAAE,MAA8C,GACxD,MAAM,CAIR;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,SAAS,GAAE,MAA8C,GACxD,OAAO,CAST;AAmBD,iFAAiF;AACjF,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,mBAAmB,CAAC;IAC/B,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,mBAAmB,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,SAAS,EAAE;QACT,+EAA+E;QAC/E,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QAC1E,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;IACF,8EAA8E;IAC9E,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE;QACR,cAAc,EAAE,MAAM,CAAC;QACvB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,sBAAsB,EAAE,MAAM,CAAC;QAC/B,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;CACH;AAUD,8EAA8E;AAC9E,wBAAgB,8BAA8B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAsBvE;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAwB;IAC1C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA0B;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuC;IAC7D,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,SAAS,CAAuB;IAGxC,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,gBAAgB,CAAgF;IACxG,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,iBAAiB,CAAS;IAGlC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,kBAAkB,CAAK;IAG/B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAA8C;gBAEpD,IAAI,EAAE,qBAAqB,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,uBAAuB,CAAC;IAM/E,OAAO,CAAC,GAAG;IAGX,OAAO,CAAC,GAAG;IAGX,OAAO,CAAC,MAAM;IASd,gEAAgE;IAChE,KAAK,IAAI,IAAI;IAOb,8EAA8E;IAC9E,IAAI,IAAI,IAAI;IAYZ,2EAA2E;IAC3E,OAAO,IAAI,OAAO;IAIlB;;;;;OAKG;IACH,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI;IAIhC,2EAA2E;IAC3E,QAAQ,IAAI,IAAI;IAahB,OAAO,CAAC,aAAa;IAqHrB,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,mBAAmB;IAO3B,OAAO,CAAC,WAAW;IAiCnB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,cAAc;IAuCtB,OAAO,CAAC,aAAa;IAQrB,iEAAiE;IACjE,MAAM,IAAI,gBAAgB;IAgD1B;;;;OAIG;IACH,aAAa,IAAI,MAAM,GAAG,IAAI;IA0B9B,gFAAgF;IAChF,oBAAoB,IAAI,IAAI;IAM5B,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,kBAAkB;IAW1B,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,aAAa;IA0BrB,OAAO,CAAC,SAAS;CAyBlB"}