instar 0.28.76 → 0.28.78

package/dist/monitoring/delivery-failure-sentinel.d.ts

@@ -0,0 +1,177 @@
+/**
+ * DeliveryFailureSentinel — Layer 3 of telegram-delivery-robustness.
+ *
+ * Spec: docs/specs/telegram-delivery-robustness.md § 4 Layer 3.
+ *
+ * Reads the SQLite queue created by Layer 2 (`pending-relay-store`),
+ * runs the recovery state machine, and is feature-flag-gated default-OFF.
+ *
+ * State machine (per row):
+ *
+ *   queued ──claim──► claimed ──/whoami──► (mismatch → retry)
+ *                                  │
+ *                                  └──tone-gate──► (422 → delivered-tone-gated)
+ *                                          │
+ *                                          └──/telegram/reply──► (200 → delivered-recovered
+ *                                                                  408 → delivered-ambiguous
+ *                                                                  5xx/conn-refused → retry)
+ *                                          │
+ *                                          └──ttl/attempts exhausted ──► escalated
+ *
+ * The sentinel never overrides the tone-gate authority (§5 signal-vs-
+ * authority compliance). On 422 it finalizes the entry as
+ * `delivered-tone-gated` and emits the fixed-template meta-notice on the
+ * original topic.
+ *
+ * Lifecycle: `start()` registers the watchdog tick + (when wsManager is
+ * provided) an SSE handler for `delivery_failed` events. `stop()` clears
+ * timers and detaches from the event stream. `tick()` is exposed for
+ * tests so the recovery loop can be driven without waiting for the
+ * 5-minute backstop interval.
+ *
+ * **Default-OFF.** The constructor enforces nothing; the AgentServer
+ * gates instantiation on `config.monitoring.deliveryFailureSentinel.enabled`.
+ * Layer 1 + Layer 2 ship unconditionally; Layer 3 is opt-in until the
+ * canary criteria in spec § 3i are met.
+ */
+import { EventEmitter } from 'node:events';
+import type { PendingRelayStore } from '../messaging/pending-relay-store.js';
+import { WhoamiCache } from '../messaging/whoami-cache.js';
+import type { MessagingToneGate } from '../core/MessagingToneGate.js';
+export interface SentinelConfig {
+    /** Backstop watchdog tick interval. Default 5 minutes (spec §3a). */
+    watchdogIntervalMs?: number;
+    /** Per-entry lease duration. Default 90s (spec §3b). */
+    leaseDurationMs?: number;
+    /** Per-topic delivery rate cap. Default 30s (spec §3c). */
+    perTopicRateMs?: number;
+    /** Max parallel cross-topic recoveries. Default 4 (spec §3c). */
+    maxConcurrent?: number;
+    /** Stampede threshold — entries on same topic before digest fires. Default 5 (spec §3c). */
+    stampedeThreshold?: number;
+    /**
+     * Circuit breaker — N consecutive escalation failures within window
+     * trips the breaker. Defaults: N=5, window=1h (spec §3f).
+     */
+    circuitBreakerCount?: number;
+    circuitBreakerWindowMs?: number;
+    /**
+     * Restore-purge threshold — entries older than this at startup are
+     * dropped, not recovered (spec §3h). Default 5 minutes.
+     */
+    restorePurgeAgeMs?: number;
+}
+export interface SentinelDeps {
+    store: PendingRelayStore;
+    /** Path to .instar/config.json — used for whoami-cache mtime invalidation. */
+    configPath: string;
+    /** Live config accessor — re-read on every tick (spec §3d step 1). */
+    readConfig: () => {
+        port: number;
+        authToken: string;
+        agentId: string;
+    };
+    /** This server's bootId (spec §3b). Required — sentinel exits if absent. */
+    bootId: string;
+    /** Tone gate authority for §3d step 3. Null = no gate configured. */
+    toneGate: MessagingToneGate | null;
+    /**
+     * Subscribe to in-process `delivery_failed` events. Returns an
+     * unsubscribe handle. The wsManager fan-out emits these events; the
+     * sentinel reacts in <1s rather than waiting for the 5-min watchdog.
+     */
+    subscribeFailureEvents?: (listener: (event: {
+        delivery_id: string;
+        topic_id: number;
+        agentId: string;
+    }) => void) => () => void;
+    /** Override clock (tests). */
+    now?: () => number;
+    /** Override fetch (tests). Default uses node:http to localhost. */
+    postReply?: (port: number, token: string, agentId: string, topicId: number, text: string, deliveryId: string, isSystem?: boolean) => Promise<{
+        status: number;
+        body: string;
+    }>;
+    /** Whoami cache — defaults to a fresh instance with default deps. */
+    whoamiCache?: WhoamiCache;
+}
+export interface SentinelEvents {
+    'sentinel:started': [];
+    'sentinel:stopped': [];
+    'sentinel:tick-complete': [{
+        processed: number;
+        recovered: number;
+        escalated: number;
+    }];
+    'sentinel:recovered': [{
+        delivery_id: string;
+        topic_id: number;
+    }];
+    'sentinel:tone-gated': [{
+        delivery_id: string;
+        topic_id: number;
+        rule?: string;
+    }];
+    'sentinel:escalated': [{
+        delivery_id: string;
+        topic_id: number;
+        category: string;
+    }];
+    'sentinel:circuit-breaker-tripped': [{
+        consecutiveFailures: number;
+    }];
+    'sentinel:circuit-breaker-resumed': [];
+}
+export declare class DeliveryFailureSentinel extends EventEmitter {
+    private readonly cfg;
+    private readonly deps;
+    private watchdog;
+    private unsubscribe;
+    private running;
+    private suspended;
+    private escalationFailures;
+    private inFlight;
+    private lastTopicDelivery;
+    /** Templates verified at start(). False disables escalation path. */
+    private templatesValid;
+    /** Restored on first start(); used to keep restore-purge a one-shot. */
+    private restorePurged;
+    constructor(deps: SentinelDeps, config?: SentinelConfig);
+    /**
+     * Start the sentinel. Verifies template integrity, performs a one-shot
+     * restore-purge of stale rows, registers the SSE listener (if available),
+     * and arms the watchdog tick. Idempotent — calling start() while running
+     * is a no-op.
+     */
+    start(): Promise<void>;
+    /** Stop the sentinel, clearing the watchdog and detaching listeners. */
+    stop(): Promise<void>;
+    /** True iff the sentinel's circuit breaker is currently tripped. */
+    isSuspended(): boolean;
+    /**
+     * Run one recovery tick. Public so tests can drive recovery without
+     * waiting for the watchdog interval.
+     *
+     * Returns aggregate counters useful for assertions.
+     */
+    tick(): Promise<{
+        processed: number;
+        recovered: number;
+        escalated: number;
+    }>;
+    private selectClaimable;
+    private isLeaseStale;
+    private processRow;
+    private finalizeToneGated;
+    private handlePolicyDecision;
+    private escalate;
+    private recordEscalationFailure;
+    private lastConfigHash;
+    private maybeResume;
+    private computeConfigAuthHash;
+    /** Manual resume — used by `instar sentinel resume` CLI handler. */
+    resume(): void;
+    private handleStampede;
+    private purgeStaleRows;
+}
+//# sourceMappingURL=delivery-failure-sentinel.d.ts.map

package/dist/monitoring/delivery-failure-sentinel.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delivery-failure-sentinel.d.ts","sourceRoot":"","sources":["../../src/monitoring/delivery-failure-sentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAK3C,OAAO,KAAK,EAAE,iBAAiB,EAAmB,MAAM,qCAAqC,CAAC;AAG9F,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AActE,MAAM,WAAW,cAAc;IAC7B,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,wDAAwD;IACxD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,2DAA2D;IAC3D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4FAA4F;IAC5F,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,iBAAiB,CAAC;IACzB,8EAA8E;IAC9E,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,UAAU,EAAE,MAAM;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACvE,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACnC;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,CACvB,QAAQ,EAAE,CAAC,KAAK,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,KAClF,MAAM,IAAI,CAAC;IAChB,8BAA8B;IAC9B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,mEAAmE;IACnE,SAAS,CAAC,EAAE,CACV,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,OAAO,KACf,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,qEAAqE;IACrE,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAeD,MAAM,WAAW,cAAc;IAC7B,kBAAkB,EAAE,EAAE,CAAC;IACvB,kBAAkB,EAAE,EAAE,CAAC;IACvB,wBAAwB,EAAE,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxF,oBAAoB,EAAE,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClE,qBAAqB,EAAE,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClF,oBAAoB,EAAE,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpF,kCAAkC,EAAE,CAAC;QAAE,mBAAmB,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtE,kCAAkC,EAAE,EAAE,CAAC;CACxC;AAED,qBAAa,uBAAwB,SAAQ,YAAY;IACvD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA2B;IAC/C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAEnB;IACF,OAAO,CAAC,QAAQ,CAA+C;IAC/D,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,kBAAkB,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,iBAAiB,CAA6B;IACtD,qEAAqE;IACrE,OAAO,CAAC,cAAc,CAAQ;IAC9B,wEAAwE;IACxE,OAAO,CAAC,aAAa,CAAS;gBAElB,IAAI,EAAE,YAAY,EAAE,MAAM,GAAE,cAAmB;IAyB3D;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAwD5B,wEAAwE;IAClE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB3B,oEAAoE;IACpE,WAAW,IAAI,OAAO;IAItB;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IA+DlF,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;YAYN,UAAU;YAwGV,iBAAiB;YAkBjB,oBAAoB;YAkCpB,QAAQ;IAqCtB,OAAO,CAAC,uBAAuB;IAwB/B,OAAO,CAAC,cAAc,CAAuB;IAE7C,OAAO,CAAC,WAAW;IAYnB,OAAO,CAAC,qBAAqB;IAW7B,oEAAoE;IACpE,MAAM,IAAI,IAAI;YAMA,cAAc;IAsB5B,OAAO,CAAC,cAAc;CAWvB"}