aweshare 0.6.8 → 0.7.0

package/packages/producer-core/dist/health.js

@@ -1,5 +1,15 @@
+/** Consecutive upstream 5xx failures >= threshold -> 'unstable': report-only
+ * middle state — the alias stays listed and dispatch continues, consumers just
+ * see the warning and the errors keep passing through verbatim. */
+const UNSTABLE_THRESHOLD = 2;
 /** AUTH/QUOTA consecutive failures >= threshold -> degraded until a success. */
 const DEGRADE_THRESHOLD = 2;
+/** SERVER (upstream 5xx) consecutive failures >= threshold -> degraded, and
+ * only when `degradeOn5xx` opted in. Higher than AUTH/QUOTA's 2: a 401 is a
+ * deterministic account verdict, while 5xx pairs also come from transient
+ * upstream blips — degrading those would turn a momentary hiccup into a
+ * full 503 outage for every consumer on the alias. */
+const SERVER_DEGRADE_THRESHOLD = 3;
 // Recovery-probe backoff, awewarm-style: while degraded, each failed probe
 // doubles the wait before the next one, so an exhausted quota isn't pinged at
 // the base cadence for hours on end. Any success clears the degraded state
@@ -21,14 +31,20 @@ const PROBE_CAP_SCHEDULE = [
 ];
 /**
  * Per-backend health state. Degraded backends are reported in heartbeats and
- * the hub stops dispatching to their offerings until they recover. The
- * ladder is two rungs — healthy -> degraded, latched until a success — which
+ * the hub stops dispatching to their offerings until they recover. The ladder
+ * is two rungs — healthy -> degraded, latched until a success — which
  * for a relay is what awewarm's deeper ladder collapses to: its intermediate
  * rungs trade probe expense against recovery speed, and a relay's probes are
  * cheap pings, not quota-consuming activations. What does carry over from
  * awewarm is the escalating probe schedule above: the longer a backend has
  * been continuously degraded, the quieter its recovery probes become, down
  * to a daily last gasp that never fully gives up.
+ *
+ * Upstream 5xx gets its own two-step path: 2 consecutive SERVER failures
+ * enter the report-only 'unstable' state (alias stays listed and dispatched,
+ * consumers see the warning), and with `degradeOn5xx` opted in, 3
+ * consecutive degrade for real. AUTH/QUOTA verdicts skip the middle state —
+ * they are deterministic account facts, not symptoms.
  */
 export class HealthGate {
     now;
@@ -38,6 +54,13 @@ export class HealthGate {
     probeDelayMs = new Map();
     probeNotBefore = new Map();
     lastOk = new Map();
+    /** Backends sitting in the report-only middle state (2+ consecutive 5xx). */
+    unstable = new Set();
+    /** Opt-in enforcement step for upstream 5xx: 3+ consecutive SERVER failures
+     * degrade the backend (dispatch stops). Off by default — a 5xx is a symptom,
+     * not a verdict, so by default it only ever reports (unstable). Hot-reloaded
+     * by the config watcher; flipping it mid-flight affects the next failure. */
+    degradeOn5xx = false;
     /** The clock is injected so tests can walk the cap ladder without waiting. */
     constructor(now = Date.now) {
         this.now = now;
@@ -47,10 +70,11 @@ export class HealthGate {
             this.lastOk.set(backendId, this.now());
             this.consecutive.delete(backendId);
             const wasDegraded = this.degraded.delete(backendId);
+            const wasUnstable = this.unstable.delete(backendId);
             this.degradedSince.delete(backendId);
             this.probeDelayMs.delete(backendId);
             this.probeNotBefore.delete(backendId);
-            return wasDegraded ? 'recovered' : null;
+            return wasDegraded || wasUnstable ? 'recovered' : null;
         }
         if (reason === undefined || reason === 'CANCELLED')
             return null;
@@ -64,12 +88,23 @@ export class HealthGate {
         const count = current?.reason === reason ? current.count + 1 : 1;
         this.consecutive.set(backendId, { reason, count });
         let transition = null;
-        if ((reason === 'AUTH' || reason === 'QUOTA') && count >= DEGRADE_THRESHOLD) {
+        const degradeNow = ((reason === 'AUTH' || reason === 'QUOTA') && count >= DEGRADE_THRESHOLD) ||
+            (reason === 'SERVER' && this.degradeOn5xx && count >= SERVER_DEGRADE_THRESHOLD);
+        if (degradeNow) {
             if (!wasDegraded) {
                 transition = 'degraded';
                 this.degradedSince.set(backendId, this.now());
             }
             this.degraded.set(backendId, { state: 'degraded', reason });
+            this.unstable.delete(backendId);
+        }
+        else if (reason === 'SERVER' && count >= UNSTABLE_THRESHOLD && !wasDegraded) {
+            // first crossing of the 5xx threshold: report only — dispatch keeps
+            // flowing so a transient upstream blip never becomes a full outage
+            if (!this.unstable.has(backendId)) {
+                this.unstable.add(backendId);
+                transition = 'unstable';
+            }
         }
         if (wasDegraded) {
             const since = this.degradedSince.get(backendId) ?? this.now();
@@ -90,10 +125,17 @@ export class HealthGate {
     isDegraded(backendId) {
         return this.degraded.has(backendId);
     }
+    /** The report-only middle state — listed and dispatched, just flagged. */
+    isUnstable(backendId) {
+        return this.unstable.has(backendId);
+    }
     statusMap() {
         const out = {};
         for (const [id, status] of this.degraded)
             out[id] = status;
+        for (const id of this.unstable)
+            if (out[id] === undefined)
+                out[id] = { state: 'unstable' };
         return out;
     }
     /** Epoch ms of the backend's last success (request or recovery probe); null
@@ -104,8 +146,9 @@ export class HealthGate {
     }
     /** Heartbeat payload covering every backend, not just the degraded ones:
      * one 'ok' entry carrying lastOkAt per healthy backend, the degraded entry
-     * as statusMap would emit it. Old hubs ignore the extra 'ok' entries (their
-     * status check only looks for state === 'degraded'). */
+     * as statusMap would emit it, and 'unstable' for 2+ consecutive-5xx
+     * backends. Old hubs ignore the extra entries (their status check only
+     * looks for state === 'degraded', so unstable renders as online there). */
     snapshot(ids) {
         const out = {};
         for (const id of ids) {
@@ -115,7 +158,10 @@ export class HealthGate {
                 continue;
             }
             const at = this.lastOk.get(id);
-            out[id] = at === undefined ? { state: 'ok' } : { state: 'ok', lastOkAt: at };
+            const lastOkAt = at === undefined ? {} : { lastOkAt: at };
+            out[id] = this.unstable.has(id)
+                ? { state: 'unstable', ...lastOkAt }
+                : { state: 'ok', ...lastOkAt };
         }
         return out;
     }
@@ -126,11 +172,18 @@ export function logHealthTransition(log, backendId, transition, detail) {
     if (transition === 'degraded') {
         log.warn(`backend ${backendId} degraded${detail ? ` (${detail})` : ''}`);
     }
+    else if (transition === 'unstable') {
+        log.warn(`backend ${backendId} unstable — upstream failing, still dispatching${detail ? ` (${detail})` : ''}`);
+    }
     else if (transition === 'recovered') {
         log.info(`backend ${backendId} recovered${detail ? ` — ${detail}` : ''}`);
     }
 }
-/** Map an upstream HTTP status to a failure class (ok responses pass null). */
+/** Map an upstream HTTP status to a failure class (ok responses pass null).
+ * SERVER (5xx) means the upstream itself is failing — a symptom that reports
+ * early (unstable) and, opted-in, degrades when persistent. HTTP (other 4xx)
+ * is request-shaped trouble — one consumer's malformed request — and never
+ * touches the ladder. */
 export function classifyStatus(status) {
     if (status < 400)
         return null;
@@ -138,6 +191,8 @@ export function classifyStatus(status) {
         return 'AUTH';
     if (status === 402 || status === 429)
         return 'QUOTA';
+    if (status >= 500)
+        return 'SERVER';
     return 'HTTP';
 }
 //# sourceMappingURL=health.js.map

package/packages/producer-core/dist/health.js.map

@@ -1 +1 @@
-{"version":3,"file":"health.js","sourceRoot":"","sources":["../src/health.ts"],"names":[],"mappings":"AASA,gFAAgF;AAChF,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAE3B,2EAA2E;AAC3E,8EAA8E;AAC9E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,yDAAyD;AACzD,4EAA4E;AAC5E,uEAAuE;AACvE,8DAA8D;AAC9D,yEAAyE;AACzE,MAAM,qBAAqB,GAAG,MAAM,CAAA;AACpC,MAAM,qBAAqB,GAAG,EAAE,GAAG,SAAS,CAAA;AAC5C,MAAM,kBAAkB,GAAsD;IAC5E,EAAE,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,CAAC,GAAG,MAAM,EAAE;IAC9C,EAAE,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE;IAC7C,EAAE,OAAO,EAAE,GAAG,GAAG,SAAS,EAAE,KAAK,EAAE,CAAC,GAAG,SAAS,EAAE;IAClD,EAAE,OAAO,EAAE,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,qBAAqB,EAAE;CACpE,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,UAAU;IASQ;IARrB,WAAW,GAAG,IAAI,GAAG,EAAoD,CAAA;IACzE,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAA;IAC3C,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IACzC,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC1C,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE1C,8EAA8E;IAC9E,YAA6B,MAAoB,IAAI,CAAC,GAAG;QAA5B,QAAG,GAAH,GAAG,CAAyB;IAAG,CAAC;IAE7D,MAAM,CAAC,SAAiB,EAAE,EAAW,EAAE,MAAsB;QAC3D,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YAClC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACnD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACpC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACnC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACrC,OAAO,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAA;QACzC,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,WAAW;YAAE,OAAO,IAAI,CAAA;QAC/D,yEAAyE;QACzE,sEAAsE;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,iCAAiC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC/C,MAAM,KAAK,GAAG,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,UAAU,GAAqB,IAAI,CAAA;QACvC,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,OAAO,CAAC,IAAI,KAAK,IAAI,iBAAiB,EAAE,CAAC;YAC5E,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,UAAU,GAAG,UAAU,CAAA;gBACvB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;YAC/C,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;QACD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAA;YAC7D,yEAAyE;YACzE,MAAM,GAAG,GACP,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK;gBAC3E,qBAAqB,CAAA;YACvB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,qBAAqB,GAAG,CAAC,CAAC,GAAG,CAAC,EACnE,GAAG,CACJ,CAAA;YACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAA;QACxD,CAAC;QACD,OAAO,UAAU,CAAA;IACnB,CAAC;IAED;uEACmE;IACnE,gBAAgB,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAChD,CAAC;IAED,UAAU,CAAC,SAAiB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACrC,CAAC;IAED,SAAS;QACP,MAAM,GAAG,GAAkC,EAAE,CAAA;QAC7C,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAA;QAC1D,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;wEAEoE;IACpE,QAAQ,CAAC,SAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAA;IAC3C,CAAC;IAED;;;4DAGwD;IACxD,QAAQ,CAAC,GAAqB;QAC5B,MAAM,GAAG,GAAkC,EAAE,CAAA;QAC7C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAA;gBAClB,SAAQ;YACV,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC9B,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAA;QAC9E,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF;AAED;uEACuE;AACvE,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,SAAiB,EACjB,UAA4B,EAC5B,MAAe;IAEf,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,WAAW,SAAS,YAAY,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC1E,CAAC;SAAM,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,WAAW,SAAS,aAAa,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC3E,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAA;IAC7B,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,MAAM,CAAA;IACnD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,OAAO,CAAA;IACpD,OAAO,MAAM,CAAA;AACf,CAAC"}
+{"version":3,"file":"health.js","sourceRoot":"","sources":["../src/health.ts"],"names":[],"mappings":"AAKA;;mEAEmE;AACnE,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAM5B,gFAAgF;AAChF,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAE3B;;;;sDAIsD;AACtD,MAAM,wBAAwB,GAAG,CAAC,CAAA;AAElC,2EAA2E;AAC3E,8EAA8E;AAC9E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,yDAAyD;AACzD,4EAA4E;AAC5E,uEAAuE;AACvE,8DAA8D;AAC9D,yEAAyE;AACzE,MAAM,qBAAqB,GAAG,MAAM,CAAA;AACpC,MAAM,qBAAqB,GAAG,EAAE,GAAG,SAAS,CAAA;AAC5C,MAAM,kBAAkB,GAAsD;IAC5E,EAAE,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,CAAC,GAAG,MAAM,EAAE;IAC9C,EAAE,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE;IAC7C,EAAE,OAAO,EAAE,GAAG,GAAG,SAAS,EAAE,KAAK,EAAE,CAAC,GAAG,SAAS,EAAE;IAClD,EAAE,OAAO,EAAE,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,qBAAqB,EAAE;CACpE,CAAA;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,UAAU;IAiBQ;IAhBrB,WAAW,GAAG,IAAI,GAAG,EAAoD,CAAA;IACzE,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAA;IAC3C,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IACzC,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC1C,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC1C,6EAA6E;IACrE,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAEpC;;;iFAG6E;IAC7E,YAAY,GAAG,KAAK,CAAA;IAEpB,8EAA8E;IAC9E,YAA6B,MAAoB,IAAI,CAAC,GAAG;QAA5B,QAAG,GAAH,GAAG,CAAyB;IAAG,CAAC;IAE7D,MAAM,CAAC,SAAiB,EAAE,EAAW,EAAE,MAAsB;QAC3D,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YAClC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACnD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACnD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACpC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACnC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACrC,OAAO,WAAW,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAA;QACxD,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,WAAW;YAAE,OAAO,IAAI,CAAA;QAC/D,yEAAyE;QACzE,sEAAsE;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,iCAAiC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC/C,MAAM,KAAK,GAAG,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,UAAU,GAAqB,IAAI,CAAA;QACvC,MAAM,UAAU,GACd,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,OAAO,CAAC,IAAI,KAAK,IAAI,iBAAiB,CAAC;YACzE,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,YAAY,IAAI,KAAK,IAAI,wBAAwB,CAAC,CAAA;QACjF,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,UAAU,GAAG,UAAU,CAAA;gBACvB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;YAC/C,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAA;YAC3D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACjC,CAAC;aAAM,IAAI,MAAM,KAAK,QAAQ,IAAI,KAAK,IAAI,kBAAkB,IAAI,CAAC,WAAW,EAAE,CAAC;YAC9E,oEAAoE;YACpE,mEAAmE;YACnE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBAC5B,UAAU,GAAG,UAAU,CAAA;YACzB,CAAC;QACH,CAAC;QACD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAA;YAC7D,yEAAyE;YACzE,MAAM,GAAG,GACP,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK;gBAC3E,qBAAqB,CAAA;YACvB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,qBAAqB,GAAG,CAAC,CAAC,GAAG,CAAC,EACnE,GAAG,CACJ,CAAA;YACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAA;QACxD,CAAC;QACD,OAAO,UAAU,CAAA;IACnB,CAAC;IAED;uEACmE;IACnE,gBAAgB,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAChD,CAAC;IAED,UAAU,CAAC,SAAiB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACrC,CAAC;IAED,0EAA0E;IAC1E,UAAU,CAAC,SAAiB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACrC,CAAC;IAED,SAAS;QACP,MAAM,GAAG,GAAkC,EAAE,CAAA;QAC7C,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAA;QAC1D,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,GAAG,CAAC,EAAE,CAAC,KAAK,SAAS;gBAAE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,CAAA;QAC1F,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;wEAEoE;IACpE,QAAQ,CAAC,SAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAA;IAC3C,CAAC;IAED;;;;+EAI2E;IAC3E,QAAQ,CAAC,GAAqB;QAC5B,MAAM,GAAG,GAAkC,EAAE,CAAA;QAC7C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAA;gBAClB,SAAQ;YACV,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC9B,MAAM,QAAQ,GAAG,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAA;YACzD,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,QAAQ,EAAE;gBACpC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,CAAA;QAClC,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF;AAED;uEACuE;AACvE,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,SAAiB,EACjB,UAA4B,EAC5B,MAAe;IAEf,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,WAAW,SAAS,YAAY,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC1E,CAAC;SAAM,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QACrC,GAAG,CAAC,IAAI,CACN,WAAW,SAAS,kDAAkD,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACrG,CAAA;IACH,CAAC;SAAM,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,WAAW,SAAS,aAAa,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC3E,CAAC;AACH,CAAC;AAED;;;;yBAIyB;AACzB,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAA;IAC7B,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,MAAM,CAAA;IACnD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,OAAO,CAAA;IACpD,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,QAAQ,CAAA;IAClC,OAAO,MAAM,CAAA;AACf,CAAC"}

package/packages/protocol/dist/format.d.ts

@@ -0,0 +1,6 @@
+/** Human-facing number rendering, kept alongside the time formatters: the
+ * wire, the SQLite rows and `--json` stay raw integers — only what a human
+ * reads is grouped. */
+/** Thousands grouping with a fixed ',' — locale-independent so CLI output
+ * stays stable and diffable ('1000000' → '1,000,000'). Integers only. */
+export declare function fmtTokens(n: number): string;

package/packages/protocol/dist/format.js

@@ -0,0 +1,9 @@
+/** Human-facing number rendering, kept alongside the time formatters: the
+ * wire, the SQLite rows and `--json` stay raw integers — only what a human
+ * reads is grouped. */
+/** Thousands grouping with a fixed ',' — locale-independent so CLI output
+ * stays stable and diffable ('1000000' → '1,000,000'). Integers only. */
+export function fmtTokens(n) {
+    return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
+}
+//# sourceMappingURL=format.js.map

package/packages/protocol/dist/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAAA;;uBAEuB;AAEvB;yEACyE;AACzE,MAAM,UAAU,SAAS,CAAC,CAAS;IACjC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAA;AACxD,CAAC"}

package/packages/protocol/dist/index.d.ts

@@ -1,5 +1,6 @@
 export * from './alias.js';
 export * from './errors.js';
+export * from './format.js';
 export * from './frames.js';
 export * from './messages.js';
 export * from './models.js';

package/packages/protocol/dist/index.js

@@ -1,5 +1,6 @@
 export * from './alias.js';
 export * from './errors.js';
+export * from './format.js';
 export * from './frames.js';
 export * from './messages.js';
 export * from './models.js';

package/packages/protocol/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,eAAe,CAAA;AAC7B,cAAc,aAAa,CAAA;AAC3B,cAAc,WAAW,CAAA;AACzB,cAAc,aAAa,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,eAAe,CAAA;AAC7B,cAAc,aAAa,CAAA;AAC3B,cAAc,WAAW,CAAA;AACzB,cAAc,aAAa,CAAA"}

package/packages/protocol/dist/messages.d.ts

@@ -25,14 +25,18 @@ export interface Offering {
     /** Shared per-alias token budget per UTC day; hub default 1000000, 0 = unlimited. */
     dailyTokens?: number;
 }
-export type BackendState = 'ok' | 'degraded';
+export type BackendState = 'ok' | 'unstable' | 'degraded';
 export interface BackendStatus {
     state: BackendState;
-    /** Failure class when degraded: AUTH | QUOTA | HTTP | NETWORK | CANCELLED */
+    /** Failure class when degraded: AUTH | QUOTA | SERVER | HTTP | NETWORK | CANCELLED.
+     * 'unstable' entries carry no reason — they are always 2+ consecutive
+     * upstream 5xx (SERVER). Old hubs only check for state === 'degraded', so
+     * the extra 'unstable' state degrades gracefully to online there. */
     reason?: string;
     /** Epoch ms of the last successful upstream interaction (a relayed request
      * or a recovery probe); absent when the backend has never succeeded. Only
-     * meaningful on 'ok' entries — the hub's freshness evidence for consumers. */
+     * meaningful on non-degraded entries — the hub's freshness evidence for
+     * consumers. */
     lastOkAt?: number;
 }
 export interface RegisterMessage {

package/packages/protocol/dist/messages.js.map

@@ -1 +1 @@
-{"version":3,"file":"messages.js","sourceRoot":"","sources":["../src/messages.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAA;AAI7B,MAAM,CAAC,MAAM,SAAS,GAAwB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;AAElF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAA6B;IACvD,MAAM,EAAE,aAAa;IACrB,SAAS,EAAE,kBAAkB;IAC7B,SAAS,EAAE,WAAW;CACvB,CAAA;AAED,gFAAgF;AAChF,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAQ,eAA0C,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAA;AAC1E,CAAC;AAqHD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAS;IAClC,UAAU;IACV,WAAW;IACX,eAAe;IACf,cAAc;IACd,gBAAgB;IAChB,OAAO;IACP,YAAY;IACZ,eAAe;IACf,aAAa;IACb,gBAAgB;CACjB,CAAC,CAAA;AAEF,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,IAAI,MAAe,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IACpD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;IACnD,CAAC;IACD,MAAM,IAAI,GAAI,MAA6B,CAAC,IAAI,CAAA;IAChD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,MAAwB,CAAA;AACjC,CAAC"}
+{"version":3,"file":"messages.js","sourceRoot":"","sources":["../src/messages.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAA;AAI7B,MAAM,CAAC,MAAM,SAAS,GAAwB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;AAElF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAA6B;IACvD,MAAM,EAAE,aAAa;IACrB,SAAS,EAAE,kBAAkB;IAC7B,SAAS,EAAE,WAAW;CACvB,CAAA;AAED,gFAAgF;AAChF,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAQ,eAA0C,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAA;AAC1E,CAAC;AAyHD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAS;IAClC,UAAU;IACV,WAAW;IACX,eAAe;IACf,cAAc;IACd,gBAAgB;IAChB,OAAO;IACP,YAAY;IACZ,eAAe;IACf,aAAa;IACb,gBAAgB;CACjB,CAAC,CAAA;AAEF,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,IAAI,MAAe,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IACpD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;IACnD,CAAC;IACD,MAAM,IAAI,GAAI,MAA6B,CAAC,IAAI,CAAA;IAChD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,MAAwB,CAAA;AACjC,CAAC"}