instar 1.3.639 → 1.3.641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +75 -19
- package/dist/commands/server.js.map +1 -1
- package/dist/core/IntelligenceRouter.d.ts +39 -1
- package/dist/core/IntelligenceRouter.d.ts.map +1 -1
- package/dist/core/IntelligenceRouter.js +74 -4
- package/dist/core/IntelligenceRouter.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +13 -1
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/types.d.ts +12 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/DegradationReporter.d.ts +42 -0
- package/dist/monitoring/DegradationReporter.d.ts.map +1 -1
- package/dist/monitoring/DegradationReporter.js +117 -1
- package/dist/monitoring/DegradationReporter.js.map +1 -1
- package/dist/monitoring/LlmQueue.d.ts +17 -0
- package/dist/monitoring/LlmQueue.d.ts.map +1 -1
- package/dist/monitoring/LlmQueue.js +36 -0
- package/dist/monitoring/LlmQueue.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/upgrades/1.3.640.md +46 -0
- package/upgrades/1.3.641.md +51 -0
- package/upgrades/side-effects/resilient-degradation-ladder-increment-2.md +97 -0
- package/upgrades/side-effects/resilient-degradation-ladder-increment-3.md +108 -0
|
@@ -53,7 +53,7 @@ function monotonicNow() {
|
|
|
53
53
|
return performance.now();
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
|
-
catch { /*
|
|
56
|
+
catch { /* @silent-fallback-ok: environment-capability probe, not a degradation — a runtime without the `performance` API falls back to `Date.now()`; a clock-source probe is low-risk and not a reportable degradation. */ }
|
|
57
57
|
return Date.now();
|
|
58
58
|
}
|
|
59
59
|
// Durable RestartPending queue caps (per §A5). When `_setRestartPending(true)`
|
|
@@ -89,6 +89,20 @@ export class DegradationReporter {
|
|
|
89
89
|
healers = new Map();
|
|
90
90
|
// Dedup: track last alert time per feature to avoid spamming Telegram
|
|
91
91
|
lastAlertTime = new Map();
|
|
92
|
+
// ── Never-silent degradation lifecycle (Resilient Degradation Ladder §4) ──────
|
|
93
|
+
// Tracks open heuristic-fallback degradations so one can NEVER silently persist
|
|
94
|
+
// indefinitely (the operator's principle). Designed to NOT repeat the 2026-06-21
|
|
95
|
+
// wedge: bounded (MAX_OPEN), O(1) per open/resolve (no full-map serialize), the
|
|
96
|
+
// escalation sweep NEVER calls report()/reportEvent()/gateHealthAlert (it surfaces
|
|
97
|
+
// via telegramSender directly), and it is liveness-gated (a run-once/idle component
|
|
98
|
+
// auto-closes via TTL rather than escalating a false alarm).
|
|
99
|
+
openDegradations = new Map();
|
|
100
|
+
neverSilentEnabled = false;
|
|
101
|
+
neverSilentEscalateMs = 15 * 60_000; // 15m before a persistent open escalates
|
|
102
|
+
neverSilentTtlMs = 30 * 60_000; // idle/run-once auto-close window
|
|
103
|
+
neverSilentMaxOpen = 500; // hard cap on the open map (anti-wedge)
|
|
104
|
+
/** Injectable clock for the lifecycle (testing). Default Date.now. */
|
|
105
|
+
neverSilentNow = () => Date.now();
|
|
92
106
|
// F-3 additions ─────────────────────────────────────────────
|
|
93
107
|
// The Remediator dispatch hook (F-8 wires the consumer). When set, the
|
|
94
108
|
// legacy `.report(...)` and the new `.reportStructured(...)` both route
|
|
@@ -154,6 +168,108 @@ export class DegradationReporter {
|
|
|
154
168
|
registerHealer(feature, healer) {
|
|
155
169
|
this.healers.set(feature, healer);
|
|
156
170
|
}
|
|
171
|
+
// ── Never-silent degradation lifecycle (Resilient Degradation Ladder §4) ──────
|
|
172
|
+
/** Configure (and enable) never-silent tracking. Called once at server startup with the
|
|
173
|
+
* dev-gate-resolved `enabled`. `now` is injectable for tests. */
|
|
174
|
+
configureNeverSilent(opts) {
|
|
175
|
+
this.neverSilentEnabled = opts.enabled === true;
|
|
176
|
+
if (opts.escalateMs && opts.escalateMs > 0)
|
|
177
|
+
this.neverSilentEscalateMs = opts.escalateMs;
|
|
178
|
+
if (opts.ttlMs && opts.ttlMs > 0)
|
|
179
|
+
this.neverSilentTtlMs = opts.ttlMs;
|
|
180
|
+
if (opts.maxOpen && opts.maxOpen > 0)
|
|
181
|
+
this.neverSilentMaxOpen = opts.maxOpen;
|
|
182
|
+
if (opts.now)
|
|
183
|
+
this.neverSilentNow = opts.now;
|
|
184
|
+
this.openDegradations.clear(); // fresh config = fresh state (startup is empty; gives test isolation)
|
|
185
|
+
}
|
|
186
|
+
nsKey(component, framework) {
|
|
187
|
+
return component + '::' + framework;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* A heuristic fallback fired for (component, framework) — open a degradation, or if one is
|
|
191
|
+
* already open, increment its retry-attempts (the liveness signal: it tried again and STILL
|
|
192
|
+
* failed → stuck). Bounded by MAX_OPEN (oldest evicted), O(1). No-op when disabled.
|
|
193
|
+
*/
|
|
194
|
+
openDegradation(component, framework) {
|
|
195
|
+
if (!this.neverSilentEnabled)
|
|
196
|
+
return;
|
|
197
|
+
const k = this.nsKey(component, framework);
|
|
198
|
+
const existing = this.openDegradations.get(k);
|
|
199
|
+
if (existing) {
|
|
200
|
+
existing.retryAttempts++; // a re-attempt that also fell to heuristic = it's genuinely stuck
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
if (this.openDegradations.size >= this.neverSilentMaxOpen) {
|
|
204
|
+
const oldest = this.openDegradations.keys().next().value; // Map preserves insertion order
|
|
205
|
+
if (oldest !== undefined)
|
|
206
|
+
this.openDegradations.delete(oldest);
|
|
207
|
+
}
|
|
208
|
+
this.openDegradations.set(k, {
|
|
209
|
+
component, framework, openedAt: this.neverSilentNow(), retryAttempts: 0, lastEscalatedAt: null,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* A real-LLM call for (component, framework) SUCCEEDED — auto-resolve the open degradation
|
|
214
|
+
* (the never-silent recovery). Returns the degraded duration (ms) or null if none was open.
|
|
215
|
+
* O(1). No-op when disabled.
|
|
216
|
+
*/
|
|
217
|
+
resolveDegradation(component, framework) {
|
|
218
|
+
if (!this.neverSilentEnabled)
|
|
219
|
+
return null;
|
|
220
|
+
const k = this.nsKey(component, framework);
|
|
221
|
+
const d = this.openDegradations.get(k);
|
|
222
|
+
if (!d)
|
|
223
|
+
return null;
|
|
224
|
+
this.openDegradations.delete(k);
|
|
225
|
+
return this.neverSilentNow() - d.openedAt;
|
|
226
|
+
}
|
|
227
|
+
/** Count of currently-open degradations (observability). */
|
|
228
|
+
openDegradationCount() {
|
|
229
|
+
return this.openDegradations.size;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Level-triggered sweep (timer-driven): escalate a degradation OPEN past escalateMs that has
|
|
233
|
+
* genuinely retried (≥1), and TTL-auto-close an idle/run-once one (0 retries past ttlMs). Deduped
|
|
234
|
+
* per episode (re-escalates only after another full window). REENTRANCY-SAFE: it NEVER calls
|
|
235
|
+
* report()/reportEvent()/gateHealthAlert — it surfaces via telegramSender directly (the 2026-06-21
|
|
236
|
+
* wedge was that recursion). No-op when disabled.
|
|
237
|
+
*/
|
|
238
|
+
sweepOpenDegradations() {
|
|
239
|
+
if (!this.neverSilentEnabled)
|
|
240
|
+
return;
|
|
241
|
+
const now = this.neverSilentNow();
|
|
242
|
+
for (const [k, d] of this.openDegradations) {
|
|
243
|
+
const age = now - d.openedAt;
|
|
244
|
+
if (d.retryAttempts === 0) {
|
|
245
|
+
// Idle / run-once: degraded once, never retried → done, not stuck. Auto-close at the TTL.
|
|
246
|
+
if (age > this.neverSilentTtlMs)
|
|
247
|
+
this.openDegradations.delete(k);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
if (age > this.neverSilentEscalateMs &&
|
|
251
|
+
(d.lastEscalatedAt === null || now - d.lastEscalatedAt > this.neverSilentEscalateMs)) {
|
|
252
|
+
d.lastEscalatedAt = now;
|
|
253
|
+
this.escalatePersistentDegradation(d, age);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/** Surface a persistent-degradation attention item DIRECTLY (fixed template, no toneGate, no
|
|
258
|
+
* report — reentrancy-safe per §4). Best-effort; never throws into the sweep. */
|
|
259
|
+
escalatePersistentDegradation(d, ageMs) {
|
|
260
|
+
try {
|
|
261
|
+
if (!this.telegramSender || this.alertTopicId === null)
|
|
262
|
+
return;
|
|
263
|
+
const mins = Math.round(ageMs / 60_000);
|
|
264
|
+
const msg = `⚠ ${d.component} has been on its heuristic fallback for ~${mins}m — the real-LLM path ` +
|
|
265
|
+
`(${d.framework}) hasn't recovered. It auto-clears the next time an LLM call for it succeeds.`;
|
|
266
|
+
void this.telegramSender(this.alertTopicId, msg);
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
// @silent-fallback-ok: escalation is best-effort; a send failure must never throw into the
|
|
270
|
+
// level-triggered sweep (which would stall the timer). The next sweep retries.
|
|
271
|
+
}
|
|
272
|
+
}
|
|
157
273
|
/**
|
|
158
274
|
* Report a degradation event.
|
|
159
275
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DegradationReporter.js","sourceRoot":"","sources":["../../src/monitoring/DegradationReporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEzD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAwB,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAmG3D;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,sGAAsG,CAAC;AAY1I,2EAA2E;AAC3E,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAEnD;;;;;GAKG;AACH,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,IAAI,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,WAAW,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YAChF,OAAO,WAAW,CAAC,GAAG,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,0CAA0C,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;AACpB,CAAC;AAED,+EAA+E;AAC/E,6EAA6E;AAC7E,0EAA0E;AAC1E,4EAA4E;AAC5E,wEAAwE;AACxE,kDAAkD;AAClD,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AACzD,MAAM,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;AAEpF,MAAM,OAAO,mBAAmB;IACtB,MAAM,CAAC,QAAQ,GAA+B,IAAI,CAAC;IAEnD,MAAM,GAAuB,EAAE,CAAC;IACxC,+EAA+E;IACvE,kBAAkB,GAAG,KAAK,CAAC;IACnC;;;;;;OAMG;IACK,MAAM,CAAU,UAAU,GAAG,GAAG,CAAC;IACjC,QAAQ,GAAkB,IAAI,CAAC;IAC/B,SAAS,GAAW,SAAS,CAAC;IAC9B,aAAa,GAAW,OAAO,CAAC;IAExC,6DAA6D;IACrD,iBAAiB,GAA6B,IAAI,CAAC;IACnD,cAAc,GAA0B,IAAI,CAAC;IAC7C,YAAY,GAAkB,IAAI,CAAC;IACnC,QAAQ,GAA6B,IAAI,CAAC;IAC1C,OAAO,GAA4B,IAAI,GAAG,EAAE,CAAC;IAErD,sEAAsE;IAC9D,aAAa,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEvD,8DAA8D;IAC9D,uEAAuE;IACvE,wEAAwE;IACxE,oEAAoE;IACpE,sEAAsE;IACtE,qEAAqE;IAC7D,UAAU,GAA0B,IAAI,CAAC;IAEjD,qEAAqE;IACrE,2EAA2E;IAC3E,iCAAiC;IACzB,QAAQ,GAAoB,IAAI,CAAC;IAEzC,2EAA2E;IAC3E,+EAA+E;IACvE,cAAc,GAAG,KAAK,CAAC;IAE/B,sEAAsE;IACtE,0EAA0E;IAC1E,8CAA8C;IACtC,cAAc,GAAG,CAAC,CAAC;IAE3B,gBAAuB,CAAC;IAExB,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,CAAC;YAClC,mBAAmB,CAAC,QAAQ,GAAG,IAAI,mBAAmB,EAAE,CAAC;QAC3D,CAAC;QACD,OAAO,mBAAmB,CAAC,QAAQ,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QACpB,mBAAmB,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,IAIT;QACC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAKjB;QACC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QAEtC,gDAAgD;QAChD,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,OAAe,EAAE,MAAkB;QAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAmE;QACxE,MAAM,IAAI,GAAqB;YAC7B,GAAG,KAAK;YACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,KAAK;SACf,CAAC;QAEF,uCAAuC;QACvC,OAAO,CAAC,IAAI,CACV,iBAAiB,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,IAAI;YACnD,cAAc,KAAK,CAAC,OAAO,IAAI;YAC/B,eAAe,KAAK,CAAC,QAAQ,IAAI;YACjC,aAAa,KAAK,CAAC,MAAM,EAAE,CAC5B,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,0EAA0E;QAC1E,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAEzB,qEAAqE;QACrE,iEAAiE;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAEzC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,oEAAoE;YACpE,kEAAkE;YAClE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACjD,OAAO,CAAC,KAAK,CACX,gDAAgD,IAAI,CAAC,OAAO,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAC5G,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,CAAC,KAAiC;QAChD,MAAM,OAAO,GAA+B;YAC1C,GAAG,KAAK;YACR,0EAA0E;YAC1E,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACtD,WAAW,EAAE,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;gBAChD,CAAC,CAAC,KAAK,CAAC,WAAW;gBACnB,CAAC,CAAC,YAAY,EAAE;SACnB,CAAC;QAEF,OAAO,CAAC,IAAI,CACV,4BAA4B,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,gBAAgB,OAAO,CAAC,UAAU,GAAG,CACzG,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC9C,OAAO,CAAC,KAAK,CACX,gDAAgD,OAAO,CAAC,SAAS,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CACjH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,aAAa,CAAC,UAAiC;QAC7C,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,MAAwB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE1D,oEAAoE;QACpE,qEAAqE;QACrE,mEAAmE;QACnE,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,KAAK,eAAe;YAClD,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;QAEnB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,OAAO;YACzB,SAAS;YACT,UAAU,EAAE,WAAW;YACvB,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE;YACzC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,YAAY,EAAE;YAC3B,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,OAAgB;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAC9B,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,oDAAoD;YACpD,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,wBAAwB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,OAAO,GAAG;iBACP,KAAK,CAAC,IAAI,CAAC;iBACX,MAAM,CAAC,OAAO,CAAC;iBACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,CAAC;oBAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA+B,CAAC;gBAAC,CAAC;gBAC9D,MAAM,CAAC;oBAAC,OAAO,IAAI,CAAC;gBAAC,CAAC;YACxB,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,CAAC,EAAmC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAChE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,kBAAkB;QAChB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,YAAY,CAAC,KAAuB;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAEnD,oEAAoE;QACpE,oEAAoE;QACpE,MAAM,cAAc,GAAG,8FAA8F,CAAC,IAAI,CAAC,aAAa,CAAC;eACpI,6EAA6E,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEvG,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,MAAM,mEAAmE,CAAC;QACtF,CAAC;QAED,8DAA8D;QAC9D,iEAAiE;QACjE,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ;aAC1B,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;aACjC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC;aACrC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEtB,wFAAwF;QACxF,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAE5D,OAAO,GAAG,MAAM,WAAW,QAAQ,qDAAqD,CAAC;IAC3F,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,cAA+B;QAC1C,MAAM,OAAO,GAAG,OAAO,cAAc,KAAK,QAAQ;YAChD,CAAC,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,cAAc;YAC3C,CAAC,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACtB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,6DAA6D;IAErD,KAAK,CAAC,WAAW,CAAC,KAAuB;QAC/C,4BAA4B;QAC5B,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,iBAAiB,CAAC;oBAC3B,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE;oBACxD,WAAW,EAAE;wBACX,0EAA0E;wBAC1E,EAAE;wBACF,gBAAgB,KAAK,CAAC,OAAO,EAAE;wBAC/B,qBAAqB,KAAK,CAAC,OAAO,EAAE;wBACpC,sBAAsB,KAAK,CAAC,QAAQ,EAAE;wBACtC,eAAe,KAAK,CAAC,MAAM,EAAE;wBAC7B,eAAe,KAAK,CAAC,MAAM,EAAE;wBAC7B,kBAAkB,KAAK,CAAC,SAAS,EAAE;qBACpC,CAAC,IAAI,CAAC,IAAI,CAAC;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,WAAW,EAAE,OAAO,CAAC,OAAO;oBAC5B,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;oBACtC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACtB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ,EAAE,OAAO,CAAC,IAAI;wBACtB,WAAW,EAAE,OAAO,CAAC,OAAO;qBAC7B,CAAC;iBACH,CAAC,CAAC;gBACH,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,sEAAsE;gBACtE,uEAAuE;gBACvE,OAAO,CAAC,KAAK,CAAC,4CAA4C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEvB,IAAI,GAAG,GAAG,SAAS,IAAI,iBAAiB,EAAE,CAAC;gBACzC,6DAA6D;gBAC7D,0DAA0D;gBAC1D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;gBACrD,IAAI,UAAU,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAClC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;oBACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBAC3C,OAAO,CAAC,IAAI,CACV,iBAAiB,KAAK,CAAC,OAAO,+BAA+B,UAAU,CAAC,QAAQ,qCAAqC,CACtH,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,6DAA6D;oBAC7D,mEAAmE;oBACnE,MAAM,SAAS,GAAG,mBAAmB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC1D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;oBACpE,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;wBACxD,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;wBACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBAC7C,CAAC;oBAAC,MAAM,CAAC;wBACP,kCAAkC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,2DAA2D;gBAC3D,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,eAAe,CAAC,KAAuB;QAKnD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;YAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CACV,iBAAiB,KAAK,CAAC,OAAO,yBAAyB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAClG,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC5D,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,eAAe,CAC3B,SAAiB,EACjB,UAA+E;QAE/E,iFAAiF;QACjF,oFAAoF;QACpF,oFAAoF;QACpF,qFAAqF;QACrF,+EAA+E;QAC/E,oFAAoF;QACpF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE;oBACnD,OAAO,EAAE,UAAU;oBACnB,WAAW,EAAE,cAAc;oBAC3B,OAAO,EAAE;wBACP,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE;wBAC/E,QAAQ,EAAE,UAAU;qBACrB;iBACF,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,OAAO,0BAA0B,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;gBAChE,mCAAmC;gBACnC,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,UAAU;QAChB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,iEAAiE;IACzD,WAAW;QACjB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QACnD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACK,qBAAqB,CAAC,KAAiC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,qEAAqE;YACrE,2CAA2C;YAC3C,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;YACzB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAED,gDAAgD;YAChD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACpC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;gBACrB,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAChD,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAE7B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YAC1C,IAAI,UAAU,IAAI,yBAAyB,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;gBAChG,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;gBACzB,2DAA2D;gBAC3D,IAAI,CAAC;oBACH,EAAE,CAAC,aAAa,CACd,SAAS,GAAG,aAAa,EACzB,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACxF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC7B,OAAO;YACT,CAAC;YAED,EAAE,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,mEAAmE;YACnE,uDAAuD;YACvD,OAAO,CAAC,KAAK,CACX,gDAAgD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAC3F,CAAC;YACF,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACK,yBAAyB;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO;QAEpD,MAAM,MAAM,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAC/C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,0CAA0C;YAC1C,IAAI,CAAC;gBACH,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE;oBACvC,SAAS,EAAE,oEAAoE;iBAChF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,mEAAmE;gBACnE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC5C,OAAO,CAAC,KAAK,CACX,4CAA4C,KAAK,CAAC,SAAS,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAC3G,CAAC;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxB,sEAAsE;gBACtE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,mDAAmD;QACnD,IAAI,CAAC;YACH,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE;gBACvC,SAAS,EAAE,yEAAyE;aACrF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IAEO,aAAa,CAAC,KAAuB;QAC3C,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE3B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;YAC/D,IAAI,QAAQ,GAAuB,EAAE,CAAC;YACtC,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAE7B,mBAAmB;YACnB,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAC5B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,CACpE,CAAC;YACF,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;gBACb,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YAED,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC1B,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;YAED,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"DegradationReporter.js","sourceRoot":"","sources":["../../src/monitoring/DegradationReporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEzD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAwB,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAmG3D;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,sGAAsG,CAAC;AAY1I,2EAA2E;AAC3E,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAEnD;;;;;GAKG;AACH,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,IAAI,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,WAAW,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YAChF,OAAO,WAAW,CAAC,GAAG,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,mNAAmN,CAAC,CAAC;IAC/N,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;AACpB,CAAC;AAED,+EAA+E;AAC/E,6EAA6E;AAC7E,0EAA0E;AAC1E,4EAA4E;AAC5E,wEAAwE;AACxE,kDAAkD;AAClD,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AACzD,MAAM,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;AAgBpF,MAAM,OAAO,mBAAmB;IACtB,MAAM,CAAC,QAAQ,GAA+B,IAAI,CAAC;IAEnD,MAAM,GAAuB,EAAE,CAAC;IACxC,+EAA+E;IACvE,kBAAkB,GAAG,KAAK,CAAC;IACnC;;;;;;OAMG;IACK,MAAM,CAAU,UAAU,GAAG,GAAG,CAAC;IACjC,QAAQ,GAAkB,IAAI,CAAC;IAC/B,SAAS,GAAW,SAAS,CAAC;IAC9B,aAAa,GAAW,OAAO,CAAC;IAExC,6DAA6D;IACrD,iBAAiB,GAA6B,IAAI,CAAC;IACnD,cAAc,GAA0B,IAAI,CAAC;IAC7C,YAAY,GAAkB,IAAI,CAAC;IACnC,QAAQ,GAA6B,IAAI,CAAC;IAC1C,OAAO,GAA4B,IAAI,GAAG,EAAE,CAAC;IAErD,sEAAsE;IAC9D,aAAa,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEvD,iFAAiF;IACjF,gFAAgF;IAChF,iFAAiF;IACjF,gFAAgF;IAChF,mFAAmF;IACnF,oFAAoF;IACpF,6DAA6D;IACrD,gBAAgB,GAAiC,IAAI,GAAG,EAAE,CAAC;IAC3D,kBAAkB,GAAG,KAAK,CAAC;IAC3B,qBAAqB,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,yCAAyC;IAC9E,gBAAgB,GAAG,EAAE,GAAG,MAAM,CAAC,CAAO,kCAAkC;IACxE,kBAAkB,GAAG,GAAG,CAAC,CAAa,wCAAwC;IACtF,sEAAsE;IAC9D,cAAc,GAAiB,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAExD,8DAA8D;IAC9D,uEAAuE;IACvE,wEAAwE;IACxE,oEAAoE;IACpE,sEAAsE;IACtE,qEAAqE;IAC7D,UAAU,GAA0B,IAAI,CAAC;IAEjD,qEAAqE;IACrE,2EAA2E;IAC3E,iCAAiC;IACzB,QAAQ,GAAoB,IAAI,CAAC;IAEzC,2EAA2E;IAC3E,+EAA+E;IACvE,cAAc,GAAG,KAAK,CAAC;IAE/B,sEAAsE;IACtE,0EAA0E;IAC1E,8CAA8C;IACtC,cAAc,GAAG,CAAC,CAAC;IAE3B,gBAAuB,CAAC;IAExB,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,CAAC;YAClC,mBAAmB,CAAC,QAAQ,GAAG,IAAI,mBAAmB,EAAE,CAAC;QAC3D,CAAC;QACD,OAAO,mBAAmB,CAAC,QAAQ,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QACpB,mBAAmB,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,IAIT;QACC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAKjB;QACC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC;QAClD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QAEtC,gDAAgD;QAChD,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,OAAe,EAAE,MAAkB;QAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,iFAAiF;IAEjF;sEACkE;IAClE,oBAAoB,CAAC,IAMpB;QACC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC;QAChD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,UAAU,CAAC;QACzF,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC;YAAE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC;QACrE,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC;YAAE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7E,IAAI,IAAI,CAAC,GAAG;YAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,sEAAsE;IACvG,CAAC;IAEO,KAAK,CAAC,SAAiB,EAAE,SAAiB;QAChD,OAAO,SAAS,GAAG,IAAI,GAAG,SAAS,CAAC;IACtC,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,SAAiB,EAAE,SAAiB;QAClD,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,kEAAkE;YAC5F,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,gCAAgC;YAC1F,IAAI,MAAM,KAAK,SAAS;gBAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE;YAC3B,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI;SAC/F,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;QACrD,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACpB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC5C,CAAC;IAED,4DAA4D;IAC5D,oBAAoB;QAClB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;IACpC,CAAC;IAED;;;;;;OAMG;IACH,qBAAqB;QACnB,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7B,IAAI,CAAC,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;gBAC1B,0FAA0F;gBAC1F,IAAI,GAAG,GAAG,IAAI,CAAC,gBAAgB;oBAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjE,SAAS;YACX,CAAC;YACD,IAAI,GAAG,GAAG,IAAI,CAAC,qBAAqB;gBAChC,CAAC,CAAC,CAAC,eAAe,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBACzF,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC;gBACxB,IAAI,CAAC,6BAA6B,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED;sFACkF;IAC1E,6BAA6B,CAAC,CAAkB,EAAE,KAAa;QACrE,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI;gBAAE,OAAO;YAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;YACxC,MAAM,GAAG,GACP,KAAK,CAAC,CAAC,SAAS,4CAA4C,IAAI,wBAAwB;gBACxF,IAAI,CAAC,CAAC,SAAS,+EAA+E,CAAC;YACjG,KAAK,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,2FAA2F;YAC3F,+EAA+E;QACjF,CAAC;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAmE;QACxE,MAAM,IAAI,GAAqB;YAC7B,GAAG,KAAK;YACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,KAAK;SACf,CAAC;QAEF,uCAAuC;QACvC,OAAO,CAAC,IAAI,CACV,iBAAiB,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,IAAI;YACnD,cAAc,KAAK,CAAC,OAAO,IAAI;YAC/B,eAAe,KAAK,CAAC,QAAQ,IAAI;YACjC,aAAa,KAAK,CAAC,MAAM,EAAE,CAC5B,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,0EAA0E;QAC1E,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAEzB,qEAAqE;QACrE,iEAAiE;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAEzC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,oEAAoE;YACpE,kEAAkE;YAClE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACjD,OAAO,CAAC,KAAK,CACX,gDAAgD,IAAI,CAAC,OAAO,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAC5G,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,CAAC,KAAiC;QAChD,MAAM,OAAO,GAA+B;YAC1C,GAAG,KAAK;YACR,0EAA0E;YAC1E,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACtD,WAAW,EAAE,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;gBAChD,CAAC,CAAC,KAAK,CAAC,WAAW;gBACnB,CAAC,CAAC,YAAY,EAAE;SACnB,CAAC;QAEF,OAAO,CAAC,IAAI,CACV,4BAA4B,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,gBAAgB,OAAO,CAAC,UAAU,GAAG,CACzG,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC9C,OAAO,CAAC,KAAK,CACX,gDAAgD,OAAO,CAAC,SAAS,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CACjH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,aAAa,CAAC,UAAiC;QAC7C,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,MAAwB;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE1D,oEAAoE;QACpE,qEAAqE;QACrE,mEAAmE;QACnE,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,KAAK,eAAe;YAClD,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;QAEnB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,OAAO;YACzB,SAAS;YACT,UAAU,EAAE,WAAW;YACvB,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE;YACzC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,YAAY,EAAE;YAC3B,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,OAAgB;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAC9B,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,oDAAoD;YACpD,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,wBAAwB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,OAAO,GAAG;iBACP,KAAK,CAAC,IAAI,CAAC;iBACX,MAAM,CAAC,OAAO,CAAC;iBACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,CAAC;oBAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA+B,CAAC;gBAAC,CAAC;gBAC9D,MAAM,CAAC;oBAAC,OAAO,IAAI,CAAC;gBAAC,CAAC;YACxB,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,CAAC,EAAmC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAChE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,kBAAkB;QAChB,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,YAAY,CAAC,KAAuB;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAEnD,oEAAoE;QACpE,oEAAoE;QACpE,MAAM,cAAc,GAAG,8FAA8F,CAAC,IAAI,CAAC,aAAa,CAAC;eACpI,6EAA6E,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEvG,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,MAAM,mEAAmE,CAAC;QACtF,CAAC;QAED,8DAA8D;QAC9D,iEAAiE;QACjE,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ;aAC1B,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;aACjC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC;aACrC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEtB,wFAAwF;QACxF,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAE5D,OAAO,GAAG,MAAM,WAAW,QAAQ,qDAAqD,CAAC;IAC3F,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,cAA+B;QAC1C,MAAM,OAAO,GAAG,OAAO,cAAc,KAAK,QAAQ;YAChD,CAAC,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,cAAc;YAC3C,CAAC,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACtB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,6DAA6D;IAErD,KAAK,CAAC,WAAW,CAAC,KAAuB;QAC/C,4BAA4B;QAC5B,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,iBAAiB,CAAC;oBAC3B,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE;oBACxD,WAAW,EAAE;wBACX,0EAA0E;wBAC1E,EAAE;wBACF,gBAAgB,KAAK,CAAC,OAAO,EAAE;wBAC/B,qBAAqB,KAAK,CAAC,OAAO,EAAE;wBACpC,sBAAsB,KAAK,CAAC,QAAQ,EAAE;wBACtC,eAAe,KAAK,CAAC,MAAM,EAAE;wBAC7B,eAAe,KAAK,CAAC,MAAM,EAAE;wBAC7B,kBAAkB,KAAK,CAAC,SAAS,EAAE;qBACpC,CAAC,IAAI,CAAC,IAAI,CAAC;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,WAAW,EAAE,OAAO,CAAC,OAAO;oBAC5B,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;oBACtC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;wBACtB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ,EAAE,OAAO,CAAC,IAAI;wBACtB,WAAW,EAAE,OAAO,CAAC,OAAO;qBAC7B,CAAC;iBACH,CAAC,CAAC;gBACH,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,sEAAsE;gBACtE,uEAAuE;gBACvE,OAAO,CAAC,KAAK,CAAC,4CAA4C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEvB,IAAI,GAAG,GAAG,SAAS,IAAI,iBAAiB,EAAE,CAAC;gBACzC,6DAA6D;gBAC7D,0DAA0D;gBAC1D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;gBACrD,IAAI,UAAU,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAClC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;oBACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBAC3C,OAAO,CAAC,IAAI,CACV,iBAAiB,KAAK,CAAC,OAAO,+BAA+B,UAAU,CAAC,QAAQ,qCAAqC,CACtH,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,6DAA6D;oBAC7D,mEAAmE;oBACnE,MAAM,SAAS,GAAG,mBAAmB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;oBAC1D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;oBACpE,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;wBACxD,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;wBACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBAC7C,CAAC;oBAAC,MAAM,CAAC;wBACP,kCAAkC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,2DAA2D;gBAC3D,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,eAAe,CAAC,KAAuB;QAKnD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;YAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CACV,iBAAiB,KAAK,CAAC,OAAO,yBAAyB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAClG,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC5D,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,eAAe,CAC3B,SAAiB,EACjB,UAA+E;QAE/E,iFAAiF;QACjF,oFAAoF;QACpF,oFAAoF;QACpF,qFAAqF;QACrF,+EAA+E;QAC/E,oFAAoF;QACpF,gFAAgF;QAChF,mFAAmF;QACnF,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE;oBACnD,OAAO,EAAE,UAAU;oBACnB,WAAW,EAAE,cAAc;oBAC3B,OAAO,EAAE;wBACP,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE;wBAC/E,QAAQ,EAAE,UAAU;qBACrB;iBACF,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,OAAO,0BAA0B,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;gBAChE,mCAAmC;gBACnC,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,UAAU;QAChB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,iEAAiE;IACzD,WAAW;QACjB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QACnD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACK,qBAAqB,CAAC,KAAiC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,qEAAqE;YACrE,2CAA2C;YAC3C,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;YACzB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAED,gDAAgD;YAChD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACpC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;gBACrB,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAChD,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAE7B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YAC1C,IAAI,UAAU,IAAI,yBAAyB,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;gBAChG,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;gBACzB,2DAA2D;gBAC3D,IAAI,CAAC;oBACH,EAAE,CAAC,aAAa,CACd,SAAS,GAAG,aAAa,EACzB,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACxF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC7B,OAAO;YACT,CAAC;YAED,EAAE,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,mEAAmE;YACnE,uDAAuD;YACvD,OAAO,CAAC,KAAK,CACX,gDAAgD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAC3F,CAAC;YACF,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACK,yBAAyB;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO;QAEpD,MAAM,MAAM,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAC/C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,0CAA0C;YAC1C,IAAI,CAAC;gBACH,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE;oBACvC,SAAS,EAAE,oEAAoE;iBAChF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,mEAAmE;gBACnE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC5C,OAAO,CAAC,KAAK,CACX,4CAA4C,KAAK,CAAC,SAAS,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAC3G,CAAC;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxB,sEAAsE;gBACtE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,mDAAmD;QACnD,IAAI,CAAC;YACH,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE;gBACvC,SAAS,EAAE,yEAAyE;aACrF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IAEO,aAAa,CAAC,KAAuB;QAC3C,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE3B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;YAC/D,IAAI,QAAQ,GAAuB,EAAE,CAAC;YACtC,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAE7B,mBAAmB;YACnB,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAC5B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,CACpE,CAAC;YACF,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;gBACb,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YAED,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC1B,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;YAED,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC"}
|
|
@@ -26,6 +26,13 @@ export interface LlmQueueOptions {
|
|
|
26
26
|
interactiveReservePct?: number;
|
|
27
27
|
/** Daily spend cap in cents across both lanes. Default 100. */
|
|
28
28
|
maxDailyCents?: number;
|
|
29
|
+
/**
|
|
30
|
+
* Herd-aware drain pacing (Resilient Degradation Ladder §3c). A jittered minimum gap (ms) between
|
|
31
|
+
* BACKGROUND-lane dispatches so a burst of queued background calls can't re-trip a just-recovered
|
|
32
|
+
* provider on recovery. Interactive-lane dispatches BYPASS pacing (latency-sensitive + preemptive).
|
|
33
|
+
* Default 0 = OFF (today's greedy drain, zero behavior change for existing callers).
|
|
34
|
+
*/
|
|
35
|
+
backgroundDispatchMinGapMs?: number;
|
|
29
36
|
/** Provide `Date.now()` — injectable for tests. */
|
|
30
37
|
now?: () => number;
|
|
31
38
|
}
|
|
@@ -33,9 +40,14 @@ export declare class LlmQueue {
|
|
|
33
40
|
private maxConcurrent;
|
|
34
41
|
private interactiveReservePct;
|
|
35
42
|
private maxDailyCents;
|
|
43
|
+
private backgroundDispatchMinGapMs;
|
|
36
44
|
private now;
|
|
37
45
|
private inFlight;
|
|
38
46
|
private waiters;
|
|
47
|
+
/** Wall-clock (via `now`) of the last BACKGROUND-lane dispatch — herd-pacing window anchor. */
|
|
48
|
+
private lastBackgroundStartAt;
|
|
49
|
+
/** At most one pending paced re-drain timer; null when none scheduled. */
|
|
50
|
+
private pacingTimer;
|
|
39
51
|
/** Daily spend ledger: { dateKey: 'YYYY-MM-DD', cents: number, interactive: number } */
|
|
40
52
|
private dailySpendCents;
|
|
41
53
|
private dailyInteractiveCents;
|
|
@@ -57,6 +69,11 @@ export declare class LlmQueue {
|
|
|
57
69
|
* abort() of an in-flight background call.
|
|
58
70
|
*/
|
|
59
71
|
private drain;
|
|
72
|
+
/**
|
|
73
|
+
* Schedule a single paced re-drain (§3c herd guard). Coalesces to at most one pending timer so a
|
|
74
|
+
* burst of background enqueues can't stack timers. `unref`'d so it never holds the process open.
|
|
75
|
+
*/
|
|
76
|
+
private schedulePacedDrain;
|
|
60
77
|
private start;
|
|
61
78
|
private rollDateIfNeeded;
|
|
62
79
|
/** Test / diagnostic accessors. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LlmQueue.d.ts","sourceRoot":"","sources":["../../src/monitoring/LlmQueue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,OAAO,GAAG,aAAa,GAAG,YAAY,CAAC;AAEnD,qBAAa,eAAgB,SAAQ,KAAK;;CAKzC;AAED,MAAM,WAAW,eAAe;IAC9B,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+DAA+D;IAC/D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mDAAmD;IACnD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAgBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,GAAG,CAAe;IAE1B,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,OAAO,CAAgB;IAE/B,wFAAwF;IACxF,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,YAAY,CAAM;gBAEd,IAAI,GAAE,eAAoB;
|
|
1
|
+
{"version":3,"file":"LlmQueue.d.ts","sourceRoot":"","sources":["../../src/monitoring/LlmQueue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,OAAO,GAAG,aAAa,GAAG,YAAY,CAAC;AAEnD,qBAAa,eAAgB,SAAQ,KAAK;;CAKzC;AAED,MAAM,WAAW,eAAe;IAC9B,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+DAA+D;IAC/D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,mDAAmD;IACnD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAgBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,0BAA0B,CAAS;IAC3C,OAAO,CAAC,GAAG,CAAe;IAE1B,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,OAAO,CAAgB;IAE/B,+FAA+F;IAC/F,OAAO,CAAC,qBAAqB,CAAK;IAClC,0EAA0E;IAC1E,OAAO,CAAC,WAAW,CAA8C;IAEjE,wFAAwF;IACxF,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,YAAY,CAAM;gBAEd,IAAI,GAAE,eAAoB;IAQtC;;;;;;;;OAQG;IACG,OAAO,CACX,IAAI,EAAE,OAAO,EACb,EAAE,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,MAAM,CAAC,EAC5C,SAAS,SAAI,GACZ,OAAO,CAAC,MAAM,CAAC;IAwBlB;;;;OAIG;IACH,OAAO,CAAC,KAAK;IAgDb;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,KAAK;IA0Bb,OAAO,CAAC,gBAAgB;IASxB,mCAAmC;IACnC,kBAAkB,IAAI,MAAM;IAI5B,gBAAgB,IAAI,MAAM;IAG1B,eAAe,IAAI,MAAM;IAGzB,gBAAgB,IAAI,OAAO,EAAE;CAG9B"}
|
|
@@ -8,9 +8,14 @@ export class LlmQueue {
|
|
|
8
8
|
maxConcurrent;
|
|
9
9
|
interactiveReservePct;
|
|
10
10
|
maxDailyCents;
|
|
11
|
+
backgroundDispatchMinGapMs;
|
|
11
12
|
now;
|
|
12
13
|
inFlight = new Set();
|
|
13
14
|
waiters = [];
|
|
15
|
+
/** Wall-clock (via `now`) of the last BACKGROUND-lane dispatch — herd-pacing window anchor. */
|
|
16
|
+
lastBackgroundStartAt = 0;
|
|
17
|
+
/** At most one pending paced re-drain timer; null when none scheduled. */
|
|
18
|
+
pacingTimer = null;
|
|
14
19
|
/** Daily spend ledger: { dateKey: 'YYYY-MM-DD', cents: number, interactive: number } */
|
|
15
20
|
dailySpendCents = 0;
|
|
16
21
|
dailyInteractiveCents = 0;
|
|
@@ -19,6 +24,7 @@ export class LlmQueue {
|
|
|
19
24
|
this.maxConcurrent = opts.maxConcurrent ?? 3;
|
|
20
25
|
this.interactiveReservePct = opts.interactiveReservePct ?? 0.4;
|
|
21
26
|
this.maxDailyCents = opts.maxDailyCents ?? 100;
|
|
27
|
+
this.backgroundDispatchMinGapMs = opts.backgroundDispatchMinGapMs ?? 0;
|
|
22
28
|
this.now = opts.now ?? (() => Date.now());
|
|
23
29
|
}
|
|
24
30
|
/**
|
|
@@ -62,6 +68,23 @@ export class LlmQueue {
|
|
|
62
68
|
while (this.waiters.length > 0) {
|
|
63
69
|
const next = this.waiters[0];
|
|
64
70
|
if (this.inFlight.size < this.maxConcurrent) {
|
|
71
|
+
// Herd guard (§3c): pace BACKGROUND-lane dispatches so a burst of queued calls can't re-trip
|
|
72
|
+
// a just-recovered provider. Interactive bypasses (latency-sensitive + preemptive). When a
|
|
73
|
+
// background dispatch is too soon after the previous one, schedule a paced re-drain instead of
|
|
74
|
+
// starting now. Because interactive waiters sort ahead, a background head means no interactive
|
|
75
|
+
// is waiting — so deferring it blocks nothing else. OFF when backgroundDispatchMinGapMs <= 0.
|
|
76
|
+
if (next.lane === 'background' && this.backgroundDispatchMinGapMs > 0 && this.lastBackgroundStartAt > 0) {
|
|
77
|
+
const since = this.now() - this.lastBackgroundStartAt;
|
|
78
|
+
// Jitter the gap (full-jitter low half) so independent recoveries don't re-sync into a herd.
|
|
79
|
+
const gap = Math.floor(this.backgroundDispatchMinGapMs * (0.5 + Math.random() * 0.5));
|
|
80
|
+
if (since < gap) {
|
|
81
|
+
this.schedulePacedDrain(gap - since);
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (next.lane === 'background' && this.backgroundDispatchMinGapMs > 0) {
|
|
86
|
+
this.lastBackgroundStartAt = this.now();
|
|
87
|
+
}
|
|
65
88
|
this.waiters.shift();
|
|
66
89
|
this.start(next);
|
|
67
90
|
continue;
|
|
@@ -82,6 +105,19 @@ export class LlmQueue {
|
|
|
82
105
|
break;
|
|
83
106
|
}
|
|
84
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Schedule a single paced re-drain (§3c herd guard). Coalesces to at most one pending timer so a
|
|
110
|
+
* burst of background enqueues can't stack timers. `unref`'d so it never holds the process open.
|
|
111
|
+
*/
|
|
112
|
+
schedulePacedDrain(delayMs) {
|
|
113
|
+
if (this.pacingTimer)
|
|
114
|
+
return;
|
|
115
|
+
this.pacingTimer = setTimeout(() => {
|
|
116
|
+
this.pacingTimer = null;
|
|
117
|
+
this.drain();
|
|
118
|
+
}, Math.max(1, delayMs));
|
|
119
|
+
this.pacingTimer.unref?.();
|
|
120
|
+
}
|
|
85
121
|
start(w) {
|
|
86
122
|
const controller = new AbortController();
|
|
87
123
|
const inflight = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LlmQueue.js","sourceRoot":"","sources":["../../src/monitoring/LlmQueue.ts"],"names":[],"mappings":"AAmBA,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC;QACE,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAClD,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;
|
|
1
|
+
{"version":3,"file":"LlmQueue.js","sourceRoot":"","sources":["../../src/monitoring/LlmQueue.ts"],"names":[],"mappings":"AAmBA,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC;QACE,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAClD,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAkCD,MAAM,OAAO,QAAQ;IACX,aAAa,CAAS;IACtB,qBAAqB,CAAS;IAC9B,aAAa,CAAS;IACtB,0BAA0B,CAAS;IACnC,GAAG,CAAe;IAElB,QAAQ,GAAkB,IAAI,GAAG,EAAE,CAAC;IACpC,OAAO,GAAa,EAAE,CAAC;IAE/B,+FAA+F;IACvF,qBAAqB,GAAG,CAAC,CAAC;IAClC,0EAA0E;IAClE,WAAW,GAAyC,IAAI,CAAC;IAEjE,wFAAwF;IAChF,eAAe,GAAG,CAAC,CAAC;IACpB,qBAAqB,GAAG,CAAC,CAAC;IAC1B,YAAY,GAAG,EAAE,CAAC;IAE1B,YAAY,OAAwB,EAAE;QACpC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,IAAI,GAAG,CAAC;QAC/D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC;QAC/C,IAAI,CAAC,0BAA0B,GAAG,IAAI,CAAC,0BAA0B,IAAI,CAAC,CAAC;QACvE,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,OAAO,CACX,IAAa,EACb,EAA4C,EAC5C,SAAS,GAAG,CAAC;QAEb,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAExB,mBAAmB;QACnB,IAAI,IAAI,CAAC,eAAe,GAAG,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QACD,4EAA4E;QAC5E,0EAA0E;QAC1E,YAAY;QACZ,MAAM,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC3F,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;YAC/E,IAAI,cAAc,GAAG,sBAAsB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBACzE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,KAAK;QACX,2CAA2C;QAC3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzF,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAE7B,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBAC5C,6FAA6F;gBAC7F,2FAA2F;gBAC3F,+FAA+F;gBAC/F,+FAA+F;gBAC/F,8FAA8F;gBAC9F,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,0BAA0B,GAAG,CAAC,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBACxG,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC;oBACtD,6FAA6F;oBAC7F,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;oBACtF,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;wBAChB,IAAI,CAAC,kBAAkB,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;wBACrC,MAAM;oBACR,CAAC;gBACH,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,0BAA0B,GAAG,CAAC,EAAE,CAAC;oBACtE,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC1C,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjB,SAAS;YACX,CAAC;YAED,mEAAmE;YACnE,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;gBACrE,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;oBAC1B,MAAM,CAAC,MAAM,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;oBACrC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC7B,4DAA4D;oBAC5D,SAAS;gBACX,CAAC;YACH,CAAC;YAED,oEAAoE;YACpE,wDAAwD;YACxD,MAAM;QACR,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,OAAe;QACxC,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,EAAE;YACjC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,CAAS;QACrB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAa;YACzB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,UAAU;YACV,MAAM,EAAE,CAAC,CAAC,MAAM;SACjB,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5B,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;aACpB,IAAI,CAAC,MAAM,CAAC,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,OAAO,CAAC,UAAU;YACpD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,SAAS,CAAC;YACpC,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa;gBAAE,IAAI,CAAC,qBAAqB,IAAI,CAAC,CAAC,SAAS,CAAC;YACxE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,OAAO,CAAC,4BAA4B;YACtE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,gBAAgB;QACtB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9D,IAAI,KAAK,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,qBAAqB,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,kBAAkB;QAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IACD,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IACD,eAAe;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IACD,gBAAgB;QACd,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-22T03:53:34.493Z",
|
|
5
|
+
"instarVersion": "1.3.641",
|
|
6
6
|
"entryCount": 202,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -1586,7 +1586,7 @@
|
|
|
1586
1586
|
"type": "subsystem",
|
|
1587
1587
|
"domain": "monitoring",
|
|
1588
1588
|
"sourcePath": "src/monitoring/DegradationReporter.ts",
|
|
1589
|
-
"contentHash": "
|
|
1589
|
+
"contentHash": "3895b7eb8c6d3caf4337014ad6aeda537ae5f2507fa22a2357609be319811326",
|
|
1590
1590
|
"since": "2025-01-01"
|
|
1591
1591
|
},
|
|
1592
1592
|
"subsystem:telegram-lifeline": {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: minor -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
The other half of the rate-limit principle: **never let the agent silently stay stuck on a
|
|
9
|
+
fallback.** When an internal call exhausts its options and the caller drops to a simple rule-of-thumb
|
|
10
|
+
instead of real AI, the agent now tracks that. The moment a real AI call for that component succeeds
|
|
11
|
+
again, it clears itself automatically. And if it stays stuck for too long (default 15 minutes) AND
|
|
12
|
+
has genuinely retried, the agent raises one calm heads-up that the AI path hasn't recovered — so a
|
|
13
|
+
degraded state can't quietly rot. A component that just ran once and finished (not stuck) quietly
|
|
14
|
+
closes itself with no false alarm.
|
|
15
|
+
|
|
16
|
+
This is **off by default** (on for the dev agent first), and it's built carefully to NOT repeat an
|
|
17
|
+
event-loop freeze the same subsystem caused earlier: it's bounded, it never re-enters the alert
|
|
18
|
+
machinery, and it only alerts when something is genuinely stuck.
|
|
19
|
+
|
|
20
|
+
## What to Tell Your User
|
|
21
|
+
|
|
22
|
+
Nothing changes unless you turn it on. Once on, if your agent ever falls back from real AI to a
|
|
23
|
+
basic rule on some internal check and stays that way, you'll get one quiet heads-up instead of it
|
|
24
|
+
silently persisting — and it clears itself the moment the AI path recovers.
|
|
25
|
+
|
|
26
|
+
## Summary of New Capabilities
|
|
27
|
+
|
|
28
|
+
- `DegradationReporter` open-degradation lifecycle: `openDegradation` / `resolveDegradation` /
|
|
29
|
+
`sweepOpenDegradations` / `configureNeverSilent` + `openDegradationCount`, keyed on
|
|
30
|
+
`(component, framework)`, bounded by a hard cap, O(1) per open/resolve.
|
|
31
|
+
- `IntelligenceRouter` `onHeuristicFallthrough` (fires when a non-gating call exhausts the ladder)
|
|
32
|
+
+ `onResolved` (fires on a successful real-LLM answer) hooks, wired to the reporter.
|
|
33
|
+
- A dev-gated 60-second sweep that escalates a genuinely-stuck degradation (≥1 retry, open past
|
|
34
|
+
`escalateMs`, deduped) and auto-closes an idle/run-once one at the TTL.
|
|
35
|
+
- Dark/dev-gated (`resolveDevAgentGate` + a `DEV_GATED_FEATURES` entry); no-op when off.
|
|
36
|
+
|
|
37
|
+
## Evidence
|
|
38
|
+
|
|
39
|
+
**Before:** `DegradationReporter` reported a degradation once (`report()` + a one-shot self-healer)
|
|
40
|
+
but had NO continuous "is this still degraded?" tracking — a fallback could persist indefinitely with
|
|
41
|
+
nobody knowing. **After:** a non-gating heuristic fallthrough opens a tracked degradation; a real
|
|
42
|
+
answer auto-resolves it; a genuinely-stuck one escalates once (deduped); a run-once one auto-closes.
|
|
43
|
+
Reproduced by `tests/unit/degradation-never-silent.test.ts` (6 cases) + 3 router-hook cases in
|
|
44
|
+
`degradation-ladder.test.ts`. Critically, the escalation path surfaces via `telegramSender` directly
|
|
45
|
+
and NEVER calls `report()` — the reentrancy that caused the 2026-06-21 event-loop wedge cannot recur.
|
|
46
|
+
Full lint + tsc green; the existing reporter/router tests are unaffected.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: minor -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
The last rung of the rate-limit principle: **wait for capacity before falling back.** When an internal
|
|
9
|
+
background check gets rate-limited and has already tried slowing-down-and-retrying and switching
|
|
10
|
+
frameworks, it can now WAIT in a small queue for capacity to free up — instead of immediately giving up
|
|
11
|
+
and dropping to a basic rule. The wait is bounded and rate-aware (it rides the same circuit-breaker
|
|
12
|
+
timing that protects the account), so a burst of queued checks can't re-trip the limit the moment it
|
|
13
|
+
recovers. Only background, non-urgent checks queue; anything the agent is actively waiting on still
|
|
14
|
+
fails fast. If the queue is full or over its daily budget, the check falls through to its basic rule
|
|
15
|
+
exactly as before — never silently dropped.
|
|
16
|
+
|
|
17
|
+
This is **off by default** (on for the dev agent first), reuses the existing wedge-safe queue, and
|
|
18
|
+
changes nothing for any other queue user.
|
|
19
|
+
|
|
20
|
+
## What to Tell Your User
|
|
21
|
+
|
|
22
|
+
Nothing changes unless you turn it on. Once on, if a background check hits a rate limit, it will
|
|
23
|
+
briefly wait its turn for capacity to come back before falling back to a simpler rule — so you lose the
|
|
24
|
+
smart answer less often. Urgent checks you are waiting on still respond fast, and if there is no spare
|
|
25
|
+
capacity the check just uses its basic rule like before.
|
|
26
|
+
|
|
27
|
+
## Summary of New Capabilities
|
|
28
|
+
|
|
29
|
+
- `IntelligenceRouter` DEFERRABLE queue rung (`tryDeferrableQueue`): a non-gating call that exhausts
|
|
30
|
+
backoff + framework-swap enqueues onto a dedicated `LlmQueue` (background lane) and WAITS for
|
|
31
|
+
capacity (the enqueued `provider.evaluate` honors the account-global breaker's `retryAfterMs`)
|
|
32
|
+
before dropping to the caller's heuristic. Inserted at both fall-through points.
|
|
33
|
+
- A GATING call is NEVER queued (`deferrable = !gating && …`, code-enforced — D5). An enqueue
|
|
34
|
+
rejection (daily-cap / interactive-reserve) or queued-call failure falls through to the heuristic,
|
|
35
|
+
never dropped. Two new `onDegrade` reasons (`queued` / `queue-rejected`) for `/metrics/features` (D7).
|
|
36
|
+
- `LlmQueue` opt-in `backgroundDispatchMinGapMs` (§3c herd guard): a jittered minimum gap between
|
|
37
|
+
background dispatches; interactive bypasses; default 0 = off (zero change for existing callers).
|
|
38
|
+
- Config `intelligence.degradationLadder.queue` → `{ enabled?, attemptTimeoutMs?, drainMinGapMs? }`;
|
|
39
|
+
`DEV_GATED_FEATURES` entry `degradationLadderQueue` (live-on-dev / dark-on-fleet).
|
|
40
|
+
|
|
41
|
+
## Evidence
|
|
42
|
+
|
|
43
|
+
**Before:** a deferrable internal call that exhausted backoff + framework-swap dropped STRAIGHT to its
|
|
44
|
+
heuristic — even when capacity would have freed up moments later. **After:** with the rung enabled, it
|
|
45
|
+
WAITS for capacity in a bounded, rate-aware queue first, and only falls back if the queue rejects it or
|
|
46
|
+
the queued call fails. Reproduced by `tests/unit/degradation-ladder.test.ts` (7 queue cases: success +
|
|
47
|
+
auto-resolve; order backoff/swap → queue → success; GATING never queued; enqueue-rejection ⇒ heuristic
|
|
48
|
+
fallthrough; non-deferrable never queued; no-llmQueue no-op; queueEnabled:false not enqueued) and
|
|
49
|
+
`tests/unit/LlmQueue.test.ts` (3 pacing cases). The rung reuses the existing wedge-safe `LlmQueue` and
|
|
50
|
+
never calls the DegradationReporter alert machinery, so the 2026-06-21 event-loop wedge cannot recur.
|
|
51
|
+
Full `tsc` + dark-gate lint green; `no-silent-fallbacks` holds at baseline 476.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# Side-Effects Review — Resilient Degradation Ladder Increment 2 (never-silent tracking)
|
|
2
|
+
|
|
3
|
+
**Slug:** `resilient-degradation-ladder-increment-2` · **Tier:** 2 (spec-driven; converged +
|
|
4
|
+
operator-approved). **Spec:** `docs/specs/resilient-degradation-ladder.md` §4, D6.
|
|
5
|
+
|
|
6
|
+
## Summary of the change
|
|
7
|
+
|
|
8
|
+
The never-silent half of the operator's principle ("never silently remain degraded indefinitely"),
|
|
9
|
+
dark/dev-gated:
|
|
10
|
+
- `DegradationReporter` gains an open-degradation lifecycle (§4): `openDegradation` /
|
|
11
|
+
`resolveDegradation` / `sweepOpenDegradations` / `configureNeverSilent`, keyed on
|
|
12
|
+
`(component, framework)`, bounded by MAX_OPEN, O(1) per open/resolve.
|
|
13
|
+
- `IntelligenceRouter` gains `onHeuristicFallthrough` (fired at each NON-gating throw — the caller
|
|
14
|
+
will use its heuristic) + `onResolved` (fired at each successful real-LLM answer) hooks.
|
|
15
|
+
- Server wires the hooks to the reporter + calls `configureNeverSilent` (dev-gated via
|
|
16
|
+
`resolveDevAgentGate`) + starts a 60s `unref`'d sweep timer.
|
|
17
|
+
- `DEV_GATED_FEATURES` registration (`degradationLadderNeverSilent`).
|
|
18
|
+
|
|
19
|
+
## Decision-point inventory
|
|
20
|
+
|
|
21
|
+
Frozen in spec §5 D6: escalateMs 15m, TTL 30m, MAX_OPEN 500, key (component, framework),
|
|
22
|
+
liveness-gated (≥1 retry to escalate; TTL-close otherwise), deduped per episode.
|
|
23
|
+
|
|
24
|
+
## 1. Over-block / false positive
|
|
25
|
+
|
|
26
|
+
The big false-positive class (round-1): a run-once / idle component that degrades once and never
|
|
27
|
+
calls again. Closed by the LIVENESS gate — a degradation with 0 retries since open AUTO-CLOSES at
|
|
28
|
+
the TTL instead of escalating (tested). Only a degradation that genuinely RE-attempted and still
|
|
29
|
+
fell to heuristic (≥1 retry) escalates. Escalation is deduped per episode (re-escalates only after
|
|
30
|
+
another full window — tested).
|
|
31
|
+
|
|
32
|
+
## 2. Under-block
|
|
33
|
+
|
|
34
|
+
Tracking only opens on a NON-gating heuristic fallthrough (where the heuristic actually runs); a
|
|
35
|
+
gating fail-closed is NOT tracked (it's the safe outcome, not a heuristic — tested:
|
|
36
|
+
onHeuristicFallthrough does NOT fire for a gating call). No-op when disabled.
|
|
37
|
+
|
|
38
|
+
## 4. Signal vs authority
|
|
39
|
+
|
|
40
|
+
The escalation is an attention SIGNAL (a deduped fixed-template Telegram line), never a gate. It
|
|
41
|
+
takes no destructive action. The auto-resolve is observational (a successful call clears the open
|
|
42
|
+
state). Consistent with Signal-vs-Authority.
|
|
43
|
+
|
|
44
|
+
## 5. Interactions — THE WEDGE-CRITICAL SECTION
|
|
45
|
+
|
|
46
|
+
§4 extends the exact `DegradationReporter` subsystem that caused the 2026-06-21 event-loop wedge
|
|
47
|
+
(`gateHealthAlert`→`toneGate.review`→router→`report`→`reportEvent`→recursion + growing-array
|
|
48
|
+
JSON.stringify). This increment is designed to NOT repeat it:
|
|
49
|
+
- The sweep NEVER calls `report()`/`reportEvent()`/`gateHealthAlert` — it surfaces an attention item
|
|
50
|
+
via `telegramSender` DIRECTLY (no toneGate, no events-array growth, no recursion). Tested: the
|
|
51
|
+
escalation path sends via the fake telegramSender, never through report.
|
|
52
|
+
- The open map is BOUNDED (MAX_OPEN; oldest evicted — tested) — no unbounded growth.
|
|
53
|
+
- open/resolve are O(1) Map mutations — no full-map serialize per event.
|
|
54
|
+
- The existing `_gatingHealthAlert` reentrancy guard + `MAX_EVENTS` cap are untouched.
|
|
55
|
+
|
|
56
|
+
## 6. External surfaces
|
|
57
|
+
|
|
58
|
+
No new route. The only new egress is the deduped escalation line (the operator's explicit want).
|
|
59
|
+
The new config (`intelligence.degradationLadder.neverSilent`) shipped in Increment 1's types.
|
|
60
|
+
|
|
61
|
+
## 6b. Operator-surface quality
|
|
62
|
+
|
|
63
|
+
N/A — no dashboard/approval surface. `openDegradationCount()` is a read-only observability getter.
|
|
64
|
+
|
|
65
|
+
## Framework generality
|
|
66
|
+
|
|
67
|
+
Framework-agnostic — the hooks key on the resolved framework; works for whichever framework the
|
|
68
|
+
component routes to.
|
|
69
|
+
|
|
70
|
+
## 7. Multi-machine posture
|
|
71
|
+
|
|
72
|
+
Machine-local: each machine's reporter tracks its own degradations + its own sweep timer. No
|
|
73
|
+
replicated state. (A sustained per-machine degradation could double-alert on a 2-machine setup for
|
|
74
|
+
one provider outage — accepted as a minor local-dedup limitation, noted in the spec §6.)
|
|
75
|
+
|
|
76
|
+
## 8. Rollback cost
|
|
77
|
+
|
|
78
|
+
Trivial: dark on the fleet (`configureNeverSilent({enabled:false})` when not dev/configured) — the
|
|
79
|
+
lifecycle methods are all `if (!enabled) return` no-ops and no sweep timer is started. The router
|
|
80
|
+
hooks are no-ops downstream when disabled. Revert = remove an unused-on-fleet code path.
|
|
81
|
+
|
|
82
|
+
## Evidence pointers
|
|
83
|
+
|
|
84
|
+
- `tests/unit/degradation-never-silent.test.ts` (6): open→resolve duration; run-once TTL-auto-close
|
|
85
|
+
(NO escalation); stuck (≥1 retry) escalates once past the window + deduped + re-escalates after a
|
|
86
|
+
new window; (component,framework) keying (no cross-resolution); bounded (MAX_OPEN evict);
|
|
87
|
+
disabled=no-op.
|
|
88
|
+
- `tests/unit/degradation-ladder.test.ts` (+3 hook cases): onResolved on success; onHeuristicFallthrough
|
|
89
|
+
on a non-gating exhaustion; onHeuristicFallthrough does NOT fire for a gating call.
|
|
90
|
+
- The `DEV_GATED_FEATURES` both-sides wiring test confirms `degradationLadderNeverSilent` resolves
|
|
91
|
+
live-on-dev / dark-on-fleet. Full `npm run lint` (incl. lint-dev-agent-dark-gate) + `tsc` green.
|
|
92
|
+
|
|
93
|
+
## Conclusion
|
|
94
|
+
|
|
95
|
+
Delivers the never-silent guarantee — a heuristic fallback can no longer silently persist
|
|
96
|
+
indefinitely: it auto-resolves on recovery and escalates if genuinely stuck — built carefully to NOT
|
|
97
|
+
repeat the wedge it extends. Dark/dev-gated, no-op when off. Ship.
|