@elisym/sdk 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +27 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +39 -1
- package/dist/index.d.ts +39 -1
- package/dist/index.js +27 -1
- package/dist/index.js.map +1 -1
- package/dist/llm-health.cjs +121 -1
- package/dist/llm-health.cjs.map +1 -1
- package/dist/llm-health.d.cts +70 -2
- package/dist/llm-health.d.ts +70 -2
- package/dist/llm-health.js +121 -1
- package/dist/llm-health.js.map +1 -1
- package/package.json +1 -1
package/dist/llm-health.d.ts
CHANGED
|
@@ -96,6 +96,13 @@ declare class LlmHealthMonitor {
|
|
|
96
96
|
* the one captured at startup. Skips an extra probe on the first
|
|
97
97
|
* `assertReady` call within the TTL window. No-op if the pair is not
|
|
98
98
|
* registered.
|
|
99
|
+
*
|
|
100
|
+
* In production `seed` is only called from the CLI startup path, which
|
|
101
|
+
* itself `process.exit(1)`s on `invalid` / `billing` before any other
|
|
102
|
+
* pair is reached - so the cascade in `applyVerification` is benign at
|
|
103
|
+
* startup. In tests, seeding `invalid` / `billing` will cascade across
|
|
104
|
+
* sibling pairs of the same provider, which matches the production
|
|
105
|
+
* runtime behavior we want to assert.
|
|
99
106
|
*/
|
|
100
107
|
seed(provider: string, model: string, verification: LlmKeyVerification): void;
|
|
101
108
|
/**
|
|
@@ -113,6 +120,12 @@ declare class LlmHealthMonitor {
|
|
|
113
120
|
* Force the next `assertReady` for this pair to re-probe regardless of
|
|
114
121
|
* TTL. Used when a real LLM call surfaces 401/402 mid-job - the cached
|
|
115
122
|
* `healthy` is stale and we want to catch the next request.
|
|
123
|
+
*
|
|
124
|
+
* NOTE: this method has no callers in the current production code
|
|
125
|
+
* (`packages/sdk/src` and `packages/cli/src` both prefer the explicit
|
|
126
|
+
* `markUnhealthyFromJob` below). It does NOT propagate to sibling
|
|
127
|
+
* pairs; if reintroduced, consider whether the cascade in
|
|
128
|
+
* `markUnhealthyFromJob` should also apply here.
|
|
116
129
|
*/
|
|
117
130
|
markFailureFromJob(provider: string, model: string): void;
|
|
118
131
|
/**
|
|
@@ -124,9 +137,15 @@ declare class LlmHealthMonitor {
|
|
|
124
137
|
* lazy recovery loop notices on its own. No-op if the pair is not
|
|
125
138
|
* registered.
|
|
126
139
|
*
|
|
140
|
+
* For `invalid` / `billing` reasons this propagates to every sibling
|
|
141
|
+
* pair sharing the same `provider` via the cascade in
|
|
142
|
+
* `applyVerification` - they share the same API key, so a revoked or
|
|
143
|
+
* exhausted key affects all models. `unavailable` stays per-pair.
|
|
144
|
+
*
|
|
127
145
|
* Recovery from this state happens through a successful probe (typically
|
|
128
146
|
* fired by `startLlmRecovery`) which flips the pair back to healthy via
|
|
129
|
-
* `applyVerification`.
|
|
147
|
+
* `applyVerification`. The recovery cascade lifts sibling pairs at the
|
|
148
|
+
* same time so the gate doesn't stay half-broken.
|
|
130
149
|
*/
|
|
131
150
|
markUnhealthyFromJob(provider: string, model: string, reason: 'billing' | 'invalid' | 'unavailable', detail?: string): void;
|
|
132
151
|
/**
|
|
@@ -140,6 +159,17 @@ declare class LlmHealthMonitor {
|
|
|
140
159
|
* loop so a billing outage on one provider does not trigger throwaway
|
|
141
160
|
* probes on every other healthy pair the agent has registered.
|
|
142
161
|
* Errors thrown by `verifyFn` are caught and recorded as `unavailable`.
|
|
162
|
+
*
|
|
163
|
+
* Provider-deduplicated for `invalid` / `billing`: every pair under
|
|
164
|
+
* the same provider shares the same API key, so a single probe per
|
|
165
|
+
* provider is enough - a successful result will cascade to siblings
|
|
166
|
+
* via the recovery cascade in `applyVerification`. Without this dedup,
|
|
167
|
+
* a 3-skill Anthropic agent in a billing outage would burn 3x the
|
|
168
|
+
* billing-token quota per recovery tick.
|
|
169
|
+
*
|
|
170
|
+
* `unavailable` entries probe individually because the failure is
|
|
171
|
+
* model-specific (e.g. one endpoint returning 5xx) and cascading the
|
|
172
|
+
* result would corrupt sibling state.
|
|
143
173
|
*/
|
|
144
174
|
refreshUnhealthy(): Promise<readonly LlmHealthSnapshotEntry[]>;
|
|
145
175
|
/** Read-only view, primarily for logs and tests. */
|
|
@@ -147,7 +177,40 @@ declare class LlmHealthMonitor {
|
|
|
147
177
|
private probeIfNeeded;
|
|
148
178
|
private synthesizeFailureFromCache;
|
|
149
179
|
private probe;
|
|
180
|
+
/**
|
|
181
|
+
* Apply a verification to a single entry AND cascade across siblings
|
|
182
|
+
* sharing the same `provider`. Two symmetric cascades:
|
|
183
|
+
*
|
|
184
|
+
* - **Failure cascade** (`invalid` / `billing`): a revoked or
|
|
185
|
+
* billing-exhausted API key affects every model the operator could
|
|
186
|
+
* reach with that key. Structurally enforced by
|
|
187
|
+
* `resolveProviderApiKey` in `packages/cli/src/llm/keys.ts:27`,
|
|
188
|
+
* which returns at most one key per provider (no per-skill
|
|
189
|
+
* injection). If a future change introduces per-skill keys, this
|
|
190
|
+
* cascade must be revisited.
|
|
191
|
+
*
|
|
192
|
+
* - **Recovery cascade** (`ok: true`): when a healthy verification
|
|
193
|
+
* arrives (typically from `refreshUnhealthy` after the operator
|
|
194
|
+
* fixed the key), lift sibling pairs that were stuck `invalid` /
|
|
195
|
+
* `billing` so the next `assertReady` for any model under this
|
|
196
|
+
* provider stops refusing jobs. Without this, the gate would stay
|
|
197
|
+
* half-broken until each sibling's own probe round-trips.
|
|
198
|
+
*
|
|
199
|
+
* `unavailable` does NOT cascade - it is treated as a transient,
|
|
200
|
+
* model-specific issue (e.g. specific endpoint returning 5xx).
|
|
201
|
+
*
|
|
202
|
+
* Cascaded sibling verifications are tagged with `(cascaded from
|
|
203
|
+
* <triggering-model>)` in their body so `snapshot()` and operator
|
|
204
|
+
* logs remain diagnosable.
|
|
205
|
+
*/
|
|
150
206
|
private applyVerification;
|
|
207
|
+
/**
|
|
208
|
+
* Per-entry state transition. Used by `applyVerification` (which then
|
|
209
|
+
* orchestrates the cascade) and by the cascade loop itself for sibling
|
|
210
|
+
* entries (which must NOT trigger a recursive cascade).
|
|
211
|
+
*/
|
|
212
|
+
private mutateEntry;
|
|
213
|
+
private tagCascaded;
|
|
151
214
|
}
|
|
152
215
|
|
|
153
216
|
/**
|
|
@@ -165,7 +228,12 @@ declare class LlmHealthMonitor {
|
|
|
165
228
|
*
|
|
166
229
|
* Logging policy lives here so the monitor stays a pure state-machine.
|
|
167
230
|
* Status transitions (healthy <-> unhealthy) are logged once per change;
|
|
168
|
-
* routine successful re-probes are not logged.
|
|
231
|
+
* routine successful re-probes are not logged. When the monitor's
|
|
232
|
+
* provider-wide cascade (see `applyVerification` in `monitor.ts`)
|
|
233
|
+
* flips multiple sibling pairs at once, each pair still produces its
|
|
234
|
+
* own transition line here - intentional asymmetry vs. the runtime's
|
|
235
|
+
* single aggregated "marking unhealthy" line, since each sibling is
|
|
236
|
+
* separately observed by the snapshot diff.
|
|
169
237
|
*/
|
|
170
238
|
|
|
171
239
|
interface HeartbeatHandle {
|
package/dist/llm-health.js
CHANGED
|
@@ -89,6 +89,13 @@ var LlmHealthMonitor = class {
|
|
|
89
89
|
* the one captured at startup. Skips an extra probe on the first
|
|
90
90
|
* `assertReady` call within the TTL window. No-op if the pair is not
|
|
91
91
|
* registered.
|
|
92
|
+
*
|
|
93
|
+
* In production `seed` is only called from the CLI startup path, which
|
|
94
|
+
* itself `process.exit(1)`s on `invalid` / `billing` before any other
|
|
95
|
+
* pair is reached - so the cascade in `applyVerification` is benign at
|
|
96
|
+
* startup. In tests, seeding `invalid` / `billing` will cascade across
|
|
97
|
+
* sibling pairs of the same provider, which matches the production
|
|
98
|
+
* runtime behavior we want to assert.
|
|
92
99
|
*/
|
|
93
100
|
seed(provider, model, verification) {
|
|
94
101
|
const entry = this.entries.get(keyOf(provider, model));
|
|
@@ -127,6 +134,12 @@ var LlmHealthMonitor = class {
|
|
|
127
134
|
* Force the next `assertReady` for this pair to re-probe regardless of
|
|
128
135
|
* TTL. Used when a real LLM call surfaces 401/402 mid-job - the cached
|
|
129
136
|
* `healthy` is stale and we want to catch the next request.
|
|
137
|
+
*
|
|
138
|
+
* NOTE: this method has no callers in the current production code
|
|
139
|
+
* (`packages/sdk/src` and `packages/cli/src` both prefer the explicit
|
|
140
|
+
* `markUnhealthyFromJob` below). It does NOT propagate to sibling
|
|
141
|
+
* pairs; if reintroduced, consider whether the cascade in
|
|
142
|
+
* `markUnhealthyFromJob` should also apply here.
|
|
130
143
|
*/
|
|
131
144
|
markFailureFromJob(provider, model) {
|
|
132
145
|
const entry = this.entries.get(keyOf(provider, model));
|
|
@@ -144,9 +157,15 @@ var LlmHealthMonitor = class {
|
|
|
144
157
|
* lazy recovery loop notices on its own. No-op if the pair is not
|
|
145
158
|
* registered.
|
|
146
159
|
*
|
|
160
|
+
* For `invalid` / `billing` reasons this propagates to every sibling
|
|
161
|
+
* pair sharing the same `provider` via the cascade in
|
|
162
|
+
* `applyVerification` - they share the same API key, so a revoked or
|
|
163
|
+
* exhausted key affects all models. `unavailable` stays per-pair.
|
|
164
|
+
*
|
|
147
165
|
* Recovery from this state happens through a successful probe (typically
|
|
148
166
|
* fired by `startLlmRecovery`) which flips the pair back to healthy via
|
|
149
|
-
* `applyVerification`.
|
|
167
|
+
* `applyVerification`. The recovery cascade lifts sibling pairs at the
|
|
168
|
+
* same time so the gate doesn't stay half-broken.
|
|
150
169
|
*/
|
|
151
170
|
markUnhealthyFromJob(provider, model, reason, detail) {
|
|
152
171
|
const entry = this.entries.get(keyOf(provider, model));
|
|
@@ -194,13 +213,33 @@ var LlmHealthMonitor = class {
|
|
|
194
213
|
* loop so a billing outage on one provider does not trigger throwaway
|
|
195
214
|
* probes on every other healthy pair the agent has registered.
|
|
196
215
|
* Errors thrown by `verifyFn` are caught and recorded as `unavailable`.
|
|
216
|
+
*
|
|
217
|
+
* Provider-deduplicated for `invalid` / `billing`: every pair under
|
|
218
|
+
* the same provider shares the same API key, so a single probe per
|
|
219
|
+
* provider is enough - a successful result will cascade to siblings
|
|
220
|
+
* via the recovery cascade in `applyVerification`. Without this dedup,
|
|
221
|
+
* a 3-skill Anthropic agent in a billing outage would burn 3x the
|
|
222
|
+
* billing-token quota per recovery tick.
|
|
223
|
+
*
|
|
224
|
+
* `unavailable` entries probe individually because the failure is
|
|
225
|
+
* model-specific (e.g. one endpoint returning 5xx) and cascading the
|
|
226
|
+
* result would corrupt sibling state.
|
|
197
227
|
*/
|
|
198
228
|
async refreshUnhealthy() {
|
|
199
229
|
const probes = [];
|
|
230
|
+
const sharedKeyProviders = /* @__PURE__ */ new Set();
|
|
200
231
|
for (const entry of this.entries.values()) {
|
|
201
232
|
if (entry.status === "healthy" || entry.status === "unknown") {
|
|
202
233
|
continue;
|
|
203
234
|
}
|
|
235
|
+
if (entry.status === "unavailable") {
|
|
236
|
+
probes.push(this.probe(entry).then(() => void 0));
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
if (sharedKeyProviders.has(entry.provider)) {
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
sharedKeyProviders.add(entry.provider);
|
|
204
243
|
probes.push(this.probe(entry).then(() => void 0));
|
|
205
244
|
}
|
|
206
245
|
await Promise.all(probes);
|
|
@@ -262,7 +301,69 @@ var LlmHealthMonitor = class {
|
|
|
262
301
|
entry.inFlight = promise;
|
|
263
302
|
return promise;
|
|
264
303
|
}
|
|
304
|
+
/**
|
|
305
|
+
* Apply a verification to a single entry AND cascade across siblings
|
|
306
|
+
* sharing the same `provider`. Two symmetric cascades:
|
|
307
|
+
*
|
|
308
|
+
* - **Failure cascade** (`invalid` / `billing`): a revoked or
|
|
309
|
+
* billing-exhausted API key affects every model the operator could
|
|
310
|
+
* reach with that key. Structurally enforced by
|
|
311
|
+
* `resolveProviderApiKey` in `packages/cli/src/llm/keys.ts:27`,
|
|
312
|
+
* which returns at most one key per provider (no per-skill
|
|
313
|
+
* injection). If a future change introduces per-skill keys, this
|
|
314
|
+
* cascade must be revisited.
|
|
315
|
+
*
|
|
316
|
+
* - **Recovery cascade** (`ok: true`): when a healthy verification
|
|
317
|
+
* arrives (typically from `refreshUnhealthy` after the operator
|
|
318
|
+
* fixed the key), lift sibling pairs that were stuck `invalid` /
|
|
319
|
+
* `billing` so the next `assertReady` for any model under this
|
|
320
|
+
* provider stops refusing jobs. Without this, the gate would stay
|
|
321
|
+
* half-broken until each sibling's own probe round-trips.
|
|
322
|
+
*
|
|
323
|
+
* `unavailable` does NOT cascade - it is treated as a transient,
|
|
324
|
+
* model-specific issue (e.g. specific endpoint returning 5xx).
|
|
325
|
+
*
|
|
326
|
+
* Cascaded sibling verifications are tagged with `(cascaded from
|
|
327
|
+
* <triggering-model>)` in their body so `snapshot()` and operator
|
|
328
|
+
* logs remain diagnosable.
|
|
329
|
+
*/
|
|
265
330
|
applyVerification(entry, verification) {
|
|
331
|
+
this.mutateEntry(entry, verification);
|
|
332
|
+
if (verification.ok) {
|
|
333
|
+
for (const sibling of this.entries.values()) {
|
|
334
|
+
if (sibling === entry) {
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
if (sibling.provider !== entry.provider) {
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
if (sibling.status !== "invalid" && sibling.status !== "billing") {
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
this.mutateEntry(sibling, { ok: true });
|
|
344
|
+
}
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
if (verification.reason !== "invalid" && verification.reason !== "billing") {
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const cascaded = this.tagCascaded(verification, entry.model);
|
|
351
|
+
for (const sibling of this.entries.values()) {
|
|
352
|
+
if (sibling === entry) {
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
if (sibling.provider !== entry.provider) {
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
this.mutateEntry(sibling, cascaded);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Per-entry state transition. Used by `applyVerification` (which then
|
|
363
|
+
* orchestrates the cascade) and by the cascade loop itself for sibling
|
|
364
|
+
* entries (which must NOT trigger a recursive cascade).
|
|
365
|
+
*/
|
|
366
|
+
mutateEntry(entry, verification) {
|
|
266
367
|
entry.lastVerifiedAt = this.now();
|
|
267
368
|
if (verification.ok) {
|
|
268
369
|
entry.status = "healthy";
|
|
@@ -279,6 +380,25 @@ var LlmHealthMonitor = class {
|
|
|
279
380
|
entry.status = verification.reason;
|
|
280
381
|
entry.consecutiveFailures = 0;
|
|
281
382
|
}
|
|
383
|
+
tagCascaded(verification, fromModel) {
|
|
384
|
+
const tag = `(cascaded from ${fromModel})`;
|
|
385
|
+
if (verification.ok) {
|
|
386
|
+
return verification;
|
|
387
|
+
}
|
|
388
|
+
if (verification.reason === "invalid") {
|
|
389
|
+
return {
|
|
390
|
+
ok: false,
|
|
391
|
+
reason: "invalid",
|
|
392
|
+
status: verification.status,
|
|
393
|
+
body: verification.body ? `${verification.body} ${tag}` : tag
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
if (verification.reason === "billing") {
|
|
397
|
+
const body = verification.body ? `${verification.body} ${tag}` : tag;
|
|
398
|
+
return verification.status !== void 0 ? { ok: false, reason: "billing", status: verification.status, body } : { ok: false, reason: "billing", body };
|
|
399
|
+
}
|
|
400
|
+
return verification;
|
|
401
|
+
}
|
|
282
402
|
};
|
|
283
403
|
|
|
284
404
|
// src/llm-health/heartbeat.ts
|
package/dist/llm-health.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/llm-health/constants.ts","../src/llm-health/types.ts","../src/llm-health/monitor.ts","../src/llm-health/heartbeat.ts","../src/primitives/rateLimiter.ts","../src/llm-health/free-llm-rate-limiter.ts"],"names":[],"mappings":";AAOO,IAAM,qBAAA,GAAwB,KAAK,EAAA,GAAK;AACxC,IAAM,6BAAA,GAAgC,KAAK,EAAA,GAAK;AAShD,IAAM,yBAAA,GAA4B,IAAI,EAAA,GAAK;AAmB3C,IAAM,6BAAA,GAAgC;AAOtC,IAAM,qBAAA,GAAwB;AAM9B,IAAM,uCAAA,GAA0C,KAAK,EAAA,GAAK;AAC1D,IAAM,iCAAA,GAAoC;AAE1C,IAAM,oCAAoC,EAAA,GAAK;AAC/C,IAAM,2BAAA,GAA8B;AAEpC,IAAM,iCAAA,GAAoC;;;ACvB1C,IAAM,cAAA,GAAN,cAA6B,KAAA,CAAM;AAAA,EAC/B,MAAA;AAAA,EACA,QAAA;AAAA,EACA,KAAA;AAAA,EAET,WAAA,CAAY,MAAA,EAA8B,QAAA,EAAkB,KAAA,EAAe,MAAA,EAAgB;AACzF,IAAA,KAAA,CAAM,CAAA,IAAA,EAAO,QAAQ,CAAA,CAAA,EAAI,KAAK,IAAI,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACrD,IAAA,IAAA,CAAK,IAAA,GAAO,gBAAA;AACZ,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,KAAA,GAAQ,KAAA;AAAA,EACf;AACF;AAcO,IAAM,2BAAA,GAAN,cAA0C,KAAA,CAAM;AAAA,EAC5C,QAAA;AAAA,EACA,MAAA;AAAA,EACA,MAAA;AAAA,EAET,WAAA,CAAY,QAAA,EAAkB,MAAA,EAAgB,MAAA,EAAgB;AAC5D,IAAA,MAAM,SAAS,MAAA,CAAO,IAAA,EAAK,IAAK,MAAA,CAAO,MAAK,IAAK,aAAA;AACjD,IAAA,KAAA,CAAM,CAAA,0CAAA,EAA6C,QAAQ,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACxE,IAAA,IAAA,CAAK,IAAA,GAAO,6BAAA;AACZ,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AACF;;;ACpBA,SAAS,KAAA,CAAM,UAAkB,KAAA,EAAuB;AACtD,EAAA,OAAO,CAAA,EAAG,QAAQ,CAAA,EAAA,EAAK,KAAK,CAAA,CAAA;AAC9B;AAEA,SAAS,aAAa,YAAA,EAA0C;AAC9D,EAAA,IAAI,aAAa,EAAA,EAAI;AACnB,IAAA,OAAO,IAAA;AAAA,EACT;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,OAAO,CAAA,KAAA,EAAQ,aAAa,MAAM,CAAA,EAAA,EAAK,aAAa,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,EACxE;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,MAAM,MAAA,GAAS,aAAa,MAAA,IAAU,CAAA;AACtC,IAAA,MAAM,QAAQ,YAAA,CAAa,IAAA,IAAQ,EAAA,EAAI,KAAA,CAAM,GAAG,GAAG,CAAA;AACnD,IAAA,OAAO,CAAA,KAAA,EAAQ,MAAM,CAAA,EAAA,EAAK,IAAI,CAAA,CAAA;AAAA,EAChC;AACA,EAAA,OAAO,YAAA,CAAa,KAAA;AACtB;AAEO,IAAM,mBAAN,MAAuB;AAAA,EACX,OAAA,uBAAc,GAAA,EAAmB;AAAA,EACjC,KAAA;AAAA,EACA,oBAAA;AAAA,EACA,GAAA;AAAA,EAEjB,WAAA,CAAY,OAAA,GAAmC,EAAC,EAAG;AACjD,IAAA,IAAA,CAAK,KAAA,GAAQ,QAAQ,KAAA,IAAS,qBAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,QAAQ,oBAAA,IAAwB,qBAAA;AAC5D,IAAA,IAAA,CAAK,GAAA,GAAM,OAAA,CAAQ,GAAA,IAAO,IAAA,CAAK,GAAA;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAS,IAAA,EAA0B;AACjC,IAAA,MAAM,GAAA,GAAM,KAAA,CAAM,IAAA,CAAK,QAAA,EAAU,KAAK,KAAK,CAAA;AAC3C,IAAA,IAAA,CAAK,OAAA,CAAQ,IAAI,GAAA,EAAK;AAAA,MACpB,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,OAAO,IAAA,CAAK,KAAA;AAAA,MACZ,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,MAAA,EAAQ,SAAA;AAAA,MACR,cAAA,EAAgB,CAAA;AAAA,MAChB,UAAA,EAAY,MAAA;AAAA,MACZ,QAAA,EAAU,IAAA;AAAA,MACV,mBAAA,EAAqB;AAAA,KACtB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,IAAA,CAAK,QAAA,EAAkB,KAAA,EAAe,YAAA,EAAwC;AAC5E,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAAA,EAC5C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,WAAA,CAAY,QAAA,EAAkB,KAAA,EAA8B;AAChE,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,MAAM,IAAI,cAAA,CAAe,SAAA,EAAW,QAAA,EAAU,OAAO,qBAAqB,CAAA;AAAA,IAC5E;AAEA,IAAA,MAAM,YAAA,GAAe,MAAM,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA;AAEnD,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,MAAA,KAAW,SAAA,IAAa,YAAA,CAAa,WAAW,SAAA,EAAW;AAC1E,MAAA,MAAM,IAAI,eAAe,YAAA,CAAa,MAAA,EAAQ,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IAC3F;AACA,IAAA,IAAI,KAAA,CAAM,mBAAA,IAAuB,IAAA,CAAK,oBAAA,EAAsB;AAC1D,MAAA,MAAM,IAAI,cAAA,CAAe,aAAA,EAAe,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IACrF;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,kBAAA,CAAmB,UAAkB,KAAA,EAAqB;AACxD,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,cAAA,GAAiB,CAAA;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,oBAAA,CACE,QAAA,EACA,KAAA,EACA,MAAA,EACA,MAAA,EACM;AACN,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAW,SAAA,EAAW;AACxB,MAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,QAC5B,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,MAAA,EAAQ,CAAA;AAAA,QACR,MAAM,MAAA,IAAU;AAAA,OACjB,CAAA;AACD,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAW,SAAA,EAAW;AACxB,MAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,QAC5B,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,MAAM,MAAA,IAAU;AAAA,OACjB,CAAA;AACD,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,MAC5B,EAAA,EAAI,KAAA;AAAA,MACJ,MAAA,EAAQ,aAAA;AAAA,MACR,OAAO,MAAA,IAAU;AAAA,KAClB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAA,GAAyD;AAC7D,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,gBAAA,GAA+D;AACnE,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,IAAI,KAAA,CAAM,MAAA,KAAW,SAAA,IAAa,KAAA,CAAM,WAAW,SAAA,EAAW;AAC5D,QAAA;AAAA,MACF;AACA,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA,EAGA,QAAA,GAA8C;AAC5C,IAAA,MAAM,MAAgC,EAAC;AACvC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,OAAO,KAAA,CAAM,KAAA;AAAA,QACb,QAAQ,KAAA,CAAM,MAAA;AAAA,QACd,gBAAgB,KAAA,CAAM,cAAA;AAAA,QACtB,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,qBAAqB,KAAA,CAAM;AAAA,OAC5B,CAAA;AAAA,IACH;AACA,IAAA,OAAO,GAAA;AAAA,EACT;AAAA,EAEQ,cAAc,KAAA,EAA2C;AAC/D,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,QAAQ,IAAA,CAAK,GAAA,EAAI,GAAI,KAAA,CAAM,iBAAiB,IAAA,CAAK,KAAA;AACvD,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,IAAI,KAAA,CAAM,WAAW,SAAA,EAAW;AAC9B,QAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,EAAE,EAAA,EAAI,MAAM,CAAA;AAAA,MACrC;AAYA,MAAA,IAAI,KAAA,CAAM,MAAA,KAAW,SAAA,IAAa,KAAA,CAAM,WAAW,SAAA,EAAW;AAC5D,QAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,IAAA,CAAK,0BAAA,CAA2B,KAAK,CAAC,CAAA;AAAA,MAC/D;AAAA,IACF;AACA,IAAA,OAAO,IAAA,CAAK,MAAM,KAAK,CAAA;AAAA,EACzB;AAAA,EAEQ,2BAA2B,KAAA,EAAkC;AACnE,IAAA,MAAM,MAAA,GAAS,MAAM,UAAA,IAAc,gBAAA;AACnC,IAAA,IAAI,KAAA,CAAM,WAAW,SAAA,EAAW;AAC9B,MAAA,OAAO,EAAE,IAAI,KAAA,EAAO,MAAA,EAAQ,WAAW,MAAA,EAAQ,CAAA,EAAG,MAAM,MAAA,EAAO;AAAA,IACjE;AAEA,IAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,SAAA,EAAW,MAAM,MAAA,EAAO;AAAA,EACtD;AAAA,EAEQ,MAAM,KAAA,EAA2C;AACvD,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,WAAW,YAAyC;AACxD,MAAA,IAAI;AACF,QAAA,OAAO,MAAM,MAAM,QAAA,EAAS;AAAA,MAC9B,SAAS,KAAA,EAAO;AACd,QAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,QAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,aAAA,EAAe,OAAO,OAAA,EAAQ;AAAA,MAC5D;AAAA,IACF,CAAA,GAAG,CAAE,IAAA,CAAK,CAAC,YAAA,KAAiB;AAC1B,MAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAC1C,MAAA,KAAA,CAAM,QAAA,GAAW,IAAA;AACjB,MAAA,OAAO,YAAA;AAAA,IACT,CAAC,CAAA;AACD,IAAA,KAAA,CAAM,QAAA,GAAW,OAAA;AACjB,IAAA,OAAO,OAAA;AAAA,EACT;AAAA,EAEQ,iBAAA,CAAkB,OAAc,YAAA,EAAwC;AAC9E,IAAA,KAAA,CAAM,cAAA,GAAiB,KAAK,GAAA,EAAI;AAChC,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA,KAAA,CAAM,MAAA,GAAS,SAAA;AACf,MAAA,KAAA,CAAM,UAAA,GAAa,MAAA;AACnB,MAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAC5B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,UAAA,GAAa,aAAa,YAAY,CAAA;AAC5C,IAAA,IAAI,YAAA,CAAa,WAAW,aAAA,EAAe;AACzC,MAAA,KAAA,CAAM,MAAA,GAAS,aAAA;AACf,MAAA,KAAA,CAAM,mBAAA,IAAuB,CAAA;AAC7B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,SAAS,YAAA,CAAa,MAAA;AAC5B,IAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAAA,EAC9B;AACF;;;AC7RA,IAAM,QAAA,GAAW,CAAC,IAAA,KAAuB;AAAC,CAAA;AAE1C,SAAS,eAAe,MAAA,EAAkC;AACxD,EAAA,OAAO,MAAA,KAAW,aAAa,MAAA,KAAW,SAAA;AAC5C;AAEA,SAAS,kBAAA,CACP,MACA,IAAA,EACoB;AACpB,EAAA,MAAM,UAAA,GAAa,eAAe,IAAI,CAAA;AACtC,EAAA,MAAM,UAAA,GAAa,cAAA,CAAe,IAAA,CAAK,MAAM,CAAA;AAC7C,EAAA,IAAI,eAAe,UAAA,EAAY;AAC7B,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,UAAA,IAAc,CAAC,UAAA,EAAY;AAC7B,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,UAAA,IAAc,IAAA,CAAK,MAAA;AACvC,IAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,IAAA,CAAK,KAAK,CAAA,mBAAA,EAAsB,IAAA,CAAK,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,qCAAA,CAAA;AAAA,EACxG;AACA,EAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,KAAK,KAAK,CAAA,WAAA,CAAA;AAC5D;AAgBO,SAAS,iBAAiB,OAAA,EAAmD;AAClF,EAAA,MAAM,UAAA,GAAa,QAAQ,UAAA,IAAc,yBAAA;AACzC,EAAA,MAAM,GAAA,GAAM,QAAQ,GAAA,IAAO,QAAA;AAE3B,EAAA,IAAI,gBAAA,uBAAuB,GAAA,EAA6B;AACxD,EAAA,KAAA,MAAW,KAAA,IAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS,EAAG;AAC9C,IAAA,gBAAA,CAAiB,GAAA,CAAI,GAAG,KAAA,CAAM,QAAQ,KAAK,KAAA,CAAM,KAAK,CAAA,CAAA,EAAI,KAAA,CAAM,MAAM,CAAA;AAAA,EACxE;AAEA,EAAA,IAAI,OAAA,GAAU,KAAA;AAEd,EAAA,MAAM,OAAO,YAA2B;AACtC,IAAA,IAAI,OAAA,EAAS;AACX,MAAA;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS;AACxC,IAAA,MAAM,YAAA,GAAe,OAAO,IAAA,CAAK,CAAC,UAAU,CAAC,cAAA,CAAe,KAAA,CAAM,MAAM,CAAC,CAAA;AACzE,IAAA,IAAI,CAAC,YAAA,EAAc;AAIjB,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,OAAA,CAAQ,OAAA,CAAQ,gBAAA,EAAiB;AAAA,IACpD,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,MAAA,GAAA,CAAI,CAAA,6BAAA,EAAgC,OAAO,CAAA,CAAE,CAAA;AAC7C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,uBAAW,GAAA,EAA6B;AAC9C,IAAA,KAAA,MAAW,SAAS,QAAA,EAAU;AAC5B,MAAA,MAAM,MAAM,CAAA,EAAG,KAAA,CAAM,QAAQ,CAAA,EAAA,EAAK,MAAM,KAAK,CAAA,CAAA;AAC7C,MAAA,MAAM,IAAA,GAAO,gBAAA,CAAiB,GAAA,CAAI,GAAG,CAAA,IAAK,SAAA;AAC1C,MAAA,MAAM,iBAAA,GAAoB,kBAAA,CAAmB,IAAA,EAAM,KAAK,CAAA;AACxD,MAAA,IAAI,iBAAA,EAAmB;AACrB,QAAA,GAAA,CAAI,iBAAiB,CAAA;AAAA,MACvB;AACA,MAAA,IAAA,CAAK,GAAA,CAAI,GAAA,EAAK,KAAA,CAAM,MAAM,CAAA;AAAA,IAC5B;AACA,IAAA,gBAAA,GAAmB,IAAA;AAAA,EACrB,CAAA;AAEA,EAAA,MAAM,MAAA,GAAyB,YAAY,MAAM;AAC/C,IAAA,KAAK,IAAA,EAAK;AAAA,EACZ,GAAG,UAAU,CAAA;AAEb,EAAA,OAAO;AAAA,IACL,MAAM,MAAM;AACV,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AACA,MAAA,OAAA,GAAU,IAAA;AACV,MAAA,aAAA,CAAc,MAAM,CAAA;AAAA,IACtB;AAAA,GACF;AACF;AASO,IAAM,iBAAA,GAAoB;;;AChG1B,SAAS,2BACd,OAAA,EACsB;AACtB,EAAA,MAAM,EAAE,QAAA,EAAU,YAAA,EAAc,OAAA,EAAQ,GAAI,OAAA;AAC5C,EAAA,IAAI,YAAY,CAAA,EAAG;AACjB,IAAA,MAAM,IAAI,WAAW,sBAAsB,CAAA;AAAA,EAC7C;AACA,EAAA,IAAI,gBAAgB,CAAA,EAAG;AACrB,IAAA,MAAM,IAAI,WAAW,0BAA0B,CAAA;AAAA,EACjD;AACA,EAAA,IAAI,WAAW,CAAA,EAAG;AAChB,IAAA,MAAM,IAAI,WAAW,qBAAqB,CAAA;AAAA,EAC5C;AAIA,EAAA,MAAM,OAAA,uBAAc,GAAA,EAAmB;AAEvC,EAAA,SAAS,aAAA,GAAsB;AAC7B,IAAA,OAAO,OAAA,CAAQ,OAAO,OAAA,EAAS;AAC7B,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,IAAA,EAAK,CAAE,MAAK,CAAE,KAAA;AACxC,MAAA,IAAI,cAAc,MAAA,EAAW;AAC3B,QAAA;AAAA,MACF;AACA,MAAA,OAAA,CAAQ,OAAO,SAAS,CAAA;AAAA,IAC1B;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,IAAA,CAAK,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC7C,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,GAAA,CAAI,GAAG,CAAA;AAC7B,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,OAAO,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,GAAA,GAAM,QAAA,EAAU,OAAO,CAAA,EAAE;AAAA,MAC5D;AACA,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,MAAM,MAAA,GAAS,YAAA;AAAA,QACxB,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC9C,MAAA,MAAM,KAAA,GAAQ,QAAQ,GAAA,CAAI,GAAG,KAAK,EAAE,IAAA,EAAM,EAAC,EAAE;AAC7C,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AAEjE,MAAA,IAAI,KAAA,CAAM,UAAU,YAAA,EAAc;AAGhC,QAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,QAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,QAAA,OAAO;AAAA,UACL,OAAA,EAAS,KAAA;AAAA,UACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,UAC7B,OAAO,KAAA,CAAM;AAAA,SACf;AAAA,MACF;AACA,MAAA,KAAA,CAAM,KAAK,GAAG,CAAA;AACd,MAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,MAAA,aAAA,EAAc;AACd,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,IAAA;AAAA,QACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,GAAM,IAAA,CAAK,GAAA,EAAI,EAAS;AAC5B,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,KAAA,MAAW,CAAC,GAAA,EAAK,KAAK,CAAA,IAAK,OAAA,EAAS;AAClC,QAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,QAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,UAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAAA,QACpB,CAAA,MAAA,IAAW,KAAA,CAAM,MAAA,KAAW,KAAA,CAAM,KAAK,MAAA,EAAQ;AAC7C,UAAA,KAAA,CAAM,IAAA,GAAO,KAAA;AAAA,QACf;AAAA,MACF;AAAA,IACF,CAAA;AAAA,IACA,IAAA,GAAe;AACb,MAAA,OAAO,OAAA,CAAQ,IAAA;AAAA,IACjB,CAAA;AAAA,IACA,KAAA,GAAc;AACZ,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,GACF;AACF;;;ACpHO,IAAM,mBAAA,GAAsB;AAqC5B,SAAS,yBACd,cAAA,EAC4B;AAC5B,EAAA,IAAI,CAAC,cAAA,EAAgB;AACnB,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,cAAA,CAAe,YAAA,IAAgB,CAAA,IAAK,cAAA,CAAe,eAAe,CAAA,EAAG;AACvE,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,OAAO,cAAA;AACT;AAEO,SAAS,uBAAA,CAAwB,OAAA,GAAiC,EAAC,EAAsB;AAC9F,EAAA,MAAM,kBAAA,GAAqC,QAAQ,kBAAA,IAAsB;AAAA,IACvE,WAAA,EAAa,uCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,MAAA,GAAyB,QAAQ,MAAA,IAAU;AAAA,IAC/C,WAAA,EAAa,iCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,OAAA,GAAU,QAAQ,OAAA,IAAW,iCAAA;AAEnC,EAAA,MAAM,gBAAgB,0BAAA,CAA2B;AAAA,IAC/C,UAAU,MAAA,CAAO,WAAA;AAAA,IACjB,cAAc,MAAA,CAAO,YAAA;AAAA,IACrB,OAAA,EAAS;AAAA,GACV,CAAA;AAED,EAAA,MAAM,iBAAiB,0BAAA,CAA2B;AAAA,IAChD,UAAU,kBAAA,CAAmB,WAAA;AAAA,IAC7B,cAAc,kBAAA,CAAmB,YAAA;AAAA,IACjC;AAAA,GACD,CAAA;AAED,EAAA,MAAM,gBAAA,uBAAuB,GAAA,EAAkC;AAE/D,EAAA,SAAS,qBAAA,CACP,WACA,QAAA,EACsB;AACtB,IAAA,MAAM,SAAA,GAAY,yBAAyB,QAAQ,CAAA;AACnD,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,OAAO,cAAA;AAAA,IACT;AACA,IAAA,MAAM,MAAA,GAAS,gBAAA,CAAiB,GAAA,CAAI,SAAS,CAAA;AAC7C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,OAAO,MAAA;AAAA,IACT;AACA,IAAA,MAAM,UAAU,0BAAA,CAA2B;AAAA,MACzC,UAAU,SAAA,CAAU,WAAA;AAAA,MACpB,cAAc,SAAA,CAAU,YAAA;AAAA,MACxB;AAAA,KACD,CAAA;AACD,IAAA,gBAAA,CAAiB,GAAA,CAAI,WAAW,OAAO,CAAA;AACvC,IAAA,OAAO,OAAA;AAAA,EACT;AAEA,EAAA,SAAS,gBAAA,GAAyB;AAChC,IAAA,cAAA,CAAe,KAAA,EAAM;AACrB,IAAA,KAAA,MAAW,OAAA,IAAW,gBAAA,CAAiB,MAAA,EAAO,EAAG;AAC/C,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,aAAA;AAAA,IACA,qBAAA;AAAA,IACA,gBAAA;AAAA,IACA,kBAAA;AAAA,IACA;AAAA,GACF;AACF;AAUO,SAAS,kBAAA,CAAmB,YAAoB,SAAA,EAA2B;AAChF,EAAA,OAAO,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,SAAS,CAAA,CAAA;AACnC","file":"llm-health.js","sourcesContent":["/**\n * LLM health monitor and heartbeat tunable defaults. CLI and plugin consumers\n * can override via `process.env.ELISYM_LLM_HEALTH_TTL_MS` and\n * `ELISYM_LLM_HEARTBEAT_INTERVAL_MS` and pass the resolved values into\n * the monitor/heartbeat options.\n */\n\nexport const DEFAULT_HEALTH_TTL_MS = 10 * 60 * 1000;\nexport const DEFAULT_HEARTBEAT_INTERVAL_MS = 10 * 60 * 1000;\n\n/**\n * Interval between recovery probes after the LLM health monitor enters an\n * unhealthy state. The recovery loop is paused while the pair is healthy\n * and only kicks in reactively (after `markUnhealthyFromJob` or a failed\n * job). On the first successful probe the monitor returns to healthy and\n * the loop stops on its own.\n */\nexport const LAZY_RECOVERY_INTERVAL_MS = 5 * 60 * 1000;\n\n/**\n * Exit code contract: a `dynamic-script` / `static-script` skill returns\n * this code from the script process to signal that the upstream LLM\n * provider rejected the request because credits / billing are exhausted.\n * The agent runtime treats this as the script's equivalent of the\n * `mode: 'llm'` 402 path: it calls `markUnhealthyFromJob(provider, model)`\n * on the health monitor (which starts the lazy recovery loop) and rejects\n * subsequent jobs against the same pair until a recovery probe succeeds.\n *\n * Any other non-zero exit is a generic failure and does NOT touch health\n * state - operators should reserve this code for billing-exhausted only.\n *\n * 42 was chosen because it sits outside POSIX shell conventions (1-2\n * generic, 126-128 shell-internal, 130+ signals) and `sysexits.h`\n * (64-78 - usage / data / host / config errors), so it doesn't collide\n * with other meaningful exit codes a script might naturally produce.\n */\nexport const SCRIPT_EXIT_BILLING_EXHAUSTED = 42;\n\n/**\n * Number of consecutive `unavailable` results tolerated before\n * `assertReady` starts throwing. The first `unavailable - 1` are treated\n * as transient blips so a brief network hiccup does not block jobs.\n */\nexport const UNAVAILABLE_TOLERANCE = 3;\n\n/**\n * Free-LLM rate-limit defaults. Applied when a SKILL.md with\n * `mode: 'llm'` and `price: 0` does not declare its own `rate_limit`.\n */\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS = 60 * 60 * 1000;\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_MAX = 3;\n\nexport const DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS = 60 * 1000;\nexport const DEFAULT_FREE_LLM_GLOBAL_MAX = 30;\n\nexport const DEFAULT_FREE_LLM_MAX_TRACKED_KEYS = 1000;\n","/**\n * Shared types for the LLM health monitor. Provider-agnostic: the\n * actual HTTP probe lives in CLI/plugin and is supplied via dependency\n * injection (`verifyFn`).\n */\n\nexport type LlmHealthStatus = 'unknown' | 'healthy' | 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Result of probing an API key with a specific model. Discriminated on\n * `ok`, then on `reason` for failures. Adding a new failure reason is a\n * breaking change for exhaustive switches; update consumers in lockstep.\n *\n * - `invalid`: HTTP 401/403 - key rejected outright.\n * - `billing`: HTTP 402, or 400 with credit/billing/insufficient marker,\n * or OpenAI's 429 with `insufficient_quota`. Operator out of credits.\n * - `unavailable`: transient (HTTP 429 without quota marker, 5xx, network\n * error). May resolve on retry.\n */\nexport type LlmKeyVerification =\n | { ok: true }\n | { ok: false; reason: 'invalid'; status: number; body: string }\n | { ok: false; reason: 'billing'; status?: number; body?: string }\n | { ok: false; reason: 'unavailable'; error: string };\n\nexport type LlmHealthErrorReason = 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Thrown by `LlmHealthMonitor.assertReady` when the gate refuses a job.\n * Carries the operator-facing reason so callers can log it; customer-facing\n * messages should be sanitized at the call site (see `runtime.ts` preflight).\n */\nexport class LlmHealthError extends Error {\n readonly reason: LlmHealthErrorReason;\n readonly provider: string;\n readonly model: string;\n\n constructor(reason: LlmHealthErrorReason, provider: string, model: string, detail: string) {\n super(`LLM ${provider}/${model} ${reason}: ${detail}`);\n this.name = 'LlmHealthError';\n this.reason = reason;\n this.provider = provider;\n this.model = model;\n }\n}\n\n/**\n * Thrown by SDK script skills (`static-script`, `dynamic-script`) when the\n * spawned process exits with `SCRIPT_EXIT_BILLING_EXHAUSTED` (= 42). The\n * runtime catches this and calls `markUnhealthyFromJob` on the matching\n * (provider, model) pair declared in SKILL.md, so subsequent jobs are\n * gated until the lazy recovery loop re-probes the key. The exit code is\n * the script-side equivalent of the LLM-mode 402 path.\n *\n * The error does NOT carry provider/model itself - the script doesn't\n * know which pair the agent registered with the monitor; the runtime\n * reads them from the matched skill's `llmOverride` declaration.\n */\nexport class ScriptBillingExhaustedError extends Error {\n readonly exitCode: number;\n readonly stderr: string;\n readonly stdout: string;\n\n constructor(exitCode: number, stdout: string, stderr: string) {\n const detail = stderr.trim() || stdout.trim() || '(no output)';\n super(`script exited with billing-exhausted code ${exitCode}: ${detail}`);\n this.name = 'ScriptBillingExhaustedError';\n this.exitCode = exitCode;\n this.stdout = stdout;\n this.stderr = stderr;\n }\n}\n\n/**\n * Per-skill rate-limit declaration. Snake-case in SKILL.md frontmatter,\n * camelCase here. Applies to any skill mode but the framework adds a\n * default cap only for free LLM skills.\n */\nexport interface SkillRateLimit {\n perWindowMs: number;\n maxPerWindow: number;\n}\n\n/** Read-only health snapshot for logs/tests. */\nexport interface LlmHealthSnapshotEntry {\n provider: string;\n model: string;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n consecutiveFailures: number;\n}\n","/**\n * Provider-agnostic health monitor for LLM API keys. State machine per\n * (provider, model) pair: caches the last verification result for\n * `ttlMs`, deduplicates concurrent probes via an in-flight promise, and\n * tolerates a bounded number of consecutive `unavailable` results\n * before `assertReady` starts throwing.\n *\n * The monitor itself never touches the network. Each pair must be\n * registered with a `verifyFn` lambda that performs the actual probe;\n * callers (CLI/plugin) supply provider-specific HTTP from their layer.\n */\n\nimport {\n DEFAULT_HEALTH_TTL_MS,\n UNAVAILABLE_TOLERANCE as DEFAULT_UNAVAILABLE_TOLERANCE,\n} from './constants';\nimport {\n LlmHealthError,\n type LlmHealthSnapshotEntry,\n type LlmHealthStatus,\n type LlmKeyVerification,\n} from './types';\n\nexport type LlmKeyVerifyFn = (signal?: AbortSignal) => Promise<LlmKeyVerification>;\n\nexport interface LlmHealthMonitorOptions {\n /** Time after which a cached `healthy` result is re-probed. Default 10 min. */\n ttlMs?: number;\n /** Number of consecutive unavailable results tolerated. Default 3. */\n unavailableTolerance?: number;\n /** Optional clock injection for tests. */\n now?: () => number;\n}\n\nexport interface RegisterArgs {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n}\n\ninterface Entry {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n inFlight: Promise<LlmKeyVerification> | null;\n consecutiveFailures: number;\n}\n\nfunction keyOf(provider: string, model: string): string {\n return `${provider}::${model}`;\n}\n\nfunction reasonDetail(verification: LlmKeyVerification): string {\n if (verification.ok) {\n return 'ok';\n }\n if (verification.reason === 'invalid') {\n return `HTTP ${verification.status}: ${verification.body.slice(0, 200)}`;\n }\n if (verification.reason === 'billing') {\n const status = verification.status ?? 0;\n const body = (verification.body ?? '').slice(0, 200);\n return `HTTP ${status}: ${body}`;\n }\n return verification.error;\n}\n\nexport class LlmHealthMonitor {\n private readonly entries = new Map<string, Entry>();\n private readonly ttlMs: number;\n private readonly unavailableTolerance: number;\n private readonly now: () => number;\n\n constructor(options: LlmHealthMonitorOptions = {}) {\n this.ttlMs = options.ttlMs ?? DEFAULT_HEALTH_TTL_MS;\n this.unavailableTolerance = options.unavailableTolerance ?? DEFAULT_UNAVAILABLE_TOLERANCE;\n this.now = options.now ?? Date.now;\n }\n\n /**\n * Register a (provider, model) pair with its probe function. Idempotent\n * on the (provider, model) key: re-registering replaces the verifyFn\n * and resets state to `unknown`. Callers typically re-register only\n * when the operator rotates the API key.\n */\n register(args: RegisterArgs): void {\n const key = keyOf(args.provider, args.model);\n this.entries.set(key, {\n provider: args.provider,\n model: args.model,\n verifyFn: args.verifyFn,\n status: 'unknown',\n lastVerifiedAt: 0,\n lastReason: undefined,\n inFlight: null,\n consecutiveFailures: 0,\n });\n }\n\n /**\n * Seed the monitor with an already-known verification result, e.g.\n * the one captured at startup. Skips an extra probe on the first\n * `assertReady` call within the TTL window. No-op if the pair is not\n * registered.\n */\n seed(provider: string, model: string, verification: LlmKeyVerification): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n this.applyVerification(entry, verification);\n }\n\n /**\n * Main gate before doing LLM work. Throws `LlmHealthError` on terminal\n * states (`invalid`, `billing`, or `unavailable` past tolerance);\n * resolves silently on `healthy` (cache hit or fresh probe) or\n * tolerated `unavailable`.\n *\n * Concurrent `assertReady` calls for the same pair are deduplicated:\n * the second caller awaits the first probe rather than launching a\n * parallel one.\n */\n async assertReady(provider: string, model: string): Promise<void> {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n throw new LlmHealthError('invalid', provider, model, 'pair not registered');\n }\n\n const verification = await this.probeIfNeeded(entry);\n\n if (verification.ok) {\n return;\n }\n if (verification.reason === 'invalid' || verification.reason === 'billing') {\n throw new LlmHealthError(verification.reason, provider, model, reasonDetail(verification));\n }\n if (entry.consecutiveFailures >= this.unavailableTolerance) {\n throw new LlmHealthError('unavailable', provider, model, reasonDetail(verification));\n }\n // Tolerated unavailable: caller proceeds, real LLM call will surface\n // any transient issue with its own error path.\n }\n\n /**\n * Force the next `assertReady` for this pair to re-probe regardless of\n * TTL. Used when a real LLM call surfaces 401/402 mid-job - the cached\n * `healthy` is stale and we want to catch the next request.\n */\n markFailureFromJob(provider: string, model: string): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n entry.lastVerifiedAt = 0;\n }\n\n /**\n * Reactively flip a (provider, model) pair to unhealthy without doing a\n * fresh probe. Called from the runtime when a job's actual LLM call (or\n * a script's `SCRIPT_EXIT_BILLING_EXHAUSTED` exit) surfaces a billing /\n * invalid signal: the cached `healthy` snapshot is wrong and we want\n * subsequent `assertReady` calls to refuse jobs immediately, before the\n * lazy recovery loop notices on its own. No-op if the pair is not\n * registered.\n *\n * Recovery from this state happens through a successful probe (typically\n * fired by `startLlmRecovery`) which flips the pair back to healthy via\n * `applyVerification`.\n */\n markUnhealthyFromJob(\n provider: string,\n model: string,\n reason: 'billing' | 'invalid' | 'unavailable',\n detail?: string,\n ): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n if (reason === 'invalid') {\n this.applyVerification(entry, {\n ok: false,\n reason: 'invalid',\n status: 0,\n body: detail ?? 'reactive markUnhealthyFromJob',\n });\n return;\n }\n if (reason === 'billing') {\n this.applyVerification(entry, {\n ok: false,\n reason: 'billing',\n body: detail ?? 'reactive markUnhealthyFromJob',\n });\n return;\n }\n this.applyVerification(entry, {\n ok: false,\n reason: 'unavailable',\n error: detail ?? 'reactive markUnhealthyFromJob',\n });\n }\n\n /**\n * Refresh every registered pair concurrently. Errors thrown by\n * `verifyFn` are caught and recorded as `unavailable`.\n */\n async refreshAll(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n for (const entry of this.entries.values()) {\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /**\n * Refresh only the pairs whose current status is non-healthy\n * (`invalid`, `billing`, or `unavailable`). Used by the lazy recovery\n * loop so a billing outage on one provider does not trigger throwaway\n * probes on every other healthy pair the agent has registered.\n * Errors thrown by `verifyFn` are caught and recorded as `unavailable`.\n */\n async refreshUnhealthy(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n for (const entry of this.entries.values()) {\n if (entry.status === 'healthy' || entry.status === 'unknown') {\n continue;\n }\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /** Read-only view, primarily for logs and tests. */\n snapshot(): readonly LlmHealthSnapshotEntry[] {\n const out: LlmHealthSnapshotEntry[] = [];\n for (const entry of this.entries.values()) {\n out.push({\n provider: entry.provider,\n model: entry.model,\n status: entry.status,\n lastVerifiedAt: entry.lastVerifiedAt,\n lastReason: entry.lastReason,\n consecutiveFailures: entry.consecutiveFailures,\n });\n }\n return out;\n }\n\n private probeIfNeeded(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const fresh = this.now() - entry.lastVerifiedAt < this.ttlMs;\n if (fresh) {\n if (entry.status === 'healthy') {\n return Promise.resolve({ ok: true });\n }\n // Terminal failure states (`invalid`, `billing`) are cached\n // aggressively: once the operator's key is rejected or out of\n // credits, the lazy recovery loop is the only path back to healthy.\n // Re-probing on every `assertReady` would burn one billing token\n // per job rejected during a billing outage, which defeats the\n // gate's whole purpose. `unavailable` is treated as transient and\n // falls through so the existing tolerance counter still works\n // (each call re-probes, increments `consecutiveFailures`, and\n // crosses the threshold on the configured number of attempts).\n // `unknown` also falls through - a freshly registered pair that\n // was never seeded should probe on first ask.\n if (entry.status === 'invalid' || entry.status === 'billing') {\n return Promise.resolve(this.synthesizeFailureFromCache(entry));\n }\n }\n return this.probe(entry);\n }\n\n private synthesizeFailureFromCache(entry: Entry): LlmKeyVerification {\n const detail = entry.lastReason ?? 'cached failure';\n if (entry.status === 'invalid') {\n return { ok: false, reason: 'invalid', status: 0, body: detail };\n }\n // 'billing' (only other status reachable here)\n return { ok: false, reason: 'billing', body: detail };\n }\n\n private probe(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const promise = (async (): Promise<LlmKeyVerification> => {\n try {\n return await entry.verifyFn();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n return { ok: false, reason: 'unavailable', error: message };\n }\n })().then((verification) => {\n this.applyVerification(entry, verification);\n entry.inFlight = null;\n return verification;\n });\n entry.inFlight = promise;\n return promise;\n }\n\n private applyVerification(entry: Entry, verification: LlmKeyVerification): void {\n entry.lastVerifiedAt = this.now();\n if (verification.ok) {\n entry.status = 'healthy';\n entry.lastReason = undefined;\n entry.consecutiveFailures = 0;\n return;\n }\n entry.lastReason = reasonDetail(verification);\n if (verification.reason === 'unavailable') {\n entry.status = 'unavailable';\n entry.consecutiveFailures += 1;\n return;\n }\n entry.status = verification.reason;\n entry.consecutiveFailures = 0;\n }\n}\n","/**\n * Lazy LLM recovery probe. While every registered (provider, model) pair\n * is healthy, ticks are no-ops: zero API traffic, zero billing tokens\n * burned. The loop only does real work after a pair has flipped to\n * unhealthy (via `markUnhealthyFromJob` from the runtime, or from a\n * preflight probe that returned a non-ok verification): each tick\n * re-probes only the unhealthy pairs and, on success, flips them back to\n * healthy via the monitor's normal `applyVerification` path.\n *\n * The recovery loop is the only path back to `healthy` after a reactive\n * markUnhealthy. Without it, the agent would stay locked out until\n * restart even after the operator pops their billing back up.\n *\n * Logging policy lives here so the monitor stays a pure state-machine.\n * Status transitions (healthy <-> unhealthy) are logged once per change;\n * routine successful re-probes are not logged.\n */\n\nimport { LAZY_RECOVERY_INTERVAL_MS } from './constants';\nimport type { LlmHealthMonitor } from './monitor';\nimport type { LlmHealthSnapshotEntry, LlmHealthStatus } from './types';\n\nexport interface HeartbeatHandle {\n stop(): void;\n}\n\nexport interface StartLlmRecoveryOptions {\n monitor: LlmHealthMonitor;\n /** Defaults to {@link LAZY_RECOVERY_INTERVAL_MS} (5 minutes). */\n intervalMs?: number;\n /** Operator log sink. Defaults to no-op (silent). */\n log?: (msg: string) => void;\n}\n\n/**\n * @deprecated Renamed to {@link StartLlmRecoveryOptions}. Kept as an alias\n * so external consumers keep building during the rename. The semantics\n * have changed (lazy, recovery-only) but the option surface is identical.\n */\nexport type StartLlmHeartbeatOptions = StartLlmRecoveryOptions;\n\ntype IntervalHandle = ReturnType<typeof setInterval>;\n\nconst NOOP_LOG = (_msg: string): void => {};\n\nfunction isHealthyState(status: LlmHealthStatus): boolean {\n return status === 'healthy' || status === 'unknown';\n}\n\nfunction describeTransition(\n prev: LlmHealthStatus,\n next: LlmHealthSnapshotEntry,\n): string | undefined {\n const wasHealthy = isHealthyState(prev);\n const nowHealthy = isHealthyState(next.status);\n if (wasHealthy === nowHealthy) {\n return undefined;\n }\n if (wasHealthy && !nowHealthy) {\n const reason = next.lastReason ?? next.status;\n return `! LLM provider ${next.provider} model ${next.model} became unhealthy: ${next.status} (${reason}). Refusing LLM jobs until recovered.`;\n }\n return `* LLM provider ${next.provider} model ${next.model} recovered.`;\n}\n\n/**\n * Start the lazy recovery loop. Returns a handle whose `stop()` cancels\n * the timer (idempotent). The loop ticks every `intervalMs` ms; each\n * tick scans `monitor.snapshot()` and, if any pair is non-healthy, asks\n * the monitor to re-probe just those non-healthy pairs via\n * `refreshUnhealthy()`. Healthy pairs are never re-probed by the loop -\n * those keep their cached `healthy` until TTL expiry forces a probe at\n * the next `assertReady` call. When all pairs are healthy the tick is a\n * single Map walk and no API calls are made.\n *\n * The function is named `startLlmRecovery` but the legacy export\n * `startLlmHeartbeat` (below) is preserved as an alias so external\n * code that already imports it keeps working unchanged.\n */\nexport function startLlmRecovery(options: StartLlmRecoveryOptions): HeartbeatHandle {\n const intervalMs = options.intervalMs ?? LAZY_RECOVERY_INTERVAL_MS;\n const log = options.log ?? NOOP_LOG;\n\n let lastStatusByPair = new Map<string, LlmHealthStatus>();\n for (const entry of options.monitor.snapshot()) {\n lastStatusByPair.set(`${entry.provider}::${entry.model}`, entry.status);\n }\n\n let stopped = false;\n\n const tick = async (): Promise<void> => {\n if (stopped) {\n return;\n }\n const before = options.monitor.snapshot();\n const anyUnhealthy = before.some((entry) => !isHealthyState(entry.status));\n if (!anyUnhealthy) {\n // Lazy: nothing to do. Refreshing healthy pairs would burn one\n // billing-token per pair per tick for no benefit (the monitor's\n // own TTL would re-probe at assertReady time anyway).\n return;\n }\n\n let snapshot: readonly LlmHealthSnapshotEntry[];\n try {\n snapshot = await options.monitor.refreshUnhealthy();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n log(`! LLM recovery probe failed: ${message}`);\n return;\n }\n\n const next = new Map<string, LlmHealthStatus>();\n for (const entry of snapshot) {\n const key = `${entry.provider}::${entry.model}`;\n const prev = lastStatusByPair.get(key) ?? 'unknown';\n const transitionMessage = describeTransition(prev, entry);\n if (transitionMessage) {\n log(transitionMessage);\n }\n next.set(key, entry.status);\n }\n lastStatusByPair = next;\n };\n\n const handle: IntervalHandle = setInterval(() => {\n void tick();\n }, intervalMs);\n\n return {\n stop: () => {\n if (stopped) {\n return;\n }\n stopped = true;\n clearInterval(handle);\n },\n };\n}\n\n/**\n * @deprecated Renamed to {@link startLlmRecovery}. The old name is\n * preserved as an alias so existing imports keep working during the\n * rename. Behavior changed materially: the new loop is lazy\n * (no API calls while all pairs are healthy) and defaults to\n * `LAZY_RECOVERY_INTERVAL_MS` (5 min) instead of 10 min.\n */\nexport const startLlmHeartbeat = startLlmRecovery;\n","/**\n * Sliding-window rate limiter keyed by an arbitrary string (typically a\n * customer pubkey). Each key gets at most `maxPerWindow` requests inside a\n * rolling `windowMs`. Stale timestamps are pruned lazily on every `check`.\n * When the tracked-key set grows past `maxKeys`, the least-recently-used\n * key is evicted so an attacker cannot exhaust memory by cycling keys.\n *\n * Thread-safety: not required. Designed for single-threaded JS consumers\n * (Node/Bun event loops, browser main thread). No timers - pruning happens\n * inside `check` and `prune`.\n */\n\nexport interface SlidingWindowLimiterOptions {\n /** Rolling window width, in ms. */\n windowMs: number;\n /** Max hits allowed per key inside the window. */\n maxPerWindow: number;\n /** Cap on total tracked keys. LRU-evicted past this cap. */\n maxKeys: number;\n}\n\nexport interface RateLimitDecision {\n allowed: boolean;\n /** Wall-clock timestamp (ms) when the limit window will reset for this key. */\n resetAt: number;\n /** Number of hits inside the current window after this call (or the attempted hit if denied). */\n count: number;\n}\n\nexport interface SlidingWindowLimiter {\n /** Record a hit against `key`; return whether it was allowed. */\n check(key: string, now?: number): RateLimitDecision;\n /**\n * Inspect the current state for `key` without recording a hit. Useful\n * when a single request must clear multiple limiters and callers want\n * to avoid double-counting against one limiter after another denies.\n */\n peek(key: string, now?: number): RateLimitDecision;\n /** Drop entries whose windows have fully elapsed. Bounded memory hygiene. */\n prune(now?: number): void;\n /** Current number of tracked keys. */\n size(): number;\n /** Clear all state. */\n reset(): void;\n}\n\ninterface Entry {\n /** Sliding-window timestamps in ms. Sorted ascending. */\n hits: number[];\n}\n\nexport function createSlidingWindowLimiter(\n options: SlidingWindowLimiterOptions,\n): SlidingWindowLimiter {\n const { windowMs, maxPerWindow, maxKeys } = options;\n if (windowMs <= 0) {\n throw new RangeError('windowMs must be > 0');\n }\n if (maxPerWindow <= 0) {\n throw new RangeError('maxPerWindow must be > 0');\n }\n if (maxKeys <= 0) {\n throw new RangeError('maxKeys must be > 0');\n }\n\n // LRU is implemented via Map's insertion-order: every check refreshes\n // the entry by deleting and re-setting it, moving it to the tail.\n const entries = new Map<string, Entry>();\n\n function evictIfNeeded(): void {\n while (entries.size > maxKeys) {\n const oldestKey = entries.keys().next().value as string | undefined;\n if (oldestKey === undefined) {\n return;\n }\n entries.delete(oldestKey);\n }\n }\n\n return {\n peek(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key);\n if (!entry) {\n return { allowed: true, resetAt: now + windowMs, count: 0 };\n }\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n return {\n allowed: fresh.length < maxPerWindow,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n check(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key) ?? { hits: [] };\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n\n if (fresh.length >= maxPerWindow) {\n // Refresh LRU order even on denial so an attacker hammering the\n // same key cannot push other tracked keys out via eviction.\n entries.delete(key);\n entries.set(key, { hits: fresh });\n return {\n allowed: false,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n }\n fresh.push(now);\n entries.delete(key);\n entries.set(key, { hits: fresh });\n evictIfNeeded();\n return {\n allowed: true,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n prune(now = Date.now()): void {\n const cutoff = now - windowMs;\n for (const [key, entry] of entries) {\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n if (fresh.length === 0) {\n entries.delete(key);\n } else if (fresh.length !== entry.hits.length) {\n entry.hits = fresh;\n }\n }\n },\n size(): number {\n return entries.size;\n },\n reset(): void {\n entries.clear();\n },\n };\n}\n","/**\n * Two-tier rate limiter for free LLM skills (mode='llm', price=0).\n * Provides Sybil-resistant global cap plus per-customer-per-skill cap\n * that respects an optional skill-level override.\n *\n * Both limiters are independent of the existing per-customer general\n * limiter; callers (CLI runtime, plugin handler) check all of them in\n * sequence and only `check()` after every tier passes `peek()` so a\n * single denial does not consume slots in earlier tiers.\n */\n\nimport { createSlidingWindowLimiter, type SlidingWindowLimiter } from '../primitives/rateLimiter';\nimport {\n DEFAULT_FREE_LLM_GLOBAL_MAX,\n DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n DEFAULT_FREE_LLM_MAX_TRACKED_KEYS,\n DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n} from './constants';\nimport type { SkillRateLimit } from './types';\n\nexport const FREE_LLM_GLOBAL_KEY = '__free_llm_global__';\n\nexport interface FreeLlmLimiterOptions {\n /** Max tracked (customer, skill) keys. Default 1000 (LRU evicted past cap). */\n maxKeys?: number;\n /** Default per-customer cap when a free LLM skill omits `rate_limit`. */\n defaultPerCustomer?: SkillRateLimit;\n /** Global Sybil cap across all free LLM jobs. */\n global?: SkillRateLimit;\n}\n\nexport interface FreeLlmLimiterSet {\n /** Sybil-protection limiter keyed on `FREE_LLM_GLOBAL_KEY`. */\n globalLimiter: SlidingWindowLimiter;\n /**\n * Return the per-customer limiter for a given skill. Skills that\n * declare a `rate_limit` get their own limiter sized to that\n * (window, cap); skills that don't share a default-window limiter.\n * Each call uses `peek/check` keyed on `customerId` alone since each\n * skill has a dedicated limiter store.\n */\n getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter;\n /** Drop expired hits from every per-customer limiter (default + per-skill). */\n prunePerCustomer(): void;\n /** Default cap to apply when a skill omits `rate_limit`. */\n defaultPerCustomer: SkillRateLimit;\n /** Sybil-cap settings (echo of global option for diagnostics). */\n global: SkillRateLimit;\n}\n\n/**\n * Effective per-skill rate limit: the override if it's well-formed,\n * else `undefined` so callers can fall through to the default limiter.\n */\nexport function resolvePerSkillRateLimit(\n skillRateLimit: SkillRateLimit | undefined,\n): SkillRateLimit | undefined {\n if (!skillRateLimit) {\n return undefined;\n }\n if (skillRateLimit.maxPerWindow <= 0 || skillRateLimit.perWindowMs <= 0) {\n return undefined;\n }\n return skillRateLimit;\n}\n\nexport function createFreeLlmLimiterSet(options: FreeLlmLimiterOptions = {}): FreeLlmLimiterSet {\n const defaultPerCustomer: SkillRateLimit = options.defaultPerCustomer ?? {\n perWindowMs: DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n };\n const global: SkillRateLimit = options.global ?? {\n perWindowMs: DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_GLOBAL_MAX,\n };\n const maxKeys = options.maxKeys ?? DEFAULT_FREE_LLM_MAX_TRACKED_KEYS;\n\n const globalLimiter = createSlidingWindowLimiter({\n windowMs: global.perWindowMs,\n maxPerWindow: global.maxPerWindow,\n maxKeys: 1,\n });\n\n const defaultLimiter = createSlidingWindowLimiter({\n windowMs: defaultPerCustomer.perWindowMs,\n maxPerWindow: defaultPerCustomer.maxPerWindow,\n maxKeys,\n });\n\n const perSkillLimiters = new Map<string, SlidingWindowLimiter>();\n\n function getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter {\n const effective = resolvePerSkillRateLimit(override);\n if (!effective) {\n return defaultLimiter;\n }\n const cached = perSkillLimiters.get(skillName);\n if (cached) {\n return cached;\n }\n const limiter = createSlidingWindowLimiter({\n windowMs: effective.perWindowMs,\n maxPerWindow: effective.maxPerWindow,\n maxKeys,\n });\n perSkillLimiters.set(skillName, limiter);\n return limiter;\n }\n\n function prunePerCustomer(): void {\n defaultLimiter.prune();\n for (const limiter of perSkillLimiters.values()) {\n limiter.prune();\n }\n }\n\n return {\n globalLimiter,\n getPerCustomerLimiter,\n prunePerCustomer,\n defaultPerCustomer,\n global,\n };\n}\n\n/**\n * Compose a per-skill key for the per-customer limiter. The default\n * limiter is shared across skills without an override, so the skill\n * name is needed to keep each (customer, skill) pair counted\n * independently. Per-skill limiters use the same key for consistency\n * even though the skill component is redundant inside a dedicated\n * limiter store.\n */\nexport function freeLlmCustomerKey(customerId: string, skillName: string): string {\n return `${customerId}|${skillName}`;\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../src/llm-health/constants.ts","../src/llm-health/types.ts","../src/llm-health/monitor.ts","../src/llm-health/heartbeat.ts","../src/primitives/rateLimiter.ts","../src/llm-health/free-llm-rate-limiter.ts"],"names":[],"mappings":";AAOO,IAAM,qBAAA,GAAwB,KAAK,EAAA,GAAK;AACxC,IAAM,6BAAA,GAAgC,KAAK,EAAA,GAAK;AAShD,IAAM,yBAAA,GAA4B,IAAI,EAAA,GAAK;AAmB3C,IAAM,6BAAA,GAAgC;AAOtC,IAAM,qBAAA,GAAwB;AAM9B,IAAM,uCAAA,GAA0C,KAAK,EAAA,GAAK;AAC1D,IAAM,iCAAA,GAAoC;AAE1C,IAAM,oCAAoC,EAAA,GAAK;AAC/C,IAAM,2BAAA,GAA8B;AAEpC,IAAM,iCAAA,GAAoC;;;ACvB1C,IAAM,cAAA,GAAN,cAA6B,KAAA,CAAM;AAAA,EAC/B,MAAA;AAAA,EACA,QAAA;AAAA,EACA,KAAA;AAAA,EAET,WAAA,CAAY,MAAA,EAA8B,QAAA,EAAkB,KAAA,EAAe,MAAA,EAAgB;AACzF,IAAA,KAAA,CAAM,CAAA,IAAA,EAAO,QAAQ,CAAA,CAAA,EAAI,KAAK,IAAI,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACrD,IAAA,IAAA,CAAK,IAAA,GAAO,gBAAA;AACZ,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,KAAA,GAAQ,KAAA;AAAA,EACf;AACF;AAcO,IAAM,2BAAA,GAAN,cAA0C,KAAA,CAAM;AAAA,EAC5C,QAAA;AAAA,EACA,MAAA;AAAA,EACA,MAAA;AAAA,EAET,WAAA,CAAY,QAAA,EAAkB,MAAA,EAAgB,MAAA,EAAgB;AAC5D,IAAA,MAAM,SAAS,MAAA,CAAO,IAAA,EAAK,IAAK,MAAA,CAAO,MAAK,IAAK,aAAA;AACjD,IAAA,KAAA,CAAM,CAAA,0CAAA,EAA6C,QAAQ,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACxE,IAAA,IAAA,CAAK,IAAA,GAAO,6BAAA;AACZ,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AACF;;;ACpBA,SAAS,KAAA,CAAM,UAAkB,KAAA,EAAuB;AACtD,EAAA,OAAO,CAAA,EAAG,QAAQ,CAAA,EAAA,EAAK,KAAK,CAAA,CAAA;AAC9B;AAEA,SAAS,aAAa,YAAA,EAA0C;AAC9D,EAAA,IAAI,aAAa,EAAA,EAAI;AACnB,IAAA,OAAO,IAAA;AAAA,EACT;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,OAAO,CAAA,KAAA,EAAQ,aAAa,MAAM,CAAA,EAAA,EAAK,aAAa,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,EACxE;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,MAAM,MAAA,GAAS,aAAa,MAAA,IAAU,CAAA;AACtC,IAAA,MAAM,QAAQ,YAAA,CAAa,IAAA,IAAQ,EAAA,EAAI,KAAA,CAAM,GAAG,GAAG,CAAA;AACnD,IAAA,OAAO,CAAA,KAAA,EAAQ,MAAM,CAAA,EAAA,EAAK,IAAI,CAAA,CAAA;AAAA,EAChC;AACA,EAAA,OAAO,YAAA,CAAa,KAAA;AACtB;AAEO,IAAM,mBAAN,MAAuB;AAAA,EACX,OAAA,uBAAc,GAAA,EAAmB;AAAA,EACjC,KAAA;AAAA,EACA,oBAAA;AAAA,EACA,GAAA;AAAA,EAEjB,WAAA,CAAY,OAAA,GAAmC,EAAC,EAAG;AACjD,IAAA,IAAA,CAAK,KAAA,GAAQ,QAAQ,KAAA,IAAS,qBAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,QAAQ,oBAAA,IAAwB,qBAAA;AAC5D,IAAA,IAAA,CAAK,GAAA,GAAM,OAAA,CAAQ,GAAA,IAAO,IAAA,CAAK,GAAA;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAS,IAAA,EAA0B;AACjC,IAAA,MAAM,GAAA,GAAM,KAAA,CAAM,IAAA,CAAK,QAAA,EAAU,KAAK,KAAK,CAAA;AAC3C,IAAA,IAAA,CAAK,OAAA,CAAQ,IAAI,GAAA,EAAK;AAAA,MACpB,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,OAAO,IAAA,CAAK,KAAA;AAAA,MACZ,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,MAAA,EAAQ,SAAA;AAAA,MACR,cAAA,EAAgB,CAAA;AAAA,MAChB,UAAA,EAAY,MAAA;AAAA,MACZ,QAAA,EAAU,IAAA;AAAA,MACV,mBAAA,EAAqB;AAAA,KACtB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,IAAA,CAAK,QAAA,EAAkB,KAAA,EAAe,YAAA,EAAwC;AAC5E,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAAA,EAC5C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,WAAA,CAAY,QAAA,EAAkB,KAAA,EAA8B;AAChE,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,MAAM,IAAI,cAAA,CAAe,SAAA,EAAW,QAAA,EAAU,OAAO,qBAAqB,CAAA;AAAA,IAC5E;AAEA,IAAA,MAAM,YAAA,GAAe,MAAM,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA;AAEnD,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,MAAA,KAAW,SAAA,IAAa,YAAA,CAAa,WAAW,SAAA,EAAW;AAC1E,MAAA,MAAM,IAAI,eAAe,YAAA,CAAa,MAAA,EAAQ,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IAC3F;AACA,IAAA,IAAI,KAAA,CAAM,mBAAA,IAAuB,IAAA,CAAK,oBAAA,EAAsB;AAC1D,MAAA,MAAM,IAAI,cAAA,CAAe,aAAA,EAAe,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IACrF;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,kBAAA,CAAmB,UAAkB,KAAA,EAAqB;AACxD,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,cAAA,GAAiB,CAAA;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAqBA,oBAAA,CACE,QAAA,EACA,KAAA,EACA,MAAA,EACA,MAAA,EACM;AACN,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAW,SAAA,EAAW;AACxB,MAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,QAC5B,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,MAAA,EAAQ,CAAA;AAAA,QACR,MAAM,MAAA,IAAU;AAAA,OACjB,CAAA;AACD,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAW,SAAA,EAAW;AACxB,MAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,QAC5B,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,MAAM,MAAA,IAAU;AAAA,OACjB,CAAA;AACD,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,MAC5B,EAAA,EAAI,KAAA;AAAA,MACJ,MAAA,EAAQ,aAAA;AAAA,MACR,OAAO,MAAA,IAAU;AAAA,KAClB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAA,GAAyD;AAC7D,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAoBA,MAAM,gBAAA,GAA+D;AACnE,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,MAAM,kBAAA,uBAAyB,GAAA,EAAY;AAC3C,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,IAAI,KAAA,CAAM,MAAA,KAAW,SAAA,IAAa,KAAA,CAAM,WAAW,SAAA,EAAW;AAC5D,QAAA;AAAA,MACF;AACA,MAAA,IAAI,KAAA,CAAM,WAAW,aAAA,EAAe;AAClC,QAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AACnD,QAAA;AAAA,MACF;AAEA,MAAA,IAAI,kBAAA,CAAmB,GAAA,CAAI,KAAA,CAAM,QAAQ,CAAA,EAAG;AAC1C,QAAA;AAAA,MACF;AACA,MAAA,kBAAA,CAAmB,GAAA,CAAI,MAAM,QAAQ,CAAA;AACrC,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA,EAGA,QAAA,GAA8C;AAC5C,IAAA,MAAM,MAAgC,EAAC;AACvC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,OAAO,KAAA,CAAM,KAAA;AAAA,QACb,QAAQ,KAAA,CAAM,MAAA;AAAA,QACd,gBAAgB,KAAA,CAAM,cAAA;AAAA,QACtB,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,qBAAqB,KAAA,CAAM;AAAA,OAC5B,CAAA;AAAA,IACH;AACA,IAAA,OAAO,GAAA;AAAA,EACT;AAAA,EAEQ,cAAc,KAAA,EAA2C;AAC/D,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,QAAQ,IAAA,CAAK,GAAA,EAAI,GAAI,KAAA,CAAM,iBAAiB,IAAA,CAAK,KAAA;AACvD,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,IAAI,KAAA,CAAM,WAAW,SAAA,EAAW;AAC9B,QAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,EAAE,EAAA,EAAI,MAAM,CAAA;AAAA,MACrC;AAYA,MAAA,IAAI,KAAA,CAAM,MAAA,KAAW,SAAA,IAAa,KAAA,CAAM,WAAW,SAAA,EAAW;AAC5D,QAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,IAAA,CAAK,0BAAA,CAA2B,KAAK,CAAC,CAAA;AAAA,MAC/D;AAAA,IACF;AACA,IAAA,OAAO,IAAA,CAAK,MAAM,KAAK,CAAA;AAAA,EACzB;AAAA,EAEQ,2BAA2B,KAAA,EAAkC;AACnE,IAAA,MAAM,MAAA,GAAS,MAAM,UAAA,IAAc,gBAAA;AACnC,IAAA,IAAI,KAAA,CAAM,WAAW,SAAA,EAAW;AAC9B,MAAA,OAAO,EAAE,IAAI,KAAA,EAAO,MAAA,EAAQ,WAAW,MAAA,EAAQ,CAAA,EAAG,MAAM,MAAA,EAAO;AAAA,IACjE;AAEA,IAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,SAAA,EAAW,MAAM,MAAA,EAAO;AAAA,EACtD;AAAA,EAEQ,MAAM,KAAA,EAA2C;AACvD,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,WAAW,YAAyC;AACxD,MAAA,IAAI;AACF,QAAA,OAAO,MAAM,MAAM,QAAA,EAAS;AAAA,MAC9B,SAAS,KAAA,EAAO;AACd,QAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,QAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,aAAA,EAAe,OAAO,OAAA,EAAQ;AAAA,MAC5D;AAAA,IACF,CAAA,GAAG,CAAE,IAAA,CAAK,CAAC,YAAA,KAAiB;AAC1B,MAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAC1C,MAAA,KAAA,CAAM,QAAA,GAAW,IAAA;AACjB,MAAA,OAAO,YAAA;AAAA,IACT,CAAC,CAAA;AACD,IAAA,KAAA,CAAM,QAAA,GAAW,OAAA;AACjB,IAAA,OAAO,OAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA4BQ,iBAAA,CAAkB,OAAc,YAAA,EAAwC;AAC9E,IAAA,IAAA,CAAK,WAAA,CAAY,OAAO,YAAY,CAAA;AAEpC,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA,KAAA,MAAW,OAAA,IAAW,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AAC3C,QAAA,IAAI,YAAY,KAAA,EAAO;AACrB,UAAA;AAAA,QACF;AACA,QAAA,IAAI,OAAA,CAAQ,QAAA,KAAa,KAAA,CAAM,QAAA,EAAU;AACvC,UAAA;AAAA,QACF;AACA,QAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,SAAA,IAAa,OAAA,CAAQ,WAAW,SAAA,EAAW;AAChE,UAAA;AAAA,QACF;AACA,QAAA,IAAA,CAAK,WAAA,CAAY,OAAA,EAAS,EAAE,EAAA,EAAI,MAAM,CAAA;AAAA,MACxC;AACA,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,YAAA,CAAa,MAAA,KAAW,SAAA,IAAa,YAAA,CAAa,WAAW,SAAA,EAAW;AAC1E,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,WAAA,CAAY,YAAA,EAAc,MAAM,KAAK,CAAA;AAC3D,IAAA,KAAA,MAAW,OAAA,IAAW,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AAC3C,MAAA,IAAI,YAAY,KAAA,EAAO;AACrB,QAAA;AAAA,MACF;AACA,MAAA,IAAI,OAAA,CAAQ,QAAA,KAAa,KAAA,CAAM,QAAA,EAAU;AACvC,QAAA;AAAA,MACF;AACA,MAAA,IAAA,CAAK,WAAA,CAAY,SAAS,QAAQ,CAAA;AAAA,IACpC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOQ,WAAA,CAAY,OAAc,YAAA,EAAwC;AACxE,IAAA,KAAA,CAAM,cAAA,GAAiB,KAAK,GAAA,EAAI;AAChC,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA,KAAA,CAAM,MAAA,GAAS,SAAA;AACf,MAAA,KAAA,CAAM,UAAA,GAAa,MAAA;AACnB,MAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAC5B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,UAAA,GAAa,aAAa,YAAY,CAAA;AAC5C,IAAA,IAAI,YAAA,CAAa,WAAW,aAAA,EAAe;AACzC,MAAA,KAAA,CAAM,MAAA,GAAS,aAAA;AACf,MAAA,KAAA,CAAM,mBAAA,IAAuB,CAAA;AAC7B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,SAAS,YAAA,CAAa,MAAA;AAC5B,IAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAAA,EAC9B;AAAA,EAEQ,WAAA,CAAY,cAAkC,SAAA,EAAuC;AAC3F,IAAA,MAAM,GAAA,GAAM,kBAAkB,SAAS,CAAA,CAAA,CAAA;AACvC,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA,OAAO,YAAA;AAAA,IACT;AACA,IAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,MAAA,OAAO;AAAA,QACL,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,QAAQ,YAAA,CAAa,MAAA;AAAA,QACrB,IAAA,EAAM,aAAa,IAAA,GAAO,CAAA,EAAG,aAAa,IAAI,CAAA,CAAA,EAAI,GAAG,CAAA,CAAA,GAAK;AAAA,OAC5D;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,MAAA,MAAM,IAAA,GAAO,aAAa,IAAA,GAAO,CAAA,EAAG,aAAa,IAAI,CAAA,CAAA,EAAI,GAAG,CAAA,CAAA,GAAK,GAAA;AACjE,MAAA,OAAO,aAAa,MAAA,KAAW,MAAA,GAC3B,EAAE,EAAA,EAAI,KAAA,EAAO,QAAQ,SAAA,EAAW,MAAA,EAAQ,YAAA,CAAa,MAAA,EAAQ,MAAK,GAClE,EAAE,IAAI,KAAA,EAAO,MAAA,EAAQ,WAAW,IAAA,EAAK;AAAA,IAC3C;AACA,IAAA,OAAO,YAAA;AAAA,EACT;AACF;;;ACxZA,IAAM,QAAA,GAAW,CAAC,IAAA,KAAuB;AAAC,CAAA;AAE1C,SAAS,eAAe,MAAA,EAAkC;AACxD,EAAA,OAAO,MAAA,KAAW,aAAa,MAAA,KAAW,SAAA;AAC5C;AAEA,SAAS,kBAAA,CACP,MACA,IAAA,EACoB;AACpB,EAAA,MAAM,UAAA,GAAa,eAAe,IAAI,CAAA;AACtC,EAAA,MAAM,UAAA,GAAa,cAAA,CAAe,IAAA,CAAK,MAAM,CAAA;AAC7C,EAAA,IAAI,eAAe,UAAA,EAAY;AAC7B,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,UAAA,IAAc,CAAC,UAAA,EAAY;AAC7B,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,UAAA,IAAc,IAAA,CAAK,MAAA;AACvC,IAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,IAAA,CAAK,KAAK,CAAA,mBAAA,EAAsB,IAAA,CAAK,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,qCAAA,CAAA;AAAA,EACxG;AACA,EAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,KAAK,KAAK,CAAA,WAAA,CAAA;AAC5D;AAgBO,SAAS,iBAAiB,OAAA,EAAmD;AAClF,EAAA,MAAM,UAAA,GAAa,QAAQ,UAAA,IAAc,yBAAA;AACzC,EAAA,MAAM,GAAA,GAAM,QAAQ,GAAA,IAAO,QAAA;AAE3B,EAAA,IAAI,gBAAA,uBAAuB,GAAA,EAA6B;AACxD,EAAA,KAAA,MAAW,KAAA,IAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS,EAAG;AAC9C,IAAA,gBAAA,CAAiB,GAAA,CAAI,GAAG,KAAA,CAAM,QAAQ,KAAK,KAAA,CAAM,KAAK,CAAA,CAAA,EAAI,KAAA,CAAM,MAAM,CAAA;AAAA,EACxE;AAEA,EAAA,IAAI,OAAA,GAAU,KAAA;AAEd,EAAA,MAAM,OAAO,YAA2B;AACtC,IAAA,IAAI,OAAA,EAAS;AACX,MAAA;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS;AACxC,IAAA,MAAM,YAAA,GAAe,OAAO,IAAA,CAAK,CAAC,UAAU,CAAC,cAAA,CAAe,KAAA,CAAM,MAAM,CAAC,CAAA;AACzE,IAAA,IAAI,CAAC,YAAA,EAAc;AAIjB,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,OAAA,CAAQ,OAAA,CAAQ,gBAAA,EAAiB;AAAA,IACpD,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,MAAA,GAAA,CAAI,CAAA,6BAAA,EAAgC,OAAO,CAAA,CAAE,CAAA;AAC7C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,uBAAW,GAAA,EAA6B;AAC9C,IAAA,KAAA,MAAW,SAAS,QAAA,EAAU;AAC5B,MAAA,MAAM,MAAM,CAAA,EAAG,KAAA,CAAM,QAAQ,CAAA,EAAA,EAAK,MAAM,KAAK,CAAA,CAAA;AAC7C,MAAA,MAAM,IAAA,GAAO,gBAAA,CAAiB,GAAA,CAAI,GAAG,CAAA,IAAK,SAAA;AAC1C,MAAA,MAAM,iBAAA,GAAoB,kBAAA,CAAmB,IAAA,EAAM,KAAK,CAAA;AACxD,MAAA,IAAI,iBAAA,EAAmB;AACrB,QAAA,GAAA,CAAI,iBAAiB,CAAA;AAAA,MACvB;AACA,MAAA,IAAA,CAAK,GAAA,CAAI,GAAA,EAAK,KAAA,CAAM,MAAM,CAAA;AAAA,IAC5B;AACA,IAAA,gBAAA,GAAmB,IAAA;AAAA,EACrB,CAAA;AAEA,EAAA,MAAM,MAAA,GAAyB,YAAY,MAAM;AAC/C,IAAA,KAAK,IAAA,EAAK;AAAA,EACZ,GAAG,UAAU,CAAA;AAEb,EAAA,OAAO;AAAA,IACL,MAAM,MAAM;AACV,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AACA,MAAA,OAAA,GAAU,IAAA;AACV,MAAA,aAAA,CAAc,MAAM,CAAA;AAAA,IACtB;AAAA,GACF;AACF;AASO,IAAM,iBAAA,GAAoB;;;ACrG1B,SAAS,2BACd,OAAA,EACsB;AACtB,EAAA,MAAM,EAAE,QAAA,EAAU,YAAA,EAAc,OAAA,EAAQ,GAAI,OAAA;AAC5C,EAAA,IAAI,YAAY,CAAA,EAAG;AACjB,IAAA,MAAM,IAAI,WAAW,sBAAsB,CAAA;AAAA,EAC7C;AACA,EAAA,IAAI,gBAAgB,CAAA,EAAG;AACrB,IAAA,MAAM,IAAI,WAAW,0BAA0B,CAAA;AAAA,EACjD;AACA,EAAA,IAAI,WAAW,CAAA,EAAG;AAChB,IAAA,MAAM,IAAI,WAAW,qBAAqB,CAAA;AAAA,EAC5C;AAIA,EAAA,MAAM,OAAA,uBAAc,GAAA,EAAmB;AAEvC,EAAA,SAAS,aAAA,GAAsB;AAC7B,IAAA,OAAO,OAAA,CAAQ,OAAO,OAAA,EAAS;AAC7B,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,IAAA,EAAK,CAAE,MAAK,CAAE,KAAA;AACxC,MAAA,IAAI,cAAc,MAAA,EAAW;AAC3B,QAAA;AAAA,MACF;AACA,MAAA,OAAA,CAAQ,OAAO,SAAS,CAAA;AAAA,IAC1B;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,IAAA,CAAK,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC7C,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,GAAA,CAAI,GAAG,CAAA;AAC7B,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,OAAO,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,GAAA,GAAM,QAAA,EAAU,OAAO,CAAA,EAAE;AAAA,MAC5D;AACA,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,MAAM,MAAA,GAAS,YAAA;AAAA,QACxB,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC9C,MAAA,MAAM,KAAA,GAAQ,QAAQ,GAAA,CAAI,GAAG,KAAK,EAAE,IAAA,EAAM,EAAC,EAAE;AAC7C,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AAEjE,MAAA,IAAI,KAAA,CAAM,UAAU,YAAA,EAAc;AAGhC,QAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,QAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,QAAA,OAAO;AAAA,UACL,OAAA,EAAS,KAAA;AAAA,UACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,UAC7B,OAAO,KAAA,CAAM;AAAA,SACf;AAAA,MACF;AACA,MAAA,KAAA,CAAM,KAAK,GAAG,CAAA;AACd,MAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,MAAA,aAAA,EAAc;AACd,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,IAAA;AAAA,QACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,GAAM,IAAA,CAAK,GAAA,EAAI,EAAS;AAC5B,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,KAAA,MAAW,CAAC,GAAA,EAAK,KAAK,CAAA,IAAK,OAAA,EAAS;AAClC,QAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,QAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,UAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAAA,QACpB,CAAA,MAAA,IAAW,KAAA,CAAM,MAAA,KAAW,KAAA,CAAM,KAAK,MAAA,EAAQ;AAC7C,UAAA,KAAA,CAAM,IAAA,GAAO,KAAA;AAAA,QACf;AAAA,MACF;AAAA,IACF,CAAA;AAAA,IACA,IAAA,GAAe;AACb,MAAA,OAAO,OAAA,CAAQ,IAAA;AAAA,IACjB,CAAA;AAAA,IACA,KAAA,GAAc;AACZ,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,GACF;AACF;;;ACpHO,IAAM,mBAAA,GAAsB;AAqC5B,SAAS,yBACd,cAAA,EAC4B;AAC5B,EAAA,IAAI,CAAC,cAAA,EAAgB;AACnB,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,cAAA,CAAe,YAAA,IAAgB,CAAA,IAAK,cAAA,CAAe,eAAe,CAAA,EAAG;AACvE,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,OAAO,cAAA;AACT;AAEO,SAAS,uBAAA,CAAwB,OAAA,GAAiC,EAAC,EAAsB;AAC9F,EAAA,MAAM,kBAAA,GAAqC,QAAQ,kBAAA,IAAsB;AAAA,IACvE,WAAA,EAAa,uCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,MAAA,GAAyB,QAAQ,MAAA,IAAU;AAAA,IAC/C,WAAA,EAAa,iCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,OAAA,GAAU,QAAQ,OAAA,IAAW,iCAAA;AAEnC,EAAA,MAAM,gBAAgB,0BAAA,CAA2B;AAAA,IAC/C,UAAU,MAAA,CAAO,WAAA;AAAA,IACjB,cAAc,MAAA,CAAO,YAAA;AAAA,IACrB,OAAA,EAAS;AAAA,GACV,CAAA;AAED,EAAA,MAAM,iBAAiB,0BAAA,CAA2B;AAAA,IAChD,UAAU,kBAAA,CAAmB,WAAA;AAAA,IAC7B,cAAc,kBAAA,CAAmB,YAAA;AAAA,IACjC;AAAA,GACD,CAAA;AAED,EAAA,MAAM,gBAAA,uBAAuB,GAAA,EAAkC;AAE/D,EAAA,SAAS,qBAAA,CACP,WACA,QAAA,EACsB;AACtB,IAAA,MAAM,SAAA,GAAY,yBAAyB,QAAQ,CAAA;AACnD,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,OAAO,cAAA;AAAA,IACT;AACA,IAAA,MAAM,MAAA,GAAS,gBAAA,CAAiB,GAAA,CAAI,SAAS,CAAA;AAC7C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,OAAO,MAAA;AAAA,IACT;AACA,IAAA,MAAM,UAAU,0BAAA,CAA2B;AAAA,MACzC,UAAU,SAAA,CAAU,WAAA;AAAA,MACpB,cAAc,SAAA,CAAU,YAAA;AAAA,MACxB;AAAA,KACD,CAAA;AACD,IAAA,gBAAA,CAAiB,GAAA,CAAI,WAAW,OAAO,CAAA;AACvC,IAAA,OAAO,OAAA;AAAA,EACT;AAEA,EAAA,SAAS,gBAAA,GAAyB;AAChC,IAAA,cAAA,CAAe,KAAA,EAAM;AACrB,IAAA,KAAA,MAAW,OAAA,IAAW,gBAAA,CAAiB,MAAA,EAAO,EAAG;AAC/C,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,aAAA;AAAA,IACA,qBAAA;AAAA,IACA,gBAAA;AAAA,IACA,kBAAA;AAAA,IACA;AAAA,GACF;AACF;AAUO,SAAS,kBAAA,CAAmB,YAAoB,SAAA,EAA2B;AAChF,EAAA,OAAO,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,SAAS,CAAA,CAAA;AACnC","file":"llm-health.js","sourcesContent":["/**\n * LLM health monitor and heartbeat tunable defaults. CLI and plugin consumers\n * can override via `process.env.ELISYM_LLM_HEALTH_TTL_MS` and\n * `ELISYM_LLM_HEARTBEAT_INTERVAL_MS` and pass the resolved values into\n * the monitor/heartbeat options.\n */\n\nexport const DEFAULT_HEALTH_TTL_MS = 10 * 60 * 1000;\nexport const DEFAULT_HEARTBEAT_INTERVAL_MS = 10 * 60 * 1000;\n\n/**\n * Interval between recovery probes after the LLM health monitor enters an\n * unhealthy state. The recovery loop is paused while the pair is healthy\n * and only kicks in reactively (after `markUnhealthyFromJob` or a failed\n * job). On the first successful probe the monitor returns to healthy and\n * the loop stops on its own.\n */\nexport const LAZY_RECOVERY_INTERVAL_MS = 5 * 60 * 1000;\n\n/**\n * Exit code contract: a `dynamic-script` / `static-script` skill returns\n * this code from the script process to signal that the upstream LLM\n * provider rejected the request because credits / billing are exhausted.\n * The agent runtime treats this as the script's equivalent of the\n * `mode: 'llm'` 402 path: it calls `markUnhealthyFromJob(provider, model)`\n * on the health monitor (which starts the lazy recovery loop) and rejects\n * subsequent jobs against the same pair until a recovery probe succeeds.\n *\n * Any other non-zero exit is a generic failure and does NOT touch health\n * state - operators should reserve this code for billing-exhausted only.\n *\n * 42 was chosen because it sits outside POSIX shell conventions (1-2\n * generic, 126-128 shell-internal, 130+ signals) and `sysexits.h`\n * (64-78 - usage / data / host / config errors), so it doesn't collide\n * with other meaningful exit codes a script might naturally produce.\n */\nexport const SCRIPT_EXIT_BILLING_EXHAUSTED = 42;\n\n/**\n * Number of consecutive `unavailable` results tolerated before\n * `assertReady` starts throwing. The first `unavailable - 1` are treated\n * as transient blips so a brief network hiccup does not block jobs.\n */\nexport const UNAVAILABLE_TOLERANCE = 3;\n\n/**\n * Free-LLM rate-limit defaults. Applied when a SKILL.md with\n * `mode: 'llm'` and `price: 0` does not declare its own `rate_limit`.\n */\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS = 60 * 60 * 1000;\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_MAX = 3;\n\nexport const DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS = 60 * 1000;\nexport const DEFAULT_FREE_LLM_GLOBAL_MAX = 30;\n\nexport const DEFAULT_FREE_LLM_MAX_TRACKED_KEYS = 1000;\n","/**\n * Shared types for the LLM health monitor. Provider-agnostic: the\n * actual HTTP probe lives in CLI/plugin and is supplied via dependency\n * injection (`verifyFn`).\n */\n\nexport type LlmHealthStatus = 'unknown' | 'healthy' | 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Result of probing an API key with a specific model. Discriminated on\n * `ok`, then on `reason` for failures. Adding a new failure reason is a\n * breaking change for exhaustive switches; update consumers in lockstep.\n *\n * - `invalid`: HTTP 401/403 - key rejected outright.\n * - `billing`: HTTP 402, or 400 with credit/billing/insufficient marker,\n * or OpenAI's 429 with `insufficient_quota`. Operator out of credits.\n * - `unavailable`: transient (HTTP 429 without quota marker, 5xx, network\n * error). May resolve on retry.\n */\nexport type LlmKeyVerification =\n | { ok: true }\n | { ok: false; reason: 'invalid'; status: number; body: string }\n | { ok: false; reason: 'billing'; status?: number; body?: string }\n | { ok: false; reason: 'unavailable'; error: string };\n\nexport type LlmHealthErrorReason = 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Thrown by `LlmHealthMonitor.assertReady` when the gate refuses a job.\n * Carries the operator-facing reason so callers can log it; customer-facing\n * messages should be sanitized at the call site (see `runtime.ts` preflight).\n */\nexport class LlmHealthError extends Error {\n readonly reason: LlmHealthErrorReason;\n readonly provider: string;\n readonly model: string;\n\n constructor(reason: LlmHealthErrorReason, provider: string, model: string, detail: string) {\n super(`LLM ${provider}/${model} ${reason}: ${detail}`);\n this.name = 'LlmHealthError';\n this.reason = reason;\n this.provider = provider;\n this.model = model;\n }\n}\n\n/**\n * Thrown by SDK script skills (`static-script`, `dynamic-script`) when the\n * spawned process exits with `SCRIPT_EXIT_BILLING_EXHAUSTED` (= 42). The\n * runtime catches this and calls `markUnhealthyFromJob` on the matching\n * (provider, model) pair declared in SKILL.md, so subsequent jobs are\n * gated until the lazy recovery loop re-probes the key. The exit code is\n * the script-side equivalent of the LLM-mode 402 path.\n *\n * The error does NOT carry provider/model itself - the script doesn't\n * know which pair the agent registered with the monitor; the runtime\n * reads them from the matched skill's `llmOverride` declaration.\n */\nexport class ScriptBillingExhaustedError extends Error {\n readonly exitCode: number;\n readonly stderr: string;\n readonly stdout: string;\n\n constructor(exitCode: number, stdout: string, stderr: string) {\n const detail = stderr.trim() || stdout.trim() || '(no output)';\n super(`script exited with billing-exhausted code ${exitCode}: ${detail}`);\n this.name = 'ScriptBillingExhaustedError';\n this.exitCode = exitCode;\n this.stdout = stdout;\n this.stderr = stderr;\n }\n}\n\n/**\n * Per-skill rate-limit declaration. Snake-case in SKILL.md frontmatter,\n * camelCase here. Applies to any skill mode but the framework adds a\n * default cap only for free LLM skills.\n */\nexport interface SkillRateLimit {\n perWindowMs: number;\n maxPerWindow: number;\n}\n\n/** Read-only health snapshot for logs/tests. */\nexport interface LlmHealthSnapshotEntry {\n provider: string;\n model: string;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n consecutiveFailures: number;\n}\n","/**\n * Provider-agnostic health monitor for LLM API keys. State machine per\n * (provider, model) pair: caches the last verification result for\n * `ttlMs`, deduplicates concurrent probes via an in-flight promise, and\n * tolerates a bounded number of consecutive `unavailable` results\n * before `assertReady` starts throwing.\n *\n * The monitor itself never touches the network. Each pair must be\n * registered with a `verifyFn` lambda that performs the actual probe;\n * callers (CLI/plugin) supply provider-specific HTTP from their layer.\n */\n\nimport {\n DEFAULT_HEALTH_TTL_MS,\n UNAVAILABLE_TOLERANCE as DEFAULT_UNAVAILABLE_TOLERANCE,\n} from './constants';\nimport {\n LlmHealthError,\n type LlmHealthSnapshotEntry,\n type LlmHealthStatus,\n type LlmKeyVerification,\n} from './types';\n\nexport type LlmKeyVerifyFn = (signal?: AbortSignal) => Promise<LlmKeyVerification>;\n\nexport interface LlmHealthMonitorOptions {\n /** Time after which a cached `healthy` result is re-probed. Default 10 min. */\n ttlMs?: number;\n /** Number of consecutive unavailable results tolerated. Default 3. */\n unavailableTolerance?: number;\n /** Optional clock injection for tests. */\n now?: () => number;\n}\n\nexport interface RegisterArgs {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n}\n\ninterface Entry {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n inFlight: Promise<LlmKeyVerification> | null;\n consecutiveFailures: number;\n}\n\nfunction keyOf(provider: string, model: string): string {\n return `${provider}::${model}`;\n}\n\nfunction reasonDetail(verification: LlmKeyVerification): string {\n if (verification.ok) {\n return 'ok';\n }\n if (verification.reason === 'invalid') {\n return `HTTP ${verification.status}: ${verification.body.slice(0, 200)}`;\n }\n if (verification.reason === 'billing') {\n const status = verification.status ?? 0;\n const body = (verification.body ?? '').slice(0, 200);\n return `HTTP ${status}: ${body}`;\n }\n return verification.error;\n}\n\nexport class LlmHealthMonitor {\n private readonly entries = new Map<string, Entry>();\n private readonly ttlMs: number;\n private readonly unavailableTolerance: number;\n private readonly now: () => number;\n\n constructor(options: LlmHealthMonitorOptions = {}) {\n this.ttlMs = options.ttlMs ?? DEFAULT_HEALTH_TTL_MS;\n this.unavailableTolerance = options.unavailableTolerance ?? DEFAULT_UNAVAILABLE_TOLERANCE;\n this.now = options.now ?? Date.now;\n }\n\n /**\n * Register a (provider, model) pair with its probe function. Idempotent\n * on the (provider, model) key: re-registering replaces the verifyFn\n * and resets state to `unknown`. Callers typically re-register only\n * when the operator rotates the API key.\n */\n register(args: RegisterArgs): void {\n const key = keyOf(args.provider, args.model);\n this.entries.set(key, {\n provider: args.provider,\n model: args.model,\n verifyFn: args.verifyFn,\n status: 'unknown',\n lastVerifiedAt: 0,\n lastReason: undefined,\n inFlight: null,\n consecutiveFailures: 0,\n });\n }\n\n /**\n * Seed the monitor with an already-known verification result, e.g.\n * the one captured at startup. Skips an extra probe on the first\n * `assertReady` call within the TTL window. No-op if the pair is not\n * registered.\n *\n * In production `seed` is only called from the CLI startup path, which\n * itself `process.exit(1)`s on `invalid` / `billing` before any other\n * pair is reached - so the cascade in `applyVerification` is benign at\n * startup. In tests, seeding `invalid` / `billing` will cascade across\n * sibling pairs of the same provider, which matches the production\n * runtime behavior we want to assert.\n */\n seed(provider: string, model: string, verification: LlmKeyVerification): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n this.applyVerification(entry, verification);\n }\n\n /**\n * Main gate before doing LLM work. Throws `LlmHealthError` on terminal\n * states (`invalid`, `billing`, or `unavailable` past tolerance);\n * resolves silently on `healthy` (cache hit or fresh probe) or\n * tolerated `unavailable`.\n *\n * Concurrent `assertReady` calls for the same pair are deduplicated:\n * the second caller awaits the first probe rather than launching a\n * parallel one.\n */\n async assertReady(provider: string, model: string): Promise<void> {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n throw new LlmHealthError('invalid', provider, model, 'pair not registered');\n }\n\n const verification = await this.probeIfNeeded(entry);\n\n if (verification.ok) {\n return;\n }\n if (verification.reason === 'invalid' || verification.reason === 'billing') {\n throw new LlmHealthError(verification.reason, provider, model, reasonDetail(verification));\n }\n if (entry.consecutiveFailures >= this.unavailableTolerance) {\n throw new LlmHealthError('unavailable', provider, model, reasonDetail(verification));\n }\n // Tolerated unavailable: caller proceeds, real LLM call will surface\n // any transient issue with its own error path.\n }\n\n /**\n * Force the next `assertReady` for this pair to re-probe regardless of\n * TTL. Used when a real LLM call surfaces 401/402 mid-job - the cached\n * `healthy` is stale and we want to catch the next request.\n *\n * NOTE: this method has no callers in the current production code\n * (`packages/sdk/src` and `packages/cli/src` both prefer the explicit\n * `markUnhealthyFromJob` below). It does NOT propagate to sibling\n * pairs; if reintroduced, consider whether the cascade in\n * `markUnhealthyFromJob` should also apply here.\n */\n markFailureFromJob(provider: string, model: string): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n entry.lastVerifiedAt = 0;\n }\n\n /**\n * Reactively flip a (provider, model) pair to unhealthy without doing a\n * fresh probe. Called from the runtime when a job's actual LLM call (or\n * a script's `SCRIPT_EXIT_BILLING_EXHAUSTED` exit) surfaces a billing /\n * invalid signal: the cached `healthy` snapshot is wrong and we want\n * subsequent `assertReady` calls to refuse jobs immediately, before the\n * lazy recovery loop notices on its own. No-op if the pair is not\n * registered.\n *\n * For `invalid` / `billing` reasons this propagates to every sibling\n * pair sharing the same `provider` via the cascade in\n * `applyVerification` - they share the same API key, so a revoked or\n * exhausted key affects all models. `unavailable` stays per-pair.\n *\n * Recovery from this state happens through a successful probe (typically\n * fired by `startLlmRecovery`) which flips the pair back to healthy via\n * `applyVerification`. The recovery cascade lifts sibling pairs at the\n * same time so the gate doesn't stay half-broken.\n */\n markUnhealthyFromJob(\n provider: string,\n model: string,\n reason: 'billing' | 'invalid' | 'unavailable',\n detail?: string,\n ): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n if (reason === 'invalid') {\n this.applyVerification(entry, {\n ok: false,\n reason: 'invalid',\n status: 0,\n body: detail ?? 'reactive markUnhealthyFromJob',\n });\n return;\n }\n if (reason === 'billing') {\n this.applyVerification(entry, {\n ok: false,\n reason: 'billing',\n body: detail ?? 'reactive markUnhealthyFromJob',\n });\n return;\n }\n this.applyVerification(entry, {\n ok: false,\n reason: 'unavailable',\n error: detail ?? 'reactive markUnhealthyFromJob',\n });\n }\n\n /**\n * Refresh every registered pair concurrently. Errors thrown by\n * `verifyFn` are caught and recorded as `unavailable`.\n */\n async refreshAll(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n for (const entry of this.entries.values()) {\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /**\n * Refresh only the pairs whose current status is non-healthy\n * (`invalid`, `billing`, or `unavailable`). Used by the lazy recovery\n * loop so a billing outage on one provider does not trigger throwaway\n * probes on every other healthy pair the agent has registered.\n * Errors thrown by `verifyFn` are caught and recorded as `unavailable`.\n *\n * Provider-deduplicated for `invalid` / `billing`: every pair under\n * the same provider shares the same API key, so a single probe per\n * provider is enough - a successful result will cascade to siblings\n * via the recovery cascade in `applyVerification`. Without this dedup,\n * a 3-skill Anthropic agent in a billing outage would burn 3x the\n * billing-token quota per recovery tick.\n *\n * `unavailable` entries probe individually because the failure is\n * model-specific (e.g. one endpoint returning 5xx) and cascading the\n * result would corrupt sibling state.\n */\n async refreshUnhealthy(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n const sharedKeyProviders = new Set<string>();\n for (const entry of this.entries.values()) {\n if (entry.status === 'healthy' || entry.status === 'unknown') {\n continue;\n }\n if (entry.status === 'unavailable') {\n probes.push(this.probe(entry).then(() => undefined));\n continue;\n }\n // invalid / billing - probe one representative per provider only.\n if (sharedKeyProviders.has(entry.provider)) {\n continue;\n }\n sharedKeyProviders.add(entry.provider);\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /** Read-only view, primarily for logs and tests. */\n snapshot(): readonly LlmHealthSnapshotEntry[] {\n const out: LlmHealthSnapshotEntry[] = [];\n for (const entry of this.entries.values()) {\n out.push({\n provider: entry.provider,\n model: entry.model,\n status: entry.status,\n lastVerifiedAt: entry.lastVerifiedAt,\n lastReason: entry.lastReason,\n consecutiveFailures: entry.consecutiveFailures,\n });\n }\n return out;\n }\n\n private probeIfNeeded(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const fresh = this.now() - entry.lastVerifiedAt < this.ttlMs;\n if (fresh) {\n if (entry.status === 'healthy') {\n return Promise.resolve({ ok: true });\n }\n // Terminal failure states (`invalid`, `billing`) are cached\n // aggressively: once the operator's key is rejected or out of\n // credits, the lazy recovery loop is the only path back to healthy.\n // Re-probing on every `assertReady` would burn one billing token\n // per job rejected during a billing outage, which defeats the\n // gate's whole purpose. `unavailable` is treated as transient and\n // falls through so the existing tolerance counter still works\n // (each call re-probes, increments `consecutiveFailures`, and\n // crosses the threshold on the configured number of attempts).\n // `unknown` also falls through - a freshly registered pair that\n // was never seeded should probe on first ask.\n if (entry.status === 'invalid' || entry.status === 'billing') {\n return Promise.resolve(this.synthesizeFailureFromCache(entry));\n }\n }\n return this.probe(entry);\n }\n\n private synthesizeFailureFromCache(entry: Entry): LlmKeyVerification {\n const detail = entry.lastReason ?? 'cached failure';\n if (entry.status === 'invalid') {\n return { ok: false, reason: 'invalid', status: 0, body: detail };\n }\n // 'billing' (only other status reachable here)\n return { ok: false, reason: 'billing', body: detail };\n }\n\n private probe(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const promise = (async (): Promise<LlmKeyVerification> => {\n try {\n return await entry.verifyFn();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n return { ok: false, reason: 'unavailable', error: message };\n }\n })().then((verification) => {\n this.applyVerification(entry, verification);\n entry.inFlight = null;\n return verification;\n });\n entry.inFlight = promise;\n return promise;\n }\n\n /**\n * Apply a verification to a single entry AND cascade across siblings\n * sharing the same `provider`. Two symmetric cascades:\n *\n * - **Failure cascade** (`invalid` / `billing`): a revoked or\n * billing-exhausted API key affects every model the operator could\n * reach with that key. Structurally enforced by\n * `resolveProviderApiKey` in `packages/cli/src/llm/keys.ts:27`,\n * which returns at most one key per provider (no per-skill\n * injection). If a future change introduces per-skill keys, this\n * cascade must be revisited.\n *\n * - **Recovery cascade** (`ok: true`): when a healthy verification\n * arrives (typically from `refreshUnhealthy` after the operator\n * fixed the key), lift sibling pairs that were stuck `invalid` /\n * `billing` so the next `assertReady` for any model under this\n * provider stops refusing jobs. Without this, the gate would stay\n * half-broken until each sibling's own probe round-trips.\n *\n * `unavailable` does NOT cascade - it is treated as a transient,\n * model-specific issue (e.g. specific endpoint returning 5xx).\n *\n * Cascaded sibling verifications are tagged with `(cascaded from\n * <triggering-model>)` in their body so `snapshot()` and operator\n * logs remain diagnosable.\n */\n private applyVerification(entry: Entry, verification: LlmKeyVerification): void {\n this.mutateEntry(entry, verification);\n\n if (verification.ok) {\n for (const sibling of this.entries.values()) {\n if (sibling === entry) {\n continue;\n }\n if (sibling.provider !== entry.provider) {\n continue;\n }\n if (sibling.status !== 'invalid' && sibling.status !== 'billing') {\n continue;\n }\n this.mutateEntry(sibling, { ok: true });\n }\n return;\n }\n\n if (verification.reason !== 'invalid' && verification.reason !== 'billing') {\n return;\n }\n\n const cascaded = this.tagCascaded(verification, entry.model);\n for (const sibling of this.entries.values()) {\n if (sibling === entry) {\n continue;\n }\n if (sibling.provider !== entry.provider) {\n continue;\n }\n this.mutateEntry(sibling, cascaded);\n }\n }\n\n /**\n * Per-entry state transition. Used by `applyVerification` (which then\n * orchestrates the cascade) and by the cascade loop itself for sibling\n * entries (which must NOT trigger a recursive cascade).\n */\n private mutateEntry(entry: Entry, verification: LlmKeyVerification): void {\n entry.lastVerifiedAt = this.now();\n if (verification.ok) {\n entry.status = 'healthy';\n entry.lastReason = undefined;\n entry.consecutiveFailures = 0;\n return;\n }\n entry.lastReason = reasonDetail(verification);\n if (verification.reason === 'unavailable') {\n entry.status = 'unavailable';\n entry.consecutiveFailures += 1;\n return;\n }\n entry.status = verification.reason;\n entry.consecutiveFailures = 0;\n }\n\n private tagCascaded(verification: LlmKeyVerification, fromModel: string): LlmKeyVerification {\n const tag = `(cascaded from ${fromModel})`;\n if (verification.ok) {\n return verification;\n }\n if (verification.reason === 'invalid') {\n return {\n ok: false,\n reason: 'invalid',\n status: verification.status,\n body: verification.body ? `${verification.body} ${tag}` : tag,\n };\n }\n if (verification.reason === 'billing') {\n const body = verification.body ? `${verification.body} ${tag}` : tag;\n return verification.status !== undefined\n ? { ok: false, reason: 'billing', status: verification.status, body }\n : { ok: false, reason: 'billing', body };\n }\n return verification;\n }\n}\n","/**\n * Lazy LLM recovery probe. While every registered (provider, model) pair\n * is healthy, ticks are no-ops: zero API traffic, zero billing tokens\n * burned. The loop only does real work after a pair has flipped to\n * unhealthy (via `markUnhealthyFromJob` from the runtime, or from a\n * preflight probe that returned a non-ok verification): each tick\n * re-probes only the unhealthy pairs and, on success, flips them back to\n * healthy via the monitor's normal `applyVerification` path.\n *\n * The recovery loop is the only path back to `healthy` after a reactive\n * markUnhealthy. Without it, the agent would stay locked out until\n * restart even after the operator pops their billing back up.\n *\n * Logging policy lives here so the monitor stays a pure state-machine.\n * Status transitions (healthy <-> unhealthy) are logged once per change;\n * routine successful re-probes are not logged. When the monitor's\n * provider-wide cascade (see `applyVerification` in `monitor.ts`)\n * flips multiple sibling pairs at once, each pair still produces its\n * own transition line here - intentional asymmetry vs. the runtime's\n * single aggregated \"marking unhealthy\" line, since each sibling is\n * separately observed by the snapshot diff.\n */\n\nimport { LAZY_RECOVERY_INTERVAL_MS } from './constants';\nimport type { LlmHealthMonitor } from './monitor';\nimport type { LlmHealthSnapshotEntry, LlmHealthStatus } from './types';\n\nexport interface HeartbeatHandle {\n stop(): void;\n}\n\nexport interface StartLlmRecoveryOptions {\n monitor: LlmHealthMonitor;\n /** Defaults to {@link LAZY_RECOVERY_INTERVAL_MS} (5 minutes). */\n intervalMs?: number;\n /** Operator log sink. Defaults to no-op (silent). */\n log?: (msg: string) => void;\n}\n\n/**\n * @deprecated Renamed to {@link StartLlmRecoveryOptions}. Kept as an alias\n * so external consumers keep building during the rename. The semantics\n * have changed (lazy, recovery-only) but the option surface is identical.\n */\nexport type StartLlmHeartbeatOptions = StartLlmRecoveryOptions;\n\ntype IntervalHandle = ReturnType<typeof setInterval>;\n\nconst NOOP_LOG = (_msg: string): void => {};\n\nfunction isHealthyState(status: LlmHealthStatus): boolean {\n return status === 'healthy' || status === 'unknown';\n}\n\nfunction describeTransition(\n prev: LlmHealthStatus,\n next: LlmHealthSnapshotEntry,\n): string | undefined {\n const wasHealthy = isHealthyState(prev);\n const nowHealthy = isHealthyState(next.status);\n if (wasHealthy === nowHealthy) {\n return undefined;\n }\n if (wasHealthy && !nowHealthy) {\n const reason = next.lastReason ?? next.status;\n return `! LLM provider ${next.provider} model ${next.model} became unhealthy: ${next.status} (${reason}). Refusing LLM jobs until recovered.`;\n }\n return `* LLM provider ${next.provider} model ${next.model} recovered.`;\n}\n\n/**\n * Start the lazy recovery loop. Returns a handle whose `stop()` cancels\n * the timer (idempotent). The loop ticks every `intervalMs` ms; each\n * tick scans `monitor.snapshot()` and, if any pair is non-healthy, asks\n * the monitor to re-probe just those non-healthy pairs via\n * `refreshUnhealthy()`. Healthy pairs are never re-probed by the loop -\n * those keep their cached `healthy` until TTL expiry forces a probe at\n * the next `assertReady` call. When all pairs are healthy the tick is a\n * single Map walk and no API calls are made.\n *\n * The function is named `startLlmRecovery` but the legacy export\n * `startLlmHeartbeat` (below) is preserved as an alias so external\n * code that already imports it keeps working unchanged.\n */\nexport function startLlmRecovery(options: StartLlmRecoveryOptions): HeartbeatHandle {\n const intervalMs = options.intervalMs ?? LAZY_RECOVERY_INTERVAL_MS;\n const log = options.log ?? NOOP_LOG;\n\n let lastStatusByPair = new Map<string, LlmHealthStatus>();\n for (const entry of options.monitor.snapshot()) {\n lastStatusByPair.set(`${entry.provider}::${entry.model}`, entry.status);\n }\n\n let stopped = false;\n\n const tick = async (): Promise<void> => {\n if (stopped) {\n return;\n }\n const before = options.monitor.snapshot();\n const anyUnhealthy = before.some((entry) => !isHealthyState(entry.status));\n if (!anyUnhealthy) {\n // Lazy: nothing to do. Refreshing healthy pairs would burn one\n // billing-token per pair per tick for no benefit (the monitor's\n // own TTL would re-probe at assertReady time anyway).\n return;\n }\n\n let snapshot: readonly LlmHealthSnapshotEntry[];\n try {\n snapshot = await options.monitor.refreshUnhealthy();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n log(`! LLM recovery probe failed: ${message}`);\n return;\n }\n\n const next = new Map<string, LlmHealthStatus>();\n for (const entry of snapshot) {\n const key = `${entry.provider}::${entry.model}`;\n const prev = lastStatusByPair.get(key) ?? 'unknown';\n const transitionMessage = describeTransition(prev, entry);\n if (transitionMessage) {\n log(transitionMessage);\n }\n next.set(key, entry.status);\n }\n lastStatusByPair = next;\n };\n\n const handle: IntervalHandle = setInterval(() => {\n void tick();\n }, intervalMs);\n\n return {\n stop: () => {\n if (stopped) {\n return;\n }\n stopped = true;\n clearInterval(handle);\n },\n };\n}\n\n/**\n * @deprecated Renamed to {@link startLlmRecovery}. The old name is\n * preserved as an alias so existing imports keep working during the\n * rename. Behavior changed materially: the new loop is lazy\n * (no API calls while all pairs are healthy) and defaults to\n * `LAZY_RECOVERY_INTERVAL_MS` (5 min) instead of 10 min.\n */\nexport const startLlmHeartbeat = startLlmRecovery;\n","/**\n * Sliding-window rate limiter keyed by an arbitrary string (typically a\n * customer pubkey). Each key gets at most `maxPerWindow` requests inside a\n * rolling `windowMs`. Stale timestamps are pruned lazily on every `check`.\n * When the tracked-key set grows past `maxKeys`, the least-recently-used\n * key is evicted so an attacker cannot exhaust memory by cycling keys.\n *\n * Thread-safety: not required. Designed for single-threaded JS consumers\n * (Node/Bun event loops, browser main thread). No timers - pruning happens\n * inside `check` and `prune`.\n */\n\nexport interface SlidingWindowLimiterOptions {\n /** Rolling window width, in ms. */\n windowMs: number;\n /** Max hits allowed per key inside the window. */\n maxPerWindow: number;\n /** Cap on total tracked keys. LRU-evicted past this cap. */\n maxKeys: number;\n}\n\nexport interface RateLimitDecision {\n allowed: boolean;\n /** Wall-clock timestamp (ms) when the limit window will reset for this key. */\n resetAt: number;\n /** Number of hits inside the current window after this call (or the attempted hit if denied). */\n count: number;\n}\n\nexport interface SlidingWindowLimiter {\n /** Record a hit against `key`; return whether it was allowed. */\n check(key: string, now?: number): RateLimitDecision;\n /**\n * Inspect the current state for `key` without recording a hit. Useful\n * when a single request must clear multiple limiters and callers want\n * to avoid double-counting against one limiter after another denies.\n */\n peek(key: string, now?: number): RateLimitDecision;\n /** Drop entries whose windows have fully elapsed. Bounded memory hygiene. */\n prune(now?: number): void;\n /** Current number of tracked keys. */\n size(): number;\n /** Clear all state. */\n reset(): void;\n}\n\ninterface Entry {\n /** Sliding-window timestamps in ms. Sorted ascending. */\n hits: number[];\n}\n\nexport function createSlidingWindowLimiter(\n options: SlidingWindowLimiterOptions,\n): SlidingWindowLimiter {\n const { windowMs, maxPerWindow, maxKeys } = options;\n if (windowMs <= 0) {\n throw new RangeError('windowMs must be > 0');\n }\n if (maxPerWindow <= 0) {\n throw new RangeError('maxPerWindow must be > 0');\n }\n if (maxKeys <= 0) {\n throw new RangeError('maxKeys must be > 0');\n }\n\n // LRU is implemented via Map's insertion-order: every check refreshes\n // the entry by deleting and re-setting it, moving it to the tail.\n const entries = new Map<string, Entry>();\n\n function evictIfNeeded(): void {\n while (entries.size > maxKeys) {\n const oldestKey = entries.keys().next().value as string | undefined;\n if (oldestKey === undefined) {\n return;\n }\n entries.delete(oldestKey);\n }\n }\n\n return {\n peek(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key);\n if (!entry) {\n return { allowed: true, resetAt: now + windowMs, count: 0 };\n }\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n return {\n allowed: fresh.length < maxPerWindow,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n check(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key) ?? { hits: [] };\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n\n if (fresh.length >= maxPerWindow) {\n // Refresh LRU order even on denial so an attacker hammering the\n // same key cannot push other tracked keys out via eviction.\n entries.delete(key);\n entries.set(key, { hits: fresh });\n return {\n allowed: false,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n }\n fresh.push(now);\n entries.delete(key);\n entries.set(key, { hits: fresh });\n evictIfNeeded();\n return {\n allowed: true,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n prune(now = Date.now()): void {\n const cutoff = now - windowMs;\n for (const [key, entry] of entries) {\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n if (fresh.length === 0) {\n entries.delete(key);\n } else if (fresh.length !== entry.hits.length) {\n entry.hits = fresh;\n }\n }\n },\n size(): number {\n return entries.size;\n },\n reset(): void {\n entries.clear();\n },\n };\n}\n","/**\n * Two-tier rate limiter for free LLM skills (mode='llm', price=0).\n * Provides Sybil-resistant global cap plus per-customer-per-skill cap\n * that respects an optional skill-level override.\n *\n * Both limiters are independent of the existing per-customer general\n * limiter; callers (CLI runtime, plugin handler) check all of them in\n * sequence and only `check()` after every tier passes `peek()` so a\n * single denial does not consume slots in earlier tiers.\n */\n\nimport { createSlidingWindowLimiter, type SlidingWindowLimiter } from '../primitives/rateLimiter';\nimport {\n DEFAULT_FREE_LLM_GLOBAL_MAX,\n DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n DEFAULT_FREE_LLM_MAX_TRACKED_KEYS,\n DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n} from './constants';\nimport type { SkillRateLimit } from './types';\n\nexport const FREE_LLM_GLOBAL_KEY = '__free_llm_global__';\n\nexport interface FreeLlmLimiterOptions {\n /** Max tracked (customer, skill) keys. Default 1000 (LRU evicted past cap). */\n maxKeys?: number;\n /** Default per-customer cap when a free LLM skill omits `rate_limit`. */\n defaultPerCustomer?: SkillRateLimit;\n /** Global Sybil cap across all free LLM jobs. */\n global?: SkillRateLimit;\n}\n\nexport interface FreeLlmLimiterSet {\n /** Sybil-protection limiter keyed on `FREE_LLM_GLOBAL_KEY`. */\n globalLimiter: SlidingWindowLimiter;\n /**\n * Return the per-customer limiter for a given skill. Skills that\n * declare a `rate_limit` get their own limiter sized to that\n * (window, cap); skills that don't share a default-window limiter.\n * Each call uses `peek/check` keyed on `customerId` alone since each\n * skill has a dedicated limiter store.\n */\n getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter;\n /** Drop expired hits from every per-customer limiter (default + per-skill). */\n prunePerCustomer(): void;\n /** Default cap to apply when a skill omits `rate_limit`. */\n defaultPerCustomer: SkillRateLimit;\n /** Sybil-cap settings (echo of global option for diagnostics). */\n global: SkillRateLimit;\n}\n\n/**\n * Effective per-skill rate limit: the override if it's well-formed,\n * else `undefined` so callers can fall through to the default limiter.\n */\nexport function resolvePerSkillRateLimit(\n skillRateLimit: SkillRateLimit | undefined,\n): SkillRateLimit | undefined {\n if (!skillRateLimit) {\n return undefined;\n }\n if (skillRateLimit.maxPerWindow <= 0 || skillRateLimit.perWindowMs <= 0) {\n return undefined;\n }\n return skillRateLimit;\n}\n\nexport function createFreeLlmLimiterSet(options: FreeLlmLimiterOptions = {}): FreeLlmLimiterSet {\n const defaultPerCustomer: SkillRateLimit = options.defaultPerCustomer ?? {\n perWindowMs: DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n };\n const global: SkillRateLimit = options.global ?? {\n perWindowMs: DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_GLOBAL_MAX,\n };\n const maxKeys = options.maxKeys ?? DEFAULT_FREE_LLM_MAX_TRACKED_KEYS;\n\n const globalLimiter = createSlidingWindowLimiter({\n windowMs: global.perWindowMs,\n maxPerWindow: global.maxPerWindow,\n maxKeys: 1,\n });\n\n const defaultLimiter = createSlidingWindowLimiter({\n windowMs: defaultPerCustomer.perWindowMs,\n maxPerWindow: defaultPerCustomer.maxPerWindow,\n maxKeys,\n });\n\n const perSkillLimiters = new Map<string, SlidingWindowLimiter>();\n\n function getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter {\n const effective = resolvePerSkillRateLimit(override);\n if (!effective) {\n return defaultLimiter;\n }\n const cached = perSkillLimiters.get(skillName);\n if (cached) {\n return cached;\n }\n const limiter = createSlidingWindowLimiter({\n windowMs: effective.perWindowMs,\n maxPerWindow: effective.maxPerWindow,\n maxKeys,\n });\n perSkillLimiters.set(skillName, limiter);\n return limiter;\n }\n\n function prunePerCustomer(): void {\n defaultLimiter.prune();\n for (const limiter of perSkillLimiters.values()) {\n limiter.prune();\n }\n }\n\n return {\n globalLimiter,\n getPerCustomerLimiter,\n prunePerCustomer,\n defaultPerCustomer,\n global,\n };\n}\n\n/**\n * Compose a per-skill key for the per-customer limiter. The default\n * limiter is shared across skills without an override, so the skill\n * name is needed to keep each (customer, skill) pair counted\n * independently. Per-skill limiters use the same key for consistency\n * even though the skill component is redundant inside a dedicated\n * limiter store.\n */\nexport function freeLlmCustomerKey(customerId: string, skillName: string): string {\n return `${customerId}|${skillName}`;\n}\n"]}
|