@elisym/sdk 0.15.1 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,8 @@
3
3
  // src/llm-health/constants.ts
4
4
  var DEFAULT_HEALTH_TTL_MS = 10 * 60 * 1e3;
5
5
  var DEFAULT_HEARTBEAT_INTERVAL_MS = 10 * 60 * 1e3;
6
+ var LAZY_RECOVERY_INTERVAL_MS = 5 * 60 * 1e3;
7
+ var SCRIPT_EXIT_BILLING_EXHAUSTED = 42;
6
8
  var UNAVAILABLE_TOLERANCE = 3;
7
9
  var DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS = 60 * 60 * 1e3;
8
10
  var DEFAULT_FREE_LLM_PER_CUSTOMER_MAX = 3;
@@ -23,10 +25,23 @@ var LlmHealthError = class extends Error {
23
25
  this.model = model;
24
26
  }
25
27
  };
28
+ var ScriptBillingExhaustedError = class extends Error {
29
+ exitCode;
30
+ stderr;
31
+ stdout;
32
+ constructor(exitCode, stdout, stderr) {
33
+ const detail = stderr.trim() || stdout.trim() || "(no output)";
34
+ super(`script exited with billing-exhausted code ${exitCode}: ${detail}`);
35
+ this.name = "ScriptBillingExhaustedError";
36
+ this.exitCode = exitCode;
37
+ this.stdout = stdout;
38
+ this.stderr = stderr;
39
+ }
40
+ };
26
41
 
27
42
  // src/llm-health/monitor.ts
28
43
  function keyOf(provider, model) {
29
- return `${provider}\0${model}`;
44
+ return `${provider}::${model}`;
30
45
  }
31
46
  function reasonDetail(verification) {
32
47
  if (verification.ok) {
@@ -123,8 +138,49 @@ var LlmHealthMonitor = class {
123
138
  entry.lastVerifiedAt = 0;
124
139
  }
125
140
  /**
126
- * Refresh every registered pair concurrently. Heartbeat hook. Errors
127
- * thrown by `verifyFn` are caught and recorded as `unavailable`.
141
+ * Reactively flip a (provider, model) pair to unhealthy without doing a
142
+ * fresh probe. Called from the runtime when a job's actual LLM call (or
143
+ * a script's `SCRIPT_EXIT_BILLING_EXHAUSTED` exit) surfaces a billing /
144
+ * invalid signal: the cached `healthy` snapshot is wrong and we want
145
+ * subsequent `assertReady` calls to refuse jobs immediately, before the
146
+ * lazy recovery loop notices on its own. No-op if the pair is not
147
+ * registered.
148
+ *
149
+ * Recovery from this state happens through a successful probe (typically
150
+ * fired by `startLlmRecovery`) which flips the pair back to healthy via
151
+ * `applyVerification`.
152
+ */
153
+ markUnhealthyFromJob(provider, model, reason, detail) {
154
+ const entry = this.entries.get(keyOf(provider, model));
155
+ if (!entry) {
156
+ return;
157
+ }
158
+ if (reason === "invalid") {
159
+ this.applyVerification(entry, {
160
+ ok: false,
161
+ reason: "invalid",
162
+ status: 0,
163
+ body: detail ?? "reactive markUnhealthyFromJob"
164
+ });
165
+ return;
166
+ }
167
+ if (reason === "billing") {
168
+ this.applyVerification(entry, {
169
+ ok: false,
170
+ reason: "billing",
171
+ body: detail ?? "reactive markUnhealthyFromJob"
172
+ });
173
+ return;
174
+ }
175
+ this.applyVerification(entry, {
176
+ ok: false,
177
+ reason: "unavailable",
178
+ error: detail ?? "reactive markUnhealthyFromJob"
179
+ });
180
+ }
181
+ /**
182
+ * Refresh every registered pair concurrently. Errors thrown by
183
+ * `verifyFn` are caught and recorded as `unavailable`.
128
184
  */
129
185
  async refreshAll() {
130
186
  const probes = [];
@@ -134,6 +190,24 @@ var LlmHealthMonitor = class {
134
190
  await Promise.all(probes);
135
191
  return this.snapshot();
136
192
  }
193
+ /**
194
+ * Refresh only the pairs whose current status is non-healthy
195
+ * (`invalid`, `billing`, or `unavailable`). Used by the lazy recovery
196
+ * loop so a billing outage on one provider does not trigger throwaway
197
+ * probes on every other healthy pair the agent has registered.
198
+ * Errors thrown by `verifyFn` are caught and recorded as `unavailable`.
199
+ */
200
+ async refreshUnhealthy() {
201
+ const probes = [];
202
+ for (const entry of this.entries.values()) {
203
+ if (entry.status === "healthy" || entry.status === "unknown") {
204
+ continue;
205
+ }
206
+ probes.push(this.probe(entry).then(() => void 0));
207
+ }
208
+ await Promise.all(probes);
209
+ return this.snapshot();
210
+ }
137
211
  /** Read-only view, primarily for logs and tests. */
138
212
  snapshot() {
139
213
  const out = [];
@@ -154,11 +228,23 @@ var LlmHealthMonitor = class {
154
228
  return entry.inFlight;
155
229
  }
156
230
  const fresh = this.now() - entry.lastVerifiedAt < this.ttlMs;
157
- if (fresh && entry.status === "healthy") {
158
- return Promise.resolve({ ok: true });
231
+ if (fresh) {
232
+ if (entry.status === "healthy") {
233
+ return Promise.resolve({ ok: true });
234
+ }
235
+ if (entry.status === "invalid" || entry.status === "billing") {
236
+ return Promise.resolve(this.synthesizeFailureFromCache(entry));
237
+ }
159
238
  }
160
239
  return this.probe(entry);
161
240
  }
241
+ synthesizeFailureFromCache(entry) {
242
+ const detail = entry.lastReason ?? "cached failure";
243
+ if (entry.status === "invalid") {
244
+ return { ok: false, reason: "invalid", status: 0, body: detail };
245
+ }
246
+ return { ok: false, reason: "billing", body: detail };
247
+ }
162
248
  probe(entry) {
163
249
  if (entry.inFlight) {
164
250
  return entry.inFlight;
@@ -215,29 +301,34 @@ function describeTransition(prev, next) {
215
301
  }
216
302
  return `* LLM provider ${next.provider} model ${next.model} recovered.`;
217
303
  }
218
- function startLlmHeartbeat(options) {
219
- const intervalMs = options.intervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
304
+ function startLlmRecovery(options) {
305
+ const intervalMs = options.intervalMs ?? LAZY_RECOVERY_INTERVAL_MS;
220
306
  const log = options.log ?? NOOP_LOG;
221
307
  let lastStatusByPair = /* @__PURE__ */ new Map();
222
308
  for (const entry of options.monitor.snapshot()) {
223
- lastStatusByPair.set(`${entry.provider}|${entry.model}`, entry.status);
309
+ lastStatusByPair.set(`${entry.provider}::${entry.model}`, entry.status);
224
310
  }
225
311
  let stopped = false;
226
312
  const tick = async () => {
227
313
  if (stopped) {
228
314
  return;
229
315
  }
316
+ const before = options.monitor.snapshot();
317
+ const anyUnhealthy = before.some((entry) => !isHealthyState(entry.status));
318
+ if (!anyUnhealthy) {
319
+ return;
320
+ }
230
321
  let snapshot;
231
322
  try {
232
- snapshot = await options.monitor.refreshAll();
323
+ snapshot = await options.monitor.refreshUnhealthy();
233
324
  } catch (error) {
234
325
  const message = error instanceof Error ? error.message : String(error);
235
- log(`! LLM heartbeat refreshAll failed: ${message}`);
326
+ log(`! LLM recovery probe failed: ${message}`);
236
327
  return;
237
328
  }
238
329
  const next = /* @__PURE__ */ new Map();
239
330
  for (const entry of snapshot) {
240
- const key = `${entry.provider}|${entry.model}`;
331
+ const key = `${entry.provider}::${entry.model}`;
241
332
  const prev = lastStatusByPair.get(key) ?? "unknown";
242
333
  const transitionMessage = describeTransition(prev, entry);
243
334
  if (transitionMessage) {
@@ -260,6 +351,7 @@ function startLlmHeartbeat(options) {
260
351
  }
261
352
  };
262
353
  }
354
+ var startLlmHeartbeat = startLlmRecovery;
263
355
 
264
356
  // src/primitives/rateLimiter.ts
265
357
  function createSlidingWindowLimiter(options) {
@@ -415,12 +507,16 @@ exports.DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS = DEFAULT_FREE_LLM_PER_CUSTOMER_
415
507
  exports.DEFAULT_HEALTH_TTL_MS = DEFAULT_HEALTH_TTL_MS;
416
508
  exports.DEFAULT_HEARTBEAT_INTERVAL_MS = DEFAULT_HEARTBEAT_INTERVAL_MS;
417
509
  exports.FREE_LLM_GLOBAL_KEY = FREE_LLM_GLOBAL_KEY;
510
+ exports.LAZY_RECOVERY_INTERVAL_MS = LAZY_RECOVERY_INTERVAL_MS;
418
511
  exports.LlmHealthError = LlmHealthError;
419
512
  exports.LlmHealthMonitor = LlmHealthMonitor;
513
+ exports.SCRIPT_EXIT_BILLING_EXHAUSTED = SCRIPT_EXIT_BILLING_EXHAUSTED;
514
+ exports.ScriptBillingExhaustedError = ScriptBillingExhaustedError;
420
515
  exports.UNAVAILABLE_TOLERANCE = UNAVAILABLE_TOLERANCE;
421
516
  exports.createFreeLlmLimiterSet = createFreeLlmLimiterSet;
422
517
  exports.freeLlmCustomerKey = freeLlmCustomerKey;
423
518
  exports.resolvePerSkillRateLimit = resolvePerSkillRateLimit;
424
519
  exports.startLlmHeartbeat = startLlmHeartbeat;
520
+ exports.startLlmRecovery = startLlmRecovery;
425
521
  //# sourceMappingURL=llm-health.cjs.map
426
522
  //# sourceMappingURL=llm-health.cjs.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/llm-health/constants.ts","../src/llm-health/types.ts","../src/llm-health/monitor.ts","../src/llm-health/heartbeat.ts","../src/primitives/rateLimiter.ts","../src/llm-health/free-llm-rate-limiter.ts"],"names":[],"mappings":";;;AAOO,IAAM,qBAAA,GAAwB,KAAK,EAAA,GAAK;AACxC,IAAM,6BAAA,GAAgC,KAAK,EAAA,GAAK;AAOhD,IAAM,qBAAA,GAAwB;AAM9B,IAAM,uCAAA,GAA0C,KAAK,EAAA,GAAK;AAC1D,IAAM,iCAAA,GAAoC;AAE1C,IAAM,oCAAoC,EAAA,GAAK;AAC/C,IAAM,2BAAA,GAA8B;AAEpC,IAAM,iCAAA,GAAoC;;;ACK1C,IAAM,cAAA,GAAN,cAA6B,KAAA,CAAM;AAAA,EAC/B,MAAA;AAAA,EACA,QAAA;AAAA,EACA,KAAA;AAAA,EAET,WAAA,CAAY,MAAA,EAA8B,QAAA,EAAkB,KAAA,EAAe,MAAA,EAAgB;AACzF,IAAA,KAAA,CAAM,CAAA,IAAA,EAAO,QAAQ,CAAA,CAAA,EAAI,KAAK,IAAI,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACrD,IAAA,IAAA,CAAK,IAAA,GAAO,gBAAA;AACZ,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,KAAA,GAAQ,KAAA;AAAA,EACf;AACF;;;ACOA,SAAS,KAAA,CAAM,UAAkB,KAAA,EAAuB;AACtD,EAAA,OAAO,CAAA,EAAG,QAAQ,CAAA,EAAA,EAAI,KAAK,CAAA,CAAA;AAC7B;AAEA,SAAS,aAAa,YAAA,EAA0C;AAC9D,EAAA,IAAI,aAAa,EAAA,EAAI;AACnB,IAAA,OAAO,IAAA;AAAA,EACT;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,OAAO,CAAA,KAAA,EAAQ,aAAa,MAAM,CAAA,EAAA,EAAK,aAAa,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,EACxE;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,MAAM,MAAA,GAAS,aAAa,MAAA,IAAU,CAAA;AACtC,IAAA,MAAM,QAAQ,YAAA,CAAa,IAAA,IAAQ,EAAA,EAAI,KAAA,CAAM,GAAG,GAAG,CAAA;AACnD,IAAA,OAAO,CAAA,KAAA,EAAQ,MAAM,CAAA,EAAA,EAAK,IAAI,CAAA,CAAA;AAAA,EAChC;AACA,EAAA,OAAO,YAAA,CAAa,KAAA;AACtB;AAEO,IAAM,mBAAN,MAAuB;AAAA,EACX,OAAA,uBAAc,GAAA,EAAmB;AAAA,EACjC,KAAA;AAAA,EACA,oBAAA;AAAA,EACA,GAAA;AAAA,EAEjB,WAAA,CAAY,OAAA,GAAmC,EAAC,EAAG;AACjD,IAAA,IAAA,CAAK,KAAA,GAAQ,QAAQ,KAAA,IAAS,qBAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,QAAQ,oBAAA,IAAwB,qBAAA;AAC5D,IAAA,IAAA,CAAK,GAAA,GAAM,OAAA,CAAQ,GAAA,IAAO,IAAA,CAAK,GAAA;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAS,IAAA,EAA0B;AACjC,IAAA,MAAM,GAAA,GAAM,KAAA,CAAM,IAAA,CAAK,QAAA,EAAU,KAAK,KAAK,CAAA;AAC3C,IAAA,IAAA,CAAK,OAAA,CAAQ,IAAI,GAAA,EAAK;AAAA,MACpB,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,OAAO,IAAA,CAAK,KAAA;AAAA,MACZ,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,MAAA,EAAQ,SAAA;AAAA,MACR,cAAA,EAAgB,CAAA;AAAA,MAChB,UAAA,EAAY,MAAA;AAAA,MACZ,QAAA,EAAU,IAAA;AAAA,MACV,mBAAA,EAAqB;AAAA,KACtB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,IAAA,CAAK,QAAA,EAAkB,KAAA,EAAe,YAAA,EAAwC;AAC5E,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAAA,EAC5C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,WAAA,CAAY,QAAA,EAAkB,KAAA,EAA8B;AAChE,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,MAAM,IAAI,cAAA,CAAe,SAAA,EAAW,QAAA,EAAU,OAAO,qBAAqB,CAAA;AAAA,IAC5E;AAEA,IAAA,MAAM,YAAA,GAAe,MAAM,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA;AAEnD,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,MAAA,KAAW,SAAA,IAAa,YAAA,CAAa,WAAW,SAAA,EAAW;AAC1E,MAAA,MAAM,IAAI,eAAe,YAAA,CAAa,MAAA,EAAQ,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IAC3F;AACA,IAAA,IAAI,KAAA,CAAM,mBAAA,IAAuB,IAAA,CAAK,oBAAA,EAAsB;AAC1D,MAAA,MAAM,IAAI,cAAA,CAAe,aAAA,EAAe,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IACrF;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,kBAAA,CAAmB,UAAkB,KAAA,EAAqB;AACxD,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,cAAA,GAAiB,CAAA;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAA,GAAyD;AAC7D,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA,EAGA,QAAA,GAA8C;AAC5C,IAAA,MAAM,MAAgC,EAAC;AACvC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,OAAO,KAAA,CAAM,KAAA;AAAA,QACb,QAAQ,KAAA,CAAM,MAAA;AAAA,QACd,gBAAgB,KAAA,CAAM,cAAA;AAAA,QACtB,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,qBAAqB,KAAA,CAAM;AAAA,OAC5B,CAAA;AAAA,IACH;AACA,IAAA,OAAO,GAAA;AAAA,EACT;AAAA,EAEQ,cAAc,KAAA,EAA2C;AAC/D,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,QAAQ,IAAA,CAAK,GAAA,EAAI,GAAI,KAAA,CAAM,iBAAiB,IAAA,CAAK,KAAA;AACvD,IAAA,IAAI,KAAA,IAAS,KAAA,CAAM,MAAA,KAAW,SAAA,EAAW;AACvC,MAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,EAAE,EAAA,EAAI,MAAM,CAAA;AAAA,IACrC;AACA,IAAA,OAAO,IAAA,CAAK,MAAM,KAAK,CAAA;AAAA,EACzB;AAAA,EAEQ,MAAM,KAAA,EAA2C;AACvD,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,WAAW,YAAyC;AACxD,MAAA,IAAI;AACF,QAAA,OAAO,MAAM,MAAM,QAAA,EAAS;AAAA,MAC9B,SAAS,KAAA,EAAO;AACd,QAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,QAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,aAAA,EAAe,OAAO,OAAA,EAAQ;AAAA,MAC5D;AAAA,IACF,CAAA,GAAG,CAAE,IAAA,CAAK,CAAC,YAAA,KAAiB;AAC1B,MAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAC1C,MAAA,KAAA,CAAM,QAAA,GAAW,IAAA;AACjB,MAAA,OAAO,YAAA;AAAA,IACT,CAAC,CAAA;AACD,IAAA,KAAA,CAAM,QAAA,GAAW,OAAA;AACjB,IAAA,OAAO,OAAA;AAAA,EACT;AAAA,EAEQ,iBAAA,CAAkB,OAAc,YAAA,EAAwC;AAC9E,IAAA,KAAA,CAAM,cAAA,GAAiB,KAAK,GAAA,EAAI;AAChC,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA,KAAA,CAAM,MAAA,GAAS,SAAA;AACf,MAAA,KAAA,CAAM,UAAA,GAAa,MAAA;AACnB,MAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAC5B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,UAAA,GAAa,aAAa,YAAY,CAAA;AAC5C,IAAA,IAAI,YAAA,CAAa,WAAW,aAAA,EAAe;AACzC,MAAA,KAAA,CAAM,MAAA,GAAS,aAAA;AACf,MAAA,KAAA,CAAM,mBAAA,IAAuB,CAAA;AAC7B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,SAAS,YAAA,CAAa,MAAA;AAC5B,IAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAAA,EAC9B;AACF;;;AClNA,IAAM,QAAA,GAAW,CAAC,IAAA,KAAuB;AAAC,CAAA;AAE1C,SAAS,eAAe,MAAA,EAAkC;AACxD,EAAA,OAAO,MAAA,KAAW,aAAa,MAAA,KAAW,SAAA;AAC5C;AAEA,SAAS,kBAAA,CACP,MACA,IAAA,EACoB;AACpB,EAAA,MAAM,UAAA,GAAa,eAAe,IAAI,CAAA;AACtC,EAAA,MAAM,UAAA,GAAa,cAAA,CAAe,IAAA,CAAK,MAAM,CAAA;AAC7C,EAAA,IAAI,eAAe,UAAA,EAAY;AAC7B,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,UAAA,IAAc,CAAC,UAAA,EAAY;AAC7B,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,UAAA,IAAc,IAAA,CAAK,MAAA;AACvC,IAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,IAAA,CAAK,KAAK,CAAA,mBAAA,EAAsB,IAAA,CAAK,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,qCAAA,CAAA;AAAA,EACxG;AACA,EAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,KAAK,KAAK,CAAA,WAAA,CAAA;AAC5D;AAEO,SAAS,kBAAkB,OAAA,EAAoD;AACpF,EAAA,MAAM,UAAA,GAAa,QAAQ,UAAA,IAAc,6BAAA;AACzC,EAAA,MAAM,GAAA,GAAM,QAAQ,GAAA,IAAO,QAAA;AAE3B,EAAA,IAAI,gBAAA,uBAAuB,GAAA,EAA6B;AACxD,EAAA,KAAA,MAAW,KAAA,IAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS,EAAG;AAC9C,IAAA,gBAAA,CAAiB,GAAA,CAAI,GAAG,KAAA,CAAM,QAAQ,IAAI,KAAA,CAAM,KAAK,CAAA,CAAA,EAAI,KAAA,CAAM,MAAM,CAAA;AAAA,EACvE;AAEA,EAAA,IAAI,OAAA,GAAU,KAAA;AAEd,EAAA,MAAM,OAAO,YAA2B;AACtC,IAAA,IAAI,OAAA,EAAS;AACX,MAAA;AAAA,IACF;AACA,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,OAAA,CAAQ,OAAA,CAAQ,UAAA,EAAW;AAAA,IAC9C,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,MAAA,GAAA,CAAI,CAAA,mCAAA,EAAsC,OAAO,CAAA,CAAE,CAAA;AACnD,MAAA;AAAA,IACF;AACA,IAAA,MAAM,IAAA,uBAAW,GAAA,EAA6B;AAC9C,IAAA,KAAA,MAAW,SAAS,QAAA,EAAU;AAC5B,MAAA,MAAM,MAAM,CAAA,EAAG,KAAA,CAAM,QAAQ,CAAA,CAAA,EAAI,MAAM,KAAK,CAAA,CAAA;AAC5C,MAAA,MAAM,IAAA,GAAO,gBAAA,CAAiB,GAAA,CAAI,GAAG,CAAA,IAAK,SAAA;AAC1C,MAAA,MAAM,iBAAA,GAAoB,kBAAA,CAAmB,IAAA,EAAM,KAAK,CAAA;AACxD,MAAA,IAAI,iBAAA,EAAmB;AACrB,QAAA,GAAA,CAAI,iBAAiB,CAAA;AAAA,MACvB;AACA,MAAA,IAAA,CAAK,GAAA,CAAI,GAAA,EAAK,KAAA,CAAM,MAAM,CAAA;AAAA,IAC5B;AACA,IAAA,gBAAA,GAAmB,IAAA;AAAA,EACrB,CAAA;AAEA,EAAA,MAAM,MAAA,GAAyB,YAAY,MAAM;AAC/C,IAAA,KAAK,IAAA,EAAK;AAAA,EACZ,GAAG,UAAU,CAAA;AAEb,EAAA,OAAO;AAAA,IACL,MAAM,MAAM;AACV,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AACA,MAAA,OAAA,GAAU,IAAA;AACV,MAAA,aAAA,CAAc,MAAM,CAAA;AAAA,IACtB;AAAA,GACF;AACF;;;AC/CO,SAAS,2BACd,OAAA,EACsB;AACtB,EAAA,MAAM,EAAE,QAAA,EAAU,YAAA,EAAc,OAAA,EAAQ,GAAI,OAAA;AAC5C,EAAA,IAAI,YAAY,CAAA,EAAG;AACjB,IAAA,MAAM,IAAI,WAAW,sBAAsB,CAAA;AAAA,EAC7C;AACA,EAAA,IAAI,gBAAgB,CAAA,EAAG;AACrB,IAAA,MAAM,IAAI,WAAW,0BAA0B,CAAA;AAAA,EACjD;AACA,EAAA,IAAI,WAAW,CAAA,EAAG;AAChB,IAAA,MAAM,IAAI,WAAW,qBAAqB,CAAA;AAAA,EAC5C;AAIA,EAAA,MAAM,OAAA,uBAAc,GAAA,EAAmB;AAEvC,EAAA,SAAS,aAAA,GAAsB;AAC7B,IAAA,OAAO,OAAA,CAAQ,OAAO,OAAA,EAAS;AAC7B,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,IAAA,EAAK,CAAE,MAAK,CAAE,KAAA;AACxC,MAAA,IAAI,cAAc,MAAA,EAAW;AAC3B,QAAA;AAAA,MACF;AACA,MAAA,OAAA,CAAQ,OAAO,SAAS,CAAA;AAAA,IAC1B;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,IAAA,CAAK,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC7C,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,GAAA,CAAI,GAAG,CAAA;AAC7B,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,OAAO,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,GAAA,GAAM,QAAA,EAAU,OAAO,CAAA,EAAE;AAAA,MAC5D;AACA,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,MAAM,MAAA,GAAS,YAAA;AAAA,QACxB,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC9C,MAAA,MAAM,KAAA,GAAQ,QAAQ,GAAA,CAAI,GAAG,KAAK,EAAE,IAAA,EAAM,EAAC,EAAE;AAC7C,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AAEjE,MAAA,IAAI,KAAA,CAAM,UAAU,YAAA,EAAc;AAGhC,QAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,QAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,QAAA,OAAO;AAAA,UACL,OAAA,EAAS,KAAA;AAAA,UACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,UAC7B,OAAO,KAAA,CAAM;AAAA,SACf;AAAA,MACF;AACA,MAAA,KAAA,CAAM,KAAK,GAAG,CAAA;AACd,MAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,MAAA,aAAA,EAAc;AACd,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,IAAA;AAAA,QACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,GAAM,IAAA,CAAK,GAAA,EAAI,EAAS;AAC5B,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,KAAA,MAAW,CAAC,GAAA,EAAK,KAAK,CAAA,IAAK,OAAA,EAAS;AAClC,QAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,QAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,UAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAAA,QACpB,CAAA,MAAA,IAAW,KAAA,CAAM,MAAA,KAAW,KAAA,CAAM,KAAK,MAAA,EAAQ;AAC7C,UAAA,KAAA,CAAM,IAAA,GAAO,KAAA;AAAA,QACf;AAAA,MACF;AAAA,IACF,CAAA;AAAA,IACA,IAAA,GAAe;AACb,MAAA,OAAO,OAAA,CAAQ,IAAA;AAAA,IACjB,CAAA;AAAA,IACA,KAAA,GAAc;AACZ,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,GACF;AACF;;;ACpHO,IAAM,mBAAA,GAAsB;AAqC5B,SAAS,yBACd,cAAA,EAC4B;AAC5B,EAAA,IAAI,CAAC,cAAA,EAAgB;AACnB,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,cAAA,CAAe,YAAA,IAAgB,CAAA,IAAK,cAAA,CAAe,eAAe,CAAA,EAAG;AACvE,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,OAAO,cAAA;AACT;AAEO,SAAS,uBAAA,CAAwB,OAAA,GAAiC,EAAC,EAAsB;AAC9F,EAAA,MAAM,kBAAA,GAAqC,QAAQ,kBAAA,IAAsB;AAAA,IACvE,WAAA,EAAa,uCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,MAAA,GAAyB,QAAQ,MAAA,IAAU;AAAA,IAC/C,WAAA,EAAa,iCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,OAAA,GAAU,QAAQ,OAAA,IAAW,iCAAA;AAEnC,EAAA,MAAM,gBAAgB,0BAAA,CAA2B;AAAA,IAC/C,UAAU,MAAA,CAAO,WAAA;AAAA,IACjB,cAAc,MAAA,CAAO,YAAA;AAAA,IACrB,OAAA,EAAS;AAAA,GACV,CAAA;AAED,EAAA,MAAM,iBAAiB,0BAAA,CAA2B;AAAA,IAChD,UAAU,kBAAA,CAAmB,WAAA;AAAA,IAC7B,cAAc,kBAAA,CAAmB,YAAA;AAAA,IACjC;AAAA,GACD,CAAA;AAED,EAAA,MAAM,gBAAA,uBAAuB,GAAA,EAAkC;AAE/D,EAAA,SAAS,qBAAA,CACP,WACA,QAAA,EACsB;AACtB,IAAA,MAAM,SAAA,GAAY,yBAAyB,QAAQ,CAAA;AACnD,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,OAAO,cAAA;AAAA,IACT;AACA,IAAA,MAAM,MAAA,GAAS,gBAAA,CAAiB,GAAA,CAAI,SAAS,CAAA;AAC7C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,OAAO,MAAA;AAAA,IACT;AACA,IAAA,MAAM,UAAU,0BAAA,CAA2B;AAAA,MACzC,UAAU,SAAA,CAAU,WAAA;AAAA,MACpB,cAAc,SAAA,CAAU,YAAA;AAAA,MACxB;AAAA,KACD,CAAA;AACD,IAAA,gBAAA,CAAiB,GAAA,CAAI,WAAW,OAAO,CAAA;AACvC,IAAA,OAAO,OAAA;AAAA,EACT;AAEA,EAAA,SAAS,gBAAA,GAAyB;AAChC,IAAA,cAAA,CAAe,KAAA,EAAM;AACrB,IAAA,KAAA,MAAW,OAAA,IAAW,gBAAA,CAAiB,MAAA,EAAO,EAAG;AAC/C,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,aAAA;AAAA,IACA,qBAAA;AAAA,IACA,gBAAA;AAAA,IACA,kBAAA;AAAA,IACA;AAAA,GACF;AACF;AAUO,SAAS,kBAAA,CAAmB,YAAoB,SAAA,EAA2B;AAChF,EAAA,OAAO,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,SAAS,CAAA,CAAA;AACnC","file":"llm-health.cjs","sourcesContent":["/**\n * LLM health monitor and heartbeat tunable defaults. CLI and plugin consumers\n * can override via `process.env.ELISYM_LLM_HEALTH_TTL_MS` and\n * `ELISYM_LLM_HEARTBEAT_INTERVAL_MS` and pass the resolved values into\n * the monitor/heartbeat options.\n */\n\nexport const DEFAULT_HEALTH_TTL_MS = 10 * 60 * 1000;\nexport const DEFAULT_HEARTBEAT_INTERVAL_MS = 10 * 60 * 1000;\n\n/**\n * Number of consecutive `unavailable` results tolerated before\n * `assertReady` starts throwing. The first `unavailable - 1` are treated\n * as transient blips so a brief network hiccup does not block jobs.\n */\nexport const UNAVAILABLE_TOLERANCE = 3;\n\n/**\n * Free-LLM rate-limit defaults. Applied when a SKILL.md with\n * `mode: 'llm'` and `price: 0` does not declare its own `rate_limit`.\n */\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS = 60 * 60 * 1000;\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_MAX = 3;\n\nexport const DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS = 60 * 1000;\nexport const DEFAULT_FREE_LLM_GLOBAL_MAX = 30;\n\nexport const DEFAULT_FREE_LLM_MAX_TRACKED_KEYS = 1000;\n","/**\n * Shared types for the LLM health monitor. Provider-agnostic: the\n * actual HTTP probe lives in CLI/plugin and is supplied via dependency\n * injection (`verifyFn`).\n */\n\nexport type LlmHealthStatus = 'unknown' | 'healthy' | 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Result of probing an API key with a specific model. Discriminated on\n * `ok`, then on `reason` for failures. Adding a new failure reason is a\n * breaking change for exhaustive switches; update consumers in lockstep.\n *\n * - `invalid`: HTTP 401/403 - key rejected outright.\n * - `billing`: HTTP 402, or 400 with credit/billing/insufficient marker,\n * or OpenAI's 429 with `insufficient_quota`. Operator out of credits.\n * - `unavailable`: transient (HTTP 429 without quota marker, 5xx, network\n * error). May resolve on retry.\n */\nexport type LlmKeyVerification =\n | { ok: true }\n | { ok: false; reason: 'invalid'; status: number; body: string }\n | { ok: false; reason: 'billing'; status?: number; body?: string }\n | { ok: false; reason: 'unavailable'; error: string };\n\nexport type LlmHealthErrorReason = 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Thrown by `LlmHealthMonitor.assertReady` when the gate refuses a job.\n * Carries the operator-facing reason so callers can log it; customer-facing\n * messages should be sanitized at the call site (see `runtime.ts` preflight).\n */\nexport class LlmHealthError extends Error {\n readonly reason: LlmHealthErrorReason;\n readonly provider: string;\n readonly model: string;\n\n constructor(reason: LlmHealthErrorReason, provider: string, model: string, detail: string) {\n super(`LLM ${provider}/${model} ${reason}: ${detail}`);\n this.name = 'LlmHealthError';\n this.reason = reason;\n this.provider = provider;\n this.model = model;\n }\n}\n\n/**\n * Per-skill rate-limit declaration. Snake-case in SKILL.md frontmatter,\n * camelCase here. Applies to any skill mode but the framework adds a\n * default cap only for free LLM skills.\n */\nexport interface SkillRateLimit {\n perWindowMs: number;\n maxPerWindow: number;\n}\n\n/** Read-only health snapshot for logs/tests. */\nexport interface LlmHealthSnapshotEntry {\n provider: string;\n model: string;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n consecutiveFailures: number;\n}\n","/**\n * Provider-agnostic health monitor for LLM API keys. State machine per\n * (provider, model) pair: caches the last verification result for\n * `ttlMs`, deduplicates concurrent probes via an in-flight promise, and\n * tolerates a bounded number of consecutive `unavailable` results\n * before `assertReady` starts throwing.\n *\n * The monitor itself never touches the network. Each pair must be\n * registered with a `verifyFn` lambda that performs the actual probe;\n * callers (CLI/plugin) supply provider-specific HTTP from their layer.\n */\n\nimport {\n DEFAULT_HEALTH_TTL_MS,\n UNAVAILABLE_TOLERANCE as DEFAULT_UNAVAILABLE_TOLERANCE,\n} from './constants';\nimport {\n LlmHealthError,\n type LlmHealthSnapshotEntry,\n type LlmHealthStatus,\n type LlmKeyVerification,\n} from './types';\n\nexport type LlmKeyVerifyFn = (signal?: AbortSignal) => Promise<LlmKeyVerification>;\n\nexport interface LlmHealthMonitorOptions {\n /** Time after which a cached `healthy` result is re-probed. Default 10 min. */\n ttlMs?: number;\n /** Number of consecutive unavailable results tolerated. Default 3. */\n unavailableTolerance?: number;\n /** Optional clock injection for tests. */\n now?: () => number;\n}\n\nexport interface RegisterArgs {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n}\n\ninterface Entry {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n inFlight: Promise<LlmKeyVerification> | null;\n consecutiveFailures: number;\n}\n\nfunction keyOf(provider: string, model: string): string {\n return `${provider}\u0000${model}`;\n}\n\nfunction reasonDetail(verification: LlmKeyVerification): string {\n if (verification.ok) {\n return 'ok';\n }\n if (verification.reason === 'invalid') {\n return `HTTP ${verification.status}: ${verification.body.slice(0, 200)}`;\n }\n if (verification.reason === 'billing') {\n const status = verification.status ?? 0;\n const body = (verification.body ?? '').slice(0, 200);\n return `HTTP ${status}: ${body}`;\n }\n return verification.error;\n}\n\nexport class LlmHealthMonitor {\n private readonly entries = new Map<string, Entry>();\n private readonly ttlMs: number;\n private readonly unavailableTolerance: number;\n private readonly now: () => number;\n\n constructor(options: LlmHealthMonitorOptions = {}) {\n this.ttlMs = options.ttlMs ?? DEFAULT_HEALTH_TTL_MS;\n this.unavailableTolerance = options.unavailableTolerance ?? DEFAULT_UNAVAILABLE_TOLERANCE;\n this.now = options.now ?? Date.now;\n }\n\n /**\n * Register a (provider, model) pair with its probe function. Idempotent\n * on the (provider, model) key: re-registering replaces the verifyFn\n * and resets state to `unknown`. Callers typically re-register only\n * when the operator rotates the API key.\n */\n register(args: RegisterArgs): void {\n const key = keyOf(args.provider, args.model);\n this.entries.set(key, {\n provider: args.provider,\n model: args.model,\n verifyFn: args.verifyFn,\n status: 'unknown',\n lastVerifiedAt: 0,\n lastReason: undefined,\n inFlight: null,\n consecutiveFailures: 0,\n });\n }\n\n /**\n * Seed the monitor with an already-known verification result, e.g.\n * the one captured at startup. Skips an extra probe on the first\n * `assertReady` call within the TTL window. No-op if the pair is not\n * registered.\n */\n seed(provider: string, model: string, verification: LlmKeyVerification): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n this.applyVerification(entry, verification);\n }\n\n /**\n * Main gate before doing LLM work. Throws `LlmHealthError` on terminal\n * states (`invalid`, `billing`, or `unavailable` past tolerance);\n * resolves silently on `healthy` (cache hit or fresh probe) or\n * tolerated `unavailable`.\n *\n * Concurrent `assertReady` calls for the same pair are deduplicated:\n * the second caller awaits the first probe rather than launching a\n * parallel one.\n */\n async assertReady(provider: string, model: string): Promise<void> {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n throw new LlmHealthError('invalid', provider, model, 'pair not registered');\n }\n\n const verification = await this.probeIfNeeded(entry);\n\n if (verification.ok) {\n return;\n }\n if (verification.reason === 'invalid' || verification.reason === 'billing') {\n throw new LlmHealthError(verification.reason, provider, model, reasonDetail(verification));\n }\n if (entry.consecutiveFailures >= this.unavailableTolerance) {\n throw new LlmHealthError('unavailable', provider, model, reasonDetail(verification));\n }\n // Tolerated unavailable: caller proceeds, real LLM call will surface\n // any transient issue with its own error path.\n }\n\n /**\n * Force the next `assertReady` for this pair to re-probe regardless of\n * TTL. Used when a real LLM call surfaces 401/402 mid-job - the cached\n * `healthy` is stale and we want to catch the next request.\n */\n markFailureFromJob(provider: string, model: string): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n entry.lastVerifiedAt = 0;\n }\n\n /**\n * Refresh every registered pair concurrently. Heartbeat hook. Errors\n * thrown by `verifyFn` are caught and recorded as `unavailable`.\n */\n async refreshAll(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n for (const entry of this.entries.values()) {\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /** Read-only view, primarily for logs and tests. */\n snapshot(): readonly LlmHealthSnapshotEntry[] {\n const out: LlmHealthSnapshotEntry[] = [];\n for (const entry of this.entries.values()) {\n out.push({\n provider: entry.provider,\n model: entry.model,\n status: entry.status,\n lastVerifiedAt: entry.lastVerifiedAt,\n lastReason: entry.lastReason,\n consecutiveFailures: entry.consecutiveFailures,\n });\n }\n return out;\n }\n\n private probeIfNeeded(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const fresh = this.now() - entry.lastVerifiedAt < this.ttlMs;\n if (fresh && entry.status === 'healthy') {\n return Promise.resolve({ ok: true });\n }\n return this.probe(entry);\n }\n\n private probe(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const promise = (async (): Promise<LlmKeyVerification> => {\n try {\n return await entry.verifyFn();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n return { ok: false, reason: 'unavailable', error: message };\n }\n })().then((verification) => {\n this.applyVerification(entry, verification);\n entry.inFlight = null;\n return verification;\n });\n entry.inFlight = promise;\n return promise;\n }\n\n private applyVerification(entry: Entry, verification: LlmKeyVerification): void {\n entry.lastVerifiedAt = this.now();\n if (verification.ok) {\n entry.status = 'healthy';\n entry.lastReason = undefined;\n entry.consecutiveFailures = 0;\n return;\n }\n entry.lastReason = reasonDetail(verification);\n if (verification.reason === 'unavailable') {\n entry.status = 'unavailable';\n entry.consecutiveFailures += 1;\n return;\n }\n entry.status = verification.reason;\n entry.consecutiveFailures = 0;\n }\n}\n","/**\n * Periodic LLM health probe. Stops on demand. Logs status transitions\n * (healthy <-> unhealthy) but does not log every successful tick to keep\n * the operator log quiet.\n *\n * The heartbeat pure-delegates to `monitor.refreshAll()`; both timing and\n * logging policy live here so the monitor stays a pure state-machine.\n */\n\nimport { DEFAULT_HEARTBEAT_INTERVAL_MS } from './constants';\nimport type { LlmHealthMonitor } from './monitor';\nimport type { LlmHealthSnapshotEntry, LlmHealthStatus } from './types';\n\nexport interface HeartbeatHandle {\n stop(): void;\n}\n\nexport interface StartLlmHeartbeatOptions {\n monitor: LlmHealthMonitor;\n /** Defaults to 10 minutes. */\n intervalMs?: number;\n /** Operator log sink. Defaults to no-op (silent). */\n log?: (msg: string) => void;\n}\n\ntype IntervalHandle = ReturnType<typeof setInterval>;\n\nconst NOOP_LOG = (_msg: string): void => {};\n\nfunction isHealthyState(status: LlmHealthStatus): boolean {\n return status === 'healthy' || status === 'unknown';\n}\n\nfunction describeTransition(\n prev: LlmHealthStatus,\n next: LlmHealthSnapshotEntry,\n): string | undefined {\n const wasHealthy = isHealthyState(prev);\n const nowHealthy = isHealthyState(next.status);\n if (wasHealthy === nowHealthy) {\n return undefined;\n }\n if (wasHealthy && !nowHealthy) {\n const reason = next.lastReason ?? next.status;\n return `! LLM provider ${next.provider} model ${next.model} became unhealthy: ${next.status} (${reason}). Refusing LLM jobs until recovered.`;\n }\n return `* LLM provider ${next.provider} model ${next.model} recovered.`;\n}\n\nexport function startLlmHeartbeat(options: StartLlmHeartbeatOptions): HeartbeatHandle {\n const intervalMs = options.intervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;\n const log = options.log ?? NOOP_LOG;\n\n let lastStatusByPair = new Map<string, LlmHealthStatus>();\n for (const entry of options.monitor.snapshot()) {\n lastStatusByPair.set(`${entry.provider}|${entry.model}`, entry.status);\n }\n\n let stopped = false;\n\n const tick = async (): Promise<void> => {\n if (stopped) {\n return;\n }\n let snapshot: readonly LlmHealthSnapshotEntry[];\n try {\n snapshot = await options.monitor.refreshAll();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n log(`! LLM heartbeat refreshAll failed: ${message}`);\n return;\n }\n const next = new Map<string, LlmHealthStatus>();\n for (const entry of snapshot) {\n const key = `${entry.provider}|${entry.model}`;\n const prev = lastStatusByPair.get(key) ?? 'unknown';\n const transitionMessage = describeTransition(prev, entry);\n if (transitionMessage) {\n log(transitionMessage);\n }\n next.set(key, entry.status);\n }\n lastStatusByPair = next;\n };\n\n const handle: IntervalHandle = setInterval(() => {\n void tick();\n }, intervalMs);\n\n return {\n stop: () => {\n if (stopped) {\n return;\n }\n stopped = true;\n clearInterval(handle);\n },\n };\n}\n","/**\n * Sliding-window rate limiter keyed by an arbitrary string (typically a\n * customer pubkey). Each key gets at most `maxPerWindow` requests inside a\n * rolling `windowMs`. Stale timestamps are pruned lazily on every `check`.\n * When the tracked-key set grows past `maxKeys`, the least-recently-used\n * key is evicted so an attacker cannot exhaust memory by cycling keys.\n *\n * Thread-safety: not required. Designed for single-threaded JS consumers\n * (Node/Bun event loops, browser main thread). No timers - pruning happens\n * inside `check` and `prune`.\n */\n\nexport interface SlidingWindowLimiterOptions {\n /** Rolling window width, in ms. */\n windowMs: number;\n /** Max hits allowed per key inside the window. */\n maxPerWindow: number;\n /** Cap on total tracked keys. LRU-evicted past this cap. */\n maxKeys: number;\n}\n\nexport interface RateLimitDecision {\n allowed: boolean;\n /** Wall-clock timestamp (ms) when the limit window will reset for this key. */\n resetAt: number;\n /** Number of hits inside the current window after this call (or the attempted hit if denied). */\n count: number;\n}\n\nexport interface SlidingWindowLimiter {\n /** Record a hit against `key`; return whether it was allowed. */\n check(key: string, now?: number): RateLimitDecision;\n /**\n * Inspect the current state for `key` without recording a hit. Useful\n * when a single request must clear multiple limiters and callers want\n * to avoid double-counting against one limiter after another denies.\n */\n peek(key: string, now?: number): RateLimitDecision;\n /** Drop entries whose windows have fully elapsed. Bounded memory hygiene. */\n prune(now?: number): void;\n /** Current number of tracked keys. */\n size(): number;\n /** Clear all state. */\n reset(): void;\n}\n\ninterface Entry {\n /** Sliding-window timestamps in ms. Sorted ascending. */\n hits: number[];\n}\n\nexport function createSlidingWindowLimiter(\n options: SlidingWindowLimiterOptions,\n): SlidingWindowLimiter {\n const { windowMs, maxPerWindow, maxKeys } = options;\n if (windowMs <= 0) {\n throw new RangeError('windowMs must be > 0');\n }\n if (maxPerWindow <= 0) {\n throw new RangeError('maxPerWindow must be > 0');\n }\n if (maxKeys <= 0) {\n throw new RangeError('maxKeys must be > 0');\n }\n\n // LRU is implemented via Map's insertion-order: every check refreshes\n // the entry by deleting and re-setting it, moving it to the tail.\n const entries = new Map<string, Entry>();\n\n function evictIfNeeded(): void {\n while (entries.size > maxKeys) {\n const oldestKey = entries.keys().next().value as string | undefined;\n if (oldestKey === undefined) {\n return;\n }\n entries.delete(oldestKey);\n }\n }\n\n return {\n peek(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key);\n if (!entry) {\n return { allowed: true, resetAt: now + windowMs, count: 0 };\n }\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n return {\n allowed: fresh.length < maxPerWindow,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n check(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key) ?? { hits: [] };\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n\n if (fresh.length >= maxPerWindow) {\n // Refresh LRU order even on denial so an attacker hammering the\n // same key cannot push other tracked keys out via eviction.\n entries.delete(key);\n entries.set(key, { hits: fresh });\n return {\n allowed: false,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n }\n fresh.push(now);\n entries.delete(key);\n entries.set(key, { hits: fresh });\n evictIfNeeded();\n return {\n allowed: true,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n prune(now = Date.now()): void {\n const cutoff = now - windowMs;\n for (const [key, entry] of entries) {\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n if (fresh.length === 0) {\n entries.delete(key);\n } else if (fresh.length !== entry.hits.length) {\n entry.hits = fresh;\n }\n }\n },\n size(): number {\n return entries.size;\n },\n reset(): void {\n entries.clear();\n },\n };\n}\n","/**\n * Two-tier rate limiter for free LLM skills (mode='llm', price=0).\n * Provides Sybil-resistant global cap plus per-customer-per-skill cap\n * that respects an optional skill-level override.\n *\n * Both limiters are independent of the existing per-customer general\n * limiter; callers (CLI runtime, plugin handler) check all of them in\n * sequence and only `check()` after every tier passes `peek()` so a\n * single denial does not consume slots in earlier tiers.\n */\n\nimport { createSlidingWindowLimiter, type SlidingWindowLimiter } from '../primitives/rateLimiter';\nimport {\n DEFAULT_FREE_LLM_GLOBAL_MAX,\n DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n DEFAULT_FREE_LLM_MAX_TRACKED_KEYS,\n DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n} from './constants';\nimport type { SkillRateLimit } from './types';\n\nexport const FREE_LLM_GLOBAL_KEY = '__free_llm_global__';\n\nexport interface FreeLlmLimiterOptions {\n /** Max tracked (customer, skill) keys. Default 1000 (LRU evicted past cap). */\n maxKeys?: number;\n /** Default per-customer cap when a free LLM skill omits `rate_limit`. */\n defaultPerCustomer?: SkillRateLimit;\n /** Global Sybil cap across all free LLM jobs. */\n global?: SkillRateLimit;\n}\n\nexport interface FreeLlmLimiterSet {\n /** Sybil-protection limiter keyed on `FREE_LLM_GLOBAL_KEY`. */\n globalLimiter: SlidingWindowLimiter;\n /**\n * Return the per-customer limiter for a given skill. Skills that\n * declare a `rate_limit` get their own limiter sized to that\n * (window, cap); skills that don't share a default-window limiter.\n * Each call uses `peek/check` keyed on `customerId` alone since each\n * skill has a dedicated limiter store.\n */\n getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter;\n /** Drop expired hits from every per-customer limiter (default + per-skill). */\n prunePerCustomer(): void;\n /** Default cap to apply when a skill omits `rate_limit`. */\n defaultPerCustomer: SkillRateLimit;\n /** Sybil-cap settings (echo of global option for diagnostics). */\n global: SkillRateLimit;\n}\n\n/**\n * Effective per-skill rate limit: the override if it's well-formed,\n * else `undefined` so callers can fall through to the default limiter.\n */\nexport function resolvePerSkillRateLimit(\n skillRateLimit: SkillRateLimit | undefined,\n): SkillRateLimit | undefined {\n if (!skillRateLimit) {\n return undefined;\n }\n if (skillRateLimit.maxPerWindow <= 0 || skillRateLimit.perWindowMs <= 0) {\n return undefined;\n }\n return skillRateLimit;\n}\n\nexport function createFreeLlmLimiterSet(options: FreeLlmLimiterOptions = {}): FreeLlmLimiterSet {\n const defaultPerCustomer: SkillRateLimit = options.defaultPerCustomer ?? {\n perWindowMs: DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n };\n const global: SkillRateLimit = options.global ?? {\n perWindowMs: DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_GLOBAL_MAX,\n };\n const maxKeys = options.maxKeys ?? DEFAULT_FREE_LLM_MAX_TRACKED_KEYS;\n\n const globalLimiter = createSlidingWindowLimiter({\n windowMs: global.perWindowMs,\n maxPerWindow: global.maxPerWindow,\n maxKeys: 1,\n });\n\n const defaultLimiter = createSlidingWindowLimiter({\n windowMs: defaultPerCustomer.perWindowMs,\n maxPerWindow: defaultPerCustomer.maxPerWindow,\n maxKeys,\n });\n\n const perSkillLimiters = new Map<string, SlidingWindowLimiter>();\n\n function getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter {\n const effective = resolvePerSkillRateLimit(override);\n if (!effective) {\n return defaultLimiter;\n }\n const cached = perSkillLimiters.get(skillName);\n if (cached) {\n return cached;\n }\n const limiter = createSlidingWindowLimiter({\n windowMs: effective.perWindowMs,\n maxPerWindow: effective.maxPerWindow,\n maxKeys,\n });\n perSkillLimiters.set(skillName, limiter);\n return limiter;\n }\n\n function prunePerCustomer(): void {\n defaultLimiter.prune();\n for (const limiter of perSkillLimiters.values()) {\n limiter.prune();\n }\n }\n\n return {\n globalLimiter,\n getPerCustomerLimiter,\n prunePerCustomer,\n defaultPerCustomer,\n global,\n };\n}\n\n/**\n * Compose a per-skill key for the per-customer limiter. The default\n * limiter is shared across skills without an override, so the skill\n * name is needed to keep each (customer, skill) pair counted\n * independently. Per-skill limiters use the same key for consistency\n * even though the skill component is redundant inside a dedicated\n * limiter store.\n */\nexport function freeLlmCustomerKey(customerId: string, skillName: string): string {\n return `${customerId}|${skillName}`;\n}\n"]}
1
+ {"version":3,"sources":["../src/llm-health/constants.ts","../src/llm-health/types.ts","../src/llm-health/monitor.ts","../src/llm-health/heartbeat.ts","../src/primitives/rateLimiter.ts","../src/llm-health/free-llm-rate-limiter.ts"],"names":[],"mappings":";;;AAOO,IAAM,qBAAA,GAAwB,KAAK,EAAA,GAAK;AACxC,IAAM,6BAAA,GAAgC,KAAK,EAAA,GAAK;AAShD,IAAM,yBAAA,GAA4B,IAAI,EAAA,GAAK;AAmB3C,IAAM,6BAAA,GAAgC;AAOtC,IAAM,qBAAA,GAAwB;AAM9B,IAAM,uCAAA,GAA0C,KAAK,EAAA,GAAK;AAC1D,IAAM,iCAAA,GAAoC;AAE1C,IAAM,oCAAoC,EAAA,GAAK;AAC/C,IAAM,2BAAA,GAA8B;AAEpC,IAAM,iCAAA,GAAoC;;;ACvB1C,IAAM,cAAA,GAAN,cAA6B,KAAA,CAAM;AAAA,EAC/B,MAAA;AAAA,EACA,QAAA;AAAA,EACA,KAAA;AAAA,EAET,WAAA,CAAY,MAAA,EAA8B,QAAA,EAAkB,KAAA,EAAe,MAAA,EAAgB;AACzF,IAAA,KAAA,CAAM,CAAA,IAAA,EAAO,QAAQ,CAAA,CAAA,EAAI,KAAK,IAAI,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACrD,IAAA,IAAA,CAAK,IAAA,GAAO,gBAAA;AACZ,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,KAAA,GAAQ,KAAA;AAAA,EACf;AACF;AAcO,IAAM,2BAAA,GAAN,cAA0C,KAAA,CAAM;AAAA,EAC5C,QAAA;AAAA,EACA,MAAA;AAAA,EACA,MAAA;AAAA,EAET,WAAA,CAAY,QAAA,EAAkB,MAAA,EAAgB,MAAA,EAAgB;AAC5D,IAAA,MAAM,SAAS,MAAA,CAAO,IAAA,EAAK,IAAK,MAAA,CAAO,MAAK,IAAK,aAAA;AACjD,IAAA,KAAA,CAAM,CAAA,0CAAA,EAA6C,QAAQ,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AACxE,IAAA,IAAA,CAAK,IAAA,GAAO,6BAAA;AACZ,IAAA,IAAA,CAAK,QAAA,GAAW,QAAA;AAChB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AACF;;;ACpBA,SAAS,KAAA,CAAM,UAAkB,KAAA,EAAuB;AACtD,EAAA,OAAO,CAAA,EAAG,QAAQ,CAAA,EAAA,EAAK,KAAK,CAAA,CAAA;AAC9B;AAEA,SAAS,aAAa,YAAA,EAA0C;AAC9D,EAAA,IAAI,aAAa,EAAA,EAAI;AACnB,IAAA,OAAO,IAAA;AAAA,EACT;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,OAAO,CAAA,KAAA,EAAQ,aAAa,MAAM,CAAA,EAAA,EAAK,aAAa,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,EACxE;AACA,EAAA,IAAI,YAAA,CAAa,WAAW,SAAA,EAAW;AACrC,IAAA,MAAM,MAAA,GAAS,aAAa,MAAA,IAAU,CAAA;AACtC,IAAA,MAAM,QAAQ,YAAA,CAAa,IAAA,IAAQ,EAAA,EAAI,KAAA,CAAM,GAAG,GAAG,CAAA;AACnD,IAAA,OAAO,CAAA,KAAA,EAAQ,MAAM,CAAA,EAAA,EAAK,IAAI,CAAA,CAAA;AAAA,EAChC;AACA,EAAA,OAAO,YAAA,CAAa,KAAA;AACtB;AAEO,IAAM,mBAAN,MAAuB;AAAA,EACX,OAAA,uBAAc,GAAA,EAAmB;AAAA,EACjC,KAAA;AAAA,EACA,oBAAA;AAAA,EACA,GAAA;AAAA,EAEjB,WAAA,CAAY,OAAA,GAAmC,EAAC,EAAG;AACjD,IAAA,IAAA,CAAK,KAAA,GAAQ,QAAQ,KAAA,IAAS,qBAAA;AAC9B,IAAA,IAAA,CAAK,oBAAA,GAAuB,QAAQ,oBAAA,IAAwB,qBAAA;AAC5D,IAAA,IAAA,CAAK,GAAA,GAAM,OAAA,CAAQ,GAAA,IAAO,IAAA,CAAK,GAAA;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAS,IAAA,EAA0B;AACjC,IAAA,MAAM,GAAA,GAAM,KAAA,CAAM,IAAA,CAAK,QAAA,EAAU,KAAK,KAAK,CAAA;AAC3C,IAAA,IAAA,CAAK,OAAA,CAAQ,IAAI,GAAA,EAAK;AAAA,MACpB,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,OAAO,IAAA,CAAK,KAAA;AAAA,MACZ,UAAU,IAAA,CAAK,QAAA;AAAA,MACf,MAAA,EAAQ,SAAA;AAAA,MACR,cAAA,EAAgB,CAAA;AAAA,MAChB,UAAA,EAAY,MAAA;AAAA,MACZ,QAAA,EAAU,IAAA;AAAA,MACV,mBAAA,EAAqB;AAAA,KACtB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,IAAA,CAAK,QAAA,EAAkB,KAAA,EAAe,YAAA,EAAwC;AAC5E,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAAA,EAC5C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,WAAA,CAAY,QAAA,EAAkB,KAAA,EAA8B;AAChE,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,MAAM,IAAI,cAAA,CAAe,SAAA,EAAW,QAAA,EAAU,OAAO,qBAAqB,CAAA;AAAA,IAC5E;AAEA,IAAA,MAAM,YAAA,GAAe,MAAM,IAAA,CAAK,aAAA,CAAc,KAAK,CAAA;AAEnD,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,MAAA,KAAW,SAAA,IAAa,YAAA,CAAa,WAAW,SAAA,EAAW;AAC1E,MAAA,MAAM,IAAI,eAAe,YAAA,CAAa,MAAA,EAAQ,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IAC3F;AACA,IAAA,IAAI,KAAA,CAAM,mBAAA,IAAuB,IAAA,CAAK,oBAAA,EAAsB;AAC1D,MAAA,MAAM,IAAI,cAAA,CAAe,aAAA,EAAe,UAAU,KAAA,EAAO,YAAA,CAAa,YAAY,CAAC,CAAA;AAAA,IACrF;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,kBAAA,CAAmB,UAAkB,KAAA,EAAqB;AACxD,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,cAAA,GAAiB,CAAA;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,oBAAA,CACE,QAAA,EACA,KAAA,EACA,MAAA,EACA,MAAA,EACM;AACN,IAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,IAAI,KAAA,CAAM,QAAA,EAAU,KAAK,CAAC,CAAA;AACrD,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAW,SAAA,EAAW;AACxB,MAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,QAC5B,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,MAAA,EAAQ,CAAA;AAAA,QACR,MAAM,MAAA,IAAU;AAAA,OACjB,CAAA;AACD,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAW,SAAA,EAAW;AACxB,MAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,QAC5B,EAAA,EAAI,KAAA;AAAA,QACJ,MAAA,EAAQ,SAAA;AAAA,QACR,MAAM,MAAA,IAAU;AAAA,OACjB,CAAA;AACD,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,kBAAkB,KAAA,EAAO;AAAA,MAC5B,EAAA,EAAI,KAAA;AAAA,MACJ,MAAA,EAAQ,aAAA;AAAA,MACR,OAAO,MAAA,IAAU;AAAA,KAClB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAA,GAAyD;AAC7D,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,gBAAA,GAA+D;AACnE,IAAA,MAAM,SAA+B,EAAC;AACtC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,IAAI,KAAA,CAAM,MAAA,KAAW,SAAA,IAAa,KAAA,CAAM,WAAW,SAAA,EAAW;AAC5D,QAAA;AAAA,MACF;AACA,MAAA,MAAA,CAAO,IAAA,CAAK,KAAK,KAAA,CAAM,KAAK,EAAE,IAAA,CAAK,MAAM,MAAS,CAAC,CAAA;AAAA,IACrD;AACA,IAAA,MAAM,OAAA,CAAQ,IAAI,MAAM,CAAA;AACxB,IAAA,OAAO,KAAK,QAAA,EAAS;AAAA,EACvB;AAAA;AAAA,EAGA,QAAA,GAA8C;AAC5C,IAAA,MAAM,MAAgC,EAAC;AACvC,IAAA,KAAA,MAAW,KAAA,IAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,EAAO,EAAG;AACzC,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,OAAO,KAAA,CAAM,KAAA;AAAA,QACb,QAAQ,KAAA,CAAM,MAAA;AAAA,QACd,gBAAgB,KAAA,CAAM,cAAA;AAAA,QACtB,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,qBAAqB,KAAA,CAAM;AAAA,OAC5B,CAAA;AAAA,IACH;AACA,IAAA,OAAO,GAAA;AAAA,EACT;AAAA,EAEQ,cAAc,KAAA,EAA2C;AAC/D,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,QAAQ,IAAA,CAAK,GAAA,EAAI,GAAI,KAAA,CAAM,iBAAiB,IAAA,CAAK,KAAA;AACvD,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,IAAI,KAAA,CAAM,WAAW,SAAA,EAAW;AAC9B,QAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,EAAE,EAAA,EAAI,MAAM,CAAA;AAAA,MACrC;AAYA,MAAA,IAAI,KAAA,CAAM,MAAA,KAAW,SAAA,IAAa,KAAA,CAAM,WAAW,SAAA,EAAW;AAC5D,QAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,IAAA,CAAK,0BAAA,CAA2B,KAAK,CAAC,CAAA;AAAA,MAC/D;AAAA,IACF;AACA,IAAA,OAAO,IAAA,CAAK,MAAM,KAAK,CAAA;AAAA,EACzB;AAAA,EAEQ,2BAA2B,KAAA,EAAkC;AACnE,IAAA,MAAM,MAAA,GAAS,MAAM,UAAA,IAAc,gBAAA;AACnC,IAAA,IAAI,KAAA,CAAM,WAAW,SAAA,EAAW;AAC9B,MAAA,OAAO,EAAE,IAAI,KAAA,EAAO,MAAA,EAAQ,WAAW,MAAA,EAAQ,CAAA,EAAG,MAAM,MAAA,EAAO;AAAA,IACjE;AAEA,IAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,SAAA,EAAW,MAAM,MAAA,EAAO;AAAA,EACtD;AAAA,EAEQ,MAAM,KAAA,EAA2C;AACvD,IAAA,IAAI,MAAM,QAAA,EAAU;AAClB,MAAA,OAAO,KAAA,CAAM,QAAA;AAAA,IACf;AACA,IAAA,MAAM,WAAW,YAAyC;AACxD,MAAA,IAAI;AACF,QAAA,OAAO,MAAM,MAAM,QAAA,EAAS;AAAA,MAC9B,SAAS,KAAA,EAAO;AACd,QAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,QAAA,OAAO,EAAE,EAAA,EAAI,KAAA,EAAO,MAAA,EAAQ,aAAA,EAAe,OAAO,OAAA,EAAQ;AAAA,MAC5D;AAAA,IACF,CAAA,GAAG,CAAE,IAAA,CAAK,CAAC,YAAA,KAAiB;AAC1B,MAAA,IAAA,CAAK,iBAAA,CAAkB,OAAO,YAAY,CAAA;AAC1C,MAAA,KAAA,CAAM,QAAA,GAAW,IAAA;AACjB,MAAA,OAAO,YAAA;AAAA,IACT,CAAC,CAAA;AACD,IAAA,KAAA,CAAM,QAAA,GAAW,OAAA;AACjB,IAAA,OAAO,OAAA;AAAA,EACT;AAAA,EAEQ,iBAAA,CAAkB,OAAc,YAAA,EAAwC;AAC9E,IAAA,KAAA,CAAM,cAAA,GAAiB,KAAK,GAAA,EAAI;AAChC,IAAA,IAAI,aAAa,EAAA,EAAI;AACnB,MAAA,KAAA,CAAM,MAAA,GAAS,SAAA;AACf,MAAA,KAAA,CAAM,UAAA,GAAa,MAAA;AACnB,MAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAC5B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,UAAA,GAAa,aAAa,YAAY,CAAA;AAC5C,IAAA,IAAI,YAAA,CAAa,WAAW,aAAA,EAAe;AACzC,MAAA,KAAA,CAAM,MAAA,GAAS,aAAA;AACf,MAAA,KAAA,CAAM,mBAAA,IAAuB,CAAA;AAC7B,MAAA;AAAA,IACF;AACA,IAAA,KAAA,CAAM,SAAS,YAAA,CAAa,MAAA;AAC5B,IAAA,KAAA,CAAM,mBAAA,GAAsB,CAAA;AAAA,EAC9B;AACF;;;AC7RA,IAAM,QAAA,GAAW,CAAC,IAAA,KAAuB;AAAC,CAAA;AAE1C,SAAS,eAAe,MAAA,EAAkC;AACxD,EAAA,OAAO,MAAA,KAAW,aAAa,MAAA,KAAW,SAAA;AAC5C;AAEA,SAAS,kBAAA,CACP,MACA,IAAA,EACoB;AACpB,EAAA,MAAM,UAAA,GAAa,eAAe,IAAI,CAAA;AACtC,EAAA,MAAM,UAAA,GAAa,cAAA,CAAe,IAAA,CAAK,MAAM,CAAA;AAC7C,EAAA,IAAI,eAAe,UAAA,EAAY;AAC7B,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,UAAA,IAAc,CAAC,UAAA,EAAY;AAC7B,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,UAAA,IAAc,IAAA,CAAK,MAAA;AACvC,IAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,IAAA,CAAK,KAAK,CAAA,mBAAA,EAAsB,IAAA,CAAK,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA,qCAAA,CAAA;AAAA,EACxG;AACA,EAAA,OAAO,CAAA,eAAA,EAAkB,IAAA,CAAK,QAAQ,CAAA,OAAA,EAAU,KAAK,KAAK,CAAA,WAAA,CAAA;AAC5D;AAgBO,SAAS,iBAAiB,OAAA,EAAmD;AAClF,EAAA,MAAM,UAAA,GAAa,QAAQ,UAAA,IAAc,yBAAA;AACzC,EAAA,MAAM,GAAA,GAAM,QAAQ,GAAA,IAAO,QAAA;AAE3B,EAAA,IAAI,gBAAA,uBAAuB,GAAA,EAA6B;AACxD,EAAA,KAAA,MAAW,KAAA,IAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS,EAAG;AAC9C,IAAA,gBAAA,CAAiB,GAAA,CAAI,GAAG,KAAA,CAAM,QAAQ,KAAK,KAAA,CAAM,KAAK,CAAA,CAAA,EAAI,KAAA,CAAM,MAAM,CAAA;AAAA,EACxE;AAEA,EAAA,IAAI,OAAA,GAAU,KAAA;AAEd,EAAA,MAAM,OAAO,YAA2B;AACtC,IAAA,IAAI,OAAA,EAAS;AACX,MAAA;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,OAAA,CAAQ,QAAA,EAAS;AACxC,IAAA,MAAM,YAAA,GAAe,OAAO,IAAA,CAAK,CAAC,UAAU,CAAC,cAAA,CAAe,KAAA,CAAM,MAAM,CAAC,CAAA;AACzE,IAAA,IAAI,CAAC,YAAA,EAAc;AAIjB,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,QAAA;AACJ,IAAA,IAAI;AACF,MAAA,QAAA,GAAW,MAAM,OAAA,CAAQ,OAAA,CAAQ,gBAAA,EAAiB;AAAA,IACpD,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,KAAA,CAAM,OAAA,GAAU,OAAO,KAAK,CAAA;AACrE,MAAA,GAAA,CAAI,CAAA,6BAAA,EAAgC,OAAO,CAAA,CAAE,CAAA;AAC7C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,uBAAW,GAAA,EAA6B;AAC9C,IAAA,KAAA,MAAW,SAAS,QAAA,EAAU;AAC5B,MAAA,MAAM,MAAM,CAAA,EAAG,KAAA,CAAM,QAAQ,CAAA,EAAA,EAAK,MAAM,KAAK,CAAA,CAAA;AAC7C,MAAA,MAAM,IAAA,GAAO,gBAAA,CAAiB,GAAA,CAAI,GAAG,CAAA,IAAK,SAAA;AAC1C,MAAA,MAAM,iBAAA,GAAoB,kBAAA,CAAmB,IAAA,EAAM,KAAK,CAAA;AACxD,MAAA,IAAI,iBAAA,EAAmB;AACrB,QAAA,GAAA,CAAI,iBAAiB,CAAA;AAAA,MACvB;AACA,MAAA,IAAA,CAAK,GAAA,CAAI,GAAA,EAAK,KAAA,CAAM,MAAM,CAAA;AAAA,IAC5B;AACA,IAAA,gBAAA,GAAmB,IAAA;AAAA,EACrB,CAAA;AAEA,EAAA,MAAM,MAAA,GAAyB,YAAY,MAAM;AAC/C,IAAA,KAAK,IAAA,EAAK;AAAA,EACZ,GAAG,UAAU,CAAA;AAEb,EAAA,OAAO;AAAA,IACL,MAAM,MAAM;AACV,MAAA,IAAI,OAAA,EAAS;AACX,QAAA;AAAA,MACF;AACA,MAAA,OAAA,GAAU,IAAA;AACV,MAAA,aAAA,CAAc,MAAM,CAAA;AAAA,IACtB;AAAA,GACF;AACF;AASO,IAAM,iBAAA,GAAoB;;;AChG1B,SAAS,2BACd,OAAA,EACsB;AACtB,EAAA,MAAM,EAAE,QAAA,EAAU,YAAA,EAAc,OAAA,EAAQ,GAAI,OAAA;AAC5C,EAAA,IAAI,YAAY,CAAA,EAAG;AACjB,IAAA,MAAM,IAAI,WAAW,sBAAsB,CAAA;AAAA,EAC7C;AACA,EAAA,IAAI,gBAAgB,CAAA,EAAG;AACrB,IAAA,MAAM,IAAI,WAAW,0BAA0B,CAAA;AAAA,EACjD;AACA,EAAA,IAAI,WAAW,CAAA,EAAG;AAChB,IAAA,MAAM,IAAI,WAAW,qBAAqB,CAAA;AAAA,EAC5C;AAIA,EAAA,MAAM,OAAA,uBAAc,GAAA,EAAmB;AAEvC,EAAA,SAAS,aAAA,GAAsB;AAC7B,IAAA,OAAO,OAAA,CAAQ,OAAO,OAAA,EAAS;AAC7B,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,IAAA,EAAK,CAAE,MAAK,CAAE,KAAA;AACxC,MAAA,IAAI,cAAc,MAAA,EAAW;AAC3B,QAAA;AAAA,MACF;AACA,MAAA,OAAA,CAAQ,OAAO,SAAS,CAAA;AAAA,IAC1B;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,IAAA,CAAK,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC7C,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,GAAA,CAAI,GAAG,CAAA;AAC7B,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,OAAO,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,GAAA,GAAM,QAAA,EAAU,OAAO,CAAA,EAAE;AAAA,MAC5D;AACA,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,MAAM,MAAA,GAAS,YAAA;AAAA,QACxB,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,EAAK,GAAA,GAAM,IAAA,CAAK,KAAI,EAAsB;AAC9C,MAAA,MAAM,KAAA,GAAQ,QAAQ,GAAA,CAAI,GAAG,KAAK,EAAE,IAAA,EAAM,EAAC,EAAE;AAC7C,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AAEjE,MAAA,IAAI,KAAA,CAAM,UAAU,YAAA,EAAc;AAGhC,QAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,QAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,QAAA,OAAO;AAAA,UACL,OAAA,EAAS,KAAA;AAAA,UACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,UAC7B,OAAO,KAAA,CAAM;AAAA,SACf;AAAA,MACF;AACA,MAAA,KAAA,CAAM,KAAK,GAAG,CAAA;AACd,MAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAClB,MAAA,OAAA,CAAQ,GAAA,CAAI,GAAA,EAAK,EAAE,IAAA,EAAM,OAAO,CAAA;AAChC,MAAA,aAAA,EAAc;AACd,MAAA,OAAO;AAAA,QACL,OAAA,EAAS,IAAA;AAAA,QACT,OAAA,EAAA,CAAU,KAAA,CAAM,CAAC,CAAA,IAAK,GAAA,IAAO,QAAA;AAAA,QAC7B,OAAO,KAAA,CAAM;AAAA,OACf;AAAA,IACF,CAAA;AAAA,IACA,KAAA,CAAM,GAAA,GAAM,IAAA,CAAK,GAAA,EAAI,EAAS;AAC5B,MAAA,MAAM,SAAS,GAAA,GAAM,QAAA;AACrB,MAAA,KAAA,MAAW,CAAC,GAAA,EAAK,KAAK,CAAA,IAAK,OAAA,EAAS;AAClC,QAAA,MAAM,QAAQ,KAAA,CAAM,IAAA,CAAK,OAAO,CAAC,SAAA,KAAc,YAAY,MAAM,CAAA;AACjE,QAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,UAAA,OAAA,CAAQ,OAAO,GAAG,CAAA;AAAA,QACpB,CAAA,MAAA,IAAW,KAAA,CAAM,MAAA,KAAW,KAAA,CAAM,KAAK,MAAA,EAAQ;AAC7C,UAAA,KAAA,CAAM,IAAA,GAAO,KAAA;AAAA,QACf;AAAA,MACF;AAAA,IACF,CAAA;AAAA,IACA,IAAA,GAAe;AACb,MAAA,OAAO,OAAA,CAAQ,IAAA;AAAA,IACjB,CAAA;AAAA,IACA,KAAA,GAAc;AACZ,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,GACF;AACF;;;ACpHO,IAAM,mBAAA,GAAsB;AAqC5B,SAAS,yBACd,cAAA,EAC4B;AAC5B,EAAA,IAAI,CAAC,cAAA,EAAgB;AACnB,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,IAAI,cAAA,CAAe,YAAA,IAAgB,CAAA,IAAK,cAAA,CAAe,eAAe,CAAA,EAAG;AACvE,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,OAAO,cAAA;AACT;AAEO,SAAS,uBAAA,CAAwB,OAAA,GAAiC,EAAC,EAAsB;AAC9F,EAAA,MAAM,kBAAA,GAAqC,QAAQ,kBAAA,IAAsB;AAAA,IACvE,WAAA,EAAa,uCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,MAAA,GAAyB,QAAQ,MAAA,IAAU;AAAA,IAC/C,WAAA,EAAa,iCAAA;AAAA,IACb,YAAA,EAAc;AAAA,GAChB;AACA,EAAA,MAAM,OAAA,GAAU,QAAQ,OAAA,IAAW,iCAAA;AAEnC,EAAA,MAAM,gBAAgB,0BAAA,CAA2B;AAAA,IAC/C,UAAU,MAAA,CAAO,WAAA;AAAA,IACjB,cAAc,MAAA,CAAO,YAAA;AAAA,IACrB,OAAA,EAAS;AAAA,GACV,CAAA;AAED,EAAA,MAAM,iBAAiB,0BAAA,CAA2B;AAAA,IAChD,UAAU,kBAAA,CAAmB,WAAA;AAAA,IAC7B,cAAc,kBAAA,CAAmB,YAAA;AAAA,IACjC;AAAA,GACD,CAAA;AAED,EAAA,MAAM,gBAAA,uBAAuB,GAAA,EAAkC;AAE/D,EAAA,SAAS,qBAAA,CACP,WACA,QAAA,EACsB;AACtB,IAAA,MAAM,SAAA,GAAY,yBAAyB,QAAQ,CAAA;AACnD,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,OAAO,cAAA;AAAA,IACT;AACA,IAAA,MAAM,MAAA,GAAS,gBAAA,CAAiB,GAAA,CAAI,SAAS,CAAA;AAC7C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,OAAO,MAAA;AAAA,IACT;AACA,IAAA,MAAM,UAAU,0BAAA,CAA2B;AAAA,MACzC,UAAU,SAAA,CAAU,WAAA;AAAA,MACpB,cAAc,SAAA,CAAU,YAAA;AAAA,MACxB;AAAA,KACD,CAAA;AACD,IAAA,gBAAA,CAAiB,GAAA,CAAI,WAAW,OAAO,CAAA;AACvC,IAAA,OAAO,OAAA;AAAA,EACT;AAEA,EAAA,SAAS,gBAAA,GAAyB;AAChC,IAAA,cAAA,CAAe,KAAA,EAAM;AACrB,IAAA,KAAA,MAAW,OAAA,IAAW,gBAAA,CAAiB,MAAA,EAAO,EAAG;AAC/C,MAAA,OAAA,CAAQ,KAAA,EAAM;AAAA,IAChB;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,aAAA;AAAA,IACA,qBAAA;AAAA,IACA,gBAAA;AAAA,IACA,kBAAA;AAAA,IACA;AAAA,GACF;AACF;AAUO,SAAS,kBAAA,CAAmB,YAAoB,SAAA,EAA2B;AAChF,EAAA,OAAO,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,SAAS,CAAA,CAAA;AACnC","file":"llm-health.cjs","sourcesContent":["/**\n * LLM health monitor and heartbeat tunable defaults. CLI and plugin consumers\n * can override via `process.env.ELISYM_LLM_HEALTH_TTL_MS` and\n * `ELISYM_LLM_HEARTBEAT_INTERVAL_MS` and pass the resolved values into\n * the monitor/heartbeat options.\n */\n\nexport const DEFAULT_HEALTH_TTL_MS = 10 * 60 * 1000;\nexport const DEFAULT_HEARTBEAT_INTERVAL_MS = 10 * 60 * 1000;\n\n/**\n * Interval between recovery probes after the LLM health monitor enters an\n * unhealthy state. The recovery loop is paused while the pair is healthy\n * and only kicks in reactively (after `markUnhealthyFromJob` or a failed\n * job). On the first successful probe the monitor returns to healthy and\n * the loop stops on its own.\n */\nexport const LAZY_RECOVERY_INTERVAL_MS = 5 * 60 * 1000;\n\n/**\n * Exit code contract: a `dynamic-script` / `static-script` skill returns\n * this code from the script process to signal that the upstream LLM\n * provider rejected the request because credits / billing are exhausted.\n * The agent runtime treats this as the script's equivalent of the\n * `mode: 'llm'` 402 path: it calls `markUnhealthyFromJob(provider, model)`\n * on the health monitor (which starts the lazy recovery loop) and rejects\n * subsequent jobs against the same pair until a recovery probe succeeds.\n *\n * Any other non-zero exit is a generic failure and does NOT touch health\n * state - operators should reserve this code for billing-exhausted only.\n *\n * 42 was chosen because it sits outside POSIX shell conventions (1-2\n * generic, 126-128 shell-internal, 130+ signals) and `sysexits.h`\n * (64-78 - usage / data / host / config errors), so it doesn't collide\n * with other meaningful exit codes a script might naturally produce.\n */\nexport const SCRIPT_EXIT_BILLING_EXHAUSTED = 42;\n\n/**\n * Number of consecutive `unavailable` results tolerated before\n * `assertReady` starts throwing. The first `unavailable - 1` are treated\n * as transient blips so a brief network hiccup does not block jobs.\n */\nexport const UNAVAILABLE_TOLERANCE = 3;\n\n/**\n * Free-LLM rate-limit defaults. Applied when a SKILL.md with\n * `mode: 'llm'` and `price: 0` does not declare its own `rate_limit`.\n */\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS = 60 * 60 * 1000;\nexport const DEFAULT_FREE_LLM_PER_CUSTOMER_MAX = 3;\n\nexport const DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS = 60 * 1000;\nexport const DEFAULT_FREE_LLM_GLOBAL_MAX = 30;\n\nexport const DEFAULT_FREE_LLM_MAX_TRACKED_KEYS = 1000;\n","/**\n * Shared types for the LLM health monitor. Provider-agnostic: the\n * actual HTTP probe lives in CLI/plugin and is supplied via dependency\n * injection (`verifyFn`).\n */\n\nexport type LlmHealthStatus = 'unknown' | 'healthy' | 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Result of probing an API key with a specific model. Discriminated on\n * `ok`, then on `reason` for failures. Adding a new failure reason is a\n * breaking change for exhaustive switches; update consumers in lockstep.\n *\n * - `invalid`: HTTP 401/403 - key rejected outright.\n * - `billing`: HTTP 402, or 400 with credit/billing/insufficient marker,\n * or OpenAI's 429 with `insufficient_quota`. Operator out of credits.\n * - `unavailable`: transient (HTTP 429 without quota marker, 5xx, network\n * error). May resolve on retry.\n */\nexport type LlmKeyVerification =\n | { ok: true }\n | { ok: false; reason: 'invalid'; status: number; body: string }\n | { ok: false; reason: 'billing'; status?: number; body?: string }\n | { ok: false; reason: 'unavailable'; error: string };\n\nexport type LlmHealthErrorReason = 'invalid' | 'billing' | 'unavailable';\n\n/**\n * Thrown by `LlmHealthMonitor.assertReady` when the gate refuses a job.\n * Carries the operator-facing reason so callers can log it; customer-facing\n * messages should be sanitized at the call site (see `runtime.ts` preflight).\n */\nexport class LlmHealthError extends Error {\n readonly reason: LlmHealthErrorReason;\n readonly provider: string;\n readonly model: string;\n\n constructor(reason: LlmHealthErrorReason, provider: string, model: string, detail: string) {\n super(`LLM ${provider}/${model} ${reason}: ${detail}`);\n this.name = 'LlmHealthError';\n this.reason = reason;\n this.provider = provider;\n this.model = model;\n }\n}\n\n/**\n * Thrown by SDK script skills (`static-script`, `dynamic-script`) when the\n * spawned process exits with `SCRIPT_EXIT_BILLING_EXHAUSTED` (= 42). The\n * runtime catches this and calls `markUnhealthyFromJob` on the matching\n * (provider, model) pair declared in SKILL.md, so subsequent jobs are\n * gated until the lazy recovery loop re-probes the key. The exit code is\n * the script-side equivalent of the LLM-mode 402 path.\n *\n * The error does NOT carry provider/model itself - the script doesn't\n * know which pair the agent registered with the monitor; the runtime\n * reads them from the matched skill's `llmOverride` declaration.\n */\nexport class ScriptBillingExhaustedError extends Error {\n readonly exitCode: number;\n readonly stderr: string;\n readonly stdout: string;\n\n constructor(exitCode: number, stdout: string, stderr: string) {\n const detail = stderr.trim() || stdout.trim() || '(no output)';\n super(`script exited with billing-exhausted code ${exitCode}: ${detail}`);\n this.name = 'ScriptBillingExhaustedError';\n this.exitCode = exitCode;\n this.stdout = stdout;\n this.stderr = stderr;\n }\n}\n\n/**\n * Per-skill rate-limit declaration. Snake-case in SKILL.md frontmatter,\n * camelCase here. Applies to any skill mode but the framework adds a\n * default cap only for free LLM skills.\n */\nexport interface SkillRateLimit {\n perWindowMs: number;\n maxPerWindow: number;\n}\n\n/** Read-only health snapshot for logs/tests. */\nexport interface LlmHealthSnapshotEntry {\n provider: string;\n model: string;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n consecutiveFailures: number;\n}\n","/**\n * Provider-agnostic health monitor for LLM API keys. State machine per\n * (provider, model) pair: caches the last verification result for\n * `ttlMs`, deduplicates concurrent probes via an in-flight promise, and\n * tolerates a bounded number of consecutive `unavailable` results\n * before `assertReady` starts throwing.\n *\n * The monitor itself never touches the network. Each pair must be\n * registered with a `verifyFn` lambda that performs the actual probe;\n * callers (CLI/plugin) supply provider-specific HTTP from their layer.\n */\n\nimport {\n DEFAULT_HEALTH_TTL_MS,\n UNAVAILABLE_TOLERANCE as DEFAULT_UNAVAILABLE_TOLERANCE,\n} from './constants';\nimport {\n LlmHealthError,\n type LlmHealthSnapshotEntry,\n type LlmHealthStatus,\n type LlmKeyVerification,\n} from './types';\n\nexport type LlmKeyVerifyFn = (signal?: AbortSignal) => Promise<LlmKeyVerification>;\n\nexport interface LlmHealthMonitorOptions {\n /** Time after which a cached `healthy` result is re-probed. Default 10 min. */\n ttlMs?: number;\n /** Number of consecutive unavailable results tolerated. Default 3. */\n unavailableTolerance?: number;\n /** Optional clock injection for tests. */\n now?: () => number;\n}\n\nexport interface RegisterArgs {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n}\n\ninterface Entry {\n provider: string;\n model: string;\n verifyFn: LlmKeyVerifyFn;\n status: LlmHealthStatus;\n lastVerifiedAt: number;\n lastReason: string | undefined;\n inFlight: Promise<LlmKeyVerification> | null;\n consecutiveFailures: number;\n}\n\nfunction keyOf(provider: string, model: string): string {\n return `${provider}::${model}`;\n}\n\nfunction reasonDetail(verification: LlmKeyVerification): string {\n if (verification.ok) {\n return 'ok';\n }\n if (verification.reason === 'invalid') {\n return `HTTP ${verification.status}: ${verification.body.slice(0, 200)}`;\n }\n if (verification.reason === 'billing') {\n const status = verification.status ?? 0;\n const body = (verification.body ?? '').slice(0, 200);\n return `HTTP ${status}: ${body}`;\n }\n return verification.error;\n}\n\nexport class LlmHealthMonitor {\n private readonly entries = new Map<string, Entry>();\n private readonly ttlMs: number;\n private readonly unavailableTolerance: number;\n private readonly now: () => number;\n\n constructor(options: LlmHealthMonitorOptions = {}) {\n this.ttlMs = options.ttlMs ?? DEFAULT_HEALTH_TTL_MS;\n this.unavailableTolerance = options.unavailableTolerance ?? DEFAULT_UNAVAILABLE_TOLERANCE;\n this.now = options.now ?? Date.now;\n }\n\n /**\n * Register a (provider, model) pair with its probe function. Idempotent\n * on the (provider, model) key: re-registering replaces the verifyFn\n * and resets state to `unknown`. Callers typically re-register only\n * when the operator rotates the API key.\n */\n register(args: RegisterArgs): void {\n const key = keyOf(args.provider, args.model);\n this.entries.set(key, {\n provider: args.provider,\n model: args.model,\n verifyFn: args.verifyFn,\n status: 'unknown',\n lastVerifiedAt: 0,\n lastReason: undefined,\n inFlight: null,\n consecutiveFailures: 0,\n });\n }\n\n /**\n * Seed the monitor with an already-known verification result, e.g.\n * the one captured at startup. Skips an extra probe on the first\n * `assertReady` call within the TTL window. No-op if the pair is not\n * registered.\n */\n seed(provider: string, model: string, verification: LlmKeyVerification): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n this.applyVerification(entry, verification);\n }\n\n /**\n * Main gate before doing LLM work. Throws `LlmHealthError` on terminal\n * states (`invalid`, `billing`, or `unavailable` past tolerance);\n * resolves silently on `healthy` (cache hit or fresh probe) or\n * tolerated `unavailable`.\n *\n * Concurrent `assertReady` calls for the same pair are deduplicated:\n * the second caller awaits the first probe rather than launching a\n * parallel one.\n */\n async assertReady(provider: string, model: string): Promise<void> {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n throw new LlmHealthError('invalid', provider, model, 'pair not registered');\n }\n\n const verification = await this.probeIfNeeded(entry);\n\n if (verification.ok) {\n return;\n }\n if (verification.reason === 'invalid' || verification.reason === 'billing') {\n throw new LlmHealthError(verification.reason, provider, model, reasonDetail(verification));\n }\n if (entry.consecutiveFailures >= this.unavailableTolerance) {\n throw new LlmHealthError('unavailable', provider, model, reasonDetail(verification));\n }\n // Tolerated unavailable: caller proceeds, real LLM call will surface\n // any transient issue with its own error path.\n }\n\n /**\n * Force the next `assertReady` for this pair to re-probe regardless of\n * TTL. Used when a real LLM call surfaces 401/402 mid-job - the cached\n * `healthy` is stale and we want to catch the next request.\n */\n markFailureFromJob(provider: string, model: string): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n entry.lastVerifiedAt = 0;\n }\n\n /**\n * Reactively flip a (provider, model) pair to unhealthy without doing a\n * fresh probe. Called from the runtime when a job's actual LLM call (or\n * a script's `SCRIPT_EXIT_BILLING_EXHAUSTED` exit) surfaces a billing /\n * invalid signal: the cached `healthy` snapshot is wrong and we want\n * subsequent `assertReady` calls to refuse jobs immediately, before the\n * lazy recovery loop notices on its own. No-op if the pair is not\n * registered.\n *\n * Recovery from this state happens through a successful probe (typically\n * fired by `startLlmRecovery`) which flips the pair back to healthy via\n * `applyVerification`.\n */\n markUnhealthyFromJob(\n provider: string,\n model: string,\n reason: 'billing' | 'invalid' | 'unavailable',\n detail?: string,\n ): void {\n const entry = this.entries.get(keyOf(provider, model));\n if (!entry) {\n return;\n }\n if (reason === 'invalid') {\n this.applyVerification(entry, {\n ok: false,\n reason: 'invalid',\n status: 0,\n body: detail ?? 'reactive markUnhealthyFromJob',\n });\n return;\n }\n if (reason === 'billing') {\n this.applyVerification(entry, {\n ok: false,\n reason: 'billing',\n body: detail ?? 'reactive markUnhealthyFromJob',\n });\n return;\n }\n this.applyVerification(entry, {\n ok: false,\n reason: 'unavailable',\n error: detail ?? 'reactive markUnhealthyFromJob',\n });\n }\n\n /**\n * Refresh every registered pair concurrently. Errors thrown by\n * `verifyFn` are caught and recorded as `unavailable`.\n */\n async refreshAll(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n for (const entry of this.entries.values()) {\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /**\n * Refresh only the pairs whose current status is non-healthy\n * (`invalid`, `billing`, or `unavailable`). Used by the lazy recovery\n * loop so a billing outage on one provider does not trigger throwaway\n * probes on every other healthy pair the agent has registered.\n * Errors thrown by `verifyFn` are caught and recorded as `unavailable`.\n */\n async refreshUnhealthy(): Promise<readonly LlmHealthSnapshotEntry[]> {\n const probes: Array<Promise<void>> = [];\n for (const entry of this.entries.values()) {\n if (entry.status === 'healthy' || entry.status === 'unknown') {\n continue;\n }\n probes.push(this.probe(entry).then(() => undefined));\n }\n await Promise.all(probes);\n return this.snapshot();\n }\n\n /** Read-only view, primarily for logs and tests. */\n snapshot(): readonly LlmHealthSnapshotEntry[] {\n const out: LlmHealthSnapshotEntry[] = [];\n for (const entry of this.entries.values()) {\n out.push({\n provider: entry.provider,\n model: entry.model,\n status: entry.status,\n lastVerifiedAt: entry.lastVerifiedAt,\n lastReason: entry.lastReason,\n consecutiveFailures: entry.consecutiveFailures,\n });\n }\n return out;\n }\n\n private probeIfNeeded(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const fresh = this.now() - entry.lastVerifiedAt < this.ttlMs;\n if (fresh) {\n if (entry.status === 'healthy') {\n return Promise.resolve({ ok: true });\n }\n // Terminal failure states (`invalid`, `billing`) are cached\n // aggressively: once the operator's key is rejected or out of\n // credits, the lazy recovery loop is the only path back to healthy.\n // Re-probing on every `assertReady` would burn one billing token\n // per job rejected during a billing outage, which defeats the\n // gate's whole purpose. `unavailable` is treated as transient and\n // falls through so the existing tolerance counter still works\n // (each call re-probes, increments `consecutiveFailures`, and\n // crosses the threshold on the configured number of attempts).\n // `unknown` also falls through - a freshly registered pair that\n // was never seeded should probe on first ask.\n if (entry.status === 'invalid' || entry.status === 'billing') {\n return Promise.resolve(this.synthesizeFailureFromCache(entry));\n }\n }\n return this.probe(entry);\n }\n\n private synthesizeFailureFromCache(entry: Entry): LlmKeyVerification {\n const detail = entry.lastReason ?? 'cached failure';\n if (entry.status === 'invalid') {\n return { ok: false, reason: 'invalid', status: 0, body: detail };\n }\n // 'billing' (only other status reachable here)\n return { ok: false, reason: 'billing', body: detail };\n }\n\n private probe(entry: Entry): Promise<LlmKeyVerification> {\n if (entry.inFlight) {\n return entry.inFlight;\n }\n const promise = (async (): Promise<LlmKeyVerification> => {\n try {\n return await entry.verifyFn();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n return { ok: false, reason: 'unavailable', error: message };\n }\n })().then((verification) => {\n this.applyVerification(entry, verification);\n entry.inFlight = null;\n return verification;\n });\n entry.inFlight = promise;\n return promise;\n }\n\n private applyVerification(entry: Entry, verification: LlmKeyVerification): void {\n entry.lastVerifiedAt = this.now();\n if (verification.ok) {\n entry.status = 'healthy';\n entry.lastReason = undefined;\n entry.consecutiveFailures = 0;\n return;\n }\n entry.lastReason = reasonDetail(verification);\n if (verification.reason === 'unavailable') {\n entry.status = 'unavailable';\n entry.consecutiveFailures += 1;\n return;\n }\n entry.status = verification.reason;\n entry.consecutiveFailures = 0;\n }\n}\n","/**\n * Lazy LLM recovery probe. While every registered (provider, model) pair\n * is healthy, ticks are no-ops: zero API traffic, zero billing tokens\n * burned. The loop only does real work after a pair has flipped to\n * unhealthy (via `markUnhealthyFromJob` from the runtime, or from a\n * preflight probe that returned a non-ok verification): each tick\n * re-probes only the unhealthy pairs and, on success, flips them back to\n * healthy via the monitor's normal `applyVerification` path.\n *\n * The recovery loop is the only path back to `healthy` after a reactive\n * markUnhealthy. Without it, the agent would stay locked out until\n * restart even after the operator pops their billing back up.\n *\n * Logging policy lives here so the monitor stays a pure state-machine.\n * Status transitions (healthy <-> unhealthy) are logged once per change;\n * routine successful re-probes are not logged.\n */\n\nimport { LAZY_RECOVERY_INTERVAL_MS } from './constants';\nimport type { LlmHealthMonitor } from './monitor';\nimport type { LlmHealthSnapshotEntry, LlmHealthStatus } from './types';\n\nexport interface HeartbeatHandle {\n stop(): void;\n}\n\nexport interface StartLlmRecoveryOptions {\n monitor: LlmHealthMonitor;\n /** Defaults to {@link LAZY_RECOVERY_INTERVAL_MS} (5 minutes). */\n intervalMs?: number;\n /** Operator log sink. Defaults to no-op (silent). */\n log?: (msg: string) => void;\n}\n\n/**\n * @deprecated Renamed to {@link StartLlmRecoveryOptions}. Kept as an alias\n * so external consumers keep building during the rename. The semantics\n * have changed (lazy, recovery-only) but the option surface is identical.\n */\nexport type StartLlmHeartbeatOptions = StartLlmRecoveryOptions;\n\ntype IntervalHandle = ReturnType<typeof setInterval>;\n\nconst NOOP_LOG = (_msg: string): void => {};\n\nfunction isHealthyState(status: LlmHealthStatus): boolean {\n return status === 'healthy' || status === 'unknown';\n}\n\nfunction describeTransition(\n prev: LlmHealthStatus,\n next: LlmHealthSnapshotEntry,\n): string | undefined {\n const wasHealthy = isHealthyState(prev);\n const nowHealthy = isHealthyState(next.status);\n if (wasHealthy === nowHealthy) {\n return undefined;\n }\n if (wasHealthy && !nowHealthy) {\n const reason = next.lastReason ?? next.status;\n return `! LLM provider ${next.provider} model ${next.model} became unhealthy: ${next.status} (${reason}). Refusing LLM jobs until recovered.`;\n }\n return `* LLM provider ${next.provider} model ${next.model} recovered.`;\n}\n\n/**\n * Start the lazy recovery loop. Returns a handle whose `stop()` cancels\n * the timer (idempotent). The loop ticks every `intervalMs` ms; each\n * tick scans `monitor.snapshot()` and, if any pair is non-healthy, asks\n * the monitor to re-probe just those non-healthy pairs via\n * `refreshUnhealthy()`. Healthy pairs are never re-probed by the loop -\n * those keep their cached `healthy` until TTL expiry forces a probe at\n * the next `assertReady` call. When all pairs are healthy the tick is a\n * single Map walk and no API calls are made.\n *\n * The function is named `startLlmRecovery` but the legacy export\n * `startLlmHeartbeat` (below) is preserved as an alias so external\n * code that already imports it keeps working unchanged.\n */\nexport function startLlmRecovery(options: StartLlmRecoveryOptions): HeartbeatHandle {\n const intervalMs = options.intervalMs ?? LAZY_RECOVERY_INTERVAL_MS;\n const log = options.log ?? NOOP_LOG;\n\n let lastStatusByPair = new Map<string, LlmHealthStatus>();\n for (const entry of options.monitor.snapshot()) {\n lastStatusByPair.set(`${entry.provider}::${entry.model}`, entry.status);\n }\n\n let stopped = false;\n\n const tick = async (): Promise<void> => {\n if (stopped) {\n return;\n }\n const before = options.monitor.snapshot();\n const anyUnhealthy = before.some((entry) => !isHealthyState(entry.status));\n if (!anyUnhealthy) {\n // Lazy: nothing to do. Refreshing healthy pairs would burn one\n // billing-token per pair per tick for no benefit (the monitor's\n // own TTL would re-probe at assertReady time anyway).\n return;\n }\n\n let snapshot: readonly LlmHealthSnapshotEntry[];\n try {\n snapshot = await options.monitor.refreshUnhealthy();\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n log(`! LLM recovery probe failed: ${message}`);\n return;\n }\n\n const next = new Map<string, LlmHealthStatus>();\n for (const entry of snapshot) {\n const key = `${entry.provider}::${entry.model}`;\n const prev = lastStatusByPair.get(key) ?? 'unknown';\n const transitionMessage = describeTransition(prev, entry);\n if (transitionMessage) {\n log(transitionMessage);\n }\n next.set(key, entry.status);\n }\n lastStatusByPair = next;\n };\n\n const handle: IntervalHandle = setInterval(() => {\n void tick();\n }, intervalMs);\n\n return {\n stop: () => {\n if (stopped) {\n return;\n }\n stopped = true;\n clearInterval(handle);\n },\n };\n}\n\n/**\n * @deprecated Renamed to {@link startLlmRecovery}. The old name is\n * preserved as an alias so existing imports keep working during the\n * rename. Behavior changed materially: the new loop is lazy\n * (no API calls while all pairs are healthy) and defaults to\n * `LAZY_RECOVERY_INTERVAL_MS` (5 min) instead of 10 min.\n */\nexport const startLlmHeartbeat = startLlmRecovery;\n","/**\n * Sliding-window rate limiter keyed by an arbitrary string (typically a\n * customer pubkey). Each key gets at most `maxPerWindow` requests inside a\n * rolling `windowMs`. Stale timestamps are pruned lazily on every `check`.\n * When the tracked-key set grows past `maxKeys`, the least-recently-used\n * key is evicted so an attacker cannot exhaust memory by cycling keys.\n *\n * Thread-safety: not required. Designed for single-threaded JS consumers\n * (Node/Bun event loops, browser main thread). No timers - pruning happens\n * inside `check` and `prune`.\n */\n\nexport interface SlidingWindowLimiterOptions {\n /** Rolling window width, in ms. */\n windowMs: number;\n /** Max hits allowed per key inside the window. */\n maxPerWindow: number;\n /** Cap on total tracked keys. LRU-evicted past this cap. */\n maxKeys: number;\n}\n\nexport interface RateLimitDecision {\n allowed: boolean;\n /** Wall-clock timestamp (ms) when the limit window will reset for this key. */\n resetAt: number;\n /** Number of hits inside the current window after this call (or the attempted hit if denied). */\n count: number;\n}\n\nexport interface SlidingWindowLimiter {\n /** Record a hit against `key`; return whether it was allowed. */\n check(key: string, now?: number): RateLimitDecision;\n /**\n * Inspect the current state for `key` without recording a hit. Useful\n * when a single request must clear multiple limiters and callers want\n * to avoid double-counting against one limiter after another denies.\n */\n peek(key: string, now?: number): RateLimitDecision;\n /** Drop entries whose windows have fully elapsed. Bounded memory hygiene. */\n prune(now?: number): void;\n /** Current number of tracked keys. */\n size(): number;\n /** Clear all state. */\n reset(): void;\n}\n\ninterface Entry {\n /** Sliding-window timestamps in ms. Sorted ascending. */\n hits: number[];\n}\n\nexport function createSlidingWindowLimiter(\n options: SlidingWindowLimiterOptions,\n): SlidingWindowLimiter {\n const { windowMs, maxPerWindow, maxKeys } = options;\n if (windowMs <= 0) {\n throw new RangeError('windowMs must be > 0');\n }\n if (maxPerWindow <= 0) {\n throw new RangeError('maxPerWindow must be > 0');\n }\n if (maxKeys <= 0) {\n throw new RangeError('maxKeys must be > 0');\n }\n\n // LRU is implemented via Map's insertion-order: every check refreshes\n // the entry by deleting and re-setting it, moving it to the tail.\n const entries = new Map<string, Entry>();\n\n function evictIfNeeded(): void {\n while (entries.size > maxKeys) {\n const oldestKey = entries.keys().next().value as string | undefined;\n if (oldestKey === undefined) {\n return;\n }\n entries.delete(oldestKey);\n }\n }\n\n return {\n peek(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key);\n if (!entry) {\n return { allowed: true, resetAt: now + windowMs, count: 0 };\n }\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n return {\n allowed: fresh.length < maxPerWindow,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n check(key, now = Date.now()): RateLimitDecision {\n const entry = entries.get(key) ?? { hits: [] };\n const cutoff = now - windowMs;\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n\n if (fresh.length >= maxPerWindow) {\n // Refresh LRU order even on denial so an attacker hammering the\n // same key cannot push other tracked keys out via eviction.\n entries.delete(key);\n entries.set(key, { hits: fresh });\n return {\n allowed: false,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n }\n fresh.push(now);\n entries.delete(key);\n entries.set(key, { hits: fresh });\n evictIfNeeded();\n return {\n allowed: true,\n resetAt: (fresh[0] ?? now) + windowMs,\n count: fresh.length,\n };\n },\n prune(now = Date.now()): void {\n const cutoff = now - windowMs;\n for (const [key, entry] of entries) {\n const fresh = entry.hits.filter((timestamp) => timestamp > cutoff);\n if (fresh.length === 0) {\n entries.delete(key);\n } else if (fresh.length !== entry.hits.length) {\n entry.hits = fresh;\n }\n }\n },\n size(): number {\n return entries.size;\n },\n reset(): void {\n entries.clear();\n },\n };\n}\n","/**\n * Two-tier rate limiter for free LLM skills (mode='llm', price=0).\n * Provides Sybil-resistant global cap plus per-customer-per-skill cap\n * that respects an optional skill-level override.\n *\n * Both limiters are independent of the existing per-customer general\n * limiter; callers (CLI runtime, plugin handler) check all of them in\n * sequence and only `check()` after every tier passes `peek()` so a\n * single denial does not consume slots in earlier tiers.\n */\n\nimport { createSlidingWindowLimiter, type SlidingWindowLimiter } from '../primitives/rateLimiter';\nimport {\n DEFAULT_FREE_LLM_GLOBAL_MAX,\n DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n DEFAULT_FREE_LLM_MAX_TRACKED_KEYS,\n DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n} from './constants';\nimport type { SkillRateLimit } from './types';\n\nexport const FREE_LLM_GLOBAL_KEY = '__free_llm_global__';\n\nexport interface FreeLlmLimiterOptions {\n /** Max tracked (customer, skill) keys. Default 1000 (LRU evicted past cap). */\n maxKeys?: number;\n /** Default per-customer cap when a free LLM skill omits `rate_limit`. */\n defaultPerCustomer?: SkillRateLimit;\n /** Global Sybil cap across all free LLM jobs. */\n global?: SkillRateLimit;\n}\n\nexport interface FreeLlmLimiterSet {\n /** Sybil-protection limiter keyed on `FREE_LLM_GLOBAL_KEY`. */\n globalLimiter: SlidingWindowLimiter;\n /**\n * Return the per-customer limiter for a given skill. Skills that\n * declare a `rate_limit` get their own limiter sized to that\n * (window, cap); skills that don't share a default-window limiter.\n * Each call uses `peek/check` keyed on `customerId` alone since each\n * skill has a dedicated limiter store.\n */\n getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter;\n /** Drop expired hits from every per-customer limiter (default + per-skill). */\n prunePerCustomer(): void;\n /** Default cap to apply when a skill omits `rate_limit`. */\n defaultPerCustomer: SkillRateLimit;\n /** Sybil-cap settings (echo of global option for diagnostics). */\n global: SkillRateLimit;\n}\n\n/**\n * Effective per-skill rate limit: the override if it's well-formed,\n * else `undefined` so callers can fall through to the default limiter.\n */\nexport function resolvePerSkillRateLimit(\n skillRateLimit: SkillRateLimit | undefined,\n): SkillRateLimit | undefined {\n if (!skillRateLimit) {\n return undefined;\n }\n if (skillRateLimit.maxPerWindow <= 0 || skillRateLimit.perWindowMs <= 0) {\n return undefined;\n }\n return skillRateLimit;\n}\n\nexport function createFreeLlmLimiterSet(options: FreeLlmLimiterOptions = {}): FreeLlmLimiterSet {\n const defaultPerCustomer: SkillRateLimit = options.defaultPerCustomer ?? {\n perWindowMs: DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_PER_CUSTOMER_MAX,\n };\n const global: SkillRateLimit = options.global ?? {\n perWindowMs: DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS,\n maxPerWindow: DEFAULT_FREE_LLM_GLOBAL_MAX,\n };\n const maxKeys = options.maxKeys ?? DEFAULT_FREE_LLM_MAX_TRACKED_KEYS;\n\n const globalLimiter = createSlidingWindowLimiter({\n windowMs: global.perWindowMs,\n maxPerWindow: global.maxPerWindow,\n maxKeys: 1,\n });\n\n const defaultLimiter = createSlidingWindowLimiter({\n windowMs: defaultPerCustomer.perWindowMs,\n maxPerWindow: defaultPerCustomer.maxPerWindow,\n maxKeys,\n });\n\n const perSkillLimiters = new Map<string, SlidingWindowLimiter>();\n\n function getPerCustomerLimiter(\n skillName: string,\n override: SkillRateLimit | undefined,\n ): SlidingWindowLimiter {\n const effective = resolvePerSkillRateLimit(override);\n if (!effective) {\n return defaultLimiter;\n }\n const cached = perSkillLimiters.get(skillName);\n if (cached) {\n return cached;\n }\n const limiter = createSlidingWindowLimiter({\n windowMs: effective.perWindowMs,\n maxPerWindow: effective.maxPerWindow,\n maxKeys,\n });\n perSkillLimiters.set(skillName, limiter);\n return limiter;\n }\n\n function prunePerCustomer(): void {\n defaultLimiter.prune();\n for (const limiter of perSkillLimiters.values()) {\n limiter.prune();\n }\n }\n\n return {\n globalLimiter,\n getPerCustomerLimiter,\n prunePerCustomer,\n defaultPerCustomer,\n global,\n };\n}\n\n/**\n * Compose a per-skill key for the per-customer limiter. The default\n * limiter is shared across skills without an override, so the skill\n * name is needed to keep each (customer, skill) pair counted\n * independently. Per-skill limiters use the same key for consistency\n * even though the skill component is redundant inside a dedicated\n * limiter store.\n */\nexport function freeLlmCustomerKey(customerId: string, skillName: string): string {\n return `${customerId}|${skillName}`;\n}\n"]}
@@ -1,5 +1,5 @@
1
- import { L as LlmKeyVerification, a as LlmHealthSnapshotEntry, S as SkillRateLimit } from './types-8vJ1I2KQ.cjs';
2
- export { b as LlmHealthError, c as LlmHealthErrorReason, d as LlmHealthStatus } from './types-8vJ1I2KQ.cjs';
1
+ import { L as LlmKeyVerification, a as LlmHealthSnapshotEntry, S as SkillRateLimit } from './types-COvV499T.cjs';
2
+ export { b as LlmHealthError, c as LlmHealthErrorReason, d as LlmHealthStatus, e as ScriptBillingExhaustedError } from './types-COvV499T.cjs';
3
3
  import { S as SlidingWindowLimiter } from './rateLimiter-CoEmZkSX.cjs';
4
4
 
5
5
  /**
@@ -10,6 +10,32 @@ import { S as SlidingWindowLimiter } from './rateLimiter-CoEmZkSX.cjs';
10
10
  */
11
11
  declare const DEFAULT_HEALTH_TTL_MS: number;
12
12
  declare const DEFAULT_HEARTBEAT_INTERVAL_MS: number;
13
+ /**
14
+ * Interval between recovery probes after the LLM health monitor enters an
15
+ * unhealthy state. The recovery loop is paused while the pair is healthy
16
+ * and only kicks in reactively (after `markUnhealthyFromJob` or a failed
17
+ * job). On the first successful probe the monitor returns to healthy and
18
+ * the loop stops on its own.
19
+ */
20
+ declare const LAZY_RECOVERY_INTERVAL_MS: number;
21
+ /**
22
+ * Exit code contract: a `dynamic-script` / `static-script` skill returns
23
+ * this code from the script process to signal that the upstream LLM
24
+ * provider rejected the request because credits / billing are exhausted.
25
+ * The agent runtime treats this as the script's equivalent of the
26
+ * `mode: 'llm'` 402 path: it calls `markUnhealthyFromJob(provider, model)`
27
+ * on the health monitor (which starts the lazy recovery loop) and rejects
28
+ * subsequent jobs against the same pair until a recovery probe succeeds.
29
+ *
30
+ * Any other non-zero exit is a generic failure and does NOT touch health
31
+ * state - operators should reserve this code for billing-exhausted only.
32
+ *
33
+ * 42 was chosen because it sits outside POSIX shell conventions (1-2
34
+ * generic, 126-128 shell-internal, 130+ signals) and `sysexits.h`
35
+ * (64-78 - usage / data / host / config errors), so it doesn't collide
36
+ * with other meaningful exit codes a script might naturally produce.
37
+ */
38
+ declare const SCRIPT_EXIT_BILLING_EXHAUSTED = 42;
13
39
  /**
14
40
  * Number of consecutive `unavailable` results tolerated before
15
41
  * `assertReady` starts throwing. The first `unavailable - 1` are treated
@@ -90,37 +116,97 @@ declare class LlmHealthMonitor {
90
116
  */
91
117
  markFailureFromJob(provider: string, model: string): void;
92
118
  /**
93
- * Refresh every registered pair concurrently. Heartbeat hook. Errors
94
- * thrown by `verifyFn` are caught and recorded as `unavailable`.
119
+ * Reactively flip a (provider, model) pair to unhealthy without doing a
120
+ * fresh probe. Called from the runtime when a job's actual LLM call (or
121
+ * a script's `SCRIPT_EXIT_BILLING_EXHAUSTED` exit) surfaces a billing /
122
+ * invalid signal: the cached `healthy` snapshot is wrong and we want
123
+ * subsequent `assertReady` calls to refuse jobs immediately, before the
124
+ * lazy recovery loop notices on its own. No-op if the pair is not
125
+ * registered.
126
+ *
127
+ * Recovery from this state happens through a successful probe (typically
128
+ * fired by `startLlmRecovery`) which flips the pair back to healthy via
129
+ * `applyVerification`.
130
+ */
131
+ markUnhealthyFromJob(provider: string, model: string, reason: 'billing' | 'invalid' | 'unavailable', detail?: string): void;
132
+ /**
133
+ * Refresh every registered pair concurrently. Errors thrown by
134
+ * `verifyFn` are caught and recorded as `unavailable`.
95
135
  */
96
136
  refreshAll(): Promise<readonly LlmHealthSnapshotEntry[]>;
137
+ /**
138
+ * Refresh only the pairs whose current status is non-healthy
139
+ * (`invalid`, `billing`, or `unavailable`). Used by the lazy recovery
140
+ * loop so a billing outage on one provider does not trigger throwaway
141
+ * probes on every other healthy pair the agent has registered.
142
+ * Errors thrown by `verifyFn` are caught and recorded as `unavailable`.
143
+ */
144
+ refreshUnhealthy(): Promise<readonly LlmHealthSnapshotEntry[]>;
97
145
  /** Read-only view, primarily for logs and tests. */
98
146
  snapshot(): readonly LlmHealthSnapshotEntry[];
99
147
  private probeIfNeeded;
148
+ private synthesizeFailureFromCache;
100
149
  private probe;
101
150
  private applyVerification;
102
151
  }
103
152
 
104
153
  /**
105
- * Periodic LLM health probe. Stops on demand. Logs status transitions
106
- * (healthy <-> unhealthy) but does not log every successful tick to keep
107
- * the operator log quiet.
154
+ * Lazy LLM recovery probe. While every registered (provider, model) pair
155
+ * is healthy, ticks are no-ops: zero API traffic, zero billing tokens
156
+ * burned. The loop only does real work after a pair has flipped to
157
+ * unhealthy (via `markUnhealthyFromJob` from the runtime, or from a
158
+ * preflight probe that returned a non-ok verification): each tick
159
+ * re-probes only the unhealthy pairs and, on success, flips them back to
160
+ * healthy via the monitor's normal `applyVerification` path.
161
+ *
162
+ * The recovery loop is the only path back to `healthy` after a reactive
163
+ * markUnhealthy. Without it, the agent would stay locked out until
164
+ * restart even after the operator pops their billing back up.
108
165
  *
109
- * The heartbeat pure-delegates to `monitor.refreshAll()`; both timing and
110
- * logging policy live here so the monitor stays a pure state-machine.
166
+ * Logging policy lives here so the monitor stays a pure state-machine.
167
+ * Status transitions (healthy <-> unhealthy) are logged once per change;
168
+ * routine successful re-probes are not logged.
111
169
  */
112
170
 
113
171
  interface HeartbeatHandle {
114
172
  stop(): void;
115
173
  }
116
- interface StartLlmHeartbeatOptions {
174
+ interface StartLlmRecoveryOptions {
117
175
  monitor: LlmHealthMonitor;
118
- /** Defaults to 10 minutes. */
176
+ /** Defaults to {@link LAZY_RECOVERY_INTERVAL_MS} (5 minutes). */
119
177
  intervalMs?: number;
120
178
  /** Operator log sink. Defaults to no-op (silent). */
121
179
  log?: (msg: string) => void;
122
180
  }
123
- declare function startLlmHeartbeat(options: StartLlmHeartbeatOptions): HeartbeatHandle;
181
+ /**
182
+ * @deprecated Renamed to {@link StartLlmRecoveryOptions}. Kept as an alias
183
+ * so external consumers keep building during the rename. The semantics
184
+ * have changed (lazy, recovery-only) but the option surface is identical.
185
+ */
186
+ type StartLlmHeartbeatOptions = StartLlmRecoveryOptions;
187
+ /**
188
+ * Start the lazy recovery loop. Returns a handle whose `stop()` cancels
189
+ * the timer (idempotent). The loop ticks every `intervalMs` ms; each
190
+ * tick scans `monitor.snapshot()` and, if any pair is non-healthy, asks
191
+ * the monitor to re-probe just those non-healthy pairs via
192
+ * `refreshUnhealthy()`. Healthy pairs are never re-probed by the loop -
193
+ * those keep their cached `healthy` until TTL expiry forces a probe at
194
+ * the next `assertReady` call. When all pairs are healthy the tick is a
195
+ * single Map walk and no API calls are made.
196
+ *
197
+ * The function is named `startLlmRecovery` but the legacy export
198
+ * `startLlmHeartbeat` (below) is preserved as an alias so external
199
+ * code that already imports it keeps working unchanged.
200
+ */
201
+ declare function startLlmRecovery(options: StartLlmRecoveryOptions): HeartbeatHandle;
202
+ /**
203
+ * @deprecated Renamed to {@link startLlmRecovery}. The old name is
204
+ * preserved as an alias so existing imports keep working during the
205
+ * rename. Behavior changed materially: the new loop is lazy
206
+ * (no API calls while all pairs are healthy) and defaults to
207
+ * `LAZY_RECOVERY_INTERVAL_MS` (5 min) instead of 10 min.
208
+ */
209
+ declare const startLlmHeartbeat: typeof startLlmRecovery;
124
210
 
125
211
  /**
126
212
  * Two-tier rate limiter for free LLM skills (mode='llm', price=0).
@@ -176,4 +262,4 @@ declare function createFreeLlmLimiterSet(options?: FreeLlmLimiterOptions): FreeL
176
262
  */
177
263
  declare function freeLlmCustomerKey(customerId: string, skillName: string): string;
178
264
 
179
- export { DEFAULT_FREE_LLM_GLOBAL_MAX, DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS, DEFAULT_FREE_LLM_MAX_TRACKED_KEYS, DEFAULT_FREE_LLM_PER_CUSTOMER_MAX, DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS, DEFAULT_HEALTH_TTL_MS, DEFAULT_HEARTBEAT_INTERVAL_MS, FREE_LLM_GLOBAL_KEY, type FreeLlmLimiterOptions, type FreeLlmLimiterSet, type HeartbeatHandle, LlmHealthMonitor, type LlmHealthMonitorOptions, LlmHealthSnapshotEntry, LlmKeyVerification, type LlmKeyVerifyFn, type RegisterArgs, SkillRateLimit, type StartLlmHeartbeatOptions, UNAVAILABLE_TOLERANCE, createFreeLlmLimiterSet, freeLlmCustomerKey, resolvePerSkillRateLimit, startLlmHeartbeat };
265
+ export { DEFAULT_FREE_LLM_GLOBAL_MAX, DEFAULT_FREE_LLM_GLOBAL_WINDOW_MS, DEFAULT_FREE_LLM_MAX_TRACKED_KEYS, DEFAULT_FREE_LLM_PER_CUSTOMER_MAX, DEFAULT_FREE_LLM_PER_CUSTOMER_WINDOW_MS, DEFAULT_HEALTH_TTL_MS, DEFAULT_HEARTBEAT_INTERVAL_MS, FREE_LLM_GLOBAL_KEY, type FreeLlmLimiterOptions, type FreeLlmLimiterSet, type HeartbeatHandle, LAZY_RECOVERY_INTERVAL_MS, LlmHealthMonitor, type LlmHealthMonitorOptions, LlmHealthSnapshotEntry, LlmKeyVerification, type LlmKeyVerifyFn, type RegisterArgs, SCRIPT_EXIT_BILLING_EXHAUSTED, SkillRateLimit, type StartLlmHeartbeatOptions, type StartLlmRecoveryOptions, UNAVAILABLE_TOLERANCE, createFreeLlmLimiterSet, freeLlmCustomerKey, resolvePerSkillRateLimit, startLlmHeartbeat, startLlmRecovery };