llm-relay 0.52.0 → 0.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/availability-snapshot.js +5 -4
- package/dist/availability-snapshot.js.map +1 -1
- package/dist/availability.d.ts +21 -1
- package/dist/availability.js +45 -4
- package/dist/availability.js.map +1 -1
- package/dist/breaker-persistence.d.ts +39 -0
- package/dist/breaker-persistence.js +166 -0
- package/dist/breaker-persistence.js.map +1 -0
- package/dist/candidates.d.ts +7 -1
- package/dist/candidates.js.map +1 -1
- package/dist/circuit-breaker.d.ts +44 -0
- package/dist/circuit-breaker.js +91 -0
- package/dist/circuit-breaker.js.map +1 -1
- package/dist/cli.d.ts +23 -0
- package/dist/cli.js +152 -9
- package/dist/cli.js.map +1 -1
- package/dist/ping/metrics.d.ts +25 -1
- package/dist/ping/metrics.js +36 -9
- package/dist/ping/metrics.js.map +1 -1
- package/dist/quota-demotion.d.ts +10 -8
- package/dist/quota-demotion.js +23 -9
- package/dist/quota-demotion.js.map +1 -1
- package/dist/server.js +14 -0
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
- package/skills/llm-relay/SKILL.md +26 -0
package/dist/ping/metrics.js
CHANGED
|
@@ -4,9 +4,12 @@
|
|
|
4
4
|
*
|
|
5
5
|
* ⚠ Latency only — this is NOT an availability set. Availability is decided by `getUptime()`
|
|
6
6
|
* below, which counts `"200"` and nothing else, so a target answering only 401s reports 0%
|
|
7
|
-
* uptime and its composite
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* uptime and `getStabilityScore` MULTIPLIES its composite to 0. That claim used to read "capped
|
|
8
|
+
* accordingly" while the arithmetic merely subtracted a 20% term, which capped an all-401 target
|
|
9
|
+
* at ~80 — above every all-success target with mediocre latency. The cap is real now because
|
|
10
|
+
* availability scales the score rather than contributing a share of it. Two other sites used to
|
|
11
|
+
* treat 401 as equivalent to 200 for availability (`probe-cache.ts`, `cadence.ts`) and a provider
|
|
12
|
+
* with a revoked key read as healthy; both now follow the 200-only rule.
|
|
10
13
|
*/
|
|
11
14
|
const MEASURABLE_CODES = new Set(["200", "401"]);
|
|
12
15
|
/** Calculate average latency from measurable pings (HTTP 200/401). Returns Infinity if none. */
|
|
@@ -50,21 +53,45 @@ export function getUptime(pings) {
|
|
|
50
53
|
const successful = pings.filter((p) => p.code === "200").length;
|
|
51
54
|
return Math.round((successful / pings.length) * 100);
|
|
52
55
|
}
|
|
53
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Composite Stability Score (0–100). Returns -1 only when the deployment has NEVER been probed.
|
|
58
|
+
*
|
|
59
|
+
* Shape: **latency quality, SCALED by availability** — not latency quality plus a fifth of
|
|
60
|
+
* availability. The distinction is the whole point, because `MEASURABLE_CODES` is a latency set:
|
|
61
|
+
* 403/404/429/5xx leave p95, jitter and spike entirely, so under the old additive form
|
|
62
|
+
* (`0.3*p95 + 0.3*jitter + 0.2*spike + 0.2*uptime`) a failing deployment kept a clean latency
|
|
63
|
+
* profile and paid only 20%. Measured on live probe data: 1 success in 12 scored **81** while
|
|
64
|
+
* 3 of 3 scored **27**, and 27 zero-success deployments scored above 50 — on a machine whose
|
|
65
|
+
* pools are all `{include: "free"}`, so this score IS the pool order. The comment above
|
|
66
|
+
* `MEASURABLE_CODES` claimed such a target was "capped accordingly"; only a multiplier delivers
|
|
67
|
+
* that cap.
|
|
68
|
+
*
|
|
69
|
+
* The latency weights are renormalized to sum to 1 across the three latency terms, so a
|
|
70
|
+
* deployment at 100% uptime scores exactly its latency quality and nothing is silently rescaled.
|
|
71
|
+
*
|
|
72
|
+
* ⚠ 401 deliberately STAYS in the latency terms: the response came back from the provider, so it
|
|
73
|
+
* really did time the network path. It contributes nothing to uptime, so the multiplier is what
|
|
74
|
+
* stops a revoked key from reading as a fast healthy target — no need to discard real timing data.
|
|
75
|
+
*
|
|
76
|
+
* ⚠ `-1` means "no samples", not "no MEASURABLE samples". Twelve consecutive 402s is evidence that
|
|
77
|
+
* this deployment fails, not absence of evidence: returning -1 there made every consumer read
|
|
78
|
+
* "unmeasured" and the ordering substitute a neutral 50, which put a fully-exhausted deployment
|
|
79
|
+
* above one that answered every probe.
|
|
80
|
+
*/
|
|
54
81
|
export function getStabilityScore(pings) {
|
|
55
|
-
|
|
56
|
-
if (measurable.length === 0)
|
|
82
|
+
if (pings.length === 0)
|
|
57
83
|
return -1;
|
|
58
84
|
const p95 = getP95(pings);
|
|
59
85
|
const jitter = getJitter(pings);
|
|
60
86
|
const uptime = getUptime(pings);
|
|
61
87
|
const spikeRate = getSpikeRate(pings);
|
|
88
|
+
// With no measurable sample `getP95` returns Infinity, so `p95Score` clamps to 0 rather than
|
|
89
|
+
// producing NaN. Such a deployment also has 0% uptime, so the product is 0 either way.
|
|
62
90
|
const p95Score = Math.max(0, Math.min(100, 100 * (1 - p95 / 5000)));
|
|
63
91
|
const jitterScore = Math.max(0, Math.min(100, 100 * (1 - jitter / 2000)));
|
|
64
92
|
const spikeScore = Math.max(0, 100 * (1 - spikeRate));
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
return Math.round(score);
|
|
93
|
+
const latencyQuality = 0.4 * p95Score + 0.4 * jitterScore + 0.2 * spikeScore;
|
|
94
|
+
return Math.round(latencyQuality * (uptime / 100));
|
|
68
95
|
}
|
|
69
96
|
/**
|
|
70
97
|
* How many of the MOST RECENT probes failed, consecutively.
|
package/dist/ping/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/ping/metrics.ts"],"names":[],"mappings":"AAgBA
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/ping/metrics.ts"],"names":[],"mappings":"AAgBA;;;;;;;;;;;;GAYG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAEjD,gGAAgG;AAChG,MAAM,UAAU,MAAM,CAAC,KAAmB;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;AAC7C,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,MAAM,CAAC,KAAmB;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC;AACnC,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,SAAS,CAAC,KAAmB;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAC9E,MAAM,QAAQ,GACZ,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IACjF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,KAAmB;IAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;AACpC,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,SAAS,CAAC,KAAmB;IAC3C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IAChE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;AACvD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAmB;IACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IAElC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAEtC,6FAA6F;IAC7F,uFAAuF;IACvF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAEtD,MAAM,cAAc,GAAG,GAAG,GAAG,QAAQ,GAAG,GAAG,GAAG,WAAW,GAAG,GAAG,GAAG,UAAU,CAAC;IAC7E,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAmB;IAClD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC,EAAE;QAAE,CAAC,EAAE,CAAC;IAC5E,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,CAAC;AAEjD;;;;;;;;;GASG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAEjD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAmB;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IACtC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtD,OAAO,gBAAgB,CAAC,KAAK,CAAC,IAAI,+BAA+B,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;AAC7F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CACxB,KAAmB,EACnB,OAAuD,EAAE;IAEzD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAEtD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,YAAY,CAAC;IAEhF,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IAC5E,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;IACjD,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAEvC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAE1B,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC;QACzD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,GAAG,GAAG,IAAI,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC;QACzD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,GAAG,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC;IAC9B,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
package/dist/quota-demotion.d.ts
CHANGED
|
@@ -33,8 +33,16 @@ import { type RemainingResolution, type ResetsAtResolution } from "./availabilit
|
|
|
33
33
|
/** One spent, gateable bucket — the smallest honest statement of "why this cell steps aside". */
|
|
34
34
|
export interface QuotaDemotion {
|
|
35
35
|
readonly axis: QuotaAxis;
|
|
36
|
-
/**
|
|
37
|
-
|
|
36
|
+
/**
|
|
37
|
+
* MAY be `"unknown"`, and the reason the old `Exclude` was wrong is worth keeping: it read
|
|
38
|
+
* "a bucket without a known period cannot reach a boundary to expire at", which the wire
|
|
39
|
+
* falsifies. Groq answers `x-ratelimit-limit-requests` with limit, remaining AND reset, and the
|
|
40
|
+
* reset is what a demotion expires against — rung 1 of `resolveResetsAt` reads it directly and
|
|
41
|
+
* never needs a period. An unknown-period bucket is admitted only when the provider stated that
|
|
42
|
+
* reset (`collectQuotaBuckets`), so this member can only ever carry a first-party expiry. What
|
|
43
|
+
* an unknown period still cannot do is rung-2 arithmetic; `localUsedForPeriod` enforces that.
|
|
44
|
+
*/
|
|
45
|
+
readonly period: QuotaPeriod;
|
|
38
46
|
/** ≤ 0 by construction; preserved rather than clamped — overshoot is information. */
|
|
39
47
|
readonly remaining: number;
|
|
40
48
|
/** `provider-stated` | `derived:provider-stated` | `derived:configured` | `derived:learned` — never a guess label. */
|
|
@@ -62,12 +70,6 @@ export interface QuotaDemotionDeps {
|
|
|
62
70
|
*/
|
|
63
71
|
readonly accounting?: Pick<import("./accounting-store.js").AccountingStore, "usedInWindow"> | null;
|
|
64
72
|
}
|
|
65
|
-
/**
|
|
66
|
-
* Canonical bucket order, so "first gating bucket" is deterministic across calls.
|
|
67
|
-
*
|
|
68
|
-
* Exported for `hard-cap.ts`, which orders the same (axis, period) buckets for the same reason:
|
|
69
|
-
* one definition of "requests before tokens, minute before day" rather than two to keep in step.
|
|
70
|
-
*/
|
|
71
73
|
export declare function bucketRank(axis: QuotaAxis, period: QuotaPeriod): number;
|
|
72
74
|
/**
|
|
73
75
|
* Build the request-path evaluator. The wrapper is the safety seam: NOTHING inside may throw into
|
package/dist/quota-demotion.js
CHANGED
|
@@ -1,17 +1,28 @@
|
|
|
1
1
|
import { parseCredentialId } from "./credential-id.js";
|
|
2
2
|
import { resolveConfiguredLimits } from "./configured-limits.js";
|
|
3
3
|
import { observedRateLimits } from "./rate-limits.js";
|
|
4
|
-
import { collectQuotaBuckets,
|
|
4
|
+
import { collectQuotaBuckets, localUsedForPeriod, resolveRemaining, resolveResetsAt, } from "./availability.js";
|
|
5
5
|
/**
|
|
6
6
|
* Canonical bucket order, so "first gating bucket" is deterministic across calls.
|
|
7
7
|
*
|
|
8
8
|
* Exported for `hard-cap.ts`, which orders the same (axis, period) buckets for the same reason:
|
|
9
9
|
* one definition of "requests before tokens, minute before day" rather than two to keep in step.
|
|
10
10
|
*/
|
|
11
|
+
const PERIOD_RANK = {
|
|
12
|
+
minute: 0,
|
|
13
|
+
day: 1,
|
|
14
|
+
month: 2,
|
|
15
|
+
// Last on purpose: an unknown-period bucket carries no boundary of its own, so a named period
|
|
16
|
+
// covering the same axis is the more specific statement and should be preferred when both exist.
|
|
17
|
+
unknown: 3,
|
|
18
|
+
};
|
|
11
19
|
export function bucketRank(axis, period) {
|
|
12
20
|
const axisRank = axis === "requests" ? 0 : 1;
|
|
13
|
-
|
|
14
|
-
|
|
21
|
+
// ⚠ A total table, not a ternary chain. The chain this replaced ended in a bare `: 2`, so
|
|
22
|
+
// "month" and "unknown" shared a rank and a fifth period would silently have joined them —
|
|
23
|
+
// the closed-vocabulary fall-through class CLAUDE.md records. A new member is now a compile
|
|
24
|
+
// error at the table, which is where a maintainer adding one is already looking.
|
|
25
|
+
return axisRank * 10 + PERIOD_RANK[period];
|
|
15
26
|
}
|
|
16
27
|
/** The §5.4 verdict for ONE credential×deployment cell, from whatever evidence exists. */
|
|
17
28
|
function resolveQuotaDemotion(deps, attempt, now) {
|
|
@@ -40,8 +51,11 @@ function resolveQuotaDemotion(deps, attempt, now) {
|
|
|
40
51
|
// Cache the RAW ledger window by period, so at most one read per period serves every axis
|
|
41
52
|
// projection without storing an already-projected value under an axis-free key.
|
|
42
53
|
const windowCache = new Map();
|
|
43
|
-
const localUsedFor = (period, axis) =>
|
|
44
|
-
|
|
54
|
+
const localUsedFor = (period, axis) =>
|
|
55
|
+
// `localUsedForPeriod` owns the "an unknown period reads no ledger" rule, so this caller
|
|
56
|
+
// never restates it and the three ledger consumers cannot drift apart.
|
|
57
|
+
localUsedForPeriod((namedPeriod) => {
|
|
58
|
+
let window = windowCache.get(namedPeriod);
|
|
45
59
|
if (window === undefined) {
|
|
46
60
|
window =
|
|
47
61
|
deps.accounting === undefined || deps.accounting === null
|
|
@@ -49,13 +63,13 @@ function resolveQuotaDemotion(deps, attempt, now) {
|
|
|
49
63
|
: deps.accounting.usedInWindow({
|
|
50
64
|
credentialId: attempt.credentialId,
|
|
51
65
|
...(model !== null ? { model } : {}),
|
|
52
|
-
period,
|
|
66
|
+
period: namedPeriod,
|
|
53
67
|
now,
|
|
54
68
|
});
|
|
55
|
-
windowCache.set(
|
|
69
|
+
windowCache.set(namedPeriod, window);
|
|
56
70
|
}
|
|
57
|
-
return
|
|
58
|
-
};
|
|
71
|
+
return window;
|
|
72
|
+
}, period, axis);
|
|
59
73
|
let best = null;
|
|
60
74
|
for (const bucket of [...buckets.values()].sort((a, b) => bucketRank(a.axis, a.period) - bucketRank(b.axis, b.period))) {
|
|
61
75
|
const resolution = resolveRemaining({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quota-demotion.js","sourceRoot":"","sources":["../src/quota-demotion.ts"],"names":[],"mappings":"AA+BA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAIvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EACL,mBAAmB,EACnB,
|
|
1
|
+
{"version":3,"file":"quota-demotion.js","sourceRoot":"","sources":["../src/quota-demotion.ts"],"names":[],"mappings":"AA+BA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAIvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,GAGhB,MAAM,mBAAmB,CAAC;AA6C3B;;;;;GAKG;AACH,MAAM,WAAW,GAAG;IAClB,MAAM,EAAE,CAAC;IACT,GAAG,EAAE,CAAC;IACN,KAAK,EAAE,CAAC;IACR,8FAA8F;IAC9F,iGAAiG;IACjG,OAAO,EAAE,CAAC;CACoC,CAAC;AAEjD,MAAM,UAAU,UAAU,CAAC,IAAe,EAAE,MAAmB;IAC7D,MAAM,QAAQ,GAAG,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,0FAA0F;IAC1F,2FAA2F;IAC3F,4FAA4F;IAC5F,iFAAiF;IACjF,OAAO,QAAQ,GAAG,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;AAC7C,CAAC;AAED,0FAA0F;AAC1F,SAAS,oBAAoB,CAAC,IAAuB,EAAE,OAAwB,EAAE,GAAW;IAC1F,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;IACxC,IAAI,QAAQ,EAAE,OAAO,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAC7C,MAAM,cAAc,GAAG,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAC;IAEzD,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC;IACnC,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEvD,MAAM,QAAQ,GAA2B;QACvC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,KAAK;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB,IAAI,EAAE,CAAC;IAC9E,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,CAAC,CAAC;IACpG,2FAA2F;IAC3F,sEAAsE;IACtE,MAAM,OAAO,GAAG,cAAc,IAAI,KAAK,KAAK,IAAI;QAC9C,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,YAA4B,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC;QAC3F,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,OAAO,GAAG,mBAAmB,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;IAE3E,0FAA0F;IAC1F,gFAAgF;IAChF,MAAM,WAAW,GAAG,IAAI,GAAG,EAA+D,CAAC;IAC3F,MAAM,YAAY,GAAG,CAAC,MAAmB,EAAE,IAAe,EAAE,EAAE;IAC5D,yFAAyF;IACzF,uEAAuE;IACvE,kBAAkB,CAChB,CAAC,WAAW,EAAE,EAAE;QACd,IAAI,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC1C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM;gBACJ,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;oBACvD,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;wBAC3B,YAAY,EAAE,OAAO,CAAC,YAAY;wBAClC,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACpC,MAAM,EAAE,WAAW;wBACnB,GAAG;qBACJ,CAAC,CAAC;YACT,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,EACD,MAAM,EACN,IAAI,CACL,CAAC;IAEJ,IAAI,IAAI,GAAyB,IAAI,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACvH,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS;gBAC/E,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;gBAC3B,CAAC,CAAC,EAAE,CAAC;YACP,SAAS,EAAE,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC;YACnD,GAAG;SACJ,CAAC,CAAC;QACH,IAAI,UAAU,CAAC,SAAS,KAAK,IAAI,IAAI,UAAU,CAAC,SAAS,GAAG,CAAC;YAAE,SAAS;QACxE,2FAA2F;QAC3F,yFAAyF;QACzF,2DAA2D;QAC3D,IAAI,CAAC,CAAC,UAAU,CAAC,eAAe,IAAI,UAAU,CAAC,KAAK,KAAK,iBAAiB,CAAC;YAAE,SAAS;QACtF,MAAM,MAAM,GAAG,eAAe,CAAC;YAC7B,cAAc,EAAE,UAAU,CAAC,mBAAmB,EAAE,QAAQ,IAAI,IAAI;YAChE,YAAY,EAAE,IAAI;YAClB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG;SACJ,CAAC,CAAC;QACH,2FAA2F;QAC3F,2FAA2F;QAC3F,uFAAuF;QACvF,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;YAAE,SAAS;QAChE,IACE,IAAI,KAAK,IAAI;YACb,gFAAgF;YAChF,4EAA4E;YAC5E,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAC/B,CAAC;YACD,IAAI,GAAG;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,aAAa,EAAE,MAAM,CAAC,KAAK;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAuB;IAC3D,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;QACtB,IAAI,CAAC;YACH,OAAO,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,QAAuB;IACtE,OAAO,GAAG,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,cAAc,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,KAAK,GAAG,CAAC;AAC5G,CAAC"}
|
package/dist/server.js
CHANGED
|
@@ -28,6 +28,7 @@ import { createDashboardSnapshotReadPort } from "./dashboard-snapshot.js";
|
|
|
28
28
|
import { createAvailabilityProducer } from "./availability-snapshot.js";
|
|
29
29
|
import { DASHBOARD_STATIC_SECURITY_HEADERS, DashboardStaticHandler, getProductionDashboardAssetRoot, } from "./dashboard-static.js";
|
|
30
30
|
import { CircuitBreaker, globalCircuitBreaker } from "./circuit-breaker.js";
|
|
31
|
+
import { installBreakerPersistence } from "./breaker-persistence.js";
|
|
31
32
|
import { estimateRequestTokens, assessCost, resolveMetadata } from "./metadata.js";
|
|
32
33
|
import { findTierModel, loadTierData } from "./tier-data.js";
|
|
33
34
|
import { specOfTarget } from "./benchmarks.js";
|
|
@@ -195,6 +196,19 @@ export function createProxy(cfg, deps = {}) {
|
|
|
195
196
|
const catalog = deps.catalog ?? new ModelCatalog();
|
|
196
197
|
const pingLoop = deps.pingLoop ?? new PingLoop(cfg, catalog);
|
|
197
198
|
const breaker = deps.breaker ?? new CircuitBreaker();
|
|
199
|
+
/**
|
|
200
|
+
* Cooling state survives a restart, like ping health already did.
|
|
201
|
+
*
|
|
202
|
+
* Without this the relay discarded every cooldown and the whole unexplained-429 escalation
|
|
203
|
+
* ladder on restart, then walked back into the same walls — measured here as a 19.9-hour
|
|
204
|
+
* cooldown learned from 7 consecutive 429s on one deployment and 26 on another.
|
|
205
|
+
*
|
|
206
|
+
* ⚠ Skipped under vitest for the reason the probe loop is: a test proxy must not read or write
|
|
207
|
+
* the developer's live cooling state. `getBreakerStatePath()` also redirects under vitest, so
|
|
208
|
+
* this is belt and braces — the suite exercises the mechanism directly with an explicit path.
|
|
209
|
+
*/
|
|
210
|
+
if (!process.env.VITEST)
|
|
211
|
+
installBreakerPersistence(breaker);
|
|
198
212
|
// Process-local and deliberately credential-wide: a deployment switch must not reset fairness.
|
|
199
213
|
const credentialLru = new CredentialLru();
|
|
200
214
|
const modelCallRecorder = deps.modelCallRecorder ?? (process.env.VITEST ? undefined : recordModelCall);
|