llm-relay 0.64.0 → 0.64.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -34,6 +34,18 @@
|
|
|
34
34
|
* is invisible to `/candidates`' cooldown column and to the dashboard Cooldowns panel, because
|
|
35
35
|
* those read breaker state. The response header is its surface.
|
|
36
36
|
*
|
|
37
|
+
* ⚠ **WHICH DATASET, and what follows from it.** `getDeploymentMeasurement` returns the
|
|
38
|
+
* BREAKER's pings, which are written only by `applyHealthOutcome` on the REQUEST path — they are
|
|
39
|
+
* real served-request latencies, they live in memory only, and `PingLoop` never writes them
|
|
40
|
+
* (`cadence.ts`: "PingLoop holds no breaker reference"). `breaker-persistence.ts` deliberately
|
|
41
|
+
* does not persist them either ("`probe-cache.json` is their one home"). So: **this term is inert
|
|
42
|
+
* after every relay restart** until `minSamples` real requests have been served per deployment,
|
|
43
|
+
* and it measures what callers actually waited for rather than what a 1-token probe waited for.
|
|
44
|
+
* That is defensible — request latency is the thing being complained about — but it is NOT the
|
|
45
|
+
* dataset `llm-relay candidates` displays, so the two surfaces can disagree about one deployment.
|
|
46
|
+
* Switching to the probe dataset would survive restarts and match `candidates`, at the cost of
|
|
47
|
+
* measuring a 1-token round-trip instead of a real answer.
|
|
48
|
+
*
|
|
37
49
|
* Pure and bounded: no IO and no clock of its own beyond what the caller passes. The factory wraps
|
|
38
50
|
* everything in try/catch — this runs on the request path, and a routing hint must never be able
|
|
39
51
|
* to fail a request. It logs nothing: routing is not an error stream.
|
|
@@ -46,10 +58,14 @@ import type { ResolvedAttempt } from "./resolved-attempt.js";
|
|
|
46
58
|
* "a guess must never be labelled a measurement" invariant draws — it forbids inventing an
|
|
47
59
|
* unpublished provider limit, price or context ceiling, and explicitly permits a tunable default.
|
|
48
60
|
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
61
|
+
* ⚠ **KNOWN CALIBRATION CAVEAT, stated rather than glossed.** The figures that motivated this
|
|
62
|
+
* module — a serving member at p95 23478 ms, a walk-burning member at 70364 ms — were read from
|
|
63
|
+
* `llm-relay candidates`, whose p95 column comes from `PingLoop.getModelSummary()`: the PROBE
|
|
64
|
+
* dataset in `probe-cache.json`. This module reads a DIFFERENT dataset (below), so 30000 ms is
|
|
65
|
+
* calibrated on probe latency and applied to request latency. A probe sends `max_tokens: 1`; a
|
|
66
|
+
* real request generates, so request latency runs systematically HIGHER. Expect this default to
|
|
67
|
+
* demote more readily than those two numbers suggest, and re-calibrate against request-path
|
|
68
|
+
* figures before treating 30000 as measured rather than chosen.
|
|
53
69
|
*/
|
|
54
70
|
export declare const DEFAULT_LATENCY_P95_MS = 30000;
|
|
55
71
|
/**
|
package/dist/latency-demotion.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getP95 } from "./ping/metrics.js";
|
|
1
|
+
import { MEASURABLE_CODES, getP95 } from "./ping/metrics.js";
|
|
2
2
|
export const DEFAULT_LATENCY_P95_MS = 30_000;
|
|
3
3
|
export const DEFAULT_LATENCY_MIN_SAMPLES = 5;
|
|
4
4
|
function resolveSettings(settings) {
|
|
@@ -8,7 +8,6 @@ function resolveSettings(settings) {
|
|
|
8
8
|
minSamples: settings?.minSamples ?? DEFAULT_LATENCY_MIN_SAMPLES,
|
|
9
9
|
};
|
|
10
10
|
}
|
|
11
|
-
const MEASURABLE_CODES = new Set(["200", "401"]);
|
|
12
11
|
export function resolveLatencyDemotion(deps, attempt) {
|
|
13
12
|
const { enabled, p95Ms, minSamples } = resolveSettings(deps.settings);
|
|
14
13
|
if (!enabled)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"latency-demotion.js","sourceRoot":"","sources":["../src/latency-demotion.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"latency-demotion.js","sourceRoot":"","sources":["../src/latency-demotion.ts"],"names":[],"mappings":"AA6DA,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAgB7D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAO7C,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AA2B7C,SAAS,eAAe,CAAC,QAA2C;IAKlE,OAAO;QACL,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,IAAI;QAClC,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,sBAAsB;QAChD,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,2BAA2B;KAChE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAyB,EACzB,OAAwB;IAExB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtE,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC;QACxD,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;QAKjC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI;KACpC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAErF,IAAI,OAAO,GAAG,UAAU;QAAE,OAAO,IAAI,CAAC;IAEtC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAW3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC;IAE9B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AACrD,CAAC;AAOD,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAC/D,OAAO,CAAC,OAAO,EAAE,EAAE;QACjB,IAAI,CAAC;YACH,OAAO,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAGD,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,QAAyB;IAC1E,OAAO,GAAG,IAAI,SAAS,QAAQ,CAAC,KAAK,QAAQ,QAAQ,CAAC,WAAW,WAAW,QAAQ,CAAC,OAAO,WAAW,CAAC;AAC1G,CAAC"}
|
package/dist/ping/metrics.d.ts
CHANGED
|
@@ -4,6 +4,20 @@ export interface PingRecord {
|
|
|
4
4
|
timestamp: number;
|
|
5
5
|
}
|
|
6
6
|
export type Verdict = "Perfect" | "Normal" | "Slow" | "Spiky" | "Overloaded" | "Unstable" | "Not Active" | "Pending";
|
|
7
|
+
/**
|
|
8
|
+
* Codes whose round-trip time is a real LATENCY sample. A 401 came back from the provider,
|
|
9
|
+
* so it timed the network path and belongs in avg/p95/jitter/spike.
|
|
10
|
+
*
|
|
11
|
+
* ⚠ Latency only — this is NOT an availability set. Availability is decided by `getUptime()`
|
|
12
|
+
* below, which counts `"200"` and nothing else, so a target answering only 401s reports 0%
|
|
13
|
+
* uptime and `getStabilityScore` MULTIPLIES its composite to 0. That claim used to read "capped
|
|
14
|
+
* accordingly" while the arithmetic merely subtracted a 20% term, which capped an all-401 target
|
|
15
|
+
* at ~80 — above every all-success target with mediocre latency. The cap is real now because
|
|
16
|
+
* availability scales the score rather than contributing a share of it. Two other sites used to
|
|
17
|
+
* treat 401 as equivalent to 200 for availability (`probe-cache.ts`, `cadence.ts`) and a provider
|
|
18
|
+
* with a revoked key read as healthy; both now follow the 200-only rule.
|
|
19
|
+
*/
|
|
20
|
+
export declare const MEASURABLE_CODES: ReadonlySet<string>;
|
|
7
21
|
/** Calculate average latency from measurable pings (HTTP 200/401). Returns Infinity if none. */
|
|
8
22
|
export declare function getAvg(pings: PingRecord[]): number;
|
|
9
23
|
/** Calculate 95th percentile latency from measurable pings. Returns Infinity if none. */
|
package/dist/ping/metrics.js
CHANGED
package/dist/ping/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/ping/metrics.ts"],"names":[],"mappings":"AA6BA,MAAM,gBAAgB,
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/ping/metrics.ts"],"names":[],"mappings":"AA6BA,MAAM,CAAC,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAG7E,MAAM,UAAU,MAAM,CAAC,KAAmB;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;AAC7C,CAAC;AAGD,MAAM,UAAU,MAAM,CAAC,KAAmB;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC;AACnC,CAAC;AAGD,MAAM,UAAU,SAAS,CAAC,KAAmB;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAC9E,MAAM,QAAQ,GACZ,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IACjF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzC,CAAC;AAGD,MAAM,UAAU,YAAY,CAAC,KAAmB;IAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC;IAC5D,OAAO,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;AACpC,CAAC;AAGD,MAAM,UAAU,SAAS,CAAC,KAAmB;IAC3C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IAChE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;AACvD,CAAC;AA2BD,MAAM,UAAU,iBAAiB,CAAC,KAAmB;IACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IAElC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAItC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAEtD,MAAM,cAAc,GAAG,GAAG,GAAG,QAAQ,GAAG,GAAG,GAAG,WAAW,GAAG,GAAG,GAAG,UAAU,CAAC;IAC7E,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;AACrD,CAAC;AAQD,MAAM,UAAU,gBAAgB,CAAC,KAAmB;IAClD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC,EAAE;QAAE,CAAC,EAAE,CAAC;IAC5E,OAAO,CAAC,CAAC;AACX,CAAC;AASD,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,CAAC;AAYjD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAcjD,MAAM,UAAU,kBAAkB,CAAC,KAAmB;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IACtC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtD,OAAO,gBAAgB,CAAC,KAAK,CAAC,IAAI,+BAA+B,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;AAC7F,CAAC;AAWD,MAAM,UAAU,UAAU,CACxB,KAAmB,EACnB,OAAuD,EAAE;IAEzD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAItD,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,YAAY,CAAC;IAEhF,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IAC5E,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;IACjD,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAEvC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAE1B,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC;QACzD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,GAAG,GAAG,IAAI,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI;YAAE,OAAO,OAAO,CAAC;QACzD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,GAAG,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC;IAC9B,OAAO,UAAU,CAAC;AACpB,CAAC"}
|