llm-relay 0.64.1 → 0.64.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -34,6 +34,18 @@
|
|
|
34
34
|
* is invisible to `/candidates`' cooldown column and to the dashboard Cooldowns panel, because
|
|
35
35
|
* those read breaker state. The response header is its surface.
|
|
36
36
|
*
|
|
37
|
+
* ⚠ **WHICH DATASET, and what follows from it.** `getDeploymentMeasurement` returns the
|
|
38
|
+
* BREAKER's pings, which are written only by `applyHealthOutcome` on the REQUEST path — they are
|
|
39
|
+
* real served-request latencies, they live in memory only, and `PingLoop` never writes them
|
|
40
|
+
* (`cadence.ts`: "PingLoop holds no breaker reference"). `breaker-persistence.ts` deliberately
|
|
41
|
+
* does not persist them either ("`probe-cache.json` is their one home"). So: **this term is inert
|
|
42
|
+
* after every relay restart** until `minSamples` real requests have been served per deployment,
|
|
43
|
+
* and it measures what callers actually waited for rather than what a 1-token probe waited for.
|
|
44
|
+
* That is defensible — request latency is the thing being complained about — but it is NOT the
|
|
45
|
+
* dataset `llm-relay candidates` displays, so the two surfaces can disagree about one deployment.
|
|
46
|
+
* Switching to the probe dataset would survive restarts and match `candidates`, at the cost of
|
|
47
|
+
* measuring a 1-token round-trip instead of a real answer.
|
|
48
|
+
*
|
|
37
49
|
* Pure and bounded: no IO and no clock of its own beyond what the caller passes. The factory wraps
|
|
38
50
|
* everything in try/catch — this runs on the request path, and a routing hint must never be able
|
|
39
51
|
* to fail a request. It logs nothing: routing is not an error stream.
|
|
@@ -46,10 +58,14 @@ import type { ResolvedAttempt } from "./resolved-attempt.js";
|
|
|
46
58
|
* "a guess must never be labelled a measurement" invariant draws — it forbids inventing an
|
|
47
59
|
* unpublished provider limit, price or context ceiling, and explicitly permits a tunable default.
|
|
48
60
|
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
61
|
+
* ⚠ **KNOWN CALIBRATION CAVEAT, stated rather than glossed.** The figures that motivated this
|
|
62
|
+
* module — a serving member at p95 23478 ms, a walk-burning member at 70364 ms — were read from
|
|
63
|
+
* `llm-relay candidates`, whose p95 column comes from `PingLoop.getModelSummary()`: the PROBE
|
|
64
|
+
* dataset in `probe-cache.json`. This module reads a DIFFERENT dataset (below), so 30000 ms is
|
|
65
|
+
* calibrated on probe latency and applied to request latency. A probe sends `max_tokens: 1`; a
|
|
66
|
+
* real request generates, so request latency runs systematically HIGHER. Expect this default to
|
|
67
|
+
* demote more readily than those two numbers suggest, and re-calibrate against request-path
|
|
68
|
+
* figures before treating 30000 as measured rather than chosen.
|
|
53
69
|
*/
|
|
54
70
|
export declare const DEFAULT_LATENCY_P95_MS = 30000;
|
|
55
71
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"latency-demotion.js","sourceRoot":"","sources":["../src/latency-demotion.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"latency-demotion.js","sourceRoot":"","sources":["../src/latency-demotion.ts"],"names":[],"mappings":"AA6DA,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAgB7D,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAO7C,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AA2B7C,SAAS,eAAe,CAAC,QAA2C;IAKlE,OAAO;QACL,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,IAAI;QAClC,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,sBAAsB;QAChD,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,2BAA2B;KAChE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAyB,EACzB,OAAwB;IAExB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtE,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC;QACxD,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;QAKjC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI;KACpC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAErF,IAAI,OAAO,GAAG,UAAU;QAAE,OAAO,IAAI,CAAC;IAEtC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAW3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC;IAE9B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AACrD,CAAC;AAOD,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAC/D,OAAO,CAAC,OAAO,EAAE,EAAE;QACjB,IAAI,CAAC;YACH,OAAO,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAGD,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,QAAyB;IAC1E,OAAO,GAAG,IAAI,SAAS,QAAQ,CAAC,KAAK,QAAQ,QAAQ,CAAC,WAAW,WAAW,QAAQ,CAAC,OAAO,WAAW,CAAC;AAC1G,CAAC"}
|