llm-relay 0.65.1 → 0.65.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -39,12 +39,26 @@
|
|
|
39
39
|
* overhead. So:
|
|
40
40
|
* - **per-token** (`getP95MsPerToken`) is the primary signal, over REQUEST samples only. It is
|
|
41
41
|
* what the owner asked for, and the only figure that is fair across sample kinds.
|
|
42
|
-
* - **absolute** (`getP95`) is the fallback, over
|
|
43
|
-
* deployment that is slow before it emits anything, and it works before any request
|
|
44
|
-
* exists.
|
|
42
|
+
* - **absolute** (`getP95`) is the fallback, over measurable **PROBE** samples only. It still
|
|
43
|
+
* catches a deployment that is slow before it emits anything, and it works before any request
|
|
44
|
+
* sample exists.
|
|
45
45
|
* Per-token is tested FIRST, so a deployment with real traffic is judged on the better evidence
|
|
46
46
|
* rather than on whichever ceiling happens to trip first.
|
|
47
47
|
*
|
|
48
|
+
* ⚠ **The absolute fallback reads PROBE samples ONLY, and that split is load-bearing
|
|
49
|
+
* (2026-08-30).** It briefly read every measurable sample, request samples included, which put a
|
|
50
|
+
* probe-calibrated ceiling in front of generation data — the exact comparison the paragraph above
|
|
51
|
+
* says cannot be made. Measured live: `nim/nvidia/nemotron-3-ultra-550b-a55b`, which had served 59
|
|
52
|
+
* of this machine's 62 successful requests, answered one request with 632 tokens in 34863 ms. That
|
|
53
|
+
* is **55.2 ms/token** against a 250 ceiling — healthy by the primary signal — yet it pushed the
|
|
54
|
+
* mixed absolute p95 to 34863 and DEMOTED the deployment. Probe-only, the same deployment reads
|
|
55
|
+
* 23478 ms and is not demoted. The per-token guard could not save it, because per-token engages
|
|
56
|
+
* only at `minSamples` REQUEST samples and it had three. So a deployment demoted itself by
|
|
57
|
+
* succeeding, inside a window every deployment passes through on its way to being measured.
|
|
58
|
+
* ⚠ A request sample with NO token count therefore reaches NEITHER statistic. That is deliberate:
|
|
59
|
+
* it is a generation of unknown length, so it is not normalisable and not what `p95Ms` describes.
|
|
60
|
+
* Evidence: `docs/latency-demotion-regression-2026-08-30.md`.
|
|
61
|
+
*
|
|
48
62
|
* ⚠ **NO breaker cooldown is registered, and that is the design, not an omission.** Quota demotion
|
|
49
63
|
* can register one because its evidence STATES a `resetsAt`. Latency states no reset, and this
|
|
50
64
|
* relay never invents a cooldown duration — so instead the term is re-resolved from the rolling
|
package/dist/latency-demotion.js
CHANGED
|
@@ -27,10 +27,11 @@ export function resolveLatencyDemotion(deps, attempt) {
|
|
|
27
27
|
: null;
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
-
const
|
|
30
|
+
const probeSamples = pings.filter((p) => p.source !== "request");
|
|
31
|
+
const absoluteSamples = probeSamples.filter((p) => MEASURABLE_CODES.has(p.code)).length;
|
|
31
32
|
if (absoluteSamples < minSamples)
|
|
32
33
|
return null;
|
|
33
|
-
const p95 = getP95(
|
|
34
|
+
const p95 = getP95(probeSamples);
|
|
34
35
|
if (!Number.isFinite(p95))
|
|
35
36
|
return null;
|
|
36
37
|
if (p95 <= p95Ms)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"latency-demotion.js","sourceRoot":"","sources":["../src/latency-demotion.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"latency-demotion.js","sourceRoot":"","sources":["../src/latency-demotion.ts"],"names":[],"mappings":"AA2EA,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,MAAM,EACN,gBAAgB,GAEjB,MAAM,mBAAmB,CAAC;AAkB3B,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAChD,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAQ7C,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAqC7C,SAAS,eAAe,CAAC,QAA2C;IAMlE,OAAO;QACL,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,IAAI;QAClC,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,sBAAsB;QAChD,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,4BAA4B;QAChE,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,2BAA2B;KAChE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAyB,EACzB,OAAwB;IAExB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAG1B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjE,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IAIlE,MAAM,eAAe,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,eAAe,IAAI,UAAU,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAGrC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAM1B,OAAO,IAAI,GAAG,UAAU;gBACtB,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE;gBACzF,CAAC,CAAC,IAAI,CAAC;QACX,CAAC;IACH,CAAC;IASD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IACjE,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACxF,IAAI,eAAe,GAAG,UAAU;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC;IAC9B,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;AAC1F,CAAC;AAOD,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAC/D,OAAO,CAAC,OAAO,EAAE,EAAE;QACjB,IAAI,CAAC;YACH,OAAO,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAMD,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,QAAyB;IAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,KAAK,WAAW,CAAC;IAChD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACrF,OAAO,GAAG,IAAI,SAAS,QAAQ,GAAG,IAAI,MAAM,QAAQ,CAAC,SAAS,GAAG,IAAI,SAAS,QAAQ,CAAC,OAAO,IAAI,IAAI,GAAG,CAAC;AAC5G,CAAC"}
|