@miller-tech/uap 2.11.1 → 2.16.2

package/dist/inference/analysis.js

@@ -0,0 +1,243 @@
+/**
+ * Inference-stack health analysis — the deterministic core.
+ *
+ * Written after a live incident on 2026-09-21. A llama.cpp server that had
+ * been up 18 hours was still reporting `active (running)` and GREEN from
+ * `uap doctor`, while its prefill throughput had fallen roughly 8x and five
+ * client requests had blown a 1800s deadline overnight. Liveness checks could
+ * not see it, because nothing was down.
+ *
+ * Three things were wrong, and each needs a different kind of evidence:
+ *
+ *   1. Throughput decayed with uptime. Invisible in any single sample — and
+ *      invisible in an hourly average too, because prefill tok/s depends
+ *      heavily on prompt length. It only showed up once the samples were
+ *      BUCKETED BY PROMPT SIZE and compared across time.
+ *   2. Context checkpoints were starved. `--ctx-checkpoints` is per slot, and
+ *      at 1 the checkpoint froze at n_past=11903 while 43k+ tokens were
+ *      reusable, so ~32k tokens were re-prefilled every single turn.
+ *   3. KV was pinned at the VBR floor, i.e. the pool was saturated and the
+ *      cache had silently dropped to its lowest quality tier.
+ *
+ * Everything here is pure: callers supply already-collected samples. That
+ * keeps the judgment testable without a GPU, a systemd unit, or a journal.
+ */
+export const DEFAULT_THRESHOLDS = {
+    // Measured 2026-09-21: 682 -> 77 tok/s (ratio 0.11) was a hard failure;
+    // 682 -> 375 (0.55) was already visibly hurting turn latency.
+    degradeRatioRed: 0.35,
+    degradeRatioWarn: 0.65,
+    minSamplesPerEra: 4,
+    // Restoring under half of what was reusable means most of the prefix is
+    // being recomputed every turn.
+    checkpointRecoveryWarn: 0.5,
+    kvFloorSlack: 0.15,
+    timeoutsWarn: 1,
+};
+/** Prompt-size buckets. Prefill tok/s is strongly size-dependent, so an
+ * average across mixed sizes is not comparable over time. */
+export function sizeBucket(tokens) {
+    if (tokens < 5_000)
+        return '<5k';
+    if (tokens < 15_000)
+        return '5-15k';
+    if (tokens < 30_000)
+        return '15-30k';
+    return '>30k';
+}
+/**
+ * Compare early-life against recent prefill throughput WITHIN a size bucket.
+ *
+ * Splitting the samples in half and averaging each side would be wrong: if
+ * the workload's prompt sizes drift (and they do — conversations grow), the
+ * mean moves for reasons that have nothing to do with the server. So pick the
+ * bucket that has enough samples on BOTH sides and compare only within it.
+ */
+export function analyzeTrend(samples, thresholds = DEFAULT_THRESHOLDS) {
+    if (samples.length < thresholds.minSamplesPerEra * 2) {
+        return { earlyCount: 0, recentCount: 0, note: 'not enough prefill samples for a trend' };
+    }
+    // Bucket FIRST, then split each bucket at its OWN median time.
+    //
+    // This used to split every sample at the GLOBAL median and bucket the two
+    // halves. That silently dropped any bucket whose samples were skewed in
+    // time — and a real agent workload skews hard, because conversations grow,
+    // so large prompts arrive late. Measured on a live 6h journal:
+    //
+    //   bucket   early recent  outcome
+    //   <5k         41     17  reported: 0.78 (mild)
+    //   15-30k       2     21  EXCLUDED — needed 4 per era
+    //
+    // The excluded bucket had gone 616 -> 73 tok/s across its 23 samples. An 8x
+    // collapse, invisible, while the report showed a benign 0.78. A bucket's own
+    // chronology is what "did this get slower" means; the other buckets' arrival
+    // times are irrelevant to it.
+    const byBucket = new Map();
+    for (const r of samples) {
+        // A NaN or negative reading is a parse artefact, not a measurement.
+        if (!Number.isFinite(r.tokensPerSecond) || r.tokensPerSecond < 0)
+            continue;
+        const b = sizeBucket(r.tokens);
+        (byBucket.get(b) ?? byBucket.set(b, []).get(b)).push(r);
+    }
+    const buckets = [];
+    for (const [bucket, rows] of byBucket) {
+        if (rows.length < thresholds.minSamplesPerEra * 2)
+            continue;
+        const ordered = [...rows].sort((a, b) => a.at - b.at);
+        const mid = Math.floor(ordered.length / 2);
+        const ev = ordered.slice(0, mid).map((x) => x.tokensPerSecond);
+        const rv = ordered.slice(mid).map((x) => x.tokensPerSecond);
+        if (ev.length < thresholds.minSamplesPerEra || rv.length < thresholds.minSamplesPerEra)
+            continue;
+        const earlyMean = ev.reduce((a, b) => a + b, 0) / ev.length;
+        const recentMean = rv.reduce((a, b) => a + b, 0) / rv.length;
+        // An all-zero early era cannot produce a meaningful ratio.
+        if (!(earlyMean > 0) || !Number.isFinite(recentMean))
+            continue;
+        buckets.push({
+            bucket,
+            earlyMean,
+            recentMean,
+            ratio: recentMean / earlyMean,
+            earlyCount: ev.length,
+            recentCount: rv.length,
+        });
+    }
+    if (buckets.length === 0) {
+        return {
+            earlyCount: 0,
+            recentCount: 0,
+            note: 'no prompt-size bucket had enough samples to split — cannot compare like with like',
+        };
+    }
+    // Rank by SEVERITY, not by sample count. Picking the fattest bucket hid the
+    // worst one: on a real 30h journal the 5-15k bucket sat at 0.289 (RED) while
+    // the reported >30k bucket showed 0.522 (WARN) purely because it had more
+    // samples — understating severity and flipping the remedy from "restart" to
+    // "watch it". A thin bucket collapsing 10x emitted nothing at all.
+    buckets.sort((a, b) => a.ratio - b.ratio || b.earlyCount + b.recentCount - (a.earlyCount + a.recentCount));
+    const worst = buckets[0];
+    return { ...worst, buckets };
+}
+/** How much of the reusable prefix the checkpoints actually deliver. */
+export function analyzeCheckpoints(samples) {
+    const usable = samples.filter((s) => s.reusable > 0);
+    if (usable.length === 0)
+        return { samples: 0 };
+    const recoveries = usable.map((s) => Math.min(1, s.restored / s.reusable));
+    const wasted = usable.map((s) => Math.max(0, s.reusable - s.restored));
+    return {
+        recovery: recoveries.reduce((a, b) => a + b, 0) / recoveries.length,
+        wastedTokens: wasted.reduce((a, b) => a + b, 0) / wasted.length,
+        samples: usable.length,
+    };
+}
+/** Name the other slowed buckets, so a single headline number does not read
+ *  as though only one prompt size is affected. */
+function alsoAffected(trend) {
+    const others = (trend.buckets ?? []).filter((b) => b.bucket !== trend.bucket && b.ratio < 1);
+    if (others.length === 0)
+        return '';
+    return `; also down in ${others.map((b) => `${b.bucket} (${Math.round((1 - b.ratio) * 100)}%)`).join(', ')}`;
+}
+const rank = { GREEN: 0, WARN: 1, RED: 2, UNKNOWN: 0 };
+/** Roll a snapshot up into a verdict. Pure; no I/O, no clock. */
+export function assessInference(snap, thresholds = DEFAULT_THRESHOLDS) {
+    const findings = [];
+    const trend = analyzeTrend(snap.prefill, thresholds);
+    const checkpoints = analyzeCheckpoints(snap.checkpoints);
+    // 1. Throughput decay over this process's life.
+    if (trend.ratio !== undefined) {
+        const pct = Math.round((1 - trend.ratio) * 100);
+        if (trend.ratio < thresholds.degradeRatioRed) {
+            findings.push({
+                health: 'RED',
+                code: 'prefill-decay',
+                message: `prefill in the ${trend.bucket} bucket fell ${pct}% over this process's life ` +
+                    `(${Math.round(trend.earlyMean)} -> ${Math.round(trend.recentMean)} tok/s)` +
+                    alsoAffected(trend),
+                remedy: 'restart the inference server; if it returns within days, investigate the prompt cache and idle-slot caching',
+            });
+        }
+        else if (trend.ratio < thresholds.degradeRatioWarn) {
+            findings.push({
+                health: 'WARN',
+                code: 'prefill-decay',
+                message: `prefill in the ${trend.bucket} bucket is down ${pct}% ` +
+                    `(${Math.round(trend.earlyMean)} -> ${Math.round(trend.recentMean)} tok/s)` +
+                    alsoAffected(trend),
+                remedy: 'watch it; a restart restores throughput if the decay continues',
+            });
+        }
+    }
+    // 2. Checkpoint starvation — reusable prefix that is not actually reused.
+    if (checkpoints.recovery !== undefined && checkpoints.recovery < thresholds.checkpointRecoveryWarn) {
+        const cp = snap.ctxCheckpoints;
+        findings.push({
+            health: checkpoints.wastedTokens > 20_000 ? 'RED' : 'WARN',
+            code: 'checkpoint-starved',
+            message: `checkpoints recover only ${Math.round(checkpoints.recovery * 100)}% of the reusable prefix ` +
+                `(~${Math.round(checkpoints.wastedTokens).toLocaleString()} tokens re-prefilled per turn)`,
+            remedy: cp !== undefined && cp <= 2
+                ? `raise --ctx-checkpoints (currently ${cp}); it is per SLOT, so a low value cannot track a growing conversation`
+                : 'check --ctx-checkpoints and --checkpoint-min-step against the conversation length',
+        });
+    }
+    // 3. KV pinned at the VBR floor = the pool is saturated.
+    if (snap.kvBitsPerValue !== undefined && snap.kvFloorBitsPerValue !== undefined) {
+        if (snap.kvBitsPerValue <= snap.kvFloorBitsPerValue + thresholds.kvFloorSlack) {
+            findings.push({
+                health: 'WARN',
+                code: 'kv-at-floor',
+                message: `KV is pinned at the VBR floor (${snap.kvBitsPerValue} bpv, floor ${snap.kvFloorBitsPerValue}) ` +
+                    '— the pool is saturated and cache quality is at its lowest tier',
+                remedy: 'reduce the context pool, the rail count, or the per-session cap; or raise --vbr-vram if VRAM allows',
+            });
+        }
+    }
+    // 4. Client-visible timeouts. A symptom, but the one users actually feel.
+    if ((snap.generationTimeouts ?? 0) >= thresholds.timeoutsWarn) {
+        findings.push({
+            health: 'RED',
+            code: 'generation-timeouts',
+            message: `${snap.generationTimeouts} generation timeout(s) in the window — clients lost work`,
+            remedy: 'fix the cause (prefill decay / checkpoint starvation) rather than raising the deadline',
+        });
+    }
+    // 5. A window spanning a restart makes the early/recent split meaningless:
+    //    the "decay" may simply be the boundary between two processes.
+    if ((snap.processCount ?? 1) > 1 && trend.ratio !== undefined) {
+        findings.push({
+            health: 'WARN',
+            code: 'spans-restart',
+            message: `the window spans ${snap.processCount} server processes — a trend across a restart ` +
+                'compares different process lifetimes, not decay within one',
+            remedy: 'narrow --since/--until to a single process, or omit them to use the current one',
+        });
+    }
+    // 6. Rail/pool sanity: under --kv-unified the pool is SHARED, so N rails
+    //    each believing they own it is an overcommit waiting to happen.
+    if (snap.rails !== undefined && snap.rails > 1 && snap.poolCells !== undefined) {
+        findings.push({
+            health: 'GREEN',
+            code: 'shared-pool',
+            message: `${snap.rails} rails share one ${snap.poolCells.toLocaleString()}-cell pool ` +
+                `(${Math.floor(snap.poolCells / snap.rails).toLocaleString()} each if evenly divided)`,
+        });
+    }
+    const worst = findings.reduce((acc, f) => (rank[f.health] > rank[acc] ? f.health : acc), findings.length > 0 ? 'GREEN' : 'UNKNOWN');
+    // No evidence at all is UNKNOWN, never a fabricated GREEN — the same
+    // fail-open posture the capacity doctor uses.
+    //
+    // But "no evidence" is about the SAMPLED signals (trend, checkpoints, KV).
+    // A finding that fired is itself evidence, so an actionable one must never
+    // be masked by an UNKNOWN rollup: generation timeouts are counted from the
+    // proxy journal and can fire with none of the sampled signals present,
+    // which let a RED finding pass --strict with exit 0.
+    const actionable = findings.some((f) => f.health !== 'GREEN');
+    const noSampledEvidence = trend.ratio === undefined && checkpoints.samples === 0 && snap.kvBitsPerValue === undefined;
+    const health = actionable || !noSampledEvidence ? worst : 'UNKNOWN';
+    return { health, findings, trend, checkpoints };
+}
+//# sourceMappingURL=analysis.js.map

package/dist/inference/analysis.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analysis.js","sourceRoot":"","sources":["../../src/inference/analysis.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AA4DH,MAAM,CAAC,MAAM,kBAAkB,GAAe;IAC5C,wEAAwE;IACxE,8DAA8D;IAC9D,eAAe,EAAE,IAAI;IACrB,gBAAgB,EAAE,IAAI;IACtB,gBAAgB,EAAE,CAAC;IACnB,wEAAwE;IACxE,+BAA+B;IAC/B,sBAAsB,EAAE,GAAG;IAC3B,YAAY,EAAE,IAAI;IAClB,YAAY,EAAE,CAAC;CAChB,CAAC;AAEF;6DAC6D;AAC7D,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,IAAI,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IACjC,IAAI,MAAM,GAAG,MAAM;QAAE,OAAO,OAAO,CAAC;IACpC,IAAI,MAAM,GAAG,MAAM;QAAE,OAAO,QAAQ,CAAC;IACrC,OAAO,MAAM,CAAC;AAChB,CAAC;AA0BD;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAwB,EACxB,aAAyB,kBAAkB;IAE3C,IAAI,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,wCAAwC,EAAE,CAAC;IAC3F,CAAC;IAED,+DAA+D;IAC/D,EAAE;IACF,0EAA0E;IAC1E,wEAAwE;IACxE,2EAA2E;IAC3E,+DAA+D;IAC/D,EAAE;IACF,mCAAmC;IACnC,iDAAiD;IACjD,uDAAuD;IACvD,EAAE;IACF,4EAA4E;IAC5E,6EAA6E;IAC7E,6EAA6E;IAC7E,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA2B,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,oEAAoE;QACpE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,eAAe,GAAG,CAAC;YAAE,SAAS;QAC3E,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,gBAAgB,GAAG,CAAC;YAAE,SAAS;QAC5D,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAC/D,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAC5D,IAAI,EAAE,CAAC,MAAM,GAAG,UAAU,CAAC,gBAAgB,IAAI,EAAE,CAAC,MAAM,GAAG,UAAU,CAAC,gBAAgB;YAAE,SAAS;QAEjG,MAAM,SAAS,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;QAC5D,MAAM,UAAU,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;QAC7D,2DAA2D;QAC3D,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,SAAS;QAC/D,OAAO,CAAC,IAAI,CAAC;YACX,MAAM;YACN,SAAS;YACT,UAAU;YACV,KAAK,EAAE,UAAU,GAAG,SAAS;YAC7B,UAAU,EAAE,EAAE,CAAC,MAAM;YACrB,WAAW,EAAE,EAAE,CAAC,MAAM;SACvB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,IAAI,EAAE,mFAAmF;SAC1F,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,6EAA6E;IAC7E,0EAA0E;IAC1E,4EAA4E;IAC5E,mEAAmE;IACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC3G,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAUD,wEAAwE;AACxE,MAAM,UAAU,kBAAkB,CAAC,OAA2B;IAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,OAAO;QACL,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QACnE,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QAC/D,OAAO,EAAE,MAAM,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC;AAkBD;kDACkD;AAClD,SAAS,YAAY,CAAC,KAAkB;IACtC,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC7F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,kBAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC/G,CAAC;AAED,MAAM,IAAI,GAAoC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;AAExF,iEAAiE;AACjE,MAAM,UAAU,eAAe,CAC7B,IAAuB,EACvB,aAAyB,kBAAkB;IAE3C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEzD,gDAAgD;IAChD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,eAAe,EAAE,CAAC;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,eAAe;gBACrB,OAAO,EACL,kBAAkB,KAAK,CAAC,MAAM,gBAAgB,GAAG,6BAA6B;oBAC9E,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAW,CAAC,SAAS;oBAC7E,YAAY,CAAC,KAAK,CAAC;gBACrB,MAAM,EAAE,6GAA6G;aACtH,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,gBAAgB,EAAE,CAAC;YACrD,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,eAAe;gBACrB,OAAO,EACL,kBAAkB,KAAK,CAAC,MAAM,mBAAmB,GAAG,IAAI;oBACxD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAU,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAW,CAAC,SAAS;oBAC7E,YAAY,CAAC,KAAK,CAAC;gBACrB,MAAM,EAAE,gEAAgE;aACzE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,QAAQ,GAAG,UAAU,CAAC,sBAAsB,EAAE,CAAC;QACnG,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,WAAW,CAAC,YAAa,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM;YAC3D,IAAI,EAAE,oBAAoB;YAC1B,OAAO,EACL,4BAA4B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,GAAG,GAAG,CAAC,2BAA2B;gBAC7F,KAAK,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,YAAa,CAAC,CAAC,cAAc,EAAE,gCAAgC;YAC7F,MAAM,EACJ,EAAE,KAAK,SAAS,IAAI,EAAE,IAAI,CAAC;gBACzB,CAAC,CAAC,sCAAsC,EAAE,uEAAuE;gBACjH,CAAC,CAAC,mFAAmF;SAC1F,CAAC,CAAC;IACL,CAAC;IAED,yDAAyD;IACzD,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QAChF,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,mBAAmB,GAAG,UAAU,CAAC,YAAY,EAAE,CAAC;YAC9E,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,aAAa;gBACnB,OAAO,EACL,kCAAkC,IAAI,CAAC,cAAc,eAAe,IAAI,CAAC,mBAAmB,IAAI;oBAChG,iEAAiE;gBACnE,MAAM,EAAE,qGAAqG;aAC9G,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;QAC9D,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,0DAA0D;YAC7F,MAAM,EAAE,wFAAwF;SACjG,CAAC,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,mEAAmE;IACnE,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9D,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,eAAe;YACrB,OAAO,EACL,oBAAoB,IAAI,CAAC,YAAY,+CAA+C;gBACpF,4DAA4D;YAC9D,MAAM,EAAE,iFAAiF;SAC1F,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,oEAAoE;IACpE,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC/E,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,OAAO;YACf,IAAI,EAAE,aAAa;YACnB,OAAO,EACL,GAAG,IAAI,CAAC,KAAK,oBAAoB,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,aAAa;gBAC7E,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,EAAE,0BAA0B;SACzF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAC3B,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EACzD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAC1C,CAAC;IACF,qEAAqE;IACrE,8CAA8C;IAC9C,EAAE;IACF,2EAA2E;IAC3E,2EAA2E;IAC3E,2EAA2E;IAC3E,uEAAuE;IACvE,qDAAqD;IACrD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC;IAC9D,MAAM,iBAAiB,GACrB,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,OAAO,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,CAAC;IAC9F,MAAM,MAAM,GAAoB,UAAU,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAErF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;AAClD,CAAC"}

package/dist/inference/probe.d.ts

@@ -0,0 +1,85 @@
+import type { CheckpointSample, InferenceSnapshot, PrefillSample } from './analysis.js';
+export interface ProbeDeps {
+    /** Run a command and return stdout; throw on failure. */
+    exec?: (cmd: string, args: string[]) => string;
+    /** Fetch a URL and return the body; throw on failure. */
+    fetchText?: (url: string) => Promise<string>;
+    now?: () => number;
+}
+/** Strip userinfo before a URL reaches the report. Node's fetch turns
+ *  `http://user:pass@host` into an Authorization header, and `--json` exists
+ *  to be piped into monitors and stored. */
+export declare function redactUrl(url: string): string;
+/** Pull `-np`, `-c` and `--ctx-checkpoints` out of a systemd ExecStart blob. */
+export declare function parseExecStartFlags(execStart: string): {
+    rails?: number;
+    poolCells?: number;
+    ctxCheckpoints?: number;
+};
+export interface UnitState {
+    active?: string;
+    mainPid?: number;
+    uptimeSeconds?: number;
+    execStart?: string;
+    /** ISO-ish timestamp systemd reported, passed to journalctl --since. */
+    activeEnterTimestamp?: string;
+}
+/**
+ * systemd renders timestamps as "Mon 2026-09-21 09:54:00 AEST", which
+ * Date.parse rejects outright (NaN) because of the weekday prefix and the
+ * non-standard zone abbreviation. Pull out the ISO-ish core and parse that as
+ * local time — which is the zone systemd printed it in.
+ *
+ * Returns NaN for anything unparseable, so callers keep reporting "unknown"
+ * rather than inventing an uptime.
+ */
+export declare function parseSystemdTimestamp(value?: string): number;
+export declare function probeUnit(unit: string, deps?: ProbeDeps): UnitState | null;
+export declare function parseServerJournal(text: string, year?: number): {
+    prefill: PrefillSample[];
+    checkpoints: CheckpointSample[];
+    processCount: number;
+};
+export declare function probeJournal(unit: string, since: string | undefined, deps?: ProbeDeps, until?: string): {
+    prefill: PrefillSample[];
+    checkpoints: CheckpointSample[];
+    processCount: number;
+} | null;
+export declare function countProxyTimeouts(unit: string, since: string, deps?: ProbeDeps, until?: string): number | undefined;
+export interface SlotsInfo {
+    slots?: number;
+    nCtx?: number;
+    kvBitsPerValue?: number;
+    anyProcessing?: boolean;
+}
+export declare function probeSlots(baseUrl: string, deps?: ProbeDeps): Promise<SlotsInfo | null>;
+/** VBR floor from /props, so the "pinned at the floor" test uses the server's
+ * own configured floor rather than a hardcoded constant. */
+export declare function probeVbrFloor(baseUrl: string, deps?: ProbeDeps): Promise<number | undefined>;
+export interface MetricsInfo {
+    promptTokens?: number;
+    promptSeconds?: number;
+    cachedPromptTokens?: number;
+    busySlotsPerDecode?: number;
+}
+export declare function parseMetrics(text: string): MetricsInfo;
+export declare function probeMetrics(baseUrl: string, deps?: ProbeDeps): Promise<MetricsInfo | null>;
+export interface CollectOptions {
+    serverUnit: string;
+    proxyUnit: string;
+    baseUrl: string;
+    /** journalctl --since expression; default: this process's start. */
+    since?: string;
+    /** journalctl --until expression — for analysing a PAST incident window. */
+    until?: string;
+}
+export interface Collected {
+    snapshot: InferenceSnapshot;
+    unit: UnitState | null;
+    slots: SlotsInfo | null;
+    metrics: MetricsInfo | null;
+    /** Probes that could not run, so the report can say so out loud. */
+    unavailable: string[];
+}
+export declare function collect(opts: CollectOptions, deps?: ProbeDeps): Promise<Collected>;
+//# sourceMappingURL=probe.d.ts.map

package/dist/inference/probe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/inference/probe.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAQxF,MAAM,WAAW,SAAS;IACxB,yDAAyD;IACzD,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IAC/C,yDAAyD;IACzD,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AA0DD;;4CAE4C;AAC5C,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAY7C;AAED,gFAAgF;AAChF,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAUA;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAI5D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,IAAI,CA8B1E;AA0BD,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,EACZ,IAAI,SAA2B,GAC9B;IAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CA8ErF;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,IAAI,CAAC,EAAE,SAAS,EAChB,KAAK,CAAC,EAAE,MAAM,GACb;IAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAiB5F;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,SAAS,EAChB,KAAK,CAAC,EAAE,MAAM,GACb,MAAM,GAAG,SAAS,CAYpB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAe7F;AAED;4DAC4D;AAC5D,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAUlG;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAWtD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAOjG;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,IAAI,EAAE,SAAS,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,SAAS,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oEAAoE;IACpE,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CA2ExF"}