@genee/omp-opsx-addon 0.7.0 → 0.9.0

package/lib/concurrency-tuner.ts

@@ -0,0 +1,773 @@
+/**
+ * Pure AIMD concurrency controller + host-signal parsing/aggregation/summary.
+ *
+ * Zero IO: every export here is deterministic and takes an explicit options
+ * object plus an injectable clock (`now`); nothing reads the clock on its own.
+ * Host log/DB/config access lives in `lib/concurrency-control.ts`. This module
+ * deliberately never imports `ResolvedOpsxConfig` — the assembly layer injects
+ * the parsed `concurrency_*` values as {@link TunerOptions}.
+ */
+
+/** Clean evaluation windows required before the additive increase (`M`). */
+export const UP_WINDOWS = 2;
+/** Consecutive bad evaluation windows required before the multiplicative decrease (`N`). */
+export const DOWN_BAD_STREAK = 2;
+
+/** Host log messages that bracket a queue wait. */
+export const BLOCKED_MESSAGE = 'Provider in-flight limit blocked request';
+export const WAIT_COMPLETED_MESSAGE = 'Provider in-flight limit wait completed';
+
+/**
+ * Injected configuration for the whole control loop. Field names mirror the
+ * `concurrency_*` opsx.yml keys; defaults are the spec defaults and are the
+ * single injection point for tests and for the assembly layer.
+ */
+export interface TunerOptions {
+    /**
+     * `concurrency_floor` — global lower bound of the adaptive range; MUST be >= 1.
+     * Each provider's *effective* floor is `min(floor, initial)` (see
+     * {@link effectiveFloor}): this knob sets the range but MUST NOT raise a
+     * provider's own static value.
+     */
+    floor: number;
+    /** `concurrency_ceiling` — global upper bound of the adaptive range. */
+    ceiling: number;
+    /** `concurrency_start` — start value used when no shared value exists yet. */
+    start: number;
+    /**
+     * `concurrency_up_ms` — clean-window length. The collector's evaluation
+     * cadence comes from `evalIntervalMs`; this is how much *clean time* must
+     * accumulate before {@link evaluateWindow} credits one clean window, so
+     * the two knobs move independently.
+     */
+    upMs: number;
+    /** `concurrency_down_ms` — cooldown after a multiplicative decrease. */
+    downMs: number;
+    /** `concurrency_latency_k` — inflation ratio that marks a window latency-bad. */
+    k: number;
+    /** `concurrency_min_samples` — latency samples below which latency is not judged. */
+    minSamples: number;
+    /** `concurrency_queue_wait_high_ms` — guardrail p50 threshold. */
+    queueWaitHighMs: number;
+    /** `concurrency_queue_wait_window_ms` — guardrail percentile window. */
+    queueWaitWindowMs: number;
+    /**
+     * `concurrency_eval_interval_ms` — evaluation cadence. The collector's
+     * absolute-time gate enforces it; carried here for injection parity.
+     */
+    evalIntervalMs: number;
+    /**
+     * `concurrency_decay_after_ms` — idle period after which `L` decays one
+     * notch toward `initial` per elapsed period.
+     */
+    decayAfterMs: number;
+    /** EWMA smoothing factor for the latency baseline (internal constant, not a config key). */
+    ewmaAlpha: number;
+}
+
+/** Spec defaults for every knob the control loop consumes. */
+export const TUNER_DEFAULTS: TunerOptions = {
+    floor: 4,
+    ceiling: 16,
+    start: 4,
+    upMs: 30_000,
+    downMs: 60_000,
+    k: 2,
+    minSamples: 5,
+    queueWaitHighMs: 2_000,
+    queueWaitWindowMs: 60_000,
+    evalIntervalMs: 30_000,
+    decayAfterMs: 86_400_000,
+    ewmaAlpha: 0.3,
+};
+
+/** Bad-signal classes, ordered by fixed priority (highest first). */
+export type BadEventKind = 'rate-limit' | 'latency' | 'throughput';
+
+/** Per-provider control state. Moved across eval windows by {@link evaluateWindow}. */
+export interface ProviderConcurrencyState {
+    /** Current adaptive limit `L`, always within `[floor, ceiling]`. */
+    limit: number;
+    /** Value `L` started from (decay target, restart fallback). */
+    initial: number;
+    floor: number;
+    ceiling: number;
+    /** Consecutive clean windows accumulated toward the additive increase. */
+    cleanWindows: number;
+    /** Clean time (ms) accumulated toward the next {@link cleanWindows} credit. */
+    cleanWindowMs: number;
+    /** Consecutive bad windows accumulated toward the multiplicative decrease. */
+    badStreak: number;
+    /** Epoch ms until which the additive increase is disabled (0 = not cooling). */
+    cooldownUntil: number;
+    /** EWMA baseline of provider generation duration in ms (`null` until observed). */
+    ewmaGenMs: number | null;
+    /** Epoch ms of the last window that carried any observation. */
+    lastObservationAt: number;
+}
+
+/** One host-log `blocked` / `wait completed` pair. */
+export interface QueueWaitSample {
+    provider: string;
+    /** Completion timestamp (epoch ms). */
+    at: number;
+    /** Wait duration in ms (`completion - blocked`). */
+    ms: number;
+    /** `limit` reported on the blocked line, when present. */
+    limit: number | null;
+}
+
+/** Signals observed for one provider during one evaluation window. */
+export interface WindowSignals {
+    /**
+     * R1 observation evidence: `true` when this window carried at least one
+     * generation sample for the provider. A window without evidence MUST NOT be
+     * credited as a clean window (see {@link evaluateWindow}).
+     *
+     * Criterion — the single source of truth for the freeze — is: the provider's
+     * `model_perf` delta for this window yields a non-empty {@link genSamplesMs},
+     * i.e. it really generated at least one sample in this window (the control
+     * layer derives it with `genSamplesFromPerfWindow`). Two deliberate
+     * exclusions: ① rate-limit events never qualify — such a window is
+     * classified bad before the additive branch is reached, so it can never be a
+     * *clean* window anyway; ② queue waits are load evidence, not service
+     * evidence (`L` may rise only on proof the provider served load well). This
+     * is narrower on purpose than the R5 decay anchor (`lastObservationAt`),
+     * which counts any signal as an observation.
+     */
+    observed: boolean;
+    /** ① explicit provider-level failures (429 / rate-limit / 5xx / timeout) in this window. */
+    rateLimitCount: number;
+    /** ③ generation-duration samples for this provider in this window (ms). */
+    genSamplesMs: number[];
+    /** ④ aggregated `output_tokens / gen_ms` for this window, or `null` when unavailable. */
+    tokensPerMs: number | null;
+    /** ④ EWMA baseline for {@link tokensPerMs}, or `null` until a baseline exists. */
+    tokensPerMsEwma: number | null;
+    /** ② queue-wait observations (already restricted to this provider). */
+    queueWait: QueueWaitSample[];
+    /**
+     * ② The guardrail statistic of this window, computed by the caller over
+     * `concurrency_queue_wait_window_ms` (the assembly layer passes exactly the
+     * p50 it also shows in the summary — one number, one verdict). `null` = the
+     * guardrail window carried no sample. When omitted, the evaluator derives it
+     * from {@link queueWait} with {@link queueWaitPercentiles}, so a pure caller
+     * that only holds the raw samples stays exact.
+     */
+    queueWaitP50Ms?: number | null;
+}
+
+/** Result of moving a provider state through one evaluation window. */
+export interface WindowEvaluation {
+    /** Next state. The input state is never mutated. */
+    state: ProviderConcurrencyState;
+    /** Merged bad-event class for this window (`null` = no bad signal). */
+    badKind: BadEventKind | null;
+    /** `true` when this window actually moved `limit` down. */
+    decreased: boolean;
+    /** `true` when this window raised `limit` by 1. */
+    increased: boolean;
+    /**
+     * `true` when the ② queue guardrail was detected *and* blocked this window's
+     * additive increase (it says nothing about ③④: those are masked instead, see
+     * {@link latencySuppressedByQueue}).
+     */
+    queueSuppressed: boolean;
+    /**
+     * `true` when a ③ latency / ④ throughput verdict was discarded because the
+     * queue was high: the extra generation time is attributable to our own gate,
+     * not to provider capacity. Signal ① (rate limit / explicit timeout) is
+     * never masked.
+     */
+    latencySuppressedByQueue: boolean;
+    /**
+     * `true` when this window may fold into the EWMA baselines (latency and
+     * throughput): healthy (no bad event) and not queue-masked. A masked or bad
+     * window MUST NOT move the baseline that the very next verdict is measured
+     * against.
+     */
+    baselineEligible: boolean;
+    /** `true` when the additive increase is blocked by the post-decrease cooldown. */
+    cooldownActive: boolean;
+    /**
+     * `true` when this window was frozen for lack of an observation sample
+     * (R1): no clean time accrued, clean streak left untouched.
+     */
+    frozen: boolean;
+    /** Guardrail p50 over {@link TunerOptions.queueWaitWindowMs} (`null` = no samples). */
+    queueWaitP50Ms: number | null;
+    /** Median of this window's generation samples (`null` = no samples). */
+    latencyMedianMs: number | null;
+}
+
+function truncInt(value: number, fallback: number): number {
+    return Number.isFinite(value) ? Math.trunc(value) : fallback;
+}
+
+/** Clamp an integer into `[max(1, floor), max(floor, ceiling)]`. */
+export function clampLimit(value: number, floor: number, ceiling: number): number {
+    const lo = Math.max(1, truncInt(floor, 1));
+    const hi = Math.max(lo, truncInt(ceiling, lo));
+    return Math.min(hi, Math.max(lo, truncInt(value, lo)));
+}
+
+/** Exponential moving average; the first observation seeds the baseline. */
+export function ewmaUpdate(previous: number | null, value: number, alpha: number): number {
+    if (previous === null || !Number.isFinite(previous)) return value;
+    const a = Number.isFinite(alpha) ? Math.min(1, Math.max(0, alpha)) : 1;
+    return previous + a * (value - previous);
+}
+
+/**
+ * Inclusive linear-interpolation percentile over an ascending-sorted array.
+ * `p` is in `[0, 1]`; returns `null` for an empty array.
+ */
+function percentile(sortedAsc: number[], p: number): number | null {
+    const n = sortedAsc.length;
+    if (n === 0) return null;
+    if (n === 1) return sortedAsc[0];
+    const idx = (n - 1) * Math.min(1, Math.max(0, p));
+    const lo = Math.floor(idx);
+    const hi = Math.ceil(idx);
+    if (lo === hi) return sortedAsc[lo];
+    return sortedAsc[lo] + (sortedAsc[hi] - sortedAsc[lo]) * (idx - lo);
+}
+
+/** Median of an unsorted sample list (`null` when empty). */
+function median(values: number[]): number | null {
+    if (values.length === 0) return null;
+    return percentile([...values].sort((a, b) => a - b), 0.5);
+}
+
+/**
+ * Effective lower bound of one provider's adaptive range: `min(concurrency_floor,
+ * initial)`.
+ *
+ * The provider's own starting value wins whenever it is smaller: the global floor
+ * may shape the range but MUST NOT raise a user's static value, so a provider
+ * sitting at 3 keeps `L >= 3` — as both its floor and its start — instead of
+ * being pulled up to `concurrency_floor`.
+ */
+export function effectiveFloor(configuredFloor: number, initial: number): number {
+    const configured = Math.max(1, truncInt(configuredFloor, 1));
+    const seed = Number.isFinite(initial) ? Math.trunc(initial) : configured;
+    return Math.max(1, Math.min(configured, seed));
+}
+
+/**
+ * Start state for a provider.
+ *
+ * `initial = clamp(sharedLimit ?? start, effectiveFloor(floor, seed), ceiling)`
+ * and `limit = initial`, so a provider that already has a static
+ * `maxInFlightRequests` value starts there (yielding `|new - old| = 0`, i.e. no
+ * write), while a provider only listed in `concurrency_providers` starts at the
+ * conservative `start`. `floor` is the effective floor, capped by the provider's
+ * own start and by its (possibly overridden) ceiling.
+ */
+export function createProviderState(
+    init: {
+        /** Current `providers.maxInFlightRequests[provider]`, or `null` when absent. */
+        sharedLimit: number | null;
+        /** `concurrency_start` fallback. */
+        start: number;
+        /** `concurrency_ceiling_by_provider[provider]` override. */
+        ceilingOverride?: number | null;
+        options: TunerOptions;
+    },
+): ProviderConcurrencyState {
+    const ceiling = Math.max(1, truncInt(init.ceilingOverride ?? init.options.ceiling, init.options.ceiling));
+    const seed = init.sharedLimit === null || init.sharedLimit === undefined ? init.start : init.sharedLimit;
+    const floor = Math.min(ceiling, effectiveFloor(init.options.floor, seed));
+    const limit = clampLimit(seed, floor, ceiling);
+    return {
+        limit,
+        initial: limit,
+        floor,
+        ceiling,
+        cleanWindows: 0,
+        cleanWindowMs: 0,
+        badStreak: 0,
+        cooldownUntil: 0,
+        ewmaGenMs: null,
+        lastObservationAt: 0,
+    };
+}
+
+/** Samples within `[now - windowMs, now]`. */
+function queueWaitSamplesInWindow(
+    samples: QueueWaitSample[],
+    now: number,
+    windowMs: number,
+): number[] {
+    const since = now - windowMs;
+    const values: number[] = [];
+    for (const sample of samples) {
+        if (sample.at >= since && sample.at <= now) values.push(sample.ms);
+    }
+    return values;
+}
+
+/** p50/p90 of the queue waits observed in the guardrail window. */
+export function queueWaitPercentiles(
+    samples: QueueWaitSample[],
+    now: number,
+    windowMs: number,
+): { p50: number | null; p90: number | null; count: number } {
+    const values = queueWaitSamplesInWindow(samples, now, windowMs).sort((a, b) => a - b);
+    return { p50: percentile(values, 0.5), p90: percentile(values, 0.9), count: values.length };
+}
+
+/** ③ latency inflation: median above `k × EWMA` with at least `minSamples` samples. */
+export function isLatencyBad(
+    medianMs: number | null,
+    sampleCount: number,
+    baselineMs: number | null,
+    options: TunerOptions,
+): boolean {
+    if (medianMs === null || baselineMs === null || baselineMs <= 0) return false;
+    if (sampleCount < options.minSamples) return false;
+    return medianMs > options.k * baselineMs;
+}
+
+/**
+ * ④ throughput fallback: below `0.5 × EWMA`, and only while latency samples are
+ * sparse (`< minSamples`) — a provider with enough latency samples is never
+ * judged on throughput.
+ */
+export function isThroughputBad(
+    tokensPerMs: number | null,
+    baselineTokensPerMs: number | null,
+    latencySampleCount: number,
+    options: TunerOptions,
+): boolean {
+    if (tokensPerMs === null || baselineTokensPerMs === null || baselineTokensPerMs <= 0) return false;
+    if (latencySampleCount >= options.minSamples) return false;
+    return tokensPerMs < 0.5 * baselineTokensPerMs;
+}
+
+/**
+ * Merge one window's signals into the single bad-event class that drives debounce
+ * and the multiplicative decrease. Priority ① > ③ > ④; queue wait (②) is not a
+ * bad-event source — it only feeds the guardrail.
+ *
+ * `queueHigh` is the ② verdict over `concurrency_queue_wait_window_ms`, applied
+ * *before* ③④ are judged: while the queue is high the extra generation time
+ * belongs to our own gate, so treating it as provider-capacity evidence is
+ * exactly the self-reinforcing spiral that
+ * drove three live providers down to `L = 1`. Signal ① is provider-side evidence
+ * and is never masked.
+ */
+export function classifyBadEvent(
+    signals: WindowSignals,
+    state: ProviderConcurrencyState,
+    options: TunerOptions,
+    queueHigh: boolean,
+): BadEventKind | null {
+    if (signals.rateLimitCount > 0) return 'rate-limit';
+    if (queueHigh) return null;
+    return classifyLatencyThroughputBad(signals, state, options);
+}
+
+/**
+ * The ③④ merge *without* the ② mask: what {@link classifyBadEvent} would return
+ * if the queue were not high. Shared by the classifier and by
+ * {@link WindowEvaluation.latencySuppressedByQueue}, so the masked verdict is
+ * never a second implementation of the same judgement.
+ */
+function classifyLatencyThroughputBad(
+    signals: WindowSignals,
+    state: ProviderConcurrencyState,
+    options: TunerOptions,
+): BadEventKind | null {
+    if (isLatencyBad(median(signals.genSamplesMs), signals.genSamplesMs.length, state.ewmaGenMs, options)) {
+        return 'latency';
+    }
+    if (isThroughputBad(signals.tokensPerMs, signals.tokensPerMsEwma, signals.genSamplesMs.length, options)) {
+        return 'throughput';
+    }
+    return null;
+}
+
+/**
+ * Advance one provider's state by one evaluation window.
+ *
+ * - bad window → clear clean counter, grow the bad streak; at `N = 2` halve
+ *   `L = max(floor, floor(L / 2))` and start the cooldown. At most one decrease
+ *   per cooldown cycle: bad events inside the cooldown keep accruing in the
+ *   streak but MUST NOT move `L` again until it expires (anti-windup).
+ * - queue-wait p50 above the threshold → mask ③④ for this window *before* they
+ *   are judged (signal ① still classifies and may decrease), suppress the
+ *   increase, restart the clean-window count, and leave the EWMA baselines
+ *   untouched (never a decrease on its own).
+ * - cooldown active → suppress the increase.
+ * - no observation sample in the window (R1) → freeze: no clean time accrues and
+ *   the clean streak keeps its previous value. Climbing `L` is a bet that the
+ *   provider served load well, and a window with nothing observed is no
+ *   evidence, so a purely idle provider never walks up to `ceiling` (and never
+ *   fights the R5 decay back down). See {@link WindowSignals.observed}.
+ * - otherwise the window's elapsed clean time accrues toward one
+ *   `concurrency_up_ms` clean window (at most one per call — see `elapsedMs`);
+ *   at `M = 2` clean windows `L += 1`, capped at `ceiling`.
+ *
+ * `elapsedMs` is the observed gap since the previous evaluation: it is what the
+ * clean-window length is measured against, so `concurrency_up_ms` (this
+ * function) and `concurrency_eval_interval_ms` (the caller's cadence) stay
+ * independent knobs. It defaults to one evaluation interval, which reproduces
+ * "one call == one clean window" for a caller that ticks at the default cadence.
+ */
+export function evaluateWindow(
+    state: ProviderConcurrencyState,
+    signals: WindowSignals,
+    now: number,
+    options: TunerOptions,
+    elapsedMs: number = options.evalIntervalMs,
+): WindowEvaluation {
+    const next: ProviderConcurrencyState = { ...state };
+    // ② The caller may hand in the guardrail p50 it already computed (the number it
+    // also shows in the summary); otherwise it is derived from the raw samples, so
+    // a pure caller holding only those stays exact.
+    const explicitP50 = signals.queueWaitP50Ms;
+    const guardrail = explicitP50 === undefined
+        ? queueWaitPercentiles(signals.queueWait, now, options.queueWaitWindowMs)
+        : null;
+    const p50 = guardrail ? guardrail.p50 : explicitP50 ?? null;
+    // ② guardrail verdict: no sample is never "high".
+    const queueHigh = p50 !== null && p50 > options.queueWaitHighMs;
+    const latencyMedianMs = median(signals.genSamplesMs);
+    // The mask is applied *before* the ③④ verdict is taken, not in the branch
+    // chain below: under load the latency inflation is manufactured by our own
+    // gate, and letting it win the classification is what made the queue
+    // guardrail unreachable during the incident.
+    const maskedKind = queueHigh ? classifyLatencyThroughputBad(signals, state, options) : null;
+    const badKind = classifyBadEvent(signals, state, options, queueHigh);
+
+    if (badKind !== null || latencyMedianMs !== null || signals.rateLimitCount > 0 || p50 !== null) {
+        next.lastObservationAt = now;
+    }
+    // Baseline hygiene: only a healthy, non-masked window may move the latency
+    // EWMA. A window we just judged inflated would blind the next verdict, and a
+    // masked window is our own gate's inflation — folding it in would raise the
+    // baseline and blunt the mask itself.
+    const baselineEligible = badKind === null && !queueHigh;
+    if (latencyMedianMs !== null && baselineEligible) {
+        next.ewmaGenMs = ewmaUpdate(next.ewmaGenMs, latencyMedianMs, options.ewmaAlpha);
+    }
+
+    let decreased = false;
+    let increased = false;
+    let queueSuppressed = false;
+    let frozen = false;
+    if (badKind !== null) {
+        next.cleanWindows = 0;
+        next.cleanWindowMs = 0;
+        next.badStreak += 1;
+        // Anti-windup: at most ONE multiplicative decrease per `concurrency_down_ms`
+        // cycle. Bad events observed inside the cooldown are not discarded — they
+        // keep accruing in `badStreak`, which is what decides the *next* decrease
+        // once the cooldown expires — but they MUST NOT halve `L` again while it
+        // runs: each extra halving tightens the very gate that produced the queue.
+        if (next.badStreak >= DOWN_BAD_STREAK && now >= next.cooldownUntil) {
+            next.badStreak = 0;
+            const halved = Math.max(next.floor, Math.floor(next.limit / 2));
+            decreased = halved !== next.limit;
+            next.limit = halved;
+            next.cooldownUntil = now + options.downMs;
+        }
+    } else if (queueHigh) {
+        next.cleanWindows = 0;
+        next.cleanWindowMs = 0;
+        queueSuppressed = true;
+    } else if (now < next.cooldownUntil) {
+        next.cleanWindows = 0;
+        next.cleanWindowMs = 0;
+    } else if (!signals.observed) {
+        // R1 freeze (see the doc above and {@link WindowSignals.observed}): a
+        // window without an observation sample neither earns clean time nor
+        // resets the streak — it is not evidence of a bad provider either, so
+        // the counters simply stay where the last observed window left them.
+        frozen = true;
+    } else {
+        // Effective clean window = max(up_ms, eval_interval_ms): a window can only
+        // be completed by an evaluation, and each evaluation credits at most one
+        // window (the remainder is dropped below), so any `up_ms` under the
+        // evaluation cadence behaves exactly like `eval_interval_ms`. The config
+        // layer already folds `up_ms < eval_interval_ms` into that value; this
+        // clamp keeps a direct (test/injected) `TunerOptions` equally honest.
+        const windowMs = Number.isFinite(options.upMs) && options.upMs > 0 ? options.upMs : TUNER_DEFAULTS.upMs;
+        const elapsed = Number.isFinite(elapsedMs) && elapsedMs > 0 ? elapsedMs : windowMs;
+        // A state that predates the accumulator (or came from a partial literal)
+        // starts its window now instead of poisoning the sum with NaN.
+        const accrued = Number.isFinite(next.cleanWindowMs) ? next.cleanWindowMs : 0;
+        next.cleanWindowMs = accrued + elapsed;
+        if (next.cleanWindowMs >= windowMs) {
+            next.cleanWindowMs = 0;
+            next.cleanWindows += 1;
+            if (next.cleanWindows >= UP_WINDOWS) {
+                next.cleanWindows = 0;
+                if (next.limit < next.ceiling) {
+                    next.limit += 1;
+                    increased = true;
+                }
+            }
+        }
+    }
+    next.limit = clampLimit(next.limit, next.floor, next.ceiling);
+
+    return {
+        state: next,
+        badKind,
+        decreased,
+        increased,
+        queueSuppressed,
+        latencySuppressedByQueue: maskedKind !== null && badKind === null,
+        baselineEligible,
+        cooldownActive: now < next.cooldownUntil,
+        frozen,
+        queueWaitP50Ms: p50,
+        latencyMedianMs,
+    };
+}
+
+/** Parse outcome for a batch of host log lines. */
+export interface InFlightLogParseResult {
+    /** Paired waits, ascending by completion time. */
+    samples: QueueWaitSample[];
+    /** Lines that are not JSON objects or carry unusable `message`/`provider`/`timestamp`. */
+    malformedLines: number;
+    /** Well-formed lines that are neither a `blocked` nor a `wait completed` record. */
+    ignoredLines: number;
+    /** `blocked` records left without a completion. */
+    unpairedBlocked: number;
+    /** Completions left without a preceding `blocked` record. */
+    unpairedCompletions: number;
+}
+
+function parseTimestamp(raw: unknown): number | null {
+    if (typeof raw === 'number' && Number.isFinite(raw)) return raw;
+    if (typeof raw === 'string') {
+        const ms = Date.parse(raw);
+        if (Number.isFinite(ms)) return ms;
+    }
+    return null;
+}
+
+/**
+ * Parse host JSONL records into per-provider queue-wait samples.
+ *
+ * `blocked` and `wait completed` records are paired FIFO per provider; the wait
+ * is their timestamp gap. Unpaired records are dropped and counted, malformed
+ * lines are skipped — nothing throws. Callers pass the lines of every
+ * `omp.*.log` file under the same config root so other processes' waits count.
+ */
+export function parseInFlightLogLines(lines: Iterable<string>): InFlightLogParseResult {
+    const pending = new Map<string, Array<{ at: number; limit: number | null }>>();
+    const samples: QueueWaitSample[] = [];
+    let malformedLines = 0;
+    let ignoredLines = 0;
+    let unpairedCompletions = 0;
+
+    for (const line of lines) {
+        const text = line.trim();
+        if (text === '') continue;
+        let rec: Record<string, unknown>;
+        try {
+            const parsed: unknown = JSON.parse(text);
+            if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+                malformedLines += 1;
+                continue;
+            }
+            rec = parsed as Record<string, unknown>;
+        } catch {
+            malformedLines += 1;
+            continue;
+        }
+        const message = rec.message;
+        const isBlocked = message === BLOCKED_MESSAGE;
+        const isCompleted = message === WAIT_COMPLETED_MESSAGE;
+        if (!isBlocked && !isCompleted) {
+            ignoredLines += 1;
+            continue;
+        }
+        const provider = rec.provider;
+        const at = parseTimestamp(rec.timestamp);
+        if (typeof provider !== 'string' || provider === '' || at === null) {
+            malformedLines += 1;
+            continue;
+        }
+        const limit = typeof rec.limit === 'number' && Number.isFinite(rec.limit)
+            ? Math.trunc(rec.limit)
+            : null;
+        if (isBlocked) {
+            const queue = pending.get(provider) ?? [];
+            queue.push({ at, limit });
+            pending.set(provider, queue);
+            continue;
+        }
+        const queue = pending.get(provider);
+        const blocked = queue?.shift();
+        if (!blocked) {
+            unpairedCompletions += 1;
+            continue;
+        }
+        samples.push({
+            provider,
+            at,
+            ms: Math.max(0, at - blocked.at),
+            limit: blocked.limit ?? limit,
+        });
+    }
+
+    let unpairedBlocked = 0;
+    for (const queue of pending.values()) unpairedBlocked += queue.length;
+    samples.sort((a, b) => a.at - b.at || a.provider.localeCompare(b.provider));
+    return { samples, malformedLines, ignoredLines, unpairedBlocked, unpairedCompletions };
+}
+
+/** Provider id of a `model_perf.model_key` (`<provider>/<model>`). */
+export function providerOfModelKey(modelKey: string): string {
+    const slash = modelKey.indexOf('/');
+    return slash <= 0 ? modelKey : modelKey.slice(0, slash);
+}
+
+/** Cumulative `model_perf` counters for one model_key. */
+export interface ModelPerfRow {
+    modelKey: string;
+    samples: number;
+    outputTokens: number;
+    genMs: number;
+}
+
+/** Per-provider window delta derived from cumulative `model_perf` counters. */
+export interface ProviderPerfWindow {
+    provider: string;
+    /** Δsamples for the window. */
+    samples: number;
+    /** Δoutput_tokens for the window. */
+    outputTokens: number;
+    /** Δgen_ms for the window. */
+    genMs: number;
+    /** Aggregated generation duration per sample (ms). */
+    genMsPerSample: number;
+    /** Aggregated throughput (tokens per ms). */
+    tokensPerMs: number;
+    /** `false` when `samples < concurrency_min_samples` — source unusable this window. */
+    sufficientSamples: boolean;
+}
+
+function delta(current: number, previous: number): number {
+    const d = current - previous;
+    return d > 0 ? d : 0; // counter reset / rollover → no window delta
+}
+
+/**
+ * Difference cumulative `model_perf` snapshots into per-provider windows.
+ *
+ * Rows without a previous snapshot are treated as all-new. Deltas are clamped at
+ * 0 so a table reset never produces a negative window.
+ */
+export function diffModelPerf(
+    previous: ModelPerfRow[],
+    current: ModelPerfRow[],
+    options: TunerOptions,
+): ProviderPerfWindow[] {
+    const before = new Map<string, ModelPerfRow>();
+    for (const row of previous) before.set(row.modelKey, row);
+
+    const agg = new Map<string, { samples: number; outputTokens: number; genMs: number }>();
+    for (const row of current) {
+        const prev = before.get(row.modelKey);
+        const samples = delta(row.samples, prev?.samples ?? 0);
+        const outputTokens = delta(row.outputTokens, prev?.outputTokens ?? 0);
+        const genMs = delta(row.genMs, prev?.genMs ?? 0);
+        const provider = providerOfModelKey(row.modelKey);
+        const bucket = agg.get(provider) ?? { samples: 0, outputTokens: 0, genMs: 0 };
+        bucket.samples += samples;
+        bucket.outputTokens += outputTokens;
+        bucket.genMs += genMs;
+        agg.set(provider, bucket);
+    }
+
+    const windows: ProviderPerfWindow[] = [];
+    for (const [provider, bucket] of agg) {
+        windows.push({
+            provider,
+            samples: bucket.samples,
+            outputTokens: bucket.outputTokens,
+            genMs: bucket.genMs,
+            genMsPerSample: bucket.samples > 0 ? bucket.genMs / bucket.samples : 0,
+            tokensPerMs: bucket.genMs > 0 ? bucket.outputTokens / bucket.genMs : 0,
+            sufficientSamples: bucket.samples >= options.minSamples,
+        });
+    }
+    windows.sort((a, b) => a.provider.localeCompare(b.provider));
+    return windows;
+}
+
+/** EWMA baselines kept alongside the per-provider control state. */
+export interface PerfBaseline {
+    /** EWMA of `gen_ms / samples`. */
+    genMsPerSample: number | null;
+    /** EWMA of `output_tokens / gen_ms`. */
+    tokensPerMs: number | null;
+}
+
+/** Fold a provider's perf window into its EWMA baselines (no-op when unusable). */
+export function updatePerfBaseline(
+    previous: PerfBaseline | null,
+    window: ProviderPerfWindow,
+    options: TunerOptions,
+): PerfBaseline {
+    const base = previous ?? { genMsPerSample: null, tokensPerMs: null };
+    if (!window.sufficientSamples) return base;
+    return {
+        genMsPerSample: ewmaUpdate(base.genMsPerSample, window.genMsPerSample, options.ewmaAlpha),
+        tokensPerMs: ewmaUpdate(base.tokensPerMs, window.tokensPerMs, options.ewmaAlpha),
+    };
+}
+
+/** Per-provider row shown in the concurrency summary. */
+export interface ProviderConcurrencyView {
+    provider: string;
+    limit: number;
+    floor: number;
+    ceiling: number;
+    /** Explicit rate-limit (429 / rate-limit error) observations in the last window. */
+    rateLimitCount: number;
+    /** Timeout observations in the last window. */
+    timeoutCount: number;
+    queueWaitP50Ms: number | null;
+    queueWaitP90Ms: number | null;
+    /** Epoch ms of the last successful `maxInFlightRequests` write (`null` = never). */
+    lastWriteAt: number | null;
+}
+
+export interface ConcurrencySummaryInput {
+    providers: ProviderConcurrencyView[];
+    /** Lease holder id, or `null` when no process holds it. */
+    leaseOwner: string | null;
+    /** This process's id; compared against {@link leaseOwner} for the writer identity. */
+    selfId: string;
+}
+
+/**
+ * Format the concurrency summary as one line per provider prefixed by a lease
+ * header line. Writer identity is derived from `leaseOwner === selfId`, so a
+ * writer and a non-writer describe themselves differently. Returns `[]` when
+ * there is no managed provider (empty summary).
+ */
+export function formatConcurrencySummary(input: ConcurrencySummaryInput): string[] {
+    if (input.providers.length === 0) return [];
+    const isWriter = input.leaseOwner !== null && input.leaseOwner === input.selfId;
+    const owner = isWriter ? '本进程' : input.leaseOwner ?? '无';
+    const lines = [`并发控制: 租约持有者=${owner}；本进程${isWriter ? '为写者' : '非写者'}`];
+    for (const view of input.providers) {
+        lines.push(
+            `${view.provider}: L=${view.limit} 区间=[${view.floor},${view.ceiling}] `
+            + `429=${view.rateLimitCount} 超时=${view.timeoutCount} `
+            + `排队等待p50=${view.queueWaitP50Ms === null ? '-' : `${Math.round(view.queueWaitP50Ms)}ms`} `
+            + `p90=${view.queueWaitP90Ms === null ? '-' : `${Math.round(view.queueWaitP90Ms)}ms`} `
+            + `最近写入=${view.lastWriteAt === null ? '无' : new Date(view.lastWriteAt).toISOString()}`,
+        );
+    }
+    return lines;
+}