@genee/omp-opsx-addon 0.8.0 → 0.10.0

package/lib/concurrency-state.ts

@@ -0,0 +1,397 @@
+/**
+ * Concurrency-state persistence and decay — tasks 5.1–5.4 of
+ * `adaptive-provider-concurrency` (design D7, spec Requirement
+ * 「可观测与持久化衰减」).
+ *
+ * This is the durable half of the control loop: it remembers the *learned*
+ * limit across restarts, decays it while a provider looks abandoned, and
+ * guarantees the stop path keeps the last finite value.
+ *
+ * The state file is a cache of learned values, never the shared gate — the
+ * authoritative cross-process limit lives in `providers.maxInFlightRequests`
+ * (group 4 / `lib/concurrency-writer.ts`). This module consequently has no
+ * config write path, no `remove`, and no timer: it cannot delete a managed key
+ * or turn a managed provider unlimited. Decay write-backs go through the same
+ * gate/lease/writer as every other write (task 5.3).
+ */
+
+import { dirname } from 'path';
+import { clampLimit, type ProviderConcurrencyState } from './concurrency-tuner.js';
+import { nodeWriterFileSystem, type Warn, type WriterFileSystemPort } from './concurrency-writer.js';
+import { canonicalizeProvider } from './usage-resolver.js';
+
+/** State-file name directly under the config root (design D7). */
+export const STATE_FILE_NAME = 'opsx-concurrency-state.json';
+
+/** On-disk schema version; bump when the entry shape changes. */
+export const STATE_FILE_VERSION = 1;
+
+const NO_WARN: Warn = () => {};
+const PREFIX = '[omp-opsx-addon] concurrency-state:';
+
+let stateSeq = 0;
+
+function message(err: unknown): string {
+    return err instanceof Error ? err.message : String(err);
+}
+
+/** Truncate to an integer, or `fallback` when the value is not a finite number. */
+function toInt(value: unknown, fallback: number): number {
+    return typeof value === 'number' && Number.isFinite(value) ? Math.trunc(value) : fallback;
+}
+
+/** Runtime narrowing for untrusted JSON nodes (schema parse is overkill for a flat record). */
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+/**
+ * One provider's durable state. The streak counters and the latency baseline
+ * ride along so the AIMD trajectory resumes rather than restarts after a
+ * process swap.
+ */
+export interface PersistedProviderState {
+    /** Learned limit `L` at the last save. */
+    limit: number;
+    /** Decay target — `L`'s seed value at the last save. */
+    initial: number;
+    /** Epoch ms of the save that last carried this entry. */
+    updatedAt: number;
+    /** Epoch ms of the last window that carried an observation (`0` = never). */
+    lastObservationAt: number;
+    /** Consecutive bad windows at the last save. */
+    badStreak: number;
+    /** Consecutive clean windows at the last save. */
+    cleanStreak: number;
+    /** Latency EWMA baseline in ms, or `null` until observed. */
+    ewmaGenMs: number | null;
+}
+
+/** Provider id → durable state, i.e. the body of the state file. */
+export type ConcurrencyStateFile = Record<string, PersistedProviderState>;
+
+// ── reading (5.1 restart recovery / 5.2 fallback) ────────────────────
+
+export interface LoadStateArgs {
+    /** Absolute path of the state file (config root + {@link STATE_FILE_NAME}). */
+    statePath: string;
+    fileSystem?: WriterFileSystemPort;
+    warn?: Warn;
+}
+
+/**
+ * Validate one on-disk entry. Returns `null` when the entry carries no usable
+ * learned limit — the caller then falls back to the shared config value, and
+ * the provider is never treated as unlimited.
+ */
+function sanitizeEntry(provider: string, raw: unknown, warn: Warn): PersistedProviderState | null {
+    if (!isPlainObject(raw)) {
+        warn(`${PREFIX} state entry for ${provider} is not an object; ignored (the shared config value is used)`);
+        return null;
+    }
+    const limit = raw.limit;
+    if (typeof limit !== 'number' || !Number.isFinite(limit) || Math.trunc(limit) < 1) {
+        warn(`${PREFIX} state entry for ${provider} has no usable limit; ignored (the shared config value is used)`);
+        return null;
+    }
+    const learned = Math.trunc(limit);
+    const updatedAt = Math.max(0, toInt(raw.updatedAt, 0));
+    const ewma = raw.ewmaGenMs;
+    return {
+        limit: learned,
+        initial: Math.max(1, toInt(raw.initial, learned)),
+        updatedAt,
+        lastObservationAt: Math.max(0, toInt(raw.lastObservationAt, updatedAt)),
+        badStreak: Math.max(0, toInt(raw.badStreak, 0)),
+        cleanStreak: Math.max(0, toInt(raw.cleanStreak, 0)),
+        ewmaGenMs: typeof ewma === 'number' && Number.isFinite(ewma) && ewma > 0 ? ewma : null,
+    };
+}
+
+/** Canonicalize the provider keys of an entry list, dropping unusable entries. */
+function canonicalEntries(
+    entries: readonly (readonly [string, unknown])[],
+    warn: Warn,
+): ConcurrencyStateFile {
+    const out: ConcurrencyStateFile = {};
+    for (const [rawId, rawEntry] of entries) {
+        const id = canonicalizeProvider(String(rawId).trim());
+        if (!id) {
+            warn(`${PREFIX} state entry with an empty provider id is ignored`);
+            continue;
+        }
+        if (Object.hasOwn(out, id)) {
+            warn(`${PREFIX} duplicate state entry for ${id} is ignored`);
+            continue;
+        }
+        const entry = sanitizeEntry(id, rawEntry, warn);
+        if (entry) out[id] = entry;
+    }
+    return out;
+}
+
+/** Parse state-file text; every failure degrades to an empty map plus a warn. */
+function parseStateFile(text: string, statePath: string, warn: Warn): ConcurrencyStateFile {
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(text);
+    } catch (err) {
+        warn(`${PREFIX} state file ${statePath} is corrupt (${message(err)}); falling back to the shared config value`);
+        return {};
+    }
+    if (!isPlainObject(parsed)) {
+        warn(`${PREFIX} state file ${statePath} is not an object; falling back to the shared config value`);
+        return {};
+    }
+    const providers = parsed.providers;
+    if (!isPlainObject(providers)) {
+        warn(`${PREFIX} state file ${statePath} has no providers map; falling back to the shared config value`);
+        return {};
+    }
+    return canonicalEntries(Object.entries(providers), warn);
+}
+
+/**
+ * Read the persisted per-provider state (task 5.1/5.2).
+ *
+ * A missing file is the normal first-run path — nothing has been learned yet —
+ * so it is silent and yields `{}`. Corrupt JSON, a non-object document, a
+ * missing `providers` map and entries without a usable `limit` all degrade the
+ * same way: the entry is dropped, a warn is recorded, and {@link restoreLimit}
+ * falls back to the shared config value (then `initial`). Never to unlimited.
+ */
+export function loadConcurrencyState(args: LoadStateArgs): ConcurrencyStateFile {
+    const fileSystem = args.fileSystem ?? nodeWriterFileSystem;
+    const text = fileSystem.readFile(args.statePath);
+    if (text === null) return {};
+    return parseStateFile(text, args.statePath, args.warn ?? NO_WARN);
+}
+
+export interface RestoreLimitArgs {
+    /** `concurrency_floor`. */
+    floor: number;
+    /** `concurrency_ceiling` (per-provider override already applied). */
+    ceiling: number;
+    /** Current `providers.maxInFlightRequests[provider]`, or `null` when absent. */
+    sharedValue: number | null;
+    /** The provider's `initial` (seed) value. */
+    initial: number;
+}
+
+/**
+ * `L` for a restarting control loop: the persisted learned limit first, then
+ * the shared config value, then `initial` — always clamped into
+ * `[floor, ceiling]` (and therefore always a finite value `>= 1`).
+ *
+ * Priority order matters for task 5.1: a restart MUST resume the learned value
+ * rather than the seed. It is the *takeover* path (a peer handing over the
+ * lease) that re-reads the shared config value, via
+ * `resolveTakeoverStart` in `lib/concurrency-writer.ts`.
+ */
+export function restoreLimit(
+    persisted: PersistedProviderState | null | undefined,
+    args: RestoreLimitArgs,
+): number {
+    const learned = persisted?.limit;
+    if (typeof learned === 'number' && Number.isFinite(learned) && learned >= 1) {
+        return clampLimit(learned, args.floor, args.ceiling);
+    }
+    const shared = args.sharedValue;
+    if (typeof shared === 'number' && Number.isFinite(shared) && shared >= 1) {
+        return clampLimit(shared, args.floor, args.ceiling);
+    }
+    return clampLimit(args.initial, args.floor, args.ceiling);
+}
+
+// ── decay (5.3) ──────────────────────────────────────────────────────
+
+export interface DecayStepArgs {
+    /** Injected clock (epoch ms). */
+    now: number;
+    /** `concurrency_decay_after_ms`. */
+    decayAfterMs: number;
+    /** Step target — the provider's `initial`. */
+    initial: number;
+    /** Lower bound; the step MUST NOT cross it. */
+    floor: number;
+}
+
+export interface DecayStepResult {
+    /** The advanced state; identical to the input (same reference) when nothing moved. */
+    state: ProviderConcurrencyState;
+    /** `true` when this call consumed one decay period and moved `L` by one. */
+    moved: boolean;
+}
+
+/**
+ * One decay step for a provider that has gone `decayAfterMs` without an
+ * observation: move `L` one notch toward `initial`, never below `floor`, and
+ * consume one period so the cadence does not drift (the anchor advances by
+ * exactly `decayAfterMs`, keeping a step per elapsed period).
+ *
+ * `L <= initial` (already at or below the target) and `L === floor` do not
+ * move. A non-finite/non-positive `decayAfterMs` disables decay entirely rather
+ * than decaying on every render tick — the config layer already falls back to
+ * the `concurrency_decay_after_ms` default, so a garbage value here must not be
+ * able to drive the loop.
+ *
+ * Pure: the input state is never mutated, and an unmoved result is returned
+ * by reference (no allocation on the common path).
+ */
+export function applyDecayStep(state: ProviderConcurrencyState, args: DecayStepArgs): DecayStepResult {
+    const decayAfter = args.decayAfterMs;
+    if (!Number.isFinite(decayAfter) || decayAfter <= 0) return { state, moved: false };
+    const floor = Math.max(1, toInt(args.floor, 1));
+    const anchor = Number.isFinite(state.lastObservationAt) ? state.lastObservationAt : 0;
+    if (args.now - anchor < decayAfter) return { state, moved: false };
+
+    const target = Math.max(floor, toInt(args.initial, 1));
+    if (!Number.isFinite(state.limit) || state.limit <= target) return { state, moved: false };
+
+    return {
+        state: { ...state, limit: Math.max(floor, state.limit - 1), lastObservationAt: anchor + decayAfter },
+        moved: true,
+    };
+}
+
+// ── writing (5.1) ────────────────────────────────────────────────────
+
+export interface SaveStateArgs {
+    /** Absolute path of the state file (config root + {@link STATE_FILE_NAME}). */
+    statePath: string;
+    /** Full per-provider state to persist. */
+    state: ConcurrencyStateFile;
+    fileSystem?: WriterFileSystemPort;
+    warn?: Warn;
+}
+
+/** Entry content minus `updatedAt` — the basis for 「这一轮没有变化」. */
+function samePersistedContent(a: PersistedProviderState, b: PersistedProviderState): boolean {
+    return a.limit === b.limit
+        && a.initial === b.initial
+        && a.lastObservationAt === b.lastObservationAt
+        && a.badStreak === b.badStreak
+        && a.cleanStreak === b.cleanStreak
+        && a.ewmaGenMs === b.ewmaGenMs;
+}
+
+/**
+ * Snapshot the live control-loop states into the durable file body.
+ *
+ * This is the bridge the controller's persist step uses: the field names differ
+ * (`cleanStreak` on disk vs `cleanWindows` in the loop) and the live state
+ * carries runtime-only members (`cooldownUntil`, the partial clean-window
+ * accumulator) that are deliberately not persisted — a restart restarts the
+ * current clean window (conservative) while the counted streak and the latency
+ * baseline resume.
+ *
+ * `previous` is the body handed to the last save. An entry whose content
+ * (everything but the timestamp) is unchanged keeps `previous[id].updatedAt`,
+ * so `updatedAt` advances only when the entry really changed — without that,
+ * every evaluation window would re-stamp all entries and defeat the no-op
+ * write guard in {@link saveConcurrencyState}.
+ */
+export function toStateFile(
+    states: Iterable<readonly [string, ProviderConcurrencyState]>,
+    updatedAt: number,
+    previous?: ConcurrencyStateFile | null,
+): ConcurrencyStateFile {
+    const out: ConcurrencyStateFile = {};
+    for (const [provider, state] of states) {
+        const id = canonicalizeProvider(provider);
+        if (!id) continue;
+        const entry: PersistedProviderState = {
+            limit: state.limit,
+            initial: state.initial,
+            updatedAt,
+            lastObservationAt: state.lastObservationAt,
+            badStreak: state.badStreak,
+            cleanStreak: state.cleanWindows,
+            ewmaGenMs: state.ewmaGenMs,
+        };
+        const prior = previous?.[id];
+        if (prior && samePersistedContent(prior, entry)) entry.updatedAt = prior.updatedAt;
+        out[id] = entry;
+    }
+    return out;
+}
+
+/** Serialize the state file body. The file-level `updatedAt` tracks the newest entry. */
+function serializeStateFile(state: ConcurrencyStateFile, warn: Warn): string {
+    const providers = canonicalEntries(Object.entries(state ?? {}), warn);
+    let updatedAt = 0;
+    for (const entry of Object.values(providers)) updatedAt = Math.max(updatedAt, entry.updatedAt);
+    return `${JSON.stringify({ version: STATE_FILE_VERSION, updatedAt, providers }, null, 2)}\n`;
+}
+
+/**
+ * Atomically persist the learned state (`tmp` + `rename`) — task 5.1.
+ *
+ * Fire-and-forget: the returned promise exists so a shutdown path can flush,
+ * but the render tick MUST NOT await it. Failures only warn — the previous file
+ * stays in place, so a restart keeps the last finite learned values.
+ *
+ * A window that changed nothing writes nothing: the file is only rewritten when
+ * its bytes would differ. The comparison is against the file itself (not a
+ * cached「上次写完的内容」), so a failed write is retried on the next window and
+ * a peer writer that replaced the file is still corrected. `toStateFile` keeps
+ * `updatedAt` stable for unchanged entries, which is what makes two no-op
+ * windows serialize identically.
+ */
+export async function saveConcurrencyState(args: SaveStateArgs): Promise<void> {
+    const fileSystem = args.fileSystem ?? nodeWriterFileSystem;
+    const warn = args.warn ?? NO_WARN;
+    const tmp = `${args.statePath}.tmp-${process.pid}-${stateSeq++}`;
+    try {
+        const text = serializeStateFile(args.state, warn);
+        if (fileSystem.readFile(args.statePath) === text) return;
+        fileSystem.ensureDir(dirname(args.statePath));
+        fileSystem.writeFile(tmp, text);
+        fileSystem.rename(tmp, args.statePath);
+    } catch (err) {
+        warn(`${PREFIX} state file ${args.statePath} update failed (${message(err)}); keeping the previous state`);
+        try {
+            fileSystem.remove(tmp);
+        } catch { /* the failed write left nothing to clean up */ }
+    }
+}
+
+// ── shutdown fail-safe (5.4) ─────────────────────────────────────────
+
+export interface StopFailSafeArgs {
+    /** Absolute path of the state file (config root + {@link STATE_FILE_NAME}). */
+    statePath: string;
+    fileSystem?: WriterFileSystemPort;
+    warn?: Warn;
+}
+
+/**
+ * Shutdown guard for tuner stop / plugin unload / process exit — task 5.4.
+ *
+ * Deliberately inert. The fail-safe is the *absence* of a cleanup write: this
+ * module never touches `providers.maxInFlightRequests`, so the last successful
+ * finite value survives a stop, and no managed key is ever deleted. The state
+ * file is likewise never removed nor overwritten with an empty body. All this
+ * does is surface a degradation warn when the file a restart would read is
+ * missing or unusable, so a silent loss of the learned values is observable.
+ */
+export async function stopFailSafe(args: StopFailSafeArgs): Promise<void> {
+    const fileSystem = args.fileSystem ?? nodeWriterFileSystem;
+    const warn = args.warn ?? NO_WARN;
+    try {
+        const text = fileSystem.readFile(args.statePath);
+        if (text === null) {
+            warn(
+                `${PREFIX} state file ${args.statePath} is missing at shutdown; the last shared config values are kept and no keys are removed`,
+            );
+            return;
+        }
+        if (Object.keys(parseStateFile(text, args.statePath, warn)).length === 0) {
+            warn(
+                `${PREFIX} state file ${args.statePath} holds no usable learned limits at shutdown; the last shared config values are kept`,
+            );
+        }
+    } catch (err) {
+        warn(`${PREFIX} shutdown state check failed (${message(err)}); the last shared config values are kept`);
+    }
+}