@camstack/types 1.2.86 → 1.2.88

package/dist/utils/pool-memory-watchdog.d.ts

@@ -0,0 +1,178 @@
+/**
+ * Pool memory watchdog — bounds the RSS of the long-lived Python inference
+ * subprocesses (detection `inference_pool.py`, audio `yamnet_audio.py`,
+ * post-analysis `raw_tensor_inference.py` / `text_encoder_inference.py`)
+ * BEFORE the kernel's OOM killer does, and emits the periodic per-pool
+ * memory line that makes an hours-long leak legible after the fact.
+ *
+ * Why this exists (2026-08-18 audit): the pools leak over hours — measured
+ * 97 / 78 / 171 / 689 MB with the 689 MB one climbing ~7 MB per 30 min —
+ * then get OOM-killed (anon-rss 3.8 GB inside the CamStack cgroup at 11:45,
+ * 8.8 GB host-wide at 17:02, same day), respawn at ~100 MB and repeat. The
+ * host is 62 GiB with NO swap and the pressure killed the co-hosted Home
+ * Assistant VM twice. The kernel was the only bound; this module makes the
+ * bound ours, and makes every step of the ramp visible in Loki.
+ *
+ * Placement: this lives in `@camstack/types` because BOTH `addon-pipeline`
+ * and `addon-post-analysis` need it, addons must not import each other, and
+ * self-contained addon bundles INLINE the bare `@camstack/types` entry at
+ * build time (see tools/build/vite-lib.preset.ts) — so the code ships with a
+ * plain `camstack deploy` of each addon, no framework train required. It is
+ * deliberately free of Node imports (`fs`, `child_process`): the pure policy
+ * below is also consumed through the types index that UI builds import.
+ * The `/proc` READ lives in each addon; the parse lives here.
+ */
+import type { IScopedLogger } from '../interfaces/logging.js';
+export interface PoolMemoryPolicy {
+    /** Sweep cadence. 60 s makes a 6-hour ramp 360 points (legible) at ~4
+     *  lines/min across all pools (no flood); the addon log ring only holds
+     *  ~10 min, but these lines are shipped to Loki which holds days. */
+    readonly sampleIntervalMs: number;
+    /** Samples IGNORED after a pool (re)appears before baseline capture
+     *  starts. Model loads land in the first minutes and inflate RSS —
+     *  a baseline taken at spawn would be the empty interpreter (~40 MB)
+     *  and every loaded pool would instantly look leaky. */
+    readonly baselineSettleSamples: number;
+    /** Post-settle samples whose MEDIAN becomes the baseline. Median, not
+     *  mean: one inference burst mid-window must not drag the baseline up. */
+    readonly baselineSampleCount: number;
+    /** Restart when RSS exceeds `restartMultiple × baseline`. Relative, not
+     *  absolute: an absolute MB number ages badly as models change, while
+     *  "4× what this pool settled at with its own models loaded" survives a
+     *  model swap unchanged. */
+    readonly restartMultiple: number;
+    /** Never restart below this RSS regardless of the multiple. The pools
+     *  settle at 78–171 MB today; without a floor, 4× a 78 MB baseline
+     *  (312 MB) would flap on legitimate working-set growth (a second model
+     *  loaded, a larger frame cache). Below the floor the pool is not a
+     *  host-level threat on a 62 GiB machine. */
+    readonly floorBytes: number;
+    /** Restart at this RSS even when the baseline is unknown or huge. This is
+     *  the backstop for (a) a pool that ramps before its baseline exists and
+     *  (b) a baseline poisoned by an already-leaking pool at guard start.
+     *  3 GiB sits well under the observed kill sizes (3.8 / 8.8 GB anon-rss)
+     *  and far above any legitimate settled pool observed on this fleet. */
+    readonly ceilingBytes: number;
+    /** Minimum gap between watchdog restarts of ONE pool. A pool that
+     *  re-crosses within minutes of a restart is leaking faster than a
+     *  restart can pay for — restarting it in a tight loop would trade an
+     *  OOM for a detection outage. */
+    readonly cooldownMs: number;
+    /** Watchdog restarts allowed per pool per `restartWindowMs`; past this
+     *  the watchdog STOPS restarting and escalates (error log). Mirrors the
+     *  CrashSupervisor circuit-breaker rule (D6) at pool granularity: never
+     *  an unbounded restarter. */
+    readonly maxRestartsPerWindow: number;
+    /** Window for `maxRestartsPerWindow`. */
+    readonly restartWindowMs: number;
+}
+export declare const DEFAULT_POOL_MEMORY_POLICY: PoolMemoryPolicy;
+/**
+ * Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
+ * absent values keep the default — an operator typo must never disable the
+ * bound or set it to zero.
+ */
+export declare function resolvePoolMemoryPolicy(env: Readonly<Record<string, string | undefined>>): PoolMemoryPolicy;
+export interface ProcMemory {
+    readonly rssBytes: number;
+    /** Virtual size. The 2026-08-18 kills showed total-vm 47.5 GB vs 11.3 GB at
+     *  similar RSS — tracking VmSize alongside Threads is the cheap lead on
+     *  "allocator arenas vs thread stacks" the audit flagged. */
+    readonly vmBytes: number;
+    /** Peak RSS (VmHWM) — the `resource.getrusage` maxrss equivalent, readable
+     *  from OUTSIDE the process, so it covers the pools whose wire protocol has
+     *  no command channel (yamnet / raw-tensor). */
+    readonly hwmBytes: number;
+    readonly swapBytes: number;
+    readonly threads: number;
+}
+/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
+ *  Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
+export declare function parseProcStatus(text: string): ProcMemory | null;
+export interface PoolMemoryState {
+    readonly settleSeen: number;
+    readonly baselineWindow: readonly number[];
+    readonly baselineBytes: number | null;
+    /** Timestamps of WATCHDOG restarts (never kernel kills — those reset the
+     *  baseline via the pid-change path but must not eat the budget). */
+    readonly restartsAt: readonly number[];
+    readonly lastRestartAt: number | null;
+}
+export declare function initialPoolMemoryState(): PoolMemoryState;
+export type PoolMemoryAction = 'ok' | 'baseline-pending' | 'restart' | 'cooldown' | 'exhausted';
+export interface PoolMemoryVerdict {
+    readonly state: PoolMemoryState;
+    readonly action: PoolMemoryAction;
+    readonly baselineBytes: number | null;
+    readonly thresholdBytes: number;
+}
+/** The one place a pool's restart threshold is computed. */
+export declare function poolMemoryThreshold(baselineBytes: number | null, policy: PoolMemoryPolicy): number;
+/**
+ * Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
+ * The caller performs (and logs) the restart, then commits it via
+ * {@link commitWatchdogRestart}.
+ */
+export declare function evaluatePoolMemory(state: PoolMemoryState, rssBytes: number, nowMs: number, policy: PoolMemoryPolicy): PoolMemoryVerdict;
+/** Record a performed watchdog restart: budget consumed, baseline reset —
+ *  the recreated pool is a NEW process and gets a fresh settle window. */
+export declare function commitWatchdogRestart(state: PoolMemoryState, nowMs: number): PoolMemoryState;
+/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
+ *  kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
+ *  restart budget is kept: it bounds the WATCHDOG, not the pool. */
+export declare function resetPoolBaseline(state: PoolMemoryState): PoolMemoryState;
+export interface RestartCandidate {
+    readonly key: string;
+    readonly rssBytes: number;
+    readonly thresholdBytes: number;
+}
+/**
+ * Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
+ * several pools at once (e.g. after a model rollout grew every baseline)
+ * would take detection down on every camera simultaneously; one per sweep
+ * means the survivors keep serving while the worst offender reloads, and the
+ * next sweep (one interval later) takes the next one.
+ */
+export declare function pickRestartCandidate(candidates: readonly RestartCandidate[]): string | null;
+/** One pool's sample for a sweep. `meta` carries the owner's cheap internals
+ *  (backlog, shed counts, models loaded, Python-side mem stats) verbatim into
+ *  the periodic `pool memory` line. */
+export interface PoolMemoryTelemetry {
+    readonly key: string;
+    readonly pids: readonly number[];
+    /** Max worker RSS for the pool — the kernel kills a PROCESS, so the bound
+     *  tracks the biggest worker, not the pool sum. */
+    readonly rssBytes: number;
+    readonly meta?: Readonly<Record<string, unknown>>;
+}
+export interface PoolMemoryWatchdogOptions {
+    readonly policy: PoolMemoryPolicy;
+    /** Enumerate the live pools with their current RSS + internals. */
+    readonly sample: () => Promise<readonly PoolMemoryTelemetry[]>;
+    /** Dispose-and-recreate the named pool via the owner's EXISTING lifecycle
+     *  machinery (idle-reaper dispose path, lazy re-init) — the watchdog is a
+     *  trigger, never a second process supervisor. */
+    readonly restart: (key: string) => Promise<void>;
+    readonly log: IScopedLogger;
+    readonly now?: () => number;
+    readonly setTimer?: (fn: () => void, ms: number) => ReturnType<typeof setTimeout>;
+    readonly clearTimer?: (timer: ReturnType<typeof setTimeout>) => void;
+}
+export declare class PoolMemoryWatchdog {
+    private readonly pools;
+    private timer;
+    private sweeping;
+    private stopped;
+    private readonly opts;
+    private readonly now;
+    private readonly setTimer;
+    private readonly clearTimer;
+    constructor(opts: PoolMemoryWatchdogOptions);
+    start(): void;
+    stop(): void;
+    private arm;
+    /** One sweep: sample → log every pool → restart at most one. Public so the
+     *  loop is testable without fake global timers. */
+    sweep(): Promise<void>;
+    private doSweep;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/types",
-  "version": "1.2.86",
+  "version": "1.2.88",
   "description": "Shared types, interfaces, and model catalogs for the CamStack detection ecosystem",
   "keywords": [
     "camstack",