@camstack/system 1.2.102 → 1.2.104

package/dist/kernel/config-schema.d.ts

@@ -85,18 +85,22 @@ export type AppConfig = BootstrapConfig & {
          */
         perAddonLogBufferSize?: number;
         /**
-         * SOFT ceiling on retained entries across every addon bucket. `null` (the
-         * default) means the per-addon rings are the only bound — the historical
-         * behaviour, unchanged unless you set this.
+         * Ceiling on retained entries across every addon bucket.
          *
-         * Set it when `perAddonLogBufferSize × addons` is more than the box can spare:
-         * 70 addons × 5000 is 350k retained entries, which is a lot on a mini PC.
+         * UNSET (the normal case) takes `DEFAULT_MAX_TOTAL_LOG_ENTRIES` — the buffer is
+         * bounded in aggregate out of the box. It has to be: `perAddonLogBufferSize ×
+         * addons` grows with the roster, and hub-main ingests from every local runner
+         * AND every agent, so the measured ~60-bucket hub retained ~300k entries
+         * (~92MB at ~306 B/entry) with no aggregate bound at all.
          *
-         * Soft, and deliberately so: it is met by discarding only entries at or below
-         * `logBufferPruneLevel`. If pruning every prunable entry still leaves the total
-         * above this number, the buffer stays above it. A cap that could evict an
-         * `error` would trade the lines that explain an incident for lines about frame
-         * timings.
+         * Set a number to tune it. Set an explicit `null` to opt out entirely — the
+         * per-addon rings then become the only bound again.
+         *
+         * It is met by discarding entries at or below `logBufferPruneLevel` FIRST, so a
+         * live addon's `warn`/`error` are never traded for lines about frame timings.
+         * Only when that cannot reach the target are whole least-recently-WRITTEN
+         * buckets dropped — a bound that pruning alone cannot hold is not a bound, and
+         * the bucket nobody has written to is the cheapest thing in the buffer to lose.
          */
         maxTotalLogBufferSize?: number | null;
         /**

package/dist/kernel/heap-watch.d.ts

@@ -59,14 +59,46 @@ export declare function nextMode(current: HeapWatchMode, usedRatio: number, esca
  * Live calibration, both readings from the same hub minutes apart: 2491MB
  * stranded before the reclaim, 438MB after. 438MB is this process's irreducible
  * native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
- * collecting against it would buy nothing and cost a stall every minute. 1536
- * sits clear of that floor and still fires ~1.1GB before the ratchet is worth
- * anything, so it triggers on the real condition and stops as soon as it is fixed.
+ * collecting against it would buy nothing and cost a stall every minute.
+ *
+ * ## Why this came down from 1536 (2026-08-17)
+ *
+ * The trigger was originally set to fire only on an unmistakable strand. It
+ * does, and the pass on the record works exactly as designed:
+ *
+ *     [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
+ *
+ * But the number it was protecting against changed. The host now runs with a
+ * 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
+ * RSS this process is holding and not using is no longer merely untidy — it is
+ * the headroom something else needed, and the OOM is triggered by the PEAK, not
+ * by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
+ * above the floor before anything happens.
+ *
+ * 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
+ * can never trip it, and every pass that does fire still returns roughly
+ * `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
+ * not a stall spent collecting noise.
  */
-export declare const HEAP_RECLAIM_TRIGGER_MB = 1536;
-/** Floor on how often the stop-the-world may run. 250-450ms measured, so ten
- *  minutes keeps the worst case under 0.1% of the router's time. */
-export declare const HEAP_RECLAIM_MIN_INTERVAL_MS = 600000;
+export declare const HEAP_RECLAIM_TRIGGER_MB = 1024;
+/**
+ * Floor on how often the stop-the-world may run.
+ *
+ * Ten minutes was chosen to keep the worst case under 0.1% of the router's
+ * time. The worst case is not what governs the cost, though: the reclaim only
+ * runs when {@link shouldReclaim} says the process is genuinely stranded, and
+ * after a pass it is at the ~438MB floor. Live off-heap growth was measured at
+ * roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
+ * takes on the order of **20 minutes** — well outside any floor considered
+ * here. In steady state this constant therefore costs nothing at all.
+ *
+ * What it does govern is a BURST, which is the only case that matters on a host
+ * with no swap: the ten-minute floor let a peak that re-formed in seconds stand
+ * for ten minutes. Two minutes bounds that, and even the pathological case of a
+ * process that re-strands instantly costs 654ms per 120s — 0.55% of the
+ * router's time, against a ~1.5GiB peak it removes each pass.
+ */
+export declare const HEAP_RECLAIM_MIN_INTERVAL_MS = 120000;
 export interface HeapSample {
     readonly rssMb: number;
     readonly heapUsedMb: number;
@@ -136,3 +168,70 @@ export interface HeapReclaimOptions {
  * a diagnostic that can crash the process it watches is worse than none.
  */
 export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions): () => void;
+/**
+ * Steady heartbeat period for a HEAVY addon runner.
+ *
+ * hub-main logs once a minute because it is ONE process. The heavy runners are
+ * ~7 per node sharing the same hub log stream; at 60s each they would put 7
+ * routine lines a minute in it forever, which is how a log stops being read.
+ * Five minutes keeps the per-runner record (a leak is still a visible climb
+ * across samples) at ~1.4 lines/min across the class.
+ *
+ * This slows LOGGING only. The probe keeps running at
+ * {@link HEAP_WATCH_FAST_INTERVAL_MS} (probe and log cadences are separate —
+ * see the comment inside {@link startHeapWatch}), so the reclaim gate and the
+ * escalated burst-sampling react exactly as fast as hub-main's.
+ */
+export declare const RUNNER_HEAP_WATCH_INTERVAL_MS = 300000;
+/** Wiring for {@link startRunnerHeapWatch}. */
+export interface RunnerHeapWatchOptions {
+    /**
+     * `CAMSTACK_RUNNER_HEAP_PROFILE` as the parent passed it. Anything but
+     * `'heavy'` (including unset — the un-annotated default is light) starts
+     * nothing: light runners are small, already carry V8 heap flags, and a
+     * quiet runner must never pay a stop-the-world.
+     */
+    readonly heapProfile: string | undefined;
+    /** Log label — the runner's nodeId, so `[mem]` lines group per runner. */
+    readonly label: string;
+    readonly sink: HeapWatchSink;
+    /**
+     * `CAMSTACK_RUNNER_HEAP_RECLAIM` as the environment carries it. `'off'`
+     * keeps the heartbeat and disables only the reclaim pass — the operator
+     * kill-switch for the media path, usable without a rebuild.
+     */
+    readonly reclaimSwitch?: string;
+    /** Heartbeat period override. Defaults to {@link RUNNER_HEAP_WATCH_INTERVAL_MS}. */
+    readonly intervalMs?: number;
+    /** Test injection. Production omits it and gets {@link createV8Reclaimer}. */
+    readonly reclaimOptions?: HeapReclaimOptions;
+}
+/**
+ * Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
+ * is a heavy one. Returns the stop function, or undefined when nothing started.
+ *
+ * ## Why heavy runners get hub-main's mechanism
+ *
+ * The media-path runners deliberately run with NO old-space ceiling
+ * (`maxOldSpaceMb: 0` — a flat cap turned a disk-stall backlog into
+ * `Ineffective mark-compacts near heap limit` and killed recording, twice on
+ * 2026-08-17). Uncapped, V8 feels no pressure, so a one-off burst becomes the
+ * runner's permanent RSS: the recorder measured 2122MB holding a ~50MB index.
+ * hub-main had exactly this shape and its reclaimer is proven live — 11 passes
+ * in 4h, 375–1652MB returned per pass at 550–680ms each. Same mechanism, same
+ * `shouldReclaim` stranded-over-trigger gate; never a second implementation.
+ *
+ * ## The stall, priced for the runners that get it
+ *
+ * A pass is a sub-second stop-the-world at most once per
+ * {@link HEAP_RECLAIM_MIN_INTERVAL_MS}, and only while >1GiB is stranded. For
+ * the recorder — finalizing a segment every ~300ms across 34 profile-writers —
+ * the pause delays WATCHER work, not video: ffmpeg children keep writing to
+ * disk independently, the playlist tail is the durable queue, and the next
+ * tick drains what accrued. For stream-broker a pass can hitch live frames
+ * once per qualifying window; after a pass the runner sits at its floor and
+ * re-stranding the trigger takes tens of minutes, so in practice the class
+ * pays a few passes per hour against the multi-GiB peak that was OOM-killing
+ * a neighbouring VM on a host with no swap.
+ */
+export declare function startRunnerHeapWatch(options: RunnerHeapWatchOptions): (() => void) | undefined;

package/dist/kernel/moleculer/process-service.d.ts

@@ -5,6 +5,40 @@ interface RunnerAddonSpec {
     readonly addonId: string;
     readonly addonDir: string;
 }
+/**
+ * Default old-space ceiling for a HEAVY runner, in MB.
+ *
+ * ## Why a heavy runner needs a ceiling at all
+ *
+ * `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
+ * host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
+ * not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
+ * one-off burst becomes the runner's permanent RSS.
+ *
+ * Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
+ * there after the deferred archive walk (99,809 segments in 84s), while the `RecordingIndex` it
+ * actually retains is arithmetically ~21 MB (a `SegmentRow` is 5 numbers + 3 strings ≈ 207 B). The
+ * same addon on two agent nodes carrying the same code and a smaller archive sits at **58 MB**.
+ * The ~2 GB is walk garbage held as high-water in a heap that had no reason to collect it.
+ *
+ * ## Why 1024
+ *
+ * From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
+ * over-estimate of its old space, since it also contains native allocations, `external` and
+ * `arrayBuffers`, none of which `--max-old-space-size` bounds:
+ *
+ *   detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
+ *   snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
+ *
+ * 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
+ * more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
+ * as it does now, because V8 only collects harder as it approaches the limit.
+ *
+ * A runner whose working set genuinely does not fit declares its own number in its manifest
+ * (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
+ * `stream-broker` (1323 MB measured, frame buffers) is the one shipped addon that does.
+ */
+export declare const HEAVY_MAX_OLD_MB_DEFAULT = 1024;
 interface SpawnedProcess {
     readonly pid: number;
     readonly nodeId: string;

package/dist/logging/log-manager.d.ts

@@ -54,11 +54,13 @@ export declare class LogManager {
      * `perAddonCapacity` bounds EACH addon's bucket, not the total — a chatty addon
      * evicts only its own lines, so quiet addons keep their sparse history.
      *
-     * `options.maxTotalEntries` adds a soft ceiling across all buckets, because the
-     * per-bucket bound scales with addon count. When it is exceeded, only entries at
-     * or below `options.pruneLevel` (default `debug`) are discarded — `warn` and
-     * `error` are never evicted to satisfy it. Both default to the previous
-     * behaviour: no total cap. See {@link PartitionedLogBuffer}.
+     * `options.maxTotalEntries` ceilings the total across all buckets, because the
+     * per-bucket bound scales with addon count, and `options.maxBuckets` ceilings the
+     * bucket count itself. When the total is exceeded, entries at or below
+     * `options.pruneLevel` (default `debug`) are discarded first — a live addon's
+     * `warn`/`error` are never evicted to satisfy it — and only if that cannot reach
+     * the target are whole least-recently-written buckets dropped. Both bounds are ON
+     * by default; pass an explicit `null` to opt out. See {@link PartitionedLogBuffer}.
      */
     constructor(perAddonCapacity?: number, options?: PartitionedLogBufferOptions);
     /** Entries currently retained across every bucket. */

package/dist/logging/partitioned-log-buffer.d.ts

@@ -1,14 +1,52 @@
 import { LogEntry, LogFilter, LogLevel } from '@camstack/types';
+/**
+ * Default ceiling on retained entries across every bucket.
+ *
+ * This used to be `null` — no aggregate bound at all — and the only thing
+ * holding hub-main's log memory was `perAddonCapacity x buckets`, a product
+ * that grows with the roster. Measured on the live cluster: 37 forked runners
+ * on the hub plus its in-process builtins, and hub-main additionally ingests
+ * every line from both agents, so ~60 buckets is the ordinary case and
+ * 60 x 5000 = 300k retained entries.
+ *
+ * A realistic entry (timestamp, level, scope, ~70-char message, three tags, no
+ * `meta`) measures **~306 B retained** in V8. So the old unbounded case is
+ * ~92MB of a hub process that is being asked to fit inside a cgroup with a
+ * neighbour to protect, and 50_000 is **~15MB**.
+ *
+ * 50_000 across a ~60-bucket roster is ~830 entries per addon on average while
+ * still letting a single chatty addon hold its full 5000-entry ring — the
+ * partitioning is not weakened, only the total is held.
+ */
+export declare const DEFAULT_MAX_TOTAL_LOG_ENTRIES = 50000;
+/**
+ * Default ceiling on the NUMBER of buckets.
+ *
+ * Buckets were created on first write and never removed, so every addon id
+ * that ever logged — a removed addon, a renamed one, a one-off worker id —
+ * kept a ring alive for the life of the process. {@link DEFAULT_MAX_TOTAL_LOG_ENTRIES}
+ * alone does not fix that: the total sweep may only discard entries at or below
+ * `pruneLevel`, so a dead addon's 5000 `info` lines are untouchable by it.
+ *
+ * 128 is ~2x the ~60-bucket live roster, so no real deployment ever evicts on
+ * bucket count alone; it exists to make an unbounded id space impossible.
+ */
+export declare const DEFAULT_MAX_LOG_BUCKETS = 128;
 export interface PartitionedLogBufferOptions {
     /** Hard ring size for EACH addon's bucket. */
     readonly perAddonCapacity?: number;
     /**
-     * Soft ceiling on entries across all buckets. `null` (the default) means the
-     * only bound is the per-addon rings.
+     * Soft ceiling on entries across all buckets. `null` means the only bound is
+     * the per-addon rings. Defaults to {@link DEFAULT_MAX_TOTAL_LOG_ENTRIES}.
      */
     readonly maxTotalEntries?: number | null;
     /** Highest severity the total-cap sweep may discard. Default `debug`. */
     readonly pruneLevel?: LogLevel;
+    /**
+     * Hard ceiling on the number of buckets. `null` means unbounded. Defaults to
+     * {@link DEFAULT_MAX_LOG_BUCKETS}.
+     */
+    readonly maxBuckets?: number | null;
 }
 /**
  * A log buffer partitioned into one fixed-capacity {@link LogRingBuffer} per
@@ -19,32 +57,48 @@ export interface PartitionedLogBufferOptions {
  * Drop-in for {@link LogRingBuffer} from {@link LogManager}'s perspective —
  * exposes the same `push` / `getAll` / `query` / `clear` surface.
  *
- * ## Two bounds, doing different jobs
+ * ## Three bounds, doing different jobs
  *
  * **`perAddonCapacity`** is a HARD bound: each bucket is a fixed-size ring, so
  * total memory can never exceed `perAddonCapacity × buckets` whatever else
  * happens. This is the safety net.
  *
- * **`maxTotalEntries`** is a SOFT target across all buckets, and it exists because
- * the hard bound scales with addon count — 70 addons × 5000 is a lot of retained
- * entries on a small box. When the total is exceeded, the sweep discards only
- * entries at or below `pruneLevel` (default `debug`), oldest first, from the
+ * **`maxTotalEntries`** targets the total across all buckets, and it exists because
+ * the hard bound scales with addon count — 60 buckets × 5000 was 300k retained
+ * entries on the live hub. When the total is exceeded, the sweep first discards
+ * only entries at or below `pruneLevel` (default `debug`), oldest first, from the
  * largest buckets first.
  *
- * It is deliberately soft. Meeting a total cap by discarding `warn` and `error`
- * would trade the entries that explain an incident for entries about frame
- * timings; if pruning every prunable entry still leaves the total above the cap,
- * the buffer stays above it and the hard per-bucket bound is what holds. A cap
- * that can evict an error is not a feature.
+ * It is deliberately soft ON A LIVE BUCKET. Meeting a total cap by discarding
+ * `warn` and `error` from an addon that is still logging would trade the entries
+ * that explain an incident for entries about frame timings. A cap that can evict
+ * a live addon's error is not a feature.
  *
  * Largest-bucket-first because the noisy addon is the one creating the pressure —
  * the same principle that motivated partitioning in the first place.
+ *
+ * **`maxBuckets`, and bucket eviction, are the hard backstop.** Pruning alone
+ * cannot bound this structure: it may only discard at or below `pruneLevel`, so
+ * a bucket full of `info` from an addon that no longer exists was immortal, and
+ * the roster the product scales with is not fixed. When pruning cannot reach the
+ * target — or the bucket count exceeds `maxBuckets` — whole buckets are dropped
+ * **least-recently-WRITTEN first**, and never the bucket currently being written
+ * to. That is precisely the dead addon's bucket, and it is precisely not the
+ * bucket of the addon whose incident is in progress.
  */
 export declare class PartitionedLogBuffer {
     private readonly buffers;
+    /**
+     * Bucket key → the write ordinal of its last `push`. A monotonic counter, not
+     * a clock: eviction order must follow the order writes actually happened, and
+     * a wall clock that steps backwards would pick the wrong victim.
+     */
+    private readonly lastWriteSeq;
+    private writeSeq;
     private perAddonCapacity;
     private maxTotalEntries;
     private pruneLevel;
+    private maxBuckets;
     /** Running total, so `push` does not walk every bucket to decide. */
     private totalEntries;
     constructor(perAddonCapacity?: number, options?: PartitionedLogBufferOptions);
@@ -59,18 +113,40 @@ export declare class PartitionedLogBuffer {
         readonly perAddonCapacity?: number;
         readonly maxTotalEntries?: number | null;
         readonly pruneLevel?: LogLevel;
+        readonly maxBuckets?: number | null;
     }): void;
     private bufferFor;
     push(entry: LogEntry): void;
     /**
-     * Bring the total down to `PRUNE_TARGET_RATIO × cap` by discarding prunable
-     * entries, largest bucket first. Stops early when nothing prunable is left —
-     * see the class docblock on why that is the correct outcome rather than a
-     * failure to enforce.
+     * Bring the total down to `PRUNE_TARGET_RATIO × cap`.
+     *
+     * Two passes, in this order and no other: first discard prunable entries,
+     * largest bucket first, which is free of information cost. Only if that
+     * cannot reach the target does the second pass drop WHOLE buckets,
+     * least-recently-written first — the bound has to hold even when every
+     * retained entry is above `pruneLevel`, and a bucket nobody has written to is
+     * the cheapest thing in the structure to lose.
      */
     private enforceTotalCap;
+    /**
+     * Drop whole buckets, least-recently-WRITTEN first, until the total is at or
+     * below `target`. Never touches `protectedKey` (the bucket being written to
+     * right now) and always leaves at least one bucket standing.
+     */
+    private evictBucketsUntil;
+    /** Drop exactly `count` buckets, least-recently-written first. */
+    private evictBuckets;
+    /** Bucket keys ordered least-recently-written first, excluding `protectedKey`.
+     *  A bucket with no recorded write sorts first — it can only be a leftover. */
+    private evictionOrder;
+    /** Remove a bucket and everything that indexes it. An empty ring left behind
+     *  is the leak this class shipped with — the map only ever grew. */
+    private dropBucket;
     /** Entries retained across every bucket. */
     size(): number;
+    /** Buckets currently retained. Observability for the aggregate bound — an
+     *  addon roster that only ever grows is what this class had to stop doing. */
+    bucketCount(): number;
     /** Every retained entry across all buckets, newest-first (mirrors
      *  {@link LogRingBuffer.getAll}). Used to replay history to a destination. */
     getAll(): LogEntry[];
@@ -83,6 +159,9 @@ export declare class PartitionedLogBuffer {
      */
     query(filter: LogFilter): LogEntry[];
     clear(filter?: LogFilter): number;
+    /** An emptied bucket is removed rather than left as an empty ring: the map
+     *  used to grow with every addon id that ever logged and never shrink. */
+    private dropIfEmpty;
     /** The bucket key a filter pins to via `tags.addonId`, or null when the
      *  filter doesn't constrain the addon (→ scan all buckets). */
     private pinnedAddonKey;