@camstack/system 1.2.151 → 1.2.154

package/dist/kernel/heap-watch.d.ts

@@ -1,3 +1,5 @@
+import { EventPlaneReader } from './event-plane-report.js';
+import { HeapSpaceReader } from './heap-spaces.js';
 /**
  * What hub-main calls itself in a `[mem]` line.
  *
@@ -137,12 +139,95 @@ export declare const HEAP_RECLAIM_MIN_INTERVAL_MS = 120000;
  * Six is 12 minutes of back-to-back passes: long enough that a genuine burst
  * (which subsides) never reaches it, short enough that a process which has
  * quietly made the stop-the-world part of its steady state is named within the
- * quarter-hour. The warn changes no behaviour — the pass is load-bearing and
- * keeps running. It exists so the OPERATOR is told that the safety net has
- * become the mechanism, which is the moment to go after the allocation source
- * instead of the symptom.
+ * quarter-hour.
+ *
+ * ## Until 2026-08-29 this counter changed nothing, and that was the bug
+ *
+ * It used to WARN and leave the cadence alone — "the pass is load-bearing and
+ * keeps running". Then hub-main's first 25 minutes of telemetry arrived and
+ * showed what that costs. See {@link HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS}:
+ * the system already knew the difference between a rescue and a steady state
+ * and simply refused to act on it.
  */
 export declare const RECLAIM_STEADY_STATE_PASSES = 6;
+/**
+ * Floor on the pass once it has PROVEN it is steady state, not a rescue.
+ *
+ * ## Why `stranded` cannot discriminate on hub-main
+ *
+ * {@link strandedMb} is `rss − heapUsed − external`: everything the process
+ * holds that those two numbers do not name. That lumps together two things with
+ * opposite fixes —
+ *
+ *  - V8 pages committed at a sawtooth peak, which `gc(last-resort)` RETURNS;
+ *  - glibc arena memory already freed and never handed back, which **no V8
+ *    flavor can reach** (nothing in this repository calls `malloc_trim`).
+ *
+ * {@link HEAP_RECLAIM_TRIGGER_MB} was calibrated against a hub-main whose strand
+ * was mostly the first kind, and a pass then took it back under the trigger. Once
+ * the second kind grows past 1024 MB the quantity can never fall back under it,
+ * and the gate degenerates. Measured on the live hub 2026-08-29, first 25
+ * minutes of hub-main telemetry ever collected:
+ *
+ *  - `stranded` sat at **1031–2313 MB for the entire boot**, i.e.
+ *    {@link shouldReclaim} never once said no. **What is written as a threshold
+ *    is, on this process, an unconditional 120-second timer.**
+ *  - Nine minutes out of nine that contained a reclaim show `loopMax` ≈ the
+ *    reclaim's own duration (555–869 ms). Every minute without one is
+ *    107–435 ms. **The largest recurring event-loop stall on the process that
+ *    mediates every cross-runner call is the memory patch it applies to itself.**
+ *  - Each pass freed less than the last (1193 → 462 MB) while the post-pass RSS
+ *    floor ratcheted **1940 → 3510 MB in fourteen minutes**, with `heapUsed`
+ *    never moving. It is compacting a heap that is not the problem.
+ *
+ * ## The fix is not a bigger trigger
+ *
+ * Raising {@link HEAP_RECLAIM_TRIGGER_MB} would move the number at which the same
+ * degeneration happens, and it would silence `hub/recorder` — whose single pass
+ * in six hours is the only cheap proof anyone has that the recorder's 1.4 GB is
+ * RETAINED (a forced compaction freed 507 MB and V8 still held 1.44 GB live).
+ * The discriminator is not the level of the strand; it is **whether the strand
+ * ever falls back under the trigger between passes**, which is precisely what
+ * {@link RECLAIM_STEADY_STATE_PASSES} already counts and used only to talk about.
+ *
+ * So the counter becomes load-bearing: once the pass has fired at the floor
+ * {@link RECLAIM_STEADY_STATE_PASSES} times running, the trigger has demonstrably
+ * stopped discriminating on this process, and the cadence falls back to a period
+ * chosen for its COST rather than to a threshold pretending to be a gate.
+ *
+ * ## Why 600_000, and why that is not a regression
+ *
+ * It is the value {@link HEAP_RECLAIM_MIN_INTERVAL_MS} held before 2026-08-17,
+ * and the reason it came down is recorded there: *"the ten-minute floor let a
+ * peak that re-formed in seconds stand for ten minutes"*. That argument is about
+ * a BURST — and a burst subsides, which resets the run and restores the 120 s
+ * floor. It never applied to a process in proven steady state. So the burst
+ * bound is untouched and the steady state gets the period the burst argument was
+ * never about.
+ *
+ * ## What each process does after this change
+ *
+ *  - **hub-main**: first six passes at 120 s, unchanged. From the seventh
+ *    consecutive floor-to-floor pass, every 600 s. Stop-the-world cost on the
+ *    router falls from ~0.58% of wall clock to ~0.12%. The ArrayBuffer probe
+ *    survives at six passes an hour — the finding it produced needed ten passes
+ *    total, not ten in twenty-five minutes.
+ *  - **`hub/recorder`**: zero change, by construction. `passesAtFloor` only
+ *    increments when a pass fires within `minIntervalMs * 2` of the previous
+ *    one; the recorder's strand takes six hours to re-form, so the counter is
+ *    reset to 1 on every pass and can never reach six. The 507 MB probe is
+ *    intact.
+ *  - **Anything that recovers**: the run resets the moment the strand falls back
+ *    under the trigger, and the 120 s floor returns with it.
+ */
+export declare const HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS = 600000;
+/**
+ * The floor in force for the NEXT pass, given how many have fired at the floor.
+ *
+ * Pure, so the cadence change is testable without stranding a gigabyte to reach
+ * it — and so the one place that decides it can be read on its own.
+ */
+export declare function reclaimIntervalMs(passesAtFloor: number, minIntervalMs?: number, steadyStateIntervalMs?: number, steadyStatePasses?: number): number;
 /**
  * Event-loop delay over the window since the previous read, in milliseconds.
  *
@@ -181,6 +266,37 @@ export interface LoopDelayMeter {
  * naively would print `loopMax=9223372036854ms` and be read as a catastrophe on
  * every boot. `count === 0` short-circuits to {@link ZERO_LOOP_DELAY}.
  *
+ * ## `loopP50=20ms` was the SAMPLER, not the loop (fixed 2026-08-29)
+ *
+ * `monitorEventLoopDelay` records the whole INTERVAL between its own timer
+ * firings, not the excess over the interval it asked for. So on a loop with
+ * nothing wrong the histogram reads `resolution`, and every `[mem]` line this
+ * process has ever emitted said `loopP50=20ms` — which was read, repeatedly in
+ * one session, as "hub-main is chronically 20 ms behind". It is not; 20 was the
+ * default `resolutionMs` of this very function.
+ *
+ * The delay is `interval − resolution`, and that is not a fudge factor — it is
+ * the only reading the four resolutions agree on. Measured here, one 120 ms
+ * block observed by four histograms at once:
+ *
+ * | resolution | raw p50 | raw max | max − resolution |
+ * | --- | --- | --- | --- |
+ * | 20 ms | 20.40 ms | 137.63 ms | 117.6 ms |
+ * | 10 ms | 10.22 ms | 123.60 ms | 113.6 ms |
+ * | 5 ms | 5.11 ms | 123.60 ms | 118.6 ms |
+ * | 1 ms | 1.27 ms | 121.04 ms | 120.0 ms |
+ *
+ * Raw, the four disagree by 16 ms about the same event and none of them reports
+ * the block. Corrected, they agree within 7 ms and all four name it. The
+ * subtraction costs nothing and needs no faster sampler — a 1 ms resolution
+ * would be 1 000 extra timer wakeups per second on the process that already
+ * serves ~44 000 socket operations per second, to buy 2 ms of accuracy on a
+ * quantity nobody reads below 100 ms.
+ *
+ * **Readings from before this change are 20 ms high.** `loopP99=21–28ms` in the
+ * 2026-08-29 census is really 1–8 ms; `loopMax=2804ms` is really 2784 ms. Do not
+ * compare a pre-fix series with a post-fix one without shifting it.
+ *
  * Returns undefined if the platform declines, so the caller keeps a heartbeat
  * either way.
  */
@@ -396,6 +512,9 @@ export interface HeapReclaimOptions {
     readonly heapLimitBytes?: number;
     readonly triggerMb?: number;
     readonly minIntervalMs?: number;
+    /** Floor once the pass is proven steady state. See
+     *  {@link HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS}. */
+    readonly steadyStateIntervalMs?: number;
     readonly now?: () => number;
     /** Sampling cadence once escalated. See {@link HEAP_WATCH_FAST_INTERVAL_MS}. */
     readonly fastIntervalMs?: number;
@@ -462,13 +581,31 @@ export interface RssBudgetOptions {
     /** See {@link RSS_BUDGET_REANNOUNCE_MIN_MS}. */
     readonly reannounceMinMs?: number;
 }
+/**
+ * The optional per-line probes.
+ *
+ * An OBJECT rather than two more positional parameters: `startHeapWatch` already
+ * takes eight, and a ninth and tenth `undefined` at a call site is how a caller
+ * ends up passing the loop meter into the space reader. Every field is optional,
+ * so a caller that wants neither passes nothing.
+ */
+export interface HeapWatchProbes {
+    /** Per-V8-space breakdown. Defaults to the real reader. */
+    readonly readSpaces?: HeapSpaceReader;
+    /**
+     * UDS + cross-node event fan-out. No default: a forked runner has no child
+     * registry, and printing zeros for a plane that does not exist is worse than
+     * printing nothing. Only hub-main and an agent's main pass one.
+     */
+    readonly eventPlane?: EventPlaneReader;
+}
 /**
  * Start the heartbeat. Returns a stop function.
  *
  * `unref`ed so it can never hold the process open, and every read is wrapped:
  * a diagnostic that can crash the process it watches is worse than none.
  */
-export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions, loopDelay?: LoopDelayMeter | undefined, execArgv?: readonly string[], announceCeilingOrigin?: boolean, rssBudget?: RssBudgetOptions): () => void;
+export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions, loopDelay?: LoopDelayMeter | undefined, execArgv?: readonly string[], announceCeilingOrigin?: boolean, rssBudget?: RssBudgetOptions, probes?: HeapWatchProbes): () => void;
 /**
  * Steady heartbeat period for a HEAVY addon runner.
  *

package/dist/kernel/index.d.ts

@@ -40,6 +40,8 @@ export { createCoreCapService, CORE_CAP_SERVICE_NAME } from './moleculer/core-ca
 export type { CoreCapAction, CoreCapCallOrigin, CoreCapServiceOptions, } from './moleculer/core-cap-service.js';
 export { createAddonContext, createUdsAddonContext, setHubConnected, adaptBrokerToCluster, registerEventBusService, setNodeEventInterest, getMoleculerEventStats, subscribePassthrough, getBrokerEventBus, clusterEventTopic, EVENT_TOPIC_PREFIX, } from './moleculer/addon-context-factory.js';
 export type { MoleculerEventStats } from './moleculer/addon-context-factory.js';
+export type { MoleculerEventFanoutMode } from './moleculer/event-bus-core.js';
+export { readMoleculerFanoutMode } from './moleculer/event-bus-core.js';
 export { HubLogForwarder } from './moleculer/hub-log-forwarder.js';
 export type { AddonContextOptions } from './moleculer/addon-context-factory.js';
 export { createHubService } from './moleculer/hub-service.js';

package/dist/kernel/transport/local-child-registry.d.ts

@@ -183,6 +183,28 @@ export declare class LocalChildRegistry {
      * surface (shadow burn-in + operator debug).
      */
     getChildEventStats(childId: string): ChildEventStats | null;
+    /**
+     * The regime the counters above were produced under, read once from
+     * `CAMSTACK_UDS_EVENT_FANOUT` at construction.
+     *
+     * A suppressed count means "nothing WAS dropped" under `filter` and "nothing
+     * WOULD have been dropped" under `shadow`. Those are different claims, and
+     * until this getter existed the mode was invisible at runtime — the counters
+     * were readable and the regime that produced them was not.
+     */
+    get eventFanoutMode(): UdsEventFanoutMode;
+    /**
+     * Connected children that never declared a subscription pattern set.
+     *
+     * Each one is a fail-open on BOTH planes: `childWantsEvent` returns true for
+     * an undeclared child ({@link ChildEntry.eventPatterns} `=== null`), and
+     * `aggregateEventInterest` returns `null` — accept everything — if ANY
+     * connected child is undeclared, which disables the node's inbound cross-node
+     * gate as well. **One legacy runner silently turns off the filtering for the
+     * whole node**, and nothing reported it, so a zero here is as load-bearing as
+     * a non-zero one.
+     */
+    undeclaredChildCount(): number;
     /**
      * Does `entry` want to receive an event in `category`? Honours the fan-out
      * mode: `broadcast` → always; undeclared patterns (`null`) → always