@camstack/system 1.2.150 → 1.2.153

package/dist/{manifest-python-deps-B5rrxUQv.mjs → manifest-python-deps-QompQxhG.mjs}

@@ -1,4 +1,4 @@
-import "./dist-CbRciAbu.mjs";
+import "./dist-BLhrrxIs.mjs";
 import * as crypto$1 from "node:crypto";
 import { randomBytes, randomUUID } from "node:crypto";
 import { ensureBinary, ensureFfmpeg, ensurePython, installPythonPackages, installPythonRequirements } from "@camstack/types/node";
@@ -7,16 +7,358 @@ import * as path$1 from "node:path";
 import { isAbsolute, join } from "node:path";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
+import * as fs from "node:fs";
 import * as os from "node:os";
 import { tmpdir } from "node:os";
-import * as fs from "node:fs";
 import { createServer } from "node:http";
-import { monitorEventLoopDelay } from "node:perf_hooks";
 import * as v8 from "node:v8";
+import { monitorEventLoopDelay } from "node:perf_hooks";
 import * as vm from "node:vm";
 import { BOOT_RECOVERY_BACKOFF_MS, DATAPLANE_SECRET_HEADER, DeviceType, DisposerChain, EventCategory, ReadinessRegistry, asJsonObject, asString, createDeviceProxy, createEventBusSliceSource, deviceOpsCapability, emitReadiness, errMsg, expandCapMethods, readNodePin, scopeKey, sleep, withCapInputDefaults } from "@camstack/types/addon";
 import { connect, createServer as createServer$1 } from "node:net";
 import { TRPCClientError, createTRPCClient } from "@trpc/client";
+//#region src/kernel/event-plane-report.ts
+/**
+* Build a reader over the two registries.
+*
+* Returns `undefined` until the child registry exists — the heartbeat starts as
+* the FIRST statement of `bootstrap()` (a process that dies early still leaves
+* samples) and `LocalChildRegistry` is built hundreds of lines later inside
+* `MoleculerService.onModuleInit`. Printing zeros in that window would report a
+* silent plane where there is no plane at all.
+*
+* Wrapped: a registry mid-teardown may throw, and a diagnostic that can crash
+* the process it watches is worse than none.
+*/
+function createEventPlaneReader(sources) {
+	return () => {
+		try {
+			const registry = sources.registry();
+			if (registry === null) return void 0;
+			const children = [];
+			const listed = registry.listChildren();
+			for (const child of listed) {
+				const stats = registry.getChildEventStats(child.childId);
+				if (stats === null) continue;
+				children.push({
+					childId: child.childId,
+					sent: stats.sent,
+					suppressed: stats.suppressed
+				});
+			}
+			const crossNode = sources.crossNodeStats();
+			return {
+				udsMode: registry.eventFanoutMode,
+				crossNodeMode: sources.crossNodeMode(),
+				children,
+				childrenConnected: listed.length,
+				childrenUndeclared: registry.undeclaredChildCount(),
+				crossNodeDelivered: crossNode?.delivered ?? 0,
+				crossNodeSuppressed: crossNode?.suppressed ?? 0
+			};
+		} catch {
+			return;
+		}
+	};
+}
+/**
+* How many children get named on the line.
+*
+* Five, because the question is which peer DOMINATES and the measured
+* distribution is not flat: one peer accounted for 78–82% of outbound bytes and
+* the runner-up for ~6%. Five names the dominant peer, its challenger and enough
+* of the tail to see the shape change; naming all ~38 would put a kilobyte-long
+* line in the hub's shared log every minute, which is how a log stops being read.
+*/
+var EVENT_PLANE_TOP_N = 5;
+/** A process that has never read a plane. */
+var EMPTY_EVENT_PLANE_BASELINE = {
+	children: /* @__PURE__ */ new Map(),
+	crossNodeDelivered: 0,
+	crossNodeSuppressed: 0
+};
+/**
+* Subtract the previous cumulative reading from the current one.
+*
+* A child that RESTARTED has counters that went backwards — the registry keys by
+* childId and a respawned runner reconnects under the same id with a fresh
+* entry. Clamping at 0 reports the restart as a quiet window rather than as a
+* negative rate: the window's traffic is genuinely unknown, and a negative
+* number on a rate reads as a broken meter.
+*/
+function diffEventPlane(baseline, current, topN = 5) {
+	const deltas = [];
+	let fanoutSent = 0;
+	let fanoutSuppressed = 0;
+	for (const sample of current.children) {
+		const before = baseline.children.get(sample.childId);
+		const sent = Math.max(0, sample.sent - (before?.sent ?? 0));
+		const suppressed = Math.max(0, sample.suppressed - (before?.suppressed ?? 0));
+		fanoutSent += sent;
+		fanoutSuppressed += suppressed;
+		if (sent > 0 || suppressed > 0) deltas.push({
+			childId: sample.childId,
+			sent,
+			suppressed
+		});
+	}
+	return {
+		udsMode: current.udsMode,
+		crossNodeMode: current.crossNodeMode,
+		childrenConnected: current.childrenConnected,
+		childrenUndeclared: current.childrenUndeclared,
+		fanoutSent,
+		fanoutSuppressed,
+		crossNodeDelivered: Math.max(0, current.crossNodeDelivered - baseline.crossNodeDelivered),
+		crossNodeSuppressed: Math.max(0, current.crossNodeSuppressed - baseline.crossNodeSuppressed),
+		top: deltas.toSorted((a, b) => b.sent - a.sent || a.childId.localeCompare(b.childId)).slice(0, topN)
+	};
+}
+function createEventPlaneMeter(reader, topN = 5) {
+	let baseline = EMPTY_EVENT_PLANE_BASELINE;
+	return { read: () => {
+		const current = reader();
+		if (current === void 0) return void 0;
+		const window = diffEventPlane(baseline, current, topN);
+		baseline = {
+			children: new Map(current.children.map((s) => [s.childId, s])),
+			crossNodeDelivered: current.crossNodeDelivered,
+			crossNodeSuppressed: current.crossNodeSuppressed
+		};
+		return window;
+	} };
+}
+/**
+* The suffix appended to a `[mem]` line — leading space included, empty when
+* this process has no event plane.
+*
+* Only two processes in the cluster have one (hub-main, and an agent's main), so
+* these fields never appear on the ~38 runner lines that share the same log
+* stream. On the two that do, every field is printed on every line INCLUDING at
+* zero: the modes because a suppressed count without its regime is unreadable,
+* `childrenUndeclared` because one undeclared child silently disables both
+* gates, and the deltas because "the plane went quiet for a minute" is a
+* reading, not a blank.
+*
+* `fanoutTop` is the one conditional field — a ranked list of nothing is noise —
+* and it is a single comma-separated value so the field COUNT does not change
+* from line to line: `childId:sent/suppressed`, largest first.
+*/
+function formatEventPlane(window) {
+	if (window === void 0) return "";
+	const top = window.top.map((d) => `${d.childId}:${d.sent}/${d.suppressed}`).join(",");
+	return ` fanoutMode=uds:${window.udsMode}/cross-node:${window.crossNodeMode} children=${window.childrenConnected} childrenUndeclared=${window.childrenUndeclared} fanoutSent=${window.fanoutSent} fanoutSuppressed=${window.fanoutSuppressed} crossNodeDelivered=${window.crossNodeDelivered} crossNodeSuppressed=${window.crossNodeSuppressed}` + (top.length === 0 ? "" : ` fanoutTop=${top}`);
+}
+//#endregion
+//#region src/kernel/heap-spaces.ts
+/**
+* heap-spaces — WHERE in the V8 heap the bytes are, on the ordinary heartbeat.
+*
+* ## The question this exists to answer, and why it was unobtainable
+*
+* `hub/recorder` holds ~1.4 GB of live, retained JS heap that nobody can name.
+* That is measured, not inferred: a forced compacting
+* `gc({ flavor: 'last-resort' })` returned 507 MB of RSS and left V8 with
+* 1870 MB committed and 1439 MB of it LIVE forty seconds later. Everything the
+* 2026-08-29 pass could price — index rows 60–67 MB, runner shell ~50 MB, boot
+* parse 35–50 MB transient, write path 0 retained — sums to 115–125 MB, under
+* 9%. The same addon build running as `recorder` on an agent with no cameras
+* and no archive sits at 52 MB.
+*
+* The next question is cheap and binary, and it halves the search:
+*
+*  - **`old_space` holds it** ⇒ millions of small objects. The hunt becomes
+*    "what allocates ~7 M objects", and the row counters the recorder's index
+*    already exposes are the next thing to log.
+*  - **`large_object_space` holds it** ⇒ a handful of objects over V8's
+*    ~128 KB regular-object ceiling. The hunt becomes "who holds one huge array
+*    or string", which is a very short list in any one addon.
+*
+* V8 answers it for free, and this repository has been computing the answer
+* since the SIGUSR2 handler was written — and then throwing it away, because
+* `addon-runner.ts` welded the free read to `v8.writeHeapSnapshot`. One signal,
+* both actions, so the cheap answer could not be had without the expensive one
+* an operator has to authorise on a production hub. {@link emitHeapDiagnosticReport}
+* splits them; this module puts the same breakdown on the periodic `[mem]` line
+* so no signal is needed at all.
+*
+* ## Why bytes and not MB in the model
+*
+* The two consumers want different resolution — the heartbeat wants whole MB
+* next to `heapUsed`, the SIGUSR2 report keeps the 0.1 MB it has always
+* printed — so the reading stays raw and rounding happens at the edge. A model
+* that pre-rounds cannot serve both without lying to one of them.
+*/
+/**
+* The two spaces that ANSWER the question, printed always — including at zero.
+*
+* Node 24 reports thirteen spaces (measured on 24.17: `read_only_space`,
+* `new_space`, `old_space`, `code_space`, `shared_space`, `trusted_space`,
+* `shared_trusted_space`, `new_large_object_space`, `large_object_space`,
+* `code_large_object_space`, `shared_large_object_space`,
+* `shared_trusted_large_object_space`, `trusted_large_object_space`). Printing
+* all thirteen once a minute per process would add ~13 fields to a line that
+* already carries eleven, on every runner, forever — and eleven of the thirteen
+* cannot be the answer to a 1.4 GB retained-set question.
+*
+* These two are printed unconditionally because **absence must never be
+* ambiguous**. A threshold-only rule would drop `large_object_space` at 20 MB,
+* and a reader would not be able to tell "small" from "this build does not
+* report it" — which is the same class of mistake as a heartbeat with no
+* destination (D285). `largeObjectSpace=0MB` is a load-bearing reading: it says
+* the retainer is NOT a handful of huge objects.
+*/
+var DECISIVE_HEAP_SPACES = ["old_space", "large_object_space"];
+/**
+* A non-decisive space is printed only once it holds at least this many MB.
+*
+* 32 MB, and the number comes from the smallest heap this line is ever asked to
+* explain: the control in the 2026-08-29 pass is an idle `recorder` runner at
+* **52 MB** of `heapUsed`. A threshold below that means no space that could
+* dominate even the smallest watched process is ever hidden, while a quiet
+* process — measured on a bare Node 24.17: `new_space` 0.7 MB, `code_space`
+* 0.25 MB, `trusted_space` 0.9 MB — prints nothing but the decisive pair.
+*
+* What this drops, and why it is right to drop it: `new_space` is the scavenger's
+* semi-space and is bounded by `--max-semi-space-size`, so it can never be a
+* retained set; `code_space` and `read_only_space` are JIT output and V8's own
+* immutable roots, both bounded by the code that exists rather than by the data
+* it touches; the `shared_*` spaces are empty outside a worker-threads
+* deployment, which this cluster does not run. None of them has ever been the
+* answer to "what is this process holding", and each would cost a field on
+* every `[mem]` line in the fleet.
+*
+* They are not censored, though — a space over the threshold prints, whatever it
+* is. If `code_space` ever reaches 32 MB that is itself the finding.
+*/
+var HEAP_SPACE_REPORT_MIN_MB = 32;
+var BYTES_PER_MB = 1048576;
+/** Whole MB, matching the rest of the `[mem]` line. */
+function mb(bytes) {
+	return Math.round(bytes / BYTES_PER_MB);
+}
+/** Tenths of an MB — the resolution the SIGUSR2 report has always printed. */
+function tenthsMb(bytes) {
+	return Math.round(bytes / BYTES_PER_MB * 10) / 10;
+}
+/**
+* Read every space from V8.
+*
+* Wrapped: a diagnostic that can crash the process it watches is worse than
+* none, and this is called from a heartbeat whose whole contract is that it
+* survives whatever it is measuring. An empty reading prints no fields, which
+* is exactly what a heartbeat that could not read the heap should say.
+*/
+function readHeapSpaces() {
+	try {
+		return v8.getHeapSpaceStatistics().map((space) => ({
+			name: space.space_name,
+			usedBytes: space.space_used_size
+		}));
+	} catch {
+		return [];
+	}
+}
+/**
+* `old_space` → `oldSpace`. Mechanical and total, so a space V8 adds tomorrow
+* gets a field name without anyone editing a table.
+*
+* camelCase because that is the vocabulary of every other field on this line
+* (`heapUsed`, `heapTotal`, `arrayBuffers`, `rssBudget`, `loopP99`). A line that
+* mixed `heapUsed=` with `old_space=` would be two conventions for one record.
+*/
+function heapSpaceField(v8Name) {
+	return v8Name.replace(/_([a-z])/g, (_match, letter) => letter.toUpperCase());
+}
+/**
+* The spaces worth a field on this line: the decisive pair (in that order,
+* always), then every other space at or above `minMb`, largest first.
+*
+* Largest-first for the tail because the tail only ever appears when something
+* unexpected is holding memory, and the biggest surprise should be the first
+* thing read. The decisive pair keeps a FIXED position so the two numbers a
+* reader is comparing are always in the same place on every line.
+*/
+function selectReportedSpaces(spaces, minMb = 32) {
+	const decisive = DECISIVE_HEAP_SPACES.map((name) => spaces.find((space) => space.name === name)).filter((space) => space !== void 0);
+	const rest = spaces.filter((space) => !DECISIVE_HEAP_SPACES.includes(space.name)).filter((space) => mb(space.usedBytes) >= minMb).toSorted((a, b) => b.usedBytes - a.usedBytes || a.name.localeCompare(b.name));
+	return [...decisive, ...rest];
+}
+/**
+* The suffix appended to a `[mem]` line — leading space included, empty when
+* there is nothing to say.
+*
+* APPENDED, never interleaved: every existing field keeps its name AND its
+* position, so a `hl loki-query` pipeline written against the old line still
+* matches. Prefix stability is the only change to a log line that cannot break
+* a query someone already wrote.
+*/
+function formatHeapSpaces(spaces, minMb = 32) {
+	const selected = selectReportedSpaces(spaces, minMb);
+	if (selected.length === 0) return "";
+	return ` ${selected.map((s) => `${heapSpaceField(s.name)}=${mb(s.usedBytes)}MB`).join(" ")}`;
+}
+/**
+* Env var that authorises a heap snapshot on SIGUSR2.
+*
+* A snapshot is an admin mutation on a production hub: `writeHeapSnapshot`
+* streams a file the size of the heap to `/tmp` and stops the world while it
+* walks it — on the recorder that is a multi-GB file and a stall measured in
+* seconds. The per-space breakdown is ~1.2 µs and changes nothing. Welding them
+* to one signal priced the free answer at the cost of the expensive one.
+*/
+var RUNNER_HEAP_SNAPSHOT_ENV = "CAMSTACK_RUNNER_HEAP_SNAPSHOT";
+/**
+* Whether the operator has authorised the snapshot half of SIGUSR2.
+*
+* Explicit `'on'` only. Anything else — unset, empty, `'true'`, `'1'` — is NOT
+* authorisation: a multi-GB stop-the-world write must never be reachable by a
+* value someone set for a different purpose.
+*/
+function heapSnapshotAuthorised(env = process.env) {
+	return env[RUNNER_HEAP_SNAPSHOT_ENV] === "on";
+}
+var defaultDeps = {
+	readMemory: () => process.memoryUsage(),
+	readSpaces: readHeapSpaces,
+	writeSnapshot: (path) => {
+		v8.writeHeapSnapshot(path);
+	},
+	env: process.env
+};
+/**
+* The SIGUSR2 report — memory + the FULL per-space breakdown, always; the
+* snapshot only when {@link heapSnapshotAuthorised}.
+*
+* The breakdown here keeps every non-empty space at 0.1 MB, unlike the
+* heartbeat's filtered whole-MB pair: this is a hand-fired diagnostic read once
+* by a human who asked for it, so completeness costs nothing. The heartbeat runs
+* unattended on every runner forever, so it pays for every field it prints.
+*
+* A refused snapshot SAYS SO. A branch that drops work silently reads as "it was
+* never requested", and this one is fired by an operator who is standing there
+* waiting for a file.
+*/
+function emitHeapDiagnosticReport(request, deps = defaultDeps) {
+	try {
+		const mu = deps.readMemory();
+		const spaces = deps.readSpaces().filter((space) => space.usedBytes > 0).map((space) => `${space.name}=${tenthsMb(space.usedBytes)}`).join(" ");
+		request.sink.info(`[mem] node="${request.label}" addons=[${request.addons.join(",")}] rss=${tenthsMb(mu.rss)} heapUsed=${tenthsMb(mu.heapUsed)} heapTotal=${tenthsMb(mu.heapTotal)} external=${tenthsMb(mu.external)} arrayBuffers=${tenthsMb(mu.arrayBuffers)} | ${spaces}`);
+	} catch (error) {
+		request.sink.warn(`[mem] node="${request.label}" space report failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	if (!heapSnapshotAuthorised(deps.env)) {
+		request.sink.info(`[mem] node="${request.label}" heap snapshot NOT taken — it is an admin mutation (a multi-GB write and a stop-the-world walk of the heap). The per-space breakdown above is the free half and needs no authorisation. Set ${RUNNER_HEAP_SNAPSHOT_ENV}=on for this runner and signal again to take one.`);
+		return;
+	}
+	try {
+		deps.writeSnapshot(request.snapshotPath);
+		request.sink.info(`[mem] node="${request.label}" heap snapshot written -> ${request.snapshotPath}`);
+	} catch (error) {
+		request.sink.warn(`[mem] node="${request.label}" snapshot failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
+}
+//#endregion
 //#region src/kernel/heap-watch.ts
 /**
 * heap-watch — hub-main's own memory, on the record, and returned to the OS.
@@ -41,6 +383,13 @@ import { TRPCClientError, createTRPCClient } from "@trpc/client";
 * `addon-runner.ts`) — hub-main, the one process that actually OOMs, was the
 * one process with no telemetry at all.
 *
+* **And for 27 days after that it still had none where anyone could read it.**
+* The watch was started correctly and passed NO SINK, so it fell back to
+* `console.log`, which is not an `ILogDestination` — 42 hours of Loki carried a
+* line for every runner and none for hub-main. Read
+* {@link createDeferredHeapWatchSink} before trusting anything below: a
+* heartbeat with no destination is indistinguishable from no heartbeat (D285).
+*
 * ## What it gives
 *
 * A heartbeat carrying the numbers that separate the two possible stories, and
@@ -51,6 +400,14 @@ import { TRPCClientError, createTRPCClient } from "@trpc/client";
 *  - `heapUsedMb` flat with a sharp jump under load → a burst allocation, and
 *    the timestamp says what was being served.
 *
+* Since 2026-08-29 the line also says **WHERE in the V8 heap** the bytes are —
+* `oldSpace` and `largeObjectSpace`, appended after everything already on it.
+* `heapUsed` alone cannot separate "millions of small objects" from "a handful
+* of huge ones", and those lead to opposite searches; the recorder's unnamed
+* ~1.4 GB retained set is the case that proved it. See
+* {@link formatHeapSpaces} for which spaces are printed and why the rest are
+* not.
+*
 * `external`/`arrayBuffers` ride along because they are NOT bounded by
 * `--max-old-space-size` and are most of the gap between a 1.5GB cap and the
 * ~2.05GB RSS observed live: a diagnosis that only looks at the JS heap
@@ -92,6 +449,30 @@ import { TRPCClientError, createTRPCClient } from "@trpc/client";
 * `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a ratchet to
 * 7.8GB and the OOM that ends it.
 */
+/**
+* What hub-main calls itself in a `[mem]` line.
+*
+* ## Why not `hub/main`
+*
+* A runner's label is its nodeId, and a nodeId is built in exactly one place as
+* `` `${parentNodeId}/${runnerId}` `` (`kernel/moleculer/addon-runner.ts:222`) —
+* so every runner label contains a `/` and every hub-local one is `hub/<runner>`.
+* A label with NO slash therefore cannot be produced by that rule, which makes
+* `hub-main` unambiguous by construction rather than by convention: no future
+* addon, group or agent can ever claim it.
+*
+* It also sorts where it belongs. `-` (0x2D) precedes `/` (0x2F), so a sorted
+* `[mem]` census puts `hub-main` immediately before `hub/decoder`,
+* `hub/recorder`, `hub/stream-broker` — the composition root first, then the
+* runners it forked. `hub/main` would sort into the middle of its own children
+* and would be one addon id away from a collision.
+*
+* And it is the name the rest of this repository already uses — the docblocks,
+* {@link HUB_MAIN_RSS_BUDGET_MB}, {@link HUB_RSS_BUDGET_ENV}, every ADR. A log
+* label that disagreed with the prose would be a second vocabulary for one
+* process, which is the exact failure this telemetry exists to remove.
+*/
+var HUB_MAIN_HEAP_WATCH_LABEL = "hub-main";
 /** Heartbeat period. Slow enough to be free, fast enough that the sample before
 *  a fatal is recent. */
 var HEAP_WATCH_INTERVAL_MS = 6e4;
@@ -168,6 +549,86 @@ var HEAP_RECLAIM_TRIGGER_MB = 1024;
 * router's time, against a ~1.5GiB peak it removes each pass.
 */
 var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
+/**
+* Floor on the pass once it has PROVEN it is steady state, not a rescue.
+*
+* ## Why `stranded` cannot discriminate on hub-main
+*
+* {@link strandedMb} is `rss − heapUsed − external`: everything the process
+* holds that those two numbers do not name. That lumps together two things with
+* opposite fixes —
+*
+*  - V8 pages committed at a sawtooth peak, which `gc(last-resort)` RETURNS;
+*  - glibc arena memory already freed and never handed back, which **no V8
+*    flavor can reach** (nothing in this repository calls `malloc_trim`).
+*
+* {@link HEAP_RECLAIM_TRIGGER_MB} was calibrated against a hub-main whose strand
+* was mostly the first kind, and a pass then took it back under the trigger. Once
+* the second kind grows past 1024 MB the quantity can never fall back under it,
+* and the gate degenerates. Measured on the live hub 2026-08-29, first 25
+* minutes of hub-main telemetry ever collected:
+*
+*  - `stranded` sat at **1031–2313 MB for the entire boot**, i.e.
+*    {@link shouldReclaim} never once said no. **What is written as a threshold
+*    is, on this process, an unconditional 120-second timer.**
+*  - Nine minutes out of nine that contained a reclaim show `loopMax` ≈ the
+*    reclaim's own duration (555–869 ms). Every minute without one is
+*    107–435 ms. **The largest recurring event-loop stall on the process that
+*    mediates every cross-runner call is the memory patch it applies to itself.**
+*  - Each pass freed less than the last (1193 → 462 MB) while the post-pass RSS
+*    floor ratcheted **1940 → 3510 MB in fourteen minutes**, with `heapUsed`
+*    never moving. It is compacting a heap that is not the problem.
+*
+* ## The fix is not a bigger trigger
+*
+* Raising {@link HEAP_RECLAIM_TRIGGER_MB} would move the number at which the same
+* degeneration happens, and it would silence `hub/recorder` — whose single pass
+* in six hours is the only cheap proof anyone has that the recorder's 1.4 GB is
+* RETAINED (a forced compaction freed 507 MB and V8 still held 1.44 GB live).
+* The discriminator is not the level of the strand; it is **whether the strand
+* ever falls back under the trigger between passes**, which is precisely what
+* {@link RECLAIM_STEADY_STATE_PASSES} already counts and used only to talk about.
+*
+* So the counter becomes load-bearing: once the pass has fired at the floor
+* {@link RECLAIM_STEADY_STATE_PASSES} times running, the trigger has demonstrably
+* stopped discriminating on this process, and the cadence falls back to a period
+* chosen for its COST rather than to a threshold pretending to be a gate.
+*
+* ## Why 600_000, and why that is not a regression
+*
+* It is the value {@link HEAP_RECLAIM_MIN_INTERVAL_MS} held before 2026-08-17,
+* and the reason it came down is recorded there: *"the ten-minute floor let a
+* peak that re-formed in seconds stand for ten minutes"*. That argument is about
+* a BURST — and a burst subsides, which resets the run and restores the 120 s
+* floor. It never applied to a process in proven steady state. So the burst
+* bound is untouched and the steady state gets the period the burst argument was
+* never about.
+*
+* ## What each process does after this change
+*
+*  - **hub-main**: first six passes at 120 s, unchanged. From the seventh
+*    consecutive floor-to-floor pass, every 600 s. Stop-the-world cost on the
+*    router falls from ~0.58% of wall clock to ~0.12%. The ArrayBuffer probe
+*    survives at six passes an hour — the finding it produced needed ten passes
+*    total, not ten in twenty-five minutes.
+*  - **`hub/recorder`**: zero change, by construction. `passesAtFloor` only
+*    increments when a pass fires within `minIntervalMs * 2` of the previous
+*    one; the recorder's strand takes six hours to re-form, so the counter is
+*    reset to 1 on every pass and can never reach six. The 507 MB probe is
+*    intact.
+*  - **Anything that recovers**: the run resets the moment the strand falls back
+*    under the trigger, and the 120 s floor returns with it.
+*/
+var HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS = 6e5;
+/**
+* The floor in force for the NEXT pass, given how many have fired at the floor.
+*
+* Pure, so the cadence change is testable without stranding a gigabyte to reach
+* it — and so the one place that decides it can be read on its own.
+*/
+function reclaimIntervalMs(passesAtFloor, minIntervalMs = HEAP_RECLAIM_MIN_INTERVAL_MS, steadyStateIntervalMs = HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS, steadyStatePasses = 6) {
+	return passesAtFloor >= steadyStatePasses ? Math.max(minIntervalMs, steadyStateIntervalMs) : minIntervalMs;
+}
 /** What a loop with nothing recorded yet reports. */
 var ZERO_LOOP_DELAY = {
 	p50Ms: 0,
@@ -187,6 +648,37 @@ var ZERO_LOOP_DELAY = {
 * naively would print `loopMax=9223372036854ms` and be read as a catastrophe on
 * every boot. `count === 0` short-circuits to {@link ZERO_LOOP_DELAY}.
 *
+* ## `loopP50=20ms` was the SAMPLER, not the loop (fixed 2026-08-29)
+*
+* `monitorEventLoopDelay` records the whole INTERVAL between its own timer
+* firings, not the excess over the interval it asked for. So on a loop with
+* nothing wrong the histogram reads `resolution`, and every `[mem]` line this
+* process has ever emitted said `loopP50=20ms` — which was read, repeatedly in
+* one session, as "hub-main is chronically 20 ms behind". It is not; 20 was the
+* default `resolutionMs` of this very function.
+*
+* The delay is `interval − resolution`, and that is not a fudge factor — it is
+* the only reading the four resolutions agree on. Measured here, one 120 ms
+* block observed by four histograms at once:
+*
+* | resolution | raw p50 | raw max | max − resolution |
+* | --- | --- | --- | --- |
+* | 20 ms | 20.40 ms | 137.63 ms | 117.6 ms |
+* | 10 ms | 10.22 ms | 123.60 ms | 113.6 ms |
+* | 5 ms | 5.11 ms | 123.60 ms | 118.6 ms |
+* | 1 ms | 1.27 ms | 121.04 ms | 120.0 ms |
+*
+* Raw, the four disagree by 16 ms about the same event and none of them reports
+* the block. Corrected, they agree within 7 ms and all four name it. The
+* subtraction costs nothing and needs no faster sampler — a 1 ms resolution
+* would be 1 000 extra timer wakeups per second on the process that already
+* serves ~44 000 socket operations per second, to buy 2 ms of accuracy on a
+* quantity nobody reads below 100 ms.
+*
+* **Readings from before this change are 20 ms high.** `loopP99=21–28ms` in the
+* 2026-08-29 census is really 1–8 ms; `loopMax=2804ms` is really 2784 ms. Do not
+* compare a pre-fix series with a post-fix one without shifting it.
+*
 * Returns undefined if the platform declines, so the caller keeps a heartbeat
 * either way.
 */
@@ -194,14 +686,15 @@ function createLoopDelayMeter(resolutionMs = 20) {
 	try {
 		const histogram = monitorEventLoopDelay({ resolution: resolutionMs });
 		histogram.enable();
+		const delay = (nanoseconds) => Math.max(0, Math.round(nanoseconds / 1e6) - resolutionMs);
 		return {
 			read: () => {
 				try {
 					if (histogram.count === 0) return ZERO_LOOP_DELAY;
 					const sample = {
-						p50Ms: Math.round(histogram.percentile(50) / 1e6),
-						p99Ms: Math.round(histogram.percentile(99) / 1e6),
-						maxMs: Math.round(histogram.max / 1e6)
+						p50Ms: delay(histogram.percentile(50)),
+						p99Ms: delay(histogram.percentile(99)),
+						maxMs: delay(histogram.max)
 					};
 					histogram.reset();
 					return sample;
@@ -257,6 +750,33 @@ var consoleSink = {
 	warn: (line) => console.warn(line)
 };
 /**
+* Build a {@link DeferredHeapWatchSink}.
+*
+* The handover announces itself, once, through the NEW target: that line is the
+* only in-band evidence that hub-main's telemetry actually reached the log
+* manager on a given boot. Its absence from Loki means the wiring is broken —
+* which is precisely the state this module was in, undetected, for 42 hours.
+*/
+function createDeferredHeapWatchSink(label = HUB_MAIN_HEAP_WATCH_LABEL, fallback = consoleSink) {
+	let target = fallback;
+	const sink = {
+		info: (line) => {
+			target.info(line);
+		},
+		warn: (line) => {
+			target.warn(line);
+		}
+	};
+	const attach = (next) => {
+		target = next;
+		next.info(`[mem] ${label} heartbeat attached to the log manager — earlier lines went to stdout only and are not in the log buffer, the rotated file or Loki.`);
+	};
+	return {
+		sink,
+		attach
+	};
+}
+/**
 * RSS the process is holding but not using.
 *
 * The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
@@ -385,10 +905,22 @@ function createV8Reclaimer() {
 function formatOverBudget(label, b, source) {
 	return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
 }
-function format(label, s, loop, budgetMb) {
+/**
+* The heartbeat line.
+*
+* ## Nothing here may be renamed or moved
+*
+* `heapUsed` says HOW MUCH; the per-space suffix says WHERE, which is the half
+* that names a retained set. The suffix is APPENDED after the loop fields for
+* one reason: every existing field then keeps its name and its position, so the
+* `hl loki-query` pipelines written against this line keep matching. A rename or
+* a reorder is the one change to a log format that silently breaks a query
+* nobody will re-run until they need it.
+*/
+function format(label, s, loop, budgetMb, suffix) {
 	const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
-	if (loop === void 0) return line;
-	return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
+	if (loop === void 0) return `${line}${suffix}`;
+	return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms${suffix}`;
 }
 /**
 * Start the heartbeat. Returns a stop function.
@@ -396,11 +928,14 @@ function format(label, s, loop, budgetMb) {
 * `unref`ed so it can never hold the process open, and every read is wrapped:
 * a diagnostic that can crash the process it watches is worse than none.
 */
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
+function startHeapWatch(label = HUB_MAIN_HEAP_WATCH_LABEL, sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget, probes) {
+	const readSpaces = probes?.readSpaces ?? readHeapSpaces;
+	const eventPlane = probes?.eventPlane === void 0 ? void 0 : createEventPlaneMeter(probes.eventPlane);
 	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
 	const now = reclaimOptions?.now ?? (() => Date.now());
 	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
 	const minIntervalMs = reclaimOptions?.minIntervalMs ?? 12e4;
+	const steadyStateIntervalMs = reclaimOptions?.steadyStateIntervalMs ?? 6e5;
 	const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
 	const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
 	const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
@@ -420,9 +955,10 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 			steadyStateAnnounced = false;
 			return;
 		}
+		const floorMs = reclaimIntervalMs(passesAtFloor, minIntervalMs, steadyStateIntervalMs);
 		const sinceLast = now() - lastReclaimAt;
-		if (sinceLast < minIntervalMs) return;
-		passesAtFloor = sinceLast <= minIntervalMs * 2 ? passesAtFloor + 1 : 1;
+		if (sinceLast < floorMs) return;
+		passesAtFloor = sinceLast <= floorMs * 2 ? passesAtFloor + 1 : 1;
 		lastReclaimAt = now();
 		const startedAt = now();
 		try {
@@ -435,7 +971,8 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 		sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
 		if (passesAtFloor >= 6 && !steadyStateAnnounced) {
 			steadyStateAnnounced = true;
-			sink.warn(`[mem] reclaim ${label} has run at the floor for ${passesAtFloor} consecutive passes (every ${minIntervalMs}ms) — the stop-the-world is now STEADY STATE, not a rescue. The allocation source is what needs the fix; this pass only bounds its peak.`);
+			const nextFloorMs = reclaimIntervalMs(passesAtFloor, minIntervalMs, steadyStateIntervalMs);
+			sink.warn(`[mem] reclaim ${label} has run at the floor for ${passesAtFloor} consecutive passes (every ${minIntervalMs}ms) — the stop-the-world is now STEADY STATE, not a rescue: stranded never fell back under ${triggerMb}MB between passes, so the trigger is not discriminating on this process and what is left deciding is the timer. BACKING OFF to every ${nextFloorMs}ms — the pass keeps running (it is still the only measurement of this process's live ArrayBuffer set) at a cost the router can pay. The allocation source is what needs the fix; this pass only bounds its peak.`);
 		}
 	};
 	let mode = "steady";
@@ -498,7 +1035,7 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 			const due = at - lastLoggedAt >= intervalMs;
 			if (mode === "escalated" || due) {
 				lastLoggedAt = at;
-				const line = format(label, sample, loopDelay?.read(), budgetMb);
+				const line = format(label, sample, loopDelay?.read(), budgetMb, `${formatHeapSpaces(readSpaces())}${formatEventPlane(eventPlane?.read())}`);
 				if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
 				else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
 				else sink.info(line);
@@ -5417,6 +5954,34 @@ var LocalChildRegistry = class {
 		};
 	}
 	/**
+	* The regime the counters above were produced under, read once from
+	* `CAMSTACK_UDS_EVENT_FANOUT` at construction.
+	*
+	* A suppressed count means "nothing WAS dropped" under `filter` and "nothing
+	* WOULD have been dropped" under `shadow`. Those are different claims, and
+	* until this getter existed the mode was invisible at runtime — the counters
+	* were readable and the regime that produced them was not.
+	*/
+	get eventFanoutMode() {
+		return this.fanoutMode;
+	}
+	/**
+	* Connected children that never declared a subscription pattern set.
+	*
+	* Each one is a fail-open on BOTH planes: `childWantsEvent` returns true for
+	* an undeclared child ({@link ChildEntry.eventPatterns} `=== null`), and
+	* `aggregateEventInterest` returns `null` — accept everything — if ANY
+	* connected child is undeclared, which disables the node's inbound cross-node
+	* gate as well. **One legacy runner silently turns off the filtering for the
+	* whole node**, and nothing reported it, so a zero here is as load-bearing as
+	* a non-zero one.
+	*/
+	undeclaredChildCount() {
+		let count = 0;
+		for (const entry of this.children.values()) if (entry.eventPatterns === null) count += 1;
+		return count;
+	}
+	/**
 	* Does `entry` want to receive an event in `category`? Honours the fan-out
 	* mode: `broadcast` → always; undeclared patterns (`null`) → always
 	* (fail-open); else the child's declared set must match via `matchesPattern`
@@ -8582,4 +9147,4 @@ async function installManifestPythonDeps(declaration, addonDir, deps, logger) {
 	await deps.installPythonRequirements(reqAbs);
 }
 //#endregion
-export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A, RSS_BUDGET_REANNOUNCE_MIN_MS as At, AGENT_CAP_FWD_SERVICE as B, shouldReclaim as Bt, brokerTransportLink as C, createAddonDataPlaneFacility as Ct, HUB_CAP_FWD_ACTION as D, HEAP_WATCH_WARN_RATIO as Dt, localProviderLink as E, HEAP_WATCH_INTERVAL_MS as Et, createUdsLoggerWithControl as F, createV8Reclaimer as Ft, createLocalTransport as G, CapRouteError as H, startRunnerHeapWatch as Ht, LocalChildClient as I, describeRss as It, SocketChannel as J, UdsLocalTransportClient as K, LocalChildRegistry as L, hubMainRssBudget as Lt, createUdsEventBus as M, RUNNER_HEAP_WATCH_INTERVAL_MS as Mt, udsChildLogToWorkerEntry as N, RUNNER_RSS_BUDGET_ENV as Nt, HUB_CAP_FWD_SERVICE as O, HUB_MAIN_RSS_BUDGET_MB as Ot, createUdsLogger as P, buildHeapSample as Pt, buildNativeCapProxy as Q, UDS_NO_ROUTE_PREFIX as R, nextRssBudgetState as Rt, brokerCallForCap as S, runNpm as St, ipcParentLink as T, HEAP_RECLAIM_TRIGGER_MB as Tt, classifyCapRoute as U, strandedMb as Ut, CapRouteResolver as V, startHeapWatch as Vt, callWithServiceDiscovery as W, FrameDecoder as X, localEndpointPath as Y, encodeFrame as Z, createKernelHwAccel as _, CapabilityUnavailableError as _t, getWorkerDeviceRegistry as a, createAddonService as at, __resetCapUsageRegistryForTests as b, resolveAddonClass as bt, setHubConnected as c, capActionName as ct, getBrokerEventBus as d, capServiceName as dt, createBrokerDeviceManagerApi as et, getMoleculerEventStats as f, parseCapAction as ft, AddonDepsManager as g, CapabilityHandle as gt, subscribePassthrough as h, DeviceRegistry as ht, createUdsAddonContext as i, setWorkerNativeCapsChangeListener as it, createUdsEventBridge as j, RSS_BUDGET_RELEASE_RATIO as jt, createHubCapForwardService as k, HUB_RSS_BUDGET_ENV as kt, EVENT_TOPIC_PREFIX as l, capActionSuffix as lt, setNodeEventInterest as m, serializeTypedArrays as mt, adaptBrokerToCluster as n, getWorkerNativeCapSnapshot as nt, getOrInitReadinessRegistry as o, validateProviderRegistrations as ot, registerEventBusService as p, deserializeTypedArrays as pt, UdsLocalTransportServer as q, createAddonContext as r, mountNativeCapService as rt, getOrInitReadinessRegistryForClient as s, NATIVE_PROVIDER_SERVICE_INFIX as st, installManifestPythonDeps as t, getWorkerNativeCapProvider as tt, clusterEventTopic as u, capBareAction as ut, resolveHwAccel as v, copyBundledNativeModules as vt, buildLinkChain as w, HEAP_RECLAIM_MIN_INTERVAL_MS as wt, getCapUsageRegistry as x, resolveNpmInvocation as xt, CapUsageRegistry as y, installManifestNativeDeps as yt, AGENT_CAP_FWD_ACTION as z, parseRssBudgetMb as zt };
+export { buildNativeCapProxy as $, RUNNER_HEAP_SNAPSHOT_ENV as $t, createParentUnownedCallHandler as A, HUB_MAIN_HEAP_WATCH_LABEL as At, AGENT_CAP_FWD_SERVICE as B, createLoopDelayMeter as Bt, brokerTransportLink as C, runNpm as Ct, HUB_CAP_FWD_ACTION as D, HEAP_RECLAIM_TRIGGER_MB as Dt, localProviderLink as E, HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS as Et, createUdsLoggerWithControl as F, RUNNER_HEAP_WATCH_INTERVAL_MS as Ft, readMoleculerFanoutMode as G, parseRssBudgetMb as Gt, CapRouteError as H, describeRss as Ht, LocalChildClient as I, RUNNER_RSS_BUDGET_ENV as It, UdsLocalTransportServer as J, startHeapWatch as Jt, createLocalTransport as K, reclaimIntervalMs as Kt, LocalChildRegistry as L, ZERO_LOOP_DELAY as Lt, createUdsEventBus as M, HUB_RSS_BUDGET_ENV as Mt, udsChildLogToWorkerEntry as N, RSS_BUDGET_REANNOUNCE_MIN_MS as Nt, HUB_CAP_FWD_SERVICE as O, HEAP_WATCH_INTERVAL_MS as Ot, createUdsLogger as P, RSS_BUDGET_RELEASE_RATIO as Pt, encodeFrame as Q, HEAP_SPACE_REPORT_MIN_MB as Qt, UDS_NO_ROUTE_PREFIX as R, buildHeapSample as Rt, brokerCallForCap as S, resolveNpmInvocation as St, ipcParentLink as T, HEAP_RECLAIM_MIN_INTERVAL_MS as Tt, classifyCapRoute as U, hubMainRssBudget as Ut, CapRouteResolver as V, createV8Reclaimer as Vt, callWithServiceDiscovery as W, nextRssBudgetState as Wt, localEndpointPath as X, strandedMb as Xt, SocketChannel as Y, startRunnerHeapWatch as Yt, FrameDecoder as Z, DECISIVE_HEAP_SPACES as Zt, createKernelHwAccel as _, CapabilityHandle as _t, getWorkerDeviceRegistry as a, selectReportedSpaces as an, setWorkerNativeCapsChangeListener as at, __resetCapUsageRegistryForTests as b, installManifestNativeDeps as bt, setHubConnected as c, createEventPlaneReader as cn, NATIVE_PROVIDER_SERVICE_INFIX as ct, getBrokerEventBus as d, capBareAction as dt, emitHeapDiagnosticReport as en, buildUdsNativeCapProxy as et, getMoleculerEventStats as f, capServiceName as ft, AddonDepsManager as g, DeviceRegistry as gt, subscribePassthrough as h, serializeTypedArrays as ht, createUdsAddonContext as i, readHeapSpaces as in, mountNativeCapService as it, createUdsEventBridge as j, HUB_MAIN_RSS_BUDGET_MB as jt, createHubCapForwardService as k, HEAP_WATCH_WARN_RATIO as kt, EVENT_TOPIC_PREFIX as l, diffEventPlane as ln, capActionName as lt, setNodeEventInterest as m, deserializeTypedArrays as mt, adaptBrokerToCluster as n, heapSnapshotAuthorised as nn, getWorkerNativeCapProvider as nt, getOrInitReadinessRegistry as o, EVENT_PLANE_TOP_N as on, createAddonService as ot, registerEventBusService as p, parseCapAction as pt, UdsLocalTransportClient as q, shouldReclaim as qt, createAddonContext as r, heapSpaceField as rn, getWorkerNativeCapSnapshot as rt, getOrInitReadinessRegistryForClient as s, createEventPlaneMeter as sn, validateProviderRegistrations as st, installManifestPythonDeps as t, formatHeapSpaces as tn, createBrokerDeviceManagerApi as tt, clusterEventTopic as u, formatEventPlane as un, capActionSuffix as ut, resolveHwAccel as v, CapabilityUnavailableError as vt, buildLinkChain as w, createAddonDataPlaneFacility as wt, getCapUsageRegistry as x, resolveAddonClass as xt, CapUsageRegistry as y, copyBundledNativeModules as yt, AGENT_CAP_FWD_ACTION as z, createDeferredHeapWatchSink as zt };