@camstack/system 1.2.102 → 1.2.104

package/dist/{manifest-python-deps-Bt_J04b0.mjs → manifest-python-deps-5ei2xVUh.mjs}

@@ -10,9 +10,363 @@ import * as os from "node:os";
 import { tmpdir } from "node:os";
 import * as fs from "node:fs";
 import { createServer } from "node:http";
+import * as v8 from "node:v8";
+import * as vm from "node:vm";
 import { BOOT_RECOVERY_BACKOFF_MS, DATAPLANE_SECRET_HEADER, DeviceType, DisposerChain, EventCategory, ReadinessRegistry, asJsonObject, asString, createDeviceProxy, deviceOpsCapability, emitReadiness, errMsg, expandCapMethods, readNodePin, scopeKey, sleep, withCapInputDefaults } from "@camstack/types/addon";
 import { TRPCClientError, createTRPCClient } from "@trpc/client";
 import { connect, createServer as createServer$1 } from "node:net";
+//#region src/kernel/heap-watch.ts
+/**
+* heap-watch — hub-main's own memory, on the record, and returned to the OS.
+*
+* ## Why this exists
+*
+* hub-main used to be launched with `--max-old-space-size=1536` (docker-entrypoint;
+* the cap was removed on 2026-08-02, so the live ceiling is now V8's own default —
+* 4288MB in this container — and this module reads it at runtime rather than
+* assuming a number). It logged NOTHING about its own heap. When it died on
+* 2026-08-01/02 the only trace was V8's own stderr line:
+*
+*     FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
+*
+* which carries no process name, no heap number and no history — the whole
+* server restarts and the evidence is gone. Four of those in ~15h, and the one
+* at 06:57:57 landed nine seconds after a viewer opened a WebRTC session, which
+* is why the app sat in a permanent "loading": the hub was dying as it
+* connected.
+*
+* `[mem]` reports already exist for forked addon runners (SIGUSR2 in
+* `addon-runner.ts`) — hub-main, the one process that actually OOMs, was the
+* one process with no telemetry at all.
+*
+* ## What it gives
+*
+* A heartbeat carrying the numbers that separate the two possible stories, and
+* a WARN as the heap approaches the ceiling — the line that will exist BEFORE
+* the next fatal instead of after it:
+*
+*  - `heapUsedMb` climbing steadily between restarts → a leak;
+*  - `heapUsedMb` flat with a sharp jump under load → a burst allocation, and
+*    the timestamp says what was being served.
+*
+* `external`/`arrayBuffers` ride along because they are NOT bounded by
+* `--max-old-space-size` and are most of the gap between a 1.5GB cap and the
+* ~2.05GB RSS observed live: a diagnosis that only looks at the JS heap
+* misreads that process by 500MB.
+*
+* ## And then the heartbeat explained the "off-heap leak", which was not one
+*
+* 2026-08-08, measured on the live hub over 1019 consecutive samples and seven
+* separate boots: **hub-main's RSS is a HIGH-WATER MARK, not a leak.** In every
+* one of those 1019 samples RSS was >= the running maximum of
+* `heapTotal + external`, never once below it, and
+* `RSS_peak - peak(heapTotal + external)` landed between 874MB and 1639MB on
+* every boot — a roughly constant native floor on top of a number that only
+* ever goes up. RSS tracked the highest point the process ever reached and
+* stayed there: 3.3GB -> 4.5GB -> 5.2GB, and 7.8GB on the worst boot.
+*
+* Two things push that peak, and neither is a retained object:
+*
+*  - the JS heap sawtooths to 2.0-2.8GB because the ceiling is 4288MB and
+*    nothing creates pressure before then;
+*  - `arrayBuffers` accumulates DEAD Buffers — up to 3.65GB — because V8 only
+*    sweeps them on a major GC. Eleven collapses were captured, and every
+*    single one coincides with `heapUsed` dropping to its ~650MB floor. No code
+*    released a reference at 13:23:03; a mark-compact ran.
+*
+* At the peak V8 has committed the pages, and **an ordinary major GC does not
+* give them back.** Proven on the production hub: `smaps` showed 10,819 mapped
+* 256KB pages (V8's `kRegularPageSize`) holding 2704MB while V8 itself reported
+* `total_physical_size` of only 846MB — ~7,800 pages committed, resident, and
+* outside V8's own accounting. One `gc({ flavor: 'last-resort' })` took 417ms
+* and returned RSS 3355MB -> 1237MB, pages 10,819 -> 2,963.
+*
+* So the fix for the "~3GB/2h off-heap growth" is not a retainer hunt. It is to
+* ask V8 for the pages back when the process is visibly holding more than it is
+* using — which is what `shouldReclaim` decides and what the heartbeat now does.
+*
+* The cost is a stop-the-world on the hub's router process — 250-450ms when
+* first measured, 654ms on the largest pass seen since — bounded to once per
+* `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a ratchet to
+* 7.8GB and the OOM that ends it.
+*/
+/** Heartbeat period. Slow enough to be free, fast enough that the sample before
+*  a fatal is recent. */
+var HEAP_WATCH_INTERVAL_MS = 6e4;
+/** Warn above this share of the heap ceiling. Chosen so the warning precedes
+*  the fatal by more than one interval under realistic growth. */
+var HEAP_WATCH_WARN_RATIO = .8;
+/**
+* Escalate above this share of the ceiling.
+*
+* 0.70 is 3002MB: clear of the measured routine max (2847MB) so steady state
+* never escalates, and below {@link HEAP_WATCH_WARN_RATIO} so the fast cadence
+* is already running by the time the warn is reachable.
+*/
+var HEAP_WATCH_ESCALATE_RATIO = .7;
+/**
+* Release escalation below this share.
+*
+* Hysteresis, not a second threshold: hub-main's heap sawtooths continuously,
+* so a single ratio would flap the mode on every GC. 0.60 is 2573MB — a real
+* retreat rather than the trough of one collection.
+*/
+var HEAP_WATCH_DEESCALATE_RATIO = .6;
+/**
+* The cadence state machine, pure so the thresholds are testable without
+* allocating three gigabytes to reach them.
+*/
+function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO, deescalateRatio = HEAP_WATCH_DEESCALATE_RATIO) {
+	if (current === "escalated") return usedRatio < deescalateRatio ? "steady" : "escalated";
+	return usedRatio >= escalateRatio ? "escalated" : "steady";
+}
+/**
+* Reclaim once the process holds this many MB beyond what it is USING.
+*
+* Live calibration, both readings from the same hub minutes apart: 2491MB
+* stranded before the reclaim, 438MB after. 438MB is this process's irreducible
+* native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
+* collecting against it would buy nothing and cost a stall every minute.
+*
+* ## Why this came down from 1536 (2026-08-17)
+*
+* The trigger was originally set to fire only on an unmistakable strand. It
+* does, and the pass on the record works exactly as designed:
+*
+*     [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
+*
+* But the number it was protecting against changed. The host now runs with a
+* 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
+* RSS this process is holding and not using is no longer merely untidy — it is
+* the headroom something else needed, and the OOM is triggered by the PEAK, not
+* by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
+* above the floor before anything happens.
+*
+* 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
+* can never trip it, and every pass that does fire still returns roughly
+* `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
+* not a stall spent collecting noise.
+*/
+var HEAP_RECLAIM_TRIGGER_MB = 1024;
+/**
+* Floor on how often the stop-the-world may run.
+*
+* Ten minutes was chosen to keep the worst case under 0.1% of the router's
+* time. The worst case is not what governs the cost, though: the reclaim only
+* runs when {@link shouldReclaim} says the process is genuinely stranded, and
+* after a pass it is at the ~438MB floor. Live off-heap growth was measured at
+* roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
+* takes on the order of **20 minutes** — well outside any floor considered
+* here. In steady state this constant therefore costs nothing at all.
+*
+* What it does govern is a BURST, which is the only case that matters on a host
+* with no swap: the ten-minute floor let a peak that re-formed in seconds stand
+* for ten minutes. Two minutes bounds that, and even the pathological case of a
+* process that re-strands instantly costs 654ms per 120s — 0.55% of the
+* router's time, against a ~1.5GiB peak it removes each pass.
+*/
+var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
+var MB = (bytes) => Math.round(bytes / 1048576);
+/**
+* Build a sample from raw readings. Pure so the thresholds are testable without
+* allocating a gigabyte to reach them.
+*/
+function buildHeapSample(mem, heapLimitBytes, warnRatio = HEAP_WATCH_WARN_RATIO) {
+	const usedRatio = heapLimitBytes > 0 ? mem.heapUsed / heapLimitBytes : 0;
+	return {
+		rssMb: MB(mem.rss),
+		heapUsedMb: MB(mem.heapUsed),
+		heapTotalMb: MB(mem.heapTotal),
+		heapLimitMb: MB(heapLimitBytes),
+		externalMb: MB(mem.external),
+		arrayBuffersMb: MB(mem.arrayBuffers),
+		usedRatio: Math.round(usedRatio * 100) / 100,
+		nearLimit: usedRatio >= warnRatio
+	};
+}
+var consoleSink = {
+	info: (line) => console.log(line),
+	warn: (line) => console.warn(line)
+};
+/**
+* RSS the process is holding but not using.
+*
+* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
+* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
+* and they are precisely what this number exists to see.
+*/
+function strandedMb(s) {
+	return s.rssMb - s.heapUsedMb - s.externalMb;
+}
+/** Whether it is worth paying a stop-the-world to get those pages back. */
+function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
+	return strandedMb(s) > triggerMb;
+}
+function isGcFunction(value) {
+	return typeof value === "function";
+}
+/**
+* Build the reclaimer without requiring `--expose-gc` on argv.
+*
+* The flag is turned on, the function captured, and the flag turned straight
+* back off — a global `gc` left reachable by application code is a footgun, and
+* requiring an argv change would mean this cannot ship without a new image.
+* Verified against the live hub before being written: 266ms, RSS 1453 -> 1237.
+*
+* Returns undefined if V8 declines, so the caller keeps a heartbeat either way.
+*/
+function createV8Reclaimer() {
+	try {
+		v8.setFlagsFromString("--expose-gc");
+		const gc = vm.runInNewContext("gc");
+		v8.setFlagsFromString("--no-expose-gc");
+		if (!isGcFunction(gc)) return void 0;
+		return () => gc({
+			execution: "sync",
+			flavor: "last-resort"
+		});
+	} catch {
+		return;
+	}
+}
+function format(label, s) {
+	return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
+}
+/**
+* Start the heartbeat. Returns a stop function.
+*
+* `unref`ed so it can never hold the process open, and every read is wrapped:
+* a diagnostic that can crash the process it watches is worse than none.
+*/
+function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
+	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
+	const now = reclaimOptions?.now ?? (() => Date.now());
+	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
+	const minIntervalMs = reclaimOptions?.minIntervalMs ?? 12e4;
+	const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
+	const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
+	const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
+	let lastReclaimAt = Number.NEGATIVE_INFINITY;
+	const read = () => {
+		const limit = reclaimOptions?.heapLimitBytes ?? v8.getHeapStatistics().heap_size_limit;
+		return buildHeapSample(readMemory(), limit);
+	};
+	const maybeReclaim = (sample) => {
+		if (reclaimOptions === void 0) return;
+		if (!shouldReclaim(sample, triggerMb)) return;
+		if (now() - lastReclaimAt < minIntervalMs) return;
+		lastReclaimAt = now();
+		const startedAt = now();
+		try {
+			reclaimOptions.reclaim();
+		} catch (error) {
+			sink.warn(`[mem] reclaim failed — ${error instanceof Error ? error.message : String(error)}`);
+			return;
+		}
+		const after = read();
+		sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
+	};
+	let mode = "steady";
+	let lastLoggedAt = Number.NEGATIVE_INFINITY;
+	/**
+	* PROBING and LOGGING are separate cadences, and that separation is the whole
+	* point.
+	*
+	* The 2026-08-08 fatal escaped a 60s heartbeat because the burst began and
+	* ended between two samples. Deciding "is the heap elevated?" only when a
+	* line is due would reproduce that exactly: the process would still be read
+	* once a minute, and a 62-second climb would still be invisible. So the timer
+	* runs at the FAST period always — `process.memoryUsage()` is tens of
+	* microseconds, i.e. free at this rate — and the slow period governs only how
+	* often a steady process is allowed to say so.
+	*
+	* Never probe slower than the log cadence, or a caller passing a short
+	* `intervalMs` would silently get the long one.
+	*/
+	const probeIntervalMs = Math.min(fastIntervalMs, intervalMs);
+	const tick = () => {
+		try {
+			const sample = read();
+			const previous = mode;
+			mode = nextMode(previous, sample.usedRatio, escalateRatio, deescalateRatio);
+			const at = now();
+			const due = at - lastLoggedAt >= intervalMs;
+			if (mode === "escalated" || due) {
+				lastLoggedAt = at;
+				const line = format(label, sample);
+				if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
+				else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
+				else sink.info(line);
+			}
+			if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
+			maybeReclaim(sample);
+		} catch {}
+	};
+	const timer = setInterval(tick, probeIntervalMs);
+	timer.unref?.();
+	tick();
+	let stopped = false;
+	return () => {
+		if (stopped) return;
+		stopped = true;
+		clearInterval(timer);
+	};
+}
+/**
+* Steady heartbeat period for a HEAVY addon runner.
+*
+* hub-main logs once a minute because it is ONE process. The heavy runners are
+* ~7 per node sharing the same hub log stream; at 60s each they would put 7
+* routine lines a minute in it forever, which is how a log stops being read.
+* Five minutes keeps the per-runner record (a leak is still a visible climb
+* across samples) at ~1.4 lines/min across the class.
+*
+* This slows LOGGING only. The probe keeps running at
+* {@link HEAP_WATCH_FAST_INTERVAL_MS} (probe and log cadences are separate —
+* see the comment inside {@link startHeapWatch}), so the reclaim gate and the
+* escalated burst-sampling react exactly as fast as hub-main's.
+*/
+var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
+/**
+* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
+* is a heavy one. Returns the stop function, or undefined when nothing started.
+*
+* ## Why heavy runners get hub-main's mechanism
+*
+* The media-path runners deliberately run with NO old-space ceiling
+* (`maxOldSpaceMb: 0` — a flat cap turned a disk-stall backlog into
+* `Ineffective mark-compacts near heap limit` and killed recording, twice on
+* 2026-08-17). Uncapped, V8 feels no pressure, so a one-off burst becomes the
+* runner's permanent RSS: the recorder measured 2122MB holding a ~50MB index.
+* hub-main had exactly this shape and its reclaimer is proven live — 11 passes
+* in 4h, 375–1652MB returned per pass at 550–680ms each. Same mechanism, same
+* `shouldReclaim` stranded-over-trigger gate; never a second implementation.
+*
+* ## The stall, priced for the runners that get it
+*
+* A pass is a sub-second stop-the-world at most once per
+* {@link HEAP_RECLAIM_MIN_INTERVAL_MS}, and only while >1GiB is stranded. For
+* the recorder — finalizing a segment every ~300ms across 34 profile-writers —
+* the pause delays WATCHER work, not video: ffmpeg children keep writing to
+* disk independently, the playlist tail is the durable queue, and the next
+* tick drains what accrued. For stream-broker a pass can hitch live frames
+* once per qualifying window; after a pass the runner sits at its floor and
+* re-stranding the trigger takes tens of minutes, so in practice the class
+* pays a few passes per hour against the multi-GiB peak that was OOM-killing
+* a neighbouring VM on a host with no swap.
+*/
+function startRunnerHeapWatch(options) {
+	if (options.heapProfile !== "heavy") return void 0;
+	const intervalMs = options.intervalMs ?? 3e5;
+	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs);
+	let reclaimOptions = options.reclaimOptions;
+	if (reclaimOptions === void 0) {
+		const reclaimer = createV8Reclaimer();
+		reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
+	}
+	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions);
+}
+//#endregion
 //#region src/kernel/moleculer/addon-data-plane-facility.ts
 /**
 * Kernel side of the addon HTTP **data-plane** (see {@link AddonDataPlane}).
@@ -7694,4 +8048,4 @@ async function installManifestPythonDeps(declaration, addonDir, deps, logger) {
 	await deps.installPythonRequirements(reqAbs);
 }
 //#endregion
-export { buildNativeCapProxy as $, createHubCapForwardService as A, AGENT_CAP_FWD_ACTION as B, brokerTransportLink as C, runNpm as Ct, localProviderLink as D, ipcParentLink as E, createUdsLogger as F, callWithServiceDiscovery as G, CapRouteResolver as H, createUdsLoggerWithControl as I, UdsLocalTransportServer as J, createLocalTransport as K, LocalChildClient as L, createUdsEventBridge as M, createUdsEventBus as N, HUB_CAP_FWD_ACTION as O, udsChildLogToWorkerEntry as P, encodeFrame as Q, LocalChildRegistry as R, brokerCallForCap as S, resolveNpmInvocation as St, ipcChildLink as T, CapRouteError as U, AGENT_CAP_FWD_SERVICE as V, classifyCapRoute as W, localEndpointPath as X, SocketChannel as Y, FrameDecoder as Z, createKernelHwAccel as _, CapabilityHandle as _t, getWorkerDeviceRegistry as a, setWorkerNativeCapsChangeListener as at, __resetCapUsageRegistryForTests as b, installManifestNativeDeps as bt, setHubConnected as c, NATIVE_PROVIDER_SERVICE_INFIX as ct, getBrokerEventBus as d, capBareAction as dt, buildUdsNativeCapProxy as et, getMoleculerEventStats as f, capServiceName as ft, AddonDepsManager as g, DeviceRegistry as gt, subscribePassthrough as h, serializeTypedArrays as ht, createUdsAddonContext as i, mountNativeCapService as it, createParentUnownedCallHandler as j, HUB_CAP_FWD_SERVICE as k, EVENT_TOPIC_PREFIX as l, capActionName as lt, setNodeEventInterest as m, deserializeTypedArrays as mt, adaptBrokerToCluster as n, getWorkerNativeCapProvider as nt, getOrInitReadinessRegistry as o, createAddonService as ot, registerEventBusService as p, parseCapAction as pt, UdsLocalTransportClient as q, createAddonContext as r, getWorkerNativeCapSnapshot as rt, getOrInitReadinessRegistryForClient as s, validateProviderRegistrations as st, installManifestPythonDeps as t, createBrokerDeviceManagerApi as tt, clusterEventTopic as u, capActionSuffix as ut, resolveHwAccel as v, CapabilityUnavailableError as vt, buildLinkChain as w, createAddonDataPlaneFacility as wt, getCapUsageRegistry as x, resolveAddonClass as xt, CapUsageRegistry as y, copyBundledNativeModules as yt, UDS_NO_ROUTE_PREFIX as z };
+export { buildNativeCapProxy as $, createHubCapForwardService as A, buildHeapSample as At, AGENT_CAP_FWD_ACTION as B, brokerTransportLink as C, runNpm as Ct, localProviderLink as D, HEAP_WATCH_INTERVAL_MS as Dt, ipcParentLink as E, HEAP_RECLAIM_TRIGGER_MB as Et, createUdsLogger as F, strandedMb as Ft, callWithServiceDiscovery as G, CapRouteResolver as H, createUdsLoggerWithControl as I, UdsLocalTransportServer as J, createLocalTransport as K, LocalChildClient as L, createUdsEventBridge as M, shouldReclaim as Mt, createUdsEventBus as N, startHeapWatch as Nt, HUB_CAP_FWD_ACTION as O, HEAP_WATCH_WARN_RATIO as Ot, udsChildLogToWorkerEntry as P, startRunnerHeapWatch as Pt, encodeFrame as Q, LocalChildRegistry as R, brokerCallForCap as S, resolveNpmInvocation as St, ipcChildLink as T, HEAP_RECLAIM_MIN_INTERVAL_MS as Tt, CapRouteError as U, AGENT_CAP_FWD_SERVICE as V, classifyCapRoute as W, localEndpointPath as X, SocketChannel as Y, FrameDecoder as Z, createKernelHwAccel as _, CapabilityHandle as _t, getWorkerDeviceRegistry as a, setWorkerNativeCapsChangeListener as at, __resetCapUsageRegistryForTests as b, installManifestNativeDeps as bt, setHubConnected as c, NATIVE_PROVIDER_SERVICE_INFIX as ct, getBrokerEventBus as d, capBareAction as dt, buildUdsNativeCapProxy as et, getMoleculerEventStats as f, capServiceName as ft, AddonDepsManager as g, DeviceRegistry as gt, subscribePassthrough as h, serializeTypedArrays as ht, createUdsAddonContext as i, mountNativeCapService as it, createParentUnownedCallHandler as j, createV8Reclaimer as jt, HUB_CAP_FWD_SERVICE as k, RUNNER_HEAP_WATCH_INTERVAL_MS as kt, EVENT_TOPIC_PREFIX as l, capActionName as lt, setNodeEventInterest as m, deserializeTypedArrays as mt, adaptBrokerToCluster as n, getWorkerNativeCapProvider as nt, getOrInitReadinessRegistry as o, createAddonService as ot, registerEventBusService as p, parseCapAction as pt, UdsLocalTransportClient as q, createAddonContext as r, getWorkerNativeCapSnapshot as rt, getOrInitReadinessRegistryForClient as s, validateProviderRegistrations as st, installManifestPythonDeps as t, createBrokerDeviceManagerApi as tt, clusterEventTopic as u, capActionSuffix as ut, resolveHwAccel as v, CapabilityUnavailableError as vt, buildLinkChain as w, createAddonDataPlaneFacility as wt, getCapUsageRegistry as x, resolveAddonClass as xt, CapUsageRegistry as y, copyBundledNativeModules as yt, UDS_NO_ROUTE_PREFIX as z };