@camstack/system 1.2.103 → 1.2.104

package/dist/index.js

@@ -1,6 +1,6 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_chunk = require("./chunk-Cek0wNdY.js");
-const require_dist = require("./dist-qnwy_iWM.js");
+const require_dist = require("./dist-DR1VmGz6.js");
 const require_builtins_alerts_alerts_addon = require("./builtins/alerts/alerts.addon.js");
 require("./builtins/alerts/index.js");
 const require_formatter = require("./formatter-DqAKDlvN.js");
@@ -25,7 +25,7 @@ const require_builtins_system_config_system_config_addon = require("./builtins/s
 require("./builtins/system-config/index.js");
 const require_builtins_winston_logging_index = require("./builtins/winston-logging/index.js");
 const require_file_data_plane = require("./file-data-plane-DUHPHa-Y.js");
-const require_manifest_python_deps = require("./manifest-python-deps-BtVjvfvj.js");
+const require_manifest_python_deps = require("./manifest-python-deps-GejnH--L.js");
 const require_resource_monitor = require("./resource-monitor-CdnzxBLP.js");
 const require_custom_action_registry = require("./custom-action-registry-jY0NOZK8.js");
 let zod = require("zod");
@@ -43,8 +43,6 @@ let node_fs = require("node:fs");
 let node_fs$1 = require_chunk.__toESM(node_fs, 1);
 node_fs = require_chunk.__toESM(node_fs);
 let node_http = require("node:http");
-let node_v8 = require("node:v8");
-node_v8 = require_chunk.__toESM(node_v8);
 let node_vm = require("node:vm");
 node_vm = require_chunk.__toESM(node_vm);
 let _camstack_types_addon = require("@camstack/types/addon");
@@ -431,304 +429,6 @@ function proxyToUpstream(opts) {
 	else clientReq.pipe(upstream);
 }
 //#endregion
-//#region src/kernel/heap-watch.ts
-/**
-* heap-watch — hub-main's own memory, on the record, and returned to the OS.
-*
-* ## Why this exists
-*
-* hub-main used to be launched with `--max-old-space-size=1536` (docker-entrypoint;
-* the cap was removed on 2026-08-02, so the live ceiling is now V8's own default —
-* 4288MB in this container — and this module reads it at runtime rather than
-* assuming a number). It logged NOTHING about its own heap. When it died on
-* 2026-08-01/02 the only trace was V8's own stderr line:
-*
-*     FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
-*
-* which carries no process name, no heap number and no history — the whole
-* server restarts and the evidence is gone. Four of those in ~15h, and the one
-* at 06:57:57 landed nine seconds after a viewer opened a WebRTC session, which
-* is why the app sat in a permanent "loading": the hub was dying as it
-* connected.
-*
-* `[mem]` reports already exist for forked addon runners (SIGUSR2 in
-* `addon-runner.ts`) — hub-main, the one process that actually OOMs, was the
-* one process with no telemetry at all.
-*
-* ## What it gives
-*
-* A heartbeat carrying the numbers that separate the two possible stories, and
-* a WARN as the heap approaches the ceiling — the line that will exist BEFORE
-* the next fatal instead of after it:
-*
-*  - `heapUsedMb` climbing steadily between restarts → a leak;
-*  - `heapUsedMb` flat with a sharp jump under load → a burst allocation, and
-*    the timestamp says what was being served.
-*
-* `external`/`arrayBuffers` ride along because they are NOT bounded by
-* `--max-old-space-size` and are most of the gap between a 1.5GB cap and the
-* ~2.05GB RSS observed live: a diagnosis that only looks at the JS heap
-* misreads that process by 500MB.
-*
-* ## And then the heartbeat explained the "off-heap leak", which was not one
-*
-* 2026-08-08, measured on the live hub over 1019 consecutive samples and seven
-* separate boots: **hub-main's RSS is a HIGH-WATER MARK, not a leak.** In every
-* one of those 1019 samples RSS was >= the running maximum of
-* `heapTotal + external`, never once below it, and
-* `RSS_peak - peak(heapTotal + external)` landed between 874MB and 1639MB on
-* every boot — a roughly constant native floor on top of a number that only
-* ever goes up. RSS tracked the highest point the process ever reached and
-* stayed there: 3.3GB -> 4.5GB -> 5.2GB, and 7.8GB on the worst boot.
-*
-* Two things push that peak, and neither is a retained object:
-*
-*  - the JS heap sawtooths to 2.0-2.8GB because the ceiling is 4288MB and
-*    nothing creates pressure before then;
-*  - `arrayBuffers` accumulates DEAD Buffers — up to 3.65GB — because V8 only
-*    sweeps them on a major GC. Eleven collapses were captured, and every
-*    single one coincides with `heapUsed` dropping to its ~650MB floor. No code
-*    released a reference at 13:23:03; a mark-compact ran.
-*
-* At the peak V8 has committed the pages, and **an ordinary major GC does not
-* give them back.** Proven on the production hub: `smaps` showed 10,819 mapped
-* 256KB pages (V8's `kRegularPageSize`) holding 2704MB while V8 itself reported
-* `total_physical_size` of only 846MB — ~7,800 pages committed, resident, and
-* outside V8's own accounting. One `gc({ flavor: 'last-resort' })` took 417ms
-* and returned RSS 3355MB -> 1237MB, pages 10,819 -> 2,963.
-*
-* So the fix for the "~3GB/2h off-heap growth" is not a retainer hunt. It is to
-* ask V8 for the pages back when the process is visibly holding more than it is
-* using — which is what `shouldReclaim` decides and what the heartbeat now does.
-*
-* The cost is a stop-the-world on the hub's router process — 250-450ms when
-* first measured, 654ms on the largest pass seen since — bounded to once per
-* `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a ratchet to
-* 7.8GB and the OOM that ends it.
-*/
-/** Heartbeat period. Slow enough to be free, fast enough that the sample before
-*  a fatal is recent. */
-var HEAP_WATCH_INTERVAL_MS = 6e4;
-/** Warn above this share of the heap ceiling. Chosen so the warning precedes
-*  the fatal by more than one interval under realistic growth. */
-var HEAP_WATCH_WARN_RATIO = .8;
-/**
-* Escalate above this share of the ceiling.
-*
-* 0.70 is 3002MB: clear of the measured routine max (2847MB) so steady state
-* never escalates, and below {@link HEAP_WATCH_WARN_RATIO} so the fast cadence
-* is already running by the time the warn is reachable.
-*/
-var HEAP_WATCH_ESCALATE_RATIO = .7;
-/**
-* Release escalation below this share.
-*
-* Hysteresis, not a second threshold: hub-main's heap sawtooths continuously,
-* so a single ratio would flap the mode on every GC. 0.60 is 2573MB — a real
-* retreat rather than the trough of one collection.
-*/
-var HEAP_WATCH_DEESCALATE_RATIO = .6;
-/**
-* The cadence state machine, pure so the thresholds are testable without
-* allocating three gigabytes to reach them.
-*/
-function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO, deescalateRatio = HEAP_WATCH_DEESCALATE_RATIO) {
-	if (current === "escalated") return usedRatio < deescalateRatio ? "steady" : "escalated";
-	return usedRatio >= escalateRatio ? "escalated" : "steady";
-}
-/**
-* Reclaim once the process holds this many MB beyond what it is USING.
-*
-* Live calibration, both readings from the same hub minutes apart: 2491MB
-* stranded before the reclaim, 438MB after. 438MB is this process's irreducible
-* native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
-* collecting against it would buy nothing and cost a stall every minute.
-*
-* ## Why this came down from 1536 (2026-08-17)
-*
-* The trigger was originally set to fire only on an unmistakable strand. It
-* does, and the pass on the record works exactly as designed:
-*
-*     [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
-*
-* But the number it was protecting against changed. The host now runs with a
-* 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
-* RSS this process is holding and not using is no longer merely untidy — it is
-* the headroom something else needed, and the OOM is triggered by the PEAK, not
-* by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
-* above the floor before anything happens.
-*
-* 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
-* can never trip it, and every pass that does fire still returns roughly
-* `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
-* not a stall spent collecting noise.
-*/
-var HEAP_RECLAIM_TRIGGER_MB = 1024;
-/**
-* Floor on how often the stop-the-world may run.
-*
-* Ten minutes was chosen to keep the worst case under 0.1% of the router's
-* time. The worst case is not what governs the cost, though: the reclaim only
-* runs when {@link shouldReclaim} says the process is genuinely stranded, and
-* after a pass it is at the ~438MB floor. Live off-heap growth was measured at
-* roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
-* takes on the order of **20 minutes** — well outside any floor considered
-* here. In steady state this constant therefore costs nothing at all.
-*
-* What it does govern is a BURST, which is the only case that matters on a host
-* with no swap: the ten-minute floor let a peak that re-formed in seconds stand
-* for ten minutes. Two minutes bounds that, and even the pathological case of a
-* process that re-strands instantly costs 654ms per 120s — 0.55% of the
-* router's time, against a ~1.5GiB peak it removes each pass.
-*/
-var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
-var MB = (bytes) => Math.round(bytes / 1048576);
-/**
-* Build a sample from raw readings. Pure so the thresholds are testable without
-* allocating a gigabyte to reach them.
-*/
-function buildHeapSample(mem, heapLimitBytes, warnRatio = HEAP_WATCH_WARN_RATIO) {
-	const usedRatio = heapLimitBytes > 0 ? mem.heapUsed / heapLimitBytes : 0;
-	return {
-		rssMb: MB(mem.rss),
-		heapUsedMb: MB(mem.heapUsed),
-		heapTotalMb: MB(mem.heapTotal),
-		heapLimitMb: MB(heapLimitBytes),
-		externalMb: MB(mem.external),
-		arrayBuffersMb: MB(mem.arrayBuffers),
-		usedRatio: Math.round(usedRatio * 100) / 100,
-		nearLimit: usedRatio >= warnRatio
-	};
-}
-var consoleSink = {
-	info: (line) => console.log(line),
-	warn: (line) => console.warn(line)
-};
-/**
-* RSS the process is holding but not using.
-*
-* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
-* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
-* and they are precisely what this number exists to see.
-*/
-function strandedMb(s) {
-	return s.rssMb - s.heapUsedMb - s.externalMb;
-}
-/** Whether it is worth paying a stop-the-world to get those pages back. */
-function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
-	return strandedMb(s) > triggerMb;
-}
-function isGcFunction(value) {
-	return typeof value === "function";
-}
-/**
-* Build the reclaimer without requiring `--expose-gc` on argv.
-*
-* The flag is turned on, the function captured, and the flag turned straight
-* back off — a global `gc` left reachable by application code is a footgun, and
-* requiring an argv change would mean this cannot ship without a new image.
-* Verified against the live hub before being written: 266ms, RSS 1453 -> 1237.
-*
-* Returns undefined if V8 declines, so the caller keeps a heartbeat either way.
-*/
-function createV8Reclaimer() {
-	try {
-		node_v8.setFlagsFromString("--expose-gc");
-		const gc = node_vm.runInNewContext("gc");
-		node_v8.setFlagsFromString("--no-expose-gc");
-		if (!isGcFunction(gc)) return void 0;
-		return () => gc({
-			execution: "sync",
-			flavor: "last-resort"
-		});
-	} catch {
-		return;
-	}
-}
-function format$1(label, s) {
-	return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
-}
-/**
-* Start the heartbeat. Returns a stop function.
-*
-* `unref`ed so it can never hold the process open, and every read is wrapped:
-* a diagnostic that can crash the process it watches is worse than none.
-*/
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
-	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
-	const now = reclaimOptions?.now ?? (() => Date.now());
-	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
-	const minIntervalMs = reclaimOptions?.minIntervalMs ?? 12e4;
-	const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
-	const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
-	const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
-	let lastReclaimAt = Number.NEGATIVE_INFINITY;
-	const read = () => {
-		const limit = reclaimOptions?.heapLimitBytes ?? node_v8.getHeapStatistics().heap_size_limit;
-		return buildHeapSample(readMemory(), limit);
-	};
-	const maybeReclaim = (sample) => {
-		if (reclaimOptions === void 0) return;
-		if (!shouldReclaim(sample, triggerMb)) return;
-		if (now() - lastReclaimAt < minIntervalMs) return;
-		lastReclaimAt = now();
-		const startedAt = now();
-		try {
-			reclaimOptions.reclaim();
-		} catch (error) {
-			sink.warn(`[mem] reclaim failed — ${error instanceof Error ? error.message : String(error)}`);
-			return;
-		}
-		const after = read();
-		sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
-	};
-	let mode = "steady";
-	let lastLoggedAt = Number.NEGATIVE_INFINITY;
-	/**
-	* PROBING and LOGGING are separate cadences, and that separation is the whole
-	* point.
-	*
-	* The 2026-08-08 fatal escaped a 60s heartbeat because the burst began and
-	* ended between two samples. Deciding "is the heap elevated?" only when a
-	* line is due would reproduce that exactly: the process would still be read
-	* once a minute, and a 62-second climb would still be invisible. So the timer
-	* runs at the FAST period always — `process.memoryUsage()` is tens of
-	* microseconds, i.e. free at this rate — and the slow period governs only how
-	* often a steady process is allowed to say so.
-	*
-	* Never probe slower than the log cadence, or a caller passing a short
-	* `intervalMs` would silently get the long one.
-	*/
-	const probeIntervalMs = Math.min(fastIntervalMs, intervalMs);
-	const tick = () => {
-		try {
-			const sample = read();
-			const previous = mode;
-			mode = nextMode(previous, sample.usedRatio, escalateRatio, deescalateRatio);
-			const at = now();
-			const due = at - lastLoggedAt >= intervalMs;
-			if (mode === "escalated" || due) {
-				lastLoggedAt = at;
-				const line = format$1(label, sample);
-				if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
-				else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
-				else sink.info(line);
-			}
-			if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
-			maybeReclaim(sample);
-		} catch {}
-	};
-	const timer = setInterval(tick, probeIntervalMs);
-	timer.unref?.();
-	tick();
-	let stopped = false;
-	return () => {
-		if (stopped) return;
-		stopped = true;
-		clearInterval(timer);
-	};
-}
-//#endregion
 //#region src/lifecycle/lifecycle-state-machine.ts
 var VALID_TRANSITIONS = {
 	stopped: ["starting", "disabled"],
@@ -94653,19 +94353,21 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
 	function spawnRunner(runnerId, addons, env) {
 		const nodeId = buildNodeId(runnerId);
 		const runnerPath = node_path.resolve(__dirname, "addon-runner.js");
+		const heavy = isHeavyRunner(addons);
 		const childEnv = {
 			...process.env,
 			CAMSTACK_RUNNER_ID: runnerId,
 			CAMSTACK_RUNNER_ADDONS: JSON.stringify(addons),
 			CAMSTACK_PARENT_NODE_ID: parentNodeId,
 			CAMSTACK_DATA_DIR: dataDir,
+			CAMSTACK_RUNNER_HEAP_PROFILE: heavy ? "heavy" : "light",
 			CAMSTACK_LOG_LEVEL: process.env["CAMSTACK_LOG_LEVEL"] ?? "info",
 			...parentTcpPort !== void 0 ? { CAMSTACK_PARENT_TCP_PORT: String(parentTcpPort) } : {},
 			...parentUdsPath !== void 0 ? { CAMSTACK_PARENT_UDS_PATH: parentUdsPath } : {},
 			...env
 		};
 		const heapFlags = runnerHeapFlags(addons);
-		capturedBroker?.logger.info(`[${runnerId}] heap profile: ${isHeavyRunner(addons) ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
+		capturedBroker?.logger.info(`[${runnerId}] heap profile: ${heavy ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
 		const child = spawnFn(process.execPath, [...heapFlags, runnerPath], {
 			env: childEnv,
 			stdio: [
@@ -94709,6 +94411,15 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
 				capturedBroker?.logger.error(`[${runnerId}] Crash circuit-breaker tripped — ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms. Not respawning.`);
 				processes.set(runnerId, entry);
 				if (capturedBroker) forceMoleculerDisconnect(capturedBroker, entry.nodeId);
+				emitRunnerEvent(require_dist.EventCategory.AddonRunnerFailed, runnerId, {
+					runnerId,
+					nodeId: parentNodeId,
+					addonIds: entry.runnerAddons.map((addon) => addon.addonId),
+					crashesInWindow: decision.crashesInWindow,
+					windowMs: CRASH_WINDOW_MS,
+					...code !== null ? { exitCode: code } : {},
+					...signal !== null ? { signal } : {}
+				});
 				for (const addon of entry.runnerAddons) emitRunnerEvent(require_dist.EventCategory.AddonCrashed, addon.addonId, {
 					addonId: addon.addonId,
 					error: `Crash circuit-breaker tripped after ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms`
@@ -95435,10 +95146,10 @@ exports.FsStorageBackend = FsStorageBackend;
 exports.HEALTH_MONITOR_GRACE_PERIOD_MS = HEALTH_MONITOR_GRACE_PERIOD_MS;
 exports.HEALTH_MONITOR_RETRY_INTERVALS_MS = HEALTH_MONITOR_RETRY_INTERVALS_MS;
 exports.HEALTH_MONITOR_TICK_MS = HEALTH_MONITOR_TICK_MS;
-exports.HEAP_RECLAIM_MIN_INTERVAL_MS = HEAP_RECLAIM_MIN_INTERVAL_MS;
-exports.HEAP_RECLAIM_TRIGGER_MB = HEAP_RECLAIM_TRIGGER_MB;
-exports.HEAP_WATCH_INTERVAL_MS = HEAP_WATCH_INTERVAL_MS;
-exports.HEAP_WATCH_WARN_RATIO = HEAP_WATCH_WARN_RATIO;
+exports.HEAP_RECLAIM_MIN_INTERVAL_MS = require_manifest_python_deps.HEAP_RECLAIM_MIN_INTERVAL_MS;
+exports.HEAP_RECLAIM_TRIGGER_MB = require_manifest_python_deps.HEAP_RECLAIM_TRIGGER_MB;
+exports.HEAP_WATCH_INTERVAL_MS = require_manifest_python_deps.HEAP_WATCH_INTERVAL_MS;
+exports.HEAP_WATCH_WARN_RATIO = require_manifest_python_deps.HEAP_WATCH_WARN_RATIO;
 exports.HUB_CAP_FWD_ACTION = require_manifest_python_deps.HUB_CAP_FWD_ACTION;
 exports.HUB_CAP_FWD_SERVICE = require_manifest_python_deps.HUB_CAP_FWD_SERVICE;
 exports.HubForwarderAddon = require_builtins_hub_forwarder_index.HubForwarderAddon$1;
@@ -95477,6 +95188,7 @@ exports.PipelineValidator = PipelineValidator;
 exports.PythonEnvManager = PythonEnvManager;
 exports.QUARANTINE_DIRNAME = QUARANTINE_DIRNAME;
 exports.RESTART_MARKER_FILE = RESTART_MARKER_FILE;
+exports.RUNNER_HEAP_WATCH_INTERVAL_MS = require_manifest_python_deps.RUNNER_HEAP_WATCH_INTERVAL_MS;
 exports.RUNTIME_DEFAULTS = require_dist.RUNTIME_DEFAULTS;
 exports.ReadinessRegistry = require_dist.ReadinessRegistry;
 exports.ReadinessTimeoutError = require_dist.ReadinessTimeoutError;
@@ -95515,7 +95227,7 @@ Object.defineProperty(exports, "buildBinaryPath", {
 	}
 });
 exports.buildCapRouters = buildCapRouters;
-exports.buildHeapSample = buildHeapSample;
+exports.buildHeapSample = require_manifest_python_deps.buildHeapSample;
 exports.buildLinkChain = require_manifest_python_deps.buildLinkChain;
 exports.buildNativeCapProxy = require_manifest_python_deps.buildNativeCapProxy;
 exports.buildNodeManifest = buildNodeManifest;
@@ -95560,7 +95272,7 @@ exports.createUdsEventBridge = require_manifest_python_deps.createUdsEventBridge
 exports.createUdsEventBus = require_manifest_python_deps.createUdsEventBus;
 exports.createUdsLogger = require_manifest_python_deps.createUdsLogger;
 exports.createUdsLoggerWithControl = require_manifest_python_deps.createUdsLoggerWithControl;
-exports.createV8Reclaimer = createV8Reclaimer;
+exports.createV8Reclaimer = require_manifest_python_deps.createV8Reclaimer;
 exports.deleteModelFromDisk = require_file_data_plane.deleteModelFromDisk;
 exports.deriveAgentListenPort = deriveAgentListenPort;
 exports.describeProviderKindDrift = describeProviderKindDrift;
@@ -95686,9 +95398,10 @@ exports.selectAddonResidue = selectAddonResidue;
 exports.serializeTypedArrays = require_manifest_python_deps.serializeTypedArrays;
 exports.setHubConnected = require_manifest_python_deps.setHubConnected;
 exports.setNodeEventInterest = require_manifest_python_deps.setNodeEventInterest;
-exports.shouldReclaim = shouldReclaim;
-exports.startHeapWatch = startHeapWatch;
-exports.strandedMb = strandedMb;
+exports.shouldReclaim = require_manifest_python_deps.shouldReclaim;
+exports.startHeapWatch = require_manifest_python_deps.startHeapWatch;
+exports.startRunnerHeapWatch = require_manifest_python_deps.startRunnerHeapWatch;
+exports.strandedMb = require_manifest_python_deps.strandedMb;
 exports.stripCamstackDeps = stripCamstackDeps;
 exports.subscribePassthrough = require_manifest_python_deps.subscribePassthrough;
 exports.udsChildLogToWorkerEntry = require_manifest_python_deps.udsChildLogToWorkerEntry;