@camstack/system 1.2.122 → 1.2.124

package/dist/{manifest-python-deps-DZSNlHjU.mjs → manifest-python-deps-D1Ydw7J0.mjs}

@@ -10,6 +10,7 @@ import * as os from "node:os";
 import { tmpdir } from "node:os";
 import * as fs from "node:fs";
 import { createServer } from "node:http";
+import { monitorEventLoopDelay } from "node:perf_hooks";
 import * as v8 from "node:v8";
 import * as vm from "node:vm";
 import { BOOT_RECOVERY_BACKOFF_MS, DATAPLANE_SECRET_HEADER, DeviceType, DisposerChain, EventCategory, ReadinessRegistry, asJsonObject, asString, createDeviceProxy, createEventBusSliceSource, deviceOpsCapability, emitReadiness, errMsg, expandCapMethods, readNodePin, scopeKey, sleep, withCapInputDefaults } from "@camstack/types/addon";
@@ -166,6 +167,72 @@ var HEAP_RECLAIM_TRIGGER_MB = 1024;
 * router's time, against a ~1.5GiB peak it removes each pass.
 */
 var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
+/** What a loop with nothing recorded yet reports. */
+var ZERO_LOOP_DELAY = {
+	p50Ms: 0,
+	p99Ms: 0,
+	maxMs: 0
+};
+/**
+* A `perf_hooks` interval histogram, read once per LOGGED line.
+*
+* The histogram samples in libuv at `resolutionMs` and costs nothing to keep
+* enabled; the expensive part would be reading it on every 2s probe, which
+* would also destroy the window the line claims to describe (probe and log
+* cadences are deliberately different — see {@link startHeapWatch}).
+*
+* An EMPTY histogram is the trap: Node answers `min = 9223372036854776000` and
+* `percentile()` throws on some versions, so a first heartbeat that read it
+* naively would print `loopMax=9223372036854ms` and be read as a catastrophe on
+* every boot. `count === 0` short-circuits to {@link ZERO_LOOP_DELAY}.
+*
+* Returns undefined if the platform declines, so the caller keeps a heartbeat
+* either way.
+*/
+function createLoopDelayMeter(resolutionMs = 20) {
+	try {
+		const histogram = monitorEventLoopDelay({ resolution: resolutionMs });
+		histogram.enable();
+		return {
+			read: () => {
+				try {
+					if (histogram.count === 0) return ZERO_LOOP_DELAY;
+					const sample = {
+						p50Ms: Math.round(histogram.percentile(50) / 1e6),
+						p99Ms: Math.round(histogram.percentile(99) / 1e6),
+						maxMs: Math.round(histogram.max / 1e6)
+					};
+					histogram.reset();
+					return sample;
+				} catch {
+					return ZERO_LOOP_DELAY;
+				}
+			},
+			stop: () => {
+				histogram.disable();
+			}
+		};
+	} catch {
+		return;
+	}
+}
+/**
+* Read the origin of the old-space ceiling off node's own options.
+*
+* hub-main is the only process in the cluster whose ceiling nobody chose. Every
+* forked runner is given V8 heap flags by `process-service.ts`; hub-main gets
+* `--max-old-space-size` from `docker-entrypoint.sh` ONLY when the operator sets
+* `CAMSTACK_HUB_HEAP_MB`, and on the live container it is unset — verified
+* 2026-08-25 by reading pid 7's argv, a bare `node .../starter.js`. So the
+* ceiling in force is whatever V8 derives from the host (4288MB there), and the
+* only record of that was a comment inside an image nobody rebuilds to read.
+*
+* A fact about the running process belongs in the running process's log, and
+* `execArgv` answers it for free at boot.
+*/
+function heapCeilingOrigin(execArgv = process.execArgv) {
+	return execArgv.some((arg) => /^--max[-_]old[-_]space[-_]size(=|$)/.test(arg)) ? "explicit" : "v8-default";
+}
 var MB = (bytes) => Math.round(bytes / 1048576);
 /**
 * Build a sample from raw readings. Pure so the thresholds are testable without
@@ -229,8 +296,10 @@ function createV8Reclaimer() {
 		return;
 	}
 }
-function format(label, s) {
-	return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
+function format(label, s, loop) {
+	const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
+	if (loop === void 0) return line;
+	return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
 }
 /**
 * Start the heartbeat. Returns a stop function.
@@ -238,7 +307,7 @@ function format(label, s) {
 * `unref`ed so it can never hold the process open, and every read is wrapped:
 * a diagnostic that can crash the process it watches is worse than none.
 */
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
+function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
 	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
 	const now = reclaimOptions?.now ?? (() => Date.now());
 	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
@@ -247,14 +316,24 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 	const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
 	const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
 	let lastReclaimAt = Number.NEGATIVE_INFINITY;
+	/** Consecutive passes that fired the instant the floor elapsed. See
+	*  {@link RECLAIM_STEADY_STATE_PASSES}. */
+	let passesAtFloor = 0;
+	let steadyStateAnnounced = false;
 	const read = () => {
 		const limit = reclaimOptions?.heapLimitBytes ?? v8.getHeapStatistics().heap_size_limit;
 		return buildHeapSample(readMemory(), limit);
 	};
 	const maybeReclaim = (sample) => {
 		if (reclaimOptions === void 0) return;
-		if (!shouldReclaim(sample, triggerMb)) return;
-		if (now() - lastReclaimAt < minIntervalMs) return;
+		if (!shouldReclaim(sample, triggerMb)) {
+			passesAtFloor = 0;
+			steadyStateAnnounced = false;
+			return;
+		}
+		const sinceLast = now() - lastReclaimAt;
+		if (sinceLast < minIntervalMs) return;
+		passesAtFloor = sinceLast <= minIntervalMs * 2 ? passesAtFloor + 1 : 1;
 		lastReclaimAt = now();
 		const startedAt = now();
 		try {
@@ -265,6 +344,10 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 		}
 		const after = read();
 		sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
+		if (passesAtFloor >= 6 && !steadyStateAnnounced) {
+			steadyStateAnnounced = true;
+			sink.warn(`[mem] reclaim ${label} has run at the floor for ${passesAtFloor} consecutive passes (every ${minIntervalMs}ms) — the stop-the-world is now STEADY STATE, not a rescue. The allocation source is what needs the fix; this pass only bounds its peak.`);
+		}
 	};
 	let mode = "steady";
 	let lastLoggedAt = Number.NEGATIVE_INFINITY;
@@ -293,7 +376,7 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 			const due = at - lastLoggedAt >= intervalMs;
 			if (mode === "escalated" || due) {
 				lastLoggedAt = at;
-				const line = format(label, sample);
+				const line = format(label, sample, loopDelay?.read());
 				if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
 				else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
 				else sink.info(line);
@@ -305,11 +388,13 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 	const timer = setInterval(tick, probeIntervalMs);
 	timer.unref?.();
 	tick();
+	if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
 	let stopped = false;
 	return () => {
 		if (stopped) return;
 		stopped = true;
 		clearInterval(timer);
+		loopDelay?.stop();
 	};
 }
 /**
@@ -358,13 +443,13 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
 function startRunnerHeapWatch(options) {
 	if (options.heapProfile !== "heavy") return void 0;
 	const intervalMs = options.intervalMs ?? 3e5;
-	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs);
+	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false);
 	let reclaimOptions = options.reclaimOptions;
 	if (reclaimOptions === void 0) {
 		const reclaimer = createV8Reclaimer();
 		reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
 	}
-	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions);
+	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, false);
 }
 //#endregion
 //#region src/kernel/moleculer/addon-data-plane-facility.ts

package/dist/{manifest-python-deps-76dZLxD0.js → manifest-python-deps-DB_4H3tR.js}

@@ -12,6 +12,7 @@ node_os = require_chunk.__toESM(node_os);
 let node_fs = require("node:fs");
 node_fs = require_chunk.__toESM(node_fs);
 let node_http = require("node:http");
+let node_perf_hooks = require("node:perf_hooks");
 let node_v8 = require("node:v8");
 node_v8 = require_chunk.__toESM(node_v8);
 let node_vm = require("node:vm");
@@ -170,6 +171,72 @@ var HEAP_RECLAIM_TRIGGER_MB = 1024;
 * router's time, against a ~1.5GiB peak it removes each pass.
 */
 var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
+/** What a loop with nothing recorded yet reports. */
+var ZERO_LOOP_DELAY = {
+	p50Ms: 0,
+	p99Ms: 0,
+	maxMs: 0
+};
+/**
+* A `perf_hooks` interval histogram, read once per LOGGED line.
+*
+* The histogram samples in libuv at `resolutionMs` and costs nothing to keep
+* enabled; the expensive part would be reading it on every 2s probe, which
+* would also destroy the window the line claims to describe (probe and log
+* cadences are deliberately different — see {@link startHeapWatch}).
+*
+* An EMPTY histogram is the trap: Node answers `min = 9223372036854776000` and
+* `percentile()` throws on some versions, so a first heartbeat that read it
+* naively would print `loopMax=9223372036854ms` and be read as a catastrophe on
+* every boot. `count === 0` short-circuits to {@link ZERO_LOOP_DELAY}.
+*
+* Returns undefined if the platform declines, so the caller keeps a heartbeat
+* either way.
+*/
+function createLoopDelayMeter(resolutionMs = 20) {
+	try {
+		const histogram = (0, node_perf_hooks.monitorEventLoopDelay)({ resolution: resolutionMs });
+		histogram.enable();
+		return {
+			read: () => {
+				try {
+					if (histogram.count === 0) return ZERO_LOOP_DELAY;
+					const sample = {
+						p50Ms: Math.round(histogram.percentile(50) / 1e6),
+						p99Ms: Math.round(histogram.percentile(99) / 1e6),
+						maxMs: Math.round(histogram.max / 1e6)
+					};
+					histogram.reset();
+					return sample;
+				} catch {
+					return ZERO_LOOP_DELAY;
+				}
+			},
+			stop: () => {
+				histogram.disable();
+			}
+		};
+	} catch {
+		return;
+	}
+}
+/**
+* Read the origin of the old-space ceiling off node's own options.
+*
+* hub-main is the only process in the cluster whose ceiling nobody chose. Every
+* forked runner is given V8 heap flags by `process-service.ts`; hub-main gets
+* `--max-old-space-size` from `docker-entrypoint.sh` ONLY when the operator sets
+* `CAMSTACK_HUB_HEAP_MB`, and on the live container it is unset — verified
+* 2026-08-25 by reading pid 7's argv, a bare `node .../starter.js`. So the
+* ceiling in force is whatever V8 derives from the host (4288MB there), and the
+* only record of that was a comment inside an image nobody rebuilds to read.
+*
+* A fact about the running process belongs in the running process's log, and
+* `execArgv` answers it for free at boot.
+*/
+function heapCeilingOrigin(execArgv = process.execArgv) {
+	return execArgv.some((arg) => /^--max[-_]old[-_]space[-_]size(=|$)/.test(arg)) ? "explicit" : "v8-default";
+}
 var MB = (bytes) => Math.round(bytes / 1048576);
 /**
 * Build a sample from raw readings. Pure so the thresholds are testable without
@@ -233,8 +300,10 @@ function createV8Reclaimer() {
 		return;
 	}
 }
-function format(label, s) {
-	return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
+function format(label, s, loop) {
+	const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
+	if (loop === void 0) return line;
+	return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
 }
 /**
 * Start the heartbeat. Returns a stop function.
@@ -242,7 +311,7 @@ function format(label, s) {
 * `unref`ed so it can never hold the process open, and every read is wrapped:
 * a diagnostic that can crash the process it watches is worse than none.
 */
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
+function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
 	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
 	const now = reclaimOptions?.now ?? (() => Date.now());
 	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
@@ -251,14 +320,24 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 	const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
 	const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
 	let lastReclaimAt = Number.NEGATIVE_INFINITY;
+	/** Consecutive passes that fired the instant the floor elapsed. See
+	*  {@link RECLAIM_STEADY_STATE_PASSES}. */
+	let passesAtFloor = 0;
+	let steadyStateAnnounced = false;
 	const read = () => {
 		const limit = reclaimOptions?.heapLimitBytes ?? node_v8.getHeapStatistics().heap_size_limit;
 		return buildHeapSample(readMemory(), limit);
 	};
 	const maybeReclaim = (sample) => {
 		if (reclaimOptions === void 0) return;
-		if (!shouldReclaim(sample, triggerMb)) return;
-		if (now() - lastReclaimAt < minIntervalMs) return;
+		if (!shouldReclaim(sample, triggerMb)) {
+			passesAtFloor = 0;
+			steadyStateAnnounced = false;
+			return;
+		}
+		const sinceLast = now() - lastReclaimAt;
+		if (sinceLast < minIntervalMs) return;
+		passesAtFloor = sinceLast <= minIntervalMs * 2 ? passesAtFloor + 1 : 1;
 		lastReclaimAt = now();
 		const startedAt = now();
 		try {
@@ -269,6 +348,10 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 		}
 		const after = read();
 		sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
+		if (passesAtFloor >= 6 && !steadyStateAnnounced) {
+			steadyStateAnnounced = true;
+			sink.warn(`[mem] reclaim ${label} has run at the floor for ${passesAtFloor} consecutive passes (every ${minIntervalMs}ms) — the stop-the-world is now STEADY STATE, not a rescue. The allocation source is what needs the fix; this pass only bounds its peak.`);
+		}
 	};
 	let mode = "steady";
 	let lastLoggedAt = Number.NEGATIVE_INFINITY;
@@ -297,7 +380,7 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 			const due = at - lastLoggedAt >= intervalMs;
 			if (mode === "escalated" || due) {
 				lastLoggedAt = at;
-				const line = format(label, sample);
+				const line = format(label, sample, loopDelay?.read());
 				if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
 				else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
 				else sink.info(line);
@@ -309,11 +392,13 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 	const timer = setInterval(tick, probeIntervalMs);
 	timer.unref?.();
 	tick();
+	if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
 	let stopped = false;
 	return () => {
 		if (stopped) return;
 		stopped = true;
 		clearInterval(timer);
+		loopDelay?.stop();
 	};
 }
 /**
@@ -362,13 +447,13 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
 function startRunnerHeapWatch(options) {
 	if (options.heapProfile !== "heavy") return void 0;
 	const intervalMs = options.intervalMs ?? 3e5;
-	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs);
+	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false);
 	let reclaimOptions = options.reclaimOptions;
 	if (reclaimOptions === void 0) {
 		const reclaimer = createV8Reclaimer();
 		reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
 	}
-	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions);
+	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, false);
 }
 //#endregion
 //#region src/kernel/moleculer/addon-data-plane-facility.ts

package/dist/{retired-settings-keys-VAyOhu_a.mjs → retired-settings-keys-B1aZcoS0.mjs}

@@ -1,4 +1,4 @@
-import { kt as asJsonObject } from "./dist-CEhrHp3u.mjs";
+import { kt as asJsonObject } from "./dist-CvJrtLkw.mjs";
 //#region src/builtins/sqlite-storage/retired-settings-keys.ts
 /**
 * Is THIS node the one whose settings store is the cluster's authority?

package/dist/{retired-settings-keys-D6Jy_SsO.js → retired-settings-keys-D7c0n9jO.js}

@@ -1,4 +1,4 @@
-const require_dist = require("./dist-DnhGRFEn.js");
+const require_dist = require("./dist-CIDgcxPk.js");
 //#region src/builtins/sqlite-storage/retired-settings-keys.ts
 /**
 * Is THIS node the one whose settings store is the cluster's authority?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/system",
-  "version": "1.2.122",
+  "version": "1.2.124",
   "description": "Core addon for CamStack — builtins, pipeline, process management, auth, logging, events",
   "keywords": [
     "camstack",