@camstack/system 1.2.123 → 1.2.125

package/dist/kernel/moleculer/runner-native-allocator.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Native-allocator environment for a forked addon runner.
+ *
+ * ## The half of a runner's memory that no V8 flag reaches
+ *
+ * `--max-old-space-size` bounds V8's old space and nothing else. It does not
+ * bound `external`, it does not bound `arrayBuffers`, and it does not bound what
+ * a native addon (`sharp`/`libvips`, decoders, codecs) allocates through `malloc`.
+ *
+ * Measured on the live hub, 2026-08-25, over a ~6h window of `[mem]` heartbeats:
+ *
+ *   hub/pipeline-analytics  rss 309-7334MB   heapTotal max 343MB   external max 5479MB
+ *
+ * It already carried `--max-old-space-size=1024`, so the flag was in force the
+ * whole time and the process still reached 7.3GB — because 96% of it was never
+ * V8's to bound. Its reclaim line names the shape exactly:
+ *
+ *   reclaim hub/pipeline-analytics stranded=2552MB rss=2765MB→2647MB
+ *           freed=118MB arrayBuffers=81MB→29MB took=58ms
+ *
+ * The `arrayBuffers` FLOOR returns to ~1MB every cycle, so this is not a leak.
+ * It is glibc holding freed pages: with `MALLOC_ARENA_MAX` unset, glibc opens up
+ * to `8 × ncpu` arenas — 160 on this 20-core host — and each keeps its own free
+ * list, which is returned to the OS only when the top of that arena is free.
+ *
+ * ## Why this matters more than the cgroup number suggests
+ *
+ * The container is capped at 24GiB and `memory.current` rides at 21-24GB. Near
+ * the limit the kernel performs direct reclaim IN THE CONTEXT OF THE TASK THAT
+ * ALLOCATES, so every process in the container stalls together — measured as
+ * four idle, mutually unrelated runners sharing one ~39s stall in the same
+ * sample. Returning stranded pages is therefore not a tidiness win; it is what
+ * keeps unrelated runners off the direct-reclaim path.
+ *
+ * ## Why `VIPS_CONCURRENCY` is welded to it
+ *
+ * `sharp` reads `MALLOC_ARENA_MAX` in its NATIVE binding (not in JS — grepping
+ * the bundle finds nothing). Unset on glibc, sharp pins libvips concurrency to 1
+ * as its own fragmentation defence; the moment the variable appears it restores
+ * concurrency to `availableParallelism()`. Verified in the live container:
+ *
+ *   MALLOC_ARENA_MAX unset                  → sharp.concurrency() === 1
+ *   MALLOC_ARENA_MAX=2                      → sharp.concurrency() === 20
+ *   MALLOC_ARENA_MAX=2 VIPS_CONCURRENCY=1   → sharp.concurrency() === 1
+ *
+ * Shipping the arena bound alone would therefore have quietly shipped a 1→20
+ * libvips threading change to the busiest addon on a host already at ~75% CPU,
+ * under the banner of a memory fix. This module ships the allocator change and
+ * ONLY the allocator change; raising libvips concurrency is a separate decision.
+ *
+ * ## What it costs, measured
+ *
+ * Same container, same 12-image corpus, same pipeline as analytics (decode →
+ * extract → resize → encode → raw), 3 interleaved pairs so host load cancels,
+ * concurrency pinned to 1 on both arms:
+ *
+ *   arenas default : mean 51.9ms  p50 60.0ms  p90 67.3ms  p99 70.7ms  rss 505MB
+ *   arenas=2       : mean 51.5ms  p50 57.6ms  p90 67.2ms  p99 72.9ms  rss 354MB
+ *
+ * Latency is unchanged inside noise (-0.8% mean, +3.1% p99); the same work is
+ * done in 30% less RSS.
+ */
+/** Bounded glibc arenas per forked runner. */
+export declare const RUNNER_MALLOC_ARENA_MAX_DEFAULT = 2;
+/**
+ * libvips worker threads per forked runner. `1` is not a tuning choice — it is
+ * sharp's OWN default on glibc, pinned here so that bounding the arenas does not
+ * change it as a side effect. See the module header.
+ */
+export declare const RUNNER_VIPS_CONCURRENCY_DEFAULT = 1;
+/**
+ * The native-allocator variables to add to a runner's environment. Both keys are
+ * present or both are absent — see the `VIPS_CONCURRENCY` note in the module
+ * header for why one without the other is the dangerous configuration.
+ */
+export interface RunnerNativeAllocatorEnv {
+    readonly MALLOC_ARENA_MAX?: string;
+    readonly VIPS_CONCURRENCY?: string;
+}
+/**
+ * Resolve the native-allocator env for a child runner from the parent's env.
+ *
+ * Precedence, highest first:
+ *   1. `CAMSTACK_RUNNER_NATIVE_ALLOCATOR=off` — kill switch, returns nothing and
+ *      restores the glibc/sharp defaults. Recoverable without a rebuild.
+ *   2. A value the operator already put in the environment (`MALLOC_ARENA_MAX`,
+ *      `VIPS_CONCURRENCY`) — inherited verbatim, never overwritten.
+ *   3. `CAMSTACK_RUNNER_MALLOC_ARENA_MAX` — the CamStack override.
+ *   4. The defaults above.
+ *
+ * A nonsense override falls back to the default rather than being passed through:
+ * glibc reads `MALLOC_ARENA_MAX=0` as "unbounded", which is the very state this
+ * exists to leave.
+ */
+export declare function runnerNativeAllocatorEnv(parentEnv: NodeJS.ProcessEnv): RunnerNativeAllocatorEnv;

package/dist/{manifest-python-deps-B3_4YiDK.js → manifest-python-deps-BTkFwAk_.js}

@@ -311,7 +311,7 @@ function format(label, s, loop) {
 * `unref`ed so it can never hold the process open, and every read is wrapped:
 * a diagnostic that can crash the process it watches is worse than none.
 */
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv) {
+function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
 	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
 	const now = reclaimOptions?.now ?? (() => Date.now());
 	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
@@ -392,7 +392,7 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 	const timer = setInterval(tick, probeIntervalMs);
 	timer.unref?.();
 	tick();
-	if (heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
+	if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
 	let stopped = false;
 	return () => {
 		if (stopped) return;
@@ -422,14 +422,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
 *
 * ## Why heavy runners get hub-main's mechanism
 *
-* The media-path runners deliberately run with NO old-space ceiling
-* (`maxOldSpaceMb: 0` — a flat cap turned a disk-stall backlog into
-* `Ineffective mark-compacts near heap limit` and killed recording, twice on
-* 2026-08-17). Uncapped, V8 feels no pressure, so a one-off burst becomes the
-* runner's permanent RSS: the recorder measured 2122MB holding a ~50MB index.
-* hub-main had exactly this shape and its reclaimer is proven live — 11 passes
-* in 4h, 375–1652MB returned per pass at 550–680ms each. Same mechanism, same
-* `shouldReclaim` stranded-over-trigger gate; never a second implementation.
+* Because a heap ceiling cannot reach what actually strands in them. Every
+* media-path runner now carries an old-space ceiling sized from its own
+* measured peak (`addon-pipeline` manifest + its `manifest-heap-ceilings` spec),
+* and that changes nothing here: what a heavy runner strands is NATIVE —
+* `external`, `arrayBuffers`, glibc arena free lists — none of which
+* `--max-old-space-size` bounds. Measured 2026-08-25, `hub/pipeline-analytics`
+* reported `stranded=2552MB` while holding an 81MB V8 heap under a 1024MB
+* ceiling that was in force the whole time. hub-main had exactly this shape and
+* its reclaimer is proven live — 11 passes in 4h, 375–1652MB returned per pass
+* at 550–680ms each. Same mechanism, same `shouldReclaim` stranded-over-trigger
+* gate; never a second implementation.
 *
 * ## The stall, priced for the runners that get it
 *
@@ -447,13 +450,13 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
 function startRunnerHeapWatch(options) {
 	if (options.heapProfile !== "heavy") return void 0;
 	const intervalMs = options.intervalMs ?? 3e5;
-	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs);
+	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false);
 	let reclaimOptions = options.reclaimOptions;
 	if (reclaimOptions === void 0) {
 		const reclaimer = createV8Reclaimer();
 		reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
 	}
-	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions);
+	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
 }
 //#endregion
 //#region src/kernel/moleculer/addon-data-plane-facility.ts

package/dist/{manifest-python-deps-DIIXlR1x.mjs → manifest-python-deps-Dmcnb287.mjs}

@@ -307,7 +307,7 @@ function format(label, s, loop) {
 * `unref`ed so it can never hold the process open, and every read is wrapped:
 * a diagnostic that can crash the process it watches is worse than none.
 */
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv) {
+function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
 	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
 	const now = reclaimOptions?.now ?? (() => Date.now());
 	const triggerMb = reclaimOptions?.triggerMb ?? 1024;
@@ -388,7 +388,7 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
 	const timer = setInterval(tick, probeIntervalMs);
 	timer.unref?.();
 	tick();
-	if (heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
+	if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
 	let stopped = false;
 	return () => {
 		if (stopped) return;
@@ -418,14 +418,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
 *
 * ## Why heavy runners get hub-main's mechanism
 *
-* The media-path runners deliberately run with NO old-space ceiling
-* (`maxOldSpaceMb: 0` — a flat cap turned a disk-stall backlog into
-* `Ineffective mark-compacts near heap limit` and killed recording, twice on
-* 2026-08-17). Uncapped, V8 feels no pressure, so a one-off burst becomes the
-* runner's permanent RSS: the recorder measured 2122MB holding a ~50MB index.
-* hub-main had exactly this shape and its reclaimer is proven live — 11 passes
-* in 4h, 375–1652MB returned per pass at 550–680ms each. Same mechanism, same
-* `shouldReclaim` stranded-over-trigger gate; never a second implementation.
+* Because a heap ceiling cannot reach what actually strands in them. Every
+* media-path runner now carries an old-space ceiling sized from its own
+* measured peak (`addon-pipeline` manifest + its `manifest-heap-ceilings` spec),
+* and that changes nothing here: what a heavy runner strands is NATIVE —
+* `external`, `arrayBuffers`, glibc arena free lists — none of which
+* `--max-old-space-size` bounds. Measured 2026-08-25, `hub/pipeline-analytics`
+* reported `stranded=2552MB` while holding an 81MB V8 heap under a 1024MB
+* ceiling that was in force the whole time. hub-main had exactly this shape and
+* its reclaimer is proven live — 11 passes in 4h, 375–1652MB returned per pass
+* at 550–680ms each. Same mechanism, same `shouldReclaim` stranded-over-trigger
+* gate; never a second implementation.
 *
 * ## The stall, priced for the runners that get it
 *
@@ -443,13 +446,13 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
 function startRunnerHeapWatch(options) {
 	if (options.heapProfile !== "heavy") return void 0;
 	const intervalMs = options.intervalMs ?? 3e5;
-	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs);
+	if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false);
 	let reclaimOptions = options.reclaimOptions;
 	if (reclaimOptions === void 0) {
 		const reclaimer = createV8Reclaimer();
 		reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
 	}
-	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions);
+	return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
 }
 //#endregion
 //#region src/kernel/moleculer/addon-data-plane-facility.ts

package/dist/{retired-settings-keys-VAyOhu_a.mjs → retired-settings-keys-B1aZcoS0.mjs}

@@ -1,4 +1,4 @@
-import { kt as asJsonObject } from "./dist-CEhrHp3u.mjs";
+import { kt as asJsonObject } from "./dist-CvJrtLkw.mjs";
 //#region src/builtins/sqlite-storage/retired-settings-keys.ts
 /**
 * Is THIS node the one whose settings store is the cluster's authority?

package/dist/{retired-settings-keys-D6Jy_SsO.js → retired-settings-keys-D7c0n9jO.js}

@@ -1,4 +1,4 @@
-const require_dist = require("./dist-DnhGRFEn.js");
+const require_dist = require("./dist-CIDgcxPk.js");
 //#region src/builtins/sqlite-storage/retired-settings-keys.ts
 /**
 * Is THIS node the one whose settings store is the cluster's authority?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/system",
-  "version": "1.2.123",
+  "version": "1.2.125",
   "description": "Core addon for CamStack — builtins, pipeline, process management, auth, logging, events",
   "keywords": [
     "camstack",