@camstack/system 1.2.145 → 1.2.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +2 -1
- package/dist/addon-runner.mjs +2 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +44 -4
- package/dist/index.mjs +36 -5
- package/dist/kernel/heap-watch.d.ts +209 -1
- package/dist/{manifest-python-deps-BRyNCnnG.mjs → manifest-python-deps-CVeckyXx.mjs} +199 -7
- package/dist/{manifest-python-deps-GjlyPjm0.js → manifest-python-deps-CktMcXzS.js} +252 -6
- package/package.json +1 -1
package/dist/addon-runner.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
2
|
-
const require_manifest_python_deps = require("./manifest-python-deps-
|
|
2
|
+
const require_manifest_python_deps = require("./manifest-python-deps-CktMcXzS.js");
|
|
3
3
|
const require_custom_action_registry = require("./custom-action-registry-jY0NOZK8.js");
|
|
4
4
|
let node_path = require("node:path");
|
|
5
5
|
node_path = require_chunk.__toESM(node_path);
|
|
@@ -692,6 +692,7 @@ async function main() {
|
|
|
692
692
|
const stopHeapWatch = require_manifest_python_deps.startRunnerHeapWatch({
|
|
693
693
|
heapProfile: process.env["CAMSTACK_RUNNER_HEAP_PROFILE"],
|
|
694
694
|
reclaimSwitch: process.env["CAMSTACK_RUNNER_HEAP_RECLAIM"],
|
|
695
|
+
rssBudgetMb: process.env[require_manifest_python_deps.RUNNER_RSS_BUDGET_ENV],
|
|
695
696
|
label: nodeId,
|
|
696
697
|
sink: {
|
|
697
698
|
info: (line) => {
|
package/dist/addon-runner.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { F as createUdsLoggerWithControl, I as LocalChildClient, Nt as
|
|
1
|
+
import { F as createUdsLoggerWithControl, Ht as startRunnerHeapWatch, I as LocalChildClient, Nt as RUNNER_RSS_BUDGET_ENV, bt as resolveAddonClass, i as createUdsAddonContext, it as setWorkerNativeCapsChangeListener, nt as getWorkerNativeCapSnapshot, ot as validateProviderRegistrations, t as installManifestPythonDeps, tt as getWorkerNativeCapProvider, yt as installManifestNativeDeps } from "./manifest-python-deps-CVeckyXx.mjs";
|
|
2
2
|
import { t as CustomActionRegistry } from "./custom-action-registry-F__gp_VX.mjs";
|
|
3
3
|
import { register } from "node:module";
|
|
4
4
|
import * as path$1 from "node:path";
|
|
@@ -688,6 +688,7 @@ async function main() {
|
|
|
688
688
|
const stopHeapWatch = startRunnerHeapWatch({
|
|
689
689
|
heapProfile: process.env["CAMSTACK_RUNNER_HEAP_PROFILE"],
|
|
690
690
|
reclaimSwitch: process.env["CAMSTACK_RUNNER_HEAP_RECLAIM"],
|
|
691
|
+
rssBudgetMb: process.env[RUNNER_RSS_BUDGET_ENV],
|
|
691
692
|
label: nodeId,
|
|
692
693
|
sink: {
|
|
693
694
|
info: (line) => {
|
package/dist/index.d.ts
CHANGED
|
@@ -46,8 +46,8 @@ export { createFileDataPlaneHandler } from './http/file-data-plane.js';
|
|
|
46
46
|
export { allFamiliesListenHost, applyLanHttp, bindPendingLanHttp, closeLanHttp, DEFAULT_LAN_HTTP_PORT, readLanHttpState, registerLanHttpHandler, } from './http/lan-http-bind.js';
|
|
47
47
|
export type { ReverseProxyOptions } from './http/reverse-proxy.js';
|
|
48
48
|
export { proxyToUpstream } from './http/reverse-proxy.js';
|
|
49
|
-
export type { HeapReclaimer, HeapReclaimOptions, HeapSample, HeapWatchSink, RunnerHeapWatchOptions, } from './kernel/heap-watch.js';
|
|
50
|
-
export { buildHeapSample, createV8Reclaimer, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, RUNNER_HEAP_WATCH_INTERVAL_MS, shouldReclaim, startHeapWatch, startRunnerHeapWatch, strandedMb, } from './kernel/heap-watch.js';
|
|
49
|
+
export type { HeapReclaimer, HeapReclaimOptions, HeapSample, HeapWatchSink, RssBreakdown, RssBudgetOptions, RssBudgetState, RssGrowthShape, RunnerHeapWatchOptions, } from './kernel/heap-watch.js';
|
|
50
|
+
export { buildHeapSample, createV8Reclaimer, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_MAIN_RSS_BUDGET_MB, HUB_RSS_BUDGET_ENV, hubMainRssBudget, describeRss, nextRssBudgetState, parseRssBudgetMb, RSS_BUDGET_REANNOUNCE_MIN_MS, RSS_BUDGET_RELEASE_RATIO, RUNNER_HEAP_WATCH_INTERVAL_MS, RUNNER_RSS_BUDGET_ENV, shouldReclaim, startHeapWatch, startRunnerHeapWatch, strandedMb, } from './kernel/heap-watch.js';
|
|
51
51
|
export type { AddonDataPlaneFacility, DataPlaneSink, } from './kernel/moleculer/addon-data-plane-facility.js';
|
|
52
52
|
export { createAddonDataPlaneFacility } from './kernel/moleculer/addon-data-plane-facility.js';
|
|
53
53
|
export type { ElementState, ElementStatus, } from './lifecycle/lifecycle-state-machine.js';
|
package/dist/index.js
CHANGED
|
@@ -26,7 +26,7 @@ require("./builtins/system-config/index.js");
|
|
|
26
26
|
const require_builtins_winston_logging_index = require("./builtins/winston-logging/index.js");
|
|
27
27
|
const require_file_data_plane = require("./file-data-plane-DO8KbxCe.js");
|
|
28
28
|
const require_tls$1 = require("./tls-u8QCJCFE.js");
|
|
29
|
-
const require_manifest_python_deps = require("./manifest-python-deps-
|
|
29
|
+
const require_manifest_python_deps = require("./manifest-python-deps-CktMcXzS.js");
|
|
30
30
|
const require_resource_monitor = require("./resource-monitor-CdnzxBLP.js");
|
|
31
31
|
const require_custom_action_registry = require("./custom-action-registry-jY0NOZK8.js");
|
|
32
32
|
let zod = require("zod");
|
|
@@ -94270,7 +94270,8 @@ async function getPidStats$1(pids) {
|
|
|
94270
94270
|
}
|
|
94271
94271
|
var EMPTY_HEAP_DECLARATION = {
|
|
94272
94272
|
profile: void 0,
|
|
94273
|
-
maxOldSpaceMb: void 0
|
|
94273
|
+
maxOldSpaceMb: void 0,
|
|
94274
|
+
rssBudgetMb: void 0
|
|
94274
94275
|
};
|
|
94275
94276
|
/**
|
|
94276
94277
|
* Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
|
|
@@ -94291,10 +94292,12 @@ function readAddonHeapDeclaration(spec) {
|
|
|
94291
94292
|
const parsed = JSON.parse(raw);
|
|
94292
94293
|
const profile = extractHeapProfile(parsed, spec.addonId);
|
|
94293
94294
|
const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
|
|
94294
|
-
|
|
94295
|
+
const rssBudgetMb = extractRssBudgetMb(parsed, spec.addonId);
|
|
94296
|
+
if (profile !== void 0 || maxOldSpaceMb !== void 0 || rssBudgetMb !== void 0) {
|
|
94295
94297
|
declaration = {
|
|
94296
94298
|
profile,
|
|
94297
|
-
maxOldSpaceMb
|
|
94299
|
+
maxOldSpaceMb,
|
|
94300
|
+
rssBudgetMb
|
|
94298
94301
|
};
|
|
94299
94302
|
break;
|
|
94300
94303
|
}
|
|
@@ -94324,6 +94327,15 @@ function extractMaxOldSpaceMb(parsed, addonId) {
|
|
|
94324
94327
|
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
|
|
94325
94328
|
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
|
|
94326
94329
|
}
|
|
94330
|
+
/**
|
|
94331
|
+
* `execution.rssBudgetMb` for one addon. Only a POSITIVE finite number is a budget: unlike
|
|
94332
|
+
* `maxOldSpaceMb`, `0` here is not an escape hatch that lifts a cap, it is an absent budget —
|
|
94333
|
+
* and an absent budget is reported as "unwatched", never rounded into a default.
|
|
94334
|
+
*/
|
|
94335
|
+
function extractRssBudgetMb(parsed, addonId) {
|
|
94336
|
+
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.rssBudgetMb;
|
|
94337
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : void 0;
|
|
94338
|
+
}
|
|
94327
94339
|
function readManifestAddons(parsed) {
|
|
94328
94340
|
if (typeof parsed !== "object" || parsed === null) return [];
|
|
94329
94341
|
const camstack = parsed.camstack;
|
|
@@ -94408,6 +94420,22 @@ function runnerMaxOldSpaceMb(addons, heavy) {
|
|
|
94408
94420
|
return Math.max(...declared);
|
|
94409
94421
|
}
|
|
94410
94422
|
/**
|
|
94423
|
+
* The runner's declared RSS budget in MB, or `undefined` when NOBODY declared one.
|
|
94424
|
+
*
|
|
94425
|
+
* Same rule as the old-space ceiling, for the same reason (D2/D29): the runner takes the MAXIMUM
|
|
94426
|
+
* any co-located addon declares, because a runner is respawned as a whole and co-location must
|
|
94427
|
+
* never silence the hungriest member. It would be strictly worse here than for a heap ceiling —
|
|
94428
|
+
* an under-sized budget does not kill anything, it just cries wolf, and a watchdog that cries
|
|
94429
|
+
* wolf is a watchdog that gets muted.
|
|
94430
|
+
*
|
|
94431
|
+
* `undefined` propagates as an ABSENT env var, and the child then announces itself unwatched.
|
|
94432
|
+
* There is deliberately no class default: see `AddonExecution.rssBudgetMb`.
|
|
94433
|
+
*/
|
|
94434
|
+
function runnerRssBudgetMb(addons) {
|
|
94435
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).rssBudgetMb).filter((mb) => mb !== void 0);
|
|
94436
|
+
return declared.length === 0 ? void 0 : Math.max(...declared);
|
|
94437
|
+
}
|
|
94438
|
+
/**
|
|
94411
94439
|
* Per-placement V8 heap flags (Block D). LIGHT runners cap their young generation
|
|
94412
94440
|
* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — and
|
|
94413
94441
|
* keep an opt-in old-space cap (`CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`, unset = uncapped). HEAVY
|
|
@@ -94505,6 +94533,7 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
|
|
|
94505
94533
|
const nodeId = buildNodeId(runnerId);
|
|
94506
94534
|
const runnerPath = node_path.resolve(__dirname, "addon-runner.js");
|
|
94507
94535
|
const heavy = isHeavyRunner(addons);
|
|
94536
|
+
const rssBudgetMb = runnerRssBudgetMb(addons);
|
|
94508
94537
|
const childEnv = {
|
|
94509
94538
|
...process.env,
|
|
94510
94539
|
CAMSTACK_RUNNER_ID: runnerId,
|
|
@@ -94518,6 +94547,8 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
|
|
|
94518
94547
|
...applyRunnerNativeAllocator(process.env),
|
|
94519
94548
|
...env
|
|
94520
94549
|
};
|
|
94550
|
+
if (rssBudgetMb === void 0) delete childEnv[require_manifest_python_deps.RUNNER_RSS_BUDGET_ENV];
|
|
94551
|
+
else childEnv[require_manifest_python_deps.RUNNER_RSS_BUDGET_ENV] = String(rssBudgetMb);
|
|
94521
94552
|
const heapFlags = runnerHeapFlags(addons);
|
|
94522
94553
|
capturedBroker?.logger.info(`[${runnerId}] heap profile: ${heavy ? "heavy" : "light"} flags=[${heapFlags.join(" ")}] arenas=${childEnv["MALLOC_ARENA_MAX"] ?? "glibc-default"} vips=${childEnv["VIPS_CONCURRENCY"] ?? "sharp-default"}`);
|
|
94523
94554
|
const child = spawnFn(process.execPath, [...heapFlags, runnerPath], {
|
|
@@ -95317,6 +95348,8 @@ exports.HEAP_WATCH_INTERVAL_MS = require_manifest_python_deps.HEAP_WATCH_INTERVA
|
|
|
95317
95348
|
exports.HEAP_WATCH_WARN_RATIO = require_manifest_python_deps.HEAP_WATCH_WARN_RATIO;
|
|
95318
95349
|
exports.HUB_CAP_FWD_ACTION = require_manifest_python_deps.HUB_CAP_FWD_ACTION;
|
|
95319
95350
|
exports.HUB_CAP_FWD_SERVICE = require_manifest_python_deps.HUB_CAP_FWD_SERVICE;
|
|
95351
|
+
exports.HUB_MAIN_RSS_BUDGET_MB = require_manifest_python_deps.HUB_MAIN_RSS_BUDGET_MB;
|
|
95352
|
+
exports.HUB_RSS_BUDGET_ENV = require_manifest_python_deps.HUB_RSS_BUDGET_ENV;
|
|
95320
95353
|
exports.HubForwarderAddon = require_builtins_hub_forwarder_index.HubForwarderAddon$1;
|
|
95321
95354
|
exports.HubForwarderDestination = require_builtins_hub_forwarder_index.HubForwarderDestination$1;
|
|
95322
95355
|
exports.HubLogForwarder = HubLogForwarder;
|
|
@@ -95356,7 +95389,10 @@ exports.PipelineValidator = PipelineValidator;
|
|
|
95356
95389
|
exports.PythonEnvManager = PythonEnvManager;
|
|
95357
95390
|
exports.QUARANTINE_DIRNAME = QUARANTINE_DIRNAME;
|
|
95358
95391
|
exports.RESTART_MARKER_FILE = RESTART_MARKER_FILE;
|
|
95392
|
+
exports.RSS_BUDGET_REANNOUNCE_MIN_MS = require_manifest_python_deps.RSS_BUDGET_REANNOUNCE_MIN_MS;
|
|
95393
|
+
exports.RSS_BUDGET_RELEASE_RATIO = require_manifest_python_deps.RSS_BUDGET_RELEASE_RATIO;
|
|
95359
95394
|
exports.RUNNER_HEAP_WATCH_INTERVAL_MS = require_manifest_python_deps.RUNNER_HEAP_WATCH_INTERVAL_MS;
|
|
95395
|
+
exports.RUNNER_RSS_BUDGET_ENV = require_manifest_python_deps.RUNNER_RSS_BUDGET_ENV;
|
|
95360
95396
|
exports.RUNTIME_DEFAULTS = require_dist.RUNTIME_DEFAULTS;
|
|
95361
95397
|
exports.ReadinessRegistry = require_dist.ReadinessRegistry;
|
|
95362
95398
|
exports.ReadinessTimeoutError = require_dist.ReadinessTimeoutError;
|
|
@@ -95453,6 +95489,7 @@ exports.createV8Reclaimer = require_manifest_python_deps.createV8Reclaimer;
|
|
|
95453
95489
|
exports.deleteModelFromDisk = require_file_data_plane.deleteModelFromDisk;
|
|
95454
95490
|
exports.deriveAgentListenPort = deriveAgentListenPort;
|
|
95455
95491
|
exports.describeProviderKindDrift = describeProviderKindDrift;
|
|
95492
|
+
exports.describeRss = require_manifest_python_deps.describeRss;
|
|
95456
95493
|
exports.detectWorkspacePackagesDir = detectWorkspacePackagesDir;
|
|
95457
95494
|
Object.defineProperty(exports, "downloadBinary", {
|
|
95458
95495
|
enumerable: true,
|
|
@@ -95529,6 +95566,7 @@ exports.getSinglePidStats = require_resource_monitor.getSinglePidStats;
|
|
|
95529
95566
|
exports.getWorkerDeviceRegistry = require_manifest_python_deps.getWorkerDeviceRegistry;
|
|
95530
95567
|
exports.hasDotNode = hasDotNode;
|
|
95531
95568
|
exports.hashClusterSecret = hashClusterSecret;
|
|
95569
|
+
exports.hubMainRssBudget = require_manifest_python_deps.hubMainRssBudget;
|
|
95532
95570
|
exports.installManifestNativeDeps = require_manifest_python_deps.installManifestNativeDeps;
|
|
95533
95571
|
exports.installManifestPythonDeps = require_manifest_python_deps.installManifestPythonDeps;
|
|
95534
95572
|
exports.installPackageFromNpm = installPackageFromNpm;
|
|
@@ -95557,8 +95595,10 @@ exports.loadTlsCert = require_tls$1.loadTlsCert;
|
|
|
95557
95595
|
exports.localEndpointPath = require_manifest_python_deps.localEndpointPath;
|
|
95558
95596
|
exports.localProviderLink = require_manifest_python_deps.localProviderLink;
|
|
95559
95597
|
exports.mountNativeCapService = require_manifest_python_deps.mountNativeCapService;
|
|
95598
|
+
exports.nextRssBudgetState = require_manifest_python_deps.nextRssBudgetState;
|
|
95560
95599
|
exports.parseCapAction = require_manifest_python_deps.parseCapAction;
|
|
95561
95600
|
exports.parseRangeHeader = require_file_data_plane.parseRangeHeader;
|
|
95601
|
+
exports.parseRssBudgetMb = require_manifest_python_deps.parseRssBudgetMb;
|
|
95562
95602
|
exports.parseTokenizedUrl = require_file_data_plane.parseTokenizedUrl;
|
|
95563
95603
|
exports.partitionIsolatedBuiltinIds = partitionIsolatedBuiltinIds;
|
|
95564
95604
|
exports.proxyToUpstream = proxyToUpstream;
|
package/dist/index.mjs
CHANGED
|
@@ -25,7 +25,7 @@ import "./builtins/system-config/index.mjs";
|
|
|
25
25
|
import { WinstonDestination, WinstonLoggingAddon } from "./builtins/winston-logging/index.mjs";
|
|
26
26
|
import { a as parseTokenizedUrl, c as collectModelFiles, d as downloadModel, f as ensureModel, h as isModelDownloaded, i as parseRangeHeader, l as deleteModelFromDisk, m as getModelFilePath, n as contentTypeFor, o as resolveFilePath, p as fetchJson, r as createAuthenticatedFileServer, s as ModelDownloadService, t as createFileDataPlaneHandler, u as downloadFile } from "./file-data-plane-BhKdxJgf.mjs";
|
|
27
27
|
import { C as registerLanHttpHandler, S as readLanHttpState, _ as DEFAULT_LAN_HTTP_PORT, a as writeTlsMode, b as bindPendingLanHttp, c as loadTlsCert, d as collectCertIdentity, f as CA_VALIDITY_DAYS, g as evaluateExistingCert, h as SERVER_AUTH_OID, i as writeExtraSans, l as reissueTlsLeaf, m as MAX_LEAF_VALIDITY_DAYS, n as readTlsAccessStatus, o as validateUploadedTls, p as LEAF_RENEWAL_WINDOW_DAYS, r as readTlsMode, s as ensureTlsCert, t as readExtraSans, u as CA_COMMON_NAME, v as allFamiliesListenHost, x as closeLanHttp, y as applyLanHttp } from "./tls-CQhPGSJm.mjs";
|
|
28
|
-
import { $ as buildUdsNativeCapProxy, A as createParentUnownedCallHandler, At as
|
|
28
|
+
import { $ as buildUdsNativeCapProxy, A as createParentUnownedCallHandler, At as RSS_BUDGET_REANNOUNCE_MIN_MS, B as AGENT_CAP_FWD_SERVICE, Bt as shouldReclaim, C as brokerTransportLink, Ct as createAddonDataPlaneFacility, D as HUB_CAP_FWD_ACTION, Dt as HEAP_WATCH_WARN_RATIO, E as localProviderLink, Et as HEAP_WATCH_INTERVAL_MS, F as createUdsLoggerWithControl, Ft as createV8Reclaimer, G as createLocalTransport, H as CapRouteError, Ht as startRunnerHeapWatch, I as LocalChildClient, It as describeRss, J as SocketChannel, K as UdsLocalTransportClient, L as LocalChildRegistry, Lt as hubMainRssBudget, M as createUdsEventBus, Mt as RUNNER_HEAP_WATCH_INTERVAL_MS, N as udsChildLogToWorkerEntry, Nt as RUNNER_RSS_BUDGET_ENV, O as HUB_CAP_FWD_SERVICE, Ot as HUB_MAIN_RSS_BUDGET_MB, P as createUdsLogger, Pt as buildHeapSample, Q as buildNativeCapProxy, R as UDS_NO_ROUTE_PREFIX, Rt as nextRssBudgetState, S as brokerCallForCap, St as runNpm, T as ipcParentLink, Tt as HEAP_RECLAIM_TRIGGER_MB, U as classifyCapRoute, Ut as strandedMb, V as CapRouteResolver, Vt as startHeapWatch, W as callWithServiceDiscovery, X as FrameDecoder, Y as localEndpointPath, Z as encodeFrame, _ as createKernelHwAccel, _t as CapabilityUnavailableError, a as getWorkerDeviceRegistry, at as createAddonService, b as __resetCapUsageRegistryForTests, bt as resolveAddonClass, c as setHubConnected, ct as capActionName, d as getBrokerEventBus, dt as capServiceName, et as createBrokerDeviceManagerApi, f as getMoleculerEventStats, ft as parseCapAction, g as AddonDepsManager, gt as CapabilityHandle, h as subscribePassthrough, ht as DeviceRegistry, i as createUdsAddonContext, j as createUdsEventBridge, jt as RSS_BUDGET_RELEASE_RATIO, k as createHubCapForwardService, kt as HUB_RSS_BUDGET_ENV, l as EVENT_TOPIC_PREFIX, lt as capActionSuffix, m as setNodeEventInterest, mt as serializeTypedArrays, n as adaptBrokerToCluster, o as getOrInitReadinessRegistry, ot as validateProviderRegistrations, p as registerEventBusService, pt as deserializeTypedArrays, q as UdsLocalTransportServer, r as createAddonContext, rt as mountNativeCapService, s as getOrInitReadinessRegistryForClient, st as NATIVE_PROVIDER_SERVICE_INFIX, t as installManifestPythonDeps, u as clusterEventTopic, ut as capBareAction, v as resolveHwAccel, vt as copyBundledNativeModules, w as buildLinkChain, wt as HEAP_RECLAIM_MIN_INTERVAL_MS, x as getCapUsageRegistry, xt as resolveNpmInvocation, y as CapUsageRegistry, yt as installManifestNativeDeps, z as AGENT_CAP_FWD_ACTION, zt as parseRssBudgetMb } from "./manifest-python-deps-CVeckyXx.mjs";
|
|
29
29
|
import { n as getSinglePidStats, t as getPidStats } from "./resource-monitor-BWmQ5i-o.mjs";
|
|
30
30
|
import { t as CustomActionRegistry } from "./custom-action-registry-F__gp_VX.mjs";
|
|
31
31
|
import { z } from "zod";
|
|
@@ -94263,7 +94263,8 @@ async function getPidStats$1(pids) {
|
|
|
94263
94263
|
}
|
|
94264
94264
|
var EMPTY_HEAP_DECLARATION = {
|
|
94265
94265
|
profile: void 0,
|
|
94266
|
-
maxOldSpaceMb: void 0
|
|
94266
|
+
maxOldSpaceMb: void 0,
|
|
94267
|
+
rssBudgetMb: void 0
|
|
94267
94268
|
};
|
|
94268
94269
|
/**
|
|
94269
94270
|
* Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
|
|
@@ -94284,10 +94285,12 @@ function readAddonHeapDeclaration(spec) {
|
|
|
94284
94285
|
const parsed = JSON.parse(raw);
|
|
94285
94286
|
const profile = extractHeapProfile(parsed, spec.addonId);
|
|
94286
94287
|
const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
|
|
94287
|
-
|
|
94288
|
+
const rssBudgetMb = extractRssBudgetMb(parsed, spec.addonId);
|
|
94289
|
+
if (profile !== void 0 || maxOldSpaceMb !== void 0 || rssBudgetMb !== void 0) {
|
|
94288
94290
|
declaration = {
|
|
94289
94291
|
profile,
|
|
94290
|
-
maxOldSpaceMb
|
|
94292
|
+
maxOldSpaceMb,
|
|
94293
|
+
rssBudgetMb
|
|
94291
94294
|
};
|
|
94292
94295
|
break;
|
|
94293
94296
|
}
|
|
@@ -94317,6 +94320,15 @@ function extractMaxOldSpaceMb(parsed, addonId) {
|
|
|
94317
94320
|
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
|
|
94318
94321
|
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
|
|
94319
94322
|
}
|
|
94323
|
+
/**
|
|
94324
|
+
* `execution.rssBudgetMb` for one addon. Only a POSITIVE finite number is a budget: unlike
|
|
94325
|
+
* `maxOldSpaceMb`, `0` here is not an escape hatch that lifts a cap, it is an absent budget —
|
|
94326
|
+
* and an absent budget is reported as "unwatched", never rounded into a default.
|
|
94327
|
+
*/
|
|
94328
|
+
function extractRssBudgetMb(parsed, addonId) {
|
|
94329
|
+
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.rssBudgetMb;
|
|
94330
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : void 0;
|
|
94331
|
+
}
|
|
94320
94332
|
function readManifestAddons(parsed) {
|
|
94321
94333
|
if (typeof parsed !== "object" || parsed === null) return [];
|
|
94322
94334
|
const camstack = parsed.camstack;
|
|
@@ -94401,6 +94413,22 @@ function runnerMaxOldSpaceMb(addons, heavy) {
|
|
|
94401
94413
|
return Math.max(...declared);
|
|
94402
94414
|
}
|
|
94403
94415
|
/**
|
|
94416
|
+
* The runner's declared RSS budget in MB, or `undefined` when NOBODY declared one.
|
|
94417
|
+
*
|
|
94418
|
+
* Same rule as the old-space ceiling, for the same reason (D2/D29): the runner takes the MAXIMUM
|
|
94419
|
+
* any co-located addon declares, because a runner is respawned as a whole and co-location must
|
|
94420
|
+
* never silence the hungriest member. It would be strictly worse here than for a heap ceiling —
|
|
94421
|
+
* an under-sized budget does not kill anything, it just cries wolf, and a watchdog that cries
|
|
94422
|
+
* wolf is a watchdog that gets muted.
|
|
94423
|
+
*
|
|
94424
|
+
* `undefined` propagates as an ABSENT env var, and the child then announces itself unwatched.
|
|
94425
|
+
* There is deliberately no class default: see `AddonExecution.rssBudgetMb`.
|
|
94426
|
+
*/
|
|
94427
|
+
function runnerRssBudgetMb(addons) {
|
|
94428
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).rssBudgetMb).filter((mb) => mb !== void 0);
|
|
94429
|
+
return declared.length === 0 ? void 0 : Math.max(...declared);
|
|
94430
|
+
}
|
|
94431
|
+
/**
|
|
94404
94432
|
* Per-placement V8 heap flags (Block D). LIGHT runners cap their young generation
|
|
94405
94433
|
* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — and
|
|
94406
94434
|
* keep an opt-in old-space cap (`CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`, unset = uncapped). HEAVY
|
|
@@ -94498,6 +94526,7 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
|
|
|
94498
94526
|
const nodeId = buildNodeId(runnerId);
|
|
94499
94527
|
const runnerPath = path$39.resolve(__dirname, "addon-runner.js");
|
|
94500
94528
|
const heavy = isHeavyRunner(addons);
|
|
94529
|
+
const rssBudgetMb = runnerRssBudgetMb(addons);
|
|
94501
94530
|
const childEnv = {
|
|
94502
94531
|
...process.env,
|
|
94503
94532
|
CAMSTACK_RUNNER_ID: runnerId,
|
|
@@ -94511,6 +94540,8 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
|
|
|
94511
94540
|
...applyRunnerNativeAllocator(process.env),
|
|
94512
94541
|
...env
|
|
94513
94542
|
};
|
|
94543
|
+
if (rssBudgetMb === void 0) delete childEnv[RUNNER_RSS_BUDGET_ENV];
|
|
94544
|
+
else childEnv[RUNNER_RSS_BUDGET_ENV] = String(rssBudgetMb);
|
|
94514
94545
|
const heapFlags = runnerHeapFlags(addons);
|
|
94515
94546
|
capturedBroker?.logger.info(`[${runnerId}] heap profile: ${heavy ? "heavy" : "light"} flags=[${heapFlags.join(" ")}] arenas=${childEnv["MALLOC_ARENA_MAX"] ?? "glibc-default"} vips=${childEnv["VIPS_CONCURRENCY"] ?? "sharp-default"}`);
|
|
94516
94547
|
const child = spawnFn(process.execPath, [...heapFlags, runnerPath], {
|
|
@@ -95252,4 +95283,4 @@ var LifecycleJobEngine = class {
|
|
|
95252
95283
|
}
|
|
95253
95284
|
};
|
|
95254
95285
|
//#endregion
|
|
95255
|
-
export { AGENT_CAP_FWD_ACTION, AGENT_CAP_FWD_SERVICE, AGENT_READINESS_SERVICE_NAME, ALL_CAPABILITY_DEFINITIONS, AddonApiFactory, AddonDepsManager, AddonEngineManager, AddonHealthMonitor, AddonInstaller, AddonLoader, AddonManifest, AddonRouteRegistry, AlertCenterAddon, ApiKeyManager, AuthManager, CA_COMMON_NAME, CA_VALIDITY_DAYS, CLUSTER_SECRET_MISMATCH_TYPE, CLUSTER_SECRET_REJECTED_EXIT_CODE, CORE_CAP_SERVICE_NAME, CapRouteError, CapRouteResolver, CapUsageRegistry, CapabilityHandle, CapabilityRegistry, CapabilityUnavailableError, ConfigManager, ConfigStore, ConsoleDestination, ConsoleLoggingAddon, CoreBlocksAddon, CustomActionRegistry, DEFAULT_DATA_PATH, DEFAULT_LAN_HTTP_PORT, DEFAULT_LOG_LEVEL, DEVICE_SETTINGS_CONTRIBUTION_METHODS, DEVICE_STATUS_METHOD, DataPlaneRegistry, DeviceManagerAddon, DeviceRegistry, DeviceStore, EVENT_TOPIC_PREFIX, EngineManagerResolver, EventBus, FeatureManager, FilesystemStorageAddon, FilesystemStorageProvider, FrameDecoder, FsStorageBackend, HEALTH_MONITOR_GRACE_PERIOD_MS, HEALTH_MONITOR_RETRY_INTERVALS_MS, HEALTH_MONITOR_TICK_MS, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_CAP_FWD_ACTION, HUB_CAP_FWD_SERVICE, HubForwarderAddon, HubForwarderDestination, HubLogForwarder, HubNodeRegistry, INFRA_CAPABILITIES, IntegrationRegistry, JobJournal, LEAF_RENEWAL_WINDOW_DAYS, LifecycleJobEngine, LifecycleStateMachine, LivenessMonitorAddon, LocalAuthAddon, LocalChildClient, LocalChildRegistry, LogManager, LogRingBuffer, LoggingGate, LokiDestination, LokiLoggingAddon, MAX_LEAF_VALIDITY_DAYS, METHOD_ACCESS_MAP, ModelDownloadService, NATIVE_PROVIDER_SERVICE_INFIX, NATIVE_SCAN_DEPTH, NativeMetricsAddon, NativeMetricsProvider, NetworkQualityTracker, NotificationService, PYTHON_VERSION, PipelineRunner, PipelineValidator, PythonEnvManager, QUARANTINE_DIRNAME, RESTART_MARKER_FILE, RUNNER_HEAP_WATCH_INTERVAL_MS, RUNTIME_DEFAULTS, ReadinessRegistry, ReadinessTimeoutError, ReplEngine, RingBuffer, SERVER_AUTH_OID, ScopedLogger, ScopedTokenManager, SocketChannel, SqliteSettingsAddon, SqliteSettingsBackend, StagingArea, StorageLocationManager, StorageManager, StorageOrchestratorAddon, StorageOrchestratorService, SystemConfigAddon, SystemEventBus, TRADITIONAL_NATIVE_PACKAGES, ToastService, UDS_NO_ROUTE_PREFIX, UdsLocalTransportClient, UdsLocalTransportServer, UserManager, WinstonDestination, WinstonLoggingAddon, __resetCapUsageRegistryForTests, __resetLoggingGateForTests, adaptBrokerToCluster, addonSettingsCapability, allFamiliesListenHost, applyLanHttp, bindPendingLanHttp, bootstrapSchema, brokerCallForCap, brokerTransportLink, buildBinaryPath, buildCapRouters, buildHeapSample, buildLinkChain, buildNativeCapProxy, buildNodeManifest, buildStorageLocationRegistry, buildUdsNativeCapProxy, builderMountedCapNames, callRegisterNodeWithRetry, callWithServiceDiscovery, canServeDataPlane, capActionName, capActionSuffix, capBareAction, capServiceName, classifyAddonDir, classifyCapRoute, clearPendingRestart, closeLanHttp, clusterEventTopic, clusterSecretMatches, collectCertIdentity, collectModelFiles, contentTypeFor, copyDirRecursive, copyExtraFileDirs, createAddonContext, createAddonDataPlaneFacility, createAddonService, createAuthenticatedFileServer, createBroker, createBrokerDeviceManagerApi, createCoreCapService, createDoorSettingsView, createFileDataPlaneHandler, createHubCapForwardService, createHubService, createKernelHwAccel, createLocalTransport, createParentUnownedCallHandler, createProcessService, createReadinessService, createReadinessServiceForRegistry, createScopedProcessManager, createStreamProbeBrokerService, createUdsAddonContext, createUdsEventBridge, createUdsEventBus, createUdsLogger, createUdsLoggerWithControl, createV8Reclaimer, deleteModelFromDisk, deriveAgentListenPort, describeProviderKindDrift, detectWorkspacePackagesDir, downloadBinary, downloadFile, downloadModel, emitDownForOwnedCaps, encodeFrame, ensureAddonNativePrebuilds, ensureBinary, ensureDir, ensureFfmpeg, ensureLibraryBuilt, ensureModel, ensureNativePrebuilds, ensurePython, ensureTlsCert, evaluateExistingCert, expandCapMethods, fetchJson, findInPath, formatLogLine, getBrokerEventBus, getCapUsageRegistry, getFfmpegDownloadUrl, getLoggingGate, getModelFilePath, getMoleculerEventStats, getOrInitReadinessRegistry, getOrInitReadinessRegistryForClient, getPidStats, getPlatformInfo, getPythonDownloadUrl, getRestartMarkerPath, getSinglePidStats, getWorkerDeviceRegistry, hasDotNode, hashClusterSecret, installManifestNativeDeps, installManifestPythonDeps, installPackageFromNpm, installPythonPackages, installPythonRequirements, ipcParentLink, isAddonDeploySource, isArrayOutputSchema, isClusterSecretMismatchError, isCollectionArrayMethod, isInfraCapability, isModelDownloaded, isSourceNewer, isolatedBuiltinPhase, loadTlsCert, localEndpointPath, localProviderLink, mountNativeCapService, parseCapAction, parseRangeHeader, parseTokenizedUrl, partitionIsolatedBuiltinIds, proxyToUpstream, quarantineAddonResidue, readExtraSans, readLanHttpState, readPendingRestart, readTlsAccessStatus, readTlsMode, readinessKey, registerEventBusService, registerLanHttpHandler, reissueTlsLeaf, resolveFilePath, resolveHwAccel, resolveNpmInvocation, runHubAddonBoot, runNpm, scheduleSelfRestart, scopeKey, scopesAllowAddon, scopesAllowDeviceCap, selectAddonResidue, serializeTypedArrays, setHubConnected, setNodeEventInterest, shouldReclaim, startHeapWatch, startRunnerHeapWatch, strandedMb, stripCamstackDeps, subscribePassthrough, udsChildLogToWorkerEntry, validateProviderRegistrations, validateUploadedTls, waitUntilReady, writeExtraSans, writePendingRestart, writeTlsMode };
|
|
95286
|
+
export { AGENT_CAP_FWD_ACTION, AGENT_CAP_FWD_SERVICE, AGENT_READINESS_SERVICE_NAME, ALL_CAPABILITY_DEFINITIONS, AddonApiFactory, AddonDepsManager, AddonEngineManager, AddonHealthMonitor, AddonInstaller, AddonLoader, AddonManifest, AddonRouteRegistry, AlertCenterAddon, ApiKeyManager, AuthManager, CA_COMMON_NAME, CA_VALIDITY_DAYS, CLUSTER_SECRET_MISMATCH_TYPE, CLUSTER_SECRET_REJECTED_EXIT_CODE, CORE_CAP_SERVICE_NAME, CapRouteError, CapRouteResolver, CapUsageRegistry, CapabilityHandle, CapabilityRegistry, CapabilityUnavailableError, ConfigManager, ConfigStore, ConsoleDestination, ConsoleLoggingAddon, CoreBlocksAddon, CustomActionRegistry, DEFAULT_DATA_PATH, DEFAULT_LAN_HTTP_PORT, DEFAULT_LOG_LEVEL, DEVICE_SETTINGS_CONTRIBUTION_METHODS, DEVICE_STATUS_METHOD, DataPlaneRegistry, DeviceManagerAddon, DeviceRegistry, DeviceStore, EVENT_TOPIC_PREFIX, EngineManagerResolver, EventBus, FeatureManager, FilesystemStorageAddon, FilesystemStorageProvider, FrameDecoder, FsStorageBackend, HEALTH_MONITOR_GRACE_PERIOD_MS, HEALTH_MONITOR_RETRY_INTERVALS_MS, HEALTH_MONITOR_TICK_MS, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_CAP_FWD_ACTION, HUB_CAP_FWD_SERVICE, HUB_MAIN_RSS_BUDGET_MB, HUB_RSS_BUDGET_ENV, HubForwarderAddon, HubForwarderDestination, HubLogForwarder, HubNodeRegistry, INFRA_CAPABILITIES, IntegrationRegistry, JobJournal, LEAF_RENEWAL_WINDOW_DAYS, LifecycleJobEngine, LifecycleStateMachine, LivenessMonitorAddon, LocalAuthAddon, LocalChildClient, LocalChildRegistry, LogManager, LogRingBuffer, LoggingGate, LokiDestination, LokiLoggingAddon, MAX_LEAF_VALIDITY_DAYS, METHOD_ACCESS_MAP, ModelDownloadService, NATIVE_PROVIDER_SERVICE_INFIX, NATIVE_SCAN_DEPTH, NativeMetricsAddon, NativeMetricsProvider, NetworkQualityTracker, NotificationService, PYTHON_VERSION, PipelineRunner, PipelineValidator, PythonEnvManager, QUARANTINE_DIRNAME, RESTART_MARKER_FILE, RSS_BUDGET_REANNOUNCE_MIN_MS, RSS_BUDGET_RELEASE_RATIO, RUNNER_HEAP_WATCH_INTERVAL_MS, RUNNER_RSS_BUDGET_ENV, RUNTIME_DEFAULTS, ReadinessRegistry, ReadinessTimeoutError, ReplEngine, RingBuffer, SERVER_AUTH_OID, ScopedLogger, ScopedTokenManager, SocketChannel, SqliteSettingsAddon, SqliteSettingsBackend, StagingArea, StorageLocationManager, StorageManager, StorageOrchestratorAddon, StorageOrchestratorService, SystemConfigAddon, SystemEventBus, TRADITIONAL_NATIVE_PACKAGES, ToastService, UDS_NO_ROUTE_PREFIX, UdsLocalTransportClient, UdsLocalTransportServer, UserManager, WinstonDestination, WinstonLoggingAddon, __resetCapUsageRegistryForTests, __resetLoggingGateForTests, adaptBrokerToCluster, addonSettingsCapability, allFamiliesListenHost, applyLanHttp, bindPendingLanHttp, bootstrapSchema, brokerCallForCap, brokerTransportLink, buildBinaryPath, buildCapRouters, buildHeapSample, buildLinkChain, buildNativeCapProxy, buildNodeManifest, buildStorageLocationRegistry, buildUdsNativeCapProxy, builderMountedCapNames, callRegisterNodeWithRetry, callWithServiceDiscovery, canServeDataPlane, capActionName, capActionSuffix, capBareAction, capServiceName, classifyAddonDir, classifyCapRoute, clearPendingRestart, closeLanHttp, clusterEventTopic, clusterSecretMatches, collectCertIdentity, collectModelFiles, contentTypeFor, copyDirRecursive, copyExtraFileDirs, createAddonContext, createAddonDataPlaneFacility, createAddonService, createAuthenticatedFileServer, createBroker, createBrokerDeviceManagerApi, createCoreCapService, createDoorSettingsView, createFileDataPlaneHandler, createHubCapForwardService, createHubService, createKernelHwAccel, createLocalTransport, createParentUnownedCallHandler, createProcessService, createReadinessService, createReadinessServiceForRegistry, createScopedProcessManager, createStreamProbeBrokerService, createUdsAddonContext, createUdsEventBridge, createUdsEventBus, createUdsLogger, createUdsLoggerWithControl, createV8Reclaimer, deleteModelFromDisk, deriveAgentListenPort, describeProviderKindDrift, describeRss, detectWorkspacePackagesDir, downloadBinary, downloadFile, downloadModel, emitDownForOwnedCaps, encodeFrame, ensureAddonNativePrebuilds, ensureBinary, ensureDir, ensureFfmpeg, ensureLibraryBuilt, ensureModel, ensureNativePrebuilds, ensurePython, ensureTlsCert, evaluateExistingCert, expandCapMethods, fetchJson, findInPath, formatLogLine, getBrokerEventBus, getCapUsageRegistry, getFfmpegDownloadUrl, getLoggingGate, getModelFilePath, getMoleculerEventStats, getOrInitReadinessRegistry, getOrInitReadinessRegistryForClient, getPidStats, getPlatformInfo, getPythonDownloadUrl, getRestartMarkerPath, getSinglePidStats, getWorkerDeviceRegistry, hasDotNode, hashClusterSecret, hubMainRssBudget, installManifestNativeDeps, installManifestPythonDeps, installPackageFromNpm, installPythonPackages, installPythonRequirements, ipcParentLink, isAddonDeploySource, isArrayOutputSchema, isClusterSecretMismatchError, isCollectionArrayMethod, isInfraCapability, isModelDownloaded, isSourceNewer, isolatedBuiltinPhase, loadTlsCert, localEndpointPath, localProviderLink, mountNativeCapService, nextRssBudgetState, parseCapAction, parseRangeHeader, parseRssBudgetMb, parseTokenizedUrl, partitionIsolatedBuiltinIds, proxyToUpstream, quarantineAddonResidue, readExtraSans, readLanHttpState, readPendingRestart, readTlsAccessStatus, readTlsMode, readinessKey, registerEventBusService, registerLanHttpHandler, reissueTlsLeaf, resolveFilePath, resolveHwAccel, resolveNpmInvocation, runHubAddonBoot, runNpm, scheduleSelfRestart, scopeKey, scopesAllowAddon, scopesAllowDeviceCap, selectAddonResidue, serializeTypedArrays, setHubConnected, setNodeEventInterest, shouldReclaim, startHeapWatch, startRunnerHeapWatch, strandedMb, stripCamstackDeps, subscribePassthrough, udsChildLogToWorkerEntry, validateProviderRegistrations, validateUploadedTls, waitUntilReady, writeExtraSans, writePendingRestart, writeTlsMode };
|
|
@@ -207,10 +207,103 @@ export interface HeapWatchSink {
|
|
|
207
207
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
208
208
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
209
209
|
* and they are precisely what this number exists to see.
|
|
210
|
+
*
|
|
211
|
+
* ## What this number can and CANNOT be asked
|
|
212
|
+
*
|
|
213
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
214
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
215
|
+
* gate that should ever be built on it.
|
|
216
|
+
*
|
|
217
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
218
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
219
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
220
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
221
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
222
|
+
* quantity cannot represent the event.
|
|
223
|
+
*
|
|
224
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
225
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
210
226
|
*/
|
|
211
227
|
export declare function strandedMb(s: HeapSample): number;
|
|
212
228
|
/** Whether it is worth paying a stop-the-world to get those pages back. */
|
|
213
229
|
export declare function shouldReclaim(s: HeapSample, triggerMb?: number): boolean;
|
|
230
|
+
/**
|
|
231
|
+
* Which of the three MEASURED growth shapes dominates a process's footprint.
|
|
232
|
+
*
|
|
233
|
+
* These are not categories invented for a log line. They are what `smaps` found
|
|
234
|
+
* on the three live processes on 2026-08-27, and each one has a different lever:
|
|
235
|
+
*
|
|
236
|
+
* | process | evidence | shape |
|
|
237
|
+
* | --- | --- | --- |
|
|
238
|
+
* | `hub/recorder` | 1509.8MB in V8's 256KB page ranges, 53.9MB `brk`, `external` 8MB | `v8-heap` |
|
|
239
|
+
* | `hub/stream-broker` | 242.6MB V8 pages but **1159.1MB `brk`**, `arrayBuffers` 829MB | `external` |
|
|
240
|
+
* | `hub-main` | 1295.1MB V8 pages, 566.7MB `brk`, **2.7GB across 13 anonymous mappings >=64MB** | `native-residue` |
|
|
241
|
+
*
|
|
242
|
+
* - `v8-heap` — live JS retainers. `--max-old-space-size` is the bound, and the
|
|
243
|
+
* fix is the retained set (the recorder's ~7.1M `SegmentRow`s).
|
|
244
|
+
* - `external` — Buffers and native `external`. **No V8 flag bounds any of
|
|
245
|
+
* it**; `hub/pipeline-analytics` reached 5735MB with `--max-old-space-size=1024`
|
|
246
|
+
* in force the whole time.
|
|
247
|
+
* - `native-residue` — memory already freed that the allocator has not returned
|
|
248
|
+
* to the kernel. This is what the reclaim pass exists for, and the only shape
|
|
249
|
+
* where `MALLOC_ARENA_MAX` / `VIPS_CONCURRENCY` are the lever — both of which
|
|
250
|
+
* are ALREADY applied here, so a `native-residue` line means that lever is
|
|
251
|
+
* spent and the answer is upstream.
|
|
252
|
+
*
|
|
253
|
+
* Reporting only "it is big" leaves the operator to guess between three
|
|
254
|
+
* different fixes, two of which cannot work.
|
|
255
|
+
*/
|
|
256
|
+
export type RssGrowthShape = 'v8-heap' | 'external' | 'native-residue';
|
|
257
|
+
/** Where a process's footprint stands against the budget its role declared. */
|
|
258
|
+
export type RssBudgetState = 'within' | 'over';
|
|
259
|
+
/** The footprint, split into the three parts that imply different fixes. */
|
|
260
|
+
export interface RssBreakdown {
|
|
261
|
+
readonly rssMb: number;
|
|
262
|
+
readonly budgetMb: number;
|
|
263
|
+
/** How far past the budget. 0 when within — "how far over" has no meaning below it. */
|
|
264
|
+
readonly overMb: number;
|
|
265
|
+
readonly heapUsedMb: number;
|
|
266
|
+
readonly externalMb: number;
|
|
267
|
+
/** `rss − heapUsed − external`: native the other two numbers do not name. */
|
|
268
|
+
readonly nativeResidueMb: number;
|
|
269
|
+
readonly arrayBuffersMb: number;
|
|
270
|
+
readonly shape: RssGrowthShape;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
274
|
+
*
|
|
275
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
276
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
277
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
278
|
+
*/
|
|
279
|
+
export declare function describeRss(s: HeapSample, budgetMb: number): RssBreakdown;
|
|
280
|
+
/**
|
|
281
|
+
* Release the alarm below this share of the budget.
|
|
282
|
+
*
|
|
283
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
284
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
285
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
286
|
+
*/
|
|
287
|
+
export declare const RSS_BUDGET_RELEASE_RATIO = 0.9;
|
|
288
|
+
/**
|
|
289
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
290
|
+
*
|
|
291
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
292
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
293
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
294
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
295
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
296
|
+
*
|
|
297
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
298
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
299
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
300
|
+
*/
|
|
301
|
+
export declare const RSS_BUDGET_REANNOUNCE_MIN_MS = 900000;
|
|
302
|
+
/**
|
|
303
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
304
|
+
* allocating five gigabytes to reach them.
|
|
305
|
+
*/
|
|
306
|
+
export declare function nextRssBudgetState(current: RssBudgetState, rssMb: number, budgetMb: number, releaseRatio?: number): RssBudgetState;
|
|
214
307
|
/** A full collection that also asks V8 to hand the pages back to the OS. */
|
|
215
308
|
export type HeapReclaimer = () => void;
|
|
216
309
|
/**
|
|
@@ -240,13 +333,71 @@ export interface HeapReclaimOptions {
|
|
|
240
333
|
/** Ratio at which it releases. See {@link HEAP_WATCH_DEESCALATE_RATIO}. */
|
|
241
334
|
readonly deescalateRatio?: number;
|
|
242
335
|
}
|
|
336
|
+
/**
|
|
337
|
+
* The RSS budget for ONE process, and where the number came from.
|
|
338
|
+
*
|
|
339
|
+
* ## Why a DECLARED budget, and not a derived one
|
|
340
|
+
*
|
|
341
|
+
* Every bound this system had was derived, and each one is blind to most of a
|
|
342
|
+
* process: `--max-old-space-size` bounds old space only, and `strandedMb`
|
|
343
|
+
* subtracts `external` by construction. `hub/pipeline-analytics` was killed at
|
|
344
|
+
* 11.4GB with a 1024MB old-space ceiling in force and a stranded number that
|
|
345
|
+
* never moved. RSS is the quantity the host's OOM killer actually reads, so RSS
|
|
346
|
+
* is the quantity that needs a ceiling — and RSS cannot be derived from any flag,
|
|
347
|
+
* because most of it is not V8's.
|
|
348
|
+
*
|
|
349
|
+
* ## Why there is no single constant
|
|
350
|
+
*
|
|
351
|
+
* The legitimate footprint differs by an order of magnitude BY ROLE, measured
|
|
352
|
+
* on the same host on the same day:
|
|
353
|
+
*
|
|
354
|
+
* - `hub/recorder` legitimately holds ~1.5GB of live V8 index (~7.1M
|
|
355
|
+
* `SegmentRow`s, ~30 days x 13 cameras x 2 profiles) and declares a 3072MB
|
|
356
|
+
* old space to hold it;
|
|
357
|
+
* - `hub/stream-broker` legitimately holds ~900MB of `arrayBuffers` (47 RTP
|
|
358
|
+
* pre-roll rings) in the glibc heap, with an old space of only 224MB;
|
|
359
|
+
* - `hub/pipeline-analytics` has no legitimate multi-GB working set at all —
|
|
360
|
+
* it died at 11.4GB holding a 536MB heap.
|
|
361
|
+
*
|
|
362
|
+
* One constant would either wave through analytics' 11GB or scream at the
|
|
363
|
+
* recorder's entirely correct 2GB. So the number lives **next to the role that
|
|
364
|
+
* knows it**: an addon declares `execution.rssBudgetMb` in its own manifest,
|
|
365
|
+
* exactly as it already declares `execution.maxOldSpaceMb`, and the runner takes
|
|
366
|
+
* the MAXIMUM any co-located addon declares (D2/D29 — a runner is respawned as a
|
|
367
|
+
* whole, so co-location must never silence the hungriest member).
|
|
368
|
+
*
|
|
369
|
+
* hub-main has no manifest, and its entrypoint is the LEAST deployable surface
|
|
370
|
+
* in this system (the Dockerfile's own comment: a backend change is train-bound
|
|
371
|
+
* by construction, and an image build does not ship it). Its budget therefore
|
|
372
|
+
* lives in `@camstack/server`, which `applyServerUpdate` replaces the same day —
|
|
373
|
+
* see {@link HUB_MAIN_RSS_BUDGET_MB}.
|
|
374
|
+
*
|
|
375
|
+
* ## And a process with no declaration is UNWATCHED, out loud
|
|
376
|
+
*
|
|
377
|
+
* `budgetMb: undefined` does not select a default. Defaulting would be the same
|
|
378
|
+
* mistake in a new place: a number nobody measured, applied to a process nobody
|
|
379
|
+
* profiled, producing either silence or noise with equal authority. The process
|
|
380
|
+
* says once, at boot, that nothing is watching it. Most runners are in this
|
|
381
|
+
* state today and that is the honest report — measurements exist for four
|
|
382
|
+
* processes, so four processes carry a budget.
|
|
383
|
+
*/
|
|
384
|
+
export interface RssBudgetOptions {
|
|
385
|
+
/** MB. `undefined` → this process is declared UNWATCHED and says so at boot. */
|
|
386
|
+
readonly budgetMb: number | undefined;
|
|
387
|
+
/** Named in every line, so a budget that is wrong is traceable to its author. */
|
|
388
|
+
readonly source: string;
|
|
389
|
+
/** See {@link RSS_BUDGET_RELEASE_RATIO}. */
|
|
390
|
+
readonly releaseRatio?: number;
|
|
391
|
+
/** See {@link RSS_BUDGET_REANNOUNCE_MIN_MS}. */
|
|
392
|
+
readonly reannounceMinMs?: number;
|
|
393
|
+
}
|
|
243
394
|
/**
|
|
244
395
|
* Start the heartbeat. Returns a stop function.
|
|
245
396
|
*
|
|
246
397
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
247
398
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
248
399
|
*/
|
|
249
|
-
export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions, loopDelay?: LoopDelayMeter | undefined, execArgv?: readonly string[], announceCeilingOrigin?: boolean): () => void;
|
|
400
|
+
export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions, loopDelay?: LoopDelayMeter | undefined, execArgv?: readonly string[], announceCeilingOrigin?: boolean, rssBudget?: RssBudgetOptions): () => void;
|
|
250
401
|
/**
|
|
251
402
|
* Steady heartbeat period for a HEAVY addon runner.
|
|
252
403
|
*
|
|
@@ -284,7 +435,64 @@ export interface RunnerHeapWatchOptions {
|
|
|
284
435
|
readonly intervalMs?: number;
|
|
285
436
|
/** Test injection. Production omits it and gets {@link createV8Reclaimer}. */
|
|
286
437
|
readonly reclaimOptions?: HeapReclaimOptions;
|
|
438
|
+
/**
|
|
439
|
+
* `CAMSTACK_RUNNER_RSS_BUDGET_MB` as the parent passed it — the MAXIMUM
|
|
440
|
+
* `execution.rssBudgetMb` any co-located addon declares, resolved from the
|
|
441
|
+
* manifests in `process-service.ts`. Absent means no co-located addon declared
|
|
442
|
+
* one, and the runner reports itself UNWATCHED. See {@link RssBudgetOptions}.
|
|
443
|
+
*/
|
|
444
|
+
readonly rssBudgetMb?: string;
|
|
287
445
|
}
|
|
446
|
+
/**
|
|
447
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
448
|
+
*
|
|
449
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
450
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
451
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
452
|
+
*/
|
|
453
|
+
export declare function parseRssBudgetMb(raw: string | undefined): number | undefined;
|
|
454
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
455
|
+
export declare const RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
456
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
457
|
+
export declare const HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
458
|
+
/**
|
|
459
|
+
* hub-main's declared RSS budget, in MB.
|
|
460
|
+
*
|
|
461
|
+
* ## Why the number lives HERE and not in the image
|
|
462
|
+
*
|
|
463
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
464
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
465
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
466
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
467
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
468
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
469
|
+
* A budget that cannot ship is not a budget.
|
|
470
|
+
*
|
|
471
|
+
* ## Why 4096
|
|
472
|
+
*
|
|
473
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
474
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
475
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
476
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
477
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
478
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
479
|
+
*
|
|
480
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
481
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
482
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
483
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
484
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
485
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
486
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
487
|
+
* just above today's RSS would be a rubber stamp.
|
|
488
|
+
*
|
|
489
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
490
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
491
|
+
* instead of being absorbed by a threshold nobody set.
|
|
492
|
+
*/
|
|
493
|
+
export declare const HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
494
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
495
|
+
export declare function hubMainRssBudget(env?: NodeJS.ProcessEnv): RssBudgetOptions;
|
|
288
496
|
/**
|
|
289
497
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
290
498
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
@@ -262,6 +262,22 @@ var consoleSink = {
|
|
|
262
262
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
263
263
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
264
264
|
* and they are precisely what this number exists to see.
|
|
265
|
+
*
|
|
266
|
+
* ## What this number can and CANNOT be asked
|
|
267
|
+
*
|
|
268
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
269
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
270
|
+
* gate that should ever be built on it.
|
|
271
|
+
*
|
|
272
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
273
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
274
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
275
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
276
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
277
|
+
* quantity cannot represent the event.
|
|
278
|
+
*
|
|
279
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
280
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
265
281
|
*/
|
|
266
282
|
function strandedMb(s) {
|
|
267
283
|
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
@@ -270,6 +286,67 @@ function strandedMb(s) {
|
|
|
270
286
|
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
271
287
|
return strandedMb(s) > triggerMb;
|
|
272
288
|
}
|
|
289
|
+
/** Share of RSS, as a whole percent. Never divides by zero. */
|
|
290
|
+
function share(partMb, rssMb) {
|
|
291
|
+
return rssMb > 0 ? Math.round(partMb / rssMb * 100) : 0;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
295
|
+
*
|
|
296
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
297
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
298
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
299
|
+
*/
|
|
300
|
+
function describeRss(s, budgetMb) {
|
|
301
|
+
const nativeResidueMb = Math.max(0, strandedMb(s));
|
|
302
|
+
const shape = s.heapUsedMb >= s.externalMb && s.heapUsedMb >= nativeResidueMb ? "v8-heap" : s.externalMb >= nativeResidueMb ? "external" : "native-residue";
|
|
303
|
+
return {
|
|
304
|
+
rssMb: s.rssMb,
|
|
305
|
+
budgetMb,
|
|
306
|
+
overMb: Math.max(0, s.rssMb - budgetMb),
|
|
307
|
+
heapUsedMb: s.heapUsedMb,
|
|
308
|
+
externalMb: s.externalMb,
|
|
309
|
+
nativeResidueMb,
|
|
310
|
+
arrayBuffersMb: s.arrayBuffersMb,
|
|
311
|
+
shape
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
/** What the operator can actually DO about each shape. */
|
|
315
|
+
var SHAPE_ADVICE = {
|
|
316
|
+
"v8-heap": "live JS retainers — bounded by --max-old-space-size; the fix is the retained set",
|
|
317
|
+
external: "Buffers / native external — NO V8 flag bounds this, and the old-space ceiling never will",
|
|
318
|
+
"native-residue": "freed native memory the allocator has not returned — MALLOC_ARENA_MAX/VIPS_CONCURRENCY are already applied, so this lever is spent"
|
|
319
|
+
};
|
|
320
|
+
/**
|
|
321
|
+
* Release the alarm below this share of the budget.
|
|
322
|
+
*
|
|
323
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
324
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
325
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
326
|
+
*/
|
|
327
|
+
var RSS_BUDGET_RELEASE_RATIO = .9;
|
|
328
|
+
/**
|
|
329
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
330
|
+
*
|
|
331
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
332
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
333
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
334
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
335
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
336
|
+
*
|
|
337
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
338
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
339
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
340
|
+
*/
|
|
341
|
+
var RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
342
|
+
/**
|
|
343
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
344
|
+
* allocating five gigabytes to reach them.
|
|
345
|
+
*/
|
|
346
|
+
function nextRssBudgetState(current, rssMb, budgetMb, releaseRatio = RSS_BUDGET_RELEASE_RATIO) {
|
|
347
|
+
if (current === "over") return rssMb < budgetMb * releaseRatio ? "within" : "over";
|
|
348
|
+
return rssMb > budgetMb ? "over" : "within";
|
|
349
|
+
}
|
|
273
350
|
function isGcFunction(value) {
|
|
274
351
|
return typeof value === "function";
|
|
275
352
|
}
|
|
@@ -297,8 +374,19 @@ function createV8Reclaimer() {
|
|
|
297
374
|
return;
|
|
298
375
|
}
|
|
299
376
|
}
|
|
300
|
-
|
|
301
|
-
|
|
377
|
+
/**
|
|
378
|
+
* The line that says HOW the process grew, not merely that it did.
|
|
379
|
+
*
|
|
380
|
+
* `heapUsed`, `external` and the native residue each carry their share of RSS,
|
|
381
|
+
* because the shares are what separate the three fixes — and on the process this
|
|
382
|
+
* module failed to catch, the shares were 9% / 74% / 16%, which names the answer
|
|
383
|
+
* on sight.
|
|
384
|
+
*/
|
|
385
|
+
function formatOverBudget(label, b, source) {
|
|
386
|
+
return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
|
|
387
|
+
}
|
|
388
|
+
function format(label, s, loop, budgetMb) {
|
|
389
|
+
const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
|
|
302
390
|
if (loop === void 0) return line;
|
|
303
391
|
return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
|
|
304
392
|
}
|
|
@@ -308,7 +396,7 @@ function format(label, s, loop) {
|
|
|
308
396
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
309
397
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
310
398
|
*/
|
|
311
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
|
|
399
|
+
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
|
|
312
400
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
313
401
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
314
402
|
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
@@ -352,6 +440,39 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
352
440
|
};
|
|
353
441
|
let mode = "steady";
|
|
354
442
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
443
|
+
const budgetMb = rssBudget?.budgetMb !== void 0 && rssBudget.budgetMb > 0 ? rssBudget.budgetMb : void 0;
|
|
444
|
+
const budgetSource = rssBudget?.source ?? "unspecified";
|
|
445
|
+
const releaseRatio = rssBudget?.releaseRatio ?? .9;
|
|
446
|
+
const reannounceMinMs = rssBudget?.reannounceMinMs ?? 9e5;
|
|
447
|
+
let budgetState = "within";
|
|
448
|
+
let lastBudgetWarnAt = Number.NEGATIVE_INFINITY;
|
|
449
|
+
/** Only pair a "back within" with an over-budget line that was actually said. */
|
|
450
|
+
let overBudgetAnnounced = false;
|
|
451
|
+
/**
|
|
452
|
+
* One line per STATE CHANGE, never one per sample.
|
|
453
|
+
*
|
|
454
|
+
* The probe runs every {@link HEAP_WATCH_FAST_INTERVAL_MS}; warning on every
|
|
455
|
+
* sample above the budget would put 30 lines a minute in the hub's shared log
|
|
456
|
+
* for a process that is legitimately, steadily large. That is precisely the
|
|
457
|
+
* failure mode that buried two real alarms under 83,198 heartbeat lines on
|
|
458
|
+
* 2026-08-27, so the crossing is the event and the level is not.
|
|
459
|
+
*/
|
|
460
|
+
const checkRssBudget = (sample, at) => {
|
|
461
|
+
if (budgetMb === void 0) return;
|
|
462
|
+
const previous = budgetState;
|
|
463
|
+
budgetState = nextRssBudgetState(previous, sample.rssMb, budgetMb, releaseRatio);
|
|
464
|
+
if (budgetState === previous) return;
|
|
465
|
+
if (budgetState === "over") {
|
|
466
|
+
if (at - lastBudgetWarnAt < reannounceMinMs) return;
|
|
467
|
+
lastBudgetWarnAt = at;
|
|
468
|
+
overBudgetAnnounced = true;
|
|
469
|
+
sink.warn(formatOverBudget(label, describeRss(sample, budgetMb), budgetSource));
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
if (!overBudgetAnnounced) return;
|
|
473
|
+
overBudgetAnnounced = false;
|
|
474
|
+
sink.info(`[mem] ${label} back within its RSS budget — rss=${sample.rssMb}MB budget=${budgetMb}MB`);
|
|
475
|
+
};
|
|
355
476
|
/**
|
|
356
477
|
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
357
478
|
* point.
|
|
@@ -377,18 +498,20 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
377
498
|
const due = at - lastLoggedAt >= intervalMs;
|
|
378
499
|
if (mode === "escalated" || due) {
|
|
379
500
|
lastLoggedAt = at;
|
|
380
|
-
const line = format(label, sample, loopDelay?.read());
|
|
501
|
+
const line = format(label, sample, loopDelay?.read(), budgetMb);
|
|
381
502
|
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
382
503
|
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
383
504
|
else sink.info(line);
|
|
384
505
|
}
|
|
385
506
|
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
507
|
+
checkRssBudget(sample, at);
|
|
386
508
|
maybeReclaim(sample);
|
|
387
509
|
} catch {}
|
|
388
510
|
};
|
|
389
511
|
const timer = setInterval(tick, probeIntervalMs);
|
|
390
512
|
timer.unref?.();
|
|
391
513
|
tick();
|
|
514
|
+
if (rssBudget !== void 0 && budgetMb === void 0) sink.info(`[mem] ${label} is NOT WATCHED against an RSS budget — no budget declared (${budgetSource}). rss growth outside the V8 heap raises nothing here: --max-old-space-size bounds old space only, and stranded subtracts external by construction. Declare execution.rssBudgetMb once this process has a measured working set.`);
|
|
392
515
|
if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
|
|
393
516
|
let stopped = false;
|
|
394
517
|
return () => {
|
|
@@ -414,6 +537,71 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
414
537
|
*/
|
|
415
538
|
var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
416
539
|
/**
|
|
540
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
541
|
+
*
|
|
542
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
543
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
544
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
545
|
+
*/
|
|
546
|
+
function parseRssBudgetMb(raw) {
|
|
547
|
+
if (raw === void 0) return void 0;
|
|
548
|
+
const parsed = Number(raw);
|
|
549
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
550
|
+
return Math.floor(parsed);
|
|
551
|
+
}
|
|
552
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
553
|
+
var RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
554
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
555
|
+
var HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
556
|
+
/**
|
|
557
|
+
* hub-main's declared RSS budget, in MB.
|
|
558
|
+
*
|
|
559
|
+
* ## Why the number lives HERE and not in the image
|
|
560
|
+
*
|
|
561
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
562
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
563
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
564
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
565
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
566
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
567
|
+
* A budget that cannot ship is not a budget.
|
|
568
|
+
*
|
|
569
|
+
* ## Why 4096
|
|
570
|
+
*
|
|
571
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
572
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
573
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
574
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
575
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
576
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
577
|
+
*
|
|
578
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
579
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
580
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
581
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
582
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
583
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
584
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
585
|
+
* just above today's RSS would be a rubber stamp.
|
|
586
|
+
*
|
|
587
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
588
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
589
|
+
* instead of being absorbed by a threshold nobody set.
|
|
590
|
+
*/
|
|
591
|
+
var HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
592
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
593
|
+
function hubMainRssBudget(env = process.env) {
|
|
594
|
+
const override = env[HUB_RSS_BUDGET_ENV];
|
|
595
|
+
if (override !== void 0) return {
|
|
596
|
+
budgetMb: parseRssBudgetMb(override),
|
|
597
|
+
source: HUB_RSS_BUDGET_ENV
|
|
598
|
+
};
|
|
599
|
+
return {
|
|
600
|
+
budgetMb: HUB_MAIN_RSS_BUDGET_MB,
|
|
601
|
+
source: "HUB_MAIN_RSS_BUDGET_MB (kernel/heap-watch)"
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
417
605
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
418
606
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
419
607
|
*
|
|
@@ -447,13 +635,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
|
447
635
|
function startRunnerHeapWatch(options) {
|
|
448
636
|
if (options.heapProfile !== "heavy") return void 0;
|
|
449
637
|
const intervalMs = options.intervalMs ?? 3e5;
|
|
450
|
-
|
|
638
|
+
const rssBudget = {
|
|
639
|
+
budgetMb: parseRssBudgetMb(options.rssBudgetMb),
|
|
640
|
+
source: `manifest execution.rssBudgetMb (via ${RUNNER_RSS_BUDGET_ENV})`
|
|
641
|
+
};
|
|
642
|
+
if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false, rssBudget);
|
|
451
643
|
let reclaimOptions = options.reclaimOptions;
|
|
452
644
|
if (reclaimOptions === void 0) {
|
|
453
645
|
const reclaimer = createV8Reclaimer();
|
|
454
646
|
reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
|
|
455
647
|
}
|
|
456
|
-
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
|
|
648
|
+
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true, rssBudget);
|
|
457
649
|
}
|
|
458
650
|
//#endregion
|
|
459
651
|
//#region src/kernel/moleculer/addon-data-plane-facility.ts
|
|
@@ -8390,4 +8582,4 @@ async function installManifestPythonDeps(declaration, addonDir, deps, logger) {
|
|
|
8390
8582
|
await deps.installPythonRequirements(reqAbs);
|
|
8391
8583
|
}
|
|
8392
8584
|
//#endregion
|
|
8393
|
-
export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A,
|
|
8585
|
+
export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A, RSS_BUDGET_REANNOUNCE_MIN_MS as At, AGENT_CAP_FWD_SERVICE as B, shouldReclaim as Bt, brokerTransportLink as C, createAddonDataPlaneFacility as Ct, HUB_CAP_FWD_ACTION as D, HEAP_WATCH_WARN_RATIO as Dt, localProviderLink as E, HEAP_WATCH_INTERVAL_MS as Et, createUdsLoggerWithControl as F, createV8Reclaimer as Ft, createLocalTransport as G, CapRouteError as H, startRunnerHeapWatch as Ht, LocalChildClient as I, describeRss as It, SocketChannel as J, UdsLocalTransportClient as K, LocalChildRegistry as L, hubMainRssBudget as Lt, createUdsEventBus as M, RUNNER_HEAP_WATCH_INTERVAL_MS as Mt, udsChildLogToWorkerEntry as N, RUNNER_RSS_BUDGET_ENV as Nt, HUB_CAP_FWD_SERVICE as O, HUB_MAIN_RSS_BUDGET_MB as Ot, createUdsLogger as P, buildHeapSample as Pt, buildNativeCapProxy as Q, UDS_NO_ROUTE_PREFIX as R, nextRssBudgetState as Rt, brokerCallForCap as S, runNpm as St, ipcParentLink as T, HEAP_RECLAIM_TRIGGER_MB as Tt, classifyCapRoute as U, strandedMb as Ut, CapRouteResolver as V, startHeapWatch as Vt, callWithServiceDiscovery as W, FrameDecoder as X, localEndpointPath as Y, encodeFrame as Z, createKernelHwAccel as _, CapabilityUnavailableError as _t, getWorkerDeviceRegistry as a, createAddonService as at, __resetCapUsageRegistryForTests as b, resolveAddonClass as bt, setHubConnected as c, capActionName as ct, getBrokerEventBus as d, capServiceName as dt, createBrokerDeviceManagerApi as et, getMoleculerEventStats as f, parseCapAction as ft, AddonDepsManager as g, CapabilityHandle as gt, subscribePassthrough as h, DeviceRegistry as ht, createUdsAddonContext as i, setWorkerNativeCapsChangeListener as it, createUdsEventBridge as j, RSS_BUDGET_RELEASE_RATIO as jt, createHubCapForwardService as k, HUB_RSS_BUDGET_ENV as kt, EVENT_TOPIC_PREFIX as l, capActionSuffix as lt, setNodeEventInterest as m, serializeTypedArrays as mt, adaptBrokerToCluster as n, getWorkerNativeCapSnapshot as nt, getOrInitReadinessRegistry as o, validateProviderRegistrations as ot, registerEventBusService as p, deserializeTypedArrays as pt, UdsLocalTransportServer as q, createAddonContext as r, mountNativeCapService as rt, getOrInitReadinessRegistryForClient as s, NATIVE_PROVIDER_SERVICE_INFIX as st, installManifestPythonDeps as t, getWorkerNativeCapProvider as tt, clusterEventTopic as u, capBareAction as ut, resolveHwAccel as v, copyBundledNativeModules as vt, buildLinkChain as w, HEAP_RECLAIM_MIN_INTERVAL_MS as wt, getCapUsageRegistry as x, resolveNpmInvocation as xt, CapUsageRegistry as y, installManifestNativeDeps as yt, AGENT_CAP_FWD_ACTION as z, parseRssBudgetMb as zt };
|
|
@@ -266,6 +266,22 @@ var consoleSink = {
|
|
|
266
266
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
267
267
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
268
268
|
* and they are precisely what this number exists to see.
|
|
269
|
+
*
|
|
270
|
+
* ## What this number can and CANNOT be asked
|
|
271
|
+
*
|
|
272
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
273
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
274
|
+
* gate that should ever be built on it.
|
|
275
|
+
*
|
|
276
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
277
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
278
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
279
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
280
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
281
|
+
* quantity cannot represent the event.
|
|
282
|
+
*
|
|
283
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
284
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
269
285
|
*/
|
|
270
286
|
function strandedMb(s) {
|
|
271
287
|
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
@@ -274,6 +290,67 @@ function strandedMb(s) {
|
|
|
274
290
|
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
275
291
|
return strandedMb(s) > triggerMb;
|
|
276
292
|
}
|
|
293
|
+
/** Share of RSS, as a whole percent. Never divides by zero. */
|
|
294
|
+
function share(partMb, rssMb) {
|
|
295
|
+
return rssMb > 0 ? Math.round(partMb / rssMb * 100) : 0;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
299
|
+
*
|
|
300
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
301
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
302
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
303
|
+
*/
|
|
304
|
+
function describeRss(s, budgetMb) {
|
|
305
|
+
const nativeResidueMb = Math.max(0, strandedMb(s));
|
|
306
|
+
const shape = s.heapUsedMb >= s.externalMb && s.heapUsedMb >= nativeResidueMb ? "v8-heap" : s.externalMb >= nativeResidueMb ? "external" : "native-residue";
|
|
307
|
+
return {
|
|
308
|
+
rssMb: s.rssMb,
|
|
309
|
+
budgetMb,
|
|
310
|
+
overMb: Math.max(0, s.rssMb - budgetMb),
|
|
311
|
+
heapUsedMb: s.heapUsedMb,
|
|
312
|
+
externalMb: s.externalMb,
|
|
313
|
+
nativeResidueMb,
|
|
314
|
+
arrayBuffersMb: s.arrayBuffersMb,
|
|
315
|
+
shape
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
/** What the operator can actually DO about each shape. */
|
|
319
|
+
var SHAPE_ADVICE = {
|
|
320
|
+
"v8-heap": "live JS retainers — bounded by --max-old-space-size; the fix is the retained set",
|
|
321
|
+
external: "Buffers / native external — NO V8 flag bounds this, and the old-space ceiling never will",
|
|
322
|
+
"native-residue": "freed native memory the allocator has not returned — MALLOC_ARENA_MAX/VIPS_CONCURRENCY are already applied, so this lever is spent"
|
|
323
|
+
};
|
|
324
|
+
/**
|
|
325
|
+
* Release the alarm below this share of the budget.
|
|
326
|
+
*
|
|
327
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
328
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
329
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
330
|
+
*/
|
|
331
|
+
var RSS_BUDGET_RELEASE_RATIO = .9;
|
|
332
|
+
/**
|
|
333
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
334
|
+
*
|
|
335
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
336
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
337
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
338
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
339
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
340
|
+
*
|
|
341
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
342
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
343
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
344
|
+
*/
|
|
345
|
+
var RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
346
|
+
/**
|
|
347
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
348
|
+
* allocating five gigabytes to reach them.
|
|
349
|
+
*/
|
|
350
|
+
function nextRssBudgetState(current, rssMb, budgetMb, releaseRatio = RSS_BUDGET_RELEASE_RATIO) {
|
|
351
|
+
if (current === "over") return rssMb < budgetMb * releaseRatio ? "within" : "over";
|
|
352
|
+
return rssMb > budgetMb ? "over" : "within";
|
|
353
|
+
}
|
|
277
354
|
function isGcFunction(value) {
|
|
278
355
|
return typeof value === "function";
|
|
279
356
|
}
|
|
@@ -301,8 +378,19 @@ function createV8Reclaimer() {
|
|
|
301
378
|
return;
|
|
302
379
|
}
|
|
303
380
|
}
|
|
304
|
-
|
|
305
|
-
|
|
381
|
+
/**
|
|
382
|
+
* The line that says HOW the process grew, not merely that it did.
|
|
383
|
+
*
|
|
384
|
+
* `heapUsed`, `external` and the native residue each carry their share of RSS,
|
|
385
|
+
* because the shares are what separate the three fixes — and on the process this
|
|
386
|
+
* module failed to catch, the shares were 9% / 74% / 16%, which names the answer
|
|
387
|
+
* on sight.
|
|
388
|
+
*/
|
|
389
|
+
function formatOverBudget(label, b, source) {
|
|
390
|
+
return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
|
|
391
|
+
}
|
|
392
|
+
function format(label, s, loop, budgetMb) {
|
|
393
|
+
const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
|
|
306
394
|
if (loop === void 0) return line;
|
|
307
395
|
return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
|
|
308
396
|
}
|
|
@@ -312,7 +400,7 @@ function format(label, s, loop) {
|
|
|
312
400
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
313
401
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
314
402
|
*/
|
|
315
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
|
|
403
|
+
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
|
|
316
404
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
317
405
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
318
406
|
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
@@ -356,6 +444,39 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
356
444
|
};
|
|
357
445
|
let mode = "steady";
|
|
358
446
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
447
|
+
const budgetMb = rssBudget?.budgetMb !== void 0 && rssBudget.budgetMb > 0 ? rssBudget.budgetMb : void 0;
|
|
448
|
+
const budgetSource = rssBudget?.source ?? "unspecified";
|
|
449
|
+
const releaseRatio = rssBudget?.releaseRatio ?? .9;
|
|
450
|
+
const reannounceMinMs = rssBudget?.reannounceMinMs ?? 9e5;
|
|
451
|
+
let budgetState = "within";
|
|
452
|
+
let lastBudgetWarnAt = Number.NEGATIVE_INFINITY;
|
|
453
|
+
/** Only pair a "back within" with an over-budget line that was actually said. */
|
|
454
|
+
let overBudgetAnnounced = false;
|
|
455
|
+
/**
|
|
456
|
+
* One line per STATE CHANGE, never one per sample.
|
|
457
|
+
*
|
|
458
|
+
* The probe runs every {@link HEAP_WATCH_FAST_INTERVAL_MS}; warning on every
|
|
459
|
+
* sample above the budget would put 30 lines a minute in the hub's shared log
|
|
460
|
+
* for a process that is legitimately, steadily large. That is precisely the
|
|
461
|
+
* failure mode that buried two real alarms under 83,198 heartbeat lines on
|
|
462
|
+
* 2026-08-27, so the crossing is the event and the level is not.
|
|
463
|
+
*/
|
|
464
|
+
const checkRssBudget = (sample, at) => {
|
|
465
|
+
if (budgetMb === void 0) return;
|
|
466
|
+
const previous = budgetState;
|
|
467
|
+
budgetState = nextRssBudgetState(previous, sample.rssMb, budgetMb, releaseRatio);
|
|
468
|
+
if (budgetState === previous) return;
|
|
469
|
+
if (budgetState === "over") {
|
|
470
|
+
if (at - lastBudgetWarnAt < reannounceMinMs) return;
|
|
471
|
+
lastBudgetWarnAt = at;
|
|
472
|
+
overBudgetAnnounced = true;
|
|
473
|
+
sink.warn(formatOverBudget(label, describeRss(sample, budgetMb), budgetSource));
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
if (!overBudgetAnnounced) return;
|
|
477
|
+
overBudgetAnnounced = false;
|
|
478
|
+
sink.info(`[mem] ${label} back within its RSS budget — rss=${sample.rssMb}MB budget=${budgetMb}MB`);
|
|
479
|
+
};
|
|
359
480
|
/**
|
|
360
481
|
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
361
482
|
* point.
|
|
@@ -381,18 +502,20 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
381
502
|
const due = at - lastLoggedAt >= intervalMs;
|
|
382
503
|
if (mode === "escalated" || due) {
|
|
383
504
|
lastLoggedAt = at;
|
|
384
|
-
const line = format(label, sample, loopDelay?.read());
|
|
505
|
+
const line = format(label, sample, loopDelay?.read(), budgetMb);
|
|
385
506
|
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
386
507
|
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
387
508
|
else sink.info(line);
|
|
388
509
|
}
|
|
389
510
|
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
511
|
+
checkRssBudget(sample, at);
|
|
390
512
|
maybeReclaim(sample);
|
|
391
513
|
} catch {}
|
|
392
514
|
};
|
|
393
515
|
const timer = setInterval(tick, probeIntervalMs);
|
|
394
516
|
timer.unref?.();
|
|
395
517
|
tick();
|
|
518
|
+
if (rssBudget !== void 0 && budgetMb === void 0) sink.info(`[mem] ${label} is NOT WATCHED against an RSS budget — no budget declared (${budgetSource}). rss growth outside the V8 heap raises nothing here: --max-old-space-size bounds old space only, and stranded subtracts external by construction. Declare execution.rssBudgetMb once this process has a measured working set.`);
|
|
396
519
|
if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
|
|
397
520
|
let stopped = false;
|
|
398
521
|
return () => {
|
|
@@ -418,6 +541,71 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
418
541
|
*/
|
|
419
542
|
var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
420
543
|
/**
|
|
544
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
545
|
+
*
|
|
546
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
547
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
548
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
549
|
+
*/
|
|
550
|
+
function parseRssBudgetMb(raw) {
|
|
551
|
+
if (raw === void 0) return void 0;
|
|
552
|
+
const parsed = Number(raw);
|
|
553
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
554
|
+
return Math.floor(parsed);
|
|
555
|
+
}
|
|
556
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
557
|
+
var RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
558
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
559
|
+
var HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
560
|
+
/**
|
|
561
|
+
* hub-main's declared RSS budget, in MB.
|
|
562
|
+
*
|
|
563
|
+
* ## Why the number lives HERE and not in the image
|
|
564
|
+
*
|
|
565
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
566
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
567
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
568
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
569
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
570
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
571
|
+
* A budget that cannot ship is not a budget.
|
|
572
|
+
*
|
|
573
|
+
* ## Why 4096
|
|
574
|
+
*
|
|
575
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
576
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
577
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
578
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
579
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
580
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
581
|
+
*
|
|
582
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
583
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
584
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
585
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
586
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
587
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
588
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
589
|
+
* just above today's RSS would be a rubber stamp.
|
|
590
|
+
*
|
|
591
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
592
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
593
|
+
* instead of being absorbed by a threshold nobody set.
|
|
594
|
+
*/
|
|
595
|
+
var HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
596
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
597
|
+
function hubMainRssBudget(env = process.env) {
|
|
598
|
+
const override = env[HUB_RSS_BUDGET_ENV];
|
|
599
|
+
if (override !== void 0) return {
|
|
600
|
+
budgetMb: parseRssBudgetMb(override),
|
|
601
|
+
source: HUB_RSS_BUDGET_ENV
|
|
602
|
+
};
|
|
603
|
+
return {
|
|
604
|
+
budgetMb: HUB_MAIN_RSS_BUDGET_MB,
|
|
605
|
+
source: "HUB_MAIN_RSS_BUDGET_MB (kernel/heap-watch)"
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
421
609
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
422
610
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
423
611
|
*
|
|
@@ -451,13 +639,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
|
451
639
|
function startRunnerHeapWatch(options) {
|
|
452
640
|
if (options.heapProfile !== "heavy") return void 0;
|
|
453
641
|
const intervalMs = options.intervalMs ?? 3e5;
|
|
454
|
-
|
|
642
|
+
const rssBudget = {
|
|
643
|
+
budgetMb: parseRssBudgetMb(options.rssBudgetMb),
|
|
644
|
+
source: `manifest execution.rssBudgetMb (via ${RUNNER_RSS_BUDGET_ENV})`
|
|
645
|
+
};
|
|
646
|
+
if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false, rssBudget);
|
|
455
647
|
let reclaimOptions = options.reclaimOptions;
|
|
456
648
|
if (reclaimOptions === void 0) {
|
|
457
649
|
const reclaimer = createV8Reclaimer();
|
|
458
650
|
reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
|
|
459
651
|
}
|
|
460
|
-
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
|
|
652
|
+
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true, rssBudget);
|
|
461
653
|
}
|
|
462
654
|
//#endregion
|
|
463
655
|
//#region src/kernel/moleculer/addon-data-plane-facility.ts
|
|
@@ -8496,6 +8688,18 @@ Object.defineProperty(exports, "HUB_CAP_FWD_SERVICE", {
|
|
|
8496
8688
|
return HUB_CAP_FWD_SERVICE;
|
|
8497
8689
|
}
|
|
8498
8690
|
});
|
|
8691
|
+
Object.defineProperty(exports, "HUB_MAIN_RSS_BUDGET_MB", {
|
|
8692
|
+
enumerable: true,
|
|
8693
|
+
get: function() {
|
|
8694
|
+
return HUB_MAIN_RSS_BUDGET_MB;
|
|
8695
|
+
}
|
|
8696
|
+
});
|
|
8697
|
+
Object.defineProperty(exports, "HUB_RSS_BUDGET_ENV", {
|
|
8698
|
+
enumerable: true,
|
|
8699
|
+
get: function() {
|
|
8700
|
+
return HUB_RSS_BUDGET_ENV;
|
|
8701
|
+
}
|
|
8702
|
+
});
|
|
8499
8703
|
Object.defineProperty(exports, "LocalChildClient", {
|
|
8500
8704
|
enumerable: true,
|
|
8501
8705
|
get: function() {
|
|
@@ -8514,12 +8718,30 @@ Object.defineProperty(exports, "NATIVE_PROVIDER_SERVICE_INFIX", {
|
|
|
8514
8718
|
return NATIVE_PROVIDER_SERVICE_INFIX;
|
|
8515
8719
|
}
|
|
8516
8720
|
});
|
|
8721
|
+
Object.defineProperty(exports, "RSS_BUDGET_REANNOUNCE_MIN_MS", {
|
|
8722
|
+
enumerable: true,
|
|
8723
|
+
get: function() {
|
|
8724
|
+
return RSS_BUDGET_REANNOUNCE_MIN_MS;
|
|
8725
|
+
}
|
|
8726
|
+
});
|
|
8727
|
+
Object.defineProperty(exports, "RSS_BUDGET_RELEASE_RATIO", {
|
|
8728
|
+
enumerable: true,
|
|
8729
|
+
get: function() {
|
|
8730
|
+
return RSS_BUDGET_RELEASE_RATIO;
|
|
8731
|
+
}
|
|
8732
|
+
});
|
|
8517
8733
|
Object.defineProperty(exports, "RUNNER_HEAP_WATCH_INTERVAL_MS", {
|
|
8518
8734
|
enumerable: true,
|
|
8519
8735
|
get: function() {
|
|
8520
8736
|
return RUNNER_HEAP_WATCH_INTERVAL_MS;
|
|
8521
8737
|
}
|
|
8522
8738
|
});
|
|
8739
|
+
Object.defineProperty(exports, "RUNNER_RSS_BUDGET_ENV", {
|
|
8740
|
+
enumerable: true,
|
|
8741
|
+
get: function() {
|
|
8742
|
+
return RUNNER_RSS_BUDGET_ENV;
|
|
8743
|
+
}
|
|
8744
|
+
});
|
|
8523
8745
|
Object.defineProperty(exports, "SocketChannel", {
|
|
8524
8746
|
enumerable: true,
|
|
8525
8747
|
get: function() {
|
|
@@ -8724,6 +8946,12 @@ Object.defineProperty(exports, "createV8Reclaimer", {
|
|
|
8724
8946
|
return createV8Reclaimer;
|
|
8725
8947
|
}
|
|
8726
8948
|
});
|
|
8949
|
+
Object.defineProperty(exports, "describeRss", {
|
|
8950
|
+
enumerable: true,
|
|
8951
|
+
get: function() {
|
|
8952
|
+
return describeRss;
|
|
8953
|
+
}
|
|
8954
|
+
});
|
|
8727
8955
|
Object.defineProperty(exports, "deserializeTypedArrays", {
|
|
8728
8956
|
enumerable: true,
|
|
8729
8957
|
get: function() {
|
|
@@ -8784,6 +9012,12 @@ Object.defineProperty(exports, "getWorkerNativeCapSnapshot", {
|
|
|
8784
9012
|
return getWorkerNativeCapSnapshot;
|
|
8785
9013
|
}
|
|
8786
9014
|
});
|
|
9015
|
+
Object.defineProperty(exports, "hubMainRssBudget", {
|
|
9016
|
+
enumerable: true,
|
|
9017
|
+
get: function() {
|
|
9018
|
+
return hubMainRssBudget;
|
|
9019
|
+
}
|
|
9020
|
+
});
|
|
8787
9021
|
Object.defineProperty(exports, "installManifestNativeDeps", {
|
|
8788
9022
|
enumerable: true,
|
|
8789
9023
|
get: function() {
|
|
@@ -8820,12 +9054,24 @@ Object.defineProperty(exports, "mountNativeCapService", {
|
|
|
8820
9054
|
return mountNativeCapService;
|
|
8821
9055
|
}
|
|
8822
9056
|
});
|
|
9057
|
+
Object.defineProperty(exports, "nextRssBudgetState", {
|
|
9058
|
+
enumerable: true,
|
|
9059
|
+
get: function() {
|
|
9060
|
+
return nextRssBudgetState;
|
|
9061
|
+
}
|
|
9062
|
+
});
|
|
8823
9063
|
Object.defineProperty(exports, "parseCapAction", {
|
|
8824
9064
|
enumerable: true,
|
|
8825
9065
|
get: function() {
|
|
8826
9066
|
return parseCapAction;
|
|
8827
9067
|
}
|
|
8828
9068
|
});
|
|
9069
|
+
Object.defineProperty(exports, "parseRssBudgetMb", {
|
|
9070
|
+
enumerable: true,
|
|
9071
|
+
get: function() {
|
|
9072
|
+
return parseRssBudgetMb;
|
|
9073
|
+
}
|
|
9074
|
+
});
|
|
8829
9075
|
Object.defineProperty(exports, "registerEventBusService", {
|
|
8830
9076
|
enumerable: true,
|
|
8831
9077
|
get: function() {
|