@camstack/system 1.2.103 → 1.2.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +15 -1
- package/dist/addon-runner.mjs +15 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +1 -1
- package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-DHKXob1m.mjs → dist-CAzHQ03i.mjs} +20 -1
- package/dist/{dist-qnwy_iWM.js → dist-DR1VmGz6.js} +20 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +25 -312
- package/dist/index.mjs +15 -303
- package/dist/kernel/heap-watch.d.ts +67 -0
- package/dist/{manifest-python-deps-Bt_J04b0.mjs → manifest-python-deps-5ei2xVUh.mjs} +355 -1
- package/dist/{manifest-python-deps-BtVjvfvj.js → manifest-python-deps-GejnH--L.js} +422 -0
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { a as __toCommonJS, i as __require, n as __esmMin, o as __toESM$1, r as __exportAll, t as __commonJSMin$1 } from "./chunk-CNf5ZN-e.mjs";
|
|
2
|
-
import { At as asNumber, B as extractNestedAddonId, Bt as parseJsonUnknown$1, Dt as ReadinessTimeoutError, Et as ReadinessRegistry, Ft as expandCapMethods, Gt as EventCategory$1, H as isArrayOutputSchema, Ht as resolveCapMount, J as lifecycleJobSchema, K as isVoidInput, Mt as createEvent, Nt as emitDownForOwnedCaps, Q as looseSchema, St as DEVICE_STATUS_METHOD, U as isCollectionArrayMethod, Ut as scopeKey, Vt as readinessKey, W as isObjectInput, Z as logLevelAtMost, bt as DATAPLANE_SECRET_HEADER$1, d as METHOD_ACCESS_MAP, f as RUNTIME_DEFAULTS, it as procedureAuthKey, jt as asString$1, kt as asJsonObject$1, ot as scopesAllowAddon, q as kebabToCamel, st as scopesAllowDeviceCap, t as ALL_CAPABILITY_DEFINITIONS, tt as objectInputDeclaresAddonId, vt as errMsg$1, xt as DEVICE_SETTINGS_CONTRIBUTION_METHODS, y as addonSettingsCapability, zt as parseJsonObject } from "./dist-
|
|
2
|
+
import { At as asNumber, B as extractNestedAddonId, Bt as parseJsonUnknown$1, Dt as ReadinessTimeoutError, Et as ReadinessRegistry, Ft as expandCapMethods, Gt as EventCategory$1, H as isArrayOutputSchema, Ht as resolveCapMount, J as lifecycleJobSchema, K as isVoidInput, Mt as createEvent, Nt as emitDownForOwnedCaps, Q as looseSchema, St as DEVICE_STATUS_METHOD, U as isCollectionArrayMethod, Ut as scopeKey, Vt as readinessKey, W as isObjectInput, Z as logLevelAtMost, bt as DATAPLANE_SECRET_HEADER$1, d as METHOD_ACCESS_MAP, f as RUNTIME_DEFAULTS, it as procedureAuthKey, jt as asString$1, kt as asJsonObject$1, ot as scopesAllowAddon, q as kebabToCamel, st as scopesAllowDeviceCap, t as ALL_CAPABILITY_DEFINITIONS, tt as objectInputDeclaresAddonId, vt as errMsg$1, xt as DEVICE_SETTINGS_CONTRIBUTION_METHODS, y as addonSettingsCapability, zt as parseJsonObject } from "./dist-CAzHQ03i.mjs";
|
|
3
3
|
import { AlertCenterAddon } from "./builtins/alerts/alerts.addon.mjs";
|
|
4
4
|
import "./builtins/alerts/index.mjs";
|
|
5
5
|
import { t as formatLogLine } from "./formatter-B7qW8bPJ.mjs";
|
|
@@ -24,7 +24,7 @@ import { SystemConfigAddon } from "./builtins/system-config/system-config.addon.
|
|
|
24
24
|
import "./builtins/system-config/index.mjs";
|
|
25
25
|
import { WinstonDestination, WinstonLoggingAddon } from "./builtins/winston-logging/index.mjs";
|
|
26
26
|
import { a as parseTokenizedUrl, c as collectModelFiles, d as downloadModel, f as ensureModel, h as isModelDownloaded, i as parseRangeHeader, l as deleteModelFromDisk, m as getModelFilePath, n as contentTypeFor, o as resolveFilePath, p as fetchJson, r as createAuthenticatedFileServer, s as ModelDownloadService, t as createFileDataPlaneHandler, u as downloadFile } from "./file-data-plane-CuE_hBli.mjs";
|
|
27
|
-
import { $ as buildNativeCapProxy, A as createHubCapForwardService, B as AGENT_CAP_FWD_ACTION, C as brokerTransportLink, Ct as runNpm, D as localProviderLink, E as ipcParentLink, F as createUdsLogger, G as callWithServiceDiscovery, H as CapRouteResolver, I as createUdsLoggerWithControl, J as UdsLocalTransportServer, K as createLocalTransport, L as LocalChildClient, M as createUdsEventBridge, N as createUdsEventBus, O as HUB_CAP_FWD_ACTION, P as udsChildLogToWorkerEntry, Q as encodeFrame, R as LocalChildRegistry, S as brokerCallForCap, St as resolveNpmInvocation, T as ipcChildLink, U as CapRouteError, V as AGENT_CAP_FWD_SERVICE, W as classifyCapRoute, X as localEndpointPath, Y as SocketChannel, Z as FrameDecoder, _ as createKernelHwAccel, _t as CapabilityHandle, a as getWorkerDeviceRegistry, b as __resetCapUsageRegistryForTests, bt as installManifestNativeDeps, c as setHubConnected, ct as NATIVE_PROVIDER_SERVICE_INFIX, d as getBrokerEventBus, dt as capBareAction, et as buildUdsNativeCapProxy, f as getMoleculerEventStats, ft as capServiceName, g as AddonDepsManager, gt as DeviceRegistry, h as subscribePassthrough, ht as serializeTypedArrays, i as createUdsAddonContext, it as mountNativeCapService, j as createParentUnownedCallHandler, k as HUB_CAP_FWD_SERVICE, l as EVENT_TOPIC_PREFIX, lt as capActionName, m as setNodeEventInterest, mt as deserializeTypedArrays, n as adaptBrokerToCluster, o as getOrInitReadinessRegistry, ot as createAddonService, p as registerEventBusService, pt as parseCapAction, q as UdsLocalTransportClient, r as createAddonContext, s as getOrInitReadinessRegistryForClient, st as validateProviderRegistrations, t as installManifestPythonDeps, tt as createBrokerDeviceManagerApi, u as clusterEventTopic, ut as capActionSuffix, v as resolveHwAccel, vt as CapabilityUnavailableError, w as buildLinkChain, wt as createAddonDataPlaneFacility, x as getCapUsageRegistry, xt as resolveAddonClass, y as CapUsageRegistry, yt as copyBundledNativeModules, z as UDS_NO_ROUTE_PREFIX } from "./manifest-python-deps-
|
|
27
|
+
import { $ as buildNativeCapProxy, A as createHubCapForwardService, At as buildHeapSample, B as AGENT_CAP_FWD_ACTION, C as brokerTransportLink, Ct as runNpm, D as localProviderLink, Dt as HEAP_WATCH_INTERVAL_MS, E as ipcParentLink, Et as HEAP_RECLAIM_TRIGGER_MB, F as createUdsLogger, Ft as strandedMb, G as callWithServiceDiscovery, H as CapRouteResolver, I as createUdsLoggerWithControl, J as UdsLocalTransportServer, K as createLocalTransport, L as LocalChildClient, M as createUdsEventBridge, Mt as shouldReclaim, N as createUdsEventBus, Nt as startHeapWatch, O as HUB_CAP_FWD_ACTION, Ot as HEAP_WATCH_WARN_RATIO, P as udsChildLogToWorkerEntry, Pt as startRunnerHeapWatch, Q as encodeFrame, R as LocalChildRegistry, S as brokerCallForCap, St as resolveNpmInvocation, T as ipcChildLink, Tt as HEAP_RECLAIM_MIN_INTERVAL_MS, U as CapRouteError, V as AGENT_CAP_FWD_SERVICE, W as classifyCapRoute, X as localEndpointPath, Y as SocketChannel, Z as FrameDecoder, _ as createKernelHwAccel, _t as CapabilityHandle, a as getWorkerDeviceRegistry, b as __resetCapUsageRegistryForTests, bt as installManifestNativeDeps, c as setHubConnected, ct as NATIVE_PROVIDER_SERVICE_INFIX, d as getBrokerEventBus, dt as capBareAction, et as buildUdsNativeCapProxy, f as getMoleculerEventStats, ft as capServiceName, g as AddonDepsManager, gt as DeviceRegistry, h as subscribePassthrough, ht as serializeTypedArrays, i as createUdsAddonContext, it as mountNativeCapService, j as createParentUnownedCallHandler, jt as createV8Reclaimer, k as HUB_CAP_FWD_SERVICE, kt as RUNNER_HEAP_WATCH_INTERVAL_MS, l as EVENT_TOPIC_PREFIX, lt as capActionName, m as setNodeEventInterest, mt as deserializeTypedArrays, n as adaptBrokerToCluster, o as getOrInitReadinessRegistry, ot as createAddonService, p as registerEventBusService, pt as parseCapAction, q as UdsLocalTransportClient, r as createAddonContext, s as getOrInitReadinessRegistryForClient, st as validateProviderRegistrations, t as installManifestPythonDeps, tt as createBrokerDeviceManagerApi, u as clusterEventTopic, ut as capActionSuffix, v as resolveHwAccel, vt as CapabilityUnavailableError, w as buildLinkChain, wt as createAddonDataPlaneFacility, x as getCapUsageRegistry, xt as resolveAddonClass, y as CapUsageRegistry, yt as copyBundledNativeModules, z as UDS_NO_ROUTE_PREFIX } from "./manifest-python-deps-5ei2xVUh.mjs";
|
|
28
28
|
import { n as getSinglePidStats, t as getPidStats } from "./resource-monitor-BWmQ5i-o.mjs";
|
|
29
29
|
import { t as CustomActionRegistry } from "./custom-action-registry-F__gp_VX.mjs";
|
|
30
30
|
import { z } from "zod";
|
|
@@ -40,7 +40,6 @@ import * as os$18 from "node:os";
|
|
|
40
40
|
import * as fs$17 from "node:fs";
|
|
41
41
|
import { accessSync, constants, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
42
42
|
import { request } from "node:http";
|
|
43
|
-
import * as v8 from "node:v8";
|
|
44
43
|
import * as vm from "node:vm";
|
|
45
44
|
import { asJsonObject, errMsg, parseJsonUnknown } from "@camstack/types/addon";
|
|
46
45
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
@@ -423,304 +422,6 @@ function proxyToUpstream(opts) {
|
|
|
423
422
|
else clientReq.pipe(upstream);
|
|
424
423
|
}
|
|
425
424
|
//#endregion
|
|
426
|
-
//#region src/kernel/heap-watch.ts
|
|
427
|
-
/**
|
|
428
|
-
* heap-watch — hub-main's own memory, on the record, and returned to the OS.
|
|
429
|
-
*
|
|
430
|
-
* ## Why this exists
|
|
431
|
-
*
|
|
432
|
-
* hub-main used to be launched with `--max-old-space-size=1536` (docker-entrypoint;
|
|
433
|
-
* the cap was removed on 2026-08-02, so the live ceiling is now V8's own default —
|
|
434
|
-
* 4288MB in this container — and this module reads it at runtime rather than
|
|
435
|
-
* assuming a number). It logged NOTHING about its own heap. When it died on
|
|
436
|
-
* 2026-08-01/02 the only trace was V8's own stderr line:
|
|
437
|
-
*
|
|
438
|
-
* FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
|
|
439
|
-
*
|
|
440
|
-
* which carries no process name, no heap number and no history — the whole
|
|
441
|
-
* server restarts and the evidence is gone. Four of those in ~15h, and the one
|
|
442
|
-
* at 06:57:57 landed nine seconds after a viewer opened a WebRTC session, which
|
|
443
|
-
* is why the app sat in a permanent "loading": the hub was dying as it
|
|
444
|
-
* connected.
|
|
445
|
-
*
|
|
446
|
-
* `[mem]` reports already exist for forked addon runners (SIGUSR2 in
|
|
447
|
-
* `addon-runner.ts`) — hub-main, the one process that actually OOMs, was the
|
|
448
|
-
* one process with no telemetry at all.
|
|
449
|
-
*
|
|
450
|
-
* ## What it gives
|
|
451
|
-
*
|
|
452
|
-
* A heartbeat carrying the numbers that separate the two possible stories, and
|
|
453
|
-
* a WARN as the heap approaches the ceiling — the line that will exist BEFORE
|
|
454
|
-
* the next fatal instead of after it:
|
|
455
|
-
*
|
|
456
|
-
* - `heapUsedMb` climbing steadily between restarts → a leak;
|
|
457
|
-
* - `heapUsedMb` flat with a sharp jump under load → a burst allocation, and
|
|
458
|
-
* the timestamp says what was being served.
|
|
459
|
-
*
|
|
460
|
-
* `external`/`arrayBuffers` ride along because they are NOT bounded by
|
|
461
|
-
* `--max-old-space-size` and are most of the gap between a 1.5GB cap and the
|
|
462
|
-
* ~2.05GB RSS observed live: a diagnosis that only looks at the JS heap
|
|
463
|
-
* misreads that process by 500MB.
|
|
464
|
-
*
|
|
465
|
-
* ## And then the heartbeat explained the "off-heap leak", which was not one
|
|
466
|
-
*
|
|
467
|
-
* 2026-08-08, measured on the live hub over 1019 consecutive samples and seven
|
|
468
|
-
* separate boots: **hub-main's RSS is a HIGH-WATER MARK, not a leak.** In every
|
|
469
|
-
* one of those 1019 samples RSS was >= the running maximum of
|
|
470
|
-
* `heapTotal + external`, never once below it, and
|
|
471
|
-
* `RSS_peak - peak(heapTotal + external)` landed between 874MB and 1639MB on
|
|
472
|
-
* every boot — a roughly constant native floor on top of a number that only
|
|
473
|
-
* ever goes up. RSS tracked the highest point the process ever reached and
|
|
474
|
-
* stayed there: 3.3GB -> 4.5GB -> 5.2GB, and 7.8GB on the worst boot.
|
|
475
|
-
*
|
|
476
|
-
* Two things push that peak, and neither is a retained object:
|
|
477
|
-
*
|
|
478
|
-
* - the JS heap sawtooths to 2.0-2.8GB because the ceiling is 4288MB and
|
|
479
|
-
* nothing creates pressure before then;
|
|
480
|
-
* - `arrayBuffers` accumulates DEAD Buffers — up to 3.65GB — because V8 only
|
|
481
|
-
* sweeps them on a major GC. Eleven collapses were captured, and every
|
|
482
|
-
* single one coincides with `heapUsed` dropping to its ~650MB floor. No code
|
|
483
|
-
* released a reference at 13:23:03; a mark-compact ran.
|
|
484
|
-
*
|
|
485
|
-
* At the peak V8 has committed the pages, and **an ordinary major GC does not
|
|
486
|
-
* give them back.** Proven on the production hub: `smaps` showed 10,819 mapped
|
|
487
|
-
* 256KB pages (V8's `kRegularPageSize`) holding 2704MB while V8 itself reported
|
|
488
|
-
* `total_physical_size` of only 846MB — ~7,800 pages committed, resident, and
|
|
489
|
-
* outside V8's own accounting. One `gc({ flavor: 'last-resort' })` took 417ms
|
|
490
|
-
* and returned RSS 3355MB -> 1237MB, pages 10,819 -> 2,963.
|
|
491
|
-
*
|
|
492
|
-
* So the fix for the "~3GB/2h off-heap growth" is not a retainer hunt. It is to
|
|
493
|
-
* ask V8 for the pages back when the process is visibly holding more than it is
|
|
494
|
-
* using — which is what `shouldReclaim` decides and what the heartbeat now does.
|
|
495
|
-
*
|
|
496
|
-
* The cost is a stop-the-world on the hub's router process — 250-450ms when
|
|
497
|
-
* first measured, 654ms on the largest pass seen since — bounded to once per
|
|
498
|
-
* `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a ratchet to
|
|
499
|
-
* 7.8GB and the OOM that ends it.
|
|
500
|
-
*/
|
|
501
|
-
/** Heartbeat period. Slow enough to be free, fast enough that the sample before
|
|
502
|
-
* a fatal is recent. */
|
|
503
|
-
var HEAP_WATCH_INTERVAL_MS = 6e4;
|
|
504
|
-
/** Warn above this share of the heap ceiling. Chosen so the warning precedes
|
|
505
|
-
* the fatal by more than one interval under realistic growth. */
|
|
506
|
-
var HEAP_WATCH_WARN_RATIO = .8;
|
|
507
|
-
/**
|
|
508
|
-
* Escalate above this share of the ceiling.
|
|
509
|
-
*
|
|
510
|
-
* 0.70 is 3002MB: clear of the measured routine max (2847MB) so steady state
|
|
511
|
-
* never escalates, and below {@link HEAP_WATCH_WARN_RATIO} so the fast cadence
|
|
512
|
-
* is already running by the time the warn is reachable.
|
|
513
|
-
*/
|
|
514
|
-
var HEAP_WATCH_ESCALATE_RATIO = .7;
|
|
515
|
-
/**
|
|
516
|
-
* Release escalation below this share.
|
|
517
|
-
*
|
|
518
|
-
* Hysteresis, not a second threshold: hub-main's heap sawtooths continuously,
|
|
519
|
-
* so a single ratio would flap the mode on every GC. 0.60 is 2573MB — a real
|
|
520
|
-
* retreat rather than the trough of one collection.
|
|
521
|
-
*/
|
|
522
|
-
var HEAP_WATCH_DEESCALATE_RATIO = .6;
|
|
523
|
-
/**
|
|
524
|
-
* The cadence state machine, pure so the thresholds are testable without
|
|
525
|
-
* allocating three gigabytes to reach them.
|
|
526
|
-
*/
|
|
527
|
-
function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO, deescalateRatio = HEAP_WATCH_DEESCALATE_RATIO) {
|
|
528
|
-
if (current === "escalated") return usedRatio < deescalateRatio ? "steady" : "escalated";
|
|
529
|
-
return usedRatio >= escalateRatio ? "escalated" : "steady";
|
|
530
|
-
}
|
|
531
|
-
/**
|
|
532
|
-
* Reclaim once the process holds this many MB beyond what it is USING.
|
|
533
|
-
*
|
|
534
|
-
* Live calibration, both readings from the same hub minutes apart: 2491MB
|
|
535
|
-
* stranded before the reclaim, 438MB after. 438MB is this process's irreducible
|
|
536
|
-
* native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
|
|
537
|
-
* collecting against it would buy nothing and cost a stall every minute.
|
|
538
|
-
*
|
|
539
|
-
* ## Why this came down from 1536 (2026-08-17)
|
|
540
|
-
*
|
|
541
|
-
* The trigger was originally set to fire only on an unmistakable strand. It
|
|
542
|
-
* does, and the pass on the record works exactly as designed:
|
|
543
|
-
*
|
|
544
|
-
* [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
|
|
545
|
-
*
|
|
546
|
-
* But the number it was protecting against changed. The host now runs with a
|
|
547
|
-
* 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
|
|
548
|
-
* RSS this process is holding and not using is no longer merely untidy — it is
|
|
549
|
-
* the headroom something else needed, and the OOM is triggered by the PEAK, not
|
|
550
|
-
* by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
|
|
551
|
-
* above the floor before anything happens.
|
|
552
|
-
*
|
|
553
|
-
* 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
|
|
554
|
-
* can never trip it, and every pass that does fire still returns roughly
|
|
555
|
-
* `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
|
|
556
|
-
* not a stall spent collecting noise.
|
|
557
|
-
*/
|
|
558
|
-
var HEAP_RECLAIM_TRIGGER_MB = 1024;
|
|
559
|
-
/**
|
|
560
|
-
* Floor on how often the stop-the-world may run.
|
|
561
|
-
*
|
|
562
|
-
* Ten minutes was chosen to keep the worst case under 0.1% of the router's
|
|
563
|
-
* time. The worst case is not what governs the cost, though: the reclaim only
|
|
564
|
-
* runs when {@link shouldReclaim} says the process is genuinely stranded, and
|
|
565
|
-
* after a pass it is at the ~438MB floor. Live off-heap growth was measured at
|
|
566
|
-
* roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
|
|
567
|
-
* takes on the order of **20 minutes** — well outside any floor considered
|
|
568
|
-
* here. In steady state this constant therefore costs nothing at all.
|
|
569
|
-
*
|
|
570
|
-
* What it does govern is a BURST, which is the only case that matters on a host
|
|
571
|
-
* with no swap: the ten-minute floor let a peak that re-formed in seconds stand
|
|
572
|
-
* for ten minutes. Two minutes bounds that, and even the pathological case of a
|
|
573
|
-
* process that re-strands instantly costs 654ms per 120s — 0.55% of the
|
|
574
|
-
* router's time, against a ~1.5GiB peak it removes each pass.
|
|
575
|
-
*/
|
|
576
|
-
var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
|
|
577
|
-
var MB = (bytes) => Math.round(bytes / 1048576);
|
|
578
|
-
/**
|
|
579
|
-
* Build a sample from raw readings. Pure so the thresholds are testable without
|
|
580
|
-
* allocating a gigabyte to reach them.
|
|
581
|
-
*/
|
|
582
|
-
function buildHeapSample(mem, heapLimitBytes, warnRatio = HEAP_WATCH_WARN_RATIO) {
|
|
583
|
-
const usedRatio = heapLimitBytes > 0 ? mem.heapUsed / heapLimitBytes : 0;
|
|
584
|
-
return {
|
|
585
|
-
rssMb: MB(mem.rss),
|
|
586
|
-
heapUsedMb: MB(mem.heapUsed),
|
|
587
|
-
heapTotalMb: MB(mem.heapTotal),
|
|
588
|
-
heapLimitMb: MB(heapLimitBytes),
|
|
589
|
-
externalMb: MB(mem.external),
|
|
590
|
-
arrayBuffersMb: MB(mem.arrayBuffers),
|
|
591
|
-
usedRatio: Math.round(usedRatio * 100) / 100,
|
|
592
|
-
nearLimit: usedRatio >= warnRatio
|
|
593
|
-
};
|
|
594
|
-
}
|
|
595
|
-
var consoleSink = {
|
|
596
|
-
info: (line) => console.log(line),
|
|
597
|
-
warn: (line) => console.warn(line)
|
|
598
|
-
};
|
|
599
|
-
/**
|
|
600
|
-
* RSS the process is holding but not using.
|
|
601
|
-
*
|
|
602
|
-
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
603
|
-
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
604
|
-
* and they are precisely what this number exists to see.
|
|
605
|
-
*/
|
|
606
|
-
function strandedMb(s) {
|
|
607
|
-
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
608
|
-
}
|
|
609
|
-
/** Whether it is worth paying a stop-the-world to get those pages back. */
|
|
610
|
-
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
611
|
-
return strandedMb(s) > triggerMb;
|
|
612
|
-
}
|
|
613
|
-
function isGcFunction(value) {
|
|
614
|
-
return typeof value === "function";
|
|
615
|
-
}
|
|
616
|
-
/**
|
|
617
|
-
* Build the reclaimer without requiring `--expose-gc` on argv.
|
|
618
|
-
*
|
|
619
|
-
* The flag is turned on, the function captured, and the flag turned straight
|
|
620
|
-
* back off — a global `gc` left reachable by application code is a footgun, and
|
|
621
|
-
* requiring an argv change would mean this cannot ship without a new image.
|
|
622
|
-
* Verified against the live hub before being written: 266ms, RSS 1453 -> 1237.
|
|
623
|
-
*
|
|
624
|
-
* Returns undefined if V8 declines, so the caller keeps a heartbeat either way.
|
|
625
|
-
*/
|
|
626
|
-
function createV8Reclaimer() {
|
|
627
|
-
try {
|
|
628
|
-
v8.setFlagsFromString("--expose-gc");
|
|
629
|
-
const gc = vm.runInNewContext("gc");
|
|
630
|
-
v8.setFlagsFromString("--no-expose-gc");
|
|
631
|
-
if (!isGcFunction(gc)) return void 0;
|
|
632
|
-
return () => gc({
|
|
633
|
-
execution: "sync",
|
|
634
|
-
flavor: "last-resort"
|
|
635
|
-
});
|
|
636
|
-
} catch {
|
|
637
|
-
return;
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
function format$1(label, s) {
|
|
641
|
-
return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
|
|
642
|
-
}
|
|
643
|
-
/**
|
|
644
|
-
* Start the heartbeat. Returns a stop function.
|
|
645
|
-
*
|
|
646
|
-
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
647
|
-
* a diagnostic that can crash the process it watches is worse than none.
|
|
648
|
-
*/
|
|
649
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
|
|
650
|
-
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
651
|
-
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
652
|
-
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
653
|
-
const minIntervalMs = reclaimOptions?.minIntervalMs ?? 12e4;
|
|
654
|
-
const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
|
|
655
|
-
const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
|
|
656
|
-
const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
|
|
657
|
-
let lastReclaimAt = Number.NEGATIVE_INFINITY;
|
|
658
|
-
const read = () => {
|
|
659
|
-
const limit = reclaimOptions?.heapLimitBytes ?? v8.getHeapStatistics().heap_size_limit;
|
|
660
|
-
return buildHeapSample(readMemory(), limit);
|
|
661
|
-
};
|
|
662
|
-
const maybeReclaim = (sample) => {
|
|
663
|
-
if (reclaimOptions === void 0) return;
|
|
664
|
-
if (!shouldReclaim(sample, triggerMb)) return;
|
|
665
|
-
if (now() - lastReclaimAt < minIntervalMs) return;
|
|
666
|
-
lastReclaimAt = now();
|
|
667
|
-
const startedAt = now();
|
|
668
|
-
try {
|
|
669
|
-
reclaimOptions.reclaim();
|
|
670
|
-
} catch (error) {
|
|
671
|
-
sink.warn(`[mem] reclaim failed — ${error instanceof Error ? error.message : String(error)}`);
|
|
672
|
-
return;
|
|
673
|
-
}
|
|
674
|
-
const after = read();
|
|
675
|
-
sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
|
|
676
|
-
};
|
|
677
|
-
let mode = "steady";
|
|
678
|
-
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
679
|
-
/**
|
|
680
|
-
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
681
|
-
* point.
|
|
682
|
-
*
|
|
683
|
-
* The 2026-08-08 fatal escaped a 60s heartbeat because the burst began and
|
|
684
|
-
* ended between two samples. Deciding "is the heap elevated?" only when a
|
|
685
|
-
* line is due would reproduce that exactly: the process would still be read
|
|
686
|
-
* once a minute, and a 62-second climb would still be invisible. So the timer
|
|
687
|
-
* runs at the FAST period always — `process.memoryUsage()` is tens of
|
|
688
|
-
* microseconds, i.e. free at this rate — and the slow period governs only how
|
|
689
|
-
* often a steady process is allowed to say so.
|
|
690
|
-
*
|
|
691
|
-
* Never probe slower than the log cadence, or a caller passing a short
|
|
692
|
-
* `intervalMs` would silently get the long one.
|
|
693
|
-
*/
|
|
694
|
-
const probeIntervalMs = Math.min(fastIntervalMs, intervalMs);
|
|
695
|
-
const tick = () => {
|
|
696
|
-
try {
|
|
697
|
-
const sample = read();
|
|
698
|
-
const previous = mode;
|
|
699
|
-
mode = nextMode(previous, sample.usedRatio, escalateRatio, deescalateRatio);
|
|
700
|
-
const at = now();
|
|
701
|
-
const due = at - lastLoggedAt >= intervalMs;
|
|
702
|
-
if (mode === "escalated" || due) {
|
|
703
|
-
lastLoggedAt = at;
|
|
704
|
-
const line = format$1(label, sample);
|
|
705
|
-
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
706
|
-
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
707
|
-
else sink.info(line);
|
|
708
|
-
}
|
|
709
|
-
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
710
|
-
maybeReclaim(sample);
|
|
711
|
-
} catch {}
|
|
712
|
-
};
|
|
713
|
-
const timer = setInterval(tick, probeIntervalMs);
|
|
714
|
-
timer.unref?.();
|
|
715
|
-
tick();
|
|
716
|
-
let stopped = false;
|
|
717
|
-
return () => {
|
|
718
|
-
if (stopped) return;
|
|
719
|
-
stopped = true;
|
|
720
|
-
clearInterval(timer);
|
|
721
|
-
};
|
|
722
|
-
}
|
|
723
|
-
//#endregion
|
|
724
425
|
//#region src/lifecycle/lifecycle-state-machine.ts
|
|
725
426
|
var VALID_TRANSITIONS = {
|
|
726
427
|
stopped: ["starting", "disabled"],
|
|
@@ -94645,19 +94346,21 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
|
|
|
94645
94346
|
function spawnRunner(runnerId, addons, env) {
|
|
94646
94347
|
const nodeId = buildNodeId(runnerId);
|
|
94647
94348
|
const runnerPath = path$39.resolve(__dirname, "addon-runner.js");
|
|
94349
|
+
const heavy = isHeavyRunner(addons);
|
|
94648
94350
|
const childEnv = {
|
|
94649
94351
|
...process.env,
|
|
94650
94352
|
CAMSTACK_RUNNER_ID: runnerId,
|
|
94651
94353
|
CAMSTACK_RUNNER_ADDONS: JSON.stringify(addons),
|
|
94652
94354
|
CAMSTACK_PARENT_NODE_ID: parentNodeId,
|
|
94653
94355
|
CAMSTACK_DATA_DIR: dataDir,
|
|
94356
|
+
CAMSTACK_RUNNER_HEAP_PROFILE: heavy ? "heavy" : "light",
|
|
94654
94357
|
CAMSTACK_LOG_LEVEL: process.env["CAMSTACK_LOG_LEVEL"] ?? "info",
|
|
94655
94358
|
...parentTcpPort !== void 0 ? { CAMSTACK_PARENT_TCP_PORT: String(parentTcpPort) } : {},
|
|
94656
94359
|
...parentUdsPath !== void 0 ? { CAMSTACK_PARENT_UDS_PATH: parentUdsPath } : {},
|
|
94657
94360
|
...env
|
|
94658
94361
|
};
|
|
94659
94362
|
const heapFlags = runnerHeapFlags(addons);
|
|
94660
|
-
capturedBroker?.logger.info(`[${runnerId}] heap profile: ${
|
|
94363
|
+
capturedBroker?.logger.info(`[${runnerId}] heap profile: ${heavy ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
|
|
94661
94364
|
const child = spawnFn(process.execPath, [...heapFlags, runnerPath], {
|
|
94662
94365
|
env: childEnv,
|
|
94663
94366
|
stdio: [
|
|
@@ -94701,6 +94404,15 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
|
|
|
94701
94404
|
capturedBroker?.logger.error(`[${runnerId}] Crash circuit-breaker tripped — ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms. Not respawning.`);
|
|
94702
94405
|
processes.set(runnerId, entry);
|
|
94703
94406
|
if (capturedBroker) forceMoleculerDisconnect(capturedBroker, entry.nodeId);
|
|
94407
|
+
emitRunnerEvent(EventCategory$1.AddonRunnerFailed, runnerId, {
|
|
94408
|
+
runnerId,
|
|
94409
|
+
nodeId: parentNodeId,
|
|
94410
|
+
addonIds: entry.runnerAddons.map((addon) => addon.addonId),
|
|
94411
|
+
crashesInWindow: decision.crashesInWindow,
|
|
94412
|
+
windowMs: CRASH_WINDOW_MS,
|
|
94413
|
+
...code !== null ? { exitCode: code } : {},
|
|
94414
|
+
...signal !== null ? { signal } : {}
|
|
94415
|
+
});
|
|
94704
94416
|
for (const addon of entry.runnerAddons) emitRunnerEvent(EventCategory$1.AddonCrashed, addon.addonId, {
|
|
94705
94417
|
addonId: addon.addonId,
|
|
94706
94418
|
error: `Crash circuit-breaker tripped after ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms`
|
|
@@ -95379,4 +95091,4 @@ var LifecycleJobEngine = class {
|
|
|
95379
95091
|
}
|
|
95380
95092
|
};
|
|
95381
95093
|
//#endregion
|
|
95382
|
-
export { AGENT_CAP_FWD_ACTION, AGENT_CAP_FWD_SERVICE, AGENT_READINESS_SERVICE_NAME, ALL_CAPABILITY_DEFINITIONS, AddonApiFactory, AddonDepsManager, AddonEngineManager, AddonHealthMonitor, AddonInstaller, AddonLoader, AddonManifest, AddonRouteRegistry, AlertCenterAddon, ApiKeyManager, AuthManager, CLUSTER_SECRET_MISMATCH_TYPE, CLUSTER_SECRET_REJECTED_EXIT_CODE, CORE_CAP_SERVICE_NAME, CapRouteError, CapRouteResolver, CapUsageRegistry, CapabilityHandle, CapabilityRegistry, CapabilityUnavailableError, ConfigManager, ConfigStore, ConsoleDestination, ConsoleLoggingAddon, CoreBlocksAddon, CustomActionRegistry, DEFAULT_DATA_PATH, DEVICE_SETTINGS_CONTRIBUTION_METHODS, DEVICE_STATUS_METHOD, DataPlaneRegistry, DeviceManagerAddon, DeviceRegistry, DeviceStore, EVENT_TOPIC_PREFIX, EngineManagerResolver, EventBus, FeatureManager, FilesystemStorageAddon, FilesystemStorageProvider, FrameDecoder, FsStorageBackend, HEALTH_MONITOR_GRACE_PERIOD_MS, HEALTH_MONITOR_RETRY_INTERVALS_MS, HEALTH_MONITOR_TICK_MS, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_CAP_FWD_ACTION, HUB_CAP_FWD_SERVICE, HubForwarderAddon, HubForwarderDestination, HubLogForwarder, HubNodeRegistry, INFRA_CAPABILITIES, IntegrationRegistry, JobJournal, LifecycleJobEngine, LifecycleStateMachine, LivenessMonitorAddon, LocalAuthAddon, LocalChildClient, LocalChildRegistry, LogManager, LogRingBuffer, LokiDestination, LokiLoggingAddon, METHOD_ACCESS_MAP, ModelDownloadService, NATIVE_PROVIDER_SERVICE_INFIX, NATIVE_SCAN_DEPTH, NativeMetricsAddon, NativeMetricsProvider, NetworkQualityTracker, NotificationService, PYTHON_VERSION, PipelineRunner, PipelineValidator, PythonEnvManager, QUARANTINE_DIRNAME, RESTART_MARKER_FILE, RUNTIME_DEFAULTS, ReadinessRegistry, ReadinessTimeoutError, ReplEngine, RingBuffer, ScopedLogger, ScopedTokenManager, SocketChannel, SqliteSettingsAddon, SqliteSettingsBackend, StagingArea, StorageLocationManager, StorageManager, StorageOrchestratorAddon, StorageOrchestratorService, SystemConfigAddon, SystemEventBus, TRADITIONAL_NATIVE_PACKAGES, ToastService, UDS_NO_ROUTE_PREFIX, UdsLocalTransportClient, UdsLocalTransportServer, UserManager, WinstonDestination, WinstonLoggingAddon, __resetCapUsageRegistryForTests, adaptBrokerToCluster, addonSettingsCapability, bootstrapSchema, brokerCallForCap, brokerTransportLink, buildBinaryPath, buildCapRouters, buildHeapSample, buildLinkChain, buildNativeCapProxy, buildNodeManifest, buildStorageLocationRegistry, buildUdsNativeCapProxy, builderMountedCapNames, callRegisterNodeWithRetry, callWithServiceDiscovery, capActionName, capActionSuffix, capBareAction, capServiceName, classifyAddonDir, classifyCapRoute, clearPendingRestart, clusterEventTopic, clusterSecretMatches, collectModelFiles, contentTypeFor, copyDirRecursive, copyExtraFileDirs, createAddonContext, createAddonDataPlaneFacility, createAddonService, createAuthenticatedFileServer, createBroker, createBrokerDeviceManagerApi, createCoreCapService, createFileDataPlaneHandler, createHubCapForwardService, createHubService, createKernelHwAccel, createLocalTransport, createParentUnownedCallHandler, createProcessService, createReadinessService, createReadinessServiceForRegistry, createScopedProcessManager, createStreamProbeBrokerService, createUdsAddonContext, createUdsEventBridge, createUdsEventBus, createUdsLogger, createUdsLoggerWithControl, createV8Reclaimer, deleteModelFromDisk, deriveAgentListenPort, describeProviderKindDrift, detectWorkspacePackagesDir, downloadBinary, downloadFile, downloadModel, emitDownForOwnedCaps, encodeFrame, ensureAddonNativePrebuilds, ensureBinary, ensureDir, ensureFfmpeg, ensureLibraryBuilt, ensureModel, ensureNativePrebuilds, ensurePython, ensureTlsCert, expandCapMethods, fetchJson, findInPath, formatLogLine, getBrokerEventBus, getCapUsageRegistry, getFfmpegDownloadUrl, getModelFilePath, getMoleculerEventStats, getOrInitReadinessRegistry, getOrInitReadinessRegistryForClient, getPidStats, getPlatformInfo, getPythonDownloadUrl, getRestartMarkerPath, getSinglePidStats, getWorkerDeviceRegistry, hasDotNode, hashClusterSecret, installManifestNativeDeps, installManifestPythonDeps, installPackageFromNpm, installPythonPackages, installPythonRequirements, ipcChildLink, ipcParentLink, isAddonDeploySource, isArrayOutputSchema, isClusterSecretMismatchError, isCollectionArrayMethod, isInfraCapability, isModelDownloaded, isSourceNewer, loadTlsCert, localEndpointPath, localProviderLink, mountNativeCapService, parseCapAction, parseRangeHeader, parseTokenizedUrl, proxyToUpstream, quarantineAddonResidue, readPendingRestart, readinessKey, registerEventBusService, resolveFilePath, resolveHwAccel, resolveNpmInvocation, runNpm, scheduleSelfRestart, scopeKey, scopesAllowAddon, scopesAllowDeviceCap, selectAddonResidue, serializeTypedArrays, setHubConnected, setNodeEventInterest, shouldReclaim, startHeapWatch, strandedMb, stripCamstackDeps, subscribePassthrough, udsChildLogToWorkerEntry, validateProviderRegistrations, writePendingRestart };
|
|
95094
|
+
export { AGENT_CAP_FWD_ACTION, AGENT_CAP_FWD_SERVICE, AGENT_READINESS_SERVICE_NAME, ALL_CAPABILITY_DEFINITIONS, AddonApiFactory, AddonDepsManager, AddonEngineManager, AddonHealthMonitor, AddonInstaller, AddonLoader, AddonManifest, AddonRouteRegistry, AlertCenterAddon, ApiKeyManager, AuthManager, CLUSTER_SECRET_MISMATCH_TYPE, CLUSTER_SECRET_REJECTED_EXIT_CODE, CORE_CAP_SERVICE_NAME, CapRouteError, CapRouteResolver, CapUsageRegistry, CapabilityHandle, CapabilityRegistry, CapabilityUnavailableError, ConfigManager, ConfigStore, ConsoleDestination, ConsoleLoggingAddon, CoreBlocksAddon, CustomActionRegistry, DEFAULT_DATA_PATH, DEVICE_SETTINGS_CONTRIBUTION_METHODS, DEVICE_STATUS_METHOD, DataPlaneRegistry, DeviceManagerAddon, DeviceRegistry, DeviceStore, EVENT_TOPIC_PREFIX, EngineManagerResolver, EventBus, FeatureManager, FilesystemStorageAddon, FilesystemStorageProvider, FrameDecoder, FsStorageBackend, HEALTH_MONITOR_GRACE_PERIOD_MS, HEALTH_MONITOR_RETRY_INTERVALS_MS, HEALTH_MONITOR_TICK_MS, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_CAP_FWD_ACTION, HUB_CAP_FWD_SERVICE, HubForwarderAddon, HubForwarderDestination, HubLogForwarder, HubNodeRegistry, INFRA_CAPABILITIES, IntegrationRegistry, JobJournal, LifecycleJobEngine, LifecycleStateMachine, LivenessMonitorAddon, LocalAuthAddon, LocalChildClient, LocalChildRegistry, LogManager, LogRingBuffer, LokiDestination, LokiLoggingAddon, METHOD_ACCESS_MAP, ModelDownloadService, NATIVE_PROVIDER_SERVICE_INFIX, NATIVE_SCAN_DEPTH, NativeMetricsAddon, NativeMetricsProvider, NetworkQualityTracker, NotificationService, PYTHON_VERSION, PipelineRunner, PipelineValidator, PythonEnvManager, QUARANTINE_DIRNAME, RESTART_MARKER_FILE, RUNNER_HEAP_WATCH_INTERVAL_MS, RUNTIME_DEFAULTS, ReadinessRegistry, ReadinessTimeoutError, ReplEngine, RingBuffer, ScopedLogger, ScopedTokenManager, SocketChannel, SqliteSettingsAddon, SqliteSettingsBackend, StagingArea, StorageLocationManager, StorageManager, StorageOrchestratorAddon, StorageOrchestratorService, SystemConfigAddon, SystemEventBus, TRADITIONAL_NATIVE_PACKAGES, ToastService, UDS_NO_ROUTE_PREFIX, UdsLocalTransportClient, UdsLocalTransportServer, UserManager, WinstonDestination, WinstonLoggingAddon, __resetCapUsageRegistryForTests, adaptBrokerToCluster, addonSettingsCapability, bootstrapSchema, brokerCallForCap, brokerTransportLink, buildBinaryPath, buildCapRouters, buildHeapSample, buildLinkChain, buildNativeCapProxy, buildNodeManifest, buildStorageLocationRegistry, buildUdsNativeCapProxy, builderMountedCapNames, callRegisterNodeWithRetry, callWithServiceDiscovery, capActionName, capActionSuffix, capBareAction, capServiceName, classifyAddonDir, classifyCapRoute, clearPendingRestart, clusterEventTopic, clusterSecretMatches, collectModelFiles, contentTypeFor, copyDirRecursive, copyExtraFileDirs, createAddonContext, createAddonDataPlaneFacility, createAddonService, createAuthenticatedFileServer, createBroker, createBrokerDeviceManagerApi, createCoreCapService, createFileDataPlaneHandler, createHubCapForwardService, createHubService, createKernelHwAccel, createLocalTransport, createParentUnownedCallHandler, createProcessService, createReadinessService, createReadinessServiceForRegistry, createScopedProcessManager, createStreamProbeBrokerService, createUdsAddonContext, createUdsEventBridge, createUdsEventBus, createUdsLogger, createUdsLoggerWithControl, createV8Reclaimer, deleteModelFromDisk, deriveAgentListenPort, describeProviderKindDrift, detectWorkspacePackagesDir, downloadBinary, downloadFile, downloadModel, emitDownForOwnedCaps, encodeFrame, ensureAddonNativePrebuilds, ensureBinary, ensureDir, ensureFfmpeg, ensureLibraryBuilt, ensureModel, ensureNativePrebuilds, ensurePython, ensureTlsCert, expandCapMethods, fetchJson, findInPath, formatLogLine, getBrokerEventBus, getCapUsageRegistry, getFfmpegDownloadUrl, getModelFilePath, getMoleculerEventStats, getOrInitReadinessRegistry, getOrInitReadinessRegistryForClient, getPidStats, getPlatformInfo, getPythonDownloadUrl, getRestartMarkerPath, getSinglePidStats, getWorkerDeviceRegistry, hasDotNode, hashClusterSecret, installManifestNativeDeps, installManifestPythonDeps, installPackageFromNpm, installPythonPackages, installPythonRequirements, ipcChildLink, ipcParentLink, isAddonDeploySource, isArrayOutputSchema, isClusterSecretMismatchError, isCollectionArrayMethod, isInfraCapability, isModelDownloaded, isSourceNewer, loadTlsCert, localEndpointPath, localProviderLink, mountNativeCapService, parseCapAction, parseRangeHeader, parseTokenizedUrl, proxyToUpstream, quarantineAddonResidue, readPendingRestart, readinessKey, registerEventBusService, resolveFilePath, resolveHwAccel, resolveNpmInvocation, runNpm, scheduleSelfRestart, scopeKey, scopesAllowAddon, scopesAllowDeviceCap, selectAddonResidue, serializeTypedArrays, setHubConnected, setNodeEventInterest, shouldReclaim, startHeapWatch, startRunnerHeapWatch, strandedMb, stripCamstackDeps, subscribePassthrough, udsChildLogToWorkerEntry, validateProviderRegistrations, writePendingRestart };
|
|
@@ -168,3 +168,70 @@ export interface HeapReclaimOptions {
|
|
|
168
168
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
169
169
|
*/
|
|
170
170
|
export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions): () => void;
|
|
171
|
+
/**
|
|
172
|
+
* Steady heartbeat period for a HEAVY addon runner.
|
|
173
|
+
*
|
|
174
|
+
* hub-main logs once a minute because it is ONE process. The heavy runners are
|
|
175
|
+
* ~7 per node sharing the same hub log stream; at 60s each they would put 7
|
|
176
|
+
* routine lines a minute in it forever, which is how a log stops being read.
|
|
177
|
+
* Five minutes keeps the per-runner record (a leak is still a visible climb
|
|
178
|
+
* across samples) at ~1.4 lines/min across the class.
|
|
179
|
+
*
|
|
180
|
+
* This slows LOGGING only. The probe keeps running at
|
|
181
|
+
* {@link HEAP_WATCH_FAST_INTERVAL_MS} (probe and log cadences are separate —
|
|
182
|
+
* see the comment inside {@link startHeapWatch}), so the reclaim gate and the
|
|
183
|
+
* escalated burst-sampling react exactly as fast as hub-main's.
|
|
184
|
+
*/
|
|
185
|
+
export declare const RUNNER_HEAP_WATCH_INTERVAL_MS = 300000;
|
|
186
|
+
/** Wiring for {@link startRunnerHeapWatch}. */
|
|
187
|
+
export interface RunnerHeapWatchOptions {
|
|
188
|
+
/**
|
|
189
|
+
* `CAMSTACK_RUNNER_HEAP_PROFILE` as the parent passed it. Anything but
|
|
190
|
+
* `'heavy'` (including unset — the un-annotated default is light) starts
|
|
191
|
+
* nothing: light runners are small, already carry V8 heap flags, and a
|
|
192
|
+
* quiet runner must never pay a stop-the-world.
|
|
193
|
+
*/
|
|
194
|
+
readonly heapProfile: string | undefined;
|
|
195
|
+
/** Log label — the runner's nodeId, so `[mem]` lines group per runner. */
|
|
196
|
+
readonly label: string;
|
|
197
|
+
readonly sink: HeapWatchSink;
|
|
198
|
+
/**
|
|
199
|
+
* `CAMSTACK_RUNNER_HEAP_RECLAIM` as the environment carries it. `'off'`
|
|
200
|
+
* keeps the heartbeat and disables only the reclaim pass — the operator
|
|
201
|
+
* kill-switch for the media path, usable without a rebuild.
|
|
202
|
+
*/
|
|
203
|
+
readonly reclaimSwitch?: string;
|
|
204
|
+
/** Heartbeat period override. Defaults to {@link RUNNER_HEAP_WATCH_INTERVAL_MS}. */
|
|
205
|
+
readonly intervalMs?: number;
|
|
206
|
+
/** Test injection. Production omits it and gets {@link createV8Reclaimer}. */
|
|
207
|
+
readonly reclaimOptions?: HeapReclaimOptions;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
211
|
+
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
212
|
+
*
|
|
213
|
+
* ## Why heavy runners get hub-main's mechanism
|
|
214
|
+
*
|
|
215
|
+
* The media-path runners deliberately run with NO old-space ceiling
|
|
216
|
+
* (`maxOldSpaceMb: 0` — a flat cap turned a disk-stall backlog into
|
|
217
|
+
* `Ineffective mark-compacts near heap limit` and killed recording, twice on
|
|
218
|
+
* 2026-08-17). Uncapped, V8 feels no pressure, so a one-off burst becomes the
|
|
219
|
+
* runner's permanent RSS: the recorder measured 2122MB holding a ~50MB index.
|
|
220
|
+
* hub-main had exactly this shape and its reclaimer is proven live — 11 passes
|
|
221
|
+
* in 4h, 375–1652MB returned per pass at 550–680ms each. Same mechanism, same
|
|
222
|
+
* `shouldReclaim` stranded-over-trigger gate; never a second implementation.
|
|
223
|
+
*
|
|
224
|
+
* ## The stall, priced for the runners that get it
|
|
225
|
+
*
|
|
226
|
+
* A pass is a sub-second stop-the-world at most once per
|
|
227
|
+
* {@link HEAP_RECLAIM_MIN_INTERVAL_MS}, and only while >1GiB is stranded. For
|
|
228
|
+
* the recorder — finalizing a segment every ~300ms across 34 profile-writers —
|
|
229
|
+
* the pause delays WATCHER work, not video: ffmpeg children keep writing to
|
|
230
|
+
* disk independently, the playlist tail is the durable queue, and the next
|
|
231
|
+
* tick drains what accrued. For stream-broker a pass can hitch live frames
|
|
232
|
+
* once per qualifying window; after a pass the runner sits at its floor and
|
|
233
|
+
* re-stranding the trigger takes tens of minutes, so in practice the class
|
|
234
|
+
* pays a few passes per hour against the multi-GiB peak that was OOM-killing
|
|
235
|
+
* a neighbouring VM on a host with no swap.
|
|
236
|
+
*/
|
|
237
|
+
export declare function startRunnerHeapWatch(options: RunnerHeapWatchOptions): (() => void) | undefined;
|