@camstack/system 1.2.102 → 1.2.104

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/addon-runner.js +15 -1
  2. package/dist/addon-runner.mjs +15 -1
  3. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
  4. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
  5. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
  6. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
  7. package/dist/builtins/alerts/alerts.addon.js +1 -1
  8. package/dist/builtins/alerts/alerts.addon.mjs +1 -1
  9. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
  10. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
  11. package/dist/builtins/console-logging/index.js +1 -1
  12. package/dist/builtins/console-logging/index.mjs +1 -1
  13. package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
  14. package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
  15. package/dist/builtins/device-manager/device-manager.addon.js +1 -1
  16. package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
  17. package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
  18. package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
  19. package/dist/builtins/hub-forwarder/index.js +1 -1
  20. package/dist/builtins/hub-forwarder/index.mjs +1 -1
  21. package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
  22. package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
  23. package/dist/builtins/local-auth/local-auth.addon.js +1 -1
  24. package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
  25. package/dist/builtins/local-network/local-network.addon.js +1 -1
  26. package/dist/builtins/local-network/local-network.addon.mjs +1 -1
  27. package/dist/builtins/loki-logging/index.js +1 -1
  28. package/dist/builtins/loki-logging/index.mjs +1 -1
  29. package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
  30. package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
  31. package/dist/builtins/platform-probe/index.js +1 -1
  32. package/dist/builtins/platform-probe/index.mjs +1 -1
  33. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
  34. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
  35. package/dist/builtins/snapshot/index.js +150 -22
  36. package/dist/builtins/snapshot/index.mjs +150 -22
  37. package/dist/builtins/snapshot/snapshot-cache.d.ts +63 -0
  38. package/dist/builtins/snapshot/snapshot-resize.d.ts +33 -1
  39. package/dist/builtins/snapshot/snapshot.addon.d.ts +6 -0
  40. package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
  41. package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
  42. package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +1 -1
  43. package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +1 -1
  44. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
  45. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
  46. package/dist/builtins/system-config/system-config.addon.js +1 -1
  47. package/dist/builtins/system-config/system-config.addon.mjs +1 -1
  48. package/dist/builtins/winston-logging/index.js +1 -1
  49. package/dist/builtins/winston-logging/index.mjs +1 -1
  50. package/dist/{dist-3wh5LCad.mjs → dist-CAzHQ03i.mjs} +144 -21
  51. package/dist/{dist-9sC0wZw6.js → dist-DR1VmGz6.js} +144 -21
  52. package/dist/index.d.ts +2 -2
  53. package/dist/index.js +243 -328
  54. package/dist/index.mjs +233 -319
  55. package/dist/kernel/config-schema.d.ts +14 -10
  56. package/dist/kernel/heap-watch.d.ts +106 -7
  57. package/dist/kernel/moleculer/process-service.d.ts +34 -0
  58. package/dist/logging/log-manager.d.ts +7 -5
  59. package/dist/logging/partitioned-log-buffer.d.ts +95 -16
  60. package/dist/{manifest-python-deps-Bt_J04b0.mjs → manifest-python-deps-5ei2xVUh.mjs} +355 -1
  61. package/dist/{manifest-python-deps-BtVjvfvj.js → manifest-python-deps-GejnH--L.js} +422 -0
  62. package/package.json +1 -1
package/dist/index.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  import { a as __toCommonJS, i as __require, n as __esmMin, o as __toESM$1, r as __exportAll, t as __commonJSMin$1 } from "./chunk-CNf5ZN-e.mjs";
2
- import { At as asNumber, B as extractNestedAddonId, Bt as parseJsonUnknown$1, Dt as ReadinessTimeoutError, Et as ReadinessRegistry, Ft as expandCapMethods, Gt as EventCategory$1, H as isArrayOutputSchema, Ht as resolveCapMount, J as lifecycleJobSchema, K as isVoidInput, Mt as createEvent, Nt as emitDownForOwnedCaps, Q as looseSchema, St as DEVICE_STATUS_METHOD, U as isCollectionArrayMethod, Ut as scopeKey, Vt as readinessKey, W as isObjectInput, Z as logLevelAtMost, bt as DATAPLANE_SECRET_HEADER$1, d as METHOD_ACCESS_MAP, f as RUNTIME_DEFAULTS, it as procedureAuthKey, jt as asString$1, kt as asJsonObject$1, ot as scopesAllowAddon, q as kebabToCamel, st as scopesAllowDeviceCap, t as ALL_CAPABILITY_DEFINITIONS, tt as objectInputDeclaresAddonId, vt as errMsg$1, xt as DEVICE_SETTINGS_CONTRIBUTION_METHODS, y as addonSettingsCapability, zt as parseJsonObject } from "./dist-3wh5LCad.mjs";
2
+ import { At as asNumber, B as extractNestedAddonId, Bt as parseJsonUnknown$1, Dt as ReadinessTimeoutError, Et as ReadinessRegistry, Ft as expandCapMethods, Gt as EventCategory$1, H as isArrayOutputSchema, Ht as resolveCapMount, J as lifecycleJobSchema, K as isVoidInput, Mt as createEvent, Nt as emitDownForOwnedCaps, Q as looseSchema, St as DEVICE_STATUS_METHOD, U as isCollectionArrayMethod, Ut as scopeKey, Vt as readinessKey, W as isObjectInput, Z as logLevelAtMost, bt as DATAPLANE_SECRET_HEADER$1, d as METHOD_ACCESS_MAP, f as RUNTIME_DEFAULTS, it as procedureAuthKey, jt as asString$1, kt as asJsonObject$1, ot as scopesAllowAddon, q as kebabToCamel, st as scopesAllowDeviceCap, t as ALL_CAPABILITY_DEFINITIONS, tt as objectInputDeclaresAddonId, vt as errMsg$1, xt as DEVICE_SETTINGS_CONTRIBUTION_METHODS, y as addonSettingsCapability, zt as parseJsonObject } from "./dist-CAzHQ03i.mjs";
3
3
  import { AlertCenterAddon } from "./builtins/alerts/alerts.addon.mjs";
4
4
  import "./builtins/alerts/index.mjs";
5
5
  import { t as formatLogLine } from "./formatter-B7qW8bPJ.mjs";
@@ -24,7 +24,7 @@ import { SystemConfigAddon } from "./builtins/system-config/system-config.addon.
24
24
  import "./builtins/system-config/index.mjs";
25
25
  import { WinstonDestination, WinstonLoggingAddon } from "./builtins/winston-logging/index.mjs";
26
26
  import { a as parseTokenizedUrl, c as collectModelFiles, d as downloadModel, f as ensureModel, h as isModelDownloaded, i as parseRangeHeader, l as deleteModelFromDisk, m as getModelFilePath, n as contentTypeFor, o as resolveFilePath, p as fetchJson, r as createAuthenticatedFileServer, s as ModelDownloadService, t as createFileDataPlaneHandler, u as downloadFile } from "./file-data-plane-CuE_hBli.mjs";
27
- import { $ as buildNativeCapProxy, A as createHubCapForwardService, B as AGENT_CAP_FWD_ACTION, C as brokerTransportLink, Ct as runNpm, D as localProviderLink, E as ipcParentLink, F as createUdsLogger, G as callWithServiceDiscovery, H as CapRouteResolver, I as createUdsLoggerWithControl, J as UdsLocalTransportServer, K as createLocalTransport, L as LocalChildClient, M as createUdsEventBridge, N as createUdsEventBus, O as HUB_CAP_FWD_ACTION, P as udsChildLogToWorkerEntry, Q as encodeFrame, R as LocalChildRegistry, S as brokerCallForCap, St as resolveNpmInvocation, T as ipcChildLink, U as CapRouteError, V as AGENT_CAP_FWD_SERVICE, W as classifyCapRoute, X as localEndpointPath, Y as SocketChannel, Z as FrameDecoder, _ as createKernelHwAccel, _t as CapabilityHandle, a as getWorkerDeviceRegistry, b as __resetCapUsageRegistryForTests, bt as installManifestNativeDeps, c as setHubConnected, ct as NATIVE_PROVIDER_SERVICE_INFIX, d as getBrokerEventBus, dt as capBareAction, et as buildUdsNativeCapProxy, f as getMoleculerEventStats, ft as capServiceName, g as AddonDepsManager, gt as DeviceRegistry, h as subscribePassthrough, ht as serializeTypedArrays, i as createUdsAddonContext, it as mountNativeCapService, j as createParentUnownedCallHandler, k as HUB_CAP_FWD_SERVICE, l as EVENT_TOPIC_PREFIX, lt as capActionName, m as setNodeEventInterest, mt as deserializeTypedArrays, n as adaptBrokerToCluster, o as getOrInitReadinessRegistry, ot as createAddonService, p as registerEventBusService, pt as parseCapAction, q as UdsLocalTransportClient, r as createAddonContext, s as getOrInitReadinessRegistryForClient, st as validateProviderRegistrations, t as installManifestPythonDeps, tt as createBrokerDeviceManagerApi, u as clusterEventTopic, ut as capActionSuffix, v as resolveHwAccel, vt as CapabilityUnavailableError, w as buildLinkChain, wt as createAddonDataPlaneFacility, x as getCapUsageRegistry, xt as resolveAddonClass, y as CapUsageRegistry, yt as copyBundledNativeModules, z as UDS_NO_ROUTE_PREFIX } from "./manifest-python-deps-Bt_J04b0.mjs";
27
+ import { $ as buildNativeCapProxy, A as createHubCapForwardService, At as buildHeapSample, B as AGENT_CAP_FWD_ACTION, C as brokerTransportLink, Ct as runNpm, D as localProviderLink, Dt as HEAP_WATCH_INTERVAL_MS, E as ipcParentLink, Et as HEAP_RECLAIM_TRIGGER_MB, F as createUdsLogger, Ft as strandedMb, G as callWithServiceDiscovery, H as CapRouteResolver, I as createUdsLoggerWithControl, J as UdsLocalTransportServer, K as createLocalTransport, L as LocalChildClient, M as createUdsEventBridge, Mt as shouldReclaim, N as createUdsEventBus, Nt as startHeapWatch, O as HUB_CAP_FWD_ACTION, Ot as HEAP_WATCH_WARN_RATIO, P as udsChildLogToWorkerEntry, Pt as startRunnerHeapWatch, Q as encodeFrame, R as LocalChildRegistry, S as brokerCallForCap, St as resolveNpmInvocation, T as ipcChildLink, Tt as HEAP_RECLAIM_MIN_INTERVAL_MS, U as CapRouteError, V as AGENT_CAP_FWD_SERVICE, W as classifyCapRoute, X as localEndpointPath, Y as SocketChannel, Z as FrameDecoder, _ as createKernelHwAccel, _t as CapabilityHandle, a as getWorkerDeviceRegistry, b as __resetCapUsageRegistryForTests, bt as installManifestNativeDeps, c as setHubConnected, ct as NATIVE_PROVIDER_SERVICE_INFIX, d as getBrokerEventBus, dt as capBareAction, et as buildUdsNativeCapProxy, f as getMoleculerEventStats, ft as capServiceName, g as AddonDepsManager, gt as DeviceRegistry, h as subscribePassthrough, ht as serializeTypedArrays, i as createUdsAddonContext, it as mountNativeCapService, j as createParentUnownedCallHandler, jt as createV8Reclaimer, k as HUB_CAP_FWD_SERVICE, kt as RUNNER_HEAP_WATCH_INTERVAL_MS, l as EVENT_TOPIC_PREFIX, lt as capActionName, m as setNodeEventInterest, mt as deserializeTypedArrays, n as adaptBrokerToCluster, o as getOrInitReadinessRegistry, ot as createAddonService, p as registerEventBusService, pt as parseCapAction, q as UdsLocalTransportClient, r as createAddonContext, s as getOrInitReadinessRegistryForClient, st as validateProviderRegistrations, t as installManifestPythonDeps, tt as createBrokerDeviceManagerApi, u as clusterEventTopic, ut as capActionSuffix, v as resolveHwAccel, vt as CapabilityUnavailableError, w as buildLinkChain, wt as createAddonDataPlaneFacility, x as getCapUsageRegistry, xt as resolveAddonClass, y as CapUsageRegistry, yt as copyBundledNativeModules, z as UDS_NO_ROUTE_PREFIX } from "./manifest-python-deps-5ei2xVUh.mjs";
28
28
  import { n as getSinglePidStats, t as getPidStats } from "./resource-monitor-BWmQ5i-o.mjs";
29
29
  import { t as CustomActionRegistry } from "./custom-action-registry-F__gp_VX.mjs";
30
30
  import { z } from "zod";
@@ -40,7 +40,6 @@ import * as os$18 from "node:os";
40
40
  import * as fs$17 from "node:fs";
41
41
  import { accessSync, constants, existsSync, mkdirSync, readFileSync } from "node:fs";
42
42
  import { request } from "node:http";
43
- import * as v8 from "node:v8";
44
43
  import * as vm from "node:vm";
45
44
  import { asJsonObject, errMsg, parseJsonUnknown } from "@camstack/types/addon";
46
45
  import { fileURLToPath, pathToFileURL } from "node:url";
@@ -423,271 +422,6 @@ function proxyToUpstream(opts) {
423
422
  else clientReq.pipe(upstream);
424
423
  }
425
424
  //#endregion
426
- //#region src/kernel/heap-watch.ts
427
- /**
428
- * heap-watch — hub-main's own memory, on the record, and returned to the OS.
429
- *
430
- * ## Why this exists
431
- *
432
- * hub-main used to be launched with `--max-old-space-size=1536` (docker-entrypoint;
433
- * the cap was removed on 2026-08-02, so the live ceiling is now V8's own default —
434
- * 4288MB in this container — and this module reads it at runtime rather than
435
- * assuming a number). It logged NOTHING about its own heap. When it died on
436
- * 2026-08-01/02 the only trace was V8's own stderr line:
437
- *
438
- * FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
439
- *
440
- * which carries no process name, no heap number and no history — the whole
441
- * server restarts and the evidence is gone. Four of those in ~15h, and the one
442
- * at 06:57:57 landed nine seconds after a viewer opened a WebRTC session, which
443
- * is why the app sat in a permanent "loading": the hub was dying as it
444
- * connected.
445
- *
446
- * `[mem]` reports already exist for forked addon runners (SIGUSR2 in
447
- * `addon-runner.ts`) — hub-main, the one process that actually OOMs, was the
448
- * one process with no telemetry at all.
449
- *
450
- * ## What it gives
451
- *
452
- * A heartbeat carrying the numbers that separate the two possible stories, and
453
- * a WARN as the heap approaches the ceiling — the line that will exist BEFORE
454
- * the next fatal instead of after it:
455
- *
456
- * - `heapUsedMb` climbing steadily between restarts → a leak;
457
- * - `heapUsedMb` flat with a sharp jump under load → a burst allocation, and
458
- * the timestamp says what was being served.
459
- *
460
- * `external`/`arrayBuffers` ride along because they are NOT bounded by
461
- * `--max-old-space-size` and are most of the gap between a 1.5GB cap and the
462
- * ~2.05GB RSS observed live: a diagnosis that only looks at the JS heap
463
- * misreads that process by 500MB.
464
- *
465
- * ## And then the heartbeat explained the "off-heap leak", which was not one
466
- *
467
- * 2026-08-08, measured on the live hub over 1019 consecutive samples and seven
468
- * separate boots: **hub-main's RSS is a HIGH-WATER MARK, not a leak.** In every
469
- * one of those 1019 samples RSS was >= the running maximum of
470
- * `heapTotal + external`, never once below it, and
471
- * `RSS_peak - peak(heapTotal + external)` landed between 874MB and 1639MB on
472
- * every boot — a roughly constant native floor on top of a number that only
473
- * ever goes up. RSS tracked the highest point the process ever reached and
474
- * stayed there: 3.3GB -> 4.5GB -> 5.2GB, and 7.8GB on the worst boot.
475
- *
476
- * Two things push that peak, and neither is a retained object:
477
- *
478
- * - the JS heap sawtooths to 2.0-2.8GB because the ceiling is 4288MB and
479
- * nothing creates pressure before then;
480
- * - `arrayBuffers` accumulates DEAD Buffers — up to 3.65GB — because V8 only
481
- * sweeps them on a major GC. Eleven collapses were captured, and every
482
- * single one coincides with `heapUsed` dropping to its ~650MB floor. No code
483
- * released a reference at 13:23:03; a mark-compact ran.
484
- *
485
- * At the peak V8 has committed the pages, and **an ordinary major GC does not
486
- * give them back.** Proven on the production hub: `smaps` showed 10,819 mapped
487
- * 256KB pages (V8's `kRegularPageSize`) holding 2704MB while V8 itself reported
488
- * `total_physical_size` of only 846MB — ~7,800 pages committed, resident, and
489
- * outside V8's own accounting. One `gc({ flavor: 'last-resort' })` took 417ms
490
- * and returned RSS 3355MB -> 1237MB, pages 10,819 -> 2,963.
491
- *
492
- * So the fix for the "~3GB/2h off-heap growth" is not a retainer hunt. It is to
493
- * ask V8 for the pages back when the process is visibly holding more than it is
494
- * using — which is what `shouldReclaim` decides and what the heartbeat now does.
495
- *
496
- * The cost is a 250-450ms stop-the-world on the hub's router process, bounded to
497
- * once per `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a
498
- * ratchet to 7.8GB and the OOM that ends it.
499
- */
500
- /** Heartbeat period. Slow enough to be free, fast enough that the sample before
501
- * a fatal is recent. */
502
- var HEAP_WATCH_INTERVAL_MS = 6e4;
503
- /** Warn above this share of the heap ceiling. Chosen so the warning precedes
504
- * the fatal by more than one interval under realistic growth. */
505
- var HEAP_WATCH_WARN_RATIO = .8;
506
- /**
507
- * Escalate above this share of the ceiling.
508
- *
509
- * 0.70 is 3002MB: clear of the measured routine max (2847MB) so steady state
510
- * never escalates, and below {@link HEAP_WATCH_WARN_RATIO} so the fast cadence
511
- * is already running by the time the warn is reachable.
512
- */
513
- var HEAP_WATCH_ESCALATE_RATIO = .7;
514
- /**
515
- * Release escalation below this share.
516
- *
517
- * Hysteresis, not a second threshold: hub-main's heap sawtooths continuously,
518
- * so a single ratio would flap the mode on every GC. 0.60 is 2573MB — a real
519
- * retreat rather than the trough of one collection.
520
- */
521
- var HEAP_WATCH_DEESCALATE_RATIO = .6;
522
- /**
523
- * The cadence state machine, pure so the thresholds are testable without
524
- * allocating three gigabytes to reach them.
525
- */
526
- function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO, deescalateRatio = HEAP_WATCH_DEESCALATE_RATIO) {
527
- if (current === "escalated") return usedRatio < deescalateRatio ? "steady" : "escalated";
528
- return usedRatio >= escalateRatio ? "escalated" : "steady";
529
- }
530
- /**
531
- * Reclaim once the process holds this many MB beyond what it is USING.
532
- *
533
- * Live calibration, both readings from the same hub minutes apart: 2491MB
534
- * stranded before the reclaim, 438MB after. 438MB is this process's irreducible
535
- * native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
536
- * collecting against it would buy nothing and cost a stall every minute. 1536
537
- * sits clear of that floor and still fires ~1.1GB before the ratchet is worth
538
- * anything, so it triggers on the real condition and stops as soon as it is fixed.
539
- */
540
- var HEAP_RECLAIM_TRIGGER_MB = 1536;
541
- /** Floor on how often the stop-the-world may run. 250-450ms measured, so ten
542
- * minutes keeps the worst case under 0.1% of the router's time. */
543
- var HEAP_RECLAIM_MIN_INTERVAL_MS = 6e5;
544
- var MB = (bytes) => Math.round(bytes / 1048576);
545
- /**
546
- * Build a sample from raw readings. Pure so the thresholds are testable without
547
- * allocating a gigabyte to reach them.
548
- */
549
- function buildHeapSample(mem, heapLimitBytes, warnRatio = HEAP_WATCH_WARN_RATIO) {
550
- const usedRatio = heapLimitBytes > 0 ? mem.heapUsed / heapLimitBytes : 0;
551
- return {
552
- rssMb: MB(mem.rss),
553
- heapUsedMb: MB(mem.heapUsed),
554
- heapTotalMb: MB(mem.heapTotal),
555
- heapLimitMb: MB(heapLimitBytes),
556
- externalMb: MB(mem.external),
557
- arrayBuffersMb: MB(mem.arrayBuffers),
558
- usedRatio: Math.round(usedRatio * 100) / 100,
559
- nearLimit: usedRatio >= warnRatio
560
- };
561
- }
562
- var consoleSink = {
563
- info: (line) => console.log(line),
564
- warn: (line) => console.warn(line)
565
- };
566
- /**
567
- * RSS the process is holding but not using.
568
- *
569
- * The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
570
- * committed at a sawtooth peak and no longer needs are counted inside heapTotal,
571
- * and they are precisely what this number exists to see.
572
- */
573
- function strandedMb(s) {
574
- return s.rssMb - s.heapUsedMb - s.externalMb;
575
- }
576
- /** Whether it is worth paying a stop-the-world to get those pages back. */
577
- function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
578
- return strandedMb(s) > triggerMb;
579
- }
580
- function isGcFunction(value) {
581
- return typeof value === "function";
582
- }
583
- /**
584
- * Build the reclaimer without requiring `--expose-gc` on argv.
585
- *
586
- * The flag is turned on, the function captured, and the flag turned straight
587
- * back off — a global `gc` left reachable by application code is a footgun, and
588
- * requiring an argv change would mean this cannot ship without a new image.
589
- * Verified against the live hub before being written: 266ms, RSS 1453 -> 1237.
590
- *
591
- * Returns undefined if V8 declines, so the caller keeps a heartbeat either way.
592
- */
593
- function createV8Reclaimer() {
594
- try {
595
- v8.setFlagsFromString("--expose-gc");
596
- const gc = vm.runInNewContext("gc");
597
- v8.setFlagsFromString("--no-expose-gc");
598
- if (!isGcFunction(gc)) return void 0;
599
- return () => gc({
600
- execution: "sync",
601
- flavor: "last-resort"
602
- });
603
- } catch {
604
- return;
605
- }
606
- }
607
- function format$1(label, s) {
608
- return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
609
- }
610
- /**
611
- * Start the heartbeat. Returns a stop function.
612
- *
613
- * `unref`ed so it can never hold the process open, and every read is wrapped:
614
- * a diagnostic that can crash the process it watches is worse than none.
615
- */
616
- function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
617
- const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
618
- const now = reclaimOptions?.now ?? (() => Date.now());
619
- const triggerMb = reclaimOptions?.triggerMb ?? 1536;
620
- const minIntervalMs = reclaimOptions?.minIntervalMs ?? 6e5;
621
- const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
622
- const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
623
- const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
624
- let lastReclaimAt = Number.NEGATIVE_INFINITY;
625
- const read = () => {
626
- const limit = reclaimOptions?.heapLimitBytes ?? v8.getHeapStatistics().heap_size_limit;
627
- return buildHeapSample(readMemory(), limit);
628
- };
629
- const maybeReclaim = (sample) => {
630
- if (reclaimOptions === void 0) return;
631
- if (!shouldReclaim(sample, triggerMb)) return;
632
- if (now() - lastReclaimAt < minIntervalMs) return;
633
- lastReclaimAt = now();
634
- const startedAt = now();
635
- try {
636
- reclaimOptions.reclaim();
637
- } catch (error) {
638
- sink.warn(`[mem] reclaim failed — ${error instanceof Error ? error.message : String(error)}`);
639
- return;
640
- }
641
- const after = read();
642
- sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
643
- };
644
- let mode = "steady";
645
- let lastLoggedAt = Number.NEGATIVE_INFINITY;
646
- /**
647
- * PROBING and LOGGING are separate cadences, and that separation is the whole
648
- * point.
649
- *
650
- * The 2026-08-08 fatal escaped a 60s heartbeat because the burst began and
651
- * ended between two samples. Deciding "is the heap elevated?" only when a
652
- * line is due would reproduce that exactly: the process would still be read
653
- * once a minute, and a 62-second climb would still be invisible. So the timer
654
- * runs at the FAST period always — `process.memoryUsage()` is tens of
655
- * microseconds, i.e. free at this rate — and the slow period governs only how
656
- * often a steady process is allowed to say so.
657
- *
658
- * Never probe slower than the log cadence, or a caller passing a short
659
- * `intervalMs` would silently get the long one.
660
- */
661
- const probeIntervalMs = Math.min(fastIntervalMs, intervalMs);
662
- const tick = () => {
663
- try {
664
- const sample = read();
665
- const previous = mode;
666
- mode = nextMode(previous, sample.usedRatio, escalateRatio, deescalateRatio);
667
- const at = now();
668
- const due = at - lastLoggedAt >= intervalMs;
669
- if (mode === "escalated" || due) {
670
- lastLoggedAt = at;
671
- const line = format$1(label, sample);
672
- if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
673
- else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
674
- else sink.info(line);
675
- }
676
- if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
677
- maybeReclaim(sample);
678
- } catch {}
679
- };
680
- const timer = setInterval(tick, probeIntervalMs);
681
- timer.unref?.();
682
- tick();
683
- let stopped = false;
684
- return () => {
685
- if (stopped) return;
686
- stopped = true;
687
- clearInterval(timer);
688
- };
689
- }
690
- //#endregion
691
425
  //#region src/lifecycle/lifecycle-state-machine.ts
692
426
  var VALID_TRANSITIONS = {
693
427
  stopped: ["starting", "disabled"],
@@ -960,6 +694,26 @@ function addonBucketKey(addonId) {
960
694
  */
961
695
  var PRUNE_TARGET_RATIO = .9;
962
696
  /**
697
+ * Default ceiling on retained entries across every bucket.
698
+ *
699
+ * This used to be `null` — no aggregate bound at all — and the only thing
700
+ * holding hub-main's log memory was `perAddonCapacity x buckets`, a product
701
+ * that grows with the roster. Measured on the live cluster: 37 forked runners
702
+ * on the hub plus its in-process builtins, and hub-main additionally ingests
703
+ * every line from both agents, so ~60 buckets is the ordinary case and
704
+ * 60 x 5000 = 300k retained entries.
705
+ *
706
+ * A realistic entry (timestamp, level, scope, ~70-char message, three tags, no
707
+ * `meta`) measures **~306 B retained** in V8. So the old unbounded case is
708
+ * ~92MB of a hub process that is being asked to fit inside a cgroup with a
709
+ * neighbour to protect, and 50_000 is **~15MB**.
710
+ *
711
+ * 50_000 across a ~60-bucket roster is ~830 entries per addon on average while
712
+ * still letting a single chatty addon hold its full 5000-entry ring — the
713
+ * partitioning is not weakened, only the total is held.
714
+ */
715
+ var DEFAULT_MAX_TOTAL_LOG_ENTRIES = 5e4;
716
+ /**
963
717
  * A log buffer partitioned into one fixed-capacity {@link LogRingBuffer} per
964
718
  * `addonId`. A chatty addon (e.g. detection-pipeline) only ever evicts its OWN
965
719
  * bucket, so a quiet addon's sparse lines (e.g. a HomeAssistant `image` entity)
@@ -968,38 +722,55 @@ var PRUNE_TARGET_RATIO = .9;
968
722
  * Drop-in for {@link LogRingBuffer} from {@link LogManager}'s perspective —
969
723
  * exposes the same `push` / `getAll` / `query` / `clear` surface.
970
724
  *
971
- * ## Two bounds, doing different jobs
725
+ * ## Three bounds, doing different jobs
972
726
  *
973
727
  * **`perAddonCapacity`** is a HARD bound: each bucket is a fixed-size ring, so
974
728
  * total memory can never exceed `perAddonCapacity × buckets` whatever else
975
729
  * happens. This is the safety net.
976
730
  *
977
- * **`maxTotalEntries`** is a SOFT target across all buckets, and it exists because
978
- * the hard bound scales with addon count — 70 addons × 5000 is a lot of retained
979
- * entries on a small box. When the total is exceeded, the sweep discards only
980
- * entries at or below `pruneLevel` (default `debug`), oldest first, from the
731
+ * **`maxTotalEntries`** targets the total across all buckets, and it exists because
732
+ * the hard bound scales with addon count — 60 buckets × 5000 was 300k retained
733
+ * entries on the live hub. When the total is exceeded, the sweep first discards
734
+ * only entries at or below `pruneLevel` (default `debug`), oldest first, from the
981
735
  * largest buckets first.
982
736
  *
983
- * It is deliberately soft. Meeting a total cap by discarding `warn` and `error`
984
- * would trade the entries that explain an incident for entries about frame
985
- * timings; if pruning every prunable entry still leaves the total above the cap,
986
- * the buffer stays above it and the hard per-bucket bound is what holds. A cap
987
- * that can evict an error is not a feature.
737
+ * It is deliberately soft ON A LIVE BUCKET. Meeting a total cap by discarding
738
+ * `warn` and `error` from an addon that is still logging would trade the entries
739
+ * that explain an incident for entries about frame timings. A cap that can evict
740
+ * a live addon's error is not a feature.
988
741
  *
989
742
  * Largest-bucket-first because the noisy addon is the one creating the pressure —
990
743
  * the same principle that motivated partitioning in the first place.
744
+ *
745
+ * **`maxBuckets`, and bucket eviction, are the hard backstop.** Pruning alone
746
+ * cannot bound this structure: it may only discard at or below `pruneLevel`, so
747
+ * a bucket full of `info` from an addon that no longer exists was immortal, and
748
+ * the roster the product scales with is not fixed. When pruning cannot reach the
749
+ * target — or the bucket count exceeds `maxBuckets` — whole buckets are dropped
750
+ * **least-recently-WRITTEN first**, and never the bucket currently being written
751
+ * to. That is precisely the dead addon's bucket, and it is precisely not the
752
+ * bucket of the addon whose incident is in progress.
991
753
  */
992
754
  var PartitionedLogBuffer = class {
993
755
  buffers = /* @__PURE__ */ new Map();
756
+ /**
757
+ * Bucket key → the write ordinal of its last `push`. A monotonic counter, not
758
+ * a clock: eviction order must follow the order writes actually happened, and
759
+ * a wall clock that steps backwards would pick the wrong victim.
760
+ */
761
+ lastWriteSeq = /* @__PURE__ */ new Map();
762
+ writeSeq = 0;
994
763
  perAddonCapacity;
995
764
  maxTotalEntries;
996
765
  pruneLevel;
766
+ maxBuckets;
997
767
  /** Running total, so `push` does not walk every bucket to decide. */
998
768
  totalEntries = 0;
999
769
  constructor(perAddonCapacity = 5e3, options = {}) {
1000
770
  this.perAddonCapacity = options.perAddonCapacity ?? perAddonCapacity;
1001
- this.maxTotalEntries = options.maxTotalEntries ?? null;
771
+ this.maxTotalEntries = options.maxTotalEntries === void 0 ? DEFAULT_MAX_TOTAL_LOG_ENTRIES : options.maxTotalEntries;
1002
772
  this.pruneLevel = options.pruneLevel ?? "debug";
773
+ this.maxBuckets = options.maxBuckets === void 0 ? 128 : options.maxBuckets;
1003
774
  }
1004
775
  /**
1005
776
  * Apply new limits at runtime. Shrinking takes effect as entries are pushed
@@ -1012,6 +783,7 @@ var PartitionedLogBuffer = class {
1012
783
  if (options.perAddonCapacity !== void 0 && options.perAddonCapacity > 0) this.perAddonCapacity = options.perAddonCapacity;
1013
784
  if (options.maxTotalEntries !== void 0) this.maxTotalEntries = options.maxTotalEntries;
1014
785
  if (options.pruneLevel !== void 0) this.pruneLevel = options.pruneLevel;
786
+ if (options.maxBuckets !== void 0) this.maxBuckets = options.maxBuckets;
1015
787
  }
1016
788
  bufferFor(key) {
1017
789
  let buf = this.buffers.get(key);
@@ -1022,34 +794,82 @@ var PartitionedLogBuffer = class {
1022
794
  return buf;
1023
795
  }
1024
796
  push(entry) {
1025
- const buf = this.bufferFor(addonBucketKey(entry.tags?.addonId));
797
+ const key = addonBucketKey(entry.tags?.addonId);
798
+ const buf = this.bufferFor(key);
1026
799
  const before = buf.size();
1027
800
  buf.push(entry);
1028
801
  this.totalEntries += buf.size() - before;
1029
- if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries);
802
+ this.lastWriteSeq.set(key, ++this.writeSeq);
803
+ if (this.maxBuckets !== null && this.buffers.size > this.maxBuckets) this.evictBuckets(this.buffers.size - this.maxBuckets, key);
804
+ if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries, key);
1030
805
  }
1031
806
  /**
1032
- * Bring the total down to `PRUNE_TARGET_RATIO × cap` by discarding prunable
1033
- * entries, largest bucket first. Stops early when nothing prunable is left —
1034
- * see the class docblock on why that is the correct outcome rather than a
1035
- * failure to enforce.
1036
- */
1037
- enforceTotalCap(cap) {
807
+ * Bring the total down to `PRUNE_TARGET_RATIO × cap`.
808
+ *
809
+ * Two passes, in this order and no other: first discard prunable entries,
810
+ * largest bucket first, which is free of information cost. Only if that
811
+ * cannot reach the target does the second pass drop WHOLE buckets,
812
+ * least-recently-written first — the bound has to hold even when every
813
+ * retained entry is above `pruneLevel`, and a bucket nobody has written to is
814
+ * the cheapest thing in the structure to lose.
815
+ */
816
+ enforceTotalCap(cap, protectedKey) {
1038
817
  const target = Math.max(1, Math.floor(cap * PRUNE_TARGET_RATIO));
1039
818
  let deficit = this.totalEntries - target;
1040
819
  if (deficit <= 0) return;
1041
- const bySizeDesc = [...this.buffers.values()].sort((a, b) => b.size() - a.size());
1042
- for (const buf of bySizeDesc) {
820
+ const bySizeDesc = [...this.buffers.entries()].toSorted((a, b) => b[1].size() - a[1].size());
821
+ for (const [key, buf] of bySizeDesc) {
1043
822
  if (deficit <= 0) break;
1044
823
  const removed = buf.pruneOldestAtOrBelow(this.pruneLevel, deficit);
1045
824
  this.totalEntries -= removed;
1046
825
  deficit -= removed;
826
+ if (buf.size() === 0) this.dropBucket(key);
1047
827
  }
828
+ if (deficit > 0) this.evictBucketsUntil(target, protectedKey);
829
+ }
830
+ /**
831
+ * Drop whole buckets, least-recently-WRITTEN first, until the total is at or
832
+ * below `target`. Never touches `protectedKey` (the bucket being written to
833
+ * right now) and always leaves at least one bucket standing.
834
+ */
835
+ evictBucketsUntil(target, protectedKey) {
836
+ for (const key of this.evictionOrder(protectedKey)) {
837
+ if (this.totalEntries <= target || this.buffers.size <= 1) return;
838
+ this.dropBucket(key);
839
+ }
840
+ }
841
+ /** Drop exactly `count` buckets, least-recently-written first. */
842
+ evictBuckets(count, protectedKey) {
843
+ let remaining = count;
844
+ for (const key of this.evictionOrder(protectedKey)) {
845
+ if (remaining <= 0 || this.buffers.size <= 1) return;
846
+ this.dropBucket(key);
847
+ remaining -= 1;
848
+ }
849
+ }
850
+ /** Bucket keys ordered least-recently-written first, excluding `protectedKey`.
851
+ * A bucket with no recorded write sorts first — it can only be a leftover. */
852
+ evictionOrder(protectedKey) {
853
+ return [...this.buffers.keys()].filter((key) => key !== protectedKey).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
854
+ }
855
+ /** Remove a bucket and everything that indexes it. An empty ring left behind
856
+ * is the leak this class shipped with — the map only ever grew. */
857
+ dropBucket(key) {
858
+ const buf = this.buffers.get(key);
859
+ if (buf === void 0) return;
860
+ this.totalEntries -= buf.size();
861
+ this.buffers.delete(key);
862
+ this.lastWriteSeq.delete(key);
1048
863
  }
1049
864
  /** Entries retained across every bucket. */
1050
865
  size() {
1051
866
  return this.totalEntries;
1052
867
  }
868
+ /** Buckets currently retained. Observability for the aggregate bound — an
869
+ * addon roster that only ever grows is what this class had to stop doing. */
870
+ bucketCount() {
871
+ return this.buffers.size;
872
+ }
1053
873
  /** Every retained entry across all buckets, newest-first (mirrors
1054
874
  * {@link LogRingBuffer.getAll}). Used to replay history to a destination. */
1055
875
  getAll() {
@@ -1077,13 +897,25 @@ var PartitionedLogBuffer = class {
1077
897
  if (pinned !== null) {
1078
898
  const removed = this.buffers.get(pinned)?.clear(filter) ?? 0;
1079
899
  this.totalEntries -= removed;
900
+ this.dropIfEmpty(pinned);
1080
901
  return removed;
1081
902
  }
1082
903
  let removed = 0;
1083
- for (const buf of this.buffers.values()) removed += buf.clear(filter);
904
+ for (const [key, buf] of [...this.buffers.entries()]) {
905
+ removed += buf.clear(filter);
906
+ this.dropIfEmpty(key);
907
+ }
1084
908
  this.totalEntries -= removed;
1085
909
  return removed;
1086
910
  }
911
+ /** An emptied bucket is removed rather than left as an empty ring: the map
912
+ * used to grow with every addon id that ever logged and never shrink. */
913
+ dropIfEmpty(key) {
914
+ if (this.buffers.get(key)?.size() === 0) {
915
+ this.buffers.delete(key);
916
+ this.lastWriteSeq.delete(key);
917
+ }
918
+ }
1087
919
  /** The bucket key a filter pins to via `tags.addonId`, or null when the
1088
920
  * filter doesn't constrain the addon (→ scan all buckets). */
1089
921
  pinnedAddonKey(filter) {
@@ -1181,11 +1013,13 @@ var LogManager = class {
1181
1013
  * `perAddonCapacity` bounds EACH addon's bucket, not the total — a chatty addon
1182
1014
  * evicts only its own lines, so quiet addons keep their sparse history.
1183
1015
  *
1184
- * `options.maxTotalEntries` adds a soft ceiling across all buckets, because the
1185
- * per-bucket bound scales with addon count. When it is exceeded, only entries at
1186
- * or below `options.pruneLevel` (default `debug`) are discarded `warn` and
1187
- * `error` are never evicted to satisfy it. Both default to the previous
1188
- * behaviour: no total cap. See {@link PartitionedLogBuffer}.
1016
+ * `options.maxTotalEntries` ceilings the total across all buckets, because the
1017
+ * per-bucket bound scales with addon count, and `options.maxBuckets` ceilings the
1018
+ * bucket count itself. When the total is exceeded, entries at or below
1019
+ * `options.pruneLevel` (default `debug`) are discarded first a live addon's
1020
+ * `warn`/`error` are never evicted to satisfy it — and only if that cannot reach
1021
+ * the target are whole least-recently-written buckets dropped. Both bounds are ON
1022
+ * by default; pass an explicit `null` to opt out. See {@link PartitionedLogBuffer}.
1189
1023
  */
1190
1024
  constructor(perAddonCapacity = 5e3, options = {}) {
1191
1025
  this.ringBuffer = new PartitionedLogBuffer(perAddonCapacity, options);
@@ -94296,33 +94130,45 @@ async function getPidStats$1(pids) {
94296
94130
  });
94297
94131
  });
94298
94132
  }
94133
+ var EMPTY_HEAP_DECLARATION = {
94134
+ profile: void 0,
94135
+ maxOldSpaceMb: void 0
94136
+ };
94299
94137
  /**
94300
- * Resolve the declared heap profile for one addon by reading its package.json manifest
94301
- * (`camstack.addons[].execution.heapProfile`). `addonDir` points at the addon's built `dist`, so the
94302
- * manifest is one level up; we also try `addonDir` itself for layouts where it IS the package root.
94138
+ * Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
94139
+ * manifest (`camstack.addons[].execution.{heapProfile,maxOldSpaceMb}`). `addonDir` points at the
94140
+ * addon's built `dist`, so the manifest is one level up; we also try `addonDir` itself for layouts
94141
+ * where it IS the package root.
94303
94142
  * Result is cached per (dir,id) — spawns are infrequent but a runner may re-spawn on crash/restart.
94304
- * Returns `undefined` when the manifest is unreadable or declares no profile → treated as light.
94143
+ * Returns an empty declaration when the manifest is unreadable → treated as light, default ceiling.
94305
94144
  */
94306
94145
  var heapProfileCache = /* @__PURE__ */ new Map();
94307
- function readAddonHeapProfile(spec) {
94146
+ function readAddonHeapDeclaration(spec) {
94308
94147
  const cacheKey = `${spec.addonDir}::${spec.addonId}`;
94309
94148
  const cached = heapProfileCache.get(cacheKey);
94310
- if (cached !== void 0 || heapProfileCache.has(cacheKey)) return cached;
94311
- let profile;
94149
+ if (cached !== void 0) return cached;
94150
+ let declaration = EMPTY_HEAP_DECLARATION;
94312
94151
  for (const manifestPath of [path$39.join(spec.addonDir, "package.json"), path$39.join(path$39.dirname(spec.addonDir), "package.json")]) try {
94313
94152
  const raw = fs$17.readFileSync(manifestPath, "utf8");
94314
94153
  const parsed = JSON.parse(raw);
94315
- profile = extractHeapProfile(parsed, spec.addonId);
94316
- if (profile !== void 0) break;
94154
+ const profile = extractHeapProfile(parsed, spec.addonId);
94155
+ const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
94156
+ if (profile !== void 0 || maxOldSpaceMb !== void 0) {
94157
+ declaration = {
94158
+ profile,
94159
+ maxOldSpaceMb
94160
+ };
94161
+ break;
94162
+ }
94317
94163
  if (manifestHasAddon(parsed, spec.addonId)) break;
94318
94164
  } catch {}
94319
- heapProfileCache.set(cacheKey, profile);
94320
- return profile;
94165
+ heapProfileCache.set(cacheKey, declaration);
94166
+ return declaration;
94321
94167
  }
94322
94168
  /**
94323
- * Drop the cached heap profile for each addon so the NEXT `readAddonHeapProfile`
94169
+ * Drop the cached heap declaration for each addon so the NEXT `readAddonHeapDeclaration`
94324
94170
  * re-reads the manifest from disk. Called on operator-driven spawns / restarts
94325
- * (a deploy may have changed the addon's `execution.heapProfile`); the
94171
+ * (a deploy may have changed `execution.heapProfile` / `execution.maxOldSpaceMb`); the
94326
94172
  * crash-respawn fast path deliberately does NOT invalidate — a crashing runner
94327
94173
  * has the same on-disk manifest, so re-reading it is pure overhead.
94328
94174
  */
@@ -94336,6 +94182,10 @@ function extractHeapProfile(parsed, addonId) {
94336
94182
  const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.heapProfile;
94337
94183
  return value === "heavy" || value === "light" ? value : void 0;
94338
94184
  }
94185
+ function extractMaxOldSpaceMb(parsed, addonId) {
94186
+ const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
94187
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
94188
+ }
94339
94189
  function readManifestAddons(parsed) {
94340
94190
  if (typeof parsed !== "object" || parsed === null) return [];
94341
94191
  const camstack = parsed.camstack;
@@ -94348,25 +94198,78 @@ function readManifestAddons(parsed) {
94348
94198
  * Everything else (the un-annotated default) is a light control-plane runner. No hard-coded names.
94349
94199
  */
94350
94200
  function isHeavyRunner(addons) {
94351
- return addons.some((a) => readAddonHeapProfile(a) === "heavy");
94201
+ return addons.some((a) => readAddonHeapDeclaration(a).profile === "heavy");
94202
+ }
94203
+ /**
94204
+ * Default old-space ceiling for a HEAVY runner, in MB.
94205
+ *
94206
+ * ## Why a heavy runner needs a ceiling at all
94207
+ *
94208
+ * `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
94209
+ * host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
94210
+ * not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
94211
+ * one-off burst becomes the runner's permanent RSS.
94212
+ *
94213
+ * Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
94214
+ * there after the deferred archive walk (99,809 segments in 84s), while the `RecordingIndex` it
94215
+ * actually retains is arithmetically ~21 MB (a `SegmentRow` is 5 numbers + 3 strings ≈ 207 B). The
94216
+ * same addon on two agent nodes carrying the same code and a smaller archive sits at **58 MB**.
94217
+ * The ~2 GB is walk garbage held as high-water in a heap that had no reason to collect it.
94218
+ *
94219
+ * ## Why 1024
94220
+ *
94221
+ * From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
94222
+ * over-estimate of its old space, since it also contains native allocations, `external` and
94223
+ * `arrayBuffers`, none of which `--max-old-space-size` bounds:
94224
+ *
94225
+ * detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
94226
+ * snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
94227
+ *
94228
+ * 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
94229
+ * more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
94230
+ * as it does now, because V8 only collects harder as it approaches the limit.
94231
+ *
94232
+ * A runner whose working set genuinely does not fit declares its own number in its manifest
94233
+ * (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
94234
+ * `stream-broker` (1323 MB measured, frame buffers) is the one shipped addon that does.
94235
+ */
94236
+ var HEAVY_MAX_OLD_MB_DEFAULT = 1024;
94237
+ /**
94238
+ * The runner's old-space ceiling in MB, or 0 for "uncapped".
94239
+ *
94240
+ * The runner takes the MAXIMUM ceiling any co-located addon declares — co-location must never
94241
+ * silence the hungriest member (D2/D29: a runner is respawned as a whole, so an OOM caused by
94242
+ * under-sizing one member kills every addon in the group). An addon declaring `0` therefore lifts
94243
+ * the cap for the whole runner, which is the intended escape hatch.
94244
+ */
94245
+ function runnerMaxOldSpaceMb(addons, heavy) {
94246
+ const envDefault = heavy ? positiveIntEnv("CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB", HEAVY_MAX_OLD_MB_DEFAULT) : positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
94247
+ const declared = addons.map((a) => readAddonHeapDeclaration(a).maxOldSpaceMb).filter((mb) => mb !== void 0);
94248
+ if (declared.length === 0) return envDefault;
94249
+ if (declared.includes(0)) return 0;
94250
+ return Math.max(...declared);
94352
94251
  }
94353
94252
  /**
94354
94253
  * Per-placement V8 heap flags (Block D). LIGHT runners cap their young generation
94355
- * (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win.
94356
- * HEAVY runners keep V8 defaults. The old-space cap is opt-in via env (unset = uncapped = no OOM
94357
- * risk) so it can be enabled after before/after measurement on the real cluster.
94254
+ * (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — and
94255
+ * keep an opt-in old-space cap (`CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`, unset = uncapped). HEAVY
94256
+ * runners keep V8's default young generation but ALWAYS carry an old-space ceiling; see
94257
+ * {@link HEAVY_MAX_OLD_MB_DEFAULT} for why, and how a runner declares its own.
94358
94258
  *
94359
94259
  * Kill-switch: `CAMSTACK_RUNNER_HEAP_TUNING=off` disables all flags (recover without a rebuild).
94360
94260
  * Overrides: `CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB` (default 2), `CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`
94361
- * (default unset = no old-space cap).
94261
+ * (default unset = no old-space cap), `CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB`
94262
+ * (default {@link HEAVY_MAX_OLD_MB_DEFAULT}; `0` restores the old uncapped behaviour).
94362
94263
  */
94363
94264
  function runnerHeapFlags(addons) {
94364
94265
  if (process.env["CAMSTACK_RUNNER_HEAP_TUNING"] === "off") return [];
94365
- if (isHeavyRunner(addons)) return [];
94366
- const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
94367
- const oldMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
94266
+ const heavy = isHeavyRunner(addons);
94368
94267
  const flags = [];
94369
- if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
94268
+ if (!heavy) {
94269
+ const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
94270
+ if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
94271
+ }
94272
+ const oldMb = runnerMaxOldSpaceMb(addons, heavy);
94370
94273
  if (oldMb > 0) flags.push(`--max-old-space-size=${oldMb}`);
94371
94274
  return flags;
94372
94275
  }
@@ -94443,19 +94346,21 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
94443
94346
  function spawnRunner(runnerId, addons, env) {
94444
94347
  const nodeId = buildNodeId(runnerId);
94445
94348
  const runnerPath = path$39.resolve(__dirname, "addon-runner.js");
94349
+ const heavy = isHeavyRunner(addons);
94446
94350
  const childEnv = {
94447
94351
  ...process.env,
94448
94352
  CAMSTACK_RUNNER_ID: runnerId,
94449
94353
  CAMSTACK_RUNNER_ADDONS: JSON.stringify(addons),
94450
94354
  CAMSTACK_PARENT_NODE_ID: parentNodeId,
94451
94355
  CAMSTACK_DATA_DIR: dataDir,
94356
+ CAMSTACK_RUNNER_HEAP_PROFILE: heavy ? "heavy" : "light",
94452
94357
  CAMSTACK_LOG_LEVEL: process.env["CAMSTACK_LOG_LEVEL"] ?? "info",
94453
94358
  ...parentTcpPort !== void 0 ? { CAMSTACK_PARENT_TCP_PORT: String(parentTcpPort) } : {},
94454
94359
  ...parentUdsPath !== void 0 ? { CAMSTACK_PARENT_UDS_PATH: parentUdsPath } : {},
94455
94360
  ...env
94456
94361
  };
94457
94362
  const heapFlags = runnerHeapFlags(addons);
94458
- capturedBroker?.logger.info(`[${runnerId}] heap profile: ${isHeavyRunner(addons) ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
94363
+ capturedBroker?.logger.info(`[${runnerId}] heap profile: ${heavy ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
94459
94364
  const child = spawnFn(process.execPath, [...heapFlags, runnerPath], {
94460
94365
  env: childEnv,
94461
94366
  stdio: [
@@ -94499,6 +94404,15 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
94499
94404
  capturedBroker?.logger.error(`[${runnerId}] Crash circuit-breaker tripped — ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms. Not respawning.`);
94500
94405
  processes.set(runnerId, entry);
94501
94406
  if (capturedBroker) forceMoleculerDisconnect(capturedBroker, entry.nodeId);
94407
+ emitRunnerEvent(EventCategory$1.AddonRunnerFailed, runnerId, {
94408
+ runnerId,
94409
+ nodeId: parentNodeId,
94410
+ addonIds: entry.runnerAddons.map((addon) => addon.addonId),
94411
+ crashesInWindow: decision.crashesInWindow,
94412
+ windowMs: CRASH_WINDOW_MS,
94413
+ ...code !== null ? { exitCode: code } : {},
94414
+ ...signal !== null ? { signal } : {}
94415
+ });
94502
94416
  for (const addon of entry.runnerAddons) emitRunnerEvent(EventCategory$1.AddonCrashed, addon.addonId, {
94503
94417
  addonId: addon.addonId,
94504
94418
  error: `Crash circuit-breaker tripped after ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms`
@@ -95177,4 +95091,4 @@ var LifecycleJobEngine = class {
95177
95091
  }
95178
95092
  };
95179
95093
  //#endregion
95180
- export { AGENT_CAP_FWD_ACTION, AGENT_CAP_FWD_SERVICE, AGENT_READINESS_SERVICE_NAME, ALL_CAPABILITY_DEFINITIONS, AddonApiFactory, AddonDepsManager, AddonEngineManager, AddonHealthMonitor, AddonInstaller, AddonLoader, AddonManifest, AddonRouteRegistry, AlertCenterAddon, ApiKeyManager, AuthManager, CLUSTER_SECRET_MISMATCH_TYPE, CLUSTER_SECRET_REJECTED_EXIT_CODE, CORE_CAP_SERVICE_NAME, CapRouteError, CapRouteResolver, CapUsageRegistry, CapabilityHandle, CapabilityRegistry, CapabilityUnavailableError, ConfigManager, ConfigStore, ConsoleDestination, ConsoleLoggingAddon, CoreBlocksAddon, CustomActionRegistry, DEFAULT_DATA_PATH, DEVICE_SETTINGS_CONTRIBUTION_METHODS, DEVICE_STATUS_METHOD, DataPlaneRegistry, DeviceManagerAddon, DeviceRegistry, DeviceStore, EVENT_TOPIC_PREFIX, EngineManagerResolver, EventBus, FeatureManager, FilesystemStorageAddon, FilesystemStorageProvider, FrameDecoder, FsStorageBackend, HEALTH_MONITOR_GRACE_PERIOD_MS, HEALTH_MONITOR_RETRY_INTERVALS_MS, HEALTH_MONITOR_TICK_MS, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_CAP_FWD_ACTION, HUB_CAP_FWD_SERVICE, HubForwarderAddon, HubForwarderDestination, HubLogForwarder, HubNodeRegistry, INFRA_CAPABILITIES, IntegrationRegistry, JobJournal, LifecycleJobEngine, LifecycleStateMachine, LivenessMonitorAddon, LocalAuthAddon, LocalChildClient, LocalChildRegistry, LogManager, LogRingBuffer, LokiDestination, LokiLoggingAddon, METHOD_ACCESS_MAP, ModelDownloadService, NATIVE_PROVIDER_SERVICE_INFIX, NATIVE_SCAN_DEPTH, NativeMetricsAddon, NativeMetricsProvider, NetworkQualityTracker, NotificationService, PYTHON_VERSION, PipelineRunner, PipelineValidator, PythonEnvManager, QUARANTINE_DIRNAME, RESTART_MARKER_FILE, RUNTIME_DEFAULTS, ReadinessRegistry, ReadinessTimeoutError, ReplEngine, RingBuffer, ScopedLogger, ScopedTokenManager, SocketChannel, SqliteSettingsAddon, SqliteSettingsBackend, StagingArea, StorageLocationManager, StorageManager, StorageOrchestratorAddon, StorageOrchestratorService, SystemConfigAddon, SystemEventBus, TRADITIONAL_NATIVE_PACKAGES, ToastService, UDS_NO_ROUTE_PREFIX, UdsLocalTransportClient, UdsLocalTransportServer, UserManager, WinstonDestination, WinstonLoggingAddon, __resetCapUsageRegistryForTests, adaptBrokerToCluster, addonSettingsCapability, bootstrapSchema, brokerCallForCap, brokerTransportLink, buildBinaryPath, buildCapRouters, buildHeapSample, buildLinkChain, buildNativeCapProxy, buildNodeManifest, buildStorageLocationRegistry, buildUdsNativeCapProxy, builderMountedCapNames, callRegisterNodeWithRetry, callWithServiceDiscovery, capActionName, capActionSuffix, capBareAction, capServiceName, classifyAddonDir, classifyCapRoute, clearPendingRestart, clusterEventTopic, clusterSecretMatches, collectModelFiles, contentTypeFor, copyDirRecursive, copyExtraFileDirs, createAddonContext, createAddonDataPlaneFacility, createAddonService, createAuthenticatedFileServer, createBroker, createBrokerDeviceManagerApi, createCoreCapService, createFileDataPlaneHandler, createHubCapForwardService, createHubService, createKernelHwAccel, createLocalTransport, createParentUnownedCallHandler, createProcessService, createReadinessService, createReadinessServiceForRegistry, createScopedProcessManager, createStreamProbeBrokerService, createUdsAddonContext, createUdsEventBridge, createUdsEventBus, createUdsLogger, createUdsLoggerWithControl, createV8Reclaimer, deleteModelFromDisk, deriveAgentListenPort, describeProviderKindDrift, detectWorkspacePackagesDir, downloadBinary, downloadFile, downloadModel, emitDownForOwnedCaps, encodeFrame, ensureAddonNativePrebuilds, ensureBinary, ensureDir, ensureFfmpeg, ensureLibraryBuilt, ensureModel, ensureNativePrebuilds, ensurePython, ensureTlsCert, expandCapMethods, fetchJson, findInPath, formatLogLine, getBrokerEventBus, getCapUsageRegistry, getFfmpegDownloadUrl, getModelFilePath, getMoleculerEventStats, getOrInitReadinessRegistry, getOrInitReadinessRegistryForClient, getPidStats, getPlatformInfo, getPythonDownloadUrl, getRestartMarkerPath, getSinglePidStats, getWorkerDeviceRegistry, hasDotNode, hashClusterSecret, installManifestNativeDeps, installManifestPythonDeps, installPackageFromNpm, installPythonPackages, installPythonRequirements, ipcChildLink, ipcParentLink, isAddonDeploySource, isArrayOutputSchema, isClusterSecretMismatchError, isCollectionArrayMethod, isInfraCapability, isModelDownloaded, isSourceNewer, loadTlsCert, localEndpointPath, localProviderLink, mountNativeCapService, parseCapAction, parseRangeHeader, parseTokenizedUrl, proxyToUpstream, quarantineAddonResidue, readPendingRestart, readinessKey, registerEventBusService, resolveFilePath, resolveHwAccel, resolveNpmInvocation, runNpm, scheduleSelfRestart, scopeKey, scopesAllowAddon, scopesAllowDeviceCap, selectAddonResidue, serializeTypedArrays, setHubConnected, setNodeEventInterest, shouldReclaim, startHeapWatch, strandedMb, stripCamstackDeps, subscribePassthrough, udsChildLogToWorkerEntry, validateProviderRegistrations, writePendingRestart };
95094
+ export { AGENT_CAP_FWD_ACTION, AGENT_CAP_FWD_SERVICE, AGENT_READINESS_SERVICE_NAME, ALL_CAPABILITY_DEFINITIONS, AddonApiFactory, AddonDepsManager, AddonEngineManager, AddonHealthMonitor, AddonInstaller, AddonLoader, AddonManifest, AddonRouteRegistry, AlertCenterAddon, ApiKeyManager, AuthManager, CLUSTER_SECRET_MISMATCH_TYPE, CLUSTER_SECRET_REJECTED_EXIT_CODE, CORE_CAP_SERVICE_NAME, CapRouteError, CapRouteResolver, CapUsageRegistry, CapabilityHandle, CapabilityRegistry, CapabilityUnavailableError, ConfigManager, ConfigStore, ConsoleDestination, ConsoleLoggingAddon, CoreBlocksAddon, CustomActionRegistry, DEFAULT_DATA_PATH, DEVICE_SETTINGS_CONTRIBUTION_METHODS, DEVICE_STATUS_METHOD, DataPlaneRegistry, DeviceManagerAddon, DeviceRegistry, DeviceStore, EVENT_TOPIC_PREFIX, EngineManagerResolver, EventBus, FeatureManager, FilesystemStorageAddon, FilesystemStorageProvider, FrameDecoder, FsStorageBackend, HEALTH_MONITOR_GRACE_PERIOD_MS, HEALTH_MONITOR_RETRY_INTERVALS_MS, HEALTH_MONITOR_TICK_MS, HEAP_RECLAIM_MIN_INTERVAL_MS, HEAP_RECLAIM_TRIGGER_MB, HEAP_WATCH_INTERVAL_MS, HEAP_WATCH_WARN_RATIO, HUB_CAP_FWD_ACTION, HUB_CAP_FWD_SERVICE, HubForwarderAddon, HubForwarderDestination, HubLogForwarder, HubNodeRegistry, INFRA_CAPABILITIES, IntegrationRegistry, JobJournal, LifecycleJobEngine, LifecycleStateMachine, LivenessMonitorAddon, LocalAuthAddon, LocalChildClient, LocalChildRegistry, LogManager, LogRingBuffer, LokiDestination, LokiLoggingAddon, METHOD_ACCESS_MAP, ModelDownloadService, NATIVE_PROVIDER_SERVICE_INFIX, NATIVE_SCAN_DEPTH, NativeMetricsAddon, NativeMetricsProvider, NetworkQualityTracker, NotificationService, PYTHON_VERSION, PipelineRunner, PipelineValidator, PythonEnvManager, QUARANTINE_DIRNAME, RESTART_MARKER_FILE, RUNNER_HEAP_WATCH_INTERVAL_MS, RUNTIME_DEFAULTS, ReadinessRegistry, ReadinessTimeoutError, ReplEngine, RingBuffer, ScopedLogger, ScopedTokenManager, SocketChannel, SqliteSettingsAddon, SqliteSettingsBackend, StagingArea, StorageLocationManager, StorageManager, StorageOrchestratorAddon, StorageOrchestratorService, SystemConfigAddon, SystemEventBus, TRADITIONAL_NATIVE_PACKAGES, ToastService, UDS_NO_ROUTE_PREFIX, UdsLocalTransportClient, UdsLocalTransportServer, UserManager, WinstonDestination, WinstonLoggingAddon, __resetCapUsageRegistryForTests, adaptBrokerToCluster, addonSettingsCapability, bootstrapSchema, brokerCallForCap, brokerTransportLink, buildBinaryPath, buildCapRouters, buildHeapSample, buildLinkChain, buildNativeCapProxy, buildNodeManifest, buildStorageLocationRegistry, buildUdsNativeCapProxy, builderMountedCapNames, callRegisterNodeWithRetry, callWithServiceDiscovery, capActionName, capActionSuffix, capBareAction, capServiceName, classifyAddonDir, classifyCapRoute, clearPendingRestart, clusterEventTopic, clusterSecretMatches, collectModelFiles, contentTypeFor, copyDirRecursive, copyExtraFileDirs, createAddonContext, createAddonDataPlaneFacility, createAddonService, createAuthenticatedFileServer, createBroker, createBrokerDeviceManagerApi, createCoreCapService, createFileDataPlaneHandler, createHubCapForwardService, createHubService, createKernelHwAccel, createLocalTransport, createParentUnownedCallHandler, createProcessService, createReadinessService, createReadinessServiceForRegistry, createScopedProcessManager, createStreamProbeBrokerService, createUdsAddonContext, createUdsEventBridge, createUdsEventBus, createUdsLogger, createUdsLoggerWithControl, createV8Reclaimer, deleteModelFromDisk, deriveAgentListenPort, describeProviderKindDrift, detectWorkspacePackagesDir, downloadBinary, downloadFile, downloadModel, emitDownForOwnedCaps, encodeFrame, ensureAddonNativePrebuilds, ensureBinary, ensureDir, ensureFfmpeg, ensureLibraryBuilt, ensureModel, ensureNativePrebuilds, ensurePython, ensureTlsCert, expandCapMethods, fetchJson, findInPath, formatLogLine, getBrokerEventBus, getCapUsageRegistry, getFfmpegDownloadUrl, getModelFilePath, getMoleculerEventStats, getOrInitReadinessRegistry, getOrInitReadinessRegistryForClient, getPidStats, getPlatformInfo, getPythonDownloadUrl, getRestartMarkerPath, getSinglePidStats, getWorkerDeviceRegistry, hasDotNode, hashClusterSecret, installManifestNativeDeps, installManifestPythonDeps, installPackageFromNpm, installPythonPackages, installPythonRequirements, ipcChildLink, ipcParentLink, isAddonDeploySource, isArrayOutputSchema, isClusterSecretMismatchError, isCollectionArrayMethod, isInfraCapability, isModelDownloaded, isSourceNewer, loadTlsCert, localEndpointPath, localProviderLink, mountNativeCapService, parseCapAction, parseRangeHeader, parseTokenizedUrl, proxyToUpstream, quarantineAddonResidue, readPendingRestart, readinessKey, registerEventBusService, resolveFilePath, resolveHwAccel, resolveNpmInvocation, runNpm, scheduleSelfRestart, scopeKey, scopesAllowAddon, scopesAllowDeviceCap, selectAddonResidue, serializeTypedArrays, setHubConnected, setNodeEventInterest, shouldReclaim, startHeapWatch, startRunnerHeapWatch, strandedMb, stripCamstackDeps, subscribePassthrough, udsChildLogToWorkerEntry, validateProviderRegistrations, writePendingRestart };