@camstack/system 1.2.389 → 1.2.391
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon/addon-api-factory.d.ts +30 -30
- package/dist/addon-routes/addon-route-registry.d.ts +31 -31
- package/dist/addon-runner.js +1 -1
- package/dist/addon-runner.mjs +1 -1
- package/dist/addon-utils.d.ts +22 -22
- package/dist/auth/admin-token-manager.d.ts +37 -67
- package/dist/auth/api-key-manager.d.ts +8 -8
- package/dist/auth/auth-audit-log.d.ts +29 -49
- package/dist/auth/auth-manager.d.ts +127 -126
- package/dist/auth/cidr.d.ts +8 -8
- package/dist/auth/external-identity-manager.d.ts +46 -34
- package/dist/auth/jwt-signing-key.d.ts +14 -14
- package/dist/auth/login-session-manager.d.ts +95 -135
- package/dist/auth/parse-record.d.ts +13 -13
- package/dist/auth/scope-matcher.d.ts +7 -7
- package/dist/auth/scoped-token-manager.d.ts +62 -62
- package/dist/auth/token-expiry.d.ts +49 -26
- package/dist/auth/totp-manager.d.ts +48 -48
- package/dist/auth/user-management-internal.d.ts +67 -52
- package/dist/auth/user-manager.d.ts +58 -58
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.d.ts +54 -54
- package/dist/builtins/addon-pages-aggregator/dedupe-pages.d.ts +22 -4
- package/dist/builtins/addon-pages-aggregator/index.d.ts +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.d.ts +54 -54
- package/dist/builtins/addon-widgets-aggregator/index.d.ts +1 -1
- package/dist/builtins/alerts/alerts.addon.d.ts +88 -87
- package/dist/builtins/alerts/device-label.d.ts +6 -6
- package/dist/builtins/alerts/index.d.ts +1 -1
- package/dist/builtins/alerts/prune-policy.d.ts +36 -36
- package/dist/builtins/autotrack/autotrack-cameras.d.ts +102 -79
- package/dist/builtins/autotrack/autotrack-config.d.ts +81 -81
- package/dist/builtins/autotrack/autotrack-decision.d.ts +69 -82
- package/dist/builtins/autotrack/autotrack-loop.d.ts +81 -61
- package/dist/builtins/autotrack/autotrack.addon.d.ts +103 -66
- package/dist/builtins/autotrack/frame-error.d.ts +38 -38
- package/dist/builtins/autotrack/frame-measurements.d.ts +79 -79
- package/dist/builtins/autotrack/index.d.ts +2 -2
- package/dist/builtins/autotrack/ptz-mirror.d.ts +46 -17
- package/dist/builtins/autotrack/target-selection.d.ts +33 -47
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.d.ts +270 -269
- package/dist/builtins/backup-orchestrator/cron-helpers.d.ts +17 -17
- package/dist/builtins/backup-orchestrator/destination-policy.d.ts +99 -82
- package/dist/builtins/backup-orchestrator/download-helpers.d.ts +9 -9
- package/dist/builtins/backup-orchestrator/encryption-key-store.d.ts +40 -29
- package/dist/builtins/backup-orchestrator/index.d.ts +2 -2
- package/dist/builtins/backup-orchestrator/manifest-store.d.ts +88 -69
- package/dist/builtins/backup-orchestrator/restore-archive.d.ts +14 -14
- package/dist/builtins/backup-orchestrator/run-queue.d.ts +86 -86
- package/dist/builtins/backup-orchestrator/schedule-store.d.ts +53 -37
- package/dist/builtins/backup-orchestrator/staging-paths.d.ts +23 -23
- package/dist/builtins/backup-orchestrator/write-targets.d.ts +53 -53
- package/dist/builtins/calendars/calendar-color.d.ts +5 -1
- package/dist/builtins/calendars/calendar-prefs-store.d.ts +18 -12
- package/dist/builtins/calendars/calendars.addon.d.ts +16 -11
- package/dist/builtins/calendars/index.d.ts +1 -1
- package/dist/builtins/camera-grid/acquire-retry.d.ts +26 -26
- package/dist/builtins/camera-grid/addon.d.ts +200 -174
- package/dist/builtins/camera-grid/flv-join.d.ts +32 -58
- package/dist/builtins/camera-grid/grid-activity-mirror.d.ts +37 -37
- package/dist/builtins/camera-grid/grid-activity.d.ts +83 -42
- package/dist/builtins/camera-grid/grid-camera-declarations.d.ts +46 -46
- package/dist/builtins/camera-grid/grid-camera-device.d.ts +60 -34
- package/dist/builtins/camera-grid/grid-canvas-derivation.d.ts +50 -15
- package/dist/builtins/camera-grid/grid-canvas-seed.d.ts +96 -46
- package/dist/builtins/camera-grid/grid-canvas.d.ts +74 -74
- package/dist/builtins/camera-grid/grid-child.d.ts +29 -16
- package/dist/builtins/camera-grid/grid-compositor-child.d.ts +2 -2
- package/dist/builtins/camera-grid/grid-compositor-invocation.d.ts +39 -11
- package/dist/builtins/camera-grid/grid-compositor.d.ts +50 -50
- package/dist/builtins/camera-grid/grid-detection-mapping.d.ts +52 -22
- package/dist/builtins/camera-grid/grid-device-settings.d.ts +31 -7
- package/dist/builtins/camera-grid/grid-filter-graph.d.ts +41 -41
- package/dist/builtins/camera-grid/grid-frame-sample.d.ts +45 -19
- package/dist/builtins/camera-grid/grid-instances.d.ts +20 -50
- package/dist/builtins/camera-grid/grid-last-frame-store.d.ts +51 -51
- package/dist/builtins/camera-grid/grid-layout-native-provider.d.ts +30 -4
- package/dist/builtins/camera-grid/grid-plan.d.ts +30 -20
- package/dist/builtins/camera-grid/grid-profiles.d.ts +85 -45
- package/dist/builtins/camera-grid/grid-row-normalization.d.ts +6 -6
- package/dist/builtins/camera-grid/grid-sentry-sources.d.ts +57 -57
- package/dist/builtins/camera-grid/grid-snapshot.d.ts +41 -41
- package/dist/builtins/camera-grid/grid-stream-descriptor.d.ts +49 -21
- package/dist/builtins/camera-grid/grid-stream-invocation.d.ts +37 -22
- package/dist/builtins/camera-grid/grid-stream-relay.d.ts +26 -26
- package/dist/builtins/camera-grid/grid-stream-session.d.ts +123 -93
- package/dist/builtins/camera-grid/grid-tile-feed.d.ts +47 -47
- package/dist/builtins/camera-grid/grid-tile-plan.d.ts +55 -20
- package/dist/builtins/camera-grid/grid-wire-format.d.ts +22 -6
- package/dist/builtins/camera-grid/index.d.ts +44 -44
- package/dist/builtins/camera-grid/silence-analysis.d.ts +31 -14
- package/dist/builtins/composer/claim-gate.d.ts +54 -27
- package/dist/builtins/composer/composed-device-set.d.ts +40 -26
- package/dist/builtins/composer/composed-device.d.ts +39 -17
- package/dist/builtins/composer/composer-apply-waits.d.ts +20 -13
- package/dist/builtins/composer/composer-grafts.d.ts +57 -38
- package/dist/builtins/composer/composer-live-set.d.ts +15 -15
- package/dist/builtins/composer/composer-plan.d.ts +76 -60
- package/dist/builtins/composer/composer-runtime-factory.d.ts +21 -14
- package/dist/builtins/composer/composer-seeds.d.ts +20 -14
- package/dist/builtins/composer/composer-verdict.d.ts +28 -14
- package/dist/builtins/composer/composer.addon.d.ts +23 -9
- package/dist/builtins/composer/composer.d.ts +145 -100
- package/dist/builtins/composer/composition-declarations.d.ts +50 -39
- package/dist/builtins/composer/composition-evaluation.d.ts +38 -22
- package/dist/builtins/composer/composition-runtime.d.ts +178 -125
- package/dist/builtins/composer/composition-target.d.ts +38 -34
- package/dist/builtins/composer/confirmed-seed.d.ts +35 -23
- package/dist/builtins/composer/deadline-timer.d.ts +13 -13
- package/dist/builtins/composer/existing-target.d.ts +90 -73
- package/dist/builtins/composer/field-record.d.ts +26 -19
- package/dist/builtins/composer/graft-host.d.ts +56 -32
- package/dist/builtins/composer/held-claims.d.ts +26 -9
- package/dist/builtins/composer/index.d.ts +1 -1
- package/dist/builtins/composer/own-field-reads.d.ts +26 -15
- package/dist/builtins/composer/owned-fields-target.d.ts +75 -44
- package/dist/builtins/composer/slice-assembler.d.ts +19 -3
- package/dist/builtins/composer/source-readings.d.ts +67 -61
- package/dist/builtins/composer/source-tracker.d.ts +133 -69
- package/dist/builtins/console-logging/console-destination.d.ts +15 -8
- package/dist/builtins/console-logging/console-logging.addon.d.ts +20 -19
- package/dist/builtins/console-logging/index.d.ts +3 -3
- package/dist/builtins/core-blocks/block-package.d.ts +37 -37
- package/dist/builtins/core-blocks/block-settings-store.d.ts +34 -21
- package/dist/builtins/core-blocks/block-supervisor.d.ts +138 -117
- package/dist/builtins/core-blocks/block-type-defs.d.ts +21 -21
- package/dist/builtins/core-blocks/blocks-integration.d.ts +76 -49
- package/dist/builtins/core-blocks/compile-block.d.ts +37 -37
- package/dist/builtins/core-blocks/composition-api.d.ts +38 -24
- package/dist/builtins/core-blocks/composition-peers.d.ts +26 -4
- package/dist/builtins/core-blocks/composition-sources.d.ts +25 -10
- package/dist/builtins/core-blocks/core-block-store.d.ts +109 -109
- package/dist/builtins/core-blocks/core-blocks.addon.d.ts +72 -72
- package/dist/builtins/core-blocks/extract-settings-schema.d.ts +4 -4
- package/dist/builtins/core-blocks/index.d.ts +8 -8
- package/dist/builtins/device-manager/adoption-job-engine.d.ts +141 -98
- package/dist/builtins/device-manager/claimed-status-overlay.d.ts +42 -9
- package/dist/builtins/device-manager/contribution-cache.d.ts +56 -56
- package/dist/builtins/device-manager/day-night-config-schema.d.ts +26 -12
- package/dist/builtins/device-manager/device-aggregation-merge.d.ts +17 -17
- package/dist/builtins/device-manager/device-aggregation.d.ts +277 -254
- package/dist/builtins/device-manager/device-bindings-store.d.ts +232 -220
- package/dist/builtins/device-manager/device-config-contribution.d.ts +29 -29
- package/dist/builtins/device-manager/device-config-secrets.d.ts +32 -13
- package/dist/builtins/device-manager/device-event-propagator.d.ts +22 -22
- package/dist/builtins/device-manager/device-extension-linked-slot.d.ts +17 -17
- package/dist/builtins/device-manager/device-form-secrets.d.ts +14 -14
- package/dist/builtins/device-manager/device-linked-devices.d.ts +159 -98
- package/dist/builtins/device-manager/device-manager.addon.d.ts +223 -197
- package/dist/builtins/device-manager/device-meta-actions.d.ts +306 -305
- package/dist/builtins/device-manager/device-meta-store.d.ts +100 -90
- package/dist/builtins/device-manager/device-meta-types.d.ts +250 -241
- package/dist/builtins/device-manager/device-online-fallback.d.ts +30 -30
- package/dist/builtins/device-manager/device-projection.d.ts +53 -67
- package/dist/builtins/device-manager/device-provider-context.d.ts +109 -94
- package/dist/builtins/device-manager/device-queries.d.ts +63 -63
- package/dist/builtins/device-manager/device-row-store.d.ts +417 -365
- package/dist/builtins/device-manager/device-source-activation.d.ts +91 -49
- package/dist/builtins/device-manager/device-state-claim-views.d.ts +76 -61
- package/dist/builtins/device-manager/device-state-claims.d.ts +53 -30
- package/dist/builtins/device-manager/device-state-mirror.d.ts +256 -264
- package/dist/builtins/device-manager/device-sync-resolvers.d.ts +25 -25
- package/dist/builtins/device-manager/field-claims-index.d.ts +85 -58
- package/dist/builtins/device-manager/field-ownership.d.ts +54 -36
- package/dist/builtins/device-manager/fleet-read-census.d.ts +99 -61
- package/dist/builtins/device-manager/image-settings-config-schema.d.ts +18 -8
- package/dist/builtins/device-manager/index.d.ts +2 -2
- package/dist/builtins/device-manager/linked-devices-manual-backfill.d.ts +20 -9
- package/dist/builtins/device-manager/location-alignment.d.ts +46 -46
- package/dist/builtins/device-manager/migrate-device.d.ts +178 -101
- package/dist/builtins/device-manager/migrate-guard.d.ts +66 -34
- package/dist/builtins/device-manager/migrate-hardware-state.d.ts +59 -59
- package/dist/builtins/device-manager/migrate-reload.d.ts +42 -4
- package/dist/builtins/device-manager/migration-refused.d.ts +20 -20
- package/dist/builtins/device-manager/mirror-row-writer.d.ts +205 -179
- package/dist/builtins/device-manager/recording-onboard-config-schema.d.ts +56 -32
- package/dist/builtins/device-manager/removal-fence.d.ts +48 -48
- package/dist/builtins/device-manager/runtime-state-persist-gate.d.ts +54 -26
- package/dist/builtins/device-manager/secret-config-paths.d.ts +17 -17
- package/dist/builtins/device-manager/stale-wrapper-activations.d.ts +52 -25
- package/dist/builtins/device-manager/write-lock.d.ts +13 -13
- package/dist/builtins/hub-forwarder/hub-forwarder-destination.d.ts +41 -41
- package/dist/builtins/hub-forwarder/hub-forwarder.addon.d.ts +13 -12
- package/dist/builtins/hub-forwarder/index.d.ts +3 -3
- package/dist/builtins/keychain/index.d.ts +1 -1
- package/dist/builtins/keychain/key-sources.d.ts +19 -19
- package/dist/builtins/keychain/keychain-crypto.d.ts +4 -4
- package/dist/builtins/keychain/keychain-reset-trigger.d.ts +13 -13
- package/dist/builtins/keychain/keychain-service.d.ts +128 -128
- package/dist/builtins/keychain/keychain-store.d.ts +96 -87
- package/dist/builtins/keychain/keychain-usage.d.ts +13 -2
- package/dist/builtins/keychain/keychain.addon.d.ts +31 -6
- package/dist/builtins/keychain/platform-key-source.d.ts +15 -15
- package/dist/builtins/keychain/system-held-secrets.d.ts +17 -4
- package/dist/builtins/liveness-monitor/index.d.ts +2 -2
- package/dist/builtins/liveness-monitor/liveness-checks.d.ts +73 -52
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.d.ts +57 -36
- package/dist/builtins/local-auth/admin-account-boot.d.ts +14 -14
- package/dist/builtins/local-auth/auth-config-reader.d.ts +34 -17
- package/dist/builtins/local-auth/auth-schema.d.ts +92 -69
- package/dist/builtins/local-auth/index.d.ts +1 -1
- package/dist/builtins/local-auth/jwt-signing-key-keeper.d.ts +22 -22
- package/dist/builtins/local-auth/local-auth.addon.d.ts +41 -40
- package/dist/builtins/local-auth/oauth-grants.d.ts +57 -57
- package/dist/builtins/local-auth/oauth-session-manager.d.ts +62 -62
- package/dist/builtins/local-network/address-book.d.ts +61 -36
- package/dist/builtins/local-network/index.d.ts +2 -2
- package/dist/builtins/local-network/ipv6-lan.d.ts +12 -17
- package/dist/builtins/local-network/local-hostname.d.ts +10 -10
- package/dist/builtins/local-network/local-network.addon.d.ts +259 -258
- package/dist/builtins/loki-logging/index.d.ts +4 -4
- package/dist/builtins/loki-logging/loki-destination.d.ts +113 -87
- package/dist/builtins/loki-logging/loki-logging.addon.d.ts +61 -42
- package/dist/builtins/loki-logging/loki-payload.d.ts +82 -39
- package/dist/builtins/loki-logging/retire-enabled-flag.d.ts +37 -12
- package/dist/builtins/native-metrics/cgroup-memory.d.ts +206 -159
- package/dist/builtins/native-metrics/cgroup-pids.d.ts +138 -91
- package/dist/builtins/native-metrics/container-memory-series.d.ts +212 -178
- package/dist/builtins/native-metrics/gpu-probe.d.ts +9 -9
- package/dist/builtins/native-metrics/index.d.ts +2 -2
- package/dist/builtins/native-metrics/load-series-config.d.ts +29 -57
- package/dist/builtins/native-metrics/load-series-reader.d.ts +52 -26
- package/dist/builtins/native-metrics/load-series-store.d.ts +237 -176
- package/dist/builtins/native-metrics/native-metrics-provider.d.ts +43 -43
- package/dist/builtins/native-metrics/native-metrics.addon.d.ts +230 -230
- package/dist/builtins/native-metrics/node-load-ring.d.ts +196 -196
- package/dist/builtins/native-metrics/proc-process-table.d.ts +12 -12
- package/dist/builtins/native-metrics/process-classification.d.ts +95 -36
- package/dist/builtins/native-metrics/thread-cpu-sampler.d.ts +95 -95
- package/dist/builtins/platform-probe/coral-accelerators.d.ts +17 -17
- package/dist/builtins/platform-probe/index.d.ts +38 -30
- package/dist/builtins/platform-probe/inference-config-resolver.d.ts +27 -27
- package/dist/builtins/platform-probe/intel-accelerators.d.ts +8 -8
- package/dist/builtins/platform-probe/intel-npu.d.ts +30 -30
- package/dist/builtins/platform-probe/platform-scorer.d.ts +35 -35
- package/dist/builtins/platform-probe/runtime-packages.d.ts +3 -3
- package/dist/builtins/remote-access-orchestrator/enabled-providers-reconcile.d.ts +87 -87
- package/dist/builtins/remote-access-orchestrator/index.d.ts +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.d.ts +66 -35
- package/dist/builtins/shared/device-state-changed.d.ts +9 -9
- package/dist/builtins/shared/settle-sources.d.ts +16 -16
- package/dist/builtins/snapshot/index.d.ts +2 -2
- package/dist/builtins/snapshot/snapshot-cache.d.ts +92 -92
- package/dist/builtins/snapshot/snapshot-coalescing.d.ts +108 -108
- package/dist/builtins/snapshot/snapshot-courtesy.d.ts +48 -48
- package/dist/builtins/snapshot/snapshot-durable-store.d.ts +32 -32
- package/dist/builtins/snapshot/snapshot-fatigue.d.ts +34 -34
- package/dist/builtins/snapshot/snapshot-grab-pick.d.ts +14 -14
- package/dist/builtins/snapshot/snapshot-link-url.d.ts +49 -49
- package/dist/builtins/snapshot/snapshot-media-handler.d.ts +97 -97
- package/dist/builtins/snapshot/snapshot-resize.d.ts +64 -103
- package/dist/builtins/snapshot/snapshot.addon.d.ts +569 -568
- package/dist/builtins/sqlite-storage/analyze-stats.d.ts +73 -29
- package/dist/builtins/sqlite-storage/config-store.d.ts +6 -6
- package/dist/builtins/sqlite-storage/device-store.d.ts +16 -16
- package/dist/builtins/sqlite-storage/filesystem-browse-provider.d.ts +23 -23
- package/dist/builtins/sqlite-storage/filesystem-storage-provider.d.ts +79 -79
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.d.ts +27 -26
- package/dist/builtins/sqlite-storage/filter-compiler.d.ts +107 -107
- package/dist/builtins/sqlite-storage/index.d.ts +12 -12
- package/dist/builtins/sqlite-storage/integration-registry.d.ts +37 -37
- package/dist/builtins/sqlite-storage/path-guard.d.ts +5 -5
- package/dist/builtins/sqlite-storage/prefix-range.d.ts +44 -44
- package/dist/builtins/sqlite-storage/query-bounds.d.ts +100 -106
- package/dist/builtins/sqlite-storage/query-profiler.d.ts +217 -217
- package/dist/builtins/sqlite-storage/retired-collections.d.ts +124 -55
- package/dist/builtins/sqlite-storage/retired-settings-keys.d.ts +239 -189
- package/dist/builtins/sqlite-storage/shm-lock-canary.d.ts +41 -41
- package/dist/builtins/sqlite-storage/sqlite-pragmas.d.ts +162 -162
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +458 -458
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.d.ts +59 -58
- package/dist/builtins/sqlite-storage/sqlite-snapshot-protocol.d.ts +37 -32
- package/dist/builtins/sqlite-storage/sqlite-snapshot-worker.d.ts +19 -19
- package/dist/builtins/sqlite-storage/sqlite-snapshotter.d.ts +76 -76
- package/dist/builtins/sqlite-storage/statement-shape.d.ts +54 -54
- package/dist/builtins/sqlite-storage/table-sizes.d.ts +71 -71
- package/dist/builtins/sqlite-storage/vector-index-shared.d.ts +63 -54
- package/dist/builtins/sqlite-storage/vector-index-vec.d.ts +133 -96
- package/dist/builtins/sqlite-storage/wal-checkpoint-policy.d.ts +161 -161
- package/dist/builtins/sqlite-storage/wal-checkpointer.d.ts +60 -60
- package/dist/builtins/sqlite-storage/wal-maintenance.d.ts +94 -94
- package/dist/builtins/storage-orchestrator/access-guards.d.ts +37 -26
- package/dist/builtins/storage-orchestrator/data-store-dispatch.d.ts +33 -18
- package/dist/builtins/storage-orchestrator/drain-ratchet.d.ts +85 -85
- package/dist/builtins/storage-orchestrator/drain-state-store.d.ts +20 -10
- package/dist/builtins/storage-orchestrator/hub-settings-forward.d.ts +113 -53
- package/dist/builtins/storage-orchestrator/index.d.ts +7 -7
- package/dist/builtins/storage-orchestrator/location-occupancy-slice.d.ts +64 -40
- package/dist/builtins/storage-orchestrator/location-occupancy.d.ts +14 -14
- package/dist/builtins/storage-orchestrator/location-protection.d.ts +13 -13
- package/dist/builtins/storage-orchestrator/location-repoint-move.d.ts +44 -44
- package/dist/builtins/storage-orchestrator/location-root-collision.d.ts +61 -25
- package/dist/builtins/storage-orchestrator/location-secrets.d.ts +99 -58
- package/dist/builtins/storage-orchestrator/location-store.d.ts +56 -47
- package/dist/builtins/storage-orchestrator/location-volume-capacity.d.ts +37 -37
- package/dist/builtins/storage-orchestrator/provider-discovery.d.ts +20 -6
- package/dist/builtins/storage-orchestrator/storage-cleanup-coordinator.d.ts +51 -51
- package/dist/builtins/storage-orchestrator/storage-migration-coordinator.d.ts +224 -224
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.d.ts +292 -292
- package/dist/builtins/storage-orchestrator/storage-orchestrator.service.d.ts +431 -450
- package/dist/builtins/storage-orchestrator/storage-pressure-manager.d.ts +104 -104
- package/dist/builtins/system-backup/backup-encryption.d.ts +60 -23
- package/dist/builtins/system-backup/system-backup.service.d.ts +240 -240
- package/dist/builtins/system-config/cluster-secret-keeper.d.ts +54 -34
- package/dist/builtins/system-config/index.d.ts +2 -2
- package/dist/builtins/system-config/mesh-actions.d.ts +2 -30
- package/dist/builtins/system-config/system-config.addon.d.ts +57 -93
- package/dist/builtins/system-config/system-config.addon.js +46 -33
- package/dist/builtins/system-config/system-config.addon.mjs +46 -33
- package/dist/builtins/winston-logging/index.d.ts +3 -3
- package/dist/builtins/winston-logging/winston-destination.d.ts +15 -15
- package/dist/builtins/winston-logging/winston-logging.addon.d.ts +14 -13
- package/dist/{child-cap-dispatch-nURMuVTX.js → child-cap-dispatch-Bnr8brru.js} +18 -0
- package/dist/{child-cap-dispatch-B1gyE35b.mjs → child-cap-dispatch-mHO_6r0G.mjs} +1 -1
- package/dist/download/hosted-model-revisions.d.ts +8 -8
- package/dist/download/model-download-service.d.ts +32 -32
- package/dist/download/model-downloader.d.ts +88 -88
- package/dist/events/event-bus.d.ts +6 -6
- package/dist/events/system-event-bus.d.ts +11 -11
- package/dist/feature/feature-manager.d.ts +6 -6
- package/dist/http/asset-compression.d.ts +35 -73
- package/dist/http/authenticated-file-server.d.ts +32 -32
- package/dist/http/data-plane-registry.d.ts +49 -39
- package/dist/http/file-data-plane.d.ts +14 -14
- package/dist/http/lan-http-bind.d.ts +24 -13
- package/dist/http/reverse-proxy.d.ts +36 -24
- package/dist/index.d.ts +137 -137
- package/dist/index.js +4 -3
- package/dist/index.mjs +2 -4
- package/dist/kernel/addon-class-resolver.d.ts +1 -1
- package/dist/kernel/addon-engine-manager.d.ts +20 -20
- package/dist/kernel/addon-health-monitor.d.ts +151 -151
- package/dist/kernel/addon-installer.d.ts +609 -609
- package/dist/kernel/addon-loader.d.ts +133 -133
- package/dist/kernel/addon-manifest.d.ts +118 -118
- package/dist/kernel/addon-residue.d.ts +28 -32
- package/dist/kernel/array-buffer-census.d.ts +84 -88
- package/dist/kernel/bootstrap-config.d.ts +3 -3
- package/dist/kernel/cap-router-builder.d.ts +145 -101
- package/dist/kernel/capability-handle.d.ts +41 -41
- package/dist/kernel/capability-registry.d.ts +588 -576
- package/dist/kernel/config-manager.d.ts +199 -199
- package/dist/kernel/config-schema.d.ts +132 -128
- package/dist/kernel/custom-action-registry.d.ts +25 -25
- package/dist/kernel/deps/addon-deps-manager.d.ts +17 -17
- package/dist/kernel/deps/closure-dedupe.d.ts +76 -76
- package/dist/kernel/deps/ensure-addon-natives.d.ts +34 -34
- package/dist/kernel/deps/ensure-native-prebuilds.d.ts +25 -25
- package/dist/kernel/deps/manifest-native-deps.d.ts +38 -38
- package/dist/kernel/deps/manifest-python-deps.d.ts +18 -18
- package/dist/kernel/deps/manifest-system-deps.d.ts +13 -13
- package/dist/kernel/deps/npm-command.d.ts +45 -45
- package/dist/kernel/deps/system-package-allowlist.d.ts +9 -9
- package/dist/kernel/deps/system-package-broker.d.ts +20 -20
- package/dist/kernel/deps/system-package-client.d.ts +3 -3
- package/dist/kernel/device-registry.d.ts +27 -27
- package/dist/kernel/event-loop-stall-monitor.d.ts +33 -33
- package/dist/kernel/event-plane-report.d.ts +174 -174
- package/dist/kernel/fs-utils.d.ts +70 -70
- package/dist/kernel/gc-census.d.ts +119 -119
- package/dist/kernel/heap-spaces.d.ts +87 -87
- package/dist/kernel/heap-watch.d.ts +706 -706
- package/dist/kernel/host-externals.d.ts +8 -8
- package/dist/kernel/hwaccel/hwaccel-resolver.d.ts +27 -27
- package/dist/kernel/index.d.ts +115 -113
- package/dist/kernel/inference/inference-resolver.d.ts +13 -13
- package/dist/kernel/infra-capabilities.d.ts +7 -7
- package/dist/kernel/isolated-builtin-phase.d.ts +42 -42
- package/dist/kernel/lifecycle/job-journal.d.ts +17 -17
- package/dist/kernel/lifecycle/lifecycle-job-engine.d.ts +99 -99
- package/dist/kernel/lifecycle/staging-area.d.ts +22 -22
- package/dist/kernel/moleculer/addon-context-factory.d.ts +90 -89
- package/dist/kernel/moleculer/addon-data-plane-facility.d.ts +12 -12
- package/dist/kernel/moleculer/addon-deploy-source.d.ts +11 -11
- package/dist/kernel/moleculer/addon-peer-bytes-facility.d.ts +18 -18
- package/dist/kernel/moleculer/addon-service-factory.d.ts +41 -41
- package/dist/kernel/moleculer/agent-mesh-authenticator.d.ts +60 -33
- package/dist/kernel/moleculer/agent-ui-unlock.d.ts +30 -30
- package/dist/kernel/moleculer/broker-factory.d.ts +61 -61
- package/dist/kernel/moleculer/cap-usage-registry.d.ts +108 -153
- package/dist/kernel/moleculer/capabilities-access.d.ts +19 -19
- package/dist/kernel/moleculer/child-addon-call-dispatch.d.ts +43 -43
- package/dist/kernel/moleculer/child-cap-descriptors.d.ts +26 -26
- package/dist/kernel/moleculer/child-cap-dispatch.d.ts +52 -52
- package/dist/kernel/moleculer/children-batch-loader.d.ts +86 -34
- package/dist/kernel/moleculer/cluster-secret.d.ts +19 -19
- package/dist/kernel/moleculer/core-cap-service.d.ts +53 -53
- package/dist/kernel/moleculer/crash-supervisor.d.ts +39 -39
- package/dist/kernel/moleculer/device-cap-proxy.d.ts +51 -51
- package/dist/kernel/moleculer/device-event-ring.d.ts +117 -75
- package/dist/kernel/moleculer/event-bus-core.d.ts +171 -159
- package/dist/kernel/moleculer/event-bus.d.ts +73 -73
- package/dist/kernel/moleculer/hub-log-forwarder.d.ts +30 -30
- package/dist/kernel/moleculer/hub-service.d.ts +42 -42
- package/dist/kernel/moleculer/init-watchdog.d.ts +49 -49
- package/dist/kernel/moleculer/mesh-access-middleware.d.ts +36 -21
- package/dist/kernel/moleculer/mesh-auth-mode.d.ts +33 -39
- package/dist/kernel/moleculer/mesh-authenticator.d.ts +58 -58
- package/dist/kernel/moleculer/mesh-auto-enforce.d.ts +38 -18
- package/dist/kernel/moleculer/mesh-join-client.d.ts +67 -45
- package/dist/kernel/moleculer/mesh-join-proof.d.ts +8 -8
- package/dist/kernel/moleculer/mesh-mtls/constants.d.ts +1 -1
- package/dist/kernel/moleculer/mesh-mtls/index.d.ts +10 -10
- package/dist/kernel/moleculer/mesh-mtls/mesh-ca-keeper.d.ts +43 -38
- package/dist/kernel/moleculer/mesh-mtls/mesh-ca.d.ts +29 -20
- package/dist/kernel/moleculer/mesh-mtls/mesh-clear-port.d.ts +32 -32
- package/dist/kernel/moleculer/mesh-mtls/mesh-crl.d.ts +11 -11
- package/dist/kernel/moleculer/mesh-mtls/mesh-enroll.d.ts +19 -16
- package/dist/kernel/moleculer/mesh-mtls/mesh-join-token.d.ts +40 -40
- package/dist/kernel/moleculer/mesh-mtls/mesh-tls-enabler.d.ts +5 -1
- package/dist/kernel/moleculer/mesh-mtls/tls-tcp-transporter.d.ts +131 -130
- package/dist/kernel/moleculer/mesh-queue-report.d.ts +39 -39
- package/dist/kernel/moleculer/node-registry.d.ts +120 -120
- package/dist/kernel/moleculer/process-context.d.ts +2 -2
- package/dist/kernel/moleculer/process-service.d.ts +168 -168
- package/dist/kernel/moleculer/provider-registry.d.ts +9 -9
- package/dist/kernel/moleculer/readiness-context.d.ts +58 -58
- package/dist/kernel/moleculer/readiness-service.d.ts +31 -16
- package/dist/kernel/moleculer/register-framework-resolver.d.ts +6 -6
- package/dist/kernel/moleculer/register-node-client.d.ts +34 -27
- package/dist/kernel/moleculer/remote-logger.d.ts +29 -29
- package/dist/kernel/moleculer/resilient-cap-call.d.ts +21 -21
- package/dist/kernel/moleculer/runner-native-allocator.d.ts +95 -95
- package/dist/kernel/moleculer/runner-process-group.d.ts +57 -57
- package/dist/kernel/moleculer/runtime-state-load.d.ts +21 -11
- package/dist/kernel/moleculer/settings-read-guard.d.ts +39 -39
- package/dist/kernel/moleculer/stream-probe-service.d.ts +25 -6
- package/dist/kernel/moleculer/trpc-links.d.ts +203 -203
- package/dist/kernel/moleculer/typed-array-serde.d.ts +9 -20
- package/dist/kernel/moleculer/worker-device-restore.d.ts +8 -8
- package/dist/kernel/outbound-queue-watch.d.ts +74 -74
- package/dist/kernel/process-cpu-census.d.ts +55 -55
- package/dist/kernel/provider-kind-drift.d.ts +17 -8
- package/dist/kernel/restart-coordinator.d.ts +71 -71
- package/dist/kernel/settings-door-view.d.ts +36 -25
- package/dist/kernel/settings-read-error.d.ts +29 -31
- package/dist/kernel/settings-view-types.d.ts +100 -95
- package/dist/kernel/socket-plane-report.d.ts +214 -153
- package/dist/kernel/status-overlay.d.ts +31 -11
- package/dist/kernel/storage-location-registry.d.ts +50 -50
- package/dist/kernel/store-settings-view.d.ts +13 -13
- package/dist/kernel/system-settings-mirror.d.ts +40 -28
- package/dist/kernel/system-settings-policy.d.ts +15 -34
- package/dist/kernel/transport/cap-action-census.d.ts +183 -183
- package/dist/kernel/transport/cap-action-name.d.ts +84 -84
- package/dist/kernel/transport/cap-child-index.d.ts +41 -41
- package/dist/kernel/transport/cap-route-resolver.d.ts +237 -212
- package/dist/kernel/transport/cap-route.d.ts +139 -139
- package/dist/kernel/transport/cap-routing-hints.d.ts +37 -37
- package/dist/kernel/transport/child-cap-protocol.d.ts +245 -245
- package/dist/kernel/transport/claimed-get-status.d.ts +30 -6
- package/dist/kernel/transport/create-local-transport.d.ts +5 -5
- package/dist/kernel/transport/frame-codec.d.ts +109 -109
- package/dist/kernel/transport/hub-cap-forward.d.ts +81 -59
- package/dist/kernel/transport/index.d.ts +36 -36
- package/dist/kernel/transport/local-child-client.d.ts +184 -184
- package/dist/kernel/transport/local-child-registry.d.ts +592 -592
- package/dist/kernel/transport/local-endpoint-path.d.ts +4 -4
- package/dist/kernel/transport/local-transport.d.ts +92 -92
- package/dist/kernel/transport/parent-unowned-call.d.ts +266 -237
- package/dist/kernel/transport/peer-answered-error.d.ts +14 -14
- package/dist/kernel/transport/socket-channel.d.ts +110 -110
- package/dist/kernel/transport/socket-traffic.d.ts +96 -96
- package/dist/kernel/transport/uds-event-bridge.d.ts +77 -50
- package/dist/kernel/transport/uds-event-bus.d.ts +75 -26
- package/dist/kernel/transport/uds-local-transport.d.ts +14 -14
- package/dist/kernel/transport/uds-log-ingest.d.ts +38 -23
- package/dist/kernel/transport/uds-logger.d.ts +33 -33
- package/dist/kernel/utils/ring-buffer.d.ts +12 -12
- package/dist/kernel/workspace-detect.d.ts +7 -7
- package/dist/lifecycle/lifecycle-state-machine.d.ts +24 -24
- package/dist/logging/formatter.d.ts +52 -25
- package/dist/logging/log-manager.d.ts +133 -133
- package/dist/logging/log-redactor.d.ts +22 -1
- package/dist/logging/log-ring-buffer.d.ts +57 -57
- package/dist/logging/logging-gate.d.ts +69 -25
- package/dist/logging/logging-gate.singleton.d.ts +12 -1
- package/dist/logging/partitioned-log-buffer.d.ts +161 -161
- package/dist/logging/scoped-logger.d.ts +14 -14
- package/dist/network/network-quality.d.ts +8 -8
- package/dist/notification/notification-service.d.ts +37 -37
- package/dist/notification/toast-service.d.ts +18 -18
- package/dist/pipeline/engine-manager-resolver.d.ts +12 -12
- package/dist/pipeline/pipeline-runner.d.ts +6 -6
- package/dist/pipeline/pipeline-validator.d.ts +8 -8
- package/dist/process/proc-stat.d.ts +51 -51
- package/dist/process/resource-monitor.d.ts +19 -19
- package/dist/python/python-env-manager.d.ts +10 -10
- package/dist/repl/interfaces.d.ts +21 -21
- package/dist/repl/repl-engine.d.ts +6 -6
- package/dist/storage/fs-storage-backend.d.ts +35 -35
- package/dist/storage/storage-location-manager.d.ts +21 -21
- package/dist/storage/storage-manager.d.ts +68 -68
- package/dist/tls/boot-tls.d.ts +24 -24
- package/dist/tls/cert-evaluation.d.ts +25 -25
- package/dist/tls/cert-identity.d.ts +39 -39
- package/dist/tls/cert-issue.d.ts +37 -17
- package/dist/tls/cert-manager.d.ts +86 -84
- package/dist/tls/https-listener.d.ts +16 -16
- package/dist/tls/index.d.ts +8 -8
- package/dist/tls/tls-actions.d.ts +18 -18
- package/dist/tls/tls-status.d.ts +15 -15
- package/dist/tls/uploaded-cert.d.ts +9 -9
- package/package.json +9 -3
|
@@ -1,803 +1,803 @@
|
|
|
1
|
-
import { EventPlaneReader } from
|
|
2
|
-
import { SocketPlaneCounters, SocketPlaneReader } from
|
|
3
|
-
import { MeshPeerQueue, MeshQueueReader } from
|
|
4
|
-
import { OutboundQueueGateOptions, OutboundQueueReading } from
|
|
5
|
-
import { HeapSpaceReader } from
|
|
6
|
-
import { NodeGcObserver } from
|
|
7
|
-
import { TaskTicks } from
|
|
8
|
-
import { ArrayBufferCensus, ArrayBufferCensusGateOptions } from
|
|
1
|
+
import { type EventPlaneReader } from "./event-plane-report.js";
|
|
2
|
+
import { type SocketPlaneCounters, type SocketPlaneReader } from "./socket-plane-report.js";
|
|
3
|
+
import { type MeshPeerQueue, type MeshQueueReader } from "./moleculer/mesh-queue-report.js";
|
|
4
|
+
import { type OutboundQueueGateOptions, type OutboundQueueReading } from "./outbound-queue-watch.js";
|
|
5
|
+
import { type HeapSpaceReader } from "./heap-spaces.js";
|
|
6
|
+
import { type NodeGcObserver } from "./gc-census.js";
|
|
7
|
+
import { type TaskTicks } from "./process-cpu-census.js";
|
|
8
|
+
import { type ArrayBufferCensus, type ArrayBufferCensusGateOptions } from "./array-buffer-census.js";
|
|
9
9
|
/**
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
10
|
+
* What hub-main calls itself in a `[mem]` line.
|
|
11
|
+
*
|
|
12
|
+
* ## Why not `hub/main`
|
|
13
|
+
*
|
|
14
|
+
* A runner's label is its nodeId, and a nodeId is built in exactly one place as
|
|
15
|
+
* `` `${parentNodeId}/${runnerId}` `` (`kernel/moleculer/addon-runner.ts:222`) —
|
|
16
|
+
* so every runner label contains a `/` and every hub-local one is `hub/<runner>`.
|
|
17
|
+
* A label with NO slash therefore cannot be produced by that rule, which makes
|
|
18
|
+
* `hub-main` unambiguous by construction rather than by convention: no future
|
|
19
|
+
* addon, group or agent can ever claim it.
|
|
20
|
+
*
|
|
21
|
+
* It also sorts where it belongs. `-` (0x2D) precedes `/` (0x2F), so a sorted
|
|
22
|
+
* `[mem]` census puts `hub-main` immediately before `hub/decoder`,
|
|
23
|
+
* `hub/recorder`, `hub/stream-broker` — the composition root first, then the
|
|
24
|
+
* runners it forked. `hub/main` would sort into the middle of its own children
|
|
25
|
+
* and would be one addon id away from a collision.
|
|
26
|
+
*
|
|
27
|
+
* And it is the name the rest of this repository already uses — the docblocks,
|
|
28
|
+
* {@link HUB_MAIN_RSS_BUDGET_MB}, {@link HUB_RSS_BUDGET_ENV}, every ADR. A log
|
|
29
|
+
* label that disagreed with the prose would be a second vocabulary for one
|
|
30
|
+
* process, which is the exact failure this telemetry exists to remove.
|
|
31
|
+
*/
|
|
32
32
|
export declare const HUB_MAIN_HEAP_WATCH_LABEL = "hub-main";
|
|
33
33
|
/** Heartbeat period. Slow enough to be free, fast enough that the sample before
|
|
34
|
-
|
|
35
|
-
export declare const HEAP_WATCH_INTERVAL_MS =
|
|
34
|
+
* a fatal is recent. */
|
|
35
|
+
export declare const HEAP_WATCH_INTERVAL_MS = 6e4;
|
|
36
36
|
/** Warn above this share of the heap ceiling. Chosen so the warning precedes
|
|
37
|
-
|
|
38
|
-
export declare const HEAP_WATCH_WARN_RATIO =
|
|
37
|
+
* the fatal by more than one interval under realistic growth. */
|
|
38
|
+
export declare const HEAP_WATCH_WARN_RATIO = .8;
|
|
39
39
|
/**
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
40
|
+
* ## The heartbeat could not see the thing it was built to see
|
|
41
|
+
*
|
|
42
|
+
* 2026-08-08 18:28:32Z, hub-main (pid 8) died on `Reached heap limit`. The last
|
|
43
|
+
* `[mem]` line was 18:27:30.433 reading `heapUsed=1086MB` — 25%, unremarkable —
|
|
44
|
+
* and V8's own trace 62 seconds later read 4083MB with a Mark-Compact that
|
|
45
|
+
* reclaimed 4MB of 4090MB. Between those two facts there is not one sample.
|
|
46
|
+
*
|
|
47
|
+
* At `HEAP_WATCH_INTERVAL_MS` the whole event fits inside a single gap, so
|
|
48
|
+
* `HEAP_WATCH_WARN_RATIO` never had a reading to fire on. The docblock above
|
|
49
|
+
* promises "the line that will exist BEFORE the next fatal instead of after
|
|
50
|
+
* it"; on the only occasion that mattered, it did not exist.
|
|
51
|
+
*
|
|
52
|
+
* Measured the same day over 397 consecutive live samples: p50 1266MB, p95
|
|
53
|
+
* 1995MB, **max 2847MB** — 0.66 of the 4288MB ceiling. So the routine sawtooth
|
|
54
|
+
* never approaches 0.8, and the only path to 0.8 is a burst that outruns the
|
|
55
|
+
* sampler. A threshold reachable only through a blind spot is decoration.
|
|
56
|
+
*
|
|
57
|
+
* The fix is not a faster heartbeat — 30 lines a minute forever, to describe a
|
|
58
|
+
* process that is fine, is how a log stops being read. It is to sample fast
|
|
59
|
+
* only while the heap is somewhere the routine sawtooth does not go.
|
|
60
|
+
*/
|
|
61
61
|
/** Sampling period once escalated. Bounded below by log volume, above by the
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
export declare const HEAP_WATCH_FAST_INTERVAL_MS =
|
|
62
|
+
* 62s window the 2026-08-08 fatal hid in: ~30 readings across that window is
|
|
63
|
+
* a trajectory, and a trajectory separates a burst from a climb. */
|
|
64
|
+
export declare const HEAP_WATCH_FAST_INTERVAL_MS = 2e3;
|
|
65
65
|
/**
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
export declare const HEAP_WATCH_ESCALATE_RATIO =
|
|
66
|
+
* Escalate above this share of the ceiling.
|
|
67
|
+
*
|
|
68
|
+
* 0.70 is 3002MB: clear of the measured routine max (2847MB) so steady state
|
|
69
|
+
* never escalates, and below {@link HEAP_WATCH_WARN_RATIO} so the fast cadence
|
|
70
|
+
* is already running by the time the warn is reachable.
|
|
71
|
+
*/
|
|
72
|
+
export declare const HEAP_WATCH_ESCALATE_RATIO = .7;
|
|
73
73
|
/**
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
export declare const HEAP_WATCH_DEESCALATE_RATIO =
|
|
74
|
+
* Release escalation below this share.
|
|
75
|
+
*
|
|
76
|
+
* Hysteresis, not a second threshold: hub-main's heap sawtooths continuously,
|
|
77
|
+
* so a single ratio would flap the mode on every GC. 0.60 is 2573MB — a real
|
|
78
|
+
* retreat rather than the trough of one collection.
|
|
79
|
+
*/
|
|
80
|
+
export declare const HEAP_WATCH_DEESCALATE_RATIO = .6;
|
|
81
81
|
/** Sampling cadence. `escalated` costs log lines and buys resolution. */
|
|
82
|
-
export type HeapWatchMode =
|
|
82
|
+
export type HeapWatchMode = "steady" | "escalated";
|
|
83
83
|
/**
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
84
|
+
* The cadence state machine, pure so the thresholds are testable without
|
|
85
|
+
* allocating three gigabytes to reach them.
|
|
86
|
+
*/
|
|
87
87
|
export declare function nextMode(current: HeapWatchMode, usedRatio: number, escalateRatio?: number, deescalateRatio?: number): HeapWatchMode;
|
|
88
88
|
/**
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
89
|
+
* Reclaim once the process holds this many MB beyond what it is USING.
|
|
90
|
+
*
|
|
91
|
+
* Live calibration, both readings from the same hub minutes apart: 2491MB
|
|
92
|
+
* stranded before the reclaim, 438MB after. 438MB is this process's irreducible
|
|
93
|
+
* native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
|
|
94
|
+
* collecting against it would buy nothing and cost a stall every minute.
|
|
95
|
+
*
|
|
96
|
+
* ## Why this came down from 1536 (2026-08-17)
|
|
97
|
+
*
|
|
98
|
+
* The trigger was originally set to fire only on an unmistakable strand. It
|
|
99
|
+
* does, and the pass on the record works exactly as designed:
|
|
100
|
+
*
|
|
101
|
+
* [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
|
|
102
|
+
*
|
|
103
|
+
* But the number it was protecting against changed. The host now runs with a
|
|
104
|
+
* 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
|
|
105
|
+
* RSS this process is holding and not using is no longer merely untidy — it is
|
|
106
|
+
* the headroom something else needed, and the OOM is triggered by the PEAK, not
|
|
107
|
+
* by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
|
|
108
|
+
* above the floor before anything happens.
|
|
109
|
+
*
|
|
110
|
+
* 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
|
|
111
|
+
* can never trip it, and every pass that does fire still returns roughly
|
|
112
|
+
* `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
|
|
113
|
+
* not a stall spent collecting noise.
|
|
114
|
+
*/
|
|
115
115
|
export declare const HEAP_RECLAIM_TRIGGER_MB = 1024;
|
|
116
116
|
/**
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
export declare const HEAP_RECLAIM_MIN_INTERVAL_MS =
|
|
117
|
+
* Floor on how often the stop-the-world may run.
|
|
118
|
+
*
|
|
119
|
+
* Ten minutes was chosen to keep the worst case under 0.1% of the router's
|
|
120
|
+
* time. The worst case is not what governs the cost, though: the reclaim only
|
|
121
|
+
* runs when {@link shouldReclaim} says the process is genuinely stranded, and
|
|
122
|
+
* after a pass it is at the ~438MB floor. Live off-heap growth was measured at
|
|
123
|
+
* roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
|
|
124
|
+
* takes on the order of **20 minutes** — well outside any floor considered
|
|
125
|
+
* here. In steady state this constant therefore costs nothing at all.
|
|
126
|
+
*
|
|
127
|
+
* What it does govern is a BURST, which is the only case that matters on a host
|
|
128
|
+
* with no swap: the ten-minute floor let a peak that re-formed in seconds stand
|
|
129
|
+
* for ten minutes. Two minutes bounds that, and even the pathological case of a
|
|
130
|
+
* process that re-strands instantly costs 654ms per 120s — 0.55% of the
|
|
131
|
+
* router's time, against a ~1.5GiB peak it removes each pass.
|
|
132
|
+
*/
|
|
133
|
+
export declare const HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
|
|
134
134
|
/**
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
135
|
+
* Consecutive floor-to-floor passes after which the reclaim is announced as
|
|
136
|
+
* STEADY STATE rather than a rescue.
|
|
137
|
+
*
|
|
138
|
+
* The floor above is documented as a bound on a BURST, on the measured premise
|
|
139
|
+
* that re-stranding the trigger takes ~20 minutes. Measured on the live hub
|
|
140
|
+
* 2026-08-25 it takes under two: 13 passes in 30 minutes, every one of them
|
|
141
|
+
* firing the instant {@link HEAP_RECLAIM_MIN_INTERVAL_MS} elapsed, `took=`
|
|
142
|
+
* 556-695ms each. Nothing in the record said so, because a pass that fires
|
|
143
|
+
* every floor period and a pass that fires once an hour print the same line.
|
|
144
|
+
*
|
|
145
|
+
* Six is 12 minutes of back-to-back passes: long enough that a genuine burst
|
|
146
|
+
* (which subsides) never reaches it, short enough that a process which has
|
|
147
|
+
* quietly made the stop-the-world part of its steady state is named within the
|
|
148
|
+
* quarter-hour.
|
|
149
|
+
*
|
|
150
|
+
* ## Until 2026-08-29 this counter changed nothing, and that was the bug
|
|
151
|
+
*
|
|
152
|
+
* It used to WARN and leave the cadence alone — "the pass is load-bearing and
|
|
153
|
+
* keeps running". Then hub-main's first 25 minutes of telemetry arrived and
|
|
154
|
+
* showed what that costs. See {@link HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS}:
|
|
155
|
+
* the system already knew the difference between a rescue and a steady state
|
|
156
|
+
* and simply refused to act on it.
|
|
157
|
+
*/
|
|
158
158
|
export declare const RECLAIM_STEADY_STATE_PASSES = 6;
|
|
159
159
|
/**
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
export declare const HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS =
|
|
160
|
+
* Floor on the pass once it has PROVEN it is steady state, not a rescue.
|
|
161
|
+
*
|
|
162
|
+
* ## Why `stranded` cannot discriminate on hub-main
|
|
163
|
+
*
|
|
164
|
+
* {@link strandedMb} is `rss − heapUsed − external`: everything the process
|
|
165
|
+
* holds that those two numbers do not name. That lumps together two things with
|
|
166
|
+
* opposite fixes —
|
|
167
|
+
*
|
|
168
|
+
* - V8 pages committed at a sawtooth peak, which `gc(last-resort)` RETURNS;
|
|
169
|
+
* - glibc arena memory already freed and never handed back, which **no V8
|
|
170
|
+
* flavor can reach** (nothing in this repository calls `malloc_trim`).
|
|
171
|
+
*
|
|
172
|
+
* {@link HEAP_RECLAIM_TRIGGER_MB} was calibrated against a hub-main whose strand
|
|
173
|
+
* was mostly the first kind, and a pass then took it back under the trigger. Once
|
|
174
|
+
* the second kind grows past 1024 MB the quantity can never fall back under it,
|
|
175
|
+
* and the gate degenerates. Measured on the live hub 2026-08-29, first 25
|
|
176
|
+
* minutes of hub-main telemetry ever collected:
|
|
177
|
+
*
|
|
178
|
+
* - `stranded` sat at **1031–2313 MB for the entire boot**, i.e.
|
|
179
|
+
* {@link shouldReclaim} never once said no. **What is written as a threshold
|
|
180
|
+
* is, on this process, an unconditional 120-second timer.**
|
|
181
|
+
* - Nine minutes out of nine that contained a reclaim show `loopMax` ≈ the
|
|
182
|
+
* reclaim's own duration (555–869 ms). Every minute without one is
|
|
183
|
+
* 107–435 ms. **The largest recurring event-loop stall on the process that
|
|
184
|
+
* mediates every cross-runner call is the memory patch it applies to itself.**
|
|
185
|
+
* - Each pass freed less than the last (1193 → 462 MB) while the post-pass RSS
|
|
186
|
+
* floor ratcheted **1940 → 3510 MB in fourteen minutes**, with `heapUsed`
|
|
187
|
+
* never moving. It is compacting a heap that is not the problem.
|
|
188
|
+
*
|
|
189
|
+
* ## The fix is not a bigger trigger
|
|
190
|
+
*
|
|
191
|
+
* Raising {@link HEAP_RECLAIM_TRIGGER_MB} would move the number at which the same
|
|
192
|
+
* degeneration happens, and it would silence `hub/recorder` — whose single pass
|
|
193
|
+
* in six hours is the only cheap proof anyone has that the recorder's 1.4 GB is
|
|
194
|
+
* RETAINED (a forced compaction freed 507 MB and V8 still held 1.44 GB live).
|
|
195
|
+
* The discriminator is not the level of the strand; it is **whether the strand
|
|
196
|
+
* ever falls back under the trigger between passes**, which is precisely what
|
|
197
|
+
* {@link RECLAIM_STEADY_STATE_PASSES} already counts and used only to talk about.
|
|
198
|
+
*
|
|
199
|
+
* So the counter becomes load-bearing: once the pass has fired at the floor
|
|
200
|
+
* {@link RECLAIM_STEADY_STATE_PASSES} times running, the trigger has demonstrably
|
|
201
|
+
* stopped discriminating on this process, and the cadence falls back to a period
|
|
202
|
+
* chosen for its COST rather than to a threshold pretending to be a gate.
|
|
203
|
+
*
|
|
204
|
+
* ## Why 600_000, and why that is not a regression
|
|
205
|
+
*
|
|
206
|
+
* It is the value {@link HEAP_RECLAIM_MIN_INTERVAL_MS} held before 2026-08-17,
|
|
207
|
+
* and the reason it came down is recorded there: *"the ten-minute floor let a
|
|
208
|
+
* peak that re-formed in seconds stand for ten minutes"*. That argument is about
|
|
209
|
+
* a BURST — and a burst subsides, which resets the run and restores the 120 s
|
|
210
|
+
* floor. It never applied to a process in proven steady state. So the burst
|
|
211
|
+
* bound is untouched and the steady state gets the period the burst argument was
|
|
212
|
+
* never about.
|
|
213
|
+
*
|
|
214
|
+
* ## What each process does after this change
|
|
215
|
+
*
|
|
216
|
+
* - **hub-main**: first six passes at 120 s, unchanged. From the seventh
|
|
217
|
+
* consecutive floor-to-floor pass, every 600 s. Stop-the-world cost on the
|
|
218
|
+
* router falls from ~0.58% of wall clock to ~0.12%. The ArrayBuffer probe
|
|
219
|
+
* survives at six passes an hour — the finding it produced needed ten passes
|
|
220
|
+
* total, not ten in twenty-five minutes.
|
|
221
|
+
* - **`hub/recorder`**: zero change, by construction. `passesAtFloor` only
|
|
222
|
+
* increments when a pass fires within `minIntervalMs * 2` of the previous
|
|
223
|
+
* one; the recorder's strand takes six hours to re-form, so the counter is
|
|
224
|
+
* reset to 1 on every pass and can never reach six. The 507 MB probe is
|
|
225
|
+
* intact.
|
|
226
|
+
* - **Anything that recovers**: the run resets the moment the strand falls back
|
|
227
|
+
* under the trigger, and the 120 s floor returns with it.
|
|
228
|
+
*/
|
|
229
|
+
export declare const HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS = 6e5;
|
|
230
230
|
/**
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
231
|
+
* The floor in force for the NEXT pass, given how many have fired at the floor.
|
|
232
|
+
*
|
|
233
|
+
* Pure, so the cadence change is testable without stranding a gigabyte to reach
|
|
234
|
+
* it — and so the one place that decides it can be read on its own.
|
|
235
|
+
*/
|
|
236
236
|
export declare function reclaimIntervalMs(passesAtFloor: number, minIntervalMs?: number, steadyStateIntervalMs?: number, steadyStatePasses?: number): number;
|
|
237
237
|
/**
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
238
|
+
* Event-loop delay over the window since the previous read, in milliseconds.
|
|
239
|
+
*
|
|
240
|
+
* Why the heartbeat carries this at all: hub-main's event loop is the cluster's
|
|
241
|
+
* only queue (D181 — every settings read from every runner and every agent,
|
|
242
|
+
* every UDS handshake and every cross-runner cap call is served on it, and a
|
|
243
|
+
* live CPU profile of a boot put `(idle)` at 0.1%). The reclaim pass is a
|
|
244
|
+
* stop-the-world ON that loop, and `took=` prices the PASS without ever pricing
|
|
245
|
+
* what it delays. 590ms against a p99 of 4ms is a visible stall; the same 590ms
|
|
246
|
+
* against a p99 of 900ms is noise inside a queue that was already saturated.
|
|
247
|
+
* Those are opposite conclusions, and until now the log could not tell them
|
|
248
|
+
* apart.
|
|
249
|
+
*/
|
|
250
250
|
export interface LoopDelaySample {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
251
|
+
readonly p50Ms: number;
|
|
252
|
+
readonly p99Ms: number;
|
|
253
|
+
readonly maxMs: number;
|
|
254
254
|
}
|
|
255
255
|
/** What a loop with nothing recorded yet reports. */
|
|
256
256
|
export declare const ZERO_LOOP_DELAY: LoopDelaySample;
|
|
257
257
|
export interface LoopDelayMeter {
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
258
|
+
/** Percentiles over the window since the previous read. Reading RESETS it. */
|
|
259
|
+
readonly read: () => LoopDelaySample;
|
|
260
|
+
readonly stop: () => void;
|
|
261
261
|
}
|
|
262
262
|
/**
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
263
|
+
* A `perf_hooks` interval histogram, read once per LOGGED line.
|
|
264
|
+
*
|
|
265
|
+
* The histogram samples in libuv at `resolutionMs` and costs nothing to keep
|
|
266
|
+
* enabled; the expensive part would be reading it on every 2s probe, which
|
|
267
|
+
* would also destroy the window the line claims to describe (probe and log
|
|
268
|
+
* cadences are deliberately different — see {@link startHeapWatch}).
|
|
269
|
+
*
|
|
270
|
+
* An EMPTY histogram is the trap: Node answers `min = 9223372036854776000` and
|
|
271
|
+
* `percentile()` throws on some versions, so a first heartbeat that read it
|
|
272
|
+
* naively would print `loopMax=9223372036854ms` and be read as a catastrophe on
|
|
273
|
+
* every boot. `count === 0` short-circuits to {@link ZERO_LOOP_DELAY}.
|
|
274
|
+
*
|
|
275
|
+
* ## `loopP50=20ms` was the SAMPLER, not the loop (fixed 2026-08-29)
|
|
276
|
+
*
|
|
277
|
+
* `monitorEventLoopDelay` records the whole INTERVAL between its own timer
|
|
278
|
+
* firings, not the excess over the interval it asked for. So on a loop with
|
|
279
|
+
* nothing wrong the histogram reads `resolution`, and every `[mem]` line this
|
|
280
|
+
* process has ever emitted said `loopP50=20ms` — which was read, repeatedly in
|
|
281
|
+
* one session, as "hub-main is chronically 20 ms behind". It is not; 20 was the
|
|
282
|
+
* default `resolutionMs` of this very function.
|
|
283
|
+
*
|
|
284
|
+
* The delay is `interval − resolution`, and that is not a fudge factor — it is
|
|
285
|
+
* the only reading the four resolutions agree on. Measured here, one 120 ms
|
|
286
|
+
* block observed by four histograms at once:
|
|
287
|
+
*
|
|
288
|
+
* | resolution | raw p50 | raw max | max − resolution |
|
|
289
|
+
* | --- | --- | --- | --- |
|
|
290
|
+
* | 20 ms | 20.40 ms | 137.63 ms | 117.6 ms |
|
|
291
|
+
* | 10 ms | 10.22 ms | 123.60 ms | 113.6 ms |
|
|
292
|
+
* | 5 ms | 5.11 ms | 123.60 ms | 118.6 ms |
|
|
293
|
+
* | 1 ms | 1.27 ms | 121.04 ms | 120.0 ms |
|
|
294
|
+
*
|
|
295
|
+
* Raw, the four disagree by 16 ms about the same event and none of them reports
|
|
296
|
+
* the block. Corrected, they agree within 7 ms and all four name it. The
|
|
297
|
+
* subtraction costs nothing and needs no faster sampler — a 1 ms resolution
|
|
298
|
+
* would be 1 000 extra timer wakeups per second on the process that already
|
|
299
|
+
* serves ~44 000 socket operations per second, to buy 2 ms of accuracy on a
|
|
300
|
+
* quantity nobody reads below 100 ms.
|
|
301
|
+
*
|
|
302
|
+
* **Readings from before this change are 20 ms high.** `loopP99=21–28ms` in the
|
|
303
|
+
* 2026-08-29 census is really 1–8 ms; `loopMax=2804ms` is really 2784 ms. Do not
|
|
304
|
+
* compare a pre-fix series with a post-fix one without shifting it.
|
|
305
|
+
*
|
|
306
|
+
* Returns undefined if the platform declines, so the caller keeps a heartbeat
|
|
307
|
+
* either way.
|
|
308
|
+
*/
|
|
309
309
|
export declare function createLoopDelayMeter(resolutionMs?: number): LoopDelayMeter | undefined;
|
|
310
310
|
/** Whether anyone CHOSE this process's old-space ceiling, or V8 derived one. */
|
|
311
|
-
export type HeapCeilingOrigin =
|
|
311
|
+
export type HeapCeilingOrigin = "explicit" | "v8-default";
|
|
312
312
|
/**
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
313
|
+
* Read the origin of the old-space ceiling off node's own options.
|
|
314
|
+
*
|
|
315
|
+
* hub-main is the only process in the cluster whose ceiling nobody chose. Every
|
|
316
|
+
* forked runner is given V8 heap flags by `process-service.ts`; hub-main gets
|
|
317
|
+
* `--max-old-space-size` from `docker-entrypoint.sh` ONLY when the operator sets
|
|
318
|
+
* `CAMSTACK_HUB_HEAP_MB`, and on the live container it is unset — verified
|
|
319
|
+
* 2026-08-25 by reading pid 7's argv, a bare `node .../starter.js`. So the
|
|
320
|
+
* ceiling in force is whatever V8 derives from the host (4288MB there), and the
|
|
321
|
+
* only record of that was a comment inside an image nobody rebuilds to read.
|
|
322
|
+
*
|
|
323
|
+
* A fact about the running process belongs in the running process's log, and
|
|
324
|
+
* `execArgv` answers it for free at boot.
|
|
325
|
+
*/
|
|
326
326
|
export declare function heapCeilingOrigin(execArgv?: readonly string[]): HeapCeilingOrigin;
|
|
327
327
|
export interface HeapSample {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
328
|
+
readonly rssMb: number;
|
|
329
|
+
readonly heapUsedMb: number;
|
|
330
|
+
readonly heapTotalMb: number;
|
|
331
|
+
readonly heapLimitMb: number;
|
|
332
|
+
readonly externalMb: number;
|
|
333
|
+
readonly arrayBuffersMb: number;
|
|
334
|
+
/** heapUsed / heapLimit, 0-1. */
|
|
335
|
+
readonly usedRatio: number;
|
|
336
|
+
/** True once the heap is close enough to the ceiling to be worth shouting. */
|
|
337
|
+
readonly nearLimit: boolean;
|
|
338
338
|
}
|
|
339
339
|
/**
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
340
|
+
* Build a sample from raw readings. Pure so the thresholds are testable without
|
|
341
|
+
* allocating a gigabyte to reach them.
|
|
342
|
+
*/
|
|
343
343
|
export declare function buildHeapSample(mem: NodeJS.MemoryUsage, heapLimitBytes: number, warnRatio?: number): HeapSample;
|
|
344
344
|
/** Minimal sink so this module stays usable before any logger exists — the
|
|
345
|
-
|
|
345
|
+
* starter runs before the kernel's logging is wired. */
|
|
346
346
|
export interface HeapWatchSink {
|
|
347
|
-
|
|
348
|
-
|
|
347
|
+
readonly info: (line: string) => void;
|
|
348
|
+
readonly warn: (line: string) => void;
|
|
349
349
|
}
|
|
350
350
|
/**
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
351
|
+
* A sink that starts on stdout and is repointed at the real logger the moment
|
|
352
|
+
* one exists.
|
|
353
|
+
*
|
|
354
|
+
* ## The gap this closes
|
|
355
|
+
*
|
|
356
|
+
* hub-main has had this heartbeat since 2026-08-02, started from
|
|
357
|
+
* `server/backend/src/main.ts` as the very first statement of `bootstrap()`.
|
|
358
|
+
* It was never wired to the LOG MANAGER, only to {@link consoleSink} — i.e. to
|
|
359
|
+
* `console.log`, which is not a destination. Every forked runner passes a sink
|
|
360
|
+
* backed by its `IScopedLogger` (`kernel/moleculer/addon-runner.ts:729`), so a
|
|
361
|
+
* runner's `[mem]` line fans out to the buffer, the console destination, the
|
|
362
|
+
* rotated file AND the Loki destination; hub-main's went to a file descriptor
|
|
363
|
+
* and nowhere else. Result: 42 hours of Loki with a `[mem]` line for every
|
|
364
|
+
* runner in the cluster and not one for the largest process in it, whose growth
|
|
365
|
+
* to ~4.8 GB was consequently never trended, only inferred.
|
|
366
|
+
*
|
|
367
|
+
* ## Why late binding rather than moving the call
|
|
368
|
+
*
|
|
369
|
+
* The heartbeat cannot simply be started later. Its whole point is that a
|
|
370
|
+
* process which dies early still leaves samples — the fatal that motivated it
|
|
371
|
+
* landed nine seconds after a viewer connected — and `LoggingService` does not
|
|
372
|
+
* exist until the Fastify app has been booted, hundreds of lines into
|
|
373
|
+
* `bootstrap()`. So the sink is handed over instead of the start being delayed:
|
|
374
|
+
* boot lines keep going to stdout exactly as they do today, and every line from
|
|
375
|
+
* the handover on enters the log manager.
|
|
376
|
+
*
|
|
377
|
+
* Nothing before the handover is lost from Loki by accident, only by design:
|
|
378
|
+
* `LogManager.addDestination` replays its ring buffer into a destination that
|
|
379
|
+
* registers late, so the heartbeat lines emitted between the handover and the
|
|
380
|
+
* Loki addon's own registration still ship.
|
|
381
|
+
*/
|
|
382
382
|
export interface DeferredHeapWatchSink {
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
383
|
+
/** Hand this to {@link startHeapWatch}. */
|
|
384
|
+
readonly sink: HeapWatchSink;
|
|
385
|
+
/** Point every SUBSEQUENT line at `target`. Last call wins. */
|
|
386
|
+
readonly attach: (target: HeapWatchSink) => void;
|
|
387
387
|
}
|
|
388
388
|
/**
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
389
|
+
* Build a {@link DeferredHeapWatchSink}.
|
|
390
|
+
*
|
|
391
|
+
* The handover announces itself, once, through the NEW target: that line is the
|
|
392
|
+
* only in-band evidence that hub-main's telemetry actually reached the log
|
|
393
|
+
* manager on a given boot. Its absence from Loki means the wiring is broken —
|
|
394
|
+
* which is precisely the state this module was in, undetected, for 42 hours.
|
|
395
|
+
*/
|
|
396
396
|
export declare function createDeferredHeapWatchSink(label?: string, fallback?: HeapWatchSink): DeferredHeapWatchSink;
|
|
397
397
|
/**
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
398
|
+
* RSS the process is holding but not using.
|
|
399
|
+
*
|
|
400
|
+
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
401
|
+
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
402
|
+
* and they are precisely what this number exists to see.
|
|
403
|
+
*
|
|
404
|
+
* ## What this number can and CANNOT be asked
|
|
405
|
+
*
|
|
406
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
407
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
408
|
+
* gate that should ever be built on it.
|
|
409
|
+
*
|
|
410
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
411
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
412
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
413
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
414
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
415
|
+
* quantity cannot represent the event.
|
|
416
|
+
*
|
|
417
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
418
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
419
|
+
*/
|
|
420
420
|
export declare function strandedMb(s: HeapSample): number;
|
|
421
421
|
/** Whether it is worth paying a stop-the-world to get those pages back. */
|
|
422
422
|
export declare function shouldReclaim(s: HeapSample, triggerMb?: number): boolean;
|
|
423
423
|
/**
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
export type RssGrowthShape =
|
|
424
|
+
* Which of the three MEASURED growth shapes dominates a process's footprint.
|
|
425
|
+
*
|
|
426
|
+
* These are not categories invented for a log line. They are what `smaps` found
|
|
427
|
+
* on the three live processes on 2026-08-27, and each one has a different lever:
|
|
428
|
+
*
|
|
429
|
+
* | process | evidence | shape |
|
|
430
|
+
* | --- | --- | --- |
|
|
431
|
+
* | `hub/recorder` | 1509.8MB in V8's 256KB page ranges, 53.9MB `brk`, `external` 8MB | `v8-heap` |
|
|
432
|
+
* | `hub/stream-broker` | 242.6MB V8 pages but **1159.1MB `brk`**, `arrayBuffers` 829MB | `external` |
|
|
433
|
+
* | `hub-main` | 1295.1MB V8 pages, 566.7MB `brk`, **2.7GB across 13 anonymous mappings >=64MB** | `native-residue` |
|
|
434
|
+
*
|
|
435
|
+
* - `v8-heap` — live JS retainers. `--max-old-space-size` is the bound, and the
|
|
436
|
+
* fix is the retained set (the recorder's ~7.1M `SegmentRow`s).
|
|
437
|
+
* - `external` — Buffers and native `external`. **No V8 flag bounds any of
|
|
438
|
+
* it**; `hub/pipeline-analytics` reached 5735MB with `--max-old-space-size=1024`
|
|
439
|
+
* in force the whole time.
|
|
440
|
+
* - `native-residue` — memory already freed that the allocator has not returned
|
|
441
|
+
* to the kernel. This is what the reclaim pass exists for, and the only shape
|
|
442
|
+
* where `MALLOC_ARENA_MAX` / `VIPS_CONCURRENCY` are the lever — both of which
|
|
443
|
+
* are ALREADY applied here, so a `native-residue` line means that lever is
|
|
444
|
+
* spent and the answer is upstream.
|
|
445
|
+
*
|
|
446
|
+
* Reporting only "it is big" leaves the operator to guess between three
|
|
447
|
+
* different fixes, two of which cannot work.
|
|
448
|
+
*/
|
|
449
|
+
export type RssGrowthShape = "v8-heap" | "external" | "native-residue";
|
|
450
450
|
/** Where a process's footprint stands against the budget its role declared. */
|
|
451
|
-
export type RssBudgetState =
|
|
451
|
+
export type RssBudgetState = "within" | "over";
|
|
452
452
|
/** The footprint, split into the three parts that imply different fixes. */
|
|
453
453
|
export interface RssBreakdown {
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
454
|
+
readonly rssMb: number;
|
|
455
|
+
readonly budgetMb: number;
|
|
456
|
+
/** How far past the budget. 0 when within — "how far over" has no meaning below it. */
|
|
457
|
+
readonly overMb: number;
|
|
458
|
+
readonly heapUsedMb: number;
|
|
459
|
+
readonly externalMb: number;
|
|
460
|
+
/** `rss − heapUsed − external`: native the other two numbers do not name. */
|
|
461
|
+
readonly nativeResidueMb: number;
|
|
462
|
+
readonly arrayBuffersMb: number;
|
|
463
|
+
readonly shape: RssGrowthShape;
|
|
464
464
|
}
|
|
465
465
|
/**
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
466
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
467
|
+
*
|
|
468
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
469
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
470
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
471
|
+
*/
|
|
472
472
|
export declare function describeRss(s: HeapSample, budgetMb: number): RssBreakdown;
|
|
473
473
|
/**
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
export declare const RSS_BUDGET_RELEASE_RATIO =
|
|
474
|
+
* Release the alarm below this share of the budget.
|
|
475
|
+
*
|
|
476
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
477
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
478
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
479
|
+
*/
|
|
480
|
+
export declare const RSS_BUDGET_RELEASE_RATIO = .9;
|
|
481
481
|
/**
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
export declare const RSS_BUDGET_REANNOUNCE_MIN_MS =
|
|
482
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
483
|
+
*
|
|
484
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
485
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
486
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
487
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
488
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
489
|
+
*
|
|
490
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
491
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
492
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
493
|
+
*/
|
|
494
|
+
export declare const RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
495
495
|
/**
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
496
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
497
|
+
* allocating five gigabytes to reach them.
|
|
498
|
+
*/
|
|
499
499
|
export declare function nextRssBudgetState(current: RssBudgetState, rssMb: number, budgetMb: number, releaseRatio?: number): RssBudgetState;
|
|
500
500
|
/** A full collection that also asks V8 to hand the pages back to the OS. */
|
|
501
501
|
export type HeapReclaimer = () => void;
|
|
502
502
|
/**
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
503
|
+
* Build the reclaimer without requiring `--expose-gc` on argv.
|
|
504
|
+
*
|
|
505
|
+
* The flag is turned on, the function captured, and the flag turned straight
|
|
506
|
+
* back off — a global `gc` left reachable by application code is a footgun, and
|
|
507
|
+
* requiring an argv change would mean this cannot ship without a new image.
|
|
508
|
+
* Verified against the live hub before being written: 266ms, RSS 1453 -> 1237.
|
|
509
|
+
*
|
|
510
|
+
* Returns undefined if V8 declines, so the caller keeps a heartbeat either way.
|
|
511
|
+
*/
|
|
512
512
|
export declare function createV8Reclaimer(): HeapReclaimer | undefined;
|
|
513
513
|
/** Wiring for the reclaim pass. `readMemory`/`heapLimitBytes` exist so a test
|
|
514
|
-
|
|
514
|
+
* can reach a gigabyte-scale state without allocating one. */
|
|
515
515
|
export interface HeapReclaimOptions {
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
516
|
+
readonly reclaim: HeapReclaimer;
|
|
517
|
+
readonly readMemory?: () => NodeJS.MemoryUsage;
|
|
518
|
+
readonly heapLimitBytes?: number;
|
|
519
|
+
readonly triggerMb?: number;
|
|
520
|
+
readonly minIntervalMs?: number;
|
|
521
|
+
/** Floor once the pass is proven steady state. See
|
|
522
|
+
* {@link HEAP_RECLAIM_STEADY_STATE_INTERVAL_MS}. */
|
|
523
|
+
readonly steadyStateIntervalMs?: number;
|
|
524
|
+
readonly now?: () => number;
|
|
525
|
+
/** Sampling cadence once escalated. See {@link HEAP_WATCH_FAST_INTERVAL_MS}. */
|
|
526
|
+
readonly fastIntervalMs?: number;
|
|
527
|
+
/** Ratio at which the fast cadence begins. See {@link HEAP_WATCH_ESCALATE_RATIO}. */
|
|
528
|
+
readonly escalateRatio?: number;
|
|
529
|
+
/** Ratio at which it releases. See {@link HEAP_WATCH_DEESCALATE_RATIO}. */
|
|
530
|
+
readonly deescalateRatio?: number;
|
|
531
531
|
}
|
|
532
532
|
/**
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
533
|
+
* The RSS budget for ONE process, and where the number came from.
|
|
534
|
+
*
|
|
535
|
+
* ## Why a DECLARED budget, and not a derived one
|
|
536
|
+
*
|
|
537
|
+
* Every bound this system had was derived, and each one is blind to most of a
|
|
538
|
+
* process: `--max-old-space-size` bounds old space only, and `strandedMb`
|
|
539
|
+
* subtracts `external` by construction. `hub/pipeline-analytics` was killed at
|
|
540
|
+
* 11.4GB with a 1024MB old-space ceiling in force and a stranded number that
|
|
541
|
+
* never moved. RSS is the quantity the host's OOM killer actually reads, so RSS
|
|
542
|
+
* is the quantity that needs a ceiling — and RSS cannot be derived from any flag,
|
|
543
|
+
* because most of it is not V8's.
|
|
544
|
+
*
|
|
545
|
+
* ## Why there is no single constant
|
|
546
|
+
*
|
|
547
|
+
* The legitimate footprint differs by an order of magnitude BY ROLE, measured
|
|
548
|
+
* on the same host on the same day:
|
|
549
|
+
*
|
|
550
|
+
* - `hub/recorder` legitimately holds ~1.5GB of live V8 index (~7.1M
|
|
551
|
+
* `SegmentRow`s, ~30 days x 13 cameras x 2 profiles) and declares a 3072MB
|
|
552
|
+
* old space to hold it;
|
|
553
|
+
* - `hub/stream-broker` legitimately holds ~900MB of `arrayBuffers` (47 RTP
|
|
554
|
+
* pre-roll rings) in the glibc heap, with an old space of only 224MB;
|
|
555
|
+
* - `hub/pipeline-analytics` has no legitimate multi-GB working set at all —
|
|
556
|
+
* it died at 11.4GB holding a 536MB heap.
|
|
557
|
+
*
|
|
558
|
+
* One constant would either wave through analytics' 11GB or scream at the
|
|
559
|
+
* recorder's entirely correct 2GB. So the number lives **next to the role that
|
|
560
|
+
* knows it**: an addon declares `execution.rssBudgetMb` in its own manifest,
|
|
561
|
+
* exactly as it already declares `execution.maxOldSpaceMb`, and the runner takes
|
|
562
|
+
* the MAXIMUM any co-located addon declares (D2/D29 — a runner is respawned as a
|
|
563
|
+
* whole, so co-location must never silence the hungriest member).
|
|
564
|
+
*
|
|
565
|
+
* hub-main has no manifest, and its entrypoint is the LEAST deployable surface
|
|
566
|
+
* in this system (the Dockerfile's own comment: a backend change is train-bound
|
|
567
|
+
* by construction, and an image build does not ship it). Its budget therefore
|
|
568
|
+
* lives in `@camstack/server`, which `applyServerUpdate` replaces the same day —
|
|
569
|
+
* see {@link HUB_MAIN_RSS_BUDGET_MB}.
|
|
570
|
+
*
|
|
571
|
+
* ## And a process with no declaration is UNWATCHED, out loud
|
|
572
|
+
*
|
|
573
|
+
* `budgetMb: undefined` does not select a default. Defaulting would be the same
|
|
574
|
+
* mistake in a new place: a number nobody measured, applied to a process nobody
|
|
575
|
+
* profiled, producing either silence or noise with equal authority. The process
|
|
576
|
+
* says once, at boot, that nothing is watching it. Most runners are in this
|
|
577
|
+
* state today and that is the honest report — measurements exist for four
|
|
578
|
+
* processes, so four processes carry a budget.
|
|
579
|
+
*/
|
|
580
580
|
export interface RssBudgetOptions {
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
581
|
+
/** MB. `undefined` → this process is declared UNWATCHED and says so at boot. */
|
|
582
|
+
readonly budgetMb: number | undefined;
|
|
583
|
+
/** Named in every line, so a budget that is wrong is traceable to its author. */
|
|
584
|
+
readonly source: string;
|
|
585
|
+
/** See {@link RSS_BUDGET_RELEASE_RATIO}. */
|
|
586
|
+
readonly releaseRatio?: number;
|
|
587
|
+
/** See {@link RSS_BUDGET_REANNOUNCE_MIN_MS}. */
|
|
588
|
+
readonly reannounceMinMs?: number;
|
|
589
589
|
}
|
|
590
590
|
/**
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
591
|
+
* The optional per-line probes.
|
|
592
|
+
*
|
|
593
|
+
* An OBJECT rather than two more positional parameters: `startHeapWatch` already
|
|
594
|
+
* takes eight, and a ninth and tenth `undefined` at a call site is how a caller
|
|
595
|
+
* ends up passing the loop meter into the space reader. Every field is optional,
|
|
596
|
+
* so a caller that wants neither passes nothing.
|
|
597
|
+
*/
|
|
598
598
|
export interface HeapWatchProbes {
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
599
|
+
/** Per-V8-space breakdown. Defaults to the real reader. */
|
|
600
|
+
readonly readSpaces?: HeapSpaceReader;
|
|
601
|
+
/**
|
|
602
|
+
* UDS + cross-node event fan-out. No default: a forked runner has no child
|
|
603
|
+
* registry, and printing zeros for a plane that does not exist is worse than
|
|
604
|
+
* printing nothing. Only hub-main and an agent's main pass one.
|
|
605
|
+
*/
|
|
606
|
+
readonly eventPlane?: EventPlaneReader;
|
|
607
|
+
/**
|
|
608
|
+
* Per-peer UDS socket traffic, split by frame kind. No default, for the same
|
|
609
|
+
* reason as {@link eventPlane}: a forked runner has no child registry, and the
|
|
610
|
+
* plane it does have (its single channel to its parent) is already reported
|
|
611
|
+
* from the parent's side. Only hub-main and an agent's main pass one.
|
|
612
|
+
*/
|
|
613
|
+
readonly socketPlane?: SocketPlaneReader;
|
|
614
|
+
/**
|
|
615
|
+
* The write queue of every Moleculer TCP socket to a peer node. No default:
|
|
616
|
+
* only a process with a broker on a real transporter has one. Those sockets
|
|
617
|
+
* are UNREF'D by Moleculer, so a sum over `_getActiveHandles()` never listed
|
|
618
|
+
* them — the hole in the 2026-09-10 exclusion of "every socket queue"
|
|
619
|
+
* (D446). Read on every probe, judged with the UDS queues below.
|
|
620
|
+
*/
|
|
621
|
+
readonly meshQueue?: MeshQueueReader;
|
|
622
|
+
/**
|
|
623
|
+
* Thresholds for the outbound-queue gate. The gate itself is created
|
|
624
|
+
* whenever {@link socketPlane} or {@link meshQueue} is given; this only
|
|
625
|
+
* tunes it (tests, or a process with a different notion of "held").
|
|
626
|
+
*/
|
|
627
|
+
readonly outboundQueue?: OutboundQueueGateOptions;
|
|
628
|
+
/**
|
|
629
|
+
* The ArrayBuffer census that ARMS ITSELF. No default: it costs a forced
|
|
630
|
+
* full GC when it fires, and only hub-main has an episode on record (two
|
|
631
|
+
* silent linear climbs to 1.3 GB and 5 GB on 2026-09-10, each released in
|
|
632
|
+
* one instant with no line, the second two hours before the OOM). A runner
|
|
633
|
+
* that legitimately holds gigabytes of ArrayBuffers (`hub/stream-broker`,
|
|
634
|
+
* ~900 MB of RTP pre-roll rings) would fire it forever. See
|
|
635
|
+
* `array-buffer-census.ts` for the gate, the cost and what it is NOT.
|
|
636
|
+
*/
|
|
637
|
+
readonly arrayBufferCensus?: ArrayBufferCensusProbe;
|
|
638
|
+
/**
|
|
639
|
+
* How the GC census observes collections. Defaults to the real Node
|
|
640
|
+
* observer, for EVERY process — the rate is half of what a GC costs and the
|
|
641
|
+
* line has never carried it (see `gc-census.ts`). A spec passes its own so a
|
|
642
|
+
* sequence of collections can be posed; nothing in production passes one.
|
|
643
|
+
*/
|
|
644
|
+
readonly gcObserver?: NodeGcObserver;
|
|
645
|
+
/**
|
|
646
|
+
* How this process reads its own CPU. Defaults to the real
|
|
647
|
+
* `process.cpuUsage()` and the real `/proc/self/task`, for EVERY process —
|
|
648
|
+
* `docker stats` reports the cgroup, and nothing on any line said what share
|
|
649
|
+
* of it this process was (see `process-cpu-census.ts`). A spec passes its own
|
|
650
|
+
* so a cost can be posed; nothing in production passes one.
|
|
651
|
+
*/
|
|
652
|
+
readonly processCpu?: ProcessCpuProbe;
|
|
653
653
|
}
|
|
654
654
|
/** The two readings `process-cpu-census.ts` needs, injectable for a spec. */
|
|
655
655
|
export interface ProcessCpuProbe {
|
|
656
|
-
|
|
657
|
-
|
|
656
|
+
readonly cpuUsageUs: () => number;
|
|
657
|
+
readonly readTasks: () => readonly TaskTicks[] | null;
|
|
658
658
|
}
|
|
659
659
|
/** How the census is wired: the census itself plus the gate's thresholds. */
|
|
660
660
|
export interface ArrayBufferCensusProbe extends ArrayBufferCensusGateOptions {
|
|
661
|
-
|
|
661
|
+
readonly census: ArrayBufferCensus;
|
|
662
662
|
}
|
|
663
663
|
/**
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
664
|
+
* Every outbound queue the heartbeat can reach, as the gate wants them: the
|
|
665
|
+
* UDS children (only those whose channel keeps a reading — an unmeasured
|
|
666
|
+
* peer is not a zero) and the mesh writer sockets, each prefixed with its
|
|
667
|
+
* plane.
|
|
668
|
+
*/
|
|
669
669
|
export declare function outboundQueueReadings(uds: SocketPlaneCounters | undefined, mesh: readonly MeshPeerQueue[] | undefined): readonly OutboundQueueReading[];
|
|
670
670
|
/**
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
671
|
+
* Start the heartbeat. Returns a stop function.
|
|
672
|
+
*
|
|
673
|
+
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
674
|
+
* a diagnostic that can crash the process it watches is worse than none.
|
|
675
|
+
*/
|
|
676
676
|
export declare function startHeapWatch(label?: string, sink?: HeapWatchSink, intervalMs?: number, reclaimOptions?: HeapReclaimOptions, loopDelay?: LoopDelayMeter | undefined, execArgv?: readonly string[], announceCeilingOrigin?: boolean, rssBudget?: RssBudgetOptions, probes?: HeapWatchProbes): () => void;
|
|
677
677
|
/**
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
export declare const RUNNER_HEAP_WATCH_INTERVAL_MS =
|
|
678
|
+
* Steady heartbeat period for a HEAVY addon runner.
|
|
679
|
+
*
|
|
680
|
+
* hub-main logs once a minute because it is ONE process. The heavy runners are
|
|
681
|
+
* ~7 per node sharing the same hub log stream; at 60s each they would put 7
|
|
682
|
+
* routine lines a minute in it forever, which is how a log stops being read.
|
|
683
|
+
* Five minutes keeps the per-runner record (a leak is still a visible climb
|
|
684
|
+
* across samples) at ~1.4 lines/min across the class.
|
|
685
|
+
*
|
|
686
|
+
* This slows LOGGING only. The probe keeps running at
|
|
687
|
+
* {@link HEAP_WATCH_FAST_INTERVAL_MS} (probe and log cadences are separate —
|
|
688
|
+
* see the comment inside {@link startHeapWatch}), so the reclaim gate and the
|
|
689
|
+
* escalated burst-sampling react exactly as fast as hub-main's.
|
|
690
|
+
*/
|
|
691
|
+
export declare const RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
692
692
|
/** Wiring for {@link startRunnerHeapWatch}. */
|
|
693
693
|
export interface RunnerHeapWatchOptions {
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
694
|
+
/**
|
|
695
|
+
* `CAMSTACK_RUNNER_HEAP_PROFILE` as the parent passed it. Anything but
|
|
696
|
+
* `'heavy'` (including unset — the un-annotated default is light) starts
|
|
697
|
+
* nothing: light runners are small, already carry V8 heap flags, and a
|
|
698
|
+
* quiet runner must never pay a stop-the-world.
|
|
699
|
+
*/
|
|
700
|
+
readonly heapProfile: string | undefined;
|
|
701
|
+
/** Log label — the runner's nodeId, so `[mem]` lines group per runner. */
|
|
702
|
+
readonly label: string;
|
|
703
|
+
readonly sink: HeapWatchSink;
|
|
704
|
+
/**
|
|
705
|
+
* `CAMSTACK_RUNNER_HEAP_RECLAIM` as the environment carries it. `'off'`
|
|
706
|
+
* keeps the heartbeat and disables only the reclaim pass — the operator
|
|
707
|
+
* kill-switch for the media path, usable without a rebuild.
|
|
708
|
+
*/
|
|
709
|
+
readonly reclaimSwitch?: string;
|
|
710
|
+
/** Heartbeat period override. Defaults to {@link RUNNER_HEAP_WATCH_INTERVAL_MS}. */
|
|
711
|
+
readonly intervalMs?: number;
|
|
712
|
+
/** Test injection. Production omits it and gets {@link createV8Reclaimer}. */
|
|
713
|
+
readonly reclaimOptions?: HeapReclaimOptions;
|
|
714
|
+
/**
|
|
715
|
+
* `CAMSTACK_RUNNER_RSS_BUDGET_MB` as the parent passed it — the MAXIMUM
|
|
716
|
+
* `execution.rssBudgetMb` any co-located addon declares, resolved from the
|
|
717
|
+
* manifests in `process-service.ts`. Absent means no co-located addon declared
|
|
718
|
+
* one, and the runner reports itself UNWATCHED. See {@link RssBudgetOptions}.
|
|
719
|
+
*/
|
|
720
|
+
readonly rssBudgetMb?: string;
|
|
721
721
|
}
|
|
722
722
|
/**
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
723
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
724
|
+
*
|
|
725
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
726
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
727
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
728
|
+
*/
|
|
729
729
|
export declare function parseRssBudgetMb(raw: string | undefined): number | undefined;
|
|
730
730
|
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
731
731
|
export declare const RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
732
732
|
/** Operator override for hub-main's budget, applied without a release. */
|
|
733
733
|
export declare const HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
734
734
|
/**
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
735
|
+
* hub-main's declared RSS budget, in MB.
|
|
736
|
+
*
|
|
737
|
+
* ## Why the number lives HERE and not in the image
|
|
738
|
+
*
|
|
739
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
740
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
741
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
742
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
743
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
744
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
745
|
+
* A budget that cannot ship is not a budget.
|
|
746
|
+
*
|
|
747
|
+
* ## Why 4096
|
|
748
|
+
*
|
|
749
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
750
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
751
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
752
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
753
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
754
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
755
|
+
*
|
|
756
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
757
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
758
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
759
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
760
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
761
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
762
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
763
|
+
* just above today's RSS would be a rubber stamp.
|
|
764
|
+
*
|
|
765
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
766
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
767
|
+
* instead of being absorbed by a threshold nobody set.
|
|
768
|
+
*/
|
|
769
769
|
export declare const HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
770
770
|
/** hub-main's budget: the declared number, or the operator's override. */
|
|
771
771
|
export declare function hubMainRssBudget(env?: NodeJS.ProcessEnv): RssBudgetOptions;
|
|
772
772
|
/**
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
773
|
+
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
774
|
+
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
775
|
+
*
|
|
776
|
+
* ## Why heavy runners get hub-main's mechanism
|
|
777
|
+
*
|
|
778
|
+
* Because a heap ceiling cannot reach what actually strands in them. Every
|
|
779
|
+
* media-path runner now carries an old-space ceiling sized from its own
|
|
780
|
+
* measured peak (`addon-pipeline` manifest + its `manifest-heap-ceilings` spec),
|
|
781
|
+
* and that changes nothing here: what a heavy runner strands is NATIVE —
|
|
782
|
+
* `external`, `arrayBuffers`, glibc arena free lists — none of which
|
|
783
|
+
* `--max-old-space-size` bounds. Measured 2026-08-25, `hub/pipeline-analytics`
|
|
784
|
+
* reported `stranded=2552MB` while holding an 81MB V8 heap under a 1024MB
|
|
785
|
+
* ceiling that was in force the whole time. hub-main had exactly this shape and
|
|
786
|
+
* its reclaimer is proven live — 11 passes in 4h, 375–1652MB returned per pass
|
|
787
|
+
* at 550–680ms each. Same mechanism, same `shouldReclaim` stranded-over-trigger
|
|
788
|
+
* gate; never a second implementation.
|
|
789
|
+
*
|
|
790
|
+
* ## The stall, priced for the runners that get it
|
|
791
|
+
*
|
|
792
|
+
* A pass is a sub-second stop-the-world at most once per
|
|
793
|
+
* {@link HEAP_RECLAIM_MIN_INTERVAL_MS}, and only while >1GiB is stranded. For
|
|
794
|
+
* the recorder — finalizing a segment every ~300ms across 34 profile-writers —
|
|
795
|
+
* the pause delays WATCHER work, not video: ffmpeg children keep writing to
|
|
796
|
+
* disk independently, the playlist tail is the durable queue, and the next
|
|
797
|
+
* tick drains what accrued. For stream-broker a pass can hitch live frames
|
|
798
|
+
* once per qualifying window; after a pass the runner sits at its floor and
|
|
799
|
+
* re-stranding the trigger takes tens of minutes, so in practice the class
|
|
800
|
+
* pays a few passes per hour against the multi-GiB peak that was OOM-killing
|
|
801
|
+
* a neighbouring VM on a host with no swap.
|
|
802
|
+
*/
|
|
803
803
|
export declare function startRunnerHeapWatch(options: RunnerHeapWatchOptions): (() => void) | undefined;
|