@camstack/addon-pipeline 1.2.93 → 1.2.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
  2. package/dist/audio-analyzer/index.js +58 -4
  3. package/dist/audio-analyzer/index.mjs +56 -2
  4. package/dist/detection-pipeline/index.js +194 -6
  5. package/dist/detection-pipeline/index.mjs +191 -3
  6. package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
  7. package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
  8. package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
  9. package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
  10. package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
  11. package/dist/motion-wasm/index.js +2 -2
  12. package/dist/motion-wasm/index.mjs +1 -1
  13. package/dist/pipeline-runner/index.js +195 -14
  14. package/dist/pipeline-runner/index.mjs +194 -13
  15. package/dist/process-memory-B9-UD7Rp.js +78 -0
  16. package/dist/process-memory-DXOB6eHX.mjs +66 -0
  17. package/dist/recorder/index.js +943 -109
  18. package/dist/recorder/index.mjs +942 -108
  19. package/dist/session-decode/decode-worker-child.js +4 -4
  20. package/dist/session-decode/decode-worker-child.mjs +3 -3
  21. package/dist/stream-broker/_stub.js +1 -1
  22. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
  23. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
  24. package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
  25. package/dist/stream-broker/index.js +25 -7
  26. package/dist/stream-broker/index.mjs +25 -7
  27. package/dist/stream-broker/remoteEntry.js +1 -1
  28. package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
  29. package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
  30. package/package.json +13 -7
  31. package/python/inference_pool.py +178 -3
  32. package/python/test_inference_pool_backpressure.py +7 -1
  33. package/python/test_inference_pool_memstats.py +146 -0
  34. package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
  35. package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
  36. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
@@ -1,5 +1,5 @@
1
1
  import { createHash } from "node:crypto";
2
- //#region ../types/dist/event-category-C0lyLd5U.mjs
2
+ //#region ../types/dist/event-category-XfKNtfCc.mjs
3
3
  var EventCategory = /* @__PURE__ */ function(EventCategory) {
4
4
  EventCategory["SystemBoot"] = "system.boot";
5
5
  EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -38,6 +38,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
38
38
  EventCategory["AddonInstalled"] = "addon.installed";
39
39
  EventCategory["AddonUninstalled"] = "addon.uninstalled";
40
40
  EventCategory["AddonCrashed"] = "addon.crashed";
41
+ /**
42
+ * A RUNNER the D6 crash circuit-breaker gave up on — terminal.
43
+ *
44
+ * `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
45
+ * respawned"); this is the one that never resolves itself. Emitted exactly
46
+ * once per trip, by `process-service.ts`, carrying the node, the runner, the
47
+ * addons it hosted and the crash count that tripped the breaker.
48
+ *
49
+ * It exists because a runner marked terminally `failed` used to be SILENT:
50
+ * on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
51
+ * errors, the breaker stopped respawning it (correctly), `/health` kept
52
+ * returning 200, and fleet-wide recording was gone for 3h15 before the
53
+ * operator's phone told him. The Notification Center's system-event intake
54
+ * consumes this category and maps it to the `addon-crash-loop` kind.
55
+ */
56
+ EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
41
57
  EventCategory["AddonError"] = "addon.error";
42
58
  EventCategory["AddonPageReady"] = "addon.page-ready";
43
59
  EventCategory["AddonWidgetReady"] = "addon.widget-ready";
@@ -16071,6 +16087,7 @@ var NcSystemEventKindSchema = _enum([
16071
16087
  "node-offline",
16072
16088
  "node-inference-unavailable",
16073
16089
  "detection-blind",
16090
+ "addon-crash-loop",
16074
16091
  "addon-update-available",
16075
16092
  "server-update-available",
16076
16093
  "alarm-triggered",
@@ -21973,20 +21990,6 @@ var VectorStatsResultSchema = object({
21973
21990
  exact: boolean()
21974
21991
  });
21975
21992
  method(VectorDeclareIndexInputSchema, _void(), { kind: "mutation" }), method(VectorUpsertInputSchema, VectorUpsertResultSchema, { kind: "mutation" }), method(VectorQueryInputSchema, VectorQueryResultSchema), method(VectorGetInputSchema, VectorGetResultSchema), method(VectorDeleteInputSchema, VectorDeleteResultSchema, { kind: "mutation" }), method(VectorDeleteByFilterInputSchema, VectorDeleteResultSchema, { kind: "mutation" }), method(VectorStatsInputSchema, VectorStatsResultSchema);
21976
- /**
21977
- * `videoclips` — the unified, navigable-clip surface for a camera.
21978
- *
21979
- * A device-scoped WRAPPER cap (like `pipeline-analytics`): exactly one active
21980
- * provider per device, substitutable. The DEFAULT provider (registered by
21981
- * `addon-post-analysis`, `defaultActive: true`) composes the analytics event
21982
- * log (markers + thumbnails) with the recorder's `getPlaybackManifest` — a clip
21983
- * is a time-WINDOW over existing footage, never a separate file. A camera that
21984
- * exposes NATIVE onboard clips (Reolink/Hikvision NVR) can later substitute the
21985
- * wrapper provider for its device and serve its own clip catalog + URLs.
21986
- *
21987
- * A `Clip` is purely time-based (subtree-blind): playback resolves segments by
21988
- * temporal overlap, so the API never decides `continuous` vs `events`.
21989
- */
21990
21993
  var ClipSchema = object({
21991
21994
  /** Opaque, provider-namespaced id. The default provider encodes the time
21992
21995
  * window so `getClipPlayback` is self-contained (no event re-query). */
@@ -22012,12 +22015,41 @@ var ClipSchema = object({
22012
22015
  * of the event that owns `kind` (object > motion > audio). Do not extract
22013
22016
  * a keyframe from the recorded segments. Other providers (onboard, HKSV)
22014
22017
  * mint their own stills.
22018
+ *
22019
+ * **VOUCHED, never fabricated.** A provider emits this only for an event it
22020
+ * has CONFIRMED owns at least one media row (its own, or its owning track's).
22021
+ * Absent is meaningful — "this visit has no event still" — never "we did not
22022
+ * look". Stamping it from a URL template made 35% of one camera's clips point
22023
+ * at a 404 (device 3836, 2026-08-17: 64 of 179 clips dead, 63 of them motion).
22024
+ * A read that FAILS drops the claim; it never invents it.
22015
22025
  */
22016
22026
  thumbnail: string().optional(),
22017
- /** Analytics event ids that overlap this visit. Empty on footage-only clips.
22018
- * The default provider's visit grain puts many motion heartbeats on one clip
22019
- * instead of minting one clip per marker. */
22027
+ /**
22028
+ * An instant INSIDE this visit's footage, hole-safe, where a recorded still
22029
+ * can be decoded. Present whenever the visit came from recorded availability;
22030
+ * absent on a per-event padded window (there is no footage to promise).
22031
+ *
22032
+ * This is not a thumbnail and not a second byte path: it is the argument to
22033
+ * the recorder's existing still route. The surface — never the provider —
22034
+ * decides whether to use it. It is the midpoint of the visit's LONGEST
22035
+ * contiguous range, not of the visit: a visit spans its holes by
22036
+ * construction, so a naive midpoint lands in dead air.
22037
+ */
22038
+ stillAtMs: number().optional(),
22039
+ /**
22040
+ * Analytics event ids that overlap this visit — a BOUNDED sample, newest
22041
+ * first within kind (object → motion → audio), capped at
22042
+ * {@link MAX_CLIP_EVENT_IDS}. Empty on footage-only clips.
22043
+ *
22044
+ * Bounded because it is not a payload the surface pages through: one visit on
22045
+ * device 615 carried 2 345 ids, and `eventIds` was 99% of a 153 KB
22046
+ * camera-day. Read {@link eventCount} for the true total.
22047
+ */
22020
22048
  eventIds: array(string()).optional(),
22049
+ /** How many analytics events actually overlap this visit. Differs from
22050
+ * `eventIds.length` exactly when the sample was capped — so a truncated
22051
+ * list is never mistaken for a quiet visit. */
22052
+ eventCount: number().int().nonnegative().optional(),
22021
22053
  /** Intra-visit footage holes (GOP rolls, discarded segments) still shorter
22022
22054
  * than `VISIT_MERGE_GAP_MS`. Playback concatenates around them; the timeline
22023
22055
  * bar keeps showing them via `recording.getAvailability`. */
@@ -22568,7 +22600,14 @@ var SearchResultSchema = object({
22568
22600
  });
22569
22601
  var AutoUpdateSettingsSchema = object({
22570
22602
  channel: ChannelSchema,
22571
- intervalSeconds: number()
22603
+ intervalSeconds: number(),
22604
+ /**
22605
+ * Cadence of the "an update exists" POLLER, in seconds. Independent of
22606
+ * `channel`: the poller runs while auto-apply is `off`, because being told
22607
+ * about a publish and installing it are different decisions. Clamped
22608
+ * server-side to 900 s … 604800 s; defaults to 21600 s (6 h).
22609
+ */
22610
+ updateCheckIntervalSeconds: number()
22572
22611
  });
22573
22612
  var AddonAutoUpdateSchema = ChannelWithInheritSchema;
22574
22613
  var RestartAddonResultSchema = unknown();
@@ -22709,7 +22748,9 @@ method(_void(), array(AddonListItemSchema).readonly()), method(object({
22709
22748
  auth: "admin"
22710
22749
  }), method(_void(), AutoUpdateSettingsSchema, { auth: "admin" }), method(object({
22711
22750
  channel: ChannelSchema,
22712
- intervalSeconds: number().min(300).max(86400).optional()
22751
+ intervalSeconds: number().min(300).max(86400).optional(),
22752
+ /** Availability-poll cadence; see AutoUpdateSettingsSchema. */
22753
+ updateCheckIntervalSeconds: number().min(900).max(604800).optional()
22713
22754
  }), unknown(), {
22714
22755
  kind: "mutation",
22715
22756
  auth: "admin"
@@ -36401,6 +36442,303 @@ function maskUrlCredentials(rawUrl) {
36401
36442
  return rawUrl;
36402
36443
  }
36403
36444
  }
36445
+ var MB = 1024 * 1024;
36446
+ var DEFAULT_POOL_MEMORY_POLICY = {
36447
+ sampleIntervalMs: 6e4,
36448
+ baselineSettleSamples: 5,
36449
+ baselineSampleCount: 3,
36450
+ restartMultiple: 4,
36451
+ floorBytes: 1024 * MB,
36452
+ ceilingBytes: 3072 * MB,
36453
+ cooldownMs: 30 * 6e4,
36454
+ maxRestartsPerWindow: 6,
36455
+ restartWindowMs: 1440 * 6e4
36456
+ };
36457
+ /**
36458
+ * Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
36459
+ * absent values keep the default — an operator typo must never disable the
36460
+ * bound or set it to zero.
36461
+ */
36462
+ function resolvePoolMemoryPolicy(env) {
36463
+ const num = (key, fallback, min) => {
36464
+ const raw = env[key];
36465
+ if (raw === void 0) return fallback;
36466
+ const parsed = Number(raw);
36467
+ return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
36468
+ };
36469
+ const d = DEFAULT_POOL_MEMORY_POLICY;
36470
+ return {
36471
+ sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
36472
+ baselineSettleSamples: d.baselineSettleSamples,
36473
+ baselineSampleCount: d.baselineSampleCount,
36474
+ restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
36475
+ floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
36476
+ ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
36477
+ cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
36478
+ maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
36479
+ restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
36480
+ };
36481
+ }
36482
+ /** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
36483
+ * Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
36484
+ function parseProcStatus(text) {
36485
+ const kb = (label) => {
36486
+ const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
36487
+ return match ? Number(match[1]) * 1024 : null;
36488
+ };
36489
+ const rssBytes = kb("VmRSS");
36490
+ if (rssBytes === null) return null;
36491
+ const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
36492
+ return {
36493
+ rssBytes,
36494
+ vmBytes: kb("VmSize") ?? 0,
36495
+ hwmBytes: kb("VmHWM") ?? 0,
36496
+ swapBytes: kb("VmSwap") ?? 0,
36497
+ threads: threadsMatch ? Number(threadsMatch[1]) : 0
36498
+ };
36499
+ }
36500
+ function initialPoolMemoryState() {
36501
+ return {
36502
+ settleSeen: 0,
36503
+ baselineWindow: [],
36504
+ baselineBytes: null,
36505
+ restartsAt: [],
36506
+ lastRestartAt: null
36507
+ };
36508
+ }
36509
+ function median(values) {
36510
+ const sorted = [...values].sort((a, b) => a - b);
36511
+ return sorted[Math.floor(sorted.length / 2)];
36512
+ }
36513
+ function advanceBaseline(state, rssBytes, policy) {
36514
+ if (state.baselineBytes !== null) return state;
36515
+ if (state.settleSeen < policy.baselineSettleSamples) return {
36516
+ ...state,
36517
+ settleSeen: state.settleSeen + 1
36518
+ };
36519
+ const window = [...state.baselineWindow, rssBytes];
36520
+ if (window.length < policy.baselineSampleCount) return {
36521
+ ...state,
36522
+ baselineWindow: window
36523
+ };
36524
+ return {
36525
+ ...state,
36526
+ baselineWindow: window,
36527
+ baselineBytes: median(window)
36528
+ };
36529
+ }
36530
+ /** The one place a pool's restart threshold is computed. */
36531
+ function poolMemoryThreshold(baselineBytes, policy) {
36532
+ if (baselineBytes === null) return policy.ceilingBytes;
36533
+ return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
36534
+ }
36535
+ function pruneRestarts(restartsAt, nowMs, policy) {
36536
+ return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
36537
+ }
36538
+ /**
36539
+ * Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
36540
+ * The caller performs (and logs) the restart, then commits it via
36541
+ * {@link commitWatchdogRestart}.
36542
+ */
36543
+ function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
36544
+ const next = advanceBaseline(state, rssBytes, policy);
36545
+ const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
36546
+ if (rssBytes <= thresholdBytes) return {
36547
+ state: next,
36548
+ action: next.baselineBytes === null ? "baseline-pending" : "ok",
36549
+ baselineBytes: next.baselineBytes,
36550
+ thresholdBytes
36551
+ };
36552
+ const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
36553
+ const pruned = {
36554
+ ...next,
36555
+ restartsAt: inWindow
36556
+ };
36557
+ if (inWindow.length >= policy.maxRestartsPerWindow) return {
36558
+ state: pruned,
36559
+ action: "exhausted",
36560
+ baselineBytes: next.baselineBytes,
36561
+ thresholdBytes
36562
+ };
36563
+ if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
36564
+ state: pruned,
36565
+ action: "cooldown",
36566
+ baselineBytes: next.baselineBytes,
36567
+ thresholdBytes
36568
+ };
36569
+ return {
36570
+ state: pruned,
36571
+ action: "restart",
36572
+ baselineBytes: next.baselineBytes,
36573
+ thresholdBytes
36574
+ };
36575
+ }
36576
+ /** Record a performed watchdog restart: budget consumed, baseline reset —
36577
+ * the recreated pool is a NEW process and gets a fresh settle window. */
36578
+ function commitWatchdogRestart(state, nowMs) {
36579
+ return {
36580
+ ...initialPoolMemoryState(),
36581
+ restartsAt: [...state.restartsAt, nowMs],
36582
+ lastRestartAt: nowMs
36583
+ };
36584
+ }
36585
+ /** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
36586
+ * kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
36587
+ * restart budget is kept: it bounds the WATCHDOG, not the pool. */
36588
+ function resetPoolBaseline(state) {
36589
+ return {
36590
+ ...initialPoolMemoryState(),
36591
+ restartsAt: state.restartsAt,
36592
+ lastRestartAt: state.lastRestartAt
36593
+ };
36594
+ }
36595
+ /**
36596
+ * Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
36597
+ * several pools at once (e.g. after a model rollout grew every baseline)
36598
+ * would take detection down on every camera simultaneously; one per sweep
36599
+ * means the survivors keep serving while the worst offender reloads, and the
36600
+ * next sweep (one interval later) takes the next one.
36601
+ */
36602
+ function pickRestartCandidate(candidates) {
36603
+ if (candidates.length === 0) return null;
36604
+ let worst = candidates[0];
36605
+ for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
36606
+ return worst.key;
36607
+ }
36608
+ var PoolMemoryWatchdog = class {
36609
+ pools = /* @__PURE__ */ new Map();
36610
+ timer = null;
36611
+ sweeping = false;
36612
+ stopped = false;
36613
+ opts;
36614
+ now;
36615
+ setTimer;
36616
+ clearTimer;
36617
+ constructor(opts) {
36618
+ this.opts = opts;
36619
+ this.now = opts.now ?? (() => Date.now());
36620
+ this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
36621
+ this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
36622
+ }
36623
+ start() {
36624
+ if (this.timer || this.stopped) return;
36625
+ this.arm();
36626
+ }
36627
+ stop() {
36628
+ this.stopped = true;
36629
+ if (this.timer) {
36630
+ this.clearTimer(this.timer);
36631
+ this.timer = null;
36632
+ }
36633
+ }
36634
+ arm() {
36635
+ const timer = this.setTimer(() => {
36636
+ this.sweep().finally(() => {
36637
+ if (!this.stopped) this.arm();
36638
+ });
36639
+ }, this.opts.policy.sampleIntervalMs);
36640
+ if (typeof timer.unref === "function") timer.unref();
36641
+ this.timer = timer;
36642
+ }
36643
+ /** One sweep: sample → log every pool → restart at most one. Public so the
36644
+ * loop is testable without fake global timers. */
36645
+ async sweep() {
36646
+ if (this.sweeping) return;
36647
+ this.sweeping = true;
36648
+ try {
36649
+ await this.doSweep();
36650
+ } catch (err) {
36651
+ this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
36652
+ } finally {
36653
+ this.sweeping = false;
36654
+ }
36655
+ }
36656
+ async doSweep() {
36657
+ const nowMs = this.now();
36658
+ const samples = await this.opts.sample();
36659
+ const candidates = [];
36660
+ const verdictByKey = /* @__PURE__ */ new Map();
36661
+ for (const t of samples) {
36662
+ const tracked = this.pools.get(t.key);
36663
+ let state = tracked?.state ?? initialPoolMemoryState();
36664
+ if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
36665
+ this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
36666
+ poolKey: t.key,
36667
+ previousPids: tracked.pids,
36668
+ pids: t.pids
36669
+ } });
36670
+ state = resetPoolBaseline(state);
36671
+ }
36672
+ const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
36673
+ this.pools.set(t.key, {
36674
+ state: verdict.state,
36675
+ pids: t.pids
36676
+ });
36677
+ verdictByKey.set(t.key, verdict);
36678
+ this.opts.log.info("pool memory", { meta: {
36679
+ poolKey: t.key,
36680
+ pids: t.pids,
36681
+ rssMb: Math.round(t.rssBytes / MB),
36682
+ baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
36683
+ thresholdMb: Math.round(verdict.thresholdBytes / MB),
36684
+ action: verdict.action,
36685
+ ...t.meta
36686
+ } });
36687
+ if (verdict.action === "restart") candidates.push({
36688
+ key: t.key,
36689
+ rssBytes: t.rssBytes,
36690
+ thresholdBytes: verdict.thresholdBytes
36691
+ });
36692
+ else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
36693
+ poolKey: t.key,
36694
+ rssMb: Math.round(t.rssBytes / MB),
36695
+ thresholdMb: Math.round(verdict.thresholdBytes / MB),
36696
+ cooldownMs: this.opts.policy.cooldownMs
36697
+ } });
36698
+ else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
36699
+ poolKey: t.key,
36700
+ rssMb: Math.round(t.rssBytes / MB),
36701
+ thresholdMb: Math.round(verdict.thresholdBytes / MB),
36702
+ maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
36703
+ restartWindowMs: this.opts.policy.restartWindowMs
36704
+ } });
36705
+ }
36706
+ for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
36707
+ const pickedKey = pickRestartCandidate(candidates);
36708
+ if (pickedKey === null) return;
36709
+ const picked = candidates.find((c) => c.key === pickedKey);
36710
+ const verdict = verdictByKey.get(pickedKey);
36711
+ const tracked = this.pools.get(pickedKey);
36712
+ const restartsInWindow = tracked.state.restartsAt.length + 1;
36713
+ this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
36714
+ poolKey: pickedKey,
36715
+ pids: tracked.pids,
36716
+ rssMb: Math.round(picked.rssBytes / MB),
36717
+ thresholdMb: Math.round(picked.thresholdBytes / MB),
36718
+ baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
36719
+ restartsInWindow,
36720
+ deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
36721
+ } });
36722
+ try {
36723
+ await this.opts.restart(pickedKey);
36724
+ this.pools.set(pickedKey, {
36725
+ state: commitWatchdogRestart(tracked.state, this.now()),
36726
+ pids: []
36727
+ });
36728
+ } catch (err) {
36729
+ this.opts.log.error("pool memory watchdog restart failed", { meta: {
36730
+ poolKey: pickedKey,
36731
+ error: err instanceof Error ? err.message : String(err)
36732
+ } });
36733
+ }
36734
+ }
36735
+ };
36736
+ function samePids(a, b) {
36737
+ if (a.length !== b.length) return false;
36738
+ const sortedA = [...a].sort((x, y) => x - y);
36739
+ const sortedB = [...b].sort((x, y) => x - y);
36740
+ return sortedA.every((v, i) => v === sortedB[i]);
36741
+ }
36404
36742
  /**
36405
36743
  * Fixed-capacity ring buffer. When full, push() overwrites the oldest entry.
36406
36744
  * drain() returns up to maxCount items in FIFO order and removes them.
@@ -36738,4 +37076,4 @@ function enumerateInferenceDevices(hw) {
36738
37076
  return out;
36739
37077
  }
36740
37078
  //#endregion
36741
- export { resolveRecordingProfiles as $, createHwAccelCache as A, literal as At, enumerateInferenceDevices as B, RingBuffer as C, selectAssignedProfileSlots as Ct, audioAnalyzerCapability as D, boolean as Dt, audioAnalysisCapability as E, array as Et, deriveDetailCropRect as F, union as Ft, motionDetectionCapability as G, hfModelUrl as H, deriveRecordingMode as I, EventCategory as It, pipelineExecutorCapability as J, pickDetailCropConvention as K, detectionPipelineCapability as L, defaultDeviceFor as M, object as Mt, defineCustomActions as N, record as Nt, batteryCapability as O, discriminatedUnion as Ot, deriveBatteryPresence as P, string as Pt, resolveEgressDecodeHwAccel as Q, egressTranscodeSharingKey as R, RecordingConfigSchema as S, parseProfileBrokerId as St, addonWidgetsSourceCapability as T, _enum as Tt, mapAudioLabelToMacro as U, evaluateZoneRules as V, maskUrlCredentials as W, recordingCapability as X, pipelineRunnerCapability as Y, recordingExportCapability as Z, NativeLeaseSettingsSchema as _, isEvent as _t, COCO_80_LABELS as a, errMsg as at, RATE_CONTROL_TIGHT as b, nodePin as bt, DEFAULT_DETAIL_CROP_CONVENTION as c, buildFfmpegArgs as ct, DEVICE_BACKEND_TO_FORMAT as d, BaseAddon as dt, runtimeDevices as et, EVENT_PAD_MS as f, CAM_PROFILE_ORDER as ft, NativeLeaseAdmissionSchema as g, hydrateSchema as gt, HF_BASE_URL as h, createEvent as ht, BatteryStatusSchema as i, webrtcSessionCapability as it, customAction as j, number as jt, cameraStreamsCapability as k, lazy as kt, DEFAULT_EVENTS_BAND_BUFFER_SEC as l, invocationFromEncodeProfile as lt, ExportRecordSchema as m, DeviceType as mt, AUDIO_BACKEND_CHOICES as n, streamBrokerCapability as nt, COCO_TO_MACRO as o, AUDIO_PRESETS as ot, EncodeProfileSchema as p, DeviceFeature as pt, pickNativeLeaseOverride as q, AUDIO_MACRO_LABELS as r, supportedRuntimes as rt, DEFAULT_AUDIO_ANALYZER_CONFIG as s, Fmp4BoxSplitter as st, APPLE_SA_TO_MACRO as t, storageEvictableCapability as tt, DEFAULT_NATIVE_LEASE_SETTINGS as u, isSoftwareDecode as ut, OpsLogEntrySchema as v, makeProfileBrokerId as vt, YAMNET_TO_MACRO as w, sleep as wt, RECORDING_EXPORT_MAX_READ_BYTES as x, parseJsonUnknown as xt, RATE_CONTROL_RELAXED as y, makeSourceBrokerId as yt, egressTransportFromRequest as z };
37079
+ export { recordingExportCapability as $, cameraStreamsCapability as A, boolean as At, egressTransportFromRequest as B, RecordingConfigSchema as C, nodePin as Ct, audioAnalysisCapability as D, sleep as Dt, addonWidgetsSourceCapability as E, selectAssignedProfileSlots as Et, deriveBatteryPresence as F, object as Ft, maskUrlCredentials as G, evaluateZoneRules as H, deriveDetailCropRect as I, record as It, pickDetailCropConvention as J, motionDetectionCapability as K, deriveRecordingMode as L, string as Lt, customAction as M, lazy as Mt, defaultDeviceFor as N, literal as Nt, audioAnalyzerCapability as O, _enum as Ot, defineCustomActions as P, number as Pt, recordingCapability as Q, detectionPipelineCapability as R, union as Rt, RECORDING_EXPORT_MAX_READ_BYTES as S, makeSourceBrokerId as St, YAMNET_TO_MACRO as T, parseProfileBrokerId as Tt, hfModelUrl as U, enumerateInferenceDevices as V, mapAudioLabelToMacro as W, pipelineExecutorCapability as X, pickNativeLeaseOverride as Y, pipelineRunnerCapability as Z, NativeLeaseSettingsSchema as _, DeviceType as _t, COCO_80_LABELS as a, streamBrokerCapability as at, RATE_CONTROL_RELAXED as b, isEvent as bt, DEFAULT_DETAIL_CROP_CONVENTION as c, errMsg as ct, DEVICE_BACKEND_TO_FORMAT as d, buildFfmpegArgs as dt, resolveEgressDecodeHwAccel as et, EVENT_PAD_MS as f, invocationFromEncodeProfile as ft, NativeLeaseAdmissionSchema as g, DeviceFeature as gt, HF_BASE_URL as h, CAM_PROFILE_ORDER as ht, BatteryStatusSchema as i, storageEvictableCapability as it, createHwAccelCache as j, discriminatedUnion as jt, batteryCapability as k, array as kt, DEFAULT_EVENTS_BAND_BUFFER_SEC as l, AUDIO_PRESETS as lt, ExportRecordSchema as m, BaseAddon as mt, AUDIO_BACKEND_CHOICES as n, resolveRecordingProfiles as nt, COCO_TO_MACRO as o, supportedRuntimes as ot, EncodeProfileSchema as p, isSoftwareDecode as pt, parseProcStatus as q, AUDIO_MACRO_LABELS as r, runtimeDevices as rt, DEFAULT_AUDIO_ANALYZER_CONFIG as s, webrtcSessionCapability as st, APPLE_SA_TO_MACRO as t, resolvePoolMemoryPolicy as tt, DEFAULT_NATIVE_LEASE_SETTINGS as u, Fmp4BoxSplitter as ut, OpsLogEntrySchema as v, createEvent as vt, RingBuffer as w, parseJsonUnknown as wt, RATE_CONTROL_TIGHT as x, makeProfileBrokerId as xt, PoolMemoryWatchdog as y, hydrateSchema as yt, egressTranscodeSharingKey as z, EventCategory as zt };