@camstack/addon-pipeline 1.2.93 → 1.2.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
- package/dist/audio-analyzer/index.js +58 -4
- package/dist/audio-analyzer/index.mjs +56 -2
- package/dist/detection-pipeline/index.js +194 -6
- package/dist/detection-pipeline/index.mjs +191 -3
- package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
- package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
- package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
- package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +195 -14
- package/dist/pipeline-runner/index.mjs +194 -13
- package/dist/process-memory-B9-UD7Rp.js +78 -0
- package/dist/process-memory-DXOB6eHX.mjs +66 -0
- package/dist/recorder/index.js +943 -109
- package/dist/recorder/index.mjs +942 -108
- package/dist/session-decode/decode-worker-child.js +4 -4
- package/dist/session-decode/decode-worker-child.mjs +3 -3
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
- package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
- package/dist/stream-broker/index.js +25 -7
- package/dist/stream-broker/index.mjs +25 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
- package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
- package/package.json +13 -7
- package/python/inference_pool.py +178 -3
- package/python/test_inference_pool_backpressure.py +7 -1
- package/python/test_inference_pool_memstats.py +146 -0
- package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
- package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
//#region ../types/dist/event-category-
|
|
2
|
+
//#region ../types/dist/event-category-XfKNtfCc.mjs
|
|
3
3
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
4
4
|
EventCategory["SystemBoot"] = "system.boot";
|
|
5
5
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -38,6 +38,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
|
38
38
|
EventCategory["AddonInstalled"] = "addon.installed";
|
|
39
39
|
EventCategory["AddonUninstalled"] = "addon.uninstalled";
|
|
40
40
|
EventCategory["AddonCrashed"] = "addon.crashed";
|
|
41
|
+
/**
|
|
42
|
+
* A RUNNER the D6 crash circuit-breaker gave up on — terminal.
|
|
43
|
+
*
|
|
44
|
+
* `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
|
|
45
|
+
* respawned"); this is the one that never resolves itself. Emitted exactly
|
|
46
|
+
* once per trip, by `process-service.ts`, carrying the node, the runner, the
|
|
47
|
+
* addons it hosted and the crash count that tripped the breaker.
|
|
48
|
+
*
|
|
49
|
+
* It exists because a runner marked terminally `failed` used to be SILENT:
|
|
50
|
+
* on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
|
|
51
|
+
* errors, the breaker stopped respawning it (correctly), `/health` kept
|
|
52
|
+
* returning 200, and fleet-wide recording was gone for 3h15 before the
|
|
53
|
+
* operator's phone told him. The Notification Center's system-event intake
|
|
54
|
+
* consumes this category and maps it to the `addon-crash-loop` kind.
|
|
55
|
+
*/
|
|
56
|
+
EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
|
|
41
57
|
EventCategory["AddonError"] = "addon.error";
|
|
42
58
|
EventCategory["AddonPageReady"] = "addon.page-ready";
|
|
43
59
|
EventCategory["AddonWidgetReady"] = "addon.widget-ready";
|
|
@@ -16071,6 +16087,7 @@ var NcSystemEventKindSchema = _enum([
|
|
|
16071
16087
|
"node-offline",
|
|
16072
16088
|
"node-inference-unavailable",
|
|
16073
16089
|
"detection-blind",
|
|
16090
|
+
"addon-crash-loop",
|
|
16074
16091
|
"addon-update-available",
|
|
16075
16092
|
"server-update-available",
|
|
16076
16093
|
"alarm-triggered",
|
|
@@ -21973,20 +21990,6 @@ var VectorStatsResultSchema = object({
|
|
|
21973
21990
|
exact: boolean()
|
|
21974
21991
|
});
|
|
21975
21992
|
method(VectorDeclareIndexInputSchema, _void(), { kind: "mutation" }), method(VectorUpsertInputSchema, VectorUpsertResultSchema, { kind: "mutation" }), method(VectorQueryInputSchema, VectorQueryResultSchema), method(VectorGetInputSchema, VectorGetResultSchema), method(VectorDeleteInputSchema, VectorDeleteResultSchema, { kind: "mutation" }), method(VectorDeleteByFilterInputSchema, VectorDeleteResultSchema, { kind: "mutation" }), method(VectorStatsInputSchema, VectorStatsResultSchema);
|
|
21976
|
-
/**
|
|
21977
|
-
* `videoclips` — the unified, navigable-clip surface for a camera.
|
|
21978
|
-
*
|
|
21979
|
-
* A device-scoped WRAPPER cap (like `pipeline-analytics`): exactly one active
|
|
21980
|
-
* provider per device, substitutable. The DEFAULT provider (registered by
|
|
21981
|
-
* `addon-post-analysis`, `defaultActive: true`) composes the analytics event
|
|
21982
|
-
* log (markers + thumbnails) with the recorder's `getPlaybackManifest` — a clip
|
|
21983
|
-
* is a time-WINDOW over existing footage, never a separate file. A camera that
|
|
21984
|
-
* exposes NATIVE onboard clips (Reolink/Hikvision NVR) can later substitute the
|
|
21985
|
-
* wrapper provider for its device and serve its own clip catalog + URLs.
|
|
21986
|
-
*
|
|
21987
|
-
* A `Clip` is purely time-based (subtree-blind): playback resolves segments by
|
|
21988
|
-
* temporal overlap, so the API never decides `continuous` vs `events`.
|
|
21989
|
-
*/
|
|
21990
21993
|
var ClipSchema = object({
|
|
21991
21994
|
/** Opaque, provider-namespaced id. The default provider encodes the time
|
|
21992
21995
|
* window so `getClipPlayback` is self-contained (no event re-query). */
|
|
@@ -22012,12 +22015,41 @@ var ClipSchema = object({
|
|
|
22012
22015
|
* of the event that owns `kind` (object > motion > audio). Do not extract
|
|
22013
22016
|
* a keyframe from the recorded segments. Other providers (onboard, HKSV)
|
|
22014
22017
|
* mint their own stills.
|
|
22018
|
+
*
|
|
22019
|
+
* **VOUCHED, never fabricated.** A provider emits this only for an event it
|
|
22020
|
+
* has CONFIRMED owns at least one media row (its own, or its owning track's).
|
|
22021
|
+
* Absent is meaningful — "this visit has no event still" — never "we did not
|
|
22022
|
+
* look". Stamping it from a URL template made 35% of one camera's clips point
|
|
22023
|
+
* at a 404 (device 3836, 2026-08-17: 64 of 179 clips dead, 63 of them motion).
|
|
22024
|
+
* A read that FAILS drops the claim; it never invents it.
|
|
22015
22025
|
*/
|
|
22016
22026
|
thumbnail: string().optional(),
|
|
22017
|
-
/**
|
|
22018
|
-
*
|
|
22019
|
-
*
|
|
22027
|
+
/**
|
|
22028
|
+
* An instant INSIDE this visit's footage, hole-safe, where a recorded still
|
|
22029
|
+
* can be decoded. Present whenever the visit came from recorded availability;
|
|
22030
|
+
* absent on a per-event padded window (there is no footage to promise).
|
|
22031
|
+
*
|
|
22032
|
+
* This is not a thumbnail and not a second byte path: it is the argument to
|
|
22033
|
+
* the recorder's existing still route. The surface — never the provider —
|
|
22034
|
+
* decides whether to use it. It is the midpoint of the visit's LONGEST
|
|
22035
|
+
* contiguous range, not of the visit: a visit spans its holes by
|
|
22036
|
+
* construction, so a naive midpoint lands in dead air.
|
|
22037
|
+
*/
|
|
22038
|
+
stillAtMs: number().optional(),
|
|
22039
|
+
/**
|
|
22040
|
+
* Analytics event ids that overlap this visit — a BOUNDED sample, newest
|
|
22041
|
+
* first within kind (object → motion → audio), capped at
|
|
22042
|
+
* {@link MAX_CLIP_EVENT_IDS}. Empty on footage-only clips.
|
|
22043
|
+
*
|
|
22044
|
+
* Bounded because it is not a payload the surface pages through: one visit on
|
|
22045
|
+
* device 615 carried 2 345 ids, and `eventIds` was 99% of a 153 KB
|
|
22046
|
+
* camera-day. Read {@link eventCount} for the true total.
|
|
22047
|
+
*/
|
|
22020
22048
|
eventIds: array(string()).optional(),
|
|
22049
|
+
/** How many analytics events actually overlap this visit. Differs from
|
|
22050
|
+
* `eventIds.length` exactly when the sample was capped — so a truncated
|
|
22051
|
+
* list is never mistaken for a quiet visit. */
|
|
22052
|
+
eventCount: number().int().nonnegative().optional(),
|
|
22021
22053
|
/** Intra-visit footage holes (GOP rolls, discarded segments) still shorter
|
|
22022
22054
|
* than `VISIT_MERGE_GAP_MS`. Playback concatenates around them; the timeline
|
|
22023
22055
|
* bar keeps showing them via `recording.getAvailability`. */
|
|
@@ -22568,7 +22600,14 @@ var SearchResultSchema = object({
|
|
|
22568
22600
|
});
|
|
22569
22601
|
var AutoUpdateSettingsSchema = object({
|
|
22570
22602
|
channel: ChannelSchema,
|
|
22571
|
-
intervalSeconds: number()
|
|
22603
|
+
intervalSeconds: number(),
|
|
22604
|
+
/**
|
|
22605
|
+
* Cadence of the "an update exists" POLLER, in seconds. Independent of
|
|
22606
|
+
* `channel`: the poller runs while auto-apply is `off`, because being told
|
|
22607
|
+
* about a publish and installing it are different decisions. Clamped
|
|
22608
|
+
* server-side to 900 s … 604800 s; defaults to 21600 s (6 h).
|
|
22609
|
+
*/
|
|
22610
|
+
updateCheckIntervalSeconds: number()
|
|
22572
22611
|
});
|
|
22573
22612
|
var AddonAutoUpdateSchema = ChannelWithInheritSchema;
|
|
22574
22613
|
var RestartAddonResultSchema = unknown();
|
|
@@ -22709,7 +22748,9 @@ method(_void(), array(AddonListItemSchema).readonly()), method(object({
|
|
|
22709
22748
|
auth: "admin"
|
|
22710
22749
|
}), method(_void(), AutoUpdateSettingsSchema, { auth: "admin" }), method(object({
|
|
22711
22750
|
channel: ChannelSchema,
|
|
22712
|
-
intervalSeconds: number().min(300).max(86400).optional()
|
|
22751
|
+
intervalSeconds: number().min(300).max(86400).optional(),
|
|
22752
|
+
/** Availability-poll cadence; see AutoUpdateSettingsSchema. */
|
|
22753
|
+
updateCheckIntervalSeconds: number().min(900).max(604800).optional()
|
|
22713
22754
|
}), unknown(), {
|
|
22714
22755
|
kind: "mutation",
|
|
22715
22756
|
auth: "admin"
|
|
@@ -36401,6 +36442,303 @@ function maskUrlCredentials(rawUrl) {
|
|
|
36401
36442
|
return rawUrl;
|
|
36402
36443
|
}
|
|
36403
36444
|
}
|
|
36445
|
+
var MB = 1024 * 1024;
|
|
36446
|
+
var DEFAULT_POOL_MEMORY_POLICY = {
|
|
36447
|
+
sampleIntervalMs: 6e4,
|
|
36448
|
+
baselineSettleSamples: 5,
|
|
36449
|
+
baselineSampleCount: 3,
|
|
36450
|
+
restartMultiple: 4,
|
|
36451
|
+
floorBytes: 1024 * MB,
|
|
36452
|
+
ceilingBytes: 3072 * MB,
|
|
36453
|
+
cooldownMs: 30 * 6e4,
|
|
36454
|
+
maxRestartsPerWindow: 6,
|
|
36455
|
+
restartWindowMs: 1440 * 6e4
|
|
36456
|
+
};
|
|
36457
|
+
/**
|
|
36458
|
+
* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
|
|
36459
|
+
* absent values keep the default — an operator typo must never disable the
|
|
36460
|
+
* bound or set it to zero.
|
|
36461
|
+
*/
|
|
36462
|
+
function resolvePoolMemoryPolicy(env) {
|
|
36463
|
+
const num = (key, fallback, min) => {
|
|
36464
|
+
const raw = env[key];
|
|
36465
|
+
if (raw === void 0) return fallback;
|
|
36466
|
+
const parsed = Number(raw);
|
|
36467
|
+
return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
|
|
36468
|
+
};
|
|
36469
|
+
const d = DEFAULT_POOL_MEMORY_POLICY;
|
|
36470
|
+
return {
|
|
36471
|
+
sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
|
|
36472
|
+
baselineSettleSamples: d.baselineSettleSamples,
|
|
36473
|
+
baselineSampleCount: d.baselineSampleCount,
|
|
36474
|
+
restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
|
|
36475
|
+
floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
|
|
36476
|
+
ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
|
|
36477
|
+
cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
|
|
36478
|
+
maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
|
|
36479
|
+
restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
|
|
36480
|
+
};
|
|
36481
|
+
}
|
|
36482
|
+
/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
|
|
36483
|
+
* Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
|
|
36484
|
+
function parseProcStatus(text) {
|
|
36485
|
+
const kb = (label) => {
|
|
36486
|
+
const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
|
|
36487
|
+
return match ? Number(match[1]) * 1024 : null;
|
|
36488
|
+
};
|
|
36489
|
+
const rssBytes = kb("VmRSS");
|
|
36490
|
+
if (rssBytes === null) return null;
|
|
36491
|
+
const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
|
|
36492
|
+
return {
|
|
36493
|
+
rssBytes,
|
|
36494
|
+
vmBytes: kb("VmSize") ?? 0,
|
|
36495
|
+
hwmBytes: kb("VmHWM") ?? 0,
|
|
36496
|
+
swapBytes: kb("VmSwap") ?? 0,
|
|
36497
|
+
threads: threadsMatch ? Number(threadsMatch[1]) : 0
|
|
36498
|
+
};
|
|
36499
|
+
}
|
|
36500
|
+
function initialPoolMemoryState() {
|
|
36501
|
+
return {
|
|
36502
|
+
settleSeen: 0,
|
|
36503
|
+
baselineWindow: [],
|
|
36504
|
+
baselineBytes: null,
|
|
36505
|
+
restartsAt: [],
|
|
36506
|
+
lastRestartAt: null
|
|
36507
|
+
};
|
|
36508
|
+
}
|
|
36509
|
+
function median(values) {
|
|
36510
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
36511
|
+
return sorted[Math.floor(sorted.length / 2)];
|
|
36512
|
+
}
|
|
36513
|
+
function advanceBaseline(state, rssBytes, policy) {
|
|
36514
|
+
if (state.baselineBytes !== null) return state;
|
|
36515
|
+
if (state.settleSeen < policy.baselineSettleSamples) return {
|
|
36516
|
+
...state,
|
|
36517
|
+
settleSeen: state.settleSeen + 1
|
|
36518
|
+
};
|
|
36519
|
+
const window = [...state.baselineWindow, rssBytes];
|
|
36520
|
+
if (window.length < policy.baselineSampleCount) return {
|
|
36521
|
+
...state,
|
|
36522
|
+
baselineWindow: window
|
|
36523
|
+
};
|
|
36524
|
+
return {
|
|
36525
|
+
...state,
|
|
36526
|
+
baselineWindow: window,
|
|
36527
|
+
baselineBytes: median(window)
|
|
36528
|
+
};
|
|
36529
|
+
}
|
|
36530
|
+
/** The one place a pool's restart threshold is computed. */
|
|
36531
|
+
function poolMemoryThreshold(baselineBytes, policy) {
|
|
36532
|
+
if (baselineBytes === null) return policy.ceilingBytes;
|
|
36533
|
+
return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
|
|
36534
|
+
}
|
|
36535
|
+
function pruneRestarts(restartsAt, nowMs, policy) {
|
|
36536
|
+
return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
|
|
36537
|
+
}
|
|
36538
|
+
/**
|
|
36539
|
+
* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
|
|
36540
|
+
* The caller performs (and logs) the restart, then commits it via
|
|
36541
|
+
* {@link commitWatchdogRestart}.
|
|
36542
|
+
*/
|
|
36543
|
+
function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
|
|
36544
|
+
const next = advanceBaseline(state, rssBytes, policy);
|
|
36545
|
+
const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
|
|
36546
|
+
if (rssBytes <= thresholdBytes) return {
|
|
36547
|
+
state: next,
|
|
36548
|
+
action: next.baselineBytes === null ? "baseline-pending" : "ok",
|
|
36549
|
+
baselineBytes: next.baselineBytes,
|
|
36550
|
+
thresholdBytes
|
|
36551
|
+
};
|
|
36552
|
+
const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
|
|
36553
|
+
const pruned = {
|
|
36554
|
+
...next,
|
|
36555
|
+
restartsAt: inWindow
|
|
36556
|
+
};
|
|
36557
|
+
if (inWindow.length >= policy.maxRestartsPerWindow) return {
|
|
36558
|
+
state: pruned,
|
|
36559
|
+
action: "exhausted",
|
|
36560
|
+
baselineBytes: next.baselineBytes,
|
|
36561
|
+
thresholdBytes
|
|
36562
|
+
};
|
|
36563
|
+
if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
|
|
36564
|
+
state: pruned,
|
|
36565
|
+
action: "cooldown",
|
|
36566
|
+
baselineBytes: next.baselineBytes,
|
|
36567
|
+
thresholdBytes
|
|
36568
|
+
};
|
|
36569
|
+
return {
|
|
36570
|
+
state: pruned,
|
|
36571
|
+
action: "restart",
|
|
36572
|
+
baselineBytes: next.baselineBytes,
|
|
36573
|
+
thresholdBytes
|
|
36574
|
+
};
|
|
36575
|
+
}
|
|
36576
|
+
/** Record a performed watchdog restart: budget consumed, baseline reset —
|
|
36577
|
+
* the recreated pool is a NEW process and gets a fresh settle window. */
|
|
36578
|
+
function commitWatchdogRestart(state, nowMs) {
|
|
36579
|
+
return {
|
|
36580
|
+
...initialPoolMemoryState(),
|
|
36581
|
+
restartsAt: [...state.restartsAt, nowMs],
|
|
36582
|
+
lastRestartAt: nowMs
|
|
36583
|
+
};
|
|
36584
|
+
}
|
|
36585
|
+
/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
|
|
36586
|
+
* kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
|
|
36587
|
+
* restart budget is kept: it bounds the WATCHDOG, not the pool. */
|
|
36588
|
+
function resetPoolBaseline(state) {
|
|
36589
|
+
return {
|
|
36590
|
+
...initialPoolMemoryState(),
|
|
36591
|
+
restartsAt: state.restartsAt,
|
|
36592
|
+
lastRestartAt: state.lastRestartAt
|
|
36593
|
+
};
|
|
36594
|
+
}
|
|
36595
|
+
/**
|
|
36596
|
+
* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
|
|
36597
|
+
* several pools at once (e.g. after a model rollout grew every baseline)
|
|
36598
|
+
* would take detection down on every camera simultaneously; one per sweep
|
|
36599
|
+
* means the survivors keep serving while the worst offender reloads, and the
|
|
36600
|
+
* next sweep (one interval later) takes the next one.
|
|
36601
|
+
*/
|
|
36602
|
+
function pickRestartCandidate(candidates) {
|
|
36603
|
+
if (candidates.length === 0) return null;
|
|
36604
|
+
let worst = candidates[0];
|
|
36605
|
+
for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
|
|
36606
|
+
return worst.key;
|
|
36607
|
+
}
|
|
36608
|
+
var PoolMemoryWatchdog = class {
|
|
36609
|
+
pools = /* @__PURE__ */ new Map();
|
|
36610
|
+
timer = null;
|
|
36611
|
+
sweeping = false;
|
|
36612
|
+
stopped = false;
|
|
36613
|
+
opts;
|
|
36614
|
+
now;
|
|
36615
|
+
setTimer;
|
|
36616
|
+
clearTimer;
|
|
36617
|
+
constructor(opts) {
|
|
36618
|
+
this.opts = opts;
|
|
36619
|
+
this.now = opts.now ?? (() => Date.now());
|
|
36620
|
+
this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
36621
|
+
this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
|
|
36622
|
+
}
|
|
36623
|
+
start() {
|
|
36624
|
+
if (this.timer || this.stopped) return;
|
|
36625
|
+
this.arm();
|
|
36626
|
+
}
|
|
36627
|
+
stop() {
|
|
36628
|
+
this.stopped = true;
|
|
36629
|
+
if (this.timer) {
|
|
36630
|
+
this.clearTimer(this.timer);
|
|
36631
|
+
this.timer = null;
|
|
36632
|
+
}
|
|
36633
|
+
}
|
|
36634
|
+
arm() {
|
|
36635
|
+
const timer = this.setTimer(() => {
|
|
36636
|
+
this.sweep().finally(() => {
|
|
36637
|
+
if (!this.stopped) this.arm();
|
|
36638
|
+
});
|
|
36639
|
+
}, this.opts.policy.sampleIntervalMs);
|
|
36640
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
36641
|
+
this.timer = timer;
|
|
36642
|
+
}
|
|
36643
|
+
/** One sweep: sample → log every pool → restart at most one. Public so the
|
|
36644
|
+
* loop is testable without fake global timers. */
|
|
36645
|
+
async sweep() {
|
|
36646
|
+
if (this.sweeping) return;
|
|
36647
|
+
this.sweeping = true;
|
|
36648
|
+
try {
|
|
36649
|
+
await this.doSweep();
|
|
36650
|
+
} catch (err) {
|
|
36651
|
+
this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
36652
|
+
} finally {
|
|
36653
|
+
this.sweeping = false;
|
|
36654
|
+
}
|
|
36655
|
+
}
|
|
36656
|
+
async doSweep() {
|
|
36657
|
+
const nowMs = this.now();
|
|
36658
|
+
const samples = await this.opts.sample();
|
|
36659
|
+
const candidates = [];
|
|
36660
|
+
const verdictByKey = /* @__PURE__ */ new Map();
|
|
36661
|
+
for (const t of samples) {
|
|
36662
|
+
const tracked = this.pools.get(t.key);
|
|
36663
|
+
let state = tracked?.state ?? initialPoolMemoryState();
|
|
36664
|
+
if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
|
|
36665
|
+
this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
|
|
36666
|
+
poolKey: t.key,
|
|
36667
|
+
previousPids: tracked.pids,
|
|
36668
|
+
pids: t.pids
|
|
36669
|
+
} });
|
|
36670
|
+
state = resetPoolBaseline(state);
|
|
36671
|
+
}
|
|
36672
|
+
const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
|
|
36673
|
+
this.pools.set(t.key, {
|
|
36674
|
+
state: verdict.state,
|
|
36675
|
+
pids: t.pids
|
|
36676
|
+
});
|
|
36677
|
+
verdictByKey.set(t.key, verdict);
|
|
36678
|
+
this.opts.log.info("pool memory", { meta: {
|
|
36679
|
+
poolKey: t.key,
|
|
36680
|
+
pids: t.pids,
|
|
36681
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
36682
|
+
baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
|
|
36683
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
36684
|
+
action: verdict.action,
|
|
36685
|
+
...t.meta
|
|
36686
|
+
} });
|
|
36687
|
+
if (verdict.action === "restart") candidates.push({
|
|
36688
|
+
key: t.key,
|
|
36689
|
+
rssBytes: t.rssBytes,
|
|
36690
|
+
thresholdBytes: verdict.thresholdBytes
|
|
36691
|
+
});
|
|
36692
|
+
else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
|
|
36693
|
+
poolKey: t.key,
|
|
36694
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
36695
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
36696
|
+
cooldownMs: this.opts.policy.cooldownMs
|
|
36697
|
+
} });
|
|
36698
|
+
else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
|
|
36699
|
+
poolKey: t.key,
|
|
36700
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
36701
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
36702
|
+
maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
|
|
36703
|
+
restartWindowMs: this.opts.policy.restartWindowMs
|
|
36704
|
+
} });
|
|
36705
|
+
}
|
|
36706
|
+
for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
|
|
36707
|
+
const pickedKey = pickRestartCandidate(candidates);
|
|
36708
|
+
if (pickedKey === null) return;
|
|
36709
|
+
const picked = candidates.find((c) => c.key === pickedKey);
|
|
36710
|
+
const verdict = verdictByKey.get(pickedKey);
|
|
36711
|
+
const tracked = this.pools.get(pickedKey);
|
|
36712
|
+
const restartsInWindow = tracked.state.restartsAt.length + 1;
|
|
36713
|
+
this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
|
|
36714
|
+
poolKey: pickedKey,
|
|
36715
|
+
pids: tracked.pids,
|
|
36716
|
+
rssMb: Math.round(picked.rssBytes / MB),
|
|
36717
|
+
thresholdMb: Math.round(picked.thresholdBytes / MB),
|
|
36718
|
+
baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
|
|
36719
|
+
restartsInWindow,
|
|
36720
|
+
deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
|
|
36721
|
+
} });
|
|
36722
|
+
try {
|
|
36723
|
+
await this.opts.restart(pickedKey);
|
|
36724
|
+
this.pools.set(pickedKey, {
|
|
36725
|
+
state: commitWatchdogRestart(tracked.state, this.now()),
|
|
36726
|
+
pids: []
|
|
36727
|
+
});
|
|
36728
|
+
} catch (err) {
|
|
36729
|
+
this.opts.log.error("pool memory watchdog restart failed", { meta: {
|
|
36730
|
+
poolKey: pickedKey,
|
|
36731
|
+
error: err instanceof Error ? err.message : String(err)
|
|
36732
|
+
} });
|
|
36733
|
+
}
|
|
36734
|
+
}
|
|
36735
|
+
};
|
|
36736
|
+
function samePids(a, b) {
|
|
36737
|
+
if (a.length !== b.length) return false;
|
|
36738
|
+
const sortedA = [...a].sort((x, y) => x - y);
|
|
36739
|
+
const sortedB = [...b].sort((x, y) => x - y);
|
|
36740
|
+
return sortedA.every((v, i) => v === sortedB[i]);
|
|
36741
|
+
}
|
|
36404
36742
|
/**
|
|
36405
36743
|
* Fixed-capacity ring buffer. When full, push() overwrites the oldest entry.
|
|
36406
36744
|
* drain() returns up to maxCount items in FIFO order and removes them.
|
|
@@ -36738,4 +37076,4 @@ function enumerateInferenceDevices(hw) {
|
|
|
36738
37076
|
return out;
|
|
36739
37077
|
}
|
|
36740
37078
|
//#endregion
|
|
36741
|
-
export {
|
|
37079
|
+
export { recordingExportCapability as $, cameraStreamsCapability as A, boolean as At, egressTransportFromRequest as B, RecordingConfigSchema as C, nodePin as Ct, audioAnalysisCapability as D, sleep as Dt, addonWidgetsSourceCapability as E, selectAssignedProfileSlots as Et, deriveBatteryPresence as F, object as Ft, maskUrlCredentials as G, evaluateZoneRules as H, deriveDetailCropRect as I, record as It, pickDetailCropConvention as J, motionDetectionCapability as K, deriveRecordingMode as L, string as Lt, customAction as M, lazy as Mt, defaultDeviceFor as N, literal as Nt, audioAnalyzerCapability as O, _enum as Ot, defineCustomActions as P, number as Pt, recordingCapability as Q, detectionPipelineCapability as R, union as Rt, RECORDING_EXPORT_MAX_READ_BYTES as S, makeSourceBrokerId as St, YAMNET_TO_MACRO as T, parseProfileBrokerId as Tt, hfModelUrl as U, enumerateInferenceDevices as V, mapAudioLabelToMacro as W, pipelineExecutorCapability as X, pickNativeLeaseOverride as Y, pipelineRunnerCapability as Z, NativeLeaseSettingsSchema as _, DeviceType as _t, COCO_80_LABELS as a, streamBrokerCapability as at, RATE_CONTROL_RELAXED as b, isEvent as bt, DEFAULT_DETAIL_CROP_CONVENTION as c, errMsg as ct, DEVICE_BACKEND_TO_FORMAT as d, buildFfmpegArgs as dt, resolveEgressDecodeHwAccel as et, EVENT_PAD_MS as f, invocationFromEncodeProfile as ft, NativeLeaseAdmissionSchema as g, DeviceFeature as gt, HF_BASE_URL as h, CAM_PROFILE_ORDER as ht, BatteryStatusSchema as i, storageEvictableCapability as it, createHwAccelCache as j, discriminatedUnion as jt, batteryCapability as k, array as kt, DEFAULT_EVENTS_BAND_BUFFER_SEC as l, AUDIO_PRESETS as lt, ExportRecordSchema as m, BaseAddon as mt, AUDIO_BACKEND_CHOICES as n, resolveRecordingProfiles as nt, COCO_TO_MACRO as o, supportedRuntimes as ot, EncodeProfileSchema as p, isSoftwareDecode as pt, parseProcStatus as q, AUDIO_MACRO_LABELS as r, runtimeDevices as rt, DEFAULT_AUDIO_ANALYZER_CONFIG as s, webrtcSessionCapability as st, APPLE_SA_TO_MACRO as t, resolvePoolMemoryPolicy as tt, DEFAULT_NATIVE_LEASE_SETTINGS as u, Fmp4BoxSplitter as ut, OpsLogEntrySchema as v, createEvent as vt, RingBuffer as w, parseJsonUnknown as wt, RATE_CONTROL_TIGHT as x, makeProfileBrokerId as xt, PoolMemoryWatchdog as y, hydrateSchema as yt, egressTranscodeSharingKey as z, EventCategory as zt };
|