@camstack/types 1.2.85 → 1.2.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon.js +2 -2
- package/dist/addon.mjs +2 -2
- package/dist/capabilities/index.d.ts +2 -2
- package/dist/capabilities/notification-rules.cap.d.ts +16 -0
- package/dist/capabilities/osd-manager.cap.d.ts +6 -0
- package/dist/capabilities/videoclips.cap.d.ts +11 -0
- package/dist/enums/event-category.d.ts +16 -0
- package/dist/enums.js +1 -1
- package/dist/enums.mjs +1 -1
- package/dist/{event-category-DBHdQVIy.js → event-category-CRPORAAz.js} +16 -0
- package/dist/{event-category-C0lyLd5U.mjs → event-category-XfKNtfCc.mjs} +16 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +357 -6
- package/dist/index.mjs +347 -7
- package/dist/interfaces/addon.d.ts +17 -2
- package/dist/interfaces/event-bus.d.ts +17 -0
- package/dist/{sleep-DZAjGv1f.js → sleep-CMRLJj2e.js} +1 -1
- package/dist/{sleep-oYerbR6F.mjs → sleep-zMxKWD0M.mjs} +1 -1
- package/dist/utils/pool-memory-watchdog.d.ts +178 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_event_category = require("./event-category-
|
|
3
|
-
const require_sleep = require("./sleep-
|
|
2
|
+
const require_event_category = require("./event-category-CRPORAAz.js");
|
|
3
|
+
const require_sleep = require("./sleep-CMRLJj2e.js");
|
|
4
4
|
const require_canonical_hash = require("./canonical-hash-DNV8S5ET.js");
|
|
5
5
|
const require_enums = require("./enums.js");
|
|
6
6
|
const require_err_msg = require("./err-msg-COpsHMw2.js");
|
|
@@ -12252,6 +12252,7 @@ var NcSystemEventKindSchema = zod.z.enum([
|
|
|
12252
12252
|
"node-offline",
|
|
12253
12253
|
"node-inference-unavailable",
|
|
12254
12254
|
"detection-blind",
|
|
12255
|
+
"addon-crash-loop",
|
|
12255
12256
|
"addon-update-available",
|
|
12256
12257
|
"server-update-available",
|
|
12257
12258
|
"alarm-triggered",
|
|
@@ -13403,6 +13404,10 @@ var NC_CONDITION_CATALOG = [
|
|
|
13403
13404
|
value: "detection-blind",
|
|
13404
13405
|
label: "Camera detecting nothing"
|
|
13405
13406
|
},
|
|
13407
|
+
{
|
|
13408
|
+
value: "addon-crash-loop",
|
|
13409
|
+
label: "Addon stopped after repeated crashes"
|
|
13410
|
+
},
|
|
13406
13411
|
{
|
|
13407
13412
|
value: "addon-update-available",
|
|
13408
13413
|
label: "Addon update available"
|
|
@@ -20208,6 +20213,13 @@ var vectorStoreCapability = {
|
|
|
20208
20213
|
* A `Clip` is purely time-based (subtree-blind): playback resolves segments by
|
|
20209
20214
|
* temporal overlap, so the API never decides `continuous` vs `events`.
|
|
20210
20215
|
*/
|
|
20216
|
+
/**
|
|
20217
|
+
* Ceiling on `Clip.eventIds`. A visit is annotated by its events; it is not a
|
|
20218
|
+
* transport for them. At ~39 wire bytes per uuid this keeps a clip row inside
|
|
20219
|
+
* the 1.5 KB budget (`docs/design/2026-08-17-videoclips-implementation.md` §7.2)
|
|
20220
|
+
* while preserving `eventIds[0]` — the surface's thumbnail fallback.
|
|
20221
|
+
*/
|
|
20222
|
+
var MAX_CLIP_EVENT_IDS = 24;
|
|
20211
20223
|
var ClipSchema = zod.z.object({
|
|
20212
20224
|
/** Opaque, provider-namespaced id. The default provider encodes the time
|
|
20213
20225
|
* window so `getClipPlayback` is self-contained (no event re-query). */
|
|
@@ -20233,12 +20245,41 @@ var ClipSchema = zod.z.object({
|
|
|
20233
20245
|
* of the event that owns `kind` (object > motion > audio). Do not extract
|
|
20234
20246
|
* a keyframe from the recorded segments. Other providers (onboard, HKSV)
|
|
20235
20247
|
* mint their own stills.
|
|
20248
|
+
*
|
|
20249
|
+
* **VOUCHED, never fabricated.** A provider emits this only for an event it
|
|
20250
|
+
* has CONFIRMED owns at least one media row (its own, or its owning track's).
|
|
20251
|
+
* Absent is meaningful — "this visit has no event still" — never "we did not
|
|
20252
|
+
* look". Stamping it from a URL template made 35% of one camera's clips point
|
|
20253
|
+
* at a 404 (device 3836, 2026-08-17: 64 of 179 clips dead, 63 of them motion).
|
|
20254
|
+
* A read that FAILS drops the claim; it never invents it.
|
|
20236
20255
|
*/
|
|
20237
20256
|
thumbnail: zod.z.string().optional(),
|
|
20238
|
-
/**
|
|
20239
|
-
*
|
|
20240
|
-
*
|
|
20257
|
+
/**
|
|
20258
|
+
* An instant INSIDE this visit's footage, hole-safe, where a recorded still
|
|
20259
|
+
* can be decoded. Present whenever the visit came from recorded availability;
|
|
20260
|
+
* absent on a per-event padded window (there is no footage to promise).
|
|
20261
|
+
*
|
|
20262
|
+
* This is not a thumbnail and not a second byte path: it is the argument to
|
|
20263
|
+
* the recorder's existing still route. The surface — never the provider —
|
|
20264
|
+
* decides whether to use it. It is the midpoint of the visit's LONGEST
|
|
20265
|
+
* contiguous range, not of the visit: a visit spans its holes by
|
|
20266
|
+
* construction, so a naive midpoint lands in dead air.
|
|
20267
|
+
*/
|
|
20268
|
+
stillAtMs: zod.z.number().optional(),
|
|
20269
|
+
/**
|
|
20270
|
+
* Analytics event ids that overlap this visit — a BOUNDED sample, newest
|
|
20271
|
+
* first within kind (object → motion → audio), capped at
|
|
20272
|
+
* {@link MAX_CLIP_EVENT_IDS}. Empty on footage-only clips.
|
|
20273
|
+
*
|
|
20274
|
+
* Bounded because it is not a payload the surface pages through: one visit on
|
|
20275
|
+
* device 615 carried 2 345 ids, and `eventIds` was 99% of a 153 KB
|
|
20276
|
+
* camera-day. Read {@link eventCount} for the true total.
|
|
20277
|
+
*/
|
|
20241
20278
|
eventIds: zod.z.array(zod.z.string()).optional(),
|
|
20279
|
+
/** How many analytics events actually overlap this visit. Differs from
|
|
20280
|
+
* `eventIds.length` exactly when the sample was capped — so a truncated
|
|
20281
|
+
* list is never mistaken for a quiet visit. */
|
|
20282
|
+
eventCount: zod.z.number().int().nonnegative().optional(),
|
|
20242
20283
|
/** Intra-visit footage holes (GOP rolls, discarded segments) still shorter
|
|
20243
20284
|
* than `VISIT_MERGE_GAP_MS`. Playback concatenates around them; the timeline
|
|
20244
20285
|
* bar keeps showing them via `recording.getAvailability`. */
|
|
@@ -43589,7 +43630,7 @@ function systemEventFilterApplies(kind, filter) {
|
|
|
43589
43630
|
switch (filter) {
|
|
43590
43631
|
case "deviceIds": return kind.startsWith("device-") || kind.startsWith("stream-") || kind === "detection-blind" || kind === "alarm-triggered" || kind === "export-completed";
|
|
43591
43632
|
case "deviceTypes": return kind.startsWith("device-") || kind === "detection-blind";
|
|
43592
|
-
case "nodeIds": return kind.startsWith("node-") || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
|
|
43633
|
+
case "nodeIds": return kind.startsWith("node-") || kind === "addon-crash-loop" || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
|
|
43593
43634
|
case "packageNames": return kind.endsWith("update-available") || kind === "addon-updated" || kind === "server-updated";
|
|
43594
43635
|
}
|
|
43595
43636
|
}
|
|
@@ -45250,6 +45291,305 @@ function maskUrlCredentials(rawUrl) {
|
|
|
45250
45291
|
}
|
|
45251
45292
|
}
|
|
45252
45293
|
//#endregion
|
|
45294
|
+
//#region src/utils/pool-memory-watchdog.ts
|
|
45295
|
+
var MB = 1024 * 1024;
|
|
45296
|
+
var DEFAULT_POOL_MEMORY_POLICY = {
|
|
45297
|
+
sampleIntervalMs: 6e4,
|
|
45298
|
+
baselineSettleSamples: 5,
|
|
45299
|
+
baselineSampleCount: 3,
|
|
45300
|
+
restartMultiple: 4,
|
|
45301
|
+
floorBytes: 1024 * MB,
|
|
45302
|
+
ceilingBytes: 3072 * MB,
|
|
45303
|
+
cooldownMs: 30 * 6e4,
|
|
45304
|
+
maxRestartsPerWindow: 6,
|
|
45305
|
+
restartWindowMs: 1440 * 6e4
|
|
45306
|
+
};
|
|
45307
|
+
/**
|
|
45308
|
+
* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
|
|
45309
|
+
* absent values keep the default — an operator typo must never disable the
|
|
45310
|
+
* bound or set it to zero.
|
|
45311
|
+
*/
|
|
45312
|
+
function resolvePoolMemoryPolicy(env) {
|
|
45313
|
+
const num = (key, fallback, min) => {
|
|
45314
|
+
const raw = env[key];
|
|
45315
|
+
if (raw === void 0) return fallback;
|
|
45316
|
+
const parsed = Number(raw);
|
|
45317
|
+
return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
|
|
45318
|
+
};
|
|
45319
|
+
const d = DEFAULT_POOL_MEMORY_POLICY;
|
|
45320
|
+
return {
|
|
45321
|
+
sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
|
|
45322
|
+
baselineSettleSamples: d.baselineSettleSamples,
|
|
45323
|
+
baselineSampleCount: d.baselineSampleCount,
|
|
45324
|
+
restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
|
|
45325
|
+
floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
|
|
45326
|
+
ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
|
|
45327
|
+
cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
|
|
45328
|
+
maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
|
|
45329
|
+
restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
|
|
45330
|
+
};
|
|
45331
|
+
}
|
|
45332
|
+
/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
|
|
45333
|
+
* Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
|
|
45334
|
+
function parseProcStatus(text) {
|
|
45335
|
+
const kb = (label) => {
|
|
45336
|
+
const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
|
|
45337
|
+
return match ? Number(match[1]) * 1024 : null;
|
|
45338
|
+
};
|
|
45339
|
+
const rssBytes = kb("VmRSS");
|
|
45340
|
+
if (rssBytes === null) return null;
|
|
45341
|
+
const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
|
|
45342
|
+
return {
|
|
45343
|
+
rssBytes,
|
|
45344
|
+
vmBytes: kb("VmSize") ?? 0,
|
|
45345
|
+
hwmBytes: kb("VmHWM") ?? 0,
|
|
45346
|
+
swapBytes: kb("VmSwap") ?? 0,
|
|
45347
|
+
threads: threadsMatch ? Number(threadsMatch[1]) : 0
|
|
45348
|
+
};
|
|
45349
|
+
}
|
|
45350
|
+
function initialPoolMemoryState() {
|
|
45351
|
+
return {
|
|
45352
|
+
settleSeen: 0,
|
|
45353
|
+
baselineWindow: [],
|
|
45354
|
+
baselineBytes: null,
|
|
45355
|
+
restartsAt: [],
|
|
45356
|
+
lastRestartAt: null
|
|
45357
|
+
};
|
|
45358
|
+
}
|
|
45359
|
+
function median(values) {
|
|
45360
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
45361
|
+
return sorted[Math.floor(sorted.length / 2)];
|
|
45362
|
+
}
|
|
45363
|
+
function advanceBaseline(state, rssBytes, policy) {
|
|
45364
|
+
if (state.baselineBytes !== null) return state;
|
|
45365
|
+
if (state.settleSeen < policy.baselineSettleSamples) return {
|
|
45366
|
+
...state,
|
|
45367
|
+
settleSeen: state.settleSeen + 1
|
|
45368
|
+
};
|
|
45369
|
+
const window = [...state.baselineWindow, rssBytes];
|
|
45370
|
+
if (window.length < policy.baselineSampleCount) return {
|
|
45371
|
+
...state,
|
|
45372
|
+
baselineWindow: window
|
|
45373
|
+
};
|
|
45374
|
+
return {
|
|
45375
|
+
...state,
|
|
45376
|
+
baselineWindow: window,
|
|
45377
|
+
baselineBytes: median(window)
|
|
45378
|
+
};
|
|
45379
|
+
}
|
|
45380
|
+
/** The one place a pool's restart threshold is computed. */
|
|
45381
|
+
function poolMemoryThreshold(baselineBytes, policy) {
|
|
45382
|
+
if (baselineBytes === null) return policy.ceilingBytes;
|
|
45383
|
+
return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
|
|
45384
|
+
}
|
|
45385
|
+
function pruneRestarts(restartsAt, nowMs, policy) {
|
|
45386
|
+
return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
|
|
45387
|
+
}
|
|
45388
|
+
/**
|
|
45389
|
+
* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
|
|
45390
|
+
* The caller performs (and logs) the restart, then commits it via
|
|
45391
|
+
* {@link commitWatchdogRestart}.
|
|
45392
|
+
*/
|
|
45393
|
+
function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
|
|
45394
|
+
const next = advanceBaseline(state, rssBytes, policy);
|
|
45395
|
+
const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
|
|
45396
|
+
if (rssBytes <= thresholdBytes) return {
|
|
45397
|
+
state: next,
|
|
45398
|
+
action: next.baselineBytes === null ? "baseline-pending" : "ok",
|
|
45399
|
+
baselineBytes: next.baselineBytes,
|
|
45400
|
+
thresholdBytes
|
|
45401
|
+
};
|
|
45402
|
+
const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
|
|
45403
|
+
const pruned = {
|
|
45404
|
+
...next,
|
|
45405
|
+
restartsAt: inWindow
|
|
45406
|
+
};
|
|
45407
|
+
if (inWindow.length >= policy.maxRestartsPerWindow) return {
|
|
45408
|
+
state: pruned,
|
|
45409
|
+
action: "exhausted",
|
|
45410
|
+
baselineBytes: next.baselineBytes,
|
|
45411
|
+
thresholdBytes
|
|
45412
|
+
};
|
|
45413
|
+
if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
|
|
45414
|
+
state: pruned,
|
|
45415
|
+
action: "cooldown",
|
|
45416
|
+
baselineBytes: next.baselineBytes,
|
|
45417
|
+
thresholdBytes
|
|
45418
|
+
};
|
|
45419
|
+
return {
|
|
45420
|
+
state: pruned,
|
|
45421
|
+
action: "restart",
|
|
45422
|
+
baselineBytes: next.baselineBytes,
|
|
45423
|
+
thresholdBytes
|
|
45424
|
+
};
|
|
45425
|
+
}
|
|
45426
|
+
/** Record a performed watchdog restart: budget consumed, baseline reset —
|
|
45427
|
+
* the recreated pool is a NEW process and gets a fresh settle window. */
|
|
45428
|
+
function commitWatchdogRestart(state, nowMs) {
|
|
45429
|
+
return {
|
|
45430
|
+
...initialPoolMemoryState(),
|
|
45431
|
+
restartsAt: [...state.restartsAt, nowMs],
|
|
45432
|
+
lastRestartAt: nowMs
|
|
45433
|
+
};
|
|
45434
|
+
}
|
|
45435
|
+
/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
|
|
45436
|
+
* kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
|
|
45437
|
+
* restart budget is kept: it bounds the WATCHDOG, not the pool. */
|
|
45438
|
+
function resetPoolBaseline(state) {
|
|
45439
|
+
return {
|
|
45440
|
+
...initialPoolMemoryState(),
|
|
45441
|
+
restartsAt: state.restartsAt,
|
|
45442
|
+
lastRestartAt: state.lastRestartAt
|
|
45443
|
+
};
|
|
45444
|
+
}
|
|
45445
|
+
/**
|
|
45446
|
+
* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
|
|
45447
|
+
* several pools at once (e.g. after a model rollout grew every baseline)
|
|
45448
|
+
* would take detection down on every camera simultaneously; one per sweep
|
|
45449
|
+
* means the survivors keep serving while the worst offender reloads, and the
|
|
45450
|
+
* next sweep (one interval later) takes the next one.
|
|
45451
|
+
*/
|
|
45452
|
+
function pickRestartCandidate(candidates) {
|
|
45453
|
+
if (candidates.length === 0) return null;
|
|
45454
|
+
let worst = candidates[0];
|
|
45455
|
+
for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
|
|
45456
|
+
return worst.key;
|
|
45457
|
+
}
|
|
45458
|
+
var PoolMemoryWatchdog = class {
|
|
45459
|
+
pools = /* @__PURE__ */ new Map();
|
|
45460
|
+
timer = null;
|
|
45461
|
+
sweeping = false;
|
|
45462
|
+
stopped = false;
|
|
45463
|
+
opts;
|
|
45464
|
+
now;
|
|
45465
|
+
setTimer;
|
|
45466
|
+
clearTimer;
|
|
45467
|
+
constructor(opts) {
|
|
45468
|
+
this.opts = opts;
|
|
45469
|
+
this.now = opts.now ?? (() => Date.now());
|
|
45470
|
+
this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
45471
|
+
this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
|
|
45472
|
+
}
|
|
45473
|
+
start() {
|
|
45474
|
+
if (this.timer || this.stopped) return;
|
|
45475
|
+
this.arm();
|
|
45476
|
+
}
|
|
45477
|
+
stop() {
|
|
45478
|
+
this.stopped = true;
|
|
45479
|
+
if (this.timer) {
|
|
45480
|
+
this.clearTimer(this.timer);
|
|
45481
|
+
this.timer = null;
|
|
45482
|
+
}
|
|
45483
|
+
}
|
|
45484
|
+
arm() {
|
|
45485
|
+
const timer = this.setTimer(() => {
|
|
45486
|
+
this.sweep().finally(() => {
|
|
45487
|
+
if (!this.stopped) this.arm();
|
|
45488
|
+
});
|
|
45489
|
+
}, this.opts.policy.sampleIntervalMs);
|
|
45490
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
45491
|
+
this.timer = timer;
|
|
45492
|
+
}
|
|
45493
|
+
/** One sweep: sample → log every pool → restart at most one. Public so the
|
|
45494
|
+
* loop is testable without fake global timers. */
|
|
45495
|
+
async sweep() {
|
|
45496
|
+
if (this.sweeping) return;
|
|
45497
|
+
this.sweeping = true;
|
|
45498
|
+
try {
|
|
45499
|
+
await this.doSweep();
|
|
45500
|
+
} catch (err) {
|
|
45501
|
+
this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
45502
|
+
} finally {
|
|
45503
|
+
this.sweeping = false;
|
|
45504
|
+
}
|
|
45505
|
+
}
|
|
45506
|
+
async doSweep() {
|
|
45507
|
+
const nowMs = this.now();
|
|
45508
|
+
const samples = await this.opts.sample();
|
|
45509
|
+
const candidates = [];
|
|
45510
|
+
const verdictByKey = /* @__PURE__ */ new Map();
|
|
45511
|
+
for (const t of samples) {
|
|
45512
|
+
const tracked = this.pools.get(t.key);
|
|
45513
|
+
let state = tracked?.state ?? initialPoolMemoryState();
|
|
45514
|
+
if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
|
|
45515
|
+
this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
|
|
45516
|
+
poolKey: t.key,
|
|
45517
|
+
previousPids: tracked.pids,
|
|
45518
|
+
pids: t.pids
|
|
45519
|
+
} });
|
|
45520
|
+
state = resetPoolBaseline(state);
|
|
45521
|
+
}
|
|
45522
|
+
const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
|
|
45523
|
+
this.pools.set(t.key, {
|
|
45524
|
+
state: verdict.state,
|
|
45525
|
+
pids: t.pids
|
|
45526
|
+
});
|
|
45527
|
+
verdictByKey.set(t.key, verdict);
|
|
45528
|
+
this.opts.log.info("pool memory", { meta: {
|
|
45529
|
+
poolKey: t.key,
|
|
45530
|
+
pids: t.pids,
|
|
45531
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
45532
|
+
baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
|
|
45533
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
45534
|
+
action: verdict.action,
|
|
45535
|
+
...t.meta
|
|
45536
|
+
} });
|
|
45537
|
+
if (verdict.action === "restart") candidates.push({
|
|
45538
|
+
key: t.key,
|
|
45539
|
+
rssBytes: t.rssBytes,
|
|
45540
|
+
thresholdBytes: verdict.thresholdBytes
|
|
45541
|
+
});
|
|
45542
|
+
else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
|
|
45543
|
+
poolKey: t.key,
|
|
45544
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
45545
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
45546
|
+
cooldownMs: this.opts.policy.cooldownMs
|
|
45547
|
+
} });
|
|
45548
|
+
else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
|
|
45549
|
+
poolKey: t.key,
|
|
45550
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
45551
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
45552
|
+
maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
|
|
45553
|
+
restartWindowMs: this.opts.policy.restartWindowMs
|
|
45554
|
+
} });
|
|
45555
|
+
}
|
|
45556
|
+
for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
|
|
45557
|
+
const pickedKey = pickRestartCandidate(candidates);
|
|
45558
|
+
if (pickedKey === null) return;
|
|
45559
|
+
const picked = candidates.find((c) => c.key === pickedKey);
|
|
45560
|
+
const verdict = verdictByKey.get(pickedKey);
|
|
45561
|
+
const tracked = this.pools.get(pickedKey);
|
|
45562
|
+
const restartsInWindow = tracked.state.restartsAt.length + 1;
|
|
45563
|
+
this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
|
|
45564
|
+
poolKey: pickedKey,
|
|
45565
|
+
pids: tracked.pids,
|
|
45566
|
+
rssMb: Math.round(picked.rssBytes / MB),
|
|
45567
|
+
thresholdMb: Math.round(picked.thresholdBytes / MB),
|
|
45568
|
+
baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
|
|
45569
|
+
restartsInWindow,
|
|
45570
|
+
deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
|
|
45571
|
+
} });
|
|
45572
|
+
try {
|
|
45573
|
+
await this.opts.restart(pickedKey);
|
|
45574
|
+
this.pools.set(pickedKey, {
|
|
45575
|
+
state: commitWatchdogRestart(tracked.state, this.now()),
|
|
45576
|
+
pids: []
|
|
45577
|
+
});
|
|
45578
|
+
} catch (err) {
|
|
45579
|
+
this.opts.log.error("pool memory watchdog restart failed", { meta: {
|
|
45580
|
+
poolKey: pickedKey,
|
|
45581
|
+
error: err instanceof Error ? err.message : String(err)
|
|
45582
|
+
} });
|
|
45583
|
+
}
|
|
45584
|
+
}
|
|
45585
|
+
};
|
|
45586
|
+
function samePids(a, b) {
|
|
45587
|
+
if (a.length !== b.length) return false;
|
|
45588
|
+
const sortedA = [...a].sort((x, y) => x - y);
|
|
45589
|
+
const sortedB = [...b].sort((x, y) => x - y);
|
|
45590
|
+
return sortedA.every((v, i) => v === sortedB[i]);
|
|
45591
|
+
}
|
|
45592
|
+
//#endregion
|
|
45253
45593
|
//#region src/utils/privacy-grid-raster.ts
|
|
45254
45594
|
/**
|
|
45255
45595
|
* Rasterize a list of normalized rects into a row-major boolean cell grid.
|
|
@@ -46149,6 +46489,7 @@ exports.DEFAULT_EVENTS_BAND_BUFFER_SEC = DEFAULT_EVENTS_BAND_BUFFER_SEC;
|
|
|
46149
46489
|
exports.DEFAULT_EVENT_COLOR = DEFAULT_EVENT_COLOR;
|
|
46150
46490
|
exports.DEFAULT_FEATURES = DEFAULT_FEATURES;
|
|
46151
46491
|
exports.DEFAULT_NATIVE_LEASE_SETTINGS = DEFAULT_NATIVE_LEASE_SETTINGS;
|
|
46492
|
+
exports.DEFAULT_POOL_MEMORY_POLICY = DEFAULT_POOL_MEMORY_POLICY;
|
|
46152
46493
|
exports.DEFAULT_RECORDING_PROFILES = DEFAULT_RECORDING_PROFILES;
|
|
46153
46494
|
exports.DEFAULT_RETENTION = DEFAULT_RETENTION;
|
|
46154
46495
|
exports.DEFAULT_RUNTIME_STATE_DURABILITY = require_sleep.DEFAULT_RUNTIME_STATE_DURABILITY;
|
|
@@ -46340,6 +46681,7 @@ exports.LogStreamEntrySchema = LogStreamEntrySchema;
|
|
|
46340
46681
|
exports.LoginMethodContributionSchema = LoginMethodContributionSchema;
|
|
46341
46682
|
exports.LoginStageEnum = LoginStageEnum;
|
|
46342
46683
|
exports.MACRO_LABELS = MACRO_LABELS;
|
|
46684
|
+
exports.MAX_CLIP_EVENT_IDS = MAX_CLIP_EVENT_IDS;
|
|
46343
46685
|
exports.MAX_CONDITION_DEPTH = MAX_CONDITION_DEPTH;
|
|
46344
46686
|
exports.MAX_CONDITION_LEAVES = MAX_CONDITION_LEAVES;
|
|
46345
46687
|
exports.MAX_EXPRESSION_AST_NODES = MAX_EXPRESSION_AST_NODES;
|
|
@@ -46545,6 +46887,7 @@ exports.PipelineValidationIssueSchema = PipelineValidationIssueSchema;
|
|
|
46545
46887
|
exports.PipelineValidationResultSchema = PipelineValidationResultSchema;
|
|
46546
46888
|
exports.PlaceholderReasonSchema = PlaceholderReasonSchema;
|
|
46547
46889
|
exports.PolygonPointSchema = PolygonPointSchema;
|
|
46890
|
+
exports.PoolMemoryWatchdog = PoolMemoryWatchdog;
|
|
46548
46891
|
exports.PowerMeterStatusSchema = PowerMeterStatusSchema;
|
|
46549
46892
|
exports.PresenceStatusSchema = PresenceStatusSchema;
|
|
46550
46893
|
exports.PressureSensorStatusSchema = PressureSensorStatusSchema;
|
|
@@ -46912,6 +47255,7 @@ exports.collectHydratedFieldEntries = require_sleep.collectHydratedFieldEntries;
|
|
|
46912
47255
|
exports.collectHydratedFieldValues = require_sleep.collectHydratedFieldValues;
|
|
46913
47256
|
exports.colorCapability = colorCapability;
|
|
46914
47257
|
exports.colorForKind = colorForKind;
|
|
47258
|
+
exports.commitWatchdogRestart = commitWatchdogRestart;
|
|
46915
47259
|
exports.compileExpression = compileExpression;
|
|
46916
47260
|
exports.compileExpressionSafe = compileExpressionSafe;
|
|
46917
47261
|
exports.composeSwitchedOff = composeSwitchedOff;
|
|
@@ -46981,6 +47325,7 @@ exports.enumerateSchemaFields = enumerateSchemaFields;
|
|
|
46981
47325
|
exports.errMsg = require_err_msg.errMsg;
|
|
46982
47326
|
exports.evaluateAst = evaluateAst;
|
|
46983
47327
|
exports.evaluateExpressionSource = evaluateExpressionSource;
|
|
47328
|
+
exports.evaluatePoolMemory = evaluatePoolMemory;
|
|
46984
47329
|
exports.evaluateZoneRules = evaluateZoneRules;
|
|
46985
47330
|
exports.event = require_sleep.event;
|
|
46986
47331
|
exports.eventEmitterCapability = eventEmitterCapability;
|
|
@@ -47010,6 +47355,7 @@ exports.humiditySensorCapability = humiditySensorCapability;
|
|
|
47010
47355
|
exports.hydrateSchema = require_sleep.hydrateSchema;
|
|
47011
47356
|
exports.imageCapability = imageCapability;
|
|
47012
47357
|
exports.imageSettingsCapability = imageSettingsCapability;
|
|
47358
|
+
exports.initialPoolMemoryState = initialPoolMemoryState;
|
|
47013
47359
|
exports.integrationsCapability = integrationsCapability;
|
|
47014
47360
|
exports.intercomCapability = intercomCapability;
|
|
47015
47361
|
exports.invocationFromEncodeProfile = require_canonical_hash.invocationFromEncodeProfile;
|
|
@@ -47092,6 +47438,7 @@ exports.parseExpression = parseExpression;
|
|
|
47092
47438
|
exports.parseJsonArray = require_sleep.parseJsonArray;
|
|
47093
47439
|
exports.parseJsonObject = require_sleep.parseJsonObject;
|
|
47094
47440
|
exports.parseJsonUnknown = require_sleep.parseJsonUnknown;
|
|
47441
|
+
exports.parseProcStatus = parseProcStatus;
|
|
47095
47442
|
exports.parseProfileBrokerId = require_sleep.parseProfileBrokerId;
|
|
47096
47443
|
exports.parseStreamParamsFormPatch = parseStreamParamsFormPatch;
|
|
47097
47444
|
exports.patchAudio = patchAudio;
|
|
@@ -47100,6 +47447,7 @@ exports.pickAccessoryControl = pickAccessoryControl;
|
|
|
47100
47447
|
exports.pickDetailCropConvention = pickDetailCropConvention;
|
|
47101
47448
|
exports.pickNativeLeaseOverride = pickNativeLeaseOverride;
|
|
47102
47449
|
exports.pickPreferredRtspEntry = pickPreferredRtspEntry;
|
|
47450
|
+
exports.pickRestartCandidate = pickRestartCandidate;
|
|
47103
47451
|
exports.pickVideoEncoder = require_canonical_hash.pickVideoEncoder;
|
|
47104
47452
|
exports.pickerForCondition = pickerForCondition;
|
|
47105
47453
|
exports.pipelineAnalyticsCapability = pipelineAnalyticsCapability;
|
|
@@ -47108,6 +47456,7 @@ exports.pipelineOrchestratorCapability = pipelineOrchestratorCapability;
|
|
|
47108
47456
|
exports.pipelineRunnerCapability = pipelineRunnerCapability;
|
|
47109
47457
|
exports.plateGalleryCapability = plateGalleryCapability;
|
|
47110
47458
|
exports.platformProbeCapability = platformProbeCapability;
|
|
47459
|
+
exports.poolMemoryThreshold = poolMemoryThreshold;
|
|
47111
47460
|
exports.powerMeterCapability = powerMeterCapability;
|
|
47112
47461
|
exports.prepareNotification = prepareNotification;
|
|
47113
47462
|
exports.presenceCapability = presenceCapability;
|
|
@@ -47129,6 +47478,7 @@ exports.recordingCapability = recordingCapability;
|
|
|
47129
47478
|
exports.recordingExportCapability = recordingExportCapability;
|
|
47130
47479
|
exports.rectsToCells = rectsToCells;
|
|
47131
47480
|
exports.requiresPython = requiresPython;
|
|
47481
|
+
exports.resetPoolBaseline = resetPoolBaseline;
|
|
47132
47482
|
exports.resolveAddonExecution = resolveAddonExecution;
|
|
47133
47483
|
exports.resolveAddonGroup = resolveAddonGroup;
|
|
47134
47484
|
exports.resolveAddonPlacement = resolveAddonPlacement;
|
|
@@ -47142,6 +47492,7 @@ exports.resolveFormat = resolveFormat;
|
|
|
47142
47492
|
exports.resolveHydratedFieldValue = require_sleep.resolveHydratedFieldValue;
|
|
47143
47493
|
exports.resolveModelFormat = resolveModelFormat;
|
|
47144
47494
|
exports.resolveMutate = resolveMutate;
|
|
47495
|
+
exports.resolvePoolMemoryPolicy = resolvePoolMemoryPolicy;
|
|
47145
47496
|
exports.resolveRecordingProfiles = resolveRecordingProfiles;
|
|
47146
47497
|
exports.resolveRunnerId = resolveRunnerId;
|
|
47147
47498
|
exports.resolveScrubThumbnailGeometry = resolveScrubThumbnailGeometry;
|