@camstack/addon-post-analysis 1.2.92 → 1.2.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{dist-BfrtqGyh.js → dist-BdojoP7i.js} +346 -2
- package/dist/{dist-VnaTfwAL.mjs → dist-C-_TL-I1.mjs} +329 -3
- package/dist/embedding-encoder/index.js +171 -4
- package/dist/embedding-encoder/index.mjs +171 -4
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-BIDGLXRs.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-CeFPqm3h.mjs} +3 -3
- package/dist/pipeline-analytics/{hostInit-CpWvyhen.mjs → hostInit-BExsGl4h.mjs} +3 -3
- package/dist/pipeline-analytics/index.js +51 -2
- package/dist/pipeline-analytics/index.mjs +51 -2
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/package.json +3 -2
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
//#region ../types/dist/event-category-
|
|
1
|
+
//#region ../types/dist/event-category-XfKNtfCc.mjs
|
|
2
2
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
3
3
|
EventCategory["SystemBoot"] = "system.boot";
|
|
4
4
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -37,6 +37,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
|
37
37
|
EventCategory["AddonInstalled"] = "addon.installed";
|
|
38
38
|
EventCategory["AddonUninstalled"] = "addon.uninstalled";
|
|
39
39
|
EventCategory["AddonCrashed"] = "addon.crashed";
|
|
40
|
+
/**
|
|
41
|
+
* A RUNNER the D6 crash circuit-breaker gave up on — terminal.
|
|
42
|
+
*
|
|
43
|
+
* `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
|
|
44
|
+
* respawned"); this is the one that never resolves itself. Emitted exactly
|
|
45
|
+
* once per trip, by `process-service.ts`, carrying the node, the runner, the
|
|
46
|
+
* addons it hosted and the crash count that tripped the breaker.
|
|
47
|
+
*
|
|
48
|
+
* It exists because a runner marked terminally `failed` used to be SILENT:
|
|
49
|
+
* on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
|
|
50
|
+
* errors, the breaker stopped respawning it (correctly), `/health` kept
|
|
51
|
+
* returning 200, and fleet-wide recording was gone for 3h15 before the
|
|
52
|
+
* operator's phone told him. The Notification Center's system-event intake
|
|
53
|
+
* consumes this category and maps it to the `addon-crash-loop` kind.
|
|
54
|
+
*/
|
|
55
|
+
EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
|
|
40
56
|
EventCategory["AddonError"] = "addon.error";
|
|
41
57
|
EventCategory["AddonPageReady"] = "addon.page-ready";
|
|
42
58
|
EventCategory["AddonWidgetReady"] = "addon.widget-ready";
|
|
@@ -14814,6 +14830,7 @@ var NcSystemEventKindSchema = _enum([
|
|
|
14814
14830
|
"node-offline",
|
|
14815
14831
|
"node-inference-unavailable",
|
|
14816
14832
|
"detection-blind",
|
|
14833
|
+
"addon-crash-loop",
|
|
14817
14834
|
"addon-update-available",
|
|
14818
14835
|
"server-update-available",
|
|
14819
14836
|
"alarm-triggered",
|
|
@@ -15939,6 +15956,10 @@ var NC_CONDITION_CATALOG = [
|
|
|
15939
15956
|
value: "detection-blind",
|
|
15940
15957
|
label: "Camera detecting nothing"
|
|
15941
15958
|
},
|
|
15959
|
+
{
|
|
15960
|
+
value: "addon-crash-loop",
|
|
15961
|
+
label: "Addon stopped after repeated crashes"
|
|
15962
|
+
},
|
|
15942
15963
|
{
|
|
15943
15964
|
value: "addon-update-available",
|
|
15944
15965
|
label: "Addon update available"
|
|
@@ -21473,6 +21494,14 @@ var ClipSchema = object({
|
|
|
21473
21494
|
endMs: number()
|
|
21474
21495
|
}),
|
|
21475
21496
|
/**
|
|
21497
|
+
* Distinct object classes attached to this visit (`person`, `car`, …),
|
|
21498
|
+
* dominant first, capped at {@link MAX_CLIP_LABELS}. The ribbon renders these
|
|
21499
|
+
* instead of the bare kind — "Object" tells the operator nothing a colour bar
|
|
21500
|
+
* did not. Absent (never empty) when the visit attached no classified object
|
|
21501
|
+
* event, so a motion/audio-only visit keeps its kind label.
|
|
21502
|
+
*/
|
|
21503
|
+
labels: array(string()).max(3).optional(),
|
|
21504
|
+
/**
|
|
21476
21505
|
* Lazy thumbnail URL, never inlined.
|
|
21477
21506
|
*
|
|
21478
21507
|
* Recording-derived clips (events-mode keep-window, and the prepared
|
|
@@ -38537,7 +38566,7 @@ function systemEventFilterApplies(kind, filter) {
|
|
|
38537
38566
|
switch (filter) {
|
|
38538
38567
|
case "deviceIds": return kind.startsWith("device-") || kind.startsWith("stream-") || kind === "detection-blind" || kind === "alarm-triggered" || kind === "export-completed";
|
|
38539
38568
|
case "deviceTypes": return kind.startsWith("device-") || kind === "detection-blind";
|
|
38540
|
-
case "nodeIds": return kind.startsWith("node-") || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
|
|
38569
|
+
case "nodeIds": return kind.startsWith("node-") || kind === "addon-crash-loop" || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
|
|
38541
38570
|
case "packageNames": return kind.endsWith("update-available") || kind === "addon-updated" || kind === "server-updated";
|
|
38542
38571
|
}
|
|
38543
38572
|
}
|
|
@@ -38994,6 +39023,303 @@ function cosineSimilarity(a, b) {
|
|
|
38994
39023
|
function hfModelUrl(repo, path) {
|
|
38995
39024
|
return `https://huggingface.co/${repo}/resolve/main/${path}`;
|
|
38996
39025
|
}
|
|
39026
|
+
var MB = 1024 * 1024;
|
|
39027
|
+
var DEFAULT_POOL_MEMORY_POLICY = {
|
|
39028
|
+
sampleIntervalMs: 6e4,
|
|
39029
|
+
baselineSettleSamples: 5,
|
|
39030
|
+
baselineSampleCount: 3,
|
|
39031
|
+
restartMultiple: 4,
|
|
39032
|
+
floorBytes: 1024 * MB,
|
|
39033
|
+
ceilingBytes: 3072 * MB,
|
|
39034
|
+
cooldownMs: 30 * 6e4,
|
|
39035
|
+
maxRestartsPerWindow: 6,
|
|
39036
|
+
restartWindowMs: 1440 * 6e4
|
|
39037
|
+
};
|
|
39038
|
+
/**
|
|
39039
|
+
* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
|
|
39040
|
+
* absent values keep the default — an operator typo must never disable the
|
|
39041
|
+
* bound or set it to zero.
|
|
39042
|
+
*/
|
|
39043
|
+
function resolvePoolMemoryPolicy(env) {
|
|
39044
|
+
const num = (key, fallback, min) => {
|
|
39045
|
+
const raw = env[key];
|
|
39046
|
+
if (raw === void 0) return fallback;
|
|
39047
|
+
const parsed = Number(raw);
|
|
39048
|
+
return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
|
|
39049
|
+
};
|
|
39050
|
+
const d = DEFAULT_POOL_MEMORY_POLICY;
|
|
39051
|
+
return {
|
|
39052
|
+
sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
|
|
39053
|
+
baselineSettleSamples: d.baselineSettleSamples,
|
|
39054
|
+
baselineSampleCount: d.baselineSampleCount,
|
|
39055
|
+
restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
|
|
39056
|
+
floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
|
|
39057
|
+
ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
|
|
39058
|
+
cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
|
|
39059
|
+
maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
|
|
39060
|
+
restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
|
|
39061
|
+
};
|
|
39062
|
+
}
|
|
39063
|
+
/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
|
|
39064
|
+
* Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
|
|
39065
|
+
function parseProcStatus(text) {
|
|
39066
|
+
const kb = (label) => {
|
|
39067
|
+
const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
|
|
39068
|
+
return match ? Number(match[1]) * 1024 : null;
|
|
39069
|
+
};
|
|
39070
|
+
const rssBytes = kb("VmRSS");
|
|
39071
|
+
if (rssBytes === null) return null;
|
|
39072
|
+
const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
|
|
39073
|
+
return {
|
|
39074
|
+
rssBytes,
|
|
39075
|
+
vmBytes: kb("VmSize") ?? 0,
|
|
39076
|
+
hwmBytes: kb("VmHWM") ?? 0,
|
|
39077
|
+
swapBytes: kb("VmSwap") ?? 0,
|
|
39078
|
+
threads: threadsMatch ? Number(threadsMatch[1]) : 0
|
|
39079
|
+
};
|
|
39080
|
+
}
|
|
39081
|
+
function initialPoolMemoryState() {
|
|
39082
|
+
return {
|
|
39083
|
+
settleSeen: 0,
|
|
39084
|
+
baselineWindow: [],
|
|
39085
|
+
baselineBytes: null,
|
|
39086
|
+
restartsAt: [],
|
|
39087
|
+
lastRestartAt: null
|
|
39088
|
+
};
|
|
39089
|
+
}
|
|
39090
|
+
function median(values) {
|
|
39091
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
39092
|
+
return sorted[Math.floor(sorted.length / 2)];
|
|
39093
|
+
}
|
|
39094
|
+
function advanceBaseline(state, rssBytes, policy) {
|
|
39095
|
+
if (state.baselineBytes !== null) return state;
|
|
39096
|
+
if (state.settleSeen < policy.baselineSettleSamples) return {
|
|
39097
|
+
...state,
|
|
39098
|
+
settleSeen: state.settleSeen + 1
|
|
39099
|
+
};
|
|
39100
|
+
const window = [...state.baselineWindow, rssBytes];
|
|
39101
|
+
if (window.length < policy.baselineSampleCount) return {
|
|
39102
|
+
...state,
|
|
39103
|
+
baselineWindow: window
|
|
39104
|
+
};
|
|
39105
|
+
return {
|
|
39106
|
+
...state,
|
|
39107
|
+
baselineWindow: window,
|
|
39108
|
+
baselineBytes: median(window)
|
|
39109
|
+
};
|
|
39110
|
+
}
|
|
39111
|
+
/** The one place a pool's restart threshold is computed. */
|
|
39112
|
+
function poolMemoryThreshold(baselineBytes, policy) {
|
|
39113
|
+
if (baselineBytes === null) return policy.ceilingBytes;
|
|
39114
|
+
return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
|
|
39115
|
+
}
|
|
39116
|
+
function pruneRestarts(restartsAt, nowMs, policy) {
|
|
39117
|
+
return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
|
|
39118
|
+
}
|
|
39119
|
+
/**
|
|
39120
|
+
* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
|
|
39121
|
+
* The caller performs (and logs) the restart, then commits it via
|
|
39122
|
+
* {@link commitWatchdogRestart}.
|
|
39123
|
+
*/
|
|
39124
|
+
function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
|
|
39125
|
+
const next = advanceBaseline(state, rssBytes, policy);
|
|
39126
|
+
const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
|
|
39127
|
+
if (rssBytes <= thresholdBytes) return {
|
|
39128
|
+
state: next,
|
|
39129
|
+
action: next.baselineBytes === null ? "baseline-pending" : "ok",
|
|
39130
|
+
baselineBytes: next.baselineBytes,
|
|
39131
|
+
thresholdBytes
|
|
39132
|
+
};
|
|
39133
|
+
const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
|
|
39134
|
+
const pruned = {
|
|
39135
|
+
...next,
|
|
39136
|
+
restartsAt: inWindow
|
|
39137
|
+
};
|
|
39138
|
+
if (inWindow.length >= policy.maxRestartsPerWindow) return {
|
|
39139
|
+
state: pruned,
|
|
39140
|
+
action: "exhausted",
|
|
39141
|
+
baselineBytes: next.baselineBytes,
|
|
39142
|
+
thresholdBytes
|
|
39143
|
+
};
|
|
39144
|
+
if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
|
|
39145
|
+
state: pruned,
|
|
39146
|
+
action: "cooldown",
|
|
39147
|
+
baselineBytes: next.baselineBytes,
|
|
39148
|
+
thresholdBytes
|
|
39149
|
+
};
|
|
39150
|
+
return {
|
|
39151
|
+
state: pruned,
|
|
39152
|
+
action: "restart",
|
|
39153
|
+
baselineBytes: next.baselineBytes,
|
|
39154
|
+
thresholdBytes
|
|
39155
|
+
};
|
|
39156
|
+
}
|
|
39157
|
+
/** Record a performed watchdog restart: budget consumed, baseline reset —
|
|
39158
|
+
* the recreated pool is a NEW process and gets a fresh settle window. */
|
|
39159
|
+
function commitWatchdogRestart(state, nowMs) {
|
|
39160
|
+
return {
|
|
39161
|
+
...initialPoolMemoryState(),
|
|
39162
|
+
restartsAt: [...state.restartsAt, nowMs],
|
|
39163
|
+
lastRestartAt: nowMs
|
|
39164
|
+
};
|
|
39165
|
+
}
|
|
39166
|
+
/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
|
|
39167
|
+
* kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
|
|
39168
|
+
* restart budget is kept: it bounds the WATCHDOG, not the pool. */
|
|
39169
|
+
function resetPoolBaseline(state) {
|
|
39170
|
+
return {
|
|
39171
|
+
...initialPoolMemoryState(),
|
|
39172
|
+
restartsAt: state.restartsAt,
|
|
39173
|
+
lastRestartAt: state.lastRestartAt
|
|
39174
|
+
};
|
|
39175
|
+
}
|
|
39176
|
+
/**
|
|
39177
|
+
* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
|
|
39178
|
+
* several pools at once (e.g. after a model rollout grew every baseline)
|
|
39179
|
+
* would take detection down on every camera simultaneously; one per sweep
|
|
39180
|
+
* means the survivors keep serving while the worst offender reloads, and the
|
|
39181
|
+
* next sweep (one interval later) takes the next one.
|
|
39182
|
+
*/
|
|
39183
|
+
function pickRestartCandidate(candidates) {
|
|
39184
|
+
if (candidates.length === 0) return null;
|
|
39185
|
+
let worst = candidates[0];
|
|
39186
|
+
for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
|
|
39187
|
+
return worst.key;
|
|
39188
|
+
}
|
|
39189
|
+
var PoolMemoryWatchdog = class {
|
|
39190
|
+
pools = /* @__PURE__ */ new Map();
|
|
39191
|
+
timer = null;
|
|
39192
|
+
sweeping = false;
|
|
39193
|
+
stopped = false;
|
|
39194
|
+
opts;
|
|
39195
|
+
now;
|
|
39196
|
+
setTimer;
|
|
39197
|
+
clearTimer;
|
|
39198
|
+
constructor(opts) {
|
|
39199
|
+
this.opts = opts;
|
|
39200
|
+
this.now = opts.now ?? (() => Date.now());
|
|
39201
|
+
this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
39202
|
+
this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
|
|
39203
|
+
}
|
|
39204
|
+
start() {
|
|
39205
|
+
if (this.timer || this.stopped) return;
|
|
39206
|
+
this.arm();
|
|
39207
|
+
}
|
|
39208
|
+
stop() {
|
|
39209
|
+
this.stopped = true;
|
|
39210
|
+
if (this.timer) {
|
|
39211
|
+
this.clearTimer(this.timer);
|
|
39212
|
+
this.timer = null;
|
|
39213
|
+
}
|
|
39214
|
+
}
|
|
39215
|
+
arm() {
|
|
39216
|
+
const timer = this.setTimer(() => {
|
|
39217
|
+
this.sweep().finally(() => {
|
|
39218
|
+
if (!this.stopped) this.arm();
|
|
39219
|
+
});
|
|
39220
|
+
}, this.opts.policy.sampleIntervalMs);
|
|
39221
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
39222
|
+
this.timer = timer;
|
|
39223
|
+
}
|
|
39224
|
+
/** One sweep: sample → log every pool → restart at most one. Public so the
|
|
39225
|
+
* loop is testable without fake global timers. */
|
|
39226
|
+
async sweep() {
|
|
39227
|
+
if (this.sweeping) return;
|
|
39228
|
+
this.sweeping = true;
|
|
39229
|
+
try {
|
|
39230
|
+
await this.doSweep();
|
|
39231
|
+
} catch (err) {
|
|
39232
|
+
this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
39233
|
+
} finally {
|
|
39234
|
+
this.sweeping = false;
|
|
39235
|
+
}
|
|
39236
|
+
}
|
|
39237
|
+
async doSweep() {
|
|
39238
|
+
const nowMs = this.now();
|
|
39239
|
+
const samples = await this.opts.sample();
|
|
39240
|
+
const candidates = [];
|
|
39241
|
+
const verdictByKey = /* @__PURE__ */ new Map();
|
|
39242
|
+
for (const t of samples) {
|
|
39243
|
+
const tracked = this.pools.get(t.key);
|
|
39244
|
+
let state = tracked?.state ?? initialPoolMemoryState();
|
|
39245
|
+
if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
|
|
39246
|
+
this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
|
|
39247
|
+
poolKey: t.key,
|
|
39248
|
+
previousPids: tracked.pids,
|
|
39249
|
+
pids: t.pids
|
|
39250
|
+
} });
|
|
39251
|
+
state = resetPoolBaseline(state);
|
|
39252
|
+
}
|
|
39253
|
+
const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
|
|
39254
|
+
this.pools.set(t.key, {
|
|
39255
|
+
state: verdict.state,
|
|
39256
|
+
pids: t.pids
|
|
39257
|
+
});
|
|
39258
|
+
verdictByKey.set(t.key, verdict);
|
|
39259
|
+
this.opts.log.info("pool memory", { meta: {
|
|
39260
|
+
poolKey: t.key,
|
|
39261
|
+
pids: t.pids,
|
|
39262
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
39263
|
+
baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
|
|
39264
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
39265
|
+
action: verdict.action,
|
|
39266
|
+
...t.meta
|
|
39267
|
+
} });
|
|
39268
|
+
if (verdict.action === "restart") candidates.push({
|
|
39269
|
+
key: t.key,
|
|
39270
|
+
rssBytes: t.rssBytes,
|
|
39271
|
+
thresholdBytes: verdict.thresholdBytes
|
|
39272
|
+
});
|
|
39273
|
+
else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
|
|
39274
|
+
poolKey: t.key,
|
|
39275
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
39276
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
39277
|
+
cooldownMs: this.opts.policy.cooldownMs
|
|
39278
|
+
} });
|
|
39279
|
+
else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
|
|
39280
|
+
poolKey: t.key,
|
|
39281
|
+
rssMb: Math.round(t.rssBytes / MB),
|
|
39282
|
+
thresholdMb: Math.round(verdict.thresholdBytes / MB),
|
|
39283
|
+
maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
|
|
39284
|
+
restartWindowMs: this.opts.policy.restartWindowMs
|
|
39285
|
+
} });
|
|
39286
|
+
}
|
|
39287
|
+
for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
|
|
39288
|
+
const pickedKey = pickRestartCandidate(candidates);
|
|
39289
|
+
if (pickedKey === null) return;
|
|
39290
|
+
const picked = candidates.find((c) => c.key === pickedKey);
|
|
39291
|
+
const verdict = verdictByKey.get(pickedKey);
|
|
39292
|
+
const tracked = this.pools.get(pickedKey);
|
|
39293
|
+
const restartsInWindow = tracked.state.restartsAt.length + 1;
|
|
39294
|
+
this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
|
|
39295
|
+
poolKey: pickedKey,
|
|
39296
|
+
pids: tracked.pids,
|
|
39297
|
+
rssMb: Math.round(picked.rssBytes / MB),
|
|
39298
|
+
thresholdMb: Math.round(picked.thresholdBytes / MB),
|
|
39299
|
+
baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
|
|
39300
|
+
restartsInWindow,
|
|
39301
|
+
deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
|
|
39302
|
+
} });
|
|
39303
|
+
try {
|
|
39304
|
+
await this.opts.restart(pickedKey);
|
|
39305
|
+
this.pools.set(pickedKey, {
|
|
39306
|
+
state: commitWatchdogRestart(tracked.state, this.now()),
|
|
39307
|
+
pids: []
|
|
39308
|
+
});
|
|
39309
|
+
} catch (err) {
|
|
39310
|
+
this.opts.log.error("pool memory watchdog restart failed", { meta: {
|
|
39311
|
+
poolKey: pickedKey,
|
|
39312
|
+
error: err instanceof Error ? err.message : String(err)
|
|
39313
|
+
} });
|
|
39314
|
+
}
|
|
39315
|
+
}
|
|
39316
|
+
};
|
|
39317
|
+
function samePids(a, b) {
|
|
39318
|
+
if (a.length !== b.length) return false;
|
|
39319
|
+
const sortedA = [...a].sort((x, y) => x - y);
|
|
39320
|
+
const sortedB = [...b].sort((x, y) => x - y);
|
|
39321
|
+
return sortedA.every((v, i) => v === sortedB[i]);
|
|
39322
|
+
}
|
|
38997
39323
|
/**
|
|
38998
39324
|
* Vector wire codec — base64 of little-endian Float32.
|
|
38999
39325
|
*
|
|
@@ -39023,4 +39349,4 @@ function vectorDimFromBase64(encoded) {
|
|
|
39023
39349
|
return Math.floor(Buffer.from(encoded, "base64").byteLength / 4);
|
|
39024
39350
|
}
|
|
39025
39351
|
//#endregion
|
|
39026
|
-
export {
|
|
39352
|
+
export { encodeVectorBase64 as $, RetrainStatusSchema as A, literal as At, TrackSourceSchema as B, NcSnoozeSchema as C, isDeviceScopedCap as Ct, OpsLogEntrySchema as D, array as Dt, NcTaxonomySchema as E, _enum as Et, SceneMonitorSchema as F, string as Ft, audioMetricsCapability as G, addonWidgetsSourceCapability as H, TIMELAPSE_DENSE_FLOOR_SEC as I, unknown as It, cosineSimilarity as J, audioModeOf as K, TimelapseRuleInputSchema as L, EventCategory as Lt, SCENE_DEFAULT_ANCHOR_THRESHOLD as M, object as Mt, SCENE_DEFAULT_UNCOVERED_POLICY as N, partialRecord as Nt, PoolMemoryWatchdog as O, boolean as Ot, SCENE_DIVERGED as P, record as Pt, embeddingEncoderCapability as Q, TimelapseRulePatchSchema as R, NcSnoozeInputSchema as S, hydrateSchema as St, NcSystemEventKindSchema as T, sleep as Tt, alarmPanelCapability as U, VISIT_MERGE_GAP_MS as V, assertTimelapseCadences as W, defineCustomActions as X, customAction as Y, deriveRecordingMode as Z, NcRuleInputSchema as _, zoneAnalyticsCapability as _t, DETECTION_PIPELINE_CAP_NAME as a, notificationRulesCapability as at, NcRuleTargetSchema as b, DeviceType as bt, EVENT_PAD_MS as c, plateGalleryCapability as ct, NC_ALARM_SYSTEM_EVENT_KINDS as d, resolvePoolMemoryPolicy as dt, faceGalleryCapability as et, NC_CONDITION_CATALOG as f, sceneMonitorCapability as ft, NcConditionDescriptorSchema as g, videoclipsCapability as gt, NC_TAXONOMY as h, vectorDimFromBase64 as ht, DETECTION_MACRO_CLASSES as i, kebabToCamel as it, SCENE_CONFIRM_DEFAULT_TIMEOUT_MS as j, number as jt, RECORDING_EXPORT_MAX_READ_BYTES as k, discriminatedUnion as kt, LabelAttributionSchema as l, readDeviceStateFrom as lt, NC_SNOOZE_MAX_MINUTES as m, systemEventFilterApplies as mt, DEFAULT_EVENT_COLOR as n, isDetectionMacroClass as nt, DeclaredDevices as o, parseProcStatus as ot, NC_DEFAULT_SNOOZE_MINUTES as p, subKindsOf as pt, buildEventKindDescriptor as q, DEFAULT_TIMELAPSE_PREVIEW_TEXT as r, isScheduleActive as rt, EVENT_KIND_BY_CAP as s, pipelineAnalyticsCapability as st, BaseDevice as t, hfModelUrl as tt, MACRO_LABELS as u, readTimelapseGeneratedAt as ut, NcRulePatchSchema as v, errMsg as vt, NcSnoozeSuppressedSchema as w, nodePin as wt, NcScheduleSchema as x, createEvent as xt, NcRuleSchema as y, BaseAddon as yt, TimelapseRuleSchema as z };
|
|
@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_dist = require("../dist-
|
|
5
|
+
const require_dist = require("../dist-BdojoP7i.js");
|
|
6
6
|
let node_fs = require("node:fs");
|
|
7
7
|
let node_fs$1 = require_dist.__toESM(node_fs, 1);
|
|
8
8
|
node_fs = require_dist.__toESM(node_fs);
|
|
@@ -10,9 +10,56 @@ let node_path = require("node:path");
|
|
|
10
10
|
let node_path$1 = require_dist.__toESM(node_path, 1);
|
|
11
11
|
node_path = require_dist.__toESM(node_path);
|
|
12
12
|
require("node:crypto");
|
|
13
|
+
let node_child_process = require("node:child_process");
|
|
13
14
|
let node_util = require("node:util");
|
|
14
15
|
let node_zlib = require("node:zlib");
|
|
15
|
-
|
|
16
|
+
//#region src/embedding-encoder/shared/process-memory.ts
|
|
17
|
+
/**
|
|
18
|
+
* Read a child process's memory footprint for the pool memory watchdog.
|
|
19
|
+
*
|
|
20
|
+
* Deliberate twin of `packages/addon-pipeline/src/shared/process-memory.ts`:
|
|
21
|
+
* addons must not import each other, and `@camstack/system` stays EXTERNAL in
|
|
22
|
+
* addon bundles (a symbol added there would be missing at runtime until the
|
|
23
|
+
* next framework train). The pure parse (`parseProcStatus`) and the whole
|
|
24
|
+
* watchdog policy live once in `@camstack/types` (inlined at build time);
|
|
25
|
+
* only this thin `fs`/`ps` read is duplicated.
|
|
26
|
+
*
|
|
27
|
+
* Linux (the fleet): `/proc/<pid>/status` — RSS, VmSize, VmHWM (peak RSS ≈
|
|
28
|
+
* getrusage maxrss), VmSwap, Threads in one read. macOS (dev): `ps -o rss=`
|
|
29
|
+
* gives RSS alone. Returns null when the pid is gone — the CALLER logs the
|
|
30
|
+
* dropped sample.
|
|
31
|
+
*/
|
|
32
|
+
var IS_LINUX = process.platform === "linux";
|
|
33
|
+
async function readProcessMemory(pid) {
|
|
34
|
+
if (IS_LINUX) try {
|
|
35
|
+
return require_dist.parseProcStatus(await node_fs.promises.readFile(`/proc/${pid}/status`, "utf8"));
|
|
36
|
+
} catch {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
return readViaPs(pid);
|
|
40
|
+
}
|
|
41
|
+
function readViaPs(pid) {
|
|
42
|
+
return new Promise((resolve) => {
|
|
43
|
+
(0, node_child_process.execFile)("ps", [
|
|
44
|
+
"-o",
|
|
45
|
+
"rss=",
|
|
46
|
+
"-p",
|
|
47
|
+
String(pid)
|
|
48
|
+
], { timeout: 2e3 }, (err, stdout) => {
|
|
49
|
+
if (err) return resolve(null);
|
|
50
|
+
const kb = Number(stdout.trim());
|
|
51
|
+
if (!Number.isFinite(kb) || kb <= 0) return resolve(null);
|
|
52
|
+
resolve({
|
|
53
|
+
rssBytes: kb * 1024,
|
|
54
|
+
vmBytes: 0,
|
|
55
|
+
hwmBytes: 0,
|
|
56
|
+
swapBytes: 0,
|
|
57
|
+
threads: 0
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
//#endregion
|
|
16
63
|
//#region ../system/dist/file-data-plane-CuE_hBli.mjs
|
|
17
64
|
function isNonEmptyFile(filePath) {
|
|
18
65
|
return node_fs$1.existsSync(filePath) && node_fs$1.statSync(filePath).size > 0;
|
|
@@ -499,8 +546,20 @@ var PythonRawTensorEngine = class {
|
|
|
499
546
|
if (ready.length !== 1 || ready[0] !== 1) throw new Error("PythonRawTensorEngine: unexpected ready frame");
|
|
500
547
|
this.log.info("ONNX raw-tensor engine ready (embedded Python)", { meta: { modelPath: this.modelPath } });
|
|
501
548
|
}
|
|
549
|
+
/** Native pid of the Python subprocess — sampled by the pool memory
|
|
550
|
+
* watchdog (`/proc/<pid>/status`). */
|
|
551
|
+
getPid() {
|
|
552
|
+
return this.process?.pid ?? null;
|
|
553
|
+
}
|
|
554
|
+
/** Inference requests served since spawn — the denominator that separates
|
|
555
|
+
* "RSS grew per unit of work" (leak) from "RSS grew because it worked". */
|
|
556
|
+
getRequestCount() {
|
|
557
|
+
return this.requestCount;
|
|
558
|
+
}
|
|
559
|
+
requestCount = 0;
|
|
502
560
|
async run(input, inputShape) {
|
|
503
561
|
if (!this.process?.stdin) throw new Error("PythonRawTensorEngine: not initialized — call initialize() first");
|
|
562
|
+
this.requestCount++;
|
|
504
563
|
const ndims = inputShape.length;
|
|
505
564
|
const meta = Buffer.allocUnsafe(1 + ndims * 4);
|
|
506
565
|
meta.writeUInt8(ndims, 0);
|
|
@@ -625,9 +684,21 @@ var PythonTextEncoderEngine = class {
|
|
|
625
684
|
tokenizerPath: this.tokenizerPath
|
|
626
685
|
} });
|
|
627
686
|
}
|
|
687
|
+
/** Native pid of the Python subprocess — sampled by the pool memory
|
|
688
|
+
* watchdog (`/proc/<pid>/status`). */
|
|
689
|
+
getPid() {
|
|
690
|
+
return this.process?.pid ?? null;
|
|
691
|
+
}
|
|
692
|
+
/** Encode requests served since spawn — the denominator that separates
|
|
693
|
+
* "RSS grew per unit of work" (leak) from "RSS grew because it worked". */
|
|
694
|
+
getRequestCount() {
|
|
695
|
+
return this.requestCount;
|
|
696
|
+
}
|
|
697
|
+
requestCount = 0;
|
|
628
698
|
/** Tokenize + encode `text` into the model embedding (float32). */
|
|
629
699
|
async encode(text) {
|
|
630
700
|
if (!this.process?.stdin) throw new Error("PythonTextEncoderEngine: not initialized — call initialize() first");
|
|
701
|
+
this.requestCount++;
|
|
631
702
|
const payload = Buffer.from(text, "utf-8");
|
|
632
703
|
const lenBuf = Buffer.allocUnsafe(4);
|
|
633
704
|
lenBuf.writeUInt32LE(payload.length, 0);
|
|
@@ -753,6 +824,22 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
753
824
|
imageRawEngine = null;
|
|
754
825
|
textEngine = null;
|
|
755
826
|
models = null;
|
|
827
|
+
/**
|
|
828
|
+
* RSS bound + periodic memory telemetry for the two Python engines
|
|
829
|
+
* (`raw_tensor_inference.py` — one of the four leaking pools in the
|
|
830
|
+
* 2026-08-18 OOM audit — and `text_encoder_inference.py`). Shared
|
|
831
|
+
* watchdog from `@camstack/types`; a restart here is dispose + null, and
|
|
832
|
+
* the next encode lazily re-resolves the engine (the exact recovery an
|
|
833
|
+
* operator model change already uses). Started in `onInitialize`,
|
|
834
|
+
* stopped in `onShutdown`.
|
|
835
|
+
*/
|
|
836
|
+
memoryGuard = null;
|
|
837
|
+
/** Single-flight guards for the lazy engine builds — a watchdog restart
|
|
838
|
+
* followed by two concurrent encodes must not spawn the engine twice
|
|
839
|
+
* (the same bug detection-pipeline fixed three times; see its
|
|
840
|
+
* `engineFactoryInflight`). */
|
|
841
|
+
imageEngineInflight = null;
|
|
842
|
+
textEngineInflight = null;
|
|
756
843
|
constructor() {
|
|
757
844
|
super({
|
|
758
845
|
modelId: DEFAULT_CLIP_MODEL,
|
|
@@ -763,11 +850,75 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
763
850
|
async onInitialize() {
|
|
764
851
|
const modelsDir = await this.resolveModelsDir();
|
|
765
852
|
this.models = new ModelDownloadService(modelsDir, [...CLIP_IMAGE_MODELS, ...CLIP_TEXT_MODELS]);
|
|
853
|
+
this.memoryGuard = new require_dist.PoolMemoryWatchdog({
|
|
854
|
+
policy: require_dist.resolvePoolMemoryPolicy(process.env),
|
|
855
|
+
log: this.ctx.logger.child("pool-memory"),
|
|
856
|
+
sample: () => this.sampleEngineMemory(),
|
|
857
|
+
restart: (key) => this.restartEngineForMemory(key)
|
|
858
|
+
});
|
|
859
|
+
this.memoryGuard.start();
|
|
766
860
|
return [{
|
|
767
861
|
capability: require_dist.embeddingEncoderCapability,
|
|
768
862
|
provider: this
|
|
769
863
|
}];
|
|
770
864
|
}
|
|
865
|
+
/** Watchdog sample: one telemetry row per live Python engine. A pid the
|
|
866
|
+
* read cannot see is logged, never silently skipped. */
|
|
867
|
+
async sampleEngineMemory() {
|
|
868
|
+
const engines = [{
|
|
869
|
+
key: "clip-image",
|
|
870
|
+
pid: this.imageRawEngine?.getPid() ?? null,
|
|
871
|
+
requests: this.imageRawEngine?.getRequestCount() ?? 0
|
|
872
|
+
}, {
|
|
873
|
+
key: "clip-text",
|
|
874
|
+
pid: this.textEngine?.getPid() ?? null,
|
|
875
|
+
requests: this.textEngine?.getRequestCount() ?? 0
|
|
876
|
+
}];
|
|
877
|
+
const out = [];
|
|
878
|
+
for (const engine of engines) {
|
|
879
|
+
if (engine.pid === null) continue;
|
|
880
|
+
const mem = await readProcessMemory(engine.pid);
|
|
881
|
+
if (!mem) {
|
|
882
|
+
this.ctx.logger.warn("embedding engine memory sample dropped", { meta: {
|
|
883
|
+
poolKey: engine.key,
|
|
884
|
+
pid: engine.pid
|
|
885
|
+
} });
|
|
886
|
+
continue;
|
|
887
|
+
}
|
|
888
|
+
const mb = (bytes) => Math.round(bytes / (1024 * 1024));
|
|
889
|
+
out.push({
|
|
890
|
+
key: engine.key,
|
|
891
|
+
pids: [engine.pid],
|
|
892
|
+
rssBytes: mem.rssBytes,
|
|
893
|
+
meta: {
|
|
894
|
+
modelId: this.config.modelId,
|
|
895
|
+
vmMb: mb(mem.vmBytes),
|
|
896
|
+
peakRssMb: mb(mem.hwmBytes),
|
|
897
|
+
swapMb: mb(mem.swapBytes),
|
|
898
|
+
threads: mem.threads,
|
|
899
|
+
requestsServed: engine.requests
|
|
900
|
+
}
|
|
901
|
+
});
|
|
902
|
+
}
|
|
903
|
+
return out;
|
|
904
|
+
}
|
|
905
|
+
/** Watchdog restart: dispose + null; the next encode re-resolves lazily
|
|
906
|
+
* through the single-flight ensure. An in-flight encode fails once — its
|
|
907
|
+
* callers are the retry-until-landed bulk passes (D58) and RPCs that
|
|
908
|
+
* surface errors, never silent loss. */
|
|
909
|
+
async restartEngineForMemory(key) {
|
|
910
|
+
if (key === "clip-image") {
|
|
911
|
+
const engine = this.imageRawEngine;
|
|
912
|
+
if (!engine) return;
|
|
913
|
+
this.imageRawEngine = null;
|
|
914
|
+
await engine.dispose();
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
const engine = this.textEngine;
|
|
918
|
+
if (!engine) return;
|
|
919
|
+
this.textEngine = null;
|
|
920
|
+
await engine.dispose();
|
|
921
|
+
}
|
|
771
922
|
async encode(input) {
|
|
772
923
|
const { crop, width, height } = input;
|
|
773
924
|
await this.ensureImageEngine();
|
|
@@ -811,17 +962,31 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
811
962
|
}
|
|
812
963
|
async ensureImageEngine() {
|
|
813
964
|
if (this.imageRawEngine) return;
|
|
965
|
+
if (this.imageEngineInflight) return this.imageEngineInflight;
|
|
814
966
|
const meta = getModelMeta(this.config.modelId);
|
|
815
967
|
const imageEntry = CLIP_IMAGE_MODELS.find((m) => m.id === meta.imageModelId);
|
|
816
968
|
if (!imageEntry) throw new Error(`EmbeddingEncoderAddon: unknown image model "${meta.imageModelId}"`);
|
|
817
|
-
|
|
969
|
+
const inflight = this.resolveForEntry(imageEntry, "image");
|
|
970
|
+
this.imageEngineInflight = inflight;
|
|
971
|
+
try {
|
|
972
|
+
await inflight;
|
|
973
|
+
} finally {
|
|
974
|
+
this.imageEngineInflight = null;
|
|
975
|
+
}
|
|
818
976
|
}
|
|
819
977
|
async ensureTextEngine() {
|
|
820
978
|
if (this.textEngine) return;
|
|
979
|
+
if (this.textEngineInflight) return this.textEngineInflight;
|
|
821
980
|
const meta = getModelMeta(this.config.modelId);
|
|
822
981
|
const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
|
|
823
982
|
if (!textEntry) throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
|
|
824
|
-
|
|
983
|
+
const inflight = this.resolveForEntry(textEntry, "text");
|
|
984
|
+
this.textEngineInflight = inflight;
|
|
985
|
+
try {
|
|
986
|
+
await inflight;
|
|
987
|
+
} finally {
|
|
988
|
+
this.textEngineInflight = null;
|
|
989
|
+
}
|
|
825
990
|
}
|
|
826
991
|
async resolveForEntry(entry, target) {
|
|
827
992
|
const engineLogger = this.ctx.logger.withTags({
|
|
@@ -847,6 +1012,8 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
|
847
1012
|
this.textEngine = textEngine;
|
|
848
1013
|
}
|
|
849
1014
|
async onShutdown() {
|
|
1015
|
+
this.memoryGuard?.stop();
|
|
1016
|
+
this.memoryGuard = null;
|
|
850
1017
|
await this.imageRawEngine?.dispose();
|
|
851
1018
|
await this.textEngine?.dispose();
|
|
852
1019
|
}
|