@camstack/addon-post-analysis 1.2.92 → 1.2.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- //#region ../types/dist/event-category-C0lyLd5U.mjs
1
+ //#region ../types/dist/event-category-XfKNtfCc.mjs
2
2
  var EventCategory = /* @__PURE__ */ function(EventCategory) {
3
3
  EventCategory["SystemBoot"] = "system.boot";
4
4
  EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -37,6 +37,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
37
37
  EventCategory["AddonInstalled"] = "addon.installed";
38
38
  EventCategory["AddonUninstalled"] = "addon.uninstalled";
39
39
  EventCategory["AddonCrashed"] = "addon.crashed";
40
+ /**
41
+ * A RUNNER the D6 crash circuit-breaker gave up on — terminal.
42
+ *
43
+ * `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
44
+ * respawned"); this is the one that never resolves itself. Emitted exactly
45
+ * once per trip, by `process-service.ts`, carrying the node, the runner, the
46
+ * addons it hosted and the crash count that tripped the breaker.
47
+ *
48
+ * It exists because a runner marked terminally `failed` used to be SILENT:
49
+ * on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
50
+ * errors, the breaker stopped respawning it (correctly), `/health` kept
51
+ * returning 200, and fleet-wide recording was gone for 3h15 before the
52
+ * operator's phone told him. The Notification Center's system-event intake
53
+ * consumes this category and maps it to the `addon-crash-loop` kind.
54
+ */
55
+ EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
40
56
  EventCategory["AddonError"] = "addon.error";
41
57
  EventCategory["AddonPageReady"] = "addon.page-ready";
42
58
  EventCategory["AddonWidgetReady"] = "addon.widget-ready";
@@ -14814,6 +14830,7 @@ var NcSystemEventKindSchema = _enum([
14814
14830
  "node-offline",
14815
14831
  "node-inference-unavailable",
14816
14832
  "detection-blind",
14833
+ "addon-crash-loop",
14817
14834
  "addon-update-available",
14818
14835
  "server-update-available",
14819
14836
  "alarm-triggered",
@@ -15939,6 +15956,10 @@ var NC_CONDITION_CATALOG = [
15939
15956
  value: "detection-blind",
15940
15957
  label: "Camera detecting nothing"
15941
15958
  },
15959
+ {
15960
+ value: "addon-crash-loop",
15961
+ label: "Addon stopped after repeated crashes"
15962
+ },
15942
15963
  {
15943
15964
  value: "addon-update-available",
15944
15965
  label: "Addon update available"
@@ -21473,6 +21494,14 @@ var ClipSchema = object({
21473
21494
  endMs: number()
21474
21495
  }),
21475
21496
  /**
21497
+ * Distinct object classes attached to this visit (`person`, `car`, …),
21498
+ * dominant first, capped at {@link MAX_CLIP_LABELS}. The ribbon renders these
21499
+ * instead of the bare kind — "Object" tells the operator nothing a colour bar
21500
+ * did not. Absent (never empty) when the visit attached no classified object
21501
+ * event, so a motion/audio-only visit keeps its kind label.
21502
+ */
21503
+ labels: array(string()).max(3).optional(),
21504
+ /**
21476
21505
  * Lazy thumbnail URL, never inlined.
21477
21506
  *
21478
21507
  * Recording-derived clips (events-mode keep-window, and the prepared
@@ -38537,7 +38566,7 @@ function systemEventFilterApplies(kind, filter) {
38537
38566
  switch (filter) {
38538
38567
  case "deviceIds": return kind.startsWith("device-") || kind.startsWith("stream-") || kind === "detection-blind" || kind === "alarm-triggered" || kind === "export-completed";
38539
38568
  case "deviceTypes": return kind.startsWith("device-") || kind === "detection-blind";
38540
- case "nodeIds": return kind.startsWith("node-") || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
38569
+ case "nodeIds": return kind.startsWith("node-") || kind === "addon-crash-loop" || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
38541
38570
  case "packageNames": return kind.endsWith("update-available") || kind === "addon-updated" || kind === "server-updated";
38542
38571
  }
38543
38572
  }
@@ -38994,6 +39023,303 @@ function cosineSimilarity(a, b) {
38994
39023
  function hfModelUrl(repo, path) {
38995
39024
  return `https://huggingface.co/${repo}/resolve/main/${path}`;
38996
39025
  }
39026
+ var MB = 1024 * 1024;
39027
+ var DEFAULT_POOL_MEMORY_POLICY = {
39028
+ sampleIntervalMs: 6e4,
39029
+ baselineSettleSamples: 5,
39030
+ baselineSampleCount: 3,
39031
+ restartMultiple: 4,
39032
+ floorBytes: 1024 * MB,
39033
+ ceilingBytes: 3072 * MB,
39034
+ cooldownMs: 30 * 6e4,
39035
+ maxRestartsPerWindow: 6,
39036
+ restartWindowMs: 1440 * 6e4
39037
+ };
39038
+ /**
39039
+ * Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
39040
+ * absent values keep the default — an operator typo must never disable the
39041
+ * bound or set it to zero.
39042
+ */
39043
+ function resolvePoolMemoryPolicy(env) {
39044
+ const num = (key, fallback, min) => {
39045
+ const raw = env[key];
39046
+ if (raw === void 0) return fallback;
39047
+ const parsed = Number(raw);
39048
+ return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
39049
+ };
39050
+ const d = DEFAULT_POOL_MEMORY_POLICY;
39051
+ return {
39052
+ sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
39053
+ baselineSettleSamples: d.baselineSettleSamples,
39054
+ baselineSampleCount: d.baselineSampleCount,
39055
+ restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
39056
+ floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
39057
+ ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
39058
+ cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
39059
+ maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
39060
+ restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
39061
+ };
39062
+ }
39063
+ /** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
39064
+ * Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
39065
+ function parseProcStatus(text) {
39066
+ const kb = (label) => {
39067
+ const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
39068
+ return match ? Number(match[1]) * 1024 : null;
39069
+ };
39070
+ const rssBytes = kb("VmRSS");
39071
+ if (rssBytes === null) return null;
39072
+ const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
39073
+ return {
39074
+ rssBytes,
39075
+ vmBytes: kb("VmSize") ?? 0,
39076
+ hwmBytes: kb("VmHWM") ?? 0,
39077
+ swapBytes: kb("VmSwap") ?? 0,
39078
+ threads: threadsMatch ? Number(threadsMatch[1]) : 0
39079
+ };
39080
+ }
39081
+ function initialPoolMemoryState() {
39082
+ return {
39083
+ settleSeen: 0,
39084
+ baselineWindow: [],
39085
+ baselineBytes: null,
39086
+ restartsAt: [],
39087
+ lastRestartAt: null
39088
+ };
39089
+ }
39090
+ function median(values) {
39091
+ const sorted = [...values].sort((a, b) => a - b);
39092
+ return sorted[Math.floor(sorted.length / 2)];
39093
+ }
39094
+ function advanceBaseline(state, rssBytes, policy) {
39095
+ if (state.baselineBytes !== null) return state;
39096
+ if (state.settleSeen < policy.baselineSettleSamples) return {
39097
+ ...state,
39098
+ settleSeen: state.settleSeen + 1
39099
+ };
39100
+ const window = [...state.baselineWindow, rssBytes];
39101
+ if (window.length < policy.baselineSampleCount) return {
39102
+ ...state,
39103
+ baselineWindow: window
39104
+ };
39105
+ return {
39106
+ ...state,
39107
+ baselineWindow: window,
39108
+ baselineBytes: median(window)
39109
+ };
39110
+ }
39111
+ /** The one place a pool's restart threshold is computed. */
39112
+ function poolMemoryThreshold(baselineBytes, policy) {
39113
+ if (baselineBytes === null) return policy.ceilingBytes;
39114
+ return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
39115
+ }
39116
+ function pruneRestarts(restartsAt, nowMs, policy) {
39117
+ return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
39118
+ }
39119
+ /**
39120
+ * Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
39121
+ * The caller performs (and logs) the restart, then commits it via
39122
+ * {@link commitWatchdogRestart}.
39123
+ */
39124
+ function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
39125
+ const next = advanceBaseline(state, rssBytes, policy);
39126
+ const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
39127
+ if (rssBytes <= thresholdBytes) return {
39128
+ state: next,
39129
+ action: next.baselineBytes === null ? "baseline-pending" : "ok",
39130
+ baselineBytes: next.baselineBytes,
39131
+ thresholdBytes
39132
+ };
39133
+ const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
39134
+ const pruned = {
39135
+ ...next,
39136
+ restartsAt: inWindow
39137
+ };
39138
+ if (inWindow.length >= policy.maxRestartsPerWindow) return {
39139
+ state: pruned,
39140
+ action: "exhausted",
39141
+ baselineBytes: next.baselineBytes,
39142
+ thresholdBytes
39143
+ };
39144
+ if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
39145
+ state: pruned,
39146
+ action: "cooldown",
39147
+ baselineBytes: next.baselineBytes,
39148
+ thresholdBytes
39149
+ };
39150
+ return {
39151
+ state: pruned,
39152
+ action: "restart",
39153
+ baselineBytes: next.baselineBytes,
39154
+ thresholdBytes
39155
+ };
39156
+ }
39157
+ /** Record a performed watchdog restart: budget consumed, baseline reset —
39158
+ * the recreated pool is a NEW process and gets a fresh settle window. */
39159
+ function commitWatchdogRestart(state, nowMs) {
39160
+ return {
39161
+ ...initialPoolMemoryState(),
39162
+ restartsAt: [...state.restartsAt, nowMs],
39163
+ lastRestartAt: nowMs
39164
+ };
39165
+ }
39166
+ /** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
39167
+ * kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
39168
+ * restart budget is kept: it bounds the WATCHDOG, not the pool. */
39169
+ function resetPoolBaseline(state) {
39170
+ return {
39171
+ ...initialPoolMemoryState(),
39172
+ restartsAt: state.restartsAt,
39173
+ lastRestartAt: state.lastRestartAt
39174
+ };
39175
+ }
39176
+ /**
39177
+ * Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
39178
+ * several pools at once (e.g. after a model rollout grew every baseline)
39179
+ * would take detection down on every camera simultaneously; one per sweep
39180
+ * means the survivors keep serving while the worst offender reloads, and the
39181
+ * next sweep (one interval later) takes the next one.
39182
+ */
39183
+ function pickRestartCandidate(candidates) {
39184
+ if (candidates.length === 0) return null;
39185
+ let worst = candidates[0];
39186
+ for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
39187
+ return worst.key;
39188
+ }
39189
+ var PoolMemoryWatchdog = class {
39190
+ pools = /* @__PURE__ */ new Map();
39191
+ timer = null;
39192
+ sweeping = false;
39193
+ stopped = false;
39194
+ opts;
39195
+ now;
39196
+ setTimer;
39197
+ clearTimer;
39198
+ constructor(opts) {
39199
+ this.opts = opts;
39200
+ this.now = opts.now ?? (() => Date.now());
39201
+ this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
39202
+ this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
39203
+ }
39204
+ start() {
39205
+ if (this.timer || this.stopped) return;
39206
+ this.arm();
39207
+ }
39208
+ stop() {
39209
+ this.stopped = true;
39210
+ if (this.timer) {
39211
+ this.clearTimer(this.timer);
39212
+ this.timer = null;
39213
+ }
39214
+ }
39215
+ arm() {
39216
+ const timer = this.setTimer(() => {
39217
+ this.sweep().finally(() => {
39218
+ if (!this.stopped) this.arm();
39219
+ });
39220
+ }, this.opts.policy.sampleIntervalMs);
39221
+ if (typeof timer.unref === "function") timer.unref();
39222
+ this.timer = timer;
39223
+ }
39224
+ /** One sweep: sample → log every pool → restart at most one. Public so the
39225
+ * loop is testable without fake global timers. */
39226
+ async sweep() {
39227
+ if (this.sweeping) return;
39228
+ this.sweeping = true;
39229
+ try {
39230
+ await this.doSweep();
39231
+ } catch (err) {
39232
+ this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
39233
+ } finally {
39234
+ this.sweeping = false;
39235
+ }
39236
+ }
39237
+ async doSweep() {
39238
+ const nowMs = this.now();
39239
+ const samples = await this.opts.sample();
39240
+ const candidates = [];
39241
+ const verdictByKey = /* @__PURE__ */ new Map();
39242
+ for (const t of samples) {
39243
+ const tracked = this.pools.get(t.key);
39244
+ let state = tracked?.state ?? initialPoolMemoryState();
39245
+ if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
39246
+ this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
39247
+ poolKey: t.key,
39248
+ previousPids: tracked.pids,
39249
+ pids: t.pids
39250
+ } });
39251
+ state = resetPoolBaseline(state);
39252
+ }
39253
+ const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
39254
+ this.pools.set(t.key, {
39255
+ state: verdict.state,
39256
+ pids: t.pids
39257
+ });
39258
+ verdictByKey.set(t.key, verdict);
39259
+ this.opts.log.info("pool memory", { meta: {
39260
+ poolKey: t.key,
39261
+ pids: t.pids,
39262
+ rssMb: Math.round(t.rssBytes / MB),
39263
+ baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
39264
+ thresholdMb: Math.round(verdict.thresholdBytes / MB),
39265
+ action: verdict.action,
39266
+ ...t.meta
39267
+ } });
39268
+ if (verdict.action === "restart") candidates.push({
39269
+ key: t.key,
39270
+ rssBytes: t.rssBytes,
39271
+ thresholdBytes: verdict.thresholdBytes
39272
+ });
39273
+ else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
39274
+ poolKey: t.key,
39275
+ rssMb: Math.round(t.rssBytes / MB),
39276
+ thresholdMb: Math.round(verdict.thresholdBytes / MB),
39277
+ cooldownMs: this.opts.policy.cooldownMs
39278
+ } });
39279
+ else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
39280
+ poolKey: t.key,
39281
+ rssMb: Math.round(t.rssBytes / MB),
39282
+ thresholdMb: Math.round(verdict.thresholdBytes / MB),
39283
+ maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
39284
+ restartWindowMs: this.opts.policy.restartWindowMs
39285
+ } });
39286
+ }
39287
+ for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
39288
+ const pickedKey = pickRestartCandidate(candidates);
39289
+ if (pickedKey === null) return;
39290
+ const picked = candidates.find((c) => c.key === pickedKey);
39291
+ const verdict = verdictByKey.get(pickedKey);
39292
+ const tracked = this.pools.get(pickedKey);
39293
+ const restartsInWindow = tracked.state.restartsAt.length + 1;
39294
+ this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
39295
+ poolKey: pickedKey,
39296
+ pids: tracked.pids,
39297
+ rssMb: Math.round(picked.rssBytes / MB),
39298
+ thresholdMb: Math.round(picked.thresholdBytes / MB),
39299
+ baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
39300
+ restartsInWindow,
39301
+ deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
39302
+ } });
39303
+ try {
39304
+ await this.opts.restart(pickedKey);
39305
+ this.pools.set(pickedKey, {
39306
+ state: commitWatchdogRestart(tracked.state, this.now()),
39307
+ pids: []
39308
+ });
39309
+ } catch (err) {
39310
+ this.opts.log.error("pool memory watchdog restart failed", { meta: {
39311
+ poolKey: pickedKey,
39312
+ error: err instanceof Error ? err.message : String(err)
39313
+ } });
39314
+ }
39315
+ }
39316
+ };
39317
+ function samePids(a, b) {
39318
+ if (a.length !== b.length) return false;
39319
+ const sortedA = [...a].sort((x, y) => x - y);
39320
+ const sortedB = [...b].sort((x, y) => x - y);
39321
+ return sortedA.every((v, i) => v === sortedB[i]);
39322
+ }
38997
39323
  /**
38998
39324
  * Vector wire codec — base64 of little-endian Float32.
38999
39325
  *
@@ -39023,4 +39349,4 @@ function vectorDimFromBase64(encoded) {
39023
39349
  return Math.floor(Buffer.from(encoded, "base64").byteLength / 4);
39024
39350
  }
39025
39351
  //#endregion
39026
- export { faceGalleryCapability as $, SCENE_CONFIRM_DEFAULT_TIMEOUT_MS as A, partialRecord as At, VISIT_MERGE_GAP_MS as B, NcSnoozeSchema as C, _enum as Ct, OpsLogEntrySchema as D, literal as Dt, NcTaxonomySchema as E, discriminatedUnion as Et, TIMELAPSE_DENSE_FLOOR_SEC as F, audioModeOf as G, alarmPanelCapability as H, TimelapseRuleInputSchema as I, customAction as J, buildEventKindDescriptor as K, TimelapseRulePatchSchema as L, SCENE_DEFAULT_UNCOVERED_POLICY as M, string as Mt, SCENE_DIVERGED as N, unknown as Nt, RECORDING_EXPORT_MAX_READ_BYTES as O, number as Ot, SceneMonitorSchema as P, EventCategory as Pt, encodeVectorBase64 as Q, TimelapseRuleSchema as R, NcSnoozeInputSchema as S, sleep as St, NcSystemEventKindSchema as T, boolean as Tt, assertTimelapseCadences as U, addonWidgetsSourceCapability as V, audioMetricsCapability as W, deriveRecordingMode as X, defineCustomActions as Y, embeddingEncoderCapability as Z, NcRuleInputSchema as _, DeviceType as _t, DETECTION_PIPELINE_CAP_NAME as a, pipelineAnalyticsCapability as at, NcRuleTargetSchema as b, isDeviceScopedCap as bt, EVENT_PAD_MS as c, readTimelapseGeneratedAt as ct, NC_ALARM_SYSTEM_EVENT_KINDS as d, systemEventFilterApplies as dt, hfModelUrl as et, NC_CONDITION_CATALOG as f, vectorDimFromBase64 as ft, NcConditionDescriptorSchema as g, BaseAddon as gt, NC_TAXONOMY as h, errMsg as ht, DETECTION_MACRO_CLASSES as i, notificationRulesCapability as it, SCENE_DEFAULT_ANCHOR_THRESHOLD as j, record as jt, RetrainStatusSchema as k, object as kt, LabelAttributionSchema as l, sceneMonitorCapability as lt, NC_SNOOZE_MAX_MINUTES as m, zoneAnalyticsCapability as mt, DEFAULT_EVENT_COLOR as n, isScheduleActive as nt, DeclaredDevices as o, plateGalleryCapability as ot, NC_DEFAULT_SNOOZE_MINUTES as p, videoclipsCapability as pt, cosineSimilarity as q, DEFAULT_TIMELAPSE_PREVIEW_TEXT as r, kebabToCamel as rt, EVENT_KIND_BY_CAP as s, readDeviceStateFrom as st, BaseDevice as t, isDetectionMacroClass as tt, MACRO_LABELS as u, subKindsOf as ut, NcRulePatchSchema as v, createEvent as vt, NcSnoozeSuppressedSchema as w, array as wt, NcScheduleSchema as x, nodePin as xt, NcRuleSchema as y, hydrateSchema as yt, TrackSourceSchema as z };
39352
+ export { encodeVectorBase64 as $, RetrainStatusSchema as A, literal as At, TrackSourceSchema as B, NcSnoozeSchema as C, isDeviceScopedCap as Ct, OpsLogEntrySchema as D, array as Dt, NcTaxonomySchema as E, _enum as Et, SceneMonitorSchema as F, string as Ft, audioMetricsCapability as G, addonWidgetsSourceCapability as H, TIMELAPSE_DENSE_FLOOR_SEC as I, unknown as It, cosineSimilarity as J, audioModeOf as K, TimelapseRuleInputSchema as L, EventCategory as Lt, SCENE_DEFAULT_ANCHOR_THRESHOLD as M, object as Mt, SCENE_DEFAULT_UNCOVERED_POLICY as N, partialRecord as Nt, PoolMemoryWatchdog as O, boolean as Ot, SCENE_DIVERGED as P, record as Pt, embeddingEncoderCapability as Q, TimelapseRulePatchSchema as R, NcSnoozeInputSchema as S, hydrateSchema as St, NcSystemEventKindSchema as T, sleep as Tt, alarmPanelCapability as U, VISIT_MERGE_GAP_MS as V, assertTimelapseCadences as W, defineCustomActions as X, customAction as Y, deriveRecordingMode as Z, NcRuleInputSchema as _, zoneAnalyticsCapability as _t, DETECTION_PIPELINE_CAP_NAME as a, notificationRulesCapability as at, NcRuleTargetSchema as b, DeviceType as bt, EVENT_PAD_MS as c, plateGalleryCapability as ct, NC_ALARM_SYSTEM_EVENT_KINDS as d, resolvePoolMemoryPolicy as dt, faceGalleryCapability as et, NC_CONDITION_CATALOG as f, sceneMonitorCapability as ft, NcConditionDescriptorSchema as g, videoclipsCapability as gt, NC_TAXONOMY as h, vectorDimFromBase64 as ht, DETECTION_MACRO_CLASSES as i, kebabToCamel as it, SCENE_CONFIRM_DEFAULT_TIMEOUT_MS as j, number as jt, RECORDING_EXPORT_MAX_READ_BYTES as k, discriminatedUnion as kt, LabelAttributionSchema as l, readDeviceStateFrom as lt, NC_SNOOZE_MAX_MINUTES as m, systemEventFilterApplies as mt, DEFAULT_EVENT_COLOR as n, isDetectionMacroClass as nt, DeclaredDevices as o, parseProcStatus as ot, NC_DEFAULT_SNOOZE_MINUTES as p, subKindsOf as pt, buildEventKindDescriptor as q, DEFAULT_TIMELAPSE_PREVIEW_TEXT as r, isScheduleActive as rt, EVENT_KIND_BY_CAP as s, pipelineAnalyticsCapability as st, BaseDevice as t, hfModelUrl as tt, MACRO_LABELS as u, readTimelapseGeneratedAt as ut, NcRulePatchSchema as v, errMsg as vt, NcSnoozeSuppressedSchema as w, nodePin as wt, NcScheduleSchema as x, createEvent as xt, NcRuleSchema as y, BaseAddon as yt, TimelapseRuleSchema as z };
@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
2
2
  __esModule: { value: true },
3
3
  [Symbol.toStringTag]: { value: "Module" }
4
4
  });
5
- const require_dist = require("../dist-BfrtqGyh.js");
5
+ const require_dist = require("../dist-BdojoP7i.js");
6
6
  let node_fs = require("node:fs");
7
7
  let node_fs$1 = require_dist.__toESM(node_fs, 1);
8
8
  node_fs = require_dist.__toESM(node_fs);
@@ -10,9 +10,56 @@ let node_path = require("node:path");
10
10
  let node_path$1 = require_dist.__toESM(node_path, 1);
11
11
  node_path = require_dist.__toESM(node_path);
12
12
  require("node:crypto");
13
+ let node_child_process = require("node:child_process");
13
14
  let node_util = require("node:util");
14
15
  let node_zlib = require("node:zlib");
15
- let node_child_process = require("node:child_process");
16
+ //#region src/embedding-encoder/shared/process-memory.ts
17
+ /**
18
+ * Read a child process's memory footprint for the pool memory watchdog.
19
+ *
20
+ * Deliberate twin of `packages/addon-pipeline/src/shared/process-memory.ts`:
21
+ * addons must not import each other, and `@camstack/system` stays EXTERNAL in
22
+ * addon bundles (a symbol added there would be missing at runtime until the
23
+ * next framework train). The pure parse (`parseProcStatus`) and the whole
24
+ * watchdog policy live once in `@camstack/types` (inlined at build time);
25
+ * only this thin `fs`/`ps` read is duplicated.
26
+ *
27
+ * Linux (the fleet): `/proc/<pid>/status` — RSS, VmSize, VmHWM (peak RSS ≈
28
+ * getrusage maxrss), VmSwap, Threads in one read. macOS (dev): `ps -o rss=`
29
+ * gives RSS alone. Returns null when the pid is gone — the CALLER logs the
30
+ * dropped sample.
31
+ */
32
+ var IS_LINUX = process.platform === "linux";
33
+ async function readProcessMemory(pid) {
34
+ if (IS_LINUX) try {
35
+ return require_dist.parseProcStatus(await node_fs.promises.readFile(`/proc/${pid}/status`, "utf8"));
36
+ } catch {
37
+ return null;
38
+ }
39
+ return readViaPs(pid);
40
+ }
41
+ function readViaPs(pid) {
42
+ return new Promise((resolve) => {
43
+ (0, node_child_process.execFile)("ps", [
44
+ "-o",
45
+ "rss=",
46
+ "-p",
47
+ String(pid)
48
+ ], { timeout: 2e3 }, (err, stdout) => {
49
+ if (err) return resolve(null);
50
+ const kb = Number(stdout.trim());
51
+ if (!Number.isFinite(kb) || kb <= 0) return resolve(null);
52
+ resolve({
53
+ rssBytes: kb * 1024,
54
+ vmBytes: 0,
55
+ hwmBytes: 0,
56
+ swapBytes: 0,
57
+ threads: 0
58
+ });
59
+ });
60
+ });
61
+ }
62
+ //#endregion
16
63
  //#region ../system/dist/file-data-plane-CuE_hBli.mjs
17
64
  function isNonEmptyFile(filePath) {
18
65
  return node_fs$1.existsSync(filePath) && node_fs$1.statSync(filePath).size > 0;
@@ -499,8 +546,20 @@ var PythonRawTensorEngine = class {
499
546
  if (ready.length !== 1 || ready[0] !== 1) throw new Error("PythonRawTensorEngine: unexpected ready frame");
500
547
  this.log.info("ONNX raw-tensor engine ready (embedded Python)", { meta: { modelPath: this.modelPath } });
501
548
  }
549
+ /** Native pid of the Python subprocess — sampled by the pool memory
550
+ * watchdog (`/proc/<pid>/status`). */
551
+ getPid() {
552
+ return this.process?.pid ?? null;
553
+ }
554
+ /** Inference requests served since spawn — the denominator that separates
555
+ * "RSS grew per unit of work" (leak) from "RSS grew because it worked". */
556
+ getRequestCount() {
557
+ return this.requestCount;
558
+ }
559
+ requestCount = 0;
502
560
  async run(input, inputShape) {
503
561
  if (!this.process?.stdin) throw new Error("PythonRawTensorEngine: not initialized — call initialize() first");
562
+ this.requestCount++;
504
563
  const ndims = inputShape.length;
505
564
  const meta = Buffer.allocUnsafe(1 + ndims * 4);
506
565
  meta.writeUInt8(ndims, 0);
@@ -625,9 +684,21 @@ var PythonTextEncoderEngine = class {
625
684
  tokenizerPath: this.tokenizerPath
626
685
  } });
627
686
  }
687
+ /** Native pid of the Python subprocess — sampled by the pool memory
688
+ * watchdog (`/proc/<pid>/status`). */
689
+ getPid() {
690
+ return this.process?.pid ?? null;
691
+ }
692
+ /** Encode requests served since spawn — the denominator that separates
693
+ * "RSS grew per unit of work" (leak) from "RSS grew because it worked". */
694
+ getRequestCount() {
695
+ return this.requestCount;
696
+ }
697
+ requestCount = 0;
628
698
  /** Tokenize + encode `text` into the model embedding (float32). */
629
699
  async encode(text) {
630
700
  if (!this.process?.stdin) throw new Error("PythonTextEncoderEngine: not initialized — call initialize() first");
701
+ this.requestCount++;
631
702
  const payload = Buffer.from(text, "utf-8");
632
703
  const lenBuf = Buffer.allocUnsafe(4);
633
704
  lenBuf.writeUInt32LE(payload.length, 0);
@@ -753,6 +824,22 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
753
824
  imageRawEngine = null;
754
825
  textEngine = null;
755
826
  models = null;
827
+ /**
828
+ * RSS bound + periodic memory telemetry for the two Python engines
829
+ * (`raw_tensor_inference.py` — one of the four leaking pools in the
830
+ * 2026-08-18 OOM audit — and `text_encoder_inference.py`). Shared
831
+ * watchdog from `@camstack/types`; a restart here is dispose + null, and
832
+ * the next encode lazily re-resolves the engine (the exact recovery an
833
+ * operator model change already uses). Started in `onInitialize`,
834
+ * stopped in `onShutdown`.
835
+ */
836
+ memoryGuard = null;
837
+ /** Single-flight guards for the lazy engine builds — a watchdog restart
838
+ * followed by two concurrent encodes must not spawn the engine twice
839
+ * (the same bug detection-pipeline fixed three times; see its
840
+ * `engineFactoryInflight`). */
841
+ imageEngineInflight = null;
842
+ textEngineInflight = null;
756
843
  constructor() {
757
844
  super({
758
845
  modelId: DEFAULT_CLIP_MODEL,
@@ -763,11 +850,75 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
763
850
  async onInitialize() {
764
851
  const modelsDir = await this.resolveModelsDir();
765
852
  this.models = new ModelDownloadService(modelsDir, [...CLIP_IMAGE_MODELS, ...CLIP_TEXT_MODELS]);
853
+ this.memoryGuard = new require_dist.PoolMemoryWatchdog({
854
+ policy: require_dist.resolvePoolMemoryPolicy(process.env),
855
+ log: this.ctx.logger.child("pool-memory"),
856
+ sample: () => this.sampleEngineMemory(),
857
+ restart: (key) => this.restartEngineForMemory(key)
858
+ });
859
+ this.memoryGuard.start();
766
860
  return [{
767
861
  capability: require_dist.embeddingEncoderCapability,
768
862
  provider: this
769
863
  }];
770
864
  }
865
+ /** Watchdog sample: one telemetry row per live Python engine. A pid the
866
+ * read cannot see is logged, never silently skipped. */
867
+ async sampleEngineMemory() {
868
+ const engines = [{
869
+ key: "clip-image",
870
+ pid: this.imageRawEngine?.getPid() ?? null,
871
+ requests: this.imageRawEngine?.getRequestCount() ?? 0
872
+ }, {
873
+ key: "clip-text",
874
+ pid: this.textEngine?.getPid() ?? null,
875
+ requests: this.textEngine?.getRequestCount() ?? 0
876
+ }];
877
+ const out = [];
878
+ for (const engine of engines) {
879
+ if (engine.pid === null) continue;
880
+ const mem = await readProcessMemory(engine.pid);
881
+ if (!mem) {
882
+ this.ctx.logger.warn("embedding engine memory sample dropped", { meta: {
883
+ poolKey: engine.key,
884
+ pid: engine.pid
885
+ } });
886
+ continue;
887
+ }
888
+ const mb = (bytes) => Math.round(bytes / (1024 * 1024));
889
+ out.push({
890
+ key: engine.key,
891
+ pids: [engine.pid],
892
+ rssBytes: mem.rssBytes,
893
+ meta: {
894
+ modelId: this.config.modelId,
895
+ vmMb: mb(mem.vmBytes),
896
+ peakRssMb: mb(mem.hwmBytes),
897
+ swapMb: mb(mem.swapBytes),
898
+ threads: mem.threads,
899
+ requestsServed: engine.requests
900
+ }
901
+ });
902
+ }
903
+ return out;
904
+ }
905
+ /** Watchdog restart: dispose + null; the next encode re-resolves lazily
906
+ * through the single-flight ensure. An in-flight encode fails once — its
907
+ * callers are the retry-until-landed bulk passes (D58) and RPCs that
908
+ * surface errors, never silent loss. */
909
+ async restartEngineForMemory(key) {
910
+ if (key === "clip-image") {
911
+ const engine = this.imageRawEngine;
912
+ if (!engine) return;
913
+ this.imageRawEngine = null;
914
+ await engine.dispose();
915
+ return;
916
+ }
917
+ const engine = this.textEngine;
918
+ if (!engine) return;
919
+ this.textEngine = null;
920
+ await engine.dispose();
921
+ }
771
922
  async encode(input) {
772
923
  const { crop, width, height } = input;
773
924
  await this.ensureImageEngine();
@@ -811,17 +962,31 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
811
962
  }
812
963
  async ensureImageEngine() {
813
964
  if (this.imageRawEngine) return;
965
+ if (this.imageEngineInflight) return this.imageEngineInflight;
814
966
  const meta = getModelMeta(this.config.modelId);
815
967
  const imageEntry = CLIP_IMAGE_MODELS.find((m) => m.id === meta.imageModelId);
816
968
  if (!imageEntry) throw new Error(`EmbeddingEncoderAddon: unknown image model "${meta.imageModelId}"`);
817
- await this.resolveForEntry(imageEntry, "image");
969
+ const inflight = this.resolveForEntry(imageEntry, "image");
970
+ this.imageEngineInflight = inflight;
971
+ try {
972
+ await inflight;
973
+ } finally {
974
+ this.imageEngineInflight = null;
975
+ }
818
976
  }
819
977
  async ensureTextEngine() {
820
978
  if (this.textEngine) return;
979
+ if (this.textEngineInflight) return this.textEngineInflight;
821
980
  const meta = getModelMeta(this.config.modelId);
822
981
  const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
823
982
  if (!textEntry) throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
824
- await this.resolveForEntry(textEntry, "text");
983
+ const inflight = this.resolveForEntry(textEntry, "text");
984
+ this.textEngineInflight = inflight;
985
+ try {
986
+ await inflight;
987
+ } finally {
988
+ this.textEngineInflight = null;
989
+ }
825
990
  }
826
991
  async resolveForEntry(entry, target) {
827
992
  const engineLogger = this.ctx.logger.withTags({
@@ -847,6 +1012,8 @@ var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
847
1012
  this.textEngine = textEngine;
848
1013
  }
849
1014
  async onShutdown() {
1015
+ this.memoryGuard?.stop();
1016
+ this.memoryGuard = null;
850
1017
  await this.imageRawEngine?.dispose();
851
1018
  await this.textEngine?.dispose();
852
1019
  }