@camstack/addon-pipeline 1.2.93 → 1.2.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
- package/dist/audio-analyzer/index.js +58 -4
- package/dist/audio-analyzer/index.mjs +56 -2
- package/dist/detection-pipeline/index.js +194 -6
- package/dist/detection-pipeline/index.mjs +191 -3
- package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
- package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
- package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
- package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +195 -14
- package/dist/pipeline-runner/index.mjs +194 -13
- package/dist/process-memory-B9-UD7Rp.js +78 -0
- package/dist/process-memory-DXOB6eHX.mjs +66 -0
- package/dist/recorder/index.js +943 -109
- package/dist/recorder/index.mjs +942 -108
- package/dist/session-decode/decode-worker-child.js +4 -4
- package/dist/session-decode/decode-worker-child.mjs +3 -3
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
- package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
- package/dist/stream-broker/index.js +25 -7
- package/dist/stream-broker/index.mjs +25 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
- package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
- package/package.json +13 -7
- package/python/inference_pool.py +178 -3
- package/python/test_inference_pool_backpressure.py +7 -1
- package/python/test_inference_pool_memstats.py +146 -0
- package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
- package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
|
@@ -2,9 +2,9 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_dist = require("../dist-
|
|
6
|
-
const
|
|
7
|
-
const require_addon_utils = require("../addon-utils-
|
|
5
|
+
const require_dist = require("../dist-hyryByiw.js");
|
|
6
|
+
const require_process_memory = require("../process-memory-B9-UD7Rp.js");
|
|
7
|
+
const require_addon_utils = require("../addon-utils-UMWiMAxR.js");
|
|
8
8
|
let node_path = require("node:path");
|
|
9
9
|
node_path = require_dist.__toESM(node_path);
|
|
10
10
|
let node_fs = require("node:fs");
|
|
@@ -135,6 +135,9 @@ var YamnetPythonPipeline = class {
|
|
|
135
135
|
if (ready["status"] !== "ready") throw new Error(`YAMNet Python: unexpected init response: ${JSON.stringify(ready)}`);
|
|
136
136
|
this.log.info(`YAMNet Python pipeline initialized (${String(ready["labels"] ?? "?")} labels)`);
|
|
137
137
|
}
|
|
138
|
+
getPid() {
|
|
139
|
+
return this.process?.pid ?? null;
|
|
140
|
+
}
|
|
138
141
|
async classify(chunk) {
|
|
139
142
|
if (!this.process?.stdin) throw new Error("YAMNet Python: process not initialized");
|
|
140
143
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
@@ -241,6 +244,9 @@ var AppleSoundAnalysisPipeline = class {
|
|
|
241
244
|
if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
|
|
242
245
|
this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
|
|
243
246
|
}
|
|
247
|
+
getPid() {
|
|
248
|
+
return this.process?.pid ?? null;
|
|
249
|
+
}
|
|
244
250
|
async classify(chunk) {
|
|
245
251
|
if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
|
|
246
252
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
@@ -477,6 +483,53 @@ var AudioAnalyzerProvider = class {
|
|
|
477
483
|
this.log = logger;
|
|
478
484
|
this.backendName = initialBackend;
|
|
479
485
|
this.desiredBackend = initialBackend;
|
|
486
|
+
this.memoryGuard = new require_dist.PoolMemoryWatchdog({
|
|
487
|
+
policy: require_dist.resolvePoolMemoryPolicy(process.env),
|
|
488
|
+
log: this.log.child("pool-memory"),
|
|
489
|
+
sample: () => this.samplePipelineMemory(),
|
|
490
|
+
restart: () => this.restartPipelineForMemory()
|
|
491
|
+
});
|
|
492
|
+
this.memoryGuard.start();
|
|
493
|
+
}
|
|
494
|
+
/** Memory watchdog for the inference subprocess (see constructor). */
|
|
495
|
+
memoryGuard;
|
|
496
|
+
async samplePipelineMemory() {
|
|
497
|
+
const live = this.pipeline;
|
|
498
|
+
const pid = live?.getPid() ?? null;
|
|
499
|
+
if (!live || pid === null) return [];
|
|
500
|
+
const mem = await require_process_memory.readProcessMemory(pid);
|
|
501
|
+
if (!mem) {
|
|
502
|
+
this.log.warn("audio pipeline memory sample dropped", { meta: {
|
|
503
|
+
pid,
|
|
504
|
+
backend: this.backendName
|
|
505
|
+
} });
|
|
506
|
+
return [];
|
|
507
|
+
}
|
|
508
|
+
const mb = (bytes) => Math.round(bytes / (1024 * 1024));
|
|
509
|
+
return [{
|
|
510
|
+
key: "audio",
|
|
511
|
+
pids: [pid],
|
|
512
|
+
rssBytes: mem.rssBytes,
|
|
513
|
+
meta: {
|
|
514
|
+
backend: this.backendName,
|
|
515
|
+
vmMb: mb(mem.vmBytes),
|
|
516
|
+
peakRssMb: mb(mem.hwmBytes),
|
|
517
|
+
swapMb: mb(mem.swapBytes),
|
|
518
|
+
threads: mem.threads,
|
|
519
|
+
classifyCount: this.classifyCount
|
|
520
|
+
}
|
|
521
|
+
}];
|
|
522
|
+
}
|
|
523
|
+
/** Watchdog restart: tear the subprocess down NOW; the next classify
|
|
524
|
+
* re-spawns lazily (same recovery as an idle-dispose). An in-flight
|
|
525
|
+
* classify fails once and is handled by the existing classify error
|
|
526
|
+
* path — a bounded gap, unlike the kernel's OOM alternative. */
|
|
527
|
+
async restartPipelineForMemory() {
|
|
528
|
+
if (this.disposed) return;
|
|
529
|
+
const live = this.pipeline;
|
|
530
|
+
if (!live) return;
|
|
531
|
+
this.pipeline = null;
|
|
532
|
+
await live.dispose();
|
|
480
533
|
}
|
|
481
534
|
/**
|
|
482
535
|
* Lazily spawn + initialize the inference pipeline on first use. The
|
|
@@ -712,6 +765,7 @@ var AudioAnalyzerProvider = class {
|
|
|
712
765
|
}
|
|
713
766
|
async dispose() {
|
|
714
767
|
this.disposed = true;
|
|
768
|
+
this.memoryGuard.stop();
|
|
715
769
|
if (this.idleTimer) {
|
|
716
770
|
clearTimeout(this.idleTimer);
|
|
717
771
|
this.idleTimer = null;
|
|
@@ -890,7 +944,7 @@ var AudioAnalyzerAddon = class extends require_dist.BaseAddon {
|
|
|
890
944
|
const localNode = this.ctxIfReady?.kernel?.localNodeId;
|
|
891
945
|
if (!nodeId || nodeId === localNode) return process.platform;
|
|
892
946
|
try {
|
|
893
|
-
const { platform } =
|
|
947
|
+
const { platform } = require_process_memory.pickNodePlatformArch(await this.ctxIfReady?.api?.nodes?.topology.query(), nodeId);
|
|
894
948
|
if (platform) return platform;
|
|
895
949
|
} catch {}
|
|
896
950
|
return process.platform;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { n as __require } from "../chunk-DnnnRqeS.mjs";
|
|
2
|
-
import { D as
|
|
3
|
-
import {
|
|
2
|
+
import { D as audioAnalysisCapability, O as audioAnalyzerCapability, W as mapAudioLabelToMacro, ct as errMsg, h as HF_BASE_URL, mt as BaseAddon, n as AUDIO_BACKEND_CHOICES, s as DEFAULT_AUDIO_ANALYZER_CONFIG, tt as resolvePoolMemoryPolicy, y as PoolMemoryWatchdog, yt as hydrateSchema } from "../dist-C49o1k7u.mjs";
|
|
3
|
+
import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-DXOB6eHX.mjs";
|
|
4
4
|
import { i as downloadFile } from "../addon-utils-A2S9D7pu.mjs";
|
|
5
5
|
import * as path$1 from "node:path";
|
|
6
6
|
import * as fs from "node:fs";
|
|
@@ -130,6 +130,9 @@ var YamnetPythonPipeline = class {
|
|
|
130
130
|
if (ready["status"] !== "ready") throw new Error(`YAMNet Python: unexpected init response: ${JSON.stringify(ready)}`);
|
|
131
131
|
this.log.info(`YAMNet Python pipeline initialized (${String(ready["labels"] ?? "?")} labels)`);
|
|
132
132
|
}
|
|
133
|
+
getPid() {
|
|
134
|
+
return this.process?.pid ?? null;
|
|
135
|
+
}
|
|
133
136
|
async classify(chunk) {
|
|
134
137
|
if (!this.process?.stdin) throw new Error("YAMNet Python: process not initialized");
|
|
135
138
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
@@ -236,6 +239,9 @@ var AppleSoundAnalysisPipeline = class {
|
|
|
236
239
|
if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
|
|
237
240
|
this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
|
|
238
241
|
}
|
|
242
|
+
getPid() {
|
|
243
|
+
return this.process?.pid ?? null;
|
|
244
|
+
}
|
|
239
245
|
async classify(chunk) {
|
|
240
246
|
if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
|
|
241
247
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
@@ -472,6 +478,53 @@ var AudioAnalyzerProvider = class {
|
|
|
472
478
|
this.log = logger;
|
|
473
479
|
this.backendName = initialBackend;
|
|
474
480
|
this.desiredBackend = initialBackend;
|
|
481
|
+
this.memoryGuard = new PoolMemoryWatchdog({
|
|
482
|
+
policy: resolvePoolMemoryPolicy(process.env),
|
|
483
|
+
log: this.log.child("pool-memory"),
|
|
484
|
+
sample: () => this.samplePipelineMemory(),
|
|
485
|
+
restart: () => this.restartPipelineForMemory()
|
|
486
|
+
});
|
|
487
|
+
this.memoryGuard.start();
|
|
488
|
+
}
|
|
489
|
+
/** Memory watchdog for the inference subprocess (see constructor). */
|
|
490
|
+
memoryGuard;
|
|
491
|
+
async samplePipelineMemory() {
|
|
492
|
+
const live = this.pipeline;
|
|
493
|
+
const pid = live?.getPid() ?? null;
|
|
494
|
+
if (!live || pid === null) return [];
|
|
495
|
+
const mem = await readProcessMemory(pid);
|
|
496
|
+
if (!mem) {
|
|
497
|
+
this.log.warn("audio pipeline memory sample dropped", { meta: {
|
|
498
|
+
pid,
|
|
499
|
+
backend: this.backendName
|
|
500
|
+
} });
|
|
501
|
+
return [];
|
|
502
|
+
}
|
|
503
|
+
const mb = (bytes) => Math.round(bytes / (1024 * 1024));
|
|
504
|
+
return [{
|
|
505
|
+
key: "audio",
|
|
506
|
+
pids: [pid],
|
|
507
|
+
rssBytes: mem.rssBytes,
|
|
508
|
+
meta: {
|
|
509
|
+
backend: this.backendName,
|
|
510
|
+
vmMb: mb(mem.vmBytes),
|
|
511
|
+
peakRssMb: mb(mem.hwmBytes),
|
|
512
|
+
swapMb: mb(mem.swapBytes),
|
|
513
|
+
threads: mem.threads,
|
|
514
|
+
classifyCount: this.classifyCount
|
|
515
|
+
}
|
|
516
|
+
}];
|
|
517
|
+
}
|
|
518
|
+
/** Watchdog restart: tear the subprocess down NOW; the next classify
|
|
519
|
+
* re-spawns lazily (same recovery as an idle-dispose). An in-flight
|
|
520
|
+
* classify fails once and is handled by the existing classify error
|
|
521
|
+
* path — a bounded gap, unlike the kernel's OOM alternative. */
|
|
522
|
+
async restartPipelineForMemory() {
|
|
523
|
+
if (this.disposed) return;
|
|
524
|
+
const live = this.pipeline;
|
|
525
|
+
if (!live) return;
|
|
526
|
+
this.pipeline = null;
|
|
527
|
+
await live.dispose();
|
|
475
528
|
}
|
|
476
529
|
/**
|
|
477
530
|
* Lazily spawn + initialize the inference pipeline on first use. The
|
|
@@ -707,6 +760,7 @@ var AudioAnalyzerProvider = class {
|
|
|
707
760
|
}
|
|
708
761
|
async dispose() {
|
|
709
762
|
this.disposed = true;
|
|
763
|
+
this.memoryGuard.stop();
|
|
710
764
|
if (this.idleTimer) {
|
|
711
765
|
clearTimeout(this.idleTimer);
|
|
712
766
|
this.idleTimer = null;
|
|
@@ -2,11 +2,11 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_dist = require("../dist-
|
|
6
|
-
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-
|
|
7
|
-
const require_lazy_sharp = require("../lazy-sharp-
|
|
8
|
-
const
|
|
9
|
-
const require_addon_utils = require("../addon-utils-
|
|
5
|
+
const require_dist = require("../dist-hyryByiw.js");
|
|
6
|
+
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-DPNTzGOT.js");
|
|
7
|
+
const require_lazy_sharp = require("../lazy-sharp-Dxg6UeW-.js");
|
|
8
|
+
const require_process_memory = require("../process-memory-B9-UD7Rp.js");
|
|
9
|
+
const require_addon_utils = require("../addon-utils-UMWiMAxR.js");
|
|
10
10
|
let sharp = require("sharp");
|
|
11
11
|
sharp = require_dist.__toESM(sharp);
|
|
12
12
|
let node_child_process = require("node:child_process");
|
|
@@ -708,6 +708,14 @@ var PoolWorker = class {
|
|
|
708
708
|
const payload = Buffer.from(JSON.stringify(cmd), "utf8");
|
|
709
709
|
return await this.dispatch(MSG_COMMAND, payload);
|
|
710
710
|
}
|
|
711
|
+
/** Command whose reply shape is NOT the load/unload/replace/status envelope
|
|
712
|
+
* (e.g. `mem_stats`) — returns the raw JSON record, so no cast is needed
|
|
713
|
+
* at the call site. */
|
|
714
|
+
async sendRawCommand(cmd) {
|
|
715
|
+
this.ensureReady();
|
|
716
|
+
const payload = Buffer.from(JSON.stringify(cmd), "utf8");
|
|
717
|
+
return this.dispatch(MSG_COMMAND, payload);
|
|
718
|
+
}
|
|
711
719
|
async dispose() {
|
|
712
720
|
const proc = this.process;
|
|
713
721
|
if (!proc) return;
|
|
@@ -1072,6 +1080,46 @@ var SharedInferencePool = class {
|
|
|
1072
1080
|
for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to replace model at index ${index}: ${resp.error ?? "unknown"}`);
|
|
1073
1081
|
return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
|
|
1074
1082
|
}
|
|
1083
|
+
/**
|
|
1084
|
+
* Per-worker Python-side memory diagnostics (`mem_stats` command):
|
|
1085
|
+
* RSS/VmSize/peak/threads from /proc/self, getrusage maxrss, glibc malloc
|
|
1086
|
+
* arena count, gc counts, served-request counters, per-model queue depth,
|
|
1087
|
+
* cache entry counts. Queried by the pool memory watchdog once per sweep —
|
|
1088
|
+
* the internals that distinguish "leak" from "working set" and that
|
|
1089
|
+
* /proc/<pid> alone cannot see. A worker that is not ready (or whose
|
|
1090
|
+
* command fails) contributes an `error` record instead of vanishing.
|
|
1091
|
+
*/
|
|
1092
|
+
async getMemStats() {
|
|
1093
|
+
const out = [];
|
|
1094
|
+
for (const [i, w] of this.workers.entries()) {
|
|
1095
|
+
if (!w.isReady()) {
|
|
1096
|
+
out.push({
|
|
1097
|
+
worker: `w${i}`,
|
|
1098
|
+
pid: w.getPid(),
|
|
1099
|
+
stats: null,
|
|
1100
|
+
error: "worker not ready"
|
|
1101
|
+
});
|
|
1102
|
+
continue;
|
|
1103
|
+
}
|
|
1104
|
+
try {
|
|
1105
|
+
const stats = await w.sendRawCommand({ cmd: "mem_stats" });
|
|
1106
|
+
out.push({
|
|
1107
|
+
worker: `w${i}`,
|
|
1108
|
+
pid: w.getPid(),
|
|
1109
|
+
stats,
|
|
1110
|
+
error: null
|
|
1111
|
+
});
|
|
1112
|
+
} catch (err) {
|
|
1113
|
+
out.push({
|
|
1114
|
+
worker: `w${i}`,
|
|
1115
|
+
pid: w.getPid(),
|
|
1116
|
+
stats: null,
|
|
1117
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1118
|
+
});
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
return out;
|
|
1122
|
+
}
|
|
1075
1123
|
async getStatus() {
|
|
1076
1124
|
if (this.workers.length === 0) return [];
|
|
1077
1125
|
return (await this.workers[0].sendCommand({ cmd: "status" })).models ?? [];
|
|
@@ -1790,6 +1838,29 @@ var EngineFactory = class {
|
|
|
1790
1838
|
getPoolPid() {
|
|
1791
1839
|
return this.pool?.getPid() ?? null;
|
|
1792
1840
|
}
|
|
1841
|
+
/** All live worker pids of the underlying Python pool (memory watchdog). */
|
|
1842
|
+
getPoolPids() {
|
|
1843
|
+
return (this.pool?.getPids() ?? []).filter((pid) => pid !== null);
|
|
1844
|
+
}
|
|
1845
|
+
/** Summed backlog / shed / dropped across the pool's workers — the cheap
|
|
1846
|
+
* internals the memory watchdog attaches to its periodic `pool memory`
|
|
1847
|
+
* line so a leak (RSS up, work flat) is separable from load. */
|
|
1848
|
+
getPoolBacklog() {
|
|
1849
|
+
if (!this.pool) return {
|
|
1850
|
+
inFlight: 0,
|
|
1851
|
+
shed: 0,
|
|
1852
|
+
dropped: 0
|
|
1853
|
+
};
|
|
1854
|
+
return {
|
|
1855
|
+
...this.pool.getBacklog(),
|
|
1856
|
+
dropped: this.pool.getDroppedResponseCount()
|
|
1857
|
+
};
|
|
1858
|
+
}
|
|
1859
|
+
/** Python-side per-worker memory diagnostics (see SharedInferencePool). */
|
|
1860
|
+
async getPoolMemStats() {
|
|
1861
|
+
if (!this.pool) return [];
|
|
1862
|
+
return this.pool.getMemStats();
|
|
1863
|
+
}
|
|
1793
1864
|
/**
|
|
1794
1865
|
* Whether this factory exposes the batched fast path (`batchInferRaw`).
|
|
1795
1866
|
*/
|
|
@@ -1999,6 +2070,65 @@ var IdlePoolReaper = class {
|
|
|
1999
2070
|
}
|
|
2000
2071
|
};
|
|
2001
2072
|
//#endregion
|
|
2073
|
+
//#region src/detection-pipeline/engine/pool-memory-guard.ts
|
|
2074
|
+
function createDetectionPoolMemoryGuard(source, logger) {
|
|
2075
|
+
const log = logger.child("pool-memory");
|
|
2076
|
+
return new require_dist.PoolMemoryWatchdog({
|
|
2077
|
+
policy: require_dist.resolvePoolMemoryPolicy(process.env),
|
|
2078
|
+
log,
|
|
2079
|
+
restart: (key) => source.restartPool(key),
|
|
2080
|
+
sample: async () => {
|
|
2081
|
+
const out = [];
|
|
2082
|
+
for (const pool of source.listPools()) {
|
|
2083
|
+
const telemetry = await samplePool(pool, log);
|
|
2084
|
+
if (telemetry) out.push(telemetry);
|
|
2085
|
+
}
|
|
2086
|
+
return out;
|
|
2087
|
+
}
|
|
2088
|
+
});
|
|
2089
|
+
}
|
|
2090
|
+
async function samplePool(pool, log) {
|
|
2091
|
+
const pids = pool.factory.getPoolPids();
|
|
2092
|
+
if (pids.length === 0) return null;
|
|
2093
|
+
let maxRssBytes = 0;
|
|
2094
|
+
const workers = [];
|
|
2095
|
+
for (const pid of pids) {
|
|
2096
|
+
const mem = await require_process_memory.readProcessMemory(pid);
|
|
2097
|
+
if (!mem) {
|
|
2098
|
+
log.warn("pool memory sample dropped for worker pid", { meta: {
|
|
2099
|
+
poolKey: pool.key,
|
|
2100
|
+
pid
|
|
2101
|
+
} });
|
|
2102
|
+
continue;
|
|
2103
|
+
}
|
|
2104
|
+
if (mem.rssBytes > maxRssBytes) maxRssBytes = mem.rssBytes;
|
|
2105
|
+
workers.push({
|
|
2106
|
+
pid,
|
|
2107
|
+
rssMb: Math.round(mem.rssBytes / (1024 * 1024)),
|
|
2108
|
+
vmMb: Math.round(mem.vmBytes / (1024 * 1024)),
|
|
2109
|
+
peakRssMb: Math.round(mem.hwmBytes / (1024 * 1024)),
|
|
2110
|
+
swapMb: Math.round(mem.swapBytes / (1024 * 1024)),
|
|
2111
|
+
threads: mem.threads
|
|
2112
|
+
});
|
|
2113
|
+
}
|
|
2114
|
+
if (workers.length === 0) return null;
|
|
2115
|
+
const backlog = pool.factory.getPoolBacklog();
|
|
2116
|
+
const memStats = await pool.factory.getPoolMemStats().catch((err) => err instanceof Error ? err.message : String(err));
|
|
2117
|
+
return {
|
|
2118
|
+
key: pool.key,
|
|
2119
|
+
pids,
|
|
2120
|
+
rssBytes: maxRssBytes,
|
|
2121
|
+
meta: {
|
|
2122
|
+
workers,
|
|
2123
|
+
inFlight: backlog.inFlight,
|
|
2124
|
+
shed: backlog.shed,
|
|
2125
|
+
dropped: backlog.dropped,
|
|
2126
|
+
modelsLoaded: pool.factory.listLoaded().length,
|
|
2127
|
+
memStats
|
|
2128
|
+
}
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
//#endregion
|
|
2002
2132
|
//#region src/detection-pipeline/engine-provisioner.ts
|
|
2003
2133
|
/** Incremental backoff growing to a ~5 min cap; retries indefinitely at cap. */
|
|
2004
2134
|
var BACKOFF_SCHEDULE_MS = [
|
|
@@ -4790,6 +4920,55 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
4790
4920
|
} })
|
|
4791
4921
|
});
|
|
4792
4922
|
static OVERRIDE_CACHE_TTL_MS = 6e4;
|
|
4923
|
+
/**
|
|
4924
|
+
* RSS bound + periodic memory telemetry for the Python inference pools
|
|
4925
|
+
* (2026-08-18 OOM audit: the pools, not the recorder, were the host's OOM
|
|
4926
|
+
* driver — grow for hours, kernel-killed at 3.8/8.8 GB, respawn, repeat).
|
|
4927
|
+
* Policy, restart budget and the `pool memory` Loki line live in
|
|
4928
|
+
* `@camstack/types` (`PoolMemoryWatchdog`); this provider only enumerates
|
|
4929
|
+
* its pools and exposes its EXISTING dispose-and-recreate machinery as the
|
|
4930
|
+
* restart action. Started in `init()`, stopped in `shutdown()`.
|
|
4931
|
+
*/
|
|
4932
|
+
poolMemoryGuard = null;
|
|
4933
|
+
/** Watchdog view of the live pools: the node-default pool plus every
|
|
4934
|
+
* per-device pool. Keys are stable per pool identity so the watchdog's
|
|
4935
|
+
* baseline tracks one pool across sweeps. */
|
|
4936
|
+
listGuardedPools() {
|
|
4937
|
+
const pools = [];
|
|
4938
|
+
if (this.engineFactory) pools.push({
|
|
4939
|
+
key: `default:${deviceKeyOf(this.currentEngine)}`,
|
|
4940
|
+
factory: this.engineFactory
|
|
4941
|
+
});
|
|
4942
|
+
for (const [deviceKey, factory] of this.factoriesByDevice) pools.push({
|
|
4943
|
+
key: deviceKey,
|
|
4944
|
+
factory
|
|
4945
|
+
});
|
|
4946
|
+
return pools;
|
|
4947
|
+
}
|
|
4948
|
+
/**
|
|
4949
|
+
* Watchdog restart action — the same dispose-then-lazy-recreate the idle
|
|
4950
|
+
* reaper and the tuning respawn already use, never a second lifecycle. The
|
|
4951
|
+
* map/field is cleared FIRST so a concurrent dispatch rebuilds fresh via
|
|
4952
|
+
* the single-flight guards instead of grabbing the disposing instance;
|
|
4953
|
+
* models reload on the next dispatch via `ensureModelsForSteps`. Frames
|
|
4954
|
+
* arriving during the seconds-long re-spin are dropped (and counted by the
|
|
4955
|
+
* existing shed/timeout paths), not crashed — a bounded gap, unlike the
|
|
4956
|
+
* kernel's alternative.
|
|
4957
|
+
*/
|
|
4958
|
+
async restartGuardedPool(key) {
|
|
4959
|
+
if (key.startsWith("default:")) {
|
|
4960
|
+
const factory = this.engineFactory;
|
|
4961
|
+
if (!factory) return;
|
|
4962
|
+
this.engineFactory = null;
|
|
4963
|
+
await factory.dispose();
|
|
4964
|
+
return;
|
|
4965
|
+
}
|
|
4966
|
+
const factory = this.factoriesByDevice.get(key);
|
|
4967
|
+
if (!factory) return;
|
|
4968
|
+
this.factoriesByDevice.delete(key);
|
|
4969
|
+
this.deviceReaper.cancel(key);
|
|
4970
|
+
await factory.dispose();
|
|
4971
|
+
}
|
|
4793
4972
|
/** Read the addon settings store (all keys). */
|
|
4794
4973
|
readStore;
|
|
4795
4974
|
/** Write a patch to the addon settings store. */
|
|
@@ -4825,6 +5004,13 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
4825
5004
|
async init() {
|
|
4826
5005
|
this.needsAutoPick = true;
|
|
4827
5006
|
this.log.info("Node-default engine derive deferred to setApi()");
|
|
5007
|
+
if (!this.poolMemoryGuard) {
|
|
5008
|
+
this.poolMemoryGuard = createDetectionPoolMemoryGuard({
|
|
5009
|
+
listPools: () => this.listGuardedPools(),
|
|
5010
|
+
restartPool: (key) => this.restartGuardedPool(key)
|
|
5011
|
+
}, this.log);
|
|
5012
|
+
this.poolMemoryGuard.start();
|
|
5013
|
+
}
|
|
4828
5014
|
}
|
|
4829
5015
|
/**
|
|
4830
5016
|
* Lazy-install the pip requirements file matching `engine.backend` into
|
|
@@ -6655,6 +6841,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6655
6841
|
};
|
|
6656
6842
|
}
|
|
6657
6843
|
async shutdown() {
|
|
6844
|
+
this.poolMemoryGuard?.stop();
|
|
6845
|
+
this.poolMemoryGuard = null;
|
|
6658
6846
|
await this.evictOverrideCache("shutdown");
|
|
6659
6847
|
if (this.engineFactory) {
|
|
6660
6848
|
await this.engineFactory.dispose();
|
|
@@ -7798,7 +7986,7 @@ var DetectionPipelineAddon = class extends require_dist.BaseAddon {
|
|
|
7798
7986
|
*/
|
|
7799
7987
|
async resolveTopologyPlatform(nodeId) {
|
|
7800
7988
|
try {
|
|
7801
|
-
const picked =
|
|
7989
|
+
const picked = require_process_memory.pickNodePlatformArch(await this.ctxIfReady?.api?.nodes?.topology.query(), nodeId);
|
|
7802
7990
|
return {
|
|
7803
7991
|
platform: toNodePlatform(picked.platform),
|
|
7804
7992
|
arch: picked.arch
|