@camstack/addon-pipeline 1.2.18 → 1.2.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/audio-analyzer/index.js +1 -1
  2. package/dist/audio-analyzer/index.mjs +3 -2
  3. package/dist/chunk-DnnnRqeS.mjs +26 -0
  4. package/dist/detection-pipeline/index.js +2 -2
  5. package/dist/detection-pipeline/index.mjs +4 -3
  6. package/dist/{dist-OjOsoSZG.mjs → dist-CsKaXmP8.mjs} +500 -130
  7. package/dist/{dist-DDTAur0R.js → dist-DUHlZcWd.js} +505 -135
  8. package/dist/{model-download-service-Cp9f4dk6-WO-foD7W.mjs → model-download-service-Cp9f4dk6-CwUb5v3Y.mjs} +1 -26
  9. package/dist/motion-wasm/index.js +1 -1
  10. package/dist/motion-wasm/index.mjs +1 -1
  11. package/dist/pipeline-runner/index.js +162 -19
  12. package/dist/pipeline-runner/index.mjs +162 -19
  13. package/dist/recorder/index.js +2604 -458
  14. package/dist/recorder/index.mjs +2599 -459
  15. package/dist/{step-definitions-DGCuoeCj.js → step-definitions-Bo1hJQQw.js} +1 -1
  16. package/dist/{step-definitions-B9vyUNYR.mjs → step-definitions-C8A78dBi.mjs} +1 -1
  17. package/dist/stream-broker/_stub.js +2 -2
  18. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-S2nRs43G.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-tHahN-lK.mjs} +3 -3
  19. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-D1zJmtUe.mjs +26 -0
  20. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CSWUWZ8c.mjs +26 -0
  21. package/dist/stream-broker/{hostInit-p_rpDGqr.mjs → hostInit-CIwilJUZ.mjs} +3 -3
  22. package/dist/stream-broker/index.js +7374 -250
  23. package/dist/stream-broker/index.mjs +7371 -249
  24. package/dist/stream-broker/remoteEntry.js +1 -1
  25. package/embed-dist/assets/{MaskShapeCanvas-DI4BY7W2-Hg_mGtXA.js → MaskShapeCanvas-DI4BY7W2-B7VdNJlO.js} +1 -1
  26. package/embed-dist/assets/{MotionZonesSettings-NcxxQN8r-C0-pVeBX.js → MotionZonesSettings-NcxxQN8r-YyB4uLdH.js} +1 -1
  27. package/embed-dist/assets/{PrivacyMaskSettings-APgPLF7p-QVePTd0g.js → PrivacyMaskSettings-APgPLF7p-DfqixEad.js} +1 -1
  28. package/embed-dist/assets/{index-Dv5H-32C.css → index-CAe6Dg-X.css} +1 -1
  29. package/embed-dist/assets/index-zVLLVdIS.js +343 -0
  30. package/embed-dist/index.html +2 -2
  31. package/package.json +5 -1
  32. package/dist/fmp4-keyframes-5s8T9Mng.js +0 -6711
  33. package/dist/fmp4-keyframes-CV0X1mGy.mjs +0 -6694
  34. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BDeEvhV5.mjs +0 -26
  35. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BGtpZkgz.mjs +0 -26
  36. package/embed-dist/assets/index-CsaHbiy9.js +0 -116
@@ -1,33 +1,8 @@
1
- import { createRequire } from "node:module";
2
1
  import * as fs from "node:fs";
3
2
  import { createReadStream, promises } from "node:fs";
4
3
  import * as path$1 from "node:path";
5
4
  import path from "node:path";
6
5
  import "node:crypto";
7
- //#region \0rolldown/runtime.js
8
- var __create = Object.create;
9
- var __defProp = Object.defineProperty;
10
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
11
- var __getOwnPropNames = Object.getOwnPropertyNames;
12
- var __getProtoOf = Object.getPrototypeOf;
13
- var __hasOwnProp = Object.prototype.hasOwnProperty;
14
- var __commonJSMin = (cb, mod) => () => (mod || (cb((mod = { exports: {} }).exports, mod), cb = null), mod.exports);
15
- var __copyProps = (to, from, except, desc) => {
16
- if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
17
- key = keys[i];
18
- if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
19
- get: ((k) => from[k]).bind(null, key),
20
- enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
21
- });
22
- }
23
- return to;
24
- };
25
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
26
- value: mod,
27
- enumerable: true
28
- }) : target, mod));
29
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
30
- //#endregion
31
6
  //#region ../system/dist/model-download-service-Cp9f4dk6.mjs
32
7
  /**
33
8
  * Map a rel path to one candidate absolute path PER root, keeping only roots the
@@ -353,4 +328,4 @@ function deleteModelFromDisk(modelsDir, entry, format) {
353
328
  return true;
354
329
  }
355
330
  //#endregion
356
- export { ensureModel as a, __commonJSMin as c, downloadFile as i, __require as l, createFileDataPlaneHandler as n, isModelDownloaded as o, deleteModelFromDisk as r, parseRangeHeader as s, contentTypeFor as t, __toESM as u };
331
+ export { ensureModel as a, downloadFile as i, createFileDataPlaneHandler as n, isModelDownloaded as o, deleteModelFromDisk as r, parseRangeHeader as s, contentTypeFor as t };
@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
2
2
  __esModule: { value: true },
3
3
  [Symbol.toStringTag]: { value: "Module" }
4
4
  });
5
- const require_dist = require("../dist-DDTAur0R.js");
5
+ const require_dist = require("../dist-DUHlZcWd.js");
6
6
  let node_fs = require("node:fs");
7
7
  let node_path = require("node:path");
8
8
  //#region src/motion-wasm/wasm-motion-detector.ts
@@ -1,4 +1,4 @@
1
- import { A as motionDetectionCapability, K as hydrateSchema, T as evaluateZoneRules, V as BaseAddon, W as DeviceType } from "../dist-OjOsoSZG.mjs";
1
+ import { A as motionDetectionCapability, E as evaluateZoneRules, K as hydrateSchema, V as BaseAddon, W as DeviceType } from "../dist-CsKaXmP8.mjs";
2
2
  import { readFileSync } from "node:fs";
3
3
  import { join } from "node:path";
4
4
  //#region src/motion-wasm/wasm-motion-detector.ts
@@ -3,11 +3,11 @@ Object.defineProperties(exports, {
3
3
  [Symbol.toStringTag]: { value: "Module" }
4
4
  });
5
5
  const require_chunk = require("../chunk-emK7D4bc.js");
6
- const require_dist = require("../dist-DDTAur0R.js");
6
+ const require_dist = require("../dist-DUHlZcWd.js");
7
7
  const require_remote_restream = require("../remote-restream-CO36Sr30.js");
8
8
  const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
9
9
  const require_worker_protocol = require("../worker-protocol-DextwlTX.js");
10
- const require_step_definitions = require("../step-definitions-DGCuoeCj.js");
10
+ const require_step_definitions = require("../step-definitions-Bo1hJQQw.js");
11
11
  let node_child_process = require("node:child_process");
12
12
  let node_url = require("node:url");
13
13
  let sharp = require("sharp");
@@ -575,6 +575,22 @@ var PipelineTimingSampler = class {
575
575
  //#endregion
576
576
  //#region src/pipeline-runner/runner.ts
577
577
  /**
578
+ * Bound on one `processFrame` call. Chosen well above the slowest legitimate
579
+ * inference seen in production (8 617 ms, GPU contention on the hub) so the
580
+ * timeout only ever fires on a call that is genuinely never coming back.
581
+ */
582
+ var DEFAULT_INFERENCE_TIMEOUT_MS = 2e4;
583
+ /** Grace period before a permit-starved scheduler with queued work warns. */
584
+ var DEFAULT_INFERENCE_STALL_WARN_MS = 6e4;
585
+ /** Re-warn cadence while a stall persists, so a wedge stays visible. */
586
+ var INFERENCE_STALL_REWARN_MS = 6e4;
587
+ /**
588
+ * Sentinel distinguishing "the bound expired" from a legitimate `null`
589
+ * (`processFrame` returning no result). A unique symbol, so it can never
590
+ * collide with a value the executor produces.
591
+ */
592
+ var TIMED_OUT = Symbol("inference-timed-out");
593
+ /**
578
594
  * Clone a {@link DetectionQueueEntry} for the queue's copy-on-retain
579
595
  * boundary: the inner frame's pixel buffer is detached via
580
596
  * {@link ownFrame}, the `FrameHandle` is carried through unchanged (it's
@@ -657,6 +673,10 @@ var PipelineRunner = class {
657
673
  arrivalCounts = /* @__PURE__ */ new Map();
658
674
  lastGovernorRecomputeAt = Date.now();
659
675
  governorIntervalHandle = null;
676
+ /** When the scheduler first found itself with zero permits; null when healthy. */
677
+ inferenceStallSince = null;
678
+ /** Last stall warning, so a persistent wedge re-warns instead of going quiet. */
679
+ inferenceStallWarnedAt = null;
660
680
  constructor(config) {
661
681
  this.config = config;
662
682
  this.logger = config.logger;
@@ -1047,7 +1067,12 @@ var PipelineRunner = class {
1047
1067
  }
1048
1068
  tick() {
1049
1069
  this.drainMotionQueues();
1050
- if (this.semaphore.available <= 0) return;
1070
+ if (this.semaphore.available <= 0) {
1071
+ this.noteInferenceStall();
1072
+ return;
1073
+ }
1074
+ this.inferenceStallSince = null;
1075
+ this.inferenceStallWarnedAt = null;
1051
1076
  const picked = this.pickNextDetectionFrame();
1052
1077
  if (!picked) return;
1053
1078
  const { deviceId, entry, state } = picked;
@@ -1070,11 +1095,18 @@ var PipelineRunner = class {
1070
1095
  const pickedAt = Date.now();
1071
1096
  const { frame, handle } = entry;
1072
1097
  const enqueuedAt = frame._enqueuedAt ?? pickedAt;
1073
- const release = await this.semaphore.acquire();
1098
+ const rawRelease = await this.semaphore.acquire();
1099
+ let released = false;
1100
+ const release = () => {
1101
+ if (released) return;
1102
+ released = true;
1103
+ rawRelease();
1104
+ };
1074
1105
  const semAcquiredAt = Date.now();
1075
1106
  const executorHandle = state.detectionQueue.size <= HANDLE_PASSTHROUGH_MAX_QUEUE_DEPTH && this.semaphore.available > 0 ? handle : void 0;
1076
1107
  try {
1077
- const result = await this.config.processFrame(deviceId, frameInput, executorHandle);
1108
+ const result = await this.awaitInferenceBounded(deviceId, frameInput, executorHandle, release);
1109
+ if (result === TIMED_OUT) return;
1078
1110
  const inferDoneAt = Date.now();
1079
1111
  const inferenceMs = inferDoneAt - semAcquiredAt;
1080
1112
  state.inferenceTimes.push(inferenceMs);
@@ -1098,6 +1130,73 @@ var PipelineRunner = class {
1098
1130
  release();
1099
1131
  }
1100
1132
  }
1133
+ /**
1134
+ * Await `processFrame` with a hard bound. On expiry the permit is released
1135
+ * immediately (via the caller's release-once closure) and {@link TIMED_OUT}
1136
+ * is returned; the underlying call is left to settle into the void, since
1137
+ * the executor contract offers no cancellation.
1138
+ */
1139
+ async awaitInferenceBounded(deviceId, frameInput, executorHandle, release) {
1140
+ const timeoutMs = this.config.inferenceTimeoutMs ?? DEFAULT_INFERENCE_TIMEOUT_MS;
1141
+ let timer = null;
1142
+ const expiry = new Promise((resolve) => {
1143
+ timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
1144
+ timer.unref?.();
1145
+ });
1146
+ try {
1147
+ const outcome = await Promise.race([this.config.processFrame(deviceId, frameInput, executorHandle), expiry]);
1148
+ if (outcome === TIMED_OUT) {
1149
+ release();
1150
+ this.logger?.warn("inference timed out — permit released, result discarded", {
1151
+ tags: { deviceId },
1152
+ meta: {
1153
+ timeoutMs,
1154
+ availablePermits: this.semaphore.available
1155
+ }
1156
+ });
1157
+ }
1158
+ return outcome;
1159
+ } finally {
1160
+ if (timer !== null) clearTimeout(timer);
1161
+ }
1162
+ }
1163
+ /**
1164
+ * Called on every tick that finds zero available permits. Warns once the
1165
+ * starvation outlives `inferenceStallWarnMs` WITH queued work — queued work
1166
+ * is the discriminator: a runner with no permits and no backlog is simply
1167
+ * busy, while one with a backlog it cannot touch is wedged.
1168
+ */
1169
+ noteInferenceStall() {
1170
+ let queuedFrames = 0;
1171
+ for (const state of this.cameras.values()) queuedFrames += state.detectionQueue.size;
1172
+ if (queuedFrames === 0) {
1173
+ this.inferenceStallSince = null;
1174
+ this.inferenceStallWarnedAt = null;
1175
+ return;
1176
+ }
1177
+ const now = Date.now();
1178
+ if (this.inferenceStallSince === null) {
1179
+ this.inferenceStallSince = now;
1180
+ return;
1181
+ }
1182
+ const stalledMs = now - this.inferenceStallSince;
1183
+ if (stalledMs < (this.config.inferenceStallWarnMs ?? DEFAULT_INFERENCE_STALL_WARN_MS)) return;
1184
+ if (this.inferenceStallWarnedAt !== null && now - this.inferenceStallWarnedAt < INFERENCE_STALL_REWARN_MS) return;
1185
+ this.inferenceStallWarnedAt = now;
1186
+ for (const [deviceId, state] of this.cameras) {
1187
+ if (state.detectionQueue.size === 0) continue;
1188
+ this.logger?.warn("no inference permit available while frames are queued", {
1189
+ tags: { deviceId },
1190
+ meta: {
1191
+ stalledMs,
1192
+ queuedFrames: state.detectionQueue.size,
1193
+ droppedFrames: state.detectionQueue.droppedFrames,
1194
+ phase: state.phase,
1195
+ concurrency: this.semaphore.concurrency
1196
+ }
1197
+ });
1198
+ }
1199
+ }
1101
1200
  async notifyCallbacks(deviceId, frame, result, streamType, handle) {
1102
1201
  for (const callback of this.resultCallbacks) try {
1103
1202
  await callback(deviceId, frame, result, streamType, handle);
@@ -2063,10 +2162,14 @@ var NativeCropRegistry = class {
2063
2162
  * Bind `request` to `handle`; evicts the oldest entry past the cap, releasing
2064
2163
  * its worker-side RAM lease so a lease never outlives its registry entry.
2065
2164
  */
2066
- register(handle, request) {
2165
+ register(handle, request, deviceId) {
2067
2166
  const k = this.key(handle);
2068
2167
  this.entries.delete(k);
2069
- this.entries.set(k, request);
2168
+ this.entries.set(k, {
2169
+ request,
2170
+ deviceId,
2171
+ registeredAt: Date.now()
2172
+ });
2070
2173
  while (this.entries.size > this.capacity) {
2071
2174
  const oldest = this.entries.keys().next().value;
2072
2175
  if (oldest === void 0) break;
@@ -2075,7 +2178,22 @@ var NativeCropRegistry = class {
2075
2178
  }
2076
2179
  /** The requester for `handle`, or `null` if never registered / already evicted. */
2077
2180
  get(handle) {
2078
- return this.entries.get(this.key(handle)) ?? null;
2181
+ const entry = this.entries.get(this.key(handle));
2182
+ if (entry === void 0) return null;
2183
+ return {
2184
+ request: entry.request,
2185
+ deviceId: entry.deviceId,
2186
+ ageMs: Date.now() - entry.registeredAt
2187
+ };
2188
+ }
2189
+ /**
2190
+ * The deviceId a handle was registered under, even AFTER its entry was
2191
+ * evicted — kept in a small parallel ring so a `no-registry-entry` miss can
2192
+ * still be attributed to a camera (a miss line without `deviceId` cannot
2193
+ * answer "why is 617 worse than 615?").
2194
+ */
2195
+ deviceIdFor(handle) {
2196
+ return this.evictedDeviceIds.get(this.key(handle)) ?? this.entries.get(this.key(handle))?.deviceId ?? null;
2079
2197
  }
2080
2198
  /**
2081
2199
  * Release the worker-side RAM native lease for `handle` and drop its entry —
@@ -2087,15 +2205,27 @@ var NativeCropRegistry = class {
2087
2205
  }
2088
2206
  /** Drop every entry, releasing each frame's worker-side lease (node/addon shutdown). */
2089
2207
  clear() {
2090
- for (const request of this.entries.values()) safeRelease(request);
2208
+ for (const entry of this.entries.values()) safeRelease(entry.request);
2091
2209
  this.entries.clear();
2210
+ this.evictedDeviceIds.clear();
2092
2211
  }
2093
- /** Release + drop one entry by its map key. */
2212
+ /** Release + drop one entry by its map key, remembering its camera. */
2094
2213
  releaseEntry(key) {
2095
- const request = this.entries.get(key);
2096
- if (!request) return;
2214
+ const entry = this.entries.get(key);
2215
+ if (!entry) return;
2097
2216
  this.entries.delete(key);
2098
- safeRelease(request);
2217
+ this.rememberEvicted(key, entry.deviceId);
2218
+ safeRelease(entry.request);
2219
+ }
2220
+ /** Bounded memory of evicted handles → deviceId, for miss attribution. */
2221
+ evictedDeviceIds = /* @__PURE__ */ new Map();
2222
+ rememberEvicted(key, deviceId) {
2223
+ this.evictedDeviceIds.set(key, deviceId);
2224
+ while (this.evictedDeviceIds.size > this.capacity) {
2225
+ const oldest = this.evictedDeviceIds.keys().next().value;
2226
+ if (oldest === void 0) break;
2227
+ this.evictedDeviceIds.delete(oldest);
2228
+ }
2099
2229
  }
2100
2230
  };
2101
2231
  //#endregion
@@ -3301,16 +3431,29 @@ var PipelineRunnerAddon = class extends require_dist.BaseAddon {
3301
3431
  const isFullFrame = b.x <= .001 && b.y <= .001 && b.w >= .999 && b.h >= .999;
3302
3432
  const isUnboundedFullFrame = isFullFrame && input.maxWidth === void 0;
3303
3433
  let nativeResult = null;
3434
+ let missReason = null;
3304
3435
  if (!isUnboundedFullFrame) {
3305
- const requester = this.nativeCropRegistry.get(input.handle);
3306
- if (requester) {
3307
- const result = await requester(input.bbox, input.maxWidth);
3436
+ const found = this.nativeCropRegistry.get(input.handle);
3437
+ if (found) {
3438
+ const result = await found.request(input.bbox, input.maxWidth);
3308
3439
  if (result) nativeResult = {
3309
3440
  bytes: result.bytes,
3310
3441
  width: result.width,
3311
3442
  height: result.height
3312
3443
  };
3313
- }
3444
+ else missReason = "worker-lease-gone";
3445
+ } else missReason = "no-registry-entry";
3446
+ if (missReason !== null) this.ctx.logger.debug("native crop miss", {
3447
+ tags: { deviceId: found?.deviceId ?? this.nativeCropRegistry.deviceIdFor(input.handle) ?? -1 },
3448
+ meta: {
3449
+ reason: missReason,
3450
+ handle: `${input.handle.shmId}#${input.handle.slot}#${input.handle.seq}`,
3451
+ handleNodeId: input.handle.nodeId,
3452
+ handleAgeMs: found?.ageMs ?? null,
3453
+ registrySize: this.nativeCropRegistry.size,
3454
+ registryCap: DEFAULT_NATIVE_CROP_REGISTRY_CAP
3455
+ }
3456
+ });
3314
3457
  }
3315
3458
  const tier = resolveNativeCropSource(isFullFrame, nativeResult !== null);
3316
3459
  if (tier === "native" && nativeResult) return this.encodeNativeCropResult(nativeResult, tier, input.encodeJpeg);
@@ -3791,7 +3934,7 @@ var PipelineRunnerAddon = class extends require_dist.BaseAddon {
3791
3934
  onEnded: () => this.handleDetectionDecodeEnded(config.deviceId),
3792
3935
  onDecodedFrame: (frame, nativeCrop) => {
3793
3936
  const handle = retainedFrames?.write(config.deviceId, frame) ?? void 0;
3794
- if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop);
3937
+ if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
3795
3938
  this.enqueueSharedDetectionFrame(config.deviceId, frame, handle);
3796
3939
  }
3797
3940
  });
@@ -4304,7 +4447,7 @@ var PipelineRunnerAddon = class extends require_dist.BaseAddon {
4304
4447
  onDecodedFrame: (frame, nativeCrop) => {
4305
4448
  if (settled) return;
4306
4449
  const handle = retainedFrames?.write(config.deviceId, frame) ?? void 0;
4307
- if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop);
4450
+ if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
4308
4451
  runner.enqueueOccupancyFrame(deviceId, frame, handle);
4309
4452
  count++;
4310
4453
  if (count >= frames) {
@@ -1,8 +1,8 @@
1
- import { B as errMsg, G as createEvent, M as pipelineRunnerCapability, S as defineCustomActions, V as BaseAddon, X as nodePin, at as lazy, b as customAction, ct as object, ft as EventCategory, nt as array, rt as boolean, s as DEVICE_BACKEND_TO_FORMAT, st as number, tt as _enum, ut as string } from "../dist-OjOsoSZG.mjs";
1
+ import { B as errMsg, G as createEvent, M as pipelineRunnerCapability, S as defineCustomActions, V as BaseAddon, X as nodePin, at as lazy, b as customAction, ct as object, ft as EventCategory, nt as array, rt as boolean, s as DEVICE_BACKEND_TO_FORMAT, st as number, tt as _enum, ut as string } from "../dist-CsKaXmP8.mjs";
2
2
  import { n as profileForStreamId, t as isRemoteRestream } from "../remote-restream-BeHi78PZ.mjs";
3
3
  import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
4
4
  import { t as isWorkerReply } from "../worker-protocol-D7RzZIla.mjs";
5
- import { a as getStepDefinition } from "../step-definitions-B9vyUNYR.mjs";
5
+ import { a as getStepDefinition } from "../step-definitions-C8A78dBi.mjs";
6
6
  import { fork } from "node:child_process";
7
7
  import { fileURLToPath } from "node:url";
8
8
  import sharp from "sharp";
@@ -569,6 +569,22 @@ var PipelineTimingSampler = class {
569
569
  //#endregion
570
570
  //#region src/pipeline-runner/runner.ts
571
571
  /**
572
+ * Bound on one `processFrame` call. Chosen well above the slowest legitimate
573
+ * inference seen in production (8 617 ms, GPU contention on the hub) so the
574
+ * timeout only ever fires on a call that is genuinely never coming back.
575
+ */
576
+ var DEFAULT_INFERENCE_TIMEOUT_MS = 2e4;
577
+ /** Grace period before a permit-starved scheduler with queued work warns. */
578
+ var DEFAULT_INFERENCE_STALL_WARN_MS = 6e4;
579
+ /** Re-warn cadence while a stall persists, so a wedge stays visible. */
580
+ var INFERENCE_STALL_REWARN_MS = 6e4;
581
+ /**
582
+ * Sentinel distinguishing "the bound expired" from a legitimate `null`
583
+ * (`processFrame` returning no result). A unique symbol, so it can never
584
+ * collide with a value the executor produces.
585
+ */
586
+ var TIMED_OUT = Symbol("inference-timed-out");
587
+ /**
572
588
  * Clone a {@link DetectionQueueEntry} for the queue's copy-on-retain
573
589
  * boundary: the inner frame's pixel buffer is detached via
574
590
  * {@link ownFrame}, the `FrameHandle` is carried through unchanged (it's
@@ -651,6 +667,10 @@ var PipelineRunner = class {
651
667
  arrivalCounts = /* @__PURE__ */ new Map();
652
668
  lastGovernorRecomputeAt = Date.now();
653
669
  governorIntervalHandle = null;
670
+ /** When the scheduler first found itself with zero permits; null when healthy. */
671
+ inferenceStallSince = null;
672
+ /** Last stall warning, so a persistent wedge re-warns instead of going quiet. */
673
+ inferenceStallWarnedAt = null;
654
674
  constructor(config) {
655
675
  this.config = config;
656
676
  this.logger = config.logger;
@@ -1041,7 +1061,12 @@ var PipelineRunner = class {
1041
1061
  }
1042
1062
  tick() {
1043
1063
  this.drainMotionQueues();
1044
- if (this.semaphore.available <= 0) return;
1064
+ if (this.semaphore.available <= 0) {
1065
+ this.noteInferenceStall();
1066
+ return;
1067
+ }
1068
+ this.inferenceStallSince = null;
1069
+ this.inferenceStallWarnedAt = null;
1045
1070
  const picked = this.pickNextDetectionFrame();
1046
1071
  if (!picked) return;
1047
1072
  const { deviceId, entry, state } = picked;
@@ -1064,11 +1089,18 @@ var PipelineRunner = class {
1064
1089
  const pickedAt = Date.now();
1065
1090
  const { frame, handle } = entry;
1066
1091
  const enqueuedAt = frame._enqueuedAt ?? pickedAt;
1067
- const release = await this.semaphore.acquire();
1092
+ const rawRelease = await this.semaphore.acquire();
1093
+ let released = false;
1094
+ const release = () => {
1095
+ if (released) return;
1096
+ released = true;
1097
+ rawRelease();
1098
+ };
1068
1099
  const semAcquiredAt = Date.now();
1069
1100
  const executorHandle = state.detectionQueue.size <= HANDLE_PASSTHROUGH_MAX_QUEUE_DEPTH && this.semaphore.available > 0 ? handle : void 0;
1070
1101
  try {
1071
- const result = await this.config.processFrame(deviceId, frameInput, executorHandle);
1102
+ const result = await this.awaitInferenceBounded(deviceId, frameInput, executorHandle, release);
1103
+ if (result === TIMED_OUT) return;
1072
1104
  const inferDoneAt = Date.now();
1073
1105
  const inferenceMs = inferDoneAt - semAcquiredAt;
1074
1106
  state.inferenceTimes.push(inferenceMs);
@@ -1092,6 +1124,73 @@ var PipelineRunner = class {
1092
1124
  release();
1093
1125
  }
1094
1126
  }
1127
+ /**
1128
+ * Await `processFrame` with a hard bound. On expiry the permit is released
1129
+ * immediately (via the caller's release-once closure) and {@link TIMED_OUT}
1130
+ * is returned; the underlying call is left to settle into the void, since
1131
+ * the executor contract offers no cancellation.
1132
+ */
1133
+ async awaitInferenceBounded(deviceId, frameInput, executorHandle, release) {
1134
+ const timeoutMs = this.config.inferenceTimeoutMs ?? DEFAULT_INFERENCE_TIMEOUT_MS;
1135
+ let timer = null;
1136
+ const expiry = new Promise((resolve) => {
1137
+ timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
1138
+ timer.unref?.();
1139
+ });
1140
+ try {
1141
+ const outcome = await Promise.race([this.config.processFrame(deviceId, frameInput, executorHandle), expiry]);
1142
+ if (outcome === TIMED_OUT) {
1143
+ release();
1144
+ this.logger?.warn("inference timed out — permit released, result discarded", {
1145
+ tags: { deviceId },
1146
+ meta: {
1147
+ timeoutMs,
1148
+ availablePermits: this.semaphore.available
1149
+ }
1150
+ });
1151
+ }
1152
+ return outcome;
1153
+ } finally {
1154
+ if (timer !== null) clearTimeout(timer);
1155
+ }
1156
+ }
1157
+ /**
1158
+ * Called on every tick that finds zero available permits. Warns once the
1159
+ * starvation outlives `inferenceStallWarnMs` WITH queued work — queued work
1160
+ * is the discriminator: a runner with no permits and no backlog is simply
1161
+ * busy, while one with a backlog it cannot touch is wedged.
1162
+ */
1163
+ noteInferenceStall() {
1164
+ let queuedFrames = 0;
1165
+ for (const state of this.cameras.values()) queuedFrames += state.detectionQueue.size;
1166
+ if (queuedFrames === 0) {
1167
+ this.inferenceStallSince = null;
1168
+ this.inferenceStallWarnedAt = null;
1169
+ return;
1170
+ }
1171
+ const now = Date.now();
1172
+ if (this.inferenceStallSince === null) {
1173
+ this.inferenceStallSince = now;
1174
+ return;
1175
+ }
1176
+ const stalledMs = now - this.inferenceStallSince;
1177
+ if (stalledMs < (this.config.inferenceStallWarnMs ?? DEFAULT_INFERENCE_STALL_WARN_MS)) return;
1178
+ if (this.inferenceStallWarnedAt !== null && now - this.inferenceStallWarnedAt < INFERENCE_STALL_REWARN_MS) return;
1179
+ this.inferenceStallWarnedAt = now;
1180
+ for (const [deviceId, state] of this.cameras) {
1181
+ if (state.detectionQueue.size === 0) continue;
1182
+ this.logger?.warn("no inference permit available while frames are queued", {
1183
+ tags: { deviceId },
1184
+ meta: {
1185
+ stalledMs,
1186
+ queuedFrames: state.detectionQueue.size,
1187
+ droppedFrames: state.detectionQueue.droppedFrames,
1188
+ phase: state.phase,
1189
+ concurrency: this.semaphore.concurrency
1190
+ }
1191
+ });
1192
+ }
1193
+ }
1095
1194
  async notifyCallbacks(deviceId, frame, result, streamType, handle) {
1096
1195
  for (const callback of this.resultCallbacks) try {
1097
1196
  await callback(deviceId, frame, result, streamType, handle);
@@ -2057,10 +2156,14 @@ var NativeCropRegistry = class {
2057
2156
  * Bind `request` to `handle`; evicts the oldest entry past the cap, releasing
2058
2157
  * its worker-side RAM lease so a lease never outlives its registry entry.
2059
2158
  */
2060
- register(handle, request) {
2159
+ register(handle, request, deviceId) {
2061
2160
  const k = this.key(handle);
2062
2161
  this.entries.delete(k);
2063
- this.entries.set(k, request);
2162
+ this.entries.set(k, {
2163
+ request,
2164
+ deviceId,
2165
+ registeredAt: Date.now()
2166
+ });
2064
2167
  while (this.entries.size > this.capacity) {
2065
2168
  const oldest = this.entries.keys().next().value;
2066
2169
  if (oldest === void 0) break;
@@ -2069,7 +2172,22 @@ var NativeCropRegistry = class {
2069
2172
  }
2070
2173
  /** The requester for `handle`, or `null` if never registered / already evicted. */
2071
2174
  get(handle) {
2072
- return this.entries.get(this.key(handle)) ?? null;
2175
+ const entry = this.entries.get(this.key(handle));
2176
+ if (entry === void 0) return null;
2177
+ return {
2178
+ request: entry.request,
2179
+ deviceId: entry.deviceId,
2180
+ ageMs: Date.now() - entry.registeredAt
2181
+ };
2182
+ }
2183
+ /**
2184
+ * The deviceId a handle was registered under, even AFTER its entry was
2185
+ * evicted — kept in a small parallel ring so a `no-registry-entry` miss can
2186
+ * still be attributed to a camera (a miss line without `deviceId` cannot
2187
+ * answer "why is 617 worse than 615?").
2188
+ */
2189
+ deviceIdFor(handle) {
2190
+ return this.evictedDeviceIds.get(this.key(handle)) ?? this.entries.get(this.key(handle))?.deviceId ?? null;
2073
2191
  }
2074
2192
  /**
2075
2193
  * Release the worker-side RAM native lease for `handle` and drop its entry —
@@ -2081,15 +2199,27 @@ var NativeCropRegistry = class {
2081
2199
  }
2082
2200
  /** Drop every entry, releasing each frame's worker-side lease (node/addon shutdown). */
2083
2201
  clear() {
2084
- for (const request of this.entries.values()) safeRelease(request);
2202
+ for (const entry of this.entries.values()) safeRelease(entry.request);
2085
2203
  this.entries.clear();
2204
+ this.evictedDeviceIds.clear();
2086
2205
  }
2087
- /** Release + drop one entry by its map key. */
2206
+ /** Release + drop one entry by its map key, remembering its camera. */
2088
2207
  releaseEntry(key) {
2089
- const request = this.entries.get(key);
2090
- if (!request) return;
2208
+ const entry = this.entries.get(key);
2209
+ if (!entry) return;
2091
2210
  this.entries.delete(key);
2092
- safeRelease(request);
2211
+ this.rememberEvicted(key, entry.deviceId);
2212
+ safeRelease(entry.request);
2213
+ }
2214
+ /** Bounded memory of evicted handles → deviceId, for miss attribution. */
2215
+ evictedDeviceIds = /* @__PURE__ */ new Map();
2216
+ rememberEvicted(key, deviceId) {
2217
+ this.evictedDeviceIds.set(key, deviceId);
2218
+ while (this.evictedDeviceIds.size > this.capacity) {
2219
+ const oldest = this.evictedDeviceIds.keys().next().value;
2220
+ if (oldest === void 0) break;
2221
+ this.evictedDeviceIds.delete(oldest);
2222
+ }
2093
2223
  }
2094
2224
  };
2095
2225
  //#endregion
@@ -3295,16 +3425,29 @@ var PipelineRunnerAddon = class extends BaseAddon {
3295
3425
  const isFullFrame = b.x <= .001 && b.y <= .001 && b.w >= .999 && b.h >= .999;
3296
3426
  const isUnboundedFullFrame = isFullFrame && input.maxWidth === void 0;
3297
3427
  let nativeResult = null;
3428
+ let missReason = null;
3298
3429
  if (!isUnboundedFullFrame) {
3299
- const requester = this.nativeCropRegistry.get(input.handle);
3300
- if (requester) {
3301
- const result = await requester(input.bbox, input.maxWidth);
3430
+ const found = this.nativeCropRegistry.get(input.handle);
3431
+ if (found) {
3432
+ const result = await found.request(input.bbox, input.maxWidth);
3302
3433
  if (result) nativeResult = {
3303
3434
  bytes: result.bytes,
3304
3435
  width: result.width,
3305
3436
  height: result.height
3306
3437
  };
3307
- }
3438
+ else missReason = "worker-lease-gone";
3439
+ } else missReason = "no-registry-entry";
3440
+ if (missReason !== null) this.ctx.logger.debug("native crop miss", {
3441
+ tags: { deviceId: found?.deviceId ?? this.nativeCropRegistry.deviceIdFor(input.handle) ?? -1 },
3442
+ meta: {
3443
+ reason: missReason,
3444
+ handle: `${input.handle.shmId}#${input.handle.slot}#${input.handle.seq}`,
3445
+ handleNodeId: input.handle.nodeId,
3446
+ handleAgeMs: found?.ageMs ?? null,
3447
+ registrySize: this.nativeCropRegistry.size,
3448
+ registryCap: DEFAULT_NATIVE_CROP_REGISTRY_CAP
3449
+ }
3450
+ });
3308
3451
  }
3309
3452
  const tier = resolveNativeCropSource(isFullFrame, nativeResult !== null);
3310
3453
  if (tier === "native" && nativeResult) return this.encodeNativeCropResult(nativeResult, tier, input.encodeJpeg);
@@ -3785,7 +3928,7 @@ var PipelineRunnerAddon = class extends BaseAddon {
3785
3928
  onEnded: () => this.handleDetectionDecodeEnded(config.deviceId),
3786
3929
  onDecodedFrame: (frame, nativeCrop) => {
3787
3930
  const handle = retainedFrames?.write(config.deviceId, frame) ?? void 0;
3788
- if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop);
3931
+ if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
3789
3932
  this.enqueueSharedDetectionFrame(config.deviceId, frame, handle);
3790
3933
  }
3791
3934
  });
@@ -4298,7 +4441,7 @@ var PipelineRunnerAddon = class extends BaseAddon {
4298
4441
  onDecodedFrame: (frame, nativeCrop) => {
4299
4442
  if (settled) return;
4300
4443
  const handle = retainedFrames?.write(config.deviceId, frame) ?? void 0;
4301
- if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop);
4444
+ if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
4302
4445
  runner.enqueueOccupancyFrame(deviceId, frame, handle);
4303
4446
  count++;
4304
4447
  if (count >= frames) {