@camstack/addon-pipeline 1.2.294 → 1.2.296

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/THIRD_PARTY_MODELS.md +241 -0
  2. package/dist/audio-analyzer/index.js +2 -2
  3. package/dist/audio-analyzer/index.mjs +2 -2
  4. package/dist/{default-detection-model-0dPKRKUD.mjs → default-detection-model-Co578D8C.mjs} +181 -99
  5. package/dist/{default-detection-model-D24AJOTn.js → default-detection-model-D1daTtqT.js} +181 -99
  6. package/dist/detection-pipeline/index.js +1301 -531
  7. package/dist/detection-pipeline/index.mjs +1301 -531
  8. package/dist/{dist-CJR259Xf.js → dist-8up-f2TX.js} +3688 -2698
  9. package/dist/{dist-RXbmRAwP.mjs → dist-CCd0Q3nr.mjs} +3676 -2698
  10. package/dist/motion-wasm/index.js +1 -1
  11. package/dist/motion-wasm/index.mjs +1 -1
  12. package/dist/{node-atmRSHPk.mjs → node-DgMSXSWP.mjs} +1 -1
  13. package/dist/{node-DWg9zbY1.js → node-lpQgHes9.js} +1 -1
  14. package/dist/pipeline-runner/index.js +975 -270
  15. package/dist/pipeline-runner/index.mjs +975 -270
  16. package/dist/{process-memory-BJUXvTjd.js → process-memory-CX_92V_r.js} +1 -1
  17. package/dist/{process-memory-BgFOHFnx.mjs → process-memory-DFC_O5zE.mjs} +1 -1
  18. package/dist/recorder/index.js +14 -6
  19. package/dist/recorder/index.mjs +14 -6
  20. package/dist/{segment-demux-js-C_fPJub3.js → segment-demux-js-DzBx6NN2.js} +1 -1
  21. package/dist/{segment-demux-js-G7wFpHzn.mjs → segment-demux-js-FZbBuk3F.mjs} +1 -1
  22. package/dist/session-decode/{decode-worker-child.js → decode-worker-main.js} +481 -72
  23. package/dist/session-decode/{decode-worker-child.mjs → decode-worker-main.mjs} +482 -71
  24. package/dist/stream-broker/_stub.js +2 -2
  25. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-6IyM-BIn.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C_i7oFBl.mjs} +2 -2
  26. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DUGQKsKL.mjs +26 -0
  27. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CkbplMHA.mjs +26 -0
  28. package/dist/stream-broker/demux-worker-child.js +1 -1
  29. package/dist/stream-broker/demux-worker-child.mjs +1 -1
  30. package/dist/stream-broker/{hostInit-BBYHWS3M.mjs → hostInit-BPtppL3W.mjs} +2 -2
  31. package/dist/stream-broker/index.js +4 -4
  32. package/dist/stream-broker/index.mjs +4 -4
  33. package/dist/stream-broker/remoteEntry.js +1 -1
  34. package/dist/{worker-protocol-B2MfQLlu.js → worker-protocol-C-G8qmye.js} +3 -1
  35. package/dist/{worker-protocol-C_W-P_g-.mjs → worker-protocol-D_NzPcnh.mjs} +3 -1
  36. package/package.json +3 -2
  37. package/python/inference_pool.py +422 -64
  38. package/python/postprocessors/__init__.py +2 -0
  39. package/python/postprocessors/ssd.py +73 -17
  40. package/python/postprocessors/test_ssd.py +205 -0
  41. package/python/postprocessors/test_yunet.py +292 -0
  42. package/python/postprocessors/testdata/ssdlite_mobiledet_outputs.json +1 -0
  43. package/python/postprocessors/yunet.py +275 -0
  44. package/python/test_inference_pool_compile_off_loop.py +414 -0
  45. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-6IHzlLJ_.mjs +0 -26
  46. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DoyA71_q.mjs +0 -26
@@ -1,7 +1,7 @@
1
- import { $t as errMsg, An as literal, At as pickDetailCropConvention, C as FailureCounters, D as NativeLeaseAdmissionSchema, Dn as boolean, E as MOTION_CLOSE_AFTER_MS, En as array, Fn as string, Ln as EventCategory, Mn as object, Nt as pipelineRunnerCapability, O as NativeLeaseSettingsSchema, Tn as _enum, _ as DEFAULT_NATIVE_LEASE_SETTINGS, _t as loadContributionCapability, bn as nodePin, fn as createEvent, ft as failureContributionCapability, jn as number, jt as pickNativeLeaseOverride, kn as lazy, m as DEFAULT_DETAIL_CROP_CONVENTION, mn as defineCustomActions, on as BaseAddon, pn as customAction, tt as deriveDetailCropRect, v as DEVICE_BACKEND_TO_FORMAT } from "../dist-RXbmRAwP.mjs";
2
- import { i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-atmRSHPk.mjs";
3
- import { a as localFrameRegistry, d as getStepDefinition, i as rootObjectGate, o as resolveFrameViewGeometry, p as RETIRED_MODEL_IDS, r as macroGateVerdict } from "../default-detection-model-0dPKRKUD.mjs";
4
- import { t as isWorkerReply } from "../worker-protocol-C_W-P_g-.mjs";
1
+ import { D as MOTION_CLOSE_AFTER_MS, Dn as _enum, Ft as pipelineRunnerCapability, Ln as string, Mn as literal, Nn as number, Nt as pickNativeLeaseOverride, O as NativeLeaseAdmissionSchema, On as array, Pn as object, Sn as nodePin, cn as BaseAddon, gn as defineCustomActions, h as DEFAULT_DETAIL_CROP_CONVENTION, hn as customAction, i as BackgroundRunRegistry, jn as lazy, jt as pickDetailCropConvention, k as NativeLeaseSettingsSchema, kn as boolean, mn as createEvent, nt as deriveDetailCropRect, pt as failureContributionCapability, tn as errMsg, v as DEFAULT_NATIVE_LEASE_SETTINGS, vt as loadContributionCapability, w as FailureCounters, y as DEVICE_BACKEND_TO_FORMAT, zn as EventCategory } from "../dist-CCd0Q3nr.mjs";
2
+ import { i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-DgMSXSWP.mjs";
3
+ import { a as localFrameRegistry, d as getStepDefinition, i as rootObjectGate, o as resolveFrameViewGeometry, p as RETIRED_MODEL_IDS, r as macroGateVerdict } from "../default-detection-model-Co578D8C.mjs";
4
+ import { t as isWorkerReply } from "../worker-protocol-D_NzPcnh.mjs";
5
5
  import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
6
6
  import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
7
7
  import { c as withDetectionIntent, o as preRollRequestOf, t as DETECTION_PRE_ROLL_REQUEST_MS } from "../restream-intent-DyGmLAU9.mjs";
@@ -14,7 +14,7 @@ import { PerformanceObserver } from "node:perf_hooks";
14
14
  import { attributeStall } from "@camstack/system/addon-utils";
15
15
  //#region src/session-decode/fork-decode-worker.ts
16
16
  /** Absolute path to the built decode-worker child entry (a vite build entry). */
17
- var CHILD_ENTRY_PATH = fileURLToPath(new URL("../session-decode/decode-worker-child.js", import.meta.url));
17
+ var CHILD_ENTRY_PATH = fileURLToPath(new URL("../session-decode/decode-worker-main.js", import.meta.url));
18
18
  /**
19
19
  * Known kill signals the coordinator may pass, mapped to their `NodeJS.Signals`
20
20
  * literal so `ChildProcess.kill` receives a correctly-typed signal without a
@@ -35,21 +35,36 @@ var KILL_SIGNALS = {
35
35
  * every listener added out here is one somebody can forget to remove.
36
36
  */
37
37
  function wrapChild(child, onExit) {
38
- child.on("error", () => void 0);
39
- child.on("exit", onExit);
38
+ let spawned = false;
39
+ let exitDelivered = false;
40
+ const exitListeners = [];
41
+ const deliverExit = (info) => {
42
+ if (exitDelivered) return;
43
+ exitDelivered = true;
44
+ onExit();
45
+ for (const listener of exitListeners) listener(info);
46
+ };
47
+ child.on("spawn", () => {
48
+ spawned = true;
49
+ });
50
+ child.on("exit", (code, signal) => deliverExit({
51
+ code: code ?? null,
52
+ signal: signal ?? null
53
+ }));
54
+ child.on("error", () => {
55
+ if (!spawned) deliverExit({
56
+ code: null,
57
+ signal: null
58
+ });
59
+ });
40
60
  return {
41
61
  send: (msg) => {
42
62
  if (!child.connected) return;
43
63
  child.send(msg, () => void 0);
44
64
  },
45
65
  on: (ev, cb) => {
46
- if (ev === "exit") {
47
- const toInfo = (code, signal) => ({
48
- code,
49
- signal
50
- });
51
- child.on("exit", (code, signal) => cb(toInfo(code ?? null, signal ?? null)));
52
- } else child.on("message", (m) => cb(m));
66
+ if (ev === "exit") exitListeners.push(cb);
67
+ else child.on("message", (m) => cb(m));
53
68
  },
54
69
  kill: (signal) => {
55
70
  if (signal === void 0) {
@@ -250,58 +265,77 @@ var DEFAULT_ACQUIRE_RETRY_DELAYS_MS = [
250
265
  15e3,
251
266
  3e4
252
267
  ];
253
- /**
254
- * Guards the in-flight restream-acquire against a detach/unsubscribe race,
255
- * and retries an acquire that fails.
256
- *
257
- * `acquireSessionDecodeRestream` + forking the decode worker both take real
258
- * time (a broker round-trip, then a process fork). If the caller's teardown
259
- * runs while that acquire is still in flight, the closure that would release
260
- * the restream and stop the pump does not exist yet by the time the caller
261
- * stores it — a naive `await acquire(); return () => {...}` construction
262
- * leaks the acquired restream refcount and, once the acquire resolves, forks
263
- * an orphan decode worker nobody will ever stop.
264
- *
265
- * This helper returns the teardown SYNCHRONOUSLY. The acquire + pump-start
266
- * run in the background; an `aborted` flag recorded by the (possibly
267
- * already-called) teardown is checked right after each acquire resolves —
268
- * if set, any restream is released immediately and the pump is NEVER
269
- * started (no orphan worker). Otherwise the pump starts and the teardown,
270
- * once called, stops the pump then releases the restream.
271
- *
272
- * **A failed acquire is retried, not swallowed.** The teardown handed back
273
- * here is stored by the caller (as `attachment.detectionUnsubscribe`) the
274
- * instant this returns, so a `return` on failure left that handle permanently
275
- * non-null with no pump behind it: `onEnded` could never fire because nothing
276
- * had ever started, the camera's phase stayed `active`, and not one frame
277
- * arrived again. The motion path was hardened against this shape by a frame
278
- * freshness window (`MOTION_FRESHNESS_WINDOW_MS`); this is the detection
279
- * path's half.
280
- */
268
+ /** A restream release that runs once however many paths reach it. */
269
+ function onceRelease(release) {
270
+ let done = false;
271
+ return () => {
272
+ if (done) return Promise.resolve();
273
+ done = true;
274
+ return release();
275
+ };
276
+ }
281
277
  function startGuardedSessionDecode(deps) {
282
278
  let aborted = false;
279
+ /** The mode the teardown asked for — replayed if it raced `startPump`. */
280
+ let abortMode = "drain";
283
281
  let stopPump = null;
284
282
  let release = null;
285
283
  let retryTimer = null;
286
284
  /** The live-worker slot held by the RUNNING session, released on teardown. */
287
285
  let workerLease = null;
288
286
  const delays = deps.acquireRetryDelaysMs && deps.acquireRetryDelaysMs.length > 0 ? deps.acquireRetryDelaysMs : DEFAULT_ACQUIRE_RETRY_DELAYS_MS;
289
- /** Resolves after the backoff for `attempt`, or immediately once aborted. */
290
- const waitBackoff = (attempt) => new Promise((resolve) => {
287
+ /** Ends a parked start's wait when the session is torn down. */
288
+ let abortWait = null;
289
+ /**
290
+ * This start's place in the ceiling queue — taken on its first refusal, kept
291
+ * to the end so a re-park keeps the FIFO position (D652).
292
+ */
293
+ let waiter = null;
294
+ /**
295
+ * The start was handed a returned slot and has not yet ASKED for it. Set on
296
+ * a wake, cleared by the next `admitWorker` call; a start torn down while
297
+ * this is set passes the slot on (`SlotWaiter.leave(true)`), or nobody gets
298
+ * it and every other parked start waits out its full timer.
299
+ */
300
+ let wakeInHand = false;
301
+ /**
302
+ * Resolves after the backoff for `attempt` (`'timer'`) — or, when `parked`,
303
+ * as soon as the node hands this start a returned slot (`'woken'`) or the
304
+ * session is torn down (`'torn-down'`), whichever is first. The start leaves
305
+ * the queue on every exit, so a torn-down start leaves no listener behind.
306
+ */
307
+ const waitBackoff = (attempt, parked) => new Promise((resolve) => {
291
308
  const ms = delays[Math.min(attempt, delays.length - 1)] ?? delays[delays.length - 1] ?? 2e3;
292
- retryTimer = setTimeout(() => {
293
- retryTimer = null;
294
- resolve();
295
- }, ms);
309
+ const done = (why) => {
310
+ if (retryTimer !== null) {
311
+ clearTimeout(retryTimer);
312
+ retryTimer = null;
313
+ }
314
+ parked?.cancel();
315
+ abortWait = null;
316
+ resolve(why);
317
+ };
318
+ retryTimer = setTimeout(() => done("timer"), ms);
296
319
  retryTimer.unref?.();
320
+ abortWait = () => done("torn-down");
321
+ if (parked) parked.wait(() => done("woken"));
297
322
  });
298
- (async () => {
323
+ const releaseWorkerSlot = () => {
324
+ workerLease?.release();
325
+ workerLease = null;
326
+ };
327
+ const startLoop = async () => {
299
328
  for (let attempt = 0; !aborted; attempt++) {
300
329
  let lease = null;
301
330
  if (deps.admitWorker) {
331
+ wakeInHand = false;
302
332
  lease = deps.admitWorker();
303
333
  if (!lease) {
304
- await waitBackoff(attempt);
334
+ waiter ??= deps.slotWaiter?.() ?? null;
335
+ if (await waitBackoff(attempt, waiter) === "woken") {
336
+ attempt -= 1;
337
+ wakeInHand = true;
338
+ }
305
339
  continue;
306
340
  }
307
341
  }
@@ -314,7 +348,7 @@ function startGuardedSessionDecode(deps) {
314
348
  attempt: attempt + 1,
315
349
  error: errMsg(err)
316
350
  } });
317
- await waitBackoff(attempt);
351
+ await waitBackoff(attempt, null);
318
352
  continue;
319
353
  }
320
354
  if (aborted) {
@@ -325,27 +359,47 @@ function startGuardedSessionDecode(deps) {
325
359
  if (!acquired) {
326
360
  lease?.release();
327
361
  (attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: no broker profile bound yet — retrying", { meta: { attempt: attempt + 1 } });
328
- await waitBackoff(attempt);
362
+ await waitBackoff(attempt, null);
329
363
  continue;
330
364
  }
331
365
  if (attempt > 0) deps.logger.info("session-decode: acquire succeeded after retry", { meta: { attempts: attempt + 1 } });
332
- release = acquired.release;
366
+ release = onceRelease(acquired.release);
333
367
  workerLease = lease;
334
- stopPump = deps.startPump(acquired.source);
368
+ try {
369
+ stopPump = deps.startPump(acquired.source, { onWorkerGone: releaseWorkerSlot });
370
+ } catch (err) {
371
+ releaseWorkerSlot();
372
+ const restream = release;
373
+ release = null;
374
+ restream?.();
375
+ throw err;
376
+ }
377
+ if (aborted) stopPump(abortMode);
335
378
  return;
336
379
  }
337
- })().catch((err) => {
380
+ };
381
+ startLoop().catch((err) => {
338
382
  deps.logger.warn("session-decode: guarded start failed", { meta: { error: errMsg(err) } });
383
+ }).finally(() => {
384
+ waiter?.leave(wakeInHand);
385
+ waiter = null;
339
386
  });
340
- return () => {
387
+ return (mode = "drain") => {
388
+ if (aborted) {
389
+ if (mode === "immediate" && abortMode !== "immediate") {
390
+ abortMode = mode;
391
+ stopPump?.(mode);
392
+ }
393
+ return;
394
+ }
341
395
  aborted = true;
396
+ abortMode = mode;
342
397
  if (retryTimer !== null) {
343
398
  clearTimeout(retryTimer);
344
399
  retryTimer = null;
345
400
  }
346
- stopPump?.();
347
- workerLease?.release();
348
- workerLease = null;
401
+ abortWait?.();
402
+ stopPump?.(mode);
349
403
  if (release) release();
350
404
  };
351
405
  }
@@ -414,6 +468,21 @@ var LiveDecodeWorkerAdmission = class {
414
468
  limit;
415
469
  logger;
416
470
  live = 0;
471
+ /**
472
+ * Starts the ceiling refused, waiting to be told a slot came back — FIFO by
473
+ * the start's FIRST park (D652): the key's `seq` is fixed when the start
474
+ * takes its {@link SlotWaiter}, so a re-park keeps the place.
475
+ */
476
+ parked = /* @__PURE__ */ new Map();
477
+ nextSeq = 0;
478
+ /**
479
+ * Cameras currently refused. The refusal WARN fires on the transition into
480
+ * this set, not on every retry: a start woken or retried while the node is
481
+ * still full says so at debug (D652 — a wake must not become a WARN storm).
482
+ * Cleared on admit AND when a parked start leaves (`SlotWaiter.leave`): a
483
+ * camera torn down while refused must be news again next time (D391).
484
+ */
485
+ refusedDevices = /* @__PURE__ */ new Set();
417
486
  nodeBudgetMb;
418
487
  workerFootprintMb;
419
488
  constructor(options) {
@@ -422,6 +491,48 @@ var LiveDecodeWorkerAdmission = class {
422
491
  this.nodeBudgetMb = positiveOrUndefined(options.nodeBudgetMb);
423
492
  this.workerFootprintMb = positiveOrUndefined(options.workerFootprintMb);
424
493
  }
494
+ /** Starts parked on a {@link SlotWaiter} right now — diagnostics and tests. */
495
+ get slotWaiters() {
496
+ return this.parked.size;
497
+ }
498
+ /**
499
+ * This start's place in the queue. A refused start waits on it instead of a
500
+ * fixed timer, so a slot freed by a draining worker's exit is taken at once
501
+ * (D652). Take it once per start, on the first refusal; keep it to the end.
502
+ */
503
+ slotWaiter(deviceId) {
504
+ const key = {
505
+ seq: this.nextSeq,
506
+ deviceId
507
+ };
508
+ this.nextSeq += 1;
509
+ return {
510
+ wait: (listener) => {
511
+ this.parked.set(key, listener);
512
+ },
513
+ cancel: () => {
514
+ this.parked.delete(key);
515
+ },
516
+ leave: (unusedWake) => {
517
+ this.parked.delete(key);
518
+ this.refusedDevices.delete(deviceId);
519
+ if (unusedWake) this.wakeNext();
520
+ }
521
+ };
522
+ }
523
+ /**
524
+ * ONE slot, ONE wake: the longest-parked start (lowest `seq`). Waking all N
525
+ * produced N−1 refusals per freed slot. The woken start leaves the queue; if
526
+ * something else took the slot first it parks again, at the same place.
527
+ */
528
+ wakeNext() {
529
+ let first;
530
+ for (const key of this.parked.keys()) if (first === void 0 || key.seq < first.seq) first = key;
531
+ if (first === void 0) return;
532
+ const wake = this.parked.get(first);
533
+ this.parked.delete(first);
534
+ wake?.();
535
+ }
425
536
  /** Workers currently holding a slot. */
426
537
  get inFlight() {
427
538
  return this.live;
@@ -478,7 +589,9 @@ var LiveDecodeWorkerAdmission = class {
478
589
  admit(deviceId, label) {
479
590
  const ceiling = this.ceiling;
480
591
  if (this.live >= ceiling) {
481
- this.logger.warn("session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
592
+ const firstRefusal = !this.refusedDevices.has(deviceId);
593
+ this.refusedDevices.add(deviceId);
594
+ (firstRefusal ? this.logger.warn : this.logger.debug).call(this.logger, "session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
482
595
  tags: { deviceId },
483
596
  meta: {
484
597
  label,
@@ -494,6 +607,7 @@ var LiveDecodeWorkerAdmission = class {
494
607
  return null;
495
608
  }
496
609
  this.live += 1;
610
+ this.refusedDevices.delete(deviceId);
497
611
  this.logger.debug("session-decode: live decode-worker slot taken", {
498
612
  tags: { deviceId },
499
613
  meta: {
@@ -507,6 +621,7 @@ var LiveDecodeWorkerAdmission = class {
507
621
  if (released) return;
508
622
  released = true;
509
623
  this.live -= 1;
624
+ this.wakeNext();
510
625
  } };
511
626
  }
512
627
  };
@@ -1443,6 +1558,29 @@ function cropRgb(frame, bbox, maxWidth) {
1443
1558
  /** How long to wait after `kill()` before escalating to `kill('SIGKILL')`. */
1444
1559
  var SIGKILL_FALLBACK_MS = 1e3;
1445
1560
  /**
1561
+ * The longest a planned teardown keeps a session's worker alive so the frames
1562
+ * it already handed out can be read (D652).
1563
+ *
1564
+ * What is being waited for is the REST of a frame's trip through the runner —
1565
+ * semaphore wait + inference + result — for frames that were admitted before
1566
+ * the detach. Measured on the live hub 2026-09-26 over one hour of
1567
+ * `pipeline stats` (2 254 per-camera 10 s windows): the WORST enqueue→emit in a
1568
+ * window was ≤1 000 ms in 96.7% of windows and ≤1 500 ms in 98.0%; going to
1569
+ * 5 000 ms buys only another 0.9%, and that tail is the inference-pool GPU hang,
1570
+ * which no wait here can rescue. The cost of waiting is one idle worker (its
1571
+ * decode already stopped by `drain`) for at most this long, per detach.
1572
+ *
1573
+ * A frame still held when this expires is not dropped silently: the
1574
+ * coordinator logs the count, tagged with the camera, and then kills.
1575
+ */
1576
+ var SESSION_DECODE_DRAIN_BOUND_MS = 1500;
1577
+ /**
1578
+ * The miss reason for a native crop requested after its session's teardown.
1579
+ * Only crops already SENT when the drain began are waited for (D652); a later
1580
+ * one would ask a worker that is about to be killed.
1581
+ */
1582
+ var NATIVE_CROP_REFUSED_DRAINING = "session draining — crop requested after teardown";
1583
+ /**
1446
1584
  * How often the cold-start wait REPORTS that it is still waiting. Task 0
1447
1585
  * live-measured cold-start (restream dial + hwaccel context + keyframe wait) at
1448
1586
  * 5-8s, so this must tolerate a slow-but-healthy start.
@@ -1514,30 +1652,38 @@ var SessionDecodeSession = class {
1514
1652
  firstFrameTimeoutMs;
1515
1653
  frameTimeoutMs;
1516
1654
  firstFrameMaxWaitMs;
1517
- onDialTiming;
1518
- onPreRollReport;
1655
+ hooks;
1519
1656
  pendingPull = null;
1520
1657
  pendingToBuffers = /* @__PURE__ */ new Map();
1521
1658
  /** Outstanding `nativeCrop` requests, keyed by the monotonic `requestId`. */
1522
1659
  pendingNativeCrops = /* @__PURE__ */ new Map();
1523
1660
  nextNativeCropId = 1;
1524
1661
  exited = false;
1525
- /** Set by teardown() so handleExit can tell an intentional stop from a death. */
1662
+ /** Set by teardown() — the session is ending, whether or not it drains first. */
1526
1663
  stopRequested = false;
1664
+ /** Set when `kill()` is sent, so handleExit can tell an intentional stop from a death. */
1665
+ killSent = false;
1666
+ /**
1667
+ * Images yielded to the consumer and not yet closed. Each is a frame the
1668
+ * consumer still holds a reference to — queued, in flight, or being read —
1669
+ * and each dies with the worker. This is what a drain waits on.
1670
+ */
1671
+ openImages = 0;
1672
+ /** Armed while a drain waits; its expiry kills with frames still held. */
1673
+ drainTimer = null;
1527
1674
  sigkillTimer = null;
1528
1675
  pullTimeoutTimer = null;
1529
1676
  firstFrameReceived = false;
1530
1677
  /** `Date.now()` of the FIRST `pull` — the origin of the cold-start ceiling. */
1531
1678
  firstPullAt = 0;
1532
- constructor(child, logger, deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, onDialTiming, onPreRollReport) {
1679
+ constructor(child, logger, deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, hooks) {
1533
1680
  this.child = child;
1534
1681
  this.logger = logger;
1535
1682
  this.deviceId = deviceId;
1536
1683
  this.firstFrameTimeoutMs = firstFrameTimeoutMs;
1537
1684
  this.frameTimeoutMs = frameTimeoutMs;
1538
1685
  this.firstFrameMaxWaitMs = firstFrameMaxWaitMs;
1539
- this.onDialTiming = onDialTiming;
1540
- this.onPreRollReport = onPreRollReport;
1686
+ this.hooks = hooks;
1541
1687
  this.child.on("message", (m) => this.handleMessage(m));
1542
1688
  this.child.on("exit", (m) => this.handleExit(toWorkerExitInfo(m)));
1543
1689
  }
@@ -1602,6 +1748,10 @@ var SessionDecodeSession = class {
1602
1748
  */
1603
1749
  requestNativeCrop(frameId, bbox, maxWidth) {
1604
1750
  if (this.exited) return Promise.resolve({ kind: "miss" });
1751
+ if (this.stopRequested) return Promise.resolve({
1752
+ kind: "miss",
1753
+ workerReason: NATIVE_CROP_REFUSED_DRAINING
1754
+ });
1605
1755
  const requestId = this.nextNativeCropId++;
1606
1756
  return new Promise((resolve) => {
1607
1757
  this.pendingNativeCrops.set(requestId, { resolve });
@@ -1616,6 +1766,7 @@ var SessionDecodeSession = class {
1616
1766
  } catch {
1617
1767
  this.pendingNativeCrops.delete(requestId);
1618
1768
  resolve({ kind: "miss" });
1769
+ this.settleDrainIfDone();
1619
1770
  }
1620
1771
  });
1621
1772
  }
@@ -1660,20 +1811,135 @@ var SessionDecodeSession = class {
1660
1811
  frameId
1661
1812
  });
1662
1813
  }
1663
- /** Teardown: `stop`, `kill()`, arm the SIGKILL fallback unless already exited. */
1664
- teardown() {
1814
+ /** A yielded image the consumer now holds — see {@link openImages}. */
1815
+ noteImageOpened() {
1816
+ this.openImages += 1;
1817
+ }
1818
+ /** The consumer let go of a yielded image; a waiting drain may now be done. */
1819
+ noteImageClosed() {
1820
+ this.openImages = Math.max(0, this.openImages - 1);
1821
+ this.settleDrainIfDone();
1822
+ }
1823
+ /**
1824
+ * End the session.
1825
+ *
1826
+ * `'drain'` with frames still held: tell the worker to stop decoding (it
1827
+ * keeps every frame it holds), and kill only once every yielded image has
1828
+ * been closed and every read answered — or when
1829
+ * {@link SESSION_DECODE_DRAIN_BOUND_MS} expires, which is logged. Nothing
1830
+ * held: `stop` + SIGTERM now. `'immediate'`: SIGKILL now.
1831
+ *
1832
+ * A SECOND call that says `'immediate'` while a drain is in progress
1833
+ * ESCALATES (D652): addon shutdown lands on sessions already draining from
1834
+ * a detach, and the runner exits right after — a drain it does not wait for
1835
+ * is an orphaned worker holding its GPU pool. Any other repeat is ignored.
1836
+ */
1837
+ teardown(mode) {
1838
+ if (this.stopRequested) {
1839
+ if (mode === "immediate") this.escalateToImmediate();
1840
+ return;
1841
+ }
1665
1842
  this.stopRequested = true;
1666
1843
  this.clearPullTimeout();
1844
+ if (this.exited) return;
1845
+ if (mode === "drain" && this.unsettledCount() > 0) {
1846
+ this.trySend({ kind: "drain" });
1847
+ this.drainTimer = setTimeout(() => this.handleDrainBoundExpired(), SESSION_DECODE_DRAIN_BOUND_MS);
1848
+ this.drainTimer.unref?.();
1849
+ return;
1850
+ }
1851
+ this.killNow(mode);
1852
+ }
1853
+ /** The drain (or a SIGTERM still waiting on its fallback) gives way to SIGKILL now. */
1854
+ escalateToImmediate() {
1855
+ if (this.exited) return;
1856
+ if (this.drainTimer !== null) {
1857
+ clearTimeout(this.drainTimer);
1858
+ this.drainTimer = null;
1859
+ }
1860
+ if (!this.killSent) {
1861
+ this.killNow("immediate");
1862
+ return;
1863
+ }
1864
+ if (this.sigkillTimer !== null) {
1865
+ clearTimeout(this.sigkillTimer);
1866
+ this.sigkillTimer = null;
1867
+ }
1868
+ this.tryKill("SIGKILL");
1869
+ }
1870
+ /**
1871
+ * Frames the consumer still holds, reads still waiting on the worker, and
1872
+ * native crops already REQUESTED. A crop asked for before the teardown
1873
+ * request is waited for; one asked for after it is refused by name — post-analysis's
1874
+ * ordinary crop arrives 2.2–9.9 s after the frame, past any bound worth
1875
+ * holding a worker for, and is served from subject tiles that die with the
1876
+ * worker either way.
1877
+ */
1878
+ unsettledCount() {
1879
+ return this.openImages + this.pendingReadCount() + this.pendingNativeCrops.size;
1880
+ }
1881
+ pendingReadCount() {
1882
+ let n = 0;
1883
+ for (const waiters of this.pendingToBuffers.values()) n += waiters.length;
1884
+ return n;
1885
+ }
1886
+ settleDrainIfDone() {
1887
+ if (this.drainTimer === null || this.unsettledCount() > 0) return;
1888
+ clearTimeout(this.drainTimer);
1889
+ this.drainTimer = null;
1890
+ this.killNow("drain");
1891
+ }
1892
+ /**
1893
+ * The drain ran out of time with frames still held. Those frames die with
1894
+ * the worker; that is a branch that DROPS work (D391), so it says how many,
1895
+ * on which camera.
1896
+ */
1897
+ handleDrainBoundExpired() {
1898
+ this.drainTimer = null;
1899
+ if (this.exited) return;
1900
+ this.logger.warn("session-decode-coordinator: drain bound expired — killing the worker with frames still held", {
1901
+ tags: { deviceId: this.deviceId },
1902
+ meta: {
1903
+ unsettledFrames: this.openImages,
1904
+ pendingReads: this.pendingReadCount(),
1905
+ pendingCrops: this.pendingNativeCrops.size,
1906
+ boundMs: SESSION_DECODE_DRAIN_BOUND_MS
1907
+ }
1908
+ });
1909
+ this.killNow("drain");
1910
+ }
1911
+ /**
1912
+ * `stop`, then the kill — unless already exited.
1913
+ *
1914
+ * `'drain'`: SIGTERM, so the child flushes its last lines (`bounded-exit.ts`),
1915
+ * and a SIGKILL fallback. `'immediate'`: SIGKILL outright. Since D652 gave the
1916
+ * child a SIGTERM handler, SIGTERM is no longer instant death: a worker
1917
+ * blocked in a native call (a VAAPI hang) never runs the handler, and the
1918
+ * fallback was an `unref`'d timer the exiting parent did not wait for — so
1919
+ * on shutdown that worker was orphaned with its GPU pool. The fallback timer
1920
+ * is now ref'd: it exists only for a drain while the parent is alive (a
1921
+ * shutdown escalates to SIGKILL and clears it), and a timer that keeps the
1922
+ * process up for at most one second is the price of a kill that lands.
1923
+ */
1924
+ killNow(mode) {
1925
+ if (this.killSent) return;
1926
+ this.killSent = true;
1667
1927
  this.trySend({ kind: "stop" });
1928
+ if (mode === "immediate") {
1929
+ this.tryKill("SIGKILL");
1930
+ return;
1931
+ }
1668
1932
  this.tryKill(void 0);
1669
1933
  if (this.exited) return;
1670
1934
  this.sigkillTimer = setTimeout(() => {
1671
1935
  this.sigkillTimer = null;
1672
1936
  if (this.exited) return;
1673
- this.logger.warn("session-decode-coordinator: child did not exit within SIGKILL fallback window — forcing", { meta: { fallbackMs: SIGKILL_FALLBACK_MS } });
1937
+ this.logger.warn("session-decode-coordinator: child did not exit within SIGKILL fallback window — forcing", {
1938
+ tags: { deviceId: this.deviceId },
1939
+ meta: { fallbackMs: SIGKILL_FALLBACK_MS }
1940
+ });
1674
1941
  this.tryKill("SIGKILL");
1675
1942
  }, SIGKILL_FALLBACK_MS);
1676
- this.sigkillTimer.unref?.();
1677
1943
  }
1678
1944
  handleMessage(raw) {
1679
1945
  if (!isWorkerReply(raw)) {
@@ -1722,7 +1988,7 @@ var SessionDecodeSession = class {
1722
1988
  this.logger[raw.level](raw.message);
1723
1989
  return;
1724
1990
  case "dialTiming":
1725
- this.onDialTiming?.({
1991
+ this.hooks.onDialTiming?.({
1726
1992
  hwContextMs: raw.hwContextMs,
1727
1993
  demuxerOpenMs: raw.demuxerOpenMs,
1728
1994
  decoderCreateMs: raw.decoderCreateMs,
@@ -1730,25 +1996,46 @@ var SessionDecodeSession = class {
1730
1996
  streamInfoProbed: raw.streamInfoProbed
1731
1997
  });
1732
1998
  return;
1999
+ case "preRollSettled":
2000
+ this.hooks.onPreRollSettled?.({
2001
+ read: raw.read,
2002
+ released: raw.released,
2003
+ lostByReason: raw.lostByReason,
2004
+ unsettled: raw.unsettled
2005
+ });
2006
+ return;
1733
2007
  case "preRollReport":
1734
- this.onPreRollReport?.({
2008
+ this.hooks.onPreRollReport?.({
1735
2009
  offered: raw.offered,
1736
2010
  inferred: raw.inferred,
1737
2011
  dropped: raw.dropped,
1738
2012
  droppedByReason: raw.droppedByReason,
1739
2013
  strideMs: raw.strideMs,
1740
2014
  subsampled: raw.subsampled,
1741
- spanMs: raw.spanMs
2015
+ spanMs: raw.spanMs,
2016
+ budgetFrames: raw.budgetFrames
1742
2017
  });
1743
2018
  return;
1744
2019
  }
1745
2020
  }
1746
2021
  handleExit(info) {
1747
- if (!this.stopRequested && !this.exited) this.logger.warn("session-decode worker exited unexpectedly", { meta: {
1748
- code: info?.code ?? null,
1749
- signal: info?.signal ?? null
1750
- } });
2022
+ if (!this.exited) this.hooks.onWorkerExit?.();
2023
+ if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
2024
+ tags: { deviceId: this.deviceId },
2025
+ meta: {
2026
+ code: info?.code ?? null,
2027
+ signal: info?.signal ?? null,
2028
+ ...this.stopRequested ? {
2029
+ unsettledFrames: this.openImages,
2030
+ pendingReads: this.pendingReadCount()
2031
+ } : {}
2032
+ }
2033
+ });
1751
2034
  this.exited = true;
2035
+ if (this.drainTimer) {
2036
+ clearTimeout(this.drainTimer);
2037
+ this.drainTimer = null;
2038
+ }
1752
2039
  if (this.sigkillTimer) {
1753
2040
  clearTimeout(this.sigkillTimer);
1754
2041
  this.sigkillTimer = null;
@@ -1770,6 +2057,7 @@ var SessionDecodeSession = class {
1770
2057
  if (!waiter) return;
1771
2058
  this.pendingNativeCrops.delete(requestId);
1772
2059
  waiter.resolve(attempt);
2060
+ this.settleDrainIfDone();
1773
2061
  }
1774
2062
  settlePendingPull(outcome) {
1775
2063
  const waiter = this.pendingPull;
@@ -1865,6 +2153,7 @@ var SessionDecodeSession = class {
1865
2153
  const waiter = queue.shift();
1866
2154
  if (queue.length === 0) this.pendingToBuffers.delete(frameId);
1867
2155
  waiter?.resolve(bytes);
2156
+ this.settleDrainIfDone();
1868
2157
  }
1869
2158
  /**
1870
2159
  * `error` replies MAY carry a `frameId` (a `toBuffer`-scoped miss, e.g. a
@@ -1884,6 +2173,7 @@ var SessionDecodeSession = class {
1884
2173
  if (queue && queue.length > 0) {
1885
2174
  for (const waiter of queue) waiter.reject(err);
1886
2175
  this.pendingToBuffers.delete(frameId);
2176
+ this.settleDrainIfDone();
1887
2177
  return;
1888
2178
  }
1889
2179
  this.logger.debug("session-decode-coordinator: dropping toBuffer error for unknown frameId", { meta: { frameId } });
@@ -1898,6 +2188,7 @@ var SessionDecodeSession = class {
1898
2188
  }
1899
2189
  for (const waiters of this.pendingToBuffers.values()) for (const waiter of waiters) waiter.reject(err);
1900
2190
  this.pendingToBuffers.clear();
2191
+ this.settleDrainIfDone();
1901
2192
  }
1902
2193
  queueToBufferWaiter(frameId, waiter) {
1903
2194
  const queue = this.pendingToBuffers.get(frameId);
@@ -1935,6 +2226,7 @@ var SessionDecodeSession = class {
1935
2226
  /** Build the {@link FrameImage} adapter for one `frame` reply. */
1936
2227
  function buildFrameImage(session, reply) {
1937
2228
  let closed = false;
2229
+ session.noteImageOpened();
1938
2230
  return {
1939
2231
  width: reply.width,
1940
2232
  height: reply.height,
@@ -1949,7 +2241,9 @@ function buildFrameImage(session, reply) {
1949
2241
  releaseNativeLease: () => session.releaseLease(reply.frameId),
1950
2242
  markInferred: () => session.markInferred(reply.frameId),
1951
2243
  close: () => {
2244
+ if (closed) return;
1952
2245
  closed = true;
2246
+ session.noteImageClosed();
1953
2247
  }
1954
2248
  };
1955
2249
  }
@@ -1958,15 +2252,23 @@ function buildFrameImage(session, reply) {
1958
2252
  * `generateVideoFrames()` call, drives it with `pull`/`toBuffer` requests
1959
2253
  * one at a time (no pre-fetch — back-pressure is the async generator's
1960
2254
  * natural suspend-on-yield), and guarantees the child is torn down
1961
- * (`stop` + `kill()` + SIGKILL fallback) in every exit path via `finally`.
2255
+ * (bounded drain, then `stop` + `kill()` + SIGKILL fallback) in every exit
2256
+ * path via `finally`.
1962
2257
  */
1963
2258
  function createSessionDecodeCoordinator(deps) {
1964
2259
  const firstFrameTimeoutMs = deps.firstFrameTimeoutMs ?? DEFAULT_FIRST_FRAME_TIMEOUT_MS;
1965
2260
  const frameTimeoutMs = deps.frameTimeoutMs ?? DEFAULT_FRAME_TIMEOUT_MS;
1966
2261
  const firstFrameMaxWaitMs = Math.max(deps.firstFrameMaxWaitMs ?? DEFAULT_FIRST_FRAME_MAX_WAIT_MS, firstFrameTimeoutMs);
1967
2262
  return { async *generateVideoFrames(source, opts) {
1968
- const session = new SessionDecodeSession(deps.fork(source), deps.logger, source.deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, deps.onDialTiming, deps.onPreRollReport);
2263
+ const session = new SessionDecodeSession(deps.fork(source), deps.logger, source.deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, {
2264
+ ...deps.onDialTiming ? { onDialTiming: deps.onDialTiming } : {},
2265
+ ...deps.onPreRollReport ? { onPreRollReport: deps.onPreRollReport } : {},
2266
+ ...deps.onPreRollSettled ? { onPreRollSettled: deps.onPreRollSettled } : {},
2267
+ ...deps.onWorkerExit ? { onWorkerExit: deps.onWorkerExit } : {}
2268
+ });
1969
2269
  session.start(source, opts, deps.lease());
2270
+ deps.onSessionTeardown?.((mode) => session.teardown(mode));
2271
+ let failed = false;
1970
2272
  try {
1971
2273
  while (true) {
1972
2274
  const outcome = await session.pullNext();
@@ -1979,8 +2281,11 @@ function createSessionDecodeCoordinator(deps) {
1979
2281
  ...reply.preRoll === true ? { preRoll: true } : {}
1980
2282
  };
1981
2283
  }
2284
+ } catch (err) {
2285
+ failed = true;
2286
+ throw err;
1982
2287
  } finally {
1983
- session.teardown();
2288
+ session.teardown(failed ? "immediate" : deps.teardownMode?.() ?? "drain");
1984
2289
  }
1985
2290
  } };
1986
2291
  }
@@ -2024,17 +2329,39 @@ async function toDecodedFrame(frame, format, viewSpec) {
2024
2329
  * Start pulling frames from a per-session decode worker and feed each decoded
2025
2330
  * frame to `onDecodedFrame`. Returns an idempotent teardown closure that stops
2026
2331
  * the async generator (`it.return()`) — which triggers the coordinator's
2027
- * `finally` → `stop` + `kill()` + 1s SIGKILL fallback → worker exit → the OS
2028
- * reclaims the VAAPI/GPU decode pool. Fire-and-forget: the returned closure is
2029
- * synchronous; the frame loop and generator teardown run on their own tasks.
2332
+ * `finally` → a bounded DRAIN of the frames already handed out (D652), then
2333
+ * `stop` + SIGTERM + 1s SIGKILL fallback → worker exit → the OS reclaims the
2334
+ * VAAPI/GPU decode pool. `teardown('immediate')` SIGKILLs at once (shutdown),
2335
+ * and said a second time during a drain it escalates that drain to the kill.
2336
+ * Fire-and-forget: the returned closure is synchronous; the frame loop and
2337
+ * generator teardown run on their own tasks.
2030
2338
  */
2031
2339
  function startSessionDecodePump(deps) {
2032
2340
  const format = deps.opts.format ?? "rgb";
2033
2341
  let dialTiming = null;
2342
+ let teardownMode = "drain";
2343
+ let escalateSession = null;
2344
+ let forked = false;
2345
+ let workerGone = false;
2346
+ const noteWorkerGone = () => {
2347
+ if (workerGone) return;
2348
+ workerGone = true;
2349
+ deps.onWorkerGone?.();
2350
+ };
2034
2351
  const iterator = createSessionDecodeCoordinator({
2035
- fork: deps.fork,
2352
+ fork: (source) => {
2353
+ const child = deps.fork(source);
2354
+ forked = true;
2355
+ return child;
2356
+ },
2036
2357
  lease: deps.lease,
2037
2358
  logger: deps.logger,
2359
+ teardownMode: () => teardownMode,
2360
+ onSessionTeardown: (teardown) => {
2361
+ escalateSession = teardown;
2362
+ },
2363
+ onWorkerExit: noteWorkerGone,
2364
+ ...deps.onPreRollSettled ? { onPreRollSettled: deps.onPreRollSettled } : {},
2038
2365
  onDialTiming: (timing) => {
2039
2366
  dialTiming = timing;
2040
2367
  },
@@ -2140,13 +2467,21 @@ function startSessionDecodePump(deps) {
2140
2467
  });
2141
2468
  } finally {
2142
2469
  releaseSpawnSlotOnce();
2470
+ if (!forked) noteWorkerGone();
2143
2471
  if (!stopped) deps.onEnded?.();
2144
2472
  }
2145
2473
  };
2146
2474
  pump();
2147
- return () => {
2148
- if (stopped) return;
2475
+ return (mode = "drain") => {
2476
+ if (stopped) {
2477
+ if (mode === "immediate" && teardownMode !== "immediate") {
2478
+ teardownMode = "immediate";
2479
+ escalateSession?.("immediate");
2480
+ }
2481
+ return;
2482
+ }
2149
2483
  stopped = true;
2484
+ teardownMode = mode;
2150
2485
  releaseSpawnSlotOnce();
2151
2486
  iterator.return().catch(() => {});
2152
2487
  };
@@ -2228,28 +2563,17 @@ function onboardRootDetections(slot, frame, gate) {
2228
2563
  //#endregion
2229
2564
  //#region src/pipeline-runner/bench-step-schema.ts
2230
2565
  /**
2231
- * Shared bench-facing schemas: the minimal recursive step tree and the
2232
- * engine choice. Extracted from `bench-actions.ts` so the capacity-ramp
2233
- * schemas (`capacity/capacity-schemas.ts`) can reuse them without a
2234
- * bench-actions ↔ capacity-schemas import cycle.
2235
- *
2236
- * Local mirror of pipeline-executor's shapes ON PURPOSE (not a cap-package
2237
- * coupling): the shape is fixed by the runtime contract
2238
- * (runtime + backend + format + optional device / addonId + modelId +
2239
- * enabled + children).
2240
- */
2241
- var BenchEngineChoiceSchema = object({
2242
- runtime: _enum(["node", "python"]),
2243
- backend: string(),
2244
- format: _enum([
2245
- "onnx",
2246
- "coreml",
2247
- "openvino",
2248
- "tflite",
2249
- "pt"
2250
- ]),
2251
- device: string().optional()
2252
- });
2566
+ * Shared bench-facing schemas: the minimal recursive step tree. Extracted from
2567
+ * `bench-actions.ts` so the capacity-ramp schemas (`capacity/capacity-schemas.ts`)
2568
+ * can reuse it without a bench-actions ↔ capacity-schemas import cycle.
2569
+ *
2570
+ * Local mirror of pipeline-executor's step shape ON PURPOSE (not a cap-package
2571
+ * coupling): addonId + modelId + enabled + children.
2572
+ *
2573
+ * There is no engine choice here any more (D646). A bench names the pool it
2574
+ * measures by `deviceKey`; the engine override it used to carry was honoured by
2575
+ * swapping the executor's shared engine state under live cameras.
2576
+ */
2253
2577
  var BenchStepSchema = lazy(() => object({
2254
2578
  addonId: string(),
2255
2579
  modelId: string(),
@@ -2285,7 +2609,6 @@ var RunCapacityRampInputSchema = object({
2285
2609
  sessionId: string(),
2286
2610
  nodeId: string().optional(),
2287
2611
  deviceKey: string().optional(),
2288
- engine: BenchEngineChoiceSchema.optional(),
2289
2612
  slo: CapacitySloSchema.optional(),
2290
2613
  allowActiveCameras: boolean().optional(),
2291
2614
  drainTimeoutSec: number().min(1).max(300).optional()
@@ -2473,16 +2796,15 @@ var CacheBenchFrameResultSchema = object({
2473
2796
  });
2474
2797
  var ReleaseBenchFrameInputSchema = object({ frameId: string() });
2475
2798
  var ReleaseBenchFrameResultSchema = object({ released: boolean() });
2476
- var RunSyntheticBenchInputSchema = object({
2799
+ var StartSyntheticBenchInputSchema = object({
2477
2800
  frameId: string(),
2478
2801
  steps: array(BenchStepSchema).min(1),
2479
2802
  parallel: number().int().min(1).max(32),
2480
2803
  iterations: number().int().min(1).max(1e4),
2481
2804
  warmup: number().int().min(0).max(100).optional(),
2482
- sessionId: string().optional(),
2805
+ sessionId: string().min(1),
2483
2806
  simulatePipeline: boolean().optional(),
2484
2807
  nodeId: string().optional(),
2485
- engine: BenchEngineChoiceSchema.optional(),
2486
2808
  deviceKey: string().optional(),
2487
2809
  plane: _enum(["full", "frame"]).optional()
2488
2810
  });
@@ -2492,7 +2814,16 @@ var TimingSplitSchema = object({
2492
2814
  p95: number(),
2493
2815
  p99: number()
2494
2816
  });
2495
- var RunSyntheticBenchResultSchema = object({
2817
+ var BenchPoolSchema = object({
2818
+ nodeId: string(),
2819
+ deviceKey: string().nullable(),
2820
+ runtime: string().nullable(),
2821
+ numWorkers: number().nullable(),
2822
+ concurrency: number().nullable(),
2823
+ batchMode: string().nullable(),
2824
+ maxBatchSize: number().nullable()
2825
+ });
2826
+ var SyntheticBenchResultSchema = object({
2496
2827
  runs: number(),
2497
2828
  wallSec: number(),
2498
2829
  fps: number(),
@@ -2504,23 +2835,52 @@ var RunSyntheticBenchResultSchema = object({
2504
2835
  predictMs: number(),
2505
2836
  batchSizeMean: number(),
2506
2837
  batchSizeMax: number(),
2507
- engine: object({
2508
- runtime: string(),
2509
- backend: string(),
2510
- device: string().optional()
2511
- }).optional(),
2512
- tuning: object({
2513
- batchMode: string(),
2514
- windowMs: number(),
2515
- maxBatchSize: number(),
2516
- concurrency: number()
2517
- }).optional(),
2518
- path: string().optional()
2838
+ pool: BenchPoolSchema,
2839
+ path: string()
2519
2840
  });
2841
+ /** `startSyntheticBench` answers at once; the run continues in the background. */
2842
+ var StartSyntheticBenchAcceptedSchema = object({
2843
+ accepted: literal(true),
2844
+ sessionId: string()
2845
+ });
2846
+ var SyntheticBenchKeySchema = object({ sessionId: string() });
2847
+ var SyntheticBenchStatusSchema = object({
2848
+ sessionId: string(),
2849
+ state: _enum([
2850
+ "running",
2851
+ "done",
2852
+ "failed",
2853
+ "aborted"
2854
+ ]),
2855
+ startedAtMs: number(),
2856
+ finishedAtMs: number().nullable(),
2857
+ progress: object({
2858
+ runs: number(),
2859
+ totalRuns: number()
2860
+ }).nullable(),
2861
+ result: SyntheticBenchResultSchema.nullable(),
2862
+ error: string().nullable()
2863
+ }).nullable();
2520
2864
  var pipelineRunnerBenchActions = defineCustomActions({
2521
2865
  cacheBenchFrame: customAction(CacheBenchFrameInputSchema, CacheBenchFrameResultSchema, { kind: "mutation" }),
2522
2866
  releaseBenchFrame: customAction(ReleaseBenchFrameInputSchema, ReleaseBenchFrameResultSchema, { kind: "mutation" }),
2523
- runSyntheticBench: customAction(RunSyntheticBenchInputSchema, RunSyntheticBenchResultSchema, { kind: "mutation" }),
2867
+ /**
2868
+ * START a synthetic bench and return at once (D646). Up to 32 parallel ×
2869
+ * 10 000 iterations does not fit the UDS addon-call plane's 60 s request
2870
+ * timeout, and a synchronous call was abandoned by its caller while it kept
2871
+ * loading the node. Poll `getSyntheticBenchStatus {sessionId}`; stop early
2872
+ * with `abortSyntheticBench {sessionId}`. One bench per runner at a time —
2873
+ * the measurement owns the pool.
2874
+ */
2875
+ startSyntheticBench: customAction(StartSyntheticBenchInputSchema, StartSyntheticBenchAcceptedSchema, {
2876
+ kind: "mutation",
2877
+ auth: "admin"
2878
+ }),
2879
+ getSyntheticBenchStatus: customAction(SyntheticBenchKeySchema, SyntheticBenchStatusSchema),
2880
+ abortSyntheticBench: customAction(SyntheticBenchKeySchema, object({ aborted: boolean() }), {
2881
+ kind: "mutation",
2882
+ auth: "admin"
2883
+ }),
2524
2884
  /**
2525
2885
  * OPEN-loop capacity ramp (spec `2026-08-02-capacity-test-design.md` §4):
2526
2886
  * paced N×`fpsPerCamera` single-frame `runPipeline` calls with raw RGB —
@@ -2585,12 +2945,24 @@ function buildBenchRunPipelineInput(req) {
2585
2945
  return {
2586
2946
  steps: [...req.steps],
2587
2947
  frame: req.frame,
2588
- ...req.engine ? { engine: req.engine } : {},
2589
2948
  ...req.deviceKey ? { deviceKey: req.deviceKey } : {},
2590
2949
  ...req.plane ? { plane: req.plane } : {}
2591
2950
  };
2592
2951
  }
2593
2952
  /**
2953
+ * The tuning query for the pool a bench MEASURED (D646). It asked the node
2954
+ * default of whatever node answered first, so an NPU run was recorded beside
2955
+ * the onnx pool's `concurrency: 4`. Absent `deviceKey`, never
2956
+ * present-and-undefined, for the same `.loose()` reason as
2957
+ * {@link buildBenchRunPipelineInput}.
2958
+ */
2959
+ function buildBenchTuningQuery(target) {
2960
+ return {
2961
+ nodeId: target.nodeId,
2962
+ input: target.deviceKey ? { deviceKey: target.deviceKey } : {}
2963
+ };
2964
+ }
2965
+ /**
2594
2966
  * How long a box stays "seen".
2595
2967
  *
2596
2968
  * Sized on the gate's own clock, not on a guess: `maxDeferralMs` defaults to
@@ -3539,8 +3911,8 @@ async function runDetailSubtree(deps, input) {
3539
3911
  } : void 0;
3540
3912
  const details = [];
3541
3913
  for (const step of steps) {
3542
- const deviceKey = deps.resolveStepDevice?.(input.deviceId, step);
3543
- const result = await deps.runPipeline([step], crop.jpeg, input.deviceId, deviceKey, nativeCropRef);
3914
+ const placement = deps.placeStep?.(input.deviceId, step);
3915
+ const result = await deps.runPipeline([placement?.step ?? step], crop.jpeg, input.deviceId, placement?.deviceKey, nativeCropRef);
3544
3916
  if (!result) continue;
3545
3917
  for (const detection of result.detections) details.push(toDetailResult(detection, step.addonId, crop));
3546
3918
  }
@@ -3647,6 +4019,72 @@ function resolveStepDevice(input) {
3647
4019
  reason: "auto"
3648
4020
  };
3649
4021
  }
4022
+ /**
4023
+ * Re-provision a jumped subtree for its TARGET device (D650): every step's
4024
+ * `modelId` / `settings` are REPLACED by the target's entry — the orchestrator
4025
+ * computed it as what the step would carry had the camera been elected onto
4026
+ * that device (its per-(node,device) base with the camera's override for it).
4027
+ * A step absent from the map runs with neither, i.e. the target format's
4028
+ * default model. The elected device's provisioning never follows a step across:
4029
+ * it was chosen for a format this subtree does not run on.
4030
+ */
4031
+ function provisionSubtreeForDevice(step, provisioning) {
4032
+ const own = provisioning[step.addonId];
4033
+ const { modelId: _electedModelId, settings: _electedSettings, ...rest } = step;
4034
+ return {
4035
+ ...rest,
4036
+ ...own?.modelId !== void 0 ? { modelId: own.modelId } : {},
4037
+ ...own?.settings !== void 0 ? { settings: own.settings } : {},
4038
+ ...step.children !== void 0 ? { children: step.children.map((c) => provisionSubtreeForDevice(c, provisioning)) } : {}
4039
+ };
4040
+ }
4041
+ /**
4042
+ * Place ONE detail subtree (D650): which same-node device runs it
4043
+ * ({@link resolveStepDevice}) and, when it jumps, the provisioning it runs
4044
+ * with there ({@link provisionSubtreeForDevice}). Pure — the caller logs.
4045
+ *
4046
+ * A roster entry without `steps` came from an orchestrator that predates D650:
4047
+ * the step keeps what it was sent with, as it did before.
4048
+ */
4049
+ function placeDetailStep(input) {
4050
+ const { step, effectiveDeviceKey, roster } = input;
4051
+ if (!effectiveDeviceKey) return {
4052
+ deviceKey: void 0,
4053
+ step,
4054
+ decision: null,
4055
+ provisioned: false
4056
+ };
4057
+ if (!roster || roster.length < 2) return {
4058
+ deviceKey: effectiveDeviceKey,
4059
+ step,
4060
+ decision: null,
4061
+ provisioned: false
4062
+ };
4063
+ const decision = resolveStepDevice({
4064
+ effectiveDeviceKey,
4065
+ devices: roster.map((d) => ({
4066
+ deviceKey: d.deviceKey,
4067
+ weight: d.weight,
4068
+ maxSessions: d.maxSessions,
4069
+ attachedCameras: input.attachedByKey.get(d.deviceKey) ?? 0
4070
+ })),
4071
+ ...step.jumpDeviceKey !== void 0 ? { manualJumpDeviceKey: step.jumpDeviceKey } : {},
4072
+ canRun: buildSubtreeCanRun(step, input.getStepDef)
4073
+ });
4074
+ if (decision.kind !== "jumped") return {
4075
+ deviceKey: decision.deviceKey,
4076
+ step,
4077
+ decision,
4078
+ provisioned: false
4079
+ };
4080
+ const provisioning = roster.find((d) => d.deviceKey === decision.deviceKey)?.steps;
4081
+ return {
4082
+ deviceKey: decision.deviceKey,
4083
+ step: provisioning === void 0 ? step : provisionSubtreeForDevice(step, provisioning),
4084
+ decision,
4085
+ provisioned: provisioning !== void 0
4086
+ };
4087
+ }
3650
4088
  //#endregion
3651
4089
  //#region src/pipeline-runner/device-load.ts
3652
4090
  /** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
@@ -3830,6 +4268,74 @@ function release(cutter) {
3830
4268
  } catch {}
3831
4269
  }
3832
4270
  //#endregion
4271
+ //#region src/pipeline-runner/occupancy-burst-ledger.ts
4272
+ var OccupancyBurstLedger = class {
4273
+ options;
4274
+ frames = 0;
4275
+ inferred = 0;
4276
+ superseded = 0;
4277
+ settled = 0;
4278
+ closed = false;
4279
+ completed = false;
4280
+ boundTimer = null;
4281
+ constructor(options) {
4282
+ this.options = options;
4283
+ }
4284
+ /**
4285
+ * Count one collected frame and hand back its settler. A frame the runner
4286
+ * REFUSED is settled `'lost'` by the caller at once — it is a frame the
4287
+ * verdict did not see, which is what `lost` counts.
4288
+ */
4289
+ admit() {
4290
+ this.frames += 1;
4291
+ let done = false;
4292
+ return (outcome) => {
4293
+ if (done || this.completed) return;
4294
+ done = true;
4295
+ this.settled += 1;
4296
+ if (outcome === "inferred") this.inferred += 1;
4297
+ else if (outcome === "superseded") this.superseded += 1;
4298
+ this.completeIfSettled();
4299
+ };
4300
+ }
4301
+ /**
4302
+ * No more frames will be collected (count reached, safety timeout, or the
4303
+ * decode ended). Completes now when everything admitted has settled, else
4304
+ * arms the bound. Idempotent.
4305
+ */
4306
+ close() {
4307
+ if (this.closed || this.completed) return;
4308
+ this.closed = true;
4309
+ if (this.completeIfSettled()) return;
4310
+ this.boundTimer = setTimeout(() => {
4311
+ this.boundTimer = null;
4312
+ this.complete(true);
4313
+ }, this.options.settleBoundMs);
4314
+ this.boundTimer.unref?.();
4315
+ }
4316
+ completeIfSettled() {
4317
+ if (!this.closed || this.settled < this.frames) return false;
4318
+ this.complete(false);
4319
+ return true;
4320
+ }
4321
+ complete(timedOut) {
4322
+ if (this.completed) return;
4323
+ this.completed = true;
4324
+ if (this.boundTimer !== null) {
4325
+ clearTimeout(this.boundTimer);
4326
+ this.boundTimer = null;
4327
+ }
4328
+ this.options.onComplete({
4329
+ frames: this.frames,
4330
+ inferred: this.inferred,
4331
+ superseded: this.superseded,
4332
+ lost: this.frames - this.inferred - this.superseded,
4333
+ unsettled: this.frames - this.settled,
4334
+ timedOut
4335
+ });
4336
+ }
4337
+ };
4338
+ //#endregion
3833
4339
  //#region src/pipeline-runner/frame-lazy-metrics.ts
3834
4340
  function emptyFrameLazyCounters() {
3835
4341
  return {
@@ -5242,41 +5748,6 @@ var FrameGovernor = class {
5242
5748
  };
5243
5749
  //#endregion
5244
5750
  //#region src/pipeline-runner/frame-queue.ts
5245
- /**
5246
- * Latest-frame-only buffer. Keeps only the most recent item, dropping all
5247
- * older items immediately. This ensures inference always runs on the
5248
- * freshest available frame, never accumulating a backlog regardless of
5249
- * inference latency.
5250
- *
5251
- * ## Buffer ownership (Phase 5 / D9 Task 7b)
5252
- *
5253
- * Frames arrive from the shm frame plane via a `FrameRingReader`, whose
5254
- * `FrameRead.pixels` is **borrowed** — a view onto the reader's reusable
5255
- * scratch buffer, valid only until that reader's next read (see
5256
- * `@camstack/shm-ring`'s `FrameRead` contract). Both queue consumers retain
5257
- * the frame PAST the next read:
5258
- *
5259
- * - the **motion** queue is drained by the scheduler's `tick`, not by the
5260
- * poller that read the frame — and the poller drains a burst of handles
5261
- * per poll, so even handle #2's read would clobber handle #1's pixels;
5262
- * - the **detection** queue is drained asynchronously after a semaphore wait
5263
- * and a 10s–100s ms inference.
5264
- *
5265
- * In both cases holding the borrowed buffer is silent corruption. So
5266
- * `enqueue` runs the queue's injected `clone` function which copies the
5267
- * pixels into queue-owned storage at the retention boundary. The queue is
5268
- * latest-only (capacity 1), so this is exactly one allocation per *queued*
5269
- * frame — bounded by the scheduler/inference cadence, NOT per *read* frame.
5270
- * The high-frequency per-read allocation that failed the D9 perf gate is
5271
- * eliminated; this bounded copy-on-retain is the correct cost.
5272
- *
5273
- * The generic parameter `T` lets the detection path queue a
5274
- * `{ frame, handle }` entry (Task 7 — propagating the `FrameHandle`
5275
- * through to the inference-result event), while the motion path keeps
5276
- * queueing bare `DecodedFrame`s. Each instance provides its own
5277
- * `clone` so the inner pixel buffer is detached at the same retention
5278
- * boundary regardless of the wrapping entry shape.
5279
- */
5280
5751
  var FrameQueue = class {
5281
5752
  maxSize;
5282
5753
  latest = null;
@@ -5297,7 +5768,7 @@ var FrameQueue = class {
5297
5768
  enqueue(item) {
5298
5769
  if (this.latest !== null) {
5299
5770
  this._droppedFrames++;
5300
- this.dispose?.(this.latest);
5771
+ this.dispose?.(this.latest, "superseded");
5301
5772
  }
5302
5773
  this.latest = this.clone(item);
5303
5774
  }
@@ -5312,8 +5783,12 @@ var FrameQueue = class {
5312
5783
  get droppedFrames() {
5313
5784
  return this._droppedFrames;
5314
5785
  }
5786
+ /** The queued item, without taking it. */
5787
+ peek() {
5788
+ return this.latest ?? void 0;
5789
+ }
5315
5790
  clear() {
5316
- if (this.latest !== null) this.dispose?.(this.latest);
5791
+ if (this.latest !== null) this.dispose?.(this.latest, "cleared");
5317
5792
  this.latest = null;
5318
5793
  }
5319
5794
  };
@@ -5713,7 +6188,8 @@ function ownDetectionEntry(entry) {
5713
6188
  return {
5714
6189
  frame: ownFrame(entry.frame),
5715
6190
  handle: entry.handle,
5716
- occasion: entry.occasion
6191
+ occasion: entry.occasion,
6192
+ ...entry.onSettled ? { onSettled: entry.onSettled } : {}
5717
6193
  };
5718
6194
  }
5719
6195
  /**
@@ -5855,9 +6331,10 @@ var PipelineRunner = class {
5855
6331
  }
5856
6332
  registerCamera(deviceId, registration) {
5857
6333
  const motionQueue = new FrameQueue(this.config.maxQueueDepth, ownMotionEntry);
5858
- const detectionQueue = new FrameQueue(this.config.maxQueueDepth, ownDetectionEntry, (entry) => {
6334
+ const detectionQueue = new FrameQueue(this.config.maxQueueDepth, ownDetectionEntry, (entry, reason) => {
5859
6335
  if (entry.handle) this.config.releaseRetention?.(entry.handle, "drop");
5860
6336
  if (entry.frame.frameRef) this.config.releaseFrameRef?.(entry.frame.frameRef, "drop");
6337
+ entry.onSettled?.(reason === "superseded" ? "superseded" : "lost");
5861
6338
  });
5862
6339
  const initialPhase = registration.detectionMode === "disabled" ? "idle" : registration.detectionMode === "always-on" ? "active" : "watching";
5863
6340
  const preRollQueue = new PreRollLane({
@@ -5866,6 +6343,7 @@ var PipelineRunner = class {
5866
6343
  dispose: (entry) => {
5867
6344
  if (entry.handle) this.config.releaseRetention?.(entry.handle, "drop");
5868
6345
  if (entry.frame.frameRef) this.config.releaseFrameRef?.(entry.frame.frameRef, "drop");
6346
+ entry.onSettled?.("lost");
5869
6347
  }
5870
6348
  });
5871
6349
  this.logger?.info("camera registered: root detectors", {
@@ -5926,8 +6404,7 @@ var PipelineRunner = class {
5926
6404
  }
5927
6405
  if (state.phase === "active") this.detectionStreamHandler?.(deviceId, "unsubscribe");
5928
6406
  state.motionQueue.clear();
5929
- state.detectionQueue.clear();
5930
- state.preRollQueue.clear();
6407
+ this.discardQueuedDetectionFrames(deviceId, state, "detach");
5931
6408
  this.timingSampler.clearDroppedFramesSource(deviceId);
5932
6409
  this.cameras.delete(deviceId);
5933
6410
  this.lastAdmittedAt.delete(deviceId);
@@ -6051,9 +6528,14 @@ var PipelineRunner = class {
6051
6528
  preRollQueueDepth(deviceId) {
6052
6529
  return this.cameras.get(deviceId)?.preRollQueue.size ?? 0;
6053
6530
  }
6054
- /** Frames this camera lane REFUSED because it was full. */
6531
+ /**
6532
+ * Frames this camera lane REFUSED because it was full — or `null` when the
6533
+ * camera is no longer registered and the count cannot be read (a report
6534
+ * that lands after a detach). A failed read is `null`, never `0` (D393):
6535
+ * zero would say "nothing refused" about a lane nobody looked at.
6536
+ */
6055
6537
  preRollRefusedCount(deviceId) {
6056
- return this.cameras.get(deviceId)?.preRollQueue.refused ?? 0;
6538
+ return this.cameras.get(deviceId)?.preRollQueue.refused ?? null;
6057
6539
  }
6058
6540
  /**
6059
6541
  * Whether the node has a permit to spare for history right now — i.e. one
@@ -6082,7 +6564,7 @@ var PipelineRunner = class {
6082
6564
  * decode worker). `runInference` already falls back to the pixel path
6083
6565
  * when the handle is absent.
6084
6566
  */
6085
- enqueueOccupancyFrame(deviceId, frame, handle) {
6567
+ enqueueOccupancyFrame(deviceId, frame, handle, onSettled) {
6086
6568
  const state = this.cameras.get(deviceId);
6087
6569
  if (!state) return false;
6088
6570
  if (state.phase !== "watching") return false;
@@ -6090,7 +6572,8 @@ var PipelineRunner = class {
6090
6572
  state.detectionQueue.enqueue({
6091
6573
  frame,
6092
6574
  handle,
6093
- occasion: "burst"
6575
+ occasion: "burst",
6576
+ ...onSettled ? { onSettled } : {}
6094
6577
  });
6095
6578
  return true;
6096
6579
  }
@@ -6298,7 +6781,7 @@ var PipelineRunner = class {
6298
6781
  }
6299
6782
  transitionToWatching(deviceId, state, cooldownMs) {
6300
6783
  state.phase = "watching";
6301
- state.detectionQueue.clear();
6784
+ this.discardQueuedDetectionFrames(deviceId, state, "session-close");
6302
6785
  state.occupancyArmed = state.lastResultDetections > 0;
6303
6786
  state.emptyBurstStreak = 0;
6304
6787
  this.logger?.info("motion gate closed — phase=watching", {
@@ -6319,6 +6802,41 @@ var PipelineRunner = class {
6319
6802
  state.lastArmedSource = null;
6320
6803
  state.lastArmedRegions = void 0;
6321
6804
  }
6805
+ /**
6806
+ * Throw away the frames still QUEUED for a camera whose session is ending.
6807
+ *
6808
+ * A branch that drops work says so (D391, D652). These are frames the runner
6809
+ * admitted and will now never infer — the live queue's latest frame and, on a
6810
+ * detach, whatever history is left in the pre-roll lane. Frames already in
6811
+ * flight are not touched here: they finish, and the decode worker they live
6812
+ * in drains before it is killed (`SESSION_DECODE_DRAIN_BOUND_MS`).
6813
+ */
6814
+ discardQueuedDetectionFrames(deviceId, state, reason) {
6815
+ const queued = state.detectionQueue.size;
6816
+ const awaited = state.detectionQueue.peek()?.onSettled !== void 0 ? 1 : 0;
6817
+ const preRoll = reason === "detach" ? state.preRollQueue.size : 0;
6818
+ state.detectionQueue.clear();
6819
+ if (reason === "detach") state.preRollQueue.clear();
6820
+ if (queued + preRoll === 0) return;
6821
+ const meta = {
6822
+ reason,
6823
+ queued,
6824
+ awaited,
6825
+ preRoll,
6826
+ inflight: state.inflight
6827
+ };
6828
+ if (awaited + preRoll > 0) {
6829
+ this.logger?.info("detection frames discarded with the session", {
6830
+ tags: { deviceId },
6831
+ meta
6832
+ });
6833
+ return;
6834
+ }
6835
+ this.logger?.debug("detection frames discarded with the session", {
6836
+ tags: { deviceId },
6837
+ meta
6838
+ });
6839
+ }
6322
6840
  tick() {
6323
6841
  this.drainMotionQueues();
6324
6842
  if (this.semaphore.available <= 0) {
@@ -6406,6 +6924,7 @@ var PipelineRunner = class {
6406
6924
  } catch {}
6407
6925
  if (handle) this.config.releaseRetention?.(handle, frameReleaseReason);
6408
6926
  if (frame.frameRef) this.config.releaseFrameRef?.(frame.frameRef, frameReleaseReason);
6927
+ entry.onSettled?.(frameReleaseReason === "success" ? "inferred" : "lost");
6409
6928
  }
6410
6929
  }
6411
6930
  /**
@@ -6487,6 +7006,7 @@ var PipelineRunner = class {
6487
7006
  } catch {}
6488
7007
  if (handle) this.config.releaseRetention?.(handle, frameReleaseReason);
6489
7008
  if (frame.frameRef) this.config.releaseFrameRef?.(frame.frameRef, frameReleaseReason);
7009
+ entry.onSettled?.(frameReleaseReason === "success" ? "inferred" : "lost");
6490
7010
  }
6491
7011
  }
6492
7012
  /**
@@ -7030,6 +7550,19 @@ function writeRetainedDetectionFrame(store, deviceId, frame, retention) {
7030
7550
  */
7031
7551
  var OCCUPANCY_BURST_SAFETY_TIMEOUT_MS = 8e3;
7032
7552
  /**
7553
+ * How long a burst whose collection has ended waits for the frames it admitted
7554
+ * to be INFERRED before it reports anyway (D652).
7555
+ *
7556
+ * A burst's verdict is read off its frames' results, so it completes when they
7557
+ * settle, not when they are enqueued. This bound is only for a frame that never
7558
+ * settles. Measured on the live hub 2026-09-26, the worst enqueue→emit of a
7559
+ * 10 s window was ≤1.5 s in 98.0% of 2 254 windows and ≤5 s in 98.9%; what lies
7560
+ * beyond is the inference-pool GPU hang, which the runner's own 20 s inference
7561
+ * timeout already ends. A timed-out burst counts its unsettled frames as lost
7562
+ * and says so on its `burst complete` line.
7563
+ */
7564
+ var OCCUPANCY_BURST_SETTLE_BOUND_MS = 5e3;
7565
+ /**
7033
7566
  * Freshness window for `lastMotionFrameAt` (Task 10 hardening). A non-null
7034
7567
  * `attachment.motionUnsubscribe` only proves a subscription handle exists —
7035
7568
  * NOT that frames are actually flowing (guarded-session-decode can return
@@ -7567,6 +8100,26 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7567
8100
  benchFrameCache = /* @__PURE__ */ new Map();
7568
8101
  benchFrameSweeper = null;
7569
8102
  /**
8103
+ * Synthetic benches, start-then-poll (D646). One at a time on this runner —
8104
+ * the measurement owns the pool — and the last few finished ones kept for a
8105
+ * late poll. In RAM: a bench belongs to this process.
8106
+ */
8107
+ syntheticBenches = new BackgroundRunRegistry({
8108
+ maxConcurrent: 1,
8109
+ keepFinished: 5,
8110
+ onSettled: (status) => {
8111
+ const line = { meta: {
8112
+ sessionId: status.runId,
8113
+ state: status.state,
8114
+ error: status.error,
8115
+ runs: status.result?.runs ?? null,
8116
+ fps: status.result?.fps ?? null
8117
+ } };
8118
+ if (status.state === "failed") this.ctx.logger.warn("synthetic bench failed", line);
8119
+ else this.ctx.logger.info("synthetic bench finished", line);
8120
+ }
8121
+ });
8122
+ /**
7570
8123
  * In-flight + recent capacity-ramp runs, keyed by sessionId. The ramp is
7571
8124
  * fire-and-poll (a full run outlives the UDS addon-call 60s timeout), so
7572
8125
  * completed results are kept here for `getCapacityRampStatus` to collect;
@@ -7767,7 +8320,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7767
8320
  actionHandlers: {
7768
8321
  cacheBenchFrame: async (input) => this.cacheBenchFrame(input),
7769
8322
  releaseBenchFrame: async (input) => this.releaseBenchFrame(input),
7770
- runSyntheticBench: async (input) => this.runSyntheticBench(input),
8323
+ startSyntheticBench: async (input) => this.startSyntheticBench(input),
8324
+ getSyntheticBenchStatus: async (input) => this.getSyntheticBenchStatus(input.sessionId),
8325
+ abortSyntheticBench: async (input) => ({ aborted: this.syntheticBenches.abort(input.sessionId) }),
7771
8326
  runCapacityRamp: async (input) => this.runCapacityRampAction(input),
7772
8327
  getCapacityRampStatus: async (input) => this.getCapacityRampStatusAction(input),
7773
8328
  listCapacityRamps: async () => this.listCapacityRampsAction(),
@@ -7789,6 +8344,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7789
8344
  this.benchFrameSweeper = null;
7790
8345
  }
7791
8346
  this.benchFrameCache.clear();
8347
+ this.syntheticBenches.abortAll();
7792
8348
  for (const entry of this.capacityRuns.values()) entry.abortFlag.aborted = true;
7793
8349
  this.capacityRuns.clear();
7794
8350
  if (this.unsubMotionEvents) {
@@ -7816,8 +8372,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7816
8372
  this.runner = null;
7817
8373
  }
7818
8374
  for (const attachment of this.attached.values()) {
7819
- attachment.motionUnsubscribe?.();
7820
- attachment.detectionUnsubscribe?.();
8375
+ attachment.motionUnsubscribe?.("immediate");
8376
+ attachment.detectionUnsubscribe?.("immediate");
7821
8377
  }
7822
8378
  for (const deviceId of [...this.parkedDetail.keys()]) this.dropParkedDetail(deviceId);
7823
8379
  this.attached.clear();
@@ -7868,7 +8424,34 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7868
8424
  const now = Date.now();
7869
8425
  for (const [id, entry] of this.benchFrameCache) if (entry.expiresAt < now) this.benchFrameCache.delete(id);
7870
8426
  }
7871
- async runSyntheticBench(input) {
8427
+ /**
8428
+ * Validate the cheap preconditions synchronously (so a bad frame id is a
8429
+ * refusal the caller sees at once), then run the bench in the background
8430
+ * and answer with the receipt (D646).
8431
+ */
8432
+ startSyntheticBench(input) {
8433
+ if (!this.ctx.api) throw new Error("startSyntheticBench: ctx.api unavailable");
8434
+ if (!this.benchFrameCache.has(input.frameId)) throw new Error(`startSyntheticBench: frameId ${input.frameId} not cached (call cacheBenchFrame first)`);
8435
+ this.syntheticBenches.start(input.sessionId, (run) => this.runSyntheticBench(input, run));
8436
+ return {
8437
+ accepted: true,
8438
+ sessionId: input.sessionId
8439
+ };
8440
+ }
8441
+ getSyntheticBenchStatus(sessionId) {
8442
+ const status = this.syntheticBenches.status(sessionId);
8443
+ if (status === null) return null;
8444
+ return {
8445
+ sessionId: status.runId,
8446
+ state: status.state,
8447
+ startedAtMs: status.startedAtMs,
8448
+ finishedAtMs: status.finishedAtMs,
8449
+ progress: status.progress,
8450
+ result: status.result,
8451
+ error: status.error
8452
+ };
8453
+ }
8454
+ async runSyntheticBench(input, run) {
7872
8455
  const ctx = this.ctx;
7873
8456
  const api = ctx.api;
7874
8457
  if (!api) throw new Error("runSyntheticBench: ctx.api unavailable");
@@ -7918,8 +8501,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7918
8501
  }, pin)).frameId;
7919
8502
  await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
7920
8503
  steps: stepsToRun,
7921
- frame: sharedFrame,
7922
- ...input.engine ? { engine: input.engine } : {}
8504
+ frame: sharedFrame
7923
8505
  }) }, pin);
7924
8506
  const warmupCount = input.warmup ?? 1;
7925
8507
  for (let w = 0; w < warmupCount; w++) await api.pipelineExecutor.inferCached.mutate({
@@ -7933,11 +8515,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7933
8515
  const batchSizes = [];
7934
8516
  const detCounts = [];
7935
8517
  let _n = 0;
7936
- const sessionId = input.sessionId ?? `synth-${Date.now().toString(36)}`;
8518
+ const sessionId = input.sessionId;
7937
8519
  const totalRuns = input.parallel * input.iterations;
7938
8520
  const wallStart = performance.now();
7939
8521
  const worker = async () => {
7940
8522
  for (let i = 0; i < input.iterations; i++) {
8523
+ if (run.signal.aborted) break;
7941
8524
  const t0 = performance.now();
7942
8525
  const result = await api.pipelineExecutor.inferCached.mutate({
7943
8526
  stepId: rootStep.addonId,
@@ -7957,6 +8540,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
7957
8540
  batchSizes.push(bs);
7958
8541
  detCounts.push(dets);
7959
8542
  const n = ++_n;
8543
+ run.report({
8544
+ runs: n,
8545
+ totalRuns
8546
+ });
7960
8547
  if (n <= 20) ctx.logger.info("bench call trace (cached)", { meta: {
7961
8548
  n,
7962
8549
  wallMs: Math.round(wallMs),
@@ -8009,7 +8596,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8009
8596
  await Promise.all(Array.from({ length: input.parallel }, () => worker()));
8010
8597
  const wallSec = (performance.now() - wallStart) / 1e3;
8011
8598
  await api.pipelineExecutor.uncacheFrame.mutate({ frameId: poolFrameId }, pin).catch(() => {});
8012
- return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "cached");
8599
+ return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "cached", { nodeId: benchNode });
8013
8600
  }
8014
8601
  ctx.logger.info("synthetic bench: using full runPipeline path", { meta: {
8015
8602
  steps: enabledSteps.length,
@@ -8023,7 +8610,6 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8023
8610
  const result = await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
8024
8611
  steps: stepsToRun,
8025
8612
  frame: sharedFrame,
8026
- ...input.engine ? { engine: input.engine } : {},
8027
8613
  ...input.deviceKey ? { deviceKey: input.deviceKey } : {},
8028
8614
  ...input.plane ? { plane: input.plane } : {}
8029
8615
  }) }, pin);
@@ -8051,11 +8637,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8051
8637
  const predictTimings = [];
8052
8638
  const batchSizes = [];
8053
8639
  const detCounts = [];
8054
- const sessionId = input.sessionId ?? `synth-${Date.now().toString(36)}`;
8640
+ const sessionId = input.sessionId;
8055
8641
  const totalRuns = input.parallel * input.iterations;
8056
8642
  const wallStart = performance.now();
8057
8643
  const worker = async () => {
8058
8644
  for (let i = 0; i < input.iterations; i++) {
8645
+ if (run.signal.aborted) break;
8059
8646
  const { wallMs, result } = await callOnce();
8060
8647
  wallTimings.push(wallMs);
8061
8648
  serverWallTimings.push(result.debug?.wallMs ?? 0);
@@ -8065,6 +8652,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8065
8652
  batchSizes.push(result.debug?.batchSize ?? 1);
8066
8653
  detCounts.push(result.detections?.length ?? 0);
8067
8654
  const n = wallTimings.length;
8655
+ run.report({
8656
+ runs: n,
8657
+ totalRuns
8658
+ });
8068
8659
  if (n % Math.max(1, input.parallel) === 0 && ctx.eventBus) {
8069
8660
  const elapsed = (performance.now() - wallStart) / 1e3;
8070
8661
  const fps = elapsed > 0 ? n / elapsed : 0;
@@ -8105,9 +8696,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8105
8696
  };
8106
8697
  await Promise.all(Array.from({ length: input.parallel }, () => worker()));
8107
8698
  const wallSec = (performance.now() - wallStart) / 1e3;
8108
- return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "pipeline");
8699
+ return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "pipeline", {
8700
+ nodeId: benchNode,
8701
+ ...input.deviceKey ? { deviceKey: input.deviceKey } : {}
8702
+ });
8109
8703
  }
8110
- async buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, path) {
8704
+ async buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, path, target) {
8111
8705
  const meanOfArr = (xs) => xs.length > 0 ? xs.reduce((s, v) => s + v, 0) / xs.length : 0;
8112
8706
  this.ctx.logger.info("synthetic bench summary", { meta: {
8113
8707
  runs: wallTimings.length,
@@ -8143,24 +8737,47 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8143
8737
  batchSizeMean: Math.round(meanOf(batchSizes) * 100) / 100,
8144
8738
  batchSizeMax: batchSizes.length > 0 ? Math.max(...batchSizes) : 0,
8145
8739
  path,
8146
- ...await this.getEngineAndTuning()
8740
+ pool: await this.readBenchPool(target)
8147
8741
  };
8148
8742
  }
8149
- async getEngineAndTuning() {
8743
+ /**
8744
+ * The pool a bench measured, and its provisioning — asked of the node it
8745
+ * was pinned to, for the device pool it named (D646). It used to ask the
8746
+ * node DEFAULT of whichever node answered, so an NPU bench was recorded
8747
+ * beside the onnx pool's tuning. A failed read is `null` fields and a line,
8748
+ * never a default (D393).
8749
+ */
8750
+ async readBenchPool(target) {
8751
+ const unknown = {
8752
+ nodeId: target.nodeId,
8753
+ deviceKey: target.deviceKey ?? null,
8754
+ runtime: null,
8755
+ numWorkers: null,
8756
+ concurrency: null,
8757
+ batchMode: null,
8758
+ maxBatchSize: null
8759
+ };
8760
+ const api = this.ctx.api;
8761
+ if (!api) return unknown;
8762
+ const query = buildBenchTuningQuery(target);
8150
8763
  try {
8151
- const api = this.ctx.api;
8152
- if (!api) return {};
8153
- const [eng, tuning] = await Promise.all([api.pipelineExecutor.getSelectedEngine.query(), api.pipelineExecutor.getEffectiveTuning.query()]);
8764
+ const tuning = await api.pipelineExecutor.getEffectiveTuning.query({ ...query.input }, nodePin(query.nodeId));
8154
8765
  return {
8155
- engine: eng ? {
8156
- runtime: eng.runtime,
8157
- backend: eng.backend,
8158
- device: eng.device
8159
- } : void 0,
8160
- tuning: tuning ?? void 0
8766
+ nodeId: target.nodeId,
8767
+ deviceKey: tuning.deviceKey,
8768
+ runtime: tuning.runtime,
8769
+ numWorkers: tuning.numWorkers,
8770
+ concurrency: tuning.concurrency,
8771
+ batchMode: tuning.batchMode,
8772
+ maxBatchSize: tuning.maxBatchSize
8161
8773
  };
8162
- } catch {
8163
- return {};
8774
+ } catch (err) {
8775
+ this.ctx.logger.warn("synthetic bench: pool tuning read failed — reported as unknown", { meta: {
8776
+ nodeId: target.nodeId,
8777
+ deviceKey: target.deviceKey ?? null,
8778
+ error: errMsg(err)
8779
+ } });
8780
+ return unknown;
8164
8781
  }
8165
8782
  }
8166
8783
  /**
@@ -8283,7 +8900,6 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8283
8900
  settleSec,
8284
8901
  measureSec,
8285
8902
  deviceKey: input.deviceKey ?? null,
8286
- engine: input.engine ?? null,
8287
8903
  contaminatedByActiveCameras,
8288
8904
  steps: stepsToRun.map((s) => `${s.addonId}/${s.modelId}${s.enabled ? "" : "[off]"}`)
8289
8905
  } });
@@ -8300,8 +8916,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
8300
8916
  },
8301
8917
  plane: "frame",
8302
8918
  sessionId,
8303
- ...input.deviceKey ? { deviceKey: input.deviceKey } : {},
8304
- ...input.engine ? { engine: input.engine } : {}
8919
+ ...input.deviceKey ? { deviceKey: input.deviceKey } : {}
8305
8920
  }, pin);
8306
8921
  if (result.debug?.engine !== void 0) engineLabel = result.debug.engine;
8307
8922
  if (result.shed === true) return {
@@ -9103,7 +9718,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
9103
9718
  resolveChildSteps: (deviceId, parentClassName, filter) => this.resolveDetailChildSteps(deviceId, parentClassName, filter),
9104
9719
  getNativeCrop: (deviceId, handle, paddedBbox) => this.getNativeCropForDetail(deviceId, handle, paddedBbox),
9105
9720
  runPipeline: (steps, imageJpeg, deviceId, deviceKey, nativeCropRef) => this.runDetailPipeline(steps, imageJpeg, deviceId, deviceKey, nativeCropRef),
9106
- resolveStepDevice: (deviceId, step) => this.resolveDetailStepDevice(deviceId, step),
9721
+ placeStep: (deviceId, step) => this.placeDetailStepForCamera(deviceId, step),
9107
9722
  resolveCropConvention: () => convention,
9108
9723
  logger: this.detailSubtreeLogger()
9109
9724
  }, input);
@@ -9317,39 +9932,49 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
9317
9932
  return { warn: (message, extras) => this.ctxIfReady?.logger.warn(message, extras) };
9318
9933
  }
9319
9934
  /**
9320
- * Step-tree device jump (phase 1) — pick which same-node inference device an
9321
- * enrichment `step` runs on. AUTO-jumps only when the camera's elected device
9322
- * cannot run the step's model (its format has no build); an operator
9323
- * `step.jumpDeviceKey` overrides. Returns the resolved `<backend>:<device>`
9324
- * key, or `undefined` (⇒ the runner's per-camera default pool) when there is
9325
- * nothing to route: no elected device, or no jump roster on the attach.
9935
+ * Step-tree device jump — place one enrichment subtree on a same-node
9936
+ * inference device ({@link placeDetailStep}). AUTO-jumps only when the
9937
+ * camera's elected device cannot run the subtree (its format has no build);
9938
+ * an operator `step.jumpDeviceKey` overrides. A jumped subtree runs with the
9939
+ * TARGET device's provisioning carried on the roster (D650), never the
9940
+ * elected device's. `deviceKey: undefined` ⇒ the runner's per-camera default
9941
+ * pool (no elected device).
9326
9942
  *
9327
- * When NO enabled device on the node can run the step, logs a LOUD warn once
9328
- * per (camera, step) — killing the historical SILENT per-child degrade (an
9329
- * `edgetpu` root would otherwise drop faces/plates/clip with no signal).
9943
+ * Logs the jump ONCE per (camera, step, target), and — when NO enabled device
9944
+ * on the node can run the subtree — a LOUD warn once per (camera, step).
9330
9945
  */
9331
- resolveDetailStepDevice(deviceId, step) {
9946
+ placeDetailStepForCamera(deviceId, step) {
9332
9947
  const config = this.attached.get(deviceId)?.config;
9333
- const effectiveDeviceKey = config?.deviceKey;
9334
- if (!effectiveDeviceKey) return void 0;
9335
- const roster = config?.inferenceDevices;
9336
- if (!roster || roster.length < 2) return effectiveDeviceKey;
9337
- const attachedByKey = this.perDeviceAttachedCounts();
9338
- const devices = roster.map((d) => ({
9339
- deviceKey: d.deviceKey,
9340
- weight: d.weight,
9341
- maxSessions: d.maxSessions,
9342
- attachedCameras: attachedByKey.get(d.deviceKey) ?? 0
9343
- }));
9344
- const canRun = buildSubtreeCanRun(step, tryGetStepDefinition);
9345
- const result = resolveStepDevice({
9346
- effectiveDeviceKey,
9347
- devices,
9348
- ...step.jumpDeviceKey !== void 0 ? { manualJumpDeviceKey: step.jumpDeviceKey } : {},
9349
- canRun
9948
+ const placement = placeDetailStep({
9949
+ step,
9950
+ effectiveDeviceKey: config?.deviceKey,
9951
+ roster: config?.inferenceDevices,
9952
+ attachedByKey: this.perDeviceAttachedCounts(),
9953
+ getStepDef: tryGetStepDefinition
9954
+ });
9955
+ const decision = placement.decision;
9956
+ if (decision?.kind === "no-candidate") this.warnStepHasNoRunnableDevice(deviceId, step.addonId, decision.deviceKey);
9957
+ else if (decision?.kind === "jumped" && config?.deviceKey) this.logStepJumped(deviceId, config.deviceKey, decision, placement);
9958
+ return placement;
9959
+ }
9960
+ /** `${deviceId}:${addonId}:${target}` triples whose jump has been logged. */
9961
+ loggedStepJumps = /* @__PURE__ */ new Set();
9962
+ logStepJumped(deviceId, electedDeviceKey, decision, placement) {
9963
+ const step = placement.step;
9964
+ const key = `${deviceId}:${step.addonId}:${decision.deviceKey}`;
9965
+ if (this.loggedStepJumps.has(key)) return;
9966
+ this.loggedStepJumps.add(key);
9967
+ this.ctx?.logger.info("enrichment step runs on another device of this node", {
9968
+ tags: { deviceId },
9969
+ meta: {
9970
+ step: step.addonId,
9971
+ electedDeviceKey,
9972
+ targetDeviceKey: decision.deviceKey,
9973
+ reason: decision.reason,
9974
+ provisioned: placement.provisioned,
9975
+ modelId: step.modelId ?? null
9976
+ }
9350
9977
  });
9351
- if (result.kind === "no-candidate") this.warnStepHasNoRunnableDevice(deviceId, step.addonId, effectiveDeviceKey);
9352
- return result.deviceKey;
9353
9978
  }
9354
9979
  /** Live attached-camera count per inference `deviceKey` on this runner. */
9355
9980
  perDeviceAttachedCounts() {
@@ -9908,6 +10533,16 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
9908
10533
  if (!admission) return void 0;
9909
10534
  return () => admission.admit(deviceId, label);
9910
10535
  }
10536
+ /**
10537
+ * The start's place in the node's queue of refused starts (D652) — taken by
10538
+ * the guarded start on its first refusal, kept until it is over. Spread into
10539
+ * `startGuardedSessionDecode`'s deps; empty when this node runs no ceiling.
10540
+ */
10541
+ liveWorkerSlotWaiter(deviceId) {
10542
+ const admission = this.liveWorkers;
10543
+ if (!admission) return {};
10544
+ return { slotWaiter: () => admission.slotWaiter(deviceId) };
10545
+ }
9911
10546
  buildDecodeSpawnAcquire(log, label) {
9912
10547
  return async () => {
9913
10548
  const gate = this.spawnGate;
@@ -9936,8 +10571,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
9936
10571
  const teardown = startGuardedSessionDecode({
9937
10572
  logger: log,
9938
10573
  admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "detect"),
10574
+ ...this.liveWorkerSlotWaiter(config.deviceId),
9939
10575
  acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:detect:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config), true),
9940
- startPump: (source) => {
10576
+ startPump: (source, hooks) => {
9941
10577
  const dialMs = Date.now() - requestedAt;
9942
10578
  const useLocalFrameRef = this.executorCoLocated() && process.env["CAMSTACK_DISABLE_LOCAL_FRAME_REF"] !== "1";
9943
10579
  log.info("session-decode: detection routed through per-session decode worker (rgb)", { meta: {
@@ -9949,6 +10585,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
9949
10585
  return startSessionDecodePump({
9950
10586
  fork: this.forkDecodeWorkerWithLease,
9951
10587
  lease: this.resolveSessionLeaseKnobs,
10588
+ onWorkerGone: hooks.onWorkerGone,
9952
10589
  logger: log,
9953
10590
  source,
9954
10591
  opts: {
@@ -10030,10 +10667,30 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10030
10667
  strideMs: report.strideMs,
10031
10668
  subsampled: report.subsampled,
10032
10669
  laneRefused: runner.preRollRefusedCount(config.deviceId),
10033
- budgetFrames: this.preRollBudgetFor(config)
10670
+ budgetFrames: this.preRollBudgetFor(config),
10671
+ effectiveBudgetFrames: report.budgetFrames
10034
10672
  }
10035
10673
  });
10036
10674
  },
10675
+ onPreRollSettled: (summary) => {
10676
+ const lost = summary.lostByReason.evicted + summary.lostByReason.cleared + summary.lostByReason["download-failed"] + summary.lostByReason.miss;
10677
+ const meta = {
10678
+ streamId: config.detectionStreamId,
10679
+ ...summary,
10680
+ lost
10681
+ };
10682
+ if (lost > 0 || summary.unsettled > 0) {
10683
+ log.info("session-decode: pre-roll frames settled", {
10684
+ tags: { deviceId: config.deviceId },
10685
+ meta
10686
+ });
10687
+ return;
10688
+ }
10689
+ log.debug("session-decode: pre-roll frames settled", {
10690
+ tags: { deviceId: config.deviceId },
10691
+ meta
10692
+ });
10693
+ },
10037
10694
  onFirstFrame: (timing) => {
10038
10695
  const phaseActiveAt = this.lastMotionAt.get(config.deviceId);
10039
10696
  log.info("session-decode: detection first frame", {
@@ -10077,8 +10734,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10077
10734
  });
10078
10735
  }
10079
10736
  });
10080
- return () => {
10081
- teardown();
10737
+ return (mode) => {
10738
+ teardown(mode);
10082
10739
  };
10083
10740
  }
10084
10741
  /**
@@ -10099,8 +10756,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10099
10756
  return startGuardedSessionDecode({
10100
10757
  logger: log,
10101
10758
  admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "motion"),
10759
+ ...this.liveWorkerSlotWaiter(config.deviceId),
10102
10760
  acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.motionStreamId, `session-decode:motion:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
10103
- startPump: (source) => {
10761
+ startPump: (source, hooks) => {
10104
10762
  log.info("session-decode: motion routed through per-session decode worker (gray)", { meta: {
10105
10763
  streamId: config.motionStreamId,
10106
10764
  codec: source.codec
@@ -10108,6 +10766,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10108
10766
  return startSessionDecodePump({
10109
10767
  fork: this.forkDecodeWorkerWithLease,
10110
10768
  lease: this.resolveSessionLeaseKnobs,
10769
+ onWorkerGone: hooks.onWorkerGone,
10111
10770
  logger: log,
10112
10771
  source,
10113
10772
  opts: {
@@ -10775,15 +11434,36 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10775
11434
  * found/report semantics can't drift between the two admission
10776
11435
  * mechanisms.
10777
11436
  */
10778
- finishOccupancyBurst(deviceId, log, collected) {
11437
+ finishOccupancyBurst(deviceId, log, summary) {
10779
11438
  this.occupancyBurstInFlight.delete(deviceId);
10780
11439
  const found = this.occupancyBurstFound.has(deviceId);
10781
11440
  this.occupancyBurstFound.delete(deviceId);
11441
+ if (summary.inferred === 0) {
11442
+ log.info("occupancy re-check: burst complete — nothing inferred, no verdict", {
11443
+ tags: { deviceId },
11444
+ meta: {
11445
+ ...summary,
11446
+ verdict: null
11447
+ }
11448
+ });
11449
+ return;
11450
+ }
10782
11451
  this.runner?.reportOccupancyBurstOutcome(deviceId, found);
10783
- log.debug("occupancy re-check: burst complete", { meta: {
10784
- collected,
11452
+ const meta = {
11453
+ ...summary,
10785
11454
  found
10786
- } });
11455
+ };
11456
+ if (summary.lost > 0) {
11457
+ log.info("occupancy re-check: burst complete", {
11458
+ tags: { deviceId },
11459
+ meta
11460
+ });
11461
+ return;
11462
+ }
11463
+ log.debug("occupancy re-check: burst complete", {
11464
+ tags: { deviceId },
11465
+ meta
11466
+ });
10787
11467
  }
10788
11468
  /**
10789
11469
  * Task 10: arm a TAP-based occupancy burst for a device whose analyzer
@@ -10819,21 +11499,30 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10819
11499
  log.debug("occupancy re-check: tap safety timeout reached", { meta: { collected } });
10820
11500
  this.finishOccupancyTap(deviceId);
10821
11501
  }, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
11502
+ const ledger = new OccupancyBurstLedger({
11503
+ settleBoundMs: OCCUPANCY_BURST_SETTLE_BOUND_MS,
11504
+ onComplete: resolve
11505
+ });
10822
11506
  this.occupancyTaps.set(deviceId, {
10823
11507
  remaining: frames,
10824
11508
  collected: 0,
10825
11509
  safetyTimer,
11510
+ ledger,
10826
11511
  resolve
10827
11512
  });
10828
11513
  });
10829
11514
  }
10830
- /** Idempotent: a no-op once the tap has already settled (timer fired, or the budget already drained). */
11515
+ /**
11516
+ * Close the tap's COLLECTION. Idempotent: a no-op once the tap has already
11517
+ * closed (timer fired, or the budget already drained). The tap's promise
11518
+ * resolves later, from its ledger, once the frames it admitted have settled.
11519
+ */
10831
11520
  finishOccupancyTap(deviceId) {
10832
11521
  const tap = this.occupancyTaps.get(deviceId);
10833
11522
  if (!tap) return;
10834
11523
  clearTimeout(tap.safetyTimer);
10835
11524
  this.occupancyTaps.delete(deviceId);
10836
- tap.resolve(tap.collected);
11525
+ tap.ledger.close();
10837
11526
  }
10838
11527
  /**
10839
11528
  * Tee an arriving analyzer motion frame into occupancy-burst detection
@@ -10848,7 +11537,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10848
11537
  teeOccupancyTapFrame(deviceId, frame, handle) {
10849
11538
  const tap = this.occupancyTaps.get(deviceId);
10850
11539
  if (!tap) return;
10851
- this.runner?.enqueueOccupancyFrame(deviceId, frame, handle);
11540
+ const settle = tap.ledger.admit();
11541
+ if (!this.runner?.enqueueOccupancyFrame(deviceId, frame, handle, settle)) settle("lost");
10852
11542
  tap.collected++;
10853
11543
  tap.remaining--;
10854
11544
  if (tap.remaining <= 0) this.finishOccupancyTap(deviceId);
@@ -10868,9 +11558,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10868
11558
  const lastFrameAt = this.lastMotionFrameAt.get(deviceId);
10869
11559
  const motionFresh = lastFrameAt !== void 0 && Date.now() - lastFrameAt < MOTION_FRESHNESS_WINDOW_MS;
10870
11560
  if (attachment.motionUnsubscribe !== null && motionFresh) {
10871
- const collected = await this.startOccupancyTap(deviceId, frames, log);
10872
- if (collected > 0) {
10873
- this.finishOccupancyBurst(deviceId, log, collected);
11561
+ const summary = await this.startOccupancyTap(deviceId, frames, log);
11562
+ if (summary.frames > 0) {
11563
+ this.finishOccupancyBurst(deviceId, log, summary);
10874
11564
  return;
10875
11565
  }
10876
11566
  log.debug("occupancy re-check: tap collected 0 frames, falling back to dial", { meta: { frames } });
@@ -10901,76 +11591,91 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
10901
11591
  fallback: alreadyInFlight
10902
11592
  } });
10903
11593
  let count = 0;
11594
+ /** Collection is over: no further frame is admitted to this burst. */
11595
+ let admissionClosed = false;
11596
+ /** The burst has reported its verdict (or its lack of one). */
10904
11597
  let settled = false;
10905
11598
  let teardown = null;
10906
- const cleanup = () => {
11599
+ const cleanup = (summary) => {
10907
11600
  if (settled) return;
10908
11601
  settled = true;
10909
- teardown?.();
10910
- this.finishOccupancyBurst(deviceId, log, count);
11602
+ this.finishOccupancyBurst(deviceId, log, summary);
11603
+ };
11604
+ const ledger = new OccupancyBurstLedger({
11605
+ settleBoundMs: OCCUPANCY_BURST_SETTLE_BOUND_MS,
11606
+ onComplete: cleanup
11607
+ });
11608
+ const closeAdmission = () => {
11609
+ if (admissionClosed) return;
11610
+ admissionClosed = true;
11611
+ clearTimeout(safetyTimer);
11612
+ teardown?.("drain");
11613
+ ledger.close();
11614
+ };
11615
+ /** Offer one collected frame to the burst; `true` = the runner admitted it. */
11616
+ const admit = (frame, handle) => {
11617
+ const settle = ledger.admit();
11618
+ const admitted = runner.enqueueOccupancyFrame(deviceId, frame, handle, settle);
11619
+ if (!admitted) settle("lost");
11620
+ count++;
11621
+ if (count >= frames) closeAdmission();
11622
+ return admitted;
10911
11623
  };
10912
11624
  const safetyTimer = setTimeout(() => {
10913
11625
  log.debug("occupancy re-check: safety timeout reached", { meta: { collected: count } });
10914
- cleanup();
11626
+ closeAdmission();
10915
11627
  }, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
10916
11628
  teardown = startGuardedSessionDecode({
10917
11629
  logger: log,
10918
11630
  admitWorker: this.buildLiveWorkerAdmit(deviceId, "occupancy"),
11631
+ ...this.liveWorkerSlotWaiter(deviceId),
10919
11632
  acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:occupancy:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
10920
- startPump: (source) => {
11633
+ startPump: (source, hooks) => {
10921
11634
  const useLocalFrameRef = this.executorCoLocated() && process.env["CAMSTACK_DISABLE_LOCAL_FRAME_REF"] !== "1";
10922
11635
  return startSessionDecodePump({
10923
11636
  fork: this.forkDecodeWorkerWithLease,
10924
11637
  lease: this.resolveSessionLeaseKnobs,
11638
+ onWorkerGone: hooks.onWorkerGone,
10925
11639
  logger: log,
10926
11640
  source,
10927
11641
  opts: {
10928
11642
  fps: config.detectionFps,
10929
11643
  format: "rgb"
10930
11644
  },
11645
+ shouldMaterializeFrame: () => !admissionClosed,
10931
11646
  ...useLocalFrameRef ? {
10932
11647
  frameRegistry: localFrameRegistry,
10933
11648
  inlineWhen: () => runner.detectionBackpressure(config.deviceId),
10934
11649
  inlineViewSpec: SPLIT_DETECTION_VIEW,
10935
11650
  onFrameRef: (frame, nativeCrop) => {
10936
- if (settled) return false;
11651
+ if (admissionClosed) return false;
10937
11652
  const handle = writeRetainedDetectionFrame(retainedFrames, config.deviceId, frame, this.retention);
10938
11653
  if (handle && nativeCrop) {
10939
11654
  this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
10940
11655
  this.bindFrameCutter(handle, nativeCrop);
10941
11656
  }
10942
- const admitted = runner.enqueueOccupancyFrame(deviceId, frame, handle);
11657
+ const admitted = admit(frame, handle);
10943
11658
  if (admitted) nativeCrop?.markInferred();
10944
- count++;
10945
- if (count >= frames) {
10946
- clearTimeout(safetyTimer);
10947
- cleanup();
10948
- }
10949
11659
  return admitted;
10950
11660
  }
10951
11661
  } : { inlineViewSpec: SPLIT_DETECTION_VIEW },
10952
11662
  acquireSpawnSlot: this.buildDecodeSpawnAcquire(log, "occupancy"),
10953
- onEnded: () => cleanup(),
11663
+ onEnded: () => closeAdmission(),
10954
11664
  onDecodedFrame: (frame, nativeCrop) => {
10955
- if (settled) return;
11665
+ if (admissionClosed) return;
10956
11666
  const handle = writeRetainedDetectionFrame(retainedFrames, config.deviceId, frame, this.retention);
10957
11667
  if (handle && nativeCrop) {
10958
11668
  this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
10959
11669
  this.bindFrameCutter(handle, nativeCrop);
10960
11670
  }
10961
- if (runner.enqueueOccupancyFrame(deviceId, frame, handle)) nativeCrop?.markInferred();
10962
- count++;
10963
- if (count >= frames) {
10964
- clearTimeout(safetyTimer);
10965
- cleanup();
10966
- }
11671
+ if (admit(frame, handle)) nativeCrop?.markInferred();
10967
11672
  }
10968
11673
  });
10969
11674
  }
10970
11675
  });
10971
- if (settled) {
11676
+ if (admissionClosed) {
10972
11677
  clearTimeout(safetyTimer);
10973
- teardown();
11678
+ teardown("drain");
10974
11679
  }
10975
11680
  }
10976
11681
  emitInferenceResult(deviceId, frame, result, handle, source, occasion) {