@camstack/addon-pipeline 1.2.294 → 1.2.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_MODELS.md +241 -0
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/{default-detection-model-0dPKRKUD.mjs → default-detection-model-Co578D8C.mjs} +181 -99
- package/dist/{default-detection-model-D24AJOTn.js → default-detection-model-D1daTtqT.js} +181 -99
- package/dist/detection-pipeline/index.js +1301 -531
- package/dist/detection-pipeline/index.mjs +1301 -531
- package/dist/{dist-CJR259Xf.js → dist-8up-f2TX.js} +3688 -2698
- package/dist/{dist-RXbmRAwP.mjs → dist-CCd0Q3nr.mjs} +3676 -2698
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-atmRSHPk.mjs → node-DgMSXSWP.mjs} +1 -1
- package/dist/{node-DWg9zbY1.js → node-lpQgHes9.js} +1 -1
- package/dist/pipeline-runner/index.js +975 -270
- package/dist/pipeline-runner/index.mjs +975 -270
- package/dist/{process-memory-BJUXvTjd.js → process-memory-CX_92V_r.js} +1 -1
- package/dist/{process-memory-BgFOHFnx.mjs → process-memory-DFC_O5zE.mjs} +1 -1
- package/dist/recorder/index.js +14 -6
- package/dist/recorder/index.mjs +14 -6
- package/dist/{segment-demux-js-C_fPJub3.js → segment-demux-js-DzBx6NN2.js} +1 -1
- package/dist/{segment-demux-js-G7wFpHzn.mjs → segment-demux-js-FZbBuk3F.mjs} +1 -1
- package/dist/session-decode/{decode-worker-child.js → decode-worker-main.js} +481 -72
- package/dist/session-decode/{decode-worker-child.mjs → decode-worker-main.mjs} +482 -71
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-6IyM-BIn.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C_i7oFBl.mjs} +2 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DUGQKsKL.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CkbplMHA.mjs +26 -0
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-BBYHWS3M.mjs → hostInit-BPtppL3W.mjs} +2 -2
- package/dist/stream-broker/index.js +4 -4
- package/dist/stream-broker/index.mjs +4 -4
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-B2MfQLlu.js → worker-protocol-C-G8qmye.js} +3 -1
- package/dist/{worker-protocol-C_W-P_g-.mjs → worker-protocol-D_NzPcnh.mjs} +3 -1
- package/package.json +3 -2
- package/python/inference_pool.py +422 -64
- package/python/postprocessors/__init__.py +2 -0
- package/python/postprocessors/ssd.py +73 -17
- package/python/postprocessors/test_ssd.py +205 -0
- package/python/postprocessors/test_yunet.py +292 -0
- package/python/postprocessors/testdata/ssdlite_mobiledet_outputs.json +1 -0
- package/python/postprocessors/yunet.py +275 -0
- package/python/test_inference_pool_compile_off_loop.py +414 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-6IHzlLJ_.mjs +0 -26
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DoyA71_q.mjs +0 -26
|
@@ -3,10 +3,10 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../chunk-emK7D4bc.js");
|
|
6
|
-
const require_dist = require("../dist-
|
|
7
|
-
const require_node = require("../node-
|
|
8
|
-
const require_default_detection_model = require("../default-detection-model-
|
|
9
|
-
const require_worker_protocol = require("../worker-protocol-
|
|
6
|
+
const require_dist = require("../dist-8up-f2TX.js");
|
|
7
|
+
const require_node = require("../node-lpQgHes9.js");
|
|
8
|
+
const require_default_detection_model = require("../default-detection-model-D1daTtqT.js");
|
|
9
|
+
const require_worker_protocol = require("../worker-protocol-C-G8qmye.js");
|
|
10
10
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
11
11
|
const require_lazy_sharp = require("../lazy-sharp-BuwKlQLR.js");
|
|
12
12
|
const require_restream_intent = require("../restream-intent-7uDNqMKs.js");
|
|
@@ -21,7 +21,7 @@ let node_perf_hooks = require("node:perf_hooks");
|
|
|
21
21
|
let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
|
|
22
22
|
//#region src/session-decode/fork-decode-worker.ts
|
|
23
23
|
/** Absolute path to the built decode-worker child entry (a vite build entry). */
|
|
24
|
-
var CHILD_ENTRY_PATH = (0, node_url.fileURLToPath)(new URL("../session-decode/decode-worker-
|
|
24
|
+
var CHILD_ENTRY_PATH = (0, node_url.fileURLToPath)(new URL("../session-decode/decode-worker-main.js", require("url").pathToFileURL(__filename).href));
|
|
25
25
|
/**
|
|
26
26
|
* Known kill signals the coordinator may pass, mapped to their `NodeJS.Signals`
|
|
27
27
|
* literal so `ChildProcess.kill` receives a correctly-typed signal without a
|
|
@@ -42,21 +42,36 @@ var KILL_SIGNALS = {
|
|
|
42
42
|
* every listener added out here is one somebody can forget to remove.
|
|
43
43
|
*/
|
|
44
44
|
function wrapChild(child, onExit) {
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
let spawned = false;
|
|
46
|
+
let exitDelivered = false;
|
|
47
|
+
const exitListeners = [];
|
|
48
|
+
const deliverExit = (info) => {
|
|
49
|
+
if (exitDelivered) return;
|
|
50
|
+
exitDelivered = true;
|
|
51
|
+
onExit();
|
|
52
|
+
for (const listener of exitListeners) listener(info);
|
|
53
|
+
};
|
|
54
|
+
child.on("spawn", () => {
|
|
55
|
+
spawned = true;
|
|
56
|
+
});
|
|
57
|
+
child.on("exit", (code, signal) => deliverExit({
|
|
58
|
+
code: code ?? null,
|
|
59
|
+
signal: signal ?? null
|
|
60
|
+
}));
|
|
61
|
+
child.on("error", () => {
|
|
62
|
+
if (!spawned) deliverExit({
|
|
63
|
+
code: null,
|
|
64
|
+
signal: null
|
|
65
|
+
});
|
|
66
|
+
});
|
|
47
67
|
return {
|
|
48
68
|
send: (msg) => {
|
|
49
69
|
if (!child.connected) return;
|
|
50
70
|
child.send(msg, () => void 0);
|
|
51
71
|
},
|
|
52
72
|
on: (ev, cb) => {
|
|
53
|
-
if (ev === "exit")
|
|
54
|
-
|
|
55
|
-
code,
|
|
56
|
-
signal
|
|
57
|
-
});
|
|
58
|
-
child.on("exit", (code, signal) => cb(toInfo(code ?? null, signal ?? null)));
|
|
59
|
-
} else child.on("message", (m) => cb(m));
|
|
73
|
+
if (ev === "exit") exitListeners.push(cb);
|
|
74
|
+
else child.on("message", (m) => cb(m));
|
|
60
75
|
},
|
|
61
76
|
kill: (signal) => {
|
|
62
77
|
if (signal === void 0) {
|
|
@@ -257,58 +272,77 @@ var DEFAULT_ACQUIRE_RETRY_DELAYS_MS = [
|
|
|
257
272
|
15e3,
|
|
258
273
|
3e4
|
|
259
274
|
];
|
|
260
|
-
/**
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
* leaks the acquired restream refcount and, once the acquire resolves, forks
|
|
270
|
-
* an orphan decode worker nobody will ever stop.
|
|
271
|
-
*
|
|
272
|
-
* This helper returns the teardown SYNCHRONOUSLY. The acquire + pump-start
|
|
273
|
-
* run in the background; an `aborted` flag recorded by the (possibly
|
|
274
|
-
* already-called) teardown is checked right after each acquire resolves —
|
|
275
|
-
* if set, any restream is released immediately and the pump is NEVER
|
|
276
|
-
* started (no orphan worker). Otherwise the pump starts and the teardown,
|
|
277
|
-
* once called, stops the pump then releases the restream.
|
|
278
|
-
*
|
|
279
|
-
* **A failed acquire is retried, not swallowed.** The teardown handed back
|
|
280
|
-
* here is stored by the caller (as `attachment.detectionUnsubscribe`) the
|
|
281
|
-
* instant this returns, so a `return` on failure left that handle permanently
|
|
282
|
-
* non-null with no pump behind it: `onEnded` could never fire because nothing
|
|
283
|
-
* had ever started, the camera's phase stayed `active`, and not one frame
|
|
284
|
-
* arrived again. The motion path was hardened against this shape by a frame
|
|
285
|
-
* freshness window (`MOTION_FRESHNESS_WINDOW_MS`); this is the detection
|
|
286
|
-
* path's half.
|
|
287
|
-
*/
|
|
275
|
+
/** A restream release that runs once however many paths reach it. */
|
|
276
|
+
function onceRelease(release) {
|
|
277
|
+
let done = false;
|
|
278
|
+
return () => {
|
|
279
|
+
if (done) return Promise.resolve();
|
|
280
|
+
done = true;
|
|
281
|
+
return release();
|
|
282
|
+
};
|
|
283
|
+
}
|
|
288
284
|
function startGuardedSessionDecode(deps) {
|
|
289
285
|
let aborted = false;
|
|
286
|
+
/** The mode the teardown asked for — replayed if it raced `startPump`. */
|
|
287
|
+
let abortMode = "drain";
|
|
290
288
|
let stopPump = null;
|
|
291
289
|
let release = null;
|
|
292
290
|
let retryTimer = null;
|
|
293
291
|
/** The live-worker slot held by the RUNNING session, released on teardown. */
|
|
294
292
|
let workerLease = null;
|
|
295
293
|
const delays = deps.acquireRetryDelaysMs && deps.acquireRetryDelaysMs.length > 0 ? deps.acquireRetryDelaysMs : DEFAULT_ACQUIRE_RETRY_DELAYS_MS;
|
|
296
|
-
/**
|
|
297
|
-
|
|
294
|
+
/** Ends a parked start's wait when the session is torn down. */
|
|
295
|
+
let abortWait = null;
|
|
296
|
+
/**
|
|
297
|
+
* This start's place in the ceiling queue — taken on its first refusal, kept
|
|
298
|
+
* to the end so a re-park keeps the FIFO position (D652).
|
|
299
|
+
*/
|
|
300
|
+
let waiter = null;
|
|
301
|
+
/**
|
|
302
|
+
* The start was handed a returned slot and has not yet ASKED for it. Set on
|
|
303
|
+
* a wake, cleared by the next `admitWorker` call; a start torn down while
|
|
304
|
+
* this is set passes the slot on (`SlotWaiter.leave(true)`), or nobody gets
|
|
305
|
+
* it and every other parked start waits out its full timer.
|
|
306
|
+
*/
|
|
307
|
+
let wakeInHand = false;
|
|
308
|
+
/**
|
|
309
|
+
* Resolves after the backoff for `attempt` (`'timer'`) — or, when `parked`,
|
|
310
|
+
* as soon as the node hands this start a returned slot (`'woken'`) or the
|
|
311
|
+
* session is torn down (`'torn-down'`), whichever is first. The start leaves
|
|
312
|
+
* the queue on every exit, so a torn-down start leaves no listener behind.
|
|
313
|
+
*/
|
|
314
|
+
const waitBackoff = (attempt, parked) => new Promise((resolve) => {
|
|
298
315
|
const ms = delays[Math.min(attempt, delays.length - 1)] ?? delays[delays.length - 1] ?? 2e3;
|
|
299
|
-
|
|
300
|
-
retryTimer
|
|
301
|
-
|
|
302
|
-
|
|
316
|
+
const done = (why) => {
|
|
317
|
+
if (retryTimer !== null) {
|
|
318
|
+
clearTimeout(retryTimer);
|
|
319
|
+
retryTimer = null;
|
|
320
|
+
}
|
|
321
|
+
parked?.cancel();
|
|
322
|
+
abortWait = null;
|
|
323
|
+
resolve(why);
|
|
324
|
+
};
|
|
325
|
+
retryTimer = setTimeout(() => done("timer"), ms);
|
|
303
326
|
retryTimer.unref?.();
|
|
327
|
+
abortWait = () => done("torn-down");
|
|
328
|
+
if (parked) parked.wait(() => done("woken"));
|
|
304
329
|
});
|
|
305
|
-
|
|
330
|
+
const releaseWorkerSlot = () => {
|
|
331
|
+
workerLease?.release();
|
|
332
|
+
workerLease = null;
|
|
333
|
+
};
|
|
334
|
+
const startLoop = async () => {
|
|
306
335
|
for (let attempt = 0; !aborted; attempt++) {
|
|
307
336
|
let lease = null;
|
|
308
337
|
if (deps.admitWorker) {
|
|
338
|
+
wakeInHand = false;
|
|
309
339
|
lease = deps.admitWorker();
|
|
310
340
|
if (!lease) {
|
|
311
|
-
|
|
341
|
+
waiter ??= deps.slotWaiter?.() ?? null;
|
|
342
|
+
if (await waitBackoff(attempt, waiter) === "woken") {
|
|
343
|
+
attempt -= 1;
|
|
344
|
+
wakeInHand = true;
|
|
345
|
+
}
|
|
312
346
|
continue;
|
|
313
347
|
}
|
|
314
348
|
}
|
|
@@ -321,7 +355,7 @@ function startGuardedSessionDecode(deps) {
|
|
|
321
355
|
attempt: attempt + 1,
|
|
322
356
|
error: require_dist.errMsg(err)
|
|
323
357
|
} });
|
|
324
|
-
await waitBackoff(attempt);
|
|
358
|
+
await waitBackoff(attempt, null);
|
|
325
359
|
continue;
|
|
326
360
|
}
|
|
327
361
|
if (aborted) {
|
|
@@ -332,27 +366,47 @@ function startGuardedSessionDecode(deps) {
|
|
|
332
366
|
if (!acquired) {
|
|
333
367
|
lease?.release();
|
|
334
368
|
(attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: no broker profile bound yet — retrying", { meta: { attempt: attempt + 1 } });
|
|
335
|
-
await waitBackoff(attempt);
|
|
369
|
+
await waitBackoff(attempt, null);
|
|
336
370
|
continue;
|
|
337
371
|
}
|
|
338
372
|
if (attempt > 0) deps.logger.info("session-decode: acquire succeeded after retry", { meta: { attempts: attempt + 1 } });
|
|
339
|
-
release = acquired.release;
|
|
373
|
+
release = onceRelease(acquired.release);
|
|
340
374
|
workerLease = lease;
|
|
341
|
-
|
|
375
|
+
try {
|
|
376
|
+
stopPump = deps.startPump(acquired.source, { onWorkerGone: releaseWorkerSlot });
|
|
377
|
+
} catch (err) {
|
|
378
|
+
releaseWorkerSlot();
|
|
379
|
+
const restream = release;
|
|
380
|
+
release = null;
|
|
381
|
+
restream?.();
|
|
382
|
+
throw err;
|
|
383
|
+
}
|
|
384
|
+
if (aborted) stopPump(abortMode);
|
|
342
385
|
return;
|
|
343
386
|
}
|
|
344
|
-
}
|
|
387
|
+
};
|
|
388
|
+
startLoop().catch((err) => {
|
|
345
389
|
deps.logger.warn("session-decode: guarded start failed", { meta: { error: require_dist.errMsg(err) } });
|
|
390
|
+
}).finally(() => {
|
|
391
|
+
waiter?.leave(wakeInHand);
|
|
392
|
+
waiter = null;
|
|
346
393
|
});
|
|
347
|
-
return () => {
|
|
394
|
+
return (mode = "drain") => {
|
|
395
|
+
if (aborted) {
|
|
396
|
+
if (mode === "immediate" && abortMode !== "immediate") {
|
|
397
|
+
abortMode = mode;
|
|
398
|
+
stopPump?.(mode);
|
|
399
|
+
}
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
348
402
|
aborted = true;
|
|
403
|
+
abortMode = mode;
|
|
349
404
|
if (retryTimer !== null) {
|
|
350
405
|
clearTimeout(retryTimer);
|
|
351
406
|
retryTimer = null;
|
|
352
407
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
workerLease = null;
|
|
408
|
+
abortWait?.();
|
|
409
|
+
stopPump?.(mode);
|
|
356
410
|
if (release) release();
|
|
357
411
|
};
|
|
358
412
|
}
|
|
@@ -421,6 +475,21 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
421
475
|
limit;
|
|
422
476
|
logger;
|
|
423
477
|
live = 0;
|
|
478
|
+
/**
|
|
479
|
+
* Starts the ceiling refused, waiting to be told a slot came back — FIFO by
|
|
480
|
+
* the start's FIRST park (D652): the key's `seq` is fixed when the start
|
|
481
|
+
* takes its {@link SlotWaiter}, so a re-park keeps the place.
|
|
482
|
+
*/
|
|
483
|
+
parked = /* @__PURE__ */ new Map();
|
|
484
|
+
nextSeq = 0;
|
|
485
|
+
/**
|
|
486
|
+
* Cameras currently refused. The refusal WARN fires on the transition into
|
|
487
|
+
* this set, not on every retry: a start woken or retried while the node is
|
|
488
|
+
* still full says so at debug (D652 — a wake must not become a WARN storm).
|
|
489
|
+
* Cleared on admit AND when a parked start leaves (`SlotWaiter.leave`): a
|
|
490
|
+
* camera torn down while refused must be news again next time (D391).
|
|
491
|
+
*/
|
|
492
|
+
refusedDevices = /* @__PURE__ */ new Set();
|
|
424
493
|
nodeBudgetMb;
|
|
425
494
|
workerFootprintMb;
|
|
426
495
|
constructor(options) {
|
|
@@ -429,6 +498,48 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
429
498
|
this.nodeBudgetMb = positiveOrUndefined(options.nodeBudgetMb);
|
|
430
499
|
this.workerFootprintMb = positiveOrUndefined(options.workerFootprintMb);
|
|
431
500
|
}
|
|
501
|
+
/** Starts parked on a {@link SlotWaiter} right now — diagnostics and tests. */
|
|
502
|
+
get slotWaiters() {
|
|
503
|
+
return this.parked.size;
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* This start's place in the queue. A refused start waits on it instead of a
|
|
507
|
+
* fixed timer, so a slot freed by a draining worker's exit is taken at once
|
|
508
|
+
* (D652). Take it once per start, on the first refusal; keep it to the end.
|
|
509
|
+
*/
|
|
510
|
+
slotWaiter(deviceId) {
|
|
511
|
+
const key = {
|
|
512
|
+
seq: this.nextSeq,
|
|
513
|
+
deviceId
|
|
514
|
+
};
|
|
515
|
+
this.nextSeq += 1;
|
|
516
|
+
return {
|
|
517
|
+
wait: (listener) => {
|
|
518
|
+
this.parked.set(key, listener);
|
|
519
|
+
},
|
|
520
|
+
cancel: () => {
|
|
521
|
+
this.parked.delete(key);
|
|
522
|
+
},
|
|
523
|
+
leave: (unusedWake) => {
|
|
524
|
+
this.parked.delete(key);
|
|
525
|
+
this.refusedDevices.delete(deviceId);
|
|
526
|
+
if (unusedWake) this.wakeNext();
|
|
527
|
+
}
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* ONE slot, ONE wake: the longest-parked start (lowest `seq`). Waking all N
|
|
532
|
+
* produced N−1 refusals per freed slot. The woken start leaves the queue; if
|
|
533
|
+
* something else took the slot first it parks again, at the same place.
|
|
534
|
+
*/
|
|
535
|
+
wakeNext() {
|
|
536
|
+
let first;
|
|
537
|
+
for (const key of this.parked.keys()) if (first === void 0 || key.seq < first.seq) first = key;
|
|
538
|
+
if (first === void 0) return;
|
|
539
|
+
const wake = this.parked.get(first);
|
|
540
|
+
this.parked.delete(first);
|
|
541
|
+
wake?.();
|
|
542
|
+
}
|
|
432
543
|
/** Workers currently holding a slot. */
|
|
433
544
|
get inFlight() {
|
|
434
545
|
return this.live;
|
|
@@ -485,7 +596,9 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
485
596
|
admit(deviceId, label) {
|
|
486
597
|
const ceiling = this.ceiling;
|
|
487
598
|
if (this.live >= ceiling) {
|
|
488
|
-
this.
|
|
599
|
+
const firstRefusal = !this.refusedDevices.has(deviceId);
|
|
600
|
+
this.refusedDevices.add(deviceId);
|
|
601
|
+
(firstRefusal ? this.logger.warn : this.logger.debug).call(this.logger, "session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
|
|
489
602
|
tags: { deviceId },
|
|
490
603
|
meta: {
|
|
491
604
|
label,
|
|
@@ -501,6 +614,7 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
501
614
|
return null;
|
|
502
615
|
}
|
|
503
616
|
this.live += 1;
|
|
617
|
+
this.refusedDevices.delete(deviceId);
|
|
504
618
|
this.logger.debug("session-decode: live decode-worker slot taken", {
|
|
505
619
|
tags: { deviceId },
|
|
506
620
|
meta: {
|
|
@@ -514,6 +628,7 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
514
628
|
if (released) return;
|
|
515
629
|
released = true;
|
|
516
630
|
this.live -= 1;
|
|
631
|
+
this.wakeNext();
|
|
517
632
|
} };
|
|
518
633
|
}
|
|
519
634
|
};
|
|
@@ -1450,6 +1565,29 @@ function cropRgb(frame, bbox, maxWidth) {
|
|
|
1450
1565
|
/** How long to wait after `kill()` before escalating to `kill('SIGKILL')`. */
|
|
1451
1566
|
var SIGKILL_FALLBACK_MS = 1e3;
|
|
1452
1567
|
/**
|
|
1568
|
+
* The longest a planned teardown keeps a session's worker alive so the frames
|
|
1569
|
+
* it already handed out can be read (D652).
|
|
1570
|
+
*
|
|
1571
|
+
* What is being waited for is the REST of a frame's trip through the runner —
|
|
1572
|
+
* semaphore wait + inference + result — for frames that were admitted before
|
|
1573
|
+
* the detach. Measured on the live hub 2026-09-26 over one hour of
|
|
1574
|
+
* `pipeline stats` (2 254 per-camera 10 s windows): the WORST enqueue→emit in a
|
|
1575
|
+
* window was ≤1 000 ms in 96.7% of windows and ≤1 500 ms in 98.0%; going to
|
|
1576
|
+
* 5 000 ms buys only another 0.9%, and that tail is the inference-pool GPU hang,
|
|
1577
|
+
* which no wait here can rescue. The cost of waiting is one idle worker (its
|
|
1578
|
+
* decode already stopped by `drain`) for at most this long, per detach.
|
|
1579
|
+
*
|
|
1580
|
+
* A frame still held when this expires is not dropped silently: the
|
|
1581
|
+
* coordinator logs the count, tagged with the camera, and then kills.
|
|
1582
|
+
*/
|
|
1583
|
+
var SESSION_DECODE_DRAIN_BOUND_MS = 1500;
|
|
1584
|
+
/**
|
|
1585
|
+
* The miss reason for a native crop requested after its session's teardown.
|
|
1586
|
+
* Only crops already SENT when the drain began are waited for (D652); a later
|
|
1587
|
+
* one would ask a worker that is about to be killed.
|
|
1588
|
+
*/
|
|
1589
|
+
var NATIVE_CROP_REFUSED_DRAINING = "session draining — crop requested after teardown";
|
|
1590
|
+
/**
|
|
1453
1591
|
* How often the cold-start wait REPORTS that it is still waiting. Task 0
|
|
1454
1592
|
* live-measured cold-start (restream dial + hwaccel context + keyframe wait) at
|
|
1455
1593
|
* 5-8s, so this must tolerate a slow-but-healthy start.
|
|
@@ -1521,30 +1659,38 @@ var SessionDecodeSession = class {
|
|
|
1521
1659
|
firstFrameTimeoutMs;
|
|
1522
1660
|
frameTimeoutMs;
|
|
1523
1661
|
firstFrameMaxWaitMs;
|
|
1524
|
-
|
|
1525
|
-
onPreRollReport;
|
|
1662
|
+
hooks;
|
|
1526
1663
|
pendingPull = null;
|
|
1527
1664
|
pendingToBuffers = /* @__PURE__ */ new Map();
|
|
1528
1665
|
/** Outstanding `nativeCrop` requests, keyed by the monotonic `requestId`. */
|
|
1529
1666
|
pendingNativeCrops = /* @__PURE__ */ new Map();
|
|
1530
1667
|
nextNativeCropId = 1;
|
|
1531
1668
|
exited = false;
|
|
1532
|
-
/** Set by teardown()
|
|
1669
|
+
/** Set by teardown() — the session is ending, whether or not it drains first. */
|
|
1533
1670
|
stopRequested = false;
|
|
1671
|
+
/** Set when `kill()` is sent, so handleExit can tell an intentional stop from a death. */
|
|
1672
|
+
killSent = false;
|
|
1673
|
+
/**
|
|
1674
|
+
* Images yielded to the consumer and not yet closed. Each is a frame the
|
|
1675
|
+
* consumer still holds a reference to — queued, in flight, or being read —
|
|
1676
|
+
* and each dies with the worker. This is what a drain waits on.
|
|
1677
|
+
*/
|
|
1678
|
+
openImages = 0;
|
|
1679
|
+
/** Armed while a drain waits; its expiry kills with frames still held. */
|
|
1680
|
+
drainTimer = null;
|
|
1534
1681
|
sigkillTimer = null;
|
|
1535
1682
|
pullTimeoutTimer = null;
|
|
1536
1683
|
firstFrameReceived = false;
|
|
1537
1684
|
/** `Date.now()` of the FIRST `pull` — the origin of the cold-start ceiling. */
|
|
1538
1685
|
firstPullAt = 0;
|
|
1539
|
-
constructor(child, logger, deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs,
|
|
1686
|
+
constructor(child, logger, deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, hooks) {
|
|
1540
1687
|
this.child = child;
|
|
1541
1688
|
this.logger = logger;
|
|
1542
1689
|
this.deviceId = deviceId;
|
|
1543
1690
|
this.firstFrameTimeoutMs = firstFrameTimeoutMs;
|
|
1544
1691
|
this.frameTimeoutMs = frameTimeoutMs;
|
|
1545
1692
|
this.firstFrameMaxWaitMs = firstFrameMaxWaitMs;
|
|
1546
|
-
this.
|
|
1547
|
-
this.onPreRollReport = onPreRollReport;
|
|
1693
|
+
this.hooks = hooks;
|
|
1548
1694
|
this.child.on("message", (m) => this.handleMessage(m));
|
|
1549
1695
|
this.child.on("exit", (m) => this.handleExit(toWorkerExitInfo(m)));
|
|
1550
1696
|
}
|
|
@@ -1609,6 +1755,10 @@ var SessionDecodeSession = class {
|
|
|
1609
1755
|
*/
|
|
1610
1756
|
requestNativeCrop(frameId, bbox, maxWidth) {
|
|
1611
1757
|
if (this.exited) return Promise.resolve({ kind: "miss" });
|
|
1758
|
+
if (this.stopRequested) return Promise.resolve({
|
|
1759
|
+
kind: "miss",
|
|
1760
|
+
workerReason: NATIVE_CROP_REFUSED_DRAINING
|
|
1761
|
+
});
|
|
1612
1762
|
const requestId = this.nextNativeCropId++;
|
|
1613
1763
|
return new Promise((resolve) => {
|
|
1614
1764
|
this.pendingNativeCrops.set(requestId, { resolve });
|
|
@@ -1623,6 +1773,7 @@ var SessionDecodeSession = class {
|
|
|
1623
1773
|
} catch {
|
|
1624
1774
|
this.pendingNativeCrops.delete(requestId);
|
|
1625
1775
|
resolve({ kind: "miss" });
|
|
1776
|
+
this.settleDrainIfDone();
|
|
1626
1777
|
}
|
|
1627
1778
|
});
|
|
1628
1779
|
}
|
|
@@ -1667,20 +1818,135 @@ var SessionDecodeSession = class {
|
|
|
1667
1818
|
frameId
|
|
1668
1819
|
});
|
|
1669
1820
|
}
|
|
1670
|
-
/**
|
|
1671
|
-
|
|
1821
|
+
/** A yielded image the consumer now holds — see {@link openImages}. */
|
|
1822
|
+
noteImageOpened() {
|
|
1823
|
+
this.openImages += 1;
|
|
1824
|
+
}
|
|
1825
|
+
/** The consumer let go of a yielded image; a waiting drain may now be done. */
|
|
1826
|
+
noteImageClosed() {
|
|
1827
|
+
this.openImages = Math.max(0, this.openImages - 1);
|
|
1828
|
+
this.settleDrainIfDone();
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* End the session.
|
|
1832
|
+
*
|
|
1833
|
+
* `'drain'` with frames still held: tell the worker to stop decoding (it
|
|
1834
|
+
* keeps every frame it holds), and kill only once every yielded image has
|
|
1835
|
+
* been closed and every read answered — or when
|
|
1836
|
+
* {@link SESSION_DECODE_DRAIN_BOUND_MS} expires, which is logged. Nothing
|
|
1837
|
+
* held: `stop` + SIGTERM now. `'immediate'`: SIGKILL now.
|
|
1838
|
+
*
|
|
1839
|
+
* A SECOND call that says `'immediate'` while a drain is in progress
|
|
1840
|
+
* ESCALATES (D652): addon shutdown lands on sessions already draining from
|
|
1841
|
+
* a detach, and the runner exits right after — a drain it does not wait for
|
|
1842
|
+
* is an orphaned worker holding its GPU pool. Any other repeat is ignored.
|
|
1843
|
+
*/
|
|
1844
|
+
teardown(mode) {
|
|
1845
|
+
if (this.stopRequested) {
|
|
1846
|
+
if (mode === "immediate") this.escalateToImmediate();
|
|
1847
|
+
return;
|
|
1848
|
+
}
|
|
1672
1849
|
this.stopRequested = true;
|
|
1673
1850
|
this.clearPullTimeout();
|
|
1851
|
+
if (this.exited) return;
|
|
1852
|
+
if (mode === "drain" && this.unsettledCount() > 0) {
|
|
1853
|
+
this.trySend({ kind: "drain" });
|
|
1854
|
+
this.drainTimer = setTimeout(() => this.handleDrainBoundExpired(), SESSION_DECODE_DRAIN_BOUND_MS);
|
|
1855
|
+
this.drainTimer.unref?.();
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
this.killNow(mode);
|
|
1859
|
+
}
|
|
1860
|
+
/** The drain (or a SIGTERM still waiting on its fallback) gives way to SIGKILL now. */
|
|
1861
|
+
escalateToImmediate() {
|
|
1862
|
+
if (this.exited) return;
|
|
1863
|
+
if (this.drainTimer !== null) {
|
|
1864
|
+
clearTimeout(this.drainTimer);
|
|
1865
|
+
this.drainTimer = null;
|
|
1866
|
+
}
|
|
1867
|
+
if (!this.killSent) {
|
|
1868
|
+
this.killNow("immediate");
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1871
|
+
if (this.sigkillTimer !== null) {
|
|
1872
|
+
clearTimeout(this.sigkillTimer);
|
|
1873
|
+
this.sigkillTimer = null;
|
|
1874
|
+
}
|
|
1875
|
+
this.tryKill("SIGKILL");
|
|
1876
|
+
}
|
|
1877
|
+
/**
|
|
1878
|
+
* Frames the consumer still holds, reads still waiting on the worker, and
|
|
1879
|
+
* native crops already REQUESTED. A crop asked for before the teardown
|
|
1880
|
+
* request is waited for; one asked for after it is refused by name — post-analysis's
|
|
1881
|
+
* ordinary crop arrives 2.2–9.9 s after the frame, past any bound worth
|
|
1882
|
+
* holding a worker for, and is served from subject tiles that die with the
|
|
1883
|
+
* worker either way.
|
|
1884
|
+
*/
|
|
1885
|
+
unsettledCount() {
|
|
1886
|
+
return this.openImages + this.pendingReadCount() + this.pendingNativeCrops.size;
|
|
1887
|
+
}
|
|
1888
|
+
pendingReadCount() {
|
|
1889
|
+
let n = 0;
|
|
1890
|
+
for (const waiters of this.pendingToBuffers.values()) n += waiters.length;
|
|
1891
|
+
return n;
|
|
1892
|
+
}
|
|
1893
|
+
settleDrainIfDone() {
|
|
1894
|
+
if (this.drainTimer === null || this.unsettledCount() > 0) return;
|
|
1895
|
+
clearTimeout(this.drainTimer);
|
|
1896
|
+
this.drainTimer = null;
|
|
1897
|
+
this.killNow("drain");
|
|
1898
|
+
}
|
|
1899
|
+
/**
|
|
1900
|
+
* The drain ran out of time with frames still held. Those frames die with
|
|
1901
|
+
* the worker; that is a branch that DROPS work (D391), so it says how many,
|
|
1902
|
+
* on which camera.
|
|
1903
|
+
*/
|
|
1904
|
+
handleDrainBoundExpired() {
|
|
1905
|
+
this.drainTimer = null;
|
|
1906
|
+
if (this.exited) return;
|
|
1907
|
+
this.logger.warn("session-decode-coordinator: drain bound expired — killing the worker with frames still held", {
|
|
1908
|
+
tags: { deviceId: this.deviceId },
|
|
1909
|
+
meta: {
|
|
1910
|
+
unsettledFrames: this.openImages,
|
|
1911
|
+
pendingReads: this.pendingReadCount(),
|
|
1912
|
+
pendingCrops: this.pendingNativeCrops.size,
|
|
1913
|
+
boundMs: SESSION_DECODE_DRAIN_BOUND_MS
|
|
1914
|
+
}
|
|
1915
|
+
});
|
|
1916
|
+
this.killNow("drain");
|
|
1917
|
+
}
|
|
1918
|
+
/**
|
|
1919
|
+
* `stop`, then the kill — unless already exited.
|
|
1920
|
+
*
|
|
1921
|
+
* `'drain'`: SIGTERM, so the child flushes its last lines (`bounded-exit.ts`),
|
|
1922
|
+
* and a SIGKILL fallback. `'immediate'`: SIGKILL outright. Since D652 gave the
|
|
1923
|
+
* child a SIGTERM handler, SIGTERM is no longer instant death: a worker
|
|
1924
|
+
* blocked in a native call (a VAAPI hang) never runs the handler, and the
|
|
1925
|
+
* fallback was an `unref`'d timer the exiting parent did not wait for — so
|
|
1926
|
+
* on shutdown that worker was orphaned with its GPU pool. The fallback timer
|
|
1927
|
+
* is now ref'd: it exists only for a drain while the parent is alive (a
|
|
1928
|
+
* shutdown escalates to SIGKILL and clears it), and a timer that keeps the
|
|
1929
|
+
* process up for at most one second is the price of a kill that lands.
|
|
1930
|
+
*/
|
|
1931
|
+
killNow(mode) {
|
|
1932
|
+
if (this.killSent) return;
|
|
1933
|
+
this.killSent = true;
|
|
1674
1934
|
this.trySend({ kind: "stop" });
|
|
1935
|
+
if (mode === "immediate") {
|
|
1936
|
+
this.tryKill("SIGKILL");
|
|
1937
|
+
return;
|
|
1938
|
+
}
|
|
1675
1939
|
this.tryKill(void 0);
|
|
1676
1940
|
if (this.exited) return;
|
|
1677
1941
|
this.sigkillTimer = setTimeout(() => {
|
|
1678
1942
|
this.sigkillTimer = null;
|
|
1679
1943
|
if (this.exited) return;
|
|
1680
|
-
this.logger.warn("session-decode-coordinator: child did not exit within SIGKILL fallback window — forcing", {
|
|
1944
|
+
this.logger.warn("session-decode-coordinator: child did not exit within SIGKILL fallback window — forcing", {
|
|
1945
|
+
tags: { deviceId: this.deviceId },
|
|
1946
|
+
meta: { fallbackMs: SIGKILL_FALLBACK_MS }
|
|
1947
|
+
});
|
|
1681
1948
|
this.tryKill("SIGKILL");
|
|
1682
1949
|
}, SIGKILL_FALLBACK_MS);
|
|
1683
|
-
this.sigkillTimer.unref?.();
|
|
1684
1950
|
}
|
|
1685
1951
|
handleMessage(raw) {
|
|
1686
1952
|
if (!require_worker_protocol.isWorkerReply(raw)) {
|
|
@@ -1729,7 +1995,7 @@ var SessionDecodeSession = class {
|
|
|
1729
1995
|
this.logger[raw.level](raw.message);
|
|
1730
1996
|
return;
|
|
1731
1997
|
case "dialTiming":
|
|
1732
|
-
this.onDialTiming?.({
|
|
1998
|
+
this.hooks.onDialTiming?.({
|
|
1733
1999
|
hwContextMs: raw.hwContextMs,
|
|
1734
2000
|
demuxerOpenMs: raw.demuxerOpenMs,
|
|
1735
2001
|
decoderCreateMs: raw.decoderCreateMs,
|
|
@@ -1737,25 +2003,46 @@ var SessionDecodeSession = class {
|
|
|
1737
2003
|
streamInfoProbed: raw.streamInfoProbed
|
|
1738
2004
|
});
|
|
1739
2005
|
return;
|
|
2006
|
+
case "preRollSettled":
|
|
2007
|
+
this.hooks.onPreRollSettled?.({
|
|
2008
|
+
read: raw.read,
|
|
2009
|
+
released: raw.released,
|
|
2010
|
+
lostByReason: raw.lostByReason,
|
|
2011
|
+
unsettled: raw.unsettled
|
|
2012
|
+
});
|
|
2013
|
+
return;
|
|
1740
2014
|
case "preRollReport":
|
|
1741
|
-
this.onPreRollReport?.({
|
|
2015
|
+
this.hooks.onPreRollReport?.({
|
|
1742
2016
|
offered: raw.offered,
|
|
1743
2017
|
inferred: raw.inferred,
|
|
1744
2018
|
dropped: raw.dropped,
|
|
1745
2019
|
droppedByReason: raw.droppedByReason,
|
|
1746
2020
|
strideMs: raw.strideMs,
|
|
1747
2021
|
subsampled: raw.subsampled,
|
|
1748
|
-
spanMs: raw.spanMs
|
|
2022
|
+
spanMs: raw.spanMs,
|
|
2023
|
+
budgetFrames: raw.budgetFrames
|
|
1749
2024
|
});
|
|
1750
2025
|
return;
|
|
1751
2026
|
}
|
|
1752
2027
|
}
|
|
1753
2028
|
handleExit(info) {
|
|
1754
|
-
if (!this.
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
2029
|
+
if (!this.exited) this.hooks.onWorkerExit?.();
|
|
2030
|
+
if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
|
|
2031
|
+
tags: { deviceId: this.deviceId },
|
|
2032
|
+
meta: {
|
|
2033
|
+
code: info?.code ?? null,
|
|
2034
|
+
signal: info?.signal ?? null,
|
|
2035
|
+
...this.stopRequested ? {
|
|
2036
|
+
unsettledFrames: this.openImages,
|
|
2037
|
+
pendingReads: this.pendingReadCount()
|
|
2038
|
+
} : {}
|
|
2039
|
+
}
|
|
2040
|
+
});
|
|
1758
2041
|
this.exited = true;
|
|
2042
|
+
if (this.drainTimer) {
|
|
2043
|
+
clearTimeout(this.drainTimer);
|
|
2044
|
+
this.drainTimer = null;
|
|
2045
|
+
}
|
|
1759
2046
|
if (this.sigkillTimer) {
|
|
1760
2047
|
clearTimeout(this.sigkillTimer);
|
|
1761
2048
|
this.sigkillTimer = null;
|
|
@@ -1777,6 +2064,7 @@ var SessionDecodeSession = class {
|
|
|
1777
2064
|
if (!waiter) return;
|
|
1778
2065
|
this.pendingNativeCrops.delete(requestId);
|
|
1779
2066
|
waiter.resolve(attempt);
|
|
2067
|
+
this.settleDrainIfDone();
|
|
1780
2068
|
}
|
|
1781
2069
|
settlePendingPull(outcome) {
|
|
1782
2070
|
const waiter = this.pendingPull;
|
|
@@ -1872,6 +2160,7 @@ var SessionDecodeSession = class {
|
|
|
1872
2160
|
const waiter = queue.shift();
|
|
1873
2161
|
if (queue.length === 0) this.pendingToBuffers.delete(frameId);
|
|
1874
2162
|
waiter?.resolve(bytes);
|
|
2163
|
+
this.settleDrainIfDone();
|
|
1875
2164
|
}
|
|
1876
2165
|
/**
|
|
1877
2166
|
* `error` replies MAY carry a `frameId` (a `toBuffer`-scoped miss, e.g. a
|
|
@@ -1891,6 +2180,7 @@ var SessionDecodeSession = class {
|
|
|
1891
2180
|
if (queue && queue.length > 0) {
|
|
1892
2181
|
for (const waiter of queue) waiter.reject(err);
|
|
1893
2182
|
this.pendingToBuffers.delete(frameId);
|
|
2183
|
+
this.settleDrainIfDone();
|
|
1894
2184
|
return;
|
|
1895
2185
|
}
|
|
1896
2186
|
this.logger.debug("session-decode-coordinator: dropping toBuffer error for unknown frameId", { meta: { frameId } });
|
|
@@ -1905,6 +2195,7 @@ var SessionDecodeSession = class {
|
|
|
1905
2195
|
}
|
|
1906
2196
|
for (const waiters of this.pendingToBuffers.values()) for (const waiter of waiters) waiter.reject(err);
|
|
1907
2197
|
this.pendingToBuffers.clear();
|
|
2198
|
+
this.settleDrainIfDone();
|
|
1908
2199
|
}
|
|
1909
2200
|
queueToBufferWaiter(frameId, waiter) {
|
|
1910
2201
|
const queue = this.pendingToBuffers.get(frameId);
|
|
@@ -1942,6 +2233,7 @@ var SessionDecodeSession = class {
|
|
|
1942
2233
|
/** Build the {@link FrameImage} adapter for one `frame` reply. */
|
|
1943
2234
|
function buildFrameImage(session, reply) {
|
|
1944
2235
|
let closed = false;
|
|
2236
|
+
session.noteImageOpened();
|
|
1945
2237
|
return {
|
|
1946
2238
|
width: reply.width,
|
|
1947
2239
|
height: reply.height,
|
|
@@ -1956,7 +2248,9 @@ function buildFrameImage(session, reply) {
|
|
|
1956
2248
|
releaseNativeLease: () => session.releaseLease(reply.frameId),
|
|
1957
2249
|
markInferred: () => session.markInferred(reply.frameId),
|
|
1958
2250
|
close: () => {
|
|
2251
|
+
if (closed) return;
|
|
1959
2252
|
closed = true;
|
|
2253
|
+
session.noteImageClosed();
|
|
1960
2254
|
}
|
|
1961
2255
|
};
|
|
1962
2256
|
}
|
|
@@ -1965,15 +2259,23 @@ function buildFrameImage(session, reply) {
|
|
|
1965
2259
|
* `generateVideoFrames()` call, drives it with `pull`/`toBuffer` requests
|
|
1966
2260
|
* one at a time (no pre-fetch — back-pressure is the async generator's
|
|
1967
2261
|
* natural suspend-on-yield), and guarantees the child is torn down
|
|
1968
|
-
* (`stop` + `kill()` + SIGKILL fallback) in every exit
|
|
2262
|
+
* (bounded drain, then `stop` + `kill()` + SIGKILL fallback) in every exit
|
|
2263
|
+
* path via `finally`.
|
|
1969
2264
|
*/
|
|
1970
2265
|
function createSessionDecodeCoordinator(deps) {
|
|
1971
2266
|
const firstFrameTimeoutMs = deps.firstFrameTimeoutMs ?? DEFAULT_FIRST_FRAME_TIMEOUT_MS;
|
|
1972
2267
|
const frameTimeoutMs = deps.frameTimeoutMs ?? DEFAULT_FRAME_TIMEOUT_MS;
|
|
1973
2268
|
const firstFrameMaxWaitMs = Math.max(deps.firstFrameMaxWaitMs ?? DEFAULT_FIRST_FRAME_MAX_WAIT_MS, firstFrameTimeoutMs);
|
|
1974
2269
|
return { async *generateVideoFrames(source, opts) {
|
|
1975
|
-
const session = new SessionDecodeSession(deps.fork(source), deps.logger, source.deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs,
|
|
2270
|
+
const session = new SessionDecodeSession(deps.fork(source), deps.logger, source.deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, {
|
|
2271
|
+
...deps.onDialTiming ? { onDialTiming: deps.onDialTiming } : {},
|
|
2272
|
+
...deps.onPreRollReport ? { onPreRollReport: deps.onPreRollReport } : {},
|
|
2273
|
+
...deps.onPreRollSettled ? { onPreRollSettled: deps.onPreRollSettled } : {},
|
|
2274
|
+
...deps.onWorkerExit ? { onWorkerExit: deps.onWorkerExit } : {}
|
|
2275
|
+
});
|
|
1976
2276
|
session.start(source, opts, deps.lease());
|
|
2277
|
+
deps.onSessionTeardown?.((mode) => session.teardown(mode));
|
|
2278
|
+
let failed = false;
|
|
1977
2279
|
try {
|
|
1978
2280
|
while (true) {
|
|
1979
2281
|
const outcome = await session.pullNext();
|
|
@@ -1986,8 +2288,11 @@ function createSessionDecodeCoordinator(deps) {
|
|
|
1986
2288
|
...reply.preRoll === true ? { preRoll: true } : {}
|
|
1987
2289
|
};
|
|
1988
2290
|
}
|
|
2291
|
+
} catch (err) {
|
|
2292
|
+
failed = true;
|
|
2293
|
+
throw err;
|
|
1989
2294
|
} finally {
|
|
1990
|
-
session.teardown();
|
|
2295
|
+
session.teardown(failed ? "immediate" : deps.teardownMode?.() ?? "drain");
|
|
1991
2296
|
}
|
|
1992
2297
|
} };
|
|
1993
2298
|
}
|
|
@@ -2031,17 +2336,39 @@ async function toDecodedFrame(frame, format, viewSpec) {
|
|
|
2031
2336
|
* Start pulling frames from a per-session decode worker and feed each decoded
|
|
2032
2337
|
* frame to `onDecodedFrame`. Returns an idempotent teardown closure that stops
|
|
2033
2338
|
* the async generator (`it.return()`) — which triggers the coordinator's
|
|
2034
|
-
* `finally` →
|
|
2035
|
-
*
|
|
2036
|
-
*
|
|
2339
|
+
* `finally` → a bounded DRAIN of the frames already handed out (D652), then
|
|
2340
|
+
* `stop` + SIGTERM + 1s SIGKILL fallback → worker exit → the OS reclaims the
|
|
2341
|
+
* VAAPI/GPU decode pool. `teardown('immediate')` SIGKILLs at once (shutdown),
|
|
2342
|
+
* and said a second time during a drain it escalates that drain to the kill.
|
|
2343
|
+
* Fire-and-forget: the returned closure is synchronous; the frame loop and
|
|
2344
|
+
* generator teardown run on their own tasks.
|
|
2037
2345
|
*/
|
|
2038
2346
|
function startSessionDecodePump(deps) {
|
|
2039
2347
|
const format = deps.opts.format ?? "rgb";
|
|
2040
2348
|
let dialTiming = null;
|
|
2349
|
+
let teardownMode = "drain";
|
|
2350
|
+
let escalateSession = null;
|
|
2351
|
+
let forked = false;
|
|
2352
|
+
let workerGone = false;
|
|
2353
|
+
const noteWorkerGone = () => {
|
|
2354
|
+
if (workerGone) return;
|
|
2355
|
+
workerGone = true;
|
|
2356
|
+
deps.onWorkerGone?.();
|
|
2357
|
+
};
|
|
2041
2358
|
const iterator = createSessionDecodeCoordinator({
|
|
2042
|
-
fork:
|
|
2359
|
+
fork: (source) => {
|
|
2360
|
+
const child = deps.fork(source);
|
|
2361
|
+
forked = true;
|
|
2362
|
+
return child;
|
|
2363
|
+
},
|
|
2043
2364
|
lease: deps.lease,
|
|
2044
2365
|
logger: deps.logger,
|
|
2366
|
+
teardownMode: () => teardownMode,
|
|
2367
|
+
onSessionTeardown: (teardown) => {
|
|
2368
|
+
escalateSession = teardown;
|
|
2369
|
+
},
|
|
2370
|
+
onWorkerExit: noteWorkerGone,
|
|
2371
|
+
...deps.onPreRollSettled ? { onPreRollSettled: deps.onPreRollSettled } : {},
|
|
2045
2372
|
onDialTiming: (timing) => {
|
|
2046
2373
|
dialTiming = timing;
|
|
2047
2374
|
},
|
|
@@ -2147,13 +2474,21 @@ function startSessionDecodePump(deps) {
|
|
|
2147
2474
|
});
|
|
2148
2475
|
} finally {
|
|
2149
2476
|
releaseSpawnSlotOnce();
|
|
2477
|
+
if (!forked) noteWorkerGone();
|
|
2150
2478
|
if (!stopped) deps.onEnded?.();
|
|
2151
2479
|
}
|
|
2152
2480
|
};
|
|
2153
2481
|
pump();
|
|
2154
|
-
return () => {
|
|
2155
|
-
if (stopped)
|
|
2482
|
+
return (mode = "drain") => {
|
|
2483
|
+
if (stopped) {
|
|
2484
|
+
if (mode === "immediate" && teardownMode !== "immediate") {
|
|
2485
|
+
teardownMode = "immediate";
|
|
2486
|
+
escalateSession?.("immediate");
|
|
2487
|
+
}
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2156
2490
|
stopped = true;
|
|
2491
|
+
teardownMode = mode;
|
|
2157
2492
|
releaseSpawnSlotOnce();
|
|
2158
2493
|
iterator.return().catch(() => {});
|
|
2159
2494
|
};
|
|
@@ -2235,28 +2570,17 @@ function onboardRootDetections(slot, frame, gate) {
|
|
|
2235
2570
|
//#endregion
|
|
2236
2571
|
//#region src/pipeline-runner/bench-step-schema.ts
|
|
2237
2572
|
/**
|
|
2238
|
-
* Shared bench-facing schemas: the minimal recursive step tree
|
|
2239
|
-
*
|
|
2240
|
-
*
|
|
2241
|
-
*
|
|
2242
|
-
*
|
|
2243
|
-
*
|
|
2244
|
-
*
|
|
2245
|
-
*
|
|
2246
|
-
*
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
runtime: require_dist._enum(["node", "python"]),
|
|
2250
|
-
backend: require_dist.string(),
|
|
2251
|
-
format: require_dist._enum([
|
|
2252
|
-
"onnx",
|
|
2253
|
-
"coreml",
|
|
2254
|
-
"openvino",
|
|
2255
|
-
"tflite",
|
|
2256
|
-
"pt"
|
|
2257
|
-
]),
|
|
2258
|
-
device: require_dist.string().optional()
|
|
2259
|
-
});
|
|
2573
|
+
* Shared bench-facing schemas: the minimal recursive step tree. Extracted from
|
|
2574
|
+
* `bench-actions.ts` so the capacity-ramp schemas (`capacity/capacity-schemas.ts`)
|
|
2575
|
+
* can reuse it without a bench-actions ↔ capacity-schemas import cycle.
|
|
2576
|
+
*
|
|
2577
|
+
* Local mirror of pipeline-executor's step shape ON PURPOSE (not a cap-package
|
|
2578
|
+
* coupling): addonId + modelId + enabled + children.
|
|
2579
|
+
*
|
|
2580
|
+
* There is no engine choice here any more (D646). A bench names the pool it
|
|
2581
|
+
* measures by `deviceKey`; the engine override it used to carry was honoured by
|
|
2582
|
+
* swapping the executor's shared engine state under live cameras.
|
|
2583
|
+
*/
|
|
2260
2584
|
var BenchStepSchema = require_dist.lazy(() => require_dist.object({
|
|
2261
2585
|
addonId: require_dist.string(),
|
|
2262
2586
|
modelId: require_dist.string(),
|
|
@@ -2292,7 +2616,6 @@ var RunCapacityRampInputSchema = require_dist.object({
|
|
|
2292
2616
|
sessionId: require_dist.string(),
|
|
2293
2617
|
nodeId: require_dist.string().optional(),
|
|
2294
2618
|
deviceKey: require_dist.string().optional(),
|
|
2295
|
-
engine: BenchEngineChoiceSchema.optional(),
|
|
2296
2619
|
slo: CapacitySloSchema.optional(),
|
|
2297
2620
|
allowActiveCameras: require_dist.boolean().optional(),
|
|
2298
2621
|
drainTimeoutSec: require_dist.number().min(1).max(300).optional()
|
|
@@ -2481,16 +2804,15 @@ var CacheBenchFrameResultSchema = require_dist.object({
|
|
|
2481
2804
|
});
|
|
2482
2805
|
var ReleaseBenchFrameInputSchema = require_dist.object({ frameId: require_dist.string() });
|
|
2483
2806
|
var ReleaseBenchFrameResultSchema = require_dist.object({ released: require_dist.boolean() });
|
|
2484
|
-
var
|
|
2807
|
+
var StartSyntheticBenchInputSchema = require_dist.object({
|
|
2485
2808
|
frameId: require_dist.string(),
|
|
2486
2809
|
steps: require_dist.array(BenchStepSchema).min(1),
|
|
2487
2810
|
parallel: require_dist.number().int().min(1).max(32),
|
|
2488
2811
|
iterations: require_dist.number().int().min(1).max(1e4),
|
|
2489
2812
|
warmup: require_dist.number().int().min(0).max(100).optional(),
|
|
2490
|
-
sessionId: require_dist.string().
|
|
2813
|
+
sessionId: require_dist.string().min(1),
|
|
2491
2814
|
simulatePipeline: require_dist.boolean().optional(),
|
|
2492
2815
|
nodeId: require_dist.string().optional(),
|
|
2493
|
-
engine: BenchEngineChoiceSchema.optional(),
|
|
2494
2816
|
deviceKey: require_dist.string().optional(),
|
|
2495
2817
|
plane: require_dist._enum(["full", "frame"]).optional()
|
|
2496
2818
|
});
|
|
@@ -2500,7 +2822,16 @@ var TimingSplitSchema = require_dist.object({
|
|
|
2500
2822
|
p95: require_dist.number(),
|
|
2501
2823
|
p99: require_dist.number()
|
|
2502
2824
|
});
|
|
2503
|
-
var
|
|
2825
|
+
var BenchPoolSchema = require_dist.object({
|
|
2826
|
+
nodeId: require_dist.string(),
|
|
2827
|
+
deviceKey: require_dist.string().nullable(),
|
|
2828
|
+
runtime: require_dist.string().nullable(),
|
|
2829
|
+
numWorkers: require_dist.number().nullable(),
|
|
2830
|
+
concurrency: require_dist.number().nullable(),
|
|
2831
|
+
batchMode: require_dist.string().nullable(),
|
|
2832
|
+
maxBatchSize: require_dist.number().nullable()
|
|
2833
|
+
});
|
|
2834
|
+
var SyntheticBenchResultSchema = require_dist.object({
|
|
2504
2835
|
runs: require_dist.number(),
|
|
2505
2836
|
wallSec: require_dist.number(),
|
|
2506
2837
|
fps: require_dist.number(),
|
|
@@ -2512,23 +2843,52 @@ var RunSyntheticBenchResultSchema = require_dist.object({
|
|
|
2512
2843
|
predictMs: require_dist.number(),
|
|
2513
2844
|
batchSizeMean: require_dist.number(),
|
|
2514
2845
|
batchSizeMax: require_dist.number(),
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
backend: require_dist.string(),
|
|
2518
|
-
device: require_dist.string().optional()
|
|
2519
|
-
}).optional(),
|
|
2520
|
-
tuning: require_dist.object({
|
|
2521
|
-
batchMode: require_dist.string(),
|
|
2522
|
-
windowMs: require_dist.number(),
|
|
2523
|
-
maxBatchSize: require_dist.number(),
|
|
2524
|
-
concurrency: require_dist.number()
|
|
2525
|
-
}).optional(),
|
|
2526
|
-
path: require_dist.string().optional()
|
|
2846
|
+
pool: BenchPoolSchema,
|
|
2847
|
+
path: require_dist.string()
|
|
2527
2848
|
});
|
|
2849
|
+
/** `startSyntheticBench` answers at once; the run continues in the background. */
|
|
2850
|
+
var StartSyntheticBenchAcceptedSchema = require_dist.object({
|
|
2851
|
+
accepted: require_dist.literal(true),
|
|
2852
|
+
sessionId: require_dist.string()
|
|
2853
|
+
});
|
|
2854
|
+
var SyntheticBenchKeySchema = require_dist.object({ sessionId: require_dist.string() });
|
|
2855
|
+
var SyntheticBenchStatusSchema = require_dist.object({
|
|
2856
|
+
sessionId: require_dist.string(),
|
|
2857
|
+
state: require_dist._enum([
|
|
2858
|
+
"running",
|
|
2859
|
+
"done",
|
|
2860
|
+
"failed",
|
|
2861
|
+
"aborted"
|
|
2862
|
+
]),
|
|
2863
|
+
startedAtMs: require_dist.number(),
|
|
2864
|
+
finishedAtMs: require_dist.number().nullable(),
|
|
2865
|
+
progress: require_dist.object({
|
|
2866
|
+
runs: require_dist.number(),
|
|
2867
|
+
totalRuns: require_dist.number()
|
|
2868
|
+
}).nullable(),
|
|
2869
|
+
result: SyntheticBenchResultSchema.nullable(),
|
|
2870
|
+
error: require_dist.string().nullable()
|
|
2871
|
+
}).nullable();
|
|
2528
2872
|
var pipelineRunnerBenchActions = require_dist.defineCustomActions({
|
|
2529
2873
|
cacheBenchFrame: require_dist.customAction(CacheBenchFrameInputSchema, CacheBenchFrameResultSchema, { kind: "mutation" }),
|
|
2530
2874
|
releaseBenchFrame: require_dist.customAction(ReleaseBenchFrameInputSchema, ReleaseBenchFrameResultSchema, { kind: "mutation" }),
|
|
2531
|
-
|
|
2875
|
+
/**
|
|
2876
|
+
* START a synthetic bench and return at once (D646). Up to 32 parallel ×
|
|
2877
|
+
* 10 000 iterations does not fit the UDS addon-call plane's 60 s request
|
|
2878
|
+
* timeout, and a synchronous call was abandoned by its caller while it kept
|
|
2879
|
+
* loading the node. Poll `getSyntheticBenchStatus {sessionId}`; stop early
|
|
2880
|
+
* with `abortSyntheticBench {sessionId}`. One bench per runner at a time —
|
|
2881
|
+
* the measurement owns the pool.
|
|
2882
|
+
*/
|
|
2883
|
+
startSyntheticBench: require_dist.customAction(StartSyntheticBenchInputSchema, StartSyntheticBenchAcceptedSchema, {
|
|
2884
|
+
kind: "mutation",
|
|
2885
|
+
auth: "admin"
|
|
2886
|
+
}),
|
|
2887
|
+
getSyntheticBenchStatus: require_dist.customAction(SyntheticBenchKeySchema, SyntheticBenchStatusSchema),
|
|
2888
|
+
abortSyntheticBench: require_dist.customAction(SyntheticBenchKeySchema, require_dist.object({ aborted: require_dist.boolean() }), {
|
|
2889
|
+
kind: "mutation",
|
|
2890
|
+
auth: "admin"
|
|
2891
|
+
}),
|
|
2532
2892
|
/**
|
|
2533
2893
|
* OPEN-loop capacity ramp (spec `2026-08-02-capacity-test-design.md` §4):
|
|
2534
2894
|
* paced N×`fpsPerCamera` single-frame `runPipeline` calls with raw RGB —
|
|
@@ -2593,12 +2953,24 @@ function buildBenchRunPipelineInput(req) {
|
|
|
2593
2953
|
return {
|
|
2594
2954
|
steps: [...req.steps],
|
|
2595
2955
|
frame: req.frame,
|
|
2596
|
-
...req.engine ? { engine: req.engine } : {},
|
|
2597
2956
|
...req.deviceKey ? { deviceKey: req.deviceKey } : {},
|
|
2598
2957
|
...req.plane ? { plane: req.plane } : {}
|
|
2599
2958
|
};
|
|
2600
2959
|
}
|
|
2601
2960
|
/**
|
|
2961
|
+
* The tuning query for the pool a bench MEASURED (D646). It asked the node
|
|
2962
|
+
* default of whatever node answered first, so an NPU run was recorded beside
|
|
2963
|
+
* the onnx pool's `concurrency: 4`. Absent `deviceKey`, never
|
|
2964
|
+
* present-and-undefined, for the same `.loose()` reason as
|
|
2965
|
+
* {@link buildBenchRunPipelineInput}.
|
|
2966
|
+
*/
|
|
2967
|
+
function buildBenchTuningQuery(target) {
|
|
2968
|
+
return {
|
|
2969
|
+
nodeId: target.nodeId,
|
|
2970
|
+
input: target.deviceKey ? { deviceKey: target.deviceKey } : {}
|
|
2971
|
+
};
|
|
2972
|
+
}
|
|
2973
|
+
/**
|
|
2602
2974
|
* How long a box stays "seen".
|
|
2603
2975
|
*
|
|
2604
2976
|
* Sized on the gate's own clock, not on a guess: `maxDeferralMs` defaults to
|
|
@@ -3547,8 +3919,8 @@ async function runDetailSubtree(deps, input) {
|
|
|
3547
3919
|
} : void 0;
|
|
3548
3920
|
const details = [];
|
|
3549
3921
|
for (const step of steps) {
|
|
3550
|
-
const
|
|
3551
|
-
const result = await deps.runPipeline([step], crop.jpeg, input.deviceId, deviceKey, nativeCropRef);
|
|
3922
|
+
const placement = deps.placeStep?.(input.deviceId, step);
|
|
3923
|
+
const result = await deps.runPipeline([placement?.step ?? step], crop.jpeg, input.deviceId, placement?.deviceKey, nativeCropRef);
|
|
3552
3924
|
if (!result) continue;
|
|
3553
3925
|
for (const detection of result.detections) details.push(toDetailResult(detection, step.addonId, crop));
|
|
3554
3926
|
}
|
|
@@ -3655,6 +4027,72 @@ function resolveStepDevice(input) {
|
|
|
3655
4027
|
reason: "auto"
|
|
3656
4028
|
};
|
|
3657
4029
|
}
|
|
4030
|
+
/**
|
|
4031
|
+
* Re-provision a jumped subtree for its TARGET device (D650): every step's
|
|
4032
|
+
* `modelId` / `settings` are REPLACED by the target's entry — the orchestrator
|
|
4033
|
+
* computed it as what the step would carry had the camera been elected onto
|
|
4034
|
+
* that device (its per-(node,device) base with the camera's override for it).
|
|
4035
|
+
* A step absent from the map runs with neither, i.e. the target format's
|
|
4036
|
+
* default model. The elected device's provisioning never follows a step across:
|
|
4037
|
+
* it was chosen for a format this subtree does not run on.
|
|
4038
|
+
*/
|
|
4039
|
+
function provisionSubtreeForDevice(step, provisioning) {
|
|
4040
|
+
const own = provisioning[step.addonId];
|
|
4041
|
+
const { modelId: _electedModelId, settings: _electedSettings, ...rest } = step;
|
|
4042
|
+
return {
|
|
4043
|
+
...rest,
|
|
4044
|
+
...own?.modelId !== void 0 ? { modelId: own.modelId } : {},
|
|
4045
|
+
...own?.settings !== void 0 ? { settings: own.settings } : {},
|
|
4046
|
+
...step.children !== void 0 ? { children: step.children.map((c) => provisionSubtreeForDevice(c, provisioning)) } : {}
|
|
4047
|
+
};
|
|
4048
|
+
}
|
|
4049
|
+
/**
|
|
4050
|
+
* Place ONE detail subtree (D650): which same-node device runs it
|
|
4051
|
+
* ({@link resolveStepDevice}) and, when it jumps, the provisioning it runs
|
|
4052
|
+
* with there ({@link provisionSubtreeForDevice}). Pure — the caller logs.
|
|
4053
|
+
*
|
|
4054
|
+
* A roster entry without `steps` came from an orchestrator that predates D650:
|
|
4055
|
+
* the step keeps what it was sent with, as it did before.
|
|
4056
|
+
*/
|
|
4057
|
+
function placeDetailStep(input) {
|
|
4058
|
+
const { step, effectiveDeviceKey, roster } = input;
|
|
4059
|
+
if (!effectiveDeviceKey) return {
|
|
4060
|
+
deviceKey: void 0,
|
|
4061
|
+
step,
|
|
4062
|
+
decision: null,
|
|
4063
|
+
provisioned: false
|
|
4064
|
+
};
|
|
4065
|
+
if (!roster || roster.length < 2) return {
|
|
4066
|
+
deviceKey: effectiveDeviceKey,
|
|
4067
|
+
step,
|
|
4068
|
+
decision: null,
|
|
4069
|
+
provisioned: false
|
|
4070
|
+
};
|
|
4071
|
+
const decision = resolveStepDevice({
|
|
4072
|
+
effectiveDeviceKey,
|
|
4073
|
+
devices: roster.map((d) => ({
|
|
4074
|
+
deviceKey: d.deviceKey,
|
|
4075
|
+
weight: d.weight,
|
|
4076
|
+
maxSessions: d.maxSessions,
|
|
4077
|
+
attachedCameras: input.attachedByKey.get(d.deviceKey) ?? 0
|
|
4078
|
+
})),
|
|
4079
|
+
...step.jumpDeviceKey !== void 0 ? { manualJumpDeviceKey: step.jumpDeviceKey } : {},
|
|
4080
|
+
canRun: buildSubtreeCanRun(step, input.getStepDef)
|
|
4081
|
+
});
|
|
4082
|
+
if (decision.kind !== "jumped") return {
|
|
4083
|
+
deviceKey: decision.deviceKey,
|
|
4084
|
+
step,
|
|
4085
|
+
decision,
|
|
4086
|
+
provisioned: false
|
|
4087
|
+
};
|
|
4088
|
+
const provisioning = roster.find((d) => d.deviceKey === decision.deviceKey)?.steps;
|
|
4089
|
+
return {
|
|
4090
|
+
deviceKey: decision.deviceKey,
|
|
4091
|
+
step: provisioning === void 0 ? step : provisionSubtreeForDevice(step, provisioning),
|
|
4092
|
+
decision,
|
|
4093
|
+
provisioned: provisioning !== void 0
|
|
4094
|
+
};
|
|
4095
|
+
}
|
|
3658
4096
|
//#endregion
|
|
3659
4097
|
//#region src/pipeline-runner/device-load.ts
|
|
3660
4098
|
/** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
|
|
@@ -3838,6 +4276,74 @@ function release(cutter) {
|
|
|
3838
4276
|
} catch {}
|
|
3839
4277
|
}
|
|
3840
4278
|
//#endregion
|
|
4279
|
+
//#region src/pipeline-runner/occupancy-burst-ledger.ts
|
|
4280
|
+
var OccupancyBurstLedger = class {
|
|
4281
|
+
options;
|
|
4282
|
+
frames = 0;
|
|
4283
|
+
inferred = 0;
|
|
4284
|
+
superseded = 0;
|
|
4285
|
+
settled = 0;
|
|
4286
|
+
closed = false;
|
|
4287
|
+
completed = false;
|
|
4288
|
+
boundTimer = null;
|
|
4289
|
+
constructor(options) {
|
|
4290
|
+
this.options = options;
|
|
4291
|
+
}
|
|
4292
|
+
/**
|
|
4293
|
+
* Count one collected frame and hand back its settler. A frame the runner
|
|
4294
|
+
* REFUSED is settled `'lost'` by the caller at once — it is a frame the
|
|
4295
|
+
* verdict did not see, which is what `lost` counts.
|
|
4296
|
+
*/
|
|
4297
|
+
admit() {
|
|
4298
|
+
this.frames += 1;
|
|
4299
|
+
let done = false;
|
|
4300
|
+
return (outcome) => {
|
|
4301
|
+
if (done || this.completed) return;
|
|
4302
|
+
done = true;
|
|
4303
|
+
this.settled += 1;
|
|
4304
|
+
if (outcome === "inferred") this.inferred += 1;
|
|
4305
|
+
else if (outcome === "superseded") this.superseded += 1;
|
|
4306
|
+
this.completeIfSettled();
|
|
4307
|
+
};
|
|
4308
|
+
}
|
|
4309
|
+
/**
|
|
4310
|
+
* No more frames will be collected (count reached, safety timeout, or the
|
|
4311
|
+
* decode ended). Completes now when everything admitted has settled, else
|
|
4312
|
+
* arms the bound. Idempotent.
|
|
4313
|
+
*/
|
|
4314
|
+
close() {
|
|
4315
|
+
if (this.closed || this.completed) return;
|
|
4316
|
+
this.closed = true;
|
|
4317
|
+
if (this.completeIfSettled()) return;
|
|
4318
|
+
this.boundTimer = setTimeout(() => {
|
|
4319
|
+
this.boundTimer = null;
|
|
4320
|
+
this.complete(true);
|
|
4321
|
+
}, this.options.settleBoundMs);
|
|
4322
|
+
this.boundTimer.unref?.();
|
|
4323
|
+
}
|
|
4324
|
+
completeIfSettled() {
|
|
4325
|
+
if (!this.closed || this.settled < this.frames) return false;
|
|
4326
|
+
this.complete(false);
|
|
4327
|
+
return true;
|
|
4328
|
+
}
|
|
4329
|
+
complete(timedOut) {
|
|
4330
|
+
if (this.completed) return;
|
|
4331
|
+
this.completed = true;
|
|
4332
|
+
if (this.boundTimer !== null) {
|
|
4333
|
+
clearTimeout(this.boundTimer);
|
|
4334
|
+
this.boundTimer = null;
|
|
4335
|
+
}
|
|
4336
|
+
this.options.onComplete({
|
|
4337
|
+
frames: this.frames,
|
|
4338
|
+
inferred: this.inferred,
|
|
4339
|
+
superseded: this.superseded,
|
|
4340
|
+
lost: this.frames - this.inferred - this.superseded,
|
|
4341
|
+
unsettled: this.frames - this.settled,
|
|
4342
|
+
timedOut
|
|
4343
|
+
});
|
|
4344
|
+
}
|
|
4345
|
+
};
|
|
4346
|
+
//#endregion
|
|
3841
4347
|
//#region src/pipeline-runner/frame-lazy-metrics.ts
|
|
3842
4348
|
function emptyFrameLazyCounters() {
|
|
3843
4349
|
return {
|
|
@@ -5250,41 +5756,6 @@ var FrameGovernor = class {
|
|
|
5250
5756
|
};
|
|
5251
5757
|
//#endregion
|
|
5252
5758
|
//#region src/pipeline-runner/frame-queue.ts
|
|
5253
|
-
/**
|
|
5254
|
-
* Latest-frame-only buffer. Keeps only the most recent item, dropping all
|
|
5255
|
-
* older items immediately. This ensures inference always runs on the
|
|
5256
|
-
* freshest available frame, never accumulating a backlog regardless of
|
|
5257
|
-
* inference latency.
|
|
5258
|
-
*
|
|
5259
|
-
* ## Buffer ownership (Phase 5 / D9 Task 7b)
|
|
5260
|
-
*
|
|
5261
|
-
* Frames arrive from the shm frame plane via a `FrameRingReader`, whose
|
|
5262
|
-
* `FrameRead.pixels` is **borrowed** — a view onto the reader's reusable
|
|
5263
|
-
* scratch buffer, valid only until that reader's next read (see
|
|
5264
|
-
* `@camstack/shm-ring`'s `FrameRead` contract). Both queue consumers retain
|
|
5265
|
-
* the frame PAST the next read:
|
|
5266
|
-
*
|
|
5267
|
-
* - the **motion** queue is drained by the scheduler's `tick`, not by the
|
|
5268
|
-
* poller that read the frame — and the poller drains a burst of handles
|
|
5269
|
-
* per poll, so even handle #2's read would clobber handle #1's pixels;
|
|
5270
|
-
* - the **detection** queue is drained asynchronously after a semaphore wait
|
|
5271
|
-
* and a 10s–100s ms inference.
|
|
5272
|
-
*
|
|
5273
|
-
* In both cases holding the borrowed buffer is silent corruption. So
|
|
5274
|
-
* `enqueue` runs the queue's injected `clone` function which copies the
|
|
5275
|
-
* pixels into queue-owned storage at the retention boundary. The queue is
|
|
5276
|
-
* latest-only (capacity 1), so this is exactly one allocation per *queued*
|
|
5277
|
-
* frame — bounded by the scheduler/inference cadence, NOT per *read* frame.
|
|
5278
|
-
* The high-frequency per-read allocation that failed the D9 perf gate is
|
|
5279
|
-
* eliminated; this bounded copy-on-retain is the correct cost.
|
|
5280
|
-
*
|
|
5281
|
-
* The generic parameter `T` lets the detection path queue a
|
|
5282
|
-
* `{ frame, handle }` entry (Task 7 — propagating the `FrameHandle`
|
|
5283
|
-
* through to the inference-result event), while the motion path keeps
|
|
5284
|
-
* queueing bare `DecodedFrame`s. Each instance provides its own
|
|
5285
|
-
* `clone` so the inner pixel buffer is detached at the same retention
|
|
5286
|
-
* boundary regardless of the wrapping entry shape.
|
|
5287
|
-
*/
|
|
5288
5759
|
var FrameQueue = class {
|
|
5289
5760
|
maxSize;
|
|
5290
5761
|
latest = null;
|
|
@@ -5305,7 +5776,7 @@ var FrameQueue = class {
|
|
|
5305
5776
|
enqueue(item) {
|
|
5306
5777
|
if (this.latest !== null) {
|
|
5307
5778
|
this._droppedFrames++;
|
|
5308
|
-
this.dispose?.(this.latest);
|
|
5779
|
+
this.dispose?.(this.latest, "superseded");
|
|
5309
5780
|
}
|
|
5310
5781
|
this.latest = this.clone(item);
|
|
5311
5782
|
}
|
|
@@ -5320,8 +5791,12 @@ var FrameQueue = class {
|
|
|
5320
5791
|
get droppedFrames() {
|
|
5321
5792
|
return this._droppedFrames;
|
|
5322
5793
|
}
|
|
5794
|
+
/** The queued item, without taking it. */
|
|
5795
|
+
peek() {
|
|
5796
|
+
return this.latest ?? void 0;
|
|
5797
|
+
}
|
|
5323
5798
|
clear() {
|
|
5324
|
-
if (this.latest !== null) this.dispose?.(this.latest);
|
|
5799
|
+
if (this.latest !== null) this.dispose?.(this.latest, "cleared");
|
|
5325
5800
|
this.latest = null;
|
|
5326
5801
|
}
|
|
5327
5802
|
};
|
|
@@ -5721,7 +6196,8 @@ function ownDetectionEntry(entry) {
|
|
|
5721
6196
|
return {
|
|
5722
6197
|
frame: ownFrame(entry.frame),
|
|
5723
6198
|
handle: entry.handle,
|
|
5724
|
-
occasion: entry.occasion
|
|
6199
|
+
occasion: entry.occasion,
|
|
6200
|
+
...entry.onSettled ? { onSettled: entry.onSettled } : {}
|
|
5725
6201
|
};
|
|
5726
6202
|
}
|
|
5727
6203
|
/**
|
|
@@ -5863,9 +6339,10 @@ var PipelineRunner = class {
|
|
|
5863
6339
|
}
|
|
5864
6340
|
registerCamera(deviceId, registration) {
|
|
5865
6341
|
const motionQueue = new FrameQueue(this.config.maxQueueDepth, ownMotionEntry);
|
|
5866
|
-
const detectionQueue = new FrameQueue(this.config.maxQueueDepth, ownDetectionEntry, (entry) => {
|
|
6342
|
+
const detectionQueue = new FrameQueue(this.config.maxQueueDepth, ownDetectionEntry, (entry, reason) => {
|
|
5867
6343
|
if (entry.handle) this.config.releaseRetention?.(entry.handle, "drop");
|
|
5868
6344
|
if (entry.frame.frameRef) this.config.releaseFrameRef?.(entry.frame.frameRef, "drop");
|
|
6345
|
+
entry.onSettled?.(reason === "superseded" ? "superseded" : "lost");
|
|
5869
6346
|
});
|
|
5870
6347
|
const initialPhase = registration.detectionMode === "disabled" ? "idle" : registration.detectionMode === "always-on" ? "active" : "watching";
|
|
5871
6348
|
const preRollQueue = new PreRollLane({
|
|
@@ -5874,6 +6351,7 @@ var PipelineRunner = class {
|
|
|
5874
6351
|
dispose: (entry) => {
|
|
5875
6352
|
if (entry.handle) this.config.releaseRetention?.(entry.handle, "drop");
|
|
5876
6353
|
if (entry.frame.frameRef) this.config.releaseFrameRef?.(entry.frame.frameRef, "drop");
|
|
6354
|
+
entry.onSettled?.("lost");
|
|
5877
6355
|
}
|
|
5878
6356
|
});
|
|
5879
6357
|
this.logger?.info("camera registered: root detectors", {
|
|
@@ -5934,8 +6412,7 @@ var PipelineRunner = class {
|
|
|
5934
6412
|
}
|
|
5935
6413
|
if (state.phase === "active") this.detectionStreamHandler?.(deviceId, "unsubscribe");
|
|
5936
6414
|
state.motionQueue.clear();
|
|
5937
|
-
|
|
5938
|
-
state.preRollQueue.clear();
|
|
6415
|
+
this.discardQueuedDetectionFrames(deviceId, state, "detach");
|
|
5939
6416
|
this.timingSampler.clearDroppedFramesSource(deviceId);
|
|
5940
6417
|
this.cameras.delete(deviceId);
|
|
5941
6418
|
this.lastAdmittedAt.delete(deviceId);
|
|
@@ -6059,9 +6536,14 @@ var PipelineRunner = class {
|
|
|
6059
6536
|
preRollQueueDepth(deviceId) {
|
|
6060
6537
|
return this.cameras.get(deviceId)?.preRollQueue.size ?? 0;
|
|
6061
6538
|
}
|
|
6062
|
-
/**
|
|
6539
|
+
/**
|
|
6540
|
+
* Frames this camera lane REFUSED because it was full — or `null` when the
|
|
6541
|
+
* camera is no longer registered and the count cannot be read (a report
|
|
6542
|
+
* that lands after a detach). A failed read is `null`, never `0` (D393):
|
|
6543
|
+
* zero would say "nothing refused" about a lane nobody looked at.
|
|
6544
|
+
*/
|
|
6063
6545
|
preRollRefusedCount(deviceId) {
|
|
6064
|
-
return this.cameras.get(deviceId)?.preRollQueue.refused ??
|
|
6546
|
+
return this.cameras.get(deviceId)?.preRollQueue.refused ?? null;
|
|
6065
6547
|
}
|
|
6066
6548
|
/**
|
|
6067
6549
|
* Whether the node has a permit to spare for history right now — i.e. one
|
|
@@ -6090,7 +6572,7 @@ var PipelineRunner = class {
|
|
|
6090
6572
|
* decode worker). `runInference` already falls back to the pixel path
|
|
6091
6573
|
* when the handle is absent.
|
|
6092
6574
|
*/
|
|
6093
|
-
enqueueOccupancyFrame(deviceId, frame, handle) {
|
|
6575
|
+
enqueueOccupancyFrame(deviceId, frame, handle, onSettled) {
|
|
6094
6576
|
const state = this.cameras.get(deviceId);
|
|
6095
6577
|
if (!state) return false;
|
|
6096
6578
|
if (state.phase !== "watching") return false;
|
|
@@ -6098,7 +6580,8 @@ var PipelineRunner = class {
|
|
|
6098
6580
|
state.detectionQueue.enqueue({
|
|
6099
6581
|
frame,
|
|
6100
6582
|
handle,
|
|
6101
|
-
occasion: "burst"
|
|
6583
|
+
occasion: "burst",
|
|
6584
|
+
...onSettled ? { onSettled } : {}
|
|
6102
6585
|
});
|
|
6103
6586
|
return true;
|
|
6104
6587
|
}
|
|
@@ -6306,7 +6789,7 @@ var PipelineRunner = class {
|
|
|
6306
6789
|
}
|
|
6307
6790
|
transitionToWatching(deviceId, state, cooldownMs) {
|
|
6308
6791
|
state.phase = "watching";
|
|
6309
|
-
|
|
6792
|
+
this.discardQueuedDetectionFrames(deviceId, state, "session-close");
|
|
6310
6793
|
state.occupancyArmed = state.lastResultDetections > 0;
|
|
6311
6794
|
state.emptyBurstStreak = 0;
|
|
6312
6795
|
this.logger?.info("motion gate closed — phase=watching", {
|
|
@@ -6327,6 +6810,41 @@ var PipelineRunner = class {
|
|
|
6327
6810
|
state.lastArmedSource = null;
|
|
6328
6811
|
state.lastArmedRegions = void 0;
|
|
6329
6812
|
}
|
|
6813
|
+
/**
|
|
6814
|
+
* Throw away the frames still QUEUED for a camera whose session is ending.
|
|
6815
|
+
*
|
|
6816
|
+
* A branch that drops work says so (D391, D652). These are frames the runner
|
|
6817
|
+
* admitted and will now never infer — the live queue's latest frame and, on a
|
|
6818
|
+
* detach, whatever history is left in the pre-roll lane. Frames already in
|
|
6819
|
+
* flight are not touched here: they finish, and the decode worker they live
|
|
6820
|
+
* in drains before it is killed (`SESSION_DECODE_DRAIN_BOUND_MS`).
|
|
6821
|
+
*/
|
|
6822
|
+
discardQueuedDetectionFrames(deviceId, state, reason) {
|
|
6823
|
+
const queued = state.detectionQueue.size;
|
|
6824
|
+
const awaited = state.detectionQueue.peek()?.onSettled !== void 0 ? 1 : 0;
|
|
6825
|
+
const preRoll = reason === "detach" ? state.preRollQueue.size : 0;
|
|
6826
|
+
state.detectionQueue.clear();
|
|
6827
|
+
if (reason === "detach") state.preRollQueue.clear();
|
|
6828
|
+
if (queued + preRoll === 0) return;
|
|
6829
|
+
const meta = {
|
|
6830
|
+
reason,
|
|
6831
|
+
queued,
|
|
6832
|
+
awaited,
|
|
6833
|
+
preRoll,
|
|
6834
|
+
inflight: state.inflight
|
|
6835
|
+
};
|
|
6836
|
+
if (awaited + preRoll > 0) {
|
|
6837
|
+
this.logger?.info("detection frames discarded with the session", {
|
|
6838
|
+
tags: { deviceId },
|
|
6839
|
+
meta
|
|
6840
|
+
});
|
|
6841
|
+
return;
|
|
6842
|
+
}
|
|
6843
|
+
this.logger?.debug("detection frames discarded with the session", {
|
|
6844
|
+
tags: { deviceId },
|
|
6845
|
+
meta
|
|
6846
|
+
});
|
|
6847
|
+
}
|
|
6330
6848
|
tick() {
|
|
6331
6849
|
this.drainMotionQueues();
|
|
6332
6850
|
if (this.semaphore.available <= 0) {
|
|
@@ -6414,6 +6932,7 @@ var PipelineRunner = class {
|
|
|
6414
6932
|
} catch {}
|
|
6415
6933
|
if (handle) this.config.releaseRetention?.(handle, frameReleaseReason);
|
|
6416
6934
|
if (frame.frameRef) this.config.releaseFrameRef?.(frame.frameRef, frameReleaseReason);
|
|
6935
|
+
entry.onSettled?.(frameReleaseReason === "success" ? "inferred" : "lost");
|
|
6417
6936
|
}
|
|
6418
6937
|
}
|
|
6419
6938
|
/**
|
|
@@ -6495,6 +7014,7 @@ var PipelineRunner = class {
|
|
|
6495
7014
|
} catch {}
|
|
6496
7015
|
if (handle) this.config.releaseRetention?.(handle, frameReleaseReason);
|
|
6497
7016
|
if (frame.frameRef) this.config.releaseFrameRef?.(frame.frameRef, frameReleaseReason);
|
|
7017
|
+
entry.onSettled?.(frameReleaseReason === "success" ? "inferred" : "lost");
|
|
6498
7018
|
}
|
|
6499
7019
|
}
|
|
6500
7020
|
/**
|
|
@@ -7038,6 +7558,19 @@ function writeRetainedDetectionFrame(store, deviceId, frame, retention) {
|
|
|
7038
7558
|
*/
|
|
7039
7559
|
var OCCUPANCY_BURST_SAFETY_TIMEOUT_MS = 8e3;
|
|
7040
7560
|
/**
|
|
7561
|
+
* How long a burst whose collection has ended waits for the frames it admitted
|
|
7562
|
+
* to be INFERRED before it reports anyway (D652).
|
|
7563
|
+
*
|
|
7564
|
+
* A burst's verdict is read off its frames' results, so it completes when they
|
|
7565
|
+
* settle, not when they are enqueued. This bound is only for a frame that never
|
|
7566
|
+
* settles. Measured on the live hub 2026-09-26, the worst enqueue→emit of a
|
|
7567
|
+
* 10 s window was ≤1.5 s in 98.0% of 2 254 windows and ≤5 s in 98.9%; what lies
|
|
7568
|
+
* beyond is the inference-pool GPU hang, which the runner's own 20 s inference
|
|
7569
|
+
* timeout already ends. A timed-out burst counts its unsettled frames as lost
|
|
7570
|
+
* and says so on its `burst complete` line.
|
|
7571
|
+
*/
|
|
7572
|
+
var OCCUPANCY_BURST_SETTLE_BOUND_MS = 5e3;
|
|
7573
|
+
/**
|
|
7041
7574
|
* Freshness window for `lastMotionFrameAt` (Task 10 hardening). A non-null
|
|
7042
7575
|
* `attachment.motionUnsubscribe` only proves a subscription handle exists —
|
|
7043
7576
|
* NOT that frames are actually flowing (guarded-session-decode can return
|
|
@@ -7575,6 +8108,26 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7575
8108
|
benchFrameCache = /* @__PURE__ */ new Map();
|
|
7576
8109
|
benchFrameSweeper = null;
|
|
7577
8110
|
/**
|
|
8111
|
+
* Synthetic benches, start-then-poll (D646). One at a time on this runner —
|
|
8112
|
+
* the measurement owns the pool — and the last few finished ones kept for a
|
|
8113
|
+
* late poll. In RAM: a bench belongs to this process.
|
|
8114
|
+
*/
|
|
8115
|
+
syntheticBenches = new require_dist.BackgroundRunRegistry({
|
|
8116
|
+
maxConcurrent: 1,
|
|
8117
|
+
keepFinished: 5,
|
|
8118
|
+
onSettled: (status) => {
|
|
8119
|
+
const line = { meta: {
|
|
8120
|
+
sessionId: status.runId,
|
|
8121
|
+
state: status.state,
|
|
8122
|
+
error: status.error,
|
|
8123
|
+
runs: status.result?.runs ?? null,
|
|
8124
|
+
fps: status.result?.fps ?? null
|
|
8125
|
+
} };
|
|
8126
|
+
if (status.state === "failed") this.ctx.logger.warn("synthetic bench failed", line);
|
|
8127
|
+
else this.ctx.logger.info("synthetic bench finished", line);
|
|
8128
|
+
}
|
|
8129
|
+
});
|
|
8130
|
+
/**
|
|
7578
8131
|
* In-flight + recent capacity-ramp runs, keyed by sessionId. The ramp is
|
|
7579
8132
|
* fire-and-poll (a full run outlives the UDS addon-call 60s timeout), so
|
|
7580
8133
|
* completed results are kept here for `getCapacityRampStatus` to collect;
|
|
@@ -7775,7 +8328,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7775
8328
|
actionHandlers: {
|
|
7776
8329
|
cacheBenchFrame: async (input) => this.cacheBenchFrame(input),
|
|
7777
8330
|
releaseBenchFrame: async (input) => this.releaseBenchFrame(input),
|
|
7778
|
-
|
|
8331
|
+
startSyntheticBench: async (input) => this.startSyntheticBench(input),
|
|
8332
|
+
getSyntheticBenchStatus: async (input) => this.getSyntheticBenchStatus(input.sessionId),
|
|
8333
|
+
abortSyntheticBench: async (input) => ({ aborted: this.syntheticBenches.abort(input.sessionId) }),
|
|
7779
8334
|
runCapacityRamp: async (input) => this.runCapacityRampAction(input),
|
|
7780
8335
|
getCapacityRampStatus: async (input) => this.getCapacityRampStatusAction(input),
|
|
7781
8336
|
listCapacityRamps: async () => this.listCapacityRampsAction(),
|
|
@@ -7797,6 +8352,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7797
8352
|
this.benchFrameSweeper = null;
|
|
7798
8353
|
}
|
|
7799
8354
|
this.benchFrameCache.clear();
|
|
8355
|
+
this.syntheticBenches.abortAll();
|
|
7800
8356
|
for (const entry of this.capacityRuns.values()) entry.abortFlag.aborted = true;
|
|
7801
8357
|
this.capacityRuns.clear();
|
|
7802
8358
|
if (this.unsubMotionEvents) {
|
|
@@ -7824,8 +8380,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7824
8380
|
this.runner = null;
|
|
7825
8381
|
}
|
|
7826
8382
|
for (const attachment of this.attached.values()) {
|
|
7827
|
-
attachment.motionUnsubscribe?.();
|
|
7828
|
-
attachment.detectionUnsubscribe?.();
|
|
8383
|
+
attachment.motionUnsubscribe?.("immediate");
|
|
8384
|
+
attachment.detectionUnsubscribe?.("immediate");
|
|
7829
8385
|
}
|
|
7830
8386
|
for (const deviceId of [...this.parkedDetail.keys()]) this.dropParkedDetail(deviceId);
|
|
7831
8387
|
this.attached.clear();
|
|
@@ -7876,7 +8432,34 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7876
8432
|
const now = Date.now();
|
|
7877
8433
|
for (const [id, entry] of this.benchFrameCache) if (entry.expiresAt < now) this.benchFrameCache.delete(id);
|
|
7878
8434
|
}
|
|
7879
|
-
|
|
8435
|
+
/**
|
|
8436
|
+
* Validate the cheap preconditions synchronously (so a bad frame id is a
|
|
8437
|
+
* refusal the caller sees at once), then run the bench in the background
|
|
8438
|
+
* and answer with the receipt (D646).
|
|
8439
|
+
*/
|
|
8440
|
+
startSyntheticBench(input) {
|
|
8441
|
+
if (!this.ctx.api) throw new Error("startSyntheticBench: ctx.api unavailable");
|
|
8442
|
+
if (!this.benchFrameCache.has(input.frameId)) throw new Error(`startSyntheticBench: frameId ${input.frameId} not cached (call cacheBenchFrame first)`);
|
|
8443
|
+
this.syntheticBenches.start(input.sessionId, (run) => this.runSyntheticBench(input, run));
|
|
8444
|
+
return {
|
|
8445
|
+
accepted: true,
|
|
8446
|
+
sessionId: input.sessionId
|
|
8447
|
+
};
|
|
8448
|
+
}
|
|
8449
|
+
getSyntheticBenchStatus(sessionId) {
|
|
8450
|
+
const status = this.syntheticBenches.status(sessionId);
|
|
8451
|
+
if (status === null) return null;
|
|
8452
|
+
return {
|
|
8453
|
+
sessionId: status.runId,
|
|
8454
|
+
state: status.state,
|
|
8455
|
+
startedAtMs: status.startedAtMs,
|
|
8456
|
+
finishedAtMs: status.finishedAtMs,
|
|
8457
|
+
progress: status.progress,
|
|
8458
|
+
result: status.result,
|
|
8459
|
+
error: status.error
|
|
8460
|
+
};
|
|
8461
|
+
}
|
|
8462
|
+
async runSyntheticBench(input, run) {
|
|
7880
8463
|
const ctx = this.ctx;
|
|
7881
8464
|
const api = ctx.api;
|
|
7882
8465
|
if (!api) throw new Error("runSyntheticBench: ctx.api unavailable");
|
|
@@ -7926,8 +8509,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7926
8509
|
}, pin)).frameId;
|
|
7927
8510
|
await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
|
|
7928
8511
|
steps: stepsToRun,
|
|
7929
|
-
frame: sharedFrame
|
|
7930
|
-
...input.engine ? { engine: input.engine } : {}
|
|
8512
|
+
frame: sharedFrame
|
|
7931
8513
|
}) }, pin);
|
|
7932
8514
|
const warmupCount = input.warmup ?? 1;
|
|
7933
8515
|
for (let w = 0; w < warmupCount; w++) await api.pipelineExecutor.inferCached.mutate({
|
|
@@ -7941,11 +8523,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7941
8523
|
const batchSizes = [];
|
|
7942
8524
|
const detCounts = [];
|
|
7943
8525
|
let _n = 0;
|
|
7944
|
-
const sessionId = input.sessionId
|
|
8526
|
+
const sessionId = input.sessionId;
|
|
7945
8527
|
const totalRuns = input.parallel * input.iterations;
|
|
7946
8528
|
const wallStart = performance.now();
|
|
7947
8529
|
const worker = async () => {
|
|
7948
8530
|
for (let i = 0; i < input.iterations; i++) {
|
|
8531
|
+
if (run.signal.aborted) break;
|
|
7949
8532
|
const t0 = performance.now();
|
|
7950
8533
|
const result = await api.pipelineExecutor.inferCached.mutate({
|
|
7951
8534
|
stepId: rootStep.addonId,
|
|
@@ -7965,6 +8548,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
7965
8548
|
batchSizes.push(bs);
|
|
7966
8549
|
detCounts.push(dets);
|
|
7967
8550
|
const n = ++_n;
|
|
8551
|
+
run.report({
|
|
8552
|
+
runs: n,
|
|
8553
|
+
totalRuns
|
|
8554
|
+
});
|
|
7968
8555
|
if (n <= 20) ctx.logger.info("bench call trace (cached)", { meta: {
|
|
7969
8556
|
n,
|
|
7970
8557
|
wallMs: Math.round(wallMs),
|
|
@@ -8017,7 +8604,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8017
8604
|
await Promise.all(Array.from({ length: input.parallel }, () => worker()));
|
|
8018
8605
|
const wallSec = (performance.now() - wallStart) / 1e3;
|
|
8019
8606
|
await api.pipelineExecutor.uncacheFrame.mutate({ frameId: poolFrameId }, pin).catch(() => {});
|
|
8020
|
-
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "cached");
|
|
8607
|
+
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "cached", { nodeId: benchNode });
|
|
8021
8608
|
}
|
|
8022
8609
|
ctx.logger.info("synthetic bench: using full runPipeline path", { meta: {
|
|
8023
8610
|
steps: enabledSteps.length,
|
|
@@ -8031,7 +8618,6 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8031
8618
|
const result = await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
|
|
8032
8619
|
steps: stepsToRun,
|
|
8033
8620
|
frame: sharedFrame,
|
|
8034
|
-
...input.engine ? { engine: input.engine } : {},
|
|
8035
8621
|
...input.deviceKey ? { deviceKey: input.deviceKey } : {},
|
|
8036
8622
|
...input.plane ? { plane: input.plane } : {}
|
|
8037
8623
|
}) }, pin);
|
|
@@ -8059,11 +8645,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8059
8645
|
const predictTimings = [];
|
|
8060
8646
|
const batchSizes = [];
|
|
8061
8647
|
const detCounts = [];
|
|
8062
|
-
const sessionId = input.sessionId
|
|
8648
|
+
const sessionId = input.sessionId;
|
|
8063
8649
|
const totalRuns = input.parallel * input.iterations;
|
|
8064
8650
|
const wallStart = performance.now();
|
|
8065
8651
|
const worker = async () => {
|
|
8066
8652
|
for (let i = 0; i < input.iterations; i++) {
|
|
8653
|
+
if (run.signal.aborted) break;
|
|
8067
8654
|
const { wallMs, result } = await callOnce();
|
|
8068
8655
|
wallTimings.push(wallMs);
|
|
8069
8656
|
serverWallTimings.push(result.debug?.wallMs ?? 0);
|
|
@@ -8073,6 +8660,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8073
8660
|
batchSizes.push(result.debug?.batchSize ?? 1);
|
|
8074
8661
|
detCounts.push(result.detections?.length ?? 0);
|
|
8075
8662
|
const n = wallTimings.length;
|
|
8663
|
+
run.report({
|
|
8664
|
+
runs: n,
|
|
8665
|
+
totalRuns
|
|
8666
|
+
});
|
|
8076
8667
|
if (n % Math.max(1, input.parallel) === 0 && ctx.eventBus) {
|
|
8077
8668
|
const elapsed = (performance.now() - wallStart) / 1e3;
|
|
8078
8669
|
const fps = elapsed > 0 ? n / elapsed : 0;
|
|
@@ -8113,9 +8704,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8113
8704
|
};
|
|
8114
8705
|
await Promise.all(Array.from({ length: input.parallel }, () => worker()));
|
|
8115
8706
|
const wallSec = (performance.now() - wallStart) / 1e3;
|
|
8116
|
-
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "pipeline"
|
|
8707
|
+
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "pipeline", {
|
|
8708
|
+
nodeId: benchNode,
|
|
8709
|
+
...input.deviceKey ? { deviceKey: input.deviceKey } : {}
|
|
8710
|
+
});
|
|
8117
8711
|
}
|
|
8118
|
-
async buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, path) {
|
|
8712
|
+
async buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, path, target) {
|
|
8119
8713
|
const meanOfArr = (xs) => xs.length > 0 ? xs.reduce((s, v) => s + v, 0) / xs.length : 0;
|
|
8120
8714
|
this.ctx.logger.info("synthetic bench summary", { meta: {
|
|
8121
8715
|
runs: wallTimings.length,
|
|
@@ -8151,24 +8745,47 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8151
8745
|
batchSizeMean: Math.round(meanOf(batchSizes) * 100) / 100,
|
|
8152
8746
|
batchSizeMax: batchSizes.length > 0 ? Math.max(...batchSizes) : 0,
|
|
8153
8747
|
path,
|
|
8154
|
-
|
|
8748
|
+
pool: await this.readBenchPool(target)
|
|
8155
8749
|
};
|
|
8156
8750
|
}
|
|
8157
|
-
|
|
8751
|
+
/**
|
|
8752
|
+
* The pool a bench measured, and its provisioning — asked of the node it
|
|
8753
|
+
* was pinned to, for the device pool it named (D646). It used to ask the
|
|
8754
|
+
* node DEFAULT of whichever node answered, so an NPU bench was recorded
|
|
8755
|
+
* beside the onnx pool's tuning. A failed read is `null` fields and a line,
|
|
8756
|
+
* never a default (D393).
|
|
8757
|
+
*/
|
|
8758
|
+
async readBenchPool(target) {
|
|
8759
|
+
const unknown = {
|
|
8760
|
+
nodeId: target.nodeId,
|
|
8761
|
+
deviceKey: target.deviceKey ?? null,
|
|
8762
|
+
runtime: null,
|
|
8763
|
+
numWorkers: null,
|
|
8764
|
+
concurrency: null,
|
|
8765
|
+
batchMode: null,
|
|
8766
|
+
maxBatchSize: null
|
|
8767
|
+
};
|
|
8768
|
+
const api = this.ctx.api;
|
|
8769
|
+
if (!api) return unknown;
|
|
8770
|
+
const query = buildBenchTuningQuery(target);
|
|
8158
8771
|
try {
|
|
8159
|
-
const
|
|
8160
|
-
if (!api) return {};
|
|
8161
|
-
const [eng, tuning] = await Promise.all([api.pipelineExecutor.getSelectedEngine.query(), api.pipelineExecutor.getEffectiveTuning.query()]);
|
|
8772
|
+
const tuning = await api.pipelineExecutor.getEffectiveTuning.query({ ...query.input }, require_dist.nodePin(query.nodeId));
|
|
8162
8773
|
return {
|
|
8163
|
-
|
|
8164
|
-
|
|
8165
|
-
|
|
8166
|
-
|
|
8167
|
-
|
|
8168
|
-
|
|
8774
|
+
nodeId: target.nodeId,
|
|
8775
|
+
deviceKey: tuning.deviceKey,
|
|
8776
|
+
runtime: tuning.runtime,
|
|
8777
|
+
numWorkers: tuning.numWorkers,
|
|
8778
|
+
concurrency: tuning.concurrency,
|
|
8779
|
+
batchMode: tuning.batchMode,
|
|
8780
|
+
maxBatchSize: tuning.maxBatchSize
|
|
8169
8781
|
};
|
|
8170
|
-
} catch {
|
|
8171
|
-
|
|
8782
|
+
} catch (err) {
|
|
8783
|
+
this.ctx.logger.warn("synthetic bench: pool tuning read failed — reported as unknown", { meta: {
|
|
8784
|
+
nodeId: target.nodeId,
|
|
8785
|
+
deviceKey: target.deviceKey ?? null,
|
|
8786
|
+
error: require_dist.errMsg(err)
|
|
8787
|
+
} });
|
|
8788
|
+
return unknown;
|
|
8172
8789
|
}
|
|
8173
8790
|
}
|
|
8174
8791
|
/**
|
|
@@ -8291,7 +8908,6 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8291
8908
|
settleSec,
|
|
8292
8909
|
measureSec,
|
|
8293
8910
|
deviceKey: input.deviceKey ?? null,
|
|
8294
|
-
engine: input.engine ?? null,
|
|
8295
8911
|
contaminatedByActiveCameras,
|
|
8296
8912
|
steps: stepsToRun.map((s) => `${s.addonId}/${s.modelId}${s.enabled ? "" : "[off]"}`)
|
|
8297
8913
|
} });
|
|
@@ -8308,8 +8924,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8308
8924
|
},
|
|
8309
8925
|
plane: "frame",
|
|
8310
8926
|
sessionId,
|
|
8311
|
-
...input.deviceKey ? { deviceKey: input.deviceKey } : {}
|
|
8312
|
-
...input.engine ? { engine: input.engine } : {}
|
|
8927
|
+
...input.deviceKey ? { deviceKey: input.deviceKey } : {}
|
|
8313
8928
|
}, pin);
|
|
8314
8929
|
if (result.debug?.engine !== void 0) engineLabel = result.debug.engine;
|
|
8315
8930
|
if (result.shed === true) return {
|
|
@@ -9111,7 +9726,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
9111
9726
|
resolveChildSteps: (deviceId, parentClassName, filter) => this.resolveDetailChildSteps(deviceId, parentClassName, filter),
|
|
9112
9727
|
getNativeCrop: (deviceId, handle, paddedBbox) => this.getNativeCropForDetail(deviceId, handle, paddedBbox),
|
|
9113
9728
|
runPipeline: (steps, imageJpeg, deviceId, deviceKey, nativeCropRef) => this.runDetailPipeline(steps, imageJpeg, deviceId, deviceKey, nativeCropRef),
|
|
9114
|
-
|
|
9729
|
+
placeStep: (deviceId, step) => this.placeDetailStepForCamera(deviceId, step),
|
|
9115
9730
|
resolveCropConvention: () => convention,
|
|
9116
9731
|
logger: this.detailSubtreeLogger()
|
|
9117
9732
|
}, input);
|
|
@@ -9325,39 +9940,49 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
9325
9940
|
return { warn: (message, extras) => this.ctxIfReady?.logger.warn(message, extras) };
|
|
9326
9941
|
}
|
|
9327
9942
|
/**
|
|
9328
|
-
* Step-tree device jump
|
|
9329
|
-
*
|
|
9330
|
-
* cannot run the
|
|
9331
|
-
* `step.jumpDeviceKey` overrides.
|
|
9332
|
-
*
|
|
9333
|
-
*
|
|
9943
|
+
* Step-tree device jump — place one enrichment subtree on a same-node
|
|
9944
|
+
* inference device ({@link placeDetailStep}). AUTO-jumps only when the
|
|
9945
|
+
* camera's elected device cannot run the subtree (its format has no build);
|
|
9946
|
+
* an operator `step.jumpDeviceKey` overrides. A jumped subtree runs with the
|
|
9947
|
+
* TARGET device's provisioning carried on the roster (D650), never the
|
|
9948
|
+
* elected device's. `deviceKey: undefined` ⇒ the runner's per-camera default
|
|
9949
|
+
* pool (no elected device).
|
|
9334
9950
|
*
|
|
9335
|
-
*
|
|
9336
|
-
*
|
|
9337
|
-
* `edgetpu` root would otherwise drop faces/plates/clip with no signal).
|
|
9951
|
+
* Logs the jump ONCE per (camera, step, target), and — when NO enabled device
|
|
9952
|
+
* on the node can run the subtree — a LOUD warn once per (camera, step).
|
|
9338
9953
|
*/
|
|
9339
|
-
|
|
9954
|
+
placeDetailStepForCamera(deviceId, step) {
|
|
9340
9955
|
const config = this.attached.get(deviceId)?.config;
|
|
9341
|
-
const
|
|
9342
|
-
|
|
9343
|
-
|
|
9344
|
-
|
|
9345
|
-
|
|
9346
|
-
|
|
9347
|
-
|
|
9348
|
-
|
|
9349
|
-
|
|
9350
|
-
|
|
9351
|
-
|
|
9352
|
-
|
|
9353
|
-
|
|
9354
|
-
|
|
9355
|
-
|
|
9356
|
-
|
|
9357
|
-
|
|
9956
|
+
const placement = placeDetailStep({
|
|
9957
|
+
step,
|
|
9958
|
+
effectiveDeviceKey: config?.deviceKey,
|
|
9959
|
+
roster: config?.inferenceDevices,
|
|
9960
|
+
attachedByKey: this.perDeviceAttachedCounts(),
|
|
9961
|
+
getStepDef: tryGetStepDefinition
|
|
9962
|
+
});
|
|
9963
|
+
const decision = placement.decision;
|
|
9964
|
+
if (decision?.kind === "no-candidate") this.warnStepHasNoRunnableDevice(deviceId, step.addonId, decision.deviceKey);
|
|
9965
|
+
else if (decision?.kind === "jumped" && config?.deviceKey) this.logStepJumped(deviceId, config.deviceKey, decision, placement);
|
|
9966
|
+
return placement;
|
|
9967
|
+
}
|
|
9968
|
+
/** `${deviceId}:${addonId}:${target}` triples whose jump has been logged. */
|
|
9969
|
+
loggedStepJumps = /* @__PURE__ */ new Set();
|
|
9970
|
+
logStepJumped(deviceId, electedDeviceKey, decision, placement) {
|
|
9971
|
+
const step = placement.step;
|
|
9972
|
+
const key = `${deviceId}:${step.addonId}:${decision.deviceKey}`;
|
|
9973
|
+
if (this.loggedStepJumps.has(key)) return;
|
|
9974
|
+
this.loggedStepJumps.add(key);
|
|
9975
|
+
this.ctx?.logger.info("enrichment step runs on another device of this node", {
|
|
9976
|
+
tags: { deviceId },
|
|
9977
|
+
meta: {
|
|
9978
|
+
step: step.addonId,
|
|
9979
|
+
electedDeviceKey,
|
|
9980
|
+
targetDeviceKey: decision.deviceKey,
|
|
9981
|
+
reason: decision.reason,
|
|
9982
|
+
provisioned: placement.provisioned,
|
|
9983
|
+
modelId: step.modelId ?? null
|
|
9984
|
+
}
|
|
9358
9985
|
});
|
|
9359
|
-
if (result.kind === "no-candidate") this.warnStepHasNoRunnableDevice(deviceId, step.addonId, effectiveDeviceKey);
|
|
9360
|
-
return result.deviceKey;
|
|
9361
9986
|
}
|
|
9362
9987
|
/** Live attached-camera count per inference `deviceKey` on this runner. */
|
|
9363
9988
|
perDeviceAttachedCounts() {
|
|
@@ -9916,6 +10541,16 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
9916
10541
|
if (!admission) return void 0;
|
|
9917
10542
|
return () => admission.admit(deviceId, label);
|
|
9918
10543
|
}
|
|
10544
|
+
/**
|
|
10545
|
+
* The start's place in the node's queue of refused starts (D652) — taken by
|
|
10546
|
+
* the guarded start on its first refusal, kept until it is over. Spread into
|
|
10547
|
+
* `startGuardedSessionDecode`'s deps; empty when this node runs no ceiling.
|
|
10548
|
+
*/
|
|
10549
|
+
liveWorkerSlotWaiter(deviceId) {
|
|
10550
|
+
const admission = this.liveWorkers;
|
|
10551
|
+
if (!admission) return {};
|
|
10552
|
+
return { slotWaiter: () => admission.slotWaiter(deviceId) };
|
|
10553
|
+
}
|
|
9919
10554
|
buildDecodeSpawnAcquire(log, label) {
|
|
9920
10555
|
return async () => {
|
|
9921
10556
|
const gate = this.spawnGate;
|
|
@@ -9944,8 +10579,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
9944
10579
|
const teardown = startGuardedSessionDecode({
|
|
9945
10580
|
logger: log,
|
|
9946
10581
|
admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "detect"),
|
|
10582
|
+
...this.liveWorkerSlotWaiter(config.deviceId),
|
|
9947
10583
|
acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:detect:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config), true),
|
|
9948
|
-
startPump: (source) => {
|
|
10584
|
+
startPump: (source, hooks) => {
|
|
9949
10585
|
const dialMs = Date.now() - requestedAt;
|
|
9950
10586
|
const useLocalFrameRef = this.executorCoLocated() && process.env["CAMSTACK_DISABLE_LOCAL_FRAME_REF"] !== "1";
|
|
9951
10587
|
log.info("session-decode: detection routed through per-session decode worker (rgb)", { meta: {
|
|
@@ -9957,6 +10593,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
9957
10593
|
return startSessionDecodePump({
|
|
9958
10594
|
fork: this.forkDecodeWorkerWithLease,
|
|
9959
10595
|
lease: this.resolveSessionLeaseKnobs,
|
|
10596
|
+
onWorkerGone: hooks.onWorkerGone,
|
|
9960
10597
|
logger: log,
|
|
9961
10598
|
source,
|
|
9962
10599
|
opts: {
|
|
@@ -10038,10 +10675,30 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10038
10675
|
strideMs: report.strideMs,
|
|
10039
10676
|
subsampled: report.subsampled,
|
|
10040
10677
|
laneRefused: runner.preRollRefusedCount(config.deviceId),
|
|
10041
|
-
budgetFrames: this.preRollBudgetFor(config)
|
|
10678
|
+
budgetFrames: this.preRollBudgetFor(config),
|
|
10679
|
+
effectiveBudgetFrames: report.budgetFrames
|
|
10042
10680
|
}
|
|
10043
10681
|
});
|
|
10044
10682
|
},
|
|
10683
|
+
onPreRollSettled: (summary) => {
|
|
10684
|
+
const lost = summary.lostByReason.evicted + summary.lostByReason.cleared + summary.lostByReason["download-failed"] + summary.lostByReason.miss;
|
|
10685
|
+
const meta = {
|
|
10686
|
+
streamId: config.detectionStreamId,
|
|
10687
|
+
...summary,
|
|
10688
|
+
lost
|
|
10689
|
+
};
|
|
10690
|
+
if (lost > 0 || summary.unsettled > 0) {
|
|
10691
|
+
log.info("session-decode: pre-roll frames settled", {
|
|
10692
|
+
tags: { deviceId: config.deviceId },
|
|
10693
|
+
meta
|
|
10694
|
+
});
|
|
10695
|
+
return;
|
|
10696
|
+
}
|
|
10697
|
+
log.debug("session-decode: pre-roll frames settled", {
|
|
10698
|
+
tags: { deviceId: config.deviceId },
|
|
10699
|
+
meta
|
|
10700
|
+
});
|
|
10701
|
+
},
|
|
10045
10702
|
onFirstFrame: (timing) => {
|
|
10046
10703
|
const phaseActiveAt = this.lastMotionAt.get(config.deviceId);
|
|
10047
10704
|
log.info("session-decode: detection first frame", {
|
|
@@ -10085,8 +10742,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10085
10742
|
});
|
|
10086
10743
|
}
|
|
10087
10744
|
});
|
|
10088
|
-
return () => {
|
|
10089
|
-
teardown();
|
|
10745
|
+
return (mode) => {
|
|
10746
|
+
teardown(mode);
|
|
10090
10747
|
};
|
|
10091
10748
|
}
|
|
10092
10749
|
/**
|
|
@@ -10107,8 +10764,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10107
10764
|
return startGuardedSessionDecode({
|
|
10108
10765
|
logger: log,
|
|
10109
10766
|
admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "motion"),
|
|
10767
|
+
...this.liveWorkerSlotWaiter(config.deviceId),
|
|
10110
10768
|
acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.motionStreamId, `session-decode:motion:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
|
|
10111
|
-
startPump: (source) => {
|
|
10769
|
+
startPump: (source, hooks) => {
|
|
10112
10770
|
log.info("session-decode: motion routed through per-session decode worker (gray)", { meta: {
|
|
10113
10771
|
streamId: config.motionStreamId,
|
|
10114
10772
|
codec: source.codec
|
|
@@ -10116,6 +10774,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10116
10774
|
return startSessionDecodePump({
|
|
10117
10775
|
fork: this.forkDecodeWorkerWithLease,
|
|
10118
10776
|
lease: this.resolveSessionLeaseKnobs,
|
|
10777
|
+
onWorkerGone: hooks.onWorkerGone,
|
|
10119
10778
|
logger: log,
|
|
10120
10779
|
source,
|
|
10121
10780
|
opts: {
|
|
@@ -10783,15 +11442,36 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10783
11442
|
* found/report semantics can't drift between the two admission
|
|
10784
11443
|
* mechanisms.
|
|
10785
11444
|
*/
|
|
10786
|
-
finishOccupancyBurst(deviceId, log,
|
|
11445
|
+
finishOccupancyBurst(deviceId, log, summary) {
|
|
10787
11446
|
this.occupancyBurstInFlight.delete(deviceId);
|
|
10788
11447
|
const found = this.occupancyBurstFound.has(deviceId);
|
|
10789
11448
|
this.occupancyBurstFound.delete(deviceId);
|
|
11449
|
+
if (summary.inferred === 0) {
|
|
11450
|
+
log.info("occupancy re-check: burst complete — nothing inferred, no verdict", {
|
|
11451
|
+
tags: { deviceId },
|
|
11452
|
+
meta: {
|
|
11453
|
+
...summary,
|
|
11454
|
+
verdict: null
|
|
11455
|
+
}
|
|
11456
|
+
});
|
|
11457
|
+
return;
|
|
11458
|
+
}
|
|
10790
11459
|
this.runner?.reportOccupancyBurstOutcome(deviceId, found);
|
|
10791
|
-
|
|
10792
|
-
|
|
11460
|
+
const meta = {
|
|
11461
|
+
...summary,
|
|
10793
11462
|
found
|
|
10794
|
-
}
|
|
11463
|
+
};
|
|
11464
|
+
if (summary.lost > 0) {
|
|
11465
|
+
log.info("occupancy re-check: burst complete", {
|
|
11466
|
+
tags: { deviceId },
|
|
11467
|
+
meta
|
|
11468
|
+
});
|
|
11469
|
+
return;
|
|
11470
|
+
}
|
|
11471
|
+
log.debug("occupancy re-check: burst complete", {
|
|
11472
|
+
tags: { deviceId },
|
|
11473
|
+
meta
|
|
11474
|
+
});
|
|
10795
11475
|
}
|
|
10796
11476
|
/**
|
|
10797
11477
|
* Task 10: arm a TAP-based occupancy burst for a device whose analyzer
|
|
@@ -10827,21 +11507,30 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10827
11507
|
log.debug("occupancy re-check: tap safety timeout reached", { meta: { collected } });
|
|
10828
11508
|
this.finishOccupancyTap(deviceId);
|
|
10829
11509
|
}, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
|
|
11510
|
+
const ledger = new OccupancyBurstLedger({
|
|
11511
|
+
settleBoundMs: OCCUPANCY_BURST_SETTLE_BOUND_MS,
|
|
11512
|
+
onComplete: resolve
|
|
11513
|
+
});
|
|
10830
11514
|
this.occupancyTaps.set(deviceId, {
|
|
10831
11515
|
remaining: frames,
|
|
10832
11516
|
collected: 0,
|
|
10833
11517
|
safetyTimer,
|
|
11518
|
+
ledger,
|
|
10834
11519
|
resolve
|
|
10835
11520
|
});
|
|
10836
11521
|
});
|
|
10837
11522
|
}
|
|
10838
|
-
/**
|
|
11523
|
+
/**
|
|
11524
|
+
* Close the tap's COLLECTION. Idempotent: a no-op once the tap has already
|
|
11525
|
+
* closed (timer fired, or the budget already drained). The tap's promise
|
|
11526
|
+
* resolves later, from its ledger, once the frames it admitted have settled.
|
|
11527
|
+
*/
|
|
10839
11528
|
finishOccupancyTap(deviceId) {
|
|
10840
11529
|
const tap = this.occupancyTaps.get(deviceId);
|
|
10841
11530
|
if (!tap) return;
|
|
10842
11531
|
clearTimeout(tap.safetyTimer);
|
|
10843
11532
|
this.occupancyTaps.delete(deviceId);
|
|
10844
|
-
tap.
|
|
11533
|
+
tap.ledger.close();
|
|
10845
11534
|
}
|
|
10846
11535
|
/**
|
|
10847
11536
|
* Tee an arriving analyzer motion frame into occupancy-burst detection
|
|
@@ -10856,7 +11545,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10856
11545
|
teeOccupancyTapFrame(deviceId, frame, handle) {
|
|
10857
11546
|
const tap = this.occupancyTaps.get(deviceId);
|
|
10858
11547
|
if (!tap) return;
|
|
10859
|
-
|
|
11548
|
+
const settle = tap.ledger.admit();
|
|
11549
|
+
if (!this.runner?.enqueueOccupancyFrame(deviceId, frame, handle, settle)) settle("lost");
|
|
10860
11550
|
tap.collected++;
|
|
10861
11551
|
tap.remaining--;
|
|
10862
11552
|
if (tap.remaining <= 0) this.finishOccupancyTap(deviceId);
|
|
@@ -10876,9 +11566,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10876
11566
|
const lastFrameAt = this.lastMotionFrameAt.get(deviceId);
|
|
10877
11567
|
const motionFresh = lastFrameAt !== void 0 && Date.now() - lastFrameAt < MOTION_FRESHNESS_WINDOW_MS;
|
|
10878
11568
|
if (attachment.motionUnsubscribe !== null && motionFresh) {
|
|
10879
|
-
const
|
|
10880
|
-
if (
|
|
10881
|
-
this.finishOccupancyBurst(deviceId, log,
|
|
11569
|
+
const summary = await this.startOccupancyTap(deviceId, frames, log);
|
|
11570
|
+
if (summary.frames > 0) {
|
|
11571
|
+
this.finishOccupancyBurst(deviceId, log, summary);
|
|
10882
11572
|
return;
|
|
10883
11573
|
}
|
|
10884
11574
|
log.debug("occupancy re-check: tap collected 0 frames, falling back to dial", { meta: { frames } });
|
|
@@ -10909,76 +11599,91 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
10909
11599
|
fallback: alreadyInFlight
|
|
10910
11600
|
} });
|
|
10911
11601
|
let count = 0;
|
|
11602
|
+
/** Collection is over: no further frame is admitted to this burst. */
|
|
11603
|
+
let admissionClosed = false;
|
|
11604
|
+
/** The burst has reported its verdict (or its lack of one). */
|
|
10912
11605
|
let settled = false;
|
|
10913
11606
|
let teardown = null;
|
|
10914
|
-
const cleanup = () => {
|
|
11607
|
+
const cleanup = (summary) => {
|
|
10915
11608
|
if (settled) return;
|
|
10916
11609
|
settled = true;
|
|
10917
|
-
|
|
10918
|
-
|
|
11610
|
+
this.finishOccupancyBurst(deviceId, log, summary);
|
|
11611
|
+
};
|
|
11612
|
+
const ledger = new OccupancyBurstLedger({
|
|
11613
|
+
settleBoundMs: OCCUPANCY_BURST_SETTLE_BOUND_MS,
|
|
11614
|
+
onComplete: cleanup
|
|
11615
|
+
});
|
|
11616
|
+
const closeAdmission = () => {
|
|
11617
|
+
if (admissionClosed) return;
|
|
11618
|
+
admissionClosed = true;
|
|
11619
|
+
clearTimeout(safetyTimer);
|
|
11620
|
+
teardown?.("drain");
|
|
11621
|
+
ledger.close();
|
|
11622
|
+
};
|
|
11623
|
+
/** Offer one collected frame to the burst; `true` = the runner admitted it. */
|
|
11624
|
+
const admit = (frame, handle) => {
|
|
11625
|
+
const settle = ledger.admit();
|
|
11626
|
+
const admitted = runner.enqueueOccupancyFrame(deviceId, frame, handle, settle);
|
|
11627
|
+
if (!admitted) settle("lost");
|
|
11628
|
+
count++;
|
|
11629
|
+
if (count >= frames) closeAdmission();
|
|
11630
|
+
return admitted;
|
|
10919
11631
|
};
|
|
10920
11632
|
const safetyTimer = setTimeout(() => {
|
|
10921
11633
|
log.debug("occupancy re-check: safety timeout reached", { meta: { collected: count } });
|
|
10922
|
-
|
|
11634
|
+
closeAdmission();
|
|
10923
11635
|
}, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
|
|
10924
11636
|
teardown = startGuardedSessionDecode({
|
|
10925
11637
|
logger: log,
|
|
10926
11638
|
admitWorker: this.buildLiveWorkerAdmit(deviceId, "occupancy"),
|
|
11639
|
+
...this.liveWorkerSlotWaiter(deviceId),
|
|
10927
11640
|
acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:occupancy:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
|
|
10928
|
-
startPump: (source) => {
|
|
11641
|
+
startPump: (source, hooks) => {
|
|
10929
11642
|
const useLocalFrameRef = this.executorCoLocated() && process.env["CAMSTACK_DISABLE_LOCAL_FRAME_REF"] !== "1";
|
|
10930
11643
|
return startSessionDecodePump({
|
|
10931
11644
|
fork: this.forkDecodeWorkerWithLease,
|
|
10932
11645
|
lease: this.resolveSessionLeaseKnobs,
|
|
11646
|
+
onWorkerGone: hooks.onWorkerGone,
|
|
10933
11647
|
logger: log,
|
|
10934
11648
|
source,
|
|
10935
11649
|
opts: {
|
|
10936
11650
|
fps: config.detectionFps,
|
|
10937
11651
|
format: "rgb"
|
|
10938
11652
|
},
|
|
11653
|
+
shouldMaterializeFrame: () => !admissionClosed,
|
|
10939
11654
|
...useLocalFrameRef ? {
|
|
10940
11655
|
frameRegistry: require_default_detection_model.localFrameRegistry,
|
|
10941
11656
|
inlineWhen: () => runner.detectionBackpressure(config.deviceId),
|
|
10942
11657
|
inlineViewSpec: SPLIT_DETECTION_VIEW,
|
|
10943
11658
|
onFrameRef: (frame, nativeCrop) => {
|
|
10944
|
-
if (
|
|
11659
|
+
if (admissionClosed) return false;
|
|
10945
11660
|
const handle = writeRetainedDetectionFrame(retainedFrames, config.deviceId, frame, this.retention);
|
|
10946
11661
|
if (handle && nativeCrop) {
|
|
10947
11662
|
this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
|
|
10948
11663
|
this.bindFrameCutter(handle, nativeCrop);
|
|
10949
11664
|
}
|
|
10950
|
-
const admitted =
|
|
11665
|
+
const admitted = admit(frame, handle);
|
|
10951
11666
|
if (admitted) nativeCrop?.markInferred();
|
|
10952
|
-
count++;
|
|
10953
|
-
if (count >= frames) {
|
|
10954
|
-
clearTimeout(safetyTimer);
|
|
10955
|
-
cleanup();
|
|
10956
|
-
}
|
|
10957
11667
|
return admitted;
|
|
10958
11668
|
}
|
|
10959
11669
|
} : { inlineViewSpec: SPLIT_DETECTION_VIEW },
|
|
10960
11670
|
acquireSpawnSlot: this.buildDecodeSpawnAcquire(log, "occupancy"),
|
|
10961
|
-
onEnded: () =>
|
|
11671
|
+
onEnded: () => closeAdmission(),
|
|
10962
11672
|
onDecodedFrame: (frame, nativeCrop) => {
|
|
10963
|
-
if (
|
|
11673
|
+
if (admissionClosed) return;
|
|
10964
11674
|
const handle = writeRetainedDetectionFrame(retainedFrames, config.deviceId, frame, this.retention);
|
|
10965
11675
|
if (handle && nativeCrop) {
|
|
10966
11676
|
this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
|
|
10967
11677
|
this.bindFrameCutter(handle, nativeCrop);
|
|
10968
11678
|
}
|
|
10969
|
-
if (
|
|
10970
|
-
count++;
|
|
10971
|
-
if (count >= frames) {
|
|
10972
|
-
clearTimeout(safetyTimer);
|
|
10973
|
-
cleanup();
|
|
10974
|
-
}
|
|
11679
|
+
if (admit(frame, handle)) nativeCrop?.markInferred();
|
|
10975
11680
|
}
|
|
10976
11681
|
});
|
|
10977
11682
|
}
|
|
10978
11683
|
});
|
|
10979
|
-
if (
|
|
11684
|
+
if (admissionClosed) {
|
|
10980
11685
|
clearTimeout(safetyTimer);
|
|
10981
|
-
teardown();
|
|
11686
|
+
teardown("drain");
|
|
10982
11687
|
}
|
|
10983
11688
|
}
|
|
10984
11689
|
emitInferenceResult(deviceId, frame, result, handle, source, occasion) {
|