@camstack/addon-pipeline 1.2.294 → 1.2.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_MODELS.md +241 -0
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/{default-detection-model-0dPKRKUD.mjs → default-detection-model-Co578D8C.mjs} +181 -99
- package/dist/{default-detection-model-D24AJOTn.js → default-detection-model-D1daTtqT.js} +181 -99
- package/dist/detection-pipeline/index.js +1301 -531
- package/dist/detection-pipeline/index.mjs +1301 -531
- package/dist/{dist-CJR259Xf.js → dist-8up-f2TX.js} +3688 -2698
- package/dist/{dist-RXbmRAwP.mjs → dist-CCd0Q3nr.mjs} +3676 -2698
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-atmRSHPk.mjs → node-DgMSXSWP.mjs} +1 -1
- package/dist/{node-DWg9zbY1.js → node-lpQgHes9.js} +1 -1
- package/dist/pipeline-runner/index.js +975 -270
- package/dist/pipeline-runner/index.mjs +975 -270
- package/dist/{process-memory-BJUXvTjd.js → process-memory-CX_92V_r.js} +1 -1
- package/dist/{process-memory-BgFOHFnx.mjs → process-memory-DFC_O5zE.mjs} +1 -1
- package/dist/recorder/index.js +14 -6
- package/dist/recorder/index.mjs +14 -6
- package/dist/{segment-demux-js-C_fPJub3.js → segment-demux-js-DzBx6NN2.js} +1 -1
- package/dist/{segment-demux-js-G7wFpHzn.mjs → segment-demux-js-FZbBuk3F.mjs} +1 -1
- package/dist/session-decode/{decode-worker-child.js → decode-worker-main.js} +481 -72
- package/dist/session-decode/{decode-worker-child.mjs → decode-worker-main.mjs} +482 -71
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-6IyM-BIn.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C_i7oFBl.mjs} +2 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DUGQKsKL.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CkbplMHA.mjs +26 -0
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-BBYHWS3M.mjs → hostInit-BPtppL3W.mjs} +2 -2
- package/dist/stream-broker/index.js +4 -4
- package/dist/stream-broker/index.mjs +4 -4
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-B2MfQLlu.js → worker-protocol-C-G8qmye.js} +3 -1
- package/dist/{worker-protocol-C_W-P_g-.mjs → worker-protocol-D_NzPcnh.mjs} +3 -1
- package/package.json +3 -2
- package/python/inference_pool.py +422 -64
- package/python/postprocessors/__init__.py +2 -0
- package/python/postprocessors/ssd.py +73 -17
- package/python/postprocessors/test_ssd.py +205 -0
- package/python/postprocessors/test_yunet.py +292 -0
- package/python/postprocessors/testdata/ssdlite_mobiledet_outputs.json +1 -0
- package/python/postprocessors/yunet.py +275 -0
- package/python/test_inference_pool_compile_off_loop.py +414 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-6IHzlLJ_.mjs +0 -26
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DoyA71_q.mjs +0 -26
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-
|
|
3
|
-
import { a as localFrameRegistry, d as getStepDefinition, i as rootObjectGate, o as resolveFrameViewGeometry, p as RETIRED_MODEL_IDS, r as macroGateVerdict } from "../default-detection-model-
|
|
4
|
-
import { t as isWorkerReply } from "../worker-protocol-
|
|
1
|
+
import { D as MOTION_CLOSE_AFTER_MS, Dn as _enum, Ft as pipelineRunnerCapability, Ln as string, Mn as literal, Nn as number, Nt as pickNativeLeaseOverride, O as NativeLeaseAdmissionSchema, On as array, Pn as object, Sn as nodePin, cn as BaseAddon, gn as defineCustomActions, h as DEFAULT_DETAIL_CROP_CONVENTION, hn as customAction, i as BackgroundRunRegistry, jn as lazy, jt as pickDetailCropConvention, k as NativeLeaseSettingsSchema, kn as boolean, mn as createEvent, nt as deriveDetailCropRect, pt as failureContributionCapability, tn as errMsg, v as DEFAULT_NATIVE_LEASE_SETTINGS, vt as loadContributionCapability, w as FailureCounters, y as DEVICE_BACKEND_TO_FORMAT, zn as EventCategory } from "../dist-CCd0Q3nr.mjs";
|
|
2
|
+
import { i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-DgMSXSWP.mjs";
|
|
3
|
+
import { a as localFrameRegistry, d as getStepDefinition, i as rootObjectGate, o as resolveFrameViewGeometry, p as RETIRED_MODEL_IDS, r as macroGateVerdict } from "../default-detection-model-Co578D8C.mjs";
|
|
4
|
+
import { t as isWorkerReply } from "../worker-protocol-D_NzPcnh.mjs";
|
|
5
5
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
6
6
|
import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
|
|
7
7
|
import { c as withDetectionIntent, o as preRollRequestOf, t as DETECTION_PRE_ROLL_REQUEST_MS } from "../restream-intent-DyGmLAU9.mjs";
|
|
@@ -14,7 +14,7 @@ import { PerformanceObserver } from "node:perf_hooks";
|
|
|
14
14
|
import { attributeStall } from "@camstack/system/addon-utils";
|
|
15
15
|
//#region src/session-decode/fork-decode-worker.ts
|
|
16
16
|
/** Absolute path to the built decode-worker child entry (a vite build entry). */
|
|
17
|
-
var CHILD_ENTRY_PATH = fileURLToPath(new URL("../session-decode/decode-worker-
|
|
17
|
+
var CHILD_ENTRY_PATH = fileURLToPath(new URL("../session-decode/decode-worker-main.js", import.meta.url));
|
|
18
18
|
/**
|
|
19
19
|
* Known kill signals the coordinator may pass, mapped to their `NodeJS.Signals`
|
|
20
20
|
* literal so `ChildProcess.kill` receives a correctly-typed signal without a
|
|
@@ -35,21 +35,36 @@ var KILL_SIGNALS = {
|
|
|
35
35
|
* every listener added out here is one somebody can forget to remove.
|
|
36
36
|
*/
|
|
37
37
|
function wrapChild(child, onExit) {
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
let spawned = false;
|
|
39
|
+
let exitDelivered = false;
|
|
40
|
+
const exitListeners = [];
|
|
41
|
+
const deliverExit = (info) => {
|
|
42
|
+
if (exitDelivered) return;
|
|
43
|
+
exitDelivered = true;
|
|
44
|
+
onExit();
|
|
45
|
+
for (const listener of exitListeners) listener(info);
|
|
46
|
+
};
|
|
47
|
+
child.on("spawn", () => {
|
|
48
|
+
spawned = true;
|
|
49
|
+
});
|
|
50
|
+
child.on("exit", (code, signal) => deliverExit({
|
|
51
|
+
code: code ?? null,
|
|
52
|
+
signal: signal ?? null
|
|
53
|
+
}));
|
|
54
|
+
child.on("error", () => {
|
|
55
|
+
if (!spawned) deliverExit({
|
|
56
|
+
code: null,
|
|
57
|
+
signal: null
|
|
58
|
+
});
|
|
59
|
+
});
|
|
40
60
|
return {
|
|
41
61
|
send: (msg) => {
|
|
42
62
|
if (!child.connected) return;
|
|
43
63
|
child.send(msg, () => void 0);
|
|
44
64
|
},
|
|
45
65
|
on: (ev, cb) => {
|
|
46
|
-
if (ev === "exit")
|
|
47
|
-
|
|
48
|
-
code,
|
|
49
|
-
signal
|
|
50
|
-
});
|
|
51
|
-
child.on("exit", (code, signal) => cb(toInfo(code ?? null, signal ?? null)));
|
|
52
|
-
} else child.on("message", (m) => cb(m));
|
|
66
|
+
if (ev === "exit") exitListeners.push(cb);
|
|
67
|
+
else child.on("message", (m) => cb(m));
|
|
53
68
|
},
|
|
54
69
|
kill: (signal) => {
|
|
55
70
|
if (signal === void 0) {
|
|
@@ -250,58 +265,77 @@ var DEFAULT_ACQUIRE_RETRY_DELAYS_MS = [
|
|
|
250
265
|
15e3,
|
|
251
266
|
3e4
|
|
252
267
|
];
|
|
253
|
-
/**
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
* leaks the acquired restream refcount and, once the acquire resolves, forks
|
|
263
|
-
* an orphan decode worker nobody will ever stop.
|
|
264
|
-
*
|
|
265
|
-
* This helper returns the teardown SYNCHRONOUSLY. The acquire + pump-start
|
|
266
|
-
* run in the background; an `aborted` flag recorded by the (possibly
|
|
267
|
-
* already-called) teardown is checked right after each acquire resolves —
|
|
268
|
-
* if set, any restream is released immediately and the pump is NEVER
|
|
269
|
-
* started (no orphan worker). Otherwise the pump starts and the teardown,
|
|
270
|
-
* once called, stops the pump then releases the restream.
|
|
271
|
-
*
|
|
272
|
-
* **A failed acquire is retried, not swallowed.** The teardown handed back
|
|
273
|
-
* here is stored by the caller (as `attachment.detectionUnsubscribe`) the
|
|
274
|
-
* instant this returns, so a `return` on failure left that handle permanently
|
|
275
|
-
* non-null with no pump behind it: `onEnded` could never fire because nothing
|
|
276
|
-
* had ever started, the camera's phase stayed `active`, and not one frame
|
|
277
|
-
* arrived again. The motion path was hardened against this shape by a frame
|
|
278
|
-
* freshness window (`MOTION_FRESHNESS_WINDOW_MS`); this is the detection
|
|
279
|
-
* path's half.
|
|
280
|
-
*/
|
|
268
|
+
/** A restream release that runs once however many paths reach it. */
|
|
269
|
+
function onceRelease(release) {
|
|
270
|
+
let done = false;
|
|
271
|
+
return () => {
|
|
272
|
+
if (done) return Promise.resolve();
|
|
273
|
+
done = true;
|
|
274
|
+
return release();
|
|
275
|
+
};
|
|
276
|
+
}
|
|
281
277
|
function startGuardedSessionDecode(deps) {
|
|
282
278
|
let aborted = false;
|
|
279
|
+
/** The mode the teardown asked for — replayed if it raced `startPump`. */
|
|
280
|
+
let abortMode = "drain";
|
|
283
281
|
let stopPump = null;
|
|
284
282
|
let release = null;
|
|
285
283
|
let retryTimer = null;
|
|
286
284
|
/** The live-worker slot held by the RUNNING session, released on teardown. */
|
|
287
285
|
let workerLease = null;
|
|
288
286
|
const delays = deps.acquireRetryDelaysMs && deps.acquireRetryDelaysMs.length > 0 ? deps.acquireRetryDelaysMs : DEFAULT_ACQUIRE_RETRY_DELAYS_MS;
|
|
289
|
-
/**
|
|
290
|
-
|
|
287
|
+
/** Ends a parked start's wait when the session is torn down. */
|
|
288
|
+
let abortWait = null;
|
|
289
|
+
/**
|
|
290
|
+
* This start's place in the ceiling queue — taken on its first refusal, kept
|
|
291
|
+
* to the end so a re-park keeps the FIFO position (D652).
|
|
292
|
+
*/
|
|
293
|
+
let waiter = null;
|
|
294
|
+
/**
|
|
295
|
+
* The start was handed a returned slot and has not yet ASKED for it. Set on
|
|
296
|
+
* a wake, cleared by the next `admitWorker` call; a start torn down while
|
|
297
|
+
* this is set passes the slot on (`SlotWaiter.leave(true)`), or nobody gets
|
|
298
|
+
* it and every other parked start waits out its full timer.
|
|
299
|
+
*/
|
|
300
|
+
let wakeInHand = false;
|
|
301
|
+
/**
|
|
302
|
+
* Resolves after the backoff for `attempt` (`'timer'`) — or, when `parked`,
|
|
303
|
+
* as soon as the node hands this start a returned slot (`'woken'`) or the
|
|
304
|
+
* session is torn down (`'torn-down'`), whichever is first. The start leaves
|
|
305
|
+
* the queue on every exit, so a torn-down start leaves no listener behind.
|
|
306
|
+
*/
|
|
307
|
+
const waitBackoff = (attempt, parked) => new Promise((resolve) => {
|
|
291
308
|
const ms = delays[Math.min(attempt, delays.length - 1)] ?? delays[delays.length - 1] ?? 2e3;
|
|
292
|
-
|
|
293
|
-
retryTimer
|
|
294
|
-
|
|
295
|
-
|
|
309
|
+
const done = (why) => {
|
|
310
|
+
if (retryTimer !== null) {
|
|
311
|
+
clearTimeout(retryTimer);
|
|
312
|
+
retryTimer = null;
|
|
313
|
+
}
|
|
314
|
+
parked?.cancel();
|
|
315
|
+
abortWait = null;
|
|
316
|
+
resolve(why);
|
|
317
|
+
};
|
|
318
|
+
retryTimer = setTimeout(() => done("timer"), ms);
|
|
296
319
|
retryTimer.unref?.();
|
|
320
|
+
abortWait = () => done("torn-down");
|
|
321
|
+
if (parked) parked.wait(() => done("woken"));
|
|
297
322
|
});
|
|
298
|
-
|
|
323
|
+
const releaseWorkerSlot = () => {
|
|
324
|
+
workerLease?.release();
|
|
325
|
+
workerLease = null;
|
|
326
|
+
};
|
|
327
|
+
const startLoop = async () => {
|
|
299
328
|
for (let attempt = 0; !aborted; attempt++) {
|
|
300
329
|
let lease = null;
|
|
301
330
|
if (deps.admitWorker) {
|
|
331
|
+
wakeInHand = false;
|
|
302
332
|
lease = deps.admitWorker();
|
|
303
333
|
if (!lease) {
|
|
304
|
-
|
|
334
|
+
waiter ??= deps.slotWaiter?.() ?? null;
|
|
335
|
+
if (await waitBackoff(attempt, waiter) === "woken") {
|
|
336
|
+
attempt -= 1;
|
|
337
|
+
wakeInHand = true;
|
|
338
|
+
}
|
|
305
339
|
continue;
|
|
306
340
|
}
|
|
307
341
|
}
|
|
@@ -314,7 +348,7 @@ function startGuardedSessionDecode(deps) {
|
|
|
314
348
|
attempt: attempt + 1,
|
|
315
349
|
error: errMsg(err)
|
|
316
350
|
} });
|
|
317
|
-
await waitBackoff(attempt);
|
|
351
|
+
await waitBackoff(attempt, null);
|
|
318
352
|
continue;
|
|
319
353
|
}
|
|
320
354
|
if (aborted) {
|
|
@@ -325,27 +359,47 @@ function startGuardedSessionDecode(deps) {
|
|
|
325
359
|
if (!acquired) {
|
|
326
360
|
lease?.release();
|
|
327
361
|
(attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: no broker profile bound yet — retrying", { meta: { attempt: attempt + 1 } });
|
|
328
|
-
await waitBackoff(attempt);
|
|
362
|
+
await waitBackoff(attempt, null);
|
|
329
363
|
continue;
|
|
330
364
|
}
|
|
331
365
|
if (attempt > 0) deps.logger.info("session-decode: acquire succeeded after retry", { meta: { attempts: attempt + 1 } });
|
|
332
|
-
release = acquired.release;
|
|
366
|
+
release = onceRelease(acquired.release);
|
|
333
367
|
workerLease = lease;
|
|
334
|
-
|
|
368
|
+
try {
|
|
369
|
+
stopPump = deps.startPump(acquired.source, { onWorkerGone: releaseWorkerSlot });
|
|
370
|
+
} catch (err) {
|
|
371
|
+
releaseWorkerSlot();
|
|
372
|
+
const restream = release;
|
|
373
|
+
release = null;
|
|
374
|
+
restream?.();
|
|
375
|
+
throw err;
|
|
376
|
+
}
|
|
377
|
+
if (aborted) stopPump(abortMode);
|
|
335
378
|
return;
|
|
336
379
|
}
|
|
337
|
-
}
|
|
380
|
+
};
|
|
381
|
+
startLoop().catch((err) => {
|
|
338
382
|
deps.logger.warn("session-decode: guarded start failed", { meta: { error: errMsg(err) } });
|
|
383
|
+
}).finally(() => {
|
|
384
|
+
waiter?.leave(wakeInHand);
|
|
385
|
+
waiter = null;
|
|
339
386
|
});
|
|
340
|
-
return () => {
|
|
387
|
+
return (mode = "drain") => {
|
|
388
|
+
if (aborted) {
|
|
389
|
+
if (mode === "immediate" && abortMode !== "immediate") {
|
|
390
|
+
abortMode = mode;
|
|
391
|
+
stopPump?.(mode);
|
|
392
|
+
}
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
341
395
|
aborted = true;
|
|
396
|
+
abortMode = mode;
|
|
342
397
|
if (retryTimer !== null) {
|
|
343
398
|
clearTimeout(retryTimer);
|
|
344
399
|
retryTimer = null;
|
|
345
400
|
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
workerLease = null;
|
|
401
|
+
abortWait?.();
|
|
402
|
+
stopPump?.(mode);
|
|
349
403
|
if (release) release();
|
|
350
404
|
};
|
|
351
405
|
}
|
|
@@ -414,6 +468,21 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
414
468
|
limit;
|
|
415
469
|
logger;
|
|
416
470
|
live = 0;
|
|
471
|
+
/**
|
|
472
|
+
* Starts the ceiling refused, waiting to be told a slot came back — FIFO by
|
|
473
|
+
* the start's FIRST park (D652): the key's `seq` is fixed when the start
|
|
474
|
+
* takes its {@link SlotWaiter}, so a re-park keeps the place.
|
|
475
|
+
*/
|
|
476
|
+
parked = /* @__PURE__ */ new Map();
|
|
477
|
+
nextSeq = 0;
|
|
478
|
+
/**
|
|
479
|
+
* Cameras currently refused. The refusal WARN fires on the transition into
|
|
480
|
+
* this set, not on every retry: a start woken or retried while the node is
|
|
481
|
+
* still full says so at debug (D652 — a wake must not become a WARN storm).
|
|
482
|
+
* Cleared on admit AND when a parked start leaves (`SlotWaiter.leave`): a
|
|
483
|
+
* camera torn down while refused must be news again next time (D391).
|
|
484
|
+
*/
|
|
485
|
+
refusedDevices = /* @__PURE__ */ new Set();
|
|
417
486
|
nodeBudgetMb;
|
|
418
487
|
workerFootprintMb;
|
|
419
488
|
constructor(options) {
|
|
@@ -422,6 +491,48 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
422
491
|
this.nodeBudgetMb = positiveOrUndefined(options.nodeBudgetMb);
|
|
423
492
|
this.workerFootprintMb = positiveOrUndefined(options.workerFootprintMb);
|
|
424
493
|
}
|
|
494
|
+
/** Starts parked on a {@link SlotWaiter} right now — diagnostics and tests. */
|
|
495
|
+
get slotWaiters() {
|
|
496
|
+
return this.parked.size;
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* This start's place in the queue. A refused start waits on it instead of a
|
|
500
|
+
* fixed timer, so a slot freed by a draining worker's exit is taken at once
|
|
501
|
+
* (D652). Take it once per start, on the first refusal; keep it to the end.
|
|
502
|
+
*/
|
|
503
|
+
slotWaiter(deviceId) {
|
|
504
|
+
const key = {
|
|
505
|
+
seq: this.nextSeq,
|
|
506
|
+
deviceId
|
|
507
|
+
};
|
|
508
|
+
this.nextSeq += 1;
|
|
509
|
+
return {
|
|
510
|
+
wait: (listener) => {
|
|
511
|
+
this.parked.set(key, listener);
|
|
512
|
+
},
|
|
513
|
+
cancel: () => {
|
|
514
|
+
this.parked.delete(key);
|
|
515
|
+
},
|
|
516
|
+
leave: (unusedWake) => {
|
|
517
|
+
this.parked.delete(key);
|
|
518
|
+
this.refusedDevices.delete(deviceId);
|
|
519
|
+
if (unusedWake) this.wakeNext();
|
|
520
|
+
}
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
/**
|
|
524
|
+
* ONE slot, ONE wake: the longest-parked start (lowest `seq`). Waking all N
|
|
525
|
+
* produced N−1 refusals per freed slot. The woken start leaves the queue; if
|
|
526
|
+
* something else took the slot first it parks again, at the same place.
|
|
527
|
+
*/
|
|
528
|
+
wakeNext() {
|
|
529
|
+
let first;
|
|
530
|
+
for (const key of this.parked.keys()) if (first === void 0 || key.seq < first.seq) first = key;
|
|
531
|
+
if (first === void 0) return;
|
|
532
|
+
const wake = this.parked.get(first);
|
|
533
|
+
this.parked.delete(first);
|
|
534
|
+
wake?.();
|
|
535
|
+
}
|
|
425
536
|
/** Workers currently holding a slot. */
|
|
426
537
|
get inFlight() {
|
|
427
538
|
return this.live;
|
|
@@ -478,7 +589,9 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
478
589
|
admit(deviceId, label) {
|
|
479
590
|
const ceiling = this.ceiling;
|
|
480
591
|
if (this.live >= ceiling) {
|
|
481
|
-
this.
|
|
592
|
+
const firstRefusal = !this.refusedDevices.has(deviceId);
|
|
593
|
+
this.refusedDevices.add(deviceId);
|
|
594
|
+
(firstRefusal ? this.logger.warn : this.logger.debug).call(this.logger, "session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
|
|
482
595
|
tags: { deviceId },
|
|
483
596
|
meta: {
|
|
484
597
|
label,
|
|
@@ -494,6 +607,7 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
494
607
|
return null;
|
|
495
608
|
}
|
|
496
609
|
this.live += 1;
|
|
610
|
+
this.refusedDevices.delete(deviceId);
|
|
497
611
|
this.logger.debug("session-decode: live decode-worker slot taken", {
|
|
498
612
|
tags: { deviceId },
|
|
499
613
|
meta: {
|
|
@@ -507,6 +621,7 @@ var LiveDecodeWorkerAdmission = class {
|
|
|
507
621
|
if (released) return;
|
|
508
622
|
released = true;
|
|
509
623
|
this.live -= 1;
|
|
624
|
+
this.wakeNext();
|
|
510
625
|
} };
|
|
511
626
|
}
|
|
512
627
|
};
|
|
@@ -1443,6 +1558,29 @@ function cropRgb(frame, bbox, maxWidth) {
|
|
|
1443
1558
|
/** How long to wait after `kill()` before escalating to `kill('SIGKILL')`. */
|
|
1444
1559
|
var SIGKILL_FALLBACK_MS = 1e3;
|
|
1445
1560
|
/**
|
|
1561
|
+
* The longest a planned teardown keeps a session's worker alive so the frames
|
|
1562
|
+
* it already handed out can be read (D652).
|
|
1563
|
+
*
|
|
1564
|
+
* What is being waited for is the REST of a frame's trip through the runner —
|
|
1565
|
+
* semaphore wait + inference + result — for frames that were admitted before
|
|
1566
|
+
* the detach. Measured on the live hub 2026-09-26 over one hour of
|
|
1567
|
+
* `pipeline stats` (2 254 per-camera 10 s windows): the WORST enqueue→emit in a
|
|
1568
|
+
* window was ≤1 000 ms in 96.7% of windows and ≤1 500 ms in 98.0%; going to
|
|
1569
|
+
* 5 000 ms buys only another 0.9%, and that tail is the inference-pool GPU hang,
|
|
1570
|
+
* which no wait here can rescue. The cost of waiting is one idle worker (its
|
|
1571
|
+
* decode already stopped by `drain`) for at most this long, per detach.
|
|
1572
|
+
*
|
|
1573
|
+
* A frame still held when this expires is not dropped silently: the
|
|
1574
|
+
* coordinator logs the count, tagged with the camera, and then kills.
|
|
1575
|
+
*/
|
|
1576
|
+
var SESSION_DECODE_DRAIN_BOUND_MS = 1500;
|
|
1577
|
+
/**
|
|
1578
|
+
* The miss reason for a native crop requested after its session's teardown.
|
|
1579
|
+
* Only crops already SENT when the drain began are waited for (D652); a later
|
|
1580
|
+
* one would ask a worker that is about to be killed.
|
|
1581
|
+
*/
|
|
1582
|
+
var NATIVE_CROP_REFUSED_DRAINING = "session draining — crop requested after teardown";
|
|
1583
|
+
/**
|
|
1446
1584
|
* How often the cold-start wait REPORTS that it is still waiting. Task 0
|
|
1447
1585
|
* live-measured cold-start (restream dial + hwaccel context + keyframe wait) at
|
|
1448
1586
|
* 5-8s, so this must tolerate a slow-but-healthy start.
|
|
@@ -1514,30 +1652,38 @@ var SessionDecodeSession = class {
|
|
|
1514
1652
|
firstFrameTimeoutMs;
|
|
1515
1653
|
frameTimeoutMs;
|
|
1516
1654
|
firstFrameMaxWaitMs;
|
|
1517
|
-
|
|
1518
|
-
onPreRollReport;
|
|
1655
|
+
hooks;
|
|
1519
1656
|
pendingPull = null;
|
|
1520
1657
|
pendingToBuffers = /* @__PURE__ */ new Map();
|
|
1521
1658
|
/** Outstanding `nativeCrop` requests, keyed by the monotonic `requestId`. */
|
|
1522
1659
|
pendingNativeCrops = /* @__PURE__ */ new Map();
|
|
1523
1660
|
nextNativeCropId = 1;
|
|
1524
1661
|
exited = false;
|
|
1525
|
-
/** Set by teardown()
|
|
1662
|
+
/** Set by teardown() — the session is ending, whether or not it drains first. */
|
|
1526
1663
|
stopRequested = false;
|
|
1664
|
+
/** Set when `kill()` is sent, so handleExit can tell an intentional stop from a death. */
|
|
1665
|
+
killSent = false;
|
|
1666
|
+
/**
|
|
1667
|
+
* Images yielded to the consumer and not yet closed. Each is a frame the
|
|
1668
|
+
* consumer still holds a reference to — queued, in flight, or being read —
|
|
1669
|
+
* and each dies with the worker. This is what a drain waits on.
|
|
1670
|
+
*/
|
|
1671
|
+
openImages = 0;
|
|
1672
|
+
/** Armed while a drain waits; its expiry kills with frames still held. */
|
|
1673
|
+
drainTimer = null;
|
|
1527
1674
|
sigkillTimer = null;
|
|
1528
1675
|
pullTimeoutTimer = null;
|
|
1529
1676
|
firstFrameReceived = false;
|
|
1530
1677
|
/** `Date.now()` of the FIRST `pull` — the origin of the cold-start ceiling. */
|
|
1531
1678
|
firstPullAt = 0;
|
|
1532
|
-
constructor(child, logger, deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs,
|
|
1679
|
+
constructor(child, logger, deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, hooks) {
|
|
1533
1680
|
this.child = child;
|
|
1534
1681
|
this.logger = logger;
|
|
1535
1682
|
this.deviceId = deviceId;
|
|
1536
1683
|
this.firstFrameTimeoutMs = firstFrameTimeoutMs;
|
|
1537
1684
|
this.frameTimeoutMs = frameTimeoutMs;
|
|
1538
1685
|
this.firstFrameMaxWaitMs = firstFrameMaxWaitMs;
|
|
1539
|
-
this.
|
|
1540
|
-
this.onPreRollReport = onPreRollReport;
|
|
1686
|
+
this.hooks = hooks;
|
|
1541
1687
|
this.child.on("message", (m) => this.handleMessage(m));
|
|
1542
1688
|
this.child.on("exit", (m) => this.handleExit(toWorkerExitInfo(m)));
|
|
1543
1689
|
}
|
|
@@ -1602,6 +1748,10 @@ var SessionDecodeSession = class {
|
|
|
1602
1748
|
*/
|
|
1603
1749
|
requestNativeCrop(frameId, bbox, maxWidth) {
|
|
1604
1750
|
if (this.exited) return Promise.resolve({ kind: "miss" });
|
|
1751
|
+
if (this.stopRequested) return Promise.resolve({
|
|
1752
|
+
kind: "miss",
|
|
1753
|
+
workerReason: NATIVE_CROP_REFUSED_DRAINING
|
|
1754
|
+
});
|
|
1605
1755
|
const requestId = this.nextNativeCropId++;
|
|
1606
1756
|
return new Promise((resolve) => {
|
|
1607
1757
|
this.pendingNativeCrops.set(requestId, { resolve });
|
|
@@ -1616,6 +1766,7 @@ var SessionDecodeSession = class {
|
|
|
1616
1766
|
} catch {
|
|
1617
1767
|
this.pendingNativeCrops.delete(requestId);
|
|
1618
1768
|
resolve({ kind: "miss" });
|
|
1769
|
+
this.settleDrainIfDone();
|
|
1619
1770
|
}
|
|
1620
1771
|
});
|
|
1621
1772
|
}
|
|
@@ -1660,20 +1811,135 @@ var SessionDecodeSession = class {
|
|
|
1660
1811
|
frameId
|
|
1661
1812
|
});
|
|
1662
1813
|
}
|
|
1663
|
-
/**
|
|
1664
|
-
|
|
1814
|
+
/** A yielded image the consumer now holds — see {@link openImages}. */
|
|
1815
|
+
noteImageOpened() {
|
|
1816
|
+
this.openImages += 1;
|
|
1817
|
+
}
|
|
1818
|
+
/** The consumer let go of a yielded image; a waiting drain may now be done. */
|
|
1819
|
+
noteImageClosed() {
|
|
1820
|
+
this.openImages = Math.max(0, this.openImages - 1);
|
|
1821
|
+
this.settleDrainIfDone();
|
|
1822
|
+
}
|
|
1823
|
+
/**
|
|
1824
|
+
* End the session.
|
|
1825
|
+
*
|
|
1826
|
+
* `'drain'` with frames still held: tell the worker to stop decoding (it
|
|
1827
|
+
* keeps every frame it holds), and kill only once every yielded image has
|
|
1828
|
+
* been closed and every read answered — or when
|
|
1829
|
+
* {@link SESSION_DECODE_DRAIN_BOUND_MS} expires, which is logged. Nothing
|
|
1830
|
+
* held: `stop` + SIGTERM now. `'immediate'`: SIGKILL now.
|
|
1831
|
+
*
|
|
1832
|
+
* A SECOND call that says `'immediate'` while a drain is in progress
|
|
1833
|
+
* ESCALATES (D652): addon shutdown lands on sessions already draining from
|
|
1834
|
+
* a detach, and the runner exits right after — a drain it does not wait for
|
|
1835
|
+
* is an orphaned worker holding its GPU pool. Any other repeat is ignored.
|
|
1836
|
+
*/
|
|
1837
|
+
teardown(mode) {
|
|
1838
|
+
if (this.stopRequested) {
|
|
1839
|
+
if (mode === "immediate") this.escalateToImmediate();
|
|
1840
|
+
return;
|
|
1841
|
+
}
|
|
1665
1842
|
this.stopRequested = true;
|
|
1666
1843
|
this.clearPullTimeout();
|
|
1844
|
+
if (this.exited) return;
|
|
1845
|
+
if (mode === "drain" && this.unsettledCount() > 0) {
|
|
1846
|
+
this.trySend({ kind: "drain" });
|
|
1847
|
+
this.drainTimer = setTimeout(() => this.handleDrainBoundExpired(), SESSION_DECODE_DRAIN_BOUND_MS);
|
|
1848
|
+
this.drainTimer.unref?.();
|
|
1849
|
+
return;
|
|
1850
|
+
}
|
|
1851
|
+
this.killNow(mode);
|
|
1852
|
+
}
|
|
1853
|
+
/** The drain (or a SIGTERM still waiting on its fallback) gives way to SIGKILL now. */
|
|
1854
|
+
escalateToImmediate() {
|
|
1855
|
+
if (this.exited) return;
|
|
1856
|
+
if (this.drainTimer !== null) {
|
|
1857
|
+
clearTimeout(this.drainTimer);
|
|
1858
|
+
this.drainTimer = null;
|
|
1859
|
+
}
|
|
1860
|
+
if (!this.killSent) {
|
|
1861
|
+
this.killNow("immediate");
|
|
1862
|
+
return;
|
|
1863
|
+
}
|
|
1864
|
+
if (this.sigkillTimer !== null) {
|
|
1865
|
+
clearTimeout(this.sigkillTimer);
|
|
1866
|
+
this.sigkillTimer = null;
|
|
1867
|
+
}
|
|
1868
|
+
this.tryKill("SIGKILL");
|
|
1869
|
+
}
|
|
1870
|
+
/**
|
|
1871
|
+
* Frames the consumer still holds, reads still waiting on the worker, and
|
|
1872
|
+
* native crops already REQUESTED. A crop asked for before the teardown
|
|
1873
|
+
* request is waited for; one asked for after it is refused by name — post-analysis's
|
|
1874
|
+
* ordinary crop arrives 2.2–9.9 s after the frame, past any bound worth
|
|
1875
|
+
* holding a worker for, and is served from subject tiles that die with the
|
|
1876
|
+
* worker either way.
|
|
1877
|
+
*/
|
|
1878
|
+
unsettledCount() {
|
|
1879
|
+
return this.openImages + this.pendingReadCount() + this.pendingNativeCrops.size;
|
|
1880
|
+
}
|
|
1881
|
+
pendingReadCount() {
|
|
1882
|
+
let n = 0;
|
|
1883
|
+
for (const waiters of this.pendingToBuffers.values()) n += waiters.length;
|
|
1884
|
+
return n;
|
|
1885
|
+
}
|
|
1886
|
+
settleDrainIfDone() {
|
|
1887
|
+
if (this.drainTimer === null || this.unsettledCount() > 0) return;
|
|
1888
|
+
clearTimeout(this.drainTimer);
|
|
1889
|
+
this.drainTimer = null;
|
|
1890
|
+
this.killNow("drain");
|
|
1891
|
+
}
|
|
1892
|
+
/**
|
|
1893
|
+
* The drain ran out of time with frames still held. Those frames die with
|
|
1894
|
+
* the worker; that is a branch that DROPS work (D391), so it says how many,
|
|
1895
|
+
* on which camera.
|
|
1896
|
+
*/
|
|
1897
|
+
handleDrainBoundExpired() {
|
|
1898
|
+
this.drainTimer = null;
|
|
1899
|
+
if (this.exited) return;
|
|
1900
|
+
this.logger.warn("session-decode-coordinator: drain bound expired — killing the worker with frames still held", {
|
|
1901
|
+
tags: { deviceId: this.deviceId },
|
|
1902
|
+
meta: {
|
|
1903
|
+
unsettledFrames: this.openImages,
|
|
1904
|
+
pendingReads: this.pendingReadCount(),
|
|
1905
|
+
pendingCrops: this.pendingNativeCrops.size,
|
|
1906
|
+
boundMs: SESSION_DECODE_DRAIN_BOUND_MS
|
|
1907
|
+
}
|
|
1908
|
+
});
|
|
1909
|
+
this.killNow("drain");
|
|
1910
|
+
}
|
|
1911
|
+
/**
|
|
1912
|
+
* `stop`, then the kill — unless already exited.
|
|
1913
|
+
*
|
|
1914
|
+
* `'drain'`: SIGTERM, so the child flushes its last lines (`bounded-exit.ts`),
|
|
1915
|
+
* and a SIGKILL fallback. `'immediate'`: SIGKILL outright. Since D652 gave the
|
|
1916
|
+
* child a SIGTERM handler, SIGTERM is no longer instant death: a worker
|
|
1917
|
+
* blocked in a native call (a VAAPI hang) never runs the handler, and the
|
|
1918
|
+
* fallback was an `unref`'d timer the exiting parent did not wait for — so
|
|
1919
|
+
* on shutdown that worker was orphaned with its GPU pool. The fallback timer
|
|
1920
|
+
* is now ref'd: it exists only for a drain while the parent is alive (a
|
|
1921
|
+
* shutdown escalates to SIGKILL and clears it), and a timer that keeps the
|
|
1922
|
+
* process up for at most one second is the price of a kill that lands.
|
|
1923
|
+
*/
|
|
1924
|
+
killNow(mode) {
|
|
1925
|
+
if (this.killSent) return;
|
|
1926
|
+
this.killSent = true;
|
|
1667
1927
|
this.trySend({ kind: "stop" });
|
|
1928
|
+
if (mode === "immediate") {
|
|
1929
|
+
this.tryKill("SIGKILL");
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1668
1932
|
this.tryKill(void 0);
|
|
1669
1933
|
if (this.exited) return;
|
|
1670
1934
|
this.sigkillTimer = setTimeout(() => {
|
|
1671
1935
|
this.sigkillTimer = null;
|
|
1672
1936
|
if (this.exited) return;
|
|
1673
|
-
this.logger.warn("session-decode-coordinator: child did not exit within SIGKILL fallback window — forcing", {
|
|
1937
|
+
this.logger.warn("session-decode-coordinator: child did not exit within SIGKILL fallback window — forcing", {
|
|
1938
|
+
tags: { deviceId: this.deviceId },
|
|
1939
|
+
meta: { fallbackMs: SIGKILL_FALLBACK_MS }
|
|
1940
|
+
});
|
|
1674
1941
|
this.tryKill("SIGKILL");
|
|
1675
1942
|
}, SIGKILL_FALLBACK_MS);
|
|
1676
|
-
this.sigkillTimer.unref?.();
|
|
1677
1943
|
}
|
|
1678
1944
|
handleMessage(raw) {
|
|
1679
1945
|
if (!isWorkerReply(raw)) {
|
|
@@ -1722,7 +1988,7 @@ var SessionDecodeSession = class {
|
|
|
1722
1988
|
this.logger[raw.level](raw.message);
|
|
1723
1989
|
return;
|
|
1724
1990
|
case "dialTiming":
|
|
1725
|
-
this.onDialTiming?.({
|
|
1991
|
+
this.hooks.onDialTiming?.({
|
|
1726
1992
|
hwContextMs: raw.hwContextMs,
|
|
1727
1993
|
demuxerOpenMs: raw.demuxerOpenMs,
|
|
1728
1994
|
decoderCreateMs: raw.decoderCreateMs,
|
|
@@ -1730,25 +1996,46 @@ var SessionDecodeSession = class {
|
|
|
1730
1996
|
streamInfoProbed: raw.streamInfoProbed
|
|
1731
1997
|
});
|
|
1732
1998
|
return;
|
|
1999
|
+
case "preRollSettled":
|
|
2000
|
+
this.hooks.onPreRollSettled?.({
|
|
2001
|
+
read: raw.read,
|
|
2002
|
+
released: raw.released,
|
|
2003
|
+
lostByReason: raw.lostByReason,
|
|
2004
|
+
unsettled: raw.unsettled
|
|
2005
|
+
});
|
|
2006
|
+
return;
|
|
1733
2007
|
case "preRollReport":
|
|
1734
|
-
this.onPreRollReport?.({
|
|
2008
|
+
this.hooks.onPreRollReport?.({
|
|
1735
2009
|
offered: raw.offered,
|
|
1736
2010
|
inferred: raw.inferred,
|
|
1737
2011
|
dropped: raw.dropped,
|
|
1738
2012
|
droppedByReason: raw.droppedByReason,
|
|
1739
2013
|
strideMs: raw.strideMs,
|
|
1740
2014
|
subsampled: raw.subsampled,
|
|
1741
|
-
spanMs: raw.spanMs
|
|
2015
|
+
spanMs: raw.spanMs,
|
|
2016
|
+
budgetFrames: raw.budgetFrames
|
|
1742
2017
|
});
|
|
1743
2018
|
return;
|
|
1744
2019
|
}
|
|
1745
2020
|
}
|
|
1746
2021
|
handleExit(info) {
|
|
1747
|
-
if (!this.
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
2022
|
+
if (!this.exited) this.hooks.onWorkerExit?.();
|
|
2023
|
+
if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
|
|
2024
|
+
tags: { deviceId: this.deviceId },
|
|
2025
|
+
meta: {
|
|
2026
|
+
code: info?.code ?? null,
|
|
2027
|
+
signal: info?.signal ?? null,
|
|
2028
|
+
...this.stopRequested ? {
|
|
2029
|
+
unsettledFrames: this.openImages,
|
|
2030
|
+
pendingReads: this.pendingReadCount()
|
|
2031
|
+
} : {}
|
|
2032
|
+
}
|
|
2033
|
+
});
|
|
1751
2034
|
this.exited = true;
|
|
2035
|
+
if (this.drainTimer) {
|
|
2036
|
+
clearTimeout(this.drainTimer);
|
|
2037
|
+
this.drainTimer = null;
|
|
2038
|
+
}
|
|
1752
2039
|
if (this.sigkillTimer) {
|
|
1753
2040
|
clearTimeout(this.sigkillTimer);
|
|
1754
2041
|
this.sigkillTimer = null;
|
|
@@ -1770,6 +2057,7 @@ var SessionDecodeSession = class {
|
|
|
1770
2057
|
if (!waiter) return;
|
|
1771
2058
|
this.pendingNativeCrops.delete(requestId);
|
|
1772
2059
|
waiter.resolve(attempt);
|
|
2060
|
+
this.settleDrainIfDone();
|
|
1773
2061
|
}
|
|
1774
2062
|
settlePendingPull(outcome) {
|
|
1775
2063
|
const waiter = this.pendingPull;
|
|
@@ -1865,6 +2153,7 @@ var SessionDecodeSession = class {
|
|
|
1865
2153
|
const waiter = queue.shift();
|
|
1866
2154
|
if (queue.length === 0) this.pendingToBuffers.delete(frameId);
|
|
1867
2155
|
waiter?.resolve(bytes);
|
|
2156
|
+
this.settleDrainIfDone();
|
|
1868
2157
|
}
|
|
1869
2158
|
/**
|
|
1870
2159
|
* `error` replies MAY carry a `frameId` (a `toBuffer`-scoped miss, e.g. a
|
|
@@ -1884,6 +2173,7 @@ var SessionDecodeSession = class {
|
|
|
1884
2173
|
if (queue && queue.length > 0) {
|
|
1885
2174
|
for (const waiter of queue) waiter.reject(err);
|
|
1886
2175
|
this.pendingToBuffers.delete(frameId);
|
|
2176
|
+
this.settleDrainIfDone();
|
|
1887
2177
|
return;
|
|
1888
2178
|
}
|
|
1889
2179
|
this.logger.debug("session-decode-coordinator: dropping toBuffer error for unknown frameId", { meta: { frameId } });
|
|
@@ -1898,6 +2188,7 @@ var SessionDecodeSession = class {
|
|
|
1898
2188
|
}
|
|
1899
2189
|
for (const waiters of this.pendingToBuffers.values()) for (const waiter of waiters) waiter.reject(err);
|
|
1900
2190
|
this.pendingToBuffers.clear();
|
|
2191
|
+
this.settleDrainIfDone();
|
|
1901
2192
|
}
|
|
1902
2193
|
queueToBufferWaiter(frameId, waiter) {
|
|
1903
2194
|
const queue = this.pendingToBuffers.get(frameId);
|
|
@@ -1935,6 +2226,7 @@ var SessionDecodeSession = class {
|
|
|
1935
2226
|
/** Build the {@link FrameImage} adapter for one `frame` reply. */
|
|
1936
2227
|
function buildFrameImage(session, reply) {
|
|
1937
2228
|
let closed = false;
|
|
2229
|
+
session.noteImageOpened();
|
|
1938
2230
|
return {
|
|
1939
2231
|
width: reply.width,
|
|
1940
2232
|
height: reply.height,
|
|
@@ -1949,7 +2241,9 @@ function buildFrameImage(session, reply) {
|
|
|
1949
2241
|
releaseNativeLease: () => session.releaseLease(reply.frameId),
|
|
1950
2242
|
markInferred: () => session.markInferred(reply.frameId),
|
|
1951
2243
|
close: () => {
|
|
2244
|
+
if (closed) return;
|
|
1952
2245
|
closed = true;
|
|
2246
|
+
session.noteImageClosed();
|
|
1953
2247
|
}
|
|
1954
2248
|
};
|
|
1955
2249
|
}
|
|
@@ -1958,15 +2252,23 @@ function buildFrameImage(session, reply) {
|
|
|
1958
2252
|
* `generateVideoFrames()` call, drives it with `pull`/`toBuffer` requests
|
|
1959
2253
|
* one at a time (no pre-fetch — back-pressure is the async generator's
|
|
1960
2254
|
* natural suspend-on-yield), and guarantees the child is torn down
|
|
1961
|
-
* (`stop` + `kill()` + SIGKILL fallback) in every exit
|
|
2255
|
+
* (bounded drain, then `stop` + `kill()` + SIGKILL fallback) in every exit
|
|
2256
|
+
* path via `finally`.
|
|
1962
2257
|
*/
|
|
1963
2258
|
function createSessionDecodeCoordinator(deps) {
|
|
1964
2259
|
const firstFrameTimeoutMs = deps.firstFrameTimeoutMs ?? DEFAULT_FIRST_FRAME_TIMEOUT_MS;
|
|
1965
2260
|
const frameTimeoutMs = deps.frameTimeoutMs ?? DEFAULT_FRAME_TIMEOUT_MS;
|
|
1966
2261
|
const firstFrameMaxWaitMs = Math.max(deps.firstFrameMaxWaitMs ?? DEFAULT_FIRST_FRAME_MAX_WAIT_MS, firstFrameTimeoutMs);
|
|
1967
2262
|
return { async *generateVideoFrames(source, opts) {
|
|
1968
|
-
const session = new SessionDecodeSession(deps.fork(source), deps.logger, source.deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs,
|
|
2263
|
+
const session = new SessionDecodeSession(deps.fork(source), deps.logger, source.deviceId, firstFrameTimeoutMs, frameTimeoutMs, firstFrameMaxWaitMs, {
|
|
2264
|
+
...deps.onDialTiming ? { onDialTiming: deps.onDialTiming } : {},
|
|
2265
|
+
...deps.onPreRollReport ? { onPreRollReport: deps.onPreRollReport } : {},
|
|
2266
|
+
...deps.onPreRollSettled ? { onPreRollSettled: deps.onPreRollSettled } : {},
|
|
2267
|
+
...deps.onWorkerExit ? { onWorkerExit: deps.onWorkerExit } : {}
|
|
2268
|
+
});
|
|
1969
2269
|
session.start(source, opts, deps.lease());
|
|
2270
|
+
deps.onSessionTeardown?.((mode) => session.teardown(mode));
|
|
2271
|
+
let failed = false;
|
|
1970
2272
|
try {
|
|
1971
2273
|
while (true) {
|
|
1972
2274
|
const outcome = await session.pullNext();
|
|
@@ -1979,8 +2281,11 @@ function createSessionDecodeCoordinator(deps) {
|
|
|
1979
2281
|
...reply.preRoll === true ? { preRoll: true } : {}
|
|
1980
2282
|
};
|
|
1981
2283
|
}
|
|
2284
|
+
} catch (err) {
|
|
2285
|
+
failed = true;
|
|
2286
|
+
throw err;
|
|
1982
2287
|
} finally {
|
|
1983
|
-
session.teardown();
|
|
2288
|
+
session.teardown(failed ? "immediate" : deps.teardownMode?.() ?? "drain");
|
|
1984
2289
|
}
|
|
1985
2290
|
} };
|
|
1986
2291
|
}
|
|
@@ -2024,17 +2329,39 @@ async function toDecodedFrame(frame, format, viewSpec) {
|
|
|
2024
2329
|
* Start pulling frames from a per-session decode worker and feed each decoded
|
|
2025
2330
|
* frame to `onDecodedFrame`. Returns an idempotent teardown closure that stops
|
|
2026
2331
|
* the async generator (`it.return()`) — which triggers the coordinator's
|
|
2027
|
-
* `finally` →
|
|
2028
|
-
*
|
|
2029
|
-
*
|
|
2332
|
+
* `finally` → a bounded DRAIN of the frames already handed out (D652), then
|
|
2333
|
+
* `stop` + SIGTERM + 1s SIGKILL fallback → worker exit → the OS reclaims the
|
|
2334
|
+
* VAAPI/GPU decode pool. `teardown('immediate')` SIGKILLs at once (shutdown),
|
|
2335
|
+
* and said a second time during a drain it escalates that drain to the kill.
|
|
2336
|
+
* Fire-and-forget: the returned closure is synchronous; the frame loop and
|
|
2337
|
+
* generator teardown run on their own tasks.
|
|
2030
2338
|
*/
|
|
2031
2339
|
function startSessionDecodePump(deps) {
|
|
2032
2340
|
const format = deps.opts.format ?? "rgb";
|
|
2033
2341
|
let dialTiming = null;
|
|
2342
|
+
let teardownMode = "drain";
|
|
2343
|
+
let escalateSession = null;
|
|
2344
|
+
let forked = false;
|
|
2345
|
+
let workerGone = false;
|
|
2346
|
+
const noteWorkerGone = () => {
|
|
2347
|
+
if (workerGone) return;
|
|
2348
|
+
workerGone = true;
|
|
2349
|
+
deps.onWorkerGone?.();
|
|
2350
|
+
};
|
|
2034
2351
|
const iterator = createSessionDecodeCoordinator({
|
|
2035
|
-
fork:
|
|
2352
|
+
fork: (source) => {
|
|
2353
|
+
const child = deps.fork(source);
|
|
2354
|
+
forked = true;
|
|
2355
|
+
return child;
|
|
2356
|
+
},
|
|
2036
2357
|
lease: deps.lease,
|
|
2037
2358
|
logger: deps.logger,
|
|
2359
|
+
teardownMode: () => teardownMode,
|
|
2360
|
+
onSessionTeardown: (teardown) => {
|
|
2361
|
+
escalateSession = teardown;
|
|
2362
|
+
},
|
|
2363
|
+
onWorkerExit: noteWorkerGone,
|
|
2364
|
+
...deps.onPreRollSettled ? { onPreRollSettled: deps.onPreRollSettled } : {},
|
|
2038
2365
|
onDialTiming: (timing) => {
|
|
2039
2366
|
dialTiming = timing;
|
|
2040
2367
|
},
|
|
@@ -2140,13 +2467,21 @@ function startSessionDecodePump(deps) {
|
|
|
2140
2467
|
});
|
|
2141
2468
|
} finally {
|
|
2142
2469
|
releaseSpawnSlotOnce();
|
|
2470
|
+
if (!forked) noteWorkerGone();
|
|
2143
2471
|
if (!stopped) deps.onEnded?.();
|
|
2144
2472
|
}
|
|
2145
2473
|
};
|
|
2146
2474
|
pump();
|
|
2147
|
-
return () => {
|
|
2148
|
-
if (stopped)
|
|
2475
|
+
return (mode = "drain") => {
|
|
2476
|
+
if (stopped) {
|
|
2477
|
+
if (mode === "immediate" && teardownMode !== "immediate") {
|
|
2478
|
+
teardownMode = "immediate";
|
|
2479
|
+
escalateSession?.("immediate");
|
|
2480
|
+
}
|
|
2481
|
+
return;
|
|
2482
|
+
}
|
|
2149
2483
|
stopped = true;
|
|
2484
|
+
teardownMode = mode;
|
|
2150
2485
|
releaseSpawnSlotOnce();
|
|
2151
2486
|
iterator.return().catch(() => {});
|
|
2152
2487
|
};
|
|
@@ -2228,28 +2563,17 @@ function onboardRootDetections(slot, frame, gate) {
|
|
|
2228
2563
|
//#endregion
|
|
2229
2564
|
//#region src/pipeline-runner/bench-step-schema.ts
|
|
2230
2565
|
/**
|
|
2231
|
-
* Shared bench-facing schemas: the minimal recursive step tree
|
|
2232
|
-
*
|
|
2233
|
-
*
|
|
2234
|
-
*
|
|
2235
|
-
*
|
|
2236
|
-
*
|
|
2237
|
-
*
|
|
2238
|
-
*
|
|
2239
|
-
*
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
runtime: _enum(["node", "python"]),
|
|
2243
|
-
backend: string(),
|
|
2244
|
-
format: _enum([
|
|
2245
|
-
"onnx",
|
|
2246
|
-
"coreml",
|
|
2247
|
-
"openvino",
|
|
2248
|
-
"tflite",
|
|
2249
|
-
"pt"
|
|
2250
|
-
]),
|
|
2251
|
-
device: string().optional()
|
|
2252
|
-
});
|
|
2566
|
+
* Shared bench-facing schemas: the minimal recursive step tree. Extracted from
|
|
2567
|
+
* `bench-actions.ts` so the capacity-ramp schemas (`capacity/capacity-schemas.ts`)
|
|
2568
|
+
* can reuse it without a bench-actions ↔ capacity-schemas import cycle.
|
|
2569
|
+
*
|
|
2570
|
+
* Local mirror of pipeline-executor's step shape ON PURPOSE (not a cap-package
|
|
2571
|
+
* coupling): addonId + modelId + enabled + children.
|
|
2572
|
+
*
|
|
2573
|
+
* There is no engine choice here any more (D646). A bench names the pool it
|
|
2574
|
+
* measures by `deviceKey`; the engine override it used to carry was honoured by
|
|
2575
|
+
* swapping the executor's shared engine state under live cameras.
|
|
2576
|
+
*/
|
|
2253
2577
|
var BenchStepSchema = lazy(() => object({
|
|
2254
2578
|
addonId: string(),
|
|
2255
2579
|
modelId: string(),
|
|
@@ -2285,7 +2609,6 @@ var RunCapacityRampInputSchema = object({
|
|
|
2285
2609
|
sessionId: string(),
|
|
2286
2610
|
nodeId: string().optional(),
|
|
2287
2611
|
deviceKey: string().optional(),
|
|
2288
|
-
engine: BenchEngineChoiceSchema.optional(),
|
|
2289
2612
|
slo: CapacitySloSchema.optional(),
|
|
2290
2613
|
allowActiveCameras: boolean().optional(),
|
|
2291
2614
|
drainTimeoutSec: number().min(1).max(300).optional()
|
|
@@ -2473,16 +2796,15 @@ var CacheBenchFrameResultSchema = object({
|
|
|
2473
2796
|
});
|
|
2474
2797
|
var ReleaseBenchFrameInputSchema = object({ frameId: string() });
|
|
2475
2798
|
var ReleaseBenchFrameResultSchema = object({ released: boolean() });
|
|
2476
|
-
var
|
|
2799
|
+
var StartSyntheticBenchInputSchema = object({
|
|
2477
2800
|
frameId: string(),
|
|
2478
2801
|
steps: array(BenchStepSchema).min(1),
|
|
2479
2802
|
parallel: number().int().min(1).max(32),
|
|
2480
2803
|
iterations: number().int().min(1).max(1e4),
|
|
2481
2804
|
warmup: number().int().min(0).max(100).optional(),
|
|
2482
|
-
sessionId: string().
|
|
2805
|
+
sessionId: string().min(1),
|
|
2483
2806
|
simulatePipeline: boolean().optional(),
|
|
2484
2807
|
nodeId: string().optional(),
|
|
2485
|
-
engine: BenchEngineChoiceSchema.optional(),
|
|
2486
2808
|
deviceKey: string().optional(),
|
|
2487
2809
|
plane: _enum(["full", "frame"]).optional()
|
|
2488
2810
|
});
|
|
@@ -2492,7 +2814,16 @@ var TimingSplitSchema = object({
|
|
|
2492
2814
|
p95: number(),
|
|
2493
2815
|
p99: number()
|
|
2494
2816
|
});
|
|
2495
|
-
var
|
|
2817
|
+
var BenchPoolSchema = object({
|
|
2818
|
+
nodeId: string(),
|
|
2819
|
+
deviceKey: string().nullable(),
|
|
2820
|
+
runtime: string().nullable(),
|
|
2821
|
+
numWorkers: number().nullable(),
|
|
2822
|
+
concurrency: number().nullable(),
|
|
2823
|
+
batchMode: string().nullable(),
|
|
2824
|
+
maxBatchSize: number().nullable()
|
|
2825
|
+
});
|
|
2826
|
+
var SyntheticBenchResultSchema = object({
|
|
2496
2827
|
runs: number(),
|
|
2497
2828
|
wallSec: number(),
|
|
2498
2829
|
fps: number(),
|
|
@@ -2504,23 +2835,52 @@ var RunSyntheticBenchResultSchema = object({
|
|
|
2504
2835
|
predictMs: number(),
|
|
2505
2836
|
batchSizeMean: number(),
|
|
2506
2837
|
batchSizeMax: number(),
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
backend: string(),
|
|
2510
|
-
device: string().optional()
|
|
2511
|
-
}).optional(),
|
|
2512
|
-
tuning: object({
|
|
2513
|
-
batchMode: string(),
|
|
2514
|
-
windowMs: number(),
|
|
2515
|
-
maxBatchSize: number(),
|
|
2516
|
-
concurrency: number()
|
|
2517
|
-
}).optional(),
|
|
2518
|
-
path: string().optional()
|
|
2838
|
+
pool: BenchPoolSchema,
|
|
2839
|
+
path: string()
|
|
2519
2840
|
});
|
|
2841
|
+
/** `startSyntheticBench` answers at once; the run continues in the background. */
|
|
2842
|
+
var StartSyntheticBenchAcceptedSchema = object({
|
|
2843
|
+
accepted: literal(true),
|
|
2844
|
+
sessionId: string()
|
|
2845
|
+
});
|
|
2846
|
+
var SyntheticBenchKeySchema = object({ sessionId: string() });
|
|
2847
|
+
var SyntheticBenchStatusSchema = object({
|
|
2848
|
+
sessionId: string(),
|
|
2849
|
+
state: _enum([
|
|
2850
|
+
"running",
|
|
2851
|
+
"done",
|
|
2852
|
+
"failed",
|
|
2853
|
+
"aborted"
|
|
2854
|
+
]),
|
|
2855
|
+
startedAtMs: number(),
|
|
2856
|
+
finishedAtMs: number().nullable(),
|
|
2857
|
+
progress: object({
|
|
2858
|
+
runs: number(),
|
|
2859
|
+
totalRuns: number()
|
|
2860
|
+
}).nullable(),
|
|
2861
|
+
result: SyntheticBenchResultSchema.nullable(),
|
|
2862
|
+
error: string().nullable()
|
|
2863
|
+
}).nullable();
|
|
2520
2864
|
var pipelineRunnerBenchActions = defineCustomActions({
|
|
2521
2865
|
cacheBenchFrame: customAction(CacheBenchFrameInputSchema, CacheBenchFrameResultSchema, { kind: "mutation" }),
|
|
2522
2866
|
releaseBenchFrame: customAction(ReleaseBenchFrameInputSchema, ReleaseBenchFrameResultSchema, { kind: "mutation" }),
|
|
2523
|
-
|
|
2867
|
+
/**
|
|
2868
|
+
* START a synthetic bench and return at once (D646). Up to 32 parallel ×
|
|
2869
|
+
* 10 000 iterations does not fit the UDS addon-call plane's 60 s request
|
|
2870
|
+
* timeout, and a synchronous call was abandoned by its caller while it kept
|
|
2871
|
+
* loading the node. Poll `getSyntheticBenchStatus {sessionId}`; stop early
|
|
2872
|
+
* with `abortSyntheticBench {sessionId}`. One bench per runner at a time —
|
|
2873
|
+
* the measurement owns the pool.
|
|
2874
|
+
*/
|
|
2875
|
+
startSyntheticBench: customAction(StartSyntheticBenchInputSchema, StartSyntheticBenchAcceptedSchema, {
|
|
2876
|
+
kind: "mutation",
|
|
2877
|
+
auth: "admin"
|
|
2878
|
+
}),
|
|
2879
|
+
getSyntheticBenchStatus: customAction(SyntheticBenchKeySchema, SyntheticBenchStatusSchema),
|
|
2880
|
+
abortSyntheticBench: customAction(SyntheticBenchKeySchema, object({ aborted: boolean() }), {
|
|
2881
|
+
kind: "mutation",
|
|
2882
|
+
auth: "admin"
|
|
2883
|
+
}),
|
|
2524
2884
|
/**
|
|
2525
2885
|
* OPEN-loop capacity ramp (spec `2026-08-02-capacity-test-design.md` §4):
|
|
2526
2886
|
* paced N×`fpsPerCamera` single-frame `runPipeline` calls with raw RGB —
|
|
@@ -2585,12 +2945,24 @@ function buildBenchRunPipelineInput(req) {
|
|
|
2585
2945
|
return {
|
|
2586
2946
|
steps: [...req.steps],
|
|
2587
2947
|
frame: req.frame,
|
|
2588
|
-
...req.engine ? { engine: req.engine } : {},
|
|
2589
2948
|
...req.deviceKey ? { deviceKey: req.deviceKey } : {},
|
|
2590
2949
|
...req.plane ? { plane: req.plane } : {}
|
|
2591
2950
|
};
|
|
2592
2951
|
}
|
|
2593
2952
|
/**
|
|
2953
|
+
* The tuning query for the pool a bench MEASURED (D646). It asked the node
|
|
2954
|
+
* default of whatever node answered first, so an NPU run was recorded beside
|
|
2955
|
+
* the onnx pool's `concurrency: 4`. Absent `deviceKey`, never
|
|
2956
|
+
* present-and-undefined, for the same `.loose()` reason as
|
|
2957
|
+
* {@link buildBenchRunPipelineInput}.
|
|
2958
|
+
*/
|
|
2959
|
+
function buildBenchTuningQuery(target) {
|
|
2960
|
+
return {
|
|
2961
|
+
nodeId: target.nodeId,
|
|
2962
|
+
input: target.deviceKey ? { deviceKey: target.deviceKey } : {}
|
|
2963
|
+
};
|
|
2964
|
+
}
|
|
2965
|
+
/**
|
|
2594
2966
|
* How long a box stays "seen".
|
|
2595
2967
|
*
|
|
2596
2968
|
* Sized on the gate's own clock, not on a guess: `maxDeferralMs` defaults to
|
|
@@ -3539,8 +3911,8 @@ async function runDetailSubtree(deps, input) {
|
|
|
3539
3911
|
} : void 0;
|
|
3540
3912
|
const details = [];
|
|
3541
3913
|
for (const step of steps) {
|
|
3542
|
-
const
|
|
3543
|
-
const result = await deps.runPipeline([step], crop.jpeg, input.deviceId, deviceKey, nativeCropRef);
|
|
3914
|
+
const placement = deps.placeStep?.(input.deviceId, step);
|
|
3915
|
+
const result = await deps.runPipeline([placement?.step ?? step], crop.jpeg, input.deviceId, placement?.deviceKey, nativeCropRef);
|
|
3544
3916
|
if (!result) continue;
|
|
3545
3917
|
for (const detection of result.detections) details.push(toDetailResult(detection, step.addonId, crop));
|
|
3546
3918
|
}
|
|
@@ -3647,6 +4019,72 @@ function resolveStepDevice(input) {
|
|
|
3647
4019
|
reason: "auto"
|
|
3648
4020
|
};
|
|
3649
4021
|
}
|
|
4022
|
+
/**
|
|
4023
|
+
* Re-provision a jumped subtree for its TARGET device (D650): every step's
|
|
4024
|
+
* `modelId` / `settings` are REPLACED by the target's entry — the orchestrator
|
|
4025
|
+
* computed it as what the step would carry had the camera been elected onto
|
|
4026
|
+
* that device (its per-(node,device) base with the camera's override for it).
|
|
4027
|
+
* A step absent from the map runs with neither, i.e. the target format's
|
|
4028
|
+
* default model. The elected device's provisioning never follows a step across:
|
|
4029
|
+
* it was chosen for a format this subtree does not run on.
|
|
4030
|
+
*/
|
|
4031
|
+
function provisionSubtreeForDevice(step, provisioning) {
|
|
4032
|
+
const own = provisioning[step.addonId];
|
|
4033
|
+
const { modelId: _electedModelId, settings: _electedSettings, ...rest } = step;
|
|
4034
|
+
return {
|
|
4035
|
+
...rest,
|
|
4036
|
+
...own?.modelId !== void 0 ? { modelId: own.modelId } : {},
|
|
4037
|
+
...own?.settings !== void 0 ? { settings: own.settings } : {},
|
|
4038
|
+
...step.children !== void 0 ? { children: step.children.map((c) => provisionSubtreeForDevice(c, provisioning)) } : {}
|
|
4039
|
+
};
|
|
4040
|
+
}
|
|
4041
|
+
/**
|
|
4042
|
+
* Place ONE detail subtree (D650): which same-node device runs it
|
|
4043
|
+
* ({@link resolveStepDevice}) and, when it jumps, the provisioning it runs
|
|
4044
|
+
* with there ({@link provisionSubtreeForDevice}). Pure — the caller logs.
|
|
4045
|
+
*
|
|
4046
|
+
* A roster entry without `steps` came from an orchestrator that predates D650:
|
|
4047
|
+
* the step keeps what it was sent with, as it did before.
|
|
4048
|
+
*/
|
|
4049
|
+
function placeDetailStep(input) {
|
|
4050
|
+
const { step, effectiveDeviceKey, roster } = input;
|
|
4051
|
+
if (!effectiveDeviceKey) return {
|
|
4052
|
+
deviceKey: void 0,
|
|
4053
|
+
step,
|
|
4054
|
+
decision: null,
|
|
4055
|
+
provisioned: false
|
|
4056
|
+
};
|
|
4057
|
+
if (!roster || roster.length < 2) return {
|
|
4058
|
+
deviceKey: effectiveDeviceKey,
|
|
4059
|
+
step,
|
|
4060
|
+
decision: null,
|
|
4061
|
+
provisioned: false
|
|
4062
|
+
};
|
|
4063
|
+
const decision = resolveStepDevice({
|
|
4064
|
+
effectiveDeviceKey,
|
|
4065
|
+
devices: roster.map((d) => ({
|
|
4066
|
+
deviceKey: d.deviceKey,
|
|
4067
|
+
weight: d.weight,
|
|
4068
|
+
maxSessions: d.maxSessions,
|
|
4069
|
+
attachedCameras: input.attachedByKey.get(d.deviceKey) ?? 0
|
|
4070
|
+
})),
|
|
4071
|
+
...step.jumpDeviceKey !== void 0 ? { manualJumpDeviceKey: step.jumpDeviceKey } : {},
|
|
4072
|
+
canRun: buildSubtreeCanRun(step, input.getStepDef)
|
|
4073
|
+
});
|
|
4074
|
+
if (decision.kind !== "jumped") return {
|
|
4075
|
+
deviceKey: decision.deviceKey,
|
|
4076
|
+
step,
|
|
4077
|
+
decision,
|
|
4078
|
+
provisioned: false
|
|
4079
|
+
};
|
|
4080
|
+
const provisioning = roster.find((d) => d.deviceKey === decision.deviceKey)?.steps;
|
|
4081
|
+
return {
|
|
4082
|
+
deviceKey: decision.deviceKey,
|
|
4083
|
+
step: provisioning === void 0 ? step : provisionSubtreeForDevice(step, provisioning),
|
|
4084
|
+
decision,
|
|
4085
|
+
provisioned: provisioning !== void 0
|
|
4086
|
+
};
|
|
4087
|
+
}
|
|
3650
4088
|
//#endregion
|
|
3651
4089
|
//#region src/pipeline-runner/device-load.ts
|
|
3652
4090
|
/** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
|
|
@@ -3830,6 +4268,74 @@ function release(cutter) {
|
|
|
3830
4268
|
} catch {}
|
|
3831
4269
|
}
|
|
3832
4270
|
//#endregion
|
|
4271
|
+
//#region src/pipeline-runner/occupancy-burst-ledger.ts
|
|
4272
|
+
var OccupancyBurstLedger = class {
|
|
4273
|
+
options;
|
|
4274
|
+
frames = 0;
|
|
4275
|
+
inferred = 0;
|
|
4276
|
+
superseded = 0;
|
|
4277
|
+
settled = 0;
|
|
4278
|
+
closed = false;
|
|
4279
|
+
completed = false;
|
|
4280
|
+
boundTimer = null;
|
|
4281
|
+
constructor(options) {
|
|
4282
|
+
this.options = options;
|
|
4283
|
+
}
|
|
4284
|
+
/**
|
|
4285
|
+
* Count one collected frame and hand back its settler. A frame the runner
|
|
4286
|
+
* REFUSED is settled `'lost'` by the caller at once — it is a frame the
|
|
4287
|
+
* verdict did not see, which is what `lost` counts.
|
|
4288
|
+
*/
|
|
4289
|
+
admit() {
|
|
4290
|
+
this.frames += 1;
|
|
4291
|
+
let done = false;
|
|
4292
|
+
return (outcome) => {
|
|
4293
|
+
if (done || this.completed) return;
|
|
4294
|
+
done = true;
|
|
4295
|
+
this.settled += 1;
|
|
4296
|
+
if (outcome === "inferred") this.inferred += 1;
|
|
4297
|
+
else if (outcome === "superseded") this.superseded += 1;
|
|
4298
|
+
this.completeIfSettled();
|
|
4299
|
+
};
|
|
4300
|
+
}
|
|
4301
|
+
/**
|
|
4302
|
+
* No more frames will be collected (count reached, safety timeout, or the
|
|
4303
|
+
* decode ended). Completes now when everything admitted has settled, else
|
|
4304
|
+
* arms the bound. Idempotent.
|
|
4305
|
+
*/
|
|
4306
|
+
close() {
|
|
4307
|
+
if (this.closed || this.completed) return;
|
|
4308
|
+
this.closed = true;
|
|
4309
|
+
if (this.completeIfSettled()) return;
|
|
4310
|
+
this.boundTimer = setTimeout(() => {
|
|
4311
|
+
this.boundTimer = null;
|
|
4312
|
+
this.complete(true);
|
|
4313
|
+
}, this.options.settleBoundMs);
|
|
4314
|
+
this.boundTimer.unref?.();
|
|
4315
|
+
}
|
|
4316
|
+
completeIfSettled() {
|
|
4317
|
+
if (!this.closed || this.settled < this.frames) return false;
|
|
4318
|
+
this.complete(false);
|
|
4319
|
+
return true;
|
|
4320
|
+
}
|
|
4321
|
+
complete(timedOut) {
|
|
4322
|
+
if (this.completed) return;
|
|
4323
|
+
this.completed = true;
|
|
4324
|
+
if (this.boundTimer !== null) {
|
|
4325
|
+
clearTimeout(this.boundTimer);
|
|
4326
|
+
this.boundTimer = null;
|
|
4327
|
+
}
|
|
4328
|
+
this.options.onComplete({
|
|
4329
|
+
frames: this.frames,
|
|
4330
|
+
inferred: this.inferred,
|
|
4331
|
+
superseded: this.superseded,
|
|
4332
|
+
lost: this.frames - this.inferred - this.superseded,
|
|
4333
|
+
unsettled: this.frames - this.settled,
|
|
4334
|
+
timedOut
|
|
4335
|
+
});
|
|
4336
|
+
}
|
|
4337
|
+
};
|
|
4338
|
+
//#endregion
|
|
3833
4339
|
//#region src/pipeline-runner/frame-lazy-metrics.ts
|
|
3834
4340
|
function emptyFrameLazyCounters() {
|
|
3835
4341
|
return {
|
|
@@ -5242,41 +5748,6 @@ var FrameGovernor = class {
|
|
|
5242
5748
|
};
|
|
5243
5749
|
//#endregion
|
|
5244
5750
|
//#region src/pipeline-runner/frame-queue.ts
|
|
5245
|
-
/**
|
|
5246
|
-
* Latest-frame-only buffer. Keeps only the most recent item, dropping all
|
|
5247
|
-
* older items immediately. This ensures inference always runs on the
|
|
5248
|
-
* freshest available frame, never accumulating a backlog regardless of
|
|
5249
|
-
* inference latency.
|
|
5250
|
-
*
|
|
5251
|
-
* ## Buffer ownership (Phase 5 / D9 Task 7b)
|
|
5252
|
-
*
|
|
5253
|
-
* Frames arrive from the shm frame plane via a `FrameRingReader`, whose
|
|
5254
|
-
* `FrameRead.pixels` is **borrowed** — a view onto the reader's reusable
|
|
5255
|
-
* scratch buffer, valid only until that reader's next read (see
|
|
5256
|
-
* `@camstack/shm-ring`'s `FrameRead` contract). Both queue consumers retain
|
|
5257
|
-
* the frame PAST the next read:
|
|
5258
|
-
*
|
|
5259
|
-
* - the **motion** queue is drained by the scheduler's `tick`, not by the
|
|
5260
|
-
* poller that read the frame — and the poller drains a burst of handles
|
|
5261
|
-
* per poll, so even handle #2's read would clobber handle #1's pixels;
|
|
5262
|
-
* - the **detection** queue is drained asynchronously after a semaphore wait
|
|
5263
|
-
* and a 10s–100s ms inference.
|
|
5264
|
-
*
|
|
5265
|
-
* In both cases holding the borrowed buffer is silent corruption. So
|
|
5266
|
-
* `enqueue` runs the queue's injected `clone` function which copies the
|
|
5267
|
-
* pixels into queue-owned storage at the retention boundary. The queue is
|
|
5268
|
-
* latest-only (capacity 1), so this is exactly one allocation per *queued*
|
|
5269
|
-
* frame — bounded by the scheduler/inference cadence, NOT per *read* frame.
|
|
5270
|
-
* The high-frequency per-read allocation that failed the D9 perf gate is
|
|
5271
|
-
* eliminated; this bounded copy-on-retain is the correct cost.
|
|
5272
|
-
*
|
|
5273
|
-
* The generic parameter `T` lets the detection path queue a
|
|
5274
|
-
* `{ frame, handle }` entry (Task 7 — propagating the `FrameHandle`
|
|
5275
|
-
* through to the inference-result event), while the motion path keeps
|
|
5276
|
-
* queueing bare `DecodedFrame`s. Each instance provides its own
|
|
5277
|
-
* `clone` so the inner pixel buffer is detached at the same retention
|
|
5278
|
-
* boundary regardless of the wrapping entry shape.
|
|
5279
|
-
*/
|
|
5280
5751
|
var FrameQueue = class {
|
|
5281
5752
|
maxSize;
|
|
5282
5753
|
latest = null;
|
|
@@ -5297,7 +5768,7 @@ var FrameQueue = class {
|
|
|
5297
5768
|
enqueue(item) {
|
|
5298
5769
|
if (this.latest !== null) {
|
|
5299
5770
|
this._droppedFrames++;
|
|
5300
|
-
this.dispose?.(this.latest);
|
|
5771
|
+
this.dispose?.(this.latest, "superseded");
|
|
5301
5772
|
}
|
|
5302
5773
|
this.latest = this.clone(item);
|
|
5303
5774
|
}
|
|
@@ -5312,8 +5783,12 @@ var FrameQueue = class {
|
|
|
5312
5783
|
get droppedFrames() {
|
|
5313
5784
|
return this._droppedFrames;
|
|
5314
5785
|
}
|
|
5786
|
+
/** The queued item, without taking it. */
|
|
5787
|
+
peek() {
|
|
5788
|
+
return this.latest ?? void 0;
|
|
5789
|
+
}
|
|
5315
5790
|
clear() {
|
|
5316
|
-
if (this.latest !== null) this.dispose?.(this.latest);
|
|
5791
|
+
if (this.latest !== null) this.dispose?.(this.latest, "cleared");
|
|
5317
5792
|
this.latest = null;
|
|
5318
5793
|
}
|
|
5319
5794
|
};
|
|
@@ -5713,7 +6188,8 @@ function ownDetectionEntry(entry) {
|
|
|
5713
6188
|
return {
|
|
5714
6189
|
frame: ownFrame(entry.frame),
|
|
5715
6190
|
handle: entry.handle,
|
|
5716
|
-
occasion: entry.occasion
|
|
6191
|
+
occasion: entry.occasion,
|
|
6192
|
+
...entry.onSettled ? { onSettled: entry.onSettled } : {}
|
|
5717
6193
|
};
|
|
5718
6194
|
}
|
|
5719
6195
|
/**
|
|
@@ -5855,9 +6331,10 @@ var PipelineRunner = class {
|
|
|
5855
6331
|
}
|
|
5856
6332
|
registerCamera(deviceId, registration) {
|
|
5857
6333
|
const motionQueue = new FrameQueue(this.config.maxQueueDepth, ownMotionEntry);
|
|
5858
|
-
const detectionQueue = new FrameQueue(this.config.maxQueueDepth, ownDetectionEntry, (entry) => {
|
|
6334
|
+
const detectionQueue = new FrameQueue(this.config.maxQueueDepth, ownDetectionEntry, (entry, reason) => {
|
|
5859
6335
|
if (entry.handle) this.config.releaseRetention?.(entry.handle, "drop");
|
|
5860
6336
|
if (entry.frame.frameRef) this.config.releaseFrameRef?.(entry.frame.frameRef, "drop");
|
|
6337
|
+
entry.onSettled?.(reason === "superseded" ? "superseded" : "lost");
|
|
5861
6338
|
});
|
|
5862
6339
|
const initialPhase = registration.detectionMode === "disabled" ? "idle" : registration.detectionMode === "always-on" ? "active" : "watching";
|
|
5863
6340
|
const preRollQueue = new PreRollLane({
|
|
@@ -5866,6 +6343,7 @@ var PipelineRunner = class {
|
|
|
5866
6343
|
dispose: (entry) => {
|
|
5867
6344
|
if (entry.handle) this.config.releaseRetention?.(entry.handle, "drop");
|
|
5868
6345
|
if (entry.frame.frameRef) this.config.releaseFrameRef?.(entry.frame.frameRef, "drop");
|
|
6346
|
+
entry.onSettled?.("lost");
|
|
5869
6347
|
}
|
|
5870
6348
|
});
|
|
5871
6349
|
this.logger?.info("camera registered: root detectors", {
|
|
@@ -5926,8 +6404,7 @@ var PipelineRunner = class {
|
|
|
5926
6404
|
}
|
|
5927
6405
|
if (state.phase === "active") this.detectionStreamHandler?.(deviceId, "unsubscribe");
|
|
5928
6406
|
state.motionQueue.clear();
|
|
5929
|
-
|
|
5930
|
-
state.preRollQueue.clear();
|
|
6407
|
+
this.discardQueuedDetectionFrames(deviceId, state, "detach");
|
|
5931
6408
|
this.timingSampler.clearDroppedFramesSource(deviceId);
|
|
5932
6409
|
this.cameras.delete(deviceId);
|
|
5933
6410
|
this.lastAdmittedAt.delete(deviceId);
|
|
@@ -6051,9 +6528,14 @@ var PipelineRunner = class {
|
|
|
6051
6528
|
preRollQueueDepth(deviceId) {
|
|
6052
6529
|
return this.cameras.get(deviceId)?.preRollQueue.size ?? 0;
|
|
6053
6530
|
}
|
|
6054
|
-
/**
|
|
6531
|
+
/**
|
|
6532
|
+
* Frames this camera lane REFUSED because it was full — or `null` when the
|
|
6533
|
+
* camera is no longer registered and the count cannot be read (a report
|
|
6534
|
+
* that lands after a detach). A failed read is `null`, never `0` (D393):
|
|
6535
|
+
* zero would say "nothing refused" about a lane nobody looked at.
|
|
6536
|
+
*/
|
|
6055
6537
|
preRollRefusedCount(deviceId) {
|
|
6056
|
-
return this.cameras.get(deviceId)?.preRollQueue.refused ??
|
|
6538
|
+
return this.cameras.get(deviceId)?.preRollQueue.refused ?? null;
|
|
6057
6539
|
}
|
|
6058
6540
|
/**
|
|
6059
6541
|
* Whether the node has a permit to spare for history right now — i.e. one
|
|
@@ -6082,7 +6564,7 @@ var PipelineRunner = class {
|
|
|
6082
6564
|
* decode worker). `runInference` already falls back to the pixel path
|
|
6083
6565
|
* when the handle is absent.
|
|
6084
6566
|
*/
|
|
6085
|
-
enqueueOccupancyFrame(deviceId, frame, handle) {
|
|
6567
|
+
enqueueOccupancyFrame(deviceId, frame, handle, onSettled) {
|
|
6086
6568
|
const state = this.cameras.get(deviceId);
|
|
6087
6569
|
if (!state) return false;
|
|
6088
6570
|
if (state.phase !== "watching") return false;
|
|
@@ -6090,7 +6572,8 @@ var PipelineRunner = class {
|
|
|
6090
6572
|
state.detectionQueue.enqueue({
|
|
6091
6573
|
frame,
|
|
6092
6574
|
handle,
|
|
6093
|
-
occasion: "burst"
|
|
6575
|
+
occasion: "burst",
|
|
6576
|
+
...onSettled ? { onSettled } : {}
|
|
6094
6577
|
});
|
|
6095
6578
|
return true;
|
|
6096
6579
|
}
|
|
@@ -6298,7 +6781,7 @@ var PipelineRunner = class {
|
|
|
6298
6781
|
}
|
|
6299
6782
|
transitionToWatching(deviceId, state, cooldownMs) {
|
|
6300
6783
|
state.phase = "watching";
|
|
6301
|
-
|
|
6784
|
+
this.discardQueuedDetectionFrames(deviceId, state, "session-close");
|
|
6302
6785
|
state.occupancyArmed = state.lastResultDetections > 0;
|
|
6303
6786
|
state.emptyBurstStreak = 0;
|
|
6304
6787
|
this.logger?.info("motion gate closed — phase=watching", {
|
|
@@ -6319,6 +6802,41 @@ var PipelineRunner = class {
|
|
|
6319
6802
|
state.lastArmedSource = null;
|
|
6320
6803
|
state.lastArmedRegions = void 0;
|
|
6321
6804
|
}
|
|
6805
|
+
/**
|
|
6806
|
+
* Throw away the frames still QUEUED for a camera whose session is ending.
|
|
6807
|
+
*
|
|
6808
|
+
* A branch that drops work says so (D391, D652). These are frames the runner
|
|
6809
|
+
* admitted and will now never infer — the live queue's latest frame and, on a
|
|
6810
|
+
* detach, whatever history is left in the pre-roll lane. Frames already in
|
|
6811
|
+
* flight are not touched here: they finish, and the decode worker they live
|
|
6812
|
+
* in drains before it is killed (`SESSION_DECODE_DRAIN_BOUND_MS`).
|
|
6813
|
+
*/
|
|
6814
|
+
discardQueuedDetectionFrames(deviceId, state, reason) {
|
|
6815
|
+
const queued = state.detectionQueue.size;
|
|
6816
|
+
const awaited = state.detectionQueue.peek()?.onSettled !== void 0 ? 1 : 0;
|
|
6817
|
+
const preRoll = reason === "detach" ? state.preRollQueue.size : 0;
|
|
6818
|
+
state.detectionQueue.clear();
|
|
6819
|
+
if (reason === "detach") state.preRollQueue.clear();
|
|
6820
|
+
if (queued + preRoll === 0) return;
|
|
6821
|
+
const meta = {
|
|
6822
|
+
reason,
|
|
6823
|
+
queued,
|
|
6824
|
+
awaited,
|
|
6825
|
+
preRoll,
|
|
6826
|
+
inflight: state.inflight
|
|
6827
|
+
};
|
|
6828
|
+
if (awaited + preRoll > 0) {
|
|
6829
|
+
this.logger?.info("detection frames discarded with the session", {
|
|
6830
|
+
tags: { deviceId },
|
|
6831
|
+
meta
|
|
6832
|
+
});
|
|
6833
|
+
return;
|
|
6834
|
+
}
|
|
6835
|
+
this.logger?.debug("detection frames discarded with the session", {
|
|
6836
|
+
tags: { deviceId },
|
|
6837
|
+
meta
|
|
6838
|
+
});
|
|
6839
|
+
}
|
|
6322
6840
|
tick() {
|
|
6323
6841
|
this.drainMotionQueues();
|
|
6324
6842
|
if (this.semaphore.available <= 0) {
|
|
@@ -6406,6 +6924,7 @@ var PipelineRunner = class {
|
|
|
6406
6924
|
} catch {}
|
|
6407
6925
|
if (handle) this.config.releaseRetention?.(handle, frameReleaseReason);
|
|
6408
6926
|
if (frame.frameRef) this.config.releaseFrameRef?.(frame.frameRef, frameReleaseReason);
|
|
6927
|
+
entry.onSettled?.(frameReleaseReason === "success" ? "inferred" : "lost");
|
|
6409
6928
|
}
|
|
6410
6929
|
}
|
|
6411
6930
|
/**
|
|
@@ -6487,6 +7006,7 @@ var PipelineRunner = class {
|
|
|
6487
7006
|
} catch {}
|
|
6488
7007
|
if (handle) this.config.releaseRetention?.(handle, frameReleaseReason);
|
|
6489
7008
|
if (frame.frameRef) this.config.releaseFrameRef?.(frame.frameRef, frameReleaseReason);
|
|
7009
|
+
entry.onSettled?.(frameReleaseReason === "success" ? "inferred" : "lost");
|
|
6490
7010
|
}
|
|
6491
7011
|
}
|
|
6492
7012
|
/**
|
|
@@ -7030,6 +7550,19 @@ function writeRetainedDetectionFrame(store, deviceId, frame, retention) {
|
|
|
7030
7550
|
*/
|
|
7031
7551
|
var OCCUPANCY_BURST_SAFETY_TIMEOUT_MS = 8e3;
|
|
7032
7552
|
/**
|
|
7553
|
+
* How long a burst whose collection has ended waits for the frames it admitted
|
|
7554
|
+
* to be INFERRED before it reports anyway (D652).
|
|
7555
|
+
*
|
|
7556
|
+
* A burst's verdict is read off its frames' results, so it completes when they
|
|
7557
|
+
* settle, not when they are enqueued. This bound is only for a frame that never
|
|
7558
|
+
* settles. Measured on the live hub 2026-09-26, the worst enqueue→emit of a
|
|
7559
|
+
* 10 s window was ≤1.5 s in 98.0% of 2 254 windows and ≤5 s in 98.9%; what lies
|
|
7560
|
+
* beyond is the inference-pool GPU hang, which the runner's own 20 s inference
|
|
7561
|
+
* timeout already ends. A timed-out burst counts its unsettled frames as lost
|
|
7562
|
+
* and says so on its `burst complete` line.
|
|
7563
|
+
*/
|
|
7564
|
+
var OCCUPANCY_BURST_SETTLE_BOUND_MS = 5e3;
|
|
7565
|
+
/**
|
|
7033
7566
|
* Freshness window for `lastMotionFrameAt` (Task 10 hardening). A non-null
|
|
7034
7567
|
* `attachment.motionUnsubscribe` only proves a subscription handle exists —
|
|
7035
7568
|
* NOT that frames are actually flowing (guarded-session-decode can return
|
|
@@ -7567,6 +8100,26 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7567
8100
|
benchFrameCache = /* @__PURE__ */ new Map();
|
|
7568
8101
|
benchFrameSweeper = null;
|
|
7569
8102
|
/**
|
|
8103
|
+
* Synthetic benches, start-then-poll (D646). One at a time on this runner —
|
|
8104
|
+
* the measurement owns the pool — and the last few finished ones kept for a
|
|
8105
|
+
* late poll. In RAM: a bench belongs to this process.
|
|
8106
|
+
*/
|
|
8107
|
+
syntheticBenches = new BackgroundRunRegistry({
|
|
8108
|
+
maxConcurrent: 1,
|
|
8109
|
+
keepFinished: 5,
|
|
8110
|
+
onSettled: (status) => {
|
|
8111
|
+
const line = { meta: {
|
|
8112
|
+
sessionId: status.runId,
|
|
8113
|
+
state: status.state,
|
|
8114
|
+
error: status.error,
|
|
8115
|
+
runs: status.result?.runs ?? null,
|
|
8116
|
+
fps: status.result?.fps ?? null
|
|
8117
|
+
} };
|
|
8118
|
+
if (status.state === "failed") this.ctx.logger.warn("synthetic bench failed", line);
|
|
8119
|
+
else this.ctx.logger.info("synthetic bench finished", line);
|
|
8120
|
+
}
|
|
8121
|
+
});
|
|
8122
|
+
/**
|
|
7570
8123
|
* In-flight + recent capacity-ramp runs, keyed by sessionId. The ramp is
|
|
7571
8124
|
* fire-and-poll (a full run outlives the UDS addon-call 60s timeout), so
|
|
7572
8125
|
* completed results are kept here for `getCapacityRampStatus` to collect;
|
|
@@ -7767,7 +8320,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7767
8320
|
actionHandlers: {
|
|
7768
8321
|
cacheBenchFrame: async (input) => this.cacheBenchFrame(input),
|
|
7769
8322
|
releaseBenchFrame: async (input) => this.releaseBenchFrame(input),
|
|
7770
|
-
|
|
8323
|
+
startSyntheticBench: async (input) => this.startSyntheticBench(input),
|
|
8324
|
+
getSyntheticBenchStatus: async (input) => this.getSyntheticBenchStatus(input.sessionId),
|
|
8325
|
+
abortSyntheticBench: async (input) => ({ aborted: this.syntheticBenches.abort(input.sessionId) }),
|
|
7771
8326
|
runCapacityRamp: async (input) => this.runCapacityRampAction(input),
|
|
7772
8327
|
getCapacityRampStatus: async (input) => this.getCapacityRampStatusAction(input),
|
|
7773
8328
|
listCapacityRamps: async () => this.listCapacityRampsAction(),
|
|
@@ -7789,6 +8344,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7789
8344
|
this.benchFrameSweeper = null;
|
|
7790
8345
|
}
|
|
7791
8346
|
this.benchFrameCache.clear();
|
|
8347
|
+
this.syntheticBenches.abortAll();
|
|
7792
8348
|
for (const entry of this.capacityRuns.values()) entry.abortFlag.aborted = true;
|
|
7793
8349
|
this.capacityRuns.clear();
|
|
7794
8350
|
if (this.unsubMotionEvents) {
|
|
@@ -7816,8 +8372,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7816
8372
|
this.runner = null;
|
|
7817
8373
|
}
|
|
7818
8374
|
for (const attachment of this.attached.values()) {
|
|
7819
|
-
attachment.motionUnsubscribe?.();
|
|
7820
|
-
attachment.detectionUnsubscribe?.();
|
|
8375
|
+
attachment.motionUnsubscribe?.("immediate");
|
|
8376
|
+
attachment.detectionUnsubscribe?.("immediate");
|
|
7821
8377
|
}
|
|
7822
8378
|
for (const deviceId of [...this.parkedDetail.keys()]) this.dropParkedDetail(deviceId);
|
|
7823
8379
|
this.attached.clear();
|
|
@@ -7868,7 +8424,34 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7868
8424
|
const now = Date.now();
|
|
7869
8425
|
for (const [id, entry] of this.benchFrameCache) if (entry.expiresAt < now) this.benchFrameCache.delete(id);
|
|
7870
8426
|
}
|
|
7871
|
-
|
|
8427
|
+
/**
|
|
8428
|
+
* Validate the cheap preconditions synchronously (so a bad frame id is a
|
|
8429
|
+
* refusal the caller sees at once), then run the bench in the background
|
|
8430
|
+
* and answer with the receipt (D646).
|
|
8431
|
+
*/
|
|
8432
|
+
startSyntheticBench(input) {
|
|
8433
|
+
if (!this.ctx.api) throw new Error("startSyntheticBench: ctx.api unavailable");
|
|
8434
|
+
if (!this.benchFrameCache.has(input.frameId)) throw new Error(`startSyntheticBench: frameId ${input.frameId} not cached (call cacheBenchFrame first)`);
|
|
8435
|
+
this.syntheticBenches.start(input.sessionId, (run) => this.runSyntheticBench(input, run));
|
|
8436
|
+
return {
|
|
8437
|
+
accepted: true,
|
|
8438
|
+
sessionId: input.sessionId
|
|
8439
|
+
};
|
|
8440
|
+
}
|
|
8441
|
+
getSyntheticBenchStatus(sessionId) {
|
|
8442
|
+
const status = this.syntheticBenches.status(sessionId);
|
|
8443
|
+
if (status === null) return null;
|
|
8444
|
+
return {
|
|
8445
|
+
sessionId: status.runId,
|
|
8446
|
+
state: status.state,
|
|
8447
|
+
startedAtMs: status.startedAtMs,
|
|
8448
|
+
finishedAtMs: status.finishedAtMs,
|
|
8449
|
+
progress: status.progress,
|
|
8450
|
+
result: status.result,
|
|
8451
|
+
error: status.error
|
|
8452
|
+
};
|
|
8453
|
+
}
|
|
8454
|
+
async runSyntheticBench(input, run) {
|
|
7872
8455
|
const ctx = this.ctx;
|
|
7873
8456
|
const api = ctx.api;
|
|
7874
8457
|
if (!api) throw new Error("runSyntheticBench: ctx.api unavailable");
|
|
@@ -7918,8 +8501,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7918
8501
|
}, pin)).frameId;
|
|
7919
8502
|
await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
|
|
7920
8503
|
steps: stepsToRun,
|
|
7921
|
-
frame: sharedFrame
|
|
7922
|
-
...input.engine ? { engine: input.engine } : {}
|
|
8504
|
+
frame: sharedFrame
|
|
7923
8505
|
}) }, pin);
|
|
7924
8506
|
const warmupCount = input.warmup ?? 1;
|
|
7925
8507
|
for (let w = 0; w < warmupCount; w++) await api.pipelineExecutor.inferCached.mutate({
|
|
@@ -7933,11 +8515,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7933
8515
|
const batchSizes = [];
|
|
7934
8516
|
const detCounts = [];
|
|
7935
8517
|
let _n = 0;
|
|
7936
|
-
const sessionId = input.sessionId
|
|
8518
|
+
const sessionId = input.sessionId;
|
|
7937
8519
|
const totalRuns = input.parallel * input.iterations;
|
|
7938
8520
|
const wallStart = performance.now();
|
|
7939
8521
|
const worker = async () => {
|
|
7940
8522
|
for (let i = 0; i < input.iterations; i++) {
|
|
8523
|
+
if (run.signal.aborted) break;
|
|
7941
8524
|
const t0 = performance.now();
|
|
7942
8525
|
const result = await api.pipelineExecutor.inferCached.mutate({
|
|
7943
8526
|
stepId: rootStep.addonId,
|
|
@@ -7957,6 +8540,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
7957
8540
|
batchSizes.push(bs);
|
|
7958
8541
|
detCounts.push(dets);
|
|
7959
8542
|
const n = ++_n;
|
|
8543
|
+
run.report({
|
|
8544
|
+
runs: n,
|
|
8545
|
+
totalRuns
|
|
8546
|
+
});
|
|
7960
8547
|
if (n <= 20) ctx.logger.info("bench call trace (cached)", { meta: {
|
|
7961
8548
|
n,
|
|
7962
8549
|
wallMs: Math.round(wallMs),
|
|
@@ -8009,7 +8596,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8009
8596
|
await Promise.all(Array.from({ length: input.parallel }, () => worker()));
|
|
8010
8597
|
const wallSec = (performance.now() - wallStart) / 1e3;
|
|
8011
8598
|
await api.pipelineExecutor.uncacheFrame.mutate({ frameId: poolFrameId }, pin).catch(() => {});
|
|
8012
|
-
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "cached");
|
|
8599
|
+
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "cached", { nodeId: benchNode });
|
|
8013
8600
|
}
|
|
8014
8601
|
ctx.logger.info("synthetic bench: using full runPipeline path", { meta: {
|
|
8015
8602
|
steps: enabledSteps.length,
|
|
@@ -8023,7 +8610,6 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8023
8610
|
const result = await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
|
|
8024
8611
|
steps: stepsToRun,
|
|
8025
8612
|
frame: sharedFrame,
|
|
8026
|
-
...input.engine ? { engine: input.engine } : {},
|
|
8027
8613
|
...input.deviceKey ? { deviceKey: input.deviceKey } : {},
|
|
8028
8614
|
...input.plane ? { plane: input.plane } : {}
|
|
8029
8615
|
}) }, pin);
|
|
@@ -8051,11 +8637,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8051
8637
|
const predictTimings = [];
|
|
8052
8638
|
const batchSizes = [];
|
|
8053
8639
|
const detCounts = [];
|
|
8054
|
-
const sessionId = input.sessionId
|
|
8640
|
+
const sessionId = input.sessionId;
|
|
8055
8641
|
const totalRuns = input.parallel * input.iterations;
|
|
8056
8642
|
const wallStart = performance.now();
|
|
8057
8643
|
const worker = async () => {
|
|
8058
8644
|
for (let i = 0; i < input.iterations; i++) {
|
|
8645
|
+
if (run.signal.aborted) break;
|
|
8059
8646
|
const { wallMs, result } = await callOnce();
|
|
8060
8647
|
wallTimings.push(wallMs);
|
|
8061
8648
|
serverWallTimings.push(result.debug?.wallMs ?? 0);
|
|
@@ -8065,6 +8652,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8065
8652
|
batchSizes.push(result.debug?.batchSize ?? 1);
|
|
8066
8653
|
detCounts.push(result.detections?.length ?? 0);
|
|
8067
8654
|
const n = wallTimings.length;
|
|
8655
|
+
run.report({
|
|
8656
|
+
runs: n,
|
|
8657
|
+
totalRuns
|
|
8658
|
+
});
|
|
8068
8659
|
if (n % Math.max(1, input.parallel) === 0 && ctx.eventBus) {
|
|
8069
8660
|
const elapsed = (performance.now() - wallStart) / 1e3;
|
|
8070
8661
|
const fps = elapsed > 0 ? n / elapsed : 0;
|
|
@@ -8105,9 +8696,12 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8105
8696
|
};
|
|
8106
8697
|
await Promise.all(Array.from({ length: input.parallel }, () => worker()));
|
|
8107
8698
|
const wallSec = (performance.now() - wallStart) / 1e3;
|
|
8108
|
-
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "pipeline"
|
|
8699
|
+
return this.buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, "pipeline", {
|
|
8700
|
+
nodeId: benchNode,
|
|
8701
|
+
...input.deviceKey ? { deviceKey: input.deviceKey } : {}
|
|
8702
|
+
});
|
|
8109
8703
|
}
|
|
8110
|
-
async buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, path) {
|
|
8704
|
+
async buildBenchResult(wallTimings, inferTimings, preprocessTimings, predictTimings, batchSizes, detCounts, wallSec, path, target) {
|
|
8111
8705
|
const meanOfArr = (xs) => xs.length > 0 ? xs.reduce((s, v) => s + v, 0) / xs.length : 0;
|
|
8112
8706
|
this.ctx.logger.info("synthetic bench summary", { meta: {
|
|
8113
8707
|
runs: wallTimings.length,
|
|
@@ -8143,24 +8737,47 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8143
8737
|
batchSizeMean: Math.round(meanOf(batchSizes) * 100) / 100,
|
|
8144
8738
|
batchSizeMax: batchSizes.length > 0 ? Math.max(...batchSizes) : 0,
|
|
8145
8739
|
path,
|
|
8146
|
-
|
|
8740
|
+
pool: await this.readBenchPool(target)
|
|
8147
8741
|
};
|
|
8148
8742
|
}
|
|
8149
|
-
|
|
8743
|
+
/**
|
|
8744
|
+
* The pool a bench measured, and its provisioning — asked of the node it
|
|
8745
|
+
* was pinned to, for the device pool it named (D646). It used to ask the
|
|
8746
|
+
* node DEFAULT of whichever node answered, so an NPU bench was recorded
|
|
8747
|
+
* beside the onnx pool's tuning. A failed read is `null` fields and a line,
|
|
8748
|
+
* never a default (D393).
|
|
8749
|
+
*/
|
|
8750
|
+
async readBenchPool(target) {
|
|
8751
|
+
const unknown = {
|
|
8752
|
+
nodeId: target.nodeId,
|
|
8753
|
+
deviceKey: target.deviceKey ?? null,
|
|
8754
|
+
runtime: null,
|
|
8755
|
+
numWorkers: null,
|
|
8756
|
+
concurrency: null,
|
|
8757
|
+
batchMode: null,
|
|
8758
|
+
maxBatchSize: null
|
|
8759
|
+
};
|
|
8760
|
+
const api = this.ctx.api;
|
|
8761
|
+
if (!api) return unknown;
|
|
8762
|
+
const query = buildBenchTuningQuery(target);
|
|
8150
8763
|
try {
|
|
8151
|
-
const
|
|
8152
|
-
if (!api) return {};
|
|
8153
|
-
const [eng, tuning] = await Promise.all([api.pipelineExecutor.getSelectedEngine.query(), api.pipelineExecutor.getEffectiveTuning.query()]);
|
|
8764
|
+
const tuning = await api.pipelineExecutor.getEffectiveTuning.query({ ...query.input }, nodePin(query.nodeId));
|
|
8154
8765
|
return {
|
|
8155
|
-
|
|
8156
|
-
|
|
8157
|
-
|
|
8158
|
-
|
|
8159
|
-
|
|
8160
|
-
|
|
8766
|
+
nodeId: target.nodeId,
|
|
8767
|
+
deviceKey: tuning.deviceKey,
|
|
8768
|
+
runtime: tuning.runtime,
|
|
8769
|
+
numWorkers: tuning.numWorkers,
|
|
8770
|
+
concurrency: tuning.concurrency,
|
|
8771
|
+
batchMode: tuning.batchMode,
|
|
8772
|
+
maxBatchSize: tuning.maxBatchSize
|
|
8161
8773
|
};
|
|
8162
|
-
} catch {
|
|
8163
|
-
|
|
8774
|
+
} catch (err) {
|
|
8775
|
+
this.ctx.logger.warn("synthetic bench: pool tuning read failed — reported as unknown", { meta: {
|
|
8776
|
+
nodeId: target.nodeId,
|
|
8777
|
+
deviceKey: target.deviceKey ?? null,
|
|
8778
|
+
error: errMsg(err)
|
|
8779
|
+
} });
|
|
8780
|
+
return unknown;
|
|
8164
8781
|
}
|
|
8165
8782
|
}
|
|
8166
8783
|
/**
|
|
@@ -8283,7 +8900,6 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8283
8900
|
settleSec,
|
|
8284
8901
|
measureSec,
|
|
8285
8902
|
deviceKey: input.deviceKey ?? null,
|
|
8286
|
-
engine: input.engine ?? null,
|
|
8287
8903
|
contaminatedByActiveCameras,
|
|
8288
8904
|
steps: stepsToRun.map((s) => `${s.addonId}/${s.modelId}${s.enabled ? "" : "[off]"}`)
|
|
8289
8905
|
} });
|
|
@@ -8300,8 +8916,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8300
8916
|
},
|
|
8301
8917
|
plane: "frame",
|
|
8302
8918
|
sessionId,
|
|
8303
|
-
...input.deviceKey ? { deviceKey: input.deviceKey } : {}
|
|
8304
|
-
...input.engine ? { engine: input.engine } : {}
|
|
8919
|
+
...input.deviceKey ? { deviceKey: input.deviceKey } : {}
|
|
8305
8920
|
}, pin);
|
|
8306
8921
|
if (result.debug?.engine !== void 0) engineLabel = result.debug.engine;
|
|
8307
8922
|
if (result.shed === true) return {
|
|
@@ -9103,7 +9718,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
9103
9718
|
resolveChildSteps: (deviceId, parentClassName, filter) => this.resolveDetailChildSteps(deviceId, parentClassName, filter),
|
|
9104
9719
|
getNativeCrop: (deviceId, handle, paddedBbox) => this.getNativeCropForDetail(deviceId, handle, paddedBbox),
|
|
9105
9720
|
runPipeline: (steps, imageJpeg, deviceId, deviceKey, nativeCropRef) => this.runDetailPipeline(steps, imageJpeg, deviceId, deviceKey, nativeCropRef),
|
|
9106
|
-
|
|
9721
|
+
placeStep: (deviceId, step) => this.placeDetailStepForCamera(deviceId, step),
|
|
9107
9722
|
resolveCropConvention: () => convention,
|
|
9108
9723
|
logger: this.detailSubtreeLogger()
|
|
9109
9724
|
}, input);
|
|
@@ -9317,39 +9932,49 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
9317
9932
|
return { warn: (message, extras) => this.ctxIfReady?.logger.warn(message, extras) };
|
|
9318
9933
|
}
|
|
9319
9934
|
/**
|
|
9320
|
-
* Step-tree device jump
|
|
9321
|
-
*
|
|
9322
|
-
* cannot run the
|
|
9323
|
-
* `step.jumpDeviceKey` overrides.
|
|
9324
|
-
*
|
|
9325
|
-
*
|
|
9935
|
+
* Step-tree device jump — place one enrichment subtree on a same-node
|
|
9936
|
+
* inference device ({@link placeDetailStep}). AUTO-jumps only when the
|
|
9937
|
+
* camera's elected device cannot run the subtree (its format has no build);
|
|
9938
|
+
* an operator `step.jumpDeviceKey` overrides. A jumped subtree runs with the
|
|
9939
|
+
* TARGET device's provisioning carried on the roster (D650), never the
|
|
9940
|
+
* elected device's. `deviceKey: undefined` ⇒ the runner's per-camera default
|
|
9941
|
+
* pool (no elected device).
|
|
9326
9942
|
*
|
|
9327
|
-
*
|
|
9328
|
-
*
|
|
9329
|
-
* `edgetpu` root would otherwise drop faces/plates/clip with no signal).
|
|
9943
|
+
* Logs the jump ONCE per (camera, step, target), and — when NO enabled device
|
|
9944
|
+
* on the node can run the subtree — a LOUD warn once per (camera, step).
|
|
9330
9945
|
*/
|
|
9331
|
-
|
|
9946
|
+
placeDetailStepForCamera(deviceId, step) {
|
|
9332
9947
|
const config = this.attached.get(deviceId)?.config;
|
|
9333
|
-
const
|
|
9334
|
-
|
|
9335
|
-
|
|
9336
|
-
|
|
9337
|
-
|
|
9338
|
-
|
|
9339
|
-
|
|
9340
|
-
|
|
9341
|
-
|
|
9342
|
-
|
|
9343
|
-
|
|
9344
|
-
|
|
9345
|
-
|
|
9346
|
-
|
|
9347
|
-
|
|
9348
|
-
|
|
9349
|
-
|
|
9948
|
+
const placement = placeDetailStep({
|
|
9949
|
+
step,
|
|
9950
|
+
effectiveDeviceKey: config?.deviceKey,
|
|
9951
|
+
roster: config?.inferenceDevices,
|
|
9952
|
+
attachedByKey: this.perDeviceAttachedCounts(),
|
|
9953
|
+
getStepDef: tryGetStepDefinition
|
|
9954
|
+
});
|
|
9955
|
+
const decision = placement.decision;
|
|
9956
|
+
if (decision?.kind === "no-candidate") this.warnStepHasNoRunnableDevice(deviceId, step.addonId, decision.deviceKey);
|
|
9957
|
+
else if (decision?.kind === "jumped" && config?.deviceKey) this.logStepJumped(deviceId, config.deviceKey, decision, placement);
|
|
9958
|
+
return placement;
|
|
9959
|
+
}
|
|
9960
|
+
/** `${deviceId}:${addonId}:${target}` triples whose jump has been logged. */
|
|
9961
|
+
loggedStepJumps = /* @__PURE__ */ new Set();
|
|
9962
|
+
logStepJumped(deviceId, electedDeviceKey, decision, placement) {
|
|
9963
|
+
const step = placement.step;
|
|
9964
|
+
const key = `${deviceId}:${step.addonId}:${decision.deviceKey}`;
|
|
9965
|
+
if (this.loggedStepJumps.has(key)) return;
|
|
9966
|
+
this.loggedStepJumps.add(key);
|
|
9967
|
+
this.ctx?.logger.info("enrichment step runs on another device of this node", {
|
|
9968
|
+
tags: { deviceId },
|
|
9969
|
+
meta: {
|
|
9970
|
+
step: step.addonId,
|
|
9971
|
+
electedDeviceKey,
|
|
9972
|
+
targetDeviceKey: decision.deviceKey,
|
|
9973
|
+
reason: decision.reason,
|
|
9974
|
+
provisioned: placement.provisioned,
|
|
9975
|
+
modelId: step.modelId ?? null
|
|
9976
|
+
}
|
|
9350
9977
|
});
|
|
9351
|
-
if (result.kind === "no-candidate") this.warnStepHasNoRunnableDevice(deviceId, step.addonId, effectiveDeviceKey);
|
|
9352
|
-
return result.deviceKey;
|
|
9353
9978
|
}
|
|
9354
9979
|
/** Live attached-camera count per inference `deviceKey` on this runner. */
|
|
9355
9980
|
perDeviceAttachedCounts() {
|
|
@@ -9908,6 +10533,16 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
9908
10533
|
if (!admission) return void 0;
|
|
9909
10534
|
return () => admission.admit(deviceId, label);
|
|
9910
10535
|
}
|
|
10536
|
+
/**
|
|
10537
|
+
* The start's place in the node's queue of refused starts (D652) — taken by
|
|
10538
|
+
* the guarded start on its first refusal, kept until it is over. Spread into
|
|
10539
|
+
* `startGuardedSessionDecode`'s deps; empty when this node runs no ceiling.
|
|
10540
|
+
*/
|
|
10541
|
+
liveWorkerSlotWaiter(deviceId) {
|
|
10542
|
+
const admission = this.liveWorkers;
|
|
10543
|
+
if (!admission) return {};
|
|
10544
|
+
return { slotWaiter: () => admission.slotWaiter(deviceId) };
|
|
10545
|
+
}
|
|
9911
10546
|
buildDecodeSpawnAcquire(log, label) {
|
|
9912
10547
|
return async () => {
|
|
9913
10548
|
const gate = this.spawnGate;
|
|
@@ -9936,8 +10571,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
9936
10571
|
const teardown = startGuardedSessionDecode({
|
|
9937
10572
|
logger: log,
|
|
9938
10573
|
admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "detect"),
|
|
10574
|
+
...this.liveWorkerSlotWaiter(config.deviceId),
|
|
9939
10575
|
acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:detect:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config), true),
|
|
9940
|
-
startPump: (source) => {
|
|
10576
|
+
startPump: (source, hooks) => {
|
|
9941
10577
|
const dialMs = Date.now() - requestedAt;
|
|
9942
10578
|
const useLocalFrameRef = this.executorCoLocated() && process.env["CAMSTACK_DISABLE_LOCAL_FRAME_REF"] !== "1";
|
|
9943
10579
|
log.info("session-decode: detection routed through per-session decode worker (rgb)", { meta: {
|
|
@@ -9949,6 +10585,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
9949
10585
|
return startSessionDecodePump({
|
|
9950
10586
|
fork: this.forkDecodeWorkerWithLease,
|
|
9951
10587
|
lease: this.resolveSessionLeaseKnobs,
|
|
10588
|
+
onWorkerGone: hooks.onWorkerGone,
|
|
9952
10589
|
logger: log,
|
|
9953
10590
|
source,
|
|
9954
10591
|
opts: {
|
|
@@ -10030,10 +10667,30 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10030
10667
|
strideMs: report.strideMs,
|
|
10031
10668
|
subsampled: report.subsampled,
|
|
10032
10669
|
laneRefused: runner.preRollRefusedCount(config.deviceId),
|
|
10033
|
-
budgetFrames: this.preRollBudgetFor(config)
|
|
10670
|
+
budgetFrames: this.preRollBudgetFor(config),
|
|
10671
|
+
effectiveBudgetFrames: report.budgetFrames
|
|
10034
10672
|
}
|
|
10035
10673
|
});
|
|
10036
10674
|
},
|
|
10675
|
+
onPreRollSettled: (summary) => {
|
|
10676
|
+
const lost = summary.lostByReason.evicted + summary.lostByReason.cleared + summary.lostByReason["download-failed"] + summary.lostByReason.miss;
|
|
10677
|
+
const meta = {
|
|
10678
|
+
streamId: config.detectionStreamId,
|
|
10679
|
+
...summary,
|
|
10680
|
+
lost
|
|
10681
|
+
};
|
|
10682
|
+
if (lost > 0 || summary.unsettled > 0) {
|
|
10683
|
+
log.info("session-decode: pre-roll frames settled", {
|
|
10684
|
+
tags: { deviceId: config.deviceId },
|
|
10685
|
+
meta
|
|
10686
|
+
});
|
|
10687
|
+
return;
|
|
10688
|
+
}
|
|
10689
|
+
log.debug("session-decode: pre-roll frames settled", {
|
|
10690
|
+
tags: { deviceId: config.deviceId },
|
|
10691
|
+
meta
|
|
10692
|
+
});
|
|
10693
|
+
},
|
|
10037
10694
|
onFirstFrame: (timing) => {
|
|
10038
10695
|
const phaseActiveAt = this.lastMotionAt.get(config.deviceId);
|
|
10039
10696
|
log.info("session-decode: detection first frame", {
|
|
@@ -10077,8 +10734,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10077
10734
|
});
|
|
10078
10735
|
}
|
|
10079
10736
|
});
|
|
10080
|
-
return () => {
|
|
10081
|
-
teardown();
|
|
10737
|
+
return (mode) => {
|
|
10738
|
+
teardown(mode);
|
|
10082
10739
|
};
|
|
10083
10740
|
}
|
|
10084
10741
|
/**
|
|
@@ -10099,8 +10756,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10099
10756
|
return startGuardedSessionDecode({
|
|
10100
10757
|
logger: log,
|
|
10101
10758
|
admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "motion"),
|
|
10759
|
+
...this.liveWorkerSlotWaiter(config.deviceId),
|
|
10102
10760
|
acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.motionStreamId, `session-decode:motion:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
|
|
10103
|
-
startPump: (source) => {
|
|
10761
|
+
startPump: (source, hooks) => {
|
|
10104
10762
|
log.info("session-decode: motion routed through per-session decode worker (gray)", { meta: {
|
|
10105
10763
|
streamId: config.motionStreamId,
|
|
10106
10764
|
codec: source.codec
|
|
@@ -10108,6 +10766,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10108
10766
|
return startSessionDecodePump({
|
|
10109
10767
|
fork: this.forkDecodeWorkerWithLease,
|
|
10110
10768
|
lease: this.resolveSessionLeaseKnobs,
|
|
10769
|
+
onWorkerGone: hooks.onWorkerGone,
|
|
10111
10770
|
logger: log,
|
|
10112
10771
|
source,
|
|
10113
10772
|
opts: {
|
|
@@ -10775,15 +11434,36 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10775
11434
|
* found/report semantics can't drift between the two admission
|
|
10776
11435
|
* mechanisms.
|
|
10777
11436
|
*/
|
|
10778
|
-
finishOccupancyBurst(deviceId, log,
|
|
11437
|
+
finishOccupancyBurst(deviceId, log, summary) {
|
|
10779
11438
|
this.occupancyBurstInFlight.delete(deviceId);
|
|
10780
11439
|
const found = this.occupancyBurstFound.has(deviceId);
|
|
10781
11440
|
this.occupancyBurstFound.delete(deviceId);
|
|
11441
|
+
if (summary.inferred === 0) {
|
|
11442
|
+
log.info("occupancy re-check: burst complete — nothing inferred, no verdict", {
|
|
11443
|
+
tags: { deviceId },
|
|
11444
|
+
meta: {
|
|
11445
|
+
...summary,
|
|
11446
|
+
verdict: null
|
|
11447
|
+
}
|
|
11448
|
+
});
|
|
11449
|
+
return;
|
|
11450
|
+
}
|
|
10782
11451
|
this.runner?.reportOccupancyBurstOutcome(deviceId, found);
|
|
10783
|
-
|
|
10784
|
-
|
|
11452
|
+
const meta = {
|
|
11453
|
+
...summary,
|
|
10785
11454
|
found
|
|
10786
|
-
}
|
|
11455
|
+
};
|
|
11456
|
+
if (summary.lost > 0) {
|
|
11457
|
+
log.info("occupancy re-check: burst complete", {
|
|
11458
|
+
tags: { deviceId },
|
|
11459
|
+
meta
|
|
11460
|
+
});
|
|
11461
|
+
return;
|
|
11462
|
+
}
|
|
11463
|
+
log.debug("occupancy re-check: burst complete", {
|
|
11464
|
+
tags: { deviceId },
|
|
11465
|
+
meta
|
|
11466
|
+
});
|
|
10787
11467
|
}
|
|
10788
11468
|
/**
|
|
10789
11469
|
* Task 10: arm a TAP-based occupancy burst for a device whose analyzer
|
|
@@ -10819,21 +11499,30 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10819
11499
|
log.debug("occupancy re-check: tap safety timeout reached", { meta: { collected } });
|
|
10820
11500
|
this.finishOccupancyTap(deviceId);
|
|
10821
11501
|
}, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
|
|
11502
|
+
const ledger = new OccupancyBurstLedger({
|
|
11503
|
+
settleBoundMs: OCCUPANCY_BURST_SETTLE_BOUND_MS,
|
|
11504
|
+
onComplete: resolve
|
|
11505
|
+
});
|
|
10822
11506
|
this.occupancyTaps.set(deviceId, {
|
|
10823
11507
|
remaining: frames,
|
|
10824
11508
|
collected: 0,
|
|
10825
11509
|
safetyTimer,
|
|
11510
|
+
ledger,
|
|
10826
11511
|
resolve
|
|
10827
11512
|
});
|
|
10828
11513
|
});
|
|
10829
11514
|
}
|
|
10830
|
-
/**
|
|
11515
|
+
/**
|
|
11516
|
+
* Close the tap's COLLECTION. Idempotent: a no-op once the tap has already
|
|
11517
|
+
* closed (timer fired, or the budget already drained). The tap's promise
|
|
11518
|
+
* resolves later, from its ledger, once the frames it admitted have settled.
|
|
11519
|
+
*/
|
|
10831
11520
|
finishOccupancyTap(deviceId) {
|
|
10832
11521
|
const tap = this.occupancyTaps.get(deviceId);
|
|
10833
11522
|
if (!tap) return;
|
|
10834
11523
|
clearTimeout(tap.safetyTimer);
|
|
10835
11524
|
this.occupancyTaps.delete(deviceId);
|
|
10836
|
-
tap.
|
|
11525
|
+
tap.ledger.close();
|
|
10837
11526
|
}
|
|
10838
11527
|
/**
|
|
10839
11528
|
* Tee an arriving analyzer motion frame into occupancy-burst detection
|
|
@@ -10848,7 +11537,8 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10848
11537
|
teeOccupancyTapFrame(deviceId, frame, handle) {
|
|
10849
11538
|
const tap = this.occupancyTaps.get(deviceId);
|
|
10850
11539
|
if (!tap) return;
|
|
10851
|
-
|
|
11540
|
+
const settle = tap.ledger.admit();
|
|
11541
|
+
if (!this.runner?.enqueueOccupancyFrame(deviceId, frame, handle, settle)) settle("lost");
|
|
10852
11542
|
tap.collected++;
|
|
10853
11543
|
tap.remaining--;
|
|
10854
11544
|
if (tap.remaining <= 0) this.finishOccupancyTap(deviceId);
|
|
@@ -10868,9 +11558,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10868
11558
|
const lastFrameAt = this.lastMotionFrameAt.get(deviceId);
|
|
10869
11559
|
const motionFresh = lastFrameAt !== void 0 && Date.now() - lastFrameAt < MOTION_FRESHNESS_WINDOW_MS;
|
|
10870
11560
|
if (attachment.motionUnsubscribe !== null && motionFresh) {
|
|
10871
|
-
const
|
|
10872
|
-
if (
|
|
10873
|
-
this.finishOccupancyBurst(deviceId, log,
|
|
11561
|
+
const summary = await this.startOccupancyTap(deviceId, frames, log);
|
|
11562
|
+
if (summary.frames > 0) {
|
|
11563
|
+
this.finishOccupancyBurst(deviceId, log, summary);
|
|
10874
11564
|
return;
|
|
10875
11565
|
}
|
|
10876
11566
|
log.debug("occupancy re-check: tap collected 0 frames, falling back to dial", { meta: { frames } });
|
|
@@ -10901,76 +11591,91 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
10901
11591
|
fallback: alreadyInFlight
|
|
10902
11592
|
} });
|
|
10903
11593
|
let count = 0;
|
|
11594
|
+
/** Collection is over: no further frame is admitted to this burst. */
|
|
11595
|
+
let admissionClosed = false;
|
|
11596
|
+
/** The burst has reported its verdict (or its lack of one). */
|
|
10904
11597
|
let settled = false;
|
|
10905
11598
|
let teardown = null;
|
|
10906
|
-
const cleanup = () => {
|
|
11599
|
+
const cleanup = (summary) => {
|
|
10907
11600
|
if (settled) return;
|
|
10908
11601
|
settled = true;
|
|
10909
|
-
|
|
10910
|
-
|
|
11602
|
+
this.finishOccupancyBurst(deviceId, log, summary);
|
|
11603
|
+
};
|
|
11604
|
+
const ledger = new OccupancyBurstLedger({
|
|
11605
|
+
settleBoundMs: OCCUPANCY_BURST_SETTLE_BOUND_MS,
|
|
11606
|
+
onComplete: cleanup
|
|
11607
|
+
});
|
|
11608
|
+
const closeAdmission = () => {
|
|
11609
|
+
if (admissionClosed) return;
|
|
11610
|
+
admissionClosed = true;
|
|
11611
|
+
clearTimeout(safetyTimer);
|
|
11612
|
+
teardown?.("drain");
|
|
11613
|
+
ledger.close();
|
|
11614
|
+
};
|
|
11615
|
+
/** Offer one collected frame to the burst; `true` = the runner admitted it. */
|
|
11616
|
+
const admit = (frame, handle) => {
|
|
11617
|
+
const settle = ledger.admit();
|
|
11618
|
+
const admitted = runner.enqueueOccupancyFrame(deviceId, frame, handle, settle);
|
|
11619
|
+
if (!admitted) settle("lost");
|
|
11620
|
+
count++;
|
|
11621
|
+
if (count >= frames) closeAdmission();
|
|
11622
|
+
return admitted;
|
|
10911
11623
|
};
|
|
10912
11624
|
const safetyTimer = setTimeout(() => {
|
|
10913
11625
|
log.debug("occupancy re-check: safety timeout reached", { meta: { collected: count } });
|
|
10914
|
-
|
|
11626
|
+
closeAdmission();
|
|
10915
11627
|
}, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
|
|
10916
11628
|
teardown = startGuardedSessionDecode({
|
|
10917
11629
|
logger: log,
|
|
10918
11630
|
admitWorker: this.buildLiveWorkerAdmit(deviceId, "occupancy"),
|
|
11631
|
+
...this.liveWorkerSlotWaiter(deviceId),
|
|
10919
11632
|
acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:occupancy:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
|
|
10920
|
-
startPump: (source) => {
|
|
11633
|
+
startPump: (source, hooks) => {
|
|
10921
11634
|
const useLocalFrameRef = this.executorCoLocated() && process.env["CAMSTACK_DISABLE_LOCAL_FRAME_REF"] !== "1";
|
|
10922
11635
|
return startSessionDecodePump({
|
|
10923
11636
|
fork: this.forkDecodeWorkerWithLease,
|
|
10924
11637
|
lease: this.resolveSessionLeaseKnobs,
|
|
11638
|
+
onWorkerGone: hooks.onWorkerGone,
|
|
10925
11639
|
logger: log,
|
|
10926
11640
|
source,
|
|
10927
11641
|
opts: {
|
|
10928
11642
|
fps: config.detectionFps,
|
|
10929
11643
|
format: "rgb"
|
|
10930
11644
|
},
|
|
11645
|
+
shouldMaterializeFrame: () => !admissionClosed,
|
|
10931
11646
|
...useLocalFrameRef ? {
|
|
10932
11647
|
frameRegistry: localFrameRegistry,
|
|
10933
11648
|
inlineWhen: () => runner.detectionBackpressure(config.deviceId),
|
|
10934
11649
|
inlineViewSpec: SPLIT_DETECTION_VIEW,
|
|
10935
11650
|
onFrameRef: (frame, nativeCrop) => {
|
|
10936
|
-
if (
|
|
11651
|
+
if (admissionClosed) return false;
|
|
10937
11652
|
const handle = writeRetainedDetectionFrame(retainedFrames, config.deviceId, frame, this.retention);
|
|
10938
11653
|
if (handle && nativeCrop) {
|
|
10939
11654
|
this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
|
|
10940
11655
|
this.bindFrameCutter(handle, nativeCrop);
|
|
10941
11656
|
}
|
|
10942
|
-
const admitted =
|
|
11657
|
+
const admitted = admit(frame, handle);
|
|
10943
11658
|
if (admitted) nativeCrop?.markInferred();
|
|
10944
|
-
count++;
|
|
10945
|
-
if (count >= frames) {
|
|
10946
|
-
clearTimeout(safetyTimer);
|
|
10947
|
-
cleanup();
|
|
10948
|
-
}
|
|
10949
11659
|
return admitted;
|
|
10950
11660
|
}
|
|
10951
11661
|
} : { inlineViewSpec: SPLIT_DETECTION_VIEW },
|
|
10952
11662
|
acquireSpawnSlot: this.buildDecodeSpawnAcquire(log, "occupancy"),
|
|
10953
|
-
onEnded: () =>
|
|
11663
|
+
onEnded: () => closeAdmission(),
|
|
10954
11664
|
onDecodedFrame: (frame, nativeCrop) => {
|
|
10955
|
-
if (
|
|
11665
|
+
if (admissionClosed) return;
|
|
10956
11666
|
const handle = writeRetainedDetectionFrame(retainedFrames, config.deviceId, frame, this.retention);
|
|
10957
11667
|
if (handle && nativeCrop) {
|
|
10958
11668
|
this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
|
|
10959
11669
|
this.bindFrameCutter(handle, nativeCrop);
|
|
10960
11670
|
}
|
|
10961
|
-
if (
|
|
10962
|
-
count++;
|
|
10963
|
-
if (count >= frames) {
|
|
10964
|
-
clearTimeout(safetyTimer);
|
|
10965
|
-
cleanup();
|
|
10966
|
-
}
|
|
11671
|
+
if (admit(frame, handle)) nativeCrop?.markInferred();
|
|
10967
11672
|
}
|
|
10968
11673
|
});
|
|
10969
11674
|
}
|
|
10970
11675
|
});
|
|
10971
|
-
if (
|
|
11676
|
+
if (admissionClosed) {
|
|
10972
11677
|
clearTimeout(safetyTimer);
|
|
10973
|
-
teardown();
|
|
11678
|
+
teardown("drain");
|
|
10974
11679
|
}
|
|
10975
11680
|
}
|
|
10976
11681
|
emitInferenceResult(deviceId, frame, result, handle, source, occasion) {
|