@camstack/addon-pipeline 1.2.124 → 1.2.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-DjiHGIRF.js → addon-utils-BnprivSs.js} +1 -1
- package/dist/audio-analyzer/index.js +3 -3
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/detection-pipeline/index.js +5 -5
- package/dist/detection-pipeline/index.mjs +3 -3
- package/dist/{dist-DomdkRKL.js → dist-BZUXb5S5.js} +26 -4
- package/dist/{dist-B7rIp5qQ.mjs → dist-HLAOw6Pe.mjs} +26 -4
- package/dist/{event-loop-stall-monitor-mvxKgrNV.mjs → event-loop-stall-monitor-BH-cL_hg.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-ChA4AQ1D.js → event-loop-stall-monitor-BdArVUlx.js} +1 -1
- package/dist/{lazy-sharp-BSeW3qY5.js → lazy-sharp-BxMwCkCA.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +972 -871
- package/dist/pipeline-runner/index.mjs +971 -870
- package/dist/{process-memory-XHs-umae.mjs → process-memory-CDPRLalN.mjs} +1 -1
- package/dist/{process-memory-BtpLNnXq.js → process-memory-Dh1SyzdJ.js} +1 -1
- package/dist/recorder/index.js +71 -65
- package/dist/recorder/index.mjs +70 -64
- package/dist/{segment-demux-js-Cc5Jubdu.mjs → segment-demux-js-D-TLcHW5.mjs} +1 -1
- package/dist/{segment-demux-js-CFIIJsfc.js → segment-demux-js-DRutggd6.js} +1 -1
- package/dist/session-decode/decode-worker-child.js +2 -2
- package/dist/session-decode/decode-worker-child.mjs +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-P-WP2KSi.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BDR5z0Z9.mjs} +1 -1
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-D6O20fld.mjs → hostInit-Dmw5LWmP.mjs} +1 -1
- package/dist/stream-broker/index.js +2 -2
- package/dist/stream-broker/index.mjs +2 -2
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-DQ5DPOFD.js → worker-protocol-BxkCoVjw.js} +1 -1
- package/dist/{worker-protocol-CrHJXGHV.mjs → worker-protocol-h7Smb_0i.mjs} +1 -1
- package/package.json +1 -1
|
@@ -2,11 +2,11 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_dist = require("../dist-
|
|
6
|
-
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-
|
|
7
|
-
const require_worker_protocol = require("../worker-protocol-
|
|
5
|
+
const require_dist = require("../dist-BZUXb5S5.js");
|
|
6
|
+
const require_event_loop_stall_monitor = require("../event-loop-stall-monitor-BdArVUlx.js");
|
|
7
|
+
const require_worker_protocol = require("../worker-protocol-BxkCoVjw.js");
|
|
8
8
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
9
|
-
const require_lazy_sharp = require("../lazy-sharp-
|
|
9
|
+
const require_lazy_sharp = require("../lazy-sharp-BxMwCkCA.js");
|
|
10
10
|
const require_restream_intent = require("../restream-intent-Cv9x3jmu.js");
|
|
11
11
|
const require_remote_restream = require("../remote-restream-CO36Sr30.js");
|
|
12
12
|
let sharp = require("sharp");
|
|
@@ -201,6 +201,158 @@ function startGuardedSessionDecode(deps) {
|
|
|
201
201
|
if (release) release();
|
|
202
202
|
};
|
|
203
203
|
}
|
|
204
|
+
/**
|
|
205
|
+
* RAM this node lets live decode workers commit, in MB.
|
|
206
|
+
*
|
|
207
|
+
* ## Why 8192, and why 6144 was wrong
|
|
208
|
+
*
|
|
209
|
+
* 6144 shipped first, derived from the SHIPPED lease defaults (1024 + 64 + 350
|
|
210
|
+
* = 1438 MB per worker) and a 13-minute concurrency sample. Both inputs were
|
|
211
|
+
* wrong for the cluster it ran on, in opposite directions:
|
|
212
|
+
*
|
|
213
|
+
* - the operator here runs `budgetMb: 512`, so a worker is worst-cased at
|
|
214
|
+
* **926 MB**, not 1438 — the old count of 8 was authorising 7.4 GiB, not the
|
|
215
|
+
* 11.5 GiB the defaults implied;
|
|
216
|
+
* - the sample missed the peak. Measured across the change: at ceiling 8 the
|
|
217
|
+
* cluster refused 4 sessions in 30 minutes; at the ceiling 6144 derived
|
|
218
|
+
* (6 workers at 926 MB) it refused **43 in 10 minutes**, and the hub — which
|
|
219
|
+
* had never refused once — began refusing too. Demand genuinely reaches 8.
|
|
220
|
+
*
|
|
221
|
+
* A refusal is self-healing but it is not free: the retry ladder means the
|
|
222
|
+
* camera decodes a motion window that has moved on. Trading real coverage for a
|
|
223
|
+
* RAM headroom nobody measured a need for is the wrong side of the trade.
|
|
224
|
+
*
|
|
225
|
+
* 8192 restores exactly the concurrency this cluster had (`floor(8192/926) = 8`)
|
|
226
|
+
* while keeping what the change was actually for: the bound is RAM, so raising
|
|
227
|
+
* `budgetMb` back to the shipped 1024 now DROPS the ceiling to 5 instead of
|
|
228
|
+
* silently authorising 11.5 GiB, and the refusal line names the megabytes.
|
|
229
|
+
*/
|
|
230
|
+
var DEFAULT_NODE_DECODE_BUDGET_MB = 8192;
|
|
231
|
+
/**
|
|
232
|
+
* Worst-case RAM one decode worker may commit, from the lease knobs actually in
|
|
233
|
+
* force. The SINGLE place the operator-facing budgets become the number the
|
|
234
|
+
* ceiling is derived from — computing it anywhere else is exactly how the count
|
|
235
|
+
* and the budget drifted apart.
|
|
236
|
+
*
|
|
237
|
+
* `override` carries only the knobs the operator SET (see
|
|
238
|
+
* `native-lease-config.ts` on why "unset" includes a knob sitting at its
|
|
239
|
+
* default); anything absent falls back to what the worker ships with.
|
|
240
|
+
*/
|
|
241
|
+
function decodeWorkerFootprintMb(override) {
|
|
242
|
+
return (override.budgetMb ?? require_dist.DEFAULT_NATIVE_LEASE_SETTINGS.budgetMb) + (override.tileBudgetMb ?? require_dist.DEFAULT_NATIVE_LEASE_SETTINGS.tileBudgetMb) + 350;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* The node's live-decode-worker budget.
|
|
246
|
+
*
|
|
247
|
+
* One instance per runner addon. Not a semaphore: a semaphore's contract is
|
|
248
|
+
* "wait", and waiting is the thing this deliberately does not do — see
|
|
249
|
+
* {@link DEFAULT_MAX_LIVE_DECODE_WORKERS}.
|
|
250
|
+
*/
|
|
251
|
+
var LiveDecodeWorkerAdmission = class {
|
|
252
|
+
limit;
|
|
253
|
+
logger;
|
|
254
|
+
live = 0;
|
|
255
|
+
nodeBudgetMb;
|
|
256
|
+
workerFootprintMb;
|
|
257
|
+
constructor(options) {
|
|
258
|
+
this.limit = Math.max(1, options.limit);
|
|
259
|
+
this.logger = options.logger;
|
|
260
|
+
this.nodeBudgetMb = positiveOrUndefined(options.nodeBudgetMb);
|
|
261
|
+
this.workerFootprintMb = positiveOrUndefined(options.workerFootprintMb);
|
|
262
|
+
}
|
|
263
|
+
/** Workers currently holding a slot. */
|
|
264
|
+
get inFlight() {
|
|
265
|
+
return this.live;
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* The ceiling actually in force: the RAM budget divided by one worker\'s worst
|
|
269
|
+
* case, clamped by the hand-set count.
|
|
270
|
+
*
|
|
271
|
+
* Never zero. A node too small for even one worker still decodes one camera —
|
|
272
|
+
* refusing every camera turns a memory bound into a total outage, and one
|
|
273
|
+
* worker is the smallest unit of "this node does its job at all".
|
|
274
|
+
*/
|
|
275
|
+
get ceiling() {
|
|
276
|
+
const fromBudget = this.budgetDerivedCeiling();
|
|
277
|
+
return fromBudget === void 0 ? this.limit : Math.max(1, Math.min(this.limit, fromBudget));
|
|
278
|
+
}
|
|
279
|
+
/** MB currently committed by the workers holding a slot. */
|
|
280
|
+
get committedMb() {
|
|
281
|
+
return this.workerFootprintMb === void 0 ? void 0 : this.live * this.workerFootprintMb;
|
|
282
|
+
}
|
|
283
|
+
budgetDerivedCeiling() {
|
|
284
|
+
if (this.nodeBudgetMb === void 0 || this.workerFootprintMb === void 0) return void 0;
|
|
285
|
+
return Math.floor(this.nodeBudgetMb / this.workerFootprintMb);
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Apply a changed per-worker footprint — the operator moved `budgetMb` or
|
|
289
|
+
* `tileBudgetMb`. Like {@link resize} this never kills a running session; it
|
|
290
|
+
* only changes what the next admission is measured against.
|
|
291
|
+
*/
|
|
292
|
+
resizeFootprint(workerFootprintMb) {
|
|
293
|
+
this.workerFootprintMb = positiveOrUndefined(workerFootprintMb);
|
|
294
|
+
}
|
|
295
|
+
/** Apply a changed node decode budget. */
|
|
296
|
+
resizeBudget(nodeBudgetMb) {
|
|
297
|
+
this.nodeBudgetMb = positiveOrUndefined(nodeBudgetMb);
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Apply an operator-changed ceiling.
|
|
301
|
+
*
|
|
302
|
+
* Lowering it never kills a running session — a decode worker torn down
|
|
303
|
+
* mid-motion loses the event it was started for. It only stops the next
|
|
304
|
+
* admission, so the count drains to the new ceiling as sessions end.
|
|
305
|
+
*/
|
|
306
|
+
resize(limit) {
|
|
307
|
+
this.limit = Math.max(1, limit);
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Take a slot, or refuse.
|
|
311
|
+
*
|
|
312
|
+
* A refusal is a WARN and carries `tags.deviceId`, because the only useful
|
|
313
|
+
* form of this question is per-camera ("why did 617 not run when 615 did?").
|
|
314
|
+
* A branch that drops work silently reads as "it never happened".
|
|
315
|
+
*/
|
|
316
|
+
admit(deviceId, label) {
|
|
317
|
+
const ceiling = this.ceiling;
|
|
318
|
+
if (this.live >= ceiling) {
|
|
319
|
+
this.logger.warn("session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
|
|
320
|
+
tags: { deviceId },
|
|
321
|
+
meta: {
|
|
322
|
+
label,
|
|
323
|
+
live: this.live,
|
|
324
|
+
ceiling,
|
|
325
|
+
...this.committedMb === void 0 ? {} : {
|
|
326
|
+
committedMb: this.committedMb,
|
|
327
|
+
budgetMb: this.nodeBudgetMb,
|
|
328
|
+
workerFootprintMb: this.workerFootprintMb
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
});
|
|
332
|
+
return null;
|
|
333
|
+
}
|
|
334
|
+
this.live += 1;
|
|
335
|
+
this.logger.debug("session-decode: live decode-worker slot taken", {
|
|
336
|
+
tags: { deviceId },
|
|
337
|
+
meta: {
|
|
338
|
+
label,
|
|
339
|
+
live: this.live,
|
|
340
|
+
ceiling: this.ceiling
|
|
341
|
+
}
|
|
342
|
+
});
|
|
343
|
+
let released = false;
|
|
344
|
+
return { release: () => {
|
|
345
|
+
if (released) return;
|
|
346
|
+
released = true;
|
|
347
|
+
this.live -= 1;
|
|
348
|
+
} };
|
|
349
|
+
}
|
|
350
|
+
};
|
|
351
|
+
/** A knob that is absent, zero or nonsense derives nothing rather than a bound nobody chose. */
|
|
352
|
+
function positiveOrUndefined(value) {
|
|
353
|
+
if (value === void 0) return void 0;
|
|
354
|
+
return Number.isFinite(value) && value > 0 ? value : void 0;
|
|
355
|
+
}
|
|
204
356
|
//#endregion
|
|
205
357
|
//#region src/session-decode/native-crop-registry.ts
|
|
206
358
|
/**
|
|
@@ -1713,589 +1865,171 @@ function buildBenchRunPipelineInput(req) {
|
|
|
1713
1865
|
};
|
|
1714
1866
|
}
|
|
1715
1867
|
//#endregion
|
|
1716
|
-
//#region src/pipeline-runner/
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1868
|
+
//#region src/pipeline-runner/capacity/capacity-runner.ts
|
|
1869
|
+
/**
|
|
1870
|
+
* Run ONE ramp step: `cameras` independent open-loop pacers for
|
|
1871
|
+
* `settleMs + measureMs`, then a bounded drain of in-flight calls.
|
|
1872
|
+
*/
|
|
1873
|
+
async function runCapacityStep(opts) {
|
|
1874
|
+
const cameras = Math.max(1, Math.floor(opts.cameras));
|
|
1875
|
+
const intervalMs = 1e3 / Math.max(.1, opts.fpsPerCamera);
|
|
1876
|
+
const start = opts.now();
|
|
1877
|
+
const measureStart = start + Math.max(0, opts.settleMs);
|
|
1878
|
+
const end = measureStart + Math.max(0, opts.measureMs);
|
|
1879
|
+
const c = {
|
|
1880
|
+
offered: 0,
|
|
1881
|
+
dispatched: 0,
|
|
1882
|
+
completed: 0,
|
|
1883
|
+
droppedByDriver: 0,
|
|
1884
|
+
shed: 0,
|
|
1885
|
+
firstClassShed: 0,
|
|
1886
|
+
fallbackShed: 0,
|
|
1887
|
+
errors: 0,
|
|
1888
|
+
aborted: false,
|
|
1889
|
+
outstanding: 0,
|
|
1890
|
+
e2eMsList: [],
|
|
1891
|
+
inferMsList: [],
|
|
1892
|
+
predictMsList: [],
|
|
1893
|
+
detectionCounts: []
|
|
1741
1894
|
};
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
this.node.framesAdmitted += 1;
|
|
1752
|
-
this.camera(deviceId).framesAdmitted += 1;
|
|
1753
|
-
}
|
|
1754
|
-
noteDroppedPixelFree(deviceId) {
|
|
1755
|
-
this.node.framesDroppedPixelFree += 1;
|
|
1756
|
-
this.camera(deviceId).framesDroppedPixelFree += 1;
|
|
1757
|
-
}
|
|
1758
|
-
noteView(input) {
|
|
1759
|
-
if (input.materialized) {
|
|
1760
|
-
this.node.viewsMaterialized += 1;
|
|
1761
|
-
this.camera(input.deviceId).viewsMaterialized += 1;
|
|
1762
|
-
} else {
|
|
1763
|
-
this.node.viewsSkipped += 1;
|
|
1764
|
-
this.camera(input.deviceId).viewsSkipped += 1;
|
|
1765
|
-
}
|
|
1766
|
-
this.node.workerToRunnerBytes += input.workerToRunnerBytes;
|
|
1767
|
-
this.camera(input.deviceId).workerToRunnerBytes += input.workerToRunnerBytes;
|
|
1768
|
-
}
|
|
1769
|
-
notePoolBytes(input) {
|
|
1770
|
-
this.node.runnerToPoolRawBytes += input.raw;
|
|
1771
|
-
this.node.runnerToPoolJpegBytes += input.jpeg;
|
|
1772
|
-
this.camera(input.deviceId).runnerToPoolRawBytes += input.raw;
|
|
1773
|
-
this.camera(input.deviceId).runnerToPoolJpegBytes += input.jpeg;
|
|
1774
|
-
}
|
|
1775
|
-
noteOnDemandRequest(input) {
|
|
1776
|
-
if (input.kind === "full-frame") {
|
|
1777
|
-
this.node.onDemandFullFrameRequests += 1;
|
|
1778
|
-
this.camera(input.deviceId).onDemandFullFrameRequests += 1;
|
|
1779
|
-
} else {
|
|
1780
|
-
this.node.onDemandCropRequests += 1;
|
|
1781
|
-
this.camera(input.deviceId).onDemandCropRequests += 1;
|
|
1895
|
+
const recordOutcome = (counted, outcome, e2eMs) => {
|
|
1896
|
+
if (!counted) return;
|
|
1897
|
+
if (outcome.kind === "ok") {
|
|
1898
|
+
c.completed += 1;
|
|
1899
|
+
c.e2eMsList.push(e2eMs);
|
|
1900
|
+
c.inferMsList.push(outcome.inferMs ?? 0);
|
|
1901
|
+
c.predictMsList.push(outcome.predictMs ?? 0);
|
|
1902
|
+
c.detectionCounts.push(outcome.detections ?? 0);
|
|
1903
|
+
return;
|
|
1782
1904
|
}
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
noteMiss(kind, deviceId) {
|
|
1789
|
-
this.addMiss(this.node, kind, 1);
|
|
1790
|
-
this.addMiss(this.camera(deviceId), kind, 1);
|
|
1791
|
-
}
|
|
1792
|
-
noteAvoidedWrite(deviceId) {
|
|
1793
|
-
this.node.retainedWritesAvoided += 1;
|
|
1794
|
-
this.camera(deviceId).retainedWritesAvoided += 1;
|
|
1795
|
-
}
|
|
1796
|
-
noteRelease(deviceId) {
|
|
1797
|
-
this.node.releases += 1;
|
|
1798
|
-
if (deviceId !== void 0) this.camera(deviceId).releases += 1;
|
|
1799
|
-
}
|
|
1800
|
-
noteEviction(deviceId) {
|
|
1801
|
-
this.node.evictions += 1;
|
|
1802
|
-
if (deviceId !== void 0) this.camera(deviceId).evictions += 1;
|
|
1803
|
-
}
|
|
1804
|
-
noteStale(deviceId) {
|
|
1805
|
-
this.node.staleMisses += 1;
|
|
1806
|
-
if (deviceId !== void 0) this.camera(deviceId).staleMisses += 1;
|
|
1807
|
-
}
|
|
1808
|
-
/**
|
|
1809
|
-
* Node occupancy. `residentRefs` is entry count (pending + store), not
|
|
1810
|
-
* outstanding registry leases.
|
|
1811
|
-
*/
|
|
1812
|
-
setResident(input) {
|
|
1813
|
-
this.node.residentRefs = input.refs;
|
|
1814
|
-
this.node.residentBytes = input.bytes;
|
|
1815
|
-
}
|
|
1816
|
-
setCameraResident(deviceId, input) {
|
|
1817
|
-
const counters = this.camera(deviceId);
|
|
1818
|
-
counters.residentRefs = input.refs;
|
|
1819
|
-
counters.residentBytes = input.bytes;
|
|
1820
|
-
}
|
|
1821
|
-
forgetCamera(deviceId) {
|
|
1822
|
-
this.cameras.delete(deviceId);
|
|
1823
|
-
}
|
|
1824
|
-
/**
|
|
1825
|
-
* Zero occupancy for already-tracked cameras absent from `present`.
|
|
1826
|
-
* Does not create or delete entries — snapshot still does not prune,
|
|
1827
|
-
* and detach still goes through {@link forgetCamera}.
|
|
1828
|
-
*/
|
|
1829
|
-
zeroAbsentResidents(present) {
|
|
1830
|
-
for (const [deviceId, counters] of this.cameras) {
|
|
1831
|
-
if (present.has(deviceId)) continue;
|
|
1832
|
-
counters.residentRefs = 0;
|
|
1833
|
-
counters.residentBytes = 0;
|
|
1905
|
+
if (outcome.kind === "shed") {
|
|
1906
|
+
c.shed += 1;
|
|
1907
|
+
if (outcome.shedSignal === "zero-detections-fallback") c.fallbackShed += 1;
|
|
1908
|
+
else c.firstClassShed += 1;
|
|
1909
|
+
return;
|
|
1834
1910
|
}
|
|
1835
|
-
|
|
1836
|
-
snapshot(deviceIds) {
|
|
1837
|
-
return {
|
|
1838
|
-
node: { ...this.node },
|
|
1839
|
-
cameras: deviceIds.map((deviceId) => ({
|
|
1840
|
-
deviceId,
|
|
1841
|
-
...this.camera(deviceId)
|
|
1842
|
-
}))
|
|
1843
|
-
};
|
|
1844
|
-
}
|
|
1845
|
-
trackedCameraCount() {
|
|
1846
|
-
return this.cameras.size;
|
|
1847
|
-
}
|
|
1848
|
-
camera(deviceId) {
|
|
1849
|
-
const existing = this.cameras.get(deviceId);
|
|
1850
|
-
if (existing) return existing;
|
|
1851
|
-
const created = emptyFrameLazyCounters();
|
|
1852
|
-
this.cameras.set(deviceId, created);
|
|
1853
|
-
return created;
|
|
1854
|
-
}
|
|
1855
|
-
addHit(counters, kind, n) {
|
|
1856
|
-
if (kind === "native") counters.nativeHits += n;
|
|
1857
|
-
else if (kind === "tile") counters.tileHits += n;
|
|
1858
|
-
else counters.fallbackHits += n;
|
|
1859
|
-
}
|
|
1860
|
-
addMiss(counters, kind, n) {
|
|
1861
|
-
if (kind === "native") counters.nativeMisses += n;
|
|
1862
|
-
else if (kind === "tile") counters.tileMisses += n;
|
|
1863
|
-
else counters.fallbackMisses += n;
|
|
1864
|
-
}
|
|
1865
|
-
};
|
|
1866
|
-
//#endregion
|
|
1867
|
-
//#region src/pipeline-runner/frame-ref-transport.ts
|
|
1868
|
-
/** A local ref never leaves the grouped process; inline remains rollback/split compatibility. */
|
|
1869
|
-
function selectExecutorFrameSource(coLocated, frameRef, inlineFrame) {
|
|
1870
|
-
return coLocated && frameRef ? { frameRef } : { frame: inlineFrame };
|
|
1871
|
-
}
|
|
1872
|
-
//#endregion
|
|
1873
|
-
//#region src/session-decode/on-demand-jpeg-fallback.ts
|
|
1874
|
-
/** Single content-sized JPEG requested after a detection, never a continuous raw. */
|
|
1875
|
-
var ON_DEMAND_JPEG_VIEW = {
|
|
1876
|
-
content: {
|
|
1877
|
-
width: 640,
|
|
1878
|
-
height: 640
|
|
1879
|
-
},
|
|
1880
|
-
fit: "contain",
|
|
1881
|
-
format: "jpeg"
|
|
1882
|
-
};
|
|
1883
|
-
function createOnDemandJpegFallback(registry, ref) {
|
|
1884
|
-
const lease = registry.acquire(ref);
|
|
1885
|
-
let cached;
|
|
1886
|
-
let released = false;
|
|
1887
|
-
const releaseLease = (reason) => {
|
|
1888
|
-
if (released) return;
|
|
1889
|
-
released = true;
|
|
1890
|
-
lease?.release(reason);
|
|
1911
|
+
c.errors += 1;
|
|
1891
1912
|
};
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1913
|
+
const measureMarker = opts.onMeasureStart !== void 0 && opts.settleMs > 0 ? opts.sleepUntil(measureStart).then(() => opts.onMeasureStart?.()) : (opts.onMeasureStart?.(), Promise.resolve());
|
|
1914
|
+
const cameraLoop = async (cameraIndex) => {
|
|
1915
|
+
const offset = cameraIndex / cameras * intervalMs;
|
|
1916
|
+
let inFlight = false;
|
|
1917
|
+
for (let k = 0;; k++) {
|
|
1918
|
+
const tickAt = start + offset + k * intervalMs;
|
|
1919
|
+
if (tickAt >= end) break;
|
|
1920
|
+
if (opts.shouldStop()) {
|
|
1921
|
+
c.aborted = true;
|
|
1922
|
+
break;
|
|
1898
1923
|
}
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
releaseLease("drop");
|
|
1904
|
-
return null;
|
|
1905
|
-
}
|
|
1906
|
-
cached = {
|
|
1907
|
-
data: Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength),
|
|
1908
|
-
width: view.width,
|
|
1909
|
-
height: view.height,
|
|
1910
|
-
format: "jpeg",
|
|
1911
|
-
timestamp: ref.timestamp,
|
|
1912
|
-
...ref.capturedAt !== void 0 ? { capturedAt: ref.capturedAt } : {}
|
|
1913
|
-
};
|
|
1914
|
-
releaseLease("success");
|
|
1915
|
-
return cached;
|
|
1916
|
-
} catch {
|
|
1917
|
-
cached = null;
|
|
1918
|
-
releaseLease("error");
|
|
1919
|
-
return null;
|
|
1924
|
+
if (tickAt > opts.now()) await opts.sleepUntil(tickAt);
|
|
1925
|
+
if (opts.shouldStop()) {
|
|
1926
|
+
c.aborted = true;
|
|
1927
|
+
break;
|
|
1920
1928
|
}
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1929
|
+
const counted = tickAt >= measureStart;
|
|
1930
|
+
if (counted) c.offered += 1;
|
|
1931
|
+
if (inFlight) {
|
|
1932
|
+
if (counted) c.droppedByDriver += 1;
|
|
1933
|
+
continue;
|
|
1934
|
+
}
|
|
1935
|
+
inFlight = true;
|
|
1936
|
+
if (counted) c.dispatched += 1;
|
|
1937
|
+
const t0 = opts.now();
|
|
1938
|
+
c.outstanding += 1;
|
|
1939
|
+
opts.dispatch(cameraIndex).then((outcome) => recordOutcome(counted, outcome, opts.now() - t0), () => {
|
|
1940
|
+
if (counted) c.errors += 1;
|
|
1941
|
+
}).finally(() => {
|
|
1942
|
+
inFlight = false;
|
|
1943
|
+
c.outstanding -= 1;
|
|
1944
|
+
});
|
|
1924
1945
|
}
|
|
1925
1946
|
};
|
|
1947
|
+
await Promise.all(Array.from({ length: cameras }, (_, i) => cameraLoop(i)));
|
|
1948
|
+
await measureMarker;
|
|
1949
|
+
const drainDeadline = opts.now() + Math.max(0, opts.drainTimeoutMs);
|
|
1950
|
+
while (c.outstanding > 0 && opts.now() < drainDeadline) await opts.sleepUntil(Math.min(drainDeadline, opts.now() + 50));
|
|
1951
|
+
return {
|
|
1952
|
+
offered: c.offered,
|
|
1953
|
+
dispatched: c.dispatched,
|
|
1954
|
+
completed: c.completed,
|
|
1955
|
+
droppedByDriver: c.droppedByDriver,
|
|
1956
|
+
shed: c.shed,
|
|
1957
|
+
firstClassShed: c.firstClassShed,
|
|
1958
|
+
fallbackShed: c.fallbackShed,
|
|
1959
|
+
errors: c.errors,
|
|
1960
|
+
unresolvedAtDrain: c.outstanding,
|
|
1961
|
+
e2eMsList: c.e2eMsList,
|
|
1962
|
+
inferMsList: c.inferMsList,
|
|
1963
|
+
predictMsList: c.predictMsList,
|
|
1964
|
+
detectionCounts: c.detectionCounts,
|
|
1965
|
+
wallMs: opts.now() - start,
|
|
1966
|
+
aborted: c.aborted
|
|
1967
|
+
};
|
|
1926
1968
|
}
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
function
|
|
1934
|
-
if (
|
|
1935
|
-
|
|
1936
|
-
return
|
|
1969
|
+
var DEFAULT_CAPACITY_SLO = {
|
|
1970
|
+
dropRatioMax: .01,
|
|
1971
|
+
e2eP95MsMax: 250,
|
|
1972
|
+
stallsMax: 0
|
|
1973
|
+
};
|
|
1974
|
+
/** Nearest-rank percentile on an unsorted list. Returns 0 for empty input. */
|
|
1975
|
+
function percentile(values, p) {
|
|
1976
|
+
if (values.length === 0) return 0;
|
|
1977
|
+
const sorted = [...values].toSorted((a, b) => a - b);
|
|
1978
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))] ?? 0;
|
|
1937
1979
|
}
|
|
1938
|
-
function
|
|
1939
|
-
if (
|
|
1940
|
-
return
|
|
1980
|
+
function mean(values) {
|
|
1981
|
+
if (values.length === 0) return 0;
|
|
1982
|
+
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
1941
1983
|
}
|
|
1942
|
-
|
|
1943
|
-
|
|
1984
|
+
/**
|
|
1985
|
+
* Judge one step against the SLO. Pure — the whole gate is testable without
|
|
1986
|
+
* a clock or a node. An error or an unresolved call is never "just a drop":
|
|
1987
|
+
* it means something other than flow control broke, so it fails the step
|
|
1988
|
+
* regardless of ratio (spec §1: errors 0 tolerated).
|
|
1989
|
+
*/
|
|
1990
|
+
function evaluateCapacityStep(raw, stall, slo) {
|
|
1991
|
+
const dropRatio = raw.offered > 0 ? (raw.droppedByDriver + raw.shed) / raw.offered : 0;
|
|
1992
|
+
const e2eMs = {
|
|
1993
|
+
p50: percentile(raw.e2eMsList, .5),
|
|
1994
|
+
p95: percentile(raw.e2eMsList, .95),
|
|
1995
|
+
p99: percentile(raw.e2eMsList, .99),
|
|
1996
|
+
max: raw.e2eMsList.length > 0 ? Math.max(...raw.e2eMsList) : 0
|
|
1997
|
+
};
|
|
1998
|
+
const failReasons = [];
|
|
1999
|
+
if (dropRatio > slo.dropRatioMax) failReasons.push(`dropRatio ${(dropRatio * 100).toFixed(2)}% > ${(slo.dropRatioMax * 100).toFixed(2)}%`);
|
|
2000
|
+
if (e2eMs.p95 > slo.e2eP95MsMax) failReasons.push(`e2e p95 ${Math.round(e2eMs.p95)}ms > ${slo.e2eP95MsMax}ms`);
|
|
2001
|
+
if (stall.count > slo.stallsMax) failReasons.push(`event-loop stalls ${stall.count} > ${slo.stallsMax} (max ${stall.maxMs}ms)`);
|
|
2002
|
+
if (raw.errors > 0) failReasons.push(`errors ${raw.errors} (0 tolerated)`);
|
|
2003
|
+
if (raw.unresolvedAtDrain > 0) failReasons.push(`unresolved calls at drain ${raw.unresolvedAtDrain}`);
|
|
2004
|
+
return {
|
|
2005
|
+
dropRatio,
|
|
2006
|
+
e2eMs,
|
|
2007
|
+
inferMs: {
|
|
2008
|
+
avg: mean(raw.inferMsList),
|
|
2009
|
+
p95: percentile(raw.inferMsList, .95)
|
|
2010
|
+
},
|
|
2011
|
+
predictMs: {
|
|
2012
|
+
avg: mean(raw.predictMsList),
|
|
2013
|
+
p95: percentile(raw.predictMsList, .95)
|
|
2014
|
+
},
|
|
2015
|
+
detectionsAvg: mean(raw.detectionCounts),
|
|
2016
|
+
pass: failReasons.length === 0,
|
|
2017
|
+
failReasons
|
|
2018
|
+
};
|
|
1944
2019
|
}
|
|
1945
2020
|
/**
|
|
1946
|
-
*
|
|
1947
|
-
*
|
|
1948
|
-
*
|
|
2021
|
+
* `N_max` — the last PASSING step's camera count, or null when none passed.
|
|
2022
|
+
* Steps after the first failure do not resurrect the number (the spec stops
|
|
2023
|
+
* at the first failing step; a re-run confirms the knee, not this fold).
|
|
1949
2024
|
*/
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
this.registry = options.registry;
|
|
1959
|
-
this.telemetry = options.telemetry;
|
|
1960
|
-
this.continuousRetain = options.continuousRetain;
|
|
1961
|
-
}
|
|
1962
|
-
get pendingSize() {
|
|
1963
|
-
return this.pending.size;
|
|
1964
|
-
}
|
|
1965
|
-
hasPending(handle) {
|
|
1966
|
-
return this.pending.has(handleKey(handle));
|
|
1967
|
-
}
|
|
1968
|
-
admitFrame(deviceId, frame) {
|
|
1969
|
-
if (this.continuousRetain) {
|
|
1970
|
-
const fallback = frame.frameRef ? createRetainedContentFallback(this.registry, frame.frameRef) : void 0;
|
|
1971
|
-
const handle = this.store.write(deviceId, frame, fallback) ?? void 0;
|
|
1972
|
-
if (handle) this.pending.set(handleKey(handle), {
|
|
1973
|
-
deviceId,
|
|
1974
|
-
ref: frame.frameRef
|
|
1975
|
-
});
|
|
1976
|
-
this.refreshResident();
|
|
1977
|
-
return handle;
|
|
1978
|
-
}
|
|
1979
|
-
const handle = this.store.allocateHandle(deviceId, frame) ?? void 0;
|
|
1980
|
-
if (!handle) return void 0;
|
|
1981
|
-
this.pending.set(handleKey(handle), {
|
|
1982
|
-
deviceId,
|
|
1983
|
-
ref: frame.frameRef
|
|
1984
|
-
});
|
|
1985
|
-
this.telemetry.noteAvoidedWrite(deviceId);
|
|
1986
|
-
this.refreshResident();
|
|
1987
|
-
return handle;
|
|
1988
|
-
}
|
|
1989
|
-
noteFrame(input) {
|
|
1990
|
-
if (input.decoded) this.telemetry.noteDecoded(input.deviceId);
|
|
1991
|
-
if (input.admitted) this.telemetry.noteAdmitted(input.deviceId);
|
|
1992
|
-
if (input.droppedPixelFree) this.telemetry.noteDroppedPixelFree(input.deviceId);
|
|
1993
|
-
}
|
|
1994
|
-
noteInferenceResult(deviceId, handle, result) {
|
|
1995
|
-
if (!handle) return;
|
|
1996
|
-
const key = handleKey(handle);
|
|
1997
|
-
const pending = this.pending.get(key);
|
|
1998
|
-
this.forgetPending(key);
|
|
1999
|
-
if (!shouldRetainDetectionFrame({
|
|
2000
|
-
admitted: true,
|
|
2001
|
-
hasDetection: (result.detections?.length ?? 0) > 0,
|
|
2002
|
-
continuousRetain: this.continuousRetain
|
|
2003
|
-
})) {
|
|
2004
|
-
this.refreshResident();
|
|
2005
|
-
return;
|
|
2006
|
-
}
|
|
2007
|
-
if (this.continuousRetain) {
|
|
2008
|
-
this.refreshResident();
|
|
2009
|
-
return;
|
|
2010
|
-
}
|
|
2011
|
-
if (!pending?.ref) {
|
|
2012
|
-
this.refreshResident();
|
|
2013
|
-
return;
|
|
2014
|
-
}
|
|
2015
|
-
const fallback = createOnDemandJpegFallback(this.registry, pending.ref);
|
|
2016
|
-
this.store.rememberOnDemand(deviceId, handle, fallback);
|
|
2017
|
-
this.refreshResident();
|
|
2018
|
-
}
|
|
2019
|
-
async requestMedia(handle, request) {
|
|
2020
|
-
const deviceId = this.store.getDeviceId(handle) ?? this.pending.get(handleKey(handle))?.deviceId ?? 0;
|
|
2021
|
-
const kind = request.kind === "full-frame" ? "full-frame" : "crop";
|
|
2022
|
-
this.telemetry.noteOnDemandRequest({
|
|
2023
|
-
deviceId,
|
|
2024
|
-
kind
|
|
2025
|
-
});
|
|
2026
|
-
if (request.nativeHit) {
|
|
2027
|
-
this.telemetry.noteHit("native", deviceId);
|
|
2028
|
-
return {
|
|
2029
|
-
tier: "native",
|
|
2030
|
-
bytes: request.nativeHit.bytes,
|
|
2031
|
-
width: request.nativeHit.width,
|
|
2032
|
-
height: request.nativeHit.height
|
|
2033
|
-
};
|
|
2034
|
-
}
|
|
2035
|
-
this.telemetry.noteMiss("native", deviceId);
|
|
2036
|
-
if (request.tileHit) {
|
|
2037
|
-
this.telemetry.noteHit("tile", deviceId);
|
|
2038
|
-
return {
|
|
2039
|
-
tier: "tile",
|
|
2040
|
-
bytes: request.tileHit.bytes,
|
|
2041
|
-
jpeg: request.tileHit.jpeg ? request.tileHit.bytes : void 0,
|
|
2042
|
-
width: request.tileHit.width,
|
|
2043
|
-
height: request.tileHit.height
|
|
2044
|
-
};
|
|
2045
|
-
}
|
|
2046
|
-
this.telemetry.noteMiss("tile", deviceId);
|
|
2047
|
-
if (request.recordedStillHit) return {
|
|
2048
|
-
tier: "recorded-still",
|
|
2049
|
-
jpeg: request.recordedStillHit.jpeg,
|
|
2050
|
-
width: request.recordedStillHit.width,
|
|
2051
|
-
height: request.recordedStillHit.height
|
|
2052
|
-
};
|
|
2053
|
-
if (!this.continuousRetain && !this.store.has(handle)) {
|
|
2054
|
-
this.telemetry.noteMiss("fallback", deviceId);
|
|
2055
|
-
return { tier: "miss" };
|
|
2056
|
-
}
|
|
2057
|
-
const view = await this.store.ensureContent(handle);
|
|
2058
|
-
if (view && view.data.byteLength > 0) {
|
|
2059
|
-
this.telemetry.noteHit("fallback", deviceId);
|
|
2060
|
-
this.refreshResident();
|
|
2061
|
-
if (view.format === "jpeg") return {
|
|
2062
|
-
tier: "fallback-jpeg",
|
|
2063
|
-
jpeg: view.data,
|
|
2064
|
-
width: view.width,
|
|
2065
|
-
height: view.height
|
|
2066
|
-
};
|
|
2067
|
-
return {
|
|
2068
|
-
tier: "fallback-jpeg",
|
|
2069
|
-
bytes: view.data,
|
|
2070
|
-
width: view.width,
|
|
2071
|
-
height: view.height
|
|
2072
|
-
};
|
|
2073
|
-
}
|
|
2074
|
-
this.telemetry.noteMiss("fallback", deviceId);
|
|
2075
|
-
this.telemetry.noteStale(deviceId);
|
|
2076
|
-
return { tier: "miss" };
|
|
2077
|
-
}
|
|
2078
|
-
/**
|
|
2079
|
-
* Pending cleanup. Success never invalidates a store entry already
|
|
2080
|
-
* transferred for late pull. Drop/error/detach drop the store only when the
|
|
2081
|
-
* handle was still pending (not yet transferred). Eviction owns transferred
|
|
2082
|
-
* fallback cleanup.
|
|
2083
|
-
*/
|
|
2084
|
-
release(handle, reason) {
|
|
2085
|
-
const key = handleKey(handle);
|
|
2086
|
-
const pending = this.pending.get(key);
|
|
2087
|
-
this.forgetPending(key);
|
|
2088
|
-
if (reason !== "success" && pending) this.store.drop(handle);
|
|
2089
|
-
this.telemetry.noteRelease(pending?.deviceId ?? this.store.getDeviceId(handle));
|
|
2090
|
-
this.refreshResident();
|
|
2091
|
-
}
|
|
2092
|
-
forgetDevice(deviceId) {
|
|
2093
|
-
for (const [key, pending] of [...this.pending]) {
|
|
2094
|
-
if (pending.deviceId !== deviceId) continue;
|
|
2095
|
-
this.pending.delete(key);
|
|
2096
|
-
}
|
|
2097
|
-
this.telemetry.forgetCamera(deviceId);
|
|
2098
|
-
this.refreshResident();
|
|
2099
|
-
}
|
|
2100
|
-
syncResident() {
|
|
2101
|
-
this.refreshResident();
|
|
2102
|
-
}
|
|
2103
|
-
forgetPending(key) {
|
|
2104
|
-
this.pending.delete(key);
|
|
2105
|
-
}
|
|
2106
|
-
refreshResident() {
|
|
2107
|
-
const byDevice = /* @__PURE__ */ new Map();
|
|
2108
|
-
for (const pending of this.pending.values()) {
|
|
2109
|
-
const cur = byDevice.get(pending.deviceId) ?? {
|
|
2110
|
-
refs: 0,
|
|
2111
|
-
bytes: 0
|
|
2112
|
-
};
|
|
2113
|
-
cur.refs += 1;
|
|
2114
|
-
byDevice.set(pending.deviceId, cur);
|
|
2115
|
-
}
|
|
2116
|
-
for (const stats of this.store.deviceStats()) {
|
|
2117
|
-
const cur = byDevice.get(stats.deviceId) ?? {
|
|
2118
|
-
refs: 0,
|
|
2119
|
-
bytes: 0
|
|
2120
|
-
};
|
|
2121
|
-
cur.refs += stats.entries;
|
|
2122
|
-
cur.bytes += stats.bytes;
|
|
2123
|
-
byDevice.set(stats.deviceId, cur);
|
|
2124
|
-
}
|
|
2125
|
-
this.telemetry.setResident({
|
|
2126
|
-
refs: this.pending.size + this.store.size,
|
|
2127
|
-
bytes: this.registry.estimatedBytes + this.store.retainedBytes
|
|
2128
|
-
});
|
|
2129
|
-
this.telemetry.zeroAbsentResidents(new Set(byDevice.keys()));
|
|
2130
|
-
for (const [deviceId, stats] of byDevice) this.telemetry.setCameraResident(deviceId, stats);
|
|
2131
|
-
}
|
|
2132
|
-
};
|
|
2133
|
-
//#endregion
|
|
2134
|
-
//#region src/pipeline-runner/capacity/capacity-runner.ts
|
|
2135
|
-
/**
|
|
2136
|
-
* Run ONE ramp step: `cameras` independent open-loop pacers for
|
|
2137
|
-
* `settleMs + measureMs`, then a bounded drain of in-flight calls.
|
|
2138
|
-
*/
|
|
2139
|
-
async function runCapacityStep(opts) {
|
|
2140
|
-
const cameras = Math.max(1, Math.floor(opts.cameras));
|
|
2141
|
-
const intervalMs = 1e3 / Math.max(.1, opts.fpsPerCamera);
|
|
2142
|
-
const start = opts.now();
|
|
2143
|
-
const measureStart = start + Math.max(0, opts.settleMs);
|
|
2144
|
-
const end = measureStart + Math.max(0, opts.measureMs);
|
|
2145
|
-
const c = {
|
|
2146
|
-
offered: 0,
|
|
2147
|
-
dispatched: 0,
|
|
2148
|
-
completed: 0,
|
|
2149
|
-
droppedByDriver: 0,
|
|
2150
|
-
shed: 0,
|
|
2151
|
-
firstClassShed: 0,
|
|
2152
|
-
fallbackShed: 0,
|
|
2153
|
-
errors: 0,
|
|
2154
|
-
aborted: false,
|
|
2155
|
-
outstanding: 0,
|
|
2156
|
-
e2eMsList: [],
|
|
2157
|
-
inferMsList: [],
|
|
2158
|
-
predictMsList: [],
|
|
2159
|
-
detectionCounts: []
|
|
2160
|
-
};
|
|
2161
|
-
const recordOutcome = (counted, outcome, e2eMs) => {
|
|
2162
|
-
if (!counted) return;
|
|
2163
|
-
if (outcome.kind === "ok") {
|
|
2164
|
-
c.completed += 1;
|
|
2165
|
-
c.e2eMsList.push(e2eMs);
|
|
2166
|
-
c.inferMsList.push(outcome.inferMs ?? 0);
|
|
2167
|
-
c.predictMsList.push(outcome.predictMs ?? 0);
|
|
2168
|
-
c.detectionCounts.push(outcome.detections ?? 0);
|
|
2169
|
-
return;
|
|
2170
|
-
}
|
|
2171
|
-
if (outcome.kind === "shed") {
|
|
2172
|
-
c.shed += 1;
|
|
2173
|
-
if (outcome.shedSignal === "zero-detections-fallback") c.fallbackShed += 1;
|
|
2174
|
-
else c.firstClassShed += 1;
|
|
2175
|
-
return;
|
|
2176
|
-
}
|
|
2177
|
-
c.errors += 1;
|
|
2178
|
-
};
|
|
2179
|
-
const measureMarker = opts.onMeasureStart !== void 0 && opts.settleMs > 0 ? opts.sleepUntil(measureStart).then(() => opts.onMeasureStart?.()) : (opts.onMeasureStart?.(), Promise.resolve());
|
|
2180
|
-
const cameraLoop = async (cameraIndex) => {
|
|
2181
|
-
const offset = cameraIndex / cameras * intervalMs;
|
|
2182
|
-
let inFlight = false;
|
|
2183
|
-
for (let k = 0;; k++) {
|
|
2184
|
-
const tickAt = start + offset + k * intervalMs;
|
|
2185
|
-
if (tickAt >= end) break;
|
|
2186
|
-
if (opts.shouldStop()) {
|
|
2187
|
-
c.aborted = true;
|
|
2188
|
-
break;
|
|
2189
|
-
}
|
|
2190
|
-
if (tickAt > opts.now()) await opts.sleepUntil(tickAt);
|
|
2191
|
-
if (opts.shouldStop()) {
|
|
2192
|
-
c.aborted = true;
|
|
2193
|
-
break;
|
|
2194
|
-
}
|
|
2195
|
-
const counted = tickAt >= measureStart;
|
|
2196
|
-
if (counted) c.offered += 1;
|
|
2197
|
-
if (inFlight) {
|
|
2198
|
-
if (counted) c.droppedByDriver += 1;
|
|
2199
|
-
continue;
|
|
2200
|
-
}
|
|
2201
|
-
inFlight = true;
|
|
2202
|
-
if (counted) c.dispatched += 1;
|
|
2203
|
-
const t0 = opts.now();
|
|
2204
|
-
c.outstanding += 1;
|
|
2205
|
-
opts.dispatch(cameraIndex).then((outcome) => recordOutcome(counted, outcome, opts.now() - t0), () => {
|
|
2206
|
-
if (counted) c.errors += 1;
|
|
2207
|
-
}).finally(() => {
|
|
2208
|
-
inFlight = false;
|
|
2209
|
-
c.outstanding -= 1;
|
|
2210
|
-
});
|
|
2211
|
-
}
|
|
2212
|
-
};
|
|
2213
|
-
await Promise.all(Array.from({ length: cameras }, (_, i) => cameraLoop(i)));
|
|
2214
|
-
await measureMarker;
|
|
2215
|
-
const drainDeadline = opts.now() + Math.max(0, opts.drainTimeoutMs);
|
|
2216
|
-
while (c.outstanding > 0 && opts.now() < drainDeadline) await opts.sleepUntil(Math.min(drainDeadline, opts.now() + 50));
|
|
2217
|
-
return {
|
|
2218
|
-
offered: c.offered,
|
|
2219
|
-
dispatched: c.dispatched,
|
|
2220
|
-
completed: c.completed,
|
|
2221
|
-
droppedByDriver: c.droppedByDriver,
|
|
2222
|
-
shed: c.shed,
|
|
2223
|
-
firstClassShed: c.firstClassShed,
|
|
2224
|
-
fallbackShed: c.fallbackShed,
|
|
2225
|
-
errors: c.errors,
|
|
2226
|
-
unresolvedAtDrain: c.outstanding,
|
|
2227
|
-
e2eMsList: c.e2eMsList,
|
|
2228
|
-
inferMsList: c.inferMsList,
|
|
2229
|
-
predictMsList: c.predictMsList,
|
|
2230
|
-
detectionCounts: c.detectionCounts,
|
|
2231
|
-
wallMs: opts.now() - start,
|
|
2232
|
-
aborted: c.aborted
|
|
2233
|
-
};
|
|
2234
|
-
}
|
|
2235
|
-
var DEFAULT_CAPACITY_SLO = {
|
|
2236
|
-
dropRatioMax: .01,
|
|
2237
|
-
e2eP95MsMax: 250,
|
|
2238
|
-
stallsMax: 0
|
|
2239
|
-
};
|
|
2240
|
-
/** Nearest-rank percentile on an unsorted list. Returns 0 for empty input. */
|
|
2241
|
-
function percentile(values, p) {
|
|
2242
|
-
if (values.length === 0) return 0;
|
|
2243
|
-
const sorted = [...values].toSorted((a, b) => a - b);
|
|
2244
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))] ?? 0;
|
|
2245
|
-
}
|
|
2246
|
-
function mean(values) {
|
|
2247
|
-
if (values.length === 0) return 0;
|
|
2248
|
-
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
2249
|
-
}
|
|
2250
|
-
/**
|
|
2251
|
-
* Judge one step against the SLO. Pure — the whole gate is testable without
|
|
2252
|
-
* a clock or a node. An error or an unresolved call is never "just a drop":
|
|
2253
|
-
* it means something other than flow control broke, so it fails the step
|
|
2254
|
-
* regardless of ratio (spec §1: errors 0 tolerated).
|
|
2255
|
-
*/
|
|
2256
|
-
function evaluateCapacityStep(raw, stall, slo) {
|
|
2257
|
-
const dropRatio = raw.offered > 0 ? (raw.droppedByDriver + raw.shed) / raw.offered : 0;
|
|
2258
|
-
const e2eMs = {
|
|
2259
|
-
p50: percentile(raw.e2eMsList, .5),
|
|
2260
|
-
p95: percentile(raw.e2eMsList, .95),
|
|
2261
|
-
p99: percentile(raw.e2eMsList, .99),
|
|
2262
|
-
max: raw.e2eMsList.length > 0 ? Math.max(...raw.e2eMsList) : 0
|
|
2263
|
-
};
|
|
2264
|
-
const failReasons = [];
|
|
2265
|
-
if (dropRatio > slo.dropRatioMax) failReasons.push(`dropRatio ${(dropRatio * 100).toFixed(2)}% > ${(slo.dropRatioMax * 100).toFixed(2)}%`);
|
|
2266
|
-
if (e2eMs.p95 > slo.e2eP95MsMax) failReasons.push(`e2e p95 ${Math.round(e2eMs.p95)}ms > ${slo.e2eP95MsMax}ms`);
|
|
2267
|
-
if (stall.count > slo.stallsMax) failReasons.push(`event-loop stalls ${stall.count} > ${slo.stallsMax} (max ${stall.maxMs}ms)`);
|
|
2268
|
-
if (raw.errors > 0) failReasons.push(`errors ${raw.errors} (0 tolerated)`);
|
|
2269
|
-
if (raw.unresolvedAtDrain > 0) failReasons.push(`unresolved calls at drain ${raw.unresolvedAtDrain}`);
|
|
2270
|
-
return {
|
|
2271
|
-
dropRatio,
|
|
2272
|
-
e2eMs,
|
|
2273
|
-
inferMs: {
|
|
2274
|
-
avg: mean(raw.inferMsList),
|
|
2275
|
-
p95: percentile(raw.inferMsList, .95)
|
|
2276
|
-
},
|
|
2277
|
-
predictMs: {
|
|
2278
|
-
avg: mean(raw.predictMsList),
|
|
2279
|
-
p95: percentile(raw.predictMsList, .95)
|
|
2280
|
-
},
|
|
2281
|
-
detectionsAvg: mean(raw.detectionCounts),
|
|
2282
|
-
pass: failReasons.length === 0,
|
|
2283
|
-
failReasons
|
|
2284
|
-
};
|
|
2285
|
-
}
|
|
2286
|
-
/**
|
|
2287
|
-
* `N_max` — the last PASSING step's camera count, or null when none passed.
|
|
2288
|
-
* Steps after the first failure do not resurrect the number (the spec stops
|
|
2289
|
-
* at the first failing step; a re-run confirms the knee, not this fold).
|
|
2290
|
-
*/
|
|
2291
|
-
function computeNMax(steps) {
|
|
2292
|
-
let nMax = null;
|
|
2293
|
-
for (const step of steps) {
|
|
2294
|
-
if (!step.pass) return {
|
|
2295
|
-
nMax,
|
|
2296
|
-
failedStep: step.cameras
|
|
2297
|
-
};
|
|
2298
|
-
nMax = step.cameras;
|
|
2025
|
+
function computeNMax(steps) {
|
|
2026
|
+
let nMax = null;
|
|
2027
|
+
for (const step of steps) {
|
|
2028
|
+
if (!step.pass) return {
|
|
2029
|
+
nMax,
|
|
2030
|
+
failedStep: step.cameras
|
|
2031
|
+
};
|
|
2032
|
+
nMax = step.cameras;
|
|
2299
2033
|
}
|
|
2300
2034
|
return {
|
|
2301
2035
|
nMax,
|
|
@@ -2970,40 +2704,197 @@ function resolveStepDevice(input) {
|
|
|
2970
2704
|
reason: "manual"
|
|
2971
2705
|
};
|
|
2972
2706
|
}
|
|
2973
|
-
if (canRunOn(effectiveDeviceKey, canRun)) return {
|
|
2974
|
-
kind: "no-jump",
|
|
2975
|
-
deviceKey: effectiveDeviceKey
|
|
2976
|
-
};
|
|
2977
|
-
const runnable = input.devices.filter((d) => d.deviceKey !== effectiveDeviceKey && (d.maxSessions === null || d.attachedCameras < d.maxSessions) && canRunOn(d.deviceKey, canRun));
|
|
2978
|
-
if (runnable.length === 0) return {
|
|
2979
|
-
kind: "no-candidate",
|
|
2980
|
-
deviceKey: effectiveDeviceKey
|
|
2981
|
-
};
|
|
2982
|
-
return {
|
|
2983
|
-
kind: "jumped",
|
|
2984
|
-
deviceKey: pickWeightedLeastLoaded(runnable).deviceKey,
|
|
2985
|
-
reason: "auto"
|
|
2986
|
-
};
|
|
2987
|
-
}
|
|
2988
|
-
//#endregion
|
|
2989
|
-
//#region src/pipeline-runner/device-load.ts
|
|
2990
|
-
/** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
|
|
2991
|
-
function backendOf(deviceKey) {
|
|
2992
|
-
const colon = deviceKey.indexOf(":");
|
|
2993
|
-
return colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
|
|
2994
|
-
}
|
|
2995
|
-
function summarizeDeviceLoad(attachedDeviceKeys) {
|
|
2996
|
-
const counts = /* @__PURE__ */ new Map();
|
|
2997
|
-
for (const key of attachedDeviceKeys) {
|
|
2998
|
-
if (!key) continue;
|
|
2999
|
-
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
2707
|
+
if (canRunOn(effectiveDeviceKey, canRun)) return {
|
|
2708
|
+
kind: "no-jump",
|
|
2709
|
+
deviceKey: effectiveDeviceKey
|
|
2710
|
+
};
|
|
2711
|
+
const runnable = input.devices.filter((d) => d.deviceKey !== effectiveDeviceKey && (d.maxSessions === null || d.attachedCameras < d.maxSessions) && canRunOn(d.deviceKey, canRun));
|
|
2712
|
+
if (runnable.length === 0) return {
|
|
2713
|
+
kind: "no-candidate",
|
|
2714
|
+
deviceKey: effectiveDeviceKey
|
|
2715
|
+
};
|
|
2716
|
+
return {
|
|
2717
|
+
kind: "jumped",
|
|
2718
|
+
deviceKey: pickWeightedLeastLoaded(runnable).deviceKey,
|
|
2719
|
+
reason: "auto"
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
2722
|
+
//#endregion
|
|
2723
|
+
//#region src/pipeline-runner/device-load.ts
|
|
2724
|
+
/** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
|
|
2725
|
+
function backendOf(deviceKey) {
|
|
2726
|
+
const colon = deviceKey.indexOf(":");
|
|
2727
|
+
return colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
|
|
2728
|
+
}
|
|
2729
|
+
function summarizeDeviceLoad(attachedDeviceKeys) {
|
|
2730
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2731
|
+
for (const key of attachedDeviceKeys) {
|
|
2732
|
+
if (!key) continue;
|
|
2733
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
2734
|
+
}
|
|
2735
|
+
return Array.from(counts.entries()).map(([deviceKey, attachedCameras]) => ({
|
|
2736
|
+
deviceKey,
|
|
2737
|
+
backend: backendOf(deviceKey),
|
|
2738
|
+
attachedCameras,
|
|
2739
|
+
queueDepthTotal: 0
|
|
2740
|
+
})).toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey));
|
|
2741
|
+
}
|
|
2742
|
+
//#endregion
|
|
2743
|
+
//#region src/pipeline-runner/frame-lazy-metrics.ts
|
|
2744
|
+
function emptyFrameLazyCounters() {
|
|
2745
|
+
return {
|
|
2746
|
+
framesDecoded: 0,
|
|
2747
|
+
framesAdmitted: 0,
|
|
2748
|
+
framesDroppedPixelFree: 0,
|
|
2749
|
+
viewsMaterialized: 0,
|
|
2750
|
+
viewsSkipped: 0,
|
|
2751
|
+
workerToRunnerBytes: 0,
|
|
2752
|
+
runnerToPoolRawBytes: 0,
|
|
2753
|
+
runnerToPoolJpegBytes: 0,
|
|
2754
|
+
onDemandFullFrameRequests: 0,
|
|
2755
|
+
onDemandCropRequests: 0,
|
|
2756
|
+
nativeHits: 0,
|
|
2757
|
+
nativeMisses: 0,
|
|
2758
|
+
tileHits: 0,
|
|
2759
|
+
tileMisses: 0,
|
|
2760
|
+
fallbackHits: 0,
|
|
2761
|
+
fallbackMisses: 0,
|
|
2762
|
+
retainedWritesAvoided: 0,
|
|
2763
|
+
residentRefs: 0,
|
|
2764
|
+
residentBytes: 0,
|
|
2765
|
+
releases: 0,
|
|
2766
|
+
evictions: 0,
|
|
2767
|
+
staleMisses: 0
|
|
2768
|
+
};
|
|
2769
|
+
}
|
|
2770
|
+
var FrameLazyTelemetry = class {
|
|
2771
|
+
node = emptyFrameLazyCounters();
|
|
2772
|
+
cameras = /* @__PURE__ */ new Map();
|
|
2773
|
+
noteDecoded(deviceId) {
|
|
2774
|
+
this.node.framesDecoded += 1;
|
|
2775
|
+
this.camera(deviceId).framesDecoded += 1;
|
|
2776
|
+
}
|
|
2777
|
+
noteAdmitted(deviceId) {
|
|
2778
|
+
this.node.framesAdmitted += 1;
|
|
2779
|
+
this.camera(deviceId).framesAdmitted += 1;
|
|
2780
|
+
}
|
|
2781
|
+
noteDroppedPixelFree(deviceId) {
|
|
2782
|
+
this.node.framesDroppedPixelFree += 1;
|
|
2783
|
+
this.camera(deviceId).framesDroppedPixelFree += 1;
|
|
2784
|
+
}
|
|
2785
|
+
noteView(input) {
|
|
2786
|
+
if (input.materialized) {
|
|
2787
|
+
this.node.viewsMaterialized += 1;
|
|
2788
|
+
this.camera(input.deviceId).viewsMaterialized += 1;
|
|
2789
|
+
} else {
|
|
2790
|
+
this.node.viewsSkipped += 1;
|
|
2791
|
+
this.camera(input.deviceId).viewsSkipped += 1;
|
|
2792
|
+
}
|
|
2793
|
+
this.node.workerToRunnerBytes += input.workerToRunnerBytes;
|
|
2794
|
+
this.camera(input.deviceId).workerToRunnerBytes += input.workerToRunnerBytes;
|
|
2795
|
+
}
|
|
2796
|
+
notePoolBytes(input) {
|
|
2797
|
+
this.node.runnerToPoolRawBytes += input.raw;
|
|
2798
|
+
this.node.runnerToPoolJpegBytes += input.jpeg;
|
|
2799
|
+
this.camera(input.deviceId).runnerToPoolRawBytes += input.raw;
|
|
2800
|
+
this.camera(input.deviceId).runnerToPoolJpegBytes += input.jpeg;
|
|
2801
|
+
}
|
|
2802
|
+
noteOnDemandRequest(input) {
|
|
2803
|
+
if (input.kind === "full-frame") {
|
|
2804
|
+
this.node.onDemandFullFrameRequests += 1;
|
|
2805
|
+
this.camera(input.deviceId).onDemandFullFrameRequests += 1;
|
|
2806
|
+
} else {
|
|
2807
|
+
this.node.onDemandCropRequests += 1;
|
|
2808
|
+
this.camera(input.deviceId).onDemandCropRequests += 1;
|
|
2809
|
+
}
|
|
2810
|
+
}
|
|
2811
|
+
noteHit(kind, deviceId) {
|
|
2812
|
+
this.addHit(this.node, kind, 1);
|
|
2813
|
+
this.addHit(this.camera(deviceId), kind, 1);
|
|
2814
|
+
}
|
|
2815
|
+
noteMiss(kind, deviceId) {
|
|
2816
|
+
this.addMiss(this.node, kind, 1);
|
|
2817
|
+
this.addMiss(this.camera(deviceId), kind, 1);
|
|
2818
|
+
}
|
|
2819
|
+
noteAvoidedWrite(deviceId) {
|
|
2820
|
+
this.node.retainedWritesAvoided += 1;
|
|
2821
|
+
this.camera(deviceId).retainedWritesAvoided += 1;
|
|
2822
|
+
}
|
|
2823
|
+
noteRelease(deviceId) {
|
|
2824
|
+
this.node.releases += 1;
|
|
2825
|
+
if (deviceId !== void 0) this.camera(deviceId).releases += 1;
|
|
2826
|
+
}
|
|
2827
|
+
noteEviction(deviceId) {
|
|
2828
|
+
this.node.evictions += 1;
|
|
2829
|
+
if (deviceId !== void 0) this.camera(deviceId).evictions += 1;
|
|
2830
|
+
}
|
|
2831
|
+
noteStale(deviceId) {
|
|
2832
|
+
this.node.staleMisses += 1;
|
|
2833
|
+
if (deviceId !== void 0) this.camera(deviceId).staleMisses += 1;
|
|
2834
|
+
}
|
|
2835
|
+
/**
|
|
2836
|
+
* Node occupancy. `residentRefs` is entry count (pending + store), not
|
|
2837
|
+
* outstanding registry leases.
|
|
2838
|
+
*/
|
|
2839
|
+
setResident(input) {
|
|
2840
|
+
this.node.residentRefs = input.refs;
|
|
2841
|
+
this.node.residentBytes = input.bytes;
|
|
2842
|
+
}
|
|
2843
|
+
setCameraResident(deviceId, input) {
|
|
2844
|
+
const counters = this.camera(deviceId);
|
|
2845
|
+
counters.residentRefs = input.refs;
|
|
2846
|
+
counters.residentBytes = input.bytes;
|
|
2847
|
+
}
|
|
2848
|
+
forgetCamera(deviceId) {
|
|
2849
|
+
this.cameras.delete(deviceId);
|
|
2850
|
+
}
|
|
2851
|
+
/**
|
|
2852
|
+
* Zero occupancy for already-tracked cameras absent from `present`.
|
|
2853
|
+
* Does not create or delete entries — snapshot still does not prune,
|
|
2854
|
+
* and detach still goes through {@link forgetCamera}.
|
|
2855
|
+
*/
|
|
2856
|
+
zeroAbsentResidents(present) {
|
|
2857
|
+
for (const [deviceId, counters] of this.cameras) {
|
|
2858
|
+
if (present.has(deviceId)) continue;
|
|
2859
|
+
counters.residentRefs = 0;
|
|
2860
|
+
counters.residentBytes = 0;
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2863
|
+
snapshot(deviceIds) {
|
|
2864
|
+
return {
|
|
2865
|
+
node: { ...this.node },
|
|
2866
|
+
cameras: deviceIds.map((deviceId) => ({
|
|
2867
|
+
deviceId,
|
|
2868
|
+
...this.camera(deviceId)
|
|
2869
|
+
}))
|
|
2870
|
+
};
|
|
2871
|
+
}
|
|
2872
|
+
trackedCameraCount() {
|
|
2873
|
+
return this.cameras.size;
|
|
2874
|
+
}
|
|
2875
|
+
camera(deviceId) {
|
|
2876
|
+
const existing = this.cameras.get(deviceId);
|
|
2877
|
+
if (existing) return existing;
|
|
2878
|
+
const created = emptyFrameLazyCounters();
|
|
2879
|
+
this.cameras.set(deviceId, created);
|
|
2880
|
+
return created;
|
|
3000
2881
|
}
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
2882
|
+
addHit(counters, kind, n) {
|
|
2883
|
+
if (kind === "native") counters.nativeHits += n;
|
|
2884
|
+
else if (kind === "tile") counters.tileHits += n;
|
|
2885
|
+
else counters.fallbackHits += n;
|
|
2886
|
+
}
|
|
2887
|
+
addMiss(counters, kind, n) {
|
|
2888
|
+
if (kind === "native") counters.nativeMisses += n;
|
|
2889
|
+
else if (kind === "tile") counters.tileMisses += n;
|
|
2890
|
+
else counters.fallbackMisses += n;
|
|
2891
|
+
}
|
|
2892
|
+
};
|
|
2893
|
+
//#endregion
|
|
2894
|
+
//#region src/pipeline-runner/frame-ref-transport.ts
|
|
2895
|
+
/** A local ref never leaves the grouped process; inline remains rollback/split compatibility. */
|
|
2896
|
+
function selectExecutorFrameSource(coLocated, frameRef, inlineFrame) {
|
|
2897
|
+
return coLocated && frameRef ? { frameRef } : { frame: inlineFrame };
|
|
3007
2898
|
}
|
|
3008
2899
|
//#endregion
|
|
3009
2900
|
//#region src/pipeline-runner/hardware-capabilities.ts
|
|
@@ -3157,81 +3048,470 @@ var INITIAL_STATE = {
|
|
|
3157
3048
|
burstRemaining: 0,
|
|
3158
3049
|
burstNextAt: 0
|
|
3159
3050
|
};
|
|
3160
|
-
var NO_RUN = {
|
|
3161
|
-
run: false,
|
|
3162
|
-
reason: null
|
|
3051
|
+
var NO_RUN = {
|
|
3052
|
+
run: false,
|
|
3053
|
+
reason: null
|
|
3054
|
+
};
|
|
3055
|
+
var MotionSettleGovernor = class {
|
|
3056
|
+
config;
|
|
3057
|
+
state = /* @__PURE__ */ new Map();
|
|
3058
|
+
constructor(config = DEFAULT_MOTION_SETTLE_CONFIG) {
|
|
3059
|
+
this.config = config;
|
|
3060
|
+
}
|
|
3061
|
+
/**
|
|
3062
|
+
* Decide whether `package-detection` runs for this device this frame, and
|
|
3063
|
+
* advance the per-device state immutably. Never throws.
|
|
3064
|
+
*/
|
|
3065
|
+
evaluate(sample) {
|
|
3066
|
+
const prev = this.state.get(sample.deviceId) ?? INITIAL_STATE;
|
|
3067
|
+
const lastMotionAt = sample.motionInZone ? sample.now : prev.lastMotionAt;
|
|
3068
|
+
const armed = sample.motionInZone ? true : prev.armed;
|
|
3069
|
+
const settleReady = armed && !sample.motionInZone && lastMotionAt !== null && sample.now - lastMotionAt >= this.config.settleMs;
|
|
3070
|
+
const pollReady = sample.now - prev.lastRunAt >= this.config.pollIntervalMs;
|
|
3071
|
+
if (settleReady) {
|
|
3072
|
+
this.state.set(sample.deviceId, {
|
|
3073
|
+
lastMotionAt,
|
|
3074
|
+
armed: false,
|
|
3075
|
+
lastRunAt: sample.now,
|
|
3076
|
+
burstRemaining: Math.max(0, this.config.burstCount - 1),
|
|
3077
|
+
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3078
|
+
});
|
|
3079
|
+
return {
|
|
3080
|
+
run: true,
|
|
3081
|
+
reason: "settle"
|
|
3082
|
+
};
|
|
3083
|
+
}
|
|
3084
|
+
if (prev.burstRemaining > 0 && sample.now >= prev.burstNextAt) {
|
|
3085
|
+
this.state.set(sample.deviceId, {
|
|
3086
|
+
lastMotionAt,
|
|
3087
|
+
armed,
|
|
3088
|
+
lastRunAt: sample.now,
|
|
3089
|
+
burstRemaining: prev.burstRemaining - 1,
|
|
3090
|
+
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3091
|
+
});
|
|
3092
|
+
return {
|
|
3093
|
+
run: true,
|
|
3094
|
+
reason: "burst"
|
|
3095
|
+
};
|
|
3096
|
+
}
|
|
3097
|
+
if (pollReady) {
|
|
3098
|
+
this.state.set(sample.deviceId, {
|
|
3099
|
+
lastMotionAt,
|
|
3100
|
+
armed,
|
|
3101
|
+
lastRunAt: sample.now,
|
|
3102
|
+
burstRemaining: prev.burstRemaining,
|
|
3103
|
+
burstNextAt: prev.burstNextAt
|
|
3104
|
+
});
|
|
3105
|
+
return {
|
|
3106
|
+
run: true,
|
|
3107
|
+
reason: "poll"
|
|
3108
|
+
};
|
|
3109
|
+
}
|
|
3110
|
+
this.state.set(sample.deviceId, {
|
|
3111
|
+
lastMotionAt,
|
|
3112
|
+
armed,
|
|
3113
|
+
lastRunAt: prev.lastRunAt,
|
|
3114
|
+
burstRemaining: prev.burstRemaining,
|
|
3115
|
+
burstNextAt: prev.burstNextAt
|
|
3116
|
+
});
|
|
3117
|
+
return NO_RUN;
|
|
3118
|
+
}
|
|
3119
|
+
/** Drop a device's state (call on detach / step-disable) to avoid a leak. */
|
|
3120
|
+
forget(deviceId) {
|
|
3121
|
+
this.state.delete(deviceId);
|
|
3122
|
+
}
|
|
3123
|
+
/** Clear all state (e.g. runner teardown). */
|
|
3124
|
+
reset() {
|
|
3125
|
+
this.state.clear();
|
|
3126
|
+
}
|
|
3127
|
+
};
|
|
3128
|
+
//#endregion
|
|
3129
|
+
//#region src/pipeline-runner/native-crop-tier.ts
|
|
3130
|
+
/**
|
|
3131
|
+
* Pure tier decision for `getNativeCrop`. A native HIT always resolves to the
|
|
3132
|
+
* native surface. On a native MISS, only a full-frame request falls back to the
|
|
3133
|
+
* ≤640 RAM store; an ROI miss is reported as `miss` (no quality masquerade).
|
|
3134
|
+
*/
|
|
3135
|
+
function resolveNativeCropSource(isFullFrame, nativeHit) {
|
|
3136
|
+
if (nativeHit) return "native";
|
|
3137
|
+
return isFullFrame ? "ram-fullframe" : "miss";
|
|
3138
|
+
}
|
|
3139
|
+
/**
|
|
3140
|
+
* Runtime hit kind for telemetry. A worker tile is native-resolution quality
|
|
3141
|
+
* (public wire `tier` stays `native`) but must not increment `nativeHits`.
|
|
3142
|
+
*/
|
|
3143
|
+
function resolveNativeCropHitKind(source) {
|
|
3144
|
+
return source === "tile" ? "tile" : "native";
|
|
3145
|
+
}
|
|
3146
|
+
//#endregion
|
|
3147
|
+
//#region src/pipeline-runner/native-lease-settings-source.ts
|
|
3148
|
+
/**
|
|
3149
|
+
* Where the runner gets the cluster-wide native-frame LEASE knobs.
|
|
3150
|
+
*
|
|
3151
|
+
* ## Why a settings read and not the attach config
|
|
3152
|
+
*
|
|
3153
|
+
* Identical reasoning to `crop-convention-source.ts`, and worth restating
|
|
3154
|
+
* because the temptation is the same: `RunnerCameraConfig` is the obvious
|
|
3155
|
+
* orchestrator→runner channel and it is the wrong one. Eleven call sites build
|
|
3156
|
+
* one; a cluster-wide constant threaded through eleven builders eventually
|
|
3157
|
+
* reaches ten of them, and the cameras attached by the forgotten builder just
|
|
3158
|
+
* quietly run a different lease window. Instead the runner reads the OWNER
|
|
3159
|
+
* addon's global settings directly, hub-routed by `AddonCallGateway`, so an
|
|
3160
|
+
* agent runner transparently reads the hub-central store with no placement
|
|
3161
|
+
* branching.
|
|
3162
|
+
*
|
|
3163
|
+
* ## Deliberately NOT node-scoped
|
|
3164
|
+
*
|
|
3165
|
+
* The read passes no `nodeId`. With one, `addon-settings` would project that
|
|
3166
|
+
* node's scoped value and two nodes could hold different lease windows for the
|
|
3167
|
+
* same camera — so whether a stored crop is native pixels or the ≤640 fallback
|
|
3168
|
+
* would depend on where the balancer happened to place it, which is invisible
|
|
3169
|
+
* from the stored media.
|
|
3170
|
+
*
|
|
3171
|
+
* ## What it hands out
|
|
3172
|
+
*
|
|
3173
|
+
* A {@link NativeLeaseSettingsOverride} — only the knobs the operator SET. The
|
|
3174
|
+
* decode worker resolves the rest (env var, then the shipped default); see
|
|
3175
|
+
* `session-decode/native-lease-config.ts` for the precedence. Handing out a
|
|
3176
|
+
* fully-resolved triple here would erase the "operator set nothing" state and
|
|
3177
|
+
* with it the env-var escape hatch.
|
|
3178
|
+
*/
|
|
3179
|
+
/**
|
|
3180
|
+
* The addon whose GLOBAL settings own the knobs. `pipeline-orchestrator` is
|
|
3181
|
+
* hub-resident and neutral — the same owner as the detail-crop convention and
|
|
3182
|
+
* the per-node decoder-backend selector.
|
|
3183
|
+
*/
|
|
3184
|
+
var NATIVE_LEASE_OWNER_ADDON_ID = "pipeline-orchestrator";
|
|
3185
|
+
/**
|
|
3186
|
+
* Whether two overrides name the same knobs with the same values.
|
|
3187
|
+
*
|
|
3188
|
+
* **Every knob has to appear here.** This is the equality that decides whether
|
|
3189
|
+
* a re-read is ADOPTED at all: a knob missing from the comparison is read from
|
|
3190
|
+
* the hub, judged identical, and discarded, so the operator's change never
|
|
3191
|
+
* reaches a decode worker and nothing anywhere says so. `admission` shipped
|
|
3192
|
+
* missing from this list and did exactly that — the setting was stored, the
|
|
3193
|
+
* hub served it, the runner fetched it, and every worker kept spawning with the
|
|
3194
|
+
* old value.
|
|
3195
|
+
*/
|
|
3196
|
+
function sameOverride(a, b) {
|
|
3197
|
+
return a.holdFrames === b.holdFrames && a.budgetMb === b.budgetMb && a.activityMs === b.activityMs && a.admission === b.admission && a.tileBudgetMb === b.tileBudgetMb;
|
|
3198
|
+
}
|
|
3199
|
+
/**
|
|
3200
|
+
* TTL-cached read of the cluster native-lease knobs.
|
|
3201
|
+
*
|
|
3202
|
+
* {@link current} is synchronous because the fork that consumes it is;
|
|
3203
|
+
* {@link refresh} is what talks to the hub, and the runner awaits it on the
|
|
3204
|
+
* path that precedes a decode-worker spawn.
|
|
3205
|
+
*/
|
|
3206
|
+
var NativeLeaseSettingsSource = class {
|
|
3207
|
+
override = {};
|
|
3208
|
+
lastReadAtMs = Number.NEGATIVE_INFINITY;
|
|
3209
|
+
inFlight = null;
|
|
3210
|
+
logger;
|
|
3211
|
+
now;
|
|
3212
|
+
ttlMs;
|
|
3213
|
+
constructor(options) {
|
|
3214
|
+
this.logger = options.logger;
|
|
3215
|
+
this.now = options.now ?? (() => Date.now());
|
|
3216
|
+
this.ttlMs = options.ttlMs ?? 3e4;
|
|
3217
|
+
}
|
|
3218
|
+
/** The operator's overrides as last read — `{}` until a read lands. */
|
|
3219
|
+
current() {
|
|
3220
|
+
return this.override;
|
|
3221
|
+
}
|
|
3222
|
+
/**
|
|
3223
|
+
* Re-read if the cached value has expired. Concurrent callers share one
|
|
3224
|
+
* in-flight read; a failure keeps the LAST KNOWN value rather than reverting
|
|
3225
|
+
* to `{}`, because a revert would silently hand the next-spawned workers a
|
|
3226
|
+
* different RAM profile than the ones already running.
|
|
3227
|
+
*/
|
|
3228
|
+
async refresh(api) {
|
|
3229
|
+
if (api === void 0) return this.override;
|
|
3230
|
+
if (this.now() - this.lastReadAtMs < this.ttlMs) return this.override;
|
|
3231
|
+
this.inFlight ??= this.read(api).finally(() => {
|
|
3232
|
+
this.inFlight = null;
|
|
3233
|
+
});
|
|
3234
|
+
await this.inFlight;
|
|
3235
|
+
return this.override;
|
|
3236
|
+
}
|
|
3237
|
+
async read(api) {
|
|
3238
|
+
try {
|
|
3239
|
+
const next = require_dist.pickNativeLeaseOverride(await api.addonSettings.getGlobalSettings.query({ addonId: NATIVE_LEASE_OWNER_ADDON_ID }));
|
|
3240
|
+
this.lastReadAtMs = this.now();
|
|
3241
|
+
if (sameOverride(next, this.override)) return;
|
|
3242
|
+
this.logger.info("native lease settings changed — applies to NEW decode sessions", { meta: {
|
|
3243
|
+
from: { ...this.override },
|
|
3244
|
+
to: { ...next }
|
|
3245
|
+
} });
|
|
3246
|
+
this.override = next;
|
|
3247
|
+
} catch (err) {
|
|
3248
|
+
this.logger.warn("native lease settings read failed — keeping the last known value", { meta: {
|
|
3249
|
+
owner: NATIVE_LEASE_OWNER_ADDON_ID,
|
|
3250
|
+
inUse: { ...this.override },
|
|
3251
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3252
|
+
} });
|
|
3253
|
+
}
|
|
3254
|
+
}
|
|
3255
|
+
};
|
|
3256
|
+
//#endregion
|
|
3257
|
+
//#region src/session-decode/on-demand-jpeg-fallback.ts
|
|
3258
|
+
/** Single content-sized JPEG requested after a detection, never a continuous raw. */
|
|
3259
|
+
var ON_DEMAND_JPEG_VIEW = {
|
|
3260
|
+
content: {
|
|
3261
|
+
width: 640,
|
|
3262
|
+
height: 640
|
|
3263
|
+
},
|
|
3264
|
+
fit: "contain",
|
|
3265
|
+
format: "jpeg"
|
|
3163
3266
|
};
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3267
|
+
function createOnDemandJpegFallback(registry, ref) {
|
|
3268
|
+
const lease = registry.acquire(ref);
|
|
3269
|
+
let cached;
|
|
3270
|
+
let released = false;
|
|
3271
|
+
const releaseLease = (reason) => {
|
|
3272
|
+
if (released) return;
|
|
3273
|
+
released = true;
|
|
3274
|
+
lease?.release(reason);
|
|
3275
|
+
};
|
|
3276
|
+
return {
|
|
3277
|
+
async materialize() {
|
|
3278
|
+
if (cached !== void 0) return cached;
|
|
3279
|
+
if (!lease) {
|
|
3280
|
+
cached = null;
|
|
3281
|
+
return null;
|
|
3282
|
+
}
|
|
3283
|
+
try {
|
|
3284
|
+
const view = await lease.resolve(ON_DEMAND_JPEG_VIEW);
|
|
3285
|
+
if (view.kind !== "view" || view.format !== "jpeg" || view.data.byteLength === 0) {
|
|
3286
|
+
cached = null;
|
|
3287
|
+
releaseLease("drop");
|
|
3288
|
+
return null;
|
|
3289
|
+
}
|
|
3290
|
+
cached = {
|
|
3291
|
+
data: Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength),
|
|
3292
|
+
width: view.width,
|
|
3293
|
+
height: view.height,
|
|
3294
|
+
format: "jpeg",
|
|
3295
|
+
timestamp: ref.timestamp,
|
|
3296
|
+
...ref.capturedAt !== void 0 ? { capturedAt: ref.capturedAt } : {}
|
|
3297
|
+
};
|
|
3298
|
+
releaseLease("success");
|
|
3299
|
+
return cached;
|
|
3300
|
+
} catch {
|
|
3301
|
+
cached = null;
|
|
3302
|
+
releaseLease("error");
|
|
3303
|
+
return null;
|
|
3304
|
+
}
|
|
3305
|
+
},
|
|
3306
|
+
release() {
|
|
3307
|
+
releaseLease("eviction");
|
|
3308
|
+
}
|
|
3309
|
+
};
|
|
3310
|
+
}
|
|
3311
|
+
//#endregion
|
|
3312
|
+
//#region src/pipeline-runner/on-demand-retention.ts
|
|
3313
|
+
var CAMSTACK_CONTINUOUS_RETAIN = "CAMSTACK_CONTINUOUS_RETAIN";
|
|
3314
|
+
function isContinuousRetainEnabled(env = process.env) {
|
|
3315
|
+
return env[CAMSTACK_CONTINUOUS_RETAIN] === "1";
|
|
3316
|
+
}
|
|
3317
|
+
function shouldRetainDetectionFrame(input) {
|
|
3318
|
+
if (!input.admitted) return false;
|
|
3319
|
+
if (input.continuousRetain) return true;
|
|
3320
|
+
return input.hasDetection;
|
|
3321
|
+
}
|
|
3322
|
+
function encodeCrossNodeCrop(input) {
|
|
3323
|
+
if (!input.jpeg) return null;
|
|
3324
|
+
return { jpeg: input.jpeg };
|
|
3325
|
+
}
|
|
3326
|
+
function handleKey(handle) {
|
|
3327
|
+
return `${handle.shmId}#${handle.slot}#${handle.seq}`;
|
|
3328
|
+
}
|
|
3329
|
+
/**
|
|
3330
|
+
* Pending occupancy exists only admission → inference result. A detection
|
|
3331
|
+
* transfers to {@link RetainedFrameStore} (authoritative late-media owner).
|
|
3332
|
+
* No-detection leaves zero state. Store eviction owns fallback cleanup.
|
|
3333
|
+
*/
|
|
3334
|
+
var OnDemandRetentionSession = class {
|
|
3335
|
+
store;
|
|
3336
|
+
registry;
|
|
3337
|
+
telemetry;
|
|
3338
|
+
continuousRetain;
|
|
3339
|
+
pending = /* @__PURE__ */ new Map();
|
|
3340
|
+
constructor(options) {
|
|
3341
|
+
this.store = options.store;
|
|
3342
|
+
this.registry = options.registry;
|
|
3343
|
+
this.telemetry = options.telemetry;
|
|
3344
|
+
this.continuousRetain = options.continuousRetain;
|
|
3169
3345
|
}
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
armed: false,
|
|
3184
|
-
lastRunAt: sample.now,
|
|
3185
|
-
burstRemaining: Math.max(0, this.config.burstCount - 1),
|
|
3186
|
-
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3346
|
+
get pendingSize() {
|
|
3347
|
+
return this.pending.size;
|
|
3348
|
+
}
|
|
3349
|
+
hasPending(handle) {
|
|
3350
|
+
return this.pending.has(handleKey(handle));
|
|
3351
|
+
}
|
|
3352
|
+
admitFrame(deviceId, frame) {
|
|
3353
|
+
if (this.continuousRetain) {
|
|
3354
|
+
const fallback = frame.frameRef ? createRetainedContentFallback(this.registry, frame.frameRef) : void 0;
|
|
3355
|
+
const handle = this.store.write(deviceId, frame, fallback) ?? void 0;
|
|
3356
|
+
if (handle) this.pending.set(handleKey(handle), {
|
|
3357
|
+
deviceId,
|
|
3358
|
+
ref: frame.frameRef
|
|
3187
3359
|
});
|
|
3360
|
+
this.refreshResident();
|
|
3361
|
+
return handle;
|
|
3362
|
+
}
|
|
3363
|
+
const handle = this.store.allocateHandle(deviceId, frame) ?? void 0;
|
|
3364
|
+
if (!handle) return void 0;
|
|
3365
|
+
this.pending.set(handleKey(handle), {
|
|
3366
|
+
deviceId,
|
|
3367
|
+
ref: frame.frameRef
|
|
3368
|
+
});
|
|
3369
|
+
this.telemetry.noteAvoidedWrite(deviceId);
|
|
3370
|
+
this.refreshResident();
|
|
3371
|
+
return handle;
|
|
3372
|
+
}
|
|
3373
|
+
noteFrame(input) {
|
|
3374
|
+
if (input.decoded) this.telemetry.noteDecoded(input.deviceId);
|
|
3375
|
+
if (input.admitted) this.telemetry.noteAdmitted(input.deviceId);
|
|
3376
|
+
if (input.droppedPixelFree) this.telemetry.noteDroppedPixelFree(input.deviceId);
|
|
3377
|
+
}
|
|
3378
|
+
noteInferenceResult(deviceId, handle, result) {
|
|
3379
|
+
if (!handle) return;
|
|
3380
|
+
const key = handleKey(handle);
|
|
3381
|
+
const pending = this.pending.get(key);
|
|
3382
|
+
this.forgetPending(key);
|
|
3383
|
+
if (!shouldRetainDetectionFrame({
|
|
3384
|
+
admitted: true,
|
|
3385
|
+
hasDetection: (result.detections?.length ?? 0) > 0,
|
|
3386
|
+
continuousRetain: this.continuousRetain
|
|
3387
|
+
})) {
|
|
3388
|
+
this.refreshResident();
|
|
3389
|
+
return;
|
|
3390
|
+
}
|
|
3391
|
+
if (this.continuousRetain) {
|
|
3392
|
+
this.refreshResident();
|
|
3393
|
+
return;
|
|
3394
|
+
}
|
|
3395
|
+
if (!pending?.ref) {
|
|
3396
|
+
this.refreshResident();
|
|
3397
|
+
return;
|
|
3398
|
+
}
|
|
3399
|
+
const fallback = createOnDemandJpegFallback(this.registry, pending.ref);
|
|
3400
|
+
this.store.rememberOnDemand(deviceId, handle, fallback);
|
|
3401
|
+
this.refreshResident();
|
|
3402
|
+
}
|
|
3403
|
+
async requestMedia(handle, request) {
|
|
3404
|
+
const deviceId = this.store.getDeviceId(handle) ?? this.pending.get(handleKey(handle))?.deviceId ?? 0;
|
|
3405
|
+
const kind = request.kind === "full-frame" ? "full-frame" : "crop";
|
|
3406
|
+
this.telemetry.noteOnDemandRequest({
|
|
3407
|
+
deviceId,
|
|
3408
|
+
kind
|
|
3409
|
+
});
|
|
3410
|
+
if (request.nativeHit) {
|
|
3411
|
+
this.telemetry.noteHit("native", deviceId);
|
|
3188
3412
|
return {
|
|
3189
|
-
|
|
3190
|
-
|
|
3413
|
+
tier: "native",
|
|
3414
|
+
bytes: request.nativeHit.bytes,
|
|
3415
|
+
width: request.nativeHit.width,
|
|
3416
|
+
height: request.nativeHit.height
|
|
3191
3417
|
};
|
|
3192
3418
|
}
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
armed,
|
|
3197
|
-
lastRunAt: sample.now,
|
|
3198
|
-
burstRemaining: prev.burstRemaining - 1,
|
|
3199
|
-
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3200
|
-
});
|
|
3419
|
+
this.telemetry.noteMiss("native", deviceId);
|
|
3420
|
+
if (request.tileHit) {
|
|
3421
|
+
this.telemetry.noteHit("tile", deviceId);
|
|
3201
3422
|
return {
|
|
3202
|
-
|
|
3203
|
-
|
|
3423
|
+
tier: "tile",
|
|
3424
|
+
bytes: request.tileHit.bytes,
|
|
3425
|
+
jpeg: request.tileHit.jpeg ? request.tileHit.bytes : void 0,
|
|
3426
|
+
width: request.tileHit.width,
|
|
3427
|
+
height: request.tileHit.height
|
|
3428
|
+
};
|
|
3429
|
+
}
|
|
3430
|
+
this.telemetry.noteMiss("tile", deviceId);
|
|
3431
|
+
if (request.recordedStillHit) return {
|
|
3432
|
+
tier: "recorded-still",
|
|
3433
|
+
jpeg: request.recordedStillHit.jpeg,
|
|
3434
|
+
width: request.recordedStillHit.width,
|
|
3435
|
+
height: request.recordedStillHit.height
|
|
3436
|
+
};
|
|
3437
|
+
if (!this.continuousRetain && !this.store.has(handle)) {
|
|
3438
|
+
this.telemetry.noteMiss("fallback", deviceId);
|
|
3439
|
+
return { tier: "miss" };
|
|
3440
|
+
}
|
|
3441
|
+
const view = await this.store.ensureContent(handle);
|
|
3442
|
+
if (view && view.data.byteLength > 0) {
|
|
3443
|
+
this.telemetry.noteHit("fallback", deviceId);
|
|
3444
|
+
this.refreshResident();
|
|
3445
|
+
if (view.format === "jpeg") return {
|
|
3446
|
+
tier: "fallback-jpeg",
|
|
3447
|
+
jpeg: view.data,
|
|
3448
|
+
width: view.width,
|
|
3449
|
+
height: view.height
|
|
3450
|
+
};
|
|
3451
|
+
return {
|
|
3452
|
+
tier: "fallback-jpeg",
|
|
3453
|
+
bytes: view.data,
|
|
3454
|
+
width: view.width,
|
|
3455
|
+
height: view.height
|
|
3456
|
+
};
|
|
3457
|
+
}
|
|
3458
|
+
this.telemetry.noteMiss("fallback", deviceId);
|
|
3459
|
+
this.telemetry.noteStale(deviceId);
|
|
3460
|
+
return { tier: "miss" };
|
|
3461
|
+
}
|
|
3462
|
+
/**
|
|
3463
|
+
* Pending cleanup. Success never invalidates a store entry already
|
|
3464
|
+
* transferred for late pull. Drop/error/detach drop the store only when the
|
|
3465
|
+
* handle was still pending (not yet transferred). Eviction owns transferred
|
|
3466
|
+
* fallback cleanup.
|
|
3467
|
+
*/
|
|
3468
|
+
release(handle, reason) {
|
|
3469
|
+
const key = handleKey(handle);
|
|
3470
|
+
const pending = this.pending.get(key);
|
|
3471
|
+
this.forgetPending(key);
|
|
3472
|
+
if (reason !== "success" && pending) this.store.drop(handle);
|
|
3473
|
+
this.telemetry.noteRelease(pending?.deviceId ?? this.store.getDeviceId(handle));
|
|
3474
|
+
this.refreshResident();
|
|
3475
|
+
}
|
|
3476
|
+
forgetDevice(deviceId) {
|
|
3477
|
+
for (const [key, pending] of [...this.pending]) {
|
|
3478
|
+
if (pending.deviceId !== deviceId) continue;
|
|
3479
|
+
this.pending.delete(key);
|
|
3480
|
+
}
|
|
3481
|
+
this.telemetry.forgetCamera(deviceId);
|
|
3482
|
+
this.refreshResident();
|
|
3483
|
+
}
|
|
3484
|
+
syncResident() {
|
|
3485
|
+
this.refreshResident();
|
|
3486
|
+
}
|
|
3487
|
+
forgetPending(key) {
|
|
3488
|
+
this.pending.delete(key);
|
|
3489
|
+
}
|
|
3490
|
+
refreshResident() {
|
|
3491
|
+
const byDevice = /* @__PURE__ */ new Map();
|
|
3492
|
+
for (const pending of this.pending.values()) {
|
|
3493
|
+
const cur = byDevice.get(pending.deviceId) ?? {
|
|
3494
|
+
refs: 0,
|
|
3495
|
+
bytes: 0
|
|
3204
3496
|
};
|
|
3497
|
+
cur.refs += 1;
|
|
3498
|
+
byDevice.set(pending.deviceId, cur);
|
|
3205
3499
|
}
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
lastRunAt: sample.now,
|
|
3211
|
-
burstRemaining: prev.burstRemaining,
|
|
3212
|
-
burstNextAt: prev.burstNextAt
|
|
3213
|
-
});
|
|
3214
|
-
return {
|
|
3215
|
-
run: true,
|
|
3216
|
-
reason: "poll"
|
|
3500
|
+
for (const stats of this.store.deviceStats()) {
|
|
3501
|
+
const cur = byDevice.get(stats.deviceId) ?? {
|
|
3502
|
+
refs: 0,
|
|
3503
|
+
bytes: 0
|
|
3217
3504
|
};
|
|
3505
|
+
cur.refs += stats.entries;
|
|
3506
|
+
cur.bytes += stats.bytes;
|
|
3507
|
+
byDevice.set(stats.deviceId, cur);
|
|
3218
3508
|
}
|
|
3219
|
-
this.
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
lastRunAt: prev.lastRunAt,
|
|
3223
|
-
burstRemaining: prev.burstRemaining,
|
|
3224
|
-
burstNextAt: prev.burstNextAt
|
|
3509
|
+
this.telemetry.setResident({
|
|
3510
|
+
refs: this.pending.size + this.store.size,
|
|
3511
|
+
bytes: this.registry.estimatedBytes + this.store.retainedBytes
|
|
3225
3512
|
});
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
/** Drop a device's state (call on detach / step-disable) to avoid a leak. */
|
|
3229
|
-
forget(deviceId) {
|
|
3230
|
-
this.state.delete(deviceId);
|
|
3231
|
-
}
|
|
3232
|
-
/** Clear all state (e.g. runner teardown). */
|
|
3233
|
-
reset() {
|
|
3234
|
-
this.state.clear();
|
|
3513
|
+
this.telemetry.zeroAbsentResidents(new Set(byDevice.keys()));
|
|
3514
|
+
for (const [deviceId, stats] of byDevice) this.telemetry.setCameraResident(deviceId, stats);
|
|
3235
3515
|
}
|
|
3236
3516
|
};
|
|
3237
3517
|
//#endregion
|
|
@@ -3319,134 +3599,6 @@ function packageZoneUnion(polygons) {
|
|
|
3319
3599
|
};
|
|
3320
3600
|
}
|
|
3321
3601
|
//#endregion
|
|
3322
|
-
//#region src/pipeline-runner/native-crop-tier.ts
|
|
3323
|
-
/**
|
|
3324
|
-
* Pure tier decision for `getNativeCrop`. A native HIT always resolves to the
|
|
3325
|
-
* native surface. On a native MISS, only a full-frame request falls back to the
|
|
3326
|
-
* ≤640 RAM store; an ROI miss is reported as `miss` (no quality masquerade).
|
|
3327
|
-
*/
|
|
3328
|
-
function resolveNativeCropSource(isFullFrame, nativeHit) {
|
|
3329
|
-
if (nativeHit) return "native";
|
|
3330
|
-
return isFullFrame ? "ram-fullframe" : "miss";
|
|
3331
|
-
}
|
|
3332
|
-
/**
|
|
3333
|
-
* Runtime hit kind for telemetry. A worker tile is native-resolution quality
|
|
3334
|
-
* (public wire `tier` stays `native`) but must not increment `nativeHits`.
|
|
3335
|
-
*/
|
|
3336
|
-
function resolveNativeCropHitKind(source) {
|
|
3337
|
-
return source === "tile" ? "tile" : "native";
|
|
3338
|
-
}
|
|
3339
|
-
//#endregion
|
|
3340
|
-
//#region src/pipeline-runner/native-lease-settings-source.ts
|
|
3341
|
-
/**
|
|
3342
|
-
* Where the runner gets the cluster-wide native-frame LEASE knobs.
|
|
3343
|
-
*
|
|
3344
|
-
* ## Why a settings read and not the attach config
|
|
3345
|
-
*
|
|
3346
|
-
* Identical reasoning to `crop-convention-source.ts`, and worth restating
|
|
3347
|
-
* because the temptation is the same: `RunnerCameraConfig` is the obvious
|
|
3348
|
-
* orchestrator→runner channel and it is the wrong one. Eleven call sites build
|
|
3349
|
-
* one; a cluster-wide constant threaded through eleven builders eventually
|
|
3350
|
-
* reaches ten of them, and the cameras attached by the forgotten builder just
|
|
3351
|
-
* quietly run a different lease window. Instead the runner reads the OWNER
|
|
3352
|
-
* addon's global settings directly, hub-routed by `AddonCallGateway`, so an
|
|
3353
|
-
* agent runner transparently reads the hub-central store with no placement
|
|
3354
|
-
* branching.
|
|
3355
|
-
*
|
|
3356
|
-
* ## Deliberately NOT node-scoped
|
|
3357
|
-
*
|
|
3358
|
-
* The read passes no `nodeId`. With one, `addon-settings` would project that
|
|
3359
|
-
* node's scoped value and two nodes could hold different lease windows for the
|
|
3360
|
-
* same camera — so whether a stored crop is native pixels or the ≤640 fallback
|
|
3361
|
-
* would depend on where the balancer happened to place it, which is invisible
|
|
3362
|
-
* from the stored media.
|
|
3363
|
-
*
|
|
3364
|
-
* ## What it hands out
|
|
3365
|
-
*
|
|
3366
|
-
* A {@link NativeLeaseSettingsOverride} — only the knobs the operator SET. The
|
|
3367
|
-
* decode worker resolves the rest (env var, then the shipped default); see
|
|
3368
|
-
* `session-decode/native-lease-config.ts` for the precedence. Handing out a
|
|
3369
|
-
* fully-resolved triple here would erase the "operator set nothing" state and
|
|
3370
|
-
* with it the env-var escape hatch.
|
|
3371
|
-
*/
|
|
3372
|
-
/**
|
|
3373
|
-
* The addon whose GLOBAL settings own the knobs. `pipeline-orchestrator` is
|
|
3374
|
-
* hub-resident and neutral — the same owner as the detail-crop convention and
|
|
3375
|
-
* the per-node decoder-backend selector.
|
|
3376
|
-
*/
|
|
3377
|
-
var NATIVE_LEASE_OWNER_ADDON_ID = "pipeline-orchestrator";
|
|
3378
|
-
/**
|
|
3379
|
-
* Whether two overrides name the same knobs with the same values.
|
|
3380
|
-
*
|
|
3381
|
-
* **Every knob has to appear here.** This is the equality that decides whether
|
|
3382
|
-
* a re-read is ADOPTED at all: a knob missing from the comparison is read from
|
|
3383
|
-
* the hub, judged identical, and discarded, so the operator's change never
|
|
3384
|
-
* reaches a decode worker and nothing anywhere says so. `admission` shipped
|
|
3385
|
-
* missing from this list and did exactly that — the setting was stored, the
|
|
3386
|
-
* hub served it, the runner fetched it, and every worker kept spawning with the
|
|
3387
|
-
* old value.
|
|
3388
|
-
*/
|
|
3389
|
-
function sameOverride(a, b) {
|
|
3390
|
-
return a.holdFrames === b.holdFrames && a.budgetMb === b.budgetMb && a.activityMs === b.activityMs && a.admission === b.admission && a.tileBudgetMb === b.tileBudgetMb;
|
|
3391
|
-
}
|
|
3392
|
-
/**
|
|
3393
|
-
* TTL-cached read of the cluster native-lease knobs.
|
|
3394
|
-
*
|
|
3395
|
-
* {@link current} is synchronous because the fork that consumes it is;
|
|
3396
|
-
* {@link refresh} is what talks to the hub, and the runner awaits it on the
|
|
3397
|
-
* path that precedes a decode-worker spawn.
|
|
3398
|
-
*/
|
|
3399
|
-
var NativeLeaseSettingsSource = class {
|
|
3400
|
-
override = {};
|
|
3401
|
-
lastReadAtMs = Number.NEGATIVE_INFINITY;
|
|
3402
|
-
inFlight = null;
|
|
3403
|
-
logger;
|
|
3404
|
-
now;
|
|
3405
|
-
ttlMs;
|
|
3406
|
-
constructor(options) {
|
|
3407
|
-
this.logger = options.logger;
|
|
3408
|
-
this.now = options.now ?? (() => Date.now());
|
|
3409
|
-
this.ttlMs = options.ttlMs ?? 3e4;
|
|
3410
|
-
}
|
|
3411
|
-
/** The operator's overrides as last read — `{}` until a read lands. */
|
|
3412
|
-
current() {
|
|
3413
|
-
return this.override;
|
|
3414
|
-
}
|
|
3415
|
-
/**
|
|
3416
|
-
* Re-read if the cached value has expired. Concurrent callers share one
|
|
3417
|
-
* in-flight read; a failure keeps the LAST KNOWN value rather than reverting
|
|
3418
|
-
* to `{}`, because a revert would silently hand the next-spawned workers a
|
|
3419
|
-
* different RAM profile than the ones already running.
|
|
3420
|
-
*/
|
|
3421
|
-
async refresh(api) {
|
|
3422
|
-
if (api === void 0) return this.override;
|
|
3423
|
-
if (this.now() - this.lastReadAtMs < this.ttlMs) return this.override;
|
|
3424
|
-
this.inFlight ??= this.read(api).finally(() => {
|
|
3425
|
-
this.inFlight = null;
|
|
3426
|
-
});
|
|
3427
|
-
await this.inFlight;
|
|
3428
|
-
return this.override;
|
|
3429
|
-
}
|
|
3430
|
-
async read(api) {
|
|
3431
|
-
try {
|
|
3432
|
-
const next = require_dist.pickNativeLeaseOverride(await api.addonSettings.getGlobalSettings.query({ addonId: NATIVE_LEASE_OWNER_ADDON_ID }));
|
|
3433
|
-
this.lastReadAtMs = this.now();
|
|
3434
|
-
if (sameOverride(next, this.override)) return;
|
|
3435
|
-
this.logger.info("native lease settings changed — applies to NEW decode sessions", { meta: {
|
|
3436
|
-
from: { ...this.override },
|
|
3437
|
-
to: { ...next }
|
|
3438
|
-
} });
|
|
3439
|
-
this.override = next;
|
|
3440
|
-
} catch (err) {
|
|
3441
|
-
this.logger.warn("native lease settings read failed — keeping the last known value", { meta: {
|
|
3442
|
-
owner: NATIVE_LEASE_OWNER_ADDON_ID,
|
|
3443
|
-
inUse: { ...this.override },
|
|
3444
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3445
|
-
} });
|
|
3446
|
-
}
|
|
3447
|
-
}
|
|
3448
|
-
};
|
|
3449
|
-
//#endregion
|
|
3450
3602
|
//#region src/pipeline-runner/frame-governor.ts
|
|
3451
3603
|
var FrameGovernor = class {
|
|
3452
3604
|
config;
|
|
@@ -4610,76 +4762,6 @@ var PipelineRunner = class {
|
|
|
4610
4762
|
}
|
|
4611
4763
|
};
|
|
4612
4764
|
//#endregion
|
|
4613
|
-
//#region src/session-decode/live-worker-admission.ts
|
|
4614
|
-
/**
|
|
4615
|
-
* The node's live-decode-worker budget.
|
|
4616
|
-
*
|
|
4617
|
-
* One instance per runner addon. Not a semaphore: a semaphore's contract is
|
|
4618
|
-
* "wait", and waiting is the thing this deliberately does not do — see
|
|
4619
|
-
* {@link DEFAULT_MAX_LIVE_DECODE_WORKERS}.
|
|
4620
|
-
*/
|
|
4621
|
-
var LiveDecodeWorkerAdmission = class {
|
|
4622
|
-
limit;
|
|
4623
|
-
logger;
|
|
4624
|
-
live = 0;
|
|
4625
|
-
constructor(options) {
|
|
4626
|
-
this.limit = Math.max(1, options.limit);
|
|
4627
|
-
this.logger = options.logger;
|
|
4628
|
-
}
|
|
4629
|
-
/** Workers currently holding a slot. */
|
|
4630
|
-
get inFlight() {
|
|
4631
|
-
return this.live;
|
|
4632
|
-
}
|
|
4633
|
-
get ceiling() {
|
|
4634
|
-
return this.limit;
|
|
4635
|
-
}
|
|
4636
|
-
/**
|
|
4637
|
-
* Apply an operator-changed ceiling.
|
|
4638
|
-
*
|
|
4639
|
-
* Lowering it never kills a running session — a decode worker torn down
|
|
4640
|
-
* mid-motion loses the event it was started for. It only stops the next
|
|
4641
|
-
* admission, so the count drains to the new ceiling as sessions end.
|
|
4642
|
-
*/
|
|
4643
|
-
resize(limit) {
|
|
4644
|
-
this.limit = Math.max(1, limit);
|
|
4645
|
-
}
|
|
4646
|
-
/**
|
|
4647
|
-
* Take a slot, or refuse.
|
|
4648
|
-
*
|
|
4649
|
-
* A refusal is a WARN and carries `tags.deviceId`, because the only useful
|
|
4650
|
-
* form of this question is per-camera ("why did 617 not run when 615 did?").
|
|
4651
|
-
* A branch that drops work silently reads as "it never happened".
|
|
4652
|
-
*/
|
|
4653
|
-
admit(deviceId, label) {
|
|
4654
|
-
if (this.live >= this.limit) {
|
|
4655
|
-
this.logger.warn("session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
|
|
4656
|
-
tags: { deviceId },
|
|
4657
|
-
meta: {
|
|
4658
|
-
label,
|
|
4659
|
-
live: this.live,
|
|
4660
|
-
ceiling: this.limit
|
|
4661
|
-
}
|
|
4662
|
-
});
|
|
4663
|
-
return null;
|
|
4664
|
-
}
|
|
4665
|
-
this.live += 1;
|
|
4666
|
-
this.logger.debug("session-decode: live decode-worker slot taken", {
|
|
4667
|
-
tags: { deviceId },
|
|
4668
|
-
meta: {
|
|
4669
|
-
label,
|
|
4670
|
-
live: this.live,
|
|
4671
|
-
ceiling: this.limit
|
|
4672
|
-
}
|
|
4673
|
-
});
|
|
4674
|
-
let released = false;
|
|
4675
|
-
return { release: () => {
|
|
4676
|
-
if (released) return;
|
|
4677
|
-
released = true;
|
|
4678
|
-
this.live -= 1;
|
|
4679
|
-
} };
|
|
4680
|
-
}
|
|
4681
|
-
};
|
|
4682
|
-
//#endregion
|
|
4683
4765
|
//#region src/pipeline-runner/stateless-step.ts
|
|
4684
4766
|
function modelServability(step, modelId, format) {
|
|
4685
4767
|
const entry = step.models.find((m) => m.id === modelId);
|
|
@@ -4819,7 +4901,8 @@ var DEFAULT_CONFIG = {
|
|
|
4819
4901
|
targetLoadPercent: 80,
|
|
4820
4902
|
minThrottledFps: 1,
|
|
4821
4903
|
maxConcurrentDecodeSpawns: 3,
|
|
4822
|
-
maxLiveDecodeWorkers: 8
|
|
4904
|
+
maxLiveDecodeWorkers: 8,
|
|
4905
|
+
nodeDecodeBudgetMb: DEFAULT_NODE_DECODE_BUDGET_MB
|
|
4823
4906
|
};
|
|
4824
4907
|
/**
|
|
4825
4908
|
* Camera attachment record kept by the runner addon. Holds the original
|
|
@@ -5382,7 +5465,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
5382
5465
|
this.spawnGate = new Semaphore(Math.max(1, this.config.maxConcurrentDecodeSpawns));
|
|
5383
5466
|
this.liveWorkers = new LiveDecodeWorkerAdmission({
|
|
5384
5467
|
limit: this.config.maxLiveDecodeWorkers,
|
|
5385
|
-
logger: this.ctx.logger
|
|
5468
|
+
logger: this.ctx.logger,
|
|
5469
|
+
nodeBudgetMb: this.config.nodeDecodeBudgetMb,
|
|
5470
|
+
workerFootprintMb: decodeWorkerFootprintMb(this.nativeLease.current())
|
|
5386
5471
|
});
|
|
5387
5472
|
this.runner.timingSampler.setLogger(this.ctx.logger.child("timing"));
|
|
5388
5473
|
this.runner.onDetectionStreamChange((deviceId, action) => {
|
|
@@ -6586,6 +6671,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
6586
6671
|
*/
|
|
6587
6672
|
async refreshNativeLeaseSettings() {
|
|
6588
6673
|
await this.nativeLease.refresh(this.ctxIfReady?.api);
|
|
6674
|
+
this.liveWorkers?.resizeFootprint(decodeWorkerFootprintMb(this.nativeLease.current()));
|
|
6589
6675
|
}
|
|
6590
6676
|
/** A warn sink that survives a not-yet-ready ctx (boot-order dispatches). */
|
|
6591
6677
|
detailSubtreeLogger() {
|
|
@@ -8055,12 +8141,23 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8055
8141
|
type: "slider",
|
|
8056
8142
|
key: "maxLiveDecodeWorkers",
|
|
8057
8143
|
label: "Max live decode workers",
|
|
8058
|
-
description: "
|
|
8144
|
+
description: "Upper CLAMP on decode workers ALIVE at once on this node. The bound that normally decides is the RAM budget below — the ceiling in force is whichever of the two is smaller. Unlike the spawn gate above, which only limits how fast workers START, this limits how many EXIST. A camera that does not fit is REFUSED and logged, then retried automatically as soon as a slot frees — it is never queued, because a queued live session would decode a moment that has already passed.",
|
|
8059
8145
|
min: 1,
|
|
8060
8146
|
max: 32,
|
|
8061
8147
|
step: 1,
|
|
8062
8148
|
default: DEFAULT_CONFIG.maxLiveDecodeWorkers,
|
|
8063
8149
|
showValue: true
|
|
8150
|
+
},
|
|
8151
|
+
{
|
|
8152
|
+
type: "slider",
|
|
8153
|
+
key: "nodeDecodeBudgetMb",
|
|
8154
|
+
label: "Decode worker RAM budget (MB)",
|
|
8155
|
+
description: "RAM this node lets live decode workers commit. The number of workers admitted is derived from it: budget divided by one worker's worst case, which is the lease budget + tile budget + ~350 MB of baseline. Measured on this cluster a worker plateaus at 650-860 MB within ten seconds of spawning, so at the shipped lease knobs the default admits 4 — the observed peak. Raising the lease budget per worker therefore LOWERS how many run at once, instead of silently authorising more memory.",
|
|
8156
|
+
min: 1024,
|
|
8157
|
+
max: 32768,
|
|
8158
|
+
step: 512,
|
|
8159
|
+
default: DEFAULT_CONFIG.nodeDecodeBudgetMb,
|
|
8160
|
+
showValue: true
|
|
8064
8161
|
}
|
|
8065
8162
|
]
|
|
8066
8163
|
}] });
|
|
@@ -8069,13 +8166,17 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
|
|
|
8069
8166
|
this.runner?.updateLimits(this.config);
|
|
8070
8167
|
this.spawnGate?.resize(Math.max(1, this.config.maxConcurrentDecodeSpawns));
|
|
8071
8168
|
this.liveWorkers?.resize(this.config.maxLiveDecodeWorkers);
|
|
8169
|
+
this.liveWorkers?.resizeBudget(this.config.nodeDecodeBudgetMb);
|
|
8072
8170
|
this.ctx.logger.info("pipeline-runner tuning updated", { meta: {
|
|
8073
8171
|
maxQueueDepth: this.config.maxQueueDepth,
|
|
8074
8172
|
maxConcurrentInferences: this.config.maxConcurrentInferences,
|
|
8075
8173
|
targetLoadPercent: this.config.targetLoadPercent,
|
|
8076
8174
|
minThrottledFps: this.config.minThrottledFps,
|
|
8077
8175
|
maxConcurrentDecodeSpawns: this.config.maxConcurrentDecodeSpawns,
|
|
8078
|
-
maxLiveDecodeWorkers: this.config.maxLiveDecodeWorkers
|
|
8176
|
+
maxLiveDecodeWorkers: this.config.maxLiveDecodeWorkers,
|
|
8177
|
+
nodeDecodeBudgetMb: this.config.nodeDecodeBudgetMb,
|
|
8178
|
+
decodeCeilingInForce: this.liveWorkers?.ceiling,
|
|
8179
|
+
decodeWorkerFootprintMb: decodeWorkerFootprintMb(this.nativeLease.current())
|
|
8079
8180
|
} });
|
|
8080
8181
|
}
|
|
8081
8182
|
};
|