@camstack/addon-pipeline 1.2.124 → 1.2.125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-DjiHGIRF.js → addon-utils-BnprivSs.js} +1 -1
- package/dist/audio-analyzer/index.js +3 -3
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/detection-pipeline/index.js +5 -5
- package/dist/detection-pipeline/index.mjs +3 -3
- package/dist/{dist-DomdkRKL.js → dist-BZUXb5S5.js} +26 -4
- package/dist/{dist-B7rIp5qQ.mjs → dist-HLAOw6Pe.mjs} +26 -4
- package/dist/{event-loop-stall-monitor-mvxKgrNV.mjs → event-loop-stall-monitor-BH-cL_hg.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-ChA4AQ1D.js → event-loop-stall-monitor-BdArVUlx.js} +1 -1
- package/dist/{lazy-sharp-BSeW3qY5.js → lazy-sharp-BxMwCkCA.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +972 -871
- package/dist/pipeline-runner/index.mjs +971 -870
- package/dist/{process-memory-XHs-umae.mjs → process-memory-CDPRLalN.mjs} +1 -1
- package/dist/{process-memory-BtpLNnXq.js → process-memory-Dh1SyzdJ.js} +1 -1
- package/dist/recorder/index.js +2 -2
- package/dist/recorder/index.mjs +1 -1
- package/dist/{segment-demux-js-Cc5Jubdu.mjs → segment-demux-js-D-TLcHW5.mjs} +1 -1
- package/dist/{segment-demux-js-CFIIJsfc.js → segment-demux-js-DRutggd6.js} +1 -1
- package/dist/session-decode/decode-worker-child.js +2 -2
- package/dist/session-decode/decode-worker-child.mjs +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-P-WP2KSi.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BDR5z0Z9.mjs} +1 -1
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-D6O20fld.mjs → hostInit-Dmw5LWmP.mjs} +1 -1
- package/dist/stream-broker/index.js +2 -2
- package/dist/stream-broker/index.mjs +2 -2
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-DQ5DPOFD.js → worker-protocol-BxkCoVjw.js} +1 -1
- package/dist/{worker-protocol-CrHJXGHV.mjs → worker-protocol-h7Smb_0i.mjs} +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { Bt as lazy, Et as createEvent, F as customAction, Gt as string, Ht as number, It as _enum, L as defineCustomActions, Lt as array, Rt as boolean, St as BaseAddon, Ut as object, Vt as literal, gt as errMsg, it as pipelineRunnerCapability, jt as nodePin, nt as pickNativeLeaseOverride, p as DEVICE_BACKEND_TO_FORMAT, qt as EventCategory, tt as pickDetailCropConvention, u as DEFAULT_DETAIL_CROP_CONVENTION, z as deriveDetailCropRect } from "../dist-
|
|
2
|
-
import { i as resolveFrameViewGeometry, r as localFrameRegistry, t as attributeStall, u as getStepDefinition } from "../event-loop-stall-monitor-
|
|
3
|
-
import { a as nativeLeaseSettingEnv, t as isWorkerReply } from "../worker-protocol-
|
|
1
|
+
import { Bt as lazy, Et as createEvent, F as customAction, Gt as string, Ht as number, It as _enum, L as defineCustomActions, Lt as array, Rt as boolean, St as BaseAddon, Ut as object, Vt as literal, f as DEFAULT_NATIVE_LEASE_SETTINGS, gt as errMsg, it as pipelineRunnerCapability, jt as nodePin, nt as pickNativeLeaseOverride, p as DEVICE_BACKEND_TO_FORMAT, qt as EventCategory, tt as pickDetailCropConvention, u as DEFAULT_DETAIL_CROP_CONVENTION, z as deriveDetailCropRect } from "../dist-HLAOw6Pe.mjs";
|
|
2
|
+
import { i as resolveFrameViewGeometry, r as localFrameRegistry, t as attributeStall, u as getStepDefinition } from "../event-loop-stall-monitor-BH-cL_hg.mjs";
|
|
3
|
+
import { a as nativeLeaseSettingEnv, t as isWorkerReply } from "../worker-protocol-h7Smb_0i.mjs";
|
|
4
4
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
5
5
|
import { t as getSharp } from "../lazy-sharp-6oymT_yf.mjs";
|
|
6
6
|
import { n as withDetectionIntent } from "../restream-intent-B4BXZra7.mjs";
|
|
@@ -195,6 +195,158 @@ function startGuardedSessionDecode(deps) {
|
|
|
195
195
|
if (release) release();
|
|
196
196
|
};
|
|
197
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* RAM this node lets live decode workers commit, in MB.
|
|
200
|
+
*
|
|
201
|
+
* ## Why 8192, and why 6144 was wrong
|
|
202
|
+
*
|
|
203
|
+
* 6144 shipped first, derived from the SHIPPED lease defaults (1024 + 64 + 350
|
|
204
|
+
* = 1438 MB per worker) and a 13-minute concurrency sample. Both inputs were
|
|
205
|
+
* wrong for the cluster it ran on, in opposite directions:
|
|
206
|
+
*
|
|
207
|
+
* - the operator here runs `budgetMb: 512`, so a worker is worst-cased at
|
|
208
|
+
* **926 MB**, not 1438 — the old count of 8 was authorising 7.4 GiB, not the
|
|
209
|
+
* 11.5 GiB the defaults implied;
|
|
210
|
+
* - the sample missed the peak. Measured across the change: at ceiling 8 the
|
|
211
|
+
* cluster refused 4 sessions in 30 minutes; at the ceiling 6144 derived
|
|
212
|
+
* (6 workers at 926 MB) it refused **43 in 10 minutes**, and the hub — which
|
|
213
|
+
* had never refused once — began refusing too. Demand genuinely reaches 8.
|
|
214
|
+
*
|
|
215
|
+
* A refusal is self-healing but it is not free: the retry ladder means the
|
|
216
|
+
* camera decodes a motion window that has moved on. Trading real coverage for a
|
|
217
|
+
* RAM headroom nobody measured a need for is the wrong side of the trade.
|
|
218
|
+
*
|
|
219
|
+
* 8192 restores exactly the concurrency this cluster had (`floor(8192/926) = 8`)
|
|
220
|
+
* while keeping what the change was actually for: the bound is RAM, so raising
|
|
221
|
+
* `budgetMb` back to the shipped 1024 now DROPS the ceiling to 5 instead of
|
|
222
|
+
* silently authorising 11.5 GiB, and the refusal line names the megabytes.
|
|
223
|
+
*/
|
|
224
|
+
var DEFAULT_NODE_DECODE_BUDGET_MB = 8192;
|
|
225
|
+
/**
|
|
226
|
+
* Worst-case RAM one decode worker may commit, from the lease knobs actually in
|
|
227
|
+
* force. The SINGLE place the operator-facing budgets become the number the
|
|
228
|
+
* ceiling is derived from — computing it anywhere else is exactly how the count
|
|
229
|
+
* and the budget drifted apart.
|
|
230
|
+
*
|
|
231
|
+
* `override` carries only the knobs the operator SET (see
|
|
232
|
+
* `native-lease-config.ts` on why "unset" includes a knob sitting at its
|
|
233
|
+
* default); anything absent falls back to what the worker ships with.
|
|
234
|
+
*/
|
|
235
|
+
function decodeWorkerFootprintMb(override) {
|
|
236
|
+
return (override.budgetMb ?? DEFAULT_NATIVE_LEASE_SETTINGS.budgetMb) + (override.tileBudgetMb ?? DEFAULT_NATIVE_LEASE_SETTINGS.tileBudgetMb) + 350;
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* The node's live-decode-worker budget.
|
|
240
|
+
*
|
|
241
|
+
* One instance per runner addon. Not a semaphore: a semaphore's contract is
|
|
242
|
+
* "wait", and waiting is the thing this deliberately does not do — see
|
|
243
|
+
* {@link DEFAULT_MAX_LIVE_DECODE_WORKERS}.
|
|
244
|
+
*/
|
|
245
|
+
var LiveDecodeWorkerAdmission = class {
|
|
246
|
+
limit;
|
|
247
|
+
logger;
|
|
248
|
+
live = 0;
|
|
249
|
+
nodeBudgetMb;
|
|
250
|
+
workerFootprintMb;
|
|
251
|
+
constructor(options) {
|
|
252
|
+
this.limit = Math.max(1, options.limit);
|
|
253
|
+
this.logger = options.logger;
|
|
254
|
+
this.nodeBudgetMb = positiveOrUndefined(options.nodeBudgetMb);
|
|
255
|
+
this.workerFootprintMb = positiveOrUndefined(options.workerFootprintMb);
|
|
256
|
+
}
|
|
257
|
+
/** Workers currently holding a slot. */
|
|
258
|
+
get inFlight() {
|
|
259
|
+
return this.live;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* The ceiling actually in force: the RAM budget divided by one worker\'s worst
|
|
263
|
+
* case, clamped by the hand-set count.
|
|
264
|
+
*
|
|
265
|
+
* Never zero. A node too small for even one worker still decodes one camera —
|
|
266
|
+
* refusing every camera turns a memory bound into a total outage, and one
|
|
267
|
+
* worker is the smallest unit of "this node does its job at all".
|
|
268
|
+
*/
|
|
269
|
+
get ceiling() {
|
|
270
|
+
const fromBudget = this.budgetDerivedCeiling();
|
|
271
|
+
return fromBudget === void 0 ? this.limit : Math.max(1, Math.min(this.limit, fromBudget));
|
|
272
|
+
}
|
|
273
|
+
/** MB currently committed by the workers holding a slot. */
|
|
274
|
+
get committedMb() {
|
|
275
|
+
return this.workerFootprintMb === void 0 ? void 0 : this.live * this.workerFootprintMb;
|
|
276
|
+
}
|
|
277
|
+
budgetDerivedCeiling() {
|
|
278
|
+
if (this.nodeBudgetMb === void 0 || this.workerFootprintMb === void 0) return void 0;
|
|
279
|
+
return Math.floor(this.nodeBudgetMb / this.workerFootprintMb);
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Apply a changed per-worker footprint — the operator moved `budgetMb` or
|
|
283
|
+
* `tileBudgetMb`. Like {@link resize} this never kills a running session; it
|
|
284
|
+
* only changes what the next admission is measured against.
|
|
285
|
+
*/
|
|
286
|
+
resizeFootprint(workerFootprintMb) {
|
|
287
|
+
this.workerFootprintMb = positiveOrUndefined(workerFootprintMb);
|
|
288
|
+
}
|
|
289
|
+
/** Apply a changed node decode budget. */
|
|
290
|
+
resizeBudget(nodeBudgetMb) {
|
|
291
|
+
this.nodeBudgetMb = positiveOrUndefined(nodeBudgetMb);
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Apply an operator-changed ceiling.
|
|
295
|
+
*
|
|
296
|
+
* Lowering it never kills a running session — a decode worker torn down
|
|
297
|
+
* mid-motion loses the event it was started for. It only stops the next
|
|
298
|
+
* admission, so the count drains to the new ceiling as sessions end.
|
|
299
|
+
*/
|
|
300
|
+
resize(limit) {
|
|
301
|
+
this.limit = Math.max(1, limit);
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Take a slot, or refuse.
|
|
305
|
+
*
|
|
306
|
+
* A refusal is a WARN and carries `tags.deviceId`, because the only useful
|
|
307
|
+
* form of this question is per-camera ("why did 617 not run when 615 did?").
|
|
308
|
+
* A branch that drops work silently reads as "it never happened".
|
|
309
|
+
*/
|
|
310
|
+
admit(deviceId, label) {
|
|
311
|
+
const ceiling = this.ceiling;
|
|
312
|
+
if (this.live >= ceiling) {
|
|
313
|
+
this.logger.warn("session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
|
|
314
|
+
tags: { deviceId },
|
|
315
|
+
meta: {
|
|
316
|
+
label,
|
|
317
|
+
live: this.live,
|
|
318
|
+
ceiling,
|
|
319
|
+
...this.committedMb === void 0 ? {} : {
|
|
320
|
+
committedMb: this.committedMb,
|
|
321
|
+
budgetMb: this.nodeBudgetMb,
|
|
322
|
+
workerFootprintMb: this.workerFootprintMb
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
return null;
|
|
327
|
+
}
|
|
328
|
+
this.live += 1;
|
|
329
|
+
this.logger.debug("session-decode: live decode-worker slot taken", {
|
|
330
|
+
tags: { deviceId },
|
|
331
|
+
meta: {
|
|
332
|
+
label,
|
|
333
|
+
live: this.live,
|
|
334
|
+
ceiling: this.ceiling
|
|
335
|
+
}
|
|
336
|
+
});
|
|
337
|
+
let released = false;
|
|
338
|
+
return { release: () => {
|
|
339
|
+
if (released) return;
|
|
340
|
+
released = true;
|
|
341
|
+
this.live -= 1;
|
|
342
|
+
} };
|
|
343
|
+
}
|
|
344
|
+
};
|
|
345
|
+
/** A knob that is absent, zero or nonsense derives nothing rather than a bound nobody chose. */
|
|
346
|
+
function positiveOrUndefined(value) {
|
|
347
|
+
if (value === void 0) return void 0;
|
|
348
|
+
return Number.isFinite(value) && value > 0 ? value : void 0;
|
|
349
|
+
}
|
|
198
350
|
//#endregion
|
|
199
351
|
//#region src/session-decode/native-crop-registry.ts
|
|
200
352
|
/**
|
|
@@ -1706,589 +1858,171 @@ function buildBenchRunPipelineInput(req) {
|
|
|
1706
1858
|
};
|
|
1707
1859
|
}
|
|
1708
1860
|
//#endregion
|
|
1709
|
-
//#region src/pipeline-runner/
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1861
|
+
//#region src/pipeline-runner/capacity/capacity-runner.ts
|
|
1862
|
+
/**
|
|
1863
|
+
* Run ONE ramp step: `cameras` independent open-loop pacers for
|
|
1864
|
+
* `settleMs + measureMs`, then a bounded drain of in-flight calls.
|
|
1865
|
+
*/
|
|
1866
|
+
async function runCapacityStep(opts) {
|
|
1867
|
+
const cameras = Math.max(1, Math.floor(opts.cameras));
|
|
1868
|
+
const intervalMs = 1e3 / Math.max(.1, opts.fpsPerCamera);
|
|
1869
|
+
const start = opts.now();
|
|
1870
|
+
const measureStart = start + Math.max(0, opts.settleMs);
|
|
1871
|
+
const end = measureStart + Math.max(0, opts.measureMs);
|
|
1872
|
+
const c = {
|
|
1873
|
+
offered: 0,
|
|
1874
|
+
dispatched: 0,
|
|
1875
|
+
completed: 0,
|
|
1876
|
+
droppedByDriver: 0,
|
|
1877
|
+
shed: 0,
|
|
1878
|
+
firstClassShed: 0,
|
|
1879
|
+
fallbackShed: 0,
|
|
1880
|
+
errors: 0,
|
|
1881
|
+
aborted: false,
|
|
1882
|
+
outstanding: 0,
|
|
1883
|
+
e2eMsList: [],
|
|
1884
|
+
inferMsList: [],
|
|
1885
|
+
predictMsList: [],
|
|
1886
|
+
detectionCounts: []
|
|
1734
1887
|
};
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
this.node.framesAdmitted += 1;
|
|
1745
|
-
this.camera(deviceId).framesAdmitted += 1;
|
|
1746
|
-
}
|
|
1747
|
-
noteDroppedPixelFree(deviceId) {
|
|
1748
|
-
this.node.framesDroppedPixelFree += 1;
|
|
1749
|
-
this.camera(deviceId).framesDroppedPixelFree += 1;
|
|
1750
|
-
}
|
|
1751
|
-
noteView(input) {
|
|
1752
|
-
if (input.materialized) {
|
|
1753
|
-
this.node.viewsMaterialized += 1;
|
|
1754
|
-
this.camera(input.deviceId).viewsMaterialized += 1;
|
|
1755
|
-
} else {
|
|
1756
|
-
this.node.viewsSkipped += 1;
|
|
1757
|
-
this.camera(input.deviceId).viewsSkipped += 1;
|
|
1758
|
-
}
|
|
1759
|
-
this.node.workerToRunnerBytes += input.workerToRunnerBytes;
|
|
1760
|
-
this.camera(input.deviceId).workerToRunnerBytes += input.workerToRunnerBytes;
|
|
1761
|
-
}
|
|
1762
|
-
notePoolBytes(input) {
|
|
1763
|
-
this.node.runnerToPoolRawBytes += input.raw;
|
|
1764
|
-
this.node.runnerToPoolJpegBytes += input.jpeg;
|
|
1765
|
-
this.camera(input.deviceId).runnerToPoolRawBytes += input.raw;
|
|
1766
|
-
this.camera(input.deviceId).runnerToPoolJpegBytes += input.jpeg;
|
|
1767
|
-
}
|
|
1768
|
-
noteOnDemandRequest(input) {
|
|
1769
|
-
if (input.kind === "full-frame") {
|
|
1770
|
-
this.node.onDemandFullFrameRequests += 1;
|
|
1771
|
-
this.camera(input.deviceId).onDemandFullFrameRequests += 1;
|
|
1772
|
-
} else {
|
|
1773
|
-
this.node.onDemandCropRequests += 1;
|
|
1774
|
-
this.camera(input.deviceId).onDemandCropRequests += 1;
|
|
1888
|
+
const recordOutcome = (counted, outcome, e2eMs) => {
|
|
1889
|
+
if (!counted) return;
|
|
1890
|
+
if (outcome.kind === "ok") {
|
|
1891
|
+
c.completed += 1;
|
|
1892
|
+
c.e2eMsList.push(e2eMs);
|
|
1893
|
+
c.inferMsList.push(outcome.inferMs ?? 0);
|
|
1894
|
+
c.predictMsList.push(outcome.predictMs ?? 0);
|
|
1895
|
+
c.detectionCounts.push(outcome.detections ?? 0);
|
|
1896
|
+
return;
|
|
1775
1897
|
}
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
noteMiss(kind, deviceId) {
|
|
1782
|
-
this.addMiss(this.node, kind, 1);
|
|
1783
|
-
this.addMiss(this.camera(deviceId), kind, 1);
|
|
1784
|
-
}
|
|
1785
|
-
noteAvoidedWrite(deviceId) {
|
|
1786
|
-
this.node.retainedWritesAvoided += 1;
|
|
1787
|
-
this.camera(deviceId).retainedWritesAvoided += 1;
|
|
1788
|
-
}
|
|
1789
|
-
noteRelease(deviceId) {
|
|
1790
|
-
this.node.releases += 1;
|
|
1791
|
-
if (deviceId !== void 0) this.camera(deviceId).releases += 1;
|
|
1792
|
-
}
|
|
1793
|
-
noteEviction(deviceId) {
|
|
1794
|
-
this.node.evictions += 1;
|
|
1795
|
-
if (deviceId !== void 0) this.camera(deviceId).evictions += 1;
|
|
1796
|
-
}
|
|
1797
|
-
noteStale(deviceId) {
|
|
1798
|
-
this.node.staleMisses += 1;
|
|
1799
|
-
if (deviceId !== void 0) this.camera(deviceId).staleMisses += 1;
|
|
1800
|
-
}
|
|
1801
|
-
/**
|
|
1802
|
-
* Node occupancy. `residentRefs` is entry count (pending + store), not
|
|
1803
|
-
* outstanding registry leases.
|
|
1804
|
-
*/
|
|
1805
|
-
setResident(input) {
|
|
1806
|
-
this.node.residentRefs = input.refs;
|
|
1807
|
-
this.node.residentBytes = input.bytes;
|
|
1808
|
-
}
|
|
1809
|
-
setCameraResident(deviceId, input) {
|
|
1810
|
-
const counters = this.camera(deviceId);
|
|
1811
|
-
counters.residentRefs = input.refs;
|
|
1812
|
-
counters.residentBytes = input.bytes;
|
|
1813
|
-
}
|
|
1814
|
-
forgetCamera(deviceId) {
|
|
1815
|
-
this.cameras.delete(deviceId);
|
|
1816
|
-
}
|
|
1817
|
-
/**
|
|
1818
|
-
* Zero occupancy for already-tracked cameras absent from `present`.
|
|
1819
|
-
* Does not create or delete entries — snapshot still does not prune,
|
|
1820
|
-
* and detach still goes through {@link forgetCamera}.
|
|
1821
|
-
*/
|
|
1822
|
-
zeroAbsentResidents(present) {
|
|
1823
|
-
for (const [deviceId, counters] of this.cameras) {
|
|
1824
|
-
if (present.has(deviceId)) continue;
|
|
1825
|
-
counters.residentRefs = 0;
|
|
1826
|
-
counters.residentBytes = 0;
|
|
1898
|
+
if (outcome.kind === "shed") {
|
|
1899
|
+
c.shed += 1;
|
|
1900
|
+
if (outcome.shedSignal === "zero-detections-fallback") c.fallbackShed += 1;
|
|
1901
|
+
else c.firstClassShed += 1;
|
|
1902
|
+
return;
|
|
1827
1903
|
}
|
|
1828
|
-
|
|
1829
|
-
snapshot(deviceIds) {
|
|
1830
|
-
return {
|
|
1831
|
-
node: { ...this.node },
|
|
1832
|
-
cameras: deviceIds.map((deviceId) => ({
|
|
1833
|
-
deviceId,
|
|
1834
|
-
...this.camera(deviceId)
|
|
1835
|
-
}))
|
|
1836
|
-
};
|
|
1837
|
-
}
|
|
1838
|
-
trackedCameraCount() {
|
|
1839
|
-
return this.cameras.size;
|
|
1840
|
-
}
|
|
1841
|
-
camera(deviceId) {
|
|
1842
|
-
const existing = this.cameras.get(deviceId);
|
|
1843
|
-
if (existing) return existing;
|
|
1844
|
-
const created = emptyFrameLazyCounters();
|
|
1845
|
-
this.cameras.set(deviceId, created);
|
|
1846
|
-
return created;
|
|
1847
|
-
}
|
|
1848
|
-
addHit(counters, kind, n) {
|
|
1849
|
-
if (kind === "native") counters.nativeHits += n;
|
|
1850
|
-
else if (kind === "tile") counters.tileHits += n;
|
|
1851
|
-
else counters.fallbackHits += n;
|
|
1852
|
-
}
|
|
1853
|
-
addMiss(counters, kind, n) {
|
|
1854
|
-
if (kind === "native") counters.nativeMisses += n;
|
|
1855
|
-
else if (kind === "tile") counters.tileMisses += n;
|
|
1856
|
-
else counters.fallbackMisses += n;
|
|
1857
|
-
}
|
|
1858
|
-
};
|
|
1859
|
-
//#endregion
|
|
1860
|
-
//#region src/pipeline-runner/frame-ref-transport.ts
|
|
1861
|
-
/** A local ref never leaves the grouped process; inline remains rollback/split compatibility. */
|
|
1862
|
-
function selectExecutorFrameSource(coLocated, frameRef, inlineFrame) {
|
|
1863
|
-
return coLocated && frameRef ? { frameRef } : { frame: inlineFrame };
|
|
1864
|
-
}
|
|
1865
|
-
//#endregion
|
|
1866
|
-
//#region src/session-decode/on-demand-jpeg-fallback.ts
|
|
1867
|
-
/** Single content-sized JPEG requested after a detection, never a continuous raw. */
|
|
1868
|
-
var ON_DEMAND_JPEG_VIEW = {
|
|
1869
|
-
content: {
|
|
1870
|
-
width: 640,
|
|
1871
|
-
height: 640
|
|
1872
|
-
},
|
|
1873
|
-
fit: "contain",
|
|
1874
|
-
format: "jpeg"
|
|
1875
|
-
};
|
|
1876
|
-
function createOnDemandJpegFallback(registry, ref) {
|
|
1877
|
-
const lease = registry.acquire(ref);
|
|
1878
|
-
let cached;
|
|
1879
|
-
let released = false;
|
|
1880
|
-
const releaseLease = (reason) => {
|
|
1881
|
-
if (released) return;
|
|
1882
|
-
released = true;
|
|
1883
|
-
lease?.release(reason);
|
|
1904
|
+
c.errors += 1;
|
|
1884
1905
|
};
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1906
|
+
const measureMarker = opts.onMeasureStart !== void 0 && opts.settleMs > 0 ? opts.sleepUntil(measureStart).then(() => opts.onMeasureStart?.()) : (opts.onMeasureStart?.(), Promise.resolve());
|
|
1907
|
+
const cameraLoop = async (cameraIndex) => {
|
|
1908
|
+
const offset = cameraIndex / cameras * intervalMs;
|
|
1909
|
+
let inFlight = false;
|
|
1910
|
+
for (let k = 0;; k++) {
|
|
1911
|
+
const tickAt = start + offset + k * intervalMs;
|
|
1912
|
+
if (tickAt >= end) break;
|
|
1913
|
+
if (opts.shouldStop()) {
|
|
1914
|
+
c.aborted = true;
|
|
1915
|
+
break;
|
|
1891
1916
|
}
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
releaseLease("drop");
|
|
1897
|
-
return null;
|
|
1898
|
-
}
|
|
1899
|
-
cached = {
|
|
1900
|
-
data: Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength),
|
|
1901
|
-
width: view.width,
|
|
1902
|
-
height: view.height,
|
|
1903
|
-
format: "jpeg",
|
|
1904
|
-
timestamp: ref.timestamp,
|
|
1905
|
-
...ref.capturedAt !== void 0 ? { capturedAt: ref.capturedAt } : {}
|
|
1906
|
-
};
|
|
1907
|
-
releaseLease("success");
|
|
1908
|
-
return cached;
|
|
1909
|
-
} catch {
|
|
1910
|
-
cached = null;
|
|
1911
|
-
releaseLease("error");
|
|
1912
|
-
return null;
|
|
1917
|
+
if (tickAt > opts.now()) await opts.sleepUntil(tickAt);
|
|
1918
|
+
if (opts.shouldStop()) {
|
|
1919
|
+
c.aborted = true;
|
|
1920
|
+
break;
|
|
1913
1921
|
}
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1922
|
+
const counted = tickAt >= measureStart;
|
|
1923
|
+
if (counted) c.offered += 1;
|
|
1924
|
+
if (inFlight) {
|
|
1925
|
+
if (counted) c.droppedByDriver += 1;
|
|
1926
|
+
continue;
|
|
1927
|
+
}
|
|
1928
|
+
inFlight = true;
|
|
1929
|
+
if (counted) c.dispatched += 1;
|
|
1930
|
+
const t0 = opts.now();
|
|
1931
|
+
c.outstanding += 1;
|
|
1932
|
+
opts.dispatch(cameraIndex).then((outcome) => recordOutcome(counted, outcome, opts.now() - t0), () => {
|
|
1933
|
+
if (counted) c.errors += 1;
|
|
1934
|
+
}).finally(() => {
|
|
1935
|
+
inFlight = false;
|
|
1936
|
+
c.outstanding -= 1;
|
|
1937
|
+
});
|
|
1917
1938
|
}
|
|
1918
1939
|
};
|
|
1940
|
+
await Promise.all(Array.from({ length: cameras }, (_, i) => cameraLoop(i)));
|
|
1941
|
+
await measureMarker;
|
|
1942
|
+
const drainDeadline = opts.now() + Math.max(0, opts.drainTimeoutMs);
|
|
1943
|
+
while (c.outstanding > 0 && opts.now() < drainDeadline) await opts.sleepUntil(Math.min(drainDeadline, opts.now() + 50));
|
|
1944
|
+
return {
|
|
1945
|
+
offered: c.offered,
|
|
1946
|
+
dispatched: c.dispatched,
|
|
1947
|
+
completed: c.completed,
|
|
1948
|
+
droppedByDriver: c.droppedByDriver,
|
|
1949
|
+
shed: c.shed,
|
|
1950
|
+
firstClassShed: c.firstClassShed,
|
|
1951
|
+
fallbackShed: c.fallbackShed,
|
|
1952
|
+
errors: c.errors,
|
|
1953
|
+
unresolvedAtDrain: c.outstanding,
|
|
1954
|
+
e2eMsList: c.e2eMsList,
|
|
1955
|
+
inferMsList: c.inferMsList,
|
|
1956
|
+
predictMsList: c.predictMsList,
|
|
1957
|
+
detectionCounts: c.detectionCounts,
|
|
1958
|
+
wallMs: opts.now() - start,
|
|
1959
|
+
aborted: c.aborted
|
|
1960
|
+
};
|
|
1919
1961
|
}
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
function
|
|
1927
|
-
if (
|
|
1928
|
-
|
|
1929
|
-
return
|
|
1962
|
+
var DEFAULT_CAPACITY_SLO = {
|
|
1963
|
+
dropRatioMax: .01,
|
|
1964
|
+
e2eP95MsMax: 250,
|
|
1965
|
+
stallsMax: 0
|
|
1966
|
+
};
|
|
1967
|
+
/** Nearest-rank percentile on an unsorted list. Returns 0 for empty input. */
|
|
1968
|
+
function percentile(values, p) {
|
|
1969
|
+
if (values.length === 0) return 0;
|
|
1970
|
+
const sorted = [...values].toSorted((a, b) => a - b);
|
|
1971
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))] ?? 0;
|
|
1930
1972
|
}
|
|
1931
|
-
function
|
|
1932
|
-
if (
|
|
1933
|
-
return
|
|
1973
|
+
function mean(values) {
|
|
1974
|
+
if (values.length === 0) return 0;
|
|
1975
|
+
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
1934
1976
|
}
|
|
1935
|
-
|
|
1936
|
-
|
|
1977
|
+
/**
|
|
1978
|
+
* Judge one step against the SLO. Pure — the whole gate is testable without
|
|
1979
|
+
* a clock or a node. An error or an unresolved call is never "just a drop":
|
|
1980
|
+
* it means something other than flow control broke, so it fails the step
|
|
1981
|
+
* regardless of ratio (spec §1: errors 0 tolerated).
|
|
1982
|
+
*/
|
|
1983
|
+
function evaluateCapacityStep(raw, stall, slo) {
|
|
1984
|
+
const dropRatio = raw.offered > 0 ? (raw.droppedByDriver + raw.shed) / raw.offered : 0;
|
|
1985
|
+
const e2eMs = {
|
|
1986
|
+
p50: percentile(raw.e2eMsList, .5),
|
|
1987
|
+
p95: percentile(raw.e2eMsList, .95),
|
|
1988
|
+
p99: percentile(raw.e2eMsList, .99),
|
|
1989
|
+
max: raw.e2eMsList.length > 0 ? Math.max(...raw.e2eMsList) : 0
|
|
1990
|
+
};
|
|
1991
|
+
const failReasons = [];
|
|
1992
|
+
if (dropRatio > slo.dropRatioMax) failReasons.push(`dropRatio ${(dropRatio * 100).toFixed(2)}% > ${(slo.dropRatioMax * 100).toFixed(2)}%`);
|
|
1993
|
+
if (e2eMs.p95 > slo.e2eP95MsMax) failReasons.push(`e2e p95 ${Math.round(e2eMs.p95)}ms > ${slo.e2eP95MsMax}ms`);
|
|
1994
|
+
if (stall.count > slo.stallsMax) failReasons.push(`event-loop stalls ${stall.count} > ${slo.stallsMax} (max ${stall.maxMs}ms)`);
|
|
1995
|
+
if (raw.errors > 0) failReasons.push(`errors ${raw.errors} (0 tolerated)`);
|
|
1996
|
+
if (raw.unresolvedAtDrain > 0) failReasons.push(`unresolved calls at drain ${raw.unresolvedAtDrain}`);
|
|
1997
|
+
return {
|
|
1998
|
+
dropRatio,
|
|
1999
|
+
e2eMs,
|
|
2000
|
+
inferMs: {
|
|
2001
|
+
avg: mean(raw.inferMsList),
|
|
2002
|
+
p95: percentile(raw.inferMsList, .95)
|
|
2003
|
+
},
|
|
2004
|
+
predictMs: {
|
|
2005
|
+
avg: mean(raw.predictMsList),
|
|
2006
|
+
p95: percentile(raw.predictMsList, .95)
|
|
2007
|
+
},
|
|
2008
|
+
detectionsAvg: mean(raw.detectionCounts),
|
|
2009
|
+
pass: failReasons.length === 0,
|
|
2010
|
+
failReasons
|
|
2011
|
+
};
|
|
1937
2012
|
}
|
|
1938
2013
|
/**
|
|
1939
|
-
*
|
|
1940
|
-
*
|
|
1941
|
-
*
|
|
2014
|
+
* `N_max` — the last PASSING step's camera count, or null when none passed.
|
|
2015
|
+
* Steps after the first failure do not resurrect the number (the spec stops
|
|
2016
|
+
* at the first failing step; a re-run confirms the knee, not this fold).
|
|
1942
2017
|
*/
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
this.registry = options.registry;
|
|
1952
|
-
this.telemetry = options.telemetry;
|
|
1953
|
-
this.continuousRetain = options.continuousRetain;
|
|
1954
|
-
}
|
|
1955
|
-
get pendingSize() {
|
|
1956
|
-
return this.pending.size;
|
|
1957
|
-
}
|
|
1958
|
-
hasPending(handle) {
|
|
1959
|
-
return this.pending.has(handleKey(handle));
|
|
1960
|
-
}
|
|
1961
|
-
admitFrame(deviceId, frame) {
|
|
1962
|
-
if (this.continuousRetain) {
|
|
1963
|
-
const fallback = frame.frameRef ? createRetainedContentFallback(this.registry, frame.frameRef) : void 0;
|
|
1964
|
-
const handle = this.store.write(deviceId, frame, fallback) ?? void 0;
|
|
1965
|
-
if (handle) this.pending.set(handleKey(handle), {
|
|
1966
|
-
deviceId,
|
|
1967
|
-
ref: frame.frameRef
|
|
1968
|
-
});
|
|
1969
|
-
this.refreshResident();
|
|
1970
|
-
return handle;
|
|
1971
|
-
}
|
|
1972
|
-
const handle = this.store.allocateHandle(deviceId, frame) ?? void 0;
|
|
1973
|
-
if (!handle) return void 0;
|
|
1974
|
-
this.pending.set(handleKey(handle), {
|
|
1975
|
-
deviceId,
|
|
1976
|
-
ref: frame.frameRef
|
|
1977
|
-
});
|
|
1978
|
-
this.telemetry.noteAvoidedWrite(deviceId);
|
|
1979
|
-
this.refreshResident();
|
|
1980
|
-
return handle;
|
|
1981
|
-
}
|
|
1982
|
-
noteFrame(input) {
|
|
1983
|
-
if (input.decoded) this.telemetry.noteDecoded(input.deviceId);
|
|
1984
|
-
if (input.admitted) this.telemetry.noteAdmitted(input.deviceId);
|
|
1985
|
-
if (input.droppedPixelFree) this.telemetry.noteDroppedPixelFree(input.deviceId);
|
|
1986
|
-
}
|
|
1987
|
-
noteInferenceResult(deviceId, handle, result) {
|
|
1988
|
-
if (!handle) return;
|
|
1989
|
-
const key = handleKey(handle);
|
|
1990
|
-
const pending = this.pending.get(key);
|
|
1991
|
-
this.forgetPending(key);
|
|
1992
|
-
if (!shouldRetainDetectionFrame({
|
|
1993
|
-
admitted: true,
|
|
1994
|
-
hasDetection: (result.detections?.length ?? 0) > 0,
|
|
1995
|
-
continuousRetain: this.continuousRetain
|
|
1996
|
-
})) {
|
|
1997
|
-
this.refreshResident();
|
|
1998
|
-
return;
|
|
1999
|
-
}
|
|
2000
|
-
if (this.continuousRetain) {
|
|
2001
|
-
this.refreshResident();
|
|
2002
|
-
return;
|
|
2003
|
-
}
|
|
2004
|
-
if (!pending?.ref) {
|
|
2005
|
-
this.refreshResident();
|
|
2006
|
-
return;
|
|
2007
|
-
}
|
|
2008
|
-
const fallback = createOnDemandJpegFallback(this.registry, pending.ref);
|
|
2009
|
-
this.store.rememberOnDemand(deviceId, handle, fallback);
|
|
2010
|
-
this.refreshResident();
|
|
2011
|
-
}
|
|
2012
|
-
async requestMedia(handle, request) {
|
|
2013
|
-
const deviceId = this.store.getDeviceId(handle) ?? this.pending.get(handleKey(handle))?.deviceId ?? 0;
|
|
2014
|
-
const kind = request.kind === "full-frame" ? "full-frame" : "crop";
|
|
2015
|
-
this.telemetry.noteOnDemandRequest({
|
|
2016
|
-
deviceId,
|
|
2017
|
-
kind
|
|
2018
|
-
});
|
|
2019
|
-
if (request.nativeHit) {
|
|
2020
|
-
this.telemetry.noteHit("native", deviceId);
|
|
2021
|
-
return {
|
|
2022
|
-
tier: "native",
|
|
2023
|
-
bytes: request.nativeHit.bytes,
|
|
2024
|
-
width: request.nativeHit.width,
|
|
2025
|
-
height: request.nativeHit.height
|
|
2026
|
-
};
|
|
2027
|
-
}
|
|
2028
|
-
this.telemetry.noteMiss("native", deviceId);
|
|
2029
|
-
if (request.tileHit) {
|
|
2030
|
-
this.telemetry.noteHit("tile", deviceId);
|
|
2031
|
-
return {
|
|
2032
|
-
tier: "tile",
|
|
2033
|
-
bytes: request.tileHit.bytes,
|
|
2034
|
-
jpeg: request.tileHit.jpeg ? request.tileHit.bytes : void 0,
|
|
2035
|
-
width: request.tileHit.width,
|
|
2036
|
-
height: request.tileHit.height
|
|
2037
|
-
};
|
|
2038
|
-
}
|
|
2039
|
-
this.telemetry.noteMiss("tile", deviceId);
|
|
2040
|
-
if (request.recordedStillHit) return {
|
|
2041
|
-
tier: "recorded-still",
|
|
2042
|
-
jpeg: request.recordedStillHit.jpeg,
|
|
2043
|
-
width: request.recordedStillHit.width,
|
|
2044
|
-
height: request.recordedStillHit.height
|
|
2045
|
-
};
|
|
2046
|
-
if (!this.continuousRetain && !this.store.has(handle)) {
|
|
2047
|
-
this.telemetry.noteMiss("fallback", deviceId);
|
|
2048
|
-
return { tier: "miss" };
|
|
2049
|
-
}
|
|
2050
|
-
const view = await this.store.ensureContent(handle);
|
|
2051
|
-
if (view && view.data.byteLength > 0) {
|
|
2052
|
-
this.telemetry.noteHit("fallback", deviceId);
|
|
2053
|
-
this.refreshResident();
|
|
2054
|
-
if (view.format === "jpeg") return {
|
|
2055
|
-
tier: "fallback-jpeg",
|
|
2056
|
-
jpeg: view.data,
|
|
2057
|
-
width: view.width,
|
|
2058
|
-
height: view.height
|
|
2059
|
-
};
|
|
2060
|
-
return {
|
|
2061
|
-
tier: "fallback-jpeg",
|
|
2062
|
-
bytes: view.data,
|
|
2063
|
-
width: view.width,
|
|
2064
|
-
height: view.height
|
|
2065
|
-
};
|
|
2066
|
-
}
|
|
2067
|
-
this.telemetry.noteMiss("fallback", deviceId);
|
|
2068
|
-
this.telemetry.noteStale(deviceId);
|
|
2069
|
-
return { tier: "miss" };
|
|
2070
|
-
}
|
|
2071
|
-
/**
|
|
2072
|
-
* Pending cleanup. Success never invalidates a store entry already
|
|
2073
|
-
* transferred for late pull. Drop/error/detach drop the store only when the
|
|
2074
|
-
* handle was still pending (not yet transferred). Eviction owns transferred
|
|
2075
|
-
* fallback cleanup.
|
|
2076
|
-
*/
|
|
2077
|
-
release(handle, reason) {
|
|
2078
|
-
const key = handleKey(handle);
|
|
2079
|
-
const pending = this.pending.get(key);
|
|
2080
|
-
this.forgetPending(key);
|
|
2081
|
-
if (reason !== "success" && pending) this.store.drop(handle);
|
|
2082
|
-
this.telemetry.noteRelease(pending?.deviceId ?? this.store.getDeviceId(handle));
|
|
2083
|
-
this.refreshResident();
|
|
2084
|
-
}
|
|
2085
|
-
forgetDevice(deviceId) {
|
|
2086
|
-
for (const [key, pending] of [...this.pending]) {
|
|
2087
|
-
if (pending.deviceId !== deviceId) continue;
|
|
2088
|
-
this.pending.delete(key);
|
|
2089
|
-
}
|
|
2090
|
-
this.telemetry.forgetCamera(deviceId);
|
|
2091
|
-
this.refreshResident();
|
|
2092
|
-
}
|
|
2093
|
-
syncResident() {
|
|
2094
|
-
this.refreshResident();
|
|
2095
|
-
}
|
|
2096
|
-
forgetPending(key) {
|
|
2097
|
-
this.pending.delete(key);
|
|
2098
|
-
}
|
|
2099
|
-
refreshResident() {
|
|
2100
|
-
const byDevice = /* @__PURE__ */ new Map();
|
|
2101
|
-
for (const pending of this.pending.values()) {
|
|
2102
|
-
const cur = byDevice.get(pending.deviceId) ?? {
|
|
2103
|
-
refs: 0,
|
|
2104
|
-
bytes: 0
|
|
2105
|
-
};
|
|
2106
|
-
cur.refs += 1;
|
|
2107
|
-
byDevice.set(pending.deviceId, cur);
|
|
2108
|
-
}
|
|
2109
|
-
for (const stats of this.store.deviceStats()) {
|
|
2110
|
-
const cur = byDevice.get(stats.deviceId) ?? {
|
|
2111
|
-
refs: 0,
|
|
2112
|
-
bytes: 0
|
|
2113
|
-
};
|
|
2114
|
-
cur.refs += stats.entries;
|
|
2115
|
-
cur.bytes += stats.bytes;
|
|
2116
|
-
byDevice.set(stats.deviceId, cur);
|
|
2117
|
-
}
|
|
2118
|
-
this.telemetry.setResident({
|
|
2119
|
-
refs: this.pending.size + this.store.size,
|
|
2120
|
-
bytes: this.registry.estimatedBytes + this.store.retainedBytes
|
|
2121
|
-
});
|
|
2122
|
-
this.telemetry.zeroAbsentResidents(new Set(byDevice.keys()));
|
|
2123
|
-
for (const [deviceId, stats] of byDevice) this.telemetry.setCameraResident(deviceId, stats);
|
|
2124
|
-
}
|
|
2125
|
-
};
|
|
2126
|
-
//#endregion
|
|
2127
|
-
//#region src/pipeline-runner/capacity/capacity-runner.ts
|
|
2128
|
-
/**
|
|
2129
|
-
* Run ONE ramp step: `cameras` independent open-loop pacers for
|
|
2130
|
-
* `settleMs + measureMs`, then a bounded drain of in-flight calls.
|
|
2131
|
-
*/
|
|
2132
|
-
async function runCapacityStep(opts) {
|
|
2133
|
-
const cameras = Math.max(1, Math.floor(opts.cameras));
|
|
2134
|
-
const intervalMs = 1e3 / Math.max(.1, opts.fpsPerCamera);
|
|
2135
|
-
const start = opts.now();
|
|
2136
|
-
const measureStart = start + Math.max(0, opts.settleMs);
|
|
2137
|
-
const end = measureStart + Math.max(0, opts.measureMs);
|
|
2138
|
-
const c = {
|
|
2139
|
-
offered: 0,
|
|
2140
|
-
dispatched: 0,
|
|
2141
|
-
completed: 0,
|
|
2142
|
-
droppedByDriver: 0,
|
|
2143
|
-
shed: 0,
|
|
2144
|
-
firstClassShed: 0,
|
|
2145
|
-
fallbackShed: 0,
|
|
2146
|
-
errors: 0,
|
|
2147
|
-
aborted: false,
|
|
2148
|
-
outstanding: 0,
|
|
2149
|
-
e2eMsList: [],
|
|
2150
|
-
inferMsList: [],
|
|
2151
|
-
predictMsList: [],
|
|
2152
|
-
detectionCounts: []
|
|
2153
|
-
};
|
|
2154
|
-
const recordOutcome = (counted, outcome, e2eMs) => {
|
|
2155
|
-
if (!counted) return;
|
|
2156
|
-
if (outcome.kind === "ok") {
|
|
2157
|
-
c.completed += 1;
|
|
2158
|
-
c.e2eMsList.push(e2eMs);
|
|
2159
|
-
c.inferMsList.push(outcome.inferMs ?? 0);
|
|
2160
|
-
c.predictMsList.push(outcome.predictMs ?? 0);
|
|
2161
|
-
c.detectionCounts.push(outcome.detections ?? 0);
|
|
2162
|
-
return;
|
|
2163
|
-
}
|
|
2164
|
-
if (outcome.kind === "shed") {
|
|
2165
|
-
c.shed += 1;
|
|
2166
|
-
if (outcome.shedSignal === "zero-detections-fallback") c.fallbackShed += 1;
|
|
2167
|
-
else c.firstClassShed += 1;
|
|
2168
|
-
return;
|
|
2169
|
-
}
|
|
2170
|
-
c.errors += 1;
|
|
2171
|
-
};
|
|
2172
|
-
const measureMarker = opts.onMeasureStart !== void 0 && opts.settleMs > 0 ? opts.sleepUntil(measureStart).then(() => opts.onMeasureStart?.()) : (opts.onMeasureStart?.(), Promise.resolve());
|
|
2173
|
-
const cameraLoop = async (cameraIndex) => {
|
|
2174
|
-
const offset = cameraIndex / cameras * intervalMs;
|
|
2175
|
-
let inFlight = false;
|
|
2176
|
-
for (let k = 0;; k++) {
|
|
2177
|
-
const tickAt = start + offset + k * intervalMs;
|
|
2178
|
-
if (tickAt >= end) break;
|
|
2179
|
-
if (opts.shouldStop()) {
|
|
2180
|
-
c.aborted = true;
|
|
2181
|
-
break;
|
|
2182
|
-
}
|
|
2183
|
-
if (tickAt > opts.now()) await opts.sleepUntil(tickAt);
|
|
2184
|
-
if (opts.shouldStop()) {
|
|
2185
|
-
c.aborted = true;
|
|
2186
|
-
break;
|
|
2187
|
-
}
|
|
2188
|
-
const counted = tickAt >= measureStart;
|
|
2189
|
-
if (counted) c.offered += 1;
|
|
2190
|
-
if (inFlight) {
|
|
2191
|
-
if (counted) c.droppedByDriver += 1;
|
|
2192
|
-
continue;
|
|
2193
|
-
}
|
|
2194
|
-
inFlight = true;
|
|
2195
|
-
if (counted) c.dispatched += 1;
|
|
2196
|
-
const t0 = opts.now();
|
|
2197
|
-
c.outstanding += 1;
|
|
2198
|
-
opts.dispatch(cameraIndex).then((outcome) => recordOutcome(counted, outcome, opts.now() - t0), () => {
|
|
2199
|
-
if (counted) c.errors += 1;
|
|
2200
|
-
}).finally(() => {
|
|
2201
|
-
inFlight = false;
|
|
2202
|
-
c.outstanding -= 1;
|
|
2203
|
-
});
|
|
2204
|
-
}
|
|
2205
|
-
};
|
|
2206
|
-
await Promise.all(Array.from({ length: cameras }, (_, i) => cameraLoop(i)));
|
|
2207
|
-
await measureMarker;
|
|
2208
|
-
const drainDeadline = opts.now() + Math.max(0, opts.drainTimeoutMs);
|
|
2209
|
-
while (c.outstanding > 0 && opts.now() < drainDeadline) await opts.sleepUntil(Math.min(drainDeadline, opts.now() + 50));
|
|
2210
|
-
return {
|
|
2211
|
-
offered: c.offered,
|
|
2212
|
-
dispatched: c.dispatched,
|
|
2213
|
-
completed: c.completed,
|
|
2214
|
-
droppedByDriver: c.droppedByDriver,
|
|
2215
|
-
shed: c.shed,
|
|
2216
|
-
firstClassShed: c.firstClassShed,
|
|
2217
|
-
fallbackShed: c.fallbackShed,
|
|
2218
|
-
errors: c.errors,
|
|
2219
|
-
unresolvedAtDrain: c.outstanding,
|
|
2220
|
-
e2eMsList: c.e2eMsList,
|
|
2221
|
-
inferMsList: c.inferMsList,
|
|
2222
|
-
predictMsList: c.predictMsList,
|
|
2223
|
-
detectionCounts: c.detectionCounts,
|
|
2224
|
-
wallMs: opts.now() - start,
|
|
2225
|
-
aborted: c.aborted
|
|
2226
|
-
};
|
|
2227
|
-
}
|
|
2228
|
-
var DEFAULT_CAPACITY_SLO = {
|
|
2229
|
-
dropRatioMax: .01,
|
|
2230
|
-
e2eP95MsMax: 250,
|
|
2231
|
-
stallsMax: 0
|
|
2232
|
-
};
|
|
2233
|
-
/** Nearest-rank percentile on an unsorted list. Returns 0 for empty input. */
|
|
2234
|
-
function percentile(values, p) {
|
|
2235
|
-
if (values.length === 0) return 0;
|
|
2236
|
-
const sorted = [...values].toSorted((a, b) => a - b);
|
|
2237
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))] ?? 0;
|
|
2238
|
-
}
|
|
2239
|
-
function mean(values) {
|
|
2240
|
-
if (values.length === 0) return 0;
|
|
2241
|
-
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
2242
|
-
}
|
|
2243
|
-
/**
|
|
2244
|
-
* Judge one step against the SLO. Pure — the whole gate is testable without
|
|
2245
|
-
* a clock or a node. An error or an unresolved call is never "just a drop":
|
|
2246
|
-
* it means something other than flow control broke, so it fails the step
|
|
2247
|
-
* regardless of ratio (spec §1: errors 0 tolerated).
|
|
2248
|
-
*/
|
|
2249
|
-
function evaluateCapacityStep(raw, stall, slo) {
|
|
2250
|
-
const dropRatio = raw.offered > 0 ? (raw.droppedByDriver + raw.shed) / raw.offered : 0;
|
|
2251
|
-
const e2eMs = {
|
|
2252
|
-
p50: percentile(raw.e2eMsList, .5),
|
|
2253
|
-
p95: percentile(raw.e2eMsList, .95),
|
|
2254
|
-
p99: percentile(raw.e2eMsList, .99),
|
|
2255
|
-
max: raw.e2eMsList.length > 0 ? Math.max(...raw.e2eMsList) : 0
|
|
2256
|
-
};
|
|
2257
|
-
const failReasons = [];
|
|
2258
|
-
if (dropRatio > slo.dropRatioMax) failReasons.push(`dropRatio ${(dropRatio * 100).toFixed(2)}% > ${(slo.dropRatioMax * 100).toFixed(2)}%`);
|
|
2259
|
-
if (e2eMs.p95 > slo.e2eP95MsMax) failReasons.push(`e2e p95 ${Math.round(e2eMs.p95)}ms > ${slo.e2eP95MsMax}ms`);
|
|
2260
|
-
if (stall.count > slo.stallsMax) failReasons.push(`event-loop stalls ${stall.count} > ${slo.stallsMax} (max ${stall.maxMs}ms)`);
|
|
2261
|
-
if (raw.errors > 0) failReasons.push(`errors ${raw.errors} (0 tolerated)`);
|
|
2262
|
-
if (raw.unresolvedAtDrain > 0) failReasons.push(`unresolved calls at drain ${raw.unresolvedAtDrain}`);
|
|
2263
|
-
return {
|
|
2264
|
-
dropRatio,
|
|
2265
|
-
e2eMs,
|
|
2266
|
-
inferMs: {
|
|
2267
|
-
avg: mean(raw.inferMsList),
|
|
2268
|
-
p95: percentile(raw.inferMsList, .95)
|
|
2269
|
-
},
|
|
2270
|
-
predictMs: {
|
|
2271
|
-
avg: mean(raw.predictMsList),
|
|
2272
|
-
p95: percentile(raw.predictMsList, .95)
|
|
2273
|
-
},
|
|
2274
|
-
detectionsAvg: mean(raw.detectionCounts),
|
|
2275
|
-
pass: failReasons.length === 0,
|
|
2276
|
-
failReasons
|
|
2277
|
-
};
|
|
2278
|
-
}
|
|
2279
|
-
/**
|
|
2280
|
-
* `N_max` — the last PASSING step's camera count, or null when none passed.
|
|
2281
|
-
* Steps after the first failure do not resurrect the number (the spec stops
|
|
2282
|
-
* at the first failing step; a re-run confirms the knee, not this fold).
|
|
2283
|
-
*/
|
|
2284
|
-
function computeNMax(steps) {
|
|
2285
|
-
let nMax = null;
|
|
2286
|
-
for (const step of steps) {
|
|
2287
|
-
if (!step.pass) return {
|
|
2288
|
-
nMax,
|
|
2289
|
-
failedStep: step.cameras
|
|
2290
|
-
};
|
|
2291
|
-
nMax = step.cameras;
|
|
2018
|
+
function computeNMax(steps) {
|
|
2019
|
+
let nMax = null;
|
|
2020
|
+
for (const step of steps) {
|
|
2021
|
+
if (!step.pass) return {
|
|
2022
|
+
nMax,
|
|
2023
|
+
failedStep: step.cameras
|
|
2024
|
+
};
|
|
2025
|
+
nMax = step.cameras;
|
|
2292
2026
|
}
|
|
2293
2027
|
return {
|
|
2294
2028
|
nMax,
|
|
@@ -2963,40 +2697,197 @@ function resolveStepDevice(input) {
|
|
|
2963
2697
|
reason: "manual"
|
|
2964
2698
|
};
|
|
2965
2699
|
}
|
|
2966
|
-
if (canRunOn(effectiveDeviceKey, canRun)) return {
|
|
2967
|
-
kind: "no-jump",
|
|
2968
|
-
deviceKey: effectiveDeviceKey
|
|
2969
|
-
};
|
|
2970
|
-
const runnable = input.devices.filter((d) => d.deviceKey !== effectiveDeviceKey && (d.maxSessions === null || d.attachedCameras < d.maxSessions) && canRunOn(d.deviceKey, canRun));
|
|
2971
|
-
if (runnable.length === 0) return {
|
|
2972
|
-
kind: "no-candidate",
|
|
2973
|
-
deviceKey: effectiveDeviceKey
|
|
2974
|
-
};
|
|
2975
|
-
return {
|
|
2976
|
-
kind: "jumped",
|
|
2977
|
-
deviceKey: pickWeightedLeastLoaded(runnable).deviceKey,
|
|
2978
|
-
reason: "auto"
|
|
2979
|
-
};
|
|
2980
|
-
}
|
|
2981
|
-
//#endregion
|
|
2982
|
-
//#region src/pipeline-runner/device-load.ts
|
|
2983
|
-
/** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
|
|
2984
|
-
function backendOf(deviceKey) {
|
|
2985
|
-
const colon = deviceKey.indexOf(":");
|
|
2986
|
-
return colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
|
|
2987
|
-
}
|
|
2988
|
-
function summarizeDeviceLoad(attachedDeviceKeys) {
|
|
2989
|
-
const counts = /* @__PURE__ */ new Map();
|
|
2990
|
-
for (const key of attachedDeviceKeys) {
|
|
2991
|
-
if (!key) continue;
|
|
2992
|
-
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
2700
|
+
if (canRunOn(effectiveDeviceKey, canRun)) return {
|
|
2701
|
+
kind: "no-jump",
|
|
2702
|
+
deviceKey: effectiveDeviceKey
|
|
2703
|
+
};
|
|
2704
|
+
const runnable = input.devices.filter((d) => d.deviceKey !== effectiveDeviceKey && (d.maxSessions === null || d.attachedCameras < d.maxSessions) && canRunOn(d.deviceKey, canRun));
|
|
2705
|
+
if (runnable.length === 0) return {
|
|
2706
|
+
kind: "no-candidate",
|
|
2707
|
+
deviceKey: effectiveDeviceKey
|
|
2708
|
+
};
|
|
2709
|
+
return {
|
|
2710
|
+
kind: "jumped",
|
|
2711
|
+
deviceKey: pickWeightedLeastLoaded(runnable).deviceKey,
|
|
2712
|
+
reason: "auto"
|
|
2713
|
+
};
|
|
2714
|
+
}
|
|
2715
|
+
//#endregion
|
|
2716
|
+
//#region src/pipeline-runner/device-load.ts
|
|
2717
|
+
/** The backend prefix of a `<backend>:<device>` key (bare `cpu` → `cpu`). */
|
|
2718
|
+
function backendOf(deviceKey) {
|
|
2719
|
+
const colon = deviceKey.indexOf(":");
|
|
2720
|
+
return colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
|
|
2721
|
+
}
|
|
2722
|
+
function summarizeDeviceLoad(attachedDeviceKeys) {
|
|
2723
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2724
|
+
for (const key of attachedDeviceKeys) {
|
|
2725
|
+
if (!key) continue;
|
|
2726
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
2727
|
+
}
|
|
2728
|
+
return Array.from(counts.entries()).map(([deviceKey, attachedCameras]) => ({
|
|
2729
|
+
deviceKey,
|
|
2730
|
+
backend: backendOf(deviceKey),
|
|
2731
|
+
attachedCameras,
|
|
2732
|
+
queueDepthTotal: 0
|
|
2733
|
+
})).toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey));
|
|
2734
|
+
}
|
|
2735
|
+
//#endregion
|
|
2736
|
+
//#region src/pipeline-runner/frame-lazy-metrics.ts
|
|
2737
|
+
function emptyFrameLazyCounters() {
|
|
2738
|
+
return {
|
|
2739
|
+
framesDecoded: 0,
|
|
2740
|
+
framesAdmitted: 0,
|
|
2741
|
+
framesDroppedPixelFree: 0,
|
|
2742
|
+
viewsMaterialized: 0,
|
|
2743
|
+
viewsSkipped: 0,
|
|
2744
|
+
workerToRunnerBytes: 0,
|
|
2745
|
+
runnerToPoolRawBytes: 0,
|
|
2746
|
+
runnerToPoolJpegBytes: 0,
|
|
2747
|
+
onDemandFullFrameRequests: 0,
|
|
2748
|
+
onDemandCropRequests: 0,
|
|
2749
|
+
nativeHits: 0,
|
|
2750
|
+
nativeMisses: 0,
|
|
2751
|
+
tileHits: 0,
|
|
2752
|
+
tileMisses: 0,
|
|
2753
|
+
fallbackHits: 0,
|
|
2754
|
+
fallbackMisses: 0,
|
|
2755
|
+
retainedWritesAvoided: 0,
|
|
2756
|
+
residentRefs: 0,
|
|
2757
|
+
residentBytes: 0,
|
|
2758
|
+
releases: 0,
|
|
2759
|
+
evictions: 0,
|
|
2760
|
+
staleMisses: 0
|
|
2761
|
+
};
|
|
2762
|
+
}
|
|
2763
|
+
var FrameLazyTelemetry = class {
|
|
2764
|
+
node = emptyFrameLazyCounters();
|
|
2765
|
+
cameras = /* @__PURE__ */ new Map();
|
|
2766
|
+
noteDecoded(deviceId) {
|
|
2767
|
+
this.node.framesDecoded += 1;
|
|
2768
|
+
this.camera(deviceId).framesDecoded += 1;
|
|
2769
|
+
}
|
|
2770
|
+
noteAdmitted(deviceId) {
|
|
2771
|
+
this.node.framesAdmitted += 1;
|
|
2772
|
+
this.camera(deviceId).framesAdmitted += 1;
|
|
2773
|
+
}
|
|
2774
|
+
noteDroppedPixelFree(deviceId) {
|
|
2775
|
+
this.node.framesDroppedPixelFree += 1;
|
|
2776
|
+
this.camera(deviceId).framesDroppedPixelFree += 1;
|
|
2777
|
+
}
|
|
2778
|
+
noteView(input) {
|
|
2779
|
+
if (input.materialized) {
|
|
2780
|
+
this.node.viewsMaterialized += 1;
|
|
2781
|
+
this.camera(input.deviceId).viewsMaterialized += 1;
|
|
2782
|
+
} else {
|
|
2783
|
+
this.node.viewsSkipped += 1;
|
|
2784
|
+
this.camera(input.deviceId).viewsSkipped += 1;
|
|
2785
|
+
}
|
|
2786
|
+
this.node.workerToRunnerBytes += input.workerToRunnerBytes;
|
|
2787
|
+
this.camera(input.deviceId).workerToRunnerBytes += input.workerToRunnerBytes;
|
|
2788
|
+
}
|
|
2789
|
+
notePoolBytes(input) {
|
|
2790
|
+
this.node.runnerToPoolRawBytes += input.raw;
|
|
2791
|
+
this.node.runnerToPoolJpegBytes += input.jpeg;
|
|
2792
|
+
this.camera(input.deviceId).runnerToPoolRawBytes += input.raw;
|
|
2793
|
+
this.camera(input.deviceId).runnerToPoolJpegBytes += input.jpeg;
|
|
2794
|
+
}
|
|
2795
|
+
noteOnDemandRequest(input) {
|
|
2796
|
+
if (input.kind === "full-frame") {
|
|
2797
|
+
this.node.onDemandFullFrameRequests += 1;
|
|
2798
|
+
this.camera(input.deviceId).onDemandFullFrameRequests += 1;
|
|
2799
|
+
} else {
|
|
2800
|
+
this.node.onDemandCropRequests += 1;
|
|
2801
|
+
this.camera(input.deviceId).onDemandCropRequests += 1;
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2804
|
+
noteHit(kind, deviceId) {
|
|
2805
|
+
this.addHit(this.node, kind, 1);
|
|
2806
|
+
this.addHit(this.camera(deviceId), kind, 1);
|
|
2807
|
+
}
|
|
2808
|
+
noteMiss(kind, deviceId) {
|
|
2809
|
+
this.addMiss(this.node, kind, 1);
|
|
2810
|
+
this.addMiss(this.camera(deviceId), kind, 1);
|
|
2811
|
+
}
|
|
2812
|
+
noteAvoidedWrite(deviceId) {
|
|
2813
|
+
this.node.retainedWritesAvoided += 1;
|
|
2814
|
+
this.camera(deviceId).retainedWritesAvoided += 1;
|
|
2815
|
+
}
|
|
2816
|
+
noteRelease(deviceId) {
|
|
2817
|
+
this.node.releases += 1;
|
|
2818
|
+
if (deviceId !== void 0) this.camera(deviceId).releases += 1;
|
|
2819
|
+
}
|
|
2820
|
+
noteEviction(deviceId) {
|
|
2821
|
+
this.node.evictions += 1;
|
|
2822
|
+
if (deviceId !== void 0) this.camera(deviceId).evictions += 1;
|
|
2823
|
+
}
|
|
2824
|
+
noteStale(deviceId) {
|
|
2825
|
+
this.node.staleMisses += 1;
|
|
2826
|
+
if (deviceId !== void 0) this.camera(deviceId).staleMisses += 1;
|
|
2827
|
+
}
|
|
2828
|
+
/**
|
|
2829
|
+
* Node occupancy. `residentRefs` is entry count (pending + store), not
|
|
2830
|
+
* outstanding registry leases.
|
|
2831
|
+
*/
|
|
2832
|
+
setResident(input) {
|
|
2833
|
+
this.node.residentRefs = input.refs;
|
|
2834
|
+
this.node.residentBytes = input.bytes;
|
|
2835
|
+
}
|
|
2836
|
+
setCameraResident(deviceId, input) {
|
|
2837
|
+
const counters = this.camera(deviceId);
|
|
2838
|
+
counters.residentRefs = input.refs;
|
|
2839
|
+
counters.residentBytes = input.bytes;
|
|
2840
|
+
}
|
|
2841
|
+
forgetCamera(deviceId) {
|
|
2842
|
+
this.cameras.delete(deviceId);
|
|
2843
|
+
}
|
|
2844
|
+
/**
|
|
2845
|
+
* Zero occupancy for already-tracked cameras absent from `present`.
|
|
2846
|
+
* Does not create or delete entries — snapshot still does not prune,
|
|
2847
|
+
* and detach still goes through {@link forgetCamera}.
|
|
2848
|
+
*/
|
|
2849
|
+
zeroAbsentResidents(present) {
|
|
2850
|
+
for (const [deviceId, counters] of this.cameras) {
|
|
2851
|
+
if (present.has(deviceId)) continue;
|
|
2852
|
+
counters.residentRefs = 0;
|
|
2853
|
+
counters.residentBytes = 0;
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
snapshot(deviceIds) {
|
|
2857
|
+
return {
|
|
2858
|
+
node: { ...this.node },
|
|
2859
|
+
cameras: deviceIds.map((deviceId) => ({
|
|
2860
|
+
deviceId,
|
|
2861
|
+
...this.camera(deviceId)
|
|
2862
|
+
}))
|
|
2863
|
+
};
|
|
2864
|
+
}
|
|
2865
|
+
trackedCameraCount() {
|
|
2866
|
+
return this.cameras.size;
|
|
2867
|
+
}
|
|
2868
|
+
camera(deviceId) {
|
|
2869
|
+
const existing = this.cameras.get(deviceId);
|
|
2870
|
+
if (existing) return existing;
|
|
2871
|
+
const created = emptyFrameLazyCounters();
|
|
2872
|
+
this.cameras.set(deviceId, created);
|
|
2873
|
+
return created;
|
|
2993
2874
|
}
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
2875
|
+
addHit(counters, kind, n) {
|
|
2876
|
+
if (kind === "native") counters.nativeHits += n;
|
|
2877
|
+
else if (kind === "tile") counters.tileHits += n;
|
|
2878
|
+
else counters.fallbackHits += n;
|
|
2879
|
+
}
|
|
2880
|
+
addMiss(counters, kind, n) {
|
|
2881
|
+
if (kind === "native") counters.nativeMisses += n;
|
|
2882
|
+
else if (kind === "tile") counters.tileMisses += n;
|
|
2883
|
+
else counters.fallbackMisses += n;
|
|
2884
|
+
}
|
|
2885
|
+
};
|
|
2886
|
+
//#endregion
|
|
2887
|
+
//#region src/pipeline-runner/frame-ref-transport.ts
|
|
2888
|
+
/** A local ref never leaves the grouped process; inline remains rollback/split compatibility. */
|
|
2889
|
+
function selectExecutorFrameSource(coLocated, frameRef, inlineFrame) {
|
|
2890
|
+
return coLocated && frameRef ? { frameRef } : { frame: inlineFrame };
|
|
3000
2891
|
}
|
|
3001
2892
|
//#endregion
|
|
3002
2893
|
//#region src/pipeline-runner/hardware-capabilities.ts
|
|
@@ -3150,81 +3041,470 @@ var INITIAL_STATE = {
|
|
|
3150
3041
|
burstRemaining: 0,
|
|
3151
3042
|
burstNextAt: 0
|
|
3152
3043
|
};
|
|
3153
|
-
var NO_RUN = {
|
|
3154
|
-
run: false,
|
|
3155
|
-
reason: null
|
|
3044
|
+
var NO_RUN = {
|
|
3045
|
+
run: false,
|
|
3046
|
+
reason: null
|
|
3047
|
+
};
|
|
3048
|
+
var MotionSettleGovernor = class {
|
|
3049
|
+
config;
|
|
3050
|
+
state = /* @__PURE__ */ new Map();
|
|
3051
|
+
constructor(config = DEFAULT_MOTION_SETTLE_CONFIG) {
|
|
3052
|
+
this.config = config;
|
|
3053
|
+
}
|
|
3054
|
+
/**
|
|
3055
|
+
* Decide whether `package-detection` runs for this device this frame, and
|
|
3056
|
+
* advance the per-device state immutably. Never throws.
|
|
3057
|
+
*/
|
|
3058
|
+
evaluate(sample) {
|
|
3059
|
+
const prev = this.state.get(sample.deviceId) ?? INITIAL_STATE;
|
|
3060
|
+
const lastMotionAt = sample.motionInZone ? sample.now : prev.lastMotionAt;
|
|
3061
|
+
const armed = sample.motionInZone ? true : prev.armed;
|
|
3062
|
+
const settleReady = armed && !sample.motionInZone && lastMotionAt !== null && sample.now - lastMotionAt >= this.config.settleMs;
|
|
3063
|
+
const pollReady = sample.now - prev.lastRunAt >= this.config.pollIntervalMs;
|
|
3064
|
+
if (settleReady) {
|
|
3065
|
+
this.state.set(sample.deviceId, {
|
|
3066
|
+
lastMotionAt,
|
|
3067
|
+
armed: false,
|
|
3068
|
+
lastRunAt: sample.now,
|
|
3069
|
+
burstRemaining: Math.max(0, this.config.burstCount - 1),
|
|
3070
|
+
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3071
|
+
});
|
|
3072
|
+
return {
|
|
3073
|
+
run: true,
|
|
3074
|
+
reason: "settle"
|
|
3075
|
+
};
|
|
3076
|
+
}
|
|
3077
|
+
if (prev.burstRemaining > 0 && sample.now >= prev.burstNextAt) {
|
|
3078
|
+
this.state.set(sample.deviceId, {
|
|
3079
|
+
lastMotionAt,
|
|
3080
|
+
armed,
|
|
3081
|
+
lastRunAt: sample.now,
|
|
3082
|
+
burstRemaining: prev.burstRemaining - 1,
|
|
3083
|
+
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3084
|
+
});
|
|
3085
|
+
return {
|
|
3086
|
+
run: true,
|
|
3087
|
+
reason: "burst"
|
|
3088
|
+
};
|
|
3089
|
+
}
|
|
3090
|
+
if (pollReady) {
|
|
3091
|
+
this.state.set(sample.deviceId, {
|
|
3092
|
+
lastMotionAt,
|
|
3093
|
+
armed,
|
|
3094
|
+
lastRunAt: sample.now,
|
|
3095
|
+
burstRemaining: prev.burstRemaining,
|
|
3096
|
+
burstNextAt: prev.burstNextAt
|
|
3097
|
+
});
|
|
3098
|
+
return {
|
|
3099
|
+
run: true,
|
|
3100
|
+
reason: "poll"
|
|
3101
|
+
};
|
|
3102
|
+
}
|
|
3103
|
+
this.state.set(sample.deviceId, {
|
|
3104
|
+
lastMotionAt,
|
|
3105
|
+
armed,
|
|
3106
|
+
lastRunAt: prev.lastRunAt,
|
|
3107
|
+
burstRemaining: prev.burstRemaining,
|
|
3108
|
+
burstNextAt: prev.burstNextAt
|
|
3109
|
+
});
|
|
3110
|
+
return NO_RUN;
|
|
3111
|
+
}
|
|
3112
|
+
/** Drop a device's state (call on detach / step-disable) to avoid a leak. */
|
|
3113
|
+
forget(deviceId) {
|
|
3114
|
+
this.state.delete(deviceId);
|
|
3115
|
+
}
|
|
3116
|
+
/** Clear all state (e.g. runner teardown). */
|
|
3117
|
+
reset() {
|
|
3118
|
+
this.state.clear();
|
|
3119
|
+
}
|
|
3120
|
+
};
|
|
3121
|
+
//#endregion
|
|
3122
|
+
//#region src/pipeline-runner/native-crop-tier.ts
|
|
3123
|
+
/**
|
|
3124
|
+
* Pure tier decision for `getNativeCrop`. A native HIT always resolves to the
|
|
3125
|
+
* native surface. On a native MISS, only a full-frame request falls back to the
|
|
3126
|
+
* ≤640 RAM store; an ROI miss is reported as `miss` (no quality masquerade).
|
|
3127
|
+
*/
|
|
3128
|
+
function resolveNativeCropSource(isFullFrame, nativeHit) {
|
|
3129
|
+
if (nativeHit) return "native";
|
|
3130
|
+
return isFullFrame ? "ram-fullframe" : "miss";
|
|
3131
|
+
}
|
|
3132
|
+
/**
|
|
3133
|
+
* Runtime hit kind for telemetry. A worker tile is native-resolution quality
|
|
3134
|
+
* (public wire `tier` stays `native`) but must not increment `nativeHits`.
|
|
3135
|
+
*/
|
|
3136
|
+
function resolveNativeCropHitKind(source) {
|
|
3137
|
+
return source === "tile" ? "tile" : "native";
|
|
3138
|
+
}
|
|
3139
|
+
//#endregion
|
|
3140
|
+
//#region src/pipeline-runner/native-lease-settings-source.ts
|
|
3141
|
+
/**
|
|
3142
|
+
* Where the runner gets the cluster-wide native-frame LEASE knobs.
|
|
3143
|
+
*
|
|
3144
|
+
* ## Why a settings read and not the attach config
|
|
3145
|
+
*
|
|
3146
|
+
* Identical reasoning to `crop-convention-source.ts`, and worth restating
|
|
3147
|
+
* because the temptation is the same: `RunnerCameraConfig` is the obvious
|
|
3148
|
+
* orchestrator→runner channel and it is the wrong one. Eleven call sites build
|
|
3149
|
+
* one; a cluster-wide constant threaded through eleven builders eventually
|
|
3150
|
+
* reaches ten of them, and the cameras attached by the forgotten builder just
|
|
3151
|
+
* quietly run a different lease window. Instead the runner reads the OWNER
|
|
3152
|
+
* addon's global settings directly, hub-routed by `AddonCallGateway`, so an
|
|
3153
|
+
* agent runner transparently reads the hub-central store with no placement
|
|
3154
|
+
* branching.
|
|
3155
|
+
*
|
|
3156
|
+
* ## Deliberately NOT node-scoped
|
|
3157
|
+
*
|
|
3158
|
+
* The read passes no `nodeId`. With one, `addon-settings` would project that
|
|
3159
|
+
* node's scoped value and two nodes could hold different lease windows for the
|
|
3160
|
+
* same camera — so whether a stored crop is native pixels or the ≤640 fallback
|
|
3161
|
+
* would depend on where the balancer happened to place it, which is invisible
|
|
3162
|
+
* from the stored media.
|
|
3163
|
+
*
|
|
3164
|
+
* ## What it hands out
|
|
3165
|
+
*
|
|
3166
|
+
* A {@link NativeLeaseSettingsOverride} — only the knobs the operator SET. The
|
|
3167
|
+
* decode worker resolves the rest (env var, then the shipped default); see
|
|
3168
|
+
* `session-decode/native-lease-config.ts` for the precedence. Handing out a
|
|
3169
|
+
* fully-resolved triple here would erase the "operator set nothing" state and
|
|
3170
|
+
* with it the env-var escape hatch.
|
|
3171
|
+
*/
|
|
3172
|
+
/**
|
|
3173
|
+
* The addon whose GLOBAL settings own the knobs. `pipeline-orchestrator` is
|
|
3174
|
+
* hub-resident and neutral — the same owner as the detail-crop convention and
|
|
3175
|
+
* the per-node decoder-backend selector.
|
|
3176
|
+
*/
|
|
3177
|
+
var NATIVE_LEASE_OWNER_ADDON_ID = "pipeline-orchestrator";
|
|
3178
|
+
/**
|
|
3179
|
+
* Whether two overrides name the same knobs with the same values.
|
|
3180
|
+
*
|
|
3181
|
+
* **Every knob has to appear here.** This is the equality that decides whether
|
|
3182
|
+
* a re-read is ADOPTED at all: a knob missing from the comparison is read from
|
|
3183
|
+
* the hub, judged identical, and discarded, so the operator's change never
|
|
3184
|
+
* reaches a decode worker and nothing anywhere says so. `admission` shipped
|
|
3185
|
+
* missing from this list and did exactly that — the setting was stored, the
|
|
3186
|
+
* hub served it, the runner fetched it, and every worker kept spawning with the
|
|
3187
|
+
* old value.
|
|
3188
|
+
*/
|
|
3189
|
+
function sameOverride(a, b) {
|
|
3190
|
+
return a.holdFrames === b.holdFrames && a.budgetMb === b.budgetMb && a.activityMs === b.activityMs && a.admission === b.admission && a.tileBudgetMb === b.tileBudgetMb;
|
|
3191
|
+
}
|
|
3192
|
+
/**
|
|
3193
|
+
* TTL-cached read of the cluster native-lease knobs.
|
|
3194
|
+
*
|
|
3195
|
+
* {@link current} is synchronous because the fork that consumes it is;
|
|
3196
|
+
* {@link refresh} is what talks to the hub, and the runner awaits it on the
|
|
3197
|
+
* path that precedes a decode-worker spawn.
|
|
3198
|
+
*/
|
|
3199
|
+
var NativeLeaseSettingsSource = class {
|
|
3200
|
+
override = {};
|
|
3201
|
+
lastReadAtMs = Number.NEGATIVE_INFINITY;
|
|
3202
|
+
inFlight = null;
|
|
3203
|
+
logger;
|
|
3204
|
+
now;
|
|
3205
|
+
ttlMs;
|
|
3206
|
+
constructor(options) {
|
|
3207
|
+
this.logger = options.logger;
|
|
3208
|
+
this.now = options.now ?? (() => Date.now());
|
|
3209
|
+
this.ttlMs = options.ttlMs ?? 3e4;
|
|
3210
|
+
}
|
|
3211
|
+
/** The operator's overrides as last read — `{}` until a read lands. */
|
|
3212
|
+
current() {
|
|
3213
|
+
return this.override;
|
|
3214
|
+
}
|
|
3215
|
+
/**
|
|
3216
|
+
* Re-read if the cached value has expired. Concurrent callers share one
|
|
3217
|
+
* in-flight read; a failure keeps the LAST KNOWN value rather than reverting
|
|
3218
|
+
* to `{}`, because a revert would silently hand the next-spawned workers a
|
|
3219
|
+
* different RAM profile than the ones already running.
|
|
3220
|
+
*/
|
|
3221
|
+
async refresh(api) {
|
|
3222
|
+
if (api === void 0) return this.override;
|
|
3223
|
+
if (this.now() - this.lastReadAtMs < this.ttlMs) return this.override;
|
|
3224
|
+
this.inFlight ??= this.read(api).finally(() => {
|
|
3225
|
+
this.inFlight = null;
|
|
3226
|
+
});
|
|
3227
|
+
await this.inFlight;
|
|
3228
|
+
return this.override;
|
|
3229
|
+
}
|
|
3230
|
+
async read(api) {
|
|
3231
|
+
try {
|
|
3232
|
+
const next = pickNativeLeaseOverride(await api.addonSettings.getGlobalSettings.query({ addonId: NATIVE_LEASE_OWNER_ADDON_ID }));
|
|
3233
|
+
this.lastReadAtMs = this.now();
|
|
3234
|
+
if (sameOverride(next, this.override)) return;
|
|
3235
|
+
this.logger.info("native lease settings changed — applies to NEW decode sessions", { meta: {
|
|
3236
|
+
from: { ...this.override },
|
|
3237
|
+
to: { ...next }
|
|
3238
|
+
} });
|
|
3239
|
+
this.override = next;
|
|
3240
|
+
} catch (err) {
|
|
3241
|
+
this.logger.warn("native lease settings read failed — keeping the last known value", { meta: {
|
|
3242
|
+
owner: NATIVE_LEASE_OWNER_ADDON_ID,
|
|
3243
|
+
inUse: { ...this.override },
|
|
3244
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3245
|
+
} });
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
3248
|
+
};
|
|
3249
|
+
//#endregion
|
|
3250
|
+
//#region src/session-decode/on-demand-jpeg-fallback.ts
|
|
3251
|
+
/** Single content-sized JPEG requested after a detection, never a continuous raw. */
|
|
3252
|
+
var ON_DEMAND_JPEG_VIEW = {
|
|
3253
|
+
content: {
|
|
3254
|
+
width: 640,
|
|
3255
|
+
height: 640
|
|
3256
|
+
},
|
|
3257
|
+
fit: "contain",
|
|
3258
|
+
format: "jpeg"
|
|
3156
3259
|
};
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3260
|
+
function createOnDemandJpegFallback(registry, ref) {
|
|
3261
|
+
const lease = registry.acquire(ref);
|
|
3262
|
+
let cached;
|
|
3263
|
+
let released = false;
|
|
3264
|
+
const releaseLease = (reason) => {
|
|
3265
|
+
if (released) return;
|
|
3266
|
+
released = true;
|
|
3267
|
+
lease?.release(reason);
|
|
3268
|
+
};
|
|
3269
|
+
return {
|
|
3270
|
+
async materialize() {
|
|
3271
|
+
if (cached !== void 0) return cached;
|
|
3272
|
+
if (!lease) {
|
|
3273
|
+
cached = null;
|
|
3274
|
+
return null;
|
|
3275
|
+
}
|
|
3276
|
+
try {
|
|
3277
|
+
const view = await lease.resolve(ON_DEMAND_JPEG_VIEW);
|
|
3278
|
+
if (view.kind !== "view" || view.format !== "jpeg" || view.data.byteLength === 0) {
|
|
3279
|
+
cached = null;
|
|
3280
|
+
releaseLease("drop");
|
|
3281
|
+
return null;
|
|
3282
|
+
}
|
|
3283
|
+
cached = {
|
|
3284
|
+
data: Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength),
|
|
3285
|
+
width: view.width,
|
|
3286
|
+
height: view.height,
|
|
3287
|
+
format: "jpeg",
|
|
3288
|
+
timestamp: ref.timestamp,
|
|
3289
|
+
...ref.capturedAt !== void 0 ? { capturedAt: ref.capturedAt } : {}
|
|
3290
|
+
};
|
|
3291
|
+
releaseLease("success");
|
|
3292
|
+
return cached;
|
|
3293
|
+
} catch {
|
|
3294
|
+
cached = null;
|
|
3295
|
+
releaseLease("error");
|
|
3296
|
+
return null;
|
|
3297
|
+
}
|
|
3298
|
+
},
|
|
3299
|
+
release() {
|
|
3300
|
+
releaseLease("eviction");
|
|
3301
|
+
}
|
|
3302
|
+
};
|
|
3303
|
+
}
|
|
3304
|
+
//#endregion
|
|
3305
|
+
//#region src/pipeline-runner/on-demand-retention.ts
|
|
3306
|
+
var CAMSTACK_CONTINUOUS_RETAIN = "CAMSTACK_CONTINUOUS_RETAIN";
|
|
3307
|
+
function isContinuousRetainEnabled(env = process.env) {
|
|
3308
|
+
return env[CAMSTACK_CONTINUOUS_RETAIN] === "1";
|
|
3309
|
+
}
|
|
3310
|
+
function shouldRetainDetectionFrame(input) {
|
|
3311
|
+
if (!input.admitted) return false;
|
|
3312
|
+
if (input.continuousRetain) return true;
|
|
3313
|
+
return input.hasDetection;
|
|
3314
|
+
}
|
|
3315
|
+
function encodeCrossNodeCrop(input) {
|
|
3316
|
+
if (!input.jpeg) return null;
|
|
3317
|
+
return { jpeg: input.jpeg };
|
|
3318
|
+
}
|
|
3319
|
+
function handleKey(handle) {
|
|
3320
|
+
return `${handle.shmId}#${handle.slot}#${handle.seq}`;
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* Pending occupancy exists only admission → inference result. A detection
|
|
3324
|
+
* transfers to {@link RetainedFrameStore} (authoritative late-media owner).
|
|
3325
|
+
* No-detection leaves zero state. Store eviction owns fallback cleanup.
|
|
3326
|
+
*/
|
|
3327
|
+
var OnDemandRetentionSession = class {
|
|
3328
|
+
store;
|
|
3329
|
+
registry;
|
|
3330
|
+
telemetry;
|
|
3331
|
+
continuousRetain;
|
|
3332
|
+
pending = /* @__PURE__ */ new Map();
|
|
3333
|
+
constructor(options) {
|
|
3334
|
+
this.store = options.store;
|
|
3335
|
+
this.registry = options.registry;
|
|
3336
|
+
this.telemetry = options.telemetry;
|
|
3337
|
+
this.continuousRetain = options.continuousRetain;
|
|
3162
3338
|
}
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
armed: false,
|
|
3177
|
-
lastRunAt: sample.now,
|
|
3178
|
-
burstRemaining: Math.max(0, this.config.burstCount - 1),
|
|
3179
|
-
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3339
|
+
get pendingSize() {
|
|
3340
|
+
return this.pending.size;
|
|
3341
|
+
}
|
|
3342
|
+
hasPending(handle) {
|
|
3343
|
+
return this.pending.has(handleKey(handle));
|
|
3344
|
+
}
|
|
3345
|
+
admitFrame(deviceId, frame) {
|
|
3346
|
+
if (this.continuousRetain) {
|
|
3347
|
+
const fallback = frame.frameRef ? createRetainedContentFallback(this.registry, frame.frameRef) : void 0;
|
|
3348
|
+
const handle = this.store.write(deviceId, frame, fallback) ?? void 0;
|
|
3349
|
+
if (handle) this.pending.set(handleKey(handle), {
|
|
3350
|
+
deviceId,
|
|
3351
|
+
ref: frame.frameRef
|
|
3180
3352
|
});
|
|
3353
|
+
this.refreshResident();
|
|
3354
|
+
return handle;
|
|
3355
|
+
}
|
|
3356
|
+
const handle = this.store.allocateHandle(deviceId, frame) ?? void 0;
|
|
3357
|
+
if (!handle) return void 0;
|
|
3358
|
+
this.pending.set(handleKey(handle), {
|
|
3359
|
+
deviceId,
|
|
3360
|
+
ref: frame.frameRef
|
|
3361
|
+
});
|
|
3362
|
+
this.telemetry.noteAvoidedWrite(deviceId);
|
|
3363
|
+
this.refreshResident();
|
|
3364
|
+
return handle;
|
|
3365
|
+
}
|
|
3366
|
+
noteFrame(input) {
|
|
3367
|
+
if (input.decoded) this.telemetry.noteDecoded(input.deviceId);
|
|
3368
|
+
if (input.admitted) this.telemetry.noteAdmitted(input.deviceId);
|
|
3369
|
+
if (input.droppedPixelFree) this.telemetry.noteDroppedPixelFree(input.deviceId);
|
|
3370
|
+
}
|
|
3371
|
+
noteInferenceResult(deviceId, handle, result) {
|
|
3372
|
+
if (!handle) return;
|
|
3373
|
+
const key = handleKey(handle);
|
|
3374
|
+
const pending = this.pending.get(key);
|
|
3375
|
+
this.forgetPending(key);
|
|
3376
|
+
if (!shouldRetainDetectionFrame({
|
|
3377
|
+
admitted: true,
|
|
3378
|
+
hasDetection: (result.detections?.length ?? 0) > 0,
|
|
3379
|
+
continuousRetain: this.continuousRetain
|
|
3380
|
+
})) {
|
|
3381
|
+
this.refreshResident();
|
|
3382
|
+
return;
|
|
3383
|
+
}
|
|
3384
|
+
if (this.continuousRetain) {
|
|
3385
|
+
this.refreshResident();
|
|
3386
|
+
return;
|
|
3387
|
+
}
|
|
3388
|
+
if (!pending?.ref) {
|
|
3389
|
+
this.refreshResident();
|
|
3390
|
+
return;
|
|
3391
|
+
}
|
|
3392
|
+
const fallback = createOnDemandJpegFallback(this.registry, pending.ref);
|
|
3393
|
+
this.store.rememberOnDemand(deviceId, handle, fallback);
|
|
3394
|
+
this.refreshResident();
|
|
3395
|
+
}
|
|
3396
|
+
async requestMedia(handle, request) {
|
|
3397
|
+
const deviceId = this.store.getDeviceId(handle) ?? this.pending.get(handleKey(handle))?.deviceId ?? 0;
|
|
3398
|
+
const kind = request.kind === "full-frame" ? "full-frame" : "crop";
|
|
3399
|
+
this.telemetry.noteOnDemandRequest({
|
|
3400
|
+
deviceId,
|
|
3401
|
+
kind
|
|
3402
|
+
});
|
|
3403
|
+
if (request.nativeHit) {
|
|
3404
|
+
this.telemetry.noteHit("native", deviceId);
|
|
3181
3405
|
return {
|
|
3182
|
-
|
|
3183
|
-
|
|
3406
|
+
tier: "native",
|
|
3407
|
+
bytes: request.nativeHit.bytes,
|
|
3408
|
+
width: request.nativeHit.width,
|
|
3409
|
+
height: request.nativeHit.height
|
|
3184
3410
|
};
|
|
3185
3411
|
}
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
armed,
|
|
3190
|
-
lastRunAt: sample.now,
|
|
3191
|
-
burstRemaining: prev.burstRemaining - 1,
|
|
3192
|
-
burstNextAt: sample.now + this.config.burstIntervalMs
|
|
3193
|
-
});
|
|
3412
|
+
this.telemetry.noteMiss("native", deviceId);
|
|
3413
|
+
if (request.tileHit) {
|
|
3414
|
+
this.telemetry.noteHit("tile", deviceId);
|
|
3194
3415
|
return {
|
|
3195
|
-
|
|
3196
|
-
|
|
3416
|
+
tier: "tile",
|
|
3417
|
+
bytes: request.tileHit.bytes,
|
|
3418
|
+
jpeg: request.tileHit.jpeg ? request.tileHit.bytes : void 0,
|
|
3419
|
+
width: request.tileHit.width,
|
|
3420
|
+
height: request.tileHit.height
|
|
3421
|
+
};
|
|
3422
|
+
}
|
|
3423
|
+
this.telemetry.noteMiss("tile", deviceId);
|
|
3424
|
+
if (request.recordedStillHit) return {
|
|
3425
|
+
tier: "recorded-still",
|
|
3426
|
+
jpeg: request.recordedStillHit.jpeg,
|
|
3427
|
+
width: request.recordedStillHit.width,
|
|
3428
|
+
height: request.recordedStillHit.height
|
|
3429
|
+
};
|
|
3430
|
+
if (!this.continuousRetain && !this.store.has(handle)) {
|
|
3431
|
+
this.telemetry.noteMiss("fallback", deviceId);
|
|
3432
|
+
return { tier: "miss" };
|
|
3433
|
+
}
|
|
3434
|
+
const view = await this.store.ensureContent(handle);
|
|
3435
|
+
if (view && view.data.byteLength > 0) {
|
|
3436
|
+
this.telemetry.noteHit("fallback", deviceId);
|
|
3437
|
+
this.refreshResident();
|
|
3438
|
+
if (view.format === "jpeg") return {
|
|
3439
|
+
tier: "fallback-jpeg",
|
|
3440
|
+
jpeg: view.data,
|
|
3441
|
+
width: view.width,
|
|
3442
|
+
height: view.height
|
|
3443
|
+
};
|
|
3444
|
+
return {
|
|
3445
|
+
tier: "fallback-jpeg",
|
|
3446
|
+
bytes: view.data,
|
|
3447
|
+
width: view.width,
|
|
3448
|
+
height: view.height
|
|
3449
|
+
};
|
|
3450
|
+
}
|
|
3451
|
+
this.telemetry.noteMiss("fallback", deviceId);
|
|
3452
|
+
this.telemetry.noteStale(deviceId);
|
|
3453
|
+
return { tier: "miss" };
|
|
3454
|
+
}
|
|
3455
|
+
/**
|
|
3456
|
+
* Pending cleanup. Success never invalidates a store entry already
|
|
3457
|
+
* transferred for late pull. Drop/error/detach drop the store only when the
|
|
3458
|
+
* handle was still pending (not yet transferred). Eviction owns transferred
|
|
3459
|
+
* fallback cleanup.
|
|
3460
|
+
*/
|
|
3461
|
+
release(handle, reason) {
|
|
3462
|
+
const key = handleKey(handle);
|
|
3463
|
+
const pending = this.pending.get(key);
|
|
3464
|
+
this.forgetPending(key);
|
|
3465
|
+
if (reason !== "success" && pending) this.store.drop(handle);
|
|
3466
|
+
this.telemetry.noteRelease(pending?.deviceId ?? this.store.getDeviceId(handle));
|
|
3467
|
+
this.refreshResident();
|
|
3468
|
+
}
|
|
3469
|
+
forgetDevice(deviceId) {
|
|
3470
|
+
for (const [key, pending] of [...this.pending]) {
|
|
3471
|
+
if (pending.deviceId !== deviceId) continue;
|
|
3472
|
+
this.pending.delete(key);
|
|
3473
|
+
}
|
|
3474
|
+
this.telemetry.forgetCamera(deviceId);
|
|
3475
|
+
this.refreshResident();
|
|
3476
|
+
}
|
|
3477
|
+
syncResident() {
|
|
3478
|
+
this.refreshResident();
|
|
3479
|
+
}
|
|
3480
|
+
forgetPending(key) {
|
|
3481
|
+
this.pending.delete(key);
|
|
3482
|
+
}
|
|
3483
|
+
refreshResident() {
|
|
3484
|
+
const byDevice = /* @__PURE__ */ new Map();
|
|
3485
|
+
for (const pending of this.pending.values()) {
|
|
3486
|
+
const cur = byDevice.get(pending.deviceId) ?? {
|
|
3487
|
+
refs: 0,
|
|
3488
|
+
bytes: 0
|
|
3197
3489
|
};
|
|
3490
|
+
cur.refs += 1;
|
|
3491
|
+
byDevice.set(pending.deviceId, cur);
|
|
3198
3492
|
}
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
lastRunAt: sample.now,
|
|
3204
|
-
burstRemaining: prev.burstRemaining,
|
|
3205
|
-
burstNextAt: prev.burstNextAt
|
|
3206
|
-
});
|
|
3207
|
-
return {
|
|
3208
|
-
run: true,
|
|
3209
|
-
reason: "poll"
|
|
3493
|
+
for (const stats of this.store.deviceStats()) {
|
|
3494
|
+
const cur = byDevice.get(stats.deviceId) ?? {
|
|
3495
|
+
refs: 0,
|
|
3496
|
+
bytes: 0
|
|
3210
3497
|
};
|
|
3498
|
+
cur.refs += stats.entries;
|
|
3499
|
+
cur.bytes += stats.bytes;
|
|
3500
|
+
byDevice.set(stats.deviceId, cur);
|
|
3211
3501
|
}
|
|
3212
|
-
this.
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
lastRunAt: prev.lastRunAt,
|
|
3216
|
-
burstRemaining: prev.burstRemaining,
|
|
3217
|
-
burstNextAt: prev.burstNextAt
|
|
3502
|
+
this.telemetry.setResident({
|
|
3503
|
+
refs: this.pending.size + this.store.size,
|
|
3504
|
+
bytes: this.registry.estimatedBytes + this.store.retainedBytes
|
|
3218
3505
|
});
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
/** Drop a device's state (call on detach / step-disable) to avoid a leak. */
|
|
3222
|
-
forget(deviceId) {
|
|
3223
|
-
this.state.delete(deviceId);
|
|
3224
|
-
}
|
|
3225
|
-
/** Clear all state (e.g. runner teardown). */
|
|
3226
|
-
reset() {
|
|
3227
|
-
this.state.clear();
|
|
3506
|
+
this.telemetry.zeroAbsentResidents(new Set(byDevice.keys()));
|
|
3507
|
+
for (const [deviceId, stats] of byDevice) this.telemetry.setCameraResident(deviceId, stats);
|
|
3228
3508
|
}
|
|
3229
3509
|
};
|
|
3230
3510
|
//#endregion
|
|
@@ -3312,134 +3592,6 @@ function packageZoneUnion(polygons) {
|
|
|
3312
3592
|
};
|
|
3313
3593
|
}
|
|
3314
3594
|
//#endregion
|
|
3315
|
-
//#region src/pipeline-runner/native-crop-tier.ts
|
|
3316
|
-
/**
|
|
3317
|
-
* Pure tier decision for `getNativeCrop`. A native HIT always resolves to the
|
|
3318
|
-
* native surface. On a native MISS, only a full-frame request falls back to the
|
|
3319
|
-
* ≤640 RAM store; an ROI miss is reported as `miss` (no quality masquerade).
|
|
3320
|
-
*/
|
|
3321
|
-
function resolveNativeCropSource(isFullFrame, nativeHit) {
|
|
3322
|
-
if (nativeHit) return "native";
|
|
3323
|
-
return isFullFrame ? "ram-fullframe" : "miss";
|
|
3324
|
-
}
|
|
3325
|
-
/**
|
|
3326
|
-
* Runtime hit kind for telemetry. A worker tile is native-resolution quality
|
|
3327
|
-
* (public wire `tier` stays `native`) but must not increment `nativeHits`.
|
|
3328
|
-
*/
|
|
3329
|
-
function resolveNativeCropHitKind(source) {
|
|
3330
|
-
return source === "tile" ? "tile" : "native";
|
|
3331
|
-
}
|
|
3332
|
-
//#endregion
|
|
3333
|
-
//#region src/pipeline-runner/native-lease-settings-source.ts
|
|
3334
|
-
/**
|
|
3335
|
-
* Where the runner gets the cluster-wide native-frame LEASE knobs.
|
|
3336
|
-
*
|
|
3337
|
-
* ## Why a settings read and not the attach config
|
|
3338
|
-
*
|
|
3339
|
-
* Identical reasoning to `crop-convention-source.ts`, and worth restating
|
|
3340
|
-
* because the temptation is the same: `RunnerCameraConfig` is the obvious
|
|
3341
|
-
* orchestrator→runner channel and it is the wrong one. Eleven call sites build
|
|
3342
|
-
* one; a cluster-wide constant threaded through eleven builders eventually
|
|
3343
|
-
* reaches ten of them, and the cameras attached by the forgotten builder just
|
|
3344
|
-
* quietly run a different lease window. Instead the runner reads the OWNER
|
|
3345
|
-
* addon's global settings directly, hub-routed by `AddonCallGateway`, so an
|
|
3346
|
-
* agent runner transparently reads the hub-central store with no placement
|
|
3347
|
-
* branching.
|
|
3348
|
-
*
|
|
3349
|
-
* ## Deliberately NOT node-scoped
|
|
3350
|
-
*
|
|
3351
|
-
* The read passes no `nodeId`. With one, `addon-settings` would project that
|
|
3352
|
-
* node's scoped value and two nodes could hold different lease windows for the
|
|
3353
|
-
* same camera — so whether a stored crop is native pixels or the ≤640 fallback
|
|
3354
|
-
* would depend on where the balancer happened to place it, which is invisible
|
|
3355
|
-
* from the stored media.
|
|
3356
|
-
*
|
|
3357
|
-
* ## What it hands out
|
|
3358
|
-
*
|
|
3359
|
-
* A {@link NativeLeaseSettingsOverride} — only the knobs the operator SET. The
|
|
3360
|
-
* decode worker resolves the rest (env var, then the shipped default); see
|
|
3361
|
-
* `session-decode/native-lease-config.ts` for the precedence. Handing out a
|
|
3362
|
-
* fully-resolved triple here would erase the "operator set nothing" state and
|
|
3363
|
-
* with it the env-var escape hatch.
|
|
3364
|
-
*/
|
|
3365
|
-
/**
|
|
3366
|
-
* The addon whose GLOBAL settings own the knobs. `pipeline-orchestrator` is
|
|
3367
|
-
* hub-resident and neutral — the same owner as the detail-crop convention and
|
|
3368
|
-
* the per-node decoder-backend selector.
|
|
3369
|
-
*/
|
|
3370
|
-
var NATIVE_LEASE_OWNER_ADDON_ID = "pipeline-orchestrator";
|
|
3371
|
-
/**
|
|
3372
|
-
* Whether two overrides name the same knobs with the same values.
|
|
3373
|
-
*
|
|
3374
|
-
* **Every knob has to appear here.** This is the equality that decides whether
|
|
3375
|
-
* a re-read is ADOPTED at all: a knob missing from the comparison is read from
|
|
3376
|
-
* the hub, judged identical, and discarded, so the operator's change never
|
|
3377
|
-
* reaches a decode worker and nothing anywhere says so. `admission` shipped
|
|
3378
|
-
* missing from this list and did exactly that — the setting was stored, the
|
|
3379
|
-
* hub served it, the runner fetched it, and every worker kept spawning with the
|
|
3380
|
-
* old value.
|
|
3381
|
-
*/
|
|
3382
|
-
function sameOverride(a, b) {
|
|
3383
|
-
return a.holdFrames === b.holdFrames && a.budgetMb === b.budgetMb && a.activityMs === b.activityMs && a.admission === b.admission && a.tileBudgetMb === b.tileBudgetMb;
|
|
3384
|
-
}
|
|
3385
|
-
/**
|
|
3386
|
-
* TTL-cached read of the cluster native-lease knobs.
|
|
3387
|
-
*
|
|
3388
|
-
* {@link current} is synchronous because the fork that consumes it is;
|
|
3389
|
-
* {@link refresh} is what talks to the hub, and the runner awaits it on the
|
|
3390
|
-
* path that precedes a decode-worker spawn.
|
|
3391
|
-
*/
|
|
3392
|
-
var NativeLeaseSettingsSource = class {
|
|
3393
|
-
override = {};
|
|
3394
|
-
lastReadAtMs = Number.NEGATIVE_INFINITY;
|
|
3395
|
-
inFlight = null;
|
|
3396
|
-
logger;
|
|
3397
|
-
now;
|
|
3398
|
-
ttlMs;
|
|
3399
|
-
constructor(options) {
|
|
3400
|
-
this.logger = options.logger;
|
|
3401
|
-
this.now = options.now ?? (() => Date.now());
|
|
3402
|
-
this.ttlMs = options.ttlMs ?? 3e4;
|
|
3403
|
-
}
|
|
3404
|
-
/** The operator's overrides as last read — `{}` until a read lands. */
|
|
3405
|
-
current() {
|
|
3406
|
-
return this.override;
|
|
3407
|
-
}
|
|
3408
|
-
/**
|
|
3409
|
-
* Re-read if the cached value has expired. Concurrent callers share one
|
|
3410
|
-
* in-flight read; a failure keeps the LAST KNOWN value rather than reverting
|
|
3411
|
-
* to `{}`, because a revert would silently hand the next-spawned workers a
|
|
3412
|
-
* different RAM profile than the ones already running.
|
|
3413
|
-
*/
|
|
3414
|
-
async refresh(api) {
|
|
3415
|
-
if (api === void 0) return this.override;
|
|
3416
|
-
if (this.now() - this.lastReadAtMs < this.ttlMs) return this.override;
|
|
3417
|
-
this.inFlight ??= this.read(api).finally(() => {
|
|
3418
|
-
this.inFlight = null;
|
|
3419
|
-
});
|
|
3420
|
-
await this.inFlight;
|
|
3421
|
-
return this.override;
|
|
3422
|
-
}
|
|
3423
|
-
async read(api) {
|
|
3424
|
-
try {
|
|
3425
|
-
const next = pickNativeLeaseOverride(await api.addonSettings.getGlobalSettings.query({ addonId: NATIVE_LEASE_OWNER_ADDON_ID }));
|
|
3426
|
-
this.lastReadAtMs = this.now();
|
|
3427
|
-
if (sameOverride(next, this.override)) return;
|
|
3428
|
-
this.logger.info("native lease settings changed — applies to NEW decode sessions", { meta: {
|
|
3429
|
-
from: { ...this.override },
|
|
3430
|
-
to: { ...next }
|
|
3431
|
-
} });
|
|
3432
|
-
this.override = next;
|
|
3433
|
-
} catch (err) {
|
|
3434
|
-
this.logger.warn("native lease settings read failed — keeping the last known value", { meta: {
|
|
3435
|
-
owner: NATIVE_LEASE_OWNER_ADDON_ID,
|
|
3436
|
-
inUse: { ...this.override },
|
|
3437
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3438
|
-
} });
|
|
3439
|
-
}
|
|
3440
|
-
}
|
|
3441
|
-
};
|
|
3442
|
-
//#endregion
|
|
3443
3595
|
//#region src/pipeline-runner/frame-governor.ts
|
|
3444
3596
|
var FrameGovernor = class {
|
|
3445
3597
|
config;
|
|
@@ -4603,76 +4755,6 @@ var PipelineRunner = class {
|
|
|
4603
4755
|
}
|
|
4604
4756
|
};
|
|
4605
4757
|
//#endregion
|
|
4606
|
-
//#region src/session-decode/live-worker-admission.ts
|
|
4607
|
-
/**
|
|
4608
|
-
* The node's live-decode-worker budget.
|
|
4609
|
-
*
|
|
4610
|
-
* One instance per runner addon. Not a semaphore: a semaphore's contract is
|
|
4611
|
-
* "wait", and waiting is the thing this deliberately does not do — see
|
|
4612
|
-
* {@link DEFAULT_MAX_LIVE_DECODE_WORKERS}.
|
|
4613
|
-
*/
|
|
4614
|
-
var LiveDecodeWorkerAdmission = class {
|
|
4615
|
-
limit;
|
|
4616
|
-
logger;
|
|
4617
|
-
live = 0;
|
|
4618
|
-
constructor(options) {
|
|
4619
|
-
this.limit = Math.max(1, options.limit);
|
|
4620
|
-
this.logger = options.logger;
|
|
4621
|
-
}
|
|
4622
|
-
/** Workers currently holding a slot. */
|
|
4623
|
-
get inFlight() {
|
|
4624
|
-
return this.live;
|
|
4625
|
-
}
|
|
4626
|
-
get ceiling() {
|
|
4627
|
-
return this.limit;
|
|
4628
|
-
}
|
|
4629
|
-
/**
|
|
4630
|
-
* Apply an operator-changed ceiling.
|
|
4631
|
-
*
|
|
4632
|
-
* Lowering it never kills a running session — a decode worker torn down
|
|
4633
|
-
* mid-motion loses the event it was started for. It only stops the next
|
|
4634
|
-
* admission, so the count drains to the new ceiling as sessions end.
|
|
4635
|
-
*/
|
|
4636
|
-
resize(limit) {
|
|
4637
|
-
this.limit = Math.max(1, limit);
|
|
4638
|
-
}
|
|
4639
|
-
/**
|
|
4640
|
-
* Take a slot, or refuse.
|
|
4641
|
-
*
|
|
4642
|
-
* A refusal is a WARN and carries `tags.deviceId`, because the only useful
|
|
4643
|
-
* form of this question is per-camera ("why did 617 not run when 615 did?").
|
|
4644
|
-
* A branch that drops work silently reads as "it never happened".
|
|
4645
|
-
*/
|
|
4646
|
-
admit(deviceId, label) {
|
|
4647
|
-
if (this.live >= this.limit) {
|
|
4648
|
-
this.logger.warn("session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
|
|
4649
|
-
tags: { deviceId },
|
|
4650
|
-
meta: {
|
|
4651
|
-
label,
|
|
4652
|
-
live: this.live,
|
|
4653
|
-
ceiling: this.limit
|
|
4654
|
-
}
|
|
4655
|
-
});
|
|
4656
|
-
return null;
|
|
4657
|
-
}
|
|
4658
|
-
this.live += 1;
|
|
4659
|
-
this.logger.debug("session-decode: live decode-worker slot taken", {
|
|
4660
|
-
tags: { deviceId },
|
|
4661
|
-
meta: {
|
|
4662
|
-
label,
|
|
4663
|
-
live: this.live,
|
|
4664
|
-
ceiling: this.limit
|
|
4665
|
-
}
|
|
4666
|
-
});
|
|
4667
|
-
let released = false;
|
|
4668
|
-
return { release: () => {
|
|
4669
|
-
if (released) return;
|
|
4670
|
-
released = true;
|
|
4671
|
-
this.live -= 1;
|
|
4672
|
-
} };
|
|
4673
|
-
}
|
|
4674
|
-
};
|
|
4675
|
-
//#endregion
|
|
4676
4758
|
//#region src/pipeline-runner/stateless-step.ts
|
|
4677
4759
|
function modelServability(step, modelId, format) {
|
|
4678
4760
|
const entry = step.models.find((m) => m.id === modelId);
|
|
@@ -4812,7 +4894,8 @@ var DEFAULT_CONFIG = {
|
|
|
4812
4894
|
targetLoadPercent: 80,
|
|
4813
4895
|
minThrottledFps: 1,
|
|
4814
4896
|
maxConcurrentDecodeSpawns: 3,
|
|
4815
|
-
maxLiveDecodeWorkers: 8
|
|
4897
|
+
maxLiveDecodeWorkers: 8,
|
|
4898
|
+
nodeDecodeBudgetMb: DEFAULT_NODE_DECODE_BUDGET_MB
|
|
4816
4899
|
};
|
|
4817
4900
|
/**
|
|
4818
4901
|
* Camera attachment record kept by the runner addon. Holds the original
|
|
@@ -5375,7 +5458,9 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
5375
5458
|
this.spawnGate = new Semaphore(Math.max(1, this.config.maxConcurrentDecodeSpawns));
|
|
5376
5459
|
this.liveWorkers = new LiveDecodeWorkerAdmission({
|
|
5377
5460
|
limit: this.config.maxLiveDecodeWorkers,
|
|
5378
|
-
logger: this.ctx.logger
|
|
5461
|
+
logger: this.ctx.logger,
|
|
5462
|
+
nodeBudgetMb: this.config.nodeDecodeBudgetMb,
|
|
5463
|
+
workerFootprintMb: decodeWorkerFootprintMb(this.nativeLease.current())
|
|
5379
5464
|
});
|
|
5380
5465
|
this.runner.timingSampler.setLogger(this.ctx.logger.child("timing"));
|
|
5381
5466
|
this.runner.onDetectionStreamChange((deviceId, action) => {
|
|
@@ -6579,6 +6664,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
6579
6664
|
*/
|
|
6580
6665
|
async refreshNativeLeaseSettings() {
|
|
6581
6666
|
await this.nativeLease.refresh(this.ctxIfReady?.api);
|
|
6667
|
+
this.liveWorkers?.resizeFootprint(decodeWorkerFootprintMb(this.nativeLease.current()));
|
|
6582
6668
|
}
|
|
6583
6669
|
/** A warn sink that survives a not-yet-ready ctx (boot-order dispatches). */
|
|
6584
6670
|
detailSubtreeLogger() {
|
|
@@ -8048,12 +8134,23 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8048
8134
|
type: "slider",
|
|
8049
8135
|
key: "maxLiveDecodeWorkers",
|
|
8050
8136
|
label: "Max live decode workers",
|
|
8051
|
-
description: "
|
|
8137
|
+
description: "Upper CLAMP on decode workers ALIVE at once on this node. The bound that normally decides is the RAM budget below — the ceiling in force is whichever of the two is smaller. Unlike the spawn gate above, which only limits how fast workers START, this limits how many EXIST. A camera that does not fit is REFUSED and logged, then retried automatically as soon as a slot frees — it is never queued, because a queued live session would decode a moment that has already passed.",
|
|
8052
8138
|
min: 1,
|
|
8053
8139
|
max: 32,
|
|
8054
8140
|
step: 1,
|
|
8055
8141
|
default: DEFAULT_CONFIG.maxLiveDecodeWorkers,
|
|
8056
8142
|
showValue: true
|
|
8143
|
+
},
|
|
8144
|
+
{
|
|
8145
|
+
type: "slider",
|
|
8146
|
+
key: "nodeDecodeBudgetMb",
|
|
8147
|
+
label: "Decode worker RAM budget (MB)",
|
|
8148
|
+
description: "RAM this node lets live decode workers commit. The number of workers admitted is derived from it: budget divided by one worker's worst case, which is the lease budget + tile budget + ~350 MB of baseline. Measured on this cluster a worker plateaus at 650-860 MB within ten seconds of spawning, so at the shipped lease knobs the default admits 4 — the observed peak. Raising the lease budget per worker therefore LOWERS how many run at once, instead of silently authorising more memory.",
|
|
8149
|
+
min: 1024,
|
|
8150
|
+
max: 32768,
|
|
8151
|
+
step: 512,
|
|
8152
|
+
default: DEFAULT_CONFIG.nodeDecodeBudgetMb,
|
|
8153
|
+
showValue: true
|
|
8057
8154
|
}
|
|
8058
8155
|
]
|
|
8059
8156
|
}] });
|
|
@@ -8062,13 +8159,17 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
|
|
|
8062
8159
|
this.runner?.updateLimits(this.config);
|
|
8063
8160
|
this.spawnGate?.resize(Math.max(1, this.config.maxConcurrentDecodeSpawns));
|
|
8064
8161
|
this.liveWorkers?.resize(this.config.maxLiveDecodeWorkers);
|
|
8162
|
+
this.liveWorkers?.resizeBudget(this.config.nodeDecodeBudgetMb);
|
|
8065
8163
|
this.ctx.logger.info("pipeline-runner tuning updated", { meta: {
|
|
8066
8164
|
maxQueueDepth: this.config.maxQueueDepth,
|
|
8067
8165
|
maxConcurrentInferences: this.config.maxConcurrentInferences,
|
|
8068
8166
|
targetLoadPercent: this.config.targetLoadPercent,
|
|
8069
8167
|
minThrottledFps: this.config.minThrottledFps,
|
|
8070
8168
|
maxConcurrentDecodeSpawns: this.config.maxConcurrentDecodeSpawns,
|
|
8071
|
-
maxLiveDecodeWorkers: this.config.maxLiveDecodeWorkers
|
|
8169
|
+
maxLiveDecodeWorkers: this.config.maxLiveDecodeWorkers,
|
|
8170
|
+
nodeDecodeBudgetMb: this.config.nodeDecodeBudgetMb,
|
|
8171
|
+
decodeCeilingInForce: this.liveWorkers?.ceiling,
|
|
8172
|
+
decodeWorkerFootprintMb: decodeWorkerFootprintMb(this.nativeLease.current())
|
|
8072
8173
|
} });
|
|
8073
8174
|
}
|
|
8074
8175
|
};
|