@camstack/addon-pipeline 1.2.31 → 1.2.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detection-pipeline/index.js +839 -1013
- package/dist/detection-pipeline/index.mjs +839 -1013
- package/dist/pipeline-runner/index.js +30 -4
- package/dist/pipeline-runner/index.mjs +30 -4
- package/dist/{step-definitions-BkY9_IM6.mjs → step-definitions-C8tk9fPJ.mjs} +11 -2
- package/dist/{step-definitions-gC-Ovglc.js → step-definitions-i8O-hxyI.js} +16 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BUXeX-vb.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C5m4hC2K.mjs} +3 -3
- package/dist/stream-broker/{hostInit-DJ3h-b1X.mjs → hostInit-WXFrgppm.mjs} +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/embed-dist/assets/{MaskShapeCanvas-DI4BY7W2-B0DyP46h.js → MaskShapeCanvas-DI4BY7W2-Dx-Vrpyt.js} +1 -1
- package/embed-dist/assets/{MotionZonesSettings-NcxxQN8r-CzGSy_EO.js → MotionZonesSettings-NcxxQN8r-9D-BWwQP.js} +1 -1
- package/embed-dist/assets/{PrivacyMaskSettings-APgPLF7p-CEgMcOOW.js → PrivacyMaskSettings-APgPLF7p-HvAetfA4.js} +1 -1
- package/embed-dist/assets/{index-CHXOWKHF.js → index-9Ob2YrIf.js} +5 -5
- package/embed-dist/index.html +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { n as __require } from "../chunk-DnnnRqeS.mjs";
|
|
2
2
|
import { B as errMsg, E as evaluateZoneRules, F as runtimeDevices$1, G as createEvent, K as hydrateSchema, R as supportedRuntimes$1, T as enumerateInferenceDevices, V as BaseAddon, X as nodePin, Z as parseJsonUnknown, ct as object, dt as union, et as sleep, ft as EventCategory, h as YAMNET_TO_MACRO, j as pipelineExecutorCapability, nt as array, s as DEVICE_BACKEND_TO_FORMAT, t as APPLE_SA_TO_MACRO, ut as string, w as detectionPipelineCapability, x as defaultDeviceFor$1 } from "../dist-Du7_SNiR.mjs";
|
|
3
|
-
import { a as
|
|
3
|
+
import { a as getStep, i as getDefaultModelForFormatFromDef, n as ALL_STEPS, o as getStepDefinition, r as getDefaultModelForFormat, s as resolveModelForFormat, t as ALL_PIPELINE_STEPS } from "../step-definitions-C8tk9fPJ.mjs";
|
|
4
4
|
import { t as pickNodePlatformArch } from "../node-topology-platform-BkR_k6WT.mjs";
|
|
5
5
|
import { a as ensureModel, o as isModelDownloaded, r as deleteModelFromDisk } from "../model-download-service-Cp9f4dk6-CwUb5v3Y.mjs";
|
|
6
6
|
import * as os from "node:os";
|
|
@@ -243,43 +243,6 @@ function normalizeEngineNodeId(rawNodeId) {
|
|
|
243
243
|
return raw.includes("/") ? raw.split("/")[0] ?? "hub" : raw;
|
|
244
244
|
}
|
|
245
245
|
//#endregion
|
|
246
|
-
//#region src/detection-pipeline/pipeline/native-crop-compose.ts
|
|
247
|
-
/** Clamp `v` into `[lo, hi]`. */
|
|
248
|
-
function clamp(v, lo, hi) {
|
|
249
|
-
if (v < lo) return lo;
|
|
250
|
-
if (v > hi) return hi;
|
|
251
|
-
return v;
|
|
252
|
-
}
|
|
253
|
-
/**
|
|
254
|
-
* Compose a child `roi` given in the parent CROP's normalized space into the
|
|
255
|
-
* FRAME's normalized space, given the crop's frame-space pixel rectangle and the
|
|
256
|
-
* frame dimensions. The result is clamped to the unit square (origin in
|
|
257
|
-
* `[0,1]`, extent bounded so `x + w ≤ 1` and `y + h ≤ 1`). Returns `null` for
|
|
258
|
-
* degenerate inputs (non-finite frame dims, empty crop rect, or an ROI that
|
|
259
|
-
* clamps to zero area) so the caller falls back to the downscaled tile crop.
|
|
260
|
-
*
|
|
261
|
-
* Pure — never mutates its arguments.
|
|
262
|
-
*/
|
|
263
|
-
function composeCropRoiToFrameNorm(roi, cropFrameSpace, frameWidth, frameHeight) {
|
|
264
|
-
if (!(frameWidth > 0) || !(frameHeight > 0)) return null;
|
|
265
|
-
if (!(cropFrameSpace.w > 0) || !(cropFrameSpace.h > 0)) return null;
|
|
266
|
-
const fx = (cropFrameSpace.x + roi.x * cropFrameSpace.w) / frameWidth;
|
|
267
|
-
const fy = (cropFrameSpace.y + roi.y * cropFrameSpace.h) / frameHeight;
|
|
268
|
-
const fw = roi.w * cropFrameSpace.w / frameWidth;
|
|
269
|
-
const fh = roi.h * cropFrameSpace.h / frameHeight;
|
|
270
|
-
const x = clamp(fx, 0, 1);
|
|
271
|
-
const y = clamp(fy, 0, 1);
|
|
272
|
-
const w = clamp(fw, 0, 1 - x);
|
|
273
|
-
const h = clamp(fh, 0, 1 - y);
|
|
274
|
-
if (!(w > 0) || !(h > 0)) return null;
|
|
275
|
-
return {
|
|
276
|
-
x,
|
|
277
|
-
y,
|
|
278
|
-
w,
|
|
279
|
-
h
|
|
280
|
-
};
|
|
281
|
-
}
|
|
282
|
-
//#endregion
|
|
283
246
|
//#region src/detection-pipeline/default-detection-model.ts
|
|
284
247
|
/** The object-detection step id — the only slot this resolver applies to. */
|
|
285
248
|
var OBJECT_DETECTION_STEP_ID$1 = "object-detection";
|
|
@@ -330,10 +293,12 @@ function classifyAccelerator(backend, device) {
|
|
|
330
293
|
*
|
|
331
294
|
* Guarantees:
|
|
332
295
|
* - never throws (hot default-tree path);
|
|
333
|
-
* - never returns a model id
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
296
|
+
* - never returns a model id without a build for `format` (unless the step
|
|
297
|
+
* has ZERO builds for `format`, which is unloadable regardless of pick and
|
|
298
|
+
* flagged by `collectZeroBuildIssues`) — an unmapped backend, a `'cpu'`
|
|
299
|
+
* class, a mapped id missing from the catalog, or a mapped id without a
|
|
300
|
+
* `format` build all fall back to the step's PER-FORMAT default
|
|
301
|
+
* (`getDefaultModelForFormatFromDef`), NOT the bare `defaultModelId`.
|
|
337
302
|
*
|
|
338
303
|
* `getStepDef` is injectable (defaults to the real catalog lookup) so the
|
|
339
304
|
* missing-build fallback branch is unit-testable without a live catalog —
|
|
@@ -346,12 +311,12 @@ function resolveDefaultDetectionModel(backend, device, format, getStepDef = getS
|
|
|
346
311
|
} catch {
|
|
347
312
|
return "yolo26n";
|
|
348
313
|
}
|
|
349
|
-
const fallback = def.defaultModelId;
|
|
350
314
|
const candidate = MODEL_BY_CLASS[classifyAccelerator(backend, device)];
|
|
351
|
-
if (candidate
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
315
|
+
if (candidate !== null) {
|
|
316
|
+
const entry = def.models.find((m) => m.id === candidate);
|
|
317
|
+
if (entry && entry.formats[format] !== void 0) return candidate;
|
|
318
|
+
}
|
|
319
|
+
return getDefaultModelForFormatFromDef(def, format);
|
|
355
320
|
}
|
|
356
321
|
//#endregion
|
|
357
322
|
//#region src/detection-pipeline/engine/shared-inference-pool.ts
|
|
@@ -468,10 +433,7 @@ var PoolHandle = class {
|
|
|
468
433
|
}
|
|
469
434
|
async infer(input) {
|
|
470
435
|
const start = performance.now();
|
|
471
|
-
return
|
|
472
|
-
structured: input.kind === "raw" ? await this.pool.inferRaw(this.modelIndex, input.data, input.width, input.height, input.format) : await this.pool.infer(this.modelIndex, input.data),
|
|
473
|
-
inferenceMs: performance.now() - start
|
|
474
|
-
};
|
|
436
|
+
return poolResultToEngineOutput(input.kind === "raw" ? await this.pool.inferRaw(this.modelIndex, input.data, input.width, input.height, input.format) : await this.pool.infer(this.modelIndex, input.data), performance.now() - start);
|
|
475
437
|
}
|
|
476
438
|
/**
|
|
477
439
|
* Inference on a frame previously cached in the Python pool via
|
|
@@ -480,20 +442,50 @@ var PoolHandle = class {
|
|
|
480
442
|
*/
|
|
481
443
|
async inferFromCache(frameId) {
|
|
482
444
|
const start = performance.now();
|
|
483
|
-
return
|
|
484
|
-
structured: await this.pool.inferCached(this.modelIndex, frameId),
|
|
485
|
-
inferenceMs: performance.now() - start
|
|
486
|
-
};
|
|
445
|
+
return poolResultToEngineOutput(await this.pool.inferCached(this.modelIndex, frameId), performance.now() - start);
|
|
487
446
|
}
|
|
488
447
|
async dispose() {}
|
|
489
448
|
};
|
|
490
449
|
/**
|
|
450
|
+
* Map a raw pool response record to an {@link EngineOutput}.
|
|
451
|
+
*
|
|
452
|
+
* An overload shed (`{dropped: true}` — from the Python pool's per-model bound
|
|
453
|
+
* OR the TS worker in-flight cap, which deliberately mirrors it) becomes a
|
|
454
|
+
* first-class `EngineOutput.dropped` with NO `structured` payload. Before this
|
|
455
|
+
* mapping the shed record rode through as `structured`, failed the StepOutput
|
|
456
|
+
* kind guard in the postprocess funnel, and every deliberate shed was logged
|
|
457
|
+
* as `runInference failed … unexpected kind: undefined` at error level on
|
|
458
|
+
* every node (~4/s cluster-wide, 2026-08-01). A shed is flow control, not a
|
|
459
|
+
* fault — it must never reach a postprocessor. Exported for tests.
|
|
460
|
+
*/
|
|
461
|
+
function poolResultToEngineOutput(result, inferenceMs) {
|
|
462
|
+
if (result["dropped"] === true) {
|
|
463
|
+
const shedReason = result["shedReason"];
|
|
464
|
+
return {
|
|
465
|
+
dropped: true,
|
|
466
|
+
...typeof shedReason === "string" ? { shedReason } : {},
|
|
467
|
+
inferenceMs
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
return {
|
|
471
|
+
structured: result,
|
|
472
|
+
inferenceMs
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
/**
|
|
491
476
|
* How long to wait for a worker to exit on SIGTERM before escalating to
|
|
492
477
|
* SIGKILL. A worker idle between frames exits well under this; only a worker
|
|
493
478
|
* stuck in a native inference call (openvino/onnx C++) needs the escalation.
|
|
494
479
|
*/
|
|
495
480
|
var POOL_WORKER_TERM_GRACE_MS = 2e3;
|
|
496
481
|
/**
|
|
482
|
+
* Sampling stride for the "Python pool shed frame under overload" debug
|
|
483
|
+
* line — see `trackDroppedResponse`. At the observed chronic-overload rate
|
|
484
|
+
* (~13 shed/s, 2026-08-01) a stride of 500 yields roughly one line every
|
|
485
|
+
* 40 s while `droppedTotal` in the line keeps the exact count.
|
|
486
|
+
*/
|
|
487
|
+
var SHED_LOG_SAMPLE_EVERY = 500;
|
|
488
|
+
/**
|
|
497
489
|
* Terminate a spawned child GRACEFULLY THEN FORCEFULLY: end stdin, send
|
|
498
490
|
* SIGTERM, and if the process hasn't exited within `graceMs`, send SIGKILL.
|
|
499
491
|
* Resolves once the process has exited (or was already dead).
|
|
@@ -1055,13 +1047,23 @@ var SharedInferencePool = class {
|
|
|
1055
1047
|
* single-frame inference paths. Previously indistinguishable from a
|
|
1056
1048
|
* genuine empty detection result — now counted + debug-logged so
|
|
1057
1049
|
* overload is measurable. The response passes through unchanged.
|
|
1050
|
+
*
|
|
1051
|
+
* SAMPLED log: one line per shed frame turned into 45 741 rows/hour on
|
|
1052
|
+
* the hub during the 2026-08-01 chronic overload (~13 shed/s) — 2/3 of
|
|
1053
|
+
* the hub's entire log volume, burying every other signal. The counter
|
|
1054
|
+
* keeps exact totals (`droppedTotal` is authoritative, surfaced in
|
|
1055
|
+
* `getStatus` consumers via `getDroppedResponseCount`); the log line
|
|
1056
|
+
* now fires on the FIRST shed and every `SHED_LOG_SAMPLE_EVERY`th,
|
|
1057
|
+
* carrying the running total so nothing is lost — only repeated.
|
|
1058
1058
|
*/
|
|
1059
1059
|
trackDroppedResponse(result, modelIndex) {
|
|
1060
1060
|
if (result["dropped"] === true) {
|
|
1061
1061
|
this.droppedResponseCount++;
|
|
1062
|
-
|
|
1062
|
+
const total = this.droppedResponseCount;
|
|
1063
|
+
if (total === 1 || total % SHED_LOG_SAMPLE_EVERY === 0) this.log.debug("Python pool shed frame under overload", { meta: {
|
|
1063
1064
|
modelIndex,
|
|
1064
|
-
droppedTotal:
|
|
1065
|
+
droppedTotal: total,
|
|
1066
|
+
sampledEvery: SHED_LOG_SAMPLE_EVERY,
|
|
1065
1067
|
runtime: this.poolRuntime,
|
|
1066
1068
|
device: this.device ?? "default"
|
|
1067
1069
|
} });
|
|
@@ -1138,6 +1140,66 @@ function flattenEnabledVideoSteps(steps) {
|
|
|
1138
1140
|
walk(steps);
|
|
1139
1141
|
return result;
|
|
1140
1142
|
}
|
|
1143
|
+
/**
|
|
1144
|
+
* The steps that EXECUTE in a `plane: 'frame'` dispatch (two-plane design),
|
|
1145
|
+
* mirroring the executor's frame-plane skip in `executeChildren`: every
|
|
1146
|
+
* enabled top-level step runs at the root; a CHILD runs only when its catalog
|
|
1147
|
+
* definition is itself root-plane (`isDetailStep(addonId)` false). The walk
|
|
1148
|
+
* continues only through steps that execute — a skipped detail child's
|
|
1149
|
+
* descendants ride ITS per-track detail-subtree call, never this one.
|
|
1150
|
+
* Disabled and audio-classifier steps execute on neither plane.
|
|
1151
|
+
*/
|
|
1152
|
+
function collectFramePlaneSteps(steps, isDetailStep) {
|
|
1153
|
+
const result = [];
|
|
1154
|
+
const walk = (nodes, topLevel) => {
|
|
1155
|
+
for (const step of nodes) {
|
|
1156
|
+
if (!step.enabled) continue;
|
|
1157
|
+
if (step.slot === "audio-classifier") continue;
|
|
1158
|
+
if (!topLevel && isDetailStep(step.addonId)) continue;
|
|
1159
|
+
result.push(step);
|
|
1160
|
+
if (step.children?.length) walk(step.children, false);
|
|
1161
|
+
}
|
|
1162
|
+
};
|
|
1163
|
+
walk(steps, true);
|
|
1164
|
+
return result;
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* Immutably remove from a frame-plane dispatch tree every enabled DETAIL
|
|
1168
|
+
* subtree (at any depth below the top level) whose top step `isUnrunnable`.
|
|
1169
|
+
* Those steps can never load into the dispatch device's pool — leaving them
|
|
1170
|
+
* in the tree makes `ensureModelsForSteps` fail the whole call on a format
|
|
1171
|
+
* build that does not exist. They are NOT dropped work: each runs later as
|
|
1172
|
+
* its own detail-subtree dispatch, where the device-jump resolver
|
|
1173
|
+
* (`resolveStepDevice`) places it on a same-node device that can run it, or
|
|
1174
|
+
* warns no-candidate loudly. Runnable detail children are kept so their
|
|
1175
|
+
* models pre-warm the device pool as before. Disabled subtrees are kept
|
|
1176
|
+
* verbatim (never walked — `ensureModelsForSteps` skips them anyway).
|
|
1177
|
+
*/
|
|
1178
|
+
function pruneUnrunnableDetailSteps(steps, isDetailStep, isUnrunnable) {
|
|
1179
|
+
const prunedAddonIds = [];
|
|
1180
|
+
const walk = (nodes, topLevel) => {
|
|
1181
|
+
const kept = [];
|
|
1182
|
+
for (const step of nodes) {
|
|
1183
|
+
if (!step.enabled) {
|
|
1184
|
+
kept.push(step);
|
|
1185
|
+
continue;
|
|
1186
|
+
}
|
|
1187
|
+
if (!topLevel && isDetailStep(step.addonId) && isUnrunnable(step.addonId)) {
|
|
1188
|
+
prunedAddonIds.push(step.addonId);
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
kept.push(step.children?.length ? {
|
|
1192
|
+
...step,
|
|
1193
|
+
children: walk(step.children, false)
|
|
1194
|
+
} : step);
|
|
1195
|
+
}
|
|
1196
|
+
return kept;
|
|
1197
|
+
};
|
|
1198
|
+
return {
|
|
1199
|
+
steps: walk(steps, true),
|
|
1200
|
+
prunedAddonIds
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1141
1203
|
//#endregion
|
|
1142
1204
|
//#region src/detection-pipeline/engine/pipeline-model-manager.ts
|
|
1143
1205
|
var PipelineModelManager = class {
|
|
@@ -1535,7 +1597,7 @@ var RUNTIME_TO_FORMAT = {
|
|
|
1535
1597
|
edgetpu: "tflite"
|
|
1536
1598
|
};
|
|
1537
1599
|
/** The step whose detections are re-filtered downstream by the executor's
|
|
1538
|
-
* per-macro `minConfidence*` sliders (`
|
|
1600
|
+
* per-macro `minConfidence*` sliders (`macroFilterVerdict`). Its pool-side
|
|
1539
1601
|
* floor must stay BELOW those sliders or it silently pre-empts them. */
|
|
1540
1602
|
var OBJECT_DETECTION_STEP_ID = "object-detection";
|
|
1541
1603
|
/**
|
|
@@ -1980,601 +2042,6 @@ var EngineProvisioner = class {
|
|
|
1980
2042
|
}
|
|
1981
2043
|
};
|
|
1982
2044
|
//#endregion
|
|
1983
|
-
//#region src/detection-pipeline/postprocess/dispatch.ts
|
|
1984
|
-
/** Detector NMS IoU fallback — the historical hardcoded literal, used when the
|
|
1985
|
-
* step carries no `nmsIouThreshold` setting (unset == today). */
|
|
1986
|
-
var DEFAULT_NMS_IOU = .45;
|
|
1987
|
-
/**
|
|
1988
|
-
* Region plate grammars for the TS fallback CTC decoder (nodejs+onnx path).
|
|
1989
|
-
* VALIDATION-ONLY: this mirror annotates `formatValid` but does NOT rescore
|
|
1990
|
-
* confusables — the confusable repair needs the aligned per-timestep softmax
|
|
1991
|
-
* that only the Python path retains (see `python/postprocessors/ctc.py`). Keep
|
|
1992
|
-
* the pattern set in sync with `PLATE_GRAMMARS` there. `'off'`/unset disables it.
|
|
1993
|
-
*/
|
|
1994
|
-
var TS_PLATE_GRAMMARS = { DE: /^[A-ZÄÖÜ]{1,3}[- ]?[A-Z]{1,2}[- ]?[0-9]{1,4}[EH]?$/ };
|
|
1995
|
-
var VALID_KINDS = new Set([
|
|
1996
|
-
"detections",
|
|
1997
|
-
"classifications",
|
|
1998
|
-
"embedding",
|
|
1999
|
-
"text",
|
|
2000
|
-
"mask"
|
|
2001
|
-
]);
|
|
2002
|
-
/**
|
|
2003
|
-
* Type guard: validates that a structured payload from the Python pool is a
|
|
2004
|
-
* well-formed StepOutput discriminated union.
|
|
2005
|
-
*
|
|
2006
|
-
* The Python inference_pool.py always sets `output.structured` with a `kind`
|
|
2007
|
-
* field. This guard narrows `Record<string, unknown>` to `StepOutput` without
|
|
2008
|
-
* resorting to double-cast.
|
|
2009
|
-
*/
|
|
2010
|
-
function isStepOutput(value) {
|
|
2011
|
-
return typeof value === "object" && value !== null && typeof value["kind"] === "string" && VALID_KINDS.has(value["kind"]);
|
|
2012
|
-
}
|
|
2013
|
-
/**
|
|
2014
|
-
* Return the structured payload as `StepOutput`, or null if it fails validation.
|
|
2015
|
-
* Callers must handle null by proceeding to raw-tensor postprocessing.
|
|
2016
|
-
*/
|
|
2017
|
-
function tryStructured(output) {
|
|
2018
|
-
if (!output.structured) return null;
|
|
2019
|
-
if (!isStepOutput(output.structured)) throw new Error(`Python pool returned structured output with unexpected kind: ${JSON.stringify(output.structured["kind"])}`);
|
|
2020
|
-
return output.structured;
|
|
2021
|
-
}
|
|
2022
|
-
function postprocessYolo(output, stepDef, nmsIouThreshold) {
|
|
2023
|
-
const structured = tryStructured(output);
|
|
2024
|
-
if (structured) return structured;
|
|
2025
|
-
const tensor = output.tensor;
|
|
2026
|
-
if (!tensor) throw new Error("YOLO postprocessor: no tensor in engine output");
|
|
2027
|
-
const labels = stepDef.labels ?? [];
|
|
2028
|
-
const numClasses = labels.length || 80;
|
|
2029
|
-
const numBoxes = tensor.length / (4 + numClasses);
|
|
2030
|
-
const letterbox = output.letterbox;
|
|
2031
|
-
const dets = [];
|
|
2032
|
-
for (let i = 0; i < numBoxes; i++) {
|
|
2033
|
-
const cx = tensor[i];
|
|
2034
|
-
const cy = tensor[1 * numBoxes + i];
|
|
2035
|
-
const w = tensor[2 * numBoxes + i];
|
|
2036
|
-
const h = tensor[3 * numBoxes + i];
|
|
2037
|
-
let bestScore = -Infinity;
|
|
2038
|
-
let bestClass = 0;
|
|
2039
|
-
for (let j = 0; j < numClasses; j++) {
|
|
2040
|
-
const score = tensor[(4 + j) * numBoxes + i];
|
|
2041
|
-
if (score > bestScore) {
|
|
2042
|
-
bestScore = score;
|
|
2043
|
-
bestClass = j;
|
|
2044
|
-
}
|
|
2045
|
-
}
|
|
2046
|
-
if (bestScore <= 0) continue;
|
|
2047
|
-
let x1 = cx - w / 2;
|
|
2048
|
-
let y1 = cy - h / 2;
|
|
2049
|
-
let x2 = cx + w / 2;
|
|
2050
|
-
let y2 = cy + h / 2;
|
|
2051
|
-
if (letterbox) {
|
|
2052
|
-
x1 = (x1 - letterbox.padX) / letterbox.scale;
|
|
2053
|
-
y1 = (y1 - letterbox.padY) / letterbox.scale;
|
|
2054
|
-
x2 = (x2 - letterbox.padX) / letterbox.scale;
|
|
2055
|
-
y2 = (y2 - letterbox.padY) / letterbox.scale;
|
|
2056
|
-
}
|
|
2057
|
-
const label = labels[bestClass] ?? String(bestClass);
|
|
2058
|
-
dets.push({
|
|
2059
|
-
class: label,
|
|
2060
|
-
score: bestScore,
|
|
2061
|
-
bbox: [
|
|
2062
|
-
x1,
|
|
2063
|
-
y1,
|
|
2064
|
-
x2,
|
|
2065
|
-
y2
|
|
2066
|
-
]
|
|
2067
|
-
});
|
|
2068
|
-
}
|
|
2069
|
-
return {
|
|
2070
|
-
kind: "detections",
|
|
2071
|
-
detections: simpleNms(dets, nmsIouThreshold)
|
|
2072
|
-
};
|
|
2073
|
-
}
|
|
2074
|
-
function postprocessSsd(output, _stepDef) {
|
|
2075
|
-
const structured = tryStructured(output);
|
|
2076
|
-
if (structured) return structured;
|
|
2077
|
-
throw new Error("SSD postprocessing runs in the Python inference pool (ssd.py); no TypeScript raw-tensor path.");
|
|
2078
|
-
}
|
|
2079
|
-
function postprocessScrfd(output, stepDef, nmsIouThreshold) {
|
|
2080
|
-
const structured = tryStructured(output);
|
|
2081
|
-
if (structured) return structured;
|
|
2082
|
-
if (!output.tensors) throw new Error("SCRFD postprocessor: no tensors in engine output");
|
|
2083
|
-
const strides = [
|
|
2084
|
-
8,
|
|
2085
|
-
16,
|
|
2086
|
-
32
|
|
2087
|
-
];
|
|
2088
|
-
const anchorsPerStride = 2;
|
|
2089
|
-
const inputSize = Math.max(stepDef.models[0]?.inputSize.width ?? 640, stepDef.models[0]?.inputSize.height ?? 640);
|
|
2090
|
-
const strideData = matchTensorsToStrides(output.tensors, strides, inputSize, anchorsPerStride);
|
|
2091
|
-
const letterbox = output.letterbox;
|
|
2092
|
-
const scale = letterbox?.scale ?? 1;
|
|
2093
|
-
const padX = letterbox?.padX ?? 0;
|
|
2094
|
-
const padY = letterbox?.padY ?? 0;
|
|
2095
|
-
const origW = letterbox?.originalWidth ?? inputSize;
|
|
2096
|
-
const origH = letterbox?.originalHeight ?? inputSize;
|
|
2097
|
-
const candidates = [];
|
|
2098
|
-
for (const stride of strides) {
|
|
2099
|
-
const data = strideData.get(stride);
|
|
2100
|
-
if (!data) continue;
|
|
2101
|
-
const { scores, bboxes, landmarks } = data;
|
|
2102
|
-
const featSize = Math.ceil(inputSize / stride);
|
|
2103
|
-
const numAnchors = featSize * featSize * anchorsPerStride;
|
|
2104
|
-
for (let i = 0; i < numAnchors; i++) {
|
|
2105
|
-
if (i >= scores.length) break;
|
|
2106
|
-
const score = scores[i];
|
|
2107
|
-
if (score <= 0) continue;
|
|
2108
|
-
const gridIdx = Math.floor(i / anchorsPerStride);
|
|
2109
|
-
const cx = (gridIdx % featSize + .5) * stride;
|
|
2110
|
-
const cy = (Math.floor(gridIdx / featSize) + .5) * stride;
|
|
2111
|
-
const x1 = cx - bboxes[i * 4] * stride;
|
|
2112
|
-
const y1 = cy - bboxes[i * 4 + 1] * stride;
|
|
2113
|
-
const x2 = cx + bboxes[i * 4 + 2] * stride;
|
|
2114
|
-
const y2 = cy + bboxes[i * 4 + 3] * stride;
|
|
2115
|
-
const ox1 = Math.max(0, Math.min(origW, (x1 - padX) / scale));
|
|
2116
|
-
const oy1 = Math.max(0, Math.min(origH, (y1 - padY) / scale));
|
|
2117
|
-
const ox2 = Math.max(0, Math.min(origW, (x2 - padX) / scale));
|
|
2118
|
-
const oy2 = Math.max(0, Math.min(origH, (y2 - padY) / scale));
|
|
2119
|
-
if (ox2 - ox1 < 1 || oy2 - oy1 < 1) continue;
|
|
2120
|
-
const det = {
|
|
2121
|
-
class: "face",
|
|
2122
|
-
score: Math.round(score * 1e4) / 1e4,
|
|
2123
|
-
bbox: [
|
|
2124
|
-
Math.round(ox1 * 10) / 10,
|
|
2125
|
-
Math.round(oy1 * 10) / 10,
|
|
2126
|
-
Math.round(ox2 * 10) / 10,
|
|
2127
|
-
Math.round(oy2 * 10) / 10
|
|
2128
|
-
],
|
|
2129
|
-
_xyxy: [
|
|
2130
|
-
ox1,
|
|
2131
|
-
oy1,
|
|
2132
|
-
ox2,
|
|
2133
|
-
oy2
|
|
2134
|
-
]
|
|
2135
|
-
};
|
|
2136
|
-
if (landmarks && i * 10 + 9 < landmarks.length) {
|
|
2137
|
-
det.landmarks = [];
|
|
2138
|
-
for (let p = 0; p < 5; p++) {
|
|
2139
|
-
const lx = (cx + landmarks[i * 10 + p * 2] * stride - padX) / scale;
|
|
2140
|
-
const ly = (cy + landmarks[i * 10 + p * 2 + 1] * stride - padY) / scale;
|
|
2141
|
-
det.landmarks.push({
|
|
2142
|
-
x: Math.round(lx * 10) / 10,
|
|
2143
|
-
y: Math.round(ly * 10) / 10
|
|
2144
|
-
});
|
|
2145
|
-
}
|
|
2146
|
-
}
|
|
2147
|
-
candidates.push(det);
|
|
2148
|
-
}
|
|
2149
|
-
}
|
|
2150
|
-
return {
|
|
2151
|
-
kind: "detections",
|
|
2152
|
-
detections: simpleNms(candidates.map((c) => ({
|
|
2153
|
-
class: c.class,
|
|
2154
|
-
score: c.score,
|
|
2155
|
-
bbox: c.bbox,
|
|
2156
|
-
...c.landmarks ? { landmarks: c.landmarks } : {}
|
|
2157
|
-
})), nmsIouThreshold)
|
|
2158
|
-
};
|
|
2159
|
-
}
|
|
2160
|
-
/** Match ONNX output tensors to SCRFD strides by expected anchor count. */
|
|
2161
|
-
function matchTensorsToStrides(tensors, strides, inputSize, anchorsPerStride) {
|
|
2162
|
-
const scoresByN = /* @__PURE__ */ new Map();
|
|
2163
|
-
const bboxesByN = /* @__PURE__ */ new Map();
|
|
2164
|
-
const landmarksByN = /* @__PURE__ */ new Map();
|
|
2165
|
-
for (const [, tensor] of Object.entries(tensors)) for (const stride of strides) {
|
|
2166
|
-
const featSize = Math.ceil(inputSize / stride);
|
|
2167
|
-
const expectedN = featSize * featSize * anchorsPerStride;
|
|
2168
|
-
if (tensor.length === expectedN) scoresByN.set(expectedN, tensor);
|
|
2169
|
-
else if (tensor.length === expectedN * 4) bboxesByN.set(expectedN, tensor);
|
|
2170
|
-
else if (tensor.length === expectedN * 10) landmarksByN.set(expectedN, tensor);
|
|
2171
|
-
}
|
|
2172
|
-
const result = /* @__PURE__ */ new Map();
|
|
2173
|
-
for (const stride of strides) {
|
|
2174
|
-
const featSize = Math.ceil(inputSize / stride);
|
|
2175
|
-
const n = featSize * featSize * anchorsPerStride;
|
|
2176
|
-
const scores = scoresByN.get(n);
|
|
2177
|
-
const bboxes = bboxesByN.get(n);
|
|
2178
|
-
if (scores && bboxes) result.set(stride, {
|
|
2179
|
-
scores,
|
|
2180
|
-
bboxes,
|
|
2181
|
-
landmarks: landmarksByN.get(n)
|
|
2182
|
-
});
|
|
2183
|
-
}
|
|
2184
|
-
return result;
|
|
2185
|
-
}
|
|
2186
|
-
function postprocessArcface(output, _stepDef) {
|
|
2187
|
-
const structured = tryStructured(output);
|
|
2188
|
-
if (structured) return structured;
|
|
2189
|
-
const tensor = output.tensor;
|
|
2190
|
-
if (!tensor) throw new Error("ArcFace postprocessor: no tensor in engine output");
|
|
2191
|
-
let sumSq = 0;
|
|
2192
|
-
for (let i = 0; i < tensor.length; i++) sumSq += tensor[i] * tensor[i];
|
|
2193
|
-
const norm = Math.sqrt(sumSq);
|
|
2194
|
-
const normalized = Array.from({ length: tensor.length });
|
|
2195
|
-
for (let i = 0; i < tensor.length; i++) normalized[i] = norm === 0 ? 0 : tensor[i] / norm;
|
|
2196
|
-
return {
|
|
2197
|
-
kind: "embedding",
|
|
2198
|
-
embedding: Array.from(tensor),
|
|
2199
|
-
embeddingNorm: normalized
|
|
2200
|
-
};
|
|
2201
|
-
}
|
|
2202
|
-
function postprocessClip(output, _stepDef) {
|
|
2203
|
-
const structured = tryStructured(output);
|
|
2204
|
-
if (structured) return structured;
|
|
2205
|
-
const tensor = output.tensor;
|
|
2206
|
-
if (!tensor) throw new Error("CLIP postprocessor: no tensor in engine output");
|
|
2207
|
-
let sumSq = 0;
|
|
2208
|
-
for (let i = 0; i < tensor.length; i++) sumSq += tensor[i] * tensor[i];
|
|
2209
|
-
const norm = Math.sqrt(sumSq);
|
|
2210
|
-
const normalized = Array.from({ length: tensor.length });
|
|
2211
|
-
for (let i = 0; i < tensor.length; i++) normalized[i] = norm === 0 ? 0 : tensor[i] / norm;
|
|
2212
|
-
return {
|
|
2213
|
-
kind: "embedding",
|
|
2214
|
-
embedding: Array.from(tensor),
|
|
2215
|
-
embeddingNorm: normalized
|
|
2216
|
-
};
|
|
2217
|
-
}
|
|
2218
|
-
function postprocessSoftmax(output, stepDef) {
|
|
2219
|
-
const structured = tryStructured(output);
|
|
2220
|
-
if (structured) return structured;
|
|
2221
|
-
const tensor = output.tensor;
|
|
2222
|
-
if (!tensor) throw new Error("Softmax postprocessor: no tensor in engine output");
|
|
2223
|
-
const labels = stepDef.labels ?? [];
|
|
2224
|
-
let max = -Infinity;
|
|
2225
|
-
for (let i = 0; i < tensor.length; i++) if (tensor[i] > max) max = tensor[i];
|
|
2226
|
-
const exps = new Float32Array(tensor.length);
|
|
2227
|
-
let sum = 0;
|
|
2228
|
-
for (let i = 0; i < tensor.length; i++) {
|
|
2229
|
-
exps[i] = Math.exp(tensor[i] - max);
|
|
2230
|
-
sum += exps[i];
|
|
2231
|
-
}
|
|
2232
|
-
const classifications = [];
|
|
2233
|
-
for (let i = 0; i < exps.length; i++) {
|
|
2234
|
-
const score = exps[i] / sum;
|
|
2235
|
-
if (score >= .01) classifications.push({
|
|
2236
|
-
class: labels[i] ?? String(i),
|
|
2237
|
-
score
|
|
2238
|
-
});
|
|
2239
|
-
}
|
|
2240
|
-
classifications.sort((a, b) => b.score - a.score);
|
|
2241
|
-
return {
|
|
2242
|
-
kind: "classifications",
|
|
2243
|
-
classifications: classifications.slice(0, 5)
|
|
2244
|
-
};
|
|
2245
|
-
}
|
|
2246
|
-
function postprocessCtc(output, stepDef) {
|
|
2247
|
-
const structured = tryStructured(output);
|
|
2248
|
-
if (structured) return structured;
|
|
2249
|
-
const tensor = output.tensor;
|
|
2250
|
-
if (!tensor) throw new Error("CTC postprocessor: no tensor in engine output");
|
|
2251
|
-
const charset = stepDef.charset ?? [];
|
|
2252
|
-
const numChars = charset.length || 97;
|
|
2253
|
-
const seqLen = Math.floor(tensor.length / numChars);
|
|
2254
|
-
const chars = [];
|
|
2255
|
-
let totalScore = 0;
|
|
2256
|
-
let prev = -1;
|
|
2257
|
-
for (let t = 0; t < seqLen; t++) {
|
|
2258
|
-
let bestIdx = 0;
|
|
2259
|
-
let bestVal = tensor[t * numChars];
|
|
2260
|
-
for (let c = 1; c < numChars; c++) {
|
|
2261
|
-
const val = tensor[t * numChars + c];
|
|
2262
|
-
if (val > bestVal) {
|
|
2263
|
-
bestVal = val;
|
|
2264
|
-
bestIdx = c;
|
|
2265
|
-
}
|
|
2266
|
-
}
|
|
2267
|
-
totalScore += bestVal;
|
|
2268
|
-
if (bestIdx !== 0 && bestIdx !== prev) chars.push(charset[bestIdx] ?? "");
|
|
2269
|
-
prev = bestIdx;
|
|
2270
|
-
}
|
|
2271
|
-
const text = chars.join("");
|
|
2272
|
-
const region = stepDef.plateRegion;
|
|
2273
|
-
if (region !== void 0 && region !== "off") {
|
|
2274
|
-
const grammar = TS_PLATE_GRAMMARS[region];
|
|
2275
|
-
if (grammar) return {
|
|
2276
|
-
kind: "text",
|
|
2277
|
-
text,
|
|
2278
|
-
confidence: seqLen > 0 ? totalScore / seqLen : 0,
|
|
2279
|
-
formatValid: grammar.test(text)
|
|
2280
|
-
};
|
|
2281
|
-
}
|
|
2282
|
-
return {
|
|
2283
|
-
kind: "text",
|
|
2284
|
-
text,
|
|
2285
|
-
confidence: seqLen > 0 ? totalScore / seqLen : 0
|
|
2286
|
-
};
|
|
2287
|
-
}
|
|
2288
|
-
/**
|
|
2289
|
-
* Bounding box [x, y, w, h] (mask-pixel space) of the largest 4-connected
|
|
2290
|
-
* foreground component. Port of `_largest_component_bbox` from
|
|
2291
|
-
* `python/postprocessors/saliency.py`.
|
|
2292
|
-
*
|
|
2293
|
-
* Returns null when there are no foreground pixels.
|
|
2294
|
-
*/
|
|
2295
|
-
function largestComponentBbox(binary, w, h) {
|
|
2296
|
-
const seeds = [];
|
|
2297
|
-
for (let row = 0; row < h; row++) for (let col = 0; col < w; col++) if ((binary[row * w + col] ?? 0) > 0) seeds.push([row, col]);
|
|
2298
|
-
if (seeds.length === 0) return null;
|
|
2299
|
-
const visited = new Uint8Array(w * h);
|
|
2300
|
-
let best = null;
|
|
2301
|
-
for (const [sy, sx] of seeds) {
|
|
2302
|
-
const seedIdx = sy * w + sx;
|
|
2303
|
-
if (visited[seedIdx] === 1) continue;
|
|
2304
|
-
const stack = [[sy, sx]];
|
|
2305
|
-
visited[seedIdx] = 1;
|
|
2306
|
-
let minX = sx;
|
|
2307
|
-
let maxX = sx;
|
|
2308
|
-
let minY = sy;
|
|
2309
|
-
let maxY = sy;
|
|
2310
|
-
let area = 0;
|
|
2311
|
-
while (stack.length > 0) {
|
|
2312
|
-
const [cy, cx] = stack.pop();
|
|
2313
|
-
area += 1;
|
|
2314
|
-
if (cx < minX) minX = cx;
|
|
2315
|
-
if (cx > maxX) maxX = cx;
|
|
2316
|
-
if (cy < minY) minY = cy;
|
|
2317
|
-
if (cy > maxY) maxY = cy;
|
|
2318
|
-
const neighbours = [
|
|
2319
|
-
[cy - 1, cx],
|
|
2320
|
-
[cy + 1, cx],
|
|
2321
|
-
[cy, cx - 1],
|
|
2322
|
-
[cy, cx + 1]
|
|
2323
|
-
];
|
|
2324
|
-
for (const [ny, nx] of neighbours) {
|
|
2325
|
-
if (ny < 0 || ny >= h || nx < 0 || nx >= w) continue;
|
|
2326
|
-
const nIdx = ny * w + nx;
|
|
2327
|
-
if ((binary[nIdx] ?? 0) > 0 && visited[nIdx] !== 1) {
|
|
2328
|
-
visited[nIdx] = 1;
|
|
2329
|
-
stack.push([ny, nx]);
|
|
2330
|
-
}
|
|
2331
|
-
}
|
|
2332
|
-
}
|
|
2333
|
-
if (best === null || area > best.area) best = {
|
|
2334
|
-
area,
|
|
2335
|
-
minX,
|
|
2336
|
-
minY,
|
|
2337
|
-
maxX,
|
|
2338
|
-
maxY
|
|
2339
|
-
};
|
|
2340
|
-
}
|
|
2341
|
-
if (best === null) return null;
|
|
2342
|
-
return [
|
|
2343
|
-
best.minX,
|
|
2344
|
-
best.minY,
|
|
2345
|
-
best.maxX - best.minX + 1,
|
|
2346
|
-
best.maxY - best.minY + 1
|
|
2347
|
-
];
|
|
2348
|
-
}
|
|
2349
|
-
function postprocessSaliency(output, _stepDef) {
|
|
2350
|
-
const structured = tryStructured(output);
|
|
2351
|
-
if (structured) return structured;
|
|
2352
|
-
const tensor = output.tensor;
|
|
2353
|
-
if (!tensor) throw new Error("Saliency postprocessor: no tensor in engine output");
|
|
2354
|
-
const binary = new Uint8Array(tensor.length);
|
|
2355
|
-
for (let i = 0; i < tensor.length; i++) binary[i] = 1 / (1 + Math.exp(-tensor[i])) > .5 ? 255 : 0;
|
|
2356
|
-
const side = Math.round(Math.sqrt(tensor.length));
|
|
2357
|
-
const mask = Buffer.from(binary).toString("base64");
|
|
2358
|
-
const maskBbox = largestComponentBbox(binary, side, side);
|
|
2359
|
-
if (maskBbox !== null) return {
|
|
2360
|
-
kind: "mask",
|
|
2361
|
-
mask,
|
|
2362
|
-
maskWidth: side,
|
|
2363
|
-
maskHeight: side,
|
|
2364
|
-
maskBbox
|
|
2365
|
-
};
|
|
2366
|
-
return {
|
|
2367
|
-
kind: "mask",
|
|
2368
|
-
mask,
|
|
2369
|
-
maskWidth: side,
|
|
2370
|
-
maskHeight: side
|
|
2371
|
-
};
|
|
2372
|
-
}
|
|
2373
|
-
function postprocessYoloSeg(output, stepDef, nmsIouThreshold) {
|
|
2374
|
-
const structured = tryStructured(output);
|
|
2375
|
-
if (structured) return structured;
|
|
2376
|
-
const tensors = output.tensors;
|
|
2377
|
-
if (!tensors) throw new Error("YOLO-seg postprocessor: no tensors in engine output");
|
|
2378
|
-
const labels = stepDef.labels ?? [];
|
|
2379
|
-
const numClasses = labels.length || 80;
|
|
2380
|
-
const numMaskCoeffs = 32;
|
|
2381
|
-
const protoSize = 160;
|
|
2382
|
-
const letterbox = output.letterbox;
|
|
2383
|
-
let detTensor;
|
|
2384
|
-
let protoTensor;
|
|
2385
|
-
let detRows = 300;
|
|
2386
|
-
let detCols = 0;
|
|
2387
|
-
for (const [, tensor] of Object.entries(tensors)) {
|
|
2388
|
-
if (tensor.length === numMaskCoeffs * protoSize * protoSize) {
|
|
2389
|
-
protoTensor = tensor;
|
|
2390
|
-
continue;
|
|
2391
|
-
}
|
|
2392
|
-
const cols = 38;
|
|
2393
|
-
if (tensor.length % cols === 0) {
|
|
2394
|
-
detTensor = tensor;
|
|
2395
|
-
detRows = tensor.length / cols;
|
|
2396
|
-
detCols = cols;
|
|
2397
|
-
continue;
|
|
2398
|
-
}
|
|
2399
|
-
const rawCols = 4 + numClasses + numMaskCoeffs;
|
|
2400
|
-
if (tensor.length % rawCols === 0 && !detTensor) {
|
|
2401
|
-
detTensor = tensor;
|
|
2402
|
-
detRows = tensor.length / rawCols;
|
|
2403
|
-
detCols = rawCols;
|
|
2404
|
-
}
|
|
2405
|
-
}
|
|
2406
|
-
if (!detTensor) throw new Error("YOLO-seg postprocessor: could not find detection tensor");
|
|
2407
|
-
if (!protoTensor) throw new Error("YOLO-seg postprocessor: could not find prototype tensor");
|
|
2408
|
-
const isNmsFormat = detCols === 38;
|
|
2409
|
-
const dets = [];
|
|
2410
|
-
for (let i = 0; i < detRows; i++) {
|
|
2411
|
-
const offset = i * detCols;
|
|
2412
|
-
let x1, y1, x2, y2;
|
|
2413
|
-
let bestScore;
|
|
2414
|
-
let bestClass;
|
|
2415
|
-
let coeffStart;
|
|
2416
|
-
if (isNmsFormat) {
|
|
2417
|
-
x1 = detTensor[offset];
|
|
2418
|
-
y1 = detTensor[offset + 1];
|
|
2419
|
-
x2 = detTensor[offset + 2];
|
|
2420
|
-
y2 = detTensor[offset + 3];
|
|
2421
|
-
bestScore = detTensor[offset + 4];
|
|
2422
|
-
bestClass = detTensor[offset + 5];
|
|
2423
|
-
coeffStart = offset + 6;
|
|
2424
|
-
if (bestScore <= 0) continue;
|
|
2425
|
-
if (bestClass < 0) continue;
|
|
2426
|
-
} else {
|
|
2427
|
-
const cx = detTensor[offset];
|
|
2428
|
-
const cy = detTensor[offset + 1];
|
|
2429
|
-
const w = detTensor[offset + 2];
|
|
2430
|
-
const h = detTensor[offset + 3];
|
|
2431
|
-
bestScore = -Infinity;
|
|
2432
|
-
bestClass = 0;
|
|
2433
|
-
for (let j = 0; j < numClasses; j++) {
|
|
2434
|
-
const score = detTensor[offset + 4 + j];
|
|
2435
|
-
if (score > bestScore) {
|
|
2436
|
-
bestScore = score;
|
|
2437
|
-
bestClass = j;
|
|
2438
|
-
}
|
|
2439
|
-
}
|
|
2440
|
-
if (bestScore <= 0) continue;
|
|
2441
|
-
x1 = cx - w / 2;
|
|
2442
|
-
y1 = cy - h / 2;
|
|
2443
|
-
x2 = cx + w / 2;
|
|
2444
|
-
y2 = cy + h / 2;
|
|
2445
|
-
coeffStart = offset + 4 + numClasses;
|
|
2446
|
-
}
|
|
2447
|
-
const coeffs = new Float32Array(numMaskCoeffs);
|
|
2448
|
-
for (let j = 0; j < numMaskCoeffs; j++) coeffs[j] = detTensor[coeffStart + j];
|
|
2449
|
-
const maskPixels = protoSize * protoSize;
|
|
2450
|
-
const maskRaw = new Float32Array(maskPixels);
|
|
2451
|
-
for (let p = 0; p < maskPixels; p++) {
|
|
2452
|
-
let sum = 0;
|
|
2453
|
-
for (let c = 0; c < numMaskCoeffs; c++) sum += coeffs[c] * protoTensor[c * maskPixels + p];
|
|
2454
|
-
maskRaw[p] = 1 / (1 + Math.exp(-sum));
|
|
2455
|
-
}
|
|
2456
|
-
const protoScale = protoSize / (stepDef.models[0]?.inputSize.width ?? 640);
|
|
2457
|
-
const px1 = Math.max(0, Math.floor(x1 * protoScale));
|
|
2458
|
-
const py1 = Math.max(0, Math.floor(y1 * protoScale));
|
|
2459
|
-
const px2 = Math.min(protoSize, Math.ceil(x2 * protoScale));
|
|
2460
|
-
const py2 = Math.min(protoSize, Math.ceil(y2 * protoScale));
|
|
2461
|
-
const cropW = Math.max(1, px2 - px1);
|
|
2462
|
-
const cropH = Math.max(1, py2 - py1);
|
|
2463
|
-
const cropped = new Uint8Array(cropW * cropH);
|
|
2464
|
-
for (let row = 0; row < cropH; row++) for (let col = 0; col < cropW; col++) {
|
|
2465
|
-
const srcIdx = (py1 + row) * protoSize + (px1 + col);
|
|
2466
|
-
cropped[row * cropW + col] = maskRaw[srcIdx] > .5 ? 255 : 0;
|
|
2467
|
-
}
|
|
2468
|
-
let ox1 = x1;
|
|
2469
|
-
let oy1 = y1;
|
|
2470
|
-
let ox2 = x2;
|
|
2471
|
-
let oy2 = y2;
|
|
2472
|
-
if (letterbox) {
|
|
2473
|
-
ox1 = (x1 - letterbox.padX) / letterbox.scale;
|
|
2474
|
-
oy1 = (y1 - letterbox.padY) / letterbox.scale;
|
|
2475
|
-
ox2 = (x2 - letterbox.padX) / letterbox.scale;
|
|
2476
|
-
oy2 = (y2 - letterbox.padY) / letterbox.scale;
|
|
2477
|
-
}
|
|
2478
|
-
const label = labels[bestClass] ?? String(bestClass);
|
|
2479
|
-
const maskB64 = Buffer.from(cropped).toString("base64");
|
|
2480
|
-
dets.push({
|
|
2481
|
-
class: label,
|
|
2482
|
-
score: Math.round(bestScore * 1e4) / 1e4,
|
|
2483
|
-
bbox: [
|
|
2484
|
-
Math.round(ox1 * 10) / 10,
|
|
2485
|
-
Math.round(oy1 * 10) / 10,
|
|
2486
|
-
Math.round(ox2 * 10) / 10,
|
|
2487
|
-
Math.round(oy2 * 10) / 10
|
|
2488
|
-
],
|
|
2489
|
-
mask: maskB64,
|
|
2490
|
-
maskWidth: cropW,
|
|
2491
|
-
maskHeight: cropH
|
|
2492
|
-
});
|
|
2493
|
-
}
|
|
2494
|
-
const keptBoxes = simpleNms(dets.map((d) => ({
|
|
2495
|
-
class: d.class,
|
|
2496
|
-
score: d.score,
|
|
2497
|
-
bbox: d.bbox
|
|
2498
|
-
})), nmsIouThreshold);
|
|
2499
|
-
const keptSet = new Set(keptBoxes.map((d) => `${d.bbox[0]}_${d.bbox[1]}_${d.bbox[2]}_${d.bbox[3]}_${d.score}`));
|
|
2500
|
-
return {
|
|
2501
|
-
kind: "detections",
|
|
2502
|
-
detections: dets.filter((d) => keptSet.has(`${d.bbox[0]}_${d.bbox[1]}_${d.bbox[2]}_${d.bbox[3]}_${d.score}`))
|
|
2503
|
-
};
|
|
2504
|
-
}
|
|
2505
|
-
function postprocessYamnet(output, _stepDef) {
|
|
2506
|
-
const structured = tryStructured(output);
|
|
2507
|
-
if (structured) return structured;
|
|
2508
|
-
throw new Error("YAMNet TypeScript postprocessing not implemented. Use Python backend or Apple SoundAnalysis.");
|
|
2509
|
-
}
|
|
2510
|
-
var DISPATCH = {
|
|
2511
|
-
yolo: postprocessYolo,
|
|
2512
|
-
ssd: postprocessSsd,
|
|
2513
|
-
"yolo-seg": postprocessYoloSeg,
|
|
2514
|
-
scrfd: postprocessScrfd,
|
|
2515
|
-
arcface: postprocessArcface,
|
|
2516
|
-
clip: postprocessClip,
|
|
2517
|
-
softmax: postprocessSoftmax,
|
|
2518
|
-
ctc: postprocessCtc,
|
|
2519
|
-
saliency: postprocessSaliency,
|
|
2520
|
-
yamnet: postprocessYamnet
|
|
2521
|
-
};
|
|
2522
|
-
/**
|
|
2523
|
-
* Dispatch postprocessing based on step definition's postprocessor type.
|
|
2524
|
-
*
|
|
2525
|
-
* For Python backends: output.structured is already populated → passthrough.
|
|
2526
|
-
* For Node.js ONNX: raw tensors → postprocess in TypeScript.
|
|
2527
|
-
*
|
|
2528
|
-
* Both paths pass through normalizeOutput() to ensure consistent format
|
|
2529
|
-
* (e.g., classification top-K, alternates).
|
|
2530
|
-
*/
|
|
2531
|
-
function dispatchPostprocess(output, stepDef, settings) {
|
|
2532
|
-
const fn = DISPATCH[stepDef.postprocessor];
|
|
2533
|
-
if (!fn) throw new Error(`Unknown postprocessor type: "${stepDef.postprocessor}"`);
|
|
2534
|
-
const raw = settings?.["nmsIouThreshold"];
|
|
2535
|
-
return normalizeOutput(fn(output, stepDef, typeof raw === "number" && raw > 0 ? raw : DEFAULT_NMS_IOU));
|
|
2536
|
-
}
|
|
2537
|
-
/**
|
|
2538
|
-
* Normalize any StepOutput to a consistent format.
|
|
2539
|
-
* Single funnel for both Python and TypeScript postprocessor paths.
|
|
2540
|
-
*
|
|
2541
|
-
* Classifications: keep only top-1 in classifications, move full list to alternates.
|
|
2542
|
-
*/
|
|
2543
|
-
function normalizeOutput(output) {
|
|
2544
|
-
if (output.kind !== "classifications") return output;
|
|
2545
|
-
const all = output.classifications;
|
|
2546
|
-
if (all.length <= 1) return output;
|
|
2547
|
-
return {
|
|
2548
|
-
kind: "classifications",
|
|
2549
|
-
classifications: all.slice(0, 1),
|
|
2550
|
-
alternates: all
|
|
2551
|
-
};
|
|
2552
|
-
}
|
|
2553
|
-
function simpleNms(dets, iouThreshold) {
|
|
2554
|
-
if (dets.length === 0) return [];
|
|
2555
|
-
const sorted = [...dets].toSorted((a, b) => b.score - a.score);
|
|
2556
|
-
const kept = [];
|
|
2557
|
-
const suppressed = /* @__PURE__ */ new Set();
|
|
2558
|
-
for (let i = 0; i < sorted.length; i++) {
|
|
2559
|
-
if (suppressed.has(i)) continue;
|
|
2560
|
-
kept.push(sorted[i]);
|
|
2561
|
-
for (let j = i + 1; j < sorted.length; j++) {
|
|
2562
|
-
if (suppressed.has(j)) continue;
|
|
2563
|
-
if (iou(sorted[i].bbox, sorted[j].bbox) > iouThreshold) suppressed.add(j);
|
|
2564
|
-
}
|
|
2565
|
-
}
|
|
2566
|
-
return kept;
|
|
2567
|
-
}
|
|
2568
|
-
function iou(a, b) {
|
|
2569
|
-
const ix1 = Math.max(a[0], b[0]);
|
|
2570
|
-
const iy1 = Math.max(a[1], b[1]);
|
|
2571
|
-
const ix2 = Math.min(a[2], b[2]);
|
|
2572
|
-
const iy2 = Math.min(a[3], b[3]);
|
|
2573
|
-
const inter = Math.max(0, ix2 - ix1) * Math.max(0, iy2 - iy1);
|
|
2574
|
-
if (inter === 0) return 0;
|
|
2575
|
-
return inter / ((a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter);
|
|
2576
|
-
}
|
|
2577
|
-
//#endregion
|
|
2578
2045
|
//#region src/detection-pipeline/pipeline/crop-utils.ts
|
|
2579
2046
|
/**
|
|
2580
2047
|
* Crop utilities — ROI extraction and bbox coordinate transforms.
|
|
@@ -2624,40 +2091,250 @@ function isBboxDegenerate(bbox) {
|
|
|
2624
2091
|
return bbox[2] - bbox[0] < 1 || bbox[3] - bbox[1] < 1;
|
|
2625
2092
|
}
|
|
2626
2093
|
//#endregion
|
|
2627
|
-
//#region src/detection-pipeline/pipeline/
|
|
2094
|
+
//#region src/detection-pipeline/pipeline/crop-zone.ts
|
|
2628
2095
|
/**
|
|
2629
|
-
*
|
|
2630
|
-
*
|
|
2631
|
-
*
|
|
2632
|
-
*
|
|
2633
|
-
* must land on fixed canonical positions. Feeding a raw axis-aligned bbox crop
|
|
2634
|
-
* stretched to 112x112 (the previous behaviour) collapses embeddings toward a
|
|
2635
|
-
* common direction — different people end up with ~0.99 cosine similarity.
|
|
2636
|
-
*
|
|
2637
|
-
* This module computes the least-squares similarity transform (rotation +
|
|
2638
|
-
* uniform scale + translation, the cv2.estimateAffinePartial2D recipe) that
|
|
2639
|
-
* maps SCRFD's 5 facial landmarks onto the canonical template, then resamples
|
|
2640
|
-
* the source pixels into a fixed-size aligned crop with bilinear interpolation.
|
|
2096
|
+
* Resolve the absolute-pixel crop bbox for a `crop-zone` root detector: the
|
|
2097
|
+
* bounding box of the UNION of every polygon referenced by the device's
|
|
2098
|
+
* ENABLED `package`-stage zone rules. Zone polygons are normalized (0..1); the
|
|
2099
|
+
* union is clamped to [0,1] then scaled to the frame's pixel dimensions.
|
|
2641
2100
|
*
|
|
2642
|
-
*
|
|
2643
|
-
*
|
|
2644
|
-
*
|
|
2645
|
-
|
|
2646
|
-
/**
|
|
2647
|
-
* Canonical ArcFace 5-point template for a 112x112 aligned crop
|
|
2648
|
-
* (insightface reference points). Order matches SCRFD landmark output:
|
|
2649
|
-
* left-eye, right-eye, nose, left-mouth-corner, right-mouth-corner.
|
|
2101
|
+
* Returns `null` when there is nothing usable to crop to — no enabled rules, no
|
|
2102
|
+
* referenced geometry, or a degenerate result — in which case the caller falls
|
|
2103
|
+
* back to full-frame inference (behaviour identical to before the crop-zone
|
|
2104
|
+
* plumbing).
|
|
2650
2105
|
*/
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2106
|
+
function resolvePackageCropBbox(zones, packageRules, frameWidth, frameHeight) {
|
|
2107
|
+
if (frameWidth <= 0 || frameHeight <= 0) return null;
|
|
2108
|
+
const enabledRules = packageRules.filter((r) => r.enabled !== false);
|
|
2109
|
+
if (enabledRules.length === 0) return null;
|
|
2110
|
+
const zoneIds = /* @__PURE__ */ new Set();
|
|
2111
|
+
for (const rule of enabledRules) for (const id of rule.zoneIds) zoneIds.add(id);
|
|
2112
|
+
if (zoneIds.size === 0) return null;
|
|
2113
|
+
const zoneById = new Map(zones.map((z) => [z.id, z]));
|
|
2114
|
+
let minX = Number.POSITIVE_INFINITY;
|
|
2115
|
+
let minY = Number.POSITIVE_INFINITY;
|
|
2116
|
+
let maxX = Number.NEGATIVE_INFINITY;
|
|
2117
|
+
let maxY = Number.NEGATIVE_INFINITY;
|
|
2118
|
+
let sawPoint = false;
|
|
2119
|
+
for (const id of zoneIds) {
|
|
2120
|
+
const zone = zoneById.get(id);
|
|
2121
|
+
if (!zone || zone.polygon.length === 0) continue;
|
|
2122
|
+
for (const point of zone.polygon) {
|
|
2123
|
+
sawPoint = true;
|
|
2124
|
+
if (point.x < minX) minX = point.x;
|
|
2125
|
+
if (point.y < minY) minY = point.y;
|
|
2126
|
+
if (point.x > maxX) maxX = point.x;
|
|
2127
|
+
if (point.y > maxY) maxY = point.y;
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
if (!sawPoint) return null;
|
|
2131
|
+
const clamp01 = (v) => Math.max(0, Math.min(1, v));
|
|
2132
|
+
const bbox = [
|
|
2133
|
+
clamp01(minX) * frameWidth,
|
|
2134
|
+
clamp01(minY) * frameHeight,
|
|
2135
|
+
clamp01(maxX) * frameWidth,
|
|
2136
|
+
clamp01(maxY) * frameHeight
|
|
2137
|
+
];
|
|
2138
|
+
if (isBboxDegenerate(bbox)) return null;
|
|
2139
|
+
return bbox;
|
|
2140
|
+
}
|
|
2141
|
+
//#endregion
|
|
2142
|
+
//#region src/detection-pipeline/postprocess/dispatch.ts
|
|
2143
|
+
var VALID_KINDS = new Set([
|
|
2144
|
+
"detections",
|
|
2145
|
+
"classifications",
|
|
2146
|
+
"embedding",
|
|
2147
|
+
"text",
|
|
2148
|
+
"mask"
|
|
2149
|
+
]);
|
|
2150
|
+
/** Every postprocessor the Python pool implements — the dispatch table's keys
|
|
2151
|
+
* survived the raw-tensor removal as a config-drift guard: an unknown
|
|
2152
|
+
* postprocessor in a step definition still fails fast here. */
|
|
2153
|
+
var KNOWN_POSTPROCESSORS = new Set([
|
|
2154
|
+
"yolo",
|
|
2155
|
+
"ssd",
|
|
2156
|
+
"yolo-seg",
|
|
2157
|
+
"scrfd",
|
|
2158
|
+
"arcface",
|
|
2159
|
+
"clip",
|
|
2160
|
+
"softmax",
|
|
2161
|
+
"ctc",
|
|
2162
|
+
"saliency",
|
|
2163
|
+
"yamnet"
|
|
2164
|
+
]);
|
|
2165
|
+
/**
|
|
2166
|
+
* Type guard: validates that a structured payload from the Python pool is a
|
|
2167
|
+
* well-formed StepOutput discriminated union.
|
|
2168
|
+
*
|
|
2169
|
+
* The Python inference_pool.py always sets `output.structured` with a `kind`
|
|
2170
|
+
* field. This guard narrows `Record<string, unknown>` to `StepOutput` without
|
|
2171
|
+
* resorting to double-cast.
|
|
2172
|
+
*/
|
|
2173
|
+
function isStepOutput(value) {
|
|
2174
|
+
return typeof value === "object" && value !== null && typeof value["kind"] === "string" && VALID_KINDS.has(value["kind"]);
|
|
2175
|
+
}
|
|
2176
|
+
/**
|
|
2177
|
+
* Validate + normalize a Python-pool engine output into a StepOutput.
|
|
2178
|
+
*/
|
|
2179
|
+
function dispatchPostprocess(output, stepDef) {
|
|
2180
|
+
if (output.dropped === true) return {
|
|
2181
|
+
kind: "detections",
|
|
2182
|
+
detections: []
|
|
2183
|
+
};
|
|
2184
|
+
if (!KNOWN_POSTPROCESSORS.has(stepDef.postprocessor)) throw new Error(`Unknown postprocessor type: "${stepDef.postprocessor}"`);
|
|
2185
|
+
if (!output.structured) throw new Error(`Postprocessing for step "${stepDef.id}" (${stepDef.postprocessor}) runs in the Python inference pool — the engine output carried no structured payload. The Node.js raw-tensor path was removed; there is nothing to fall back to.`);
|
|
2186
|
+
if (!isStepOutput(output.structured)) throw new Error(`Python pool returned structured output with unexpected kind: ${JSON.stringify(output.structured["kind"])}`);
|
|
2187
|
+
return normalizeOutput(output.structured);
|
|
2188
|
+
}
|
|
2189
|
+
/**
|
|
2190
|
+
* Normalize any StepOutput to a consistent format — the single funnel every
|
|
2191
|
+
* pool payload passes through.
|
|
2192
|
+
*
|
|
2193
|
+
* Classifications: keep only top-1 in classifications, move full list to alternates.
|
|
2194
|
+
*/
|
|
2195
|
+
function normalizeOutput(output) {
|
|
2196
|
+
if (output.kind !== "classifications") return output;
|
|
2197
|
+
const all = output.classifications;
|
|
2198
|
+
if (all.length <= 1) return output;
|
|
2199
|
+
return {
|
|
2200
|
+
kind: "classifications",
|
|
2201
|
+
classifications: all.slice(0, 1),
|
|
2202
|
+
alternates: all
|
|
2203
|
+
};
|
|
2204
|
+
}
|
|
2205
|
+
//#endregion
|
|
2206
|
+
//#region src/detection-pipeline/pipeline/execution-trace.ts
|
|
2207
|
+
var traceCounter = 0;
|
|
2208
|
+
var ExecutionTraceBuilder = class {
|
|
2209
|
+
verbosity;
|
|
2210
|
+
deviceId;
|
|
2211
|
+
frameWidth;
|
|
2212
|
+
frameHeight;
|
|
2213
|
+
engineRuntime;
|
|
2214
|
+
steps = [];
|
|
2215
|
+
startTime;
|
|
2216
|
+
constructor(verbosity, deviceId, frameWidth, frameHeight, engineRuntime) {
|
|
2217
|
+
this.verbosity = verbosity;
|
|
2218
|
+
this.deviceId = deviceId;
|
|
2219
|
+
this.frameWidth = frameWidth;
|
|
2220
|
+
this.frameHeight = frameHeight;
|
|
2221
|
+
this.engineRuntime = engineRuntime;
|
|
2222
|
+
this.startTime = Date.now();
|
|
2223
|
+
}
|
|
2224
|
+
/** Whether trace collection is active. */
|
|
2225
|
+
get isActive() {
|
|
2226
|
+
return this.verbosity !== "off";
|
|
2227
|
+
}
|
|
2228
|
+
/**
|
|
2229
|
+
* Record a completed step execution.
|
|
2230
|
+
* No-op if verbosity is 'off'.
|
|
2231
|
+
*/
|
|
2232
|
+
addStep(params) {
|
|
2233
|
+
if (this.verbosity === "off") return;
|
|
2234
|
+
const totalMs = params.preprocessMs + params.inferenceMs + params.postprocessMs;
|
|
2235
|
+
const trace = {
|
|
2236
|
+
stepId: params.stepId,
|
|
2237
|
+
modelId: params.modelId,
|
|
2238
|
+
slot: params.slot,
|
|
2239
|
+
postprocessor: params.postprocessor,
|
|
2240
|
+
preprocessMs: Math.round(params.preprocessMs * 100) / 100,
|
|
2241
|
+
inferenceMs: Math.round(params.inferenceMs * 100) / 100,
|
|
2242
|
+
postprocessMs: Math.round(params.postprocessMs * 100) / 100,
|
|
2243
|
+
totalMs: Math.round(totalMs * 100) / 100,
|
|
2244
|
+
inputType: params.inputType,
|
|
2245
|
+
inputSize: {
|
|
2246
|
+
width: params.inputWidth,
|
|
2247
|
+
height: params.inputHeight
|
|
2248
|
+
},
|
|
2249
|
+
parentDetection: this.verbosity === "full" ? params.parentDetection : void 0,
|
|
2250
|
+
outputKind: params.output.kind,
|
|
2251
|
+
outputCount: countOutput(params.output),
|
|
2252
|
+
topResult: this.verbosity === "full" ? formatTopResult(params.output) : void 0,
|
|
2253
|
+
error: params.error
|
|
2254
|
+
};
|
|
2255
|
+
this.steps.push(trace);
|
|
2256
|
+
}
|
|
2257
|
+
/**
|
|
2258
|
+
* Finalize and return the complete execution trace.
|
|
2259
|
+
* Returns null if verbosity is 'off'.
|
|
2260
|
+
*/
|
|
2261
|
+
build(detectionCount) {
|
|
2262
|
+
if (this.verbosity === "off") return null;
|
|
2263
|
+
const totalMs = Date.now() - this.startTime;
|
|
2264
|
+
return {
|
|
2265
|
+
traceId: `trace-${Date.now()}-${++traceCounter}`,
|
|
2266
|
+
deviceId: this.deviceId,
|
|
2267
|
+
timestamp: this.startTime,
|
|
2268
|
+
frameSize: {
|
|
2269
|
+
width: this.frameWidth,
|
|
2270
|
+
height: this.frameHeight
|
|
2271
|
+
},
|
|
2272
|
+
engineRuntime: this.engineRuntime,
|
|
2273
|
+
steps: this.steps,
|
|
2274
|
+
totalMs,
|
|
2275
|
+
detectionCount
|
|
2276
|
+
};
|
|
2277
|
+
}
|
|
2278
|
+
};
|
|
2279
|
+
function countOutput(output) {
|
|
2280
|
+
switch (output.kind) {
|
|
2281
|
+
case "detections": return output.detections.length;
|
|
2282
|
+
case "classifications": return output.classifications.length;
|
|
2283
|
+
case "embedding": return 1;
|
|
2284
|
+
case "text": return output.text.length > 0 ? 1 : 0;
|
|
2285
|
+
case "mask": return 1;
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
function formatTopResult(output) {
|
|
2289
|
+
switch (output.kind) {
|
|
2290
|
+
case "detections": {
|
|
2291
|
+
const top = output.detections[0];
|
|
2292
|
+
return top ? `${top.class} ${top.score.toFixed(2)}` : void 0;
|
|
2293
|
+
}
|
|
2294
|
+
case "classifications": {
|
|
2295
|
+
const top = output.classifications[0];
|
|
2296
|
+
return top ? `${top.class} ${top.score.toFixed(2)}` : void 0;
|
|
2297
|
+
}
|
|
2298
|
+
case "embedding": return `embedding[${output.embedding.length}]`;
|
|
2299
|
+
case "text": return output.text || void 0;
|
|
2300
|
+
case "mask": return `mask ${output.maskWidth}×${output.maskHeight}`;
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
//#endregion
|
|
2304
|
+
//#region src/detection-pipeline/pipeline/face-align.ts
|
|
2305
|
+
/**
|
|
2306
|
+
* Face alignment — landmark-based similarity warp to the ArcFace template.
|
|
2307
|
+
*
|
|
2308
|
+
* ArcFace (and most face-recognition embedders) are only discriminative when
|
|
2309
|
+
* the input is a *landmark-aligned* 112x112 crop: eyes, nose and mouth corners
|
|
2310
|
+
* must land on fixed canonical positions. Feeding a raw axis-aligned bbox crop
|
|
2311
|
+
* stretched to 112x112 (the previous behaviour) collapses embeddings toward a
|
|
2312
|
+
* common direction — different people end up with ~0.99 cosine similarity.
|
|
2313
|
+
*
|
|
2314
|
+
* This module computes the least-squares similarity transform (rotation +
|
|
2315
|
+
* uniform scale + translation, the cv2.estimateAffinePartial2D recipe) that
|
|
2316
|
+
* maps SCRFD's 5 facial landmarks onto the canonical template, then resamples
|
|
2317
|
+
* the source pixels into a fixed-size aligned crop with bilinear interpolation.
|
|
2318
|
+
*
|
|
2319
|
+
* The warp is done in pure JS on a raw RGB buffer (a few thousand samples per
|
|
2320
|
+
* face — negligible cost) so it is fully deterministic and unit-testable, and
|
|
2321
|
+
* needs no native affine primitive nor any change to the Python inference pool.
|
|
2322
|
+
*/
|
|
2323
|
+
/**
|
|
2324
|
+
* Canonical ArcFace 5-point template for a 112x112 aligned crop
|
|
2325
|
+
* (insightface reference points). Order matches SCRFD landmark output:
|
|
2326
|
+
* left-eye, right-eye, nose, left-mouth-corner, right-mouth-corner.
|
|
2327
|
+
*/
|
|
2328
|
+
var ARCFACE_TEMPLATE_112 = [
|
|
2329
|
+
{
|
|
2330
|
+
x: 38.2946,
|
|
2331
|
+
y: 51.6963
|
|
2332
|
+
},
|
|
2333
|
+
{
|
|
2334
|
+
x: 73.5318,
|
|
2335
|
+
y: 51.5014
|
|
2336
|
+
},
|
|
2337
|
+
{
|
|
2661
2338
|
x: 56.0252,
|
|
2662
2339
|
y: 71.7366
|
|
2663
2340
|
},
|
|
@@ -2929,63 +2606,6 @@ async function alignFaceCrop(fullFrameJpeg, faceBbox, landmarksImageSpace, image
|
|
|
2929
2606
|
height: aligned.height
|
|
2930
2607
|
};
|
|
2931
2608
|
}
|
|
2932
|
-
//#endregion
|
|
2933
|
-
//#region src/detection-pipeline/pipeline/native-child-crop.ts
|
|
2934
|
-
/**
|
|
2935
|
-
* NATIVE-resolution crop of a LEAF crop-child ROI (plate-ocr, leaf classifiers)
|
|
2936
|
-
* — the plate-side mirror of the face path's {@link ./face-align.buildAlignedFaceCrop}.
|
|
2937
|
-
*
|
|
2938
|
-
* On the detail plane the executor runs a child model against a parent crop
|
|
2939
|
-
* (e.g. the native vehicle crop). A leaf child's ROI (`parentDetection.bbox`,
|
|
2940
|
-
* in the parent crop's PIXEL space) is what the child model reads. Cutting it
|
|
2941
|
-
* from the parent tile with {@link ./crop-utils.cropJpeg} bounds the child input
|
|
2942
|
-
* to the tile's resolution — fatal for OCR on a small, distant plate. When a
|
|
2943
|
-
* native-crop provider is bound to the frame, we instead resolve the SAME ROI
|
|
2944
|
-
* straight from the frame's retained NATIVE surface (the provider re-composes
|
|
2945
|
-
* the crop-normalized ROI into frame space — see {@link ./native-crop-compose}),
|
|
2946
|
-
* giving the model native pixels. A miss/degenerate returns `null` so the caller
|
|
2947
|
-
* falls back to the existing tile crop (never an upscale — native or the tile).
|
|
2948
|
-
*/
|
|
2949
|
-
/**
|
|
2950
|
-
* Generous native-width cap for a leaf child crop. A plate ROI is a few hundred
|
|
2951
|
-
* native pixels wide, so this never binds for plates (effectively uncapped —
|
|
2952
|
-
* the quality path the task requires); it only bounds a pathologically large
|
|
2953
|
-
* whole-object classifier ROI so the in-process native fetch/encode stays cheap.
|
|
2954
|
-
*/
|
|
2955
|
-
var NATIVE_CHILD_CROP_MAX_WIDTH = 1920;
|
|
2956
|
-
/**
|
|
2957
|
-
* Resolve `bbox` (a parent-crop PIXEL rectangle `[x1,y1,x2,y2]`) at native
|
|
2958
|
-
* resolution via `provider`, returning a JPEG crop + its native dimensions.
|
|
2959
|
-
* Returns `null` on a degenerate ROI or any provider miss (so the caller uses
|
|
2960
|
-
* the downscaled tile crop instead). Pure w.r.t. its inputs; the only effect is
|
|
2961
|
-
* the sharp encode of the returned pixels.
|
|
2962
|
-
*/
|
|
2963
|
-
async function nativeChildCrop(provider, bbox, imageWidth, imageHeight, maxWidth) {
|
|
2964
|
-
if (!(imageWidth > 0) || !(imageHeight > 0)) return null;
|
|
2965
|
-
const x1 = Math.max(0, bbox[0]);
|
|
2966
|
-
const y1 = Math.max(0, bbox[1]);
|
|
2967
|
-
const x2 = Math.min(imageWidth, bbox[2]);
|
|
2968
|
-
const y2 = Math.min(imageHeight, bbox[3]);
|
|
2969
|
-
const w = x2 - x1;
|
|
2970
|
-
const h = y2 - y1;
|
|
2971
|
-
if (w < 1 || h < 1) return null;
|
|
2972
|
-
const native = await provider({
|
|
2973
|
-
x: x1 / imageWidth,
|
|
2974
|
-
y: y1 / imageHeight,
|
|
2975
|
-
w: w / imageWidth,
|
|
2976
|
-
h: h / imageHeight
|
|
2977
|
-
}, maxWidth);
|
|
2978
|
-
if (!native || native.width < 2 || native.height < 2 || native.bytes.length < native.width * native.height * 3) return null;
|
|
2979
|
-
return {
|
|
2980
|
-
jpeg: await sharp(Buffer.from(native.bytes), { raw: {
|
|
2981
|
-
width: native.width,
|
|
2982
|
-
height: native.height,
|
|
2983
|
-
channels: 3
|
|
2984
|
-
} }).jpeg({ quality: 90 }).toBuffer(),
|
|
2985
|
-
width: native.width,
|
|
2986
|
-
height: native.height
|
|
2987
|
-
};
|
|
2988
|
-
}
|
|
2989
2609
|
/**
|
|
2990
2610
|
* Detection score at/above which a near-full-frame box is trusted as a REAL
|
|
2991
2611
|
* close subject rather than a phantom. Below this bar a full-frame box is
|
|
@@ -3067,6 +2687,63 @@ function isFullFramePhantomDetection(bbox, frameWidth, frameHeight, score, maxCo
|
|
|
3067
2687
|
return false;
|
|
3068
2688
|
}
|
|
3069
2689
|
//#endregion
|
|
2690
|
+
//#region src/detection-pipeline/pipeline/native-child-crop.ts
|
|
2691
|
+
/**
|
|
2692
|
+
* NATIVE-resolution crop of a LEAF crop-child ROI (plate-ocr, leaf classifiers)
|
|
2693
|
+
* — the plate-side mirror of the face path's {@link ./face-align.buildAlignedFaceCrop}.
|
|
2694
|
+
*
|
|
2695
|
+
* On the detail plane the executor runs a child model against a parent crop
|
|
2696
|
+
* (e.g. the native vehicle crop). A leaf child's ROI (`parentDetection.bbox`,
|
|
2697
|
+
* in the parent crop's PIXEL space) is what the child model reads. Cutting it
|
|
2698
|
+
* from the parent tile with {@link ./crop-utils.cropJpeg} bounds the child input
|
|
2699
|
+
* to the tile's resolution — fatal for OCR on a small, distant plate. When a
|
|
2700
|
+
* native-crop provider is bound to the frame, we instead resolve the SAME ROI
|
|
2701
|
+
* straight from the frame's retained NATIVE surface (the provider re-composes
|
|
2702
|
+
* the crop-normalized ROI into frame space — see {@link ./native-crop-compose}),
|
|
2703
|
+
* giving the model native pixels. A miss/degenerate returns `null` so the caller
|
|
2704
|
+
* falls back to the existing tile crop (never an upscale — native or the tile).
|
|
2705
|
+
*/
|
|
2706
|
+
/**
|
|
2707
|
+
* Generous native-width cap for a leaf child crop. A plate ROI is a few hundred
|
|
2708
|
+
* native pixels wide, so this never binds for plates (effectively uncapped —
|
|
2709
|
+
* the quality path the task requires); it only bounds a pathologically large
|
|
2710
|
+
* whole-object classifier ROI so the in-process native fetch/encode stays cheap.
|
|
2711
|
+
*/
|
|
2712
|
+
var NATIVE_CHILD_CROP_MAX_WIDTH = 1920;
|
|
2713
|
+
/**
|
|
2714
|
+
* Resolve `bbox` (a parent-crop PIXEL rectangle `[x1,y1,x2,y2]`) at native
|
|
2715
|
+
* resolution via `provider`, returning a JPEG crop + its native dimensions.
|
|
2716
|
+
* Returns `null` on a degenerate ROI or any provider miss (so the caller uses
|
|
2717
|
+
* the downscaled tile crop instead). Pure w.r.t. its inputs; the only effect is
|
|
2718
|
+
* the sharp encode of the returned pixels.
|
|
2719
|
+
*/
|
|
2720
|
+
async function nativeChildCrop(provider, bbox, imageWidth, imageHeight, maxWidth) {
|
|
2721
|
+
if (!(imageWidth > 0) || !(imageHeight > 0)) return null;
|
|
2722
|
+
const x1 = Math.max(0, bbox[0]);
|
|
2723
|
+
const y1 = Math.max(0, bbox[1]);
|
|
2724
|
+
const x2 = Math.min(imageWidth, bbox[2]);
|
|
2725
|
+
const y2 = Math.min(imageHeight, bbox[3]);
|
|
2726
|
+
const w = x2 - x1;
|
|
2727
|
+
const h = y2 - y1;
|
|
2728
|
+
if (w < 1 || h < 1) return null;
|
|
2729
|
+
const native = await provider({
|
|
2730
|
+
x: x1 / imageWidth,
|
|
2731
|
+
y: y1 / imageHeight,
|
|
2732
|
+
w: w / imageWidth,
|
|
2733
|
+
h: h / imageHeight
|
|
2734
|
+
}, maxWidth);
|
|
2735
|
+
if (!native || native.width < 2 || native.height < 2 || native.bytes.length < native.width * native.height * 3) return null;
|
|
2736
|
+
return {
|
|
2737
|
+
jpeg: await sharp(Buffer.from(native.bytes), { raw: {
|
|
2738
|
+
width: native.width,
|
|
2739
|
+
height: native.height,
|
|
2740
|
+
channels: 3
|
|
2741
|
+
} }).jpeg({ quality: 90 }).toBuffer(),
|
|
2742
|
+
width: native.width,
|
|
2743
|
+
height: native.height
|
|
2744
|
+
};
|
|
2745
|
+
}
|
|
2746
|
+
//#endregion
|
|
3070
2747
|
//#region src/detection-pipeline/pipeline/plate-deskew.ts
|
|
3071
2748
|
/**
|
|
3072
2749
|
* Plate-crop DESKEW — rotation rectification for oblique licence-plate crops.
|
|
@@ -3555,171 +3232,74 @@ function buildFrameResult(input) {
|
|
|
3555
3232
|
if (Object.keys(cleanDebug).length > 0) return {
|
|
3556
3233
|
...base,
|
|
3557
3234
|
debug: cleanDebug
|
|
3558
|
-
};
|
|
3559
|
-
}
|
|
3560
|
-
return base;
|
|
3561
|
-
};
|
|
3562
|
-
const detections = [];
|
|
3563
|
-
for (const d of input.firstLevel) detections.push(toImmutableObject(d));
|
|
3564
|
-
for (const d of input.details) detections.push(toImmutableObject(d));
|
|
3565
|
-
const frameDebug = {
|
|
3566
|
-
totalInferenceMs: input.totalMs,
|
|
3567
|
-
stepTimings: input.stepTimings,
|
|
3568
|
-
engine: input.engine,
|
|
3569
|
-
pipelineId: input.pipelineId,
|
|
3570
|
-
...input.poolTimings ? {
|
|
3571
|
-
preprocessMs: input.poolTimings.preprocessMs,
|
|
3572
|
-
predictMs: input.poolTimings.predictMs,
|
|
3573
|
-
batchSize: input.poolTimings.batchSize
|
|
3574
|
-
} : {}
|
|
3575
|
-
};
|
|
3576
|
-
return {
|
|
3577
|
-
kind: "frame",
|
|
3578
|
-
frameId: input.frameId ?? generateFrameId(),
|
|
3579
|
-
deviceId: input.deviceId,
|
|
3580
|
-
timestamp: input.timestamp,
|
|
3581
|
-
width: input.width,
|
|
3582
|
-
height: input.height,
|
|
3583
|
-
detections,
|
|
3584
|
-
debug: frameDebug
|
|
3585
|
-
};
|
|
3586
|
-
}
|
|
3587
|
-
function bboxTupleToRect(bbox) {
|
|
3588
|
-
const [x1, y1, x2, y2] = bbox;
|
|
3589
|
-
return {
|
|
3590
|
-
x: x1,
|
|
3591
|
-
y: y1,
|
|
3592
|
-
width: x2 - x1,
|
|
3593
|
-
height: y2 - y1
|
|
3594
|
-
};
|
|
3595
|
-
}
|
|
3596
|
-
function pruneUndefined(obj) {
|
|
3597
|
-
const out = {};
|
|
3598
|
-
for (const [k, v] of Object.entries(obj)) if (v !== void 0) out[k] = v;
|
|
3599
|
-
return out;
|
|
3600
|
-
}
|
|
3601
|
-
function generateFrameId() {
|
|
3602
|
-
return `frame-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
3603
|
-
}
|
|
3604
|
-
/** Seed a MutableObjectDetection from a root-level detector output. */
|
|
3605
|
-
function toMutableRootDetection(det, rootStep, idGen, rootStepLatencyMs) {
|
|
3606
|
-
return {
|
|
3607
|
-
id: idGen.next(),
|
|
3608
|
-
kind: "first-level",
|
|
3609
|
-
macroClass: det.class,
|
|
3610
|
-
originalClass: det.class,
|
|
3611
|
-
score: det.score,
|
|
3612
|
-
bbox: [...det.bbox],
|
|
3613
|
-
ownLabels: [],
|
|
3614
|
-
alternateLabels: {},
|
|
3615
|
-
stepLatencyMs: { [rootStep.stepId]: rootStepLatencyMs },
|
|
3616
|
-
modelIds: { [rootStep.stepId]: rootStep.modelId },
|
|
3617
|
-
extractMode: "full-frame",
|
|
3618
|
-
...det.mask !== void 0 ? {
|
|
3619
|
-
mask: det.mask,
|
|
3620
|
-
maskWidth: det.maskWidth,
|
|
3621
|
-
maskHeight: det.maskHeight
|
|
3622
|
-
} : {},
|
|
3623
|
-
...det.landmarks !== void 0 ? { landmarks: det.landmarks } : {}
|
|
3624
|
-
};
|
|
3625
|
-
}
|
|
3626
|
-
//#endregion
|
|
3627
|
-
//#region src/detection-pipeline/pipeline/execution-trace.ts
|
|
3628
|
-
var traceCounter = 0;
|
|
3629
|
-
var ExecutionTraceBuilder = class {
|
|
3630
|
-
verbosity;
|
|
3631
|
-
deviceId;
|
|
3632
|
-
frameWidth;
|
|
3633
|
-
frameHeight;
|
|
3634
|
-
engineRuntime;
|
|
3635
|
-
steps = [];
|
|
3636
|
-
startTime;
|
|
3637
|
-
constructor(verbosity, deviceId, frameWidth, frameHeight, engineRuntime) {
|
|
3638
|
-
this.verbosity = verbosity;
|
|
3639
|
-
this.deviceId = deviceId;
|
|
3640
|
-
this.frameWidth = frameWidth;
|
|
3641
|
-
this.frameHeight = frameHeight;
|
|
3642
|
-
this.engineRuntime = engineRuntime;
|
|
3643
|
-
this.startTime = Date.now();
|
|
3644
|
-
}
|
|
3645
|
-
/** Whether trace collection is active. */
|
|
3646
|
-
get isActive() {
|
|
3647
|
-
return this.verbosity !== "off";
|
|
3648
|
-
}
|
|
3649
|
-
/**
|
|
3650
|
-
* Record a completed step execution.
|
|
3651
|
-
* No-op if verbosity is 'off'.
|
|
3652
|
-
*/
|
|
3653
|
-
addStep(params) {
|
|
3654
|
-
if (this.verbosity === "off") return;
|
|
3655
|
-
const totalMs = params.preprocessMs + params.inferenceMs + params.postprocessMs;
|
|
3656
|
-
const trace = {
|
|
3657
|
-
stepId: params.stepId,
|
|
3658
|
-
modelId: params.modelId,
|
|
3659
|
-
slot: params.slot,
|
|
3660
|
-
postprocessor: params.postprocessor,
|
|
3661
|
-
preprocessMs: Math.round(params.preprocessMs * 100) / 100,
|
|
3662
|
-
inferenceMs: Math.round(params.inferenceMs * 100) / 100,
|
|
3663
|
-
postprocessMs: Math.round(params.postprocessMs * 100) / 100,
|
|
3664
|
-
totalMs: Math.round(totalMs * 100) / 100,
|
|
3665
|
-
inputType: params.inputType,
|
|
3666
|
-
inputSize: {
|
|
3667
|
-
width: params.inputWidth,
|
|
3668
|
-
height: params.inputHeight
|
|
3669
|
-
},
|
|
3670
|
-
parentDetection: this.verbosity === "full" ? params.parentDetection : void 0,
|
|
3671
|
-
outputKind: params.output.kind,
|
|
3672
|
-
outputCount: countOutput(params.output),
|
|
3673
|
-
topResult: this.verbosity === "full" ? formatTopResult(params.output) : void 0,
|
|
3674
|
-
error: params.error
|
|
3675
|
-
};
|
|
3676
|
-
this.steps.push(trace);
|
|
3677
|
-
}
|
|
3678
|
-
/**
|
|
3679
|
-
* Finalize and return the complete execution trace.
|
|
3680
|
-
* Returns null if verbosity is 'off'.
|
|
3681
|
-
*/
|
|
3682
|
-
build(detectionCount) {
|
|
3683
|
-
if (this.verbosity === "off") return null;
|
|
3684
|
-
const totalMs = Date.now() - this.startTime;
|
|
3685
|
-
return {
|
|
3686
|
-
traceId: `trace-${Date.now()}-${++traceCounter}`,
|
|
3687
|
-
deviceId: this.deviceId,
|
|
3688
|
-
timestamp: this.startTime,
|
|
3689
|
-
frameSize: {
|
|
3690
|
-
width: this.frameWidth,
|
|
3691
|
-
height: this.frameHeight
|
|
3692
|
-
},
|
|
3693
|
-
engineRuntime: this.engineRuntime,
|
|
3694
|
-
steps: this.steps,
|
|
3695
|
-
totalMs,
|
|
3696
|
-
detectionCount
|
|
3697
|
-
};
|
|
3698
|
-
}
|
|
3699
|
-
};
|
|
3700
|
-
function countOutput(output) {
|
|
3701
|
-
switch (output.kind) {
|
|
3702
|
-
case "detections": return output.detections.length;
|
|
3703
|
-
case "classifications": return output.classifications.length;
|
|
3704
|
-
case "embedding": return 1;
|
|
3705
|
-
case "text": return output.text.length > 0 ? 1 : 0;
|
|
3706
|
-
case "mask": return 1;
|
|
3707
|
-
}
|
|
3708
|
-
}
|
|
3709
|
-
function formatTopResult(output) {
|
|
3710
|
-
switch (output.kind) {
|
|
3711
|
-
case "detections": {
|
|
3712
|
-
const top = output.detections[0];
|
|
3713
|
-
return top ? `${top.class} ${top.score.toFixed(2)}` : void 0;
|
|
3714
|
-
}
|
|
3715
|
-
case "classifications": {
|
|
3716
|
-
const top = output.classifications[0];
|
|
3717
|
-
return top ? `${top.class} ${top.score.toFixed(2)}` : void 0;
|
|
3235
|
+
};
|
|
3718
3236
|
}
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3237
|
+
return base;
|
|
3238
|
+
};
|
|
3239
|
+
const detections = [];
|
|
3240
|
+
for (const d of input.firstLevel) detections.push(toImmutableObject(d));
|
|
3241
|
+
for (const d of input.details) detections.push(toImmutableObject(d));
|
|
3242
|
+
const frameDebug = {
|
|
3243
|
+
totalInferenceMs: input.totalMs,
|
|
3244
|
+
stepTimings: input.stepTimings,
|
|
3245
|
+
engine: input.engine,
|
|
3246
|
+
pipelineId: input.pipelineId,
|
|
3247
|
+
...input.poolTimings ? {
|
|
3248
|
+
preprocessMs: input.poolTimings.preprocessMs,
|
|
3249
|
+
predictMs: input.poolTimings.predictMs,
|
|
3250
|
+
batchSize: input.poolTimings.batchSize
|
|
3251
|
+
} : {}
|
|
3252
|
+
};
|
|
3253
|
+
return {
|
|
3254
|
+
kind: "frame",
|
|
3255
|
+
frameId: input.frameId ?? generateFrameId(),
|
|
3256
|
+
deviceId: input.deviceId,
|
|
3257
|
+
timestamp: input.timestamp,
|
|
3258
|
+
width: input.width,
|
|
3259
|
+
height: input.height,
|
|
3260
|
+
detections,
|
|
3261
|
+
...input.discarded !== void 0 && input.discarded.length > 0 ? { discarded: input.discarded } : {},
|
|
3262
|
+
debug: frameDebug
|
|
3263
|
+
};
|
|
3264
|
+
}
|
|
3265
|
+
function bboxTupleToRect(bbox) {
|
|
3266
|
+
const [x1, y1, x2, y2] = bbox;
|
|
3267
|
+
return {
|
|
3268
|
+
x: x1,
|
|
3269
|
+
y: y1,
|
|
3270
|
+
width: x2 - x1,
|
|
3271
|
+
height: y2 - y1
|
|
3272
|
+
};
|
|
3273
|
+
}
|
|
3274
|
+
function pruneUndefined(obj) {
|
|
3275
|
+
const out = {};
|
|
3276
|
+
for (const [k, v] of Object.entries(obj)) if (v !== void 0) out[k] = v;
|
|
3277
|
+
return out;
|
|
3278
|
+
}
|
|
3279
|
+
function generateFrameId() {
|
|
3280
|
+
return `frame-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
3281
|
+
}
|
|
3282
|
+
/** Seed a MutableObjectDetection from a root-level detector output. */
|
|
3283
|
+
function toMutableRootDetection(det, rootStep, idGen, rootStepLatencyMs) {
|
|
3284
|
+
return {
|
|
3285
|
+
id: idGen.next(),
|
|
3286
|
+
kind: "first-level",
|
|
3287
|
+
macroClass: det.class,
|
|
3288
|
+
originalClass: det.class,
|
|
3289
|
+
score: det.score,
|
|
3290
|
+
bbox: [...det.bbox],
|
|
3291
|
+
ownLabels: [],
|
|
3292
|
+
alternateLabels: {},
|
|
3293
|
+
stepLatencyMs: { [rootStep.stepId]: rootStepLatencyMs },
|
|
3294
|
+
modelIds: { [rootStep.stepId]: rootStep.modelId },
|
|
3295
|
+
extractMode: "full-frame",
|
|
3296
|
+
...det.mask !== void 0 ? {
|
|
3297
|
+
mask: det.mask,
|
|
3298
|
+
maskWidth: det.maskWidth,
|
|
3299
|
+
maskHeight: det.maskHeight
|
|
3300
|
+
} : {},
|
|
3301
|
+
...det.landmarks !== void 0 ? { landmarks: det.landmarks } : {}
|
|
3302
|
+
};
|
|
3723
3303
|
}
|
|
3724
3304
|
//#endregion
|
|
3725
3305
|
//#region src/detection-pipeline/pipeline/executor.ts
|
|
@@ -3871,12 +3451,14 @@ var PipelineExecutor = class {
|
|
|
3871
3451
|
idGen,
|
|
3872
3452
|
details
|
|
3873
3453
|
};
|
|
3454
|
+
let discarded;
|
|
3874
3455
|
for (const rootStep of tree.roots) {
|
|
3875
3456
|
if (debug) console.log(`[executor] rootStep=${rootStep.stepId} settings=${JSON.stringify(rootStep.settings ?? {})}`);
|
|
3876
3457
|
const cropPrep = rootStep.definition.extractMode === "crop-zone" && cropZoneBbox !== void 0 ? await this.prepareCropZoneInput(cropZoneBbox, fullFrameJpegProvider, imageWidth, imageHeight, deviceId) : null;
|
|
3877
3458
|
const rootStart = Date.now();
|
|
3878
3459
|
const rawRootOutput = await this.executeStep(rootStep, cropPrep?.input ?? rootInput, cropPrep?.width ?? imageWidth, cropPrep?.height ?? imageHeight, "full-frame", void 0, void 0, traceBuilder, stepTimings, poolAgg);
|
|
3879
3460
|
const rootMs = Date.now() - rootStart;
|
|
3461
|
+
if (rawRootOutput === null) continue;
|
|
3880
3462
|
const rootOutput = cropPrep && rawRootOutput.kind === "detections" ? reprojectCropZoneDetections(rawRootOutput, cropPrep.origin) : rawRootOutput;
|
|
3881
3463
|
if (rootOutput.kind !== "detections") {
|
|
3882
3464
|
if (isEnrichmentOutput(rootOutput)) {
|
|
@@ -3906,10 +3488,35 @@ var PipelineExecutor = class {
|
|
|
3906
3488
|
if (mapped) {
|
|
3907
3489
|
mutable.originalClass = det.class;
|
|
3908
3490
|
mutable.macroClass = mapped;
|
|
3909
|
-
} else if (!rootStep.definition.classMap.preserveOriginal)
|
|
3491
|
+
} else if (!rootStep.definition.classMap.preserveOriginal) {
|
|
3492
|
+
(discarded ??= []).push({
|
|
3493
|
+
bbox: [
|
|
3494
|
+
mutable.bbox[0],
|
|
3495
|
+
mutable.bbox[1],
|
|
3496
|
+
mutable.bbox[2],
|
|
3497
|
+
mutable.bbox[3]
|
|
3498
|
+
],
|
|
3499
|
+
macroClass: mutable.macroClass,
|
|
3500
|
+
score: mutable.score,
|
|
3501
|
+
reason: "class-filter"
|
|
3502
|
+
});
|
|
3503
|
+
continue;
|
|
3504
|
+
}
|
|
3910
3505
|
}
|
|
3911
|
-
|
|
3912
|
-
|
|
3506
|
+
const filterVerdict = this.macroFilterVerdict(mutable.macroClass, mutable.score, rootStep.settings);
|
|
3507
|
+
if (filterVerdict !== "pass") {
|
|
3508
|
+
(discarded ??= []).push({
|
|
3509
|
+
bbox: [
|
|
3510
|
+
mutable.bbox[0],
|
|
3511
|
+
mutable.bbox[1],
|
|
3512
|
+
mutable.bbox[2],
|
|
3513
|
+
mutable.bbox[3]
|
|
3514
|
+
],
|
|
3515
|
+
macroClass: mutable.macroClass,
|
|
3516
|
+
score: mutable.score,
|
|
3517
|
+
reason: filterVerdict
|
|
3518
|
+
});
|
|
3519
|
+
if (debug) console.log(`[executor] drop det macro=${mutable.macroClass} score=${mutable.score} (${filterVerdict})`);
|
|
3913
3520
|
continue;
|
|
3914
3521
|
}
|
|
3915
3522
|
const fullFrameBar = rootStep.settings?.["fullFrameGuardMaxConfidence"];
|
|
@@ -3928,9 +3535,20 @@ var PipelineExecutor = class {
|
|
|
3928
3535
|
hardAreaRatio: typeof fullFrameHardArea === "number" ? fullFrameHardArea : void 0
|
|
3929
3536
|
}
|
|
3930
3537
|
});
|
|
3538
|
+
(discarded ??= []).push({
|
|
3539
|
+
bbox: [
|
|
3540
|
+
mutable.bbox[0],
|
|
3541
|
+
mutable.bbox[1],
|
|
3542
|
+
mutable.bbox[2],
|
|
3543
|
+
mutable.bbox[3]
|
|
3544
|
+
],
|
|
3545
|
+
macroClass: mutable.macroClass,
|
|
3546
|
+
score: mutable.score,
|
|
3547
|
+
reason: "full-frame-guard"
|
|
3548
|
+
});
|
|
3931
3549
|
continue;
|
|
3932
3550
|
}
|
|
3933
|
-
if (isFullFrameSurvivor(mutable.bbox, imageWidth, imageHeight, mutable.score, typeof fullFrameBar === "number" ? fullFrameBar : void 0)) this.opts.logger?.
|
|
3551
|
+
if (isFullFrameSurvivor(mutable.bbox, imageWidth, imageHeight, mutable.score, typeof fullFrameBar === "number" ? fullFrameBar : void 0)) this.opts.logger?.info("full-frame box SURVIVED the guard on score", {
|
|
3934
3552
|
tags: { deviceId },
|
|
3935
3553
|
meta: {
|
|
3936
3554
|
macroClass: mutable.macroClass,
|
|
@@ -3957,6 +3575,7 @@ var PipelineExecutor = class {
|
|
|
3957
3575
|
firstLevel.push(mutable);
|
|
3958
3576
|
}
|
|
3959
3577
|
}
|
|
3578
|
+
const totalMs = Date.now() - startMs;
|
|
3960
3579
|
return {
|
|
3961
3580
|
result: buildFrameResult({
|
|
3962
3581
|
deviceId,
|
|
@@ -3964,8 +3583,9 @@ var PipelineExecutor = class {
|
|
|
3964
3583
|
height: imageHeight,
|
|
3965
3584
|
firstLevel,
|
|
3966
3585
|
details,
|
|
3586
|
+
discarded,
|
|
3967
3587
|
stepTimings,
|
|
3968
|
-
totalMs
|
|
3588
|
+
totalMs,
|
|
3969
3589
|
engine: this.opts.engineRuntime,
|
|
3970
3590
|
debug,
|
|
3971
3591
|
timestamp: Date.now(),
|
|
@@ -4070,13 +3690,40 @@ var PipelineExecutor = class {
|
|
|
4070
3690
|
});
|
|
4071
3691
|
throw new Error(`Inference failed for step "${step.stepId}": ${errorMsg}`, { cause: err });
|
|
4072
3692
|
}
|
|
3693
|
+
if (engineOutput.dropped === true) {
|
|
3694
|
+
stepTimings.push({
|
|
3695
|
+
source: step.stepId,
|
|
3696
|
+
modelId: step.modelId,
|
|
3697
|
+
ms: Date.now() - preprocessStart,
|
|
3698
|
+
detectionCount: 0
|
|
3699
|
+
});
|
|
3700
|
+
if (traceBuilder.isActive) traceBuilder.addStep({
|
|
3701
|
+
stepId: step.stepId,
|
|
3702
|
+
modelId: step.modelId,
|
|
3703
|
+
slot: step.definition.slot,
|
|
3704
|
+
postprocessor: step.definition.postprocessor,
|
|
3705
|
+
preprocessMs,
|
|
3706
|
+
inferenceMs: Date.now() - inferenceStart,
|
|
3707
|
+
postprocessMs: 0,
|
|
3708
|
+
inputType,
|
|
3709
|
+
inputWidth,
|
|
3710
|
+
inputHeight,
|
|
3711
|
+
parentDetection: parentClass,
|
|
3712
|
+
output: {
|
|
3713
|
+
kind: "detections",
|
|
3714
|
+
detections: []
|
|
3715
|
+
},
|
|
3716
|
+
error: `shed:${engineOutput.shedReason ?? "pool-overload"}`
|
|
3717
|
+
});
|
|
3718
|
+
return null;
|
|
3719
|
+
}
|
|
4073
3720
|
const structured = engineOutput.structured ?? {};
|
|
4074
3721
|
const inferenceMs = typeof structured.inferenceMs === "number" ? structured.inferenceMs : Date.now() - inferenceStart;
|
|
4075
3722
|
if (typeof structured.preprocessMs === "number") poolAgg.preprocessMs += structured.preprocessMs;
|
|
4076
3723
|
if (typeof structured.predictMs === "number") poolAgg.predictMs += structured.predictMs;
|
|
4077
3724
|
if (typeof structured.batchSize === "number" && structured.batchSize > poolAgg.batchSize) poolAgg.batchSize = structured.batchSize;
|
|
4078
3725
|
const postprocessStart = Date.now();
|
|
4079
|
-
const output = dispatchPostprocess(engineOutput, step.definition
|
|
3726
|
+
const output = dispatchPostprocess(engineOutput, step.definition);
|
|
4080
3727
|
const postprocessMs = Date.now() - postprocessStart;
|
|
4081
3728
|
stepTimings.push({
|
|
4082
3729
|
source: step.stepId,
|
|
@@ -4219,6 +3866,7 @@ var PipelineExecutor = class {
|
|
|
4219
3866
|
data: cropJpegBuf
|
|
4220
3867
|
}, cropW, cropH, "crop-roi", parentDetection.bbox, parentDetection.macroClass, traceBuilder, stepTimings, poolAgg);
|
|
4221
3868
|
const childMs = Date.now() - childStart;
|
|
3869
|
+
if (childOutput === null) continue;
|
|
4222
3870
|
const detailsBefore = ctx.details.length;
|
|
4223
3871
|
applyChildOutput(parentDetection, child, childOutput, childMs, ctx);
|
|
4224
3872
|
if (childOutput.kind === "detections" && child.children.length > 0) {
|
|
@@ -4252,20 +3900,25 @@ var PipelineExecutor = class {
|
|
|
4252
3900
|
* Apply the object-detection step's macro filter + per-macro
|
|
4253
3901
|
* minConfidence sliders introduced in the Phase 6 step rework.
|
|
4254
3902
|
*
|
|
3903
|
+
* Returns WHICH gate failed rather than a bare boolean so the
|
|
3904
|
+
* discard trail can label the dropped box with the honest reason
|
|
3905
|
+
* (`class-filter` vs `below-threshold`) — the two are remediated
|
|
3906
|
+
* differently (enable the class vs lower a slider).
|
|
3907
|
+
*
|
|
4255
3908
|
* Expected `settings` shape:
|
|
4256
3909
|
* - `enabledMacroClasses`: readonly string[] (e.g. ['person','vehicle','animal']).
|
|
4257
3910
|
* Empty array = all allowed (legacy behaviour).
|
|
4258
3911
|
* - `minConfidence<Macro>`: number (e.g. `minConfidencePerson: 0.5`)
|
|
4259
3912
|
*/
|
|
4260
|
-
|
|
4261
|
-
if (!settings) return
|
|
3913
|
+
macroFilterVerdict(macroClass, score, settings) {
|
|
3914
|
+
if (!settings) return "pass";
|
|
4262
3915
|
const enabled = settings["enabledMacroClasses"];
|
|
4263
3916
|
if (Array.isArray(enabled) && enabled.length > 0) {
|
|
4264
|
-
if (!enabled.includes(macroClass)) return
|
|
3917
|
+
if (!enabled.includes(macroClass)) return "class-filter";
|
|
4265
3918
|
}
|
|
4266
3919
|
const perMacroThreshold = settings[`minConfidence${capitalize(macroClass)}`];
|
|
4267
|
-
if (typeof perMacroThreshold === "number" && score < perMacroThreshold) return
|
|
4268
|
-
return
|
|
3920
|
+
if (typeof perMacroThreshold === "number" && score < perMacroThreshold) return "below-threshold";
|
|
3921
|
+
return "pass";
|
|
4269
3922
|
}
|
|
4270
3923
|
};
|
|
4271
3924
|
function capitalize(s) {
|
|
@@ -4273,52 +3926,41 @@ function capitalize(s) {
|
|
|
4273
3926
|
return s.charAt(0).toUpperCase() + s.slice(1);
|
|
4274
3927
|
}
|
|
4275
3928
|
//#endregion
|
|
4276
|
-
//#region src/detection-pipeline/pipeline/crop-
|
|
3929
|
+
//#region src/detection-pipeline/pipeline/native-crop-compose.ts
|
|
3930
|
+
/** Clamp `v` into `[lo, hi]`. */
|
|
3931
|
+
function clamp(v, lo, hi) {
|
|
3932
|
+
if (v < lo) return lo;
|
|
3933
|
+
if (v > hi) return hi;
|
|
3934
|
+
return v;
|
|
3935
|
+
}
|
|
4277
3936
|
/**
|
|
4278
|
-
*
|
|
4279
|
-
*
|
|
4280
|
-
*
|
|
4281
|
-
*
|
|
3937
|
+
* Compose a child `roi` given in the parent CROP's normalized space into the
|
|
3938
|
+
* FRAME's normalized space, given the crop's frame-space pixel rectangle and the
|
|
3939
|
+
* frame dimensions. The result is clamped to the unit square (origin in
|
|
3940
|
+
* `[0,1]`, extent bounded so `x + w ≤ 1` and `y + h ≤ 1`). Returns `null` for
|
|
3941
|
+
* degenerate inputs (non-finite frame dims, empty crop rect, or an ROI that
|
|
3942
|
+
* clamps to zero area) so the caller falls back to the downscaled tile crop.
|
|
4282
3943
|
*
|
|
4283
|
-
*
|
|
4284
|
-
* referenced geometry, or a degenerate result — in which case the caller falls
|
|
4285
|
-
* back to full-frame inference (behaviour identical to before the crop-zone
|
|
4286
|
-
* plumbing).
|
|
3944
|
+
* Pure — never mutates its arguments.
|
|
4287
3945
|
*/
|
|
4288
|
-
function
|
|
4289
|
-
if (frameWidth
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
const
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
const
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
|
|
4301
|
-
|
|
4302
|
-
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
if (point.x < minX) minX = point.x;
|
|
4307
|
-
if (point.y < minY) minY = point.y;
|
|
4308
|
-
if (point.x > maxX) maxX = point.x;
|
|
4309
|
-
if (point.y > maxY) maxY = point.y;
|
|
4310
|
-
}
|
|
4311
|
-
}
|
|
4312
|
-
if (!sawPoint) return null;
|
|
4313
|
-
const clamp01 = (v) => Math.max(0, Math.min(1, v));
|
|
4314
|
-
const bbox = [
|
|
4315
|
-
clamp01(minX) * frameWidth,
|
|
4316
|
-
clamp01(minY) * frameHeight,
|
|
4317
|
-
clamp01(maxX) * frameWidth,
|
|
4318
|
-
clamp01(maxY) * frameHeight
|
|
4319
|
-
];
|
|
4320
|
-
if (isBboxDegenerate(bbox)) return null;
|
|
4321
|
-
return bbox;
|
|
3946
|
+
function composeCropRoiToFrameNorm(roi, cropFrameSpace, frameWidth, frameHeight) {
|
|
3947
|
+
if (!(frameWidth > 0) || !(frameHeight > 0)) return null;
|
|
3948
|
+
if (!(cropFrameSpace.w > 0) || !(cropFrameSpace.h > 0)) return null;
|
|
3949
|
+
const fx = (cropFrameSpace.x + roi.x * cropFrameSpace.w) / frameWidth;
|
|
3950
|
+
const fy = (cropFrameSpace.y + roi.y * cropFrameSpace.h) / frameHeight;
|
|
3951
|
+
const fw = roi.w * cropFrameSpace.w / frameWidth;
|
|
3952
|
+
const fh = roi.h * cropFrameSpace.h / frameHeight;
|
|
3953
|
+
const x = clamp(fx, 0, 1);
|
|
3954
|
+
const y = clamp(fy, 0, 1);
|
|
3955
|
+
const w = clamp(fw, 0, 1 - x);
|
|
3956
|
+
const h = clamp(fh, 0, 1 - y);
|
|
3957
|
+
if (!(w > 0) || !(h > 0)) return null;
|
|
3958
|
+
return {
|
|
3959
|
+
x,
|
|
3960
|
+
y,
|
|
3961
|
+
w,
|
|
3962
|
+
h
|
|
3963
|
+
};
|
|
4322
3964
|
}
|
|
4323
3965
|
//#endregion
|
|
4324
3966
|
//#region src/detection-pipeline/pipeline/tree-builder.ts
|
|
@@ -4554,6 +4196,44 @@ function resolveInputSteps(steps, format, engine) {
|
|
|
4554
4196
|
};
|
|
4555
4197
|
}
|
|
4556
4198
|
//#endregion
|
|
4199
|
+
//#region src/detection-pipeline/zone-gate.ts
|
|
4200
|
+
function applyZoneRuleGate(result, zones, rules) {
|
|
4201
|
+
if (result.detections.length === 0) return result;
|
|
4202
|
+
if (zones.length === 0 || rules.length === 0) return result;
|
|
4203
|
+
const frameW = result.width;
|
|
4204
|
+
const frameH = result.height;
|
|
4205
|
+
if (frameW === 0 || frameH === 0) return result;
|
|
4206
|
+
const firstLevel = result.detections.filter((d) => d.kind === "first-level");
|
|
4207
|
+
const details = result.detections.filter((d) => d.kind === "detail");
|
|
4208
|
+
const { passed, excluded } = evaluateZoneRules(firstLevel, zones, rules, (det) => ({
|
|
4209
|
+
x: (det.bbox.x + det.bbox.width / 2) / frameW,
|
|
4210
|
+
y: (det.bbox.y + det.bbox.height / 2) / frameH
|
|
4211
|
+
}), (det) => det.macroClass);
|
|
4212
|
+
if (passed.length === firstLevel.length) return result;
|
|
4213
|
+
const passedIds = new Set(passed.map((d) => d.id));
|
|
4214
|
+
const filteredDetails = details.filter((d) => {
|
|
4215
|
+
const parentId = d.parentId;
|
|
4216
|
+
return parentId === void 0 || passedIds.has(parentId);
|
|
4217
|
+
});
|
|
4218
|
+
const zoneDiscards = excluded.map((d) => ({
|
|
4219
|
+
bbox: [
|
|
4220
|
+
d.bbox.x,
|
|
4221
|
+
d.bbox.y,
|
|
4222
|
+
d.bbox.x + d.bbox.width,
|
|
4223
|
+
d.bbox.y + d.bbox.height
|
|
4224
|
+
],
|
|
4225
|
+
macroClass: d.macroClass,
|
|
4226
|
+
score: d.score,
|
|
4227
|
+
reason: "zone-gate"
|
|
4228
|
+
}));
|
|
4229
|
+
const mergedDiscarded = [...result.discarded ?? [], ...zoneDiscards];
|
|
4230
|
+
return {
|
|
4231
|
+
...result,
|
|
4232
|
+
detections: [...passed, ...filteredDetails],
|
|
4233
|
+
...mergedDiscarded.length > 0 ? { discarded: mergedDiscarded } : {}
|
|
4234
|
+
};
|
|
4235
|
+
}
|
|
4236
|
+
//#endregion
|
|
4557
4237
|
//#region src/detection-pipeline/provider.ts
|
|
4558
4238
|
/**
|
|
4559
4239
|
* DetectionPipelineProvider — implements IPipelineExecutorProvider.
|
|
@@ -5952,19 +5632,15 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
5952
5632
|
emit(`Image: ${imageWidth}×${imageHeight}`);
|
|
5953
5633
|
}
|
|
5954
5634
|
decodeMs = performance.now() - decodeT0;
|
|
5955
|
-
const
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
backend: this.currentEngine.backend,
|
|
5965
|
-
device: this.currentEngine.device ?? null
|
|
5966
|
-
};
|
|
5967
|
-
const benchmarkSteps = this.inputStepsToPipelineSteps(input.steps, resolveFormat, resolveEngine);
|
|
5635
|
+
const dispatchResolution = this.resolveStepsForDispatch({
|
|
5636
|
+
steps: input.steps,
|
|
5637
|
+
deviceKey: input.deviceKey,
|
|
5638
|
+
engineOverride: input.engine,
|
|
5639
|
+
deviceId: input.deviceId,
|
|
5640
|
+
plane: input.plane
|
|
5641
|
+
});
|
|
5642
|
+
const dispatchDeviceKey = dispatchResolution.deviceKey;
|
|
5643
|
+
const benchmarkSteps = dispatchResolution.steps;
|
|
5968
5644
|
const stepsSignature = JSON.stringify(benchmarkSteps.map((s) => ({
|
|
5969
5645
|
id: s.addonId,
|
|
5970
5646
|
settings: s.settings ?? {}
|
|
@@ -6056,8 +5732,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6056
5732
|
logger: this.log
|
|
6057
5733
|
});
|
|
6058
5734
|
this.executor = executor;
|
|
6059
|
-
const dispatchEngine =
|
|
6060
|
-
const dispatchFactory =
|
|
5735
|
+
const dispatchEngine = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey) : runEngine;
|
|
5736
|
+
const dispatchFactory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
|
|
6061
5737
|
const needed = enabledSteps.filter((s) => !dispatchFactory.isLoadedWithModel(s.addonId, s.modelId));
|
|
6062
5738
|
if (needed.length > 0) {
|
|
6063
5739
|
this.log.info("Benchmark: models to load", { meta: {
|
|
@@ -6306,6 +5982,142 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6306
5982
|
fn();
|
|
6307
5983
|
}
|
|
6308
5984
|
/**
|
|
5985
|
+
* Resolve a dispatch's step tree against its target engine, with the
|
|
5986
|
+
* CAPABILITY GATE + node-local engine fallback (cam-615, 2026-08-01;
|
|
5987
|
+
* per-step split 2026-08-01 evening).
|
|
5988
|
+
*
|
|
5989
|
+
* A per-call `deviceKey` (orchestrator device routing / detail-plane step
|
|
5990
|
+
* jump) selects an engine whose format every step EXECUTING IN THIS CALL
|
|
5991
|
+
* must actually ship a model build for. When such a step's catalog has
|
|
5992
|
+
* ZERO builds for that format (so not even `resolveModelForFormat`'s
|
|
5993
|
+
* substitution can rescue it), dispatching anyway means every frame
|
|
5994
|
+
* fails — before this gate it failed as a doomed 3-retry model download,
|
|
5995
|
+
* ~5.5s per frame, forever.
|
|
5996
|
+
*
|
|
5997
|
+
* The gated set is per-plane — the ROOT decides the dispatch device, a
|
|
5998
|
+
* child never bounces it (operator requirement: devices compensate for
|
|
5999
|
+
* each other's missing steps; the original whole-tree gate let ONE
|
|
6000
|
+
* tflite-less child knock the root off an otherwise-capable Coral):
|
|
6001
|
+
*
|
|
6002
|
+
* - `'frame'` (the live per-frame path) executes only root-plane steps,
|
|
6003
|
+
* so only those gate the device. Detail children that can NEVER run on
|
|
6004
|
+
* this device are pruned from THIS dispatch (so `ensureModelsForSteps`
|
|
6005
|
+
* doesn't fail the call on a format build that does not exist) and
|
|
6006
|
+
* logged once — NOT dropped work: each runs later as its own per-track
|
|
6007
|
+
* detail dispatch, where the runner's device-jump resolver
|
|
6008
|
+
* (`resolveStepDevice`) places it on a same-node device that can run
|
|
6009
|
+
* it, or warns no-candidate loudly.
|
|
6010
|
+
* - any other plane (detail subtrees, benchmark, batch) executes the
|
|
6011
|
+
* whole passed tree inline on ONE pool — a single call cannot split
|
|
6012
|
+
* devices — so the whole-tree gate stays.
|
|
6013
|
+
*
|
|
6014
|
+
* Fallback is strictly NODE-LOCAL and ordered: the node's own selected
|
|
6015
|
+
* engine (or the benchmark's explicit `engineOverride`) is the one and only
|
|
6016
|
+
* fallback — frames never migrate to another node from here (that is the
|
|
6017
|
+
* orchestrator's tier). The fallback is logged ONCE per distinct
|
|
6018
|
+
* (deviceKey, steps, format) signature, with `tags: { deviceId }` when the
|
|
6019
|
+
* dispatch is camera-scoped.
|
|
6020
|
+
*
|
|
6021
|
+
* When the node's own engine ALSO cannot run a step, this throws — fast,
|
|
6022
|
+
* before any download attempt — and logs the full picture once (model,
|
|
6023
|
+
* its available formats, both engines tried). A genuinely impossible
|
|
6024
|
+
* configuration must stay loud, not vanish into a silent fallback chain.
|
|
6025
|
+
*/
|
|
6026
|
+
resolveStepsForDispatch(args) {
|
|
6027
|
+
const { deviceKey, engineOverride, deviceId } = args;
|
|
6028
|
+
const nodeFormat = engineOverride?.format ?? this.currentEngine.format;
|
|
6029
|
+
const nodeEngine = engineOverride ? {
|
|
6030
|
+
backend: engineOverride.backend,
|
|
6031
|
+
device: engineOverride.device ?? null
|
|
6032
|
+
} : {
|
|
6033
|
+
backend: this.currentEngine.backend,
|
|
6034
|
+
device: this.currentEngine.device ?? null
|
|
6035
|
+
};
|
|
6036
|
+
const nodeEngineLabel = `${nodeEngine.backend}/${nodeEngine.device ?? "default"} (${nodeFormat})`;
|
|
6037
|
+
const deviceEngine = deviceKey ? resolveDeviceEngine(deviceKey) : void 0;
|
|
6038
|
+
if (deviceKey === void 0 || deviceEngine === void 0) {
|
|
6039
|
+
const steps = this.inputStepsToPipelineSteps(args.steps, nodeFormat, nodeEngine);
|
|
6040
|
+
this.assertStepsRunnable(steps, nodeFormat, deviceId, [nodeEngineLabel]);
|
|
6041
|
+
return {
|
|
6042
|
+
deviceKey: void 0,
|
|
6043
|
+
steps
|
|
6044
|
+
};
|
|
6045
|
+
}
|
|
6046
|
+
const deviceFormat = deviceEngine.format;
|
|
6047
|
+
const deviceStepEngine = {
|
|
6048
|
+
backend: deviceEngine.backend,
|
|
6049
|
+
device: deviceEngine.device ?? null
|
|
6050
|
+
};
|
|
6051
|
+
const steps = this.inputStepsToPipelineSteps(args.steps, deviceFormat, deviceStepEngine);
|
|
6052
|
+
const framePlane = args.plane === "frame";
|
|
6053
|
+
const zeroBuild = collectZeroBuildIssues(framePlane ? collectFramePlaneSteps(steps, isDetailPlaneStep) : flattenSteps(steps), deviceFormat);
|
|
6054
|
+
if (zeroBuild.length === 0) {
|
|
6055
|
+
if (!framePlane) return {
|
|
6056
|
+
deviceKey,
|
|
6057
|
+
steps
|
|
6058
|
+
};
|
|
6059
|
+
const pruneResult = pruneUnrunnableDetailSteps(steps, isDetailPlaneStep, (addonId) => stepHasZeroBuildsFor(addonId, deviceFormat));
|
|
6060
|
+
if (pruneResult.prunedAddonIds.length > 0) {
|
|
6061
|
+
const prunedList = pruneResult.prunedAddonIds.join(",");
|
|
6062
|
+
this.logLiveDispatchIssueOnce(`detail-defer:${deviceKey}:${deviceFormat}:${prunedList}`, () => this.log.info("Detail steps have no model build for the dispatch device format — deferred to the detail-plane device jump", {
|
|
6063
|
+
...deviceId !== void 0 ? { tags: { deviceId } } : {},
|
|
6064
|
+
meta: {
|
|
6065
|
+
deviceKey,
|
|
6066
|
+
deviceFormat,
|
|
6067
|
+
steps: [...pruneResult.prunedAddonIds]
|
|
6068
|
+
}
|
|
6069
|
+
}));
|
|
6070
|
+
}
|
|
6071
|
+
return {
|
|
6072
|
+
deviceKey,
|
|
6073
|
+
steps: pruneResult.steps
|
|
6074
|
+
};
|
|
6075
|
+
}
|
|
6076
|
+
const blockedSteps = zeroBuild.map((issue) => issue.addonId);
|
|
6077
|
+
this.logLiveDispatchIssueOnce(`capability-fallback:${deviceKey}:${deviceFormat}:${blockedSteps.join(",")}`, () => this.log.warn("Device pool cannot run step (no model build for its format) — falling back to the node-local default engine", {
|
|
6078
|
+
...deviceId !== void 0 ? { tags: { deviceId } } : {},
|
|
6079
|
+
meta: {
|
|
6080
|
+
deviceKey,
|
|
6081
|
+
deviceFormat,
|
|
6082
|
+
steps: blockedSteps,
|
|
6083
|
+
fallbackEngine: nodeEngineLabel
|
|
6084
|
+
}
|
|
6085
|
+
}));
|
|
6086
|
+
const fallbackSteps = this.inputStepsToPipelineSteps(args.steps, nodeFormat, nodeEngine);
|
|
6087
|
+
this.assertStepsRunnable(fallbackSteps, nodeFormat, deviceId, [`${deviceKey} (${deviceFormat})`, nodeEngineLabel]);
|
|
6088
|
+
return {
|
|
6089
|
+
deviceKey: void 0,
|
|
6090
|
+
steps: fallbackSteps
|
|
6091
|
+
};
|
|
6092
|
+
}
|
|
6093
|
+
/**
|
|
6094
|
+
* The loud, fast half of the capability gate: throw when any ENABLED step
|
|
6095
|
+
* has ZERO model builds for `format` — BEFORE any model download runs. The
|
|
6096
|
+
* error names the step, its chosen model, the formats the catalog actually
|
|
6097
|
+
* ships, and every engine that was tried, and is logged ONCE per distinct
|
|
6098
|
+
* signature (the throw itself still surfaces per dispatch, but costs
|
|
6099
|
+
* microseconds instead of a 3-retry download backoff).
|
|
6100
|
+
*/
|
|
6101
|
+
assertStepsRunnable(steps, format, deviceId, enginesTried) {
|
|
6102
|
+
const zeroBuild = collectZeroBuildIssues(flattenSteps(steps), format);
|
|
6103
|
+
if (zeroBuild.length === 0) return;
|
|
6104
|
+
const detail = zeroBuild.map((issue) => {
|
|
6105
|
+
const step = flattenSteps(steps).find((s) => s.addonId === issue.addonId);
|
|
6106
|
+
const formats = availableFormatsForStep(issue.addonId);
|
|
6107
|
+
return `step "${issue.addonId}" (model "${step?.modelId ?? "unknown"}") has no ${format} build (available: ${formats.join(", ") || "none"})`;
|
|
6108
|
+
}).join("; ");
|
|
6109
|
+
const message = `Pipeline dispatch impossible on this node: ${detail}. Engines tried: ${enginesTried.join(" → ")}.`;
|
|
6110
|
+
this.logLiveDispatchIssueOnce(`unrunnable:${format}:${zeroBuild.map((i) => i.addonId).join(",")}`, () => this.log.error("Pipeline step cannot run on any node-local engine", {
|
|
6111
|
+
...deviceId !== void 0 ? { tags: { deviceId } } : {},
|
|
6112
|
+
meta: {
|
|
6113
|
+
format,
|
|
6114
|
+
enginesTried: [...enginesTried],
|
|
6115
|
+
detail
|
|
6116
|
+
}
|
|
6117
|
+
}));
|
|
6118
|
+
throw new Error(message);
|
|
6119
|
+
}
|
|
6120
|
+
/**
|
|
6309
6121
|
* Single-flight gate around `ensureModelsForSteps`. Concurrent
|
|
6310
6122
|
* callers (every camera that fires motion in the same window calls
|
|
6311
6123
|
* `runPipeline` → `ensureModelsForSteps`) share the same in-flight
|
|
@@ -6332,6 +6144,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6332
6144
|
for (const step of needed) {
|
|
6333
6145
|
const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
|
|
6334
6146
|
if (modelEntry && !isModelDownloaded(this.modelsDir, modelEntry, format)) {
|
|
6147
|
+
if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
|
|
6335
6148
|
this.log.info("Downloading model for step", { meta: {
|
|
6336
6149
|
modelId: step.modelId,
|
|
6337
6150
|
format,
|
|
@@ -6449,19 +6262,16 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6449
6262
|
}
|
|
6450
6263
|
async runPipelineBatchImpl(input) {
|
|
6451
6264
|
if (input.frames.length === 0) return { results: [] };
|
|
6452
|
-
const
|
|
6453
|
-
|
|
6454
|
-
|
|
6455
|
-
|
|
6456
|
-
|
|
6457
|
-
|
|
6458
|
-
|
|
6459
|
-
|
|
6460
|
-
|
|
6461
|
-
|
|
6462
|
-
device: this.currentEngine.device ?? null
|
|
6463
|
-
};
|
|
6464
|
-
const benchmarkSteps = this.inputStepsToPipelineSteps(input.steps, resolveFormat, resolveEngine);
|
|
6265
|
+
const dispatchResolution = this.resolveStepsForDispatch({
|
|
6266
|
+
steps: input.steps,
|
|
6267
|
+
deviceKey: input.deviceKey,
|
|
6268
|
+
engineOverride: input.engine,
|
|
6269
|
+
deviceId: input.deviceId,
|
|
6270
|
+
plane: void 0
|
|
6271
|
+
});
|
|
6272
|
+
const dispatchDeviceKey = dispatchResolution.deviceKey;
|
|
6273
|
+
const resolveFormat = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey).format : input.engine?.format ?? this.currentEngine.format;
|
|
6274
|
+
const benchmarkSteps = dispatchResolution.steps;
|
|
6465
6275
|
const enabledSteps = flattenEnabledVideoSteps(benchmarkSteps);
|
|
6466
6276
|
if (enabledSteps.length === 0) throw new Error("runPipelineBatch: no enabled steps");
|
|
6467
6277
|
const rootStep = enabledSteps[0];
|
|
@@ -6483,7 +6293,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6483
6293
|
await this.ensureEngineFactory();
|
|
6484
6294
|
const restoreEngine = await this.applyEngineOverride(input.engine);
|
|
6485
6295
|
try {
|
|
6486
|
-
const factory =
|
|
6296
|
+
const factory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
|
|
6487
6297
|
if (!factory) throw new Error("runPipelineBatch: factory not initialised");
|
|
6488
6298
|
if (enabledSteps.filter((s) => !factory.isLoadedWithModel(s.addonId, s.modelId)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat);
|
|
6489
6299
|
const canFastPath = singleRoot && allRaw && uniformDims && factory.supportsBatch();
|
|
@@ -6500,7 +6310,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6500
6310
|
frame,
|
|
6501
6311
|
deviceId: input.deviceId,
|
|
6502
6312
|
sessionId: input.sessionId,
|
|
6503
|
-
deviceKey:
|
|
6313
|
+
deviceKey: dispatchDeviceKey
|
|
6504
6314
|
}))) };
|
|
6505
6315
|
const items = input.frames.map((frame) => ({
|
|
6506
6316
|
raw: Buffer.from(frame.data),
|
|
@@ -6685,31 +6495,10 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
|
|
|
6685
6495
|
* detections).
|
|
6686
6496
|
*/
|
|
6687
6497
|
gateDetectionsByZoneRules(deviceId, result) {
|
|
6688
|
-
if (deviceId <= 0
|
|
6498
|
+
if (deviceId <= 0) return result;
|
|
6689
6499
|
const proxy = this.deviceProxies.get(deviceId);
|
|
6690
6500
|
if (!proxy) return result;
|
|
6691
|
-
|
|
6692
|
-
const rules = proxy.state.zoneRules.value?.detection ?? [];
|
|
6693
|
-
if (zones.length === 0 || rules.length === 0) return result;
|
|
6694
|
-
const frameW = result.width;
|
|
6695
|
-
const frameH = result.height;
|
|
6696
|
-
if (frameW === 0 || frameH === 0) return result;
|
|
6697
|
-
const firstLevel = result.detections.filter((d) => d.kind === "first-level");
|
|
6698
|
-
const details = result.detections.filter((d) => d.kind === "detail");
|
|
6699
|
-
const { passed } = evaluateZoneRules(firstLevel, zones, rules, (det) => ({
|
|
6700
|
-
x: (det.bbox.x + det.bbox.width / 2) / frameW,
|
|
6701
|
-
y: (det.bbox.y + det.bbox.height / 2) / frameH
|
|
6702
|
-
}), (det) => det.macroClass);
|
|
6703
|
-
if (passed.length === firstLevel.length) return result;
|
|
6704
|
-
const passedIds = new Set(passed.map((d) => d.id));
|
|
6705
|
-
const filteredDetails = details.filter((d) => {
|
|
6706
|
-
const parentId = d.parentId;
|
|
6707
|
-
return parentId === void 0 || passedIds.has(parentId);
|
|
6708
|
-
});
|
|
6709
|
-
return {
|
|
6710
|
-
...result,
|
|
6711
|
-
detections: [...passed, ...filteredDetails]
|
|
6712
|
-
};
|
|
6501
|
+
return applyZoneRuleGate(result, proxy.state.zones.value?.zones ?? [], proxy.state.zoneRules.value?.detection ?? []);
|
|
6713
6502
|
}
|
|
6714
6503
|
async cacheFrameInPool(input) {
|
|
6715
6504
|
await this.ensureEngineFactory();
|
|
@@ -7299,12 +7088,49 @@ function stepsToPipelineConfig(steps, engine) {
|
|
|
7299
7088
|
} : {}
|
|
7300
7089
|
};
|
|
7301
7090
|
}
|
|
7302
|
-
/** Flatten a step tree into a flat list (DFS). */
|
|
7303
7091
|
/** Flatten a step tree into a flat list (DFS) — delegates to shared utility. */
|
|
7304
7092
|
function flattenSteps(steps) {
|
|
7305
7093
|
return flattenEnabledVideoSteps(steps);
|
|
7306
7094
|
}
|
|
7307
7095
|
/**
|
|
7096
|
+
* Catalog root-vs-detail plane of a step: a non-null `inputClasses` on the
|
|
7097
|
+
* DEFINITION marks a crop child served per-track on the detail plane —
|
|
7098
|
+
* mirrors the executor's frame-plane skip (`executeChildren`) and the
|
|
7099
|
+
* runner's `deriveDetailSteps`. Unknown addonIds (already dropped from
|
|
7100
|
+
* resolved trees) default to root-plane.
|
|
7101
|
+
*/
|
|
7102
|
+
function isDetailPlaneStep(addonId) {
|
|
7103
|
+
try {
|
|
7104
|
+
return getStepDefinition(addonId).inputClasses !== null;
|
|
7105
|
+
} catch {
|
|
7106
|
+
return false;
|
|
7107
|
+
}
|
|
7108
|
+
}
|
|
7109
|
+
/**
|
|
7110
|
+
* True when the step's FULL catalog ships no model with a build for
|
|
7111
|
+
* `format` — the single-step form of `collectZeroBuildIssues`. Unknown/custom
|
|
7112
|
+
* addonIds (no catalog to check) are never zero-build, mirroring the
|
|
7113
|
+
* collector's skip.
|
|
7114
|
+
*/
|
|
7115
|
+
function stepHasZeroBuildsFor(addonId, format) {
|
|
7116
|
+
return collectZeroBuildIssues([{ addonId }], format).length > 0;
|
|
7117
|
+
}
|
|
7118
|
+
/**
|
|
7119
|
+
* The union of model formats a step's catalog ships ANY build for — the
|
|
7120
|
+
* "Available: …" half of the capability-gate error. Unknown/custom addonIds
|
|
7121
|
+
* yield an empty list (no catalog to enumerate), never a throw.
|
|
7122
|
+
*/
|
|
7123
|
+
function availableFormatsForStep(addonId) {
|
|
7124
|
+
try {
|
|
7125
|
+
const def = getStepDefinition(addonId);
|
|
7126
|
+
const formats = /* @__PURE__ */ new Set();
|
|
7127
|
+
for (const model of def.models) for (const format of Object.keys(model.formats)) formats.add(format);
|
|
7128
|
+
return Array.from(formats).toSorted();
|
|
7129
|
+
} catch {
|
|
7130
|
+
return [];
|
|
7131
|
+
}
|
|
7132
|
+
}
|
|
7133
|
+
/**
|
|
7308
7134
|
* Return a shallow-cloned executable tree with `rootStepId`'s `settings`
|
|
7309
7135
|
* merged with `overrides` (per-device values win). Non-matching root steps
|
|
7310
7136
|
* pass through unchanged. Used by `runFrame` to apply per-device detection
|