@camstack/addon-post-analysis 1.1.25 → 1.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{dist-Csk_yJr_.mjs → dist-BqOBYSWs.mjs} +181 -8
- package/dist/{dist-CCC79h7t.js → dist-CtnFKuWh.js} +186 -7
- package/dist/embedding-encoder/index.js +1 -1
- package/dist/embedding-encoder/index.mjs +1 -1
- package/dist/enrichment-engine/index.js +2 -2
- package/dist/enrichment-engine/index.mjs +1 -1
- package/dist/{node-CIEkEv1F.js → node-BN31iDiA.js} +1 -1
- package/dist/pipeline-analytics/_stub.js +1 -1
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-Cb6Z2H5K.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-Dvz_QBOD.mjs} +3 -3
- package/dist/pipeline-analytics/{_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DtOrjE2U.mjs → _virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BXQHc7Tq.mjs} +1 -1
- package/dist/pipeline-analytics/{hostInit-BrqCuPYQ.mjs → hostInit-C7nUOwbi.mjs} +3 -3
- package/dist/pipeline-analytics/index.js +1348 -43
- package/dist/pipeline-analytics/index.mjs +1346 -41
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/dist/{resolve-frame-sKYbstL-.js → resolve-frame-BAdpVnlx.js} +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { C as
|
|
1
|
+
import { C as number, S as boolean, T as string, c as nodePin, d as videoclipsCapability, f as zoneAnalyticsCapability, g as EventCategory, h as DeviceType, i as cosineSimilarity, l as pipelineAnalyticsCapability, m as BaseAddon, n as addonWidgetsSourceCapability, o as faceGalleryCapability, p as errMsg, r as audioMetricsCapability, t as EVENT_PAD_MS, u as plateGalleryCapability, w as object, y as hydrateSchema } from "../dist-BqOBYSWs.mjs";
|
|
2
2
|
import { n as extractCrop, t as resolveFrame } from "../resolve-frame-CT1T1tWy.mjs";
|
|
3
3
|
import { FrameRingReaderCache } from "@camstack/shm-ring";
|
|
4
4
|
import sharp from "sharp";
|
|
@@ -119,6 +119,138 @@ function clusterByEmbedding(items, threshold = .5) {
|
|
|
119
119
|
})).toSorted((a, b) => b.size - a.size || b.cohesion - a.cohesion);
|
|
120
120
|
}
|
|
121
121
|
//#endregion
|
|
122
|
+
//#region src/pipeline-analytics/pipeline/key-event-scoring.ts
|
|
123
|
+
/**
|
|
124
|
+
* key-event-scoring — the SINGLE deterministic "importance" scorer for a
|
|
125
|
+
* completed track. A value in [0,1] the viewer sorts events by (events page)
|
|
126
|
+
* and a future reels feature pulls highlights from.
|
|
127
|
+
*
|
|
128
|
+
* PURE + DETERMINISTIC: no Date.now / no random. Time enters ONLY as the
|
|
129
|
+
* precomputed `durationMs` (the caller already knows a track's dwell). Given
|
|
130
|
+
* the same `ScoringInput`, `computeImportance` always returns the same
|
|
131
|
+
* `{ importance, reason }`.
|
|
132
|
+
*
|
|
133
|
+
* The score is a weighted sum of seven independent, individually-saturated
|
|
134
|
+
* signals. Every weight and knee is a NAMED EXPORT so the formula is auditable
|
|
135
|
+
* and the tests can assert exact wiring. The weights sum to exactly 1.0, so an
|
|
136
|
+
* all-signals-maxed track scores exactly 1.0 (the clamp is then a no-op) and an
|
|
137
|
+
* all-zero track scores 0.
|
|
138
|
+
*/
|
|
139
|
+
var WEIGHT_CONFIDENCE = .3;
|
|
140
|
+
var WEIGHT_CLASS = .2;
|
|
141
|
+
var WEIGHT_DWELL = .2;
|
|
142
|
+
var WEIGHT_PROXIMITY = .12;
|
|
143
|
+
var WEIGHT_TRAVEL = .08;
|
|
144
|
+
var WEIGHT_ZONE = .05;
|
|
145
|
+
var WEIGHT_IDENTITY = .05;
|
|
146
|
+
/** Dwell (ms) that saturates the dwell term. */
|
|
147
|
+
var DWELL_FULL_MS = 6e4;
|
|
148
|
+
/** Peak bbox area (as a fraction of frame area) that saturates the size term. */
|
|
149
|
+
var SIZE_FULL = .15;
|
|
150
|
+
var CLASS_RANK_VEHICLE = .8;
|
|
151
|
+
var CLASS_RANK_ANIMAL = .5;
|
|
152
|
+
var CLASS_RANK_DEFAULT = .25;
|
|
153
|
+
var PERSON_CLASSES = new Set(["person", "face"]);
|
|
154
|
+
var VEHICLE_CLASSES = new Set([
|
|
155
|
+
"vehicle",
|
|
156
|
+
"car",
|
|
157
|
+
"truck",
|
|
158
|
+
"bus"
|
|
159
|
+
]);
|
|
160
|
+
var ANIMAL_CLASSES = new Set([
|
|
161
|
+
"animal",
|
|
162
|
+
"dog",
|
|
163
|
+
"cat"
|
|
164
|
+
]);
|
|
165
|
+
/** Clamp to [0,1]. */
|
|
166
|
+
function clamp01(x) {
|
|
167
|
+
if (Number.isNaN(x)) return 0;
|
|
168
|
+
if (x < 0) return 0;
|
|
169
|
+
if (x > 1) return 1;
|
|
170
|
+
return x;
|
|
171
|
+
}
|
|
172
|
+
/** Rank a class name into [0,1] by how event-worthy it is. Case-insensitive. */
|
|
173
|
+
function classRank(className) {
|
|
174
|
+
const c = className.toLowerCase();
|
|
175
|
+
if (PERSON_CLASSES.has(c)) return 1;
|
|
176
|
+
if (VEHICLE_CLASSES.has(c)) return CLASS_RANK_VEHICLE;
|
|
177
|
+
if (ANIMAL_CLASSES.has(c)) return CLASS_RANK_ANIMAL;
|
|
178
|
+
return CLASS_RANK_DEFAULT;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Compute a deterministic importance score for a completed track.
|
|
182
|
+
*
|
|
183
|
+
* `reason` is the tag of the single largest weighted term. Ties resolve to the
|
|
184
|
+
* first term in a fixed, weight-descending order (confidence → class → dwell →
|
|
185
|
+
* proximity → travel → zone → identity), so the output stays deterministic.
|
|
186
|
+
*/
|
|
187
|
+
function computeImportance(input) {
|
|
188
|
+
const terms = [
|
|
189
|
+
{
|
|
190
|
+
reason: "confidence",
|
|
191
|
+
value: WEIGHT_CONFIDENCE * clamp01(input.peakConfidence)
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
reason: "class",
|
|
195
|
+
value: WEIGHT_CLASS * classRank(input.className)
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
reason: "dwell",
|
|
199
|
+
value: WEIGHT_DWELL * clamp01(input.durationMs / DWELL_FULL_MS)
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
reason: "proximity",
|
|
203
|
+
value: WEIGHT_PROXIMITY * clamp01(input.peakBboxAreaFrac / SIZE_FULL)
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
reason: "travel",
|
|
207
|
+
value: WEIGHT_TRAVEL * clamp01(input.totalDistance / 2)
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
reason: "zone",
|
|
211
|
+
value: WEIGHT_ZONE * (input.zonesVisited.length > 0 ? 1 : 0)
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
reason: "identity",
|
|
215
|
+
value: WEIGHT_IDENTITY * (input.label ? 1 : 0)
|
|
216
|
+
}
|
|
217
|
+
];
|
|
218
|
+
let sum = 0;
|
|
219
|
+
let best = terms[0];
|
|
220
|
+
for (const term of terms) {
|
|
221
|
+
sum += term.value;
|
|
222
|
+
if (term.value > best.value) best = term;
|
|
223
|
+
}
|
|
224
|
+
return {
|
|
225
|
+
importance: clamp01(sum),
|
|
226
|
+
reason: best.reason
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
//#endregion
|
|
230
|
+
//#region src/pipeline-analytics/pipeline/key-event-recompute.ts
|
|
231
|
+
/**
|
|
232
|
+
* Recompute + persist a persisted track's importance from its current fields
|
|
233
|
+
* (including any newly-assigned label) and re-stamp its object events. No-op
|
|
234
|
+
* when the track is not (yet) persisted.
|
|
235
|
+
*/
|
|
236
|
+
async function recomputeTrackImportance(deps, trackId) {
|
|
237
|
+
const track = await deps.trackStore.getPersistedByTrackId(trackId);
|
|
238
|
+
if (track === null) return;
|
|
239
|
+
const peak = await deps.eventStore.peakForTrack(trackId);
|
|
240
|
+
const { importance, reason } = computeImportance({
|
|
241
|
+
peakConfidence: peak.peakConfidence,
|
|
242
|
+
className: track.className,
|
|
243
|
+
durationMs: track.lastSeen - track.firstSeen,
|
|
244
|
+
peakBboxAreaFrac: peak.peakBboxAreaFrac,
|
|
245
|
+
totalDistance: track.totalDistance,
|
|
246
|
+
zonesVisited: track.zonesVisited,
|
|
247
|
+
...track.label !== void 0 ? { label: track.label } : {}
|
|
248
|
+
});
|
|
249
|
+
const bestEventId = track.bestEventId ?? peak.bestEventId;
|
|
250
|
+
await deps.trackStore.setImportance(trackId, importance, reason, bestEventId);
|
|
251
|
+
await deps.eventStore.setImportanceForTrack(trackId, importance);
|
|
252
|
+
}
|
|
253
|
+
//#endregion
|
|
122
254
|
//#region src/pipeline-analytics/face-gallery-provider.ts
|
|
123
255
|
/** Embedding model id used when enrolling face crops as identity samples. */
|
|
124
256
|
var MODEL_ID = "arcface-r100";
|
|
@@ -239,7 +371,9 @@ var FaceGalleryProvider = class {
|
|
|
239
371
|
...face.recognizedIdentityId != null ? { recognizedIdentityId: face.recognizedIdentityId } : {},
|
|
240
372
|
...identityName !== void 0 ? { identityName } : {},
|
|
241
373
|
assigned: face.assigned,
|
|
242
|
-
...base64 !== void 0 ? { base64 } : {}
|
|
374
|
+
...base64 !== void 0 ? { base64 } : {},
|
|
375
|
+
...face.faceBbox !== void 0 ? { faceBbox: face.faceBbox } : {},
|
|
376
|
+
...face.keyFrameMediaKey !== void 0 ? { keyFrameMediaKey: face.keyFrameMediaKey } : {}
|
|
243
377
|
});
|
|
244
378
|
}
|
|
245
379
|
return result;
|
|
@@ -267,7 +401,9 @@ var FaceGalleryProvider = class {
|
|
|
267
401
|
...face.recognizedIdentityId != null ? { recognizedIdentityId: face.recognizedIdentityId } : {},
|
|
268
402
|
...identityName !== void 0 ? { identityName } : {},
|
|
269
403
|
assigned: face.assigned,
|
|
270
|
-
...base64 !== void 0 ? { base64 } : {}
|
|
404
|
+
...base64 !== void 0 ? { base64 } : {},
|
|
405
|
+
...face.faceBbox !== void 0 ? { faceBbox: face.faceBbox } : {},
|
|
406
|
+
...face.keyFrameMediaKey !== void 0 ? { keyFrameMediaKey: face.keyFrameMediaKey } : {}
|
|
271
407
|
};
|
|
272
408
|
}
|
|
273
409
|
/**
|
|
@@ -344,6 +480,10 @@ var FaceGalleryProvider = class {
|
|
|
344
480
|
if (identityName !== void 0) {
|
|
345
481
|
await this.trackStore.setLabel(currentFace.trackId, identityName);
|
|
346
482
|
await this.eventStore.setLabelForTrack(currentFace.trackId, identityName);
|
|
483
|
+
await recomputeTrackImportance({
|
|
484
|
+
trackStore: this.trackStore,
|
|
485
|
+
eventStore: this.eventStore
|
|
486
|
+
}, currentFace.trackId);
|
|
347
487
|
}
|
|
348
488
|
this.refreshGallery();
|
|
349
489
|
}
|
|
@@ -1382,6 +1522,7 @@ var FrameProcessor = class {
|
|
|
1382
1522
|
const embeddingByBbox = /* @__PURE__ */ new Map();
|
|
1383
1523
|
const firstLevelBboxById = /* @__PURE__ */ new Map();
|
|
1384
1524
|
const faceBboxByBbox = /* @__PURE__ */ new Map();
|
|
1525
|
+
const faceAlignedCropByBbox = /* @__PURE__ */ new Map();
|
|
1385
1526
|
const plateByBbox = /* @__PURE__ */ new Map();
|
|
1386
1527
|
const maskByBbox = /* @__PURE__ */ new Map();
|
|
1387
1528
|
const flatDetections = frame.detections.filter((d) => d.kind === "first-level").map((det) => {
|
|
@@ -1424,6 +1565,11 @@ var FrameProcessor = class {
|
|
|
1424
1565
|
w: det.bbox.width,
|
|
1425
1566
|
h: det.bbox.height
|
|
1426
1567
|
});
|
|
1568
|
+
if (det.faceAlignedCrop !== void 0) faceAlignedCropByBbox.set(parentBbox, det.faceAlignedCrop);
|
|
1569
|
+
if (det.embedding !== void 0 && !embeddingByBbox.has(parentBbox)) embeddingByBbox.set(parentBbox, {
|
|
1570
|
+
embedding: det.embedding,
|
|
1571
|
+
...det.embeddingModelId !== void 0 ? { embeddingModelId: det.embeddingModelId } : {}
|
|
1572
|
+
});
|
|
1427
1573
|
}
|
|
1428
1574
|
for (const det of frame.detections) {
|
|
1429
1575
|
if (det.kind !== "detail" || det.macroClass !== "plate" || !det.parentId) continue;
|
|
@@ -1462,6 +1608,7 @@ var FrameProcessor = class {
|
|
|
1462
1608
|
});
|
|
1463
1609
|
const emb = embeddingByBbox.get(td.bbox);
|
|
1464
1610
|
const faceBbox = faceBboxByBbox.get(td.bbox);
|
|
1611
|
+
const faceAlignedCrop = faceAlignedCropByBbox.get(td.bbox);
|
|
1465
1612
|
const plate = plateByBbox.get(td.bbox);
|
|
1466
1613
|
return {
|
|
1467
1614
|
trackId: td.trackId,
|
|
@@ -1476,6 +1623,7 @@ var FrameProcessor = class {
|
|
|
1476
1623
|
...emb.embeddingModelId !== void 0 ? { embeddingModelId: emb.embeddingModelId } : {}
|
|
1477
1624
|
} : {},
|
|
1478
1625
|
...faceBbox !== void 0 ? { faceBbox } : {},
|
|
1626
|
+
...faceAlignedCrop !== void 0 ? { faceAlignedCrop } : {},
|
|
1479
1627
|
...plate !== void 0 ? {
|
|
1480
1628
|
plateText: plate.text,
|
|
1481
1629
|
plateScore: plate.score,
|
|
@@ -1564,6 +1712,115 @@ var BestDetectionTracker = class {
|
|
|
1564
1712
|
}
|
|
1565
1713
|
};
|
|
1566
1714
|
//#endregion
|
|
1715
|
+
//#region src/pipeline-analytics/pipeline/track-best-detection.ts
|
|
1716
|
+
/**
|
|
1717
|
+
* `TrackBestSelector` — the ONE unified "best detection per track" primitive.
|
|
1718
|
+
*
|
|
1719
|
+
* Post-analysis derives several per-track "best" artefacts (best FRAME thumbnail,
|
|
1720
|
+
* best OBJECT/CLIP embedding + its crop, best FACE crop). They all rank the same
|
|
1721
|
+
* way: highest detector confidence per `trackId`. This selector composes the
|
|
1722
|
+
* canonical {@link BestDetectionTracker} ranking with the RESOLVED payload the
|
|
1723
|
+
* consumers need at the peak (bbox / className / embedding / faceBbox), so a
|
|
1724
|
+
* single "new best" DECISION can drive one capture whose output is shared:
|
|
1725
|
+
* - the boxed best-frame `thumbnail`,
|
|
1726
|
+
* - the tight object crop written onto the CLIP embedding row's `mediaKey`
|
|
1727
|
+
* (so a search hit's thumbnail IS the embedded crop).
|
|
1728
|
+
*
|
|
1729
|
+
* It is deliberately in-memory (the peak is per live track, dropped at track
|
|
1730
|
+
* end). The CLIP embedding's cross-restart persistence stays a SEPARATE store-
|
|
1731
|
+
* side gate (`ObjectEmbeddingStore.upsertIfBetter`): this selector unifies the
|
|
1732
|
+
* DECISION, not the durable store (see best-detection-tracker.ts docstring).
|
|
1733
|
+
*
|
|
1734
|
+
* The face path keeps its own hold buffer because a face crop can only come
|
|
1735
|
+
* from a face-bearing frame (the documented FRAME↔FACE seam) — but it shares
|
|
1736
|
+
* this ranking so best-frame and best-face agree on which frame is "best".
|
|
1737
|
+
*/
|
|
1738
|
+
var TrackBestSelector = class {
|
|
1739
|
+
tracker;
|
|
1740
|
+
payloads = /* @__PURE__ */ new Map();
|
|
1741
|
+
constructor(options = {}) {
|
|
1742
|
+
this.tracker = new BestDetectionTracker(options);
|
|
1743
|
+
}
|
|
1744
|
+
/**
|
|
1745
|
+
* Record an observation for its track. Returns true when it becomes the
|
|
1746
|
+
* track's new best (first sighting, or a confidence that beats the held peak
|
|
1747
|
+
* per the tracker's hysteresis / minGap rules). On acceptance the held payload
|
|
1748
|
+
* advances to this observation so `peak(trackId)` returns the winning frame's
|
|
1749
|
+
* bbox / embedding / faceBbox.
|
|
1750
|
+
*/
|
|
1751
|
+
observe(obs) {
|
|
1752
|
+
const isNewBest = this.tracker.observe(obs.trackId, obs.confidence, obs.atMs);
|
|
1753
|
+
if (isNewBest) {
|
|
1754
|
+
const { trackId, ...payload } = obs;
|
|
1755
|
+
this.payloads.set(trackId, payload);
|
|
1756
|
+
}
|
|
1757
|
+
return isNewBest;
|
|
1758
|
+
}
|
|
1759
|
+
/** The held best payload for a track (undefined if never observed). */
|
|
1760
|
+
peak(trackId) {
|
|
1761
|
+
return this.payloads.get(trackId);
|
|
1762
|
+
}
|
|
1763
|
+
/** The held peak confidence for a track (undefined if never observed). */
|
|
1764
|
+
peakConfidence(trackId) {
|
|
1765
|
+
return this.tracker.peak(trackId)?.confidence;
|
|
1766
|
+
}
|
|
1767
|
+
/** Drop a track's peak + payload (call at track end). */
|
|
1768
|
+
delete(trackId) {
|
|
1769
|
+
this.tracker.delete(trackId);
|
|
1770
|
+
this.payloads.delete(trackId);
|
|
1771
|
+
}
|
|
1772
|
+
clear() {
|
|
1773
|
+
this.tracker.clear();
|
|
1774
|
+
this.payloads.clear();
|
|
1775
|
+
}
|
|
1776
|
+
};
|
|
1777
|
+
//#endregion
|
|
1778
|
+
//#region src/pipeline-analytics/pipeline/object-embedding-selection.ts
|
|
1779
|
+
function isClipObjectEmbedding(t) {
|
|
1780
|
+
return Array.isArray(t.embedding) && t.embedding.length > 0 && typeof t.embeddingModelId === "string" && t.embeddingModelId.startsWith("mobileclip-");
|
|
1781
|
+
}
|
|
1782
|
+
function resolveSearchThumbnailUrl(input) {
|
|
1783
|
+
if (input.baseUrl === null) return void 0;
|
|
1784
|
+
const id = input.embeddingMediaKey ?? input.eventId;
|
|
1785
|
+
return `${input.baseUrl}/${encodeURIComponent(id)}`;
|
|
1786
|
+
}
|
|
1787
|
+
//#endregion
|
|
1788
|
+
//#region src/pipeline-analytics/pipeline/key-event-query.ts
|
|
1789
|
+
async function rankKeyEvents(candidates, options, peakLookup) {
|
|
1790
|
+
const scored = [];
|
|
1791
|
+
for (const t of candidates) {
|
|
1792
|
+
if (options.classFilter !== void 0 && t.className !== options.classFilter) continue;
|
|
1793
|
+
let importance = t.importance;
|
|
1794
|
+
let bestEventId = t.bestEventId;
|
|
1795
|
+
if (importance === void 0) {
|
|
1796
|
+
const peak = await peakLookup(t.trackId);
|
|
1797
|
+
importance = computeImportance({
|
|
1798
|
+
peakConfidence: peak.peakConfidence,
|
|
1799
|
+
className: t.className,
|
|
1800
|
+
durationMs: t.lastSeen - t.firstSeen,
|
|
1801
|
+
peakBboxAreaFrac: peak.peakBboxAreaFrac,
|
|
1802
|
+
totalDistance: t.totalDistance,
|
|
1803
|
+
zonesVisited: t.zonesVisited,
|
|
1804
|
+
...t.label !== void 0 ? { label: t.label } : {}
|
|
1805
|
+
}).importance;
|
|
1806
|
+
bestEventId = bestEventId ?? peak.bestEventId;
|
|
1807
|
+
}
|
|
1808
|
+
if (options.minImportance !== void 0 && importance < options.minImportance) continue;
|
|
1809
|
+
scored.push({
|
|
1810
|
+
id: bestEventId ?? t.trackId,
|
|
1811
|
+
trackId: t.trackId,
|
|
1812
|
+
timestamp: t.firstSeen,
|
|
1813
|
+
className: t.className,
|
|
1814
|
+
...t.label !== void 0 ? { label: t.label } : {},
|
|
1815
|
+
importance,
|
|
1816
|
+
bestEventId: bestEventId ?? "",
|
|
1817
|
+
windowMs: t.lastSeen - t.firstSeen
|
|
1818
|
+
});
|
|
1819
|
+
}
|
|
1820
|
+
scored.sort((a, b) => b.importance - a.importance);
|
|
1821
|
+
return scored.slice(0, options.limit);
|
|
1822
|
+
}
|
|
1823
|
+
//#endregion
|
|
1567
1824
|
//#region src/pipeline-analytics/pipeline/native-detection.ts
|
|
1568
1825
|
/**
|
|
1569
1826
|
* Nominal frame size used to denormalize native `[0,1]` boxes when a
|
|
@@ -1733,6 +1990,18 @@ var TRACKS_COLUMNS = [
|
|
|
1733
1990
|
{
|
|
1734
1991
|
name: "state",
|
|
1735
1992
|
type: "TEXT"
|
|
1993
|
+
},
|
|
1994
|
+
{
|
|
1995
|
+
name: "importance",
|
|
1996
|
+
type: "REAL"
|
|
1997
|
+
},
|
|
1998
|
+
{
|
|
1999
|
+
name: "bestEventId",
|
|
2000
|
+
type: "TEXT"
|
|
2001
|
+
},
|
|
2002
|
+
{
|
|
2003
|
+
name: "importanceReason",
|
|
2004
|
+
type: "TEXT"
|
|
1736
2005
|
}
|
|
1737
2006
|
];
|
|
1738
2007
|
var TRACKS_INDEXES = [{
|
|
@@ -1764,7 +2033,10 @@ function cloneTrack(t) {
|
|
|
1764
2033
|
zonesVisited: [...t.zonesVisited],
|
|
1765
2034
|
totalDistance: t.totalDistance,
|
|
1766
2035
|
state: t.state,
|
|
1767
|
-
active: t.active
|
|
2036
|
+
active: t.active,
|
|
2037
|
+
...t.importance !== void 0 ? { importance: t.importance } : {},
|
|
2038
|
+
...t.bestEventId !== void 0 ? { bestEventId: t.bestEventId } : {},
|
|
2039
|
+
...t.importanceReason !== void 0 ? { importanceReason: t.importanceReason } : {}
|
|
1768
2040
|
};
|
|
1769
2041
|
}
|
|
1770
2042
|
var TrackStore = class {
|
|
@@ -1898,6 +2170,37 @@ var TrackStore = class {
|
|
|
1898
2170
|
}
|
|
1899
2171
|
}
|
|
1900
2172
|
/**
|
|
2173
|
+
* Stamp a track's importance score (+ dominant reason and best-event pointer).
|
|
2174
|
+
* Updates the in-memory active entry (so the value is carried into an eventual
|
|
2175
|
+
* (re)persist) AND patches the already-persisted row. Mirrors `setLabel`.
|
|
2176
|
+
* Forward-only — never rewrites history beyond these fields.
|
|
2177
|
+
*/
|
|
2178
|
+
async setImportance(trackId, importance, reason, bestEventId) {
|
|
2179
|
+
const active = this.active.get(trackId);
|
|
2180
|
+
if (active) {
|
|
2181
|
+
active.importance = importance;
|
|
2182
|
+
active.importanceReason = reason;
|
|
2183
|
+
if (bestEventId !== void 0) active.bestEventId = bestEventId;
|
|
2184
|
+
}
|
|
2185
|
+
const data = {
|
|
2186
|
+
importance,
|
|
2187
|
+
importanceReason: reason
|
|
2188
|
+
};
|
|
2189
|
+
if (bestEventId !== void 0) data["bestEventId"] = bestEventId;
|
|
2190
|
+
try {
|
|
2191
|
+
await this.store.update.mutate({
|
|
2192
|
+
collection: TRACKS_COLLECTION,
|
|
2193
|
+
id: trackId,
|
|
2194
|
+
data
|
|
2195
|
+
});
|
|
2196
|
+
} catch (err) {
|
|
2197
|
+
this.logger.warn("setImportance persist failed", { meta: {
|
|
2198
|
+
trackId,
|
|
2199
|
+
error: String(err)
|
|
2200
|
+
} });
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
/**
|
|
1901
2204
|
* Clear a track's label. Sets label to null in the persisted row (so
|
|
1902
2205
|
* rowToTrack's `typeof label === 'string'` guard omits it on read → label
|
|
1903
2206
|
* is absent/undefined). Also clears the in-memory active entry if present.
|
|
@@ -1968,7 +2271,10 @@ var TrackStore = class {
|
|
|
1968
2271
|
snapshots: [...t.snapshots],
|
|
1969
2272
|
zonesVisited: [...t.zonesVisited],
|
|
1970
2273
|
totalDistance: t.totalDistance,
|
|
1971
|
-
state: t.state
|
|
2274
|
+
state: t.state,
|
|
2275
|
+
...t.importance !== void 0 ? { importance: t.importance } : {},
|
|
2276
|
+
...t.bestEventId !== void 0 ? { bestEventId: t.bestEventId } : {},
|
|
2277
|
+
...t.importanceReason !== void 0 ? { importanceReason: t.importanceReason } : {}
|
|
1972
2278
|
}
|
|
1973
2279
|
});
|
|
1974
2280
|
}
|
|
@@ -1977,6 +2283,9 @@ var TrackStore = class {
|
|
|
1977
2283
|
const snapshots = data["snapshots"] ?? [];
|
|
1978
2284
|
const zones = data["zonesVisited"] ?? [];
|
|
1979
2285
|
const label = data["label"];
|
|
2286
|
+
const importance = data["importance"];
|
|
2287
|
+
const bestEventId = data["bestEventId"];
|
|
2288
|
+
const importanceReason = data["importanceReason"];
|
|
1980
2289
|
return {
|
|
1981
2290
|
trackId: id,
|
|
1982
2291
|
deviceId: Number(data["deviceId"]),
|
|
@@ -1989,7 +2298,10 @@ var TrackStore = class {
|
|
|
1989
2298
|
zonesVisited: zones,
|
|
1990
2299
|
totalDistance: Number(data["totalDistance"] ?? 0),
|
|
1991
2300
|
state: data["state"] ?? "idle",
|
|
1992
|
-
active: false
|
|
2301
|
+
active: false,
|
|
2302
|
+
...typeof importance === "number" ? { importance } : {},
|
|
2303
|
+
...typeof bestEventId === "string" ? { bestEventId } : {},
|
|
2304
|
+
...typeof importanceReason === "string" ? { importanceReason } : {}
|
|
1993
2305
|
};
|
|
1994
2306
|
}
|
|
1995
2307
|
};
|
|
@@ -2492,6 +2804,10 @@ var OBJECT_COLUMNS = [
|
|
|
2492
2804
|
{
|
|
2493
2805
|
name: "mediaKey",
|
|
2494
2806
|
type: "TEXT"
|
|
2807
|
+
},
|
|
2808
|
+
{
|
|
2809
|
+
name: "importance",
|
|
2810
|
+
type: "REAL"
|
|
2495
2811
|
}
|
|
2496
2812
|
];
|
|
2497
2813
|
var AUDIO_COLUMNS = [
|
|
@@ -2689,6 +3005,63 @@ var EventStore = class {
|
|
|
2689
3005
|
return updated;
|
|
2690
3006
|
}
|
|
2691
3007
|
/**
|
|
3008
|
+
* Forward-only: stamp `importance` on every already-emitted object event of a
|
|
3009
|
+
* track. Returns the number of events updated. Best-effort per row. Mirrors
|
|
3010
|
+
* `setLabelForTrack`. Called when a track's key-event score is computed (at
|
|
3011
|
+
* expiry) or recomputed (late label) so an event row carries the parent
|
|
3012
|
+
* track's importance without a join.
|
|
3013
|
+
*/
|
|
3014
|
+
async setImportanceForTrack(trackId, importance) {
|
|
3015
|
+
const rows = await this.store.query.query({
|
|
3016
|
+
collection: OBJECT_EVENTS_COLLECTION,
|
|
3017
|
+
filter: { where: { trackId } }
|
|
3018
|
+
});
|
|
3019
|
+
let updated = 0;
|
|
3020
|
+
for (const row of rows) try {
|
|
3021
|
+
await this.store.update.mutate({
|
|
3022
|
+
collection: OBJECT_EVENTS_COLLECTION,
|
|
3023
|
+
id: row.id,
|
|
3024
|
+
data: { importance }
|
|
3025
|
+
});
|
|
3026
|
+
updated++;
|
|
3027
|
+
} catch (err) {
|
|
3028
|
+
this.logger.warn("setImportanceForTrack update failed", { meta: {
|
|
3029
|
+
trackId,
|
|
3030
|
+
eventId: row.id,
|
|
3031
|
+
error: String(err)
|
|
3032
|
+
} });
|
|
3033
|
+
}
|
|
3034
|
+
return updated;
|
|
3035
|
+
}
|
|
3036
|
+
/**
|
|
3037
|
+
* The track's highest-confidence object event, its bbox area (as a fraction of
|
|
3038
|
+
* frame area), and that event's id — the SHARED per-track ranking already used
|
|
3039
|
+
* for the best frame, read back from the persisted object events (index
|
|
3040
|
+
* `idx_object_track`). Returns zeros + undefined id when the track has none.
|
|
3041
|
+
* Used by the importance scorer at expiry and by `getKeyEvents` compute-on-read.
|
|
3042
|
+
*/
|
|
3043
|
+
async peakForTrack(trackId) {
|
|
3044
|
+
const rows = await this.store.query.query({
|
|
3045
|
+
collection: OBJECT_EVENTS_COLLECTION,
|
|
3046
|
+
filter: { where: { trackId } }
|
|
3047
|
+
});
|
|
3048
|
+
let bestConf = -1;
|
|
3049
|
+
let bestEventId;
|
|
3050
|
+
let peakBboxAreaFrac = 0;
|
|
3051
|
+
for (const row of rows) {
|
|
3052
|
+
const conf = typeof row.data["confidence"] === "number" ? row.data["confidence"] : 0;
|
|
3053
|
+
if (conf <= bestConf) continue;
|
|
3054
|
+
bestConf = conf;
|
|
3055
|
+
bestEventId = row.id;
|
|
3056
|
+
peakBboxAreaFrac = bboxAreaFrac(row.data);
|
|
3057
|
+
}
|
|
3058
|
+
return {
|
|
3059
|
+
peakConfidence: bestConf < 0 ? 0 : bestConf,
|
|
3060
|
+
peakBboxAreaFrac,
|
|
3061
|
+
bestEventId
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
/**
|
|
2692
3065
|
* Clear `label` on every already-emitted object event of a track (sets label
|
|
2693
3066
|
* to null so stripNulls/slimObject omit it on read → label is absent/undefined).
|
|
2694
3067
|
* Returns the number of events updated. Best-effort per row. Mirrors
|
|
@@ -2883,7 +3256,8 @@ function slimObject(id, data) {
|
|
|
2883
3256
|
timestamp: data["timestamp"],
|
|
2884
3257
|
className: data["className"],
|
|
2885
3258
|
...typeof data["frameId"] === "string" ? { frameId: data["frameId"] } : {},
|
|
2886
|
-
...typeof data["trackId"] === "string" ? { trackId: data["trackId"] } : {}
|
|
3259
|
+
...typeof data["trackId"] === "string" ? { trackId: data["trackId"] } : {},
|
|
3260
|
+
...typeof data["importance"] === "number" ? { importance: data["importance"] } : {}
|
|
2887
3261
|
};
|
|
2888
3262
|
if (typeof data["label"] === "string") return {
|
|
2889
3263
|
...base,
|
|
@@ -2913,6 +3287,17 @@ function slimAudio(id, data) {
|
|
|
2913
3287
|
}
|
|
2914
3288
|
return base;
|
|
2915
3289
|
}
|
|
3290
|
+
function bboxAreaFrac(data) {
|
|
3291
|
+
const bbox = data["bbox"];
|
|
3292
|
+
const fw = data["frameWidth"];
|
|
3293
|
+
const fh = data["frameHeight"];
|
|
3294
|
+
if (bbox === null || typeof bbox !== "object") return 0;
|
|
3295
|
+
if (typeof fw !== "number" || typeof fh !== "number" || fw <= 0 || fh <= 0) return 0;
|
|
3296
|
+
const w = "w" in bbox && typeof bbox.w === "number" ? bbox.w : 0;
|
|
3297
|
+
const h = "h" in bbox && typeof bbox.h === "number" ? bbox.h : 0;
|
|
3298
|
+
if (w <= 0 || h <= 0) return 0;
|
|
3299
|
+
return w * h / (fw * fh);
|
|
3300
|
+
}
|
|
2916
3301
|
function stripNulls(data) {
|
|
2917
3302
|
const out = {};
|
|
2918
3303
|
for (const [k, v] of Object.entries(data)) if (v !== null) out[k] = v;
|
|
@@ -4532,6 +4917,14 @@ var FACE_COLUMNS = [
|
|
|
4532
4917
|
{
|
|
4533
4918
|
name: "assignedSampleId",
|
|
4534
4919
|
type: "TEXT"
|
|
4920
|
+
},
|
|
4921
|
+
{
|
|
4922
|
+
name: "keyFrameMediaKey",
|
|
4923
|
+
type: "TEXT"
|
|
4924
|
+
},
|
|
4925
|
+
{
|
|
4926
|
+
name: "faceBbox",
|
|
4927
|
+
type: "JSON"
|
|
4535
4928
|
}
|
|
4536
4929
|
];
|
|
4537
4930
|
var FACE_INDEXES = [{
|
|
@@ -4604,7 +4997,9 @@ var FaceStore = class {
|
|
|
4604
4997
|
assigned: Boolean(r.data.assigned),
|
|
4605
4998
|
mediaKey: data.mediaKey ?? void 0,
|
|
4606
4999
|
recognizedIdentityId: data.recognizedIdentityId ?? void 0,
|
|
4607
|
-
assignedSampleId: data.assignedSampleId ?? void 0
|
|
5000
|
+
assignedSampleId: data.assignedSampleId ?? void 0,
|
|
5001
|
+
keyFrameMediaKey: data.keyFrameMediaKey ?? void 0,
|
|
5002
|
+
faceBbox: data.faceBbox ?? void 0
|
|
4608
5003
|
};
|
|
4609
5004
|
}).filter((f) => !f.assigned);
|
|
4610
5005
|
}
|
|
@@ -4768,8 +5163,11 @@ var FaceStore = class {
|
|
|
4768
5163
|
id: faceId,
|
|
4769
5164
|
...data,
|
|
4770
5165
|
assigned: Boolean(raw.assigned),
|
|
5166
|
+
mediaKey: data.mediaKey ?? void 0,
|
|
4771
5167
|
recognizedIdentityId: data.recognizedIdentityId ?? void 0,
|
|
4772
|
-
assignedSampleId: data.assignedSampleId ?? void 0
|
|
5168
|
+
assignedSampleId: data.assignedSampleId ?? void 0,
|
|
5169
|
+
keyFrameMediaKey: data.keyFrameMediaKey ?? void 0,
|
|
5170
|
+
faceBbox: data.faceBbox ?? void 0
|
|
4773
5171
|
};
|
|
4774
5172
|
}
|
|
4775
5173
|
/**
|
|
@@ -4813,7 +5211,9 @@ var FaceStore = class {
|
|
|
4813
5211
|
assigned: Boolean(r.data.assigned),
|
|
4814
5212
|
mediaKey: data.mediaKey ?? void 0,
|
|
4815
5213
|
recognizedIdentityId: data.recognizedIdentityId ?? void 0,
|
|
4816
|
-
assignedSampleId: data.assignedSampleId ?? void 0
|
|
5214
|
+
assignedSampleId: data.assignedSampleId ?? void 0,
|
|
5215
|
+
keyFrameMediaKey: data.keyFrameMediaKey ?? void 0,
|
|
5216
|
+
faceBbox: data.faceBbox ?? void 0
|
|
4817
5217
|
};
|
|
4818
5218
|
});
|
|
4819
5219
|
const filterMode = input.filter ?? "all";
|
|
@@ -4878,6 +5278,10 @@ var OBJECT_EMBEDDING_COLUMNS = [
|
|
|
4878
5278
|
{
|
|
4879
5279
|
name: "mediaKey",
|
|
4880
5280
|
type: "TEXT"
|
|
5281
|
+
},
|
|
5282
|
+
{
|
|
5283
|
+
name: "keyFrameMediaKey",
|
|
5284
|
+
type: "TEXT"
|
|
4881
5285
|
}
|
|
4882
5286
|
];
|
|
4883
5287
|
var ObjectEmbeddingStore = class {
|
|
@@ -4924,7 +5328,8 @@ var ObjectEmbeddingStore = class {
|
|
|
4924
5328
|
modelId: input.modelId,
|
|
4925
5329
|
dim: input.embedding.length,
|
|
4926
5330
|
confidence: input.confidence,
|
|
4927
|
-
...input.mediaKey !== void 0 ? { mediaKey: input.mediaKey } : {}
|
|
5331
|
+
...input.mediaKey !== void 0 ? { mediaKey: input.mediaKey } : {},
|
|
5332
|
+
...input.keyFrameMediaKey !== void 0 ? { keyFrameMediaKey: input.keyFrameMediaKey } : {}
|
|
4928
5333
|
};
|
|
4929
5334
|
try {
|
|
4930
5335
|
await this.store.set.mutate({
|
|
@@ -4962,7 +5367,8 @@ var ObjectEmbeddingStore = class {
|
|
|
4962
5367
|
return {
|
|
4963
5368
|
id: r.id,
|
|
4964
5369
|
...data,
|
|
4965
|
-
mediaKey: data.mediaKey ?? void 0
|
|
5370
|
+
mediaKey: data.mediaKey ?? void 0,
|
|
5371
|
+
keyFrameMediaKey: data.keyFrameMediaKey ?? void 0
|
|
4966
5372
|
};
|
|
4967
5373
|
});
|
|
4968
5374
|
}
|
|
@@ -5091,6 +5497,16 @@ function updateTrackAggregate(prev, match, opts) {
|
|
|
5091
5497
|
}
|
|
5092
5498
|
//#endregion
|
|
5093
5499
|
//#region src/pipeline-analytics/face-recognizer.ts
|
|
5500
|
+
/** At most one "dropping imageless track" log per this interval, per recognizer. */
|
|
5501
|
+
var FACE_IMAGELESS_LOG_THROTTLE_MS = 6e4;
|
|
5502
|
+
/**
|
|
5503
|
+
* arcface model id stamped on a detail-plane face candidate when the gallery is
|
|
5504
|
+
* empty (collect-only). The detail-subtree result carries no `embeddingModelId`
|
|
5505
|
+
* (the two-plane `DetailResult` schema omits it), so recognition uses the
|
|
5506
|
+
* gallery's own model id (all enrolled samples share one) and this constant is
|
|
5507
|
+
* only a placeholder for the collect-only case where the id is never compared.
|
|
5508
|
+
*/
|
|
5509
|
+
var FALLBACK_FACE_MODEL_ID = "arcface";
|
|
5094
5510
|
var FaceRecognizer = class {
|
|
5095
5511
|
deps;
|
|
5096
5512
|
gallery = [];
|
|
@@ -5102,6 +5518,9 @@ var FaceRecognizer = class {
|
|
|
5102
5518
|
* true highest-confidence face (holding a buffer in memory is cheap, and a
|
|
5103
5519
|
* track may last well under the best-frame rate-limit window). */
|
|
5104
5520
|
bestTracker = new BestDetectionTracker();
|
|
5521
|
+
/** Throttle for the "dropping imageless track" log — one line per minute at
|
|
5522
|
+
* most, so a busy scene that never produces a face crop can't flood logs. */
|
|
5523
|
+
lastImagelessLogAt = 0;
|
|
5105
5524
|
constructor(deps) {
|
|
5106
5525
|
this.deps = deps;
|
|
5107
5526
|
}
|
|
@@ -5115,6 +5534,38 @@ var FaceRecognizer = class {
|
|
|
5115
5534
|
this.deps.logger.warn("FaceRecognizer.refreshGallery failed", { meta: { error: String(err) } });
|
|
5116
5535
|
}
|
|
5117
5536
|
}
|
|
5537
|
+
/**
|
|
5538
|
+
* Two-plane detail feed: ingest ONE `runDetailSubtree` face result for a
|
|
5539
|
+
* track and run it through the SAME `processFrame` logic (candidate → best
|
|
5540
|
+
* face → gallery match → crop hold). The result is synthesized into a single
|
|
5541
|
+
* `TrackedDetectionOut` candidate so no recognizer logic changes — only the
|
|
5542
|
+
* input source moves from the per-frame plane to this per-track call.
|
|
5543
|
+
*/
|
|
5544
|
+
async ingestFaceDetail(input) {
|
|
5545
|
+
const modelId = this.gallery[0]?.modelId ?? FALLBACK_FACE_MODEL_ID;
|
|
5546
|
+
const candidate = {
|
|
5547
|
+
trackId: input.trackId,
|
|
5548
|
+
className: "face",
|
|
5549
|
+
confidence: input.score,
|
|
5550
|
+
bbox: input.parentBbox,
|
|
5551
|
+
zones: [],
|
|
5552
|
+
state: "moving",
|
|
5553
|
+
embedding: input.embedding,
|
|
5554
|
+
embeddingModelId: modelId,
|
|
5555
|
+
...input.faceBbox !== void 0 ? { faceBbox: input.faceBbox } : {},
|
|
5556
|
+
...input.alignedCropJpeg !== void 0 ? { faceAlignedCrop: input.alignedCropJpeg } : {}
|
|
5557
|
+
};
|
|
5558
|
+
await this.processFrame({
|
|
5559
|
+
deviceId: input.deviceId,
|
|
5560
|
+
timestamp: input.timestamp,
|
|
5561
|
+
frameWidth: input.frameWidth,
|
|
5562
|
+
frameHeight: input.frameHeight,
|
|
5563
|
+
tracked: [candidate],
|
|
5564
|
+
settings: input.settings,
|
|
5565
|
+
cropPadding: input.cropPadding,
|
|
5566
|
+
...input.frameHandle !== void 0 ? { frameHandle: input.frameHandle } : {}
|
|
5567
|
+
});
|
|
5568
|
+
}
|
|
5118
5569
|
async processFrame(input) {
|
|
5119
5570
|
const { settings } = input;
|
|
5120
5571
|
const candidates = input.tracked.filter((t) => Array.isArray(t.embedding) && t.embedding.length > 0 && typeof t.embeddingModelId === "string" && t.confidence >= settings.minFaceConfidence);
|
|
@@ -5152,7 +5603,8 @@ var FaceRecognizer = class {
|
|
|
5152
5603
|
if (isNewBest || needsCrop) {
|
|
5153
5604
|
const cropBbox = c.faceBbox ?? c.bbox;
|
|
5154
5605
|
let crop;
|
|
5155
|
-
if (c.
|
|
5606
|
+
if (c.faceAlignedCrop !== void 0) crop = Buffer.from(c.faceAlignedCrop, "base64");
|
|
5607
|
+
else if (c.faceBbox !== void 0 && input.frameHandle !== void 0) try {
|
|
5156
5608
|
crop = await this.deps.captureCrop(input.frameHandle, c.faceBbox, input.frameWidth, input.frameHeight, input.cropPadding) ?? void 0;
|
|
5157
5609
|
} catch (err) {
|
|
5158
5610
|
this.deps.logger.debug("FaceRecognizer crop capture failed", {
|
|
@@ -5231,6 +5683,17 @@ var FaceRecognizer = class {
|
|
|
5231
5683
|
}
|
|
5232
5684
|
});
|
|
5233
5685
|
}
|
|
5686
|
+
try {
|
|
5687
|
+
await this.deps.recomputeImportance?.(work.trackId);
|
|
5688
|
+
} catch (err) {
|
|
5689
|
+
this.deps.logger.warn("recomputeImportance failed", {
|
|
5690
|
+
tags: { deviceId: input.deviceId },
|
|
5691
|
+
meta: {
|
|
5692
|
+
trackId: work.trackId,
|
|
5693
|
+
error: String(err)
|
|
5694
|
+
}
|
|
5695
|
+
});
|
|
5696
|
+
}
|
|
5234
5697
|
}
|
|
5235
5698
|
}
|
|
5236
5699
|
/**
|
|
@@ -5250,9 +5713,23 @@ var FaceRecognizer = class {
|
|
|
5250
5713
|
} });
|
|
5251
5714
|
return;
|
|
5252
5715
|
}
|
|
5716
|
+
if (held.crop === void 0) {
|
|
5717
|
+
const now = Date.now();
|
|
5718
|
+
if (now - this.lastImagelessLogAt >= FACE_IMAGELESS_LOG_THROTTLE_MS) {
|
|
5719
|
+
this.lastImagelessLogAt = now;
|
|
5720
|
+
this.deps.logger.info("face: dropping imageless track (no crop captured)", {
|
|
5721
|
+
tags: {
|
|
5722
|
+
deviceId,
|
|
5723
|
+
trackId
|
|
5724
|
+
},
|
|
5725
|
+
meta: { score: held.score }
|
|
5726
|
+
});
|
|
5727
|
+
}
|
|
5728
|
+
return;
|
|
5729
|
+
}
|
|
5253
5730
|
const faceId = `face-${trackId}`;
|
|
5254
5731
|
let mediaKey;
|
|
5255
|
-
|
|
5732
|
+
try {
|
|
5256
5733
|
mediaKey = await this.deps.mediaStore.put({
|
|
5257
5734
|
deviceId,
|
|
5258
5735
|
ownerKind: "face",
|
|
@@ -5270,6 +5747,17 @@ var FaceRecognizer = class {
|
|
|
5270
5747
|
}
|
|
5271
5748
|
});
|
|
5272
5749
|
}
|
|
5750
|
+
if (mediaKey === void 0) {
|
|
5751
|
+
this.deps.logger.warn("face: crop store failed — dropping face row", {
|
|
5752
|
+
tags: {
|
|
5753
|
+
deviceId,
|
|
5754
|
+
trackId
|
|
5755
|
+
},
|
|
5756
|
+
meta: { faceId }
|
|
5757
|
+
});
|
|
5758
|
+
return;
|
|
5759
|
+
}
|
|
5760
|
+
const keyFrameMediaKey = this.deps.getKeyFrameMediaKey?.(trackId);
|
|
5273
5761
|
try {
|
|
5274
5762
|
await this.deps.faceStore.insert({
|
|
5275
5763
|
id: faceId,
|
|
@@ -5277,9 +5765,11 @@ var FaceRecognizer = class {
|
|
|
5277
5765
|
trackId,
|
|
5278
5766
|
timestamp: held.timestamp,
|
|
5279
5767
|
embedding: held.embedding,
|
|
5280
|
-
|
|
5768
|
+
mediaKey,
|
|
5281
5769
|
...held.recognizedIdentityId !== void 0 ? { recognizedIdentityId: held.recognizedIdentityId } : {},
|
|
5282
|
-
assigned: false
|
|
5770
|
+
assigned: false,
|
|
5771
|
+
faceBbox: held.bbox,
|
|
5772
|
+
...keyFrameMediaKey !== void 0 ? { keyFrameMediaKey } : {}
|
|
5283
5773
|
});
|
|
5284
5774
|
this.deps.logger.info("face: buffered to gallery", {
|
|
5285
5775
|
tags: {
|
|
@@ -5288,7 +5778,7 @@ var FaceRecognizer = class {
|
|
|
5288
5778
|
},
|
|
5289
5779
|
meta: {
|
|
5290
5780
|
faceId,
|
|
5291
|
-
hasCrop:
|
|
5781
|
+
hasCrop: true,
|
|
5292
5782
|
recognizedIdentityId: held.recognizedIdentityId ?? null,
|
|
5293
5783
|
score: held.score
|
|
5294
5784
|
}
|
|
@@ -5305,6 +5795,346 @@ var FaceRecognizer = class {
|
|
|
5305
5795
|
}
|
|
5306
5796
|
};
|
|
5307
5797
|
//#endregion
|
|
5798
|
+
//#region src/pipeline-analytics/detail-scheduler.ts
|
|
5799
|
+
/** Default backoff/period when a step's cadence omits `minIntervalMs`. */
|
|
5800
|
+
var DEFAULT_MIN_INTERVAL_MS = 1e3;
|
|
5801
|
+
/** Default fire cap for a `once` step when its announce omits `maxPerTrack`. */
|
|
5802
|
+
var DEFAULT_ONCE_MAX_PER_TRACK = 3;
|
|
5803
|
+
/**
|
|
5804
|
+
* Pure per-(track, step) scheduling state machine for detail-subtree
|
|
5805
|
+
* dispatch. Given a camera's announced child steps (`DetailStepAnnounce[]`,
|
|
5806
|
+
* read off `PipelineInferenceResultPayload.detailSteps`), decides WHEN each
|
|
5807
|
+
* step should run for a given track — independent of transport, I/O, or
|
|
5808
|
+
* timers. The caller drives it with wall-clock `nowMs` and dispatches the
|
|
5809
|
+
* returned `DetailRequest[]`.
|
|
5810
|
+
*/
|
|
5811
|
+
var DetailScheduler = class {
|
|
5812
|
+
tracks = /* @__PURE__ */ new Map();
|
|
5813
|
+
/** Track appeared with class + announce; returns immediate requests. */
|
|
5814
|
+
onTrackStarted(trackId, className, announce, nowMs) {
|
|
5815
|
+
const steps = /* @__PURE__ */ new Map();
|
|
5816
|
+
const requests = [];
|
|
5817
|
+
for (const stepAnnounce of announce) {
|
|
5818
|
+
if (!stepAnnounce.inputClasses.includes(className)) continue;
|
|
5819
|
+
const state = {
|
|
5820
|
+
announce: stepAnnounce,
|
|
5821
|
+
firedCount: 1,
|
|
5822
|
+
lastFiredAt: nowMs,
|
|
5823
|
+
sticky: false,
|
|
5824
|
+
retryPending: false
|
|
5825
|
+
};
|
|
5826
|
+
steps.set(stepAnnounce.stepId, state);
|
|
5827
|
+
requests.push({
|
|
5828
|
+
trackId,
|
|
5829
|
+
stepId: stepAnnounce.stepId,
|
|
5830
|
+
reason: "new-track"
|
|
5831
|
+
});
|
|
5832
|
+
}
|
|
5833
|
+
this.tracks.set(trackId, steps);
|
|
5834
|
+
return requests;
|
|
5835
|
+
}
|
|
5836
|
+
/** Better candidate crop observed for the track. */
|
|
5837
|
+
onCandidateImproved(trackId, nowMs) {
|
|
5838
|
+
const steps = this.tracks.get(trackId);
|
|
5839
|
+
if (!steps) return [];
|
|
5840
|
+
const requests = [];
|
|
5841
|
+
for (const state of steps.values()) {
|
|
5842
|
+
if (state.announce.cadence.trigger !== "improve") continue;
|
|
5843
|
+
if (!this.canFire(state, nowMs)) continue;
|
|
5844
|
+
this.markFired(state, nowMs);
|
|
5845
|
+
requests.push({
|
|
5846
|
+
trackId,
|
|
5847
|
+
stepId: state.announce.stepId,
|
|
5848
|
+
reason: "improve"
|
|
5849
|
+
});
|
|
5850
|
+
}
|
|
5851
|
+
return requests;
|
|
5852
|
+
}
|
|
5853
|
+
/** Periodic tick (call ~1/s). Also carries pending retries for any trigger kind. */
|
|
5854
|
+
tick(nowMs) {
|
|
5855
|
+
const requests = [];
|
|
5856
|
+
for (const [trackId, steps] of this.tracks) for (const state of steps.values()) {
|
|
5857
|
+
if (state.sticky) continue;
|
|
5858
|
+
if (state.retryPending) {
|
|
5859
|
+
if (!this.intervalElapsed(state, nowMs)) continue;
|
|
5860
|
+
if (!this.underMaxPerTrack(state)) {
|
|
5861
|
+
state.retryPending = false;
|
|
5862
|
+
continue;
|
|
5863
|
+
}
|
|
5864
|
+
this.markFired(state, nowMs);
|
|
5865
|
+
requests.push({
|
|
5866
|
+
trackId,
|
|
5867
|
+
stepId: state.announce.stepId,
|
|
5868
|
+
reason: "retry"
|
|
5869
|
+
});
|
|
5870
|
+
continue;
|
|
5871
|
+
}
|
|
5872
|
+
if (state.announce.cadence.trigger !== "periodic") continue;
|
|
5873
|
+
if (!this.canFire(state, nowMs)) continue;
|
|
5874
|
+
this.markFired(state, nowMs);
|
|
5875
|
+
requests.push({
|
|
5876
|
+
trackId,
|
|
5877
|
+
stepId: state.announce.stepId,
|
|
5878
|
+
reason: "periodic"
|
|
5879
|
+
});
|
|
5880
|
+
}
|
|
5881
|
+
return requests;
|
|
5882
|
+
}
|
|
5883
|
+
/**
|
|
5884
|
+
* Result arrived; confidence drives sticky/retry. null = failed (retry per
|
|
5885
|
+
* policy). `_nowMs` is part of the public signature for symmetry with the
|
|
5886
|
+
* other methods but isn't needed here — retry backoff is anchored to
|
|
5887
|
+
* `lastFiredAt` (set when the step was actually dispatched), not to when
|
|
5888
|
+
* its result came back.
|
|
5889
|
+
*/
|
|
5890
|
+
onResult(trackId, stepId, confidence, _nowMs) {
|
|
5891
|
+
const steps = this.tracks.get(trackId);
|
|
5892
|
+
if (!steps) return;
|
|
5893
|
+
const state = steps.get(stepId);
|
|
5894
|
+
if (!state) return;
|
|
5895
|
+
if (state.sticky) return;
|
|
5896
|
+
const { stickyOnConfidence } = state.announce.cadence;
|
|
5897
|
+
if (confidence !== null && stickyOnConfidence !== void 0 && confidence >= stickyOnConfidence) {
|
|
5898
|
+
state.sticky = true;
|
|
5899
|
+
state.retryPending = false;
|
|
5900
|
+
return;
|
|
5901
|
+
}
|
|
5902
|
+
if (confidence === null) {
|
|
5903
|
+
if (this.underMaxPerTrack(state)) state.retryPending = true;
|
|
5904
|
+
return;
|
|
5905
|
+
}
|
|
5906
|
+
if (state.announce.cadence.trigger === "once" && stickyOnConfidence !== void 0) {
|
|
5907
|
+
if (this.underMaxPerTrack(state)) state.retryPending = true;
|
|
5908
|
+
}
|
|
5909
|
+
}
|
|
5910
|
+
onTrackEnded(trackId) {
|
|
5911
|
+
this.tracks.delete(trackId);
|
|
5912
|
+
}
|
|
5913
|
+
canFire(state, nowMs) {
|
|
5914
|
+
if (state.sticky) return false;
|
|
5915
|
+
if (!this.underMaxPerTrack(state)) return false;
|
|
5916
|
+
return this.intervalElapsed(state, nowMs);
|
|
5917
|
+
}
|
|
5918
|
+
intervalElapsed(state, nowMs) {
|
|
5919
|
+
if (state.lastFiredAt === null) return true;
|
|
5920
|
+
const minIntervalMs = state.announce.cadence.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS;
|
|
5921
|
+
return nowMs - state.lastFiredAt >= minIntervalMs;
|
|
5922
|
+
}
|
|
5923
|
+
underMaxPerTrack(state) {
|
|
5924
|
+
const maxPerTrack = state.announce.cadence.maxPerTrack ?? (state.announce.cadence.trigger === "once" ? DEFAULT_ONCE_MAX_PER_TRACK : Infinity);
|
|
5925
|
+
return state.firedCount < maxPerTrack;
|
|
5926
|
+
}
|
|
5927
|
+
markFired(state, nowMs) {
|
|
5928
|
+
state.firedCount += 1;
|
|
5929
|
+
state.lastFiredAt = nowMs;
|
|
5930
|
+
state.retryPending = false;
|
|
5931
|
+
}
|
|
5932
|
+
};
|
|
5933
|
+
//#endregion
|
|
5934
|
+
//#region src/pipeline-analytics/detail-dispatcher.ts
|
|
5935
|
+
/** Throttle for the per-device "detail call failed" warn — one line / minute. */
|
|
5936
|
+
var FAIL_WARN_THROTTLE_MS = 6e4;
|
|
5937
|
+
var TrackDetailDispatcher = class {
|
|
5938
|
+
deps;
|
|
5939
|
+
devices = /* @__PURE__ */ new Map();
|
|
5940
|
+
maxInFlight;
|
|
5941
|
+
tickIntervalMs;
|
|
5942
|
+
disposed = false;
|
|
5943
|
+
constructor(deps) {
|
|
5944
|
+
this.deps = deps;
|
|
5945
|
+
this.maxInFlight = deps.maxInFlightPerDevice ?? 2;
|
|
5946
|
+
this.tickIntervalMs = deps.tickIntervalMs ?? 1e3;
|
|
5947
|
+
}
|
|
5948
|
+
/** A track appeared: record its frame, seed its best-confidence, and dispatch
|
|
5949
|
+
* the scheduler's immediate (new-track) requests. */
|
|
5950
|
+
onTrackStarted(deviceId, trackId, className, announce, frame, nowMs) {
|
|
5951
|
+
if (this.disposed) return;
|
|
5952
|
+
const dev = this.ensureDevice(deviceId);
|
|
5953
|
+
dev.tracks.set(trackId, frame);
|
|
5954
|
+
dev.candidateBest.observe(trackId, frame.confidence, frame.timestamp);
|
|
5955
|
+
const requests = dev.scheduler.onTrackStarted(trackId, className, announce, nowMs);
|
|
5956
|
+
this.enqueue(deviceId, dev, requests);
|
|
5957
|
+
this.ensureTimer(deviceId, dev);
|
|
5958
|
+
}
|
|
5959
|
+
/** A subsequent frame for a live track: refresh its frame + fire
|
|
5960
|
+
* `improve`-cadence steps when the detector confidence strictly improves.
|
|
5961
|
+
*
|
|
5962
|
+
* `announce` is the frame's currently-announced detail chain. When a track
|
|
5963
|
+
* is alive but has NO dispatcher state yet — it existed before `detailSteps`
|
|
5964
|
+
* first appeared (a mid-track redeploy / config change) — this adopts it as a
|
|
5965
|
+
* new track so it starts getting scheduled instead of starving for its whole
|
|
5966
|
+
* life. Idempotent: guarded by the per-track state check, so a track that
|
|
5967
|
+
* already has state is never reset. */
|
|
5968
|
+
onFrame(deviceId, trackId, announce, frame, nowMs) {
|
|
5969
|
+
if (this.disposed) return;
|
|
5970
|
+
const dev = this.devices.get(deviceId);
|
|
5971
|
+
if (!dev || !dev.tracks.has(trackId)) {
|
|
5972
|
+
if (announce.length > 0) this.onTrackStarted(deviceId, trackId, frame.className, announce, frame, nowMs);
|
|
5973
|
+
return;
|
|
5974
|
+
}
|
|
5975
|
+
dev.tracks.set(trackId, frame);
|
|
5976
|
+
if (!dev.candidateBest.observe(trackId, frame.confidence, frame.timestamp)) return;
|
|
5977
|
+
const requests = dev.scheduler.onCandidateImproved(trackId, nowMs);
|
|
5978
|
+
this.enqueue(deviceId, dev, requests);
|
|
5979
|
+
}
|
|
5980
|
+
/** A track ended (durable TTL expiry): drop its scheduler + frame state. Any
|
|
5981
|
+
* queued request for it is discarded at dequeue. */
|
|
5982
|
+
onTrackEnded(deviceId, trackId) {
|
|
5983
|
+
const dev = this.devices.get(deviceId);
|
|
5984
|
+
if (!dev) return;
|
|
5985
|
+
dev.scheduler.onTrackEnded(trackId);
|
|
5986
|
+
dev.tracks.delete(trackId);
|
|
5987
|
+
dev.candidateBest.delete(trackId);
|
|
5988
|
+
if (dev.tracks.size === 0) this.clearTimer(dev);
|
|
5989
|
+
}
|
|
5990
|
+
dispose() {
|
|
5991
|
+
this.disposed = true;
|
|
5992
|
+
for (const dev of this.devices.values()) {
|
|
5993
|
+
this.clearTimer(dev);
|
|
5994
|
+
dev.queue.length = 0;
|
|
5995
|
+
dev.tracks.clear();
|
|
5996
|
+
dev.candidateBest.clear();
|
|
5997
|
+
}
|
|
5998
|
+
this.devices.clear();
|
|
5999
|
+
}
|
|
6000
|
+
ensureDevice(deviceId) {
|
|
6001
|
+
let dev = this.devices.get(deviceId);
|
|
6002
|
+
if (!dev) {
|
|
6003
|
+
dev = {
|
|
6004
|
+
scheduler: new DetailScheduler(),
|
|
6005
|
+
tracks: /* @__PURE__ */ new Map(),
|
|
6006
|
+
candidateBest: new BestDetectionTracker(),
|
|
6007
|
+
queue: [],
|
|
6008
|
+
inFlight: 0,
|
|
6009
|
+
timer: null,
|
|
6010
|
+
lastFailWarnAt: 0
|
|
6011
|
+
};
|
|
6012
|
+
this.devices.set(deviceId, dev);
|
|
6013
|
+
}
|
|
6014
|
+
return dev;
|
|
6015
|
+
}
|
|
6016
|
+
ensureTimer(deviceId, dev) {
|
|
6017
|
+
if (dev.timer) return;
|
|
6018
|
+
const timer = setInterval(() => this.tick(deviceId, dev), this.tickIntervalMs);
|
|
6019
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
6020
|
+
dev.timer = timer;
|
|
6021
|
+
}
|
|
6022
|
+
clearTimer(dev) {
|
|
6023
|
+
if (dev.timer) {
|
|
6024
|
+
clearInterval(dev.timer);
|
|
6025
|
+
dev.timer = null;
|
|
6026
|
+
}
|
|
6027
|
+
}
|
|
6028
|
+
tick(deviceId, dev) {
|
|
6029
|
+
if (this.disposed) return;
|
|
6030
|
+
const requests = dev.scheduler.tick(Date.now());
|
|
6031
|
+
this.enqueue(deviceId, dev, requests);
|
|
6032
|
+
}
|
|
6033
|
+
enqueue(deviceId, dev, requests) {
|
|
6034
|
+
if (requests.length === 0) return;
|
|
6035
|
+
for (const r of requests) dev.queue.push(r);
|
|
6036
|
+
this.pump(deviceId, dev);
|
|
6037
|
+
}
|
|
6038
|
+
pump(deviceId, dev) {
|
|
6039
|
+
while (dev.inFlight < this.maxInFlight && dev.queue.length > 0) {
|
|
6040
|
+
const req = dev.queue.shift();
|
|
6041
|
+
if (req === void 0) break;
|
|
6042
|
+
const frame = dev.tracks.get(req.trackId);
|
|
6043
|
+
if (frame === void 0) continue;
|
|
6044
|
+
dev.inFlight += 1;
|
|
6045
|
+
this.dispatch(deviceId, dev, req, frame).finally(() => {
|
|
6046
|
+
dev.inFlight -= 1;
|
|
6047
|
+
this.pump(deviceId, dev);
|
|
6048
|
+
});
|
|
6049
|
+
}
|
|
6050
|
+
}
|
|
6051
|
+
async dispatch(deviceId, dev, req, frame) {
|
|
6052
|
+
const details = await this.runOnce(deviceId, dev, req, frame);
|
|
6053
|
+
let topScore = null;
|
|
6054
|
+
if (details !== null && details.length > 0) {
|
|
6055
|
+
topScore = details.reduce((max, d) => d.score > max ? d.score : max, Number.NEGATIVE_INFINITY);
|
|
6056
|
+
try {
|
|
6057
|
+
await this.deps.routeResults(deviceId, req.trackId, details, frame);
|
|
6058
|
+
} catch (err) {
|
|
6059
|
+
this.deps.logger.warn("detail result routing failed", {
|
|
6060
|
+
tags: { deviceId },
|
|
6061
|
+
meta: {
|
|
6062
|
+
trackId: req.trackId,
|
|
6063
|
+
stepId: req.stepId,
|
|
6064
|
+
error: String(err)
|
|
6065
|
+
}
|
|
6066
|
+
});
|
|
6067
|
+
}
|
|
6068
|
+
}
|
|
6069
|
+
dev.scheduler.onResult(req.trackId, req.stepId, topScore, Date.now());
|
|
6070
|
+
}
|
|
6071
|
+
/**
|
|
6072
|
+
* Run the request once via the frameHandle, and — on a miss (null OR throw)
|
|
6073
|
+
* — retry ONCE with a `cropJpeg` fallback when one can be captured. Returns
|
|
6074
|
+
* the detail list, or `null` when both attempts fail to produce a result.
|
|
6075
|
+
*/
|
|
6076
|
+
async runOnce(deviceId, dev, req, frame) {
|
|
6077
|
+
const parent = {
|
|
6078
|
+
bbox: { ...frame.bbox },
|
|
6079
|
+
className: frame.className
|
|
6080
|
+
};
|
|
6081
|
+
if (frame.frameHandle !== void 0) try {
|
|
6082
|
+
const primary = await this.deps.runDetailSubtree({
|
|
6083
|
+
deviceId,
|
|
6084
|
+
frameHandle: frame.frameHandle,
|
|
6085
|
+
parent,
|
|
6086
|
+
steps: [req.stepId]
|
|
6087
|
+
}, frame.nodeId);
|
|
6088
|
+
if (primary !== null) return primary.details;
|
|
6089
|
+
} catch (err) {
|
|
6090
|
+
this.deps.logger.debug("detail primary call failed — trying crop fallback", {
|
|
6091
|
+
tags: { deviceId },
|
|
6092
|
+
meta: {
|
|
6093
|
+
trackId: req.trackId,
|
|
6094
|
+
stepId: req.stepId,
|
|
6095
|
+
error: String(err)
|
|
6096
|
+
}
|
|
6097
|
+
});
|
|
6098
|
+
}
|
|
6099
|
+
if (this.deps.captureCropBase64 !== void 0) try {
|
|
6100
|
+
const cropJpeg = await this.deps.captureCropBase64(frame);
|
|
6101
|
+
if (cropJpeg !== null) {
|
|
6102
|
+
const retry = await this.deps.runDetailSubtree({
|
|
6103
|
+
deviceId,
|
|
6104
|
+
cropJpeg,
|
|
6105
|
+
parent,
|
|
6106
|
+
steps: [req.stepId]
|
|
6107
|
+
}, frame.nodeId);
|
|
6108
|
+
if (retry !== null) return retry.details;
|
|
6109
|
+
}
|
|
6110
|
+
} catch (err) {
|
|
6111
|
+
this.deps.logger.debug("detail crop-fallback call failed", {
|
|
6112
|
+
tags: { deviceId },
|
|
6113
|
+
meta: {
|
|
6114
|
+
trackId: req.trackId,
|
|
6115
|
+
stepId: req.stepId,
|
|
6116
|
+
error: String(err)
|
|
6117
|
+
}
|
|
6118
|
+
});
|
|
6119
|
+
}
|
|
6120
|
+
this.warnFailThrottled(deviceId, dev, req);
|
|
6121
|
+
return null;
|
|
6122
|
+
}
|
|
6123
|
+
warnFailThrottled(deviceId, dev, req) {
|
|
6124
|
+
const now = Date.now();
|
|
6125
|
+
if (now - dev.lastFailWarnAt < FAIL_WARN_THROTTLE_MS) return;
|
|
6126
|
+
dev.lastFailWarnAt = now;
|
|
6127
|
+
this.deps.logger.warn("runDetailSubtree produced no result (frame + crop both missed)", {
|
|
6128
|
+
tags: { deviceId },
|
|
6129
|
+
meta: {
|
|
6130
|
+
trackId: req.trackId,
|
|
6131
|
+
stepId: req.stepId,
|
|
6132
|
+
reason: req.reason
|
|
6133
|
+
}
|
|
6134
|
+
});
|
|
6135
|
+
}
|
|
6136
|
+
};
|
|
6137
|
+
//#endregion
|
|
5308
6138
|
//#region src/pipeline-analytics/store/plate-store.ts
|
|
5309
6139
|
var PLATES_COLLECTION = "pipeline-analytics:plates";
|
|
5310
6140
|
var PLATE_COLUMNS = [
|
|
@@ -5620,6 +6450,38 @@ var PlateRecognizer = class {
|
|
|
5620
6450
|
}
|
|
5621
6451
|
};
|
|
5622
6452
|
//#endregion
|
|
6453
|
+
//#region src/shared/frame/shared-frame-resolver.ts
|
|
6454
|
+
function frameHandleKey(h) {
|
|
6455
|
+
return `${h.nodeId}:${h.shmId}:${h.slot}:${h.seq}`;
|
|
6456
|
+
}
|
|
6457
|
+
function createSharedFrameResolver(resolve) {
|
|
6458
|
+
let cache = null;
|
|
6459
|
+
return (handle) => {
|
|
6460
|
+
const key = frameHandleKey(handle);
|
|
6461
|
+
if (cache === null || cache.key !== key) cache = {
|
|
6462
|
+
key,
|
|
6463
|
+
value: resolve(handle)
|
|
6464
|
+
};
|
|
6465
|
+
return cache.value;
|
|
6466
|
+
};
|
|
6467
|
+
}
|
|
6468
|
+
//#endregion
|
|
6469
|
+
//#region src/shared/frame/encode-crop.ts
|
|
6470
|
+
/**
|
|
6471
|
+
* JPEG-encode an already-cropped raw RGB (24-bit) buffer. Used for the
|
|
6472
|
+
* NATIVE-resolution crop path: the decode worker returns the exact ROI pixels
|
|
6473
|
+
* at native res, so there is nothing left to crop — only to encode to the same
|
|
6474
|
+
* JPEG contract `extractCrop` produces (quality 90). Kept separate from
|
|
6475
|
+
* `extractCrop` (which crops a FULL frame) so the native path never re-crops.
|
|
6476
|
+
*/
|
|
6477
|
+
async function encodeRgbCropToJpeg(bytes, width, height) {
|
|
6478
|
+
return sharp(Buffer.from(bytes), { raw: {
|
|
6479
|
+
width,
|
|
6480
|
+
height,
|
|
6481
|
+
channels: 3
|
|
6482
|
+
} }).jpeg({ quality: 90 }).toBuffer();
|
|
6483
|
+
}
|
|
6484
|
+
//#endregion
|
|
5623
6485
|
//#region src/pipeline-analytics/pipeline/event-child-crops.ts
|
|
5624
6486
|
/**
|
|
5625
6487
|
* Compute Intersection-over-Union between two (x,y,w,h) bounding boxes.
|
|
@@ -5917,6 +6779,12 @@ function createEventMediaHandler(deps) {
|
|
|
5917
6779
|
* surface to turn the refinement pipeline on/off for a camera.
|
|
5918
6780
|
*/
|
|
5919
6781
|
var TTL_SWEEP_INTERVAL_MS = 5e3;
|
|
6782
|
+
/** Padding applied to the parent bbox for the `cropJpeg` retry fallback when a
|
|
6783
|
+
* scheduled detail call's frameHandle lease is already gone. */
|
|
6784
|
+
var DETAIL_FALLBACK_CROP_PADDING = .15;
|
|
6785
|
+
/** How long the active CLIP model id (from the embedding-encoder) is cached
|
|
6786
|
+
* before re-reading. */
|
|
6787
|
+
var CLIP_MODEL_ID_CACHE_TTL_MS = 6e4;
|
|
5920
6788
|
var SETTINGS_CACHE_TTL_MS = 5e3;
|
|
5921
6789
|
/** §5 best-frame: a track's `thumbnail` is overwritten only when the current
|
|
5922
6790
|
* detection confidence beats the held best by at least this margin (hysteresis
|
|
@@ -5924,6 +6792,18 @@ var SETTINGS_CACHE_TTL_MS = 5e3;
|
|
|
5924
6792
|
var BEST_FRAME_HYSTERESIS = .05;
|
|
5925
6793
|
/** §5 best-frame: at most one best-thumbnail capture per this interval per track. */
|
|
5926
6794
|
var BEST_FRAME_MIN_GAP_MS = 2e3;
|
|
6795
|
+
/** Design B: cap (px) on the width of the native KEY FRAME (full-frame native
|
|
6796
|
+
* capture). Native resolution is the point, but a full 4K RGB surface over the
|
|
6797
|
+
* transport per new-best is wasteful for a web detail view — 1920px keeps a
|
|
6798
|
+
* sharp native frame while bounding the copy (a miss falls back to the
|
|
6799
|
+
* detection-res frame, which is already ≤640px). */
|
|
6800
|
+
var KEYFRAME_NATIVE_MAX_WIDTH = 1920;
|
|
6801
|
+
/** getKeyEvents: max completed tracks pulled from a window before importance
|
|
6802
|
+
* ranking. Ordering is by importance (not firstSeen) and legacy rows score on
|
|
6803
|
+
* read, so we over-fetch candidates and trim to `limit` after sorting. */
|
|
6804
|
+
var KEY_EVENT_CANDIDATE_CAP = 500;
|
|
6805
|
+
/** getKeyEvents: default page size when the caller omits `limit`. */
|
|
6806
|
+
var KEY_EVENT_DEFAULT_LIMIT = 50;
|
|
5927
6807
|
/** Cluster setting key (in the centralized addon store) selecting the SINGLE
|
|
5928
6808
|
* node that runs post-analysis (event/media/audio/motion generation). All
|
|
5929
6809
|
* other nodes are fully inert. No multi-node balancing. Default: the hub. */
|
|
@@ -5937,6 +6817,15 @@ var RETENTION_SWEEP_INTERVAL_MS = 5 * 6e4;
|
|
|
5937
6817
|
var AUDIO_EVENT_HEARTBEAT_MS = 5e3;
|
|
5938
6818
|
var MOTION_EVENT_HEARTBEAT_MS = 5e3;
|
|
5939
6819
|
/**
|
|
6820
|
+
* Decode a base64 little-endian float32 vector (the `DetailResult.embedding`
|
|
6821
|
+
* wire encoding produced by `runDetailSubtree`) back into a plain number[].
|
|
6822
|
+
*/
|
|
6823
|
+
function decodeEmbeddingBase64(base64) {
|
|
6824
|
+
const bytes = Buffer.from(base64, "base64");
|
|
6825
|
+
const view = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
|
|
6826
|
+
return Array.from(view);
|
|
6827
|
+
}
|
|
6828
|
+
/**
|
|
5940
6829
|
* Re-home the global analytics sections into the per-device `Analytics`
|
|
5941
6830
|
* top-tab. Every section defaults to the `Analytics` tab (so the
|
|
5942
6831
|
* device-manager aggregator groups them) AND is marked
|
|
@@ -5986,6 +6875,14 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
5986
6875
|
plateStore = null;
|
|
5987
6876
|
plateRecognizer = null;
|
|
5988
6877
|
objectEmbeddingStore = null;
|
|
6878
|
+
/** Two-plane detail scheduler/dispatcher: per-track on-demand face/clip/
|
|
6879
|
+
* classifier/plate enrichment via `pipelineRunner.runDetailSubtree`. Replaces
|
|
6880
|
+
* the per-frame child consumption the executor no longer emits. */
|
|
6881
|
+
detailDispatcher = null;
|
|
6882
|
+
/** Active CLIP image-embedding model id (== the text encoder's), TTL-cached.
|
|
6883
|
+
* Stamped on object-embedding rows from the detail plane so semantic search's
|
|
6884
|
+
* same-model gate keeps matching. */
|
|
6885
|
+
clipModelIdCache = null;
|
|
5989
6886
|
/** Frame-based event/track media (crop + boxed full-frame) from the
|
|
5990
6887
|
* detection-pipeline DECODED frame — the ONLY image source (never the
|
|
5991
6888
|
* snapshot cap). Null when shm frame access is unavailable. */
|
|
@@ -6043,6 +6940,23 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6043
6940
|
hysteresis: BEST_FRAME_HYSTERESIS,
|
|
6044
6941
|
minGapMs: BEST_FRAME_MIN_GAP_MS
|
|
6045
6942
|
});
|
|
6943
|
+
/** Best (highest-confidence) CLIP-object detection per track — drives ONE
|
|
6944
|
+
* tight object-crop capture whose media key is written onto the embedding
|
|
6945
|
+
* row (so a semantic-search hit's thumbnail IS the embedded crop). Shares the
|
|
6946
|
+
* unified best-per-track decision (`TrackBestSelector`); the persistent
|
|
6947
|
+
* cross-restart "best" gate stays in `ObjectEmbeddingStore.upsertIfBetter`. */
|
|
6948
|
+
objectEmbeddingBestSelector = new TrackBestSelector();
|
|
6949
|
+
/** Design B: the track's shared native key-frame media key, captured at the
|
|
6950
|
+
* best-detection moment (object-embedding best path). Read by the face path
|
|
6951
|
+
* at track end so a face row links to the SAME single key frame. Cleared on
|
|
6952
|
+
* track end. */
|
|
6953
|
+
keyFrameKeyByTrackId = /* @__PURE__ */ new Map();
|
|
6954
|
+
/** The shared crop extractor (native-res first, detection-frame fallback),
|
|
6955
|
+
* captured in the constructor so `processFrame` can crop object thumbnails in
|
|
6956
|
+
* the same live-frame window as the face/plate/event-media captures. The
|
|
6957
|
+
* optional `maxWidth` caps the native crop width (used for the full-frame key
|
|
6958
|
+
* frame so a 4K native surface never floods the transport). */
|
|
6959
|
+
captureCrop = null;
|
|
6046
6960
|
shuttingDown = false;
|
|
6047
6961
|
/** True only on the cluster's designated post-processing node. When false the
|
|
6048
6962
|
* addon subscribes to NOTHING — fully inert (no event/media generation). */
|
|
@@ -6142,23 +7056,62 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6142
7056
|
});
|
|
6143
7057
|
const ownNodeIdForFaces = ownNodeId;
|
|
6144
7058
|
const frameReadersForFaces = this.frameReaders;
|
|
6145
|
-
const
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
7059
|
+
const pipelineRunnerApi = api.pipelineRunner;
|
|
7060
|
+
const cropMetricLogger = logger.child("NativeCrop");
|
|
7061
|
+
let nativeHits = 0;
|
|
7062
|
+
let nativeFallbacks = 0;
|
|
7063
|
+
let lastCropMetricAt = 0;
|
|
7064
|
+
const NATIVE_CROP_METRIC_INTERVAL_MS = 3e4;
|
|
7065
|
+
const bumpCropMetric = (hit) => {
|
|
7066
|
+
if (hit) nativeHits += 1;
|
|
7067
|
+
else nativeFallbacks += 1;
|
|
7068
|
+
const now = Date.now();
|
|
7069
|
+
if (now - lastCropMetricAt < NATIVE_CROP_METRIC_INTERVAL_MS) return;
|
|
7070
|
+
lastCropMetricAt = now;
|
|
7071
|
+
cropMetricLogger.info("native-crop window", { meta: {
|
|
7072
|
+
nativeHits,
|
|
7073
|
+
detectionFrameFallbacks: nativeFallbacks
|
|
7074
|
+
} });
|
|
7075
|
+
};
|
|
7076
|
+
const tryNativeCrop = async (frameHandle, paddedNorm, maxWidth) => {
|
|
7077
|
+
if (!pipelineRunnerApi?.getNativeCrop) return null;
|
|
7078
|
+
try {
|
|
7079
|
+
const native = await pipelineRunnerApi.getNativeCrop.query({
|
|
7080
|
+
handle: frameHandle,
|
|
7081
|
+
bbox: paddedNorm,
|
|
7082
|
+
...maxWidth !== void 0 ? { maxWidth } : {}
|
|
7083
|
+
}, nodePin(frameHandle.nodeId));
|
|
7084
|
+
if (!native || native.width <= 0 || native.height <= 0) return null;
|
|
7085
|
+
return await encodeRgbCropToJpeg(Buffer.from(native.bytes), native.width, native.height);
|
|
7086
|
+
} catch (err) {
|
|
7087
|
+
cropMetricLogger.debug("native crop request failed — falling back", { meta: { error: errMsg(err) } });
|
|
7088
|
+
return null;
|
|
7089
|
+
}
|
|
7090
|
+
};
|
|
7091
|
+
const resolveFrameShared = createSharedFrameResolver((frameHandle) => resolveFrame(frameHandle, {
|
|
7092
|
+
ownNodeId: ownNodeIdForFaces,
|
|
7093
|
+
readers: frameReadersForFaces,
|
|
7094
|
+
getRemoteFrame
|
|
7095
|
+
}));
|
|
7096
|
+
const captureCrop = async (frameHandle, bbox, frameWidth, frameHeight, padding, maxWidth) => {
|
|
6153
7097
|
const paddedNorm = padBbox({
|
|
6154
7098
|
x: bbox.x / frameWidth,
|
|
6155
7099
|
y: bbox.y / frameHeight,
|
|
6156
7100
|
w: bbox.w / frameWidth,
|
|
6157
7101
|
h: bbox.h / frameHeight
|
|
6158
7102
|
}, padding);
|
|
6159
|
-
const
|
|
7103
|
+
const nativeCrop = await tryNativeCrop(frameHandle, paddedNorm, maxWidth);
|
|
7104
|
+
if (nativeCrop) {
|
|
7105
|
+
bumpCropMetric(true);
|
|
7106
|
+
return nativeCrop;
|
|
7107
|
+
}
|
|
7108
|
+
bumpCropMetric(false);
|
|
7109
|
+
const decoded = await resolveFrameShared(frameHandle);
|
|
7110
|
+
if (!decoded || decoded.format !== "rgb") return null;
|
|
7111
|
+
const { crop } = await extractCrop(Buffer.isBuffer(decoded.data) ? decoded.data : Buffer.from(decoded.data), decoded.width, decoded.height, paddedNorm);
|
|
6160
7112
|
return crop;
|
|
6161
7113
|
};
|
|
7114
|
+
this.captureCrop = captureCrop;
|
|
6162
7115
|
this.faceRecognizer = new FaceRecognizer({
|
|
6163
7116
|
identityStore: this.identityStore,
|
|
6164
7117
|
faceStore: this.faceStore,
|
|
@@ -6166,6 +7119,16 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6166
7119
|
trackStore: this.trackStore,
|
|
6167
7120
|
eventStore: this.eventStore,
|
|
6168
7121
|
captureCrop,
|
|
7122
|
+
recomputeImportance: (trackId) => {
|
|
7123
|
+
const trackStore = this.trackStore;
|
|
7124
|
+
const eventStore = this.eventStore;
|
|
7125
|
+
if (!trackStore || !eventStore) return Promise.resolve();
|
|
7126
|
+
return recomputeTrackImportance({
|
|
7127
|
+
trackStore,
|
|
7128
|
+
eventStore
|
|
7129
|
+
}, trackId);
|
|
7130
|
+
},
|
|
7131
|
+
getKeyFrameMediaKey: (trackId) => this.keyFrameKeyByTrackId.get(trackId),
|
|
6169
7132
|
logger: logger.child("FaceRecognizer")
|
|
6170
7133
|
});
|
|
6171
7134
|
this.faceRecognizer.refreshGallery();
|
|
@@ -6179,6 +7142,21 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6179
7142
|
store: api.settingsStore,
|
|
6180
7143
|
logger: logger.child("ObjectEmbeddingStore")
|
|
6181
7144
|
});
|
|
7145
|
+
const runnerApi = api.pipelineRunner;
|
|
7146
|
+
this.detailDispatcher = new TrackDetailDispatcher({
|
|
7147
|
+
logger: logger.child("DetailDispatcher"),
|
|
7148
|
+
runDetailSubtree: async (input, nodeId) => {
|
|
7149
|
+
if (!runnerApi?.runDetailSubtree) return null;
|
|
7150
|
+
if (nodeId !== void 0) return runnerApi.runDetailSubtree.mutate(input, nodePin(nodeId));
|
|
7151
|
+
return runnerApi.runDetailSubtree.mutate(input);
|
|
7152
|
+
},
|
|
7153
|
+
routeResults: (deviceId, trackId, details, frame) => this.routeDetailResults(deviceId, trackId, details, frame),
|
|
7154
|
+
captureCropBase64: async (frame) => {
|
|
7155
|
+
if (frame.frameHandle === void 0 || !this.captureCrop) return null;
|
|
7156
|
+
const buf = await this.captureCrop(frame.frameHandle, { ...frame.bbox }, frame.frameWidth, frame.frameHeight, DETAIL_FALLBACK_CROP_PADDING);
|
|
7157
|
+
return buf ? buf.toString("base64") : null;
|
|
7158
|
+
}
|
|
7159
|
+
});
|
|
6182
7160
|
this.bindingCache = new BindingCache({
|
|
6183
7161
|
api,
|
|
6184
7162
|
logger: logger.child("BindingCache")
|
|
@@ -6203,7 +7181,7 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6203
7181
|
try {
|
|
6204
7182
|
const handler = createEventMediaHandler({ getMedia: async (id) => {
|
|
6205
7183
|
try {
|
|
6206
|
-
return await this.
|
|
7184
|
+
return await this.readMediaByEventOrKey(id);
|
|
6207
7185
|
} catch (err) {
|
|
6208
7186
|
this.ctx.logger.warn("readEventThumbnail failed", { meta: {
|
|
6209
7187
|
eventId: id,
|
|
@@ -6583,10 +7561,13 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6583
7561
|
}
|
|
6584
7562
|
this.zoneAnalytics?.destroy();
|
|
6585
7563
|
this.audioMetrics?.destroy();
|
|
7564
|
+
this.detailDispatcher?.dispose();
|
|
7565
|
+
this.detailDispatcher = null;
|
|
6586
7566
|
this.processors.clear();
|
|
6587
7567
|
this.lastActiveTrackIds.clear();
|
|
6588
7568
|
this.dropoutSkipsByKey.clear();
|
|
6589
7569
|
this.bestFrameTracker.clear();
|
|
7570
|
+
this.objectEmbeddingBestSelector.clear();
|
|
6590
7571
|
this.levelStateByDevice.clear();
|
|
6591
7572
|
this.settingsCacheByDevice.clear();
|
|
6592
7573
|
this.sensitivityCacheByDevice.clear();
|
|
@@ -6606,7 +7587,7 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6606
7587
|
async handleInferenceResult(data) {
|
|
6607
7588
|
if (this.shuttingDown) return;
|
|
6608
7589
|
const { deviceId, frame } = data;
|
|
6609
|
-
await this.processFrame(deviceId, frame, "pipeline", data.frameHandle);
|
|
7590
|
+
await this.processFrame(deviceId, frame, "pipeline", data.frameHandle, data.detailSteps);
|
|
6610
7591
|
}
|
|
6611
7592
|
/**
|
|
6612
7593
|
* Run one detection frame through the analysis layers for a given
|
|
@@ -6616,7 +7597,7 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6616
7597
|
* tracking/zone/event state never crosses between sources. Emits the
|
|
6617
7598
|
* SAME canonical events, distinguished only by `source`.
|
|
6618
7599
|
*/
|
|
6619
|
-
async processFrame(deviceId, frame, source, frameHandle) {
|
|
7600
|
+
async processFrame(deviceId, frame, source, frameHandle, detailSteps) {
|
|
6620
7601
|
if (this.shuttingDown) return;
|
|
6621
7602
|
if (!await this.bindingCache.isActive(deviceId)) return;
|
|
6622
7603
|
const key = this.procKey(deviceId, source);
|
|
@@ -6728,6 +7709,26 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6728
7709
|
} });
|
|
6729
7710
|
}
|
|
6730
7711
|
this.lastActiveTrackIds.set(key, currentTrackIds);
|
|
7712
|
+
if (this.detailDispatcher && detailSteps && detailSteps.length > 0) {
|
|
7713
|
+
const dispatcher = this.detailDispatcher;
|
|
7714
|
+
const steps = detailSteps;
|
|
7715
|
+
for (const t of result.tracked) {
|
|
7716
|
+
const detailFrame = {
|
|
7717
|
+
bbox: { ...t.bbox },
|
|
7718
|
+
frameWidth: result.frameWidth,
|
|
7719
|
+
frameHeight: result.frameHeight,
|
|
7720
|
+
className: t.className,
|
|
7721
|
+
confidence: t.confidence,
|
|
7722
|
+
timestamp: result.timestamp,
|
|
7723
|
+
...frameHandle !== void 0 ? {
|
|
7724
|
+
frameHandle,
|
|
7725
|
+
nodeId: frameHandle.nodeId
|
|
7726
|
+
} : {}
|
|
7727
|
+
};
|
|
7728
|
+
if (prevIds.has(t.trackId)) dispatcher.onFrame(deviceId, t.trackId, steps, detailFrame, result.timestamp);
|
|
7729
|
+
else dispatcher.onTrackStarted(deviceId, t.trackId, t.className, steps, detailFrame, result.timestamp);
|
|
7730
|
+
}
|
|
7731
|
+
}
|
|
6731
7732
|
if (newTrackCount > 0 || lostTrackCount > 0 || result.objectEvents.length > 0) {
|
|
6732
7733
|
const byState = {};
|
|
6733
7734
|
for (const t of result.tracked) byState[t.state] = (byState[t.state] ?? 0) + 1;
|
|
@@ -6742,16 +7743,18 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6742
7743
|
} });
|
|
6743
7744
|
}
|
|
6744
7745
|
await Promise.all(result.objectEvents.map((e) => this.eventStore.insertObject(e)));
|
|
6745
|
-
|
|
6746
|
-
|
|
7746
|
+
const objectEmbeddingBests = [];
|
|
7747
|
+
if (this.objectEmbeddingStore) for (const t of result.tracked) {
|
|
7748
|
+
if (!isClipObjectEmbedding(t)) continue;
|
|
7749
|
+
if (this.objectEmbeddingBestSelector.observe({
|
|
6747
7750
|
trackId: t.trackId,
|
|
6748
|
-
|
|
6749
|
-
|
|
7751
|
+
confidence: t.confidence,
|
|
7752
|
+
atMs: result.timestamp,
|
|
6750
7753
|
className: t.className,
|
|
7754
|
+
bbox: t.bbox,
|
|
6751
7755
|
embedding: t.embedding,
|
|
6752
|
-
|
|
6753
|
-
|
|
6754
|
-
});
|
|
7756
|
+
embeddingModelId: t.embeddingModelId
|
|
7757
|
+
})) objectEmbeddingBests.push(t);
|
|
6755
7758
|
}
|
|
6756
7759
|
const faceSettings = this.faceRecognizer ? await this.resolveDeviceFaceSettings(deviceId) : null;
|
|
6757
7760
|
const faceGloballyEnabled = this.faceRecognizer ? await this.resolveGlobalFaceEnabled() : false;
|
|
@@ -6824,6 +7827,7 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6824
7827
|
cropPadding: mediaSettings.cropPadding,
|
|
6825
7828
|
...frameHandle !== void 0 ? { frameHandle } : {}
|
|
6826
7829
|
});
|
|
7830
|
+
if (objectEmbeddingBests.length > 0) this.persistObjectEmbeddingBests(deviceId, result.timestamp, objectEmbeddingBests, frameHandle, result.frameWidth, result.frameHeight, mediaSettings.cropPadding);
|
|
6827
7831
|
for (const e of result.objectEvents) this.ctx.eventBus.emit({
|
|
6828
7832
|
id: `pa-${e.id}`,
|
|
6829
7833
|
timestamp: new Date(e.timestamp),
|
|
@@ -6936,6 +7940,142 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6936
7940
|
return settings;
|
|
6937
7941
|
}
|
|
6938
7942
|
/**
|
|
7943
|
+
* Route one track's `runDetailSubtree` results (two-plane detail dispatch)
|
|
7944
|
+
* into the EXISTING per-track consumers, discriminated by payload SHAPE:
|
|
7945
|
+
* • embedding + alignedCropJpeg → FACE (arcface): the FaceRecognizer's
|
|
7946
|
+
* candidate/best-face/gallery/keyFrame path (input source cut-over);
|
|
7947
|
+
* • embedding only → CLIP object embedding → the object-embedding store
|
|
7948
|
+
* (semantic search);
|
|
7949
|
+
* • label only → classifier answer / plate OCR text → the track's
|
|
7950
|
+
* enrichment label the notifier/UI already read.
|
|
7951
|
+
* Best-effort (D8): a per-detail failure is logged and never propagated.
|
|
7952
|
+
*/
|
|
7953
|
+
async routeDetailResults(deviceId, trackId, details, frame) {
|
|
7954
|
+
for (const d of details) try {
|
|
7955
|
+
if (d.className === "face" && d.embedding !== void 0) await this.routeFaceDetail(deviceId, trackId, d, frame);
|
|
7956
|
+
else if (d.embedding !== void 0) await this.routeClipDetail(deviceId, trackId, d, frame.timestamp);
|
|
7957
|
+
else if (d.label !== void 0 && d.label.length > 0) await this.applyTrackEnrichmentLabel(deviceId, trackId, d.label);
|
|
7958
|
+
} catch (err) {
|
|
7959
|
+
this.ctx.logger.warn("detail result route failed", {
|
|
7960
|
+
tags: { deviceId },
|
|
7961
|
+
meta: {
|
|
7962
|
+
trackId,
|
|
7963
|
+
stepId: d.stepId,
|
|
7964
|
+
error: errMsg(err)
|
|
7965
|
+
}
|
|
7966
|
+
});
|
|
7967
|
+
}
|
|
7968
|
+
}
|
|
7969
|
+
/** Face-embedding detail → the FaceRecognizer (same gate + logic as the
|
|
7970
|
+
* former per-frame face path; only the input source moved). */
|
|
7971
|
+
async routeFaceDetail(deviceId, trackId, detail, frame) {
|
|
7972
|
+
if (!this.faceRecognizer || detail.embedding === void 0) return;
|
|
7973
|
+
if (!await this.resolveGlobalFaceEnabled()) return;
|
|
7974
|
+
const [settings, media] = await Promise.all([this.resolveDeviceFaceSettings(deviceId), this.resolveDeviceMediaSettings(deviceId)]);
|
|
7975
|
+
await this.faceRecognizer.ingestFaceDetail({
|
|
7976
|
+
deviceId,
|
|
7977
|
+
trackId,
|
|
7978
|
+
timestamp: frame.timestamp,
|
|
7979
|
+
frameWidth: frame.frameWidth,
|
|
7980
|
+
frameHeight: frame.frameHeight,
|
|
7981
|
+
score: detail.score,
|
|
7982
|
+
embedding: decodeEmbeddingBase64(detail.embedding),
|
|
7983
|
+
parentBbox: { ...frame.bbox },
|
|
7984
|
+
...detail.bbox !== void 0 ? { faceBbox: { ...detail.bbox } } : {},
|
|
7985
|
+
...detail.alignedCropJpeg !== void 0 ? { alignedCropJpeg: detail.alignedCropJpeg } : {},
|
|
7986
|
+
settings,
|
|
7987
|
+
cropPadding: media.cropPadding,
|
|
7988
|
+
...frame.frameHandle !== void 0 ? { frameHandle: frame.frameHandle } : {}
|
|
7989
|
+
});
|
|
7990
|
+
}
|
|
7991
|
+
/** CLIP object-embedding detail → the object-embedding store (semantic
|
|
7992
|
+
* search). Stamps the active encoder's model id so the same-model search
|
|
7993
|
+
* gate keeps matching. */
|
|
7994
|
+
async routeClipDetail(deviceId, trackId, detail, timestamp) {
|
|
7995
|
+
const store = this.objectEmbeddingStore;
|
|
7996
|
+
if (!store || detail.embedding === void 0) return;
|
|
7997
|
+
const modelId = await this.resolveClipModelId();
|
|
7998
|
+
if (modelId === null) {
|
|
7999
|
+
this.ctx.logger.debug("clip detail dropped — no active embedding model id", {
|
|
8000
|
+
tags: { deviceId },
|
|
8001
|
+
meta: {
|
|
8002
|
+
trackId,
|
|
8003
|
+
stepId: detail.stepId
|
|
8004
|
+
}
|
|
8005
|
+
});
|
|
8006
|
+
return;
|
|
8007
|
+
}
|
|
8008
|
+
await store.upsertIfBetter({
|
|
8009
|
+
trackId,
|
|
8010
|
+
deviceId,
|
|
8011
|
+
timestamp,
|
|
8012
|
+
className: detail.className,
|
|
8013
|
+
embedding: decodeEmbeddingBase64(detail.embedding),
|
|
8014
|
+
modelId,
|
|
8015
|
+
confidence: detail.score
|
|
8016
|
+
});
|
|
8017
|
+
}
|
|
8018
|
+
/** Classifier answer / plate OCR text → the track's enrichment label
|
|
8019
|
+
* (TrackStore + persisted events + importance), mirroring the FaceRecognizer
|
|
8020
|
+
* label-propagation path. */
|
|
8021
|
+
async applyTrackEnrichmentLabel(deviceId, trackId, label) {
|
|
8022
|
+
try {
|
|
8023
|
+
await this.trackStore?.setLabel(trackId, label);
|
|
8024
|
+
} catch (err) {
|
|
8025
|
+
this.ctx.logger.warn("detail label setLabel failed", {
|
|
8026
|
+
tags: { deviceId },
|
|
8027
|
+
meta: {
|
|
8028
|
+
trackId,
|
|
8029
|
+
error: errMsg(err)
|
|
8030
|
+
}
|
|
8031
|
+
});
|
|
8032
|
+
}
|
|
8033
|
+
try {
|
|
8034
|
+
await this.eventStore?.setLabelForTrack(trackId, label);
|
|
8035
|
+
} catch (err) {
|
|
8036
|
+
this.ctx.logger.warn("detail label setLabelForTrack failed", {
|
|
8037
|
+
tags: { deviceId },
|
|
8038
|
+
meta: {
|
|
8039
|
+
trackId,
|
|
8040
|
+
error: errMsg(err)
|
|
8041
|
+
}
|
|
8042
|
+
});
|
|
8043
|
+
}
|
|
8044
|
+
try {
|
|
8045
|
+
const trackStore = this.trackStore;
|
|
8046
|
+
const eventStore = this.eventStore;
|
|
8047
|
+
if (trackStore && eventStore) await recomputeTrackImportance({
|
|
8048
|
+
trackStore,
|
|
8049
|
+
eventStore
|
|
8050
|
+
}, trackId);
|
|
8051
|
+
} catch (err) {
|
|
8052
|
+
this.ctx.logger.debug("detail label recomputeImportance failed", {
|
|
8053
|
+
tags: { deviceId },
|
|
8054
|
+
meta: {
|
|
8055
|
+
trackId,
|
|
8056
|
+
error: errMsg(err)
|
|
8057
|
+
}
|
|
8058
|
+
});
|
|
8059
|
+
}
|
|
8060
|
+
}
|
|
8061
|
+
/** Active CLIP image-embedding model id (== the text encoder's), TTL-cached.
|
|
8062
|
+
* Returns null when the embedding-encoder cap is unavailable. */
|
|
8063
|
+
async resolveClipModelId() {
|
|
8064
|
+
const now = Date.now();
|
|
8065
|
+
if (this.clipModelIdCache && now < this.clipModelIdCache.expiresAt) return this.clipModelIdCache.value;
|
|
8066
|
+
let value = null;
|
|
8067
|
+
try {
|
|
8068
|
+
value = (await this.ctx.api.embeddingEncoder.getInfo.query())?.modelId ?? null;
|
|
8069
|
+
} catch (err) {
|
|
8070
|
+
this.ctx.logger.debug("resolveClipModelId: getInfo failed", { meta: { error: errMsg(err) } });
|
|
8071
|
+
}
|
|
8072
|
+
this.clipModelIdCache = {
|
|
8073
|
+
value,
|
|
8074
|
+
expiresAt: now + CLIP_MODEL_ID_CACHE_TTL_MS
|
|
8075
|
+
};
|
|
8076
|
+
return value;
|
|
8077
|
+
}
|
|
8078
|
+
/**
|
|
6939
8079
|
* §5 — decide which active tracks need periodic media THIS frame. Pure over
|
|
6940
8080
|
* TrackStore.lastSnapshotAt + the per-track best-confidence map:
|
|
6941
8081
|
* • `snapshot` (append) + `lastFrame` (rolling overwrite) fire together on
|
|
@@ -6945,6 +8085,83 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
6945
8085
|
* NOT gated by saveThumbnails (a single best still-frame is always useful).
|
|
6946
8086
|
* The per-track best-confidence map is updated here as a side effect.
|
|
6947
8087
|
*/
|
|
8088
|
+
/**
|
|
8089
|
+
* Capture the tight object crop for each new-best CLIP track (native-res via
|
|
8090
|
+
* the shared extractor) and upsert the embedding row with that crop's media
|
|
8091
|
+
* key. The crop uses `putReplacing` so exactly ONE object crop is kept per
|
|
8092
|
+
* track (the current peak). The embedding is upserted even when no crop was
|
|
8093
|
+
* captured (no frame handle) so semantic search still works — the crop just
|
|
8094
|
+
* enhances the search-hit thumbnail. `upsertIfBetter` remains the durable
|
|
8095
|
+
* cross-restart best gate (R3). Best-effort; issued in the live-frame window.
|
|
8096
|
+
*/
|
|
8097
|
+
async persistObjectEmbeddingBests(deviceId, timestamp, bests, frameHandle, frameWidth, frameHeight, cropPadding) {
|
|
8098
|
+
const store = this.objectEmbeddingStore;
|
|
8099
|
+
if (!store) return;
|
|
8100
|
+
await Promise.all(bests.map(async (t) => {
|
|
8101
|
+
if (!isClipObjectEmbedding(t)) return;
|
|
8102
|
+
let mediaKey;
|
|
8103
|
+
let keyFrameMediaKey;
|
|
8104
|
+
if (frameHandle !== void 0 && this.captureCrop && this.mediaStore) {
|
|
8105
|
+
try {
|
|
8106
|
+
const crop = await this.captureCrop(frameHandle, t.bbox, frameWidth, frameHeight, cropPadding);
|
|
8107
|
+
if (crop) mediaKey = await this.mediaStore.putReplacing({
|
|
8108
|
+
deviceId,
|
|
8109
|
+
ownerKind: "track",
|
|
8110
|
+
ownerId: t.trackId,
|
|
8111
|
+
kind: "crop",
|
|
8112
|
+
timestamp,
|
|
8113
|
+
data: crop
|
|
8114
|
+
});
|
|
8115
|
+
} catch (err) {
|
|
8116
|
+
this.ctx.logger.debug("object-embedding crop capture failed", {
|
|
8117
|
+
tags: { deviceId },
|
|
8118
|
+
meta: {
|
|
8119
|
+
trackId: t.trackId,
|
|
8120
|
+
error: errMsg(err)
|
|
8121
|
+
}
|
|
8122
|
+
});
|
|
8123
|
+
}
|
|
8124
|
+
try {
|
|
8125
|
+
const keyFrame = await this.captureCrop(frameHandle, {
|
|
8126
|
+
x: 0,
|
|
8127
|
+
y: 0,
|
|
8128
|
+
w: frameWidth,
|
|
8129
|
+
h: frameHeight
|
|
8130
|
+
}, frameWidth, frameHeight, 0, KEYFRAME_NATIVE_MAX_WIDTH);
|
|
8131
|
+
if (keyFrame) {
|
|
8132
|
+
keyFrameMediaKey = await this.mediaStore.putReplacing({
|
|
8133
|
+
deviceId,
|
|
8134
|
+
ownerKind: "track",
|
|
8135
|
+
ownerId: t.trackId,
|
|
8136
|
+
kind: "keyFrame",
|
|
8137
|
+
timestamp,
|
|
8138
|
+
data: keyFrame
|
|
8139
|
+
});
|
|
8140
|
+
this.keyFrameKeyByTrackId.set(t.trackId, keyFrameMediaKey);
|
|
8141
|
+
}
|
|
8142
|
+
} catch (err) {
|
|
8143
|
+
this.ctx.logger.debug("key-frame capture failed", {
|
|
8144
|
+
tags: { deviceId },
|
|
8145
|
+
meta: {
|
|
8146
|
+
trackId: t.trackId,
|
|
8147
|
+
error: errMsg(err)
|
|
8148
|
+
}
|
|
8149
|
+
});
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
8152
|
+
await store.upsertIfBetter({
|
|
8153
|
+
trackId: t.trackId,
|
|
8154
|
+
deviceId,
|
|
8155
|
+
timestamp,
|
|
8156
|
+
className: t.className,
|
|
8157
|
+
embedding: t.embedding,
|
|
8158
|
+
modelId: t.embeddingModelId,
|
|
8159
|
+
confidence: t.confidence,
|
|
8160
|
+
...mediaKey !== void 0 ? { mediaKey } : {},
|
|
8161
|
+
...keyFrameMediaKey !== void 0 ? { keyFrameMediaKey } : {}
|
|
8162
|
+
});
|
|
8163
|
+
}));
|
|
8164
|
+
}
|
|
6948
8165
|
buildSnapshotTargets(tracked, timestamp, media) {
|
|
6949
8166
|
const targets = [];
|
|
6950
8167
|
for (const t of tracked) {
|
|
@@ -7216,9 +8433,37 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
7216
8433
|
positions: t.positions.length
|
|
7217
8434
|
}
|
|
7218
8435
|
});
|
|
7219
|
-
this.faceRecognizer?.onTrackEnd(t.deviceId, t.trackId);
|
|
8436
|
+
const faceEnd = this.faceRecognizer?.onTrackEnd(t.deviceId, t.trackId);
|
|
8437
|
+
const dropKeyFrameKey = () => {
|
|
8438
|
+
this.keyFrameKeyByTrackId.delete(t.trackId);
|
|
8439
|
+
};
|
|
8440
|
+
if (faceEnd) faceEnd.finally(dropKeyFrameKey);
|
|
8441
|
+
else dropKeyFrameKey();
|
|
7220
8442
|
this.plateRecognizer?.onTrackEnd(t.deviceId, t.trackId);
|
|
8443
|
+
try {
|
|
8444
|
+
const peak = await this.eventStore?.peakForTrack(t.trackId);
|
|
8445
|
+
if (peak) {
|
|
8446
|
+
const { importance, reason } = computeImportance({
|
|
8447
|
+
peakConfidence: this.bestFrameTracker.peak(t.trackId)?.confidence ?? peak.peakConfidence,
|
|
8448
|
+
className: t.className,
|
|
8449
|
+
durationMs: duration,
|
|
8450
|
+
peakBboxAreaFrac: peak.peakBboxAreaFrac,
|
|
8451
|
+
totalDistance: t.totalDistance,
|
|
8452
|
+
zonesVisited: t.zonesVisited,
|
|
8453
|
+
...t.label !== void 0 ? { label: t.label } : {}
|
|
8454
|
+
});
|
|
8455
|
+
await this.trackStore?.setImportance(t.trackId, importance, reason, peak.bestEventId);
|
|
8456
|
+
if (peak.bestEventId !== void 0) await this.eventStore?.setImportanceForTrack(t.trackId, importance);
|
|
8457
|
+
}
|
|
8458
|
+
} catch (err) {
|
|
8459
|
+
this.ctx.logger.debug("importance scoring failed", { meta: {
|
|
8460
|
+
trackId: t.trackId,
|
|
8461
|
+
error: String(err)
|
|
8462
|
+
} });
|
|
8463
|
+
}
|
|
7221
8464
|
this.bestFrameTracker.delete(t.trackId);
|
|
8465
|
+
this.objectEmbeddingBestSelector.delete(t.trackId);
|
|
8466
|
+
this.detailDispatcher?.onTrackEnded(t.deviceId, t.trackId);
|
|
7222
8467
|
this.ctx.eventBus.emit({
|
|
7223
8468
|
id: `pa-end-${t.trackId}`,
|
|
7224
8469
|
timestamp: new Date(t.lastSeen),
|
|
@@ -7441,6 +8686,32 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
7441
8686
|
if (input.projection !== "slim" || this.eventMediaBaseUrl === null) return events;
|
|
7442
8687
|
return this.withMediaUrl(events);
|
|
7443
8688
|
}
|
|
8689
|
+
async getKeyEvents(input) {
|
|
8690
|
+
const trackStore = this.trackStore;
|
|
8691
|
+
if (!trackStore) return [];
|
|
8692
|
+
const eventStore = this.eventStore;
|
|
8693
|
+
try {
|
|
8694
|
+
const candidates = await trackStore.queryHistorical({
|
|
8695
|
+
deviceId: input.deviceId,
|
|
8696
|
+
since: input.since,
|
|
8697
|
+
until: input.until,
|
|
8698
|
+
limit: KEY_EVENT_CANDIDATE_CAP
|
|
8699
|
+
});
|
|
8700
|
+
const peakLookup = (trackId) => eventStore ? eventStore.peakForTrack(trackId) : Promise.resolve({
|
|
8701
|
+
peakConfidence: 0,
|
|
8702
|
+
peakBboxAreaFrac: 0,
|
|
8703
|
+
bestEventId: void 0
|
|
8704
|
+
});
|
|
8705
|
+
return await rankKeyEvents(candidates, {
|
|
8706
|
+
limit: input.limit ?? KEY_EVENT_DEFAULT_LIMIT,
|
|
8707
|
+
...input.minImportance !== void 0 ? { minImportance: input.minImportance } : {},
|
|
8708
|
+
...input.classFilter !== void 0 ? { classFilter: input.classFilter } : {}
|
|
8709
|
+
}, peakLookup);
|
|
8710
|
+
} catch (err) {
|
|
8711
|
+
this.ctx.logger.debug("getKeyEvents failed", { meta: { error: String(err) } });
|
|
8712
|
+
return [];
|
|
8713
|
+
}
|
|
8714
|
+
}
|
|
7444
8715
|
async getAudioEvents(input) {
|
|
7445
8716
|
const events = await (this.eventStore?.queryAudio(input) ?? Promise.resolve([]));
|
|
7446
8717
|
if (input.projection !== "slim" || this.eventMediaBaseUrl === null) return events;
|
|
@@ -7483,6 +8754,12 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
7483
8754
|
className: input.classFilter
|
|
7484
8755
|
});
|
|
7485
8756
|
if (embeddingRows.length === 0) return [];
|
|
8757
|
+
const embeddingMediaKeyByTrackId = /* @__PURE__ */ new Map();
|
|
8758
|
+
const keyFrameKeyByTrackId = /* @__PURE__ */ new Map();
|
|
8759
|
+
for (const row of embeddingRows) {
|
|
8760
|
+
if (row.mediaKey !== void 0) embeddingMediaKeyByTrackId.set(row.trackId, row.mediaKey);
|
|
8761
|
+
if (row.keyFrameMediaKey !== void 0) keyFrameKeyByTrackId.set(row.trackId, row.keyFrameMediaKey);
|
|
8762
|
+
}
|
|
7486
8763
|
const encoderInfo = await api.embeddingEncoder.getInfo.query().catch((err) => {
|
|
7487
8764
|
this.ctx.logger.warn("searchObjectEvents: getInfo failed, proceeding without modelId gate", { meta: { error: String(err) } });
|
|
7488
8765
|
return null;
|
|
@@ -7529,10 +8806,20 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
7529
8806
|
score
|
|
7530
8807
|
});
|
|
7531
8808
|
}
|
|
7532
|
-
for (const { event, score } of bestEventByTrackId.values())
|
|
7533
|
-
|
|
7534
|
-
|
|
7535
|
-
|
|
8809
|
+
for (const { event, score } of bestEventByTrackId.values()) {
|
|
8810
|
+
const mediaUrl = resolveSearchThumbnailUrl({
|
|
8811
|
+
baseUrl: this.eventMediaBaseUrl,
|
|
8812
|
+
eventId: event.id,
|
|
8813
|
+
...event.trackId !== void 0 && embeddingMediaKeyByTrackId.has(event.trackId) ? { embeddingMediaKey: embeddingMediaKeyByTrackId.get(event.trackId) } : {}
|
|
8814
|
+
});
|
|
8815
|
+
const keyFrameMediaKey = event.trackId !== void 0 ? keyFrameKeyByTrackId.get(event.trackId) : void 0;
|
|
8816
|
+
scored.push({
|
|
8817
|
+
...event,
|
|
8818
|
+
score,
|
|
8819
|
+
...mediaUrl !== void 0 ? { mediaUrl } : {},
|
|
8820
|
+
...keyFrameMediaKey !== void 0 ? { keyFrameMediaKey } : {}
|
|
8821
|
+
});
|
|
8822
|
+
}
|
|
7536
8823
|
}
|
|
7537
8824
|
scored.sort((a, b) => b.score - a.score);
|
|
7538
8825
|
return scored.slice(0, input.limit);
|
|
@@ -7564,6 +8851,24 @@ var PipelineAnalyticsAddon = class extends BaseAddon {
|
|
|
7564
8851
|
* (native-res boxed frame) → any available file. Returns `null` if the
|
|
7565
8852
|
* event has no media at all.
|
|
7566
8853
|
*/
|
|
8854
|
+
/**
|
|
8855
|
+
* Data-plane resolver: an id is EITHER a bare event id (a UUID → the event's
|
|
8856
|
+
* crop, today's behaviour) OR a MediaStore key (`ownerKind:ownerId:kind:ts`,
|
|
8857
|
+
* contains ':' → served directly by key). The object-embedding search hit
|
|
8858
|
+
* points its thumbnail at the embedding row's crop key (a `track:…:crop:…`
|
|
8859
|
+
* key), so this resolves that crop; event ids stay on the event-crop path.
|
|
8860
|
+
*/
|
|
8861
|
+
async readMediaByEventOrKey(id) {
|
|
8862
|
+
if (id.includes(":")) {
|
|
8863
|
+
const file = await (this.mediaStore?.getByKey(id) ?? Promise.resolve(null));
|
|
8864
|
+
if (!file) return null;
|
|
8865
|
+
return {
|
|
8866
|
+
bytes: Buffer.from(file.base64, "base64"),
|
|
8867
|
+
key: file.key
|
|
8868
|
+
};
|
|
8869
|
+
}
|
|
8870
|
+
return this.readEventThumbnail(id);
|
|
8871
|
+
}
|
|
7567
8872
|
async readEventThumbnail(eventId) {
|
|
7568
8873
|
const files = await (this.mediaStore?.listByOwner("event", eventId) ?? Promise.resolve([]));
|
|
7569
8874
|
const chosen = files.find((f) => f.kind === "crop") ?? files.find((f) => f.kind === "fullFrameBoxed") ?? files[0];
|