@camstack/addon-post-analysis 1.2.341 → 1.2.343
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{clip-model-registry-BOAYW16o.js → clip-model-registry-Cd_y9S4U.js} +382 -8
- package/dist/{clip-model-registry-BP-BF3CK.mjs → clip-model-registry-DZz3e781.mjs} +371 -9
- package/dist/embedding-encoder/index.js +1 -1
- package/dist/embedding-encoder/index.mjs +1 -1
- package/dist/pipeline-analytics/_stub.js +2 -2
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-B2hR0cv6.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-DrdO3Pxp.mjs} +3 -3
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DEq-B6oq.mjs +26 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CiTUMvVf.mjs +26 -0
- package/dist/pipeline-analytics/{hostInit-BgudaQ7W.mjs → hostInit-DPNnWr-m.mjs} +3 -3
- package/dist/pipeline-analytics/index.js +876 -239
- package/dist/pipeline-analytics/index.mjs +868 -231
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/package.json +2 -2
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-B9vPD32p.mjs +0 -26
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CY24cHgW.mjs +0 -26
|
@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_clip_model_registry = require("../clip-model-registry-
|
|
5
|
+
const require_clip_model_registry = require("../clip-model-registry-Cd_y9S4U.js");
|
|
6
6
|
let node_fs = require("node:fs");
|
|
7
7
|
node_fs = require_clip_model_registry.__toESM(node_fs, 1);
|
|
8
8
|
let node_path = require("node:path");
|
|
@@ -23,6 +23,8 @@ let node_timers_promises = require("node:timers/promises");
|
|
|
23
23
|
let node_util = require("node:util");
|
|
24
24
|
let node_buffer = require("node:buffer");
|
|
25
25
|
let node_zlib = require("node:zlib");
|
|
26
|
+
let node_http = require("node:http");
|
|
27
|
+
let node_https = require("node:https");
|
|
26
28
|
//#region src/pipeline-analytics/media-occupancy.ts
|
|
27
29
|
/** The bare type ref legacy NULL-stamped media rows belong to. */
|
|
28
30
|
var LEGACY_MEDIA_TYPE = "eventMedia";
|
|
@@ -7028,18 +7030,44 @@ function clockCondition(now) {
|
|
|
7028
7030
|
* does not.
|
|
7029
7031
|
*/
|
|
7030
7032
|
function resolveCondition(input) {
|
|
7031
|
-
|
|
7032
|
-
|
|
7033
|
+
return resolveConditionDetailed(input).condition;
|
|
7034
|
+
}
|
|
7035
|
+
/**
|
|
7036
|
+
* {@link resolveCondition}, plus the layer that answered. A consumer that
|
|
7037
|
+
* reports a day/night split (the retrain export, D785) must be able to tell a
|
|
7038
|
+
* measured IR frame from an almanac or a clock guess.
|
|
7039
|
+
*/
|
|
7040
|
+
function resolveConditionDetailed(input) {
|
|
7041
|
+
if (input.forcedMode === "day" || input.forcedMode === "night") return {
|
|
7042
|
+
condition: input.forcedMode,
|
|
7043
|
+
source: "forced"
|
|
7044
|
+
};
|
|
7045
|
+
if (input.chromaSpread !== void 0 && input.chromaSpread < IR_CHROMA_THRESHOLD) return {
|
|
7046
|
+
condition: IR,
|
|
7047
|
+
source: "ir-chroma"
|
|
7048
|
+
};
|
|
7033
7049
|
if (input.coordinates) {
|
|
7034
7050
|
const times = sunTimes(input.coordinates.lat, input.coordinates.lng, input.now);
|
|
7035
7051
|
if (times) {
|
|
7036
7052
|
const t = input.now.getTime();
|
|
7037
|
-
if (Math.abs(t - times.rise) <= TWILIGHT_BAND_MS) return
|
|
7038
|
-
|
|
7039
|
-
|
|
7053
|
+
if (Math.abs(t - times.rise) <= TWILIGHT_BAND_MS) return {
|
|
7054
|
+
condition: DAWN,
|
|
7055
|
+
source: "sun"
|
|
7056
|
+
};
|
|
7057
|
+
if (Math.abs(t - times.set) <= TWILIGHT_BAND_MS) return {
|
|
7058
|
+
condition: DUSK,
|
|
7059
|
+
source: "sun"
|
|
7060
|
+
};
|
|
7061
|
+
return {
|
|
7062
|
+
condition: t >= times.rise && t < times.set ? DAY : NIGHT,
|
|
7063
|
+
source: "sun"
|
|
7064
|
+
};
|
|
7040
7065
|
}
|
|
7041
7066
|
}
|
|
7042
|
-
return
|
|
7067
|
+
return {
|
|
7068
|
+
condition: clockCondition(input.now),
|
|
7069
|
+
source: "clock"
|
|
7070
|
+
};
|
|
7043
7071
|
}
|
|
7044
7072
|
//#endregion
|
|
7045
7073
|
//#region src/shared/llm-vision/parse-model-json.ts
|
|
@@ -7172,8 +7200,8 @@ async function raceTimeout(call, timeoutMs) {
|
|
|
7172
7200
|
* detail the encoder can see.
|
|
7173
7201
|
*/
|
|
7174
7202
|
async function downscaleJpeg(bytes, maxPx) {
|
|
7175
|
-
const { default: sharp$
|
|
7176
|
-
const out = await sharp$
|
|
7203
|
+
const { default: sharp$20 } = await import("sharp");
|
|
7204
|
+
const out = await sharp$20(Buffer.from(bytes)).resize({
|
|
7177
7205
|
width: maxPx,
|
|
7178
7206
|
height: maxPx,
|
|
7179
7207
|
fit: "inside",
|
|
@@ -19384,8 +19412,8 @@ async function deriveSyntheticMedia(jpeg, bbox, timestamp) {
|
|
|
19384
19412
|
];
|
|
19385
19413
|
if (bbox === void 0) return files;
|
|
19386
19414
|
try {
|
|
19387
|
-
const { default: sharp$
|
|
19388
|
-
const meta = await sharp$
|
|
19415
|
+
const { default: sharp$19 } = await import("sharp");
|
|
19416
|
+
const meta = await sharp$19(Buffer.from(jpeg)).metadata();
|
|
19389
19417
|
const W = meta.width ?? 0;
|
|
19390
19418
|
const H = meta.height ?? 0;
|
|
19391
19419
|
if (W === 0 || H === 0) return files;
|
|
@@ -19393,7 +19421,7 @@ async function deriveSyntheticMedia(jpeg, bbox, timestamp) {
|
|
|
19393
19421
|
const top = Math.max(0, Math.min(H - 1, Math.floor(bbox.y * H)));
|
|
19394
19422
|
const width = Math.max(1, Math.min(W - left, Math.ceil(bbox.w * W)));
|
|
19395
19423
|
const height = Math.max(1, Math.min(H - top, Math.ceil(bbox.h * H)));
|
|
19396
|
-
const cropBuf = await sharp$
|
|
19424
|
+
const cropBuf = await sharp$19(Buffer.from(jpeg)).extract({
|
|
19397
19425
|
left,
|
|
19398
19426
|
top,
|
|
19399
19427
|
width,
|
|
@@ -19415,7 +19443,7 @@ async function deriveSyntheticMedia(jpeg, bbox, timestamp) {
|
|
|
19415
19443
|
});
|
|
19416
19444
|
const stroke = Math.max(2, Math.round(Math.min(W, H) / 240));
|
|
19417
19445
|
const overlay = Buffer.from(`<svg width="${W}" height="${H}"><rect x="${left}" y="${top}" width="${width}" height="${height}" fill="none" stroke="#22d3ee" stroke-width="${stroke}"/></svg>`);
|
|
19418
|
-
const boxedBuf = await sharp$
|
|
19446
|
+
const boxedBuf = await sharp$19(Buffer.from(jpeg)).composite([{
|
|
19419
19447
|
input: overlay,
|
|
19420
19448
|
top: 0,
|
|
19421
19449
|
left: 0
|
|
@@ -26452,8 +26480,8 @@ var NcDispatcher = class {
|
|
|
26452
26480
|
try {
|
|
26453
26481
|
const points = (await this.deps.getZonePolygons?.(deviceId, zoneIds) ?? []).flat();
|
|
26454
26482
|
if (points.length === 0) return null;
|
|
26455
|
-
const { default: sharp$
|
|
26456
|
-
const img = sharp$
|
|
26483
|
+
const { default: sharp$18 } = await import("sharp");
|
|
26484
|
+
const img = sharp$18(Buffer.from(jpeg));
|
|
26457
26485
|
const meta = await img.metadata();
|
|
26458
26486
|
const W = meta.width ?? 0;
|
|
26459
26487
|
const H = meta.height ?? 0;
|
|
@@ -70560,8 +70588,11 @@ var TrackStore = class {
|
|
|
70560
70588
|
* RetrainFrameStore — the COPIES the retrain dataset owns.
|
|
70561
70589
|
*
|
|
70562
70590
|
* Collection: `pipeline-analytics:retrain-frames`.
|
|
70563
|
-
* Blobs: `retrain/<deviceId>/<trackId>/<hash>.jpg
|
|
70564
|
-
*
|
|
70591
|
+
* Blobs: `retrain/<deviceId>/<trackId>/<hash>.jpg` under the `artifacts`
|
|
70592
|
+
* class root — the dataset is that class's fourth tenant (D341), addressed
|
|
70593
|
+
* through the permanent row ref {@link RETRAIN_MEDIA_LOCATION}. Never the
|
|
70594
|
+
* footage volume: the 2026-08-17 recordings wipe took the whole dataset with
|
|
70595
|
+
* it while it lived on `eventMedia` (D197). See D781.
|
|
70565
70596
|
*
|
|
70566
70597
|
* ## Why a copy, and why at selection time
|
|
70567
70598
|
*
|
|
@@ -70592,16 +70623,37 @@ var TrackStore = class {
|
|
|
70592
70623
|
* copy is taken eagerly at selection (copyFrame), not referenced, so the row
|
|
70593
70624
|
* records a blob the dataset owns outright; re-selecting the same source reuses
|
|
70594
70625
|
* the existing copy rather than writing a second one."
|
|
70595
|
-
* retention="none — the blobs sit
|
|
70596
|
-
* age sweep walks, which is exactly what makes a 'trained' track evictable
|
|
70626
|
+
* retention="none — the blobs sit in the 'retrain/' subtree of the artifacts class
|
|
70627
|
+
* (D781), which no owner cascade and no age sweep walks, which is exactly what makes a 'trained' track evictable
|
|
70597
70628
|
* again (D81). Rows go only when the operator deselects the frame (deleteFrame,
|
|
70598
70629
|
* after its annotations). Bounded only by operator selection — nothing caps the
|
|
70599
70630
|
* table or the disk it consumes."
|
|
70600
70631
|
*/
|
|
70601
70632
|
var RETRAIN_FRAMES_COLLECTION = "pipeline-analytics:retrain-frames";
|
|
70602
|
-
/**
|
|
70603
|
-
*
|
|
70604
|
-
|
|
70633
|
+
/**
|
|
70634
|
+
* The PERMANENT location ref every retrain copy is written under (D781).
|
|
70635
|
+
*
|
|
70636
|
+
* Not a declared storage class: it is the rows' `locationId`, which the
|
|
70637
|
+
* media-storage wiring maps onto the `artifacts` class root — exactly how the
|
|
70638
|
+
* enrolled gallery's `galleryMedia` ref works (D341). The copies are curation
|
|
70639
|
+
* work with NO retention, so they share the one durable class rather than the
|
|
70640
|
+
* footage class, which is quota-evicted, drained and wipeable per disk. Rows
|
|
70641
|
+
* are never restamped for a root move: the ref is remapped instead, and the
|
|
70642
|
+
* singleton repoint carries the whole `artifacts` tree, `retrain/` included.
|
|
70643
|
+
*/
|
|
70644
|
+
var RETRAIN_MEDIA_LOCATION = "retrainMedia";
|
|
70645
|
+
/**
|
|
70646
|
+
* Where copies taken before D781 live: a NULL `locationId` meant "wherever
|
|
70647
|
+
* `eventMedia` points", and a concrete `eventMedia:*` id was stamped by the
|
|
70648
|
+
* seal or a relocate pass. The boot reconcile
|
|
70649
|
+
* ({@link RetrainFrameStore.relocateLegacyCopies}) moves them onto
|
|
70650
|
+
* {@link RETRAIN_MEDIA_LOCATION}; until it has, reads resolve them here.
|
|
70651
|
+
*/
|
|
70652
|
+
var LEGACY_RETRAIN_MEDIA_LOCATION = "eventMedia";
|
|
70653
|
+
/** Is this row already on the durable retrain tenant? */
|
|
70654
|
+
function rowIsDurableRetrainCopy(data) {
|
|
70655
|
+
return data?.["locationId"] === RETRAIN_MEDIA_LOCATION;
|
|
70656
|
+
}
|
|
70605
70657
|
/** Path prefix of every retrain copy. Never `<deviceId>/events/…`: the orphan
|
|
70606
70658
|
* audit resolves ownership from INDEX rows, but a future filesystem-side tool
|
|
70607
70659
|
* must be able to tell the two corpora apart by path alone. */
|
|
@@ -70659,11 +70711,32 @@ var RETRAIN_FRAME_COLUMNS = [
|
|
|
70659
70711
|
notNull: true
|
|
70660
70712
|
},
|
|
70661
70713
|
(
|
|
70662
|
-
/**
|
|
70663
|
-
*
|
|
70714
|
+
/** `retrainMedia` for every copy taken since D781. NULL (the eventMedia
|
|
70715
|
+
* default) or a concrete `eventMedia:*` id only on a pre-D781 copy the boot
|
|
70716
|
+
* reconcile has not moved yet. */
|
|
70664
70717
|
{
|
|
70665
70718
|
name: "locationId",
|
|
70666
70719
|
type: "TEXT"
|
|
70720
|
+
}),
|
|
70721
|
+
(
|
|
70722
|
+
/** When the CAMERA captured the frame — the source media row's timestamp.
|
|
70723
|
+
* NULL on a copy taken before D785. */
|
|
70724
|
+
{
|
|
70725
|
+
name: "capturedAt",
|
|
70726
|
+
type: "INTEGER"
|
|
70727
|
+
}),
|
|
70728
|
+
(
|
|
70729
|
+
/** Mean channel spread of the copy (`chromaSpread`), measured once at copy
|
|
70730
|
+
* time; below `IR_CHROMA_THRESHOLD` the frame is IR. NULL = not measured. */
|
|
70731
|
+
{
|
|
70732
|
+
name: "chromaSpread",
|
|
70733
|
+
type: "REAL"
|
|
70734
|
+
}),
|
|
70735
|
+
(
|
|
70736
|
+
/** The operator's "every subject in this frame is boxed". NULL = no. */
|
|
70737
|
+
{
|
|
70738
|
+
name: "complete",
|
|
70739
|
+
type: "BOOLEAN"
|
|
70667
70740
|
})
|
|
70668
70741
|
];
|
|
70669
70742
|
var RETRAIN_FRAME_INDEXES = [
|
|
@@ -70704,6 +70777,10 @@ function retrainFrameId(trackId, sourceMediaId) {
|
|
|
70704
70777
|
function framePath(deviceId, trackId, frameId) {
|
|
70705
70778
|
return `${RETRAIN_PATH_PREFIX}/${deviceId}/${trackId}/${frameId.slice(frameId.lastIndexOf(":") + 1)}.jpg`;
|
|
70706
70779
|
}
|
|
70780
|
+
/** A stored number, or `undefined` for NULL / absent / junk. */
|
|
70781
|
+
function finiteNumber(value) {
|
|
70782
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
70783
|
+
}
|
|
70707
70784
|
var RetrainFrameStore = class {
|
|
70708
70785
|
store;
|
|
70709
70786
|
storage;
|
|
@@ -70711,7 +70788,7 @@ var RetrainFrameStore = class {
|
|
|
70711
70788
|
readSource;
|
|
70712
70789
|
now;
|
|
70713
70790
|
writeGate;
|
|
70714
|
-
|
|
70791
|
+
measureChroma;
|
|
70715
70792
|
constructor(deps) {
|
|
70716
70793
|
this.store = deps.store;
|
|
70717
70794
|
this.storage = deps.storage;
|
|
@@ -70719,28 +70796,20 @@ var RetrainFrameStore = class {
|
|
|
70719
70796
|
this.readSource = deps.readSource;
|
|
70720
70797
|
this.now = deps.now;
|
|
70721
70798
|
this.writeGate = deps.writeGate;
|
|
70722
|
-
this.
|
|
70799
|
+
this.measureChroma = deps.measureChroma;
|
|
70723
70800
|
}
|
|
70724
|
-
/**
|
|
70725
|
-
|
|
70726
|
-
|
|
70727
|
-
* `null` = "no concrete location was resolved", which persists as the legacy
|
|
70728
|
-
* NULL row. A resolver that throws must not refuse the copy — losing an
|
|
70729
|
-
* operator's selection is worse than an unstamped row the seal can fix — but
|
|
70730
|
-
* it says so, because that branch re-creates the exact population a
|
|
70731
|
-
* non-blocking migration is gated on.
|
|
70732
|
-
*/
|
|
70733
|
-
async resolveWriteLocation(deviceId) {
|
|
70734
|
-
if (!this.resolveEventMediaLocation) return null;
|
|
70801
|
+
/** The copy's chroma, or `undefined` when it cannot be measured. */
|
|
70802
|
+
async chromaOf(bytes, deviceId) {
|
|
70803
|
+
if (!this.measureChroma) return void 0;
|
|
70735
70804
|
try {
|
|
70736
|
-
const
|
|
70737
|
-
return
|
|
70805
|
+
const value = await this.measureChroma(bytes);
|
|
70806
|
+
return Number.isFinite(value) ? value : void 0;
|
|
70738
70807
|
} catch (err) {
|
|
70739
|
-
this.logger.warn("retrain copy could not
|
|
70808
|
+
this.logger.warn("retrain copy chroma could not be measured — its day/night falls back to the sun", {
|
|
70740
70809
|
tags: { deviceId },
|
|
70741
70810
|
meta: { error: String(err) }
|
|
70742
70811
|
});
|
|
70743
|
-
return
|
|
70812
|
+
return;
|
|
70744
70813
|
}
|
|
70745
70814
|
}
|
|
70746
70815
|
static async declare(store) {
|
|
@@ -70761,7 +70830,10 @@ var RetrainFrameStore = class {
|
|
|
70761
70830
|
width: Number(data["width"] ?? 0),
|
|
70762
70831
|
height: Number(data["height"] ?? 0),
|
|
70763
70832
|
copiedAt: Number(data["copiedAt"] ?? 0),
|
|
70764
|
-
locationId: typeof data["locationId"] === "string" && data["locationId"].length > 0 ? data["locationId"] : null
|
|
70833
|
+
locationId: typeof data["locationId"] === "string" && data["locationId"].length > 0 ? data["locationId"] : null,
|
|
70834
|
+
...finiteNumber(data["capturedAt"]) !== void 0 ? { capturedAt: finiteNumber(data["capturedAt"]) } : {},
|
|
70835
|
+
...finiteNumber(data["chromaSpread"]) !== void 0 ? { chromaSpread: finiteNumber(data["chromaSpread"]) } : {},
|
|
70836
|
+
complete: data["complete"] === true || data["complete"] === 1
|
|
70765
70837
|
};
|
|
70766
70838
|
}
|
|
70767
70839
|
/** The copy with this id, or `null`. Index read only — no blob is touched. */
|
|
@@ -70801,6 +70873,31 @@ var RetrainFrameStore = class {
|
|
|
70801
70873
|
return [];
|
|
70802
70874
|
}
|
|
70803
70875
|
}
|
|
70876
|
+
/**
|
|
70877
|
+
* The copies of one camera the operator marked complete. A complete frame
|
|
70878
|
+
* with no box is a hard negative, so an export cannot find these through
|
|
70879
|
+
* the annotation table alone.
|
|
70880
|
+
*/
|
|
70881
|
+
async listCompleteByDevice(deviceId, limit) {
|
|
70882
|
+
try {
|
|
70883
|
+
return (await this.store.query.query({
|
|
70884
|
+
collection: RETRAIN_FRAMES_COLLECTION,
|
|
70885
|
+
filter: {
|
|
70886
|
+
where: {
|
|
70887
|
+
deviceId,
|
|
70888
|
+
complete: true
|
|
70889
|
+
},
|
|
70890
|
+
limit
|
|
70891
|
+
}
|
|
70892
|
+
})).filter((r) => typeof r.id === "string").map((r) => this.rowToFrame(r.id, r.data));
|
|
70893
|
+
} catch (err) {
|
|
70894
|
+
this.logger.warn("RetrainFrameStore.listCompleteByDevice failed", {
|
|
70895
|
+
tags: { deviceId },
|
|
70896
|
+
meta: { error: String(err) }
|
|
70897
|
+
});
|
|
70898
|
+
return [];
|
|
70899
|
+
}
|
|
70900
|
+
}
|
|
70804
70901
|
/** How many copies each of these tracks already owns — one query, so the
|
|
70805
70902
|
* staging list can badge "worked / not worked" without N round trips. */
|
|
70806
70903
|
async countsByTrack(trackIds) {
|
|
@@ -70829,7 +70926,7 @@ var RetrainFrameStore = class {
|
|
|
70829
70926
|
async readFrame(frame) {
|
|
70830
70927
|
try {
|
|
70831
70928
|
return await this.storage.read({
|
|
70832
|
-
location: frame.locationId ??
|
|
70929
|
+
location: frame.locationId ?? LEGACY_RETRAIN_MEDIA_LOCATION,
|
|
70833
70930
|
relativePath: framePath(frame.deviceId, frame.trackId, frame.frameId)
|
|
70834
70931
|
});
|
|
70835
70932
|
} catch (err) {
|
|
@@ -70900,7 +70997,8 @@ var RetrainFrameStore = class {
|
|
|
70900
70997
|
};
|
|
70901
70998
|
}
|
|
70902
70999
|
const path = framePath(input.deviceId, input.trackId, frameId);
|
|
70903
|
-
const locationId =
|
|
71000
|
+
const locationId = RETRAIN_MEDIA_LOCATION;
|
|
71001
|
+
const chromaSpread = await this.chromaOf(bytes, input.deviceId);
|
|
70904
71002
|
const frame = {
|
|
70905
71003
|
frameId,
|
|
70906
71004
|
deviceId: input.deviceId,
|
|
@@ -70911,11 +71009,14 @@ var RetrainFrameStore = class {
|
|
|
70911
71009
|
width: size.width,
|
|
70912
71010
|
height: size.height,
|
|
70913
71011
|
copiedAt: this.now(),
|
|
70914
|
-
locationId
|
|
71012
|
+
locationId,
|
|
71013
|
+
...input.capturedAt !== void 0 ? { capturedAt: input.capturedAt } : {},
|
|
71014
|
+
...chromaSpread !== void 0 ? { chromaSpread } : {},
|
|
71015
|
+
complete: false
|
|
70915
71016
|
};
|
|
70916
71017
|
const persist = async () => {
|
|
70917
71018
|
await this.storage.write({
|
|
70918
|
-
location: locationId
|
|
71019
|
+
location: locationId,
|
|
70919
71020
|
relativePath: path,
|
|
70920
71021
|
data: bytes
|
|
70921
71022
|
});
|
|
@@ -70932,7 +71033,10 @@ var RetrainFrameStore = class {
|
|
|
70932
71033
|
width: frame.width,
|
|
70933
71034
|
height: frame.height,
|
|
70934
71035
|
copiedAt: frame.copiedAt,
|
|
70935
|
-
locationId: frame.locationId
|
|
71036
|
+
locationId: frame.locationId,
|
|
71037
|
+
capturedAt: frame.capturedAt ?? null,
|
|
71038
|
+
chromaSpread: frame.chromaSpread ?? null,
|
|
71039
|
+
complete: false
|
|
70936
71040
|
}
|
|
70937
71041
|
});
|
|
70938
71042
|
};
|
|
@@ -70972,6 +71076,27 @@ var RetrainFrameStore = class {
|
|
|
70972
71076
|
};
|
|
70973
71077
|
}
|
|
70974
71078
|
/**
|
|
71079
|
+
* The operator's "every subject in this frame is boxed" (D785 context: an
|
|
71080
|
+
* unboxed subject is a false positive to a detector evaluation). Only the
|
|
71081
|
+
* operator sets it; nothing derives it from the annotation count, because a
|
|
71082
|
+
* frame with zero boxes may be a complete hard negative.
|
|
71083
|
+
*
|
|
71084
|
+
* @returns the updated frame, or `null` when no such copy exists.
|
|
71085
|
+
*/
|
|
71086
|
+
async setComplete(frameId, complete) {
|
|
71087
|
+
const existing = await this.get(frameId);
|
|
71088
|
+
if (existing === null) return null;
|
|
71089
|
+
await this.store.update.mutate({
|
|
71090
|
+
collection: RETRAIN_FRAMES_COLLECTION,
|
|
71091
|
+
id: frameId,
|
|
71092
|
+
data: { complete }
|
|
71093
|
+
});
|
|
71094
|
+
return {
|
|
71095
|
+
...existing,
|
|
71096
|
+
complete
|
|
71097
|
+
};
|
|
71098
|
+
}
|
|
71099
|
+
/**
|
|
70975
71100
|
* Drop a copy and its blob — the operator un-selecting a frame.
|
|
70976
71101
|
*
|
|
70977
71102
|
* The caller must have removed the frame's annotations first; this store does
|
|
@@ -70982,7 +71107,7 @@ var RetrainFrameStore = class {
|
|
|
70982
71107
|
const remove = async () => {
|
|
70983
71108
|
try {
|
|
70984
71109
|
await this.storage.delete({
|
|
70985
|
-
location: frame.locationId ??
|
|
71110
|
+
location: frame.locationId ?? LEGACY_RETRAIN_MEDIA_LOCATION,
|
|
70986
71111
|
relativePath: framePath(frame.deviceId, frame.trackId, frame.frameId)
|
|
70987
71112
|
});
|
|
70988
71113
|
} catch {}
|
|
@@ -71004,6 +71129,95 @@ var RetrainFrameStore = class {
|
|
|
71004
71129
|
if (this.writeGate) await this.writeGate.run(remove);
|
|
71005
71130
|
else await remove();
|
|
71006
71131
|
}
|
|
71132
|
+
/**
|
|
71133
|
+
* Move every copy still on the footage class onto the durable retrain
|
|
71134
|
+
* tenant (D781): copy → read back → restamp → delete the source.
|
|
71135
|
+
*
|
|
71136
|
+
* A reconcile, not a one-shot: one query of a collection bounded by
|
|
71137
|
+
* operator selection, so it is cheap when converged, and it is what would
|
|
71138
|
+
* notice a pre-D781 copy restored from anywhere. The order is the gallery
|
|
71139
|
+
* reconcile's (D341) and for the same reason: the row is restamped only
|
|
71140
|
+
* after the new blob reads back, so a crash at any point leaves a row that
|
|
71141
|
+
* names a location holding its bytes.
|
|
71142
|
+
*
|
|
71143
|
+
* A copy whose bytes are already gone is COUNTED and LOGGED, never deleted:
|
|
71144
|
+
* its annotations are the operator's work, and only the operator decides
|
|
71145
|
+
* to drop them (deselecting the frame).
|
|
71146
|
+
*/
|
|
71147
|
+
async relocateLegacyCopies() {
|
|
71148
|
+
const rows = await this.store.query.query({ collection: RETRAIN_FRAMES_COLLECTION });
|
|
71149
|
+
let relocated = 0;
|
|
71150
|
+
let missing = 0;
|
|
71151
|
+
let failed = 0;
|
|
71152
|
+
for (const row of rows) {
|
|
71153
|
+
if (typeof row.id !== "string") continue;
|
|
71154
|
+
const data = row.data;
|
|
71155
|
+
if (rowIsDurableRetrainCopy(data)) continue;
|
|
71156
|
+
const frame = this.rowToFrame(row.id, data);
|
|
71157
|
+
const from = frame.locationId ?? LEGACY_RETRAIN_MEDIA_LOCATION;
|
|
71158
|
+
const relativePath = framePath(frame.deviceId, frame.trackId, frame.frameId);
|
|
71159
|
+
let bytes;
|
|
71160
|
+
try {
|
|
71161
|
+
bytes = await this.storage.read({
|
|
71162
|
+
location: from,
|
|
71163
|
+
relativePath
|
|
71164
|
+
});
|
|
71165
|
+
} catch (err) {
|
|
71166
|
+
missing += 1;
|
|
71167
|
+
this.logger.warn("retrain copy has NO bytes on the footage class — its row and annotations are kept; deselect the frame to drop it", {
|
|
71168
|
+
tags: { deviceId: frame.deviceId },
|
|
71169
|
+
meta: {
|
|
71170
|
+
frameId: frame.frameId,
|
|
71171
|
+
trackId: frame.trackId,
|
|
71172
|
+
from,
|
|
71173
|
+
error: String(err)
|
|
71174
|
+
}
|
|
71175
|
+
});
|
|
71176
|
+
continue;
|
|
71177
|
+
}
|
|
71178
|
+
const move = async () => {
|
|
71179
|
+
await this.storage.write({
|
|
71180
|
+
location: RETRAIN_MEDIA_LOCATION,
|
|
71181
|
+
relativePath,
|
|
71182
|
+
data: bytes
|
|
71183
|
+
});
|
|
71184
|
+
if (!(await this.storage.read({
|
|
71185
|
+
location: RETRAIN_MEDIA_LOCATION,
|
|
71186
|
+
relativePath
|
|
71187
|
+
})).equals(bytes)) throw new Error("read-back differs from the source");
|
|
71188
|
+
await this.store.update.mutate({
|
|
71189
|
+
collection: RETRAIN_FRAMES_COLLECTION,
|
|
71190
|
+
id: frame.frameId,
|
|
71191
|
+
data: { locationId: RETRAIN_MEDIA_LOCATION }
|
|
71192
|
+
});
|
|
71193
|
+
await this.storage.delete({
|
|
71194
|
+
location: from,
|
|
71195
|
+
relativePath
|
|
71196
|
+
});
|
|
71197
|
+
};
|
|
71198
|
+
try {
|
|
71199
|
+
if (this.writeGate) await this.writeGate.run(move);
|
|
71200
|
+
else await move();
|
|
71201
|
+
relocated += 1;
|
|
71202
|
+
} catch (err) {
|
|
71203
|
+
failed += 1;
|
|
71204
|
+
this.logger.error("retrain copy could not be moved off the footage class", {
|
|
71205
|
+
tags: { deviceId: frame.deviceId },
|
|
71206
|
+
meta: {
|
|
71207
|
+
frameId: frame.frameId,
|
|
71208
|
+
trackId: frame.trackId,
|
|
71209
|
+
from,
|
|
71210
|
+
error: String(err)
|
|
71211
|
+
}
|
|
71212
|
+
});
|
|
71213
|
+
}
|
|
71214
|
+
}
|
|
71215
|
+
return {
|
|
71216
|
+
relocated,
|
|
71217
|
+
missing,
|
|
71218
|
+
failed
|
|
71219
|
+
};
|
|
71220
|
+
}
|
|
71007
71221
|
};
|
|
71008
71222
|
//#endregion
|
|
71009
71223
|
//#region src/pipeline-analytics/retention/location-class-wipe.ts
|
|
@@ -82543,6 +82757,322 @@ var RetrainCollectionStore = class {
|
|
|
82543
82757
|
}
|
|
82544
82758
|
};
|
|
82545
82759
|
//#endregion
|
|
82760
|
+
//#region src/pipeline-analytics/retrain/retrain-archive-upload.ts
|
|
82761
|
+
/**
|
|
82762
|
+
* The HTTP sink behind `pipelineAnalytics.writeRetrainArchive`: ONE request
|
|
82763
|
+
* whose body is the archive, written chunk by chunk with backpressure honoured
|
|
82764
|
+
* (D447) — the archive is never assembled in memory or on disk.
|
|
82765
|
+
*
|
|
82766
|
+
* The target is admin-supplied (the training orchestrator hands over a
|
|
82767
|
+
* provider's presigned URL or its upload route). Its headers may carry a
|
|
82768
|
+
* credential, so they are sent and never logged.
|
|
82769
|
+
*/
|
|
82770
|
+
function openRetrainUploadSink(target) {
|
|
82771
|
+
const url = new URL(target.url);
|
|
82772
|
+
const req = (url.protocol === "https:" ? node_https.request : node_http.request)(url, {
|
|
82773
|
+
method: target.method,
|
|
82774
|
+
headers: {
|
|
82775
|
+
"content-type": "application/x-tar",
|
|
82776
|
+
...target.headers
|
|
82777
|
+
}
|
|
82778
|
+
});
|
|
82779
|
+
let failure = null;
|
|
82780
|
+
const answer = new Promise((resolve, reject) => {
|
|
82781
|
+
req.on("response", resolve);
|
|
82782
|
+
req.on("error", (err) => {
|
|
82783
|
+
failure = err;
|
|
82784
|
+
reject(err);
|
|
82785
|
+
});
|
|
82786
|
+
});
|
|
82787
|
+
answer.catch(() => void 0);
|
|
82788
|
+
return {
|
|
82789
|
+
sink: { write: (chunk) => new Promise((resolve, reject) => {
|
|
82790
|
+
if (failure !== null) {
|
|
82791
|
+
reject(failure);
|
|
82792
|
+
return;
|
|
82793
|
+
}
|
|
82794
|
+
if (req.write(chunk, (err) => {
|
|
82795
|
+
if (err) reject(err);
|
|
82796
|
+
})) resolve();
|
|
82797
|
+
else req.once("drain", resolve);
|
|
82798
|
+
}) },
|
|
82799
|
+
finish: async () => {
|
|
82800
|
+
req.end();
|
|
82801
|
+
const res = await answer;
|
|
82802
|
+
res.resume();
|
|
82803
|
+
const status = res.statusCode ?? 0;
|
|
82804
|
+
if (status < 200 || status >= 300) throw new Error(`retrain upload refused by the target: HTTP ${status}`);
|
|
82805
|
+
return status;
|
|
82806
|
+
},
|
|
82807
|
+
abort: (reason) => {
|
|
82808
|
+
req.destroy(new Error(reason));
|
|
82809
|
+
}
|
|
82810
|
+
};
|
|
82811
|
+
}
|
|
82812
|
+
//#endregion
|
|
82813
|
+
//#region src/pipeline-analytics/training-export/tar-stream.ts
|
|
82814
|
+
/**
|
|
82815
|
+
* A minimal POSIX ustar writer — enough to emit a dataset archive entry by
|
|
82816
|
+
* entry, without ever holding the archive in memory.
|
|
82817
|
+
*
|
|
82818
|
+
* ## Why hand-rolled rather than the `tar` package
|
|
82819
|
+
*
|
|
82820
|
+
* `tar` is in the workspace and is the right tool for what it does: archive a
|
|
82821
|
+
* set of paths under one `cwd`. This is a different job. A track's media lives
|
|
82822
|
+
* across several storage LOCATION roots (`eventMedia` and whatever else the
|
|
82823
|
+
* operator relocated to), under paths that encode nothing a training set wants —
|
|
82824
|
+
* `615/events/track/<uuid>/keyFrame.jpg` — and every entry has to be RENAMED on
|
|
82825
|
+
* the way in, to `tracks/615/<trackId>/keyFrame.jpg`. `tar.create` takes a list
|
|
82826
|
+
* of paths relative to a single cwd and gives no rename hook, so using it would
|
|
82827
|
+
* mean staging a whole shadow tree on disk first: a full second copy of the
|
|
82828
|
+
* dataset, written and then deleted, to avoid writing 100 lines.
|
|
82829
|
+
*
|
|
82830
|
+
* So: ustar, which is a 512-byte header and a 512-byte-padded body per entry,
|
|
82831
|
+
* fully specified, and asserted against the system `tar` binary in the spec next
|
|
82832
|
+
* door rather than against my reading of the format.
|
|
82833
|
+
*
|
|
82834
|
+
* ## What this deliberately does NOT do
|
|
82835
|
+
*
|
|
82836
|
+
* No gzip — the payload is JPEG, already compressed; a gzip pass would burn CPU
|
|
82837
|
+
* on the hub for a percent or two. No long-name (GNU/PAX) extension: ustar
|
|
82838
|
+
* splits a path across a 155-byte prefix and a 100-byte name, and
|
|
82839
|
+
* {@link tarNameTooLong} reports a path that will not fit so the caller can skip
|
|
82840
|
+
* it and SAY it skipped it, rather than emitting a silently truncated entry.
|
|
82841
|
+
*/
|
|
82842
|
+
var BLOCK_SIZE = 512;
|
|
82843
|
+
var NAME_MAX = 100;
|
|
82844
|
+
var PREFIX_MAX = 155;
|
|
82845
|
+
/**
|
|
82846
|
+
* Split a path into the ustar `prefix` + `name` pair, or `null` when it cannot
|
|
82847
|
+
* be represented. The split must land on a `/`, and each half has its own cap.
|
|
82848
|
+
*/
|
|
82849
|
+
function splitUstarName(name) {
|
|
82850
|
+
if (Buffer.byteLength(name) <= NAME_MAX) return {
|
|
82851
|
+
prefix: "",
|
|
82852
|
+
name
|
|
82853
|
+
};
|
|
82854
|
+
for (let i = name.length - 1; i >= 0; i--) {
|
|
82855
|
+
if (name[i] !== "/") continue;
|
|
82856
|
+
const head = name.slice(0, i);
|
|
82857
|
+
const tail = name.slice(i + 1);
|
|
82858
|
+
if (Buffer.byteLength(tail) > NAME_MAX) break;
|
|
82859
|
+
if (Buffer.byteLength(head) <= PREFIX_MAX) return {
|
|
82860
|
+
prefix: head,
|
|
82861
|
+
name: tail
|
|
82862
|
+
};
|
|
82863
|
+
}
|
|
82864
|
+
return null;
|
|
82865
|
+
}
|
|
82866
|
+
/** True when a path cannot be represented as a ustar entry name. */
|
|
82867
|
+
function tarNameTooLong(name) {
|
|
82868
|
+
return splitUstarName(name) === null;
|
|
82869
|
+
}
|
|
82870
|
+
function writeString(block, value, offset, length) {
|
|
82871
|
+
block.write(value.slice(0, length - 1), offset, length - 1, "utf8");
|
|
82872
|
+
}
|
|
82873
|
+
/** ustar numeric field: zero-padded octal, NUL-terminated. */
|
|
82874
|
+
function writeOctal(block, value, offset, length) {
|
|
82875
|
+
const text = Math.floor(value).toString(8).padStart(length - 1, "0");
|
|
82876
|
+
block.write(text.slice(-(length - 1)), offset, length - 1, "ascii");
|
|
82877
|
+
}
|
|
82878
|
+
/**
|
|
82879
|
+
* The 512-byte header for one regular file.
|
|
82880
|
+
*
|
|
82881
|
+
* The checksum is computed with the checksum field itself read as eight spaces —
|
|
82882
|
+
* that is the format's rule, and getting it wrong produces an archive that looks
|
|
82883
|
+
* fine until something tries to read it.
|
|
82884
|
+
*/
|
|
82885
|
+
function tarHeader(entry) {
|
|
82886
|
+
const split = splitUstarName(entry.name);
|
|
82887
|
+
if (split === null) throw new Error(`tar: entry name too long for ustar: ${entry.name}`);
|
|
82888
|
+
const block = Buffer.alloc(BLOCK_SIZE);
|
|
82889
|
+
writeString(block, split.name, 0, NAME_MAX);
|
|
82890
|
+
writeOctal(block, 420, 100, 8);
|
|
82891
|
+
writeOctal(block, 0, 108, 8);
|
|
82892
|
+
writeOctal(block, 0, 116, 8);
|
|
82893
|
+
writeOctal(block, entry.body.length, 124, 12);
|
|
82894
|
+
writeOctal(block, Math.floor(entry.mtimeMs / 1e3), 136, 12);
|
|
82895
|
+
block.write(" ", 148, 8, "ascii");
|
|
82896
|
+
block.write("0", 156, 1, "ascii");
|
|
82897
|
+
block.write("ustar\0", 257, 6, "ascii");
|
|
82898
|
+
block.write("00", 263, 2, "ascii");
|
|
82899
|
+
writeString(block, split.prefix, 345, PREFIX_MAX);
|
|
82900
|
+
let sum = 0;
|
|
82901
|
+
for (const byte of block) sum += byte;
|
|
82902
|
+
writeOctal(block, sum, 148, 7);
|
|
82903
|
+
block.write("\0", 154, 1, "ascii");
|
|
82904
|
+
block.write(" ", 155, 1, "ascii");
|
|
82905
|
+
return block;
|
|
82906
|
+
}
|
|
82907
|
+
/** Zero padding that rounds a body up to the 512-byte block. */
|
|
82908
|
+
function tarPadding(size) {
|
|
82909
|
+
const remainder = size % BLOCK_SIZE;
|
|
82910
|
+
return remainder === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK_SIZE - remainder);
|
|
82911
|
+
}
|
|
82912
|
+
/** Header + body + padding for one entry, ready to write. */
|
|
82913
|
+
function tarBlocks(entry) {
|
|
82914
|
+
return [
|
|
82915
|
+
tarHeader(entry),
|
|
82916
|
+
entry.body,
|
|
82917
|
+
tarPadding(entry.body.length)
|
|
82918
|
+
];
|
|
82919
|
+
}
|
|
82920
|
+
/** The two zero blocks that terminate an archive. */
|
|
82921
|
+
function tarTrailer() {
|
|
82922
|
+
return Buffer.alloc(BLOCK_SIZE * 2);
|
|
82923
|
+
}
|
|
82924
|
+
//#endregion
|
|
82925
|
+
//#region src/pipeline-analytics/retrain/retrain-archive-writer.ts
|
|
82926
|
+
/**
|
|
82927
|
+
* The annotated archive body, written to ANY sink.
|
|
82928
|
+
*
|
|
82929
|
+
* One body, two consumers: the data-plane download (`retrain-export-handler`,
|
|
82930
|
+
* sink = the HTTP response) and `pipelineAnalytics.writeRetrainArchive` (sink
|
|
82931
|
+
* = one streamed HTTP upload to a training provider). The disciplines are the
|
|
82932
|
+
* training export's: the manifest is the first entry, one JPEG is in memory at
|
|
82933
|
+
* a time, a pause between files keeps the hub's one thread available, and the
|
|
82934
|
+
* batch id is stamped only on what the sink actually received.
|
|
82935
|
+
*/
|
|
82936
|
+
var RETRAIN_MANIFEST_ENTRY = "annotations.json";
|
|
82937
|
+
/** Pause between files so a large export cannot monopolise the hub's thread. */
|
|
82938
|
+
var RETRAIN_ARCHIVE_PACING_MS = 25;
|
|
82939
|
+
async function writeRetrainArchiveTo(deps) {
|
|
82940
|
+
const { plan, logger } = deps;
|
|
82941
|
+
const pacingMs = deps.pacingMs ?? RETRAIN_ARCHIVE_PACING_MS;
|
|
82942
|
+
const hash = (0, node_crypto.createHash)("sha256");
|
|
82943
|
+
let bytes = 0;
|
|
82944
|
+
const write = async (chunk) => {
|
|
82945
|
+
hash.update(chunk);
|
|
82946
|
+
bytes += chunk.length;
|
|
82947
|
+
await deps.sink.write(chunk);
|
|
82948
|
+
};
|
|
82949
|
+
const writeEntry = async (name, body, mtimeMs) => {
|
|
82950
|
+
for (const block of tarBlocks({
|
|
82951
|
+
name,
|
|
82952
|
+
body,
|
|
82953
|
+
mtimeMs
|
|
82954
|
+
})) await write(block);
|
|
82955
|
+
};
|
|
82956
|
+
const mtimeMs = deps.now();
|
|
82957
|
+
await writeEntry(RETRAIN_MANIFEST_ENTRY, Buffer.from(JSON.stringify(plan.manifest, null, 2), "utf8"), mtimeMs);
|
|
82958
|
+
const byId = new Map(plan.manifest.frames.map((f) => [f.frameId, f]));
|
|
82959
|
+
const shipped = [];
|
|
82960
|
+
let missing = 0;
|
|
82961
|
+
let crops = 0;
|
|
82962
|
+
for (const frame of plan.frames) {
|
|
82963
|
+
const entry = byId.get(frame.frameId);
|
|
82964
|
+
if (entry === void 0) continue;
|
|
82965
|
+
if (tarNameTooLong(entry.path)) {
|
|
82966
|
+
missing += 1;
|
|
82967
|
+
logger.warn("retrain export skipped a frame — path too long for the archive", {
|
|
82968
|
+
tags: { deviceId: frame.deviceId },
|
|
82969
|
+
meta: {
|
|
82970
|
+
trackId: frame.trackId,
|
|
82971
|
+
frameId: frame.frameId,
|
|
82972
|
+
path: entry.path
|
|
82973
|
+
}
|
|
82974
|
+
});
|
|
82975
|
+
continue;
|
|
82976
|
+
}
|
|
82977
|
+
const frameBytes = await deps.readFrame(frame);
|
|
82978
|
+
if (frameBytes === null) {
|
|
82979
|
+
missing += 1;
|
|
82980
|
+
logger.warn("retrain export skipped a frame — the COPY is unreadable", {
|
|
82981
|
+
tags: { deviceId: frame.deviceId },
|
|
82982
|
+
meta: {
|
|
82983
|
+
trackId: frame.trackId,
|
|
82984
|
+
frameId: frame.frameId
|
|
82985
|
+
}
|
|
82986
|
+
});
|
|
82987
|
+
continue;
|
|
82988
|
+
}
|
|
82989
|
+
await writeEntry(entry.path, frameBytes, mtimeMs);
|
|
82990
|
+
for (const annotation of entry.annotations) {
|
|
82991
|
+
if (annotation.cropPath === void 0) continue;
|
|
82992
|
+
const crop = deps.cutCrop ? await deps.cutCrop(frameBytes, frame, annotation) : null;
|
|
82993
|
+
if (crop === null) {
|
|
82994
|
+
logger.warn("retrain export skipped a crop — it could not be cut", {
|
|
82995
|
+
tags: { deviceId: frame.deviceId },
|
|
82996
|
+
meta: {
|
|
82997
|
+
frameId: frame.frameId,
|
|
82998
|
+
annotationId: annotation.id,
|
|
82999
|
+
cropPath: annotation.cropPath
|
|
83000
|
+
}
|
|
83001
|
+
});
|
|
83002
|
+
continue;
|
|
83003
|
+
}
|
|
83004
|
+
await writeEntry(annotation.cropPath, crop, mtimeMs);
|
|
83005
|
+
crops += 1;
|
|
83006
|
+
}
|
|
83007
|
+
shipped.push(frame.frameId);
|
|
83008
|
+
if (pacingMs > 0) await new Promise((resolve) => setTimeout(resolve, pacingMs));
|
|
83009
|
+
}
|
|
83010
|
+
await write(tarTrailer());
|
|
83011
|
+
await deps.markExported(shipped, plan.manifest.batchId);
|
|
83012
|
+
const stats = {
|
|
83013
|
+
batchId: plan.manifest.batchId,
|
|
83014
|
+
frames: shipped.length,
|
|
83015
|
+
crops,
|
|
83016
|
+
annotations: plan.manifest.annotationCount,
|
|
83017
|
+
missing,
|
|
83018
|
+
bytes,
|
|
83019
|
+
sha256: hash.digest("hex")
|
|
83020
|
+
};
|
|
83021
|
+
logger.info("retrain export complete", { meta: {
|
|
83022
|
+
...stats,
|
|
83023
|
+
layout: plan.manifest.layout,
|
|
83024
|
+
modelErrors: plan.manifest.modelErrorCount
|
|
83025
|
+
} });
|
|
83026
|
+
return stats;
|
|
83027
|
+
}
|
|
83028
|
+
//#endregion
|
|
83029
|
+
//#region src/pipeline-analytics/retrain/retrain-crop-cut.ts
|
|
83030
|
+
/**
|
|
83031
|
+
* The `crops` layout's cutter: one subject crop from a retrain frame copy.
|
|
83032
|
+
*
|
|
83033
|
+
* A classifier must be trained on the crop the runtime feeds it, so the
|
|
83034
|
+
* rectangle comes from THE detail-crop derivation (`deriveDetailCropRect`,
|
|
83035
|
+
* `types/src/pipeline/detail-crop.ts`) with the convention the manifest
|
|
83036
|
+
* records — never a second rectangle maths. This is the second permitted
|
|
83037
|
+
* caller of that derivation (`scripts/check-clip-crop-single-owner.ts`): it
|
|
83038
|
+
* writes training files, never a vector, so it cannot split a feature space;
|
|
83039
|
+
* what it must not do is cut differently, and calling the one function is how
|
|
83040
|
+
* it doesn't.
|
|
83041
|
+
*/
|
|
83042
|
+
/** The pixel window to extract, or `null` when nothing of it is inside the frame. */
|
|
83043
|
+
function retrainCropWindow(input) {
|
|
83044
|
+
const { frameWidth: fw, frameHeight: fh, bbox } = input;
|
|
83045
|
+
const rect = require_clip_model_registry.deriveDetailCropRect({
|
|
83046
|
+
x: bbox.x * fw,
|
|
83047
|
+
y: bbox.y * fh,
|
|
83048
|
+
w: bbox.w * fw,
|
|
83049
|
+
h: bbox.h * fh
|
|
83050
|
+
}, fw, fh, input.convention);
|
|
83051
|
+
const left = Math.max(0, Math.floor(rect.x));
|
|
83052
|
+
const top = Math.max(0, Math.floor(rect.y));
|
|
83053
|
+
const right = Math.min(fw, Math.ceil(rect.x + rect.w));
|
|
83054
|
+
const bottom = Math.min(fh, Math.ceil(rect.y + rect.h));
|
|
83055
|
+
if (right - left < 1 || bottom - top < 1) return null;
|
|
83056
|
+
return {
|
|
83057
|
+
left,
|
|
83058
|
+
top,
|
|
83059
|
+
width: right - left,
|
|
83060
|
+
height: bottom - top
|
|
83061
|
+
};
|
|
83062
|
+
}
|
|
83063
|
+
async function cutRetrainCrop(input) {
|
|
83064
|
+
const window = retrainCropWindow(input);
|
|
83065
|
+
if (window === null) return null;
|
|
83066
|
+
let pipeline = (0, sharp.default)(input.frameBytes).extract(window);
|
|
83067
|
+
if (input.maxSide > 0) pipeline = pipeline.resize({
|
|
83068
|
+
width: input.maxSide,
|
|
83069
|
+
height: input.maxSide,
|
|
83070
|
+
fit: "inside",
|
|
83071
|
+
withoutEnlargement: true
|
|
83072
|
+
});
|
|
83073
|
+
return pipeline.jpeg({ quality: 92 }).toBuffer();
|
|
83074
|
+
}
|
|
83075
|
+
//#endregion
|
|
82546
83076
|
//#region src/pipeline-analytics/retrain/retrain-geometry.ts
|
|
82547
83077
|
function clamp01(v) {
|
|
82548
83078
|
return v < 0 ? 0 : v > 1 ? 1 : v;
|
|
@@ -82638,15 +83168,99 @@ function deriveSubjectCropRect(bbox, padding) {
|
|
|
82638
83168
|
};
|
|
82639
83169
|
}
|
|
82640
83170
|
//#endregion
|
|
83171
|
+
//#region src/pipeline-analytics/retrain/retrain-lighting.ts
|
|
83172
|
+
function isNight(condition) {
|
|
83173
|
+
return condition === "night" || condition === "ir";
|
|
83174
|
+
}
|
|
83175
|
+
function retrainFrameLighting(input) {
|
|
83176
|
+
if (input.capturedAt === void 0) {
|
|
83177
|
+
const measured = resolveConditionDetailed({
|
|
83178
|
+
...input.chromaSpread !== void 0 ? { chromaSpread: input.chromaSpread } : {},
|
|
83179
|
+
now: /* @__PURE__ */ new Date(0)
|
|
83180
|
+
});
|
|
83181
|
+
return measured.source === "ir-chroma" ? {
|
|
83182
|
+
condition: measured.condition,
|
|
83183
|
+
night: true,
|
|
83184
|
+
source: measured.source
|
|
83185
|
+
} : null;
|
|
83186
|
+
}
|
|
83187
|
+
const resolved = resolveConditionDetailed({
|
|
83188
|
+
...input.chromaSpread !== void 0 ? { chromaSpread: input.chromaSpread } : {},
|
|
83189
|
+
...input.coordinates !== null ? { coordinates: input.coordinates } : {},
|
|
83190
|
+
now: new Date(input.capturedAt)
|
|
83191
|
+
});
|
|
83192
|
+
return {
|
|
83193
|
+
...resolved,
|
|
83194
|
+
night: isNight(resolved.condition)
|
|
83195
|
+
};
|
|
83196
|
+
}
|
|
83197
|
+
//#endregion
|
|
82641
83198
|
//#region src/pipeline-analytics/retrain/retrain-export.ts
|
|
83199
|
+
/**
|
|
83200
|
+
* The ANNOTATED dataset export — the retrain page's own archive.
|
|
83201
|
+
*
|
|
83202
|
+
* Sibling of `training-export/`, deliberately not the same thing. That one
|
|
83203
|
+
* streams a marked track's stored media verbatim and knows nothing about what
|
|
83204
|
+
* is in it; this one streams the frames the operator SELECTED (already copied
|
|
83205
|
+
* into retrain storage) plus the ground truth they drew.
|
|
83206
|
+
*
|
|
83207
|
+
* ## The one interesting decision
|
|
83208
|
+
*
|
|
83209
|
+
* `annotations.json` carries, for every subject, the canonical full-frame box
|
|
83210
|
+
* AND the geometry derived for each model shape. **Derived here, at export,
|
|
83211
|
+
* never stored** — one box in, three shapes out. Storing the derived shapes is
|
|
83212
|
+
* how two crops of the same subject end up in one feature space, which is the
|
|
83213
|
+
* failure already recorded for CLIP (D52). A consumer that wants a different
|
|
83214
|
+
* input size or a different padding re-derives from the canonical box; nothing
|
|
83215
|
+
* in the database has to change and nothing can disagree.
|
|
83216
|
+
*
|
|
83217
|
+
* Everything else follows the export next door: index reads to plan, one JPEG
|
|
83218
|
+
* in memory at a time, a yield between files, and NO inference — a training
|
|
83219
|
+
* set's pixels are the pixels the pipeline saw.
|
|
83220
|
+
*/
|
|
82642
83221
|
var RETRAIN_EXPORT_PREFIX = "retrain-export";
|
|
82643
|
-
/**
|
|
82644
|
-
|
|
83222
|
+
/**
|
|
83223
|
+
* Padding a subject-crop classifier is trained with: the RUNTIME detail-crop
|
|
83224
|
+
* padding, read from the one convention `deriveDetailCropRect` cuts with
|
|
83225
|
+
* (`types/src/pipeline/detail-crop.ts`), never a second literal. It was `0.2`
|
|
83226
|
+
* here while the runtime fed `0.15`, so a classifier trained from this export
|
|
83227
|
+
* learned on more context than it is ever given live.
|
|
83228
|
+
*/
|
|
83229
|
+
var CLASSIFIER_CROP_PADDING = require_clip_model_registry.DEFAULT_DETAIL_CROP_CONVENTION.paddingRatio;
|
|
83230
|
+
/** The convention a `crops` layout cuts with — the runtime default, recorded
|
|
83231
|
+
* in the manifest so a consumer never has to assume it. */
|
|
83232
|
+
var RETRAIN_CROP_CONVENTION = require_clip_model_registry.DEFAULT_DETAIL_CROP_CONVENTION;
|
|
83233
|
+
/** Longest side of a `crops`-layout JPEG. Classifier inputs are 224–260 px;
|
|
83234
|
+
* the trainer resizes per model, so this only bounds the upload. */
|
|
83235
|
+
var RETRAIN_CROP_MAX_SIDE = 256;
|
|
83236
|
+
/** Manifest schema id. v2 = capturedAt / complete / lighting / layout. */
|
|
83237
|
+
var RETRAIN_MANIFEST_SCHEMA = "camstack.retrain-export/v2";
|
|
82645
83238
|
/** Archive path of a frame: `frames/<deviceId>/<trackId>/<frameId leaf>.jpg`. */
|
|
82646
83239
|
function retrainFramePath(frame) {
|
|
82647
83240
|
const leaf = frame.frameId.slice(frame.frameId.lastIndexOf(":") + 1);
|
|
82648
83241
|
return `frames/${frame.deviceId}/${frame.trackId}/${leaf}.jpg`;
|
|
82649
83242
|
}
|
|
83243
|
+
/** Archive path of a subject crop: `crops/<macroClass>/<annotationId>.jpg`. */
|
|
83244
|
+
function retrainCropPath(macroClass, annotationId) {
|
|
83245
|
+
return `crops/${macroClass}/${annotationId}.jpg`;
|
|
83246
|
+
}
|
|
83247
|
+
/**
|
|
83248
|
+
* Narrow the planned frames to a filter. Pure. Every field narrows; an empty
|
|
83249
|
+
* filter keeps everything. Order is the input's.
|
|
83250
|
+
*/
|
|
83251
|
+
function filterRetrainExport(frames, annotationsByFrame, filter) {
|
|
83252
|
+
const wanted = filter.frameIds !== void 0 ? new Set(filter.frameIds) : null;
|
|
83253
|
+
const devices = filter.deviceIds !== void 0 && filter.deviceIds.length > 0 ? new Set(filter.deviceIds) : null;
|
|
83254
|
+
return frames.filter((frame) => {
|
|
83255
|
+
if (wanted !== null && !wanted.has(frame.frameId)) return false;
|
|
83256
|
+
if (devices !== null && !devices.has(frame.deviceId)) return false;
|
|
83257
|
+
if (filter.completeOnly === true && !frame.complete) return false;
|
|
83258
|
+
if (filter.batchId !== void 0) {
|
|
83259
|
+
if (!(annotationsByFrame.get(frame.frameId) ?? []).some((a) => a.exportedInBatch === filter.batchId)) return false;
|
|
83260
|
+
}
|
|
83261
|
+
return true;
|
|
83262
|
+
});
|
|
83263
|
+
}
|
|
82650
83264
|
/**
|
|
82651
83265
|
* Derive every per-model shape for one annotation.
|
|
82652
83266
|
*
|
|
@@ -82667,6 +83281,8 @@ function deriveAll(annotation, frame, zone) {
|
|
|
82667
83281
|
}
|
|
82668
83282
|
/** Build the manifest. Pure — index rows in, JSON out, no blob touched. */
|
|
82669
83283
|
function buildRetrainManifest(input) {
|
|
83284
|
+
const layout = input.layout ?? "frames";
|
|
83285
|
+
const coordinates = input.coordinates ?? null;
|
|
82670
83286
|
let annotationCount = 0;
|
|
82671
83287
|
let modelErrorCount = 0;
|
|
82672
83288
|
const frames = input.frames.map((frame) => {
|
|
@@ -82674,6 +83290,7 @@ function buildRetrainManifest(input) {
|
|
|
82674
83290
|
const annotations = (input.annotationsByFrame.get(frame.frameId) ?? []).map((a) => {
|
|
82675
83291
|
annotationCount += 1;
|
|
82676
83292
|
if (a.kind === "model_error") modelErrorCount += 1;
|
|
83293
|
+
const derived = deriveAll(a, frame, zone);
|
|
82677
83294
|
return {
|
|
82678
83295
|
id: a.id,
|
|
82679
83296
|
macroClass: a.macroClass,
|
|
@@ -82684,7 +83301,8 @@ function buildRetrainManifest(input) {
|
|
|
82684
83301
|
...a.assistModelId !== void 0 ? { assistModelId: a.assistModelId } : {},
|
|
82685
83302
|
...a.assistScore !== void 0 ? { assistScore: a.assistScore } : {},
|
|
82686
83303
|
bbox: a.bbox,
|
|
82687
|
-
derived:
|
|
83304
|
+
...layout === "crops" && derived.subjectCrop !== void 0 ? { cropPath: retrainCropPath(a.macroClass, a.id) } : {},
|
|
83305
|
+
derived
|
|
82688
83306
|
};
|
|
82689
83307
|
});
|
|
82690
83308
|
return {
|
|
@@ -82697,11 +83315,22 @@ function buildRetrainManifest(input) {
|
|
|
82697
83315
|
sizeBytes: frame.sizeBytes,
|
|
82698
83316
|
sourceKind: frame.sourceKind,
|
|
82699
83317
|
copiedAt: frame.copiedAt,
|
|
83318
|
+
...frame.capturedAt !== void 0 ? { capturedAt: frame.capturedAt } : {},
|
|
83319
|
+
complete: frame.complete,
|
|
83320
|
+
lighting: retrainFrameLighting({
|
|
83321
|
+
...frame.capturedAt !== void 0 ? { capturedAt: frame.capturedAt } : {},
|
|
83322
|
+
...frame.chromaSpread !== void 0 ? { chromaSpread: frame.chromaSpread } : {},
|
|
83323
|
+
coordinates
|
|
83324
|
+
}),
|
|
82700
83325
|
...zone !== null ? { zoneUsed: zone } : {},
|
|
82701
83326
|
annotations
|
|
82702
83327
|
};
|
|
82703
83328
|
});
|
|
82704
83329
|
return {
|
|
83330
|
+
schema: RETRAIN_MANIFEST_SCHEMA,
|
|
83331
|
+
layout,
|
|
83332
|
+
detailCropConvention: RETRAIN_CROP_CONVENTION,
|
|
83333
|
+
...layout === "crops" ? { cropMaxSide: RETRAIN_CROP_MAX_SIDE } : {},
|
|
82705
83334
|
generatedAt: input.generatedAt,
|
|
82706
83335
|
batchId: input.batchId,
|
|
82707
83336
|
frameCount: frames.length,
|
|
@@ -82712,123 +83341,36 @@ function buildRetrainManifest(input) {
|
|
|
82712
83341
|
};
|
|
82713
83342
|
}
|
|
82714
83343
|
//#endregion
|
|
82715
|
-
//#region src/pipeline-analytics/
|
|
82716
|
-
/**
|
|
82717
|
-
|
|
82718
|
-
|
|
82719
|
-
|
|
82720
|
-
|
|
82721
|
-
|
|
82722
|
-
|
|
82723
|
-
|
|
82724
|
-
|
|
82725
|
-
|
|
82726
|
-
|
|
82727
|
-
|
|
82728
|
-
|
|
82729
|
-
|
|
82730
|
-
|
|
82731
|
-
*
|
|
82732
|
-
* So: ustar, which is a 512-byte header and a 512-byte-padded body per entry,
|
|
82733
|
-
* fully specified, and asserted against the system `tar` binary in the spec next
|
|
82734
|
-
* door rather than against my reading of the format.
|
|
82735
|
-
*
|
|
82736
|
-
* ## What this deliberately does NOT do
|
|
82737
|
-
*
|
|
82738
|
-
* No gzip — the payload is JPEG, already compressed; a gzip pass would burn CPU
|
|
82739
|
-
* on the hub for a percent or two. No long-name (GNU/PAX) extension: ustar
|
|
82740
|
-
* splits a path across a 155-byte prefix and a 100-byte name, and
|
|
82741
|
-
* {@link tarNameTooLong} reports a path that will not fit so the caller can skip
|
|
82742
|
-
* it and SAY it skipped it, rather than emitting a silently truncated entry.
|
|
82743
|
-
*/
|
|
82744
|
-
var BLOCK_SIZE = 512;
|
|
82745
|
-
var NAME_MAX = 100;
|
|
82746
|
-
var PREFIX_MAX = 155;
|
|
82747
|
-
/**
|
|
82748
|
-
* Split a path into the ustar `prefix` + `name` pair, or `null` when it cannot
|
|
82749
|
-
* be represented. The split must land on a `/`, and each half has its own cap.
|
|
82750
|
-
*/
|
|
82751
|
-
function splitUstarName(name) {
|
|
82752
|
-
if (Buffer.byteLength(name) <= NAME_MAX) return {
|
|
82753
|
-
prefix: "",
|
|
82754
|
-
name
|
|
83344
|
+
//#region src/pipeline-analytics/retrain/retrain-export-handler.ts
|
|
83345
|
+
/** The export request a URL carries (`getRetrainExportUrl` composes it). */
|
|
83346
|
+
function parseRetrainExportRequest(url) {
|
|
83347
|
+
const query = url.includes("?") ? url.slice(url.indexOf("?") + 1) : "";
|
|
83348
|
+
const params = new URLSearchParams(query);
|
|
83349
|
+
const deviceIds = parseRetrainDeviceIds(url);
|
|
83350
|
+
const frameIds = (params.get("frameIds") ?? "").split(",").map((id) => id.trim()).filter((id) => id.length > 0);
|
|
83351
|
+
const batchId = params.get("batchId")?.trim() ?? "";
|
|
83352
|
+
return {
|
|
83353
|
+
filter: {
|
|
83354
|
+
...deviceIds.length > 0 ? { deviceIds: [...deviceIds] } : {},
|
|
83355
|
+
...frameIds.length > 0 ? { frameIds } : {},
|
|
83356
|
+
...batchId.length > 0 ? { batchId } : {},
|
|
83357
|
+
...params.get("completeOnly") === "1" ? { completeOnly: true } : {}
|
|
83358
|
+
},
|
|
83359
|
+
layout: params.get("layout") === "crops" ? "crops" : "frames"
|
|
82755
83360
|
};
|
|
82756
|
-
for (let i = name.length - 1; i >= 0; i--) {
|
|
82757
|
-
if (name[i] !== "/") continue;
|
|
82758
|
-
const head = name.slice(0, i);
|
|
82759
|
-
const tail = name.slice(i + 1);
|
|
82760
|
-
if (Buffer.byteLength(tail) > NAME_MAX) break;
|
|
82761
|
-
if (Buffer.byteLength(head) <= PREFIX_MAX) return {
|
|
82762
|
-
prefix: head,
|
|
82763
|
-
name: tail
|
|
82764
|
-
};
|
|
82765
|
-
}
|
|
82766
|
-
return null;
|
|
82767
83361
|
}
|
|
82768
|
-
/**
|
|
82769
|
-
function
|
|
82770
|
-
|
|
82771
|
-
}
|
|
82772
|
-
|
|
82773
|
-
|
|
82774
|
-
|
|
82775
|
-
|
|
82776
|
-
|
|
82777
|
-
const text =
|
|
82778
|
-
|
|
83362
|
+
/** The query string `parseRetrainExportRequest` reads back. */
|
|
83363
|
+
function retrainExportQuery(request) {
|
|
83364
|
+
const params = new URLSearchParams();
|
|
83365
|
+
const { filter } = request;
|
|
83366
|
+
if (filter.deviceIds !== void 0 && filter.deviceIds.length > 0) params.set("deviceIds", filter.deviceIds.join(","));
|
|
83367
|
+
if (filter.frameIds !== void 0 && filter.frameIds.length > 0) params.set("frameIds", filter.frameIds.join(","));
|
|
83368
|
+
if (filter.batchId !== void 0) params.set("batchId", filter.batchId);
|
|
83369
|
+
if (filter.completeOnly === true) params.set("completeOnly", "1");
|
|
83370
|
+
if (request.layout === "crops") params.set("layout", "crops");
|
|
83371
|
+
const text = params.toString();
|
|
83372
|
+
return text.length > 0 ? `?${text}` : "";
|
|
82779
83373
|
}
|
|
82780
|
-
/**
|
|
82781
|
-
* The 512-byte header for one regular file.
|
|
82782
|
-
*
|
|
82783
|
-
* The checksum is computed with the checksum field itself read as eight spaces —
|
|
82784
|
-
* that is the format's rule, and getting it wrong produces an archive that looks
|
|
82785
|
-
* fine until something tries to read it.
|
|
82786
|
-
*/
|
|
82787
|
-
function tarHeader(entry) {
|
|
82788
|
-
const split = splitUstarName(entry.name);
|
|
82789
|
-
if (split === null) throw new Error(`tar: entry name too long for ustar: ${entry.name}`);
|
|
82790
|
-
const block = Buffer.alloc(BLOCK_SIZE);
|
|
82791
|
-
writeString(block, split.name, 0, NAME_MAX);
|
|
82792
|
-
writeOctal(block, 420, 100, 8);
|
|
82793
|
-
writeOctal(block, 0, 108, 8);
|
|
82794
|
-
writeOctal(block, 0, 116, 8);
|
|
82795
|
-
writeOctal(block, entry.body.length, 124, 12);
|
|
82796
|
-
writeOctal(block, Math.floor(entry.mtimeMs / 1e3), 136, 12);
|
|
82797
|
-
block.write(" ", 148, 8, "ascii");
|
|
82798
|
-
block.write("0", 156, 1, "ascii");
|
|
82799
|
-
block.write("ustar\0", 257, 6, "ascii");
|
|
82800
|
-
block.write("00", 263, 2, "ascii");
|
|
82801
|
-
writeString(block, split.prefix, 345, PREFIX_MAX);
|
|
82802
|
-
let sum = 0;
|
|
82803
|
-
for (const byte of block) sum += byte;
|
|
82804
|
-
writeOctal(block, sum, 148, 7);
|
|
82805
|
-
block.write("\0", 154, 1, "ascii");
|
|
82806
|
-
block.write(" ", 155, 1, "ascii");
|
|
82807
|
-
return block;
|
|
82808
|
-
}
|
|
82809
|
-
/** Zero padding that rounds a body up to the 512-byte block. */
|
|
82810
|
-
function tarPadding(size) {
|
|
82811
|
-
const remainder = size % BLOCK_SIZE;
|
|
82812
|
-
return remainder === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK_SIZE - remainder);
|
|
82813
|
-
}
|
|
82814
|
-
/** Header + body + padding for one entry, ready to write. */
|
|
82815
|
-
function tarBlocks(entry) {
|
|
82816
|
-
return [
|
|
82817
|
-
tarHeader(entry),
|
|
82818
|
-
entry.body,
|
|
82819
|
-
tarPadding(entry.body.length)
|
|
82820
|
-
];
|
|
82821
|
-
}
|
|
82822
|
-
/** The two zero blocks that terminate an archive. */
|
|
82823
|
-
function tarTrailer() {
|
|
82824
|
-
return Buffer.alloc(BLOCK_SIZE * 2);
|
|
82825
|
-
}
|
|
82826
|
-
//#endregion
|
|
82827
|
-
//#region src/pipeline-analytics/retrain/retrain-export-handler.ts
|
|
82828
|
-
var RETRAIN_MANIFEST_ENTRY = "annotations.json";
|
|
82829
|
-
/** Yield between files so a large export cannot monopolise the hub's one
|
|
82830
|
-
* thread — the same discipline the training export follows. */
|
|
82831
|
-
var PACING_MS = 25;
|
|
82832
83374
|
function parseRetrainDeviceIds(url) {
|
|
82833
83375
|
const query = url.includes("?") ? url.slice(url.indexOf("?") + 1) : "";
|
|
82834
83376
|
const raw = new URLSearchParams(query).get("deviceIds");
|
|
@@ -82851,10 +83393,10 @@ function createRetrainExportHandler(deps) {
|
|
|
82851
83393
|
res.end();
|
|
82852
83394
|
return;
|
|
82853
83395
|
}
|
|
82854
|
-
const
|
|
83396
|
+
const request = parseRetrainExportRequest(req.url ?? "");
|
|
82855
83397
|
let plan;
|
|
82856
83398
|
try {
|
|
82857
|
-
plan = await deps.buildPlan(
|
|
83399
|
+
plan = await deps.buildPlan(request);
|
|
82858
83400
|
} catch (err) {
|
|
82859
83401
|
deps.logger.warn("retrain export plan failed", { meta: { error: String(err) } });
|
|
82860
83402
|
res.writeHead(500, { "content-type": "text/plain" });
|
|
@@ -82868,7 +83410,8 @@ function createRetrainExportHandler(deps) {
|
|
|
82868
83410
|
"x-camstack-retrain-frames": String(plan.manifest.frameCount),
|
|
82869
83411
|
"x-camstack-retrain-annotations": String(plan.manifest.annotationCount),
|
|
82870
83412
|
"x-camstack-retrain-model-errors": String(plan.manifest.modelErrorCount),
|
|
82871
|
-
"x-camstack-retrain-batch": plan.manifest.batchId
|
|
83413
|
+
"x-camstack-retrain-batch": plan.manifest.batchId,
|
|
83414
|
+
"x-camstack-retrain-layout": plan.manifest.layout
|
|
82872
83415
|
});
|
|
82873
83416
|
if (method === "HEAD") {
|
|
82874
83417
|
res.end();
|
|
@@ -82881,61 +83424,18 @@ function createRetrainExportHandler(deps) {
|
|
|
82881
83424
|
});
|
|
82882
83425
|
if (!ok) res.once("drain", resolve);
|
|
82883
83426
|
});
|
|
82884
|
-
|
|
82885
|
-
|
|
82886
|
-
|
|
82887
|
-
|
|
82888
|
-
|
|
82889
|
-
|
|
82890
|
-
|
|
82891
|
-
|
|
82892
|
-
|
|
82893
|
-
const entry = plan.manifest.frames.find((f) => f.frameId === frame.frameId);
|
|
82894
|
-
if (entry === void 0) continue;
|
|
82895
|
-
if (tarNameTooLong(entry.path)) {
|
|
82896
|
-
missing += 1;
|
|
82897
|
-
deps.logger.warn("retrain export skipped a frame — path too long for the archive", {
|
|
82898
|
-
tags: { deviceId: frame.deviceId },
|
|
82899
|
-
meta: {
|
|
82900
|
-
trackId: frame.trackId,
|
|
82901
|
-
frameId: frame.frameId,
|
|
82902
|
-
path: entry.path
|
|
82903
|
-
}
|
|
82904
|
-
});
|
|
82905
|
-
continue;
|
|
82906
|
-
}
|
|
82907
|
-
const bytes = await deps.readFrame(frame);
|
|
82908
|
-
if (bytes === null) {
|
|
82909
|
-
missing += 1;
|
|
82910
|
-
deps.logger.warn("retrain export skipped a frame — the COPY is unreadable", {
|
|
82911
|
-
tags: { deviceId: frame.deviceId },
|
|
82912
|
-
meta: {
|
|
82913
|
-
trackId: frame.trackId,
|
|
82914
|
-
frameId: frame.frameId
|
|
82915
|
-
}
|
|
82916
|
-
});
|
|
82917
|
-
continue;
|
|
82918
|
-
}
|
|
82919
|
-
for (const block of tarBlocks({
|
|
82920
|
-
name: entry.path,
|
|
82921
|
-
body: bytes,
|
|
82922
|
-
mtimeMs
|
|
82923
|
-
})) await write(block);
|
|
82924
|
-
shipped.push(frame.frameId);
|
|
82925
|
-
await new Promise((resolve) => setTimeout(resolve, PACING_MS));
|
|
82926
|
-
}
|
|
82927
|
-
await write(tarTrailer());
|
|
83427
|
+
await writeRetrainArchiveTo({
|
|
83428
|
+
plan,
|
|
83429
|
+
readFrame: deps.readFrame,
|
|
83430
|
+
cutCrop: deps.cutCrop,
|
|
83431
|
+
sink: { write },
|
|
83432
|
+
markExported: deps.markExported,
|
|
83433
|
+
logger: deps.logger,
|
|
83434
|
+
now: deps.now
|
|
83435
|
+
});
|
|
82928
83436
|
await new Promise((resolve) => {
|
|
82929
83437
|
res.end(resolve);
|
|
82930
83438
|
});
|
|
82931
|
-
await deps.markExported(shipped, plan.manifest.batchId);
|
|
82932
|
-
deps.logger.info("retrain export complete", { meta: {
|
|
82933
|
-
batchId: plan.manifest.batchId,
|
|
82934
|
-
frames: shipped.length,
|
|
82935
|
-
missing,
|
|
82936
|
-
annotations: plan.manifest.annotationCount,
|
|
82937
|
-
modelErrors: plan.manifest.modelErrorCount
|
|
82938
|
-
} });
|
|
82939
83439
|
};
|
|
82940
83440
|
}
|
|
82941
83441
|
//#endregion
|
|
@@ -83253,7 +83753,9 @@ function toWireFrame(f) {
|
|
|
83253
83753
|
sizeBytes: f.sizeBytes,
|
|
83254
83754
|
width: f.width,
|
|
83255
83755
|
height: f.height,
|
|
83256
|
-
copiedAt: f.copiedAt
|
|
83756
|
+
copiedAt: f.copiedAt,
|
|
83757
|
+
...f.capturedAt !== void 0 ? { capturedAt: f.capturedAt } : {},
|
|
83758
|
+
complete: f.complete
|
|
83257
83759
|
};
|
|
83258
83760
|
}
|
|
83259
83761
|
function toWireAnnotation(a) {
|
|
@@ -83356,6 +83858,7 @@ var RetrainService = class {
|
|
|
83356
83858
|
async selectFrames(input) {
|
|
83357
83859
|
const media = await this.deps.media.listTrackMedia(input.trackId);
|
|
83358
83860
|
const kindOf = new Map(media.map((m) => [m.mediaId, m.kind]));
|
|
83861
|
+
const capturedAtOf = new Map(media.map((m) => [m.mediaId, m.timestamp]));
|
|
83359
83862
|
const copied = [];
|
|
83360
83863
|
const refused = [];
|
|
83361
83864
|
for (const mediaId of input.mediaIds) {
|
|
@@ -83363,7 +83866,8 @@ var RetrainService = class {
|
|
|
83363
83866
|
deviceId: input.deviceId,
|
|
83364
83867
|
trackId: input.trackId,
|
|
83365
83868
|
sourceMediaId: mediaId,
|
|
83366
|
-
sourceKind: kindOf.get(mediaId) ?? "keyFrame"
|
|
83869
|
+
sourceKind: kindOf.get(mediaId) ?? "keyFrame",
|
|
83870
|
+
...capturedAtOf.has(mediaId) ? { capturedAt: capturedAtOf.get(mediaId) } : {}
|
|
83367
83871
|
});
|
|
83368
83872
|
if (result.kind === "copied") copied.push(toWireFrame(result.frame));
|
|
83369
83873
|
else refused.push({
|
|
@@ -83376,6 +83880,34 @@ var RetrainService = class {
|
|
|
83376
83880
|
refused
|
|
83377
83881
|
};
|
|
83378
83882
|
}
|
|
83883
|
+
/**
|
|
83884
|
+
* The operator's "every subject in this frame is boxed". Scoped to the
|
|
83885
|
+
* track the caller names: a frame of another track is refused, loudly.
|
|
83886
|
+
*/
|
|
83887
|
+
async setFrameComplete(input) {
|
|
83888
|
+
const frame = await this.deps.frames.get(input.frameId);
|
|
83889
|
+
if (frame === null || frame.trackId !== input.trackId || frame.deviceId !== input.deviceId) {
|
|
83890
|
+
this.deps.logger.warn("retrain complete refused: the frame does not belong to that track", {
|
|
83891
|
+
tags: { deviceId: input.deviceId },
|
|
83892
|
+
meta: {
|
|
83893
|
+
trackId: input.trackId,
|
|
83894
|
+
frameId: input.frameId
|
|
83895
|
+
}
|
|
83896
|
+
});
|
|
83897
|
+
throw new RetrainAnnotationError(`retrain frame ${input.frameId} does not belong to track ${input.trackId}`);
|
|
83898
|
+
}
|
|
83899
|
+
const updated = await this.deps.frames.setComplete(input.frameId, input.complete);
|
|
83900
|
+
if (updated === null) throw new RetrainAnnotationError(`retrain frame ${input.frameId} vanished`);
|
|
83901
|
+
this.deps.logger.info("retrain frame completeness set", {
|
|
83902
|
+
tags: { deviceId: input.deviceId },
|
|
83903
|
+
meta: {
|
|
83904
|
+
trackId: input.trackId,
|
|
83905
|
+
frameId: input.frameId,
|
|
83906
|
+
complete: input.complete
|
|
83907
|
+
}
|
|
83908
|
+
});
|
|
83909
|
+
return toWireFrame(updated);
|
|
83910
|
+
}
|
|
83379
83911
|
/** Un-select: annotations first, then the copy. Never the other way round —
|
|
83380
83912
|
* an annotation whose frame is gone is unopenable and unfixable. */
|
|
83381
83913
|
async deselectFrame(input) {
|
|
@@ -84588,7 +85120,8 @@ function selectsRow(mode, toLocationId, data, isRetrain, fromLocationId) {
|
|
|
84588
85120
|
if (mode === "seal") return rowIsUnstamped(data);
|
|
84589
85121
|
if (mediaRowLocation(data) === toLocationId) return false;
|
|
84590
85122
|
if (mode === "gallery") return !isRetrain && rowIsGallery(data);
|
|
84591
|
-
|
|
85123
|
+
if (isRetrain) return !rowIsDurableRetrainCopy(data);
|
|
85124
|
+
return !rowIsGallery(data);
|
|
84592
85125
|
}
|
|
84593
85126
|
var MediaRelocateEngine = class {
|
|
84594
85127
|
deps;
|
|
@@ -97247,7 +97780,7 @@ var ANALYTICS_COLLECTIONS = [
|
|
|
97247
97780
|
collection: RETRAIN_FRAMES_COLLECTION,
|
|
97248
97781
|
classification: {
|
|
97249
97782
|
kind: "never-orphaned",
|
|
97250
|
-
why: "the frame COPIES the dataset owns outright,
|
|
97783
|
+
why: "the frame COPIES the dataset owns outright, in the `retrain/` subtree of the durable `artifacts` class (D781), which no owner cascade and no age sweep walks (D81). Also declares `trackId` as provenance. Rows go only through `deleteFrame`, after their annotations, on operator deselection. NOTE: nothing caps this table or the disk it consumes — that is a deliberate decision, not an oversight, and it is the one collection here whose bound is an operator's habit."
|
|
97251
97784
|
}
|
|
97252
97785
|
},
|
|
97253
97786
|
{
|
|
@@ -108514,6 +109047,9 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
108514
109047
|
resolveRoot: (locationId) => locationId === LEGACY_ADDON_DATA_GALLERY_LOCATION ? Promise.resolve(node_path.default.join(this.ctx.dataDir, "gallery")) : locationId === GALLERY_MEDIA_LOCATION ? this.ctx.api.storage.resolve.query({
|
|
108515
109048
|
location: ARTIFACTS_MEDIA_LOCATION,
|
|
108516
109049
|
relativePath: GALLERY_TENANT_SUBPATH
|
|
109050
|
+
}) : locationId === RETRAIN_MEDIA_LOCATION ? this.ctx.api.storage.resolve.query({
|
|
109051
|
+
location: ARTIFACTS_MEDIA_LOCATION,
|
|
109052
|
+
relativePath: ""
|
|
108517
109053
|
}) : this.ctx.api.storage.resolve.query({
|
|
108518
109054
|
location: locationId,
|
|
108519
109055
|
relativePath: ""
|
|
@@ -108692,7 +109228,7 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
108692
109228
|
logger: logger.child("RetrainFrames"),
|
|
108693
109229
|
now: () => Date.now(),
|
|
108694
109230
|
writeGate: this.mediaWriteGate,
|
|
108695
|
-
|
|
109231
|
+
measureChroma: (bytes) => chromaSpread(bytes),
|
|
108696
109232
|
readSource: async (mediaKey) => {
|
|
108697
109233
|
const file = await mediaStore.getById(mediaKey);
|
|
108698
109234
|
return file === null ? null : Buffer.from(file.base64, "base64");
|
|
@@ -109929,10 +110465,59 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
109929
110465
|
/** Where to stream the ANNOTATED dataset from. Bytes never cross the cap. */
|
|
109930
110466
|
async getRetrainExportUrl(input) {
|
|
109931
110467
|
if (this.retrainExportDataPlane === null) throw new Error("retrain export: data plane unavailable on this node");
|
|
109932
|
-
const
|
|
109933
|
-
const query =
|
|
110468
|
+
const { layout, ...filter } = input;
|
|
110469
|
+
const query = retrainExportQuery({
|
|
110470
|
+
filter,
|
|
110471
|
+
layout: layout ?? "frames"
|
|
110472
|
+
});
|
|
109934
110473
|
return { url: `/addon/${this.ctx.id}/${RETRAIN_EXPORT_PREFIX}${query}` };
|
|
109935
110474
|
}
|
|
110475
|
+
async setRetrainFrameComplete(input) {
|
|
110476
|
+
return this.requireRetrain().setFrameComplete(input);
|
|
110477
|
+
}
|
|
110478
|
+
/**
|
|
110479
|
+
* Stream the annotated archive into ONE HTTP request to `target` — the
|
|
110480
|
+
* training upload. Same bytes as the download; never on the hub's disk.
|
|
110481
|
+
*/
|
|
110482
|
+
async writeRetrainArchive(input) {
|
|
110483
|
+
const frameStore = this.retrainFrames;
|
|
110484
|
+
if (frameStore === null) throw new Error("retrain export: analytics stores unavailable");
|
|
110485
|
+
const plan = await this.buildRetrainExportPlan({
|
|
110486
|
+
filter: input.filter,
|
|
110487
|
+
layout: input.layout
|
|
110488
|
+
});
|
|
110489
|
+
const upload = openRetrainUploadSink(input.target);
|
|
110490
|
+
const host = new URL(input.target.url).host;
|
|
110491
|
+
try {
|
|
110492
|
+
const stats = await writeRetrainArchiveTo({
|
|
110493
|
+
plan,
|
|
110494
|
+
readFrame: (frame) => frameStore.readFrame(frame),
|
|
110495
|
+
cutCrop: this.retrainCropCutter(),
|
|
110496
|
+
sink: upload.sink,
|
|
110497
|
+
markExported: (frameIds, batchId) => this.markRetrainExported(frameIds, batchId),
|
|
110498
|
+
logger: this.ctx.logger,
|
|
110499
|
+
now: () => Date.now()
|
|
110500
|
+
});
|
|
110501
|
+
const status = await upload.finish();
|
|
110502
|
+
this.ctx.logger.info("retrain archive uploaded", { meta: {
|
|
110503
|
+
host,
|
|
110504
|
+
status,
|
|
110505
|
+
...stats,
|
|
110506
|
+
layout: input.layout
|
|
110507
|
+
} });
|
|
110508
|
+
return {
|
|
110509
|
+
...stats,
|
|
110510
|
+
status
|
|
110511
|
+
};
|
|
110512
|
+
} catch (err) {
|
|
110513
|
+
upload.abort(require_clip_model_registry.errMsg(err));
|
|
110514
|
+
this.ctx.logger.warn("retrain archive upload failed", { meta: {
|
|
110515
|
+
host,
|
|
110516
|
+
error: require_clip_model_registry.errMsg(err)
|
|
110517
|
+
} });
|
|
110518
|
+
throw err;
|
|
110519
|
+
}
|
|
110520
|
+
}
|
|
109936
110521
|
/**
|
|
109937
110522
|
* The store seam the `debug.*` actions and the debug bundle both read through.
|
|
109938
110523
|
*
|
|
@@ -110214,15 +110799,10 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
110214
110799
|
async serveRetrainExportPlane() {
|
|
110215
110800
|
try {
|
|
110216
110801
|
const handler = createRetrainExportHandler({
|
|
110217
|
-
buildPlan: (
|
|
110802
|
+
buildPlan: (request) => this.buildRetrainExportPlan(request),
|
|
110218
110803
|
readFrame: (frame) => this.retrainFrames?.readFrame(frame) ?? Promise.resolve(null),
|
|
110219
|
-
|
|
110220
|
-
|
|
110221
|
-
if (!store || frameIds.length === 0) return;
|
|
110222
|
-
const ids = [];
|
|
110223
|
-
for (const frameId of frameIds) for (const a of await store.listByFrame(frameId)) ids.push(a.id);
|
|
110224
|
-
await store.markExported(ids, batchId);
|
|
110225
|
-
},
|
|
110804
|
+
cutCrop: this.retrainCropCutter(),
|
|
110805
|
+
markExported: (frameIds, batchId) => this.markRetrainExported(frameIds, batchId),
|
|
110226
110806
|
logger: this.ctx.logger,
|
|
110227
110807
|
now: () => Date.now()
|
|
110228
110808
|
});
|
|
@@ -110244,13 +110824,14 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
110244
110824
|
* reference rectangle is implicit is a box nobody can check. A camera with no
|
|
110245
110825
|
* zones simply has no `zoneCrop` geometry — absent, never a clamped guess.
|
|
110246
110826
|
*/
|
|
110247
|
-
async buildRetrainExportPlan(
|
|
110827
|
+
async buildRetrainExportPlan(request) {
|
|
110248
110828
|
const frameStore = this.retrainFrames;
|
|
110249
110829
|
const annotationStore = this.retrainAnnotations;
|
|
110250
110830
|
const trackStore = this.trackStore;
|
|
110251
110831
|
if (!frameStore || !annotationStore || !trackStore) throw new Error("retrain export: analytics stores unavailable");
|
|
110832
|
+
const deviceIds = request.filter.deviceIds ?? [];
|
|
110252
110833
|
const devices = deviceIds.length > 0 ? deviceIds : await trackStore.listDeviceIds();
|
|
110253
|
-
const
|
|
110834
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
110254
110835
|
const annotationsByFrame = /* @__PURE__ */ new Map();
|
|
110255
110836
|
const zones = /* @__PURE__ */ new Map();
|
|
110256
110837
|
for (const deviceId of devices) {
|
|
@@ -110259,22 +110840,56 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
110259
110840
|
const existing = annotationsByFrame.get(annotation.mediaKey) ?? [];
|
|
110260
110841
|
annotationsByFrame.set(annotation.mediaKey, [...existing, annotation]);
|
|
110261
110842
|
}
|
|
110843
|
+
for (const frame of await frameStore.listCompleteByDevice(deviceId, RETRAIN_EXPORT_MAX)) candidates.set(frame.frameId, frame);
|
|
110262
110844
|
}
|
|
110263
110845
|
for (const frameId of annotationsByFrame.keys()) {
|
|
110846
|
+
if (candidates.has(frameId)) continue;
|
|
110264
110847
|
const frame = await frameStore.get(frameId);
|
|
110265
|
-
if (frame !== null)
|
|
110848
|
+
if (frame !== null) candidates.set(frameId, frame);
|
|
110266
110849
|
}
|
|
110850
|
+
const frames = filterRetrainExport([...candidates.values()], annotationsByFrame, request.filter);
|
|
110267
110851
|
return {
|
|
110268
110852
|
manifest: buildRetrainManifest({
|
|
110269
110853
|
frames,
|
|
110270
110854
|
annotationsByFrame,
|
|
110271
110855
|
zoneForDevice: (deviceId) => zones.get(deviceId) ?? null,
|
|
110272
110856
|
generatedAt: Date.now(),
|
|
110273
|
-
batchId: (0, node_crypto.randomUUID)()
|
|
110857
|
+
batchId: (0, node_crypto.randomUUID)(),
|
|
110858
|
+
layout: request.layout,
|
|
110859
|
+
coordinates: await this.siteCoordinates()
|
|
110274
110860
|
}),
|
|
110275
110861
|
frames
|
|
110276
110862
|
};
|
|
110277
110863
|
}
|
|
110864
|
+
/** Stamp the batch on the annotations of the frames an archive carried. */
|
|
110865
|
+
async markRetrainExported(frameIds, batchId) {
|
|
110866
|
+
const store = this.retrainAnnotations;
|
|
110867
|
+
if (!store || frameIds.length === 0) return;
|
|
110868
|
+
const ids = [];
|
|
110869
|
+
for (const frameId of frameIds) for (const a of await store.listByFrame(frameId)) ids.push(a.id);
|
|
110870
|
+
await store.markExported(ids, batchId);
|
|
110871
|
+
}
|
|
110872
|
+
/** The `crops` layout's cutter, with the convention the manifest records. */
|
|
110873
|
+
retrainCropCutter() {
|
|
110874
|
+
return (frameBytes, frame, annotation) => cutRetrainCrop({
|
|
110875
|
+
frameBytes,
|
|
110876
|
+
frameWidth: frame.width,
|
|
110877
|
+
frameHeight: frame.height,
|
|
110878
|
+
bbox: annotation.bbox,
|
|
110879
|
+
convention: RETRAIN_CROP_CONVENTION,
|
|
110880
|
+
maxSide: RETRAIN_CROP_MAX_SIDE
|
|
110881
|
+
}).catch((err) => {
|
|
110882
|
+
this.ctx.logger.warn("retrain crop could not be cut", {
|
|
110883
|
+
tags: { deviceId: frame.deviceId },
|
|
110884
|
+
meta: {
|
|
110885
|
+
frameId: frame.frameId,
|
|
110886
|
+
annotationId: annotation.id,
|
|
110887
|
+
error: require_clip_model_registry.errMsg(err)
|
|
110888
|
+
}
|
|
110889
|
+
});
|
|
110890
|
+
return null;
|
|
110891
|
+
});
|
|
110892
|
+
}
|
|
110278
110893
|
/** Bounding box of a camera's zones, or null when it declares none. */
|
|
110279
110894
|
async retrainZoneForDevice(deviceId) {
|
|
110280
110895
|
try {
|
|
@@ -110422,6 +111037,28 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
110422
111037
|
}, LTS_FLUSH_INTERVAL_MS);
|
|
110423
111038
|
this.startOrphanAuditTimer();
|
|
110424
111039
|
this.reconcileGalleryMediaLocation();
|
|
111040
|
+
this.reconcileRetrainCopiesLocation();
|
|
111041
|
+
}
|
|
111042
|
+
/**
|
|
111043
|
+
* Move pre-D781 retrain copies off the footage class onto the durable
|
|
111044
|
+
* `retrainMedia` tenant, and say how many have no bytes left.
|
|
111045
|
+
*/
|
|
111046
|
+
async reconcileRetrainCopiesLocation() {
|
|
111047
|
+
const retrainFrames = this.retrainFrames;
|
|
111048
|
+
if (!retrainFrames) return;
|
|
111049
|
+
try {
|
|
111050
|
+
const result = await retrainFrames.relocateLegacyCopies();
|
|
111051
|
+
if (result.relocated === 0 && result.missing === 0 && result.failed === 0) return;
|
|
111052
|
+
this.ctx.logger.info("retrain copies reconciled onto the durable retrain tenant", { meta: {
|
|
111053
|
+
...result,
|
|
111054
|
+
location: RETRAIN_MEDIA_LOCATION
|
|
111055
|
+
} });
|
|
111056
|
+
} catch (err) {
|
|
111057
|
+
this.ctx.logger.warn("retrain copies reconcile failed — the copies stay on eventMedia", { meta: {
|
|
111058
|
+
error: require_clip_model_registry.errMsg(err),
|
|
111059
|
+
location: ARTIFACTS_MEDIA_LOCATION
|
|
111060
|
+
} });
|
|
111061
|
+
}
|
|
110425
111062
|
}
|
|
110426
111063
|
/**
|
|
110427
111064
|
* Move enrolled-gallery blobs off the footage volume, and say out loud how
|