@camstack/addon-post-analysis 1.2.342 → 1.2.343
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{clip-model-registry-ClEoe-WP.js → clip-model-registry-Cd_y9S4U.js} +301 -7
- package/dist/{clip-model-registry-Dhjpee-Q.mjs → clip-model-registry-DZz3e781.mjs} +290 -8
- package/dist/embedding-encoder/index.js +1 -1
- package/dist/embedding-encoder/index.mjs +1 -1
- package/dist/pipeline-analytics/_stub.js +2 -2
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-XzFWnlu-.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-DrdO3Pxp.mjs} +2 -2
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DEq-B6oq.mjs +26 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CiTUMvVf.mjs +26 -0
- package/dist/pipeline-analytics/{hostInit-C6P0R93E.mjs → hostInit-DPNnWr-m.mjs} +2 -2
- package/dist/pipeline-analytics/index.js +876 -239
- package/dist/pipeline-analytics/index.mjs +868 -231
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/package.json +2 -2
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CsAksj5u.mjs +0 -26
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-Cp4eQz57.mjs +0 -26
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as NC_ALARM_DEFAULT_PROFILE_ID, $r as subKindsOf, $t as RECORDING_EXPORT_MAX_READ_BYTES, A as DetectionSourceSchema, Ai as boolean, An as applyBlockingEdit, At as NcConditionsSchema, B as FailureCounters, Bi as templateLiteral, Bn as consumableLowPercentOf, Br as resolveModeNotifyPolicy, Bt as NcSharedSequenceSchema, C as CoverStateSchema, Ci as isDeviceScopedCap, Cn as TrackRelationSchema, Cr as isVectorIndexDimConflict, Ct as NC_TEMPLATE_VARS, D as DETECTION_MACRO_CLASSES, Di as ZodObject, Dn as addonWidgetsSourceCapability, Dr as memoizePinnedFfmpeg, Dt as NcAlarmSensorsSchema, E as DEFAULT_TIMELAPSE_PREVIEW_TEXT, Ei as sleep$1, En as activateAlarmProfile, Er as mayWriteToLocation, Et as NcAlarmSensorsInputSchema, F as EVENT_KIND_BY_CAP, Fi as object, Fn as audioMetricsCapability, Fr as readTimelapseGeneratedAt, Ft as NcRulePatchSchema, G as MAX_ARCHIVED_DEBUG_NOTES, Gn as deriveDetailCropRect, Gr as ruleDeviceCompatCapQueries, Gt as NcSystemEventKindSchema, H as ListTrackRelationsInputSchema, Hi as union, Hn as cosineSimilarity$1, Hr as rootSupplyServes, Ht as NcSnoozeSchema, I as EVENT_OWNER_TYPES, Ii as partialRecord, In as audioModeOf, Ir as renameAlarmProfile, It as NcRuleSchema, J as MAX_TRACK_PURITIES, Jn as describeSystemEventDeliveryProblem, Jr as sceneMonitorCapability, Jt as NcTemplatePreviewSchema, K as MAX_BIRTH_DECISION_RECORDS, Kn as derivePostTriggeredAction, Kr as ruleGroupingApplies, Kt as NcTaxonomySchema, L as EXPORT_DENSE_MAX_RANGES, Li as record, Ln as buildEventKindDescriptor, Lr as resolveClipSearchMinScore, Lt as NcRuleTargetSchema, M as EMBEDDING_REQUIREMENTS_FAILED, Mi as literal, Mn as applyNonBlockingEdit, Mr as plateGalleryCapability, Mt as NcRuleActionSchema, N as EMBEDDING_REQUIREMENTS_UNAVAILABLE, Ni as looseObject, Nn as applySecureOnArmEdit, Nr as readDeviceStateFrom, Nt as NcRuleActionsSchema, O as DETECTION_PIPELINE_CAP_NAME, Oi as _enum, On as alarmDeviceSecureOnArm, Or as notificationRulesCapability, Ot as NcAlarmSettingsPatchSchema, P as EVENT_DENSITY_BATCH_MAX, Pi as number, Pn as assertTimelapseCadences, Pr as readMotionTriggeredBy, Pt as NcRuleInputSchema, Q as MotionTriggeredBySchema, Qn as embeddingRefusalOf, Qr as storageOccupancyCapability, R as FIRST_LEVEL_MACRO_CLASSES, Ri as strictObject, Rn as buildViewerLink, Rr as resolveLocationMode, Rt as NcRuleUpdateInputSchema, S as ClearTrackRelationResultSchema, Si as hydrateSchema, Sn as TrackRelationKindSchema, Sr as isSystemEventSummaryRule, St as NC_TAXONOMY, T as DEFAULT_EVENT_COLOR, Tn as TrackSourceSchema, Tr as linkedStepOf, Tt as NcAlarmPostTriggeredActionSchema, U as ListTrackRelationsResultSchema, Ui as unknown, Un as createAlarmProfile, Ur as ruleAutoTriggersAlarm, Ut as NcSnoozeSuppressedSchema, V as LabelAttributionSchema, Vi as tuple, Vn as coreBlockIdFromAddonId, Vt as NcSnoozeInputSchema, W as LockStateSchema, Wi as EventCategory, Wn as deleteAlarmProfile, Wr as ruleBelongsToAlarm$1, Wt as NcStateSummarySectionIdSchema, X as MOTION_CLOSE_AFTER_MS, Xr as sensorRaisedOf, Xt as OPS_LOG_DEFAULT_LIMIT, Y as MAX_TRACK_RELATIONS, Yn as describeSystemEventFamilyProblem, Yr as secureActuationFor, Yt as NcTemplateVarDescriptorSchema, Z as MediaFileKindEnum, Zr as servedTemplateVars, Zt as OpsLogEntrySchema, _ as BIRTH_DECISION_DEFAULT_LIMIT, _i as BaseAddon, _n as TRACK_RELATIONS_DEFAULT_LIMIT, _r as isMediaOwnerType, _t as NC_GROUPING_MIN_SEC, a as CLIP_EMBEDDING_STEP_ID, ai as systemEventSummarySplitOf, an as SCENE_DEFAULT_ANCHOR_THRESHOLD, ar as faceGalleryCapability, at as NC_ALARM_TRIGGERED_MAX_SEC, b as COCO_TO_MACRO, bi as customAction, bn as TimelapseRuleSchema, br as isScheduleActive, bt as NC_SYSTEM_EVENT_KIND_GROUP_SPECS, c as resolveClusterModelPin, ci as vectorDimFromBase64, cn as SCENE_DEFAULT_QUIET_SECONDS, cr as gifFromMp4Args, ct as NC_CLIP_POST_ROLL_DEFAULT_SEC, d as APPEARANCE_BLOCKS, di as DEFAULT_FIRST_SIGHTING_FRESHNESS_MS, dn as SCENE_RESET_RECAPTURES, dr as isAudioClassName, dt as NC_CONDITION_CATALOG, ei as syncActiveAlarmProfileSettings, en as RECORDING_SOURCE_CAMSTACK_ADDON, et as NC_ALARM_DEFAULT_TRIGGERED_DURATION_SEC, f as APPEARANCE_RESOLVES_AMBIGUITY_DEFAULT, fi as canonicalHash, fn as SceneMonitorSchema, fr as isDefaultAlarmProfileId, ft as NC_CONSUMABLE_LOW_PERCENT_DEFAULT, g as ArchivedDebugNoteSchema, gi as errMsg, gn as TIMELAPSE_DENSE_FLOOR_SEC, gr as isKnownTemplateVar, gt as NC_GROUPING_MAX_SEC, h as AlarmArmModeSchema, hi as DeviceType, hn as SetTrackRelationInputSchema, hr as isEventOwnerType, ht as NC_GROUPING_DEFAULT_SEC, ii as systemEventSummaryScheduleOf, in as SCENE_CONFIRM_DEFAULT_TIMEOUT_MS, ir as evictionPolicyOfLocation, it as NC_ALARM_SYSTEM_EVENT_KINDS, j as EMBEDDING_MODEL_FAILED, ji as discriminatedUnion, jn as applyModeDelaysEdits, jr as pipelineAnalyticsCapability, jt as NcLegacyRuleActionSequenceSchema, k as DeclaredDevices, ki as array, kn as alarmPanelCapability, kr as occupancyScope, kt as NcConditionDescriptorSchema, l as ALL_CAPABILITY_DEFINITIONS, li as viewerLinkBaseOf, ln as SCENE_DEFAULT_UNCOVERED_POLICY, lr as isAlarmControllableCap, lt as NC_CLIP_PRE_ROLL_DEFAULT_SEC, m as AUDIO_MACRO_LABELS, mi as isSourceCap, mn as SetTrackPurityResultSchema, mr as isEventKindLabelKey, mt as NC_DEFAULT_SNOOZE_MINUTES, ni as systemEventFamilyProblems, nn as RetrainStatusSchema, nr as ensureAlarmProfiles, nt as NC_ALARM_DELAY_MAX_SEC, o as FACE_EMBEDDING_STEP_ID, oi as templateCarriesSummary, on as SCENE_DEFAULT_CHECK_INTERVAL_SEC, or as failureContributionCapability, ot as NC_AUDIO_CONFIRM_HITS_DEFAULT, p as ARCHIVED_DEBUG_NOTES_DEFAULT_LIMIT, pi as evaluateSensorEdge, pn as SetTrackPurityInputSchema, pr as isDetectionMacroClass, pt as NC_CONSUMABLE_SYSTEM_EVENT_KINDS, q as MAX_BIRTH_LATENCY_PROXY_MS, qn as deriveRecordingMode, qr as ruleKindOf, qt as NcTemplatePreviewInputSchema, ri as systemEventFilterApplies, rn as SCENE_CONFIRM_DEFAULT_MAX_IMAGE_PX, rr as eventKindLabel, rt as NC_ALARM_NOTIFY_FIELDS, s as readClusterModelPinStrict, si as templateVarsFor, sn as SCENE_DEFAULT_OBSERVATION_SPACING_SEC, sr as gifDeriveSpeed, st as NC_AUDIO_CONFIRM_WINDOW_SEC_DEFAULT, t as BUILTIN_CLIP_MODELS, ti as systemEventDeliveryProblems, tn as RebuildObjectEmbeddingsRefusalSchema, tr as encodeVectorBase64, tt as NC_ALARM_DELAY_FIELDS, u as APPEARANCE_BIN_COUNT, ui as zoneAnalyticsCapability, un as SCENE_DIVERGED, ur as isAppLinkBase, ut as NC_CLIP_RETENTION_MAX_SEC, v as BaseDevice, vi as CamProfileSchema, vn as TimelapseRuleInputSchema, vr as isRetiredClearKey, vt as NC_SENSOR_TRANSITION_KINDS, w as DEFAULT_DETAIL_CROP_CONVENTION, wi as nodePin, wn as TrackSideSchema, wr as kebabToCamel, wt as NcAlarmConfigSchema, x as ClearTrackRelationInputSchema, xi as defineCustomActions, xn as TrackPuritySchema, xr as isSystemDelivery, xt as NC_TANK_SYSTEM_EVENT_KINDS, y as BirthDecisionRecordSchema, yi as createEvent, yn as TimelapseRulePatchSchema, yr as isRootSuppliableStepId, yt as NC_SNOOZE_MAX_MINUTES, z as FULL_IMAGE_BBOX, zi as string, zn as cloneAlarmSettings, zr as resolveModeDelays, zt as NcScheduleSchema } from "../clip-model-registry-DZz3e781.mjs";
|
|
2
2
|
import { t as __exportAll } from "../embedding-encoder/index.mjs";
|
|
3
3
|
import * as fs$1 from "node:fs";
|
|
4
4
|
import { createWriteStream, promises } from "node:fs";
|
|
@@ -16,6 +16,8 @@ import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
|
16
16
|
import { isDeepStrictEqual, promisify } from "node:util";
|
|
17
17
|
import { Buffer as Buffer$1 } from "node:buffer";
|
|
18
18
|
import { createGzip, gunzip, gunzipSync, gzip, gzipSync } from "node:zlib";
|
|
19
|
+
import { request } from "node:http";
|
|
20
|
+
import { request as request$1 } from "node:https";
|
|
19
21
|
//#region src/pipeline-analytics/media-occupancy.ts
|
|
20
22
|
/** The bare type ref legacy NULL-stamped media rows belong to. */
|
|
21
23
|
var LEGACY_MEDIA_TYPE = "eventMedia";
|
|
@@ -7020,18 +7022,44 @@ function clockCondition(now) {
|
|
|
7020
7022
|
* does not.
|
|
7021
7023
|
*/
|
|
7022
7024
|
function resolveCondition(input) {
|
|
7023
|
-
|
|
7024
|
-
|
|
7025
|
+
return resolveConditionDetailed(input).condition;
|
|
7026
|
+
}
|
|
7027
|
+
/**
|
|
7028
|
+
* {@link resolveCondition}, plus the layer that answered. A consumer that
|
|
7029
|
+
* reports a day/night split (the retrain export, D785) must be able to tell a
|
|
7030
|
+
* measured IR frame from an almanac or a clock guess.
|
|
7031
|
+
*/
|
|
7032
|
+
function resolveConditionDetailed(input) {
|
|
7033
|
+
if (input.forcedMode === "day" || input.forcedMode === "night") return {
|
|
7034
|
+
condition: input.forcedMode,
|
|
7035
|
+
source: "forced"
|
|
7036
|
+
};
|
|
7037
|
+
if (input.chromaSpread !== void 0 && input.chromaSpread < IR_CHROMA_THRESHOLD) return {
|
|
7038
|
+
condition: IR,
|
|
7039
|
+
source: "ir-chroma"
|
|
7040
|
+
};
|
|
7025
7041
|
if (input.coordinates) {
|
|
7026
7042
|
const times = sunTimes(input.coordinates.lat, input.coordinates.lng, input.now);
|
|
7027
7043
|
if (times) {
|
|
7028
7044
|
const t = input.now.getTime();
|
|
7029
|
-
if (Math.abs(t - times.rise) <= TWILIGHT_BAND_MS) return
|
|
7030
|
-
|
|
7031
|
-
|
|
7045
|
+
if (Math.abs(t - times.rise) <= TWILIGHT_BAND_MS) return {
|
|
7046
|
+
condition: DAWN,
|
|
7047
|
+
source: "sun"
|
|
7048
|
+
};
|
|
7049
|
+
if (Math.abs(t - times.set) <= TWILIGHT_BAND_MS) return {
|
|
7050
|
+
condition: DUSK,
|
|
7051
|
+
source: "sun"
|
|
7052
|
+
};
|
|
7053
|
+
return {
|
|
7054
|
+
condition: t >= times.rise && t < times.set ? DAY : NIGHT,
|
|
7055
|
+
source: "sun"
|
|
7056
|
+
};
|
|
7032
7057
|
}
|
|
7033
7058
|
}
|
|
7034
|
-
return
|
|
7059
|
+
return {
|
|
7060
|
+
condition: clockCondition(input.now),
|
|
7061
|
+
source: "clock"
|
|
7062
|
+
};
|
|
7035
7063
|
}
|
|
7036
7064
|
//#endregion
|
|
7037
7065
|
//#region src/shared/llm-vision/parse-model-json.ts
|
|
@@ -70516,8 +70544,11 @@ var TrackStore = class {
|
|
|
70516
70544
|
* RetrainFrameStore — the COPIES the retrain dataset owns.
|
|
70517
70545
|
*
|
|
70518
70546
|
* Collection: `pipeline-analytics:retrain-frames`.
|
|
70519
|
-
* Blobs: `retrain/<deviceId>/<trackId>/<hash>.jpg
|
|
70520
|
-
*
|
|
70547
|
+
* Blobs: `retrain/<deviceId>/<trackId>/<hash>.jpg` under the `artifacts`
|
|
70548
|
+
* class root — the dataset is that class's fourth tenant (D341), addressed
|
|
70549
|
+
* through the permanent row ref {@link RETRAIN_MEDIA_LOCATION}. Never the
|
|
70550
|
+
* footage volume: the 2026-08-17 recordings wipe took the whole dataset with
|
|
70551
|
+
* it while it lived on `eventMedia` (D197). See D781.
|
|
70521
70552
|
*
|
|
70522
70553
|
* ## Why a copy, and why at selection time
|
|
70523
70554
|
*
|
|
@@ -70548,16 +70579,37 @@ var TrackStore = class {
|
|
|
70548
70579
|
* copy is taken eagerly at selection (copyFrame), not referenced, so the row
|
|
70549
70580
|
* records a blob the dataset owns outright; re-selecting the same source reuses
|
|
70550
70581
|
* the existing copy rather than writing a second one."
|
|
70551
|
-
* retention="none — the blobs sit
|
|
70552
|
-
* age sweep walks, which is exactly what makes a 'trained' track evictable
|
|
70582
|
+
* retention="none — the blobs sit in the 'retrain/' subtree of the artifacts class
|
|
70583
|
+
* (D781), which no owner cascade and no age sweep walks, which is exactly what makes a 'trained' track evictable
|
|
70553
70584
|
* again (D81). Rows go only when the operator deselects the frame (deleteFrame,
|
|
70554
70585
|
* after its annotations). Bounded only by operator selection — nothing caps the
|
|
70555
70586
|
* table or the disk it consumes."
|
|
70556
70587
|
*/
|
|
70557
70588
|
var RETRAIN_FRAMES_COLLECTION = "pipeline-analytics:retrain-frames";
|
|
70558
|
-
/**
|
|
70559
|
-
*
|
|
70560
|
-
|
|
70589
|
+
/**
|
|
70590
|
+
* The PERMANENT location ref every retrain copy is written under (D781).
|
|
70591
|
+
*
|
|
70592
|
+
* Not a declared storage class: it is the rows' `locationId`, which the
|
|
70593
|
+
* media-storage wiring maps onto the `artifacts` class root — exactly how the
|
|
70594
|
+
* enrolled gallery's `galleryMedia` ref works (D341). The copies are curation
|
|
70595
|
+
* work with NO retention, so they share the one durable class rather than the
|
|
70596
|
+
* footage class, which is quota-evicted, drained and wipeable per disk. Rows
|
|
70597
|
+
* are never restamped for a root move: the ref is remapped instead, and the
|
|
70598
|
+
* singleton repoint carries the whole `artifacts` tree, `retrain/` included.
|
|
70599
|
+
*/
|
|
70600
|
+
var RETRAIN_MEDIA_LOCATION = "retrainMedia";
|
|
70601
|
+
/**
|
|
70602
|
+
* Where copies taken before D781 live: a NULL `locationId` meant "wherever
|
|
70603
|
+
* `eventMedia` points", and a concrete `eventMedia:*` id was stamped by the
|
|
70604
|
+
* seal or a relocate pass. The boot reconcile
|
|
70605
|
+
* ({@link RetrainFrameStore.relocateLegacyCopies}) moves them onto
|
|
70606
|
+
* {@link RETRAIN_MEDIA_LOCATION}; until it has, reads resolve them here.
|
|
70607
|
+
*/
|
|
70608
|
+
var LEGACY_RETRAIN_MEDIA_LOCATION = "eventMedia";
|
|
70609
|
+
/** Is this row already on the durable retrain tenant? */
|
|
70610
|
+
function rowIsDurableRetrainCopy(data) {
|
|
70611
|
+
return data?.["locationId"] === RETRAIN_MEDIA_LOCATION;
|
|
70612
|
+
}
|
|
70561
70613
|
/** Path prefix of every retrain copy. Never `<deviceId>/events/…`: the orphan
|
|
70562
70614
|
* audit resolves ownership from INDEX rows, but a future filesystem-side tool
|
|
70563
70615
|
* must be able to tell the two corpora apart by path alone. */
|
|
@@ -70615,11 +70667,32 @@ var RETRAIN_FRAME_COLUMNS = [
|
|
|
70615
70667
|
notNull: true
|
|
70616
70668
|
},
|
|
70617
70669
|
(
|
|
70618
|
-
/**
|
|
70619
|
-
*
|
|
70670
|
+
/** `retrainMedia` for every copy taken since D781. NULL (the eventMedia
|
|
70671
|
+
* default) or a concrete `eventMedia:*` id only on a pre-D781 copy the boot
|
|
70672
|
+
* reconcile has not moved yet. */
|
|
70620
70673
|
{
|
|
70621
70674
|
name: "locationId",
|
|
70622
70675
|
type: "TEXT"
|
|
70676
|
+
}),
|
|
70677
|
+
(
|
|
70678
|
+
/** When the CAMERA captured the frame — the source media row's timestamp.
|
|
70679
|
+
* NULL on a copy taken before D785. */
|
|
70680
|
+
{
|
|
70681
|
+
name: "capturedAt",
|
|
70682
|
+
type: "INTEGER"
|
|
70683
|
+
}),
|
|
70684
|
+
(
|
|
70685
|
+
/** Mean channel spread of the copy (`chromaSpread`), measured once at copy
|
|
70686
|
+
* time; below `IR_CHROMA_THRESHOLD` the frame is IR. NULL = not measured. */
|
|
70687
|
+
{
|
|
70688
|
+
name: "chromaSpread",
|
|
70689
|
+
type: "REAL"
|
|
70690
|
+
}),
|
|
70691
|
+
(
|
|
70692
|
+
/** The operator's "every subject in this frame is boxed". NULL = no. */
|
|
70693
|
+
{
|
|
70694
|
+
name: "complete",
|
|
70695
|
+
type: "BOOLEAN"
|
|
70623
70696
|
})
|
|
70624
70697
|
];
|
|
70625
70698
|
var RETRAIN_FRAME_INDEXES = [
|
|
@@ -70660,6 +70733,10 @@ function retrainFrameId(trackId, sourceMediaId) {
|
|
|
70660
70733
|
function framePath(deviceId, trackId, frameId) {
|
|
70661
70734
|
return `${RETRAIN_PATH_PREFIX}/${deviceId}/${trackId}/${frameId.slice(frameId.lastIndexOf(":") + 1)}.jpg`;
|
|
70662
70735
|
}
|
|
70736
|
+
/** A stored number, or `undefined` for NULL / absent / junk. */
|
|
70737
|
+
function finiteNumber(value) {
|
|
70738
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
70739
|
+
}
|
|
70663
70740
|
var RetrainFrameStore = class {
|
|
70664
70741
|
store;
|
|
70665
70742
|
storage;
|
|
@@ -70667,7 +70744,7 @@ var RetrainFrameStore = class {
|
|
|
70667
70744
|
readSource;
|
|
70668
70745
|
now;
|
|
70669
70746
|
writeGate;
|
|
70670
|
-
|
|
70747
|
+
measureChroma;
|
|
70671
70748
|
constructor(deps) {
|
|
70672
70749
|
this.store = deps.store;
|
|
70673
70750
|
this.storage = deps.storage;
|
|
@@ -70675,28 +70752,20 @@ var RetrainFrameStore = class {
|
|
|
70675
70752
|
this.readSource = deps.readSource;
|
|
70676
70753
|
this.now = deps.now;
|
|
70677
70754
|
this.writeGate = deps.writeGate;
|
|
70678
|
-
this.
|
|
70755
|
+
this.measureChroma = deps.measureChroma;
|
|
70679
70756
|
}
|
|
70680
|
-
/**
|
|
70681
|
-
|
|
70682
|
-
|
|
70683
|
-
* `null` = "no concrete location was resolved", which persists as the legacy
|
|
70684
|
-
* NULL row. A resolver that throws must not refuse the copy — losing an
|
|
70685
|
-
* operator's selection is worse than an unstamped row the seal can fix — but
|
|
70686
|
-
* it says so, because that branch re-creates the exact population a
|
|
70687
|
-
* non-blocking migration is gated on.
|
|
70688
|
-
*/
|
|
70689
|
-
async resolveWriteLocation(deviceId) {
|
|
70690
|
-
if (!this.resolveEventMediaLocation) return null;
|
|
70757
|
+
/** The copy's chroma, or `undefined` when it cannot be measured. */
|
|
70758
|
+
async chromaOf(bytes, deviceId) {
|
|
70759
|
+
if (!this.measureChroma) return void 0;
|
|
70691
70760
|
try {
|
|
70692
|
-
const
|
|
70693
|
-
return
|
|
70761
|
+
const value = await this.measureChroma(bytes);
|
|
70762
|
+
return Number.isFinite(value) ? value : void 0;
|
|
70694
70763
|
} catch (err) {
|
|
70695
|
-
this.logger.warn("retrain copy could not
|
|
70764
|
+
this.logger.warn("retrain copy chroma could not be measured — its day/night falls back to the sun", {
|
|
70696
70765
|
tags: { deviceId },
|
|
70697
70766
|
meta: { error: String(err) }
|
|
70698
70767
|
});
|
|
70699
|
-
return
|
|
70768
|
+
return;
|
|
70700
70769
|
}
|
|
70701
70770
|
}
|
|
70702
70771
|
static async declare(store) {
|
|
@@ -70717,7 +70786,10 @@ var RetrainFrameStore = class {
|
|
|
70717
70786
|
width: Number(data["width"] ?? 0),
|
|
70718
70787
|
height: Number(data["height"] ?? 0),
|
|
70719
70788
|
copiedAt: Number(data["copiedAt"] ?? 0),
|
|
70720
|
-
locationId: typeof data["locationId"] === "string" && data["locationId"].length > 0 ? data["locationId"] : null
|
|
70789
|
+
locationId: typeof data["locationId"] === "string" && data["locationId"].length > 0 ? data["locationId"] : null,
|
|
70790
|
+
...finiteNumber(data["capturedAt"]) !== void 0 ? { capturedAt: finiteNumber(data["capturedAt"]) } : {},
|
|
70791
|
+
...finiteNumber(data["chromaSpread"]) !== void 0 ? { chromaSpread: finiteNumber(data["chromaSpread"]) } : {},
|
|
70792
|
+
complete: data["complete"] === true || data["complete"] === 1
|
|
70721
70793
|
};
|
|
70722
70794
|
}
|
|
70723
70795
|
/** The copy with this id, or `null`. Index read only — no blob is touched. */
|
|
@@ -70757,6 +70829,31 @@ var RetrainFrameStore = class {
|
|
|
70757
70829
|
return [];
|
|
70758
70830
|
}
|
|
70759
70831
|
}
|
|
70832
|
+
/**
|
|
70833
|
+
* The copies of one camera the operator marked complete. A complete frame
|
|
70834
|
+
* with no box is a hard negative, so an export cannot find these through
|
|
70835
|
+
* the annotation table alone.
|
|
70836
|
+
*/
|
|
70837
|
+
async listCompleteByDevice(deviceId, limit) {
|
|
70838
|
+
try {
|
|
70839
|
+
return (await this.store.query.query({
|
|
70840
|
+
collection: RETRAIN_FRAMES_COLLECTION,
|
|
70841
|
+
filter: {
|
|
70842
|
+
where: {
|
|
70843
|
+
deviceId,
|
|
70844
|
+
complete: true
|
|
70845
|
+
},
|
|
70846
|
+
limit
|
|
70847
|
+
}
|
|
70848
|
+
})).filter((r) => typeof r.id === "string").map((r) => this.rowToFrame(r.id, r.data));
|
|
70849
|
+
} catch (err) {
|
|
70850
|
+
this.logger.warn("RetrainFrameStore.listCompleteByDevice failed", {
|
|
70851
|
+
tags: { deviceId },
|
|
70852
|
+
meta: { error: String(err) }
|
|
70853
|
+
});
|
|
70854
|
+
return [];
|
|
70855
|
+
}
|
|
70856
|
+
}
|
|
70760
70857
|
/** How many copies each of these tracks already owns — one query, so the
|
|
70761
70858
|
* staging list can badge "worked / not worked" without N round trips. */
|
|
70762
70859
|
async countsByTrack(trackIds) {
|
|
@@ -70785,7 +70882,7 @@ var RetrainFrameStore = class {
|
|
|
70785
70882
|
async readFrame(frame) {
|
|
70786
70883
|
try {
|
|
70787
70884
|
return await this.storage.read({
|
|
70788
|
-
location: frame.locationId ??
|
|
70885
|
+
location: frame.locationId ?? LEGACY_RETRAIN_MEDIA_LOCATION,
|
|
70789
70886
|
relativePath: framePath(frame.deviceId, frame.trackId, frame.frameId)
|
|
70790
70887
|
});
|
|
70791
70888
|
} catch (err) {
|
|
@@ -70856,7 +70953,8 @@ var RetrainFrameStore = class {
|
|
|
70856
70953
|
};
|
|
70857
70954
|
}
|
|
70858
70955
|
const path = framePath(input.deviceId, input.trackId, frameId);
|
|
70859
|
-
const locationId =
|
|
70956
|
+
const locationId = RETRAIN_MEDIA_LOCATION;
|
|
70957
|
+
const chromaSpread = await this.chromaOf(bytes, input.deviceId);
|
|
70860
70958
|
const frame = {
|
|
70861
70959
|
frameId,
|
|
70862
70960
|
deviceId: input.deviceId,
|
|
@@ -70867,11 +70965,14 @@ var RetrainFrameStore = class {
|
|
|
70867
70965
|
width: size.width,
|
|
70868
70966
|
height: size.height,
|
|
70869
70967
|
copiedAt: this.now(),
|
|
70870
|
-
locationId
|
|
70968
|
+
locationId,
|
|
70969
|
+
...input.capturedAt !== void 0 ? { capturedAt: input.capturedAt } : {},
|
|
70970
|
+
...chromaSpread !== void 0 ? { chromaSpread } : {},
|
|
70971
|
+
complete: false
|
|
70871
70972
|
};
|
|
70872
70973
|
const persist = async () => {
|
|
70873
70974
|
await this.storage.write({
|
|
70874
|
-
location: locationId
|
|
70975
|
+
location: locationId,
|
|
70875
70976
|
relativePath: path,
|
|
70876
70977
|
data: bytes
|
|
70877
70978
|
});
|
|
@@ -70888,7 +70989,10 @@ var RetrainFrameStore = class {
|
|
|
70888
70989
|
width: frame.width,
|
|
70889
70990
|
height: frame.height,
|
|
70890
70991
|
copiedAt: frame.copiedAt,
|
|
70891
|
-
locationId: frame.locationId
|
|
70992
|
+
locationId: frame.locationId,
|
|
70993
|
+
capturedAt: frame.capturedAt ?? null,
|
|
70994
|
+
chromaSpread: frame.chromaSpread ?? null,
|
|
70995
|
+
complete: false
|
|
70892
70996
|
}
|
|
70893
70997
|
});
|
|
70894
70998
|
};
|
|
@@ -70928,6 +71032,27 @@ var RetrainFrameStore = class {
|
|
|
70928
71032
|
};
|
|
70929
71033
|
}
|
|
70930
71034
|
/**
|
|
71035
|
+
* The operator's "every subject in this frame is boxed" (D785 context: an
|
|
71036
|
+
* unboxed subject is a false positive to a detector evaluation). Only the
|
|
71037
|
+
* operator sets it; nothing derives it from the annotation count, because a
|
|
71038
|
+
* frame with zero boxes may be a complete hard negative.
|
|
71039
|
+
*
|
|
71040
|
+
* @returns the updated frame, or `null` when no such copy exists.
|
|
71041
|
+
*/
|
|
71042
|
+
async setComplete(frameId, complete) {
|
|
71043
|
+
const existing = await this.get(frameId);
|
|
71044
|
+
if (existing === null) return null;
|
|
71045
|
+
await this.store.update.mutate({
|
|
71046
|
+
collection: RETRAIN_FRAMES_COLLECTION,
|
|
71047
|
+
id: frameId,
|
|
71048
|
+
data: { complete }
|
|
71049
|
+
});
|
|
71050
|
+
return {
|
|
71051
|
+
...existing,
|
|
71052
|
+
complete
|
|
71053
|
+
};
|
|
71054
|
+
}
|
|
71055
|
+
/**
|
|
70931
71056
|
* Drop a copy and its blob — the operator un-selecting a frame.
|
|
70932
71057
|
*
|
|
70933
71058
|
* The caller must have removed the frame's annotations first; this store does
|
|
@@ -70938,7 +71063,7 @@ var RetrainFrameStore = class {
|
|
|
70938
71063
|
const remove = async () => {
|
|
70939
71064
|
try {
|
|
70940
71065
|
await this.storage.delete({
|
|
70941
|
-
location: frame.locationId ??
|
|
71066
|
+
location: frame.locationId ?? LEGACY_RETRAIN_MEDIA_LOCATION,
|
|
70942
71067
|
relativePath: framePath(frame.deviceId, frame.trackId, frame.frameId)
|
|
70943
71068
|
});
|
|
70944
71069
|
} catch {}
|
|
@@ -70960,6 +71085,95 @@ var RetrainFrameStore = class {
|
|
|
70960
71085
|
if (this.writeGate) await this.writeGate.run(remove);
|
|
70961
71086
|
else await remove();
|
|
70962
71087
|
}
|
|
71088
|
+
/**
|
|
71089
|
+
* Move every copy still on the footage class onto the durable retrain
|
|
71090
|
+
* tenant (D781): copy → read back → restamp → delete the source.
|
|
71091
|
+
*
|
|
71092
|
+
* A reconcile, not a one-shot: one query of a collection bounded by
|
|
71093
|
+
* operator selection, so it is cheap when converged, and it is what would
|
|
71094
|
+
* notice a pre-D781 copy restored from anywhere. The order is the gallery
|
|
71095
|
+
* reconcile's (D341) and for the same reason: the row is restamped only
|
|
71096
|
+
* after the new blob reads back, so a crash at any point leaves a row that
|
|
71097
|
+
* names a location holding its bytes.
|
|
71098
|
+
*
|
|
71099
|
+
* A copy whose bytes are already gone is COUNTED and LOGGED, never deleted:
|
|
71100
|
+
* its annotations are the operator's work, and only the operator decides
|
|
71101
|
+
* to drop them (deselecting the frame).
|
|
71102
|
+
*/
|
|
71103
|
+
async relocateLegacyCopies() {
|
|
71104
|
+
const rows = await this.store.query.query({ collection: RETRAIN_FRAMES_COLLECTION });
|
|
71105
|
+
let relocated = 0;
|
|
71106
|
+
let missing = 0;
|
|
71107
|
+
let failed = 0;
|
|
71108
|
+
for (const row of rows) {
|
|
71109
|
+
if (typeof row.id !== "string") continue;
|
|
71110
|
+
const data = row.data;
|
|
71111
|
+
if (rowIsDurableRetrainCopy(data)) continue;
|
|
71112
|
+
const frame = this.rowToFrame(row.id, data);
|
|
71113
|
+
const from = frame.locationId ?? LEGACY_RETRAIN_MEDIA_LOCATION;
|
|
71114
|
+
const relativePath = framePath(frame.deviceId, frame.trackId, frame.frameId);
|
|
71115
|
+
let bytes;
|
|
71116
|
+
try {
|
|
71117
|
+
bytes = await this.storage.read({
|
|
71118
|
+
location: from,
|
|
71119
|
+
relativePath
|
|
71120
|
+
});
|
|
71121
|
+
} catch (err) {
|
|
71122
|
+
missing += 1;
|
|
71123
|
+
this.logger.warn("retrain copy has NO bytes on the footage class — its row and annotations are kept; deselect the frame to drop it", {
|
|
71124
|
+
tags: { deviceId: frame.deviceId },
|
|
71125
|
+
meta: {
|
|
71126
|
+
frameId: frame.frameId,
|
|
71127
|
+
trackId: frame.trackId,
|
|
71128
|
+
from,
|
|
71129
|
+
error: String(err)
|
|
71130
|
+
}
|
|
71131
|
+
});
|
|
71132
|
+
continue;
|
|
71133
|
+
}
|
|
71134
|
+
const move = async () => {
|
|
71135
|
+
await this.storage.write({
|
|
71136
|
+
location: RETRAIN_MEDIA_LOCATION,
|
|
71137
|
+
relativePath,
|
|
71138
|
+
data: bytes
|
|
71139
|
+
});
|
|
71140
|
+
if (!(await this.storage.read({
|
|
71141
|
+
location: RETRAIN_MEDIA_LOCATION,
|
|
71142
|
+
relativePath
|
|
71143
|
+
})).equals(bytes)) throw new Error("read-back differs from the source");
|
|
71144
|
+
await this.store.update.mutate({
|
|
71145
|
+
collection: RETRAIN_FRAMES_COLLECTION,
|
|
71146
|
+
id: frame.frameId,
|
|
71147
|
+
data: { locationId: RETRAIN_MEDIA_LOCATION }
|
|
71148
|
+
});
|
|
71149
|
+
await this.storage.delete({
|
|
71150
|
+
location: from,
|
|
71151
|
+
relativePath
|
|
71152
|
+
});
|
|
71153
|
+
};
|
|
71154
|
+
try {
|
|
71155
|
+
if (this.writeGate) await this.writeGate.run(move);
|
|
71156
|
+
else await move();
|
|
71157
|
+
relocated += 1;
|
|
71158
|
+
} catch (err) {
|
|
71159
|
+
failed += 1;
|
|
71160
|
+
this.logger.error("retrain copy could not be moved off the footage class", {
|
|
71161
|
+
tags: { deviceId: frame.deviceId },
|
|
71162
|
+
meta: {
|
|
71163
|
+
frameId: frame.frameId,
|
|
71164
|
+
trackId: frame.trackId,
|
|
71165
|
+
from,
|
|
71166
|
+
error: String(err)
|
|
71167
|
+
}
|
|
71168
|
+
});
|
|
71169
|
+
}
|
|
71170
|
+
}
|
|
71171
|
+
return {
|
|
71172
|
+
relocated,
|
|
71173
|
+
missing,
|
|
71174
|
+
failed
|
|
71175
|
+
};
|
|
71176
|
+
}
|
|
70963
71177
|
};
|
|
70964
71178
|
//#endregion
|
|
70965
71179
|
//#region src/pipeline-analytics/retention/location-class-wipe.ts
|
|
@@ -82448,6 +82662,322 @@ var RetrainCollectionStore = class {
|
|
|
82448
82662
|
}
|
|
82449
82663
|
};
|
|
82450
82664
|
//#endregion
|
|
82665
|
+
//#region src/pipeline-analytics/retrain/retrain-archive-upload.ts
|
|
82666
|
+
/**
|
|
82667
|
+
* The HTTP sink behind `pipelineAnalytics.writeRetrainArchive`: ONE request
|
|
82668
|
+
* whose body is the archive, written chunk by chunk with backpressure honoured
|
|
82669
|
+
* (D447) — the archive is never assembled in memory or on disk.
|
|
82670
|
+
*
|
|
82671
|
+
* The target is admin-supplied (the training orchestrator hands over a
|
|
82672
|
+
* provider's presigned URL or its upload route). Its headers may carry a
|
|
82673
|
+
* credential, so they are sent and never logged.
|
|
82674
|
+
*/
|
|
82675
|
+
function openRetrainUploadSink(target) {
|
|
82676
|
+
const url = new URL(target.url);
|
|
82677
|
+
const req = (url.protocol === "https:" ? request$1 : request)(url, {
|
|
82678
|
+
method: target.method,
|
|
82679
|
+
headers: {
|
|
82680
|
+
"content-type": "application/x-tar",
|
|
82681
|
+
...target.headers
|
|
82682
|
+
}
|
|
82683
|
+
});
|
|
82684
|
+
let failure = null;
|
|
82685
|
+
const answer = new Promise((resolve, reject) => {
|
|
82686
|
+
req.on("response", resolve);
|
|
82687
|
+
req.on("error", (err) => {
|
|
82688
|
+
failure = err;
|
|
82689
|
+
reject(err);
|
|
82690
|
+
});
|
|
82691
|
+
});
|
|
82692
|
+
answer.catch(() => void 0);
|
|
82693
|
+
return {
|
|
82694
|
+
sink: { write: (chunk) => new Promise((resolve, reject) => {
|
|
82695
|
+
if (failure !== null) {
|
|
82696
|
+
reject(failure);
|
|
82697
|
+
return;
|
|
82698
|
+
}
|
|
82699
|
+
if (req.write(chunk, (err) => {
|
|
82700
|
+
if (err) reject(err);
|
|
82701
|
+
})) resolve();
|
|
82702
|
+
else req.once("drain", resolve);
|
|
82703
|
+
}) },
|
|
82704
|
+
finish: async () => {
|
|
82705
|
+
req.end();
|
|
82706
|
+
const res = await answer;
|
|
82707
|
+
res.resume();
|
|
82708
|
+
const status = res.statusCode ?? 0;
|
|
82709
|
+
if (status < 200 || status >= 300) throw new Error(`retrain upload refused by the target: HTTP ${status}`);
|
|
82710
|
+
return status;
|
|
82711
|
+
},
|
|
82712
|
+
abort: (reason) => {
|
|
82713
|
+
req.destroy(new Error(reason));
|
|
82714
|
+
}
|
|
82715
|
+
};
|
|
82716
|
+
}
|
|
82717
|
+
//#endregion
|
|
82718
|
+
//#region src/pipeline-analytics/training-export/tar-stream.ts
|
|
82719
|
+
/**
|
|
82720
|
+
* A minimal POSIX ustar writer — enough to emit a dataset archive entry by
|
|
82721
|
+
* entry, without ever holding the archive in memory.
|
|
82722
|
+
*
|
|
82723
|
+
* ## Why hand-rolled rather than the `tar` package
|
|
82724
|
+
*
|
|
82725
|
+
* `tar` is in the workspace and is the right tool for what it does: archive a
|
|
82726
|
+
* set of paths under one `cwd`. This is a different job. A track's media lives
|
|
82727
|
+
* across several storage LOCATION roots (`eventMedia` and whatever else the
|
|
82728
|
+
* operator relocated to), under paths that encode nothing a training set wants —
|
|
82729
|
+
* `615/events/track/<uuid>/keyFrame.jpg` — and every entry has to be RENAMED on
|
|
82730
|
+
* the way in, to `tracks/615/<trackId>/keyFrame.jpg`. `tar.create` takes a list
|
|
82731
|
+
* of paths relative to a single cwd and gives no rename hook, so using it would
|
|
82732
|
+
* mean staging a whole shadow tree on disk first: a full second copy of the
|
|
82733
|
+
* dataset, written and then deleted, to avoid writing 100 lines.
|
|
82734
|
+
*
|
|
82735
|
+
* So: ustar, which is a 512-byte header and a 512-byte-padded body per entry,
|
|
82736
|
+
* fully specified, and asserted against the system `tar` binary in the spec next
|
|
82737
|
+
* door rather than against my reading of the format.
|
|
82738
|
+
*
|
|
82739
|
+
* ## What this deliberately does NOT do
|
|
82740
|
+
*
|
|
82741
|
+
* No gzip — the payload is JPEG, already compressed; a gzip pass would burn CPU
|
|
82742
|
+
* on the hub for a percent or two. No long-name (GNU/PAX) extension: ustar
|
|
82743
|
+
* splits a path across a 155-byte prefix and a 100-byte name, and
|
|
82744
|
+
* {@link tarNameTooLong} reports a path that will not fit so the caller can skip
|
|
82745
|
+
* it and SAY it skipped it, rather than emitting a silently truncated entry.
|
|
82746
|
+
*/
|
|
82747
|
+
var BLOCK_SIZE = 512;
|
|
82748
|
+
var NAME_MAX = 100;
|
|
82749
|
+
var PREFIX_MAX = 155;
|
|
82750
|
+
/**
|
|
82751
|
+
* Split a path into the ustar `prefix` + `name` pair, or `null` when it cannot
|
|
82752
|
+
* be represented. The split must land on a `/`, and each half has its own cap.
|
|
82753
|
+
*/
|
|
82754
|
+
function splitUstarName(name) {
|
|
82755
|
+
if (Buffer.byteLength(name) <= NAME_MAX) return {
|
|
82756
|
+
prefix: "",
|
|
82757
|
+
name
|
|
82758
|
+
};
|
|
82759
|
+
for (let i = name.length - 1; i >= 0; i--) {
|
|
82760
|
+
if (name[i] !== "/") continue;
|
|
82761
|
+
const head = name.slice(0, i);
|
|
82762
|
+
const tail = name.slice(i + 1);
|
|
82763
|
+
if (Buffer.byteLength(tail) > NAME_MAX) break;
|
|
82764
|
+
if (Buffer.byteLength(head) <= PREFIX_MAX) return {
|
|
82765
|
+
prefix: head,
|
|
82766
|
+
name: tail
|
|
82767
|
+
};
|
|
82768
|
+
}
|
|
82769
|
+
return null;
|
|
82770
|
+
}
|
|
82771
|
+
/** True when a path cannot be represented as a ustar entry name. */
|
|
82772
|
+
function tarNameTooLong(name) {
|
|
82773
|
+
return splitUstarName(name) === null;
|
|
82774
|
+
}
|
|
82775
|
+
function writeString(block, value, offset, length) {
|
|
82776
|
+
block.write(value.slice(0, length - 1), offset, length - 1, "utf8");
|
|
82777
|
+
}
|
|
82778
|
+
/** ustar numeric field: zero-padded octal, NUL-terminated. */
|
|
82779
|
+
function writeOctal(block, value, offset, length) {
|
|
82780
|
+
const text = Math.floor(value).toString(8).padStart(length - 1, "0");
|
|
82781
|
+
block.write(text.slice(-(length - 1)), offset, length - 1, "ascii");
|
|
82782
|
+
}
|
|
82783
|
+
/**
|
|
82784
|
+
* The 512-byte header for one regular file.
|
|
82785
|
+
*
|
|
82786
|
+
* The checksum is computed with the checksum field itself read as eight spaces —
|
|
82787
|
+
* that is the format's rule, and getting it wrong produces an archive that looks
|
|
82788
|
+
* fine until something tries to read it.
|
|
82789
|
+
*/
|
|
82790
|
+
function tarHeader(entry) {
|
|
82791
|
+
const split = splitUstarName(entry.name);
|
|
82792
|
+
if (split === null) throw new Error(`tar: entry name too long for ustar: ${entry.name}`);
|
|
82793
|
+
const block = Buffer.alloc(BLOCK_SIZE);
|
|
82794
|
+
writeString(block, split.name, 0, NAME_MAX);
|
|
82795
|
+
writeOctal(block, 420, 100, 8);
|
|
82796
|
+
writeOctal(block, 0, 108, 8);
|
|
82797
|
+
writeOctal(block, 0, 116, 8);
|
|
82798
|
+
writeOctal(block, entry.body.length, 124, 12);
|
|
82799
|
+
writeOctal(block, Math.floor(entry.mtimeMs / 1e3), 136, 12);
|
|
82800
|
+
block.write(" ", 148, 8, "ascii");
|
|
82801
|
+
block.write("0", 156, 1, "ascii");
|
|
82802
|
+
block.write("ustar\0", 257, 6, "ascii");
|
|
82803
|
+
block.write("00", 263, 2, "ascii");
|
|
82804
|
+
writeString(block, split.prefix, 345, PREFIX_MAX);
|
|
82805
|
+
let sum = 0;
|
|
82806
|
+
for (const byte of block) sum += byte;
|
|
82807
|
+
writeOctal(block, sum, 148, 7);
|
|
82808
|
+
block.write("\0", 154, 1, "ascii");
|
|
82809
|
+
block.write(" ", 155, 1, "ascii");
|
|
82810
|
+
return block;
|
|
82811
|
+
}
|
|
82812
|
+
/** Zero padding that rounds a body up to the 512-byte block. */
|
|
82813
|
+
function tarPadding(size) {
|
|
82814
|
+
const remainder = size % BLOCK_SIZE;
|
|
82815
|
+
return remainder === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK_SIZE - remainder);
|
|
82816
|
+
}
|
|
82817
|
+
/** Header + body + padding for one entry, ready to write. */
|
|
82818
|
+
function tarBlocks(entry) {
|
|
82819
|
+
return [
|
|
82820
|
+
tarHeader(entry),
|
|
82821
|
+
entry.body,
|
|
82822
|
+
tarPadding(entry.body.length)
|
|
82823
|
+
];
|
|
82824
|
+
}
|
|
82825
|
+
/** The two zero blocks that terminate an archive. */
|
|
82826
|
+
function tarTrailer() {
|
|
82827
|
+
return Buffer.alloc(BLOCK_SIZE * 2);
|
|
82828
|
+
}
|
|
82829
|
+
//#endregion
|
|
82830
|
+
//#region src/pipeline-analytics/retrain/retrain-archive-writer.ts
|
|
82831
|
+
/**
|
|
82832
|
+
* The annotated archive body, written to ANY sink.
|
|
82833
|
+
*
|
|
82834
|
+
* One body, two consumers: the data-plane download (`retrain-export-handler`,
|
|
82835
|
+
* sink = the HTTP response) and `pipelineAnalytics.writeRetrainArchive` (sink
|
|
82836
|
+
* = one streamed HTTP upload to a training provider). The disciplines are the
|
|
82837
|
+
* training export's: the manifest is the first entry, one JPEG is in memory at
|
|
82838
|
+
* a time, a pause between files keeps the hub's one thread available, and the
|
|
82839
|
+
* batch id is stamped only on what the sink actually received.
|
|
82840
|
+
*/
|
|
82841
|
+
var RETRAIN_MANIFEST_ENTRY = "annotations.json";
|
|
82842
|
+
/** Pause between files so a large export cannot monopolise the hub's thread. */
|
|
82843
|
+
var RETRAIN_ARCHIVE_PACING_MS = 25;
|
|
82844
|
+
async function writeRetrainArchiveTo(deps) {
|
|
82845
|
+
const { plan, logger } = deps;
|
|
82846
|
+
const pacingMs = deps.pacingMs ?? RETRAIN_ARCHIVE_PACING_MS;
|
|
82847
|
+
const hash = createHash("sha256");
|
|
82848
|
+
let bytes = 0;
|
|
82849
|
+
const write = async (chunk) => {
|
|
82850
|
+
hash.update(chunk);
|
|
82851
|
+
bytes += chunk.length;
|
|
82852
|
+
await deps.sink.write(chunk);
|
|
82853
|
+
};
|
|
82854
|
+
const writeEntry = async (name, body, mtimeMs) => {
|
|
82855
|
+
for (const block of tarBlocks({
|
|
82856
|
+
name,
|
|
82857
|
+
body,
|
|
82858
|
+
mtimeMs
|
|
82859
|
+
})) await write(block);
|
|
82860
|
+
};
|
|
82861
|
+
const mtimeMs = deps.now();
|
|
82862
|
+
await writeEntry(RETRAIN_MANIFEST_ENTRY, Buffer.from(JSON.stringify(plan.manifest, null, 2), "utf8"), mtimeMs);
|
|
82863
|
+
const byId = new Map(plan.manifest.frames.map((f) => [f.frameId, f]));
|
|
82864
|
+
const shipped = [];
|
|
82865
|
+
let missing = 0;
|
|
82866
|
+
let crops = 0;
|
|
82867
|
+
for (const frame of plan.frames) {
|
|
82868
|
+
const entry = byId.get(frame.frameId);
|
|
82869
|
+
if (entry === void 0) continue;
|
|
82870
|
+
if (tarNameTooLong(entry.path)) {
|
|
82871
|
+
missing += 1;
|
|
82872
|
+
logger.warn("retrain export skipped a frame — path too long for the archive", {
|
|
82873
|
+
tags: { deviceId: frame.deviceId },
|
|
82874
|
+
meta: {
|
|
82875
|
+
trackId: frame.trackId,
|
|
82876
|
+
frameId: frame.frameId,
|
|
82877
|
+
path: entry.path
|
|
82878
|
+
}
|
|
82879
|
+
});
|
|
82880
|
+
continue;
|
|
82881
|
+
}
|
|
82882
|
+
const frameBytes = await deps.readFrame(frame);
|
|
82883
|
+
if (frameBytes === null) {
|
|
82884
|
+
missing += 1;
|
|
82885
|
+
logger.warn("retrain export skipped a frame — the COPY is unreadable", {
|
|
82886
|
+
tags: { deviceId: frame.deviceId },
|
|
82887
|
+
meta: {
|
|
82888
|
+
trackId: frame.trackId,
|
|
82889
|
+
frameId: frame.frameId
|
|
82890
|
+
}
|
|
82891
|
+
});
|
|
82892
|
+
continue;
|
|
82893
|
+
}
|
|
82894
|
+
await writeEntry(entry.path, frameBytes, mtimeMs);
|
|
82895
|
+
for (const annotation of entry.annotations) {
|
|
82896
|
+
if (annotation.cropPath === void 0) continue;
|
|
82897
|
+
const crop = deps.cutCrop ? await deps.cutCrop(frameBytes, frame, annotation) : null;
|
|
82898
|
+
if (crop === null) {
|
|
82899
|
+
logger.warn("retrain export skipped a crop — it could not be cut", {
|
|
82900
|
+
tags: { deviceId: frame.deviceId },
|
|
82901
|
+
meta: {
|
|
82902
|
+
frameId: frame.frameId,
|
|
82903
|
+
annotationId: annotation.id,
|
|
82904
|
+
cropPath: annotation.cropPath
|
|
82905
|
+
}
|
|
82906
|
+
});
|
|
82907
|
+
continue;
|
|
82908
|
+
}
|
|
82909
|
+
await writeEntry(annotation.cropPath, crop, mtimeMs);
|
|
82910
|
+
crops += 1;
|
|
82911
|
+
}
|
|
82912
|
+
shipped.push(frame.frameId);
|
|
82913
|
+
if (pacingMs > 0) await new Promise((resolve) => setTimeout(resolve, pacingMs));
|
|
82914
|
+
}
|
|
82915
|
+
await write(tarTrailer());
|
|
82916
|
+
await deps.markExported(shipped, plan.manifest.batchId);
|
|
82917
|
+
const stats = {
|
|
82918
|
+
batchId: plan.manifest.batchId,
|
|
82919
|
+
frames: shipped.length,
|
|
82920
|
+
crops,
|
|
82921
|
+
annotations: plan.manifest.annotationCount,
|
|
82922
|
+
missing,
|
|
82923
|
+
bytes,
|
|
82924
|
+
sha256: hash.digest("hex")
|
|
82925
|
+
};
|
|
82926
|
+
logger.info("retrain export complete", { meta: {
|
|
82927
|
+
...stats,
|
|
82928
|
+
layout: plan.manifest.layout,
|
|
82929
|
+
modelErrors: plan.manifest.modelErrorCount
|
|
82930
|
+
} });
|
|
82931
|
+
return stats;
|
|
82932
|
+
}
|
|
82933
|
+
//#endregion
|
|
82934
|
+
//#region src/pipeline-analytics/retrain/retrain-crop-cut.ts
|
|
82935
|
+
/**
|
|
82936
|
+
* The `crops` layout's cutter: one subject crop from a retrain frame copy.
|
|
82937
|
+
*
|
|
82938
|
+
* A classifier must be trained on the crop the runtime feeds it, so the
|
|
82939
|
+
* rectangle comes from THE detail-crop derivation (`deriveDetailCropRect`,
|
|
82940
|
+
* `types/src/pipeline/detail-crop.ts`) with the convention the manifest
|
|
82941
|
+
* records — never a second rectangle maths. This is the second permitted
|
|
82942
|
+
* caller of that derivation (`scripts/check-clip-crop-single-owner.ts`): it
|
|
82943
|
+
* writes training files, never a vector, so it cannot split a feature space;
|
|
82944
|
+
* what it must not do is cut differently, and calling the one function is how
|
|
82945
|
+
* it doesn't.
|
|
82946
|
+
*/
|
|
82947
|
+
/** The pixel window to extract, or `null` when nothing of it is inside the frame. */
|
|
82948
|
+
function retrainCropWindow(input) {
|
|
82949
|
+
const { frameWidth: fw, frameHeight: fh, bbox } = input;
|
|
82950
|
+
const rect = deriveDetailCropRect({
|
|
82951
|
+
x: bbox.x * fw,
|
|
82952
|
+
y: bbox.y * fh,
|
|
82953
|
+
w: bbox.w * fw,
|
|
82954
|
+
h: bbox.h * fh
|
|
82955
|
+
}, fw, fh, input.convention);
|
|
82956
|
+
const left = Math.max(0, Math.floor(rect.x));
|
|
82957
|
+
const top = Math.max(0, Math.floor(rect.y));
|
|
82958
|
+
const right = Math.min(fw, Math.ceil(rect.x + rect.w));
|
|
82959
|
+
const bottom = Math.min(fh, Math.ceil(rect.y + rect.h));
|
|
82960
|
+
if (right - left < 1 || bottom - top < 1) return null;
|
|
82961
|
+
return {
|
|
82962
|
+
left,
|
|
82963
|
+
top,
|
|
82964
|
+
width: right - left,
|
|
82965
|
+
height: bottom - top
|
|
82966
|
+
};
|
|
82967
|
+
}
|
|
82968
|
+
async function cutRetrainCrop(input) {
|
|
82969
|
+
const window = retrainCropWindow(input);
|
|
82970
|
+
if (window === null) return null;
|
|
82971
|
+
let pipeline = sharp(input.frameBytes).extract(window);
|
|
82972
|
+
if (input.maxSide > 0) pipeline = pipeline.resize({
|
|
82973
|
+
width: input.maxSide,
|
|
82974
|
+
height: input.maxSide,
|
|
82975
|
+
fit: "inside",
|
|
82976
|
+
withoutEnlargement: true
|
|
82977
|
+
});
|
|
82978
|
+
return pipeline.jpeg({ quality: 92 }).toBuffer();
|
|
82979
|
+
}
|
|
82980
|
+
//#endregion
|
|
82451
82981
|
//#region src/pipeline-analytics/retrain/retrain-geometry.ts
|
|
82452
82982
|
function clamp01(v) {
|
|
82453
82983
|
return v < 0 ? 0 : v > 1 ? 1 : v;
|
|
@@ -82543,15 +83073,99 @@ function deriveSubjectCropRect(bbox, padding) {
|
|
|
82543
83073
|
};
|
|
82544
83074
|
}
|
|
82545
83075
|
//#endregion
|
|
83076
|
+
//#region src/pipeline-analytics/retrain/retrain-lighting.ts
|
|
83077
|
+
function isNight(condition) {
|
|
83078
|
+
return condition === "night" || condition === "ir";
|
|
83079
|
+
}
|
|
83080
|
+
function retrainFrameLighting(input) {
|
|
83081
|
+
if (input.capturedAt === void 0) {
|
|
83082
|
+
const measured = resolveConditionDetailed({
|
|
83083
|
+
...input.chromaSpread !== void 0 ? { chromaSpread: input.chromaSpread } : {},
|
|
83084
|
+
now: /* @__PURE__ */ new Date(0)
|
|
83085
|
+
});
|
|
83086
|
+
return measured.source === "ir-chroma" ? {
|
|
83087
|
+
condition: measured.condition,
|
|
83088
|
+
night: true,
|
|
83089
|
+
source: measured.source
|
|
83090
|
+
} : null;
|
|
83091
|
+
}
|
|
83092
|
+
const resolved = resolveConditionDetailed({
|
|
83093
|
+
...input.chromaSpread !== void 0 ? { chromaSpread: input.chromaSpread } : {},
|
|
83094
|
+
...input.coordinates !== null ? { coordinates: input.coordinates } : {},
|
|
83095
|
+
now: new Date(input.capturedAt)
|
|
83096
|
+
});
|
|
83097
|
+
return {
|
|
83098
|
+
...resolved,
|
|
83099
|
+
night: isNight(resolved.condition)
|
|
83100
|
+
};
|
|
83101
|
+
}
|
|
83102
|
+
//#endregion
|
|
82546
83103
|
//#region src/pipeline-analytics/retrain/retrain-export.ts
|
|
83104
|
+
/**
|
|
83105
|
+
* The ANNOTATED dataset export — the retrain page's own archive.
|
|
83106
|
+
*
|
|
83107
|
+
* Sibling of `training-export/`, deliberately not the same thing. That one
|
|
83108
|
+
* streams a marked track's stored media verbatim and knows nothing about what
|
|
83109
|
+
* is in it; this one streams the frames the operator SELECTED (already copied
|
|
83110
|
+
* into retrain storage) plus the ground truth they drew.
|
|
83111
|
+
*
|
|
83112
|
+
* ## The one interesting decision
|
|
83113
|
+
*
|
|
83114
|
+
* `annotations.json` carries, for every subject, the canonical full-frame box
|
|
83115
|
+
* AND the geometry derived for each model shape. **Derived here, at export,
|
|
83116
|
+
* never stored** — one box in, three shapes out. Storing the derived shapes is
|
|
83117
|
+
* how two crops of the same subject end up in one feature space, which is the
|
|
83118
|
+
* failure already recorded for CLIP (D52). A consumer that wants a different
|
|
83119
|
+
* input size or a different padding re-derives from the canonical box; nothing
|
|
83120
|
+
* in the database has to change and nothing can disagree.
|
|
83121
|
+
*
|
|
83122
|
+
* Everything else follows the export next door: index reads to plan, one JPEG
|
|
83123
|
+
* in memory at a time, a yield between files, and NO inference — a training
|
|
83124
|
+
* set's pixels are the pixels the pipeline saw.
|
|
83125
|
+
*/
|
|
82547
83126
|
var RETRAIN_EXPORT_PREFIX = "retrain-export";
|
|
82548
|
-
/**
|
|
82549
|
-
|
|
83127
|
+
/**
|
|
83128
|
+
* Padding a subject-crop classifier is trained with: the RUNTIME detail-crop
|
|
83129
|
+
* padding, read from the one convention `deriveDetailCropRect` cuts with
|
|
83130
|
+
* (`types/src/pipeline/detail-crop.ts`), never a second literal. It was `0.2`
|
|
83131
|
+
* here while the runtime fed `0.15`, so a classifier trained from this export
|
|
83132
|
+
* learned on more context than it is ever given live.
|
|
83133
|
+
*/
|
|
83134
|
+
var CLASSIFIER_CROP_PADDING = DEFAULT_DETAIL_CROP_CONVENTION.paddingRatio;
|
|
83135
|
+
/** The convention a `crops` layout cuts with — the runtime default, recorded
|
|
83136
|
+
* in the manifest so a consumer never has to assume it. */
|
|
83137
|
+
var RETRAIN_CROP_CONVENTION = DEFAULT_DETAIL_CROP_CONVENTION;
|
|
83138
|
+
/** Longest side of a `crops`-layout JPEG. Classifier inputs are 224–260 px;
|
|
83139
|
+
* the trainer resizes per model, so this only bounds the upload. */
|
|
83140
|
+
var RETRAIN_CROP_MAX_SIDE = 256;
|
|
83141
|
+
/** Manifest schema id. v2 = capturedAt / complete / lighting / layout. */
|
|
83142
|
+
var RETRAIN_MANIFEST_SCHEMA = "camstack.retrain-export/v2";
|
|
82550
83143
|
/** Archive path of a frame: `frames/<deviceId>/<trackId>/<frameId leaf>.jpg`. */
|
|
82551
83144
|
function retrainFramePath(frame) {
|
|
82552
83145
|
const leaf = frame.frameId.slice(frame.frameId.lastIndexOf(":") + 1);
|
|
82553
83146
|
return `frames/${frame.deviceId}/${frame.trackId}/${leaf}.jpg`;
|
|
82554
83147
|
}
|
|
83148
|
+
/** Archive path of a subject crop: `crops/<macroClass>/<annotationId>.jpg`. */
|
|
83149
|
+
function retrainCropPath(macroClass, annotationId) {
|
|
83150
|
+
return `crops/${macroClass}/${annotationId}.jpg`;
|
|
83151
|
+
}
|
|
83152
|
+
/**
|
|
83153
|
+
* Narrow the planned frames to a filter. Pure. Every field narrows; an empty
|
|
83154
|
+
* filter keeps everything. Order is the input's.
|
|
83155
|
+
*/
|
|
83156
|
+
function filterRetrainExport(frames, annotationsByFrame, filter) {
|
|
83157
|
+
const wanted = filter.frameIds !== void 0 ? new Set(filter.frameIds) : null;
|
|
83158
|
+
const devices = filter.deviceIds !== void 0 && filter.deviceIds.length > 0 ? new Set(filter.deviceIds) : null;
|
|
83159
|
+
return frames.filter((frame) => {
|
|
83160
|
+
if (wanted !== null && !wanted.has(frame.frameId)) return false;
|
|
83161
|
+
if (devices !== null && !devices.has(frame.deviceId)) return false;
|
|
83162
|
+
if (filter.completeOnly === true && !frame.complete) return false;
|
|
83163
|
+
if (filter.batchId !== void 0) {
|
|
83164
|
+
if (!(annotationsByFrame.get(frame.frameId) ?? []).some((a) => a.exportedInBatch === filter.batchId)) return false;
|
|
83165
|
+
}
|
|
83166
|
+
return true;
|
|
83167
|
+
});
|
|
83168
|
+
}
|
|
82555
83169
|
/**
|
|
82556
83170
|
* Derive every per-model shape for one annotation.
|
|
82557
83171
|
*
|
|
@@ -82572,6 +83186,8 @@ function deriveAll(annotation, frame, zone) {
|
|
|
82572
83186
|
}
|
|
82573
83187
|
/** Build the manifest. Pure — index rows in, JSON out, no blob touched. */
|
|
82574
83188
|
function buildRetrainManifest(input) {
|
|
83189
|
+
const layout = input.layout ?? "frames";
|
|
83190
|
+
const coordinates = input.coordinates ?? null;
|
|
82575
83191
|
let annotationCount = 0;
|
|
82576
83192
|
let modelErrorCount = 0;
|
|
82577
83193
|
const frames = input.frames.map((frame) => {
|
|
@@ -82579,6 +83195,7 @@ function buildRetrainManifest(input) {
|
|
|
82579
83195
|
const annotations = (input.annotationsByFrame.get(frame.frameId) ?? []).map((a) => {
|
|
82580
83196
|
annotationCount += 1;
|
|
82581
83197
|
if (a.kind === "model_error") modelErrorCount += 1;
|
|
83198
|
+
const derived = deriveAll(a, frame, zone);
|
|
82582
83199
|
return {
|
|
82583
83200
|
id: a.id,
|
|
82584
83201
|
macroClass: a.macroClass,
|
|
@@ -82589,7 +83206,8 @@ function buildRetrainManifest(input) {
|
|
|
82589
83206
|
...a.assistModelId !== void 0 ? { assistModelId: a.assistModelId } : {},
|
|
82590
83207
|
...a.assistScore !== void 0 ? { assistScore: a.assistScore } : {},
|
|
82591
83208
|
bbox: a.bbox,
|
|
82592
|
-
derived:
|
|
83209
|
+
...layout === "crops" && derived.subjectCrop !== void 0 ? { cropPath: retrainCropPath(a.macroClass, a.id) } : {},
|
|
83210
|
+
derived
|
|
82593
83211
|
};
|
|
82594
83212
|
});
|
|
82595
83213
|
return {
|
|
@@ -82602,11 +83220,22 @@ function buildRetrainManifest(input) {
|
|
|
82602
83220
|
sizeBytes: frame.sizeBytes,
|
|
82603
83221
|
sourceKind: frame.sourceKind,
|
|
82604
83222
|
copiedAt: frame.copiedAt,
|
|
83223
|
+
...frame.capturedAt !== void 0 ? { capturedAt: frame.capturedAt } : {},
|
|
83224
|
+
complete: frame.complete,
|
|
83225
|
+
lighting: retrainFrameLighting({
|
|
83226
|
+
...frame.capturedAt !== void 0 ? { capturedAt: frame.capturedAt } : {},
|
|
83227
|
+
...frame.chromaSpread !== void 0 ? { chromaSpread: frame.chromaSpread } : {},
|
|
83228
|
+
coordinates
|
|
83229
|
+
}),
|
|
82605
83230
|
...zone !== null ? { zoneUsed: zone } : {},
|
|
82606
83231
|
annotations
|
|
82607
83232
|
};
|
|
82608
83233
|
});
|
|
82609
83234
|
return {
|
|
83235
|
+
schema: RETRAIN_MANIFEST_SCHEMA,
|
|
83236
|
+
layout,
|
|
83237
|
+
detailCropConvention: RETRAIN_CROP_CONVENTION,
|
|
83238
|
+
...layout === "crops" ? { cropMaxSide: RETRAIN_CROP_MAX_SIDE } : {},
|
|
82610
83239
|
generatedAt: input.generatedAt,
|
|
82611
83240
|
batchId: input.batchId,
|
|
82612
83241
|
frameCount: frames.length,
|
|
@@ -82617,123 +83246,36 @@ function buildRetrainManifest(input) {
|
|
|
82617
83246
|
};
|
|
82618
83247
|
}
|
|
82619
83248
|
//#endregion
|
|
82620
|
-
//#region src/pipeline-analytics/
|
|
82621
|
-
/**
|
|
82622
|
-
|
|
82623
|
-
|
|
82624
|
-
|
|
82625
|
-
|
|
82626
|
-
|
|
82627
|
-
|
|
82628
|
-
|
|
82629
|
-
|
|
82630
|
-
|
|
82631
|
-
|
|
82632
|
-
|
|
82633
|
-
|
|
82634
|
-
|
|
82635
|
-
|
|
82636
|
-
*
|
|
82637
|
-
* So: ustar, which is a 512-byte header and a 512-byte-padded body per entry,
|
|
82638
|
-
* fully specified, and asserted against the system `tar` binary in the spec next
|
|
82639
|
-
* door rather than against my reading of the format.
|
|
82640
|
-
*
|
|
82641
|
-
* ## What this deliberately does NOT do
|
|
82642
|
-
*
|
|
82643
|
-
* No gzip — the payload is JPEG, already compressed; a gzip pass would burn CPU
|
|
82644
|
-
* on the hub for a percent or two. No long-name (GNU/PAX) extension: ustar
|
|
82645
|
-
* splits a path across a 155-byte prefix and a 100-byte name, and
|
|
82646
|
-
* {@link tarNameTooLong} reports a path that will not fit so the caller can skip
|
|
82647
|
-
* it and SAY it skipped it, rather than emitting a silently truncated entry.
|
|
82648
|
-
*/
|
|
82649
|
-
var BLOCK_SIZE = 512;
|
|
82650
|
-
var NAME_MAX = 100;
|
|
82651
|
-
var PREFIX_MAX = 155;
|
|
82652
|
-
/**
|
|
82653
|
-
* Split a path into the ustar `prefix` + `name` pair, or `null` when it cannot
|
|
82654
|
-
* be represented. The split must land on a `/`, and each half has its own cap.
|
|
82655
|
-
*/
|
|
82656
|
-
function splitUstarName(name) {
|
|
82657
|
-
if (Buffer.byteLength(name) <= NAME_MAX) return {
|
|
82658
|
-
prefix: "",
|
|
82659
|
-
name
|
|
83249
|
+
//#region src/pipeline-analytics/retrain/retrain-export-handler.ts
|
|
83250
|
+
/** The export request a URL carries (`getRetrainExportUrl` composes it). */
|
|
83251
|
+
function parseRetrainExportRequest(url) {
|
|
83252
|
+
const query = url.includes("?") ? url.slice(url.indexOf("?") + 1) : "";
|
|
83253
|
+
const params = new URLSearchParams(query);
|
|
83254
|
+
const deviceIds = parseRetrainDeviceIds(url);
|
|
83255
|
+
const frameIds = (params.get("frameIds") ?? "").split(",").map((id) => id.trim()).filter((id) => id.length > 0);
|
|
83256
|
+
const batchId = params.get("batchId")?.trim() ?? "";
|
|
83257
|
+
return {
|
|
83258
|
+
filter: {
|
|
83259
|
+
...deviceIds.length > 0 ? { deviceIds: [...deviceIds] } : {},
|
|
83260
|
+
...frameIds.length > 0 ? { frameIds } : {},
|
|
83261
|
+
...batchId.length > 0 ? { batchId } : {},
|
|
83262
|
+
...params.get("completeOnly") === "1" ? { completeOnly: true } : {}
|
|
83263
|
+
},
|
|
83264
|
+
layout: params.get("layout") === "crops" ? "crops" : "frames"
|
|
82660
83265
|
};
|
|
82661
|
-
for (let i = name.length - 1; i >= 0; i--) {
|
|
82662
|
-
if (name[i] !== "/") continue;
|
|
82663
|
-
const head = name.slice(0, i);
|
|
82664
|
-
const tail = name.slice(i + 1);
|
|
82665
|
-
if (Buffer.byteLength(tail) > NAME_MAX) break;
|
|
82666
|
-
if (Buffer.byteLength(head) <= PREFIX_MAX) return {
|
|
82667
|
-
prefix: head,
|
|
82668
|
-
name: tail
|
|
82669
|
-
};
|
|
82670
|
-
}
|
|
82671
|
-
return null;
|
|
82672
83266
|
}
|
|
82673
|
-
/**
|
|
82674
|
-
function
|
|
82675
|
-
|
|
82676
|
-
}
|
|
82677
|
-
|
|
82678
|
-
|
|
82679
|
-
|
|
82680
|
-
|
|
82681
|
-
|
|
82682
|
-
const text =
|
|
82683
|
-
|
|
83267
|
+
/** The query string `parseRetrainExportRequest` reads back. */
|
|
83268
|
+
function retrainExportQuery(request) {
|
|
83269
|
+
const params = new URLSearchParams();
|
|
83270
|
+
const { filter } = request;
|
|
83271
|
+
if (filter.deviceIds !== void 0 && filter.deviceIds.length > 0) params.set("deviceIds", filter.deviceIds.join(","));
|
|
83272
|
+
if (filter.frameIds !== void 0 && filter.frameIds.length > 0) params.set("frameIds", filter.frameIds.join(","));
|
|
83273
|
+
if (filter.batchId !== void 0) params.set("batchId", filter.batchId);
|
|
83274
|
+
if (filter.completeOnly === true) params.set("completeOnly", "1");
|
|
83275
|
+
if (request.layout === "crops") params.set("layout", "crops");
|
|
83276
|
+
const text = params.toString();
|
|
83277
|
+
return text.length > 0 ? `?${text}` : "";
|
|
82684
83278
|
}
|
|
82685
|
-
/**
|
|
82686
|
-
* The 512-byte header for one regular file.
|
|
82687
|
-
*
|
|
82688
|
-
* The checksum is computed with the checksum field itself read as eight spaces —
|
|
82689
|
-
* that is the format's rule, and getting it wrong produces an archive that looks
|
|
82690
|
-
* fine until something tries to read it.
|
|
82691
|
-
*/
|
|
82692
|
-
function tarHeader(entry) {
|
|
82693
|
-
const split = splitUstarName(entry.name);
|
|
82694
|
-
if (split === null) throw new Error(`tar: entry name too long for ustar: ${entry.name}`);
|
|
82695
|
-
const block = Buffer.alloc(BLOCK_SIZE);
|
|
82696
|
-
writeString(block, split.name, 0, NAME_MAX);
|
|
82697
|
-
writeOctal(block, 420, 100, 8);
|
|
82698
|
-
writeOctal(block, 0, 108, 8);
|
|
82699
|
-
writeOctal(block, 0, 116, 8);
|
|
82700
|
-
writeOctal(block, entry.body.length, 124, 12);
|
|
82701
|
-
writeOctal(block, Math.floor(entry.mtimeMs / 1e3), 136, 12);
|
|
82702
|
-
block.write(" ", 148, 8, "ascii");
|
|
82703
|
-
block.write("0", 156, 1, "ascii");
|
|
82704
|
-
block.write("ustar\0", 257, 6, "ascii");
|
|
82705
|
-
block.write("00", 263, 2, "ascii");
|
|
82706
|
-
writeString(block, split.prefix, 345, PREFIX_MAX);
|
|
82707
|
-
let sum = 0;
|
|
82708
|
-
for (const byte of block) sum += byte;
|
|
82709
|
-
writeOctal(block, sum, 148, 7);
|
|
82710
|
-
block.write("\0", 154, 1, "ascii");
|
|
82711
|
-
block.write(" ", 155, 1, "ascii");
|
|
82712
|
-
return block;
|
|
82713
|
-
}
|
|
82714
|
-
/** Zero padding that rounds a body up to the 512-byte block. */
|
|
82715
|
-
function tarPadding(size) {
|
|
82716
|
-
const remainder = size % BLOCK_SIZE;
|
|
82717
|
-
return remainder === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK_SIZE - remainder);
|
|
82718
|
-
}
|
|
82719
|
-
/** Header + body + padding for one entry, ready to write. */
|
|
82720
|
-
function tarBlocks(entry) {
|
|
82721
|
-
return [
|
|
82722
|
-
tarHeader(entry),
|
|
82723
|
-
entry.body,
|
|
82724
|
-
tarPadding(entry.body.length)
|
|
82725
|
-
];
|
|
82726
|
-
}
|
|
82727
|
-
/** The two zero blocks that terminate an archive. */
|
|
82728
|
-
function tarTrailer() {
|
|
82729
|
-
return Buffer.alloc(BLOCK_SIZE * 2);
|
|
82730
|
-
}
|
|
82731
|
-
//#endregion
|
|
82732
|
-
//#region src/pipeline-analytics/retrain/retrain-export-handler.ts
|
|
82733
|
-
var RETRAIN_MANIFEST_ENTRY = "annotations.json";
|
|
82734
|
-
/** Yield between files so a large export cannot monopolise the hub's one
|
|
82735
|
-
* thread — the same discipline the training export follows. */
|
|
82736
|
-
var PACING_MS = 25;
|
|
82737
83279
|
function parseRetrainDeviceIds(url) {
|
|
82738
83280
|
const query = url.includes("?") ? url.slice(url.indexOf("?") + 1) : "";
|
|
82739
83281
|
const raw = new URLSearchParams(query).get("deviceIds");
|
|
@@ -82756,10 +83298,10 @@ function createRetrainExportHandler(deps) {
|
|
|
82756
83298
|
res.end();
|
|
82757
83299
|
return;
|
|
82758
83300
|
}
|
|
82759
|
-
const
|
|
83301
|
+
const request = parseRetrainExportRequest(req.url ?? "");
|
|
82760
83302
|
let plan;
|
|
82761
83303
|
try {
|
|
82762
|
-
plan = await deps.buildPlan(
|
|
83304
|
+
plan = await deps.buildPlan(request);
|
|
82763
83305
|
} catch (err) {
|
|
82764
83306
|
deps.logger.warn("retrain export plan failed", { meta: { error: String(err) } });
|
|
82765
83307
|
res.writeHead(500, { "content-type": "text/plain" });
|
|
@@ -82773,7 +83315,8 @@ function createRetrainExportHandler(deps) {
|
|
|
82773
83315
|
"x-camstack-retrain-frames": String(plan.manifest.frameCount),
|
|
82774
83316
|
"x-camstack-retrain-annotations": String(plan.manifest.annotationCount),
|
|
82775
83317
|
"x-camstack-retrain-model-errors": String(plan.manifest.modelErrorCount),
|
|
82776
|
-
"x-camstack-retrain-batch": plan.manifest.batchId
|
|
83318
|
+
"x-camstack-retrain-batch": plan.manifest.batchId,
|
|
83319
|
+
"x-camstack-retrain-layout": plan.manifest.layout
|
|
82777
83320
|
});
|
|
82778
83321
|
if (method === "HEAD") {
|
|
82779
83322
|
res.end();
|
|
@@ -82786,61 +83329,18 @@ function createRetrainExportHandler(deps) {
|
|
|
82786
83329
|
});
|
|
82787
83330
|
if (!ok) res.once("drain", resolve);
|
|
82788
83331
|
});
|
|
82789
|
-
|
|
82790
|
-
|
|
82791
|
-
|
|
82792
|
-
|
|
82793
|
-
|
|
82794
|
-
|
|
82795
|
-
|
|
82796
|
-
|
|
82797
|
-
|
|
82798
|
-
const entry = plan.manifest.frames.find((f) => f.frameId === frame.frameId);
|
|
82799
|
-
if (entry === void 0) continue;
|
|
82800
|
-
if (tarNameTooLong(entry.path)) {
|
|
82801
|
-
missing += 1;
|
|
82802
|
-
deps.logger.warn("retrain export skipped a frame — path too long for the archive", {
|
|
82803
|
-
tags: { deviceId: frame.deviceId },
|
|
82804
|
-
meta: {
|
|
82805
|
-
trackId: frame.trackId,
|
|
82806
|
-
frameId: frame.frameId,
|
|
82807
|
-
path: entry.path
|
|
82808
|
-
}
|
|
82809
|
-
});
|
|
82810
|
-
continue;
|
|
82811
|
-
}
|
|
82812
|
-
const bytes = await deps.readFrame(frame);
|
|
82813
|
-
if (bytes === null) {
|
|
82814
|
-
missing += 1;
|
|
82815
|
-
deps.logger.warn("retrain export skipped a frame — the COPY is unreadable", {
|
|
82816
|
-
tags: { deviceId: frame.deviceId },
|
|
82817
|
-
meta: {
|
|
82818
|
-
trackId: frame.trackId,
|
|
82819
|
-
frameId: frame.frameId
|
|
82820
|
-
}
|
|
82821
|
-
});
|
|
82822
|
-
continue;
|
|
82823
|
-
}
|
|
82824
|
-
for (const block of tarBlocks({
|
|
82825
|
-
name: entry.path,
|
|
82826
|
-
body: bytes,
|
|
82827
|
-
mtimeMs
|
|
82828
|
-
})) await write(block);
|
|
82829
|
-
shipped.push(frame.frameId);
|
|
82830
|
-
await new Promise((resolve) => setTimeout(resolve, PACING_MS));
|
|
82831
|
-
}
|
|
82832
|
-
await write(tarTrailer());
|
|
83332
|
+
await writeRetrainArchiveTo({
|
|
83333
|
+
plan,
|
|
83334
|
+
readFrame: deps.readFrame,
|
|
83335
|
+
cutCrop: deps.cutCrop,
|
|
83336
|
+
sink: { write },
|
|
83337
|
+
markExported: deps.markExported,
|
|
83338
|
+
logger: deps.logger,
|
|
83339
|
+
now: deps.now
|
|
83340
|
+
});
|
|
82833
83341
|
await new Promise((resolve) => {
|
|
82834
83342
|
res.end(resolve);
|
|
82835
83343
|
});
|
|
82836
|
-
await deps.markExported(shipped, plan.manifest.batchId);
|
|
82837
|
-
deps.logger.info("retrain export complete", { meta: {
|
|
82838
|
-
batchId: plan.manifest.batchId,
|
|
82839
|
-
frames: shipped.length,
|
|
82840
|
-
missing,
|
|
82841
|
-
annotations: plan.manifest.annotationCount,
|
|
82842
|
-
modelErrors: plan.manifest.modelErrorCount
|
|
82843
|
-
} });
|
|
82844
83344
|
};
|
|
82845
83345
|
}
|
|
82846
83346
|
//#endregion
|
|
@@ -83158,7 +83658,9 @@ function toWireFrame(f) {
|
|
|
83158
83658
|
sizeBytes: f.sizeBytes,
|
|
83159
83659
|
width: f.width,
|
|
83160
83660
|
height: f.height,
|
|
83161
|
-
copiedAt: f.copiedAt
|
|
83661
|
+
copiedAt: f.copiedAt,
|
|
83662
|
+
...f.capturedAt !== void 0 ? { capturedAt: f.capturedAt } : {},
|
|
83663
|
+
complete: f.complete
|
|
83162
83664
|
};
|
|
83163
83665
|
}
|
|
83164
83666
|
function toWireAnnotation(a) {
|
|
@@ -83261,6 +83763,7 @@ var RetrainService = class {
|
|
|
83261
83763
|
async selectFrames(input) {
|
|
83262
83764
|
const media = await this.deps.media.listTrackMedia(input.trackId);
|
|
83263
83765
|
const kindOf = new Map(media.map((m) => [m.mediaId, m.kind]));
|
|
83766
|
+
const capturedAtOf = new Map(media.map((m) => [m.mediaId, m.timestamp]));
|
|
83264
83767
|
const copied = [];
|
|
83265
83768
|
const refused = [];
|
|
83266
83769
|
for (const mediaId of input.mediaIds) {
|
|
@@ -83268,7 +83771,8 @@ var RetrainService = class {
|
|
|
83268
83771
|
deviceId: input.deviceId,
|
|
83269
83772
|
trackId: input.trackId,
|
|
83270
83773
|
sourceMediaId: mediaId,
|
|
83271
|
-
sourceKind: kindOf.get(mediaId) ?? "keyFrame"
|
|
83774
|
+
sourceKind: kindOf.get(mediaId) ?? "keyFrame",
|
|
83775
|
+
...capturedAtOf.has(mediaId) ? { capturedAt: capturedAtOf.get(mediaId) } : {}
|
|
83272
83776
|
});
|
|
83273
83777
|
if (result.kind === "copied") copied.push(toWireFrame(result.frame));
|
|
83274
83778
|
else refused.push({
|
|
@@ -83281,6 +83785,34 @@ var RetrainService = class {
|
|
|
83281
83785
|
refused
|
|
83282
83786
|
};
|
|
83283
83787
|
}
|
|
83788
|
+
/**
|
|
83789
|
+
* The operator's "every subject in this frame is boxed". Scoped to the
|
|
83790
|
+
* track the caller names: a frame of another track is refused, loudly.
|
|
83791
|
+
*/
|
|
83792
|
+
async setFrameComplete(input) {
|
|
83793
|
+
const frame = await this.deps.frames.get(input.frameId);
|
|
83794
|
+
if (frame === null || frame.trackId !== input.trackId || frame.deviceId !== input.deviceId) {
|
|
83795
|
+
this.deps.logger.warn("retrain complete refused: the frame does not belong to that track", {
|
|
83796
|
+
tags: { deviceId: input.deviceId },
|
|
83797
|
+
meta: {
|
|
83798
|
+
trackId: input.trackId,
|
|
83799
|
+
frameId: input.frameId
|
|
83800
|
+
}
|
|
83801
|
+
});
|
|
83802
|
+
throw new RetrainAnnotationError(`retrain frame ${input.frameId} does not belong to track ${input.trackId}`);
|
|
83803
|
+
}
|
|
83804
|
+
const updated = await this.deps.frames.setComplete(input.frameId, input.complete);
|
|
83805
|
+
if (updated === null) throw new RetrainAnnotationError(`retrain frame ${input.frameId} vanished`);
|
|
83806
|
+
this.deps.logger.info("retrain frame completeness set", {
|
|
83807
|
+
tags: { deviceId: input.deviceId },
|
|
83808
|
+
meta: {
|
|
83809
|
+
trackId: input.trackId,
|
|
83810
|
+
frameId: input.frameId,
|
|
83811
|
+
complete: input.complete
|
|
83812
|
+
}
|
|
83813
|
+
});
|
|
83814
|
+
return toWireFrame(updated);
|
|
83815
|
+
}
|
|
83284
83816
|
/** Un-select: annotations first, then the copy. Never the other way round —
|
|
83285
83817
|
* an annotation whose frame is gone is unopenable and unfixable. */
|
|
83286
83818
|
async deselectFrame(input) {
|
|
@@ -84493,7 +85025,8 @@ function selectsRow(mode, toLocationId, data, isRetrain, fromLocationId) {
|
|
|
84493
85025
|
if (mode === "seal") return rowIsUnstamped(data);
|
|
84494
85026
|
if (mediaRowLocation(data) === toLocationId) return false;
|
|
84495
85027
|
if (mode === "gallery") return !isRetrain && rowIsGallery(data);
|
|
84496
|
-
|
|
85028
|
+
if (isRetrain) return !rowIsDurableRetrainCopy(data);
|
|
85029
|
+
return !rowIsGallery(data);
|
|
84497
85030
|
}
|
|
84498
85031
|
var MediaRelocateEngine = class {
|
|
84499
85032
|
deps;
|
|
@@ -97152,7 +97685,7 @@ var ANALYTICS_COLLECTIONS = [
|
|
|
97152
97685
|
collection: RETRAIN_FRAMES_COLLECTION,
|
|
97153
97686
|
classification: {
|
|
97154
97687
|
kind: "never-orphaned",
|
|
97155
|
-
why: "the frame COPIES the dataset owns outright,
|
|
97688
|
+
why: "the frame COPIES the dataset owns outright, in the `retrain/` subtree of the durable `artifacts` class (D781), which no owner cascade and no age sweep walks (D81). Also declares `trackId` as provenance. Rows go only through `deleteFrame`, after their annotations, on operator deselection. NOTE: nothing caps this table or the disk it consumes — that is a deliberate decision, not an oversight, and it is the one collection here whose bound is an operator's habit."
|
|
97156
97689
|
}
|
|
97157
97690
|
},
|
|
97158
97691
|
{
|
|
@@ -108419,6 +108952,9 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
108419
108952
|
resolveRoot: (locationId) => locationId === LEGACY_ADDON_DATA_GALLERY_LOCATION ? Promise.resolve(path.join(this.ctx.dataDir, "gallery")) : locationId === GALLERY_MEDIA_LOCATION ? this.ctx.api.storage.resolve.query({
|
|
108420
108953
|
location: ARTIFACTS_MEDIA_LOCATION,
|
|
108421
108954
|
relativePath: GALLERY_TENANT_SUBPATH
|
|
108955
|
+
}) : locationId === RETRAIN_MEDIA_LOCATION ? this.ctx.api.storage.resolve.query({
|
|
108956
|
+
location: ARTIFACTS_MEDIA_LOCATION,
|
|
108957
|
+
relativePath: ""
|
|
108422
108958
|
}) : this.ctx.api.storage.resolve.query({
|
|
108423
108959
|
location: locationId,
|
|
108424
108960
|
relativePath: ""
|
|
@@ -108597,7 +109133,7 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
108597
109133
|
logger: logger.child("RetrainFrames"),
|
|
108598
109134
|
now: () => Date.now(),
|
|
108599
109135
|
writeGate: this.mediaWriteGate,
|
|
108600
|
-
|
|
109136
|
+
measureChroma: (bytes) => chromaSpread(bytes),
|
|
108601
109137
|
readSource: async (mediaKey) => {
|
|
108602
109138
|
const file = await mediaStore.getById(mediaKey);
|
|
108603
109139
|
return file === null ? null : Buffer.from(file.base64, "base64");
|
|
@@ -109834,10 +110370,59 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
109834
110370
|
/** Where to stream the ANNOTATED dataset from. Bytes never cross the cap. */
|
|
109835
110371
|
async getRetrainExportUrl(input) {
|
|
109836
110372
|
if (this.retrainExportDataPlane === null) throw new Error("retrain export: data plane unavailable on this node");
|
|
109837
|
-
const
|
|
109838
|
-
const query =
|
|
110373
|
+
const { layout, ...filter } = input;
|
|
110374
|
+
const query = retrainExportQuery({
|
|
110375
|
+
filter,
|
|
110376
|
+
layout: layout ?? "frames"
|
|
110377
|
+
});
|
|
109839
110378
|
return { url: `/addon/${this.ctx.id}/${RETRAIN_EXPORT_PREFIX}${query}` };
|
|
109840
110379
|
}
|
|
110380
|
+
async setRetrainFrameComplete(input) {
|
|
110381
|
+
return this.requireRetrain().setFrameComplete(input);
|
|
110382
|
+
}
|
|
110383
|
+
/**
|
|
110384
|
+
* Stream the annotated archive into ONE HTTP request to `target` — the
|
|
110385
|
+
* training upload. Same bytes as the download; never on the hub's disk.
|
|
110386
|
+
*/
|
|
110387
|
+
async writeRetrainArchive(input) {
|
|
110388
|
+
const frameStore = this.retrainFrames;
|
|
110389
|
+
if (frameStore === null) throw new Error("retrain export: analytics stores unavailable");
|
|
110390
|
+
const plan = await this.buildRetrainExportPlan({
|
|
110391
|
+
filter: input.filter,
|
|
110392
|
+
layout: input.layout
|
|
110393
|
+
});
|
|
110394
|
+
const upload = openRetrainUploadSink(input.target);
|
|
110395
|
+
const host = new URL(input.target.url).host;
|
|
110396
|
+
try {
|
|
110397
|
+
const stats = await writeRetrainArchiveTo({
|
|
110398
|
+
plan,
|
|
110399
|
+
readFrame: (frame) => frameStore.readFrame(frame),
|
|
110400
|
+
cutCrop: this.retrainCropCutter(),
|
|
110401
|
+
sink: upload.sink,
|
|
110402
|
+
markExported: (frameIds, batchId) => this.markRetrainExported(frameIds, batchId),
|
|
110403
|
+
logger: this.ctx.logger,
|
|
110404
|
+
now: () => Date.now()
|
|
110405
|
+
});
|
|
110406
|
+
const status = await upload.finish();
|
|
110407
|
+
this.ctx.logger.info("retrain archive uploaded", { meta: {
|
|
110408
|
+
host,
|
|
110409
|
+
status,
|
|
110410
|
+
...stats,
|
|
110411
|
+
layout: input.layout
|
|
110412
|
+
} });
|
|
110413
|
+
return {
|
|
110414
|
+
...stats,
|
|
110415
|
+
status
|
|
110416
|
+
};
|
|
110417
|
+
} catch (err) {
|
|
110418
|
+
upload.abort(errMsg(err));
|
|
110419
|
+
this.ctx.logger.warn("retrain archive upload failed", { meta: {
|
|
110420
|
+
host,
|
|
110421
|
+
error: errMsg(err)
|
|
110422
|
+
} });
|
|
110423
|
+
throw err;
|
|
110424
|
+
}
|
|
110425
|
+
}
|
|
109841
110426
|
/**
|
|
109842
110427
|
* The store seam the `debug.*` actions and the debug bundle both read through.
|
|
109843
110428
|
*
|
|
@@ -110119,15 +110704,10 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
110119
110704
|
async serveRetrainExportPlane() {
|
|
110120
110705
|
try {
|
|
110121
110706
|
const handler = createRetrainExportHandler({
|
|
110122
|
-
buildPlan: (
|
|
110707
|
+
buildPlan: (request) => this.buildRetrainExportPlan(request),
|
|
110123
110708
|
readFrame: (frame) => this.retrainFrames?.readFrame(frame) ?? Promise.resolve(null),
|
|
110124
|
-
|
|
110125
|
-
|
|
110126
|
-
if (!store || frameIds.length === 0) return;
|
|
110127
|
-
const ids = [];
|
|
110128
|
-
for (const frameId of frameIds) for (const a of await store.listByFrame(frameId)) ids.push(a.id);
|
|
110129
|
-
await store.markExported(ids, batchId);
|
|
110130
|
-
},
|
|
110709
|
+
cutCrop: this.retrainCropCutter(),
|
|
110710
|
+
markExported: (frameIds, batchId) => this.markRetrainExported(frameIds, batchId),
|
|
110131
110711
|
logger: this.ctx.logger,
|
|
110132
110712
|
now: () => Date.now()
|
|
110133
110713
|
});
|
|
@@ -110149,13 +110729,14 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
110149
110729
|
* reference rectangle is implicit is a box nobody can check. A camera with no
|
|
110150
110730
|
* zones simply has no `zoneCrop` geometry — absent, never a clamped guess.
|
|
110151
110731
|
*/
|
|
110152
|
-
async buildRetrainExportPlan(
|
|
110732
|
+
async buildRetrainExportPlan(request) {
|
|
110153
110733
|
const frameStore = this.retrainFrames;
|
|
110154
110734
|
const annotationStore = this.retrainAnnotations;
|
|
110155
110735
|
const trackStore = this.trackStore;
|
|
110156
110736
|
if (!frameStore || !annotationStore || !trackStore) throw new Error("retrain export: analytics stores unavailable");
|
|
110737
|
+
const deviceIds = request.filter.deviceIds ?? [];
|
|
110157
110738
|
const devices = deviceIds.length > 0 ? deviceIds : await trackStore.listDeviceIds();
|
|
110158
|
-
const
|
|
110739
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
110159
110740
|
const annotationsByFrame = /* @__PURE__ */ new Map();
|
|
110160
110741
|
const zones = /* @__PURE__ */ new Map();
|
|
110161
110742
|
for (const deviceId of devices) {
|
|
@@ -110164,22 +110745,56 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
110164
110745
|
const existing = annotationsByFrame.get(annotation.mediaKey) ?? [];
|
|
110165
110746
|
annotationsByFrame.set(annotation.mediaKey, [...existing, annotation]);
|
|
110166
110747
|
}
|
|
110748
|
+
for (const frame of await frameStore.listCompleteByDevice(deviceId, RETRAIN_EXPORT_MAX)) candidates.set(frame.frameId, frame);
|
|
110167
110749
|
}
|
|
110168
110750
|
for (const frameId of annotationsByFrame.keys()) {
|
|
110751
|
+
if (candidates.has(frameId)) continue;
|
|
110169
110752
|
const frame = await frameStore.get(frameId);
|
|
110170
|
-
if (frame !== null)
|
|
110753
|
+
if (frame !== null) candidates.set(frameId, frame);
|
|
110171
110754
|
}
|
|
110755
|
+
const frames = filterRetrainExport([...candidates.values()], annotationsByFrame, request.filter);
|
|
110172
110756
|
return {
|
|
110173
110757
|
manifest: buildRetrainManifest({
|
|
110174
110758
|
frames,
|
|
110175
110759
|
annotationsByFrame,
|
|
110176
110760
|
zoneForDevice: (deviceId) => zones.get(deviceId) ?? null,
|
|
110177
110761
|
generatedAt: Date.now(),
|
|
110178
|
-
batchId: randomUUID()
|
|
110762
|
+
batchId: randomUUID(),
|
|
110763
|
+
layout: request.layout,
|
|
110764
|
+
coordinates: await this.siteCoordinates()
|
|
110179
110765
|
}),
|
|
110180
110766
|
frames
|
|
110181
110767
|
};
|
|
110182
110768
|
}
|
|
110769
|
+
/** Stamp the batch on the annotations of the frames an archive carried. */
|
|
110770
|
+
async markRetrainExported(frameIds, batchId) {
|
|
110771
|
+
const store = this.retrainAnnotations;
|
|
110772
|
+
if (!store || frameIds.length === 0) return;
|
|
110773
|
+
const ids = [];
|
|
110774
|
+
for (const frameId of frameIds) for (const a of await store.listByFrame(frameId)) ids.push(a.id);
|
|
110775
|
+
await store.markExported(ids, batchId);
|
|
110776
|
+
}
|
|
110777
|
+
/** The `crops` layout's cutter, with the convention the manifest records. */
|
|
110778
|
+
retrainCropCutter() {
|
|
110779
|
+
return (frameBytes, frame, annotation) => cutRetrainCrop({
|
|
110780
|
+
frameBytes,
|
|
110781
|
+
frameWidth: frame.width,
|
|
110782
|
+
frameHeight: frame.height,
|
|
110783
|
+
bbox: annotation.bbox,
|
|
110784
|
+
convention: RETRAIN_CROP_CONVENTION,
|
|
110785
|
+
maxSide: RETRAIN_CROP_MAX_SIDE
|
|
110786
|
+
}).catch((err) => {
|
|
110787
|
+
this.ctx.logger.warn("retrain crop could not be cut", {
|
|
110788
|
+
tags: { deviceId: frame.deviceId },
|
|
110789
|
+
meta: {
|
|
110790
|
+
frameId: frame.frameId,
|
|
110791
|
+
annotationId: annotation.id,
|
|
110792
|
+
error: errMsg(err)
|
|
110793
|
+
}
|
|
110794
|
+
});
|
|
110795
|
+
return null;
|
|
110796
|
+
});
|
|
110797
|
+
}
|
|
110183
110798
|
/** Bounding box of a camera's zones, or null when it declares none. */
|
|
110184
110799
|
async retrainZoneForDevice(deviceId) {
|
|
110185
110800
|
try {
|
|
@@ -110327,6 +110942,28 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends BaseAddon {
|
|
|
110327
110942
|
}, LTS_FLUSH_INTERVAL_MS);
|
|
110328
110943
|
this.startOrphanAuditTimer();
|
|
110329
110944
|
this.reconcileGalleryMediaLocation();
|
|
110945
|
+
this.reconcileRetrainCopiesLocation();
|
|
110946
|
+
}
|
|
110947
|
+
/**
|
|
110948
|
+
* Move pre-D781 retrain copies off the footage class onto the durable
|
|
110949
|
+
* `retrainMedia` tenant, and say how many have no bytes left.
|
|
110950
|
+
*/
|
|
110951
|
+
async reconcileRetrainCopiesLocation() {
|
|
110952
|
+
const retrainFrames = this.retrainFrames;
|
|
110953
|
+
if (!retrainFrames) return;
|
|
110954
|
+
try {
|
|
110955
|
+
const result = await retrainFrames.relocateLegacyCopies();
|
|
110956
|
+
if (result.relocated === 0 && result.missing === 0 && result.failed === 0) return;
|
|
110957
|
+
this.ctx.logger.info("retrain copies reconciled onto the durable retrain tenant", { meta: {
|
|
110958
|
+
...result,
|
|
110959
|
+
location: RETRAIN_MEDIA_LOCATION
|
|
110960
|
+
} });
|
|
110961
|
+
} catch (err) {
|
|
110962
|
+
this.ctx.logger.warn("retrain copies reconcile failed — the copies stay on eventMedia", { meta: {
|
|
110963
|
+
error: errMsg(err),
|
|
110964
|
+
location: ARTIFACTS_MEDIA_LOCATION
|
|
110965
|
+
} });
|
|
110966
|
+
}
|
|
110330
110967
|
}
|
|
110331
110968
|
/**
|
|
110332
110969
|
* Move enrolled-gallery blobs off the footage volume, and say out loud how
|