@camstack/addon-pipeline 1.2.134 → 1.2.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-Cg_Sl2UR.js → addon-utils-CKf8kbwh.js} +1 -1
- package/dist/audio-analyzer/index.js +3 -3
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/detection-pipeline/index.js +5 -5
- package/dist/detection-pipeline/index.mjs +3 -3
- package/dist/{dist-BnwP1K09.js → dist-BzLiQ8Ue.js} +612 -25
- package/dist/{dist-Dsn9NGO3.mjs → dist-C1dXLLQq.mjs} +595 -26
- package/dist/{event-loop-stall-monitor-CRdmHBpl.mjs → event-loop-stall-monitor-CU-xxX5c.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-DRpAtCOy.js → event-loop-stall-monitor-DjV4Bow7.js} +1 -1
- package/dist/{lazy-sharp-Div0sK_l.js → lazy-sharp-DxWmhTqk.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +4 -4
- package/dist/pipeline-runner/index.mjs +3 -3
- package/dist/{process-memory-C3Q8YXd_.js → process-memory-CZK3D3kM.js} +1 -1
- package/dist/{process-memory-B-zF-wcr.mjs → process-memory-CaEwi5Cd.mjs} +1 -1
- package/dist/recorder/index.js +703 -358
- package/dist/recorder/index.mjs +702 -357
- package/dist/{segment-demux-js-BOIExWSt.js → segment-demux-js-BbpqygHL.js} +1 -1
- package/dist/{segment-demux-js-BIyVwpSp.mjs → segment-demux-js-njRO85gd.mjs} +1 -1
- package/dist/session-decode/decode-worker-child.js +2 -2
- package/dist/session-decode/decode-worker-child.mjs +1 -1
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CASELLGa.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CLtMmF1E.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DqplWP1i.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DKMXG57X.mjs +26 -0
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-BjhhoD3z.mjs → hostInit-CP7Xhd8J.mjs} +3 -3
- package/dist/stream-broker/index.js +105 -7
- package/dist/stream-broker/index.mjs +105 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-Bo2fIjZ2.js → worker-protocol-DvTZBvh2.js} +1 -1
- package/dist/{worker-protocol-CaQCRtsw.mjs → worker-protocol-DxsVx0G7.mjs} +1 -1
- package/package.json +8 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BF-ziHjH.mjs +0 -26
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-77rGxv1J.mjs +0 -26
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $t as EventCategory, At as DeviceFeature, B as defineCustomActions, Bt as selectAssignedProfileSlots, E as RecordingConfigSchema, J as evaluateSensorEdge, Jt as number, L as customAction, Lt as nodePin, N as batteryCapability, Nt as hydrateSchema, Ot as BaseAddon, Q as isFirstLevelMacroClass, St as errMsg, T as RECORDING_EXPORT_MAX_READ_BYTES, U as deriveRecordingMode, Ut as array, V as deriveBatteryPresence, Wt as boolean, Xt as record, Yt as object, Zt as string, _ as ExportRecordSchema, d as DEFAULT_EVENTS_BAND_BUFFER_SEC, dt as recordingCapability, ft as recordingExportCapability, gt as resolveRecordingProfiles, h as EVENT_PAD_MS, i as BatteryStatusSchema, jt as DeviceType, vt as storageEvictableCapability, x as OpsLogEntrySchema } from "../dist-C1dXLLQq.mjs";
|
|
2
2
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
3
3
|
import { r as withRecordingIntent } from "../restream-intent-B4BXZra7.mjs";
|
|
4
4
|
import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-CgBCF-mE.mjs";
|
|
@@ -236,81 +236,6 @@ var EventMap = class {
|
|
|
236
236
|
}
|
|
237
237
|
};
|
|
238
238
|
//#endregion
|
|
239
|
-
//#region src/recorder/redundant-segments.ts
|
|
240
|
-
/**
|
|
241
|
-
* Return the indices (into `segs`) of segments safe to delete: those fully
|
|
242
|
-
* contained within a kept segment `[coverStart, coverEnd]`. Sorting by start
|
|
243
|
-
* ascending, then by end DESCENDING, guarantees the containing (longer) segment
|
|
244
|
-
* is seen first and kept, and any shorter segment nested inside it is removed.
|
|
245
|
-
*/
|
|
246
|
-
function selectRedundantSegments(segs) {
|
|
247
|
-
if (segs.length < 2) return [];
|
|
248
|
-
const order = segs.map((s, i) => ({
|
|
249
|
-
start: s.startMs,
|
|
250
|
-
end: s.startMs + s.durMs,
|
|
251
|
-
i
|
|
252
|
-
})).toSorted((a, b) => a.start - b.start || b.end - a.end);
|
|
253
|
-
const remove = [];
|
|
254
|
-
let coverStart = Number.NEGATIVE_INFINITY;
|
|
255
|
-
let coverEnd = Number.NEGATIVE_INFINITY;
|
|
256
|
-
for (const seg of order) if (seg.start >= coverStart && seg.end <= coverEnd && seg.end > seg.start) remove.push(seg.i);
|
|
257
|
-
else {
|
|
258
|
-
coverStart = seg.start;
|
|
259
|
-
coverEnd = seg.end;
|
|
260
|
-
}
|
|
261
|
-
return remove;
|
|
262
|
-
}
|
|
263
|
-
//#endregion
|
|
264
|
-
//#region src/recorder/janitor.ts
|
|
265
|
-
/** Group an array by a string key. */
|
|
266
|
-
function groupBy(items, key) {
|
|
267
|
-
const out = /* @__PURE__ */ new Map();
|
|
268
|
-
for (const it of items) {
|
|
269
|
-
const k = key(it);
|
|
270
|
-
const arr = out.get(k);
|
|
271
|
-
if (arr) arr.push(it);
|
|
272
|
-
else out.set(k, [it]);
|
|
273
|
-
}
|
|
274
|
-
return out;
|
|
275
|
-
}
|
|
276
|
-
async function runRedundancyJanitor(deps) {
|
|
277
|
-
let files = 0;
|
|
278
|
-
let bytes = 0;
|
|
279
|
-
for (const deviceId of deps.deviceIds) {
|
|
280
|
-
const byProfile = groupBy(deps.segmentsFor(deviceId), (r) => r.profile);
|
|
281
|
-
for (const [profile, rows] of byProfile) {
|
|
282
|
-
const sorted = [...rows].toSorted((a, b) => a.startMs - b.startMs);
|
|
283
|
-
const redundant = selectRedundantSegments(sorted);
|
|
284
|
-
if (redundant.length === 0) continue;
|
|
285
|
-
const victims = redundant.map((i) => sorted[i]);
|
|
286
|
-
for (const [location, vrows] of groupBy(victims, (v) => v.locationId)) try {
|
|
287
|
-
const reclaimed = await deps.evict(location, vrows);
|
|
288
|
-
files += vrows.length;
|
|
289
|
-
bytes += reclaimed;
|
|
290
|
-
} catch (err) {
|
|
291
|
-
deps.logger.warn("janitor: evict failed (skipping)", {
|
|
292
|
-
tags: { deviceId },
|
|
293
|
-
meta: {
|
|
294
|
-
deviceId,
|
|
295
|
-
profile,
|
|
296
|
-
location,
|
|
297
|
-
count: vrows.length,
|
|
298
|
-
error: err instanceof Error ? err.message : String(err)
|
|
299
|
-
}
|
|
300
|
-
});
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
if (files > 0) deps.logger.info("janitor: removed redundant recording segments", { meta: {
|
|
305
|
-
files,
|
|
306
|
-
bytes
|
|
307
|
-
} });
|
|
308
|
-
return {
|
|
309
|
-
files,
|
|
310
|
-
bytes
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
//#endregion
|
|
314
239
|
//#region src/recorder/init-info.ts
|
|
315
240
|
/** Walk sibling boxes in [from, to), yielding [type, start, end]. */
|
|
316
241
|
function* boxes$1(buf, view, from, to) {
|
|
@@ -1290,69 +1215,6 @@ var RecordingIndex = class {
|
|
|
1290
1215
|
for (const m of this.byDevice.values()) for (const s of m.values()) if (s.locationId === locationId) out.push(s);
|
|
1291
1216
|
return out.toSorted((a, b) => a.startMs - b.startMs);
|
|
1292
1217
|
}
|
|
1293
|
-
/**
|
|
1294
|
-
* All segments across a SET of storage locations (one eviction DOMAIN — the
|
|
1295
|
-
* locations that alias one physical filesystem root), oldest-first across the
|
|
1296
|
-
* whole pool. This is the correct unit for byte-eviction: two location ids over
|
|
1297
|
-
* the same disk must be reclaimed as ONE oldest-first stream, never partitioned.
|
|
1298
|
-
*/
|
|
1299
|
-
segmentsOnLocations(locationIds) {
|
|
1300
|
-
const out = [];
|
|
1301
|
-
for (const m of this.byDevice.values()) for (const s of m.values()) if (locationIds.has(s.locationId)) out.push(s);
|
|
1302
|
-
return out.toSorted((a, b) => a.startMs - b.startMs);
|
|
1303
|
-
}
|
|
1304
|
-
/** Aggregate accounting across a SET of storage locations (one eviction domain). */
|
|
1305
|
-
accountingForLocations(locationIds) {
|
|
1306
|
-
let bytes = 0;
|
|
1307
|
-
let count = 0;
|
|
1308
|
-
let oldestMs = Infinity;
|
|
1309
|
-
let newestMs = -Infinity;
|
|
1310
|
-
for (const m of this.byDevice.values()) for (const s of m.values()) {
|
|
1311
|
-
if (!locationIds.has(s.locationId)) continue;
|
|
1312
|
-
bytes += s.bytes;
|
|
1313
|
-
count += 1;
|
|
1314
|
-
if (s.startMs < oldestMs) oldestMs = s.startMs;
|
|
1315
|
-
if (s.startMs > newestMs) newestMs = s.startMs;
|
|
1316
|
-
}
|
|
1317
|
-
if (count === 0) return {
|
|
1318
|
-
bytes: 0,
|
|
1319
|
-
count: 0,
|
|
1320
|
-
oldestMs: null,
|
|
1321
|
-
newestMs: null
|
|
1322
|
-
};
|
|
1323
|
-
return {
|
|
1324
|
-
bytes,
|
|
1325
|
-
count,
|
|
1326
|
-
oldestMs,
|
|
1327
|
-
newestMs
|
|
1328
|
-
};
|
|
1329
|
-
}
|
|
1330
|
-
/** Aggregate accounting for a storage location across every device. */
|
|
1331
|
-
accountingForLocation(locationId) {
|
|
1332
|
-
let bytes = 0;
|
|
1333
|
-
let count = 0;
|
|
1334
|
-
let oldestMs = Infinity;
|
|
1335
|
-
let newestMs = -Infinity;
|
|
1336
|
-
for (const m of this.byDevice.values()) for (const s of m.values()) {
|
|
1337
|
-
if (s.locationId !== locationId) continue;
|
|
1338
|
-
bytes += s.bytes;
|
|
1339
|
-
count += 1;
|
|
1340
|
-
if (s.startMs < oldestMs) oldestMs = s.startMs;
|
|
1341
|
-
if (s.startMs > newestMs) newestMs = s.startMs;
|
|
1342
|
-
}
|
|
1343
|
-
if (count === 0) return {
|
|
1344
|
-
bytes: 0,
|
|
1345
|
-
count: 0,
|
|
1346
|
-
oldestMs: null,
|
|
1347
|
-
newestMs: null
|
|
1348
|
-
};
|
|
1349
|
-
return {
|
|
1350
|
-
bytes,
|
|
1351
|
-
count,
|
|
1352
|
-
oldestMs,
|
|
1353
|
-
newestMs
|
|
1354
|
-
};
|
|
1355
|
-
}
|
|
1356
1218
|
/** Binary-search a start-sorted slice for the segment containing epochMs. */
|
|
1357
1219
|
segmentAtIn(segs, epochMs) {
|
|
1358
1220
|
if (segs.length === 0) return null;
|
|
@@ -2255,12 +2117,36 @@ var StorageEvictableProvider = class {
|
|
|
2255
2117
|
for (const id of domain) if (!this.deps.isReadOnly(id)) writable.add(id);
|
|
2256
2118
|
return writable;
|
|
2257
2119
|
}
|
|
2120
|
+
/**
|
|
2121
|
+
* How many bytes this domain holds — ONE aggregation, no rows returned.
|
|
2122
|
+
*
|
|
2123
|
+
* The storage-pressure sweep asks this per managed location every 60 s, and
|
|
2124
|
+
* two locations aliasing one root make it twice a minute. It used to be a
|
|
2125
|
+
* walk over every indexed segment on the node (7.1 M rows live), which is
|
|
2126
|
+
* also why the whole archive had to stay resident to be walked.
|
|
2127
|
+
*/
|
|
2258
2128
|
async getEvictableUsage(input) {
|
|
2259
|
-
|
|
2129
|
+
const archive = this.deps.archive();
|
|
2130
|
+
if (archive === null) {
|
|
2131
|
+
this.deps.logger.warn("recorder: evictable usage unavailable — no footage archive to measure, reporting 0 evictable bytes", { meta: { locationId: input.locationId } });
|
|
2132
|
+
return { bytes: 0 };
|
|
2133
|
+
}
|
|
2134
|
+
return { bytes: (await archive.accountingForLocations(this.domainOf(input.locationId))).bytes };
|
|
2260
2135
|
}
|
|
2261
2136
|
async evict(input) {
|
|
2262
|
-
const
|
|
2263
|
-
|
|
2137
|
+
const archive = this.deps.archive();
|
|
2138
|
+
if (archive === null) {
|
|
2139
|
+
this.deps.logger.warn("recorder: disk-pressure eviction skipped — no footage archive to pick the oldest from", { meta: {
|
|
2140
|
+
locationId: input.locationId,
|
|
2141
|
+
targetBytes: input.targetBytes
|
|
2142
|
+
} });
|
|
2143
|
+
return {
|
|
2144
|
+
reclaimedBytes: 0,
|
|
2145
|
+
exhausted: true
|
|
2146
|
+
};
|
|
2147
|
+
}
|
|
2148
|
+
const oldest = await archive.oldestSegments(this.domainOf(input.locationId), input.targetBytes);
|
|
2149
|
+
const victims = selectEvictionsByBytes(oldest.rows, input.targetBytes);
|
|
2264
2150
|
const byLocation = /* @__PURE__ */ new Map();
|
|
2265
2151
|
for (const v of victims) {
|
|
2266
2152
|
const group = byLocation.get(v.locationId);
|
|
@@ -2270,7 +2156,7 @@ var StorageEvictableProvider = class {
|
|
|
2270
2156
|
let reclaimedBytes = 0;
|
|
2271
2157
|
for (const [locationId, rows] of byLocation) reclaimedBytes += await this.deps.store.evict(locationId, rows);
|
|
2272
2158
|
if (reclaimedBytes > 0 && this.deps.onEvicted) this.deps.onEvicted(summarizeEvictionsByDevice(victims));
|
|
2273
|
-
const exhausted = victims.length >=
|
|
2159
|
+
const exhausted = oldest.exhausted && victims.length >= oldest.rows.length;
|
|
2274
2160
|
return {
|
|
2275
2161
|
reclaimedBytes,
|
|
2276
2162
|
exhausted
|
|
@@ -3067,6 +2953,7 @@ var CENSUS_MAX_WINDOW_MS = 10 * 6e4;
|
|
|
3067
2953
|
var CENSUSED_PROCEDURES = new Set([
|
|
3068
2954
|
"query",
|
|
3069
2955
|
"count",
|
|
2956
|
+
"aggregate",
|
|
3070
2957
|
"get",
|
|
3071
2958
|
"histogram",
|
|
3072
2959
|
"getAll"
|
|
@@ -5556,7 +5443,7 @@ function buildRecordingProvider(deps) {
|
|
|
5556
5443
|
locationId: first.id,
|
|
5557
5444
|
locationIds: ids,
|
|
5558
5445
|
root: first.root,
|
|
5559
|
-
usedBytes: deps.
|
|
5446
|
+
usedBytes: (await deps.archiveAccounting?.(new Set(ids)))?.bytes ?? 0,
|
|
5560
5447
|
availableBytes: cap.availableBytes,
|
|
5561
5448
|
totalBytes: cap.totalBytes
|
|
5562
5449
|
};
|
|
@@ -6336,6 +6223,7 @@ Object.freeze({
|
|
|
6336
6223
|
"listProfiles",
|
|
6337
6224
|
"listRuntimeNodes"
|
|
6338
6225
|
],
|
|
6226
|
+
"log-channels": ["list"],
|
|
6339
6227
|
"log-destination": ["query"],
|
|
6340
6228
|
"login-method": ["getLoginMethods"],
|
|
6341
6229
|
"mqtt-broker": ["listBrokers"],
|
|
@@ -7086,61 +6974,6 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
|
|
|
7086
6974
|
}
|
|
7087
6975
|
/** Entries classified between yields in {@link hydrateDeviceFromStorage}. */
|
|
7088
6976
|
var HYDRATE_CHUNK = 2e4;
|
|
7089
|
-
/**
|
|
7090
|
-
* Device walks in flight at once in {@link hydrateDevicesFromStorage}.
|
|
7091
|
-
*
|
|
7092
|
-
* The walk is `readdir` latency on a network/FUSE share, not CPU: measured on
|
|
7093
|
-
* the live hub, 24 cold hour directories cost 31 s one at a time and 16.7 s
|
|
7094
|
-
* eight at a time. Four is the conservative point on that curve — enough to
|
|
7095
|
-
* hide most of the latency, few enough that the boot walk never becomes the
|
|
7096
|
-
* reason a live SegmentWriter waits. libuv's default threadpool is 4, so a
|
|
7097
|
-
* higher number here mostly queues.
|
|
7098
|
-
*/
|
|
7099
|
-
var HYDRATE_DEVICE_CONCURRENCY = 4;
|
|
7100
|
-
/**
|
|
7101
|
-
* Walk the archive for a whole fleet: bounded-parallel, and each camera becomes
|
|
7102
|
-
* authoritative the moment ITS OWN walk lands (D167).
|
|
7103
|
-
*
|
|
7104
|
-
* Two things this fixes, both measured on the live hub on 2026-08-15:
|
|
7105
|
-
*
|
|
7106
|
-
* - The caller was `for (const id of ids) await hydrateDeviceFromStorage(...)`,
|
|
7107
|
-
* so seven cameras' 357 s of `readdir` WAS the 362 s wall clock. Nothing in
|
|
7108
|
-
* the walk contends for CPU — it is share latency — so overlapping is free.
|
|
7109
|
-
* - `markHydratedAll` ran in a SECOND loop after every device. A camera whose
|
|
7110
|
-
* walk finished at minute 1 still reported `hydrationOf → 'unknown'` until
|
|
7111
|
-
* minute 6, and every read of it in that window paid for an on-demand walk
|
|
7112
|
-
* over rows already in the index, competing with the walk still running for
|
|
7113
|
-
* the others.
|
|
7114
|
-
*
|
|
7115
|
-
* A device whose walk throws is NOT marked: `unknown` is the honest answer, and
|
|
7116
|
-
* it is what keeps the read path walking that window on demand.
|
|
7117
|
-
*/
|
|
7118
|
-
async function hydrateDevicesFromStorage(api, index, deviceIds, locations, logger, options = {}) {
|
|
7119
|
-
const concurrency = Math.max(1, options.concurrency ?? HYDRATE_DEVICE_CONCURRENCY);
|
|
7120
|
-
const queue = [...deviceIds];
|
|
7121
|
-
const walkOptions = {
|
|
7122
|
-
...options.yieldBetween ? { yieldBetween: options.yieldBetween } : {},
|
|
7123
|
-
...options.chunkSize !== void 0 ? { chunkSize: options.chunkSize } : {}
|
|
7124
|
-
};
|
|
7125
|
-
const lane = async () => {
|
|
7126
|
-
for (;;) {
|
|
7127
|
-
const deviceId = queue.shift();
|
|
7128
|
-
if (deviceId === void 0) return;
|
|
7129
|
-
try {
|
|
7130
|
-
options.onDeviceStarted?.(deviceId);
|
|
7131
|
-
await hydrateDeviceFromStorage(api, index, deviceId, locations, logger, walkOptions);
|
|
7132
|
-
index.markHydratedAll(deviceId);
|
|
7133
|
-
options.onDeviceHydrated?.(deviceId);
|
|
7134
|
-
} catch (err) {
|
|
7135
|
-
logger.warn("recorder: device hydrate failed — window stays unknown, reads will walk it", {
|
|
7136
|
-
tags: { deviceId },
|
|
7137
|
-
meta: { error: errMsg(err) }
|
|
7138
|
-
});
|
|
7139
|
-
}
|
|
7140
|
-
}
|
|
7141
|
-
};
|
|
7142
|
-
await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, lane));
|
|
7143
|
-
}
|
|
7144
6977
|
var macrotask$1 = () => new Promise((resolve) => {
|
|
7145
6978
|
setImmediate(resolve);
|
|
7146
6979
|
});
|
|
@@ -7711,7 +7544,7 @@ var PlacementService = class {
|
|
|
7711
7544
|
limits.push(capacity.availableBytes - floor);
|
|
7712
7545
|
}
|
|
7713
7546
|
if (location.maxUsedGb !== void 0) {
|
|
7714
|
-
const used = this.deps.usedOnLocation?.(location.id) ?? 0;
|
|
7547
|
+
const used = await this.deps.usedOnLocation?.(location.id) ?? 0;
|
|
7715
7548
|
limits.push(location.maxUsedGb * 1e9 - used);
|
|
7716
7549
|
}
|
|
7717
7550
|
out.push({
|
|
@@ -8899,6 +8732,38 @@ var RESTART_MAX_MS = 1e4;
|
|
|
8899
8732
|
/** A process that ran at least this long is "stable" → reset the restart count,
|
|
8900
8733
|
* so sporadic blips over a long lifetime never accumulate into a give-up. */
|
|
8901
8734
|
var STABLE_RUN_MS = 3e4;
|
|
8735
|
+
/**
|
|
8736
|
+
* The window the SECOND budget counts over — and it counts EVERY restart,
|
|
8737
|
+
* whatever that run lasted.
|
|
8738
|
+
*
|
|
8739
|
+
* `STABLE_RUN_MS` above is a reset, and a reset is a blind spot: measured on
|
|
8740
|
+
* 2026-08-27, `SegmentWriter restarting` fired **1 814 times in 12 h** and
|
|
8741
|
+
* almost every `ranMs` was over 30 s (122 620 / 165 255 / 345 420 / 394 522 ms
|
|
8742
|
+
* are four consecutive real samples), so `attempt` never left 1, `onGaveUp`
|
|
8743
|
+
* never fired, and `recoverWriterGaveUp` never ran. The storm could have lasted
|
|
8744
|
+
* forever without a single line saying anything was wrong.
|
|
8745
|
+
*
|
|
8746
|
+
* A window cannot be zeroed by a long run, which is the whole point. Modelled
|
|
8747
|
+
* on `CrashSupervisor` (kernel, D6): timestamps age out of `windowMs`, and the
|
|
8748
|
+
* count inside it is the budget.
|
|
8749
|
+
*/
|
|
8750
|
+
var RESTART_WINDOW_MS = 60 * 6e4;
|
|
8751
|
+
/**
|
|
8752
|
+
* Restarts inside {@link RESTART_WINDOW_MS} before the writer stops respawning
|
|
8753
|
+
* and escalates to the controller.
|
|
8754
|
+
*
|
|
8755
|
+
* 15/hour, from the measured distribution. Per writer over the same 12 h:
|
|
8756
|
+
* 1441/high 549 (**45.8/h**), 615/high 241 (20.1/h), 618/high 217 (18.1/h),
|
|
8757
|
+
* 592/high 211 (17.6/h), 590/high 195 (16.3/h) — every one of them a storm and
|
|
8758
|
+
* every one of them silent — against `low` profiles at 4–15 per 12 h (≤1.3/h),
|
|
8759
|
+
* which is what a healthy writer looks like. 15 separates them with room for a
|
|
8760
|
+
* camera that blips hourly and is otherwise fine.
|
|
8761
|
+
*
|
|
8762
|
+
* It never shadows {@link MAX_RESTARTS}: 10 CONSECUTIVE rapid failures happen
|
|
8763
|
+
* inside ~90 s, so the rapid breaker still trips first on a hard failure. This
|
|
8764
|
+
* one exists for the slow storm the rapid breaker structurally cannot see.
|
|
8765
|
+
*/
|
|
8766
|
+
var MAX_RESTARTS_IN_WINDOW = 15;
|
|
8902
8767
|
/** Grace between the SIGTERM in stop() and the escalated SIGKILL. */
|
|
8903
8768
|
var KILL_GRACE_MS = 500;
|
|
8904
8769
|
/** Supervises one passthrough ffmpeg for a single (camera, profile). */
|
|
@@ -8908,6 +8773,12 @@ var SegmentWriter = class {
|
|
|
8908
8773
|
proc = null;
|
|
8909
8774
|
stopped = false;
|
|
8910
8775
|
restarts = 0;
|
|
8776
|
+
/**
|
|
8777
|
+
* Epoch-ms of every termination still inside {@link RESTART_WINDOW_MS}.
|
|
8778
|
+
* Replaced, never mutated in place, so a reader can never observe a
|
|
8779
|
+
* half-pruned window.
|
|
8780
|
+
*/
|
|
8781
|
+
restartWindow = [];
|
|
8911
8782
|
restartTimer = null;
|
|
8912
8783
|
startedAt = 0;
|
|
8913
8784
|
exitWait = null;
|
|
@@ -8941,10 +8812,11 @@ var SegmentWriter = class {
|
|
|
8941
8812
|
settled = true;
|
|
8942
8813
|
this.onTermination(termination, stderrTail);
|
|
8943
8814
|
};
|
|
8944
|
-
proc.on("exit", (code) => {
|
|
8815
|
+
proc.on("exit", (code, signal) => {
|
|
8945
8816
|
terminate({
|
|
8946
8817
|
reason: "exit",
|
|
8947
|
-
exitCode: typeof code === "number" ? code : null
|
|
8818
|
+
exitCode: typeof code === "number" ? code : null,
|
|
8819
|
+
signal: typeof signal === "string" ? signal : null
|
|
8948
8820
|
});
|
|
8949
8821
|
});
|
|
8950
8822
|
proc.on("error", (raw) => {
|
|
@@ -8960,32 +8832,51 @@ var SegmentWriter = class {
|
|
|
8960
8832
|
/**
|
|
8961
8833
|
* The ONE restart policy, reached from both `exit` and `error`.
|
|
8962
8834
|
*
|
|
8963
|
-
* `
|
|
8964
|
-
*
|
|
8965
|
-
*
|
|
8966
|
-
* camera, and a writer that stops recording one camera must be greppable by
|
|
8967
|
-
* that camera.
|
|
8835
|
+
* Every line below carries `tags: { deviceId }` from `deps.deviceId` — "why
|
|
8836
|
+
* is 617 worse than 615" is always asked per camera, and a writer that stops
|
|
8837
|
+
* recording one camera must be greppable by that camera.
|
|
8968
8838
|
*/
|
|
8969
8839
|
onTermination(termination, stderrTail) {
|
|
8970
8840
|
this.resolveExit?.();
|
|
8971
8841
|
this.resolveExit = null;
|
|
8972
8842
|
this.proc = null;
|
|
8973
8843
|
if (this.stopped) return;
|
|
8974
|
-
const
|
|
8844
|
+
const now = Date.now();
|
|
8845
|
+
const ranMs = now - this.startedAt;
|
|
8975
8846
|
if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
|
|
8976
|
-
|
|
8977
|
-
|
|
8847
|
+
this.restartWindow = [...this.restartWindow.filter((at) => at > now - RESTART_WINDOW_MS), now];
|
|
8848
|
+
const restartsInWindow = this.restartWindow.length;
|
|
8849
|
+
if (termination.reason === "exit") this.deps.logger.warn("SegmentWriter ffmpeg exited", {
|
|
8850
|
+
tags: { deviceId: this.deps.deviceId },
|
|
8851
|
+
meta: {
|
|
8978
8852
|
outDir: this.cfg.outDir,
|
|
8979
|
-
|
|
8980
|
-
|
|
8981
|
-
|
|
8982
|
-
|
|
8983
|
-
|
|
8984
|
-
|
|
8853
|
+
exitCode: termination.exitCode,
|
|
8854
|
+
signal: termination.signal,
|
|
8855
|
+
ranMs,
|
|
8856
|
+
restartsInWindow,
|
|
8857
|
+
windowMs: RESTART_WINDOW_MS,
|
|
8858
|
+
stderrTail: stderrTail.join(" | ")
|
|
8859
|
+
}
|
|
8860
|
+
});
|
|
8861
|
+
if (termination.reason === "spawn-error") {
|
|
8862
|
+
this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", {
|
|
8863
|
+
tags: { deviceId: this.deps.deviceId },
|
|
8864
|
+
meta: {
|
|
8985
8865
|
outDir: this.cfg.outDir,
|
|
8986
8866
|
errorCode: termination.errorCode,
|
|
8987
|
-
error: termination.message
|
|
8988
|
-
|
|
8867
|
+
error: termination.message,
|
|
8868
|
+
permanent: termination.permanent
|
|
8869
|
+
}
|
|
8870
|
+
});
|
|
8871
|
+
if (termination.permanent) {
|
|
8872
|
+
this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", {
|
|
8873
|
+
tags: { deviceId: this.deps.deviceId },
|
|
8874
|
+
meta: {
|
|
8875
|
+
outDir: this.cfg.outDir,
|
|
8876
|
+
errorCode: termination.errorCode,
|
|
8877
|
+
error: termination.message
|
|
8878
|
+
}
|
|
8879
|
+
});
|
|
8989
8880
|
this.stopped = true;
|
|
8990
8881
|
this.deps.onGaveUp?.();
|
|
8991
8882
|
return;
|
|
@@ -8993,25 +8884,51 @@ var SegmentWriter = class {
|
|
|
8993
8884
|
this.deps.onResourcePressure?.();
|
|
8994
8885
|
}
|
|
8995
8886
|
if (this.restarts >= MAX_RESTARTS) {
|
|
8996
|
-
this.deps.logger.warn("SegmentWriter giving up after max restarts", {
|
|
8997
|
-
|
|
8998
|
-
|
|
8999
|
-
|
|
9000
|
-
|
|
9001
|
-
|
|
8887
|
+
this.deps.logger.warn("SegmentWriter giving up after max restarts", {
|
|
8888
|
+
tags: { deviceId: this.deps.deviceId },
|
|
8889
|
+
meta: {
|
|
8890
|
+
outDir: this.cfg.outDir,
|
|
8891
|
+
reason: termination.reason,
|
|
8892
|
+
code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
|
|
8893
|
+
stderrTail: stderrTail.join(" | ")
|
|
8894
|
+
}
|
|
8895
|
+
});
|
|
8896
|
+
this.stopped = true;
|
|
8897
|
+
this.deps.onGaveUp?.();
|
|
8898
|
+
return;
|
|
8899
|
+
}
|
|
8900
|
+
if (restartsInWindow >= MAX_RESTARTS_IN_WINDOW) {
|
|
8901
|
+
this.deps.logger.warn("SegmentWriter giving up: restart storm", {
|
|
8902
|
+
tags: { deviceId: this.deps.deviceId },
|
|
8903
|
+
meta: {
|
|
8904
|
+
outDir: this.cfg.outDir,
|
|
8905
|
+
restartsInWindow,
|
|
8906
|
+
windowMs: RESTART_WINDOW_MS,
|
|
8907
|
+
budget: MAX_RESTARTS_IN_WINDOW,
|
|
8908
|
+
attempt: this.restarts,
|
|
8909
|
+
ranMs,
|
|
8910
|
+
reason: termination.reason,
|
|
8911
|
+
code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
|
|
8912
|
+
stderrTail: stderrTail.join(" | ")
|
|
8913
|
+
}
|
|
8914
|
+
});
|
|
9002
8915
|
this.stopped = true;
|
|
9003
8916
|
this.deps.onGaveUp?.();
|
|
9004
8917
|
return;
|
|
9005
8918
|
}
|
|
9006
8919
|
this.restarts++;
|
|
9007
8920
|
const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
|
|
9008
|
-
this.deps.logger.info("SegmentWriter restarting", {
|
|
9009
|
-
|
|
9010
|
-
|
|
9011
|
-
|
|
9012
|
-
|
|
9013
|
-
|
|
9014
|
-
|
|
8921
|
+
this.deps.logger.info("SegmentWriter restarting", {
|
|
8922
|
+
tags: { deviceId: this.deps.deviceId },
|
|
8923
|
+
meta: {
|
|
8924
|
+
outDir: this.cfg.outDir,
|
|
8925
|
+
attempt: this.restarts,
|
|
8926
|
+
restartsInWindow,
|
|
8927
|
+
delayMs,
|
|
8928
|
+
ranMs,
|
|
8929
|
+
reason: termination.reason
|
|
8930
|
+
}
|
|
8931
|
+
});
|
|
9015
8932
|
this.restartTimer = setTimeout(() => {
|
|
9016
8933
|
this.restartTimer = null;
|
|
9017
8934
|
this.start();
|
|
@@ -10275,6 +10192,7 @@ var RecordingController = class {
|
|
|
10275
10192
|
}, {
|
|
10276
10193
|
spawn: this.deps.spawn,
|
|
10277
10194
|
logger: deviceLog,
|
|
10195
|
+
deviceId,
|
|
10278
10196
|
onGaveUp: () => {
|
|
10279
10197
|
this.recoverWriterGaveUp(deviceId, profile);
|
|
10280
10198
|
},
|
|
@@ -10698,6 +10616,209 @@ function buildRecordingExportProvider(deps) {
|
|
|
10698
10616
|
*/
|
|
10699
10617
|
var resolveRecordingHubHostname = resolveHubHostname;
|
|
10700
10618
|
//#endregion
|
|
10619
|
+
//#region src/recorder/redundant-segments.ts
|
|
10620
|
+
/**
|
|
10621
|
+
* Return the indices (into `segs`) of segments safe to delete: those fully
|
|
10622
|
+
* contained within a kept segment `[coverStart, coverEnd]`. Sorting by start
|
|
10623
|
+
* ascending, then by end DESCENDING, guarantees the containing (longer) segment
|
|
10624
|
+
* is seen first and kept, and any shorter segment nested inside it is removed.
|
|
10625
|
+
*/
|
|
10626
|
+
function selectRedundantSegments(segs) {
|
|
10627
|
+
if (segs.length < 2) return [];
|
|
10628
|
+
const order = segs.map((s, i) => ({
|
|
10629
|
+
start: s.startMs,
|
|
10630
|
+
end: s.startMs + s.durMs,
|
|
10631
|
+
i
|
|
10632
|
+
})).toSorted((a, b) => a.start - b.start || b.end - a.end);
|
|
10633
|
+
const remove = [];
|
|
10634
|
+
let coverStart = Number.NEGATIVE_INFINITY;
|
|
10635
|
+
let coverEnd = Number.NEGATIVE_INFINITY;
|
|
10636
|
+
for (const seg of order) if (seg.start >= coverStart && seg.end <= coverEnd && seg.end > seg.start) remove.push(seg.i);
|
|
10637
|
+
else {
|
|
10638
|
+
coverStart = seg.start;
|
|
10639
|
+
coverEnd = seg.end;
|
|
10640
|
+
}
|
|
10641
|
+
return remove;
|
|
10642
|
+
}
|
|
10643
|
+
//#endregion
|
|
10644
|
+
//#region src/recorder/addon/redundancy-sweep.ts
|
|
10645
|
+
/**
|
|
10646
|
+
* Periodic redundancy sweep — the only thing that removes duplicated footage.
|
|
10647
|
+
*
|
|
10648
|
+
* WHY IT EXISTS AGAIN
|
|
10649
|
+
* ───────────────────
|
|
10650
|
+
* Every `SegmentWriter` restart re-dials the broker with `withRecordingIntent`,
|
|
10651
|
+
* and the broker answers a RECORDING dial with its pre-roll ring (≈10 s). So a
|
|
10652
|
+
* restart does not resume at the live edge: it re-writes ~10 s of media that is
|
|
10653
|
+
* already on disk. At the 2026-08-27 measurement — **1 814 restarts in 12 h** —
|
|
10654
|
+
* that is ~5 hours of duplicated `high`-profile footage per day, on a fuse
|
|
10655
|
+
* share that was already at 90 % full.
|
|
10656
|
+
*
|
|
10657
|
+
* The pruner that used to remove it, `runRedundancyJanitor`, was reachable ONLY
|
|
10658
|
+
* from `runFullArchiveWalk`, which was reachable only from
|
|
10659
|
+
* `scheduleDeferredFullWalk`, which had no caller at all — so on 2026-08-27
|
|
10660
|
+
* (commit `9308428b4`) all three were deleted together as dead code. That was
|
|
10661
|
+
* correct about the walk and left the system with **no duplicate pruner
|
|
10662
|
+
* whatsoever**. The precedent for letting this class of waste run unattended is
|
|
10663
|
+
* the 179 GB of stranded staging segments in `staging-reconcile.ts`.
|
|
10664
|
+
*
|
|
10665
|
+
* WHAT IS DIFFERENT FROM THE ONE THAT WAS DELETED
|
|
10666
|
+
* ───────────────────────────────────────────────
|
|
10667
|
+
* The old janitor ran once, after a walk of the WHOLE archive, and asked the
|
|
10668
|
+
* in-RAM index for `segments(deviceId)` — a full copy plus a sort of every row
|
|
10669
|
+
* the device owns. At the live shape (7.1 M rows) that copy alone was measured
|
|
10670
|
+
* at ~560 MB of transient heap, which is precisely the cost D248 had just
|
|
10671
|
+
* finished removing from the pressure sweep. Reconnecting it in that shape
|
|
10672
|
+
* would have traded one regression for another.
|
|
10673
|
+
*
|
|
10674
|
+
* So this one is bounded by a LOOKBACK WINDOW, not by the archive: duplicates
|
|
10675
|
+
* are made by a restart, restarts are now, and `segmentsStartingIn` slices the
|
|
10676
|
+
* sorted view instead of copying it. Cost grows with footage RECORDED in the
|
|
10677
|
+
* window — the invariant the recorder is supposed to satisfy — never with rows
|
|
10678
|
+
* already indexed.
|
|
10679
|
+
*
|
|
10680
|
+
* WHAT IT WILL NOT DO
|
|
10681
|
+
* ───────────────────
|
|
10682
|
+
* It never deletes a segment that contributes unique coverage:
|
|
10683
|
+
* {@link selectRedundantSegments} selects only segments fully contained inside
|
|
10684
|
+
* another segment that is KEPT, and staggered partial overlaps are always kept.
|
|
10685
|
+
* The delete itself is `SegmentStore.evict` — the same audited path
|
|
10686
|
+
* disk-pressure eviction uses, so a removed segment is de-indexed exactly as an
|
|
10687
|
+
* evicted one is. A failure for one (device, profile, location) is logged and
|
|
10688
|
+
* never aborts the pass.
|
|
10689
|
+
*/
|
|
10690
|
+
/** The profiles a recorder writes. Same set `staging-reconcile` sweeps. */
|
|
10691
|
+
var SWEPT_PROFILES = [
|
|
10692
|
+
"high",
|
|
10693
|
+
"mid",
|
|
10694
|
+
"low"
|
|
10695
|
+
];
|
|
10696
|
+
/** Group rows by the location that owns them. `evict` is per-location. */
|
|
10697
|
+
function byLocation(rows) {
|
|
10698
|
+
const out = /* @__PURE__ */ new Map();
|
|
10699
|
+
for (const row of rows) {
|
|
10700
|
+
const found = out.get(row.locationId);
|
|
10701
|
+
if (found) found.push(row);
|
|
10702
|
+
else out.set(row.locationId, [row]);
|
|
10703
|
+
}
|
|
10704
|
+
return out;
|
|
10705
|
+
}
|
|
10706
|
+
/**
|
|
10707
|
+
* Owns its timer and a single-flight guard, the same shape the retention sweep
|
|
10708
|
+
* and the export janitor already use — a slow pass is never overlapped by the
|
|
10709
|
+
* next tick.
|
|
10710
|
+
*/
|
|
10711
|
+
var RedundancySweeper = class {
|
|
10712
|
+
deps;
|
|
10713
|
+
timer = null;
|
|
10714
|
+
sweeping = false;
|
|
10715
|
+
constructor(deps) {
|
|
10716
|
+
this.deps = deps;
|
|
10717
|
+
}
|
|
10718
|
+
/**
|
|
10719
|
+
* Arm the sweep on its interval. Idempotent.
|
|
10720
|
+
*
|
|
10721
|
+
* Deliberately NOT run immediately: boot is when the writers are attaching,
|
|
10722
|
+
* the staging reconcile is relocating thousands of files over shfs and the
|
|
10723
|
+
* first viewer is painting. A duplicate that has waited an hour can wait
|
|
10724
|
+
* another one; the write path cannot.
|
|
10725
|
+
*/
|
|
10726
|
+
start() {
|
|
10727
|
+
if (this.timer !== null) return;
|
|
10728
|
+
this.timer = setInterval(() => {
|
|
10729
|
+
this.sweep();
|
|
10730
|
+
}, this.deps.intervalMs);
|
|
10731
|
+
this.timer.unref?.();
|
|
10732
|
+
}
|
|
10733
|
+
/** Stop the timer. Idempotent. An in-flight pass still completes. */
|
|
10734
|
+
stop() {
|
|
10735
|
+
if (this.timer !== null) {
|
|
10736
|
+
clearInterval(this.timer);
|
|
10737
|
+
this.timer = null;
|
|
10738
|
+
}
|
|
10739
|
+
}
|
|
10740
|
+
/**
|
|
10741
|
+
* One pass. Never throws: a device-list read failure aborts THIS pass
|
|
10742
|
+
* (logged), a per-location evict failure is logged and the pass continues.
|
|
10743
|
+
*/
|
|
10744
|
+
async sweep() {
|
|
10745
|
+
const empty = {
|
|
10746
|
+
files: 0,
|
|
10747
|
+
bytes: 0,
|
|
10748
|
+
devices: 0
|
|
10749
|
+
};
|
|
10750
|
+
if (this.sweeping) {
|
|
10751
|
+
this.deps.logger.debug("recorder: redundancy sweep already running — skipping this tick");
|
|
10752
|
+
return empty;
|
|
10753
|
+
}
|
|
10754
|
+
this.sweeping = true;
|
|
10755
|
+
try {
|
|
10756
|
+
let deviceIds;
|
|
10757
|
+
try {
|
|
10758
|
+
deviceIds = await this.deps.deviceIds();
|
|
10759
|
+
} catch (err) {
|
|
10760
|
+
this.deps.logger.warn("recorder: redundancy sweep could not read the device list", { meta: { error: errMsg(err) } });
|
|
10761
|
+
return empty;
|
|
10762
|
+
}
|
|
10763
|
+
const toMs = (this.deps.now ?? Date.now)();
|
|
10764
|
+
const fromMs = toMs - this.deps.lookbackMs;
|
|
10765
|
+
let files = 0;
|
|
10766
|
+
let bytes = 0;
|
|
10767
|
+
let devices = 0;
|
|
10768
|
+
for (const deviceId of deviceIds) {
|
|
10769
|
+
let removedHere = 0;
|
|
10770
|
+
for (const profile of SWEPT_PROFILES) {
|
|
10771
|
+
const rows = this.deps.segmentsInWindow(deviceId, profile, fromMs, toMs);
|
|
10772
|
+
const redundant = selectRedundantSegments(rows);
|
|
10773
|
+
if (redundant.length === 0) continue;
|
|
10774
|
+
const victims = redundant.map((i) => rows[i]).filter((row) => row !== void 0);
|
|
10775
|
+
for (const [locationId, group] of byLocation(victims)) try {
|
|
10776
|
+
const reclaimed = await this.deps.evict(locationId, group);
|
|
10777
|
+
files += group.length;
|
|
10778
|
+
bytes += reclaimed;
|
|
10779
|
+
removedHere += group.length;
|
|
10780
|
+
} catch (err) {
|
|
10781
|
+
this.deps.logger.warn("recorder: redundancy sweep could not evict duplicates", {
|
|
10782
|
+
tags: { deviceId },
|
|
10783
|
+
meta: {
|
|
10784
|
+
profile,
|
|
10785
|
+
locationId,
|
|
10786
|
+
count: group.length,
|
|
10787
|
+
error: errMsg(err)
|
|
10788
|
+
}
|
|
10789
|
+
});
|
|
10790
|
+
}
|
|
10791
|
+
}
|
|
10792
|
+
if (removedHere > 0) {
|
|
10793
|
+
devices += 1;
|
|
10794
|
+
this.deps.logger.info("recorder: removed duplicated segments for device", {
|
|
10795
|
+
tags: { deviceId },
|
|
10796
|
+
meta: {
|
|
10797
|
+
files: removedHere,
|
|
10798
|
+
fromMs,
|
|
10799
|
+
toMs
|
|
10800
|
+
}
|
|
10801
|
+
});
|
|
10802
|
+
}
|
|
10803
|
+
}
|
|
10804
|
+
this.deps.logger.info("recorder: redundancy sweep complete", { meta: {
|
|
10805
|
+
devices,
|
|
10806
|
+
files,
|
|
10807
|
+
bytes,
|
|
10808
|
+
lookbackMs: this.deps.lookbackMs,
|
|
10809
|
+
deviceCount: deviceIds.length
|
|
10810
|
+
} });
|
|
10811
|
+
return {
|
|
10812
|
+
files,
|
|
10813
|
+
bytes,
|
|
10814
|
+
devices
|
|
10815
|
+
};
|
|
10816
|
+
} finally {
|
|
10817
|
+
this.sweeping = false;
|
|
10818
|
+
}
|
|
10819
|
+
}
|
|
10820
|
+
};
|
|
10821
|
+
//#endregion
|
|
10701
10822
|
//#region src/recorder/addon/retention-sweep.ts
|
|
10702
10823
|
/**
|
|
10703
10824
|
* Periodic footage-retention sweep (recording-spec §6).
|
|
@@ -11090,6 +11211,14 @@ var HOUR_MS$1 = 36e5;
|
|
|
11090
11211
|
* retention="a row goes when the walk or eviction confirms the hour is empty — never by age, never from the ledger alone (D148)"
|
|
11091
11212
|
*/
|
|
11092
11213
|
var RECORDING_SEGMENT_HOURS_COLLECTION = "recorder:segment-hours";
|
|
11214
|
+
/**
|
|
11215
|
+
* The four aggregate columns exist so "how much footage is on this disk" is a
|
|
11216
|
+
* `SUM`/`MIN`/`MAX` over ~19 k hour rows instead of a walk over ~7.1 M in-RAM
|
|
11217
|
+
* segment rows. They are DERIVED from `paths` — never a second authority — and
|
|
11218
|
+
* {@link hourAggregate} is the only place that derives them, so a row loaded
|
|
11219
|
+
* from a database that predates these columns is corrected in the mirror the
|
|
11220
|
+
* moment it is read and repaired on disk by {@link SegmentHourLedger.repairAggregates}.
|
|
11221
|
+
*/
|
|
11093
11222
|
var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
11094
11223
|
{
|
|
11095
11224
|
name: "key",
|
|
@@ -11121,15 +11250,85 @@ var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
|
11121
11250
|
name: "paths",
|
|
11122
11251
|
type: "TEXT",
|
|
11123
11252
|
notNull: true
|
|
11253
|
+
},
|
|
11254
|
+
{
|
|
11255
|
+
name: "bytes",
|
|
11256
|
+
type: "INTEGER",
|
|
11257
|
+
notNull: true,
|
|
11258
|
+
defaultValue: 0
|
|
11259
|
+
},
|
|
11260
|
+
{
|
|
11261
|
+
name: "segments",
|
|
11262
|
+
type: "INTEGER",
|
|
11263
|
+
notNull: true,
|
|
11264
|
+
defaultValue: 0
|
|
11265
|
+
},
|
|
11266
|
+
{
|
|
11267
|
+
name: "minStartMs",
|
|
11268
|
+
type: "INTEGER",
|
|
11269
|
+
notNull: true,
|
|
11270
|
+
defaultValue: 0
|
|
11271
|
+
},
|
|
11272
|
+
{
|
|
11273
|
+
name: "maxStartMs",
|
|
11274
|
+
type: "INTEGER",
|
|
11275
|
+
notNull: true,
|
|
11276
|
+
defaultValue: 0
|
|
11124
11277
|
}
|
|
11125
11278
|
];
|
|
11126
|
-
var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
11127
|
-
|
|
11128
|
-
|
|
11129
|
-
|
|
11130
|
-
|
|
11131
|
-
|
|
11132
|
-
|
|
11279
|
+
var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
11280
|
+
{
|
|
11281
|
+
name: "idx_recorder_segment_hours_device",
|
|
11282
|
+
columns: ["deviceId"]
|
|
11283
|
+
},
|
|
11284
|
+
{
|
|
11285
|
+
name: "idx_recorder_segment_hours_hour",
|
|
11286
|
+
columns: ["hourStartMs"]
|
|
11287
|
+
},
|
|
11288
|
+
(
|
|
11289
|
+
/** The eviction read: one oldest-first stream across a SET of location ids.
|
|
11290
|
+
* Location leads so `locationId IN (…)` is a range scan and `hourStartMs`
|
|
11291
|
+
* supplies the order without a sort. */
|
|
11292
|
+
{
|
|
11293
|
+
name: "idx_recorder_segment_hours_loc_hour",
|
|
11294
|
+
columns: ["locationId", "hourStartMs"]
|
|
11295
|
+
})
|
|
11296
|
+
];
|
|
11297
|
+
/**
|
|
11298
|
+
* Bytes / count / oldest / newest for one hour's paths.
|
|
11299
|
+
*
|
|
11300
|
+
* Every field of a segment path encodes its own facts (`parseSegmentPath`), so
|
|
11301
|
+
* this is a derivation, not a measurement: it can never disagree with the paths
|
|
11302
|
+
* it was computed from. A path the parser refuses contributes nothing — it is
|
|
11303
|
+
* not a segment, and counting it would inflate a disk report.
|
|
11304
|
+
*/
|
|
11305
|
+
function hourAggregate(paths) {
|
|
11306
|
+
let bytes = 0;
|
|
11307
|
+
let segments = 0;
|
|
11308
|
+
let minStartMs = 0;
|
|
11309
|
+
let maxStartMs = 0;
|
|
11310
|
+
for (const path of paths) {
|
|
11311
|
+
const p = parseSegmentPath(path);
|
|
11312
|
+
if (p === null) continue;
|
|
11313
|
+
bytes += p.bytes;
|
|
11314
|
+
if (segments === 0 || p.startMs < minStartMs) minStartMs = p.startMs;
|
|
11315
|
+
if (segments === 0 || p.startMs > maxStartMs) maxStartMs = p.startMs;
|
|
11316
|
+
segments += 1;
|
|
11317
|
+
}
|
|
11318
|
+
return {
|
|
11319
|
+
bytes,
|
|
11320
|
+
segments,
|
|
11321
|
+
minStartMs,
|
|
11322
|
+
maxStartMs
|
|
11323
|
+
};
|
|
11324
|
+
}
|
|
11325
|
+
/** Build the persisted row for an hour from its paths — the ONE constructor. */
|
|
11326
|
+
function segmentHourRow(base) {
|
|
11327
|
+
return {
|
|
11328
|
+
...base,
|
|
11329
|
+
...hourAggregate(base.paths)
|
|
11330
|
+
};
|
|
11331
|
+
}
|
|
11133
11332
|
/**
|
|
11134
11333
|
* The largest share of a device's held hours a single walk may delete.
|
|
11135
11334
|
*
|
|
@@ -11141,6 +11340,49 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [{
|
|
|
11141
11340
|
* `readdir` over an unmounted root deletes all of it.
|
|
11142
11341
|
*/
|
|
11143
11342
|
var LEDGER_PRUNE_MAX_SHARE = .5;
|
|
11343
|
+
/** Backstop on the oldest-first probe. A bounded read must stay bounded even if
|
|
11344
|
+
* a caller asks for a target no archive can cover. */
|
|
11345
|
+
var OLDEST_MAX_PAGES = 16;
|
|
11346
|
+
/** Phase-one columns: everything the accumulation needs and nothing fat. */
|
|
11347
|
+
var OLDEST_PROBE_COLUMNS = [
|
|
11348
|
+
"hourStartMs",
|
|
11349
|
+
"bytes",
|
|
11350
|
+
"locationId"
|
|
11351
|
+
];
|
|
11352
|
+
/** Aggregate-repair probe: the primary key always comes back, so this asks for
|
|
11353
|
+
* the predicate column alone. */
|
|
11354
|
+
var REPAIR_PROBE_COLUMNS = ["segments"];
|
|
11355
|
+
/** Rows repaired per boot. The live archive is ~19 k hour rows. */
|
|
11356
|
+
var REPAIR_MAX_ROWS = 5e4;
|
|
11357
|
+
/** The four scalars an accounting answer is made of, asked in one statement. */
|
|
11358
|
+
var ACCOUNTING_FIELDS = [
|
|
11359
|
+
{
|
|
11360
|
+
as: "bytes",
|
|
11361
|
+
field: "bytes",
|
|
11362
|
+
op: "sum"
|
|
11363
|
+
},
|
|
11364
|
+
{
|
|
11365
|
+
as: "segments",
|
|
11366
|
+
field: "segments",
|
|
11367
|
+
op: "sum"
|
|
11368
|
+
},
|
|
11369
|
+
{
|
|
11370
|
+
as: "oldestMs",
|
|
11371
|
+
field: "minStartMs",
|
|
11372
|
+
op: "min"
|
|
11373
|
+
},
|
|
11374
|
+
{
|
|
11375
|
+
as: "newestMs",
|
|
11376
|
+
field: "maxStartMs",
|
|
11377
|
+
op: "max"
|
|
11378
|
+
}
|
|
11379
|
+
];
|
|
11380
|
+
var EMPTY_ACCOUNTING = {
|
|
11381
|
+
bytes: 0,
|
|
11382
|
+
count: 0,
|
|
11383
|
+
oldestMs: null,
|
|
11384
|
+
newestMs: null
|
|
11385
|
+
};
|
|
11144
11386
|
function hourStartMs(startMs) {
|
|
11145
11387
|
return Math.floor(startMs / HOUR_MS$1) * HOUR_MS$1;
|
|
11146
11388
|
}
|
|
@@ -11176,7 +11418,11 @@ function rowToValue(row) {
|
|
|
11176
11418
|
profile: row.profile,
|
|
11177
11419
|
locationId: row.locationId,
|
|
11178
11420
|
hourStartMs: row.hourStartMs,
|
|
11179
|
-
paths: JSON.stringify(row.paths)
|
|
11421
|
+
paths: JSON.stringify(row.paths),
|
|
11422
|
+
bytes: row.bytes,
|
|
11423
|
+
segments: row.segments,
|
|
11424
|
+
minStartMs: row.minStartMs,
|
|
11425
|
+
maxStartMs: row.maxStartMs
|
|
11180
11426
|
};
|
|
11181
11427
|
}
|
|
11182
11428
|
function recordToRow(key, data) {
|
|
@@ -11191,14 +11437,14 @@ function recordToRow(key, data) {
|
|
|
11191
11437
|
return null;
|
|
11192
11438
|
}
|
|
11193
11439
|
if (!Number.isFinite(deviceId) || deviceId <= 0 || typeof profile !== "string" || profile.length === 0 || typeof locationId !== "string" || locationId.length === 0 || !Number.isFinite(hour) || !Array.isArray(raw) || !raw.every((p) => typeof p === "string" && p.length > 0)) return null;
|
|
11194
|
-
return {
|
|
11440
|
+
return segmentHourRow({
|
|
11195
11441
|
key,
|
|
11196
11442
|
deviceId,
|
|
11197
11443
|
profile,
|
|
11198
11444
|
locationId,
|
|
11199
11445
|
hourStartMs: hour,
|
|
11200
11446
|
paths: raw
|
|
11201
|
-
};
|
|
11447
|
+
});
|
|
11202
11448
|
}
|
|
11203
11449
|
var SPEC = {
|
|
11204
11450
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
@@ -11214,9 +11460,15 @@ var SPEC = {
|
|
|
11214
11460
|
var SegmentHourLedger = class {
|
|
11215
11461
|
ledger;
|
|
11216
11462
|
logger;
|
|
11463
|
+
/** Held for the two ARCHIVE reads below, which are queries rather than mirror
|
|
11464
|
+
* lookups. Every DECISION path (`recordSegment`, `dropSegments`, the prune
|
|
11465
|
+
* refusals) still answers from the mirror alone — D49 is about gates, and an
|
|
11466
|
+
* accounting number is not a gate. */
|
|
11467
|
+
store;
|
|
11217
11468
|
isLocationUsable;
|
|
11218
11469
|
constructor(deps) {
|
|
11219
11470
|
this.logger = deps.logger;
|
|
11471
|
+
this.store = deps.store;
|
|
11220
11472
|
this.isLocationUsable = deps.isLocationUsable ?? (() => true);
|
|
11221
11473
|
this.ledger = new DurableLedger({
|
|
11222
11474
|
spec: SPEC,
|
|
@@ -11251,19 +11503,178 @@ var SegmentHourLedger = class {
|
|
|
11251
11503
|
hydrateIndex(index) {
|
|
11252
11504
|
for (const hour of this.ledger.snapshot()) index.hydrateHour(hour.deviceId, hour.locationId, hour.hourStartMs, hour.paths);
|
|
11253
11505
|
}
|
|
11506
|
+
/**
|
|
11507
|
+
* Bytes / count / oldest / newest across a SET of storage locations, as ONE
|
|
11508
|
+
* `SUM`/`COUNT`/`MIN`/`MAX` over the hour rows.
|
|
11509
|
+
*
|
|
11510
|
+
* The set is the eviction DOMAIN — the location ids that alias one physical
|
|
11511
|
+
* root. It goes into a single statement as `locationId IN (…)`, never a loop
|
|
11512
|
+
* per location: two ids over one disk are one pool, and asking twice is how
|
|
11513
|
+
* an answer about a disk gets partitioned.
|
|
11514
|
+
*/
|
|
11515
|
+
async accountingForLocations(locationIds) {
|
|
11516
|
+
const ids = [...locationIds];
|
|
11517
|
+
if (ids.length === 0) return EMPTY_ACCOUNTING;
|
|
11518
|
+
const out = await this.store.aggregate.query({
|
|
11519
|
+
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
11520
|
+
fields: ACCOUNTING_FIELDS,
|
|
11521
|
+
filter: { whereIn: { locationId: ids } }
|
|
11522
|
+
});
|
|
11523
|
+
const segments = out.values["segments"];
|
|
11524
|
+
if (out.count === 0 || segments === null || segments === void 0) return EMPTY_ACCOUNTING;
|
|
11525
|
+
return {
|
|
11526
|
+
bytes: out.values["bytes"] ?? 0,
|
|
11527
|
+
count: segments,
|
|
11528
|
+
oldestMs: out.values["oldestMs"] ?? null,
|
|
11529
|
+
newestMs: out.values["newestMs"] ?? null
|
|
11530
|
+
};
|
|
11531
|
+
}
|
|
11532
|
+
/**
|
|
11533
|
+
* The OLDEST segments across a set of storage locations — enough of them to
|
|
11534
|
+
* cover `targetBytes`, and no more.
|
|
11535
|
+
*
|
|
11536
|
+
* ## The constraint this read exists to keep
|
|
11537
|
+
*
|
|
11538
|
+
* Locations aliasing one physical root must be reclaimed as ONE oldest-first
|
|
11539
|
+
* stream. That is not a property of the CALLER here, it is a property of the
|
|
11540
|
+
* STATEMENT: every id of the domain rides in a single
|
|
11541
|
+
* `locationId IN (…) ORDER BY hourStartMs ASC`. A query per location plus a
|
|
11542
|
+
* merge would be the age-partition data-loss bug rebuilt in a second place.
|
|
11543
|
+
*
|
|
11544
|
+
* ## Why two phases
|
|
11545
|
+
*
|
|
11546
|
+
* Phase one reads only the small columns and walks oldest-first until the
|
|
11547
|
+
* accumulated bytes cover the target — normally one page. `paths` is by far
|
|
11548
|
+
* the fattest column in the row and phase one never touches it. Phase two
|
|
11549
|
+
* fetches paths for exactly the hours phase one chose.
|
|
11550
|
+
*
|
|
11551
|
+
* `exhausted` means the stream ran out before the target was met — the
|
|
11552
|
+
* caller's stop condition, and the one thing a bounded read must still be
|
|
11553
|
+
* able to say truthfully.
|
|
11554
|
+
*/
|
|
11555
|
+
async oldestSegments(locationIds, targetBytes) {
|
|
11556
|
+
const ids = [...locationIds];
|
|
11557
|
+
if (ids.length === 0 || targetBytes <= 0) return {
|
|
11558
|
+
rows: [],
|
|
11559
|
+
exhausted: true
|
|
11560
|
+
};
|
|
11561
|
+
const chosen = [];
|
|
11562
|
+
let covered = 0;
|
|
11563
|
+
let offset = 0;
|
|
11564
|
+
let exhausted = false;
|
|
11565
|
+
for (let page = 0; page < OLDEST_MAX_PAGES && covered < targetBytes; page++) {
|
|
11566
|
+
const records = await this.store.query.query({
|
|
11567
|
+
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
11568
|
+
columns: OLDEST_PROBE_COLUMNS,
|
|
11569
|
+
filter: {
|
|
11570
|
+
whereIn: { locationId: ids },
|
|
11571
|
+
orderBy: {
|
|
11572
|
+
field: "hourStartMs",
|
|
11573
|
+
direction: "asc"
|
|
11574
|
+
},
|
|
11575
|
+
limit: 32,
|
|
11576
|
+
offset
|
|
11577
|
+
}
|
|
11578
|
+
});
|
|
11579
|
+
for (const record of records) {
|
|
11580
|
+
chosen.push(record.id);
|
|
11581
|
+
const bytes = record.data["bytes"];
|
|
11582
|
+
covered += typeof bytes === "number" ? bytes : 0;
|
|
11583
|
+
}
|
|
11584
|
+
offset += records.length;
|
|
11585
|
+
if (records.length < 32) {
|
|
11586
|
+
exhausted = true;
|
|
11587
|
+
break;
|
|
11588
|
+
}
|
|
11589
|
+
}
|
|
11590
|
+
if (chosen.length === 0) return {
|
|
11591
|
+
rows: [],
|
|
11592
|
+
exhausted: true
|
|
11593
|
+
};
|
|
11594
|
+
const full = await this.store.query.query({
|
|
11595
|
+
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
11596
|
+
filter: {
|
|
11597
|
+
whereIn: { key: chosen },
|
|
11598
|
+
limit: chosen.length
|
|
11599
|
+
}
|
|
11600
|
+
});
|
|
11601
|
+
const rows = [];
|
|
11602
|
+
for (const record of full) {
|
|
11603
|
+
const hour = recordToRow(record.id, record.data);
|
|
11604
|
+
if (hour === null) continue;
|
|
11605
|
+
for (const path of hour.paths) {
|
|
11606
|
+
const parsed = parseSegmentPath(path);
|
|
11607
|
+
if (parsed === null) continue;
|
|
11608
|
+
rows.push({
|
|
11609
|
+
deviceId: parsed.deviceId,
|
|
11610
|
+
profile: parsed.profile,
|
|
11611
|
+
startMs: parsed.startMs,
|
|
11612
|
+
durMs: parsed.durMs,
|
|
11613
|
+
bytes: parsed.bytes,
|
|
11614
|
+
path,
|
|
11615
|
+
locationId: hour.locationId
|
|
11616
|
+
});
|
|
11617
|
+
}
|
|
11618
|
+
}
|
|
11619
|
+
return {
|
|
11620
|
+
rows: rows.toSorted((a, b) => a.startMs - b.startMs),
|
|
11621
|
+
exhausted
|
|
11622
|
+
};
|
|
11623
|
+
}
|
|
11624
|
+
/**
|
|
11625
|
+
* Rewrite, once, the rows whose aggregate columns are still the schema
|
|
11626
|
+
* default.
|
|
11627
|
+
*
|
|
11628
|
+
* `declareCollection` adds a column additively, so every row written before
|
|
11629
|
+
* the aggregate columns existed holds `segments = 0` — and an hour row with
|
|
11630
|
+
* no segments cannot exist any other way (`dropSegments` forgets an hour that
|
|
11631
|
+
* empties). So `segments = 0` IS the backfill predicate, and after one pass
|
|
11632
|
+
* it selects nothing. A partial repair finishes on the next boot; until then
|
|
11633
|
+
* the affected disks UNDER-report, which prunes less and never more.
|
|
11634
|
+
*/
|
|
11635
|
+
async repairAggregates() {
|
|
11636
|
+
let repaired = 0;
|
|
11637
|
+
try {
|
|
11638
|
+
const stale = await this.store.query.query({
|
|
11639
|
+
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
11640
|
+
columns: REPAIR_PROBE_COLUMNS,
|
|
11641
|
+
filter: {
|
|
11642
|
+
where: { segments: 0 },
|
|
11643
|
+
limit: REPAIR_MAX_ROWS
|
|
11644
|
+
}
|
|
11645
|
+
});
|
|
11646
|
+
for (const record of stale) {
|
|
11647
|
+
const held = this.ledger.get(record.id);
|
|
11648
|
+
if (held === void 0 || held.segments === 0) continue;
|
|
11649
|
+
await this.ledger.put(held);
|
|
11650
|
+
repaired += 1;
|
|
11651
|
+
}
|
|
11652
|
+
} catch (err) {
|
|
11653
|
+
this.logger.warn("recorder: segment-hour aggregate repair failed — disks under-report", { meta: {
|
|
11654
|
+
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
11655
|
+
error: String(err)
|
|
11656
|
+
} });
|
|
11657
|
+
return repaired;
|
|
11658
|
+
}
|
|
11659
|
+
if (repaired > 0) this.logger.info("recorder: segment-hour aggregate columns backfilled", { meta: {
|
|
11660
|
+
rows: repaired,
|
|
11661
|
+
collection: RECORDING_SEGMENT_HOURS_COLLECTION
|
|
11662
|
+
} });
|
|
11663
|
+
return repaired;
|
|
11664
|
+
}
|
|
11254
11665
|
/** Write-behind: a just-finalized segment joins its hour row. */
|
|
11255
11666
|
async recordSegment(row) {
|
|
11256
11667
|
const { key, hourStartMs: hour } = hourOf(row);
|
|
11257
11668
|
const held = this.ledger.get(key);
|
|
11258
11669
|
if (held !== void 0 && held.paths.includes(row.path)) return;
|
|
11259
|
-
await this.ledger.put({
|
|
11670
|
+
await this.ledger.put(segmentHourRow({
|
|
11260
11671
|
key,
|
|
11261
11672
|
deviceId: row.deviceId,
|
|
11262
11673
|
profile: row.profile,
|
|
11263
11674
|
locationId: row.locationId,
|
|
11264
11675
|
hourStartMs: hour,
|
|
11265
11676
|
paths: held === void 0 ? [row.path] : [...held.paths, row.path]
|
|
11266
|
-
});
|
|
11677
|
+
}));
|
|
11267
11678
|
}
|
|
11268
11679
|
/**
|
|
11269
11680
|
* Eviction confirmed these paths are gone from disk. Drop them; forget the
|
|
@@ -11289,10 +11700,10 @@ var SegmentHourLedger = class {
|
|
|
11289
11700
|
continue;
|
|
11290
11701
|
}
|
|
11291
11702
|
if (next.length === held.paths.length) continue;
|
|
11292
|
-
await this.ledger.put({
|
|
11703
|
+
await this.ledger.put(segmentHourRow({
|
|
11293
11704
|
...held,
|
|
11294
11705
|
paths: next
|
|
11295
|
-
});
|
|
11706
|
+
}));
|
|
11296
11707
|
}
|
|
11297
11708
|
}
|
|
11298
11709
|
/**
|
|
@@ -11319,18 +11730,18 @@ var SegmentHourLedger = class {
|
|
|
11319
11730
|
for (const deviceId of deviceIds) for (const seg of index.segments(deviceId)) {
|
|
11320
11731
|
const { key, hourStartMs: hour } = hourOf(seg);
|
|
11321
11732
|
const existing = fromIndex.get(key);
|
|
11322
|
-
if (existing === void 0) fromIndex.set(key, {
|
|
11733
|
+
if (existing === void 0) fromIndex.set(key, segmentHourRow({
|
|
11323
11734
|
key,
|
|
11324
11735
|
deviceId: seg.deviceId,
|
|
11325
11736
|
profile: seg.profile,
|
|
11326
11737
|
locationId: seg.locationId,
|
|
11327
11738
|
hourStartMs: hour,
|
|
11328
11739
|
paths: [seg.path]
|
|
11329
|
-
});
|
|
11330
|
-
else fromIndex.set(key, {
|
|
11740
|
+
}));
|
|
11741
|
+
else fromIndex.set(key, segmentHourRow({
|
|
11331
11742
|
...existing,
|
|
11332
11743
|
paths: [...existing.paths, seg.path]
|
|
11333
|
-
});
|
|
11744
|
+
}));
|
|
11334
11745
|
}
|
|
11335
11746
|
const refused = this.refusedPrunes(walked, fromIndex);
|
|
11336
11747
|
for (const row of this.ledger.snapshot()) {
|
|
@@ -11344,10 +11755,10 @@ var SegmentHourLedger = class {
|
|
|
11344
11755
|
}
|
|
11345
11756
|
if (additiveOnly) {
|
|
11346
11757
|
const union = uniquePaths([...disk.paths, ...row.paths]);
|
|
11347
|
-
if (!samePaths(union, row.paths)) await this.ledger.put({
|
|
11758
|
+
if (!samePaths(union, row.paths)) await this.ledger.put(segmentHourRow({
|
|
11348
11759
|
...row,
|
|
11349
11760
|
paths: union
|
|
11350
|
-
});
|
|
11761
|
+
}));
|
|
11351
11762
|
continue;
|
|
11352
11763
|
}
|
|
11353
11764
|
if (!samePaths(row.paths, disk.paths)) await this.ledger.put(disk);
|
|
@@ -11968,22 +12379,26 @@ var EXPORT_SWEEP_INTERVAL_MS = 5 * 6e4;
|
|
|
11968
12379
|
* hourly cadence keeps the storage.list/evict cost negligible while honouring
|
|
11969
12380
|
* the recording-spec §6 periodic retention. */
|
|
11970
12381
|
var RETENTION_SWEEP_INTERVAL_MS = 60 * 6e4;
|
|
11971
|
-
/**
|
|
11972
|
-
|
|
11973
|
-
|
|
11974
|
-
*
|
|
11975
|
-
*
|
|
11976
|
-
*
|
|
11977
|
-
*
|
|
11978
|
-
*
|
|
11979
|
-
*
|
|
11980
|
-
*
|
|
11981
|
-
|
|
11982
|
-
|
|
11983
|
-
|
|
11984
|
-
*
|
|
11985
|
-
|
|
11986
|
-
|
|
12382
|
+
/** How often the redundancy sweep removes footage a writer restart duplicated. */
|
|
12383
|
+
var REDUNDANCY_SWEEP_INTERVAL_MS = 60 * 6e4;
|
|
12384
|
+
/**
|
|
12385
|
+
* How far back each redundancy pass looks.
|
|
12386
|
+
*
|
|
12387
|
+
* Twice the interval, so a pass that is skipped (single-flight) or delayed by a
|
|
12388
|
+
* slow predecessor still covers the window its predecessor was supposed to.
|
|
12389
|
+
* It is deliberately NOT "the whole archive": duplicates are made by a writer
|
|
12390
|
+
* restart, restarts are recent, and a full-archive selection is the ~560 MB
|
|
12391
|
+
* transient copy D248 removed. See `redundancy-sweep.ts`.
|
|
12392
|
+
*/
|
|
12393
|
+
var REDUNDANCY_SWEEP_LOOKBACK_MS = 2 * REDUNDANCY_SWEEP_INTERVAL_MS;
|
|
12394
|
+
/** What the per-volume usage row reports when the hour ledger is unavailable:
|
|
12395
|
+
* nothing measured, rather than a total counted off a partial RAM index. */
|
|
12396
|
+
var EMPTY_ARCHIVE_ACCOUNTING = {
|
|
12397
|
+
bytes: 0,
|
|
12398
|
+
count: 0,
|
|
12399
|
+
oldestMs: null,
|
|
12400
|
+
newestMs: null
|
|
12401
|
+
};
|
|
11987
12402
|
/** Grace after a completed download before a `deleteAfterDownload` file is removed. */
|
|
11988
12403
|
var EXPORT_DELETE_GRACE_MS = 6e4;
|
|
11989
12404
|
/** Only `.mp4` export ids matching this shape are servable (traversal guard). */
|
|
@@ -12122,17 +12537,15 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12122
12537
|
exportEngine = null;
|
|
12123
12538
|
/** Periodic export-expiry sweep timer. Cleared on shutdown. */
|
|
12124
12539
|
exportSweepTimer = null;
|
|
12125
|
-
/** Timer arming the deferred full-archive index walk. Null once fired. */
|
|
12126
|
-
fullWalkTimer = null;
|
|
12127
|
-
/** Single-flight guard for the full-archive walk (onHubReachable can re-enter
|
|
12128
|
-
* on a hub reconnect while a deferred walk is still running). */
|
|
12129
|
-
fullWalkInFlight = false;
|
|
12130
12540
|
/** The `recording` cap provider — retained so the periodic retention sweep can
|
|
12131
12541
|
* call its `pruneFootage`. Null until built in `onInitialize`. */
|
|
12132
12542
|
recordingProvider = null;
|
|
12133
12543
|
/** Periodic footage-retention sweep (owns its timer + single-flight guard).
|
|
12134
12544
|
* Constructed + started in `onHubReachable`, stopped on shutdown. */
|
|
12135
12545
|
retentionSweeper = null;
|
|
12546
|
+
/** Periodic duplicate-footage sweep — the ONLY pruner of media a writer
|
|
12547
|
+
* restart re-wrote. Constructed + started in `onHubReachable`. */
|
|
12548
|
+
redundancySweeper = null;
|
|
12136
12549
|
/** Reversible migration pause lease; never reflected in RecordingConfig. */
|
|
12137
12550
|
storageMigrationLeaseId = null;
|
|
12138
12551
|
/**
|
|
@@ -12293,7 +12706,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12293
12706
|
opsLog,
|
|
12294
12707
|
alert: (input) => this.emitPlacementAlert(input),
|
|
12295
12708
|
capacityFor: (location) => this.locationCapacity(location.root),
|
|
12296
|
-
usedOnLocation: (locationId) => this.
|
|
12709
|
+
usedOnLocation: async (locationId) => (await this.segmentHours?.accountingForLocations(new Set([locationId])))?.bytes ?? 0,
|
|
12297
12710
|
dailyBytesFor: (deviceId, profile) => this.dailyBytesFor(deviceId, profile)
|
|
12298
12711
|
});
|
|
12299
12712
|
this.controller = new RecordingController({
|
|
@@ -12405,6 +12818,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12405
12818
|
locations: () => this.resolvedLocations,
|
|
12406
12819
|
refreshLocations: () => this.refreshLocations(),
|
|
12407
12820
|
capacity: (root) => this.locationCapacity(root),
|
|
12821
|
+
archiveAccounting: (locationIds) => this.segmentHours?.accountingForLocations(locationIds) ?? Promise.resolve(EMPTY_ARCHIVE_ACCOUNTING),
|
|
12408
12822
|
hydrateDevice: (deviceId, locations) => hydrateDeviceFromStorage(this.ctx.api, this.index, deviceId, locations, this.ctx.logger),
|
|
12409
12823
|
reconcileHourLedger: async (deviceId) => {
|
|
12410
12824
|
await this.segmentHours?.reconcileFromIndex(this.index, [deviceId], Date.now());
|
|
@@ -12435,7 +12849,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12435
12849
|
const recordingProvider = buildRecordingProvider(recordingDeps);
|
|
12436
12850
|
this.recordingProvider = recordingProvider;
|
|
12437
12851
|
const evictableProvider = new StorageEvictableProvider({
|
|
12438
|
-
|
|
12852
|
+
archive: () => this.segmentHours,
|
|
12853
|
+
logger: this.ctx.logger,
|
|
12439
12854
|
store: this.segmentStore,
|
|
12440
12855
|
resolveDomain: (locationId) => this.locationDomain(locationId),
|
|
12441
12856
|
isReadOnly: (locationId) => this.resolvedLocations.find((l) => l.id === locationId)?.readOnly === true,
|
|
@@ -12619,10 +13034,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12619
13034
|
}
|
|
12620
13035
|
this.retentionSweeper?.stop();
|
|
12621
13036
|
this.retentionSweeper = null;
|
|
12622
|
-
|
|
12623
|
-
|
|
12624
|
-
this.fullWalkTimer = null;
|
|
12625
|
-
}
|
|
13037
|
+
this.redundancySweeper?.stop();
|
|
13038
|
+
this.redundancySweeper = null;
|
|
12626
13039
|
this.exportEngine = null;
|
|
12627
13040
|
this.recordingProvider = null;
|
|
12628
13041
|
}
|
|
@@ -12734,6 +13147,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12734
13147
|
try {
|
|
12735
13148
|
await this.segmentHours?.load();
|
|
12736
13149
|
this.segmentHours?.hydrateIndex(this.index);
|
|
13150
|
+
this.segmentHours?.repairAggregates();
|
|
12737
13151
|
} catch (err) {
|
|
12738
13152
|
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: errMsg(err) } });
|
|
12739
13153
|
}
|
|
@@ -12761,6 +13175,16 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
12761
13175
|
intervalMs: RETENTION_SWEEP_INTERVAL_MS
|
|
12762
13176
|
});
|
|
12763
13177
|
this.retentionSweeper?.start();
|
|
13178
|
+
const store = this.segmentStore;
|
|
13179
|
+
if (store && this.redundancySweeper === null) this.redundancySweeper = new RedundancySweeper({
|
|
13180
|
+
deviceIds: async () => [...(await readDeviceConfigs(this.configStore())).keys()],
|
|
13181
|
+
segmentsInWindow: (deviceId, profile, fromMs, toMs) => this.index.segmentsStartingIn(deviceId, profile, fromMs, toMs),
|
|
13182
|
+
evict: (locationId, rows) => store.evict(locationId, rows),
|
|
13183
|
+
logger: this.ctx.logger,
|
|
13184
|
+
intervalMs: REDUNDANCY_SWEEP_INTERVAL_MS,
|
|
13185
|
+
lookbackMs: REDUNDANCY_SWEEP_LOOKBACK_MS
|
|
13186
|
+
});
|
|
13187
|
+
this.redundancySweeper?.start();
|
|
12764
13188
|
}
|
|
12765
13189
|
/**
|
|
12766
13190
|
* Rebuild the sensor trigger's reverse index from the persisted configs.
|
|
@@ -13222,85 +13646,6 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
13222
13646
|
} });
|
|
13223
13647
|
}
|
|
13224
13648
|
/**
|
|
13225
|
-
* Opt-in fleet walk — NOT armed at boot (2026-08-22: starved scrub/play).
|
|
13226
|
-
* Kept so an operator path can still request a full reconcile later.
|
|
13227
|
-
*
|
|
13228
|
-
* ⚠ That operator path DOES NOT EXIST YET. This is the only caller of
|
|
13229
|
-
* `runFullArchiveWalk`, so today the whole walk is unreachable — `tsc` was
|
|
13230
|
-
* flagging it as unused and it was one of the errors keeping this package
|
|
13231
|
-
* red. Made public rather than deleted, because deleting it would have taken
|
|
13232
|
-
* the 100-line walk with it; but a comment promising a path nobody built is
|
|
13233
|
-
* exactly the leftover that reads as verification. Wire it or drop it.
|
|
13234
|
-
*/
|
|
13235
|
-
scheduleDeferredFullWalk(deviceIds) {
|
|
13236
|
-
if (this.fullWalkTimer !== null || this.fullWalkInFlight) return;
|
|
13237
|
-
this.fullWalkTimer = setTimeout(() => {
|
|
13238
|
-
this.fullWalkTimer = null;
|
|
13239
|
-
this.runFullArchiveWalk(deviceIds);
|
|
13240
|
-
}, FULL_WALK_DEFER_MS);
|
|
13241
|
-
this.fullWalkTimer.unref?.();
|
|
13242
|
-
this.ctx.logger.info("recorder: full-archive index walk deferred", { meta: {
|
|
13243
|
-
deferMs: FULL_WALK_DEFER_MS,
|
|
13244
|
-
deviceCount: deviceIds.length
|
|
13245
|
-
} });
|
|
13246
|
-
}
|
|
13247
|
-
/**
|
|
13248
|
-
* The deferred boot walk itself: hydrate every configured device from the
|
|
13249
|
-
* recordings roots, then run the two consumers that need the COMPLETE index
|
|
13250
|
-
* (the eviction-domain span log and the redundancy janitor). Bounded-parallel
|
|
13251
|
-
* per device, and each camera is marked hydrated at ITS OWN completion — not
|
|
13252
|
-
* after the fleet (D167). Never overlaps itself.
|
|
13253
|
-
*/
|
|
13254
|
-
async runFullArchiveWalk(deviceIds) {
|
|
13255
|
-
if (this.fullWalkInFlight) return;
|
|
13256
|
-
this.fullWalkInFlight = true;
|
|
13257
|
-
const hydrateStartedMs = Date.now();
|
|
13258
|
-
try {
|
|
13259
|
-
await hydrateDevicesFromStorage(this.ctx.api, this.index, deviceIds, this.resolvedLocations, this.ctx.logger);
|
|
13260
|
-
this.ctx.logger.info("recorder: hydrated index from storage", { meta: {
|
|
13261
|
-
deviceCount: deviceIds.length,
|
|
13262
|
-
locationCount: this.resolvedLocations.length,
|
|
13263
|
-
ms: Date.now() - hydrateStartedMs
|
|
13264
|
-
} });
|
|
13265
|
-
try {
|
|
13266
|
-
await this.segmentHours?.reconcileFromIndex(this.index, deviceIds, Date.now());
|
|
13267
|
-
} catch (err) {
|
|
13268
|
-
this.ctx.logger.warn("recorder: segment-hour ledger reconcile failed — walk remains the authority", { meta: { error: errMsg(err) } });
|
|
13269
|
-
}
|
|
13270
|
-
const loggedDomains = /* @__PURE__ */ new Set();
|
|
13271
|
-
for (const loc of this.resolvedLocations) {
|
|
13272
|
-
const domain = this.locationDomain(loc.id);
|
|
13273
|
-
const domainKey = [...domain].toSorted().join("+");
|
|
13274
|
-
if (loggedDomains.has(domainKey)) continue;
|
|
13275
|
-
loggedDomains.add(domainKey);
|
|
13276
|
-
const acct = this.index.accountingForLocations(domain);
|
|
13277
|
-
this.ctx.logger.info("recorder: eviction domain footage span at hydrate", { meta: {
|
|
13278
|
-
locationIds: [...domain].toSorted(),
|
|
13279
|
-
root: loc.root,
|
|
13280
|
-
count: acct.count,
|
|
13281
|
-
bytes: acct.bytes,
|
|
13282
|
-
oldestMs: acct.oldestMs,
|
|
13283
|
-
newestMs: acct.newestMs
|
|
13284
|
-
} });
|
|
13285
|
-
}
|
|
13286
|
-
const store = this.segmentStore;
|
|
13287
|
-
if (store) try {
|
|
13288
|
-
await runRedundancyJanitor({
|
|
13289
|
-
deviceIds,
|
|
13290
|
-
segmentsFor: (deviceId, profile) => this.index.segments(deviceId, profile),
|
|
13291
|
-
evict: (location, rows) => store.evict(location, rows),
|
|
13292
|
-
logger: this.ctx.logger
|
|
13293
|
-
});
|
|
13294
|
-
} catch (err) {
|
|
13295
|
-
this.ctx.logger.warn("recorder: redundancy janitor failed", { meta: { error: errMsg(err) } });
|
|
13296
|
-
}
|
|
13297
|
-
} catch (err) {
|
|
13298
|
-
this.ctx.logger.warn("recorder: full-archive walk failed", { meta: { error: errMsg(err) } });
|
|
13299
|
-
} finally {
|
|
13300
|
-
this.fullWalkInFlight = false;
|
|
13301
|
-
}
|
|
13302
|
-
}
|
|
13303
|
-
/**
|
|
13304
13649
|
* Declare the three row collections that replaced the `recordingConfigs`,
|
|
13305
13650
|
* `recordingExports` and `recordingOpsLog` blob keys. Idempotent — a
|
|
13306
13651
|
* re-declaration of the same shape is a no-op in the engine.
|