@camstack/addon-pipeline 1.2.256 → 1.2.258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/{default-detection-model-A5DRjEr3.mjs → default-detection-model-BZ54sUY2.mjs} +1 -1
- package/dist/{default-detection-model-Bbw4nC11.js → default-detection-model-vVWcdv0g.js} +1 -1
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +4 -4
- package/dist/{dist-lqqUXDfs.js → dist-CsJJPDQW.js} +180 -6
- package/dist/{dist-MJ4ji7um.mjs → dist-D3w2fwD4.mjs} +180 -6
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-80-uOMr8.js → node-1vbEWzPZ.js} +1 -1
- package/dist/{node-DdXmHLK4.mjs → node-IHGNrWH9.mjs} +1 -1
- package/dist/pipeline-runner/index.js +4 -4
- package/dist/pipeline-runner/index.mjs +4 -4
- package/dist/{process-memory-Dac9TXlX.js → process-memory-DynU4177.js} +1 -1
- package/dist/{process-memory-Cl3ubMxY.mjs → process-memory-jhOBJEil.mjs} +1 -1
- package/dist/recorder/index.js +1069 -1015
- package/dist/recorder/index.mjs +1069 -1015
- package/dist/{restream-intent-VVJiUAkh.js → restream-intent-B0GOuR-F.js} +90 -0
- package/dist/{restream-intent-D2VT97-U.mjs → restream-intent-CgyZI565.mjs} +85 -1
- package/dist/{segment-demux-js-gIj82O9k.mjs → segment-demux-js-BZAiu_2h.mjs} +1 -1
- package/dist/{segment-demux-js-CWwlKQ3M.js → segment-demux-js-OQT6wqXH.js} +1 -1
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-pOLAb3B1.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CFcO9OoY.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CTHexfai.mjs +26 -0
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BXK7jbJm.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BbnIk-9T.mjs} +1 -1
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-BRnsXyTM.mjs → hostInit-rU23F6nl.mjs} +3 -3
- package/dist/stream-broker/index.js +46 -7
- package/dist/stream-broker/index.mjs +46 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CPFjd94C.mjs +0 -26
package/dist/recorder/index.js
CHANGED
|
@@ -3,10 +3,10 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../chunk-emK7D4bc.js");
|
|
6
|
-
const require_dist = require("../dist-
|
|
7
|
-
const require_node = require("../node-
|
|
6
|
+
const require_dist = require("../dist-CsJJPDQW.js");
|
|
7
|
+
const require_node = require("../node-1vbEWzPZ.js");
|
|
8
8
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
9
|
-
const require_restream_intent = require("../restream-intent-
|
|
9
|
+
const require_restream_intent = require("../restream-intent-B0GOuR-F.js");
|
|
10
10
|
const require_retire_root_keys = require("../retire-root-keys-Csb8z2f3.js");
|
|
11
11
|
let node_crypto = require("node:crypto");
|
|
12
12
|
let node_fs = require("node:fs");
|
|
@@ -15,7 +15,6 @@ node_path = require_chunk.__toESM(node_path);
|
|
|
15
15
|
let node_child_process = require("node:child_process");
|
|
16
16
|
let node_fs_promises = require("node:fs/promises");
|
|
17
17
|
let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
|
|
18
|
-
let node_zlib = require("node:zlib");
|
|
19
18
|
//#region src/recorder/event-capture.ts
|
|
20
19
|
/**
|
|
21
20
|
* La sottoscrizione motion del recorder: bus → `onTrigger(deviceId, atMs)`.
|
|
@@ -821,14 +820,22 @@ function parseSegmentPath(relPath) {
|
|
|
821
820
|
/** Hour bucket, the granularity the path convention (`.../YYYY/MM/DD/HH/`)
|
|
822
821
|
* already gives us — so "have I walked this?" costs one Set lookup. */
|
|
823
822
|
var HOUR_MS$9 = 36e5;
|
|
824
|
-
/** A device walked end-to-end. Cheaper and clearer than inserting every hour of
|
|
825
|
-
* the retention window into the set. */
|
|
826
|
-
var ALL_HOURS = Symbol("all-hours");
|
|
827
823
|
/** The hour buckets a half-open span touches. */
|
|
828
824
|
function* hourBuckets(fromMs, toMs) {
|
|
829
825
|
const first = Math.floor(fromMs / HOUR_MS$9) * HOUR_MS$9;
|
|
830
826
|
for (let h = first; h < toMs; h += HOUR_MS$9) yield h;
|
|
831
827
|
}
|
|
828
|
+
/** The hour bucket a segment belongs to — the same one its path encodes. */
|
|
829
|
+
function hourOf$1(startMs) {
|
|
830
|
+
return Math.floor(startMs / HOUR_MS$9) * HOUR_MS$9;
|
|
831
|
+
}
|
|
832
|
+
/** The production bound. See {@link RecordingIndexBudget} for the measurement. */
|
|
833
|
+
var DEFAULT_RECORDING_INDEX_BUDGET = {
|
|
834
|
+
residentRowBudget: 25e4,
|
|
835
|
+
residentHourBudget: 2e4,
|
|
836
|
+
evictToFraction: .8,
|
|
837
|
+
pinnedTailHours: 2
|
|
838
|
+
};
|
|
832
839
|
/**
|
|
833
840
|
* Index of the first element whose `startMs` is GREATER than `startMs` (upper
|
|
834
841
|
* bound). Inserting there keeps equal-start rows in arrival order — exactly
|
|
@@ -876,10 +883,35 @@ var RecordingIndex = class {
|
|
|
876
883
|
/** See {@link HydrateReuseStats}. */
|
|
877
884
|
rowsReused = 0;
|
|
878
885
|
rowsAllocated = 0;
|
|
879
|
-
/**
|
|
880
|
-
*
|
|
881
|
-
*
|
|
886
|
+
/**
|
|
887
|
+
* Hour buckets this device has been walked for → when anything last ASKED
|
|
888
|
+
* about that hour ({@link useTick}, 0 = never since the walk). Empty =
|
|
889
|
+
* nothing has been looked at, which is NOT the same as "there is nothing" —
|
|
890
|
+
* see {@link hydrationOf}.
|
|
891
|
+
*
|
|
892
|
+
* The tick is the eviction order, and it lives here rather than in a side
|
|
893
|
+
* map because an evicted hour must lose its marker in the SAME operation
|
|
894
|
+
* that loses its rows: a walked marker outliving its rows would report
|
|
895
|
+
* `hydrated` over an hour the index no longer holds, which is the one
|
|
896
|
+
* failure a forgetting index must not have.
|
|
897
|
+
*
|
|
898
|
+
* There used to be an `ALL_HOURS` sentinel here for a device walked
|
|
899
|
+
* end-to-end. It had no production caller (2026-08-30 census) and a bounded
|
|
900
|
+
* index can never truthfully claim it, so it is gone.
|
|
901
|
+
*/
|
|
882
902
|
walked = /* @__PURE__ */ new Map();
|
|
903
|
+
/** Monotonic "somebody asked" counter. Only ever compared, never read out. */
|
|
904
|
+
useTick = 0;
|
|
905
|
+
/** `null` until {@link armResidency}. An unarmed index never evicts. */
|
|
906
|
+
residencyDeps = null;
|
|
907
|
+
evictedRows = 0;
|
|
908
|
+
evictedHours = 0;
|
|
909
|
+
evictionPasses = 0;
|
|
910
|
+
/** Hours dropped per device, for the life of the process. One number per
|
|
911
|
+
* camera — the input to {@link droppedDuringWalk}, nothing else. */
|
|
912
|
+
evictedHoursByDevice = /* @__PURE__ */ new Map();
|
|
913
|
+
/** {@link evictedHoursByDevice} as it stood when a whole-device walk began. */
|
|
914
|
+
walkBaseline = /* @__PURE__ */ new Map();
|
|
883
915
|
/**
|
|
884
916
|
* Upper bound on the `durMs` of any row the index holds — the ONLY thing that
|
|
885
917
|
* turns "walk the whole sorted view" into "walk backwards a bounded number of
|
|
@@ -1053,7 +1085,9 @@ var RecordingIndex = class {
|
|
|
1053
1085
|
* Deliberately NOT delete-all-then-reinsert: the archive walk re-reports rows
|
|
1054
1086
|
* this index already holds, with freshly built path strings, and dropping the
|
|
1055
1087
|
* held rows first would make those fresh strings the permanent ones — a
|
|
1056
|
-
* second copy of the whole archive's paths. See {@link
|
|
1088
|
+
* second copy of the whole archive's paths. See {@link RecordingIndex.rowFor}
|
|
1089
|
+
* for the measurement (`retainedPathStrings`, which this used to point at,
|
|
1090
|
+
* never existed — a leftover that read as a verification, removed 2026-09-17).
|
|
1057
1091
|
* @residency mutation
|
|
1058
1092
|
*/
|
|
1059
1093
|
hydrateDevice(deviceId, locationId, relPaths) {
|
|
@@ -1067,6 +1101,7 @@ var RecordingIndex = class {
|
|
|
1067
1101
|
}
|
|
1068
1102
|
for (const [path, existing] of m) if (existing.locationId === locationId && !seen.has(path)) m.delete(path);
|
|
1069
1103
|
this.invalidateSorted(deviceId);
|
|
1104
|
+
this.enforceResidency();
|
|
1070
1105
|
}
|
|
1071
1106
|
/**
|
|
1072
1107
|
* Whether the index has WALKED `[fromMs, toMs)` for a device.
|
|
@@ -1080,7 +1115,6 @@ var RecordingIndex = class {
|
|
|
1080
1115
|
hydrationOf(deviceId, fromMs, toMs) {
|
|
1081
1116
|
const w = this.walked.get(deviceId);
|
|
1082
1117
|
if (w === void 0) return "unknown";
|
|
1083
|
-
if (w === ALL_HOURS) return "hydrated";
|
|
1084
1118
|
let seen = 0;
|
|
1085
1119
|
let total = 0;
|
|
1086
1120
|
for (const h of hourBuckets(fromMs, toMs)) {
|
|
@@ -1092,15 +1126,225 @@ var RecordingIndex = class {
|
|
|
1092
1126
|
}
|
|
1093
1127
|
/** Record that `[fromMs, toMs)` has been walked, whatever it contained. @residency mutation */
|
|
1094
1128
|
markHydrated(deviceId, fromMs, toMs) {
|
|
1129
|
+
let w = this.walked.get(deviceId);
|
|
1130
|
+
if (w === void 0) {
|
|
1131
|
+
w = /* @__PURE__ */ new Map();
|
|
1132
|
+
this.walked.set(deviceId, w);
|
|
1133
|
+
}
|
|
1134
|
+
const tick = ++this.useTick;
|
|
1135
|
+
for (const h of hourBuckets(fromMs, toMs)) w.set(h, tick);
|
|
1136
|
+
}
|
|
1137
|
+
/**
|
|
1138
|
+
* Bound this index, and give it somewhere to SAY what it dropped.
|
|
1139
|
+
*
|
|
1140
|
+
* Called once, from the recorder addon's initialize — `ctx.logger` does not
|
|
1141
|
+
* exist when the field is constructed. Until it runs the index never evicts,
|
|
1142
|
+
* which is honest for the pre-init window (it holds nothing) and is the one
|
|
1143
|
+
* state in which {@link ResidencyCensus.budget} is `null`.
|
|
1144
|
+
* @residency mutation
|
|
1145
|
+
*/
|
|
1146
|
+
armResidency(deps) {
|
|
1147
|
+
this.residencyDeps = deps;
|
|
1148
|
+
this.enforceResidency();
|
|
1149
|
+
}
|
|
1150
|
+
/**
|
|
1151
|
+
* A whole-device walk is starting: remember how much this index had already
|
|
1152
|
+
* forgotten about that camera, so {@link droppedDuringWalk} can say whether
|
|
1153
|
+
* the walk's result is a COMPLETE view of what it read.
|
|
1154
|
+
* @residency mutation
|
|
1155
|
+
*/
|
|
1156
|
+
beginWalk(deviceId) {
|
|
1157
|
+
this.walkBaseline.set(deviceId, this.evictedHoursByDevice.get(deviceId) ?? 0);
|
|
1158
|
+
}
|
|
1159
|
+
/**
|
|
1160
|
+
* Did this index drop anything for `deviceId` since {@link beginWalk}?
|
|
1161
|
+
*
|
|
1162
|
+
* The one question a PRUNE has to ask. An hour the index dropped is
|
|
1163
|
+
* indistinguishable, from the outside, from an hour disk no longer has — so
|
|
1164
|
+
* a consumer that deletes durable state on the index's silence must refuse
|
|
1165
|
+
* when the answer is `true`. `false` when nobody called `beginWalk`: an
|
|
1166
|
+
* unarmed index never drops, and a caller that never announced a walk is
|
|
1167
|
+
* asking about one that did not happen.
|
|
1168
|
+
* @residency resident-scoped
|
|
1169
|
+
*/
|
|
1170
|
+
droppedDuringWalk(deviceId) {
|
|
1171
|
+
const baseline = this.walkBaseline.get(deviceId);
|
|
1172
|
+
if (baseline === void 0) return false;
|
|
1173
|
+
return (this.evictedHoursByDevice.get(deviceId) ?? 0) > baseline;
|
|
1174
|
+
}
|
|
1175
|
+
/** What is held, what was let go, and under which bound. @residency resident-scoped */
|
|
1176
|
+
residency() {
|
|
1177
|
+
return {
|
|
1178
|
+
residentRows: this.residentRowCount(),
|
|
1179
|
+
residentHours: this.residentHourCount(),
|
|
1180
|
+
evictedRows: this.evictedRows,
|
|
1181
|
+
evictedHours: this.evictedHours,
|
|
1182
|
+
evictionPasses: this.evictionPasses,
|
|
1183
|
+
budget: this.residencyDeps?.budget ?? null
|
|
1184
|
+
};
|
|
1185
|
+
}
|
|
1186
|
+
residentRowCount() {
|
|
1187
|
+
let n = 0;
|
|
1188
|
+
for (const m of this.byDevice.values()) n += m.size;
|
|
1189
|
+
return n;
|
|
1190
|
+
}
|
|
1191
|
+
residentHourCount() {
|
|
1192
|
+
let n = 0;
|
|
1193
|
+
for (const w of this.walked.values()) n += w.size;
|
|
1194
|
+
return n;
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Note that something ASKED about `[fromMs, toMs)` — the only input to the
|
|
1198
|
+
* eviction order.
|
|
1199
|
+
*
|
|
1200
|
+
* Walks whichever side is smaller: a read whose span is wider than the
|
|
1201
|
+
* device's whole walked map (a month-wide `ranges`, an availability sweep)
|
|
1202
|
+
* would otherwise iterate one hour at a time across the span.
|
|
1203
|
+
*/
|
|
1204
|
+
touch(deviceId, fromMs, toMs) {
|
|
1095
1205
|
const w = this.walked.get(deviceId);
|
|
1096
|
-
if (w ===
|
|
1097
|
-
const
|
|
1098
|
-
|
|
1099
|
-
|
|
1206
|
+
if (w === void 0 || w.size === 0) return;
|
|
1207
|
+
const tick = ++this.useTick;
|
|
1208
|
+
if (Math.ceil(Math.max(0, toMs - fromMs) / HOUR_MS$9) > w.size) {
|
|
1209
|
+
for (const h of w.keys()) if (h < toMs && h + HOUR_MS$9 > fromMs) w.set(h, tick);
|
|
1210
|
+
return;
|
|
1211
|
+
}
|
|
1212
|
+
for (const h of hourBuckets(fromMs, toMs)) if (w.has(h)) w.set(h, tick);
|
|
1100
1213
|
}
|
|
1101
|
-
/**
|
|
1102
|
-
|
|
1103
|
-
|
|
1214
|
+
/**
|
|
1215
|
+
* Drop the least-recently-asked-about hours until the index is back inside
|
|
1216
|
+
* its budget.
|
|
1217
|
+
*
|
|
1218
|
+
* ## What a dropped hour costs, and why that is safe
|
|
1219
|
+
*
|
|
1220
|
+
* The directory is the authority for which segments exist — every row here
|
|
1221
|
+
* is derived from a path the walk read, and `hydrateHour` /
|
|
1222
|
+
* `hydrateWindowFromStorage` rebuild it from one shallow `readdir` (or one
|
|
1223
|
+
* ledger point read). So a drop costs a re-read, never a segment. The
|
|
1224
|
+
* property that makes it costs-a-read rather than loses-a-segment is that
|
|
1225
|
+
* the walked marker goes with the rows: `hydrationOf` then answers `unknown`
|
|
1226
|
+
* and the reader walks. A marker left behind would answer `hydrated` over an
|
|
1227
|
+
* hour holding nothing, which is exactly the silent gap this class spent
|
|
1228
|
+
* D295 learning not to report.
|
|
1229
|
+
*
|
|
1230
|
+
* ## What is never a candidate
|
|
1231
|
+
*
|
|
1232
|
+
* The live tail ({@link RecordingIndexBudget.pinnedTailHours}). Those hours
|
|
1233
|
+
* are write-through from the writer (`addSegment`), not walked, and a
|
|
1234
|
+
* `readdir` of an hour a writer is still filling cannot give back what it
|
|
1235
|
+
* had — D445's lost tail, from the other direction.
|
|
1236
|
+
*/
|
|
1237
|
+
enforceResidency() {
|
|
1238
|
+
const deps = this.residencyDeps;
|
|
1239
|
+
if (deps === null) return;
|
|
1240
|
+
const { budget } = deps;
|
|
1241
|
+
let rows = this.residentRowCount();
|
|
1242
|
+
let hours = this.residentHourCount();
|
|
1243
|
+
if (rows <= budget.residentRowBudget && hours <= budget.residentHourBudget) return;
|
|
1244
|
+
const rowTarget = Math.floor(budget.residentRowBudget * budget.evictToFraction);
|
|
1245
|
+
const hourTarget = Math.floor(budget.residentHourBudget * budget.evictToFraction);
|
|
1246
|
+
const pinnedFrom = hourOf$1(deps.now()) - Math.max(0, budget.pinnedTailHours - 1) * HOUR_MS$9;
|
|
1247
|
+
const counts = this.hourHistogram();
|
|
1248
|
+
const candidates = [];
|
|
1249
|
+
for (const [deviceId, byHour] of counts) {
|
|
1250
|
+
const walkedHours = this.walked.get(deviceId);
|
|
1251
|
+
for (const [hourStartMs, count] of byHour) {
|
|
1252
|
+
if (hourStartMs >= pinnedFrom) continue;
|
|
1253
|
+
candidates.push({
|
|
1254
|
+
deviceId,
|
|
1255
|
+
hourStartMs,
|
|
1256
|
+
rows: count,
|
|
1257
|
+
lastUse: walkedHours?.get(hourStartMs) ?? 0
|
|
1258
|
+
});
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
candidates.sort((a, b) => a.lastUse - b.lastUse || a.hourStartMs - b.hourStartMs);
|
|
1262
|
+
const victims = /* @__PURE__ */ new Map();
|
|
1263
|
+
let droppedRows = 0;
|
|
1264
|
+
let droppedHours = 0;
|
|
1265
|
+
for (const c of candidates) {
|
|
1266
|
+
if (rows <= rowTarget && hours <= hourTarget) break;
|
|
1267
|
+
let forDevice = victims.get(c.deviceId);
|
|
1268
|
+
if (!forDevice) {
|
|
1269
|
+
forDevice = /* @__PURE__ */ new Set();
|
|
1270
|
+
victims.set(c.deviceId, forDevice);
|
|
1271
|
+
}
|
|
1272
|
+
forDevice.add(c.hourStartMs);
|
|
1273
|
+
rows -= c.rows;
|
|
1274
|
+
droppedRows += c.rows;
|
|
1275
|
+
if (this.walked.get(c.deviceId)?.has(c.hourStartMs) === true) {
|
|
1276
|
+
hours -= 1;
|
|
1277
|
+
droppedHours += 1;
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
if (victims.size === 0) return;
|
|
1281
|
+
this.evictionPasses += 1;
|
|
1282
|
+
this.evictedRows += droppedRows;
|
|
1283
|
+
this.evictedHours += droppedHours;
|
|
1284
|
+
for (const [deviceId, hoursToDrop] of victims) this.dropHours(deviceId, hoursToDrop, rows, budget, deps.logger);
|
|
1285
|
+
}
|
|
1286
|
+
/** `deviceId → hourStartMs → resident rows`, including hours walked and found
|
|
1287
|
+
* empty (count 0) — those hold no rows and would otherwise never be dropped,
|
|
1288
|
+
* leaving the walked map to grow without bound on its own. */
|
|
1289
|
+
hourHistogram() {
|
|
1290
|
+
const out = /* @__PURE__ */ new Map();
|
|
1291
|
+
for (const [deviceId, m] of this.byDevice) {
|
|
1292
|
+
const byHour = /* @__PURE__ */ new Map();
|
|
1293
|
+
for (const s of m.values()) {
|
|
1294
|
+
const h = hourOf$1(s.startMs);
|
|
1295
|
+
byHour.set(h, (byHour.get(h) ?? 0) + 1);
|
|
1296
|
+
}
|
|
1297
|
+
out.set(deviceId, byHour);
|
|
1298
|
+
}
|
|
1299
|
+
for (const [deviceId, w] of this.walked) {
|
|
1300
|
+
let byHour = out.get(deviceId);
|
|
1301
|
+
if (!byHour) {
|
|
1302
|
+
byHour = /* @__PURE__ */ new Map();
|
|
1303
|
+
out.set(deviceId, byHour);
|
|
1304
|
+
}
|
|
1305
|
+
for (const h of w.keys()) if (!byHour.has(h)) byHour.set(h, 0);
|
|
1306
|
+
}
|
|
1307
|
+
return out;
|
|
1308
|
+
}
|
|
1309
|
+
/** Delete one device's victim hours — rows and walked markers together — and
|
|
1310
|
+
* say so. A structure that silently forgets is how this class of bug hides. */
|
|
1311
|
+
dropHours(deviceId, hoursToDrop, residentRowsAfter, budget, logger) {
|
|
1312
|
+
const m = this.byDevice.get(deviceId);
|
|
1313
|
+
let droppedRows = 0;
|
|
1314
|
+
if (m) {
|
|
1315
|
+
for (const [path, s] of m) {
|
|
1316
|
+
if (!hoursToDrop.has(hourOf$1(s.startMs))) continue;
|
|
1317
|
+
m.delete(path);
|
|
1318
|
+
droppedRows += 1;
|
|
1319
|
+
}
|
|
1320
|
+
if (m.size === 0) this.byDevice.delete(deviceId);
|
|
1321
|
+
}
|
|
1322
|
+
const w = this.walked.get(deviceId);
|
|
1323
|
+
if (w) {
|
|
1324
|
+
for (const h of hoursToDrop) w.delete(h);
|
|
1325
|
+
if (w.size === 0) this.walked.delete(deviceId);
|
|
1326
|
+
}
|
|
1327
|
+
this.invalidateSorted(deviceId);
|
|
1328
|
+
this.evictedHoursByDevice.set(deviceId, (this.evictedHoursByDevice.get(deviceId) ?? 0) + hoursToDrop.size);
|
|
1329
|
+
let oldestHourMs = Infinity;
|
|
1330
|
+
let newestHourMs = -Infinity;
|
|
1331
|
+
for (const h of hoursToDrop) {
|
|
1332
|
+
if (h < oldestHourMs) oldestHourMs = h;
|
|
1333
|
+
if (h > newestHourMs) newestHourMs = h;
|
|
1334
|
+
}
|
|
1335
|
+
logger.info("recorder: index dropped resident hours — over the residency budget", {
|
|
1336
|
+
tags: { deviceId },
|
|
1337
|
+
meta: {
|
|
1338
|
+
hours: hoursToDrop.size,
|
|
1339
|
+
rows: droppedRows,
|
|
1340
|
+
oldestHourMs,
|
|
1341
|
+
newestHourMs,
|
|
1342
|
+
residentRowsAfter,
|
|
1343
|
+
rowBudget: budget.residentRowBudget,
|
|
1344
|
+
hourBudget: budget.residentHourBudget,
|
|
1345
|
+
costOfRecovery: "readdir-per-hour"
|
|
1346
|
+
}
|
|
1347
|
+
});
|
|
1104
1348
|
}
|
|
1105
1349
|
/**
|
|
1106
1350
|
* Merge one hour's segments for a location, and mark that hour walked.
|
|
@@ -1122,6 +1366,39 @@ var RecordingIndex = class {
|
|
|
1122
1366
|
}
|
|
1123
1367
|
if (changed) this.invalidateSorted(deviceId);
|
|
1124
1368
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$9);
|
|
1369
|
+
this.enforceResidency();
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Make one hour of ONE location exactly `relPaths` — merge what is there and
|
|
1373
|
+
* drop what is not.
|
|
1374
|
+
*
|
|
1375
|
+
* {@link hydrateHour} merges, which is right for a reader: a listing it took
|
|
1376
|
+
* of a live hour is a lower bound, and dropping on it would delete the index
|
|
1377
|
+
* of footage the readdir merely raced. This is for the one caller that has
|
|
1378
|
+
* TRUSTED evidence the hour is over and the volume it listed is the volume it
|
|
1379
|
+
* started on — the operator-armed ledger walk — where the directory is the
|
|
1380
|
+
* whole truth about the hour and an index row naming a file that is not in it
|
|
1381
|
+
* is a ghost (D523, D148's "trusted walk" arm).
|
|
1382
|
+
*
|
|
1383
|
+
* Scoped to the (device, location, hour) triple: a path filed under another
|
|
1384
|
+
* hour, or held for another location, is none of this call's business.
|
|
1385
|
+
* @residency mutation
|
|
1386
|
+
*/
|
|
1387
|
+
resetHour(deviceId, locationId, hourStartMs, relPaths) {
|
|
1388
|
+
const m = this.mapFor(deviceId);
|
|
1389
|
+
const keep = new Set(relPaths);
|
|
1390
|
+
const dropped = [];
|
|
1391
|
+
for (const [path, row] of m) {
|
|
1392
|
+
if (row.locationId !== locationId) continue;
|
|
1393
|
+
if (Math.floor(row.startMs / HOUR_MS$9) * HOUR_MS$9 !== hourStartMs) continue;
|
|
1394
|
+
if (keep.has(path)) continue;
|
|
1395
|
+
dropped.push(path);
|
|
1396
|
+
}
|
|
1397
|
+
for (const path of dropped) m.delete(path);
|
|
1398
|
+
if (dropped.length > 0) this.invalidateSorted(deviceId);
|
|
1399
|
+
this.hydrateHour(deviceId, locationId, hourStartMs, relPaths);
|
|
1400
|
+
this.enforceResidency();
|
|
1401
|
+
return dropped;
|
|
1125
1402
|
}
|
|
1126
1403
|
/**
|
|
1127
1404
|
* Insert/replace one finalized segment (idempotent by path).
|
|
@@ -1145,6 +1422,7 @@ var RecordingIndex = class {
|
|
|
1145
1422
|
return;
|
|
1146
1423
|
}
|
|
1147
1424
|
this.insertIntoSortedViews(s);
|
|
1425
|
+
this.enforceResidency();
|
|
1148
1426
|
}
|
|
1149
1427
|
/** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. @residency mutation */
|
|
1150
1428
|
removeSegments(paths) {
|
|
@@ -1172,6 +1450,7 @@ var RecordingIndex = class {
|
|
|
1172
1450
|
* @residency resident-scoped
|
|
1173
1451
|
*/
|
|
1174
1452
|
segmentsStartingIn(deviceId, profile, fromMs, toMs) {
|
|
1453
|
+
this.touch(deviceId, fromMs, toMs);
|
|
1175
1454
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1176
1455
|
let lo = 0;
|
|
1177
1456
|
let hi = segs.length;
|
|
@@ -1200,6 +1479,7 @@ var RecordingIndex = class {
|
|
|
1200
1479
|
* @residency resident-scoped
|
|
1201
1480
|
*/
|
|
1202
1481
|
segmentStartingAt(deviceId, profile, startMs) {
|
|
1482
|
+
this.touch(deviceId, startMs, startMs + 1);
|
|
1203
1483
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1204
1484
|
const at = segs[lowerBoundByStart(segs, startMs)];
|
|
1205
1485
|
return at !== void 0 && at.startMs === startMs ? at : null;
|
|
@@ -1223,6 +1503,7 @@ var RecordingIndex = class {
|
|
|
1223
1503
|
* @residency resident-scoped
|
|
1224
1504
|
*/
|
|
1225
1505
|
segmentsOverlapping(deviceId, profile, fromMs, toMs) {
|
|
1506
|
+
this.touch(deviceId, fromMs - this.maxDurMs, toMs);
|
|
1226
1507
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1227
1508
|
const end = lowerBoundByStart(segs, toMs);
|
|
1228
1509
|
let head = lowerBoundByStart(segs, fromMs);
|
|
@@ -1384,6 +1665,7 @@ var RecordingIndex = class {
|
|
|
1384
1665
|
}
|
|
1385
1666
|
/** The segment whose [startMs, startMs+durMs) window contains `epochMs`, or null if `epochMs` falls in a gap / there is no footage. @residency resident-scoped */
|
|
1386
1667
|
segmentAt(deviceId, profile, epochMs) {
|
|
1668
|
+
this.touch(deviceId, epochMs - this.maxDurMs, epochMs + 1);
|
|
1387
1669
|
return this.segmentAtIn(this.sortedSegments(deviceId, profile), epochMs);
|
|
1388
1670
|
}
|
|
1389
1671
|
/**
|
|
@@ -1393,6 +1675,7 @@ var RecordingIndex = class {
|
|
|
1393
1675
|
* @residency resident-scoped
|
|
1394
1676
|
*/
|
|
1395
1677
|
nearestCoveredEdge(deviceId, profile, epochMs) {
|
|
1678
|
+
this.touch(deviceId, epochMs - this.maxDurMs, epochMs + 1);
|
|
1396
1679
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1397
1680
|
if (segs.length === 0) return null;
|
|
1398
1681
|
if (this.segmentAtIn(segs, epochMs) !== null) return epochMs;
|
|
@@ -1446,6 +1729,7 @@ var RecordingIndex = class {
|
|
|
1446
1729
|
* @residency resident-scoped
|
|
1447
1730
|
*/
|
|
1448
1731
|
coveredEdgeBefore(deviceId, profile, epochMs) {
|
|
1732
|
+
this.touch(deviceId, epochMs - this.maxDurMs, epochMs + 1);
|
|
1449
1733
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1450
1734
|
let best = null;
|
|
1451
1735
|
for (let i = upperBoundByStart(segs, epochMs) - 1; i >= 0; i--) {
|
|
@@ -1458,6 +1742,7 @@ var RecordingIndex = class {
|
|
|
1458
1742
|
}
|
|
1459
1743
|
/** @residency resident-scoped */
|
|
1460
1744
|
ranges(deviceId, profile, fromMs, toMs, gapToleranceMs) {
|
|
1745
|
+
this.touch(deviceId, fromMs, toMs);
|
|
1461
1746
|
return mergeRanges(this.sortedSegments(deviceId, profile).filter((s) => s.startMs + s.durMs > fromMs && s.startMs < toMs), gapToleranceMs).map((r) => ({
|
|
1462
1747
|
startMs: Math.max(r.startMs, fromMs),
|
|
1463
1748
|
endMs: Math.min(r.endMs, toMs)
|
|
@@ -1692,6 +1977,7 @@ async function resolveRecordingsLocations(api, logger) {
|
|
|
1692
1977
|
async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger, options = {}) {
|
|
1693
1978
|
const yieldBetween = options.yieldBetween ?? macrotask$3;
|
|
1694
1979
|
const chunkSize = options.chunkSize ?? HYDRATE_CHUNK;
|
|
1980
|
+
index.beginWalk(deviceId);
|
|
1695
1981
|
for (const [root, aliases] of aliasesByRoot(locations)) {
|
|
1696
1982
|
const deviceDir = node_path.default.join(root, String(deviceId));
|
|
1697
1983
|
const startedMs = Date.now();
|
|
@@ -1772,40 +2058,6 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1772
2058
|
const verify = options?.verifyClosedHours === true;
|
|
1773
2059
|
if (!verify && index.hydrationOf(deviceId, fromMs, toMs) === "hydrated") return;
|
|
1774
2060
|
const liveHour = Math.floor(Date.now() / HOUR_MS$8) * HOUR_MS$8;
|
|
1775
|
-
/** Per (locationId, profile, hour): what the LEDGER claimed, for the diff. */
|
|
1776
|
-
const claimed = /* @__PURE__ */ new Map();
|
|
1777
|
-
const claimKey = (locationId, hour) => `${locationId}:${hour}`;
|
|
1778
|
-
/**
|
|
1779
|
-
* Closed hours the ledger answered for with a row nobody SEALED (D451): what
|
|
1780
|
-
* a writer had persisted when it was last heard from, which is a legitimate
|
|
1781
|
-
* answer and not a complete one. Each is read from the array once and the
|
|
1782
|
-
* row is unioned and sealed, so the second reader pays a point read.
|
|
1783
|
-
*/
|
|
1784
|
-
const unsealed = /* @__PURE__ */ new Set();
|
|
1785
|
-
if (ledger !== void 0) {
|
|
1786
|
-
const startedMs = Date.now();
|
|
1787
|
-
const hours = await ledger.detailForWindow(deviceId, fromMs, toMs);
|
|
1788
|
-
let segments = 0;
|
|
1789
|
-
let served = 0;
|
|
1790
|
-
for (const hour of hours) {
|
|
1791
|
-
if (hour.hourStartMs >= liveHour) continue;
|
|
1792
|
-
index.hydrateHour(deviceId, hour.locationId, hour.hourStartMs, hour.paths);
|
|
1793
|
-
claimed.set(claimKey(hour.locationId, hour.hourStartMs), hour.paths.length);
|
|
1794
|
-
if (!hour.sealed) unsealed.add(hour.hourStartMs);
|
|
1795
|
-
segments += hour.paths.length;
|
|
1796
|
-
served += 1;
|
|
1797
|
-
}
|
|
1798
|
-
if (served > 0) logger.info("recorder: hydrated a window from the hour ledger", {
|
|
1799
|
-
tags: { deviceId },
|
|
1800
|
-
meta: {
|
|
1801
|
-
hours: served,
|
|
1802
|
-
segments,
|
|
1803
|
-
unsealedHours: unsealed.size,
|
|
1804
|
-
ms: Date.now() - startedMs
|
|
1805
|
-
}
|
|
1806
|
-
});
|
|
1807
|
-
if (!verify && unsealed.size === 0 && index.hydrationOf(deviceId, fromMs, toMs) === "hydrated") return;
|
|
1808
|
-
}
|
|
1809
2061
|
for (const [root, aliases] of aliasesByRoot(locations)) {
|
|
1810
2062
|
const deviceDir = node_path.default.join(root, String(deviceId));
|
|
1811
2063
|
let profiles;
|
|
@@ -1829,7 +2081,7 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1829
2081
|
let forgottenHours = 0;
|
|
1830
2082
|
let sealed = 0;
|
|
1831
2083
|
for (let hour = first; hour < toMs; hour += HOUR_MS$8) {
|
|
1832
|
-
if (!verify && hour < liveHour &&
|
|
2084
|
+
if (!verify && hour < liveHour && index.hydrationOf(deviceId, hour, hour + HOUR_MS$8) === "hydrated") continue;
|
|
1833
2085
|
for (const profile of profiles) {
|
|
1834
2086
|
const rel = hourDirRelPath(deviceId, profile, hour);
|
|
1835
2087
|
let files;
|
|
@@ -1843,11 +2095,10 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1843
2095
|
const relPaths = files.filter((f) => f.endsWith(".m4s")).map((f) => `${rel}/${f}`);
|
|
1844
2096
|
index.hydrateHour(deviceId, locationId, hour, relPaths);
|
|
1845
2097
|
if (ledger === void 0 || relPaths.length === 0 || hour + HOUR_MS$8 > Date.now()) continue;
|
|
1846
|
-
const claimedHere = claimed.get(claimKey(locationId, hour));
|
|
1847
2098
|
const outcome = await ledger.adoptWalkedHour(deviceId, profile, locationId, hour, relPaths);
|
|
1848
2099
|
if (outcome === null) continue;
|
|
1849
2100
|
if (outcome.created) repaired += 1;
|
|
1850
|
-
else if (
|
|
2101
|
+
else if (outcome.added > 0) {
|
|
1851
2102
|
forgottenSegments += outcome.added;
|
|
1852
2103
|
forgottenHours += 1;
|
|
1853
2104
|
}
|
|
@@ -2484,10 +2735,12 @@ var RelocateEngine = class {
|
|
|
2484
2735
|
*
|
|
2485
2736
|
* The order is the whole point. Between the copy and the delete both
|
|
2486
2737
|
* locations hold every byte, so the ledger re-point in the middle can only
|
|
2487
|
-
* ever
|
|
2488
|
-
* recoverable instead of a black window.
|
|
2489
|
-
*
|
|
2490
|
-
*
|
|
2738
|
+
* ever account for a location that HAS the files — which is what makes a
|
|
2739
|
+
* crash recoverable instead of a black window.
|
|
2740
|
+
*
|
|
2741
|
+
* There is no ledger refusal to handle any more (D523): the durable half is
|
|
2742
|
+
* arithmetic over a row the mirror already holds, so nothing on that path can
|
|
2743
|
+
* fail and stop an hour dead.
|
|
2491
2744
|
*/
|
|
2492
2745
|
async moveHour(job, rows, from, to, samePhysicalRoot, sleep, bytesPerMs) {
|
|
2493
2746
|
const movable = [];
|
|
@@ -2523,17 +2776,7 @@ var RelocateEngine = class {
|
|
|
2523
2776
|
toLocationId: to.id
|
|
2524
2777
|
}
|
|
2525
2778
|
});
|
|
2526
|
-
else
|
|
2527
|
-
this.deps.logger.warn("relocate: hour NOT moved — the durable ledger refused the re-point, so the source bytes are kept", {
|
|
2528
|
-
tags: { deviceId: movable[0].deviceId },
|
|
2529
|
-
meta: {
|
|
2530
|
-
jobId: job.jobId,
|
|
2531
|
-
files: movable.length,
|
|
2532
|
-
toLocationId: to.id
|
|
2533
|
-
}
|
|
2534
|
-
});
|
|
2535
|
-
return;
|
|
2536
|
-
}
|
|
2779
|
+
else await ledger.repointSegments(movable, to.id);
|
|
2537
2780
|
if (!samePhysicalRoot) for (const row of movable) await node_fs.promises.rm(node_path.default.join(from.root, row.fromPath ?? row.path), { force: true });
|
|
2538
2781
|
for (const row of movable) {
|
|
2539
2782
|
this.deps.index.removeSegments([row.fromPath ?? row.path]);
|
|
@@ -2926,23 +3169,21 @@ var SegmentStore = class SegmentStore {
|
|
|
2926
3169
|
* work now the per-bucket archive scan is gone. */
|
|
2927
3170
|
static DEFAULT_EVICT_CHUNK = 25;
|
|
2928
3171
|
/**
|
|
2929
|
-
* True when `victims` cover EVERY segment the
|
|
2930
|
-
*
|
|
3172
|
+
* True when `victims` cover EVERY segment the bucket DIRECTORY holds, AND the
|
|
3173
|
+
* bucket's hour has fully elapsed.
|
|
2931
3174
|
*
|
|
2932
3175
|
* The current-hour guard matters because `onFinalized` relocates new segments
|
|
2933
3176
|
* INTO the current bucket concurrently — an rm racing that rename would
|
|
2934
3177
|
* orphan a just-indexed segment.
|
|
2935
3178
|
*
|
|
2936
|
-
* The cover test is
|
|
2937
|
-
* `RecordingIndex
|
|
2938
|
-
*
|
|
2939
|
-
*
|
|
2940
|
-
*
|
|
2941
|
-
*
|
|
2942
|
-
* went with every file the round never named, leaving their durable rows
|
|
2943
|
-
* behind as ghosts.
|
|
3179
|
+
* The cover test is a listing of the directory the `rm -rf` is about to take,
|
|
3180
|
+
* never `RecordingIndex` and never a durable row (D523). A bucketDir is
|
|
3181
|
+
* `<dev>/<profile>/Y/M/D/H`, so one listing answers exactly the question the
|
|
3182
|
+
* delete asks. Asked of the resident index it read EQUAL whenever the hour was
|
|
3183
|
+
* partly resident — both sides short by the same rows — and the directory went
|
|
3184
|
+
* with every file the round never named.
|
|
2944
3185
|
*
|
|
2945
|
-
* An unanswerable
|
|
3186
|
+
* An unanswerable listing returns `false`: per-file deletes are slower and
|
|
2946
3187
|
* always correct, and the alternative is a recursive delete computed from a
|
|
2947
3188
|
* fraction of the truth.
|
|
2948
3189
|
*/
|
|
@@ -2951,7 +3192,7 @@ var SegmentStore = class SegmentStore {
|
|
|
2951
3192
|
if (!first) return false;
|
|
2952
3193
|
const bucketStartMs = Math.floor(first.startMs / SegmentStore.HOUR_MS) * SegmentStore.HOUR_MS;
|
|
2953
3194
|
if ((this.deps.now?.() ?? Date.now()) < bucketStartMs + SegmentStore.HOUR_MS) return false;
|
|
2954
|
-
const ask = this.deps.
|
|
3195
|
+
const ask = this.deps.hourDirectoryPaths;
|
|
2955
3196
|
if (ask === void 0) return false;
|
|
2956
3197
|
const held = await ask(first.deviceId, first.profile, location, bucketStartMs);
|
|
2957
3198
|
if (held === null || held.length === 0) return false;
|
|
@@ -4084,8 +4325,9 @@ var CalendarIndex = class {
|
|
|
4084
4325
|
*
|
|
4085
4326
|
* Also used to warm the days ADJACENT to one being viewed — changing day is a
|
|
4086
4327
|
* single click and the operator expects the summary to be there. That warm
|
|
4087
|
-
* belongs in the background:
|
|
4088
|
-
*
|
|
4328
|
+
* belongs in the background: a cold read costs whatever the array charges
|
|
4329
|
+
* that day (see the header — the ~650 ms figure is from the retired FUSE
|
|
4330
|
+
* share), and doing it inline would only move the wait onto the click.
|
|
4089
4331
|
*/
|
|
4090
4332
|
async warm(deviceId, fromMs, toMs) {
|
|
4091
4333
|
for (let day = utcDayOf(fromMs); day <= utcDayOf(toMs); day += DAY_MS$3) {
|
|
@@ -5207,22 +5449,18 @@ function createFootageDiscardJob(deps) {
|
|
|
5207
5449
|
const forgetLedger = async (location) => {
|
|
5208
5450
|
const ledger = deps.ledger;
|
|
5209
5451
|
if (ledger === null) return 0;
|
|
5210
|
-
let forgotten = 0;
|
|
5211
5452
|
const profiles = profilesForType(location.type);
|
|
5212
|
-
|
|
5453
|
+
const scope = {
|
|
5213
5454
|
locationId: location.id,
|
|
5214
5455
|
profiles
|
|
5215
|
-
}
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
});
|
|
5224
|
-
await sleep(0);
|
|
5225
|
-
}
|
|
5456
|
+
};
|
|
5457
|
+
const hours = ledger.snapshot().filter((row) => row.locationId === location.id && profiles.includes(row.profile));
|
|
5458
|
+
const forgotten = await ledger.forgetScope(scope);
|
|
5459
|
+
for (const hour of hours) deps.index.resetHour(hour.deviceId, hour.locationId, hour.hourStartMs, []);
|
|
5460
|
+
report({
|
|
5461
|
+
dbRowsDeleted: forgotten,
|
|
5462
|
+
detail: `forgot ${forgotten} ledger segments`
|
|
5463
|
+
});
|
|
5226
5464
|
return forgotten;
|
|
5227
5465
|
};
|
|
5228
5466
|
return {
|
|
@@ -8275,63 +8513,6 @@ function runFfmpeg(renderDeps, args, ext) {
|
|
|
8275
8513
|
});
|
|
8276
8514
|
}
|
|
8277
8515
|
//#endregion
|
|
8278
|
-
//#region src/recorder/addon/paths-codec.ts
|
|
8279
|
-
/**
|
|
8280
|
-
* How an hour row's `paths` becomes text, and back (D471).
|
|
8281
|
-
*
|
|
8282
|
-
* `recorder:segment-hours` is 66 MB of the settings database and **99.2 % of it
|
|
8283
|
-
* is this one column** — 15 643 bytes per row over 4 036 rows. The content is a
|
|
8284
|
-
* list of location-relative segment paths that nearly all share the same
|
|
8285
|
-
* directory prefix, repeated in full on every entry. Measured per column on a
|
|
8286
|
-
* copy of the live database, never the live file (D468).
|
|
8287
|
-
*
|
|
8288
|
-
* Compressing it is unusually safe here because of D287's two-tier mirror: boot
|
|
8289
|
-
* loads the aggregate half of every row and NOT `paths`, so this column is read
|
|
8290
|
-
* only on a point materialise, never on the hot path.
|
|
8291
|
-
*
|
|
8292
|
-
* A local copy of the same idea `pipeline-analytics` uses for `positions` —
|
|
8293
|
-
* addons never import each other (CLAUDE.md), so twenty duplicated lines are
|
|
8294
|
-
* the correct shape, not a shared dependency across an addon boundary.
|
|
8295
|
-
*/
|
|
8296
|
-
/** Marks a value this module wrote. A plain JSON array always starts `[`. */
|
|
8297
|
-
var ENCODED_PREFIX = "z1:";
|
|
8298
|
-
function encodePaths(paths) {
|
|
8299
|
-
return ENCODED_PREFIX + (0, node_zlib.gzipSync)(Buffer.from(JSON.stringify(paths), "utf8"), { level: 6 }).toString("base64");
|
|
8300
|
-
}
|
|
8301
|
-
/**
|
|
8302
|
-
* Decode whichever era wrote the value: the `z1:` string this writes, the plain
|
|
8303
|
-
* JSON string every pre-D471 row holds, or an array the store already parsed.
|
|
8304
|
-
*
|
|
8305
|
-
* `null` means "this row is unreadable", which the caller turns into a DISCARDED
|
|
8306
|
-
* row — the same contract the bare `JSON.parse` had. An hour whose paths cannot
|
|
8307
|
-
* be read must not silently become an hour with no segments: that would report
|
|
8308
|
-
* an empty disk, which is the failure `recordToRow` was already written to
|
|
8309
|
-
* avoid.
|
|
8310
|
-
*/
|
|
8311
|
-
function decodePaths(value) {
|
|
8312
|
-
if (Array.isArray(value)) return isPathList(value) ? value : null;
|
|
8313
|
-
if (typeof value !== "string") return null;
|
|
8314
|
-
const json = value.startsWith(ENCODED_PREFIX) ? tryGunzip(value.slice(3)) : value;
|
|
8315
|
-
if (json === null) return null;
|
|
8316
|
-
try {
|
|
8317
|
-
const parsed = JSON.parse(json);
|
|
8318
|
-
if (!Array.isArray(parsed)) return null;
|
|
8319
|
-
return isPathList(parsed) ? parsed : null;
|
|
8320
|
-
} catch {
|
|
8321
|
-
return null;
|
|
8322
|
-
}
|
|
8323
|
-
}
|
|
8324
|
-
function isPathList(v) {
|
|
8325
|
-
return v.every((p) => typeof p === "string" && p.length > 0);
|
|
8326
|
-
}
|
|
8327
|
-
function tryGunzip(b64) {
|
|
8328
|
-
try {
|
|
8329
|
-
return (0, node_zlib.gunzipSync)(Buffer.from(b64, "base64")).toString("utf8");
|
|
8330
|
-
} catch {
|
|
8331
|
-
return null;
|
|
8332
|
-
}
|
|
8333
|
-
}
|
|
8334
|
-
//#endregion
|
|
8335
8516
|
//#region src/durable/durable-ledger.ts
|
|
8336
8517
|
/** Default reseed cap — every current consumer's row set is installation-bounded. */
|
|
8337
8518
|
var DEFAULT_LOAD_LIMIT = 1e5;
|
|
@@ -8586,19 +8767,19 @@ var macrotask$1 = () => new Promise((resolve) => {
|
|
|
8586
8767
|
});
|
|
8587
8768
|
/**
|
|
8588
8769
|
* @durable class=mirror owner=recorder
|
|
8589
|
-
* write="write-behind on every finalized segment (hour-row upsert, serialised per hour); the writer SEALS the hour it leaves (one write per hour); a
|
|
8770
|
+
* write="write-behind on every finalized segment (hour-row upsert, serialised per hour); the writer SEALS the hour it leaves (one write per hour); a listing of a closed hour re-derives the aggregate and seals; eviction and relocation shrink it"
|
|
8590
8771
|
* retention="a row goes when the walk or eviction confirms the hour is empty — never by age, never from the ledger alone (D148)"
|
|
8591
8772
|
*/
|
|
8592
8773
|
var RECORDING_SEGMENT_HOURS_COLLECTION = "recorder:segment-hours";
|
|
8593
8774
|
/**
|
|
8594
|
-
* The four aggregate columns
|
|
8595
|
-
*
|
|
8596
|
-
*
|
|
8597
|
-
*
|
|
8598
|
-
*
|
|
8599
|
-
*
|
|
8600
|
-
*
|
|
8601
|
-
*
|
|
8775
|
+
* The four aggregate columns are what this table IS (D523).
|
|
8776
|
+
*
|
|
8777
|
+
* They exist so "how much footage is on this disk" is a `SUM`/`MIN`/`MAX` over
|
|
8778
|
+
* ~19 k hour rows instead of a walk over ~7.1 M in-RAM segment rows, and they
|
|
8779
|
+
* are derived from segment NAMES — by {@link withSegmentPath} and nothing else.
|
|
8780
|
+
* {@link hourAggregate} is that step over a whole listing and
|
|
8781
|
+
* {@link appendSegmentPath} is that step over the one path that just arrived,
|
|
8782
|
+
* so there is no second derivation to drift.
|
|
8602
8783
|
*/
|
|
8603
8784
|
var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
8604
8785
|
{
|
|
@@ -8627,11 +8808,6 @@ var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
|
8627
8808
|
type: "INTEGER",
|
|
8628
8809
|
notNull: true
|
|
8629
8810
|
},
|
|
8630
|
-
{
|
|
8631
|
-
name: "paths",
|
|
8632
|
-
type: "TEXT",
|
|
8633
|
-
notNull: true
|
|
8634
|
-
},
|
|
8635
8811
|
{
|
|
8636
8812
|
name: "bytes",
|
|
8637
8813
|
type: "INTEGER",
|
|
@@ -8690,15 +8866,9 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
|
8690
8866
|
* `MAX(maxStartMs)` over `locationId IN (…)` — one row back, and with this
|
|
8691
8867
|
* index not one table row read to produce it (D447).
|
|
8692
8868
|
*
|
|
8693
|
-
* `
|
|
8694
|
-
*
|
|
8695
|
-
*
|
|
8696
|
-
* accounting read at the price of ~30 KB of table page per hour row. On the
|
|
8697
|
-
* live hub that was `durationMs=1001…2902` once a MINUTE on
|
|
8698
|
-
* `hub/sqlite-settings` (174 slow-call WARNs in ten hours on 2026-09-10/11),
|
|
8699
|
-
* every one of them holding the thread every configuration read in the
|
|
8700
|
-
* cluster waits on. The index is ~50 bytes a row over ~19 k rows — nothing —
|
|
8701
|
-
* and the first declare after it ships builds it in well under a second.
|
|
8869
|
+
* Since D523 dropped `paths` the table row is ten scalars, so the covering
|
|
8870
|
+
* index no longer saves ~30 KB of page per row; it still saves the row
|
|
8871
|
+
* fetch, and the accounting read is the one this hub runs on a schedule.
|
|
8702
8872
|
*/
|
|
8703
8873
|
{
|
|
8704
8874
|
name: "idx_recorder_segment_hours_loc_accounting",
|
|
@@ -8723,13 +8893,13 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
|
8723
8893
|
* nominal `-segment_time` (ffmpeg cuts on the next GOP, never on the clock).
|
|
8724
8894
|
*
|
|
8725
8895
|
* It is a guess, and this is the shape of being wrong in each direction.
|
|
8726
|
-
* TIGHTER: a normal rollover stops sealing
|
|
8727
|
-
* once by
|
|
8728
|
-
*
|
|
8729
|
-
*
|
|
8730
|
-
*
|
|
8731
|
-
*
|
|
8732
|
-
*
|
|
8896
|
+
* TIGHTER: a normal rollover stops sealing and every hour of the archive is
|
|
8897
|
+
* listed once by the adoption sweep — at ~1 ms per hour directory on this hub's
|
|
8898
|
+
* SATA SSD, affordable but pointless. LOOSER: a runner respawn inside the
|
|
8899
|
+
* window seals an hour it entered late, which is exactly the 2026-09-17 defect;
|
|
8900
|
+
* the observed hole there was ~29 minutes, so anything under a few minutes
|
|
8901
|
+
* would have caught it and 90 s is the smallest number that does not fire on a
|
|
8902
|
+
* legitimate rollover.
|
|
8733
8903
|
*/
|
|
8734
8904
|
var WRITER_HOUR_ENTRY_GRACE_MS = 9e4;
|
|
8735
8905
|
/** The identity of the {@link hourAggregate} fold — an hour holding nothing. */
|
|
@@ -8743,12 +8913,11 @@ var EMPTY_HOUR_AGGREGATE = {
|
|
|
8743
8913
|
* ONE path folded into an aggregate — the whole derivation, expressed as its
|
|
8744
8914
|
* single step.
|
|
8745
8915
|
*
|
|
8746
|
-
* {@link hourAggregate} is this step over
|
|
8916
|
+
* {@link hourAggregate} is this step over a whole listing, and {@link
|
|
8747
8917
|
* SegmentHourLedger.recordSegment} is this step over the one path that just
|
|
8748
8918
|
* arrived. That is not two implementations that happen to agree: the full
|
|
8749
8919
|
* derivation is *defined* as this function repeated, so an append is the fold
|
|
8750
|
-
* resumed from the state the held row already carries
|
|
8751
|
-
* aggregate half IS the fold state over its `paths`.
|
|
8920
|
+
* resumed from the state the held row already carries.
|
|
8752
8921
|
*
|
|
8753
8922
|
* Why it had to become a step: `recordSegment` re-derived the whole hour on
|
|
8754
8923
|
* every finalize. Σ1..360 ≈ 65 k path parses per camera-profile-hour at 10 s
|
|
@@ -8771,49 +8940,57 @@ function withSegmentPath(agg, path) {
|
|
|
8771
8940
|
};
|
|
8772
8941
|
}
|
|
8773
8942
|
/**
|
|
8774
|
-
* Bytes / count / oldest / newest for one hour's
|
|
8775
|
-
* derivation,
|
|
8776
|
-
* by the append path.
|
|
8943
|
+
* Bytes / count / oldest / newest for one hour's LISTING — the sole FULL
|
|
8944
|
+
* derivation, and since D523 the only thing that can produce an EXACT row.
|
|
8777
8945
|
*
|
|
8778
8946
|
* Every field of a segment path encodes its own facts (`parseSegmentPath`), so
|
|
8779
|
-
* this is a derivation, not a measurement: it can never disagree with the
|
|
8780
|
-
* it was computed from.
|
|
8947
|
+
* this is a derivation, not a measurement: it can never disagree with the
|
|
8948
|
+
* listing it was computed from.
|
|
8781
8949
|
*/
|
|
8782
8950
|
function hourAggregate(paths) {
|
|
8783
8951
|
let agg = EMPTY_HOUR_AGGREGATE;
|
|
8784
8952
|
for (const path of paths) agg = withSegmentPath(agg, path);
|
|
8785
8953
|
return agg;
|
|
8786
8954
|
}
|
|
8787
|
-
/** Build the persisted row for an hour from
|
|
8955
|
+
/** Build the persisted row for an hour from a LISTING of it — the ONE
|
|
8956
|
+
* constructor that can produce an exact aggregate. */
|
|
8788
8957
|
function segmentHourRow(base) {
|
|
8958
|
+
const { paths, sealed, ...identity } = base;
|
|
8789
8959
|
return {
|
|
8790
|
-
...
|
|
8791
|
-
sealed:
|
|
8792
|
-
|
|
8793
|
-
|
|
8960
|
+
...identity,
|
|
8961
|
+
sealed: sealed ?? false,
|
|
8962
|
+
...hourAggregate(paths)
|
|
8963
|
+
};
|
|
8964
|
+
}
|
|
8965
|
+
/**
|
|
8966
|
+
* Build the persisted row for an hour from an aggregate somebody already folded
|
|
8967
|
+
* — a relocation moving one, or a listing that has already been reduced.
|
|
8968
|
+
*
|
|
8969
|
+
* `sealed` is a NAMED field and never a positional boolean, because
|
|
8970
|
+
* `scripts/check-seal-authority.ts` attributes every literal `sealed: true` in
|
|
8971
|
+
* the recorder to the method that wrote it. A seal smuggled in as a third
|
|
8972
|
+
* argument is a seal the guard cannot see.
|
|
8973
|
+
*/
|
|
8974
|
+
function hourRowOf(identity, row) {
|
|
8975
|
+
return {
|
|
8976
|
+
...identity,
|
|
8977
|
+
...row
|
|
8794
8978
|
};
|
|
8795
8979
|
}
|
|
8796
8980
|
/**
|
|
8797
8981
|
* The row an hour becomes when ONE more path joins it.
|
|
8798
8982
|
*
|
|
8799
|
-
* Identical to `segmentHourRow({
|
|
8800
|
-
* strictly cheaper: the aggregate half of `held` is already the fold over
|
|
8801
|
-
*
|
|
8983
|
+
* Identical to `segmentHourRow({ …held, paths: [...everything, path] })` and
|
|
8984
|
+
* strictly cheaper: the aggregate half of `held` is already the fold over the
|
|
8985
|
+
* hour's names, so this resumes that fold instead of restarting it. The
|
|
8802
8986
|
* equality is not an assumption — `segment-hour-append.spec.ts` pins it on
|
|
8803
8987
|
* generated hours including out-of-order arrival, duplicate paths, unparseable
|
|
8804
8988
|
* paths and storm-sized hours.
|
|
8805
|
-
*
|
|
8806
|
-
* The four fields are named rather than spread on purpose. {@link
|
|
8807
|
-
* withSegmentPath} hands its argument straight back for a path the parser
|
|
8808
|
-
* refuses, and `held` is a whole row — spreading that return would put the
|
|
8809
|
-
* PREVIOUS `paths` array back over the appended one. Which is exactly what it
|
|
8810
|
-
* did until the generated-hours test caught it.
|
|
8811
8989
|
*/
|
|
8812
8990
|
function appendSegmentPath(held, path) {
|
|
8813
8991
|
const { bytes, segments, minStartMs, maxStartMs } = withSegmentPath(held, path);
|
|
8814
8992
|
return {
|
|
8815
8993
|
...held,
|
|
8816
|
-
paths: [...held.paths, path],
|
|
8817
8994
|
bytes,
|
|
8818
8995
|
segments,
|
|
8819
8996
|
minStartMs,
|
|
@@ -8831,6 +9008,20 @@ function appendSegmentPath(held, path) {
|
|
|
8831
9008
|
* `readdir` over an unmounted root deletes all of it.
|
|
8832
9009
|
*/
|
|
8833
9010
|
var LEDGER_PRUNE_MAX_SHARE = .5;
|
|
9011
|
+
/**
|
|
9012
|
+
* The largest share of ONE hour's accounted segments a single listing may
|
|
9013
|
+
* erase (D523).
|
|
9014
|
+
*
|
|
9015
|
+
* The directory is the authority for WHICH segments exist, so a listing of a
|
|
9016
|
+
* closed hour replaces that hour's aggregate — including downwards, which is
|
|
9017
|
+
* what makes an over-counted row self-correcting. The guard is the same
|
|
9018
|
+
* argument {@link LEDGER_PRUNE_MAX_SHARE} makes one level up: a root that has
|
|
9019
|
+
* gone away, or a bucket half-deleted by a round that died, answers SHORT, and
|
|
9020
|
+
* a short answer that halves an hour is indistinguishable from one that is
|
|
9021
|
+
* simply right. Under the bar the listing wins; over it the row stands, the
|
|
9022
|
+
* hour stays unsealed, and the disagreement is named.
|
|
9023
|
+
*/
|
|
9024
|
+
var HOUR_LISTING_MAX_SHRINK_SHARE = .5;
|
|
8834
9025
|
/** Backstop on the oldest-first probe. A bounded read must stay bounded even if
|
|
8835
9026
|
* a caller asks for a target no archive can cover. */
|
|
8836
9027
|
var OLDEST_MAX_PAGES = 16;
|
|
@@ -8840,9 +9031,6 @@ var OLDEST_PROBE_COLUMNS = [
|
|
|
8840
9031
|
"bytes",
|
|
8841
9032
|
"locationId"
|
|
8842
9033
|
];
|
|
8843
|
-
/** Aggregate-repair probe: the primary key always comes back, so this asks for
|
|
8844
|
-
* the predicate column alone. */
|
|
8845
|
-
var REPAIR_PROBE_COLUMNS = ["segments"];
|
|
8846
9034
|
/**
|
|
8847
9035
|
* Backstop on {@link SegmentHourLedger.streamSegments}.
|
|
8848
9036
|
*
|
|
@@ -8881,47 +9069,8 @@ var CENSUS_FIELDS = [
|
|
|
8881
9069
|
op: "max"
|
|
8882
9070
|
}
|
|
8883
9071
|
];
|
|
8884
|
-
/** Rows repaired per boot. The live archive is ~19 k hour rows. */
|
|
8885
|
-
var REPAIR_MAX_ROWS = 5e4;
|
|
8886
|
-
/**
|
|
8887
|
-
* The boot projection (D287): everything the recorder's always-resident tier
|
|
8888
|
-
* needs, and deliberately NOT `paths`.
|
|
8889
|
-
*
|
|
8890
|
-
* `paths` is ~55 B per segment and the archive is the only thing that grows:
|
|
8891
|
-
* ~16 MB of JSON today, a projected ~460 MB at 28 TB of today's bitrate mix
|
|
8892
|
-
* and ~5.9 GB at a low-bitrate mix — one tRPC response, parsed on the main
|
|
8893
|
-
* thread, before the index exists. The summary half is O(writers × days) and
|
|
8894
|
-
* stays ~10–15 MB at any archive size.
|
|
8895
|
-
*/
|
|
8896
|
-
var SEGMENT_HOUR_SUMMARY_COLUMNS = [
|
|
8897
|
-
"deviceId",
|
|
8898
|
-
"profile",
|
|
8899
|
-
"locationId",
|
|
8900
|
-
"hourStartMs",
|
|
8901
|
-
"bytes",
|
|
8902
|
-
"segments",
|
|
8903
|
-
"minStartMs",
|
|
8904
|
-
"maxStartMs",
|
|
8905
|
-
"sealed"
|
|
8906
|
-
];
|
|
8907
9072
|
/** Summary rows folded into the mirror between yields. */
|
|
8908
9073
|
var LOAD_CHUNK = 5e3;
|
|
8909
|
-
/** Hour rows materialised per point-read page. One page covers a day of one
|
|
8910
|
-
* camera's three profiles with room to spare. */
|
|
8911
|
-
var HOUR_DETAIL_PAGE = 64;
|
|
8912
|
-
/**
|
|
8913
|
-
* The live tail materialised at boot: the current hour and the one before it.
|
|
8914
|
-
*
|
|
8915
|
-
* `recordSegment` must gate from RAM (D49), and the hour it appends to at boot
|
|
8916
|
-
* is exactly the hour the previous process was writing. Two hours × the active
|
|
8917
|
-
* writers is ~60 rows — one point-read page — and it is what makes the write
|
|
8918
|
-
* path's materialise a case that essentially never happens.
|
|
8919
|
-
*/
|
|
8920
|
-
var LIVE_TAIL_HOURS = 2;
|
|
8921
|
-
/** Rows the one-shot aggregate repair may materialise in a single boot. The
|
|
8922
|
-
* backfill is bounded so it cannot re-become the whole-archive read D287
|
|
8923
|
-
* removed; a partial pass finishes on the next boot. */
|
|
8924
|
-
var REPAIR_MAX_MATERIALIZE = 2e3;
|
|
8925
9074
|
/** The four scalars an accounting answer is made of, asked in one statement. */
|
|
8926
9075
|
var ACCOUNTING_FIELDS = [
|
|
8927
9076
|
{
|
|
@@ -8974,14 +9123,6 @@ function writerKey(deviceId, profile) {
|
|
|
8974
9123
|
function writerHourKey(deviceId, profile, hour) {
|
|
8975
9124
|
return `${deviceId}|${profile}|${hour}`;
|
|
8976
9125
|
}
|
|
8977
|
-
function detailOf(row) {
|
|
8978
|
-
return {
|
|
8979
|
-
locationId: row.locationId,
|
|
8980
|
-
hourStartMs: row.hourStartMs,
|
|
8981
|
-
paths: row.paths,
|
|
8982
|
-
sealed: row.sealed
|
|
8983
|
-
};
|
|
8984
|
-
}
|
|
8985
9126
|
/**
|
|
8986
9127
|
* The `hourStartMs` of one probe record.
|
|
8987
9128
|
*
|
|
@@ -8993,21 +9134,31 @@ function hourOfRecord(record) {
|
|
|
8993
9134
|
const raw = record.data["hourStartMs"];
|
|
8994
9135
|
return typeof raw === "number" && Number.isFinite(raw) ? raw : -1;
|
|
8995
9136
|
}
|
|
8996
|
-
|
|
8997
|
-
|
|
8998
|
-
|
|
8999
|
-
|
|
9000
|
-
|
|
9001
|
-
|
|
9002
|
-
|
|
9137
|
+
/** One listing turned into the segment rows a caller can act on — the only
|
|
9138
|
+
* place a `SegmentRow` is built out of a directory read. */
|
|
9139
|
+
function segmentRowsOf(locationId, paths) {
|
|
9140
|
+
const rows = [];
|
|
9141
|
+
let unparseable = 0;
|
|
9142
|
+
for (const path of paths) {
|
|
9143
|
+
const parsed = parseSegmentPath(path);
|
|
9144
|
+
if (parsed === null) {
|
|
9145
|
+
unparseable += 1;
|
|
9146
|
+
continue;
|
|
9147
|
+
}
|
|
9148
|
+
rows.push({
|
|
9149
|
+
deviceId: parsed.deviceId,
|
|
9150
|
+
profile: parsed.profile,
|
|
9151
|
+
startMs: parsed.startMs,
|
|
9152
|
+
durMs: parsed.durMs,
|
|
9153
|
+
bytes: parsed.bytes,
|
|
9154
|
+
path,
|
|
9155
|
+
locationId
|
|
9156
|
+
});
|
|
9003
9157
|
}
|
|
9004
|
-
return
|
|
9005
|
-
|
|
9006
|
-
|
|
9007
|
-
|
|
9008
|
-
const set = new Set(a);
|
|
9009
|
-
for (const p of b) if (!set.has(p)) return false;
|
|
9010
|
-
return true;
|
|
9158
|
+
return {
|
|
9159
|
+
rows,
|
|
9160
|
+
unparseable
|
|
9161
|
+
};
|
|
9011
9162
|
}
|
|
9012
9163
|
function rowToValue(row) {
|
|
9013
9164
|
return {
|
|
@@ -9015,7 +9166,6 @@ function rowToValue(row) {
|
|
|
9015
9166
|
profile: row.profile,
|
|
9016
9167
|
locationId: row.locationId,
|
|
9017
9168
|
hourStartMs: row.hourStartMs,
|
|
9018
|
-
paths: encodePaths(row.paths),
|
|
9019
9169
|
bytes: row.bytes,
|
|
9020
9170
|
segments: row.segments,
|
|
9021
9171
|
minStartMs: row.minStartMs,
|
|
@@ -9028,44 +9178,21 @@ function rowToValue(row) {
|
|
|
9028
9178
|
function sealedOf(raw) {
|
|
9029
9179
|
return raw === 1 || raw === true;
|
|
9030
9180
|
}
|
|
9031
|
-
function recordToRow(key, data) {
|
|
9032
|
-
const deviceId = Number(data["deviceId"]);
|
|
9033
|
-
const profile = data["profile"];
|
|
9034
|
-
const locationId = data["locationId"];
|
|
9035
|
-
const hour = Number(data["hourStartMs"]);
|
|
9036
|
-
const decodedPaths = decodePaths(data["paths"]);
|
|
9037
|
-
if (decodedPaths === null) return null;
|
|
9038
|
-
const raw = decodedPaths;
|
|
9039
|
-
if (!Number.isFinite(deviceId) || deviceId <= 0 || typeof profile !== "string" || profile.length === 0 || typeof locationId !== "string" || locationId.length === 0 || !Number.isFinite(hour) || !Array.isArray(raw) || !raw.every((p) => typeof p === "string" && p.length > 0)) return null;
|
|
9040
|
-
return segmentHourRow({
|
|
9041
|
-
key,
|
|
9042
|
-
deviceId,
|
|
9043
|
-
profile,
|
|
9044
|
-
locationId,
|
|
9045
|
-
hourStartMs: hour,
|
|
9046
|
-
paths: raw,
|
|
9047
|
-
sealed: sealedOf(data["sealed"])
|
|
9048
|
-
});
|
|
9049
|
-
}
|
|
9050
|
-
/** The paths of a summary row: one shared empty array, never a per-row
|
|
9051
|
-
* allocation, and never mistakable for "this hour holds nothing". */
|
|
9052
|
-
var NO_PATHS = Object.freeze([]);
|
|
9053
9181
|
function finiteOr0(raw) {
|
|
9054
9182
|
const n = Number(raw);
|
|
9055
9183
|
return Number.isFinite(n) ? n : 0;
|
|
9056
9184
|
}
|
|
9057
9185
|
/**
|
|
9058
|
-
* One row
|
|
9186
|
+
* One persisted row.
|
|
9059
9187
|
*
|
|
9060
|
-
* The identity half is validated
|
|
9061
|
-
*
|
|
9062
|
-
*
|
|
9063
|
-
*
|
|
9064
|
-
*
|
|
9065
|
-
*
|
|
9066
|
-
* {@link SegmentHourLedger.repairAggregates} rewrites it.
|
|
9188
|
+
* The identity half is validated: a row that cannot say which camera, profile,
|
|
9189
|
+
* location and hour it is about is DISCARDED, because a row keyed on garbage
|
|
9190
|
+
* would refuse the write path for an hour that is perfectly fine. The aggregate
|
|
9191
|
+
* half is taken as the database wrote it — a row that predates the aggregate
|
|
9192
|
+
* columns reports zeros, which UNDER-reports its disk (prunes less, never more)
|
|
9193
|
+
* until a listing of that hour re-derives it.
|
|
9067
9194
|
*/
|
|
9068
|
-
function
|
|
9195
|
+
function recordToRow(key, data) {
|
|
9069
9196
|
const deviceId = Number(data["deviceId"]);
|
|
9070
9197
|
const profile = data["profile"];
|
|
9071
9198
|
const locationId = data["locationId"];
|
|
@@ -9077,8 +9204,6 @@ function summaryToRow(key, data) {
|
|
|
9077
9204
|
profile,
|
|
9078
9205
|
locationId,
|
|
9079
9206
|
hourStartMs: hour,
|
|
9080
|
-
paths: NO_PATHS,
|
|
9081
|
-
pathsResident: false,
|
|
9082
9207
|
sealed: sealedOf(data["sealed"]),
|
|
9083
9208
|
bytes: finiteOr0(data["bytes"]),
|
|
9084
9209
|
segments: finiteOr0(data["segments"]),
|
|
@@ -9100,11 +9225,12 @@ var SPEC = {
|
|
|
9100
9225
|
var SegmentHourLedger = class SegmentHourLedger {
|
|
9101
9226
|
ledger;
|
|
9102
9227
|
logger;
|
|
9103
|
-
/** Held for the
|
|
9228
|
+
/** Held for the ARCHIVE reads below, which are queries rather than mirror
|
|
9104
9229
|
* lookups. Every DECISION path (`recordSegment`, `dropSegments`, the prune
|
|
9105
9230
|
* refusals) still answers from the mirror alone — D49 is about gates, and an
|
|
9106
9231
|
* accounting number is not a gate. */
|
|
9107
9232
|
store;
|
|
9233
|
+
listHour;
|
|
9108
9234
|
isLocationUsable;
|
|
9109
9235
|
now;
|
|
9110
9236
|
loadChunk;
|
|
@@ -9118,6 +9244,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9118
9244
|
lanes = /* @__PURE__ */ new Map();
|
|
9119
9245
|
/** Per (device, profile): the hour its writer last appended to. */
|
|
9120
9246
|
writerHours = /* @__PURE__ */ new Map();
|
|
9247
|
+
/**
|
|
9248
|
+
* Segment paths THIS PROCESS has already accounted for, per hour key.
|
|
9249
|
+
*
|
|
9250
|
+
* The `paths` column used to make the write path idempotent for free: an
|
|
9251
|
+
* append checked `held.paths.includes(path)`. With the column gone the
|
|
9252
|
+
* aggregate cannot say whether it already counted a name, so the dedup lives
|
|
9253
|
+
* here — bounded to the hours this process actually writes (two per writer:
|
|
9254
|
+
* the current hour and the one it just left), dropped when the writer moves
|
|
9255
|
+
* on. A duplicate that survives a RESTART is not caught here and is not
|
|
9256
|
+
* silent either: the row is unsealed, and the first listing of the closed
|
|
9257
|
+
* hour re-derives the aggregate exactly ({@link adoptWalkedHour}).
|
|
9258
|
+
*/
|
|
9259
|
+
accounted = /* @__PURE__ */ new Map();
|
|
9121
9260
|
/** Hour keys a write was REFUSED for in this process — never sealed by it. */
|
|
9122
9261
|
refused = /* @__PURE__ */ new Set();
|
|
9123
9262
|
/**
|
|
@@ -9130,6 +9269,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9130
9269
|
constructor(deps) {
|
|
9131
9270
|
this.logger = deps.logger;
|
|
9132
9271
|
this.store = deps.store;
|
|
9272
|
+
this.listHour = deps.listHour ?? null;
|
|
9133
9273
|
this.isLocationUsable = deps.isLocationUsable ?? (() => true);
|
|
9134
9274
|
this.now = deps.now ?? (() => Date.now());
|
|
9135
9275
|
this.loadChunk = deps.loadPacing?.chunkSize ?? LOAD_CHUNK;
|
|
@@ -9148,22 +9288,17 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9148
9288
|
return this.ledger.declare();
|
|
9149
9289
|
}
|
|
9150
9290
|
/**
|
|
9151
|
-
* Boot load —
|
|
9291
|
+
* Boot load — every hour row, in one query.
|
|
9152
9292
|
*
|
|
9153
|
-
*
|
|
9154
|
-
*
|
|
9155
|
-
*
|
|
9293
|
+
* Since D523 the whole row is ten scalars, so there is no second tier to
|
|
9294
|
+
* materialise and no projection to choose: the mirror this returns is the
|
|
9295
|
+
* ledger, entire. It keeps `DurableLedger.load`'s two rules — a failed read
|
|
9296
|
+
* RETURNS WHAT IS ALREADY MIRRORED rather than clearing it, and a malformed
|
|
9297
|
+
* row is skipped rather than repaired.
|
|
9156
9298
|
*
|
|
9157
|
-
* It
|
|
9158
|
-
*
|
|
9159
|
-
*
|
|
9160
|
-
* and a malformed row is skipped rather than repaired.
|
|
9161
|
-
*
|
|
9162
|
-
* It deliberately does NOT replay the archive into {@link RecordingIndex}.
|
|
9163
|
-
* `hydrated` now means "the rows are resident", so a ledger-known hour is
|
|
9164
|
-
* merely *hydratable from T3*: the first read of it pays one point read
|
|
9165
|
-
* ({@link detailForWindow}), and an hour whose tail the ledger is missing is
|
|
9166
|
-
* no longer permanently invisible to playback.
|
|
9299
|
+
* It deliberately does NOT replay the archive into {@link RecordingIndex}: a
|
|
9300
|
+
* row says how MUCH an hour holds, never which files, so only a listing can
|
|
9301
|
+
* seed the index (D523).
|
|
9167
9302
|
*/
|
|
9168
9303
|
async load() {
|
|
9169
9304
|
const startedMs = this.now();
|
|
@@ -9171,7 +9306,6 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9171
9306
|
try {
|
|
9172
9307
|
records = await this.store.query.query({
|
|
9173
9308
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9174
|
-
columns: SEGMENT_HOUR_SUMMARY_COLUMNS,
|
|
9175
9309
|
filter: { limit: this.rowCap }
|
|
9176
9310
|
});
|
|
9177
9311
|
} catch (err) {
|
|
@@ -9188,7 +9322,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9188
9322
|
const end = Math.min(i + this.loadChunk, records.length);
|
|
9189
9323
|
for (let j = i; j < end; j += 1) {
|
|
9190
9324
|
const record = records[j];
|
|
9191
|
-
const row =
|
|
9325
|
+
const row = recordToRow(record.id, record.data);
|
|
9192
9326
|
if (row === null) {
|
|
9193
9327
|
skipped += 1;
|
|
9194
9328
|
continue;
|
|
@@ -9197,7 +9331,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9197
9331
|
}
|
|
9198
9332
|
if (end < records.length) await this.yieldBetween();
|
|
9199
9333
|
}
|
|
9200
|
-
if (skipped > 0) this.logger.warn("recorder: segment-hour
|
|
9334
|
+
if (skipped > 0) this.logger.warn("recorder: segment-hour rows skipped as malformed — they gate NOTHING", { meta: {
|
|
9201
9335
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9202
9336
|
skipped
|
|
9203
9337
|
} });
|
|
@@ -9205,11 +9339,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9205
9339
|
limit: this.rowCap,
|
|
9206
9340
|
loaded: records.length
|
|
9207
9341
|
} });
|
|
9208
|
-
const tail = await this.materializeLiveTail();
|
|
9209
9342
|
this.logger.info("recorder: segment-hour ledger loaded", { meta: {
|
|
9210
9343
|
hours: this.ledger.size,
|
|
9211
|
-
pathsLoaded: false,
|
|
9212
|
-
liveTailHours: tail,
|
|
9213
9344
|
skipped,
|
|
9214
9345
|
queryMs: queriedMs - startedMs,
|
|
9215
9346
|
ms: this.now() - startedMs,
|
|
@@ -9217,8 +9348,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9217
9348
|
} });
|
|
9218
9349
|
return this.ledger.snapshot();
|
|
9219
9350
|
}
|
|
9220
|
-
/** Every hour
|
|
9221
|
-
* `pathsResident` is false carries real aggregates and empty `paths`. */
|
|
9351
|
+
/** Every hour row held. Pure RAM. */
|
|
9222
9352
|
snapshot() {
|
|
9223
9353
|
return this.ledger.snapshot();
|
|
9224
9354
|
}
|
|
@@ -9228,10 +9358,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9228
9358
|
* The archive's answer to "what should placement plan for", and the reason it
|
|
9229
9359
|
* is here rather than a `census` loop or an index read:
|
|
9230
9360
|
*
|
|
9231
|
-
* - `RecordingIndex.accountingByProfile` is the wrong authority.
|
|
9232
|
-
*
|
|
9233
|
-
*
|
|
9234
|
-
*
|
|
9361
|
+
* - `RecordingIndex.accountingByProfile` is the wrong authority. It holds
|
|
9362
|
+
* the live tail plus whatever a reader hydrated, so a camera nobody has
|
|
9363
|
+
* viewed since boot reports NO profiles — and the placement pass then
|
|
9364
|
+
* never plans it (D295 §3a #9, D299).
|
|
9235
9365
|
* - `census({ deviceId, profiles: [p] })` is the right authority but the
|
|
9236
9366
|
* wrong shape for a 30 s tick: it is one SQL round trip per (camera,
|
|
9237
9367
|
* profile), so a 16-camera fleet pays 48 SERIAL trips over the UDS door
|
|
@@ -9239,10 +9369,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9239
9369
|
* 240003` pass (see `PLACEMENT_STORE_TIMEOUT_MS`), and it grows with the
|
|
9240
9370
|
* fleet.
|
|
9241
9371
|
*
|
|
9242
|
-
* So it is answered from
|
|
9243
|
-
* precisely so that O(writers × days) questions cost no I/O: ~10–15 MB and
|
|
9244
|
-
* ~47 k rows even at 30 TB, one fold, no query. A hydrated segment row is not
|
|
9245
|
-
* needed to know that an hour exists.
|
|
9372
|
+
* So it is answered from the mirror: one fold, no query, no I/O.
|
|
9246
9373
|
*
|
|
9247
9374
|
* An EMPTY result for a device is a real answer — "the archive knows no hour
|
|
9248
9375
|
* for this camera" — and it is also what a ledger whose load failed reports,
|
|
@@ -9259,136 +9386,111 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9259
9386
|
}
|
|
9260
9387
|
return out;
|
|
9261
9388
|
}
|
|
9262
|
-
/** Does the ledger hold a row for this hour at all? Pure RAM
|
|
9263
|
-
* storage-walk repair consults before it upserts a rebuilt hour. */
|
|
9389
|
+
/** Does the ledger hold a row for this hour at all? Pure RAM. */
|
|
9264
9390
|
knowsHour(deviceId, profile, locationId, hour) {
|
|
9265
9391
|
return this.ledger.has(segmentHourKey(deviceId, profile, locationId, hour));
|
|
9266
9392
|
}
|
|
9267
|
-
/**
|
|
9268
|
-
*
|
|
9269
|
-
|
|
9270
|
-
|
|
9271
|
-
* {@link detailForWindow} answers per (device, window) and drops the profile
|
|
9272
|
-
* on the way out, which is fine for a reader assembling a playlist and wrong
|
|
9273
|
-
* for a caller that must act on one hour ROW: the ghost-location reconcile
|
|
9274
|
-
* decides per `(deviceId, profile, locationId, hour)` and hands the resulting
|
|
9275
|
-
* paths straight back to {@link dropSegments} / {@link repointSegments},
|
|
9276
|
-
* whose bucket key is that same tuple.
|
|
9277
|
-
*
|
|
9278
|
-
* `null` is "known, and its detail could not be had" — never an empty array.
|
|
9279
|
-
* The one caller FORGETS rows on this answer, and a failed point read that
|
|
9280
|
-
* came back as `[]` would read as "this hour holds nothing", which is the
|
|
9281
|
-
* D148 prune-on-no-evidence this ledger refuses to make.
|
|
9282
|
-
*/
|
|
9283
|
-
async hourPaths(deviceId, profile, locationId, hour) {
|
|
9284
|
-
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9285
|
-
if (!this.ledger.has(key)) return null;
|
|
9286
|
-
const held = await this.resident(key, deviceId);
|
|
9287
|
-
return held === null ? null : held.paths;
|
|
9393
|
+
/** One held hour row, or `undefined`. Pure RAM — since D523 a row carries
|
|
9394
|
+
* everything it has. */
|
|
9395
|
+
hourRow(deviceId, profile, locationId, hour) {
|
|
9396
|
+
return this.ledger.get(segmentHourKey(deviceId, profile, locationId, hour));
|
|
9288
9397
|
}
|
|
9289
9398
|
/**
|
|
9290
|
-
*
|
|
9291
|
-
* cold-hour detail read that replaces the boot replay (D287).
|
|
9399
|
+
* A LISTING of a CLOSED hour is adopted as that hour's account, and SEALS it.
|
|
9292
9400
|
*
|
|
9293
|
-
*
|
|
9294
|
-
*
|
|
9295
|
-
* the
|
|
9296
|
-
*
|
|
9297
|
-
*
|
|
9401
|
+
* The directory is the authority for which segments exist (D523), so a
|
|
9402
|
+
* listing of an hour that is OVER is the one complete answer there is — it
|
|
9403
|
+
* replaces the row's aggregate rather than merging with it, which is what
|
|
9404
|
+
* makes an hour that was over-counted (a duplicate finalize across a restart)
|
|
9405
|
+
* or under-counted (a writer torn down mid-hour) self-correcting. The caller
|
|
9406
|
+
* guarantees the hour is CLOSED: a live hour's directory races its writer, so
|
|
9407
|
+
* a listing of it is a lower bound.
|
|
9298
9408
|
*
|
|
9299
|
-
*
|
|
9300
|
-
*
|
|
9301
|
-
*
|
|
9302
|
-
*
|
|
9303
|
-
|
|
9304
|
-
async detailForWindow(deviceId, fromMs, toMs) {
|
|
9305
|
-
const first = hourStartMs(fromMs);
|
|
9306
|
-
const out = [];
|
|
9307
|
-
const wanted = [];
|
|
9308
|
-
for (const row of this.ledger.snapshot()) {
|
|
9309
|
-
if (row.deviceId !== deviceId) continue;
|
|
9310
|
-
if (row.hourStartMs < first || row.hourStartMs >= toMs) continue;
|
|
9311
|
-
if (row.pathsResident) {
|
|
9312
|
-
out.push(detailOf(row));
|
|
9313
|
-
continue;
|
|
9314
|
-
}
|
|
9315
|
-
wanted.push(row.key);
|
|
9316
|
-
}
|
|
9317
|
-
for (const row of await this.materialize(wanted, deviceId)) out.push(detailOf(row));
|
|
9318
|
-
return out;
|
|
9319
|
-
}
|
|
9320
|
-
/**
|
|
9321
|
-
* Union a CLOSED hour the storage walk read into its row, and SEAL it.
|
|
9322
|
-
*
|
|
9323
|
-
* The reader repairs the durable index it just reconstructed, so the next
|
|
9324
|
-
* cold read of this hour is a point read instead of another `readdir`
|
|
9325
|
-
* (`docs/design/2026-08-29-reference-nvr-recording-model.md` §4.2), and it
|
|
9326
|
-
* seals the row because a directory read of a closed hour is the one
|
|
9327
|
-
* complete answer there is. The caller guarantees the hour is CLOSED — a
|
|
9328
|
-
* live hour's directory races its writer, so a walk of it is a lower bound.
|
|
9329
|
-
*
|
|
9330
|
-
* A row the ledger already holds is UNIONED, never replaced: the walk can
|
|
9331
|
-
* only add (D148) — a path the row names and the directory lacks stays,
|
|
9332
|
-
* because "the reader did not see it" is not evidence it is gone. This is
|
|
9333
|
-
* the repair D445 described and did not perform: its predecessor returned
|
|
9334
|
-
* on any known key, so a row known INCOMPLETELY — the only kind that needs
|
|
9335
|
-
* repairing — was exactly the row it refused to touch, and the export's
|
|
9336
|
-
* `forgottenSegments=9758` was counted again the next morning.
|
|
9337
|
-
*
|
|
9338
|
-
* `null` when the row is known and its detail could not be read: nothing
|
|
9339
|
-
* is written from a row whose paths are not resident.
|
|
9409
|
+
* A listing that would erase more than {@link HOUR_LISTING_MAX_SHRINK_SHARE}
|
|
9410
|
+
* of what the row accounts for is REFUSED and named. That is the one thing a
|
|
9411
|
+
* listing cannot be trusted about: a root that has gone away and a bucket a
|
|
9412
|
+
* dead round half-deleted both answer short, and neither is distinguishable
|
|
9413
|
+
* from an hour that is simply smaller than the row thought.
|
|
9340
9414
|
*
|
|
9341
9415
|
* @seal-authority closed-hour-listing — a directory read of an hour that is
|
|
9342
9416
|
* OVER is the one complete answer there is; the caller guarantees the hour is
|
|
9343
|
-
* closed (D451, D521).
|
|
9417
|
+
* closed (D451, D521, D523).
|
|
9344
9418
|
*/
|
|
9345
9419
|
async adoptWalkedHour(deviceId, profile, locationId, hour, paths) {
|
|
9346
9420
|
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9421
|
+
const identity = {
|
|
9422
|
+
key,
|
|
9423
|
+
deviceId,
|
|
9424
|
+
profile,
|
|
9425
|
+
locationId,
|
|
9426
|
+
hourStartMs: hour
|
|
9427
|
+
};
|
|
9428
|
+
const walked = hourAggregate(paths);
|
|
9347
9429
|
return this.serialized(key, async () => {
|
|
9348
|
-
|
|
9349
|
-
|
|
9430
|
+
const held = this.ledger.get(key);
|
|
9431
|
+
if (held === void 0) {
|
|
9432
|
+
if (walked.segments === 0) return {
|
|
9350
9433
|
created: false,
|
|
9351
9434
|
added: 0,
|
|
9352
9435
|
sealedNow: false
|
|
9353
9436
|
};
|
|
9354
|
-
|
|
9355
|
-
|
|
9356
|
-
deviceId,
|
|
9357
|
-
profile,
|
|
9358
|
-
locationId,
|
|
9359
|
-
hourStartMs: hour,
|
|
9360
|
-
paths: uniquePaths(paths),
|
|
9437
|
+
await this.ledger.put(hourRowOf(identity, {
|
|
9438
|
+
...walked,
|
|
9361
9439
|
sealed: true
|
|
9362
|
-
});
|
|
9363
|
-
|
|
9364
|
-
this.logger.info("recorder: adopted a closed hour the disk walk rebuilt and the ledger never held", {
|
|
9440
|
+
}));
|
|
9441
|
+
this.logger.info("recorder: adopted a closed hour the disk listing rebuilt and the ledger never held", {
|
|
9365
9442
|
tags: { deviceId },
|
|
9366
9443
|
meta: {
|
|
9367
9444
|
profile,
|
|
9368
9445
|
locationId,
|
|
9369
9446
|
hourStartMs: hour,
|
|
9370
|
-
segments:
|
|
9371
|
-
bytes:
|
|
9447
|
+
segments: walked.segments,
|
|
9448
|
+
bytes: walked.bytes
|
|
9372
9449
|
}
|
|
9373
9450
|
});
|
|
9374
9451
|
return {
|
|
9375
9452
|
created: true,
|
|
9376
|
-
added:
|
|
9453
|
+
added: walked.segments,
|
|
9377
9454
|
sealedNow: true
|
|
9378
9455
|
};
|
|
9379
9456
|
}
|
|
9380
|
-
const
|
|
9381
|
-
if (held
|
|
9382
|
-
|
|
9383
|
-
|
|
9384
|
-
|
|
9457
|
+
const shrink = held.segments - walked.segments;
|
|
9458
|
+
if (shrink > 0 && shrink > held.segments * .5) {
|
|
9459
|
+
this.logger.warn("recorder: hour listing REFUSED — it accounts for less than half of what this hour row holds, which is what an unmounted root looks like; the row stands and stays unsealed", {
|
|
9460
|
+
tags: { deviceId },
|
|
9461
|
+
meta: {
|
|
9462
|
+
profile,
|
|
9463
|
+
locationId,
|
|
9464
|
+
hourStartMs: hour,
|
|
9465
|
+
held: held.segments,
|
|
9466
|
+
listed: walked.segments,
|
|
9467
|
+
maxShare: HOUR_LISTING_MAX_SHRINK_SHARE
|
|
9468
|
+
}
|
|
9469
|
+
});
|
|
9470
|
+
return {
|
|
9471
|
+
created: false,
|
|
9472
|
+
added: 0,
|
|
9473
|
+
sealedNow: false
|
|
9474
|
+
};
|
|
9475
|
+
}
|
|
9476
|
+
const added = Math.max(0, walked.segments - held.segments);
|
|
9477
|
+
if (added === 0 && shrink === 0 && held.sealed) return {
|
|
9385
9478
|
created: false,
|
|
9386
9479
|
added: 0,
|
|
9387
9480
|
sealedNow: false
|
|
9388
9481
|
};
|
|
9389
|
-
|
|
9390
|
-
|
|
9391
|
-
|
|
9482
|
+
if (shrink > 0) this.logger.info("recorder: hour listing accounts for FEWER segments than the row did — the directory decides", {
|
|
9483
|
+
tags: { deviceId },
|
|
9484
|
+
meta: {
|
|
9485
|
+
profile,
|
|
9486
|
+
locationId,
|
|
9487
|
+
hourStartMs: hour,
|
|
9488
|
+
held: held.segments,
|
|
9489
|
+
listed: walked.segments
|
|
9490
|
+
}
|
|
9491
|
+
});
|
|
9492
|
+
await this.ledger.put(hourRowOf(identity, {
|
|
9493
|
+
...walked,
|
|
9392
9494
|
sealed: true
|
|
9393
9495
|
}));
|
|
9394
9496
|
return {
|
|
@@ -9418,9 +9520,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9418
9520
|
}
|
|
9419
9521
|
/**
|
|
9420
9522
|
* A finalize this ledger was NEVER handed — the fire-and-forget lane refused
|
|
9421
|
-
* it. The hour's row is short by that
|
|
9422
|
-
* party that knows, so the hour must not be sealed by it; a
|
|
9423
|
-
*
|
|
9523
|
+
* it. The hour's row is short by that segment and this process is the only
|
|
9524
|
+
* party that knows, so the hour must not be sealed by it; a listing corrects
|
|
9525
|
+
* it instead, which is exactly what an unsealed row is for.
|
|
9424
9526
|
*/
|
|
9425
9527
|
markRefused(row) {
|
|
9426
9528
|
this.refused.add(hourOf(row).key);
|
|
@@ -9448,7 +9550,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9448
9550
|
marked += 1;
|
|
9449
9551
|
}
|
|
9450
9552
|
if (marked === 0) return 0;
|
|
9451
|
-
this.logger.warn("recorder: segments this writer produced never reached the hour ledger — these hours stay UNSEALED and a
|
|
9553
|
+
this.logger.warn("recorder: segments this writer produced never reached the hour ledger — these hours stay UNSEALED and a listing corrects them", {
|
|
9452
9554
|
tags: { deviceId: span.deviceId },
|
|
9453
9555
|
meta: {
|
|
9454
9556
|
profile: span.profile,
|
|
@@ -9481,10 +9583,11 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9481
9583
|
graceMs: WRITER_HOUR_ENTRY_GRACE_MS
|
|
9482
9584
|
}
|
|
9483
9585
|
});
|
|
9586
|
+
this.accounted.delete(target.key);
|
|
9484
9587
|
return;
|
|
9485
9588
|
}
|
|
9486
9589
|
if (this.refused.has(target.key) || this.refusedWriterHours.has(writerHourKey(target.deviceId, target.profile, target.hourStartMs))) {
|
|
9487
|
-
this.logger.warn("recorder: hour left unsealed — a ledger write for it was refused, so its row is short and a
|
|
9590
|
+
this.logger.warn("recorder: hour left unsealed — a ledger write for it was refused, so its row is short and a listing must correct it", {
|
|
9488
9591
|
tags: { deviceId: target.deviceId },
|
|
9489
9592
|
meta: {
|
|
9490
9593
|
profile: target.profile,
|
|
@@ -9492,10 +9595,12 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9492
9595
|
hourStartMs: target.hourStartMs
|
|
9493
9596
|
}
|
|
9494
9597
|
});
|
|
9598
|
+
this.accounted.delete(target.key);
|
|
9495
9599
|
return;
|
|
9496
9600
|
}
|
|
9497
|
-
const held =
|
|
9498
|
-
|
|
9601
|
+
const held = this.ledger.get(target.key);
|
|
9602
|
+
this.accounted.delete(target.key);
|
|
9603
|
+
if (held === void 0 || held.sealed) return;
|
|
9499
9604
|
await this.ledger.put({
|
|
9500
9605
|
...held,
|
|
9501
9606
|
sealed: true
|
|
@@ -9523,11 +9628,25 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9523
9628
|
/**
|
|
9524
9629
|
* Run `fn` after every mutation already queued on `key`.
|
|
9525
9630
|
*
|
|
9526
|
-
*
|
|
9527
|
-
*
|
|
9528
|
-
*
|
|
9529
|
-
*
|
|
9530
|
-
*
|
|
9631
|
+
* It was the lost-update guard: a row was read from the STORE, folded into
|
|
9632
|
+
* and written back with awaits in between, so two such runs on one key
|
|
9633
|
+
* interleaved and the second write carried a path set that never saw the
|
|
9634
|
+
* first — the sparse rows D451 found. Since D523 there is no store read on
|
|
9635
|
+
* any of these paths (the mirror holds the whole row) and `DurableLedger.put`
|
|
9636
|
+
* sets the mirror synchronously before it awaits the store, so every
|
|
9637
|
+
* get→fold→put here is synchronous up to the mirror and the interleaving
|
|
9638
|
+
* window is gone.
|
|
9639
|
+
*
|
|
9640
|
+
* What the lane still buys is the ORDER of the durable writes per hour key —
|
|
9641
|
+
* a rollover seal fired without an await must not land over an append that
|
|
9642
|
+
* was queued before it — and a handle for {@link flush} to wait on at
|
|
9643
|
+
* shutdown. A rejected step must not poison the lane: the next step still
|
|
9644
|
+
* runs.
|
|
9645
|
+
*
|
|
9646
|
+
* Note for anyone tempted to delete it: `segment-hour-seal.spec.ts` does NOT
|
|
9647
|
+
* catch its removal any more, and says so. Pinning durable write ORDER needs
|
|
9648
|
+
* a store whose latency a test can control, which `fakeSettingsStore` has no
|
|
9649
|
+
* way to express.
|
|
9531
9650
|
*/
|
|
9532
9651
|
serialized(key, fn) {
|
|
9533
9652
|
const run = (this.lanes.get(key) ?? Promise.resolve()).then(fn, fn);
|
|
@@ -9539,116 +9658,59 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9539
9658
|
return run;
|
|
9540
9659
|
}
|
|
9541
9660
|
/**
|
|
9542
|
-
*
|
|
9661
|
+
* List the hour directories of `rows`, paced, and expand them into segment
|
|
9662
|
+
* rows.
|
|
9543
9663
|
*
|
|
9544
|
-
*
|
|
9545
|
-
*
|
|
9546
|
-
*
|
|
9547
|
-
*
|
|
9548
|
-
* walk, or refusing a write) is strictly safer than a guess.
|
|
9549
|
-
*
|
|
9550
|
-
* A row the store returns but this cannot parse is EVICTED from the mirror
|
|
9551
|
-
* rather than left as a summary that can never become resident: it re-seeds
|
|
9552
|
-
* cold on the next write, which is `DurableLedger`'s rule 4 and the only
|
|
9553
|
-
* escape from an hour permanently frozen against its own writer.
|
|
9664
|
+
* The one place the ledger touches the array. An hour whose listing FAILED
|
|
9665
|
+
* takes no part in the caller's pass and is SAID — a branch that drops work
|
|
9666
|
+
* silently reads as "there was nothing there", which for a drain or a prune
|
|
9667
|
+
* is the difference between "moved everything" and "moved what it could see".
|
|
9554
9668
|
*/
|
|
9555
|
-
async
|
|
9669
|
+
async listHours(rows, reason) {
|
|
9556
9670
|
const out = [];
|
|
9557
|
-
if (
|
|
9558
|
-
const
|
|
9559
|
-
|
|
9560
|
-
|
|
9561
|
-
|
|
9562
|
-
|
|
9563
|
-
|
|
9564
|
-
|
|
9565
|
-
|
|
9566
|
-
|
|
9567
|
-
|
|
9568
|
-
|
|
9569
|
-
|
|
9570
|
-
|
|
9571
|
-
|
|
9572
|
-
...tags,
|
|
9671
|
+
if (rows.length === 0) return out;
|
|
9672
|
+
const lister = this.listHour;
|
|
9673
|
+
if (lister === null) {
|
|
9674
|
+
this.logger.warn("recorder: no hour lister on this ledger — nothing that needs segment NAMES can run here", { meta: {
|
|
9675
|
+
hours: rows.length,
|
|
9676
|
+
reason
|
|
9677
|
+
} });
|
|
9678
|
+
return out;
|
|
9679
|
+
}
|
|
9680
|
+
let listed = 0;
|
|
9681
|
+
for (const row of rows) {
|
|
9682
|
+
const paths = await lister(row.locationId, row.deviceId, row.profile, row.hourStartMs);
|
|
9683
|
+
if (paths === null) {
|
|
9684
|
+
this.logger.warn("recorder: hour directory could not be listed — this hour takes NO part in this pass", {
|
|
9685
|
+
tags: { deviceId: row.deviceId },
|
|
9573
9686
|
meta: {
|
|
9574
|
-
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
|
|
9687
|
+
profile: row.profile,
|
|
9688
|
+
locationId: row.locationId,
|
|
9689
|
+
hourStartMs: row.hourStartMs,
|
|
9690
|
+
accounted: row.segments,
|
|
9691
|
+
reason
|
|
9578
9692
|
}
|
|
9579
9693
|
});
|
|
9580
|
-
|
|
9694
|
+
continue;
|
|
9581
9695
|
}
|
|
9582
|
-
|
|
9583
|
-
|
|
9584
|
-
|
|
9585
|
-
|
|
9586
|
-
|
|
9587
|
-
|
|
9588
|
-
|
|
9589
|
-
|
|
9590
|
-
|
|
9696
|
+
const { rows: segments, unparseable } = segmentRowsOf(row.locationId, paths);
|
|
9697
|
+
if (unparseable > 0) this.logger.warn("recorder: hour directory holds names that are not segments", {
|
|
9698
|
+
tags: { deviceId: row.deviceId },
|
|
9699
|
+
meta: {
|
|
9700
|
+
profile: row.profile,
|
|
9701
|
+
locationId: row.locationId,
|
|
9702
|
+
hourStartMs: row.hourStartMs,
|
|
9703
|
+
unparseable,
|
|
9704
|
+
reason
|
|
9591
9705
|
}
|
|
9592
|
-
|
|
9593
|
-
|
|
9594
|
-
|
|
9595
|
-
if (
|
|
9706
|
+
});
|
|
9707
|
+
out.push(...segments);
|
|
9708
|
+
listed += 1;
|
|
9709
|
+
if (listed % 8 === 0) await this.yieldBetween();
|
|
9596
9710
|
}
|
|
9597
9711
|
return out;
|
|
9598
9712
|
}
|
|
9599
9713
|
/**
|
|
9600
|
-
* Materialise every hour these devices hold, and report how many stayed
|
|
9601
|
-
* summaries.
|
|
9602
|
-
*
|
|
9603
|
-
* The reconcile is the one path that DELETES durable rows, so it may not run
|
|
9604
|
-
* against a mirror that only half knows what it holds. This is a whole-device
|
|
9605
|
-
* read and it is deliberately confined here: `reconcileFromIndex` follows an
|
|
9606
|
-
* operator-armed rescan of that same device — a recursive `readdir` over
|
|
9607
|
-
* >1M files — so a paged point read alongside it is noise, and it is the one
|
|
9608
|
-
* caller whose correctness depends on completeness.
|
|
9609
|
-
*/
|
|
9610
|
-
async materializeDevices(deviceIds) {
|
|
9611
|
-
const keys = [];
|
|
9612
|
-
for (const row of this.ledger.snapshot()) if (!row.pathsResident && deviceIds.has(row.deviceId)) keys.push(row.key);
|
|
9613
|
-
if (keys.length === 0) return;
|
|
9614
|
-
await this.materialize(keys);
|
|
9615
|
-
const stillSummary = /* @__PURE__ */ new Map();
|
|
9616
|
-
for (const row of this.ledger.snapshot()) {
|
|
9617
|
-
if (row.pathsResident || !deviceIds.has(row.deviceId)) continue;
|
|
9618
|
-
stillSummary.set(row.deviceId, (stillSummary.get(row.deviceId) ?? 0) + 1);
|
|
9619
|
-
}
|
|
9620
|
-
for (const [deviceId, unreadable] of stillSummary) this.logger.warn("recorder: hours without readable detail take NO part in this reconcile — they are neither pruned nor unioned", {
|
|
9621
|
-
tags: { deviceId },
|
|
9622
|
-
meta: {
|
|
9623
|
-
asked: keys.length,
|
|
9624
|
-
unreadable
|
|
9625
|
-
}
|
|
9626
|
-
});
|
|
9627
|
-
}
|
|
9628
|
-
/** The hour a write path may touch: current + previous. Materialised at boot
|
|
9629
|
-
* so {@link recordSegment}'s dedup gate answers from RAM (D49). */
|
|
9630
|
-
async materializeLiveTail() {
|
|
9631
|
-
const oldest = hourStartMs(this.now()) - (LIVE_TAIL_HOURS - 1) * HOUR_MS$4;
|
|
9632
|
-
const keys = [];
|
|
9633
|
-
for (const row of this.ledger.snapshot()) if (row.hourStartMs >= oldest && !row.pathsResident) keys.push(row.key);
|
|
9634
|
-
return (await this.materialize(keys)).length;
|
|
9635
|
-
}
|
|
9636
|
-
/**
|
|
9637
|
-
* Promote one HELD row to paths-resident, or answer `null`.
|
|
9638
|
-
*
|
|
9639
|
-
* `null` means "this hour is known and its detail could not be had" — the
|
|
9640
|
-
* only honest answer to a failed point read, and every caller turns it into
|
|
9641
|
-
* "do nothing". It is deliberately distinct from `this.ledger.get(key) ===
|
|
9642
|
-
* undefined`, which means "no such hour" and is what licenses a fresh row.
|
|
9643
|
-
*/
|
|
9644
|
-
async resident(key, deviceId) {
|
|
9645
|
-
const held = this.ledger.get(key);
|
|
9646
|
-
if (held !== void 0 && held.pathsResident) return held;
|
|
9647
|
-
await this.materialize([key], deviceId);
|
|
9648
|
-
const after = this.ledger.get(key);
|
|
9649
|
-
return after !== void 0 && after.pathsResident ? after : null;
|
|
9650
|
-
}
|
|
9651
|
-
/**
|
|
9652
9714
|
* Bytes / count / oldest / newest across a SET of storage locations, as ONE
|
|
9653
9715
|
* `SUM`/`COUNT`/`MIN`/`MAX` over the hour rows.
|
|
9654
9716
|
*
|
|
@@ -9689,9 +9751,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9689
9751
|
* ## Why two phases
|
|
9690
9752
|
*
|
|
9691
9753
|
* Phase one reads only the small columns and walks oldest-first until the
|
|
9692
|
-
* accumulated bytes cover the target — normally one page.
|
|
9693
|
-
*
|
|
9694
|
-
*
|
|
9754
|
+
* accumulated bytes cover the target — normally one page. Phase two LISTS the
|
|
9755
|
+
* hour directories phase one chose, which is the only thing that can name the
|
|
9756
|
+
* files eviction is about to delete (D523).
|
|
9695
9757
|
*
|
|
9696
9758
|
* `exhausted` means the stream ran out before the target was met — the
|
|
9697
9759
|
* caller's stop condition, and the one thing a bounded read must still be
|
|
@@ -9725,11 +9787,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9725
9787
|
/**
|
|
9726
9788
|
* Every camera the ledger holds an hour for on these locations.
|
|
9727
9789
|
*
|
|
9728
|
-
* Answered from
|
|
9729
|
-
*
|
|
9730
|
-
*
|
|
9731
|
-
*
|
|
9732
|
-
* would have to be turned into a verdict.
|
|
9790
|
+
* Answered from the mirror, exactly like {@link profilesWithFootage}, and for
|
|
9791
|
+
* the same reason: this is an O(writers × days) question and it must cost no
|
|
9792
|
+
* I/O. It runs on the disk-pressure path, where an I/O that fails would have
|
|
9793
|
+
* to be turned into a verdict.
|
|
9733
9794
|
*
|
|
9734
9795
|
* An EMPTY answer is "the mirror knows no hour here", which is also what a
|
|
9735
9796
|
* ledger whose load failed reports — so the caller must treat it as "decide
|
|
@@ -9744,7 +9805,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9744
9805
|
* Bytes / segments / oldest / newest per CAMERA on these locations — the
|
|
9745
9806
|
* per-camera half of the storage page, answered by the ARCHIVE (D418).
|
|
9746
9807
|
*
|
|
9747
|
-
* ## Why the
|
|
9808
|
+
* ## Why the mirror fold and not `census({ deviceId })` per camera
|
|
9748
9809
|
*
|
|
9749
9810
|
* `census` is the right authority for a SCOPE, and it cannot express this
|
|
9750
9811
|
* scope: {@link SegmentHourLedger.predicate} takes ONE `locationId`, so
|
|
@@ -9754,14 +9815,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9754
9815
|
* accounts for. Per camera per volume that is `cameras × volumes` SQL
|
|
9755
9816
|
* aggregates for one operator click, and it grows with the fleet.
|
|
9756
9817
|
*
|
|
9757
|
-
* The
|
|
9758
|
-
*
|
|
9759
|
-
*
|
|
9760
|
-
*
|
|
9761
|
-
* over the hour rows the mirror holds — O(writers × profiles × hours), ~19 k
|
|
9762
|
-
* on the live hub (see {@link REPAIR_MAX_ROWS}) against 98 445 segments,
|
|
9763
|
-
* i.e. it does NOT scale with the segment count — and it is capped for good
|
|
9764
|
-
* by {@link SEGMENT_HOUR_SUMMARY_LIMIT}, the mirror's own ceiling.
|
|
9818
|
+
* The mirror answers all of them in ONE pass with no I/O, exactly as
|
|
9819
|
+
* {@link devicesWithFootage} and {@link profilesWithFootage} already do, and
|
|
9820
|
+
* it is the same rows `accountingForLocations` sums in SQL — the mirror is
|
|
9821
|
+
* write-behind, so it is never BEHIND the store.
|
|
9765
9822
|
*
|
|
9766
9823
|
* ## What an empty answer means
|
|
9767
9824
|
*
|
|
@@ -9835,44 +9892,28 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9835
9892
|
rows: [],
|
|
9836
9893
|
exhausted: true
|
|
9837
9894
|
};
|
|
9838
|
-
const full = await this.store.query.query({
|
|
9839
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9840
|
-
filter: {
|
|
9841
|
-
whereIn: { key: chosen },
|
|
9842
|
-
limit: chosen.length
|
|
9843
|
-
}
|
|
9844
|
-
});
|
|
9845
|
-
const rows = [];
|
|
9846
|
-
for (const record of full) {
|
|
9847
|
-
const hour = recordToRow(record.id, record.data);
|
|
9848
|
-
if (hour === null) continue;
|
|
9849
|
-
this.ledger.stage(hour);
|
|
9850
|
-
for (const path of hour.paths) {
|
|
9851
|
-
const parsed = parseSegmentPath(path);
|
|
9852
|
-
if (parsed === null) continue;
|
|
9853
|
-
rows.push({
|
|
9854
|
-
deviceId: parsed.deviceId,
|
|
9855
|
-
profile: parsed.profile,
|
|
9856
|
-
startMs: parsed.startMs,
|
|
9857
|
-
durMs: parsed.durMs,
|
|
9858
|
-
bytes: parsed.bytes,
|
|
9859
|
-
path,
|
|
9860
|
-
locationId: hour.locationId
|
|
9861
|
-
});
|
|
9862
|
-
}
|
|
9863
|
-
}
|
|
9864
9895
|
return {
|
|
9865
|
-
rows:
|
|
9896
|
+
rows: (await this.listHours(this.rowsOfKeys(chosen), "oldest-segments")).toSorted((a, b) => a.startMs - b.startMs),
|
|
9866
9897
|
exhausted
|
|
9867
9898
|
};
|
|
9868
9899
|
}
|
|
9900
|
+
/** Mirror rows for keys a probe named. A key the mirror does not hold is
|
|
9901
|
+
* skipped: the probe and the mirror describe the same table, and an hour the
|
|
9902
|
+
* mirror never loaded is one this process may not act on. */
|
|
9903
|
+
rowsOfKeys(keys) {
|
|
9904
|
+
const out = [];
|
|
9905
|
+
for (const key of keys) {
|
|
9906
|
+
const row = this.ledger.get(key);
|
|
9907
|
+
if (row !== void 0) out.push(row);
|
|
9908
|
+
}
|
|
9909
|
+
return out;
|
|
9910
|
+
}
|
|
9869
9911
|
/**
|
|
9870
9912
|
* The predicate half of an {@link ArchiveScope}, as one filter.
|
|
9871
9913
|
*
|
|
9872
9914
|
* REFUSES an empty scope rather than compiling to "the whole collection".
|
|
9873
|
-
* There is no read here that may be unscoped:
|
|
9874
|
-
*
|
|
9875
|
-
* how that query comes back under a new name.
|
|
9915
|
+
* There is no read here that may be unscoped: an unscoped stream lists every
|
|
9916
|
+
* hour directory on the hub.
|
|
9876
9917
|
*/
|
|
9877
9918
|
static predicate(scope) {
|
|
9878
9919
|
const where = {};
|
|
@@ -9885,6 +9926,15 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9885
9926
|
...profiles.length > 0 ? { whereIn: { profile: [...profiles] } } : {}
|
|
9886
9927
|
};
|
|
9887
9928
|
}
|
|
9929
|
+
/** Does this row fall inside `scope`? The mirror-side twin of
|
|
9930
|
+
* {@link SegmentHourLedger.predicate}, for the passes that fold the mirror
|
|
9931
|
+
* instead of querying. */
|
|
9932
|
+
static inScope(row, scope) {
|
|
9933
|
+
if (scope.locationId !== void 0 && row.locationId !== scope.locationId) return false;
|
|
9934
|
+
if (scope.deviceId !== void 0 && row.deviceId !== scope.deviceId) return false;
|
|
9935
|
+
const profiles = scope.profiles ?? [];
|
|
9936
|
+
return profiles.length === 0 || profiles.includes(row.profile);
|
|
9937
|
+
}
|
|
9888
9938
|
/**
|
|
9889
9939
|
* What the ARCHIVE holds in this scope — segments and bytes, as ONE
|
|
9890
9940
|
* `SUM` over the hour rows.
|
|
@@ -9928,6 +9978,15 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9928
9978
|
/**
|
|
9929
9979
|
* Every segment in `scope`, oldest-first, ONE PAGE at a time.
|
|
9930
9980
|
*
|
|
9981
|
+
* ## Two phases, and which authority answers which
|
|
9982
|
+
*
|
|
9983
|
+
* Phase one asks the TABLE which hours the scope holds, oldest-first — the
|
|
9984
|
+
* ordered aggregate no directory read can answer. Phase two LISTS those hour
|
|
9985
|
+
* directories, which is the only authority for the file names the caller is
|
|
9986
|
+
* about to move or delete (D523). An hour whose listing fails is skipped and
|
|
9987
|
+
* said; the caller's own N-versus-M report is what makes the short walk
|
|
9988
|
+
* visible.
|
|
9989
|
+
*
|
|
9931
9990
|
* ## Why a keyset cursor and not `offset`
|
|
9932
9991
|
*
|
|
9933
9992
|
* Both callers MUTATE the scope while they walk it: the drain re-points each
|
|
@@ -9944,10 +10003,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9944
10003
|
*
|
|
9945
10004
|
* ## What a failed read does
|
|
9946
10005
|
*
|
|
9947
|
-
* It ENDS the stream (D49: a fallible read destroys nothing)
|
|
9948
|
-
*
|
|
9949
|
-
*
|
|
9950
|
-
* from a scope that was genuinely empty.
|
|
10006
|
+
* It ENDS the stream (D49: a fallible read destroys nothing). A stream that
|
|
10007
|
+
* swallowed the error and returned would be indistinguishable from a scope
|
|
10008
|
+
* that was genuinely empty.
|
|
9951
10009
|
*/
|
|
9952
10010
|
async *streamSegments(scope) {
|
|
9953
10011
|
const predicate = SegmentHourLedger.predicate(scope);
|
|
@@ -10000,8 +10058,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10000
10058
|
if (nextCursor !== cursor) emittedAtCursor = /* @__PURE__ */ new Set();
|
|
10001
10059
|
for (const record of fresh) if (hourOfRecord(record) === nextCursor) emittedAtCursor.add(record.id);
|
|
10002
10060
|
cursor = nextCursor;
|
|
10003
|
-
const rows = await this.
|
|
10004
|
-
if (rows.length > 0) yield rows;
|
|
10061
|
+
const rows = await this.listHours(this.rowsOfKeys(fresh.map((record) => record.id)), "archive-stream");
|
|
10062
|
+
if (rows.length > 0) yield rows.toSorted((a, b) => a.startMs - b.startMs);
|
|
10005
10063
|
if (probe.length < 128 && fresh.length === probe.length) return;
|
|
10006
10064
|
}
|
|
10007
10065
|
this.logger.warn("recorder: archive stream hit its page backstop — the rest of this scope was NOT walked", {
|
|
@@ -10015,110 +10073,50 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10015
10073
|
});
|
|
10016
10074
|
}
|
|
10017
10075
|
/**
|
|
10018
|
-
*
|
|
10019
|
-
*
|
|
10076
|
+
* Forget every hour row in `scope` — the durable half of an operator's
|
|
10077
|
+
* "discard this location's footage".
|
|
10078
|
+
*
|
|
10079
|
+
* It exists because the files are already GONE by the time it runs: the
|
|
10080
|
+
* discard wipes the class subtrees first, so every hour directory in the
|
|
10081
|
+
* scope answers ENOENT and nothing that needs a listing can name a single
|
|
10082
|
+
* segment. The rows are forgotten by their own identity instead, and the
|
|
10083
|
+
* count returned is the SEGMENTS they accounted for, which is the number the
|
|
10084
|
+
* operator is shown.
|
|
10020
10085
|
*
|
|
10021
|
-
*
|
|
10022
|
-
*
|
|
10023
|
-
*
|
|
10024
|
-
* and which `forget`s the row as it empties — so the staging is
|
|
10025
|
-
* self-cleaning and the alternative is two reads per hour.
|
|
10086
|
+
* Never reachable except from that explicitly-armed action: this is the one
|
|
10087
|
+
* surface here that deletes durable rows with no evidence from the array at
|
|
10088
|
+
* all, because the operator's own instruction is the evidence.
|
|
10026
10089
|
*/
|
|
10027
|
-
async rowsForHours(keys, scope) {
|
|
10028
|
-
if (keys.length === 0) return [];
|
|
10029
|
-
let records;
|
|
10030
|
-
try {
|
|
10031
|
-
records = await this.store.query.query({
|
|
10032
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10033
|
-
filter: {
|
|
10034
|
-
whereIn: { key: [...keys] },
|
|
10035
|
-
limit: keys.length
|
|
10036
|
-
}
|
|
10037
|
-
});
|
|
10038
|
-
} catch (err) {
|
|
10039
|
-
this.logger.warn("recorder: archive page detail unavailable — these hours take NO part in this pass", {
|
|
10040
|
-
...scope.deviceId !== void 0 ? { tags: { deviceId: scope.deviceId } } : {},
|
|
10041
|
-
meta: {
|
|
10042
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10043
|
-
hours: keys.length,
|
|
10044
|
-
error: String(err)
|
|
10045
|
-
}
|
|
10046
|
-
});
|
|
10047
|
-
return [];
|
|
10048
|
-
}
|
|
10049
|
-
const rows = [];
|
|
10050
|
-
for (const record of records) {
|
|
10051
|
-
const hour = recordToRow(record.id, record.data);
|
|
10052
|
-
if (hour === null) continue;
|
|
10053
|
-
this.ledger.stage(hour);
|
|
10054
|
-
for (const path of hour.paths) {
|
|
10055
|
-
const parsed = parseSegmentPath(path);
|
|
10056
|
-
if (parsed === null) continue;
|
|
10057
|
-
rows.push({
|
|
10058
|
-
deviceId: parsed.deviceId,
|
|
10059
|
-
profile: parsed.profile,
|
|
10060
|
-
startMs: parsed.startMs,
|
|
10061
|
-
durMs: parsed.durMs,
|
|
10062
|
-
bytes: parsed.bytes,
|
|
10063
|
-
path,
|
|
10064
|
-
locationId: hour.locationId
|
|
10065
|
-
});
|
|
10066
|
-
}
|
|
10067
|
-
}
|
|
10068
|
-
return rows.toSorted((a, b) => a.startMs - b.startMs);
|
|
10069
|
-
}
|
|
10070
10090
|
/**
|
|
10071
|
-
*
|
|
10072
|
-
* default.
|
|
10091
|
+
* Forget ONE hour row outright, and answer the segments it accounted for.
|
|
10073
10092
|
*
|
|
10074
|
-
*
|
|
10075
|
-
*
|
|
10076
|
-
*
|
|
10077
|
-
*
|
|
10078
|
-
* it selects nothing. A partial repair finishes on the next boot; until then
|
|
10079
|
-
* the affected disks UNDER-report, which prunes less and never more.
|
|
10093
|
+
* The caller must have TRUSTED evidence that the hour holds nothing — a
|
|
10094
|
+
* directory listing under a volume anchor that agreed twice (D49), or a
|
|
10095
|
+
* per-path confirmation against every root there is. This method makes no
|
|
10096
|
+
* such check itself and can make none: it is handed a verdict, not evidence.
|
|
10080
10097
|
*
|
|
10081
|
-
*
|
|
10082
|
-
*
|
|
10083
|
-
*
|
|
10084
|
-
* unbounded materialise here would be the whole-archive read this stage
|
|
10085
|
-
* removed, wearing a different name.
|
|
10098
|
+
* It exists because {@link adoptWalkedHour} deliberately refuses an EMPTY
|
|
10099
|
+
* listing — "the walk found nothing" is what an unmounted root looks like —
|
|
10100
|
+
* so the one repair a listing cannot express is the one that empties an hour.
|
|
10086
10101
|
*/
|
|
10087
|
-
async
|
|
10088
|
-
|
|
10089
|
-
|
|
10090
|
-
const
|
|
10091
|
-
|
|
10092
|
-
|
|
10093
|
-
|
|
10094
|
-
|
|
10095
|
-
|
|
10096
|
-
|
|
10097
|
-
|
|
10098
|
-
|
|
10099
|
-
|
|
10100
|
-
|
|
10101
|
-
|
|
10102
|
-
|
|
10103
|
-
await this.materialize(pending.slice(0, REPAIR_MAX_MATERIALIZE));
|
|
10104
|
-
for (const record of stale) {
|
|
10105
|
-
const held = this.ledger.get(record.id);
|
|
10106
|
-
if (held === void 0 || !held.pathsResident || held.segments === 0) continue;
|
|
10107
|
-
await this.ledger.put(held);
|
|
10108
|
-
repaired += 1;
|
|
10109
|
-
}
|
|
10110
|
-
} catch (err) {
|
|
10111
|
-
this.logger.warn("recorder: segment-hour aggregate repair failed — disks under-report", { meta: {
|
|
10112
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10113
|
-
error: String(err)
|
|
10114
|
-
} });
|
|
10115
|
-
return repaired;
|
|
10102
|
+
async forgetHour(deviceId, profile, locationId, hour) {
|
|
10103
|
+
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
10104
|
+
return this.serialized(key, async () => {
|
|
10105
|
+
const held = this.ledger.get(key);
|
|
10106
|
+
if (held === void 0) return 0;
|
|
10107
|
+
await this.ledger.forget(key);
|
|
10108
|
+
this.accounted.delete(key);
|
|
10109
|
+
return held.segments;
|
|
10110
|
+
});
|
|
10111
|
+
}
|
|
10112
|
+
async forgetScope(scope) {
|
|
10113
|
+
let forgotten = 0;
|
|
10114
|
+
for (const row of this.ledger.snapshot()) {
|
|
10115
|
+
if (!SegmentHourLedger.inScope(row, scope)) continue;
|
|
10116
|
+
await this.ledger.forget(row.key);
|
|
10117
|
+
forgotten += row.segments;
|
|
10116
10118
|
}
|
|
10117
|
-
|
|
10118
|
-
rows: repaired,
|
|
10119
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION
|
|
10120
|
-
} });
|
|
10121
|
-
return repaired;
|
|
10119
|
+
return forgotten;
|
|
10122
10120
|
}
|
|
10123
10121
|
/**
|
|
10124
10122
|
* Write-behind: a just-finalized segment joins its hour row.
|
|
@@ -10126,8 +10124,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10126
10124
|
* THE hot path — every profile-writer lands here every `segmentSeconds`,
|
|
10127
10125
|
* ~3.4 times a second fleet-wide. The append is {@link appendSegmentPath}:
|
|
10128
10126
|
* one `parseSegmentPath` and four scalar updates, whatever the hour already
|
|
10129
|
-
* holds
|
|
10130
|
-
*
|
|
10127
|
+
* holds — and since D523 it needs no read of the store at all, because the
|
|
10128
|
+
* mirror carries the whole row.
|
|
10131
10129
|
*/
|
|
10132
10130
|
async recordSegment(row) {
|
|
10133
10131
|
const { key, hourStartMs: hour } = hourOf(row);
|
|
@@ -10140,7 +10138,12 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10140
10138
|
entryMs: row.startMs
|
|
10141
10139
|
});
|
|
10142
10140
|
await this.serialized(key, async () => {
|
|
10143
|
-
|
|
10141
|
+
const seen = this.accounted.get(key);
|
|
10142
|
+
if (seen !== void 0 && seen.has(row.path)) return;
|
|
10143
|
+
if (seen === void 0) this.accounted.set(key, new Set([row.path]));
|
|
10144
|
+
else seen.add(row.path);
|
|
10145
|
+
const held = this.ledger.get(key);
|
|
10146
|
+
if (held === void 0) {
|
|
10144
10147
|
await this.ledger.put(segmentHourRow({
|
|
10145
10148
|
key,
|
|
10146
10149
|
deviceId: row.deviceId,
|
|
@@ -10151,25 +10154,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10151
10154
|
}));
|
|
10152
10155
|
return;
|
|
10153
10156
|
}
|
|
10154
|
-
const held = await this.resident(key, row.deviceId);
|
|
10155
|
-
if (held === null) {
|
|
10156
|
-
this.refused.add(key);
|
|
10157
|
-
this.logger.warn("recorder: hour detail unavailable — this finalized segment is NOT in the durable hour row yet", {
|
|
10158
|
-
tags: { deviceId: row.deviceId },
|
|
10159
|
-
meta: {
|
|
10160
|
-
key,
|
|
10161
|
-
path: row.path
|
|
10162
|
-
}
|
|
10163
|
-
});
|
|
10164
|
-
return;
|
|
10165
|
-
}
|
|
10166
|
-
if (held.paths.includes(row.path)) return;
|
|
10167
10157
|
await this.ledger.put(appendSegmentPath(held, row.path));
|
|
10168
10158
|
});
|
|
10169
10159
|
}
|
|
10170
10160
|
/**
|
|
10171
|
-
* Eviction confirmed these
|
|
10172
|
-
* hour when
|
|
10161
|
+
* Eviction confirmed these segments are gone from disk. Take them out of the
|
|
10162
|
+
* hour's account; forget the hour when nothing is left.
|
|
10163
|
+
*
|
|
10164
|
+
* The aggregate shrinks by exactly what was removed. `minStartMs` /
|
|
10165
|
+
* `maxStartMs` are NOT re-derived — they cannot be, without a listing — so a
|
|
10166
|
+
* partly-evicted hour keeps extremes that may be older or newer than what
|
|
10167
|
+
* survives. That error is bounded by the hour itself (both values always lie
|
|
10168
|
+
* inside `[hourStartMs, hourStartMs + 1h)`), and the row is UNSEALED so the
|
|
10169
|
+
* next listing of it re-derives both exactly.
|
|
10173
10170
|
*/
|
|
10174
10171
|
async dropSegments(rows) {
|
|
10175
10172
|
const doomed = /* @__PURE__ */ new Map();
|
|
@@ -10179,70 +10176,50 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10179
10176
|
if (entry === void 0) {
|
|
10180
10177
|
doomed.set(key, {
|
|
10181
10178
|
deviceId: row.deviceId,
|
|
10182
|
-
|
|
10183
|
-
|
|
10184
|
-
continue;
|
|
10185
|
-
}
|
|
10186
|
-
entry.paths.add(row.path);
|
|
10187
|
-
}
|
|
10188
|
-
for (const [key, { deviceId, paths }] of doomed) {
|
|
10189
|
-
if (!this.ledger.has(key)) continue;
|
|
10190
|
-
const held = await this.resident(key, deviceId);
|
|
10191
|
-
if (held === null) {
|
|
10192
|
-
this.logger.warn("recorder: hour detail unavailable — these evicted paths stay in the durable hour row until a walk or the next eviction confirms them", {
|
|
10193
|
-
tags: { deviceId },
|
|
10194
|
-
meta: {
|
|
10195
|
-
key,
|
|
10196
|
-
paths: paths.size
|
|
10197
|
-
}
|
|
10179
|
+
bytes: row.bytes,
|
|
10180
|
+
segments: 1
|
|
10198
10181
|
});
|
|
10199
10182
|
continue;
|
|
10200
10183
|
}
|
|
10201
|
-
|
|
10202
|
-
|
|
10184
|
+
entry.bytes += row.bytes;
|
|
10185
|
+
entry.segments += 1;
|
|
10186
|
+
}
|
|
10187
|
+
for (const [key, dropped] of doomed) await this.serialized(key, async () => {
|
|
10188
|
+
const held = this.ledger.get(key);
|
|
10189
|
+
if (held === void 0) return;
|
|
10190
|
+
const segments = held.segments - dropped.segments;
|
|
10191
|
+
if (segments <= 0) {
|
|
10203
10192
|
await this.ledger.forget(key);
|
|
10204
|
-
|
|
10193
|
+
this.accounted.delete(key);
|
|
10194
|
+
return;
|
|
10205
10195
|
}
|
|
10206
|
-
|
|
10207
|
-
await this.ledger.put(segmentHourRow({
|
|
10196
|
+
await this.ledger.put({
|
|
10208
10197
|
...held,
|
|
10209
|
-
|
|
10210
|
-
|
|
10211
|
-
|
|
10198
|
+
sealed: false,
|
|
10199
|
+
segments,
|
|
10200
|
+
bytes: Math.max(0, held.bytes - dropped.bytes)
|
|
10201
|
+
});
|
|
10202
|
+
});
|
|
10212
10203
|
}
|
|
10213
10204
|
/**
|
|
10214
|
-
* Move `rows` from the hour
|
|
10205
|
+
* Move `rows` from the hour they are accounted under today to the SAME hour
|
|
10215
10206
|
* at `toLocationId` — the DURABLE half of a relocation.
|
|
10216
10207
|
*
|
|
10217
|
-
* Without this the mover re-points only `RecordingIndex`, which is RAM
|
|
10218
|
-
*
|
|
10219
|
-
*
|
|
10220
|
-
*
|
|
10221
|
-
* no longer holds the file. Playback does not stat (that syscall per segment
|
|
10222
|
-
* was removed deliberately), so the result is a silent black window; and once
|
|
10223
|
-
* the drained location is deleted, `collectRangeSegments` drops the rows
|
|
10224
|
-
* outright and the footage is invisible. That is the shape of the 2 131
|
|
10225
|
-
* `recordings:high` ghost segments already on this hub.
|
|
10208
|
+
* Without this the mover re-points only `RecordingIndex`, which is RAM, and
|
|
10209
|
+
* the drained location keeps the bytes on every storage page for the life of
|
|
10210
|
+
* the hub. That is the shape of the 2 131 `recordings:high` ghost rows this
|
|
10211
|
+
* hub carried.
|
|
10226
10212
|
*
|
|
10227
10213
|
* **The ordering is the invariant, not an implementation detail.** The
|
|
10228
10214
|
* destination row is written BEFORE the source row is shrunk, and the caller
|
|
10229
10215
|
* has not yet deleted the source bytes — so the only state a crash can leave
|
|
10230
|
-
* is "
|
|
10231
|
-
*
|
|
10232
|
-
*
|
|
10233
|
-
* the index `hydrated`: nothing would ever walk that hour again, and the
|
|
10234
|
-
* segment would sit on disk permanently unreachable.
|
|
10235
|
-
*
|
|
10236
|
-
* **Two path sets, not one.** A path can arrive at the target under a
|
|
10237
|
-
* DIFFERENT name than it leaves the source under: a mover that found a file
|
|
10238
|
-
* larger than its recorded size reconciles the row, and the byte count is
|
|
10239
|
-
* part of the name. So the source row is shrunk by what is LEAVING it and the
|
|
10240
|
-
* target row grows by what is ARRIVING — conflating them would leave the
|
|
10241
|
-
* source row naming a file the caller is about to delete.
|
|
10216
|
+
* is "accounted at BOTH locations, both of which hold the files", which a
|
|
10217
|
+
* listing of either hour resolves. The reverse order has a window in which
|
|
10218
|
+
* the segments are accounted NOWHERE.
|
|
10242
10219
|
*
|
|
10243
|
-
*
|
|
10244
|
-
*
|
|
10245
|
-
* the
|
|
10220
|
+
* Both rows come out UNSEALED: each carries an aggregate that arithmetic
|
|
10221
|
+
* moved rather than a listing derived, and the next walk of either hour fixes
|
|
10222
|
+
* whatever the arithmetic rounded.
|
|
10246
10223
|
*/
|
|
10247
10224
|
async repointSegments(rows, toLocationId) {
|
|
10248
10225
|
const byHour = /* @__PURE__ */ new Map();
|
|
@@ -10250,87 +10227,75 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10250
10227
|
if (row.locationId === toLocationId) continue;
|
|
10251
10228
|
const { key } = hourOf(row);
|
|
10252
10229
|
const entry = byHour.get(key);
|
|
10253
|
-
if (entry === void 0)
|
|
10254
|
-
|
|
10255
|
-
|
|
10256
|
-
|
|
10257
|
-
|
|
10258
|
-
|
|
10259
|
-
|
|
10260
|
-
entry.arriving.add(row.path);
|
|
10261
|
-
}
|
|
10262
|
-
}
|
|
10263
|
-
for (const [sourceKey, { row, leaving, arriving }] of byHour) {
|
|
10264
|
-
const held = await this.resident(sourceKey, row.deviceId);
|
|
10265
|
-
if (held === null) {
|
|
10266
|
-
this.logger.warn("recorder: hour detail unavailable — this relocation is REFUSED and nothing is deleted; the segments stay where they are", {
|
|
10267
|
-
tags: { deviceId: row.deviceId },
|
|
10268
|
-
meta: {
|
|
10269
|
-
key: sourceKey,
|
|
10270
|
-
toLocationId,
|
|
10271
|
-
paths: leaving.size
|
|
10272
|
-
}
|
|
10273
|
-
});
|
|
10274
|
-
return false;
|
|
10275
|
-
}
|
|
10276
|
-
const targetKey = segmentHourKey(row.deviceId, row.profile, toLocationId, held.hourStartMs);
|
|
10277
|
-
const target = this.ledger.has(targetKey) ? await this.resident(targetKey, row.deviceId) : void 0;
|
|
10278
|
-
if (target === null) {
|
|
10279
|
-
this.logger.warn("recorder: TARGET hour detail unavailable — this relocation is REFUSED; appending to a summary would rewrite the hour as these paths alone", {
|
|
10280
|
-
tags: { deviceId: row.deviceId },
|
|
10281
|
-
meta: {
|
|
10282
|
-
key: targetKey,
|
|
10283
|
-
fromLocationId: row.locationId,
|
|
10284
|
-
paths: arriving.size
|
|
10285
|
-
}
|
|
10230
|
+
if (entry === void 0) {
|
|
10231
|
+
byHour.set(key, {
|
|
10232
|
+
row,
|
|
10233
|
+
bytes: row.bytes,
|
|
10234
|
+
segments: 1,
|
|
10235
|
+
minStartMs: row.startMs,
|
|
10236
|
+
maxStartMs: row.startMs
|
|
10286
10237
|
});
|
|
10287
|
-
|
|
10238
|
+
continue;
|
|
10288
10239
|
}
|
|
10289
|
-
|
|
10290
|
-
|
|
10240
|
+
entry.bytes += row.bytes;
|
|
10241
|
+
entry.segments += 1;
|
|
10242
|
+
entry.minStartMs = Math.min(entry.minStartMs, row.startMs);
|
|
10243
|
+
entry.maxStartMs = Math.max(entry.maxStartMs, row.startMs);
|
|
10244
|
+
}
|
|
10245
|
+
for (const [sourceKey, moved] of byHour) {
|
|
10246
|
+
const hour = hourStartMs(moved.row.startMs);
|
|
10247
|
+
const targetKey = segmentHourKey(moved.row.deviceId, moved.row.profile, toLocationId, hour);
|
|
10248
|
+
const target = this.ledger.get(targetKey);
|
|
10249
|
+
const targetIdentity = {
|
|
10291
10250
|
key: targetKey,
|
|
10292
|
-
deviceId: row.deviceId,
|
|
10293
|
-
profile: row.profile,
|
|
10251
|
+
deviceId: moved.row.deviceId,
|
|
10252
|
+
profile: moved.row.profile,
|
|
10294
10253
|
locationId: toLocationId,
|
|
10295
|
-
hourStartMs:
|
|
10296
|
-
|
|
10254
|
+
hourStartMs: hour
|
|
10255
|
+
};
|
|
10256
|
+
const targetEmpty = target === void 0 || target.segments === 0;
|
|
10257
|
+
await this.ledger.put(hourRowOf(targetIdentity, {
|
|
10258
|
+
sealed: false,
|
|
10259
|
+
bytes: (target?.bytes ?? 0) + moved.bytes,
|
|
10260
|
+
segments: (target?.segments ?? 0) + moved.segments,
|
|
10261
|
+
minStartMs: targetEmpty ? moved.minStartMs : Math.min(target.minStartMs, moved.minStartMs),
|
|
10262
|
+
maxStartMs: targetEmpty ? moved.maxStartMs : Math.max(target.maxStartMs, moved.maxStartMs)
|
|
10297
10263
|
}));
|
|
10298
|
-
const
|
|
10299
|
-
if (
|
|
10300
|
-
|
|
10264
|
+
const held = this.ledger.get(sourceKey);
|
|
10265
|
+
if (held === void 0) continue;
|
|
10266
|
+
const segments = held.segments - moved.segments;
|
|
10267
|
+
if (segments <= 0) {
|
|
10268
|
+
await this.ledger.forget(sourceKey);
|
|
10269
|
+
continue;
|
|
10270
|
+
}
|
|
10271
|
+
await this.ledger.put({
|
|
10301
10272
|
...held,
|
|
10302
|
-
|
|
10303
|
-
|
|
10273
|
+
sealed: false,
|
|
10274
|
+
segments,
|
|
10275
|
+
bytes: Math.max(0, held.bytes - moved.bytes)
|
|
10276
|
+
});
|
|
10304
10277
|
}
|
|
10305
|
-
return true;
|
|
10306
10278
|
}
|
|
10307
10279
|
/**
|
|
10308
|
-
* After a disk walk of `deviceIds`, make the ledger match the index
|
|
10309
|
-
* two D148 exceptions:
|
|
10280
|
+
* After a disk walk of `deviceIds`, make the ledger match what the index now
|
|
10281
|
+
* holds — with two D148 exceptions:
|
|
10310
10282
|
*
|
|
10311
10283
|
* - an hour the walk did not see is dropped ONLY if it is in the past;
|
|
10312
10284
|
* the current hour may still hold a live writer whose file the readdir
|
|
10313
10285
|
* raced;
|
|
10314
|
-
* - the current hour
|
|
10315
|
-
*
|
|
10286
|
+
* - the current hour takes the LARGER of the two accounts rather than the
|
|
10287
|
+
* walk's, for the same race.
|
|
10316
10288
|
*
|
|
10317
10289
|
* Devices not in `deviceIds` are left untouched (a partial walk must not
|
|
10318
10290
|
* prune cameras it did not look at).
|
|
10319
10291
|
*
|
|
10320
10292
|
* And a walk this ledger does not TRUST prunes nothing at all — see
|
|
10321
|
-
* {@link refusedPrunes}. What such a walk FOUND is still real, so its
|
|
10322
|
-
*
|
|
10323
|
-
*
|
|
10324
|
-
* Since D287 the walked devices' hours are MATERIALISED first: every
|
|
10325
|
-
* comparison below reads `row.paths`, and a summary's empty `paths` would
|
|
10326
|
-
* read as "the ledger holds nothing here" — which is the union collapsing to
|
|
10327
|
-
* the walk's own view, i.e. the prune the refusal machinery exists to
|
|
10328
|
-
* prevent. An hour whose detail could not be read is left additive-only.
|
|
10293
|
+
* {@link refusedPrunes}. What such a walk FOUND is still real, so its larger
|
|
10294
|
+
* account is taken; what it did not find decides nothing (D49).
|
|
10329
10295
|
*/
|
|
10330
10296
|
async reconcileFromIndex(index, deviceIds, nowMs) {
|
|
10331
10297
|
const walked = new Set(deviceIds);
|
|
10332
10298
|
const currentHour = hourStartMs(nowMs);
|
|
10333
|
-
await this.materializeDevices(walked);
|
|
10334
10299
|
const fromIndex = /* @__PURE__ */ new Map();
|
|
10335
10300
|
for (const deviceId of deviceIds) for (const seg of index.segments(deviceId)) {
|
|
10336
10301
|
const { key, hourStartMs: hour } = hourOf(seg);
|
|
@@ -10347,25 +10312,26 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10347
10312
|
else fromIndex.set(key, appendSegmentPath(existing, seg.path));
|
|
10348
10313
|
}
|
|
10349
10314
|
const refused = this.refusedPrunes(walked, fromIndex);
|
|
10315
|
+
const trimmed = /* @__PURE__ */ new Set();
|
|
10316
|
+
for (const deviceId of walked) {
|
|
10317
|
+
if (!index.droppedDuringWalk(deviceId)) continue;
|
|
10318
|
+
trimmed.add(deviceId);
|
|
10319
|
+
this.logger.warn("recorder: ledger prune REFUSED — the RAM index dropped hours during this walk, so its silence is not evidence", { tags: { deviceId } });
|
|
10320
|
+
}
|
|
10350
10321
|
for (const row of this.ledger.snapshot()) {
|
|
10351
10322
|
if (!walked.has(row.deviceId)) continue;
|
|
10352
|
-
if (!row.pathsResident) continue;
|
|
10353
10323
|
const disk = fromIndex.get(row.key);
|
|
10354
|
-
const additiveOnly = row.hourStartMs === currentHour || refused.devices.has(row.deviceId) || refused.locations.has(locationKey(row.deviceId, row.locationId));
|
|
10324
|
+
const additiveOnly = row.hourStartMs === currentHour || trimmed.has(row.deviceId) || refused.devices.has(row.deviceId) || refused.locations.has(locationKey(row.deviceId, row.locationId));
|
|
10355
10325
|
if (disk === void 0) {
|
|
10356
10326
|
if (additiveOnly) continue;
|
|
10357
10327
|
await this.ledger.forget(row.key);
|
|
10358
10328
|
continue;
|
|
10359
10329
|
}
|
|
10360
10330
|
if (additiveOnly) {
|
|
10361
|
-
|
|
10362
|
-
if (!samePaths(union, row.paths)) await this.ledger.put(segmentHourRow({
|
|
10363
|
-
...row,
|
|
10364
|
-
paths: union
|
|
10365
|
-
}));
|
|
10331
|
+
if (disk.segments > row.segments) await this.ledger.put(disk);
|
|
10366
10332
|
continue;
|
|
10367
10333
|
}
|
|
10368
|
-
if (
|
|
10334
|
+
if (disk.segments !== row.segments || disk.bytes !== row.bytes) await this.ledger.put(disk);
|
|
10369
10335
|
}
|
|
10370
10336
|
for (const [key, disk] of fromIndex) {
|
|
10371
10337
|
if (this.ledger.has(key)) continue;
|
|
@@ -10484,44 +10450,45 @@ function locationKey(deviceId, locationId) {
|
|
|
10484
10450
|
* skips it forever. Under D148 as written, a ghost row can never be forgotten
|
|
10485
10451
|
* by any mechanism — which is why the live ones have stood for thirteen days.
|
|
10486
10452
|
*
|
|
10487
|
-
* The resolution is that
|
|
10488
|
-
*
|
|
10489
|
-
* (`590/high/2026/08/15/04
|
|
10490
|
-
* currently-resolvable root
|
|
10491
|
-
*
|
|
10492
|
-
*
|
|
10493
|
-
* one `readdir` at a time.** That, and nothing weaker, is what licenses the
|
|
10453
|
+
* The resolution is that the ghost row's root does not resolve but its hour
|
|
10454
|
+
* DIRECTORY is addressable anywhere: the path convention
|
|
10455
|
+
* (`590/high/2026/08/15/04/`) is a function of the row's own fields, so the same
|
|
10456
|
+
* hour can be listed under every currently-resolvable root. **A listing of the
|
|
10457
|
+
* hour directory at every candidate root is exactly the walk D148 sanctions,
|
|
10458
|
+
* done one root at a time.** That, and nothing weaker, is what licenses the
|
|
10494
10459
|
* forget. Two consequences ride with it:
|
|
10495
10460
|
*
|
|
10496
|
-
* - **
|
|
10497
|
-
*
|
|
10498
|
-
* stands.
|
|
10499
|
-
* - **An error is not an absence** (D49). A
|
|
10500
|
-
* with nothing decided — never "not
|
|
10461
|
+
* - **Every root must answer.** An hour is forgotten only when every candidate
|
|
10462
|
+
* root listed its directory and every one of them came back EMPTY. One root
|
|
10463
|
+
* that could not answer and the hour stands.
|
|
10464
|
+
* - **An error is not an absence** (D49). A listing that fails, or that reports
|
|
10465
|
+
* "I could not decide", ends the hour with nothing decided — never "not
|
|
10466
|
+
* found". Whether an absent directory means "empty" or "this disk is gone" is
|
|
10467
|
+
* the lister's question, and it answers it by reading the ROOT.
|
|
10468
|
+
*
|
|
10469
|
+
* Until D523 this probed each of the row's `paths` at each root, one
|
|
10470
|
+
* `storage.exists` at a time — up to ~1 000 syscalls to prove one hour absent.
|
|
10471
|
+
* The column is gone and the row names no file, so the unit is the directory,
|
|
10472
|
+
* which is both cheaper and the same evidence.
|
|
10501
10473
|
*
|
|
10502
10474
|
* ## Repoint versus forget
|
|
10503
10475
|
*
|
|
10504
10476
|
* The asymmetry is the whole safety argument, and it is deliberately cheap in
|
|
10505
10477
|
* the safe direction:
|
|
10506
10478
|
*
|
|
10507
|
-
* - **
|
|
10508
|
-
* repointed to that root's location
|
|
10509
|
-
*
|
|
10510
|
-
* deleting here would destroy the index of footage
|
|
10511
|
-
* failure D291 was written to prevent.
|
|
10512
|
-
* - **every
|
|
10513
|
-
*
|
|
10514
|
-
* So proving presence costs one probe and proving absence costs all of them,
|
|
10515
|
-
* which is the right way round: the expensive proof is the destructive one.
|
|
10479
|
+
* - **a non-empty listing at any root ⇒ the row is MIS-POINTED, not orphaned.**
|
|
10480
|
+
* It is repointed to that root's location, with the segments that listing
|
|
10481
|
+
* names, and the hour is finished. A renamed or re-created location produces
|
|
10482
|
+
* exactly this shape, and deleting here would destroy the index of footage
|
|
10483
|
+
* that EXISTS, which is the failure D291 was written to prevent.
|
|
10484
|
+
* - **every root lists an EMPTY hour ⇒ orphaned**, and only then forgotten.
|
|
10516
10485
|
*
|
|
10517
10486
|
* ## Pacing
|
|
10518
10487
|
*
|
|
10519
|
-
* The recordings array is
|
|
10520
|
-
*
|
|
10521
|
-
*
|
|
10522
|
-
*
|
|
10523
|
-
* Work it does not finish is left for the next pass and REPORTED, never
|
|
10524
|
-
* silently dropped.
|
|
10488
|
+
* The recordings array is the resource every live read competes for. This is a
|
|
10489
|
+
* janitor: bounded hours per pass, a hard probe ceiling, and a caller-supplied
|
|
10490
|
+
* `pace()` awaited between probe batches. Work it does not finish is left for
|
|
10491
|
+
* the next pass and REPORTED, never silently dropped.
|
|
10525
10492
|
*/
|
|
10526
10493
|
var HOUR_MS$3 = 36e5;
|
|
10527
10494
|
/**
|
|
@@ -10670,22 +10637,12 @@ var GhostLocationSweeper = class {
|
|
|
10670
10637
|
refuse(row, "live-tail", `hourStartMs=${row.hourStartMs}`);
|
|
10671
10638
|
continue;
|
|
10672
10639
|
}
|
|
10673
|
-
|
|
10674
|
-
|
|
10675
|
-
refuse(row, "detail-unavailable", "point read failed");
|
|
10676
|
-
continue;
|
|
10677
|
-
}
|
|
10678
|
-
if (paths.length === 0) {
|
|
10679
|
-
refuse(row, "no-paths", "the row materialised empty");
|
|
10680
|
-
continue;
|
|
10681
|
-
}
|
|
10682
|
-
const rows = this.rowsOf(row, paths);
|
|
10683
|
-
if (rows === null) {
|
|
10684
|
-
refuse(row, "unparseable-paths", "a path does not parse into this hour");
|
|
10640
|
+
if (row.segments === 0) {
|
|
10641
|
+
refuse(row, "no-paths", "the row accounts for no segments");
|
|
10685
10642
|
continue;
|
|
10686
10643
|
}
|
|
10687
10644
|
const before = probes;
|
|
10688
|
-
const verdict = await this.probeHour(
|
|
10645
|
+
const verdict = await this.probeHour(row, roots, maxProbes - probes, probes);
|
|
10689
10646
|
probes += verdict.probes;
|
|
10690
10647
|
tallyOf(row.deviceId).probes += probes - before;
|
|
10691
10648
|
if (verdict.outcome.kind === "refused") {
|
|
@@ -10694,10 +10651,12 @@ var GhostLocationSweeper = class {
|
|
|
10694
10651
|
}
|
|
10695
10652
|
if (verdict.outcome.kind === "found") {
|
|
10696
10653
|
const target = locationForHydratedProfile(verdict.outcome.root.aliases, row.profile);
|
|
10697
|
-
|
|
10698
|
-
|
|
10654
|
+
const rows = this.rowsOf(row, verdict.outcome.paths);
|
|
10655
|
+
if (rows === null) {
|
|
10656
|
+
refuse(row, "unparseable-paths", "a listed name does not parse into this hour");
|
|
10699
10657
|
continue;
|
|
10700
10658
|
}
|
|
10659
|
+
await this.deps.ledger.repointSegments(rows, target.id);
|
|
10701
10660
|
this.deps.applyToIndex(rows, target.id);
|
|
10702
10661
|
repointedHours += 1;
|
|
10703
10662
|
segmentsRepointed += rows.length;
|
|
@@ -10707,19 +10666,19 @@ var GhostLocationSweeper = class {
|
|
|
10707
10666
|
tally.targets.add(target.id);
|
|
10708
10667
|
continue;
|
|
10709
10668
|
}
|
|
10710
|
-
await this.deps.ledger.
|
|
10669
|
+
const forgotten = await this.deps.ledger.forgetHour(row.deviceId, row.profile, row.locationId, row.hourStartMs);
|
|
10711
10670
|
if (this.deps.ledger.knowsHour(row.deviceId, row.profile, row.locationId, row.hourStartMs)) {
|
|
10712
|
-
refuse(row, "durable-refused", "the ledger still holds the hour after the
|
|
10671
|
+
refuse(row, "durable-refused", "the ledger still holds the hour after the forget");
|
|
10713
10672
|
continue;
|
|
10714
10673
|
}
|
|
10715
|
-
this.deps.
|
|
10674
|
+
this.deps.forgetInIndex(row.deviceId, row.locationId, row.hourStartMs);
|
|
10716
10675
|
forgottenHours += 1;
|
|
10717
|
-
segmentsForgotten +=
|
|
10718
|
-
bytesForgotten +=
|
|
10676
|
+
segmentsForgotten += forgotten;
|
|
10677
|
+
bytesForgotten += row.bytes;
|
|
10719
10678
|
const tally = tallyOf(row.deviceId);
|
|
10720
10679
|
tally.forgottenHours += 1;
|
|
10721
|
-
tally.forgottenSegments +=
|
|
10722
|
-
tally.forgottenBytes +=
|
|
10680
|
+
tally.forgottenSegments += forgotten;
|
|
10681
|
+
tally.forgottenBytes += row.bytes;
|
|
10723
10682
|
}
|
|
10724
10683
|
for (const [deviceId, tally] of byDevice) this.deps.logger.info("recorder: ghost-location reconcile — camera summary", {
|
|
10725
10684
|
tags: { deviceId },
|
|
@@ -10801,60 +10760,82 @@ var GhostLocationSweeper = class {
|
|
|
10801
10760
|
return out;
|
|
10802
10761
|
}
|
|
10803
10762
|
/**
|
|
10804
|
-
*
|
|
10763
|
+
* List this hour's directory under EVERY candidate root.
|
|
10764
|
+
*
|
|
10765
|
+
* A non-empty listing at one root makes the hour mis-pointed. Two roots
|
|
10766
|
+
* holding it is a refusal, not a choice: repointing would be a guess about
|
|
10767
|
+
* which copy is the footage. Absence is only concluded when every root
|
|
10768
|
+
* answered and every answer was empty — a listing that could not decide
|
|
10769
|
+
* (`null`) or threw ends the hour with nothing decided (D49).
|
|
10805
10770
|
*
|
|
10806
|
-
*
|
|
10807
|
-
*
|
|
10808
|
-
*
|
|
10809
|
-
*
|
|
10771
|
+
* The whole loop runs even after a hit, because "found at exactly one root"
|
|
10772
|
+
* is a claim about every root and cannot be short-circuited without giving up
|
|
10773
|
+
* the multi-root refusal.
|
|
10774
|
+
*
|
|
10775
|
+
* `spent` is the listings the PASS has already made, not this hour's, and the
|
|
10776
|
+
* brake fires on that running total. Since D523 an hour costs one listing per
|
|
10777
|
+
* physical root — two or three on this hub — so a counter that reset per hour
|
|
10778
|
+
* would never reach {@link GHOST_PROBE_BATCH} and the `pace()` dep would be
|
|
10779
|
+
* dead on every real topology.
|
|
10810
10780
|
*/
|
|
10811
|
-
async probeHour(
|
|
10781
|
+
async probeHour(row, roots, budget, spent) {
|
|
10812
10782
|
let used = 0;
|
|
10813
|
-
|
|
10814
|
-
|
|
10815
|
-
|
|
10816
|
-
|
|
10783
|
+
const found = [];
|
|
10784
|
+
for (const root of roots) {
|
|
10785
|
+
if (used >= budget) return {
|
|
10786
|
+
outcome: {
|
|
10787
|
+
kind: "refused",
|
|
10788
|
+
reason: "probe-budget",
|
|
10789
|
+
detail: `stopped after ${used} listings with ${roots.length} roots to probe`
|
|
10790
|
+
},
|
|
10791
|
+
probes: used
|
|
10792
|
+
};
|
|
10793
|
+
const alias = locationForHydratedProfile(root.aliases, row.profile);
|
|
10794
|
+
used += 1;
|
|
10795
|
+
if ((spent + used) % 16 === 0) await this.deps.pace();
|
|
10796
|
+
let listing;
|
|
10797
|
+
try {
|
|
10798
|
+
listing = await this.deps.listHour(alias.id, row.deviceId, row.profile, row.hourStartMs);
|
|
10799
|
+
} catch (err) {
|
|
10800
|
+
return {
|
|
10817
10801
|
outcome: {
|
|
10818
10802
|
kind: "refused",
|
|
10819
|
-
reason: "probe-
|
|
10820
|
-
detail:
|
|
10803
|
+
reason: "probe-error",
|
|
10804
|
+
detail: `${alias.id}: ${require_dist.errMsg(err)}`
|
|
10821
10805
|
},
|
|
10822
10806
|
probes: used
|
|
10823
10807
|
};
|
|
10824
|
-
const alias = locationForHydratedProfile(root.aliases, row.profile);
|
|
10825
|
-
used += 1;
|
|
10826
|
-
if (used % 16 === 0) await this.deps.pace();
|
|
10827
|
-
try {
|
|
10828
|
-
if (await this.deps.exists(alias.id, row.path)) found.push(root);
|
|
10829
|
-
} catch (err) {
|
|
10830
|
-
return {
|
|
10831
|
-
outcome: {
|
|
10832
|
-
kind: "refused",
|
|
10833
|
-
reason: "probe-error",
|
|
10834
|
-
detail: `${alias.id}: ${require_dist.errMsg(err)}`
|
|
10835
|
-
},
|
|
10836
|
-
probes: used
|
|
10837
|
-
};
|
|
10838
|
-
}
|
|
10839
10808
|
}
|
|
10840
|
-
if (
|
|
10809
|
+
if (listing === null) return {
|
|
10841
10810
|
outcome: {
|
|
10842
10811
|
kind: "refused",
|
|
10843
|
-
reason: "
|
|
10844
|
-
detail:
|
|
10845
|
-
},
|
|
10846
|
-
probes: used
|
|
10847
|
-
};
|
|
10848
|
-
const only = found[0];
|
|
10849
|
-
if (only !== void 0) return {
|
|
10850
|
-
outcome: {
|
|
10851
|
-
kind: "found",
|
|
10852
|
-
root: only,
|
|
10853
|
-
path: row.path
|
|
10812
|
+
reason: "detail-unavailable",
|
|
10813
|
+
detail: `${alias.id} could not say what its copy of this hour holds`
|
|
10854
10814
|
},
|
|
10855
10815
|
probes: used
|
|
10856
10816
|
};
|
|
10817
|
+
if (listing.length > 0) found.push({
|
|
10818
|
+
root,
|
|
10819
|
+
paths: listing
|
|
10820
|
+
});
|
|
10857
10821
|
}
|
|
10822
|
+
if (found.length > 1) return {
|
|
10823
|
+
outcome: {
|
|
10824
|
+
kind: "refused",
|
|
10825
|
+
reason: "found-at-multiple-roots",
|
|
10826
|
+
detail: found.map((hit) => hit.root.key).join(", ")
|
|
10827
|
+
},
|
|
10828
|
+
probes: used
|
|
10829
|
+
};
|
|
10830
|
+
const only = found[0];
|
|
10831
|
+
if (only !== void 0) return {
|
|
10832
|
+
outcome: {
|
|
10833
|
+
kind: "found",
|
|
10834
|
+
root: only.root,
|
|
10835
|
+
paths: only.paths
|
|
10836
|
+
},
|
|
10837
|
+
probes: used
|
|
10838
|
+
};
|
|
10858
10839
|
return {
|
|
10859
10840
|
outcome: { kind: "absent" },
|
|
10860
10841
|
probes: used
|
|
@@ -11235,34 +11216,12 @@ function emptyTally() {
|
|
|
11235
11216
|
orphanFiles: 0
|
|
11236
11217
|
};
|
|
11237
11218
|
}
|
|
11238
|
-
/**
|
|
11239
|
-
function
|
|
11240
|
-
|
|
11241
|
-
|
|
11242
|
-
|
|
11243
|
-
|
|
11244
|
-
* rows (`streamSegments` expands them whole), so no hour is ever split. */
|
|
11245
|
-
function claimsOf(rows) {
|
|
11246
|
-
const byHour = /* @__PURE__ */ new Map();
|
|
11247
|
-
for (const row of rows) {
|
|
11248
|
-
const hour = hourStartMs(row.startMs);
|
|
11249
|
-
const key = `${row.deviceId}|${row.profile}|${hour}`;
|
|
11250
|
-
const held = byHour.get(key);
|
|
11251
|
-
if (held) held.push(row);
|
|
11252
|
-
else byHour.set(key, [row]);
|
|
11253
|
-
}
|
|
11254
|
-
const out = [];
|
|
11255
|
-
for (const group of byHour.values()) {
|
|
11256
|
-
const first = group[0];
|
|
11257
|
-
if (first === void 0) continue;
|
|
11258
|
-
out.push({
|
|
11259
|
-
deviceId: first.deviceId,
|
|
11260
|
-
profile: first.profile,
|
|
11261
|
-
hourStartMs: hourStartMs(first.startMs),
|
|
11262
|
-
rows: group
|
|
11263
|
-
});
|
|
11264
|
-
}
|
|
11265
|
-
return out.toSorted((a, b) => a.hourStartMs - b.hourStartMs || a.deviceId - b.deviceId);
|
|
11219
|
+
/** Does this hour row fall inside the scope the operator asked for? */
|
|
11220
|
+
function inScope(row, scope) {
|
|
11221
|
+
if (scope.locationId !== void 0 && row.locationId !== scope.locationId) return false;
|
|
11222
|
+
if (scope.deviceId !== void 0 && row.deviceId !== scope.deviceId) return false;
|
|
11223
|
+
const profiles = scope.profiles ?? [];
|
|
11224
|
+
return profiles.length === 0 || profiles.includes(row.profile);
|
|
11266
11225
|
}
|
|
11267
11226
|
/**
|
|
11268
11227
|
* One reconcile pass over one location's claimed hours.
|
|
@@ -11360,20 +11319,35 @@ var LedgerWalkReconciler = class {
|
|
|
11360
11319
|
this.reportPass(report);
|
|
11361
11320
|
return report;
|
|
11362
11321
|
}
|
|
11363
|
-
/**
|
|
11322
|
+
/**
|
|
11323
|
+
* Walk the hour ROWS of the scope, oldest first.
|
|
11324
|
+
*
|
|
11325
|
+
* From the MIRROR, not a query. Since D523 a row is ten scalars, so the whole
|
|
11326
|
+
* claim set is already in RAM and costs nothing to fold — and the pass mutates
|
|
11327
|
+
* what it walks, which is exactly the shape that makes `offset` paging skip
|
|
11328
|
+
* half a scope. A snapshot taken once cannot skip.
|
|
11329
|
+
*/
|
|
11364
11330
|
async walk(state, scope) {
|
|
11365
|
-
|
|
11331
|
+
const claims = state.ledger.snapshot().filter((row) => inScope(row, scope)).toSorted((a, b) => a.hourStartMs - b.hourStartMs || a.deviceId - b.deviceId);
|
|
11332
|
+
for (const claim of claims) {
|
|
11366
11333
|
if (state.hoursClaimed >= state.maxHours) {
|
|
11367
11334
|
state.bounded = true;
|
|
11368
11335
|
return;
|
|
11369
11336
|
}
|
|
11370
11337
|
state.hoursClaimed += 1;
|
|
11371
|
-
state.segmentsClaimed += claim.
|
|
11338
|
+
state.segmentsClaimed += claim.segments;
|
|
11372
11339
|
if (!await this.reconcileHour(state, claim)) return;
|
|
11373
11340
|
}
|
|
11374
11341
|
}
|
|
11375
11342
|
/**
|
|
11376
|
-
*
|
|
11343
|
+
* Reconcile ONE hour: what its row ACCOUNTS FOR against what its directory
|
|
11344
|
+
* HOLDS, and repair the account from the directory.
|
|
11345
|
+
*
|
|
11346
|
+
* Since D523 the row names no files, so this is a comparison of counts and
|
|
11347
|
+
* bytes — and it does not need to name them: the repair is "make the account
|
|
11348
|
+
* the listing's", which is exact, and the RESIDENT index is reset to the same
|
|
11349
|
+
* listing so a row it holds for a file the directory does not is dropped with
|
|
11350
|
+
* it.
|
|
11377
11351
|
*
|
|
11378
11352
|
* `false` ends the whole pass — the only thing that returns it is an anchor
|
|
11379
11353
|
* that stopped agreeing, and every listing taken after that point is about a
|
|
@@ -11386,10 +11360,6 @@ var LedgerWalkReconciler = class {
|
|
|
11386
11360
|
return true;
|
|
11387
11361
|
}
|
|
11388
11362
|
const relDir = hourDirRelPath(claim.deviceId, claim.profile, claim.hourStartMs);
|
|
11389
|
-
for (const row of claim.rows) if (parseSegmentPath(row.path) === null || row.path !== `${relDir}/${fileName(row.path)}`) {
|
|
11390
|
-
this.skip(state, claim, "path-mismatch", row.path);
|
|
11391
|
-
return true;
|
|
11392
|
-
}
|
|
11393
11363
|
let listing;
|
|
11394
11364
|
try {
|
|
11395
11365
|
listing = await this.deps.listHour(state.location.root, relDir);
|
|
@@ -11401,36 +11371,55 @@ var LedgerWalkReconciler = class {
|
|
|
11401
11371
|
if (state.listings % 8 === 0) await this.deps.pace();
|
|
11402
11372
|
state.hoursWalked += 1;
|
|
11403
11373
|
tally.hoursWalked += 1;
|
|
11404
|
-
const
|
|
11405
|
-
if (
|
|
11374
|
+
const missingDirectory = listing === null;
|
|
11375
|
+
if (missingDirectory) {
|
|
11406
11376
|
state.hoursMissing += 1;
|
|
11407
11377
|
tally.hoursMissing += 1;
|
|
11408
11378
|
}
|
|
11409
|
-
|
|
11410
|
-
|
|
11411
|
-
|
|
11412
|
-
|
|
11379
|
+
const present = [];
|
|
11380
|
+
for (const name of listing ?? []) {
|
|
11381
|
+
if (!name.endsWith(".m4s")) continue;
|
|
11382
|
+
const relPath = `${relDir}/${name}`;
|
|
11383
|
+
const parsed = parseSegmentPath(relPath);
|
|
11384
|
+
if (parsed === null || hourStartMs(parsed.startMs) !== claim.hourStartMs) {
|
|
11385
|
+
this.skip(state, claim, "path-mismatch", relPath);
|
|
11386
|
+
return true;
|
|
11387
|
+
}
|
|
11388
|
+
present.push(relPath);
|
|
11389
|
+
}
|
|
11390
|
+
const listed = hourAggregate(present);
|
|
11391
|
+
const orphans = Math.max(0, listed.segments - claim.segments);
|
|
11392
|
+
if (orphans > 0) {
|
|
11393
|
+
state.orphanFiles += orphans;
|
|
11394
|
+
tally.orphanFiles += orphans;
|
|
11395
|
+
for (const path of present) {
|
|
11413
11396
|
if (state.orphanSample.length >= 50) break;
|
|
11414
|
-
state.orphanSample.push(
|
|
11397
|
+
state.orphanSample.push(path);
|
|
11415
11398
|
}
|
|
11416
|
-
this.deps.logger.warn("recorder:
|
|
11399
|
+
this.deps.logger.warn("recorder: an hour directory holds more segments than its durable row accounts for", {
|
|
11417
11400
|
tags: { deviceId: claim.deviceId },
|
|
11418
11401
|
meta: {
|
|
11419
11402
|
locationId: state.location.id,
|
|
11420
11403
|
profile: claim.profile,
|
|
11421
11404
|
hourStartMs: claim.hourStartMs,
|
|
11422
|
-
|
|
11423
|
-
|
|
11405
|
+
accounted: claim.segments,
|
|
11406
|
+
listed: listed.segments,
|
|
11407
|
+
apply: state.apply
|
|
11424
11408
|
}
|
|
11425
11409
|
});
|
|
11426
11410
|
}
|
|
11427
|
-
|
|
11428
|
-
|
|
11429
|
-
|
|
11411
|
+
const ghosts = Math.max(0, claim.segments - listed.segments);
|
|
11412
|
+
if (ghosts === 0) {
|
|
11413
|
+
if (state.apply && listed.segments > 0 && listed.bytes !== claim.bytes) await state.ledger.adoptWalkedHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs, present);
|
|
11414
|
+
if (state.apply) this.resetIndex(state, claim, present);
|
|
11415
|
+
return true;
|
|
11416
|
+
}
|
|
11417
|
+
const ghostBytes = Math.max(0, claim.bytes - listed.bytes);
|
|
11418
|
+
state.ghostSegments += ghosts;
|
|
11430
11419
|
state.ghostBytes += ghostBytes;
|
|
11431
|
-
tally.ghostSegments +=
|
|
11420
|
+
tally.ghostSegments += ghosts;
|
|
11432
11421
|
tally.ghostBytes += ghostBytes;
|
|
11433
|
-
if (
|
|
11422
|
+
if (listed.segments === 0) state.ghostHoursWhole += 1;
|
|
11434
11423
|
const anchor = await this.readAnchor(state.location);
|
|
11435
11424
|
if (!anchor.ok || anchor.volumeId !== state.anchor) {
|
|
11436
11425
|
state.refused = anchor.ok ? "anchor-moved" : anchor.refusal;
|
|
@@ -11446,56 +11435,52 @@ var LedgerWalkReconciler = class {
|
|
|
11446
11435
|
});
|
|
11447
11436
|
return false;
|
|
11448
11437
|
}
|
|
11449
|
-
this.deps.logger.warn("recorder: durable hour row
|
|
11438
|
+
this.deps.logger.warn("recorder: durable hour row accounts for segments the directory does not hold", {
|
|
11450
11439
|
tags: { deviceId: claim.deviceId },
|
|
11451
11440
|
meta: {
|
|
11452
11441
|
locationId: state.location.id,
|
|
11453
11442
|
profile: claim.profile,
|
|
11454
11443
|
hourStartMs: claim.hourStartMs,
|
|
11455
|
-
|
|
11456
|
-
|
|
11444
|
+
accounted: claim.segments,
|
|
11445
|
+
listed: listed.segments,
|
|
11446
|
+
ghosts,
|
|
11457
11447
|
ghostBytes,
|
|
11458
|
-
wholeHour:
|
|
11459
|
-
missingDirectory
|
|
11448
|
+
wholeHour: listed.segments === 0,
|
|
11449
|
+
missingDirectory,
|
|
11460
11450
|
apply: state.apply
|
|
11461
11451
|
}
|
|
11462
11452
|
});
|
|
11463
11453
|
if (!state.apply) return true;
|
|
11464
|
-
await state.ledger.
|
|
11465
|
-
if (
|
|
11466
|
-
this.skip(state, claim, "durable-refused", "the ledger still
|
|
11454
|
+
const forgotten = listed.segments === 0 ? await state.ledger.forgetHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs) : await this.adoptLanded(state, claim, present, ghosts);
|
|
11455
|
+
if (forgotten === null) {
|
|
11456
|
+
this.skip(state, claim, "durable-refused", "the ledger still accounts for the ghosts");
|
|
11467
11457
|
return true;
|
|
11468
11458
|
}
|
|
11469
|
-
this.
|
|
11470
|
-
state.forgottenSegments +=
|
|
11459
|
+
this.resetIndex(state, claim, present);
|
|
11460
|
+
state.forgottenSegments += forgotten;
|
|
11471
11461
|
state.forgottenBytes += ghostBytes;
|
|
11472
|
-
tally.forgottenSegments +=
|
|
11462
|
+
tally.forgottenSegments += forgotten;
|
|
11473
11463
|
return true;
|
|
11474
11464
|
}
|
|
11475
|
-
/** Did the durable drop actually land? */
|
|
11476
|
-
async forgetLanded(state, claim, ghosts) {
|
|
11477
|
-
const { deviceId, profile, hourStartMs: hour } = claim;
|
|
11478
|
-
if (!state.ledger.knowsHour(deviceId, profile, state.location.id, hour)) return true;
|
|
11479
|
-
const held = await state.ledger.hourPaths(deviceId, profile, state.location.id, hour);
|
|
11480
|
-
if (held === null) return false;
|
|
11481
|
-
const remaining = new Set(held);
|
|
11482
|
-
return ghosts.every((row) => !remaining.has(row.path));
|
|
11483
|
-
}
|
|
11484
11465
|
/**
|
|
11485
|
-
*
|
|
11466
|
+
* Adopt the listing, then READ THE ROW BACK.
|
|
11486
11467
|
*
|
|
11487
|
-
*
|
|
11488
|
-
*
|
|
11489
|
-
*
|
|
11468
|
+
* `adoptWalkedHour` refuses a listing that would erase more than half of what
|
|
11469
|
+
* the row accounts for — the guard against a root that answered short — and it
|
|
11470
|
+
* refuses silently, which a caller that counted it as done would turn into
|
|
11471
|
+
* telling the resident index a row is gone that the archive still holds.
|
|
11472
|
+
* `null` is that refusal.
|
|
11490
11473
|
*/
|
|
11491
|
-
|
|
11492
|
-
|
|
11493
|
-
const
|
|
11494
|
-
return
|
|
11495
|
-
|
|
11496
|
-
|
|
11497
|
-
|
|
11498
|
-
|
|
11474
|
+
async adoptLanded(state, claim, present, ghosts) {
|
|
11475
|
+
await state.ledger.adoptWalkedHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs, present);
|
|
11476
|
+
const after = state.ledger.hourRow(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs);
|
|
11477
|
+
if (after === void 0) return claim.segments;
|
|
11478
|
+
return after.segments === claim.segments - ghosts ? ghosts : null;
|
|
11479
|
+
}
|
|
11480
|
+
/** Make the resident index's view of the hour the listing's, and say how many
|
|
11481
|
+
* index rows that dropped. */
|
|
11482
|
+
resetIndex(state, claim, present) {
|
|
11483
|
+
return this.deps.applyToIndex(claim.deviceId, state.location.id, claim.hourStartMs, present);
|
|
11499
11484
|
}
|
|
11500
11485
|
/** The volume anchor, or the refusal its absence produces. */
|
|
11501
11486
|
async readAnchor(location) {
|
|
@@ -17784,12 +17769,17 @@ var GHOST_LOCATION_PROBE_PAUSE_MS = 50;
|
|
|
17784
17769
|
/**
|
|
17785
17770
|
* Awaited every `WALK_HOUR_BATCH` hour listings by the ledger↔disk walk (D319).
|
|
17786
17771
|
*
|
|
17787
|
-
* Twice the ghost sweep's brake
|
|
17788
|
-
* heavier
|
|
17789
|
-
* `readdir`
|
|
17790
|
-
*
|
|
17791
|
-
*
|
|
17792
|
-
*
|
|
17772
|
+
* Twice the ghost sweep's brake. That used to be because the syscall between
|
|
17773
|
+
* brakes was far heavier — the sweep paced one `storage.exists`, this one an
|
|
17774
|
+
* hour `readdir` "measured at ~650 ms on the live share under load". Both
|
|
17775
|
+
* halves of that have changed: the sweep paces a directory listing too since
|
|
17776
|
+
* D523, and the 650 ms belonged to a FUSE mount that has been retired (an hour
|
|
17777
|
+
* of 345–349 files on `/recordings-2` lists in ~1 ms).
|
|
17778
|
+
*
|
|
17779
|
+
* It stays at twice anyway, for the reason that never depended on the number:
|
|
17780
|
+
* the walk is operator-armed and one-off, so the right trade is to be slower
|
|
17781
|
+
* and invisible rather than fast and felt — a `find` over this array hung an
|
|
17782
|
+
* SSH session on 2026-08-29.
|
|
17793
17783
|
*/
|
|
17794
17784
|
var LEDGER_WALK_PAUSE_MS = 100;
|
|
17795
17785
|
/**
|
|
@@ -18069,6 +18059,11 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18069
18059
|
}
|
|
18070
18060
|
async onInitialize() {
|
|
18071
18061
|
this.stopStallMonitor = (0, _camstack_system_addon_utils.startEventLoopStallMonitor)(this.ctx.logger);
|
|
18062
|
+
this.index.armResidency({
|
|
18063
|
+
logger: this.ctx.logger,
|
|
18064
|
+
now: () => Date.now(),
|
|
18065
|
+
budget: DEFAULT_RECORDING_INDEX_BUDGET
|
|
18066
|
+
});
|
|
18072
18067
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
18073
18068
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
18074
18069
|
const configuredNode = this.config.recordingNodeId;
|
|
@@ -18086,6 +18081,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18086
18081
|
this.segmentHours = new SegmentHourLedger({
|
|
18087
18082
|
store: this.ctx.api.settingsStore,
|
|
18088
18083
|
logger: this.ctx.logger,
|
|
18084
|
+
listHour: (locationId, deviceId, profile, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18089
18085
|
isLocationUsable: (locationId) => this.placement?.isLocationUsable(locationId) ?? true
|
|
18090
18086
|
});
|
|
18091
18087
|
try {
|
|
@@ -18119,7 +18115,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18119
18115
|
await node_fs.promises.unlink(fromAbs);
|
|
18120
18116
|
}
|
|
18121
18117
|
},
|
|
18122
|
-
|
|
18118
|
+
hourDirectoryPaths: (deviceId, profile, locationId, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18123
18119
|
removeDir: async (locationId, relDir) => {
|
|
18124
18120
|
const loc = requireReadableLocation(this.resolvedLocations, locationId, "recorder");
|
|
18125
18121
|
await node_fs.promises.rm(node_path.default.join(loc.root, relDir), {
|
|
@@ -18604,8 +18600,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18604
18600
|
localNodeId: this.nodeId,
|
|
18605
18601
|
index: this.index,
|
|
18606
18602
|
ledger: this.segmentHours === null ? null : {
|
|
18607
|
-
|
|
18608
|
-
|
|
18603
|
+
forgetScope: (scope) => this.segmentHours.forgetScope(scope),
|
|
18604
|
+
snapshot: () => this.segmentHours.snapshot()
|
|
18609
18605
|
},
|
|
18610
18606
|
now: () => Date.now()
|
|
18611
18607
|
});
|
|
@@ -18712,9 +18708,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18712
18708
|
}
|
|
18713
18709
|
},
|
|
18714
18710
|
anchorOf: async (root) => (await readLocationMarker(root))?.volumeId ?? null,
|
|
18715
|
-
applyToIndex: (paths) =>
|
|
18716
|
-
this.index.removeSegments(paths);
|
|
18717
|
-
},
|
|
18711
|
+
applyToIndex: (deviceId, locationId, hour, paths) => this.index.resetHour(deviceId, locationId, hour, paths),
|
|
18718
18712
|
logger: this.ctx.logger,
|
|
18719
18713
|
now: () => Date.now(),
|
|
18720
18714
|
pace: () => require_dist.sleep(LEDGER_WALK_PAUSE_MS)
|
|
@@ -18842,7 +18836,6 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18842
18836
|
});
|
|
18843
18837
|
try {
|
|
18844
18838
|
await this.segmentHours?.load();
|
|
18845
|
-
this.segmentHours?.repairAggregates();
|
|
18846
18839
|
} catch (err) {
|
|
18847
18840
|
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: require_dist.errMsg(err) } });
|
|
18848
18841
|
}
|
|
@@ -18885,18 +18878,15 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
18885
18878
|
const sweeper = new GhostLocationSweeper({
|
|
18886
18879
|
ledger,
|
|
18887
18880
|
locations: () => this.resolvedLocations,
|
|
18888
|
-
|
|
18889
|
-
location,
|
|
18890
|
-
relativePath
|
|
18891
|
-
}),
|
|
18881
|
+
listHour: (locationId, deviceId, profile, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18892
18882
|
applyToIndex: (rows, toLocationId) => {
|
|
18893
18883
|
this.index.removeSegments(rows.map((row) => row.path));
|
|
18894
|
-
if (toLocationId === null) return;
|
|
18895
18884
|
for (const row of rows) this.index.addSegment({
|
|
18896
18885
|
...row,
|
|
18897
18886
|
locationId: toLocationId
|
|
18898
18887
|
});
|
|
18899
18888
|
},
|
|
18889
|
+
forgetInIndex: (deviceId, locationId, hourStartMs) => this.index.resetHour(deviceId, locationId, hourStartMs, []),
|
|
18900
18890
|
logger: this.ctx.logger,
|
|
18901
18891
|
now: () => Date.now(),
|
|
18902
18892
|
pace: () => require_dist.sleep(GHOST_LOCATION_PROBE_PAUSE_MS)
|
|
@@ -19069,9 +19059,16 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
19069
19059
|
const census = this.index.sortedViewCensus();
|
|
19070
19060
|
const views = this.index.sortedViewStats();
|
|
19071
19061
|
const reuse = this.index.hydrateReuseStats();
|
|
19062
|
+
const residency = this.index.residency();
|
|
19072
19063
|
this.ctx.logger.info("recorder: index census", {
|
|
19073
19064
|
...census.largestDeviceId > 0 ? { tags: { deviceId: census.largestDeviceId } } : {},
|
|
19074
19065
|
meta: {
|
|
19066
|
+
residentRows: residency.residentRows,
|
|
19067
|
+
residentHours: residency.residentHours,
|
|
19068
|
+
rowBudget: residency.budget?.residentRowBudget ?? null,
|
|
19069
|
+
evictedRows: residency.evictedRows,
|
|
19070
|
+
evictedHours: residency.evictedHours,
|
|
19071
|
+
evictionPasses: residency.evictionPasses,
|
|
19075
19072
|
sortedViews: census.views,
|
|
19076
19073
|
rowSlots: census.rowSlots,
|
|
19077
19074
|
largestViewRows: census.largestRows,
|
|
@@ -19171,6 +19168,63 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
19171
19168
|
return this.resolvedLocations;
|
|
19172
19169
|
}
|
|
19173
19170
|
/**
|
|
19171
|
+
* ONE hour bucket's segment files, location-relative — the recorder's single
|
|
19172
|
+
* implementation of {@link HourLister} (D523).
|
|
19173
|
+
*
|
|
19174
|
+
* The directory is the authority for WHICH segments exist, and the segment
|
|
19175
|
+
* NAME carries start, duration and bytes, so this is a complete description
|
|
19176
|
+
* of the hour and no file is ever opened.
|
|
19177
|
+
*
|
|
19178
|
+
* Three answers, and the difference between the last two is the whole point:
|
|
19179
|
+
*
|
|
19180
|
+
* - `null` — **undecided.** The location does not resolve, may not be read,
|
|
19181
|
+
* or the read failed for a reason that is not "this directory is not
|
|
19182
|
+
* there". Every caller turns it into "this hour takes no part in this
|
|
19183
|
+
* pass".
|
|
19184
|
+
* - `[]` — **the hour holds nothing**, and the volume confirmed it: an
|
|
19185
|
+
* ENOENT on the hour directory is only reported as empty once the
|
|
19186
|
+
* location's ROOT has answered a read. A root that has gone away answers
|
|
19187
|
+
* ENOENT for every hour on it, and reporting that as "empty" is how a pass
|
|
19188
|
+
* deletes an operator's index of real footage (D148, D393).
|
|
19189
|
+
* - the paths.
|
|
19190
|
+
*/
|
|
19191
|
+
async listHourDirectory(locationId, deviceId, profile, hourStartMs) {
|
|
19192
|
+
const location = this.resolvedLocations.find((l) => l.id === locationId);
|
|
19193
|
+
if (location === void 0 || !locationMayBeRead(location)) return null;
|
|
19194
|
+
const rel = hourDirRelPath(deviceId, profile, hourStartMs);
|
|
19195
|
+
try {
|
|
19196
|
+
return (await node_fs.promises.readdir(node_path.default.join(location.root, rel))).filter((f) => f.endsWith(".m4s")).map((f) => `${rel}/${f}`);
|
|
19197
|
+
} catch (err) {
|
|
19198
|
+
if (err.code !== "ENOENT") {
|
|
19199
|
+
this.ctx.logger.warn("recorder: hour directory read failed — this hour decides nothing", {
|
|
19200
|
+
tags: { deviceId },
|
|
19201
|
+
meta: {
|
|
19202
|
+
profile,
|
|
19203
|
+
locationId,
|
|
19204
|
+
hourStartMs,
|
|
19205
|
+
error: require_dist.errMsg(err)
|
|
19206
|
+
}
|
|
19207
|
+
});
|
|
19208
|
+
return null;
|
|
19209
|
+
}
|
|
19210
|
+
try {
|
|
19211
|
+
await node_fs.promises.readdir(location.root);
|
|
19212
|
+
} catch {
|
|
19213
|
+
this.ctx.logger.warn("recorder: hour directory is absent AND its location root cannot be read — this hour decides nothing", {
|
|
19214
|
+
tags: { deviceId },
|
|
19215
|
+
meta: {
|
|
19216
|
+
profile,
|
|
19217
|
+
locationId,
|
|
19218
|
+
root: location.root,
|
|
19219
|
+
hourStartMs
|
|
19220
|
+
}
|
|
19221
|
+
});
|
|
19222
|
+
return null;
|
|
19223
|
+
}
|
|
19224
|
+
return [];
|
|
19225
|
+
}
|
|
19226
|
+
}
|
|
19227
|
+
/**
|
|
19174
19228
|
* Attach-boundary placement for one (device, profile). Falls back to the pure
|
|
19175
19229
|
* resolver when the service is not built (non-recording node / early boot),
|
|
19176
19230
|
* which is byte-identical to the pre-placement behaviour.
|