@camstack/addon-pipeline 1.2.256 → 1.2.258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/{default-detection-model-A5DRjEr3.mjs → default-detection-model-BZ54sUY2.mjs} +1 -1
- package/dist/{default-detection-model-Bbw4nC11.js → default-detection-model-vVWcdv0g.js} +1 -1
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +4 -4
- package/dist/{dist-lqqUXDfs.js → dist-CsJJPDQW.js} +180 -6
- package/dist/{dist-MJ4ji7um.mjs → dist-D3w2fwD4.mjs} +180 -6
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-80-uOMr8.js → node-1vbEWzPZ.js} +1 -1
- package/dist/{node-DdXmHLK4.mjs → node-IHGNrWH9.mjs} +1 -1
- package/dist/pipeline-runner/index.js +4 -4
- package/dist/pipeline-runner/index.mjs +4 -4
- package/dist/{process-memory-Dac9TXlX.js → process-memory-DynU4177.js} +1 -1
- package/dist/{process-memory-Cl3ubMxY.mjs → process-memory-jhOBJEil.mjs} +1 -1
- package/dist/recorder/index.js +1069 -1015
- package/dist/recorder/index.mjs +1069 -1015
- package/dist/{restream-intent-VVJiUAkh.js → restream-intent-B0GOuR-F.js} +90 -0
- package/dist/{restream-intent-D2VT97-U.mjs → restream-intent-CgyZI565.mjs} +85 -1
- package/dist/{segment-demux-js-gIj82O9k.mjs → segment-demux-js-BZAiu_2h.mjs} +1 -1
- package/dist/{segment-demux-js-CWwlKQ3M.js → segment-demux-js-OQT6wqXH.js} +1 -1
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-pOLAb3B1.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CFcO9OoY.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CTHexfai.mjs +26 -0
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BXK7jbJm.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BbnIk-9T.mjs} +1 -1
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-BRnsXyTM.mjs → hostInit-rU23F6nl.mjs} +3 -3
- package/dist/stream-broker/index.js +46 -7
- package/dist/stream-broker/index.mjs +46 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CPFjd94C.mjs +0 -26
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { $t as hydrateSchema, C as OpsLogEntrySchema, Ct as recordingExportCapability, D as RECORDING_EXPORT_MAX_READ_BYTES, Dt as resolveLocationMode, H as deriveBatteryPresence, I as batteryCapability, Kt as DeviceFeature, Lt as errMsg, Mt as storageOccupancyCapability, O as RecordingConfigSchema, Pt as stripRetiredBandPreBufferSec, Q as evictionPolicyOfLocation, St as recordingCapability, W as deriveRecordingMode, Wt as BaseAddon, Xt as customAction, Y as evaluateSensorEdge, Z as evictionPolicyForMode, Zt as defineCustomActions, _n as record, bn as EventCategory, cn as _enum, ct as mayReadLocation, d as DEFAULT_EVENTS_BAND_BUFFER_SEC, dt as modeMayWrite, et as gbToBytes, g as ExportRecordSchema, gn as preprocess, hn as object, i as BatteryStatusSchema, it as loadContributionCapability, jt as storageEvictableCapability, k as RecordingSignalStatusSchema, kt as resolveRecordingProfiles, ln as array, lt as mayWriteToLocation, mn as number, on as selectAssignedProfileSlots, qt as DeviceType, rn as nodePin, rt as isFirstLevelMacroClass, sn as sleep$1, un as boolean, ut as migrateLegacyDeviceSignalConfig, vn as string, wt as recordingSignalCapability } from "../dist-
|
|
2
|
-
import { a as physicalRootOf, i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-
|
|
1
|
+
import { $t as hydrateSchema, C as OpsLogEntrySchema, Ct as recordingExportCapability, D as RECORDING_EXPORT_MAX_READ_BYTES, Dt as resolveLocationMode, H as deriveBatteryPresence, I as batteryCapability, Kt as DeviceFeature, Lt as errMsg, Mt as storageOccupancyCapability, O as RecordingConfigSchema, Pt as stripRetiredBandPreBufferSec, Q as evictionPolicyOfLocation, St as recordingCapability, W as deriveRecordingMode, Wt as BaseAddon, Xt as customAction, Y as evaluateSensorEdge, Z as evictionPolicyForMode, Zt as defineCustomActions, _n as record, bn as EventCategory, cn as _enum, ct as mayReadLocation, d as DEFAULT_EVENTS_BAND_BUFFER_SEC, dt as modeMayWrite, et as gbToBytes, g as ExportRecordSchema, gn as preprocess, hn as object, i as BatteryStatusSchema, it as loadContributionCapability, jt as storageEvictableCapability, k as RecordingSignalStatusSchema, kt as resolveRecordingProfiles, ln as array, lt as mayWriteToLocation, mn as number, on as selectAssignedProfileSlots, qt as DeviceType, rn as nodePin, rt as isFirstLevelMacroClass, sn as sleep$1, un as boolean, ut as migrateLegacyDeviceSignalConfig, vn as string, wt as recordingSignalCapability } from "../dist-D3w2fwD4.mjs";
|
|
2
|
+
import { a as physicalRootOf, i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-IHGNrWH9.mjs";
|
|
3
3
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
4
|
-
import {
|
|
4
|
+
import { l as PREBUFFER_MAX_MS, r as RECORDING_PRE_ROLL_REQUEST_MS, s as withRecordingIntent } from "../restream-intent-CgyZI565.mjs";
|
|
5
5
|
import { n as RowMapStore, t as retireRootKeys } from "../retire-root-keys-DZvlso2l.mjs";
|
|
6
6
|
import { randomUUID } from "node:crypto";
|
|
7
7
|
import { createReadStream, promises, realpathSync } from "node:fs";
|
|
@@ -9,7 +9,6 @@ import path from "node:path";
|
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
10
|
import { open } from "node:fs/promises";
|
|
11
11
|
import { contentTypeFor, createFileDataPlaneHandler, parseRangeHeader, startEventLoopStallMonitor } from "@camstack/system/addon-utils";
|
|
12
|
-
import { gunzipSync, gzipSync } from "node:zlib";
|
|
13
12
|
//#region src/recorder/event-capture.ts
|
|
14
13
|
/**
|
|
15
14
|
* La sottoscrizione motion del recorder: bus → `onTrigger(deviceId, atMs)`.
|
|
@@ -815,14 +814,22 @@ function parseSegmentPath(relPath) {
|
|
|
815
814
|
/** Hour bucket, the granularity the path convention (`.../YYYY/MM/DD/HH/`)
|
|
816
815
|
* already gives us — so "have I walked this?" costs one Set lookup. */
|
|
817
816
|
var HOUR_MS$9 = 36e5;
|
|
818
|
-
/** A device walked end-to-end. Cheaper and clearer than inserting every hour of
|
|
819
|
-
* the retention window into the set. */
|
|
820
|
-
var ALL_HOURS = Symbol("all-hours");
|
|
821
817
|
/** The hour buckets a half-open span touches. */
|
|
822
818
|
function* hourBuckets(fromMs, toMs) {
|
|
823
819
|
const first = Math.floor(fromMs / HOUR_MS$9) * HOUR_MS$9;
|
|
824
820
|
for (let h = first; h < toMs; h += HOUR_MS$9) yield h;
|
|
825
821
|
}
|
|
822
|
+
/** The hour bucket a segment belongs to — the same one its path encodes. */
|
|
823
|
+
function hourOf$1(startMs) {
|
|
824
|
+
return Math.floor(startMs / HOUR_MS$9) * HOUR_MS$9;
|
|
825
|
+
}
|
|
826
|
+
/** The production bound. See {@link RecordingIndexBudget} for the measurement. */
|
|
827
|
+
var DEFAULT_RECORDING_INDEX_BUDGET = {
|
|
828
|
+
residentRowBudget: 25e4,
|
|
829
|
+
residentHourBudget: 2e4,
|
|
830
|
+
evictToFraction: .8,
|
|
831
|
+
pinnedTailHours: 2
|
|
832
|
+
};
|
|
826
833
|
/**
|
|
827
834
|
* Index of the first element whose `startMs` is GREATER than `startMs` (upper
|
|
828
835
|
* bound). Inserting there keeps equal-start rows in arrival order — exactly
|
|
@@ -870,10 +877,35 @@ var RecordingIndex = class {
|
|
|
870
877
|
/** See {@link HydrateReuseStats}. */
|
|
871
878
|
rowsReused = 0;
|
|
872
879
|
rowsAllocated = 0;
|
|
873
|
-
/**
|
|
874
|
-
*
|
|
875
|
-
*
|
|
880
|
+
/**
|
|
881
|
+
* Hour buckets this device has been walked for → when anything last ASKED
|
|
882
|
+
* about that hour ({@link useTick}, 0 = never since the walk). Empty =
|
|
883
|
+
* nothing has been looked at, which is NOT the same as "there is nothing" —
|
|
884
|
+
* see {@link hydrationOf}.
|
|
885
|
+
*
|
|
886
|
+
* The tick is the eviction order, and it lives here rather than in a side
|
|
887
|
+
* map because an evicted hour must lose its marker in the SAME operation
|
|
888
|
+
* that loses its rows: a walked marker outliving its rows would report
|
|
889
|
+
* `hydrated` over an hour the index no longer holds, which is the one
|
|
890
|
+
* failure a forgetting index must not have.
|
|
891
|
+
*
|
|
892
|
+
* There used to be an `ALL_HOURS` sentinel here for a device walked
|
|
893
|
+
* end-to-end. It had no production caller (2026-08-30 census) and a bounded
|
|
894
|
+
* index can never truthfully claim it, so it is gone.
|
|
895
|
+
*/
|
|
876
896
|
walked = /* @__PURE__ */ new Map();
|
|
897
|
+
/** Monotonic "somebody asked" counter. Only ever compared, never read out. */
|
|
898
|
+
useTick = 0;
|
|
899
|
+
/** `null` until {@link armResidency}. An unarmed index never evicts. */
|
|
900
|
+
residencyDeps = null;
|
|
901
|
+
evictedRows = 0;
|
|
902
|
+
evictedHours = 0;
|
|
903
|
+
evictionPasses = 0;
|
|
904
|
+
/** Hours dropped per device, for the life of the process. One number per
|
|
905
|
+
* camera — the input to {@link droppedDuringWalk}, nothing else. */
|
|
906
|
+
evictedHoursByDevice = /* @__PURE__ */ new Map();
|
|
907
|
+
/** {@link evictedHoursByDevice} as it stood when a whole-device walk began. */
|
|
908
|
+
walkBaseline = /* @__PURE__ */ new Map();
|
|
877
909
|
/**
|
|
878
910
|
* Upper bound on the `durMs` of any row the index holds — the ONLY thing that
|
|
879
911
|
* turns "walk the whole sorted view" into "walk backwards a bounded number of
|
|
@@ -1047,7 +1079,9 @@ var RecordingIndex = class {
|
|
|
1047
1079
|
* Deliberately NOT delete-all-then-reinsert: the archive walk re-reports rows
|
|
1048
1080
|
* this index already holds, with freshly built path strings, and dropping the
|
|
1049
1081
|
* held rows first would make those fresh strings the permanent ones — a
|
|
1050
|
-
* second copy of the whole archive's paths. See {@link
|
|
1082
|
+
* second copy of the whole archive's paths. See {@link RecordingIndex.rowFor}
|
|
1083
|
+
* for the measurement (`retainedPathStrings`, which this used to point at,
|
|
1084
|
+
* never existed — a leftover that read as a verification, removed 2026-09-17).
|
|
1051
1085
|
* @residency mutation
|
|
1052
1086
|
*/
|
|
1053
1087
|
hydrateDevice(deviceId, locationId, relPaths) {
|
|
@@ -1061,6 +1095,7 @@ var RecordingIndex = class {
|
|
|
1061
1095
|
}
|
|
1062
1096
|
for (const [path, existing] of m) if (existing.locationId === locationId && !seen.has(path)) m.delete(path);
|
|
1063
1097
|
this.invalidateSorted(deviceId);
|
|
1098
|
+
this.enforceResidency();
|
|
1064
1099
|
}
|
|
1065
1100
|
/**
|
|
1066
1101
|
* Whether the index has WALKED `[fromMs, toMs)` for a device.
|
|
@@ -1074,7 +1109,6 @@ var RecordingIndex = class {
|
|
|
1074
1109
|
hydrationOf(deviceId, fromMs, toMs) {
|
|
1075
1110
|
const w = this.walked.get(deviceId);
|
|
1076
1111
|
if (w === void 0) return "unknown";
|
|
1077
|
-
if (w === ALL_HOURS) return "hydrated";
|
|
1078
1112
|
let seen = 0;
|
|
1079
1113
|
let total = 0;
|
|
1080
1114
|
for (const h of hourBuckets(fromMs, toMs)) {
|
|
@@ -1086,15 +1120,225 @@ var RecordingIndex = class {
|
|
|
1086
1120
|
}
|
|
1087
1121
|
/** Record that `[fromMs, toMs)` has been walked, whatever it contained. @residency mutation */
|
|
1088
1122
|
markHydrated(deviceId, fromMs, toMs) {
|
|
1123
|
+
let w = this.walked.get(deviceId);
|
|
1124
|
+
if (w === void 0) {
|
|
1125
|
+
w = /* @__PURE__ */ new Map();
|
|
1126
|
+
this.walked.set(deviceId, w);
|
|
1127
|
+
}
|
|
1128
|
+
const tick = ++this.useTick;
|
|
1129
|
+
for (const h of hourBuckets(fromMs, toMs)) w.set(h, tick);
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Bound this index, and give it somewhere to SAY what it dropped.
|
|
1133
|
+
*
|
|
1134
|
+
* Called once, from the recorder addon's initialize — `ctx.logger` does not
|
|
1135
|
+
* exist when the field is constructed. Until it runs the index never evicts,
|
|
1136
|
+
* which is honest for the pre-init window (it holds nothing) and is the one
|
|
1137
|
+
* state in which {@link ResidencyCensus.budget} is `null`.
|
|
1138
|
+
* @residency mutation
|
|
1139
|
+
*/
|
|
1140
|
+
armResidency(deps) {
|
|
1141
|
+
this.residencyDeps = deps;
|
|
1142
|
+
this.enforceResidency();
|
|
1143
|
+
}
|
|
1144
|
+
/**
|
|
1145
|
+
* A whole-device walk is starting: remember how much this index had already
|
|
1146
|
+
* forgotten about that camera, so {@link droppedDuringWalk} can say whether
|
|
1147
|
+
* the walk's result is a COMPLETE view of what it read.
|
|
1148
|
+
* @residency mutation
|
|
1149
|
+
*/
|
|
1150
|
+
beginWalk(deviceId) {
|
|
1151
|
+
this.walkBaseline.set(deviceId, this.evictedHoursByDevice.get(deviceId) ?? 0);
|
|
1152
|
+
}
|
|
1153
|
+
/**
|
|
1154
|
+
* Did this index drop anything for `deviceId` since {@link beginWalk}?
|
|
1155
|
+
*
|
|
1156
|
+
* The one question a PRUNE has to ask. An hour the index dropped is
|
|
1157
|
+
* indistinguishable, from the outside, from an hour disk no longer has — so
|
|
1158
|
+
* a consumer that deletes durable state on the index's silence must refuse
|
|
1159
|
+
* when the answer is `true`. `false` when nobody called `beginWalk`: an
|
|
1160
|
+
* unarmed index never drops, and a caller that never announced a walk is
|
|
1161
|
+
* asking about one that did not happen.
|
|
1162
|
+
* @residency resident-scoped
|
|
1163
|
+
*/
|
|
1164
|
+
droppedDuringWalk(deviceId) {
|
|
1165
|
+
const baseline = this.walkBaseline.get(deviceId);
|
|
1166
|
+
if (baseline === void 0) return false;
|
|
1167
|
+
return (this.evictedHoursByDevice.get(deviceId) ?? 0) > baseline;
|
|
1168
|
+
}
|
|
1169
|
+
/** What is held, what was let go, and under which bound. @residency resident-scoped */
|
|
1170
|
+
residency() {
|
|
1171
|
+
return {
|
|
1172
|
+
residentRows: this.residentRowCount(),
|
|
1173
|
+
residentHours: this.residentHourCount(),
|
|
1174
|
+
evictedRows: this.evictedRows,
|
|
1175
|
+
evictedHours: this.evictedHours,
|
|
1176
|
+
evictionPasses: this.evictionPasses,
|
|
1177
|
+
budget: this.residencyDeps?.budget ?? null
|
|
1178
|
+
};
|
|
1179
|
+
}
|
|
1180
|
+
residentRowCount() {
|
|
1181
|
+
let n = 0;
|
|
1182
|
+
for (const m of this.byDevice.values()) n += m.size;
|
|
1183
|
+
return n;
|
|
1184
|
+
}
|
|
1185
|
+
residentHourCount() {
|
|
1186
|
+
let n = 0;
|
|
1187
|
+
for (const w of this.walked.values()) n += w.size;
|
|
1188
|
+
return n;
|
|
1189
|
+
}
|
|
1190
|
+
/**
|
|
1191
|
+
* Note that something ASKED about `[fromMs, toMs)` — the only input to the
|
|
1192
|
+
* eviction order.
|
|
1193
|
+
*
|
|
1194
|
+
* Walks whichever side is smaller: a read whose span is wider than the
|
|
1195
|
+
* device's whole walked map (a month-wide `ranges`, an availability sweep)
|
|
1196
|
+
* would otherwise iterate one hour at a time across the span.
|
|
1197
|
+
*/
|
|
1198
|
+
touch(deviceId, fromMs, toMs) {
|
|
1089
1199
|
const w = this.walked.get(deviceId);
|
|
1090
|
-
if (w ===
|
|
1091
|
-
const
|
|
1092
|
-
|
|
1093
|
-
|
|
1200
|
+
if (w === void 0 || w.size === 0) return;
|
|
1201
|
+
const tick = ++this.useTick;
|
|
1202
|
+
if (Math.ceil(Math.max(0, toMs - fromMs) / HOUR_MS$9) > w.size) {
|
|
1203
|
+
for (const h of w.keys()) if (h < toMs && h + HOUR_MS$9 > fromMs) w.set(h, tick);
|
|
1204
|
+
return;
|
|
1205
|
+
}
|
|
1206
|
+
for (const h of hourBuckets(fromMs, toMs)) if (w.has(h)) w.set(h, tick);
|
|
1094
1207
|
}
|
|
1095
|
-
/**
|
|
1096
|
-
|
|
1097
|
-
|
|
1208
|
+
/**
|
|
1209
|
+
* Drop the least-recently-asked-about hours until the index is back inside
|
|
1210
|
+
* its budget.
|
|
1211
|
+
*
|
|
1212
|
+
* ## What a dropped hour costs, and why that is safe
|
|
1213
|
+
*
|
|
1214
|
+
* The directory is the authority for which segments exist — every row here
|
|
1215
|
+
* is derived from a path the walk read, and `hydrateHour` /
|
|
1216
|
+
* `hydrateWindowFromStorage` rebuild it from one shallow `readdir` (or one
|
|
1217
|
+
* ledger point read). So a drop costs a re-read, never a segment. The
|
|
1218
|
+
* property that makes it costs-a-read rather than loses-a-segment is that
|
|
1219
|
+
* the walked marker goes with the rows: `hydrationOf` then answers `unknown`
|
|
1220
|
+
* and the reader walks. A marker left behind would answer `hydrated` over an
|
|
1221
|
+
* hour holding nothing, which is exactly the silent gap this class spent
|
|
1222
|
+
* D295 learning not to report.
|
|
1223
|
+
*
|
|
1224
|
+
* ## What is never a candidate
|
|
1225
|
+
*
|
|
1226
|
+
* The live tail ({@link RecordingIndexBudget.pinnedTailHours}). Those hours
|
|
1227
|
+
* are write-through from the writer (`addSegment`), not walked, and a
|
|
1228
|
+
* `readdir` of an hour a writer is still filling cannot give back what it
|
|
1229
|
+
* had — D445's lost tail, from the other direction.
|
|
1230
|
+
*/
|
|
1231
|
+
enforceResidency() {
|
|
1232
|
+
const deps = this.residencyDeps;
|
|
1233
|
+
if (deps === null) return;
|
|
1234
|
+
const { budget } = deps;
|
|
1235
|
+
let rows = this.residentRowCount();
|
|
1236
|
+
let hours = this.residentHourCount();
|
|
1237
|
+
if (rows <= budget.residentRowBudget && hours <= budget.residentHourBudget) return;
|
|
1238
|
+
const rowTarget = Math.floor(budget.residentRowBudget * budget.evictToFraction);
|
|
1239
|
+
const hourTarget = Math.floor(budget.residentHourBudget * budget.evictToFraction);
|
|
1240
|
+
const pinnedFrom = hourOf$1(deps.now()) - Math.max(0, budget.pinnedTailHours - 1) * HOUR_MS$9;
|
|
1241
|
+
const counts = this.hourHistogram();
|
|
1242
|
+
const candidates = [];
|
|
1243
|
+
for (const [deviceId, byHour] of counts) {
|
|
1244
|
+
const walkedHours = this.walked.get(deviceId);
|
|
1245
|
+
for (const [hourStartMs, count] of byHour) {
|
|
1246
|
+
if (hourStartMs >= pinnedFrom) continue;
|
|
1247
|
+
candidates.push({
|
|
1248
|
+
deviceId,
|
|
1249
|
+
hourStartMs,
|
|
1250
|
+
rows: count,
|
|
1251
|
+
lastUse: walkedHours?.get(hourStartMs) ?? 0
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
candidates.sort((a, b) => a.lastUse - b.lastUse || a.hourStartMs - b.hourStartMs);
|
|
1256
|
+
const victims = /* @__PURE__ */ new Map();
|
|
1257
|
+
let droppedRows = 0;
|
|
1258
|
+
let droppedHours = 0;
|
|
1259
|
+
for (const c of candidates) {
|
|
1260
|
+
if (rows <= rowTarget && hours <= hourTarget) break;
|
|
1261
|
+
let forDevice = victims.get(c.deviceId);
|
|
1262
|
+
if (!forDevice) {
|
|
1263
|
+
forDevice = /* @__PURE__ */ new Set();
|
|
1264
|
+
victims.set(c.deviceId, forDevice);
|
|
1265
|
+
}
|
|
1266
|
+
forDevice.add(c.hourStartMs);
|
|
1267
|
+
rows -= c.rows;
|
|
1268
|
+
droppedRows += c.rows;
|
|
1269
|
+
if (this.walked.get(c.deviceId)?.has(c.hourStartMs) === true) {
|
|
1270
|
+
hours -= 1;
|
|
1271
|
+
droppedHours += 1;
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
if (victims.size === 0) return;
|
|
1275
|
+
this.evictionPasses += 1;
|
|
1276
|
+
this.evictedRows += droppedRows;
|
|
1277
|
+
this.evictedHours += droppedHours;
|
|
1278
|
+
for (const [deviceId, hoursToDrop] of victims) this.dropHours(deviceId, hoursToDrop, rows, budget, deps.logger);
|
|
1279
|
+
}
|
|
1280
|
+
/** `deviceId → hourStartMs → resident rows`, including hours walked and found
|
|
1281
|
+
* empty (count 0) — those hold no rows and would otherwise never be dropped,
|
|
1282
|
+
* leaving the walked map to grow without bound on its own. */
|
|
1283
|
+
hourHistogram() {
|
|
1284
|
+
const out = /* @__PURE__ */ new Map();
|
|
1285
|
+
for (const [deviceId, m] of this.byDevice) {
|
|
1286
|
+
const byHour = /* @__PURE__ */ new Map();
|
|
1287
|
+
for (const s of m.values()) {
|
|
1288
|
+
const h = hourOf$1(s.startMs);
|
|
1289
|
+
byHour.set(h, (byHour.get(h) ?? 0) + 1);
|
|
1290
|
+
}
|
|
1291
|
+
out.set(deviceId, byHour);
|
|
1292
|
+
}
|
|
1293
|
+
for (const [deviceId, w] of this.walked) {
|
|
1294
|
+
let byHour = out.get(deviceId);
|
|
1295
|
+
if (!byHour) {
|
|
1296
|
+
byHour = /* @__PURE__ */ new Map();
|
|
1297
|
+
out.set(deviceId, byHour);
|
|
1298
|
+
}
|
|
1299
|
+
for (const h of w.keys()) if (!byHour.has(h)) byHour.set(h, 0);
|
|
1300
|
+
}
|
|
1301
|
+
return out;
|
|
1302
|
+
}
|
|
1303
|
+
/** Delete one device's victim hours — rows and walked markers together — and
|
|
1304
|
+
* say so. A structure that silently forgets is how this class of bug hides. */
|
|
1305
|
+
dropHours(deviceId, hoursToDrop, residentRowsAfter, budget, logger) {
|
|
1306
|
+
const m = this.byDevice.get(deviceId);
|
|
1307
|
+
let droppedRows = 0;
|
|
1308
|
+
if (m) {
|
|
1309
|
+
for (const [path, s] of m) {
|
|
1310
|
+
if (!hoursToDrop.has(hourOf$1(s.startMs))) continue;
|
|
1311
|
+
m.delete(path);
|
|
1312
|
+
droppedRows += 1;
|
|
1313
|
+
}
|
|
1314
|
+
if (m.size === 0) this.byDevice.delete(deviceId);
|
|
1315
|
+
}
|
|
1316
|
+
const w = this.walked.get(deviceId);
|
|
1317
|
+
if (w) {
|
|
1318
|
+
for (const h of hoursToDrop) w.delete(h);
|
|
1319
|
+
if (w.size === 0) this.walked.delete(deviceId);
|
|
1320
|
+
}
|
|
1321
|
+
this.invalidateSorted(deviceId);
|
|
1322
|
+
this.evictedHoursByDevice.set(deviceId, (this.evictedHoursByDevice.get(deviceId) ?? 0) + hoursToDrop.size);
|
|
1323
|
+
let oldestHourMs = Infinity;
|
|
1324
|
+
let newestHourMs = -Infinity;
|
|
1325
|
+
for (const h of hoursToDrop) {
|
|
1326
|
+
if (h < oldestHourMs) oldestHourMs = h;
|
|
1327
|
+
if (h > newestHourMs) newestHourMs = h;
|
|
1328
|
+
}
|
|
1329
|
+
logger.info("recorder: index dropped resident hours — over the residency budget", {
|
|
1330
|
+
tags: { deviceId },
|
|
1331
|
+
meta: {
|
|
1332
|
+
hours: hoursToDrop.size,
|
|
1333
|
+
rows: droppedRows,
|
|
1334
|
+
oldestHourMs,
|
|
1335
|
+
newestHourMs,
|
|
1336
|
+
residentRowsAfter,
|
|
1337
|
+
rowBudget: budget.residentRowBudget,
|
|
1338
|
+
hourBudget: budget.residentHourBudget,
|
|
1339
|
+
costOfRecovery: "readdir-per-hour"
|
|
1340
|
+
}
|
|
1341
|
+
});
|
|
1098
1342
|
}
|
|
1099
1343
|
/**
|
|
1100
1344
|
* Merge one hour's segments for a location, and mark that hour walked.
|
|
@@ -1116,6 +1360,39 @@ var RecordingIndex = class {
|
|
|
1116
1360
|
}
|
|
1117
1361
|
if (changed) this.invalidateSorted(deviceId);
|
|
1118
1362
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$9);
|
|
1363
|
+
this.enforceResidency();
|
|
1364
|
+
}
|
|
1365
|
+
/**
|
|
1366
|
+
* Make one hour of ONE location exactly `relPaths` — merge what is there and
|
|
1367
|
+
* drop what is not.
|
|
1368
|
+
*
|
|
1369
|
+
* {@link hydrateHour} merges, which is right for a reader: a listing it took
|
|
1370
|
+
* of a live hour is a lower bound, and dropping on it would delete the index
|
|
1371
|
+
* of footage the readdir merely raced. This is for the one caller that has
|
|
1372
|
+
* TRUSTED evidence the hour is over and the volume it listed is the volume it
|
|
1373
|
+
* started on — the operator-armed ledger walk — where the directory is the
|
|
1374
|
+
* whole truth about the hour and an index row naming a file that is not in it
|
|
1375
|
+
* is a ghost (D523, D148's "trusted walk" arm).
|
|
1376
|
+
*
|
|
1377
|
+
* Scoped to the (device, location, hour) triple: a path filed under another
|
|
1378
|
+
* hour, or held for another location, is none of this call's business.
|
|
1379
|
+
* @residency mutation
|
|
1380
|
+
*/
|
|
1381
|
+
resetHour(deviceId, locationId, hourStartMs, relPaths) {
|
|
1382
|
+
const m = this.mapFor(deviceId);
|
|
1383
|
+
const keep = new Set(relPaths);
|
|
1384
|
+
const dropped = [];
|
|
1385
|
+
for (const [path, row] of m) {
|
|
1386
|
+
if (row.locationId !== locationId) continue;
|
|
1387
|
+
if (Math.floor(row.startMs / HOUR_MS$9) * HOUR_MS$9 !== hourStartMs) continue;
|
|
1388
|
+
if (keep.has(path)) continue;
|
|
1389
|
+
dropped.push(path);
|
|
1390
|
+
}
|
|
1391
|
+
for (const path of dropped) m.delete(path);
|
|
1392
|
+
if (dropped.length > 0) this.invalidateSorted(deviceId);
|
|
1393
|
+
this.hydrateHour(deviceId, locationId, hourStartMs, relPaths);
|
|
1394
|
+
this.enforceResidency();
|
|
1395
|
+
return dropped;
|
|
1119
1396
|
}
|
|
1120
1397
|
/**
|
|
1121
1398
|
* Insert/replace one finalized segment (idempotent by path).
|
|
@@ -1139,6 +1416,7 @@ var RecordingIndex = class {
|
|
|
1139
1416
|
return;
|
|
1140
1417
|
}
|
|
1141
1418
|
this.insertIntoSortedViews(s);
|
|
1419
|
+
this.enforceResidency();
|
|
1142
1420
|
}
|
|
1143
1421
|
/** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. @residency mutation */
|
|
1144
1422
|
removeSegments(paths) {
|
|
@@ -1166,6 +1444,7 @@ var RecordingIndex = class {
|
|
|
1166
1444
|
* @residency resident-scoped
|
|
1167
1445
|
*/
|
|
1168
1446
|
segmentsStartingIn(deviceId, profile, fromMs, toMs) {
|
|
1447
|
+
this.touch(deviceId, fromMs, toMs);
|
|
1169
1448
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1170
1449
|
let lo = 0;
|
|
1171
1450
|
let hi = segs.length;
|
|
@@ -1194,6 +1473,7 @@ var RecordingIndex = class {
|
|
|
1194
1473
|
* @residency resident-scoped
|
|
1195
1474
|
*/
|
|
1196
1475
|
segmentStartingAt(deviceId, profile, startMs) {
|
|
1476
|
+
this.touch(deviceId, startMs, startMs + 1);
|
|
1197
1477
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1198
1478
|
const at = segs[lowerBoundByStart(segs, startMs)];
|
|
1199
1479
|
return at !== void 0 && at.startMs === startMs ? at : null;
|
|
@@ -1217,6 +1497,7 @@ var RecordingIndex = class {
|
|
|
1217
1497
|
* @residency resident-scoped
|
|
1218
1498
|
*/
|
|
1219
1499
|
segmentsOverlapping(deviceId, profile, fromMs, toMs) {
|
|
1500
|
+
this.touch(deviceId, fromMs - this.maxDurMs, toMs);
|
|
1220
1501
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1221
1502
|
const end = lowerBoundByStart(segs, toMs);
|
|
1222
1503
|
let head = lowerBoundByStart(segs, fromMs);
|
|
@@ -1378,6 +1659,7 @@ var RecordingIndex = class {
|
|
|
1378
1659
|
}
|
|
1379
1660
|
/** The segment whose [startMs, startMs+durMs) window contains `epochMs`, or null if `epochMs` falls in a gap / there is no footage. @residency resident-scoped */
|
|
1380
1661
|
segmentAt(deviceId, profile, epochMs) {
|
|
1662
|
+
this.touch(deviceId, epochMs - this.maxDurMs, epochMs + 1);
|
|
1381
1663
|
return this.segmentAtIn(this.sortedSegments(deviceId, profile), epochMs);
|
|
1382
1664
|
}
|
|
1383
1665
|
/**
|
|
@@ -1387,6 +1669,7 @@ var RecordingIndex = class {
|
|
|
1387
1669
|
* @residency resident-scoped
|
|
1388
1670
|
*/
|
|
1389
1671
|
nearestCoveredEdge(deviceId, profile, epochMs) {
|
|
1672
|
+
this.touch(deviceId, epochMs - this.maxDurMs, epochMs + 1);
|
|
1390
1673
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1391
1674
|
if (segs.length === 0) return null;
|
|
1392
1675
|
if (this.segmentAtIn(segs, epochMs) !== null) return epochMs;
|
|
@@ -1440,6 +1723,7 @@ var RecordingIndex = class {
|
|
|
1440
1723
|
* @residency resident-scoped
|
|
1441
1724
|
*/
|
|
1442
1725
|
coveredEdgeBefore(deviceId, profile, epochMs) {
|
|
1726
|
+
this.touch(deviceId, epochMs - this.maxDurMs, epochMs + 1);
|
|
1443
1727
|
const segs = this.sortedSegments(deviceId, profile);
|
|
1444
1728
|
let best = null;
|
|
1445
1729
|
for (let i = upperBoundByStart(segs, epochMs) - 1; i >= 0; i--) {
|
|
@@ -1452,6 +1736,7 @@ var RecordingIndex = class {
|
|
|
1452
1736
|
}
|
|
1453
1737
|
/** @residency resident-scoped */
|
|
1454
1738
|
ranges(deviceId, profile, fromMs, toMs, gapToleranceMs) {
|
|
1739
|
+
this.touch(deviceId, fromMs, toMs);
|
|
1455
1740
|
return mergeRanges(this.sortedSegments(deviceId, profile).filter((s) => s.startMs + s.durMs > fromMs && s.startMs < toMs), gapToleranceMs).map((r) => ({
|
|
1456
1741
|
startMs: Math.max(r.startMs, fromMs),
|
|
1457
1742
|
endMs: Math.min(r.endMs, toMs)
|
|
@@ -1686,6 +1971,7 @@ async function resolveRecordingsLocations(api, logger) {
|
|
|
1686
1971
|
async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger, options = {}) {
|
|
1687
1972
|
const yieldBetween = options.yieldBetween ?? macrotask$3;
|
|
1688
1973
|
const chunkSize = options.chunkSize ?? HYDRATE_CHUNK;
|
|
1974
|
+
index.beginWalk(deviceId);
|
|
1689
1975
|
for (const [root, aliases] of aliasesByRoot(locations)) {
|
|
1690
1976
|
const deviceDir = path.join(root, String(deviceId));
|
|
1691
1977
|
const startedMs = Date.now();
|
|
@@ -1766,40 +2052,6 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1766
2052
|
const verify = options?.verifyClosedHours === true;
|
|
1767
2053
|
if (!verify && index.hydrationOf(deviceId, fromMs, toMs) === "hydrated") return;
|
|
1768
2054
|
const liveHour = Math.floor(Date.now() / HOUR_MS$8) * HOUR_MS$8;
|
|
1769
|
-
/** Per (locationId, profile, hour): what the LEDGER claimed, for the diff. */
|
|
1770
|
-
const claimed = /* @__PURE__ */ new Map();
|
|
1771
|
-
const claimKey = (locationId, hour) => `${locationId}:${hour}`;
|
|
1772
|
-
/**
|
|
1773
|
-
* Closed hours the ledger answered for with a row nobody SEALED (D451): what
|
|
1774
|
-
* a writer had persisted when it was last heard from, which is a legitimate
|
|
1775
|
-
* answer and not a complete one. Each is read from the array once and the
|
|
1776
|
-
* row is unioned and sealed, so the second reader pays a point read.
|
|
1777
|
-
*/
|
|
1778
|
-
const unsealed = /* @__PURE__ */ new Set();
|
|
1779
|
-
if (ledger !== void 0) {
|
|
1780
|
-
const startedMs = Date.now();
|
|
1781
|
-
const hours = await ledger.detailForWindow(deviceId, fromMs, toMs);
|
|
1782
|
-
let segments = 0;
|
|
1783
|
-
let served = 0;
|
|
1784
|
-
for (const hour of hours) {
|
|
1785
|
-
if (hour.hourStartMs >= liveHour) continue;
|
|
1786
|
-
index.hydrateHour(deviceId, hour.locationId, hour.hourStartMs, hour.paths);
|
|
1787
|
-
claimed.set(claimKey(hour.locationId, hour.hourStartMs), hour.paths.length);
|
|
1788
|
-
if (!hour.sealed) unsealed.add(hour.hourStartMs);
|
|
1789
|
-
segments += hour.paths.length;
|
|
1790
|
-
served += 1;
|
|
1791
|
-
}
|
|
1792
|
-
if (served > 0) logger.info("recorder: hydrated a window from the hour ledger", {
|
|
1793
|
-
tags: { deviceId },
|
|
1794
|
-
meta: {
|
|
1795
|
-
hours: served,
|
|
1796
|
-
segments,
|
|
1797
|
-
unsealedHours: unsealed.size,
|
|
1798
|
-
ms: Date.now() - startedMs
|
|
1799
|
-
}
|
|
1800
|
-
});
|
|
1801
|
-
if (!verify && unsealed.size === 0 && index.hydrationOf(deviceId, fromMs, toMs) === "hydrated") return;
|
|
1802
|
-
}
|
|
1803
2055
|
for (const [root, aliases] of aliasesByRoot(locations)) {
|
|
1804
2056
|
const deviceDir = path.join(root, String(deviceId));
|
|
1805
2057
|
let profiles;
|
|
@@ -1823,7 +2075,7 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1823
2075
|
let forgottenHours = 0;
|
|
1824
2076
|
let sealed = 0;
|
|
1825
2077
|
for (let hour = first; hour < toMs; hour += HOUR_MS$8) {
|
|
1826
|
-
if (!verify && hour < liveHour &&
|
|
2078
|
+
if (!verify && hour < liveHour && index.hydrationOf(deviceId, hour, hour + HOUR_MS$8) === "hydrated") continue;
|
|
1827
2079
|
for (const profile of profiles) {
|
|
1828
2080
|
const rel = hourDirRelPath(deviceId, profile, hour);
|
|
1829
2081
|
let files;
|
|
@@ -1837,11 +2089,10 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1837
2089
|
const relPaths = files.filter((f) => f.endsWith(".m4s")).map((f) => `${rel}/${f}`);
|
|
1838
2090
|
index.hydrateHour(deviceId, locationId, hour, relPaths);
|
|
1839
2091
|
if (ledger === void 0 || relPaths.length === 0 || hour + HOUR_MS$8 > Date.now()) continue;
|
|
1840
|
-
const claimedHere = claimed.get(claimKey(locationId, hour));
|
|
1841
2092
|
const outcome = await ledger.adoptWalkedHour(deviceId, profile, locationId, hour, relPaths);
|
|
1842
2093
|
if (outcome === null) continue;
|
|
1843
2094
|
if (outcome.created) repaired += 1;
|
|
1844
|
-
else if (
|
|
2095
|
+
else if (outcome.added > 0) {
|
|
1845
2096
|
forgottenSegments += outcome.added;
|
|
1846
2097
|
forgottenHours += 1;
|
|
1847
2098
|
}
|
|
@@ -2478,10 +2729,12 @@ var RelocateEngine = class {
|
|
|
2478
2729
|
*
|
|
2479
2730
|
* The order is the whole point. Between the copy and the delete both
|
|
2480
2731
|
* locations hold every byte, so the ledger re-point in the middle can only
|
|
2481
|
-
* ever
|
|
2482
|
-
* recoverable instead of a black window.
|
|
2483
|
-
*
|
|
2484
|
-
*
|
|
2732
|
+
* ever account for a location that HAS the files — which is what makes a
|
|
2733
|
+
* crash recoverable instead of a black window.
|
|
2734
|
+
*
|
|
2735
|
+
* There is no ledger refusal to handle any more (D523): the durable half is
|
|
2736
|
+
* arithmetic over a row the mirror already holds, so nothing on that path can
|
|
2737
|
+
* fail and stop an hour dead.
|
|
2485
2738
|
*/
|
|
2486
2739
|
async moveHour(job, rows, from, to, samePhysicalRoot, sleep, bytesPerMs) {
|
|
2487
2740
|
const movable = [];
|
|
@@ -2517,17 +2770,7 @@ var RelocateEngine = class {
|
|
|
2517
2770
|
toLocationId: to.id
|
|
2518
2771
|
}
|
|
2519
2772
|
});
|
|
2520
|
-
else
|
|
2521
|
-
this.deps.logger.warn("relocate: hour NOT moved — the durable ledger refused the re-point, so the source bytes are kept", {
|
|
2522
|
-
tags: { deviceId: movable[0].deviceId },
|
|
2523
|
-
meta: {
|
|
2524
|
-
jobId: job.jobId,
|
|
2525
|
-
files: movable.length,
|
|
2526
|
-
toLocationId: to.id
|
|
2527
|
-
}
|
|
2528
|
-
});
|
|
2529
|
-
return;
|
|
2530
|
-
}
|
|
2773
|
+
else await ledger.repointSegments(movable, to.id);
|
|
2531
2774
|
if (!samePhysicalRoot) for (const row of movable) await promises.rm(path.join(from.root, row.fromPath ?? row.path), { force: true });
|
|
2532
2775
|
for (const row of movable) {
|
|
2533
2776
|
this.deps.index.removeSegments([row.fromPath ?? row.path]);
|
|
@@ -2920,23 +3163,21 @@ var SegmentStore = class SegmentStore {
|
|
|
2920
3163
|
* work now the per-bucket archive scan is gone. */
|
|
2921
3164
|
static DEFAULT_EVICT_CHUNK = 25;
|
|
2922
3165
|
/**
|
|
2923
|
-
* True when `victims` cover EVERY segment the
|
|
2924
|
-
*
|
|
3166
|
+
* True when `victims` cover EVERY segment the bucket DIRECTORY holds, AND the
|
|
3167
|
+
* bucket's hour has fully elapsed.
|
|
2925
3168
|
*
|
|
2926
3169
|
* The current-hour guard matters because `onFinalized` relocates new segments
|
|
2927
3170
|
* INTO the current bucket concurrently — an rm racing that rename would
|
|
2928
3171
|
* orphan a just-indexed segment.
|
|
2929
3172
|
*
|
|
2930
|
-
* The cover test is
|
|
2931
|
-
* `RecordingIndex
|
|
2932
|
-
*
|
|
2933
|
-
*
|
|
2934
|
-
*
|
|
2935
|
-
*
|
|
2936
|
-
* went with every file the round never named, leaving their durable rows
|
|
2937
|
-
* behind as ghosts.
|
|
3173
|
+
* The cover test is a listing of the directory the `rm -rf` is about to take,
|
|
3174
|
+
* never `RecordingIndex` and never a durable row (D523). A bucketDir is
|
|
3175
|
+
* `<dev>/<profile>/Y/M/D/H`, so one listing answers exactly the question the
|
|
3176
|
+
* delete asks. Asked of the resident index it read EQUAL whenever the hour was
|
|
3177
|
+
* partly resident — both sides short by the same rows — and the directory went
|
|
3178
|
+
* with every file the round never named.
|
|
2938
3179
|
*
|
|
2939
|
-
* An unanswerable
|
|
3180
|
+
* An unanswerable listing returns `false`: per-file deletes are slower and
|
|
2940
3181
|
* always correct, and the alternative is a recursive delete computed from a
|
|
2941
3182
|
* fraction of the truth.
|
|
2942
3183
|
*/
|
|
@@ -2945,7 +3186,7 @@ var SegmentStore = class SegmentStore {
|
|
|
2945
3186
|
if (!first) return false;
|
|
2946
3187
|
const bucketStartMs = Math.floor(first.startMs / SegmentStore.HOUR_MS) * SegmentStore.HOUR_MS;
|
|
2947
3188
|
if ((this.deps.now?.() ?? Date.now()) < bucketStartMs + SegmentStore.HOUR_MS) return false;
|
|
2948
|
-
const ask = this.deps.
|
|
3189
|
+
const ask = this.deps.hourDirectoryPaths;
|
|
2949
3190
|
if (ask === void 0) return false;
|
|
2950
3191
|
const held = await ask(first.deviceId, first.profile, location, bucketStartMs);
|
|
2951
3192
|
if (held === null || held.length === 0) return false;
|
|
@@ -4078,8 +4319,9 @@ var CalendarIndex = class {
|
|
|
4078
4319
|
*
|
|
4079
4320
|
* Also used to warm the days ADJACENT to one being viewed — changing day is a
|
|
4080
4321
|
* single click and the operator expects the summary to be there. That warm
|
|
4081
|
-
* belongs in the background:
|
|
4082
|
-
*
|
|
4322
|
+
* belongs in the background: a cold read costs whatever the array charges
|
|
4323
|
+
* that day (see the header — the ~650 ms figure is from the retired FUSE
|
|
4324
|
+
* share), and doing it inline would only move the wait onto the click.
|
|
4083
4325
|
*/
|
|
4084
4326
|
async warm(deviceId, fromMs, toMs) {
|
|
4085
4327
|
for (let day = utcDayOf(fromMs); day <= utcDayOf(toMs); day += DAY_MS$3) {
|
|
@@ -5200,22 +5442,18 @@ function createFootageDiscardJob(deps) {
|
|
|
5200
5442
|
const forgetLedger = async (location) => {
|
|
5201
5443
|
const ledger = deps.ledger;
|
|
5202
5444
|
if (ledger === null) return 0;
|
|
5203
|
-
let forgotten = 0;
|
|
5204
5445
|
const profiles = profilesForType(location.type);
|
|
5205
|
-
|
|
5446
|
+
const scope = {
|
|
5206
5447
|
locationId: location.id,
|
|
5207
5448
|
profiles
|
|
5208
|
-
}
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
});
|
|
5217
|
-
await sleep(0);
|
|
5218
|
-
}
|
|
5449
|
+
};
|
|
5450
|
+
const hours = ledger.snapshot().filter((row) => row.locationId === location.id && profiles.includes(row.profile));
|
|
5451
|
+
const forgotten = await ledger.forgetScope(scope);
|
|
5452
|
+
for (const hour of hours) deps.index.resetHour(hour.deviceId, hour.locationId, hour.hourStartMs, []);
|
|
5453
|
+
report({
|
|
5454
|
+
dbRowsDeleted: forgotten,
|
|
5455
|
+
detail: `forgot ${forgotten} ledger segments`
|
|
5456
|
+
});
|
|
5219
5457
|
return forgotten;
|
|
5220
5458
|
};
|
|
5221
5459
|
return {
|
|
@@ -8268,63 +8506,6 @@ function runFfmpeg(renderDeps, args, ext) {
|
|
|
8268
8506
|
});
|
|
8269
8507
|
}
|
|
8270
8508
|
//#endregion
|
|
8271
|
-
//#region src/recorder/addon/paths-codec.ts
|
|
8272
|
-
/**
|
|
8273
|
-
* How an hour row's `paths` becomes text, and back (D471).
|
|
8274
|
-
*
|
|
8275
|
-
* `recorder:segment-hours` is 66 MB of the settings database and **99.2 % of it
|
|
8276
|
-
* is this one column** — 15 643 bytes per row over 4 036 rows. The content is a
|
|
8277
|
-
* list of location-relative segment paths that nearly all share the same
|
|
8278
|
-
* directory prefix, repeated in full on every entry. Measured per column on a
|
|
8279
|
-
* copy of the live database, never the live file (D468).
|
|
8280
|
-
*
|
|
8281
|
-
* Compressing it is unusually safe here because of D287's two-tier mirror: boot
|
|
8282
|
-
* loads the aggregate half of every row and NOT `paths`, so this column is read
|
|
8283
|
-
* only on a point materialise, never on the hot path.
|
|
8284
|
-
*
|
|
8285
|
-
* A local copy of the same idea `pipeline-analytics` uses for `positions` —
|
|
8286
|
-
* addons never import each other (CLAUDE.md), so twenty duplicated lines are
|
|
8287
|
-
* the correct shape, not a shared dependency across an addon boundary.
|
|
8288
|
-
*/
|
|
8289
|
-
/** Marks a value this module wrote. A plain JSON array always starts `[`. */
|
|
8290
|
-
var ENCODED_PREFIX = "z1:";
|
|
8291
|
-
function encodePaths(paths) {
|
|
8292
|
-
return ENCODED_PREFIX + gzipSync(Buffer.from(JSON.stringify(paths), "utf8"), { level: 6 }).toString("base64");
|
|
8293
|
-
}
|
|
8294
|
-
/**
|
|
8295
|
-
* Decode whichever era wrote the value: the `z1:` string this writes, the plain
|
|
8296
|
-
* JSON string every pre-D471 row holds, or an array the store already parsed.
|
|
8297
|
-
*
|
|
8298
|
-
* `null` means "this row is unreadable", which the caller turns into a DISCARDED
|
|
8299
|
-
* row — the same contract the bare `JSON.parse` had. An hour whose paths cannot
|
|
8300
|
-
* be read must not silently become an hour with no segments: that would report
|
|
8301
|
-
* an empty disk, which is the failure `recordToRow` was already written to
|
|
8302
|
-
* avoid.
|
|
8303
|
-
*/
|
|
8304
|
-
function decodePaths(value) {
|
|
8305
|
-
if (Array.isArray(value)) return isPathList(value) ? value : null;
|
|
8306
|
-
if (typeof value !== "string") return null;
|
|
8307
|
-
const json = value.startsWith(ENCODED_PREFIX) ? tryGunzip(value.slice(3)) : value;
|
|
8308
|
-
if (json === null) return null;
|
|
8309
|
-
try {
|
|
8310
|
-
const parsed = JSON.parse(json);
|
|
8311
|
-
if (!Array.isArray(parsed)) return null;
|
|
8312
|
-
return isPathList(parsed) ? parsed : null;
|
|
8313
|
-
} catch {
|
|
8314
|
-
return null;
|
|
8315
|
-
}
|
|
8316
|
-
}
|
|
8317
|
-
function isPathList(v) {
|
|
8318
|
-
return v.every((p) => typeof p === "string" && p.length > 0);
|
|
8319
|
-
}
|
|
8320
|
-
function tryGunzip(b64) {
|
|
8321
|
-
try {
|
|
8322
|
-
return gunzipSync(Buffer.from(b64, "base64")).toString("utf8");
|
|
8323
|
-
} catch {
|
|
8324
|
-
return null;
|
|
8325
|
-
}
|
|
8326
|
-
}
|
|
8327
|
-
//#endregion
|
|
8328
8509
|
//#region src/durable/durable-ledger.ts
|
|
8329
8510
|
/** Default reseed cap — every current consumer's row set is installation-bounded. */
|
|
8330
8511
|
var DEFAULT_LOAD_LIMIT = 1e5;
|
|
@@ -8579,19 +8760,19 @@ var macrotask$1 = () => new Promise((resolve) => {
|
|
|
8579
8760
|
});
|
|
8580
8761
|
/**
|
|
8581
8762
|
* @durable class=mirror owner=recorder
|
|
8582
|
-
* write="write-behind on every finalized segment (hour-row upsert, serialised per hour); the writer SEALS the hour it leaves (one write per hour); a
|
|
8763
|
+
* write="write-behind on every finalized segment (hour-row upsert, serialised per hour); the writer SEALS the hour it leaves (one write per hour); a listing of a closed hour re-derives the aggregate and seals; eviction and relocation shrink it"
|
|
8583
8764
|
* retention="a row goes when the walk or eviction confirms the hour is empty — never by age, never from the ledger alone (D148)"
|
|
8584
8765
|
*/
|
|
8585
8766
|
var RECORDING_SEGMENT_HOURS_COLLECTION = "recorder:segment-hours";
|
|
8586
8767
|
/**
|
|
8587
|
-
* The four aggregate columns
|
|
8588
|
-
*
|
|
8589
|
-
*
|
|
8590
|
-
*
|
|
8591
|
-
*
|
|
8592
|
-
*
|
|
8593
|
-
*
|
|
8594
|
-
*
|
|
8768
|
+
* The four aggregate columns are what this table IS (D523).
|
|
8769
|
+
*
|
|
8770
|
+
* They exist so "how much footage is on this disk" is a `SUM`/`MIN`/`MAX` over
|
|
8771
|
+
* ~19 k hour rows instead of a walk over ~7.1 M in-RAM segment rows, and they
|
|
8772
|
+
* are derived from segment NAMES — by {@link withSegmentPath} and nothing else.
|
|
8773
|
+
* {@link hourAggregate} is that step over a whole listing and
|
|
8774
|
+
* {@link appendSegmentPath} is that step over the one path that just arrived,
|
|
8775
|
+
* so there is no second derivation to drift.
|
|
8595
8776
|
*/
|
|
8596
8777
|
var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
8597
8778
|
{
|
|
@@ -8620,11 +8801,6 @@ var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
|
8620
8801
|
type: "INTEGER",
|
|
8621
8802
|
notNull: true
|
|
8622
8803
|
},
|
|
8623
|
-
{
|
|
8624
|
-
name: "paths",
|
|
8625
|
-
type: "TEXT",
|
|
8626
|
-
notNull: true
|
|
8627
|
-
},
|
|
8628
8804
|
{
|
|
8629
8805
|
name: "bytes",
|
|
8630
8806
|
type: "INTEGER",
|
|
@@ -8683,15 +8859,9 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
|
8683
8859
|
* `MAX(maxStartMs)` over `locationId IN (…)` — one row back, and with this
|
|
8684
8860
|
* index not one table row read to produce it (D447).
|
|
8685
8861
|
*
|
|
8686
|
-
* `
|
|
8687
|
-
*
|
|
8688
|
-
*
|
|
8689
|
-
* accounting read at the price of ~30 KB of table page per hour row. On the
|
|
8690
|
-
* live hub that was `durationMs=1001…2902` once a MINUTE on
|
|
8691
|
-
* `hub/sqlite-settings` (174 slow-call WARNs in ten hours on 2026-09-10/11),
|
|
8692
|
-
* every one of them holding the thread every configuration read in the
|
|
8693
|
-
* cluster waits on. The index is ~50 bytes a row over ~19 k rows — nothing —
|
|
8694
|
-
* and the first declare after it ships builds it in well under a second.
|
|
8862
|
+
* Since D523 dropped `paths` the table row is ten scalars, so the covering
|
|
8863
|
+
* index no longer saves ~30 KB of page per row; it still saves the row
|
|
8864
|
+
* fetch, and the accounting read is the one this hub runs on a schedule.
|
|
8695
8865
|
*/
|
|
8696
8866
|
{
|
|
8697
8867
|
name: "idx_recorder_segment_hours_loc_accounting",
|
|
@@ -8716,13 +8886,13 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
|
8716
8886
|
* nominal `-segment_time` (ffmpeg cuts on the next GOP, never on the clock).
|
|
8717
8887
|
*
|
|
8718
8888
|
* It is a guess, and this is the shape of being wrong in each direction.
|
|
8719
|
-
* TIGHTER: a normal rollover stops sealing
|
|
8720
|
-
* once by
|
|
8721
|
-
*
|
|
8722
|
-
*
|
|
8723
|
-
*
|
|
8724
|
-
*
|
|
8725
|
-
*
|
|
8889
|
+
* TIGHTER: a normal rollover stops sealing and every hour of the archive is
|
|
8890
|
+
* listed once by the adoption sweep — at ~1 ms per hour directory on this hub's
|
|
8891
|
+
* SATA SSD, affordable but pointless. LOOSER: a runner respawn inside the
|
|
8892
|
+
* window seals an hour it entered late, which is exactly the 2026-09-17 defect;
|
|
8893
|
+
* the observed hole there was ~29 minutes, so anything under a few minutes
|
|
8894
|
+
* would have caught it and 90 s is the smallest number that does not fire on a
|
|
8895
|
+
* legitimate rollover.
|
|
8726
8896
|
*/
|
|
8727
8897
|
var WRITER_HOUR_ENTRY_GRACE_MS = 9e4;
|
|
8728
8898
|
/** The identity of the {@link hourAggregate} fold — an hour holding nothing. */
|
|
@@ -8736,12 +8906,11 @@ var EMPTY_HOUR_AGGREGATE = {
|
|
|
8736
8906
|
* ONE path folded into an aggregate — the whole derivation, expressed as its
|
|
8737
8907
|
* single step.
|
|
8738
8908
|
*
|
|
8739
|
-
* {@link hourAggregate} is this step over
|
|
8909
|
+
* {@link hourAggregate} is this step over a whole listing, and {@link
|
|
8740
8910
|
* SegmentHourLedger.recordSegment} is this step over the one path that just
|
|
8741
8911
|
* arrived. That is not two implementations that happen to agree: the full
|
|
8742
8912
|
* derivation is *defined* as this function repeated, so an append is the fold
|
|
8743
|
-
* resumed from the state the held row already carries
|
|
8744
|
-
* aggregate half IS the fold state over its `paths`.
|
|
8913
|
+
* resumed from the state the held row already carries.
|
|
8745
8914
|
*
|
|
8746
8915
|
* Why it had to become a step: `recordSegment` re-derived the whole hour on
|
|
8747
8916
|
* every finalize. Σ1..360 ≈ 65 k path parses per camera-profile-hour at 10 s
|
|
@@ -8764,49 +8933,57 @@ function withSegmentPath(agg, path) {
|
|
|
8764
8933
|
};
|
|
8765
8934
|
}
|
|
8766
8935
|
/**
|
|
8767
|
-
* Bytes / count / oldest / newest for one hour's
|
|
8768
|
-
* derivation,
|
|
8769
|
-
* by the append path.
|
|
8936
|
+
* Bytes / count / oldest / newest for one hour's LISTING — the sole FULL
|
|
8937
|
+
* derivation, and since D523 the only thing that can produce an EXACT row.
|
|
8770
8938
|
*
|
|
8771
8939
|
* Every field of a segment path encodes its own facts (`parseSegmentPath`), so
|
|
8772
|
-
* this is a derivation, not a measurement: it can never disagree with the
|
|
8773
|
-
* it was computed from.
|
|
8940
|
+
* this is a derivation, not a measurement: it can never disagree with the
|
|
8941
|
+
* listing it was computed from.
|
|
8774
8942
|
*/
|
|
8775
8943
|
function hourAggregate(paths) {
|
|
8776
8944
|
let agg = EMPTY_HOUR_AGGREGATE;
|
|
8777
8945
|
for (const path of paths) agg = withSegmentPath(agg, path);
|
|
8778
8946
|
return agg;
|
|
8779
8947
|
}
|
|
8780
|
-
/** Build the persisted row for an hour from
|
|
8948
|
+
/** Build the persisted row for an hour from a LISTING of it — the ONE
|
|
8949
|
+
* constructor that can produce an exact aggregate. */
|
|
8781
8950
|
function segmentHourRow(base) {
|
|
8951
|
+
const { paths, sealed, ...identity } = base;
|
|
8782
8952
|
return {
|
|
8783
|
-
...
|
|
8784
|
-
sealed:
|
|
8785
|
-
|
|
8786
|
-
|
|
8953
|
+
...identity,
|
|
8954
|
+
sealed: sealed ?? false,
|
|
8955
|
+
...hourAggregate(paths)
|
|
8956
|
+
};
|
|
8957
|
+
}
|
|
8958
|
+
/**
|
|
8959
|
+
* Build the persisted row for an hour from an aggregate somebody already folded
|
|
8960
|
+
* — a relocation moving one, or a listing that has already been reduced.
|
|
8961
|
+
*
|
|
8962
|
+
* `sealed` is a NAMED field and never a positional boolean, because
|
|
8963
|
+
* `scripts/check-seal-authority.ts` attributes every literal `sealed: true` in
|
|
8964
|
+
* the recorder to the method that wrote it. A seal smuggled in as a third
|
|
8965
|
+
* argument is a seal the guard cannot see.
|
|
8966
|
+
*/
|
|
8967
|
+
function hourRowOf(identity, row) {
|
|
8968
|
+
return {
|
|
8969
|
+
...identity,
|
|
8970
|
+
...row
|
|
8787
8971
|
};
|
|
8788
8972
|
}
|
|
8789
8973
|
/**
|
|
8790
8974
|
* The row an hour becomes when ONE more path joins it.
|
|
8791
8975
|
*
|
|
8792
|
-
* Identical to `segmentHourRow({
|
|
8793
|
-
* strictly cheaper: the aggregate half of `held` is already the fold over
|
|
8794
|
-
*
|
|
8976
|
+
* Identical to `segmentHourRow({ …held, paths: [...everything, path] })` and
|
|
8977
|
+
* strictly cheaper: the aggregate half of `held` is already the fold over the
|
|
8978
|
+
* hour's names, so this resumes that fold instead of restarting it. The
|
|
8795
8979
|
* equality is not an assumption — `segment-hour-append.spec.ts` pins it on
|
|
8796
8980
|
* generated hours including out-of-order arrival, duplicate paths, unparseable
|
|
8797
8981
|
* paths and storm-sized hours.
|
|
8798
|
-
*
|
|
8799
|
-
* The four fields are named rather than spread on purpose. {@link
|
|
8800
|
-
* withSegmentPath} hands its argument straight back for a path the parser
|
|
8801
|
-
* refuses, and `held` is a whole row — spreading that return would put the
|
|
8802
|
-
* PREVIOUS `paths` array back over the appended one. Which is exactly what it
|
|
8803
|
-
* did until the generated-hours test caught it.
|
|
8804
8982
|
*/
|
|
8805
8983
|
function appendSegmentPath(held, path) {
|
|
8806
8984
|
const { bytes, segments, minStartMs, maxStartMs } = withSegmentPath(held, path);
|
|
8807
8985
|
return {
|
|
8808
8986
|
...held,
|
|
8809
|
-
paths: [...held.paths, path],
|
|
8810
8987
|
bytes,
|
|
8811
8988
|
segments,
|
|
8812
8989
|
minStartMs,
|
|
@@ -8824,6 +9001,20 @@ function appendSegmentPath(held, path) {
|
|
|
8824
9001
|
* `readdir` over an unmounted root deletes all of it.
|
|
8825
9002
|
*/
|
|
8826
9003
|
var LEDGER_PRUNE_MAX_SHARE = .5;
|
|
9004
|
+
/**
|
|
9005
|
+
* The largest share of ONE hour's accounted segments a single listing may
|
|
9006
|
+
* erase (D523).
|
|
9007
|
+
*
|
|
9008
|
+
* The directory is the authority for WHICH segments exist, so a listing of a
|
|
9009
|
+
* closed hour replaces that hour's aggregate — including downwards, which is
|
|
9010
|
+
* what makes an over-counted row self-correcting. The guard is the same
|
|
9011
|
+
* argument {@link LEDGER_PRUNE_MAX_SHARE} makes one level up: a root that has
|
|
9012
|
+
* gone away, or a bucket half-deleted by a round that died, answers SHORT, and
|
|
9013
|
+
* a short answer that halves an hour is indistinguishable from one that is
|
|
9014
|
+
* simply right. Under the bar the listing wins; over it the row stands, the
|
|
9015
|
+
* hour stays unsealed, and the disagreement is named.
|
|
9016
|
+
*/
|
|
9017
|
+
var HOUR_LISTING_MAX_SHRINK_SHARE = .5;
|
|
8827
9018
|
/** Backstop on the oldest-first probe. A bounded read must stay bounded even if
|
|
8828
9019
|
* a caller asks for a target no archive can cover. */
|
|
8829
9020
|
var OLDEST_MAX_PAGES = 16;
|
|
@@ -8833,9 +9024,6 @@ var OLDEST_PROBE_COLUMNS = [
|
|
|
8833
9024
|
"bytes",
|
|
8834
9025
|
"locationId"
|
|
8835
9026
|
];
|
|
8836
|
-
/** Aggregate-repair probe: the primary key always comes back, so this asks for
|
|
8837
|
-
* the predicate column alone. */
|
|
8838
|
-
var REPAIR_PROBE_COLUMNS = ["segments"];
|
|
8839
9027
|
/**
|
|
8840
9028
|
* Backstop on {@link SegmentHourLedger.streamSegments}.
|
|
8841
9029
|
*
|
|
@@ -8874,47 +9062,8 @@ var CENSUS_FIELDS = [
|
|
|
8874
9062
|
op: "max"
|
|
8875
9063
|
}
|
|
8876
9064
|
];
|
|
8877
|
-
/** Rows repaired per boot. The live archive is ~19 k hour rows. */
|
|
8878
|
-
var REPAIR_MAX_ROWS = 5e4;
|
|
8879
|
-
/**
|
|
8880
|
-
* The boot projection (D287): everything the recorder's always-resident tier
|
|
8881
|
-
* needs, and deliberately NOT `paths`.
|
|
8882
|
-
*
|
|
8883
|
-
* `paths` is ~55 B per segment and the archive is the only thing that grows:
|
|
8884
|
-
* ~16 MB of JSON today, a projected ~460 MB at 28 TB of today's bitrate mix
|
|
8885
|
-
* and ~5.9 GB at a low-bitrate mix — one tRPC response, parsed on the main
|
|
8886
|
-
* thread, before the index exists. The summary half is O(writers × days) and
|
|
8887
|
-
* stays ~10–15 MB at any archive size.
|
|
8888
|
-
*/
|
|
8889
|
-
var SEGMENT_HOUR_SUMMARY_COLUMNS = [
|
|
8890
|
-
"deviceId",
|
|
8891
|
-
"profile",
|
|
8892
|
-
"locationId",
|
|
8893
|
-
"hourStartMs",
|
|
8894
|
-
"bytes",
|
|
8895
|
-
"segments",
|
|
8896
|
-
"minStartMs",
|
|
8897
|
-
"maxStartMs",
|
|
8898
|
-
"sealed"
|
|
8899
|
-
];
|
|
8900
9065
|
/** Summary rows folded into the mirror between yields. */
|
|
8901
9066
|
var LOAD_CHUNK = 5e3;
|
|
8902
|
-
/** Hour rows materialised per point-read page. One page covers a day of one
|
|
8903
|
-
* camera's three profiles with room to spare. */
|
|
8904
|
-
var HOUR_DETAIL_PAGE = 64;
|
|
8905
|
-
/**
|
|
8906
|
-
* The live tail materialised at boot: the current hour and the one before it.
|
|
8907
|
-
*
|
|
8908
|
-
* `recordSegment` must gate from RAM (D49), and the hour it appends to at boot
|
|
8909
|
-
* is exactly the hour the previous process was writing. Two hours × the active
|
|
8910
|
-
* writers is ~60 rows — one point-read page — and it is what makes the write
|
|
8911
|
-
* path's materialise a case that essentially never happens.
|
|
8912
|
-
*/
|
|
8913
|
-
var LIVE_TAIL_HOURS = 2;
|
|
8914
|
-
/** Rows the one-shot aggregate repair may materialise in a single boot. The
|
|
8915
|
-
* backfill is bounded so it cannot re-become the whole-archive read D287
|
|
8916
|
-
* removed; a partial pass finishes on the next boot. */
|
|
8917
|
-
var REPAIR_MAX_MATERIALIZE = 2e3;
|
|
8918
9067
|
/** The four scalars an accounting answer is made of, asked in one statement. */
|
|
8919
9068
|
var ACCOUNTING_FIELDS = [
|
|
8920
9069
|
{
|
|
@@ -8967,14 +9116,6 @@ function writerKey(deviceId, profile) {
|
|
|
8967
9116
|
function writerHourKey(deviceId, profile, hour) {
|
|
8968
9117
|
return `${deviceId}|${profile}|${hour}`;
|
|
8969
9118
|
}
|
|
8970
|
-
function detailOf(row) {
|
|
8971
|
-
return {
|
|
8972
|
-
locationId: row.locationId,
|
|
8973
|
-
hourStartMs: row.hourStartMs,
|
|
8974
|
-
paths: row.paths,
|
|
8975
|
-
sealed: row.sealed
|
|
8976
|
-
};
|
|
8977
|
-
}
|
|
8978
9119
|
/**
|
|
8979
9120
|
* The `hourStartMs` of one probe record.
|
|
8980
9121
|
*
|
|
@@ -8986,21 +9127,31 @@ function hourOfRecord(record) {
|
|
|
8986
9127
|
const raw = record.data["hourStartMs"];
|
|
8987
9128
|
return typeof raw === "number" && Number.isFinite(raw) ? raw : -1;
|
|
8988
9129
|
}
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
|
|
8992
|
-
|
|
8993
|
-
|
|
8994
|
-
|
|
8995
|
-
|
|
9130
|
+
/** One listing turned into the segment rows a caller can act on — the only
|
|
9131
|
+
* place a `SegmentRow` is built out of a directory read. */
|
|
9132
|
+
function segmentRowsOf(locationId, paths) {
|
|
9133
|
+
const rows = [];
|
|
9134
|
+
let unparseable = 0;
|
|
9135
|
+
for (const path of paths) {
|
|
9136
|
+
const parsed = parseSegmentPath(path);
|
|
9137
|
+
if (parsed === null) {
|
|
9138
|
+
unparseable += 1;
|
|
9139
|
+
continue;
|
|
9140
|
+
}
|
|
9141
|
+
rows.push({
|
|
9142
|
+
deviceId: parsed.deviceId,
|
|
9143
|
+
profile: parsed.profile,
|
|
9144
|
+
startMs: parsed.startMs,
|
|
9145
|
+
durMs: parsed.durMs,
|
|
9146
|
+
bytes: parsed.bytes,
|
|
9147
|
+
path,
|
|
9148
|
+
locationId
|
|
9149
|
+
});
|
|
8996
9150
|
}
|
|
8997
|
-
return
|
|
8998
|
-
|
|
8999
|
-
|
|
9000
|
-
|
|
9001
|
-
const set = new Set(a);
|
|
9002
|
-
for (const p of b) if (!set.has(p)) return false;
|
|
9003
|
-
return true;
|
|
9151
|
+
return {
|
|
9152
|
+
rows,
|
|
9153
|
+
unparseable
|
|
9154
|
+
};
|
|
9004
9155
|
}
|
|
9005
9156
|
function rowToValue(row) {
|
|
9006
9157
|
return {
|
|
@@ -9008,7 +9159,6 @@ function rowToValue(row) {
|
|
|
9008
9159
|
profile: row.profile,
|
|
9009
9160
|
locationId: row.locationId,
|
|
9010
9161
|
hourStartMs: row.hourStartMs,
|
|
9011
|
-
paths: encodePaths(row.paths),
|
|
9012
9162
|
bytes: row.bytes,
|
|
9013
9163
|
segments: row.segments,
|
|
9014
9164
|
minStartMs: row.minStartMs,
|
|
@@ -9021,44 +9171,21 @@ function rowToValue(row) {
|
|
|
9021
9171
|
function sealedOf(raw) {
|
|
9022
9172
|
return raw === 1 || raw === true;
|
|
9023
9173
|
}
|
|
9024
|
-
function recordToRow(key, data) {
|
|
9025
|
-
const deviceId = Number(data["deviceId"]);
|
|
9026
|
-
const profile = data["profile"];
|
|
9027
|
-
const locationId = data["locationId"];
|
|
9028
|
-
const hour = Number(data["hourStartMs"]);
|
|
9029
|
-
const decodedPaths = decodePaths(data["paths"]);
|
|
9030
|
-
if (decodedPaths === null) return null;
|
|
9031
|
-
const raw = decodedPaths;
|
|
9032
|
-
if (!Number.isFinite(deviceId) || deviceId <= 0 || typeof profile !== "string" || profile.length === 0 || typeof locationId !== "string" || locationId.length === 0 || !Number.isFinite(hour) || !Array.isArray(raw) || !raw.every((p) => typeof p === "string" && p.length > 0)) return null;
|
|
9033
|
-
return segmentHourRow({
|
|
9034
|
-
key,
|
|
9035
|
-
deviceId,
|
|
9036
|
-
profile,
|
|
9037
|
-
locationId,
|
|
9038
|
-
hourStartMs: hour,
|
|
9039
|
-
paths: raw,
|
|
9040
|
-
sealed: sealedOf(data["sealed"])
|
|
9041
|
-
});
|
|
9042
|
-
}
|
|
9043
|
-
/** The paths of a summary row: one shared empty array, never a per-row
|
|
9044
|
-
* allocation, and never mistakable for "this hour holds nothing". */
|
|
9045
|
-
var NO_PATHS = Object.freeze([]);
|
|
9046
9174
|
function finiteOr0(raw) {
|
|
9047
9175
|
const n = Number(raw);
|
|
9048
9176
|
return Number.isFinite(n) ? n : 0;
|
|
9049
9177
|
}
|
|
9050
9178
|
/**
|
|
9051
|
-
* One row
|
|
9179
|
+
* One persisted row.
|
|
9052
9180
|
*
|
|
9053
|
-
* The identity half is validated
|
|
9054
|
-
*
|
|
9055
|
-
*
|
|
9056
|
-
*
|
|
9057
|
-
*
|
|
9058
|
-
*
|
|
9059
|
-
* {@link SegmentHourLedger.repairAggregates} rewrites it.
|
|
9181
|
+
* The identity half is validated: a row that cannot say which camera, profile,
|
|
9182
|
+
* location and hour it is about is DISCARDED, because a row keyed on garbage
|
|
9183
|
+
* would refuse the write path for an hour that is perfectly fine. The aggregate
|
|
9184
|
+
* half is taken as the database wrote it — a row that predates the aggregate
|
|
9185
|
+
* columns reports zeros, which UNDER-reports its disk (prunes less, never more)
|
|
9186
|
+
* until a listing of that hour re-derives it.
|
|
9060
9187
|
*/
|
|
9061
|
-
function
|
|
9188
|
+
function recordToRow(key, data) {
|
|
9062
9189
|
const deviceId = Number(data["deviceId"]);
|
|
9063
9190
|
const profile = data["profile"];
|
|
9064
9191
|
const locationId = data["locationId"];
|
|
@@ -9070,8 +9197,6 @@ function summaryToRow(key, data) {
|
|
|
9070
9197
|
profile,
|
|
9071
9198
|
locationId,
|
|
9072
9199
|
hourStartMs: hour,
|
|
9073
|
-
paths: NO_PATHS,
|
|
9074
|
-
pathsResident: false,
|
|
9075
9200
|
sealed: sealedOf(data["sealed"]),
|
|
9076
9201
|
bytes: finiteOr0(data["bytes"]),
|
|
9077
9202
|
segments: finiteOr0(data["segments"]),
|
|
@@ -9093,11 +9218,12 @@ var SPEC = {
|
|
|
9093
9218
|
var SegmentHourLedger = class SegmentHourLedger {
|
|
9094
9219
|
ledger;
|
|
9095
9220
|
logger;
|
|
9096
|
-
/** Held for the
|
|
9221
|
+
/** Held for the ARCHIVE reads below, which are queries rather than mirror
|
|
9097
9222
|
* lookups. Every DECISION path (`recordSegment`, `dropSegments`, the prune
|
|
9098
9223
|
* refusals) still answers from the mirror alone — D49 is about gates, and an
|
|
9099
9224
|
* accounting number is not a gate. */
|
|
9100
9225
|
store;
|
|
9226
|
+
listHour;
|
|
9101
9227
|
isLocationUsable;
|
|
9102
9228
|
now;
|
|
9103
9229
|
loadChunk;
|
|
@@ -9111,6 +9237,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9111
9237
|
lanes = /* @__PURE__ */ new Map();
|
|
9112
9238
|
/** Per (device, profile): the hour its writer last appended to. */
|
|
9113
9239
|
writerHours = /* @__PURE__ */ new Map();
|
|
9240
|
+
/**
|
|
9241
|
+
* Segment paths THIS PROCESS has already accounted for, per hour key.
|
|
9242
|
+
*
|
|
9243
|
+
* The `paths` column used to make the write path idempotent for free: an
|
|
9244
|
+
* append checked `held.paths.includes(path)`. With the column gone the
|
|
9245
|
+
* aggregate cannot say whether it already counted a name, so the dedup lives
|
|
9246
|
+
* here — bounded to the hours this process actually writes (two per writer:
|
|
9247
|
+
* the current hour and the one it just left), dropped when the writer moves
|
|
9248
|
+
* on. A duplicate that survives a RESTART is not caught here and is not
|
|
9249
|
+
* silent either: the row is unsealed, and the first listing of the closed
|
|
9250
|
+
* hour re-derives the aggregate exactly ({@link adoptWalkedHour}).
|
|
9251
|
+
*/
|
|
9252
|
+
accounted = /* @__PURE__ */ new Map();
|
|
9114
9253
|
/** Hour keys a write was REFUSED for in this process — never sealed by it. */
|
|
9115
9254
|
refused = /* @__PURE__ */ new Set();
|
|
9116
9255
|
/**
|
|
@@ -9123,6 +9262,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9123
9262
|
constructor(deps) {
|
|
9124
9263
|
this.logger = deps.logger;
|
|
9125
9264
|
this.store = deps.store;
|
|
9265
|
+
this.listHour = deps.listHour ?? null;
|
|
9126
9266
|
this.isLocationUsable = deps.isLocationUsable ?? (() => true);
|
|
9127
9267
|
this.now = deps.now ?? (() => Date.now());
|
|
9128
9268
|
this.loadChunk = deps.loadPacing?.chunkSize ?? LOAD_CHUNK;
|
|
@@ -9141,22 +9281,17 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9141
9281
|
return this.ledger.declare();
|
|
9142
9282
|
}
|
|
9143
9283
|
/**
|
|
9144
|
-
* Boot load —
|
|
9284
|
+
* Boot load — every hour row, in one query.
|
|
9145
9285
|
*
|
|
9146
|
-
*
|
|
9147
|
-
*
|
|
9148
|
-
*
|
|
9286
|
+
* Since D523 the whole row is ten scalars, so there is no second tier to
|
|
9287
|
+
* materialise and no projection to choose: the mirror this returns is the
|
|
9288
|
+
* ledger, entire. It keeps `DurableLedger.load`'s two rules — a failed read
|
|
9289
|
+
* RETURNS WHAT IS ALREADY MIRRORED rather than clearing it, and a malformed
|
|
9290
|
+
* row is skipped rather than repaired.
|
|
9149
9291
|
*
|
|
9150
|
-
* It
|
|
9151
|
-
*
|
|
9152
|
-
*
|
|
9153
|
-
* and a malformed row is skipped rather than repaired.
|
|
9154
|
-
*
|
|
9155
|
-
* It deliberately does NOT replay the archive into {@link RecordingIndex}.
|
|
9156
|
-
* `hydrated` now means "the rows are resident", so a ledger-known hour is
|
|
9157
|
-
* merely *hydratable from T3*: the first read of it pays one point read
|
|
9158
|
-
* ({@link detailForWindow}), and an hour whose tail the ledger is missing is
|
|
9159
|
-
* no longer permanently invisible to playback.
|
|
9292
|
+
* It deliberately does NOT replay the archive into {@link RecordingIndex}: a
|
|
9293
|
+
* row says how MUCH an hour holds, never which files, so only a listing can
|
|
9294
|
+
* seed the index (D523).
|
|
9160
9295
|
*/
|
|
9161
9296
|
async load() {
|
|
9162
9297
|
const startedMs = this.now();
|
|
@@ -9164,7 +9299,6 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9164
9299
|
try {
|
|
9165
9300
|
records = await this.store.query.query({
|
|
9166
9301
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9167
|
-
columns: SEGMENT_HOUR_SUMMARY_COLUMNS,
|
|
9168
9302
|
filter: { limit: this.rowCap }
|
|
9169
9303
|
});
|
|
9170
9304
|
} catch (err) {
|
|
@@ -9181,7 +9315,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9181
9315
|
const end = Math.min(i + this.loadChunk, records.length);
|
|
9182
9316
|
for (let j = i; j < end; j += 1) {
|
|
9183
9317
|
const record = records[j];
|
|
9184
|
-
const row =
|
|
9318
|
+
const row = recordToRow(record.id, record.data);
|
|
9185
9319
|
if (row === null) {
|
|
9186
9320
|
skipped += 1;
|
|
9187
9321
|
continue;
|
|
@@ -9190,7 +9324,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9190
9324
|
}
|
|
9191
9325
|
if (end < records.length) await this.yieldBetween();
|
|
9192
9326
|
}
|
|
9193
|
-
if (skipped > 0) this.logger.warn("recorder: segment-hour
|
|
9327
|
+
if (skipped > 0) this.logger.warn("recorder: segment-hour rows skipped as malformed — they gate NOTHING", { meta: {
|
|
9194
9328
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9195
9329
|
skipped
|
|
9196
9330
|
} });
|
|
@@ -9198,11 +9332,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9198
9332
|
limit: this.rowCap,
|
|
9199
9333
|
loaded: records.length
|
|
9200
9334
|
} });
|
|
9201
|
-
const tail = await this.materializeLiveTail();
|
|
9202
9335
|
this.logger.info("recorder: segment-hour ledger loaded", { meta: {
|
|
9203
9336
|
hours: this.ledger.size,
|
|
9204
|
-
pathsLoaded: false,
|
|
9205
|
-
liveTailHours: tail,
|
|
9206
9337
|
skipped,
|
|
9207
9338
|
queryMs: queriedMs - startedMs,
|
|
9208
9339
|
ms: this.now() - startedMs,
|
|
@@ -9210,8 +9341,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9210
9341
|
} });
|
|
9211
9342
|
return this.ledger.snapshot();
|
|
9212
9343
|
}
|
|
9213
|
-
/** Every hour
|
|
9214
|
-
* `pathsResident` is false carries real aggregates and empty `paths`. */
|
|
9344
|
+
/** Every hour row held. Pure RAM. */
|
|
9215
9345
|
snapshot() {
|
|
9216
9346
|
return this.ledger.snapshot();
|
|
9217
9347
|
}
|
|
@@ -9221,10 +9351,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9221
9351
|
* The archive's answer to "what should placement plan for", and the reason it
|
|
9222
9352
|
* is here rather than a `census` loop or an index read:
|
|
9223
9353
|
*
|
|
9224
|
-
* - `RecordingIndex.accountingByProfile` is the wrong authority.
|
|
9225
|
-
*
|
|
9226
|
-
*
|
|
9227
|
-
*
|
|
9354
|
+
* - `RecordingIndex.accountingByProfile` is the wrong authority. It holds
|
|
9355
|
+
* the live tail plus whatever a reader hydrated, so a camera nobody has
|
|
9356
|
+
* viewed since boot reports NO profiles — and the placement pass then
|
|
9357
|
+
* never plans it (D295 §3a #9, D299).
|
|
9228
9358
|
* - `census({ deviceId, profiles: [p] })` is the right authority but the
|
|
9229
9359
|
* wrong shape for a 30 s tick: it is one SQL round trip per (camera,
|
|
9230
9360
|
* profile), so a 16-camera fleet pays 48 SERIAL trips over the UDS door
|
|
@@ -9232,10 +9362,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9232
9362
|
* 240003` pass (see `PLACEMENT_STORE_TIMEOUT_MS`), and it grows with the
|
|
9233
9363
|
* fleet.
|
|
9234
9364
|
*
|
|
9235
|
-
* So it is answered from
|
|
9236
|
-
* precisely so that O(writers × days) questions cost no I/O: ~10–15 MB and
|
|
9237
|
-
* ~47 k rows even at 30 TB, one fold, no query. A hydrated segment row is not
|
|
9238
|
-
* needed to know that an hour exists.
|
|
9365
|
+
* So it is answered from the mirror: one fold, no query, no I/O.
|
|
9239
9366
|
*
|
|
9240
9367
|
* An EMPTY result for a device is a real answer — "the archive knows no hour
|
|
9241
9368
|
* for this camera" — and it is also what a ledger whose load failed reports,
|
|
@@ -9252,136 +9379,111 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9252
9379
|
}
|
|
9253
9380
|
return out;
|
|
9254
9381
|
}
|
|
9255
|
-
/** Does the ledger hold a row for this hour at all? Pure RAM
|
|
9256
|
-
* storage-walk repair consults before it upserts a rebuilt hour. */
|
|
9382
|
+
/** Does the ledger hold a row for this hour at all? Pure RAM. */
|
|
9257
9383
|
knowsHour(deviceId, profile, locationId, hour) {
|
|
9258
9384
|
return this.ledger.has(segmentHourKey(deviceId, profile, locationId, hour));
|
|
9259
9385
|
}
|
|
9260
|
-
/**
|
|
9261
|
-
*
|
|
9262
|
-
|
|
9263
|
-
|
|
9264
|
-
* {@link detailForWindow} answers per (device, window) and drops the profile
|
|
9265
|
-
* on the way out, which is fine for a reader assembling a playlist and wrong
|
|
9266
|
-
* for a caller that must act on one hour ROW: the ghost-location reconcile
|
|
9267
|
-
* decides per `(deviceId, profile, locationId, hour)` and hands the resulting
|
|
9268
|
-
* paths straight back to {@link dropSegments} / {@link repointSegments},
|
|
9269
|
-
* whose bucket key is that same tuple.
|
|
9270
|
-
*
|
|
9271
|
-
* `null` is "known, and its detail could not be had" — never an empty array.
|
|
9272
|
-
* The one caller FORGETS rows on this answer, and a failed point read that
|
|
9273
|
-
* came back as `[]` would read as "this hour holds nothing", which is the
|
|
9274
|
-
* D148 prune-on-no-evidence this ledger refuses to make.
|
|
9275
|
-
*/
|
|
9276
|
-
async hourPaths(deviceId, profile, locationId, hour) {
|
|
9277
|
-
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9278
|
-
if (!this.ledger.has(key)) return null;
|
|
9279
|
-
const held = await this.resident(key, deviceId);
|
|
9280
|
-
return held === null ? null : held.paths;
|
|
9386
|
+
/** One held hour row, or `undefined`. Pure RAM — since D523 a row carries
|
|
9387
|
+
* everything it has. */
|
|
9388
|
+
hourRow(deviceId, profile, locationId, hour) {
|
|
9389
|
+
return this.ledger.get(segmentHourKey(deviceId, profile, locationId, hour));
|
|
9281
9390
|
}
|
|
9282
9391
|
/**
|
|
9283
|
-
*
|
|
9284
|
-
* cold-hour detail read that replaces the boot replay (D287).
|
|
9392
|
+
* A LISTING of a CLOSED hour is adopted as that hour's account, and SEALS it.
|
|
9285
9393
|
*
|
|
9286
|
-
*
|
|
9287
|
-
*
|
|
9288
|
-
* the
|
|
9289
|
-
*
|
|
9290
|
-
*
|
|
9394
|
+
* The directory is the authority for which segments exist (D523), so a
|
|
9395
|
+
* listing of an hour that is OVER is the one complete answer there is — it
|
|
9396
|
+
* replaces the row's aggregate rather than merging with it, which is what
|
|
9397
|
+
* makes an hour that was over-counted (a duplicate finalize across a restart)
|
|
9398
|
+
* or under-counted (a writer torn down mid-hour) self-correcting. The caller
|
|
9399
|
+
* guarantees the hour is CLOSED: a live hour's directory races its writer, so
|
|
9400
|
+
* a listing of it is a lower bound.
|
|
9291
9401
|
*
|
|
9292
|
-
*
|
|
9293
|
-
*
|
|
9294
|
-
*
|
|
9295
|
-
*
|
|
9296
|
-
|
|
9297
|
-
async detailForWindow(deviceId, fromMs, toMs) {
|
|
9298
|
-
const first = hourStartMs(fromMs);
|
|
9299
|
-
const out = [];
|
|
9300
|
-
const wanted = [];
|
|
9301
|
-
for (const row of this.ledger.snapshot()) {
|
|
9302
|
-
if (row.deviceId !== deviceId) continue;
|
|
9303
|
-
if (row.hourStartMs < first || row.hourStartMs >= toMs) continue;
|
|
9304
|
-
if (row.pathsResident) {
|
|
9305
|
-
out.push(detailOf(row));
|
|
9306
|
-
continue;
|
|
9307
|
-
}
|
|
9308
|
-
wanted.push(row.key);
|
|
9309
|
-
}
|
|
9310
|
-
for (const row of await this.materialize(wanted, deviceId)) out.push(detailOf(row));
|
|
9311
|
-
return out;
|
|
9312
|
-
}
|
|
9313
|
-
/**
|
|
9314
|
-
* Union a CLOSED hour the storage walk read into its row, and SEAL it.
|
|
9315
|
-
*
|
|
9316
|
-
* The reader repairs the durable index it just reconstructed, so the next
|
|
9317
|
-
* cold read of this hour is a point read instead of another `readdir`
|
|
9318
|
-
* (`docs/design/2026-08-29-reference-nvr-recording-model.md` §4.2), and it
|
|
9319
|
-
* seals the row because a directory read of a closed hour is the one
|
|
9320
|
-
* complete answer there is. The caller guarantees the hour is CLOSED — a
|
|
9321
|
-
* live hour's directory races its writer, so a walk of it is a lower bound.
|
|
9322
|
-
*
|
|
9323
|
-
* A row the ledger already holds is UNIONED, never replaced: the walk can
|
|
9324
|
-
* only add (D148) — a path the row names and the directory lacks stays,
|
|
9325
|
-
* because "the reader did not see it" is not evidence it is gone. This is
|
|
9326
|
-
* the repair D445 described and did not perform: its predecessor returned
|
|
9327
|
-
* on any known key, so a row known INCOMPLETELY — the only kind that needs
|
|
9328
|
-
* repairing — was exactly the row it refused to touch, and the export's
|
|
9329
|
-
* `forgottenSegments=9758` was counted again the next morning.
|
|
9330
|
-
*
|
|
9331
|
-
* `null` when the row is known and its detail could not be read: nothing
|
|
9332
|
-
* is written from a row whose paths are not resident.
|
|
9402
|
+
* A listing that would erase more than {@link HOUR_LISTING_MAX_SHRINK_SHARE}
|
|
9403
|
+
* of what the row accounts for is REFUSED and named. That is the one thing a
|
|
9404
|
+
* listing cannot be trusted about: a root that has gone away and a bucket a
|
|
9405
|
+
* dead round half-deleted both answer short, and neither is distinguishable
|
|
9406
|
+
* from an hour that is simply smaller than the row thought.
|
|
9333
9407
|
*
|
|
9334
9408
|
* @seal-authority closed-hour-listing — a directory read of an hour that is
|
|
9335
9409
|
* OVER is the one complete answer there is; the caller guarantees the hour is
|
|
9336
|
-
* closed (D451, D521).
|
|
9410
|
+
* closed (D451, D521, D523).
|
|
9337
9411
|
*/
|
|
9338
9412
|
async adoptWalkedHour(deviceId, profile, locationId, hour, paths) {
|
|
9339
9413
|
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9414
|
+
const identity = {
|
|
9415
|
+
key,
|
|
9416
|
+
deviceId,
|
|
9417
|
+
profile,
|
|
9418
|
+
locationId,
|
|
9419
|
+
hourStartMs: hour
|
|
9420
|
+
};
|
|
9421
|
+
const walked = hourAggregate(paths);
|
|
9340
9422
|
return this.serialized(key, async () => {
|
|
9341
|
-
|
|
9342
|
-
|
|
9423
|
+
const held = this.ledger.get(key);
|
|
9424
|
+
if (held === void 0) {
|
|
9425
|
+
if (walked.segments === 0) return {
|
|
9343
9426
|
created: false,
|
|
9344
9427
|
added: 0,
|
|
9345
9428
|
sealedNow: false
|
|
9346
9429
|
};
|
|
9347
|
-
|
|
9348
|
-
|
|
9349
|
-
deviceId,
|
|
9350
|
-
profile,
|
|
9351
|
-
locationId,
|
|
9352
|
-
hourStartMs: hour,
|
|
9353
|
-
paths: uniquePaths(paths),
|
|
9430
|
+
await this.ledger.put(hourRowOf(identity, {
|
|
9431
|
+
...walked,
|
|
9354
9432
|
sealed: true
|
|
9355
|
-
});
|
|
9356
|
-
|
|
9357
|
-
this.logger.info("recorder: adopted a closed hour the disk walk rebuilt and the ledger never held", {
|
|
9433
|
+
}));
|
|
9434
|
+
this.logger.info("recorder: adopted a closed hour the disk listing rebuilt and the ledger never held", {
|
|
9358
9435
|
tags: { deviceId },
|
|
9359
9436
|
meta: {
|
|
9360
9437
|
profile,
|
|
9361
9438
|
locationId,
|
|
9362
9439
|
hourStartMs: hour,
|
|
9363
|
-
segments:
|
|
9364
|
-
bytes:
|
|
9440
|
+
segments: walked.segments,
|
|
9441
|
+
bytes: walked.bytes
|
|
9365
9442
|
}
|
|
9366
9443
|
});
|
|
9367
9444
|
return {
|
|
9368
9445
|
created: true,
|
|
9369
|
-
added:
|
|
9446
|
+
added: walked.segments,
|
|
9370
9447
|
sealedNow: true
|
|
9371
9448
|
};
|
|
9372
9449
|
}
|
|
9373
|
-
const
|
|
9374
|
-
if (held
|
|
9375
|
-
|
|
9376
|
-
|
|
9377
|
-
|
|
9450
|
+
const shrink = held.segments - walked.segments;
|
|
9451
|
+
if (shrink > 0 && shrink > held.segments * .5) {
|
|
9452
|
+
this.logger.warn("recorder: hour listing REFUSED — it accounts for less than half of what this hour row holds, which is what an unmounted root looks like; the row stands and stays unsealed", {
|
|
9453
|
+
tags: { deviceId },
|
|
9454
|
+
meta: {
|
|
9455
|
+
profile,
|
|
9456
|
+
locationId,
|
|
9457
|
+
hourStartMs: hour,
|
|
9458
|
+
held: held.segments,
|
|
9459
|
+
listed: walked.segments,
|
|
9460
|
+
maxShare: HOUR_LISTING_MAX_SHRINK_SHARE
|
|
9461
|
+
}
|
|
9462
|
+
});
|
|
9463
|
+
return {
|
|
9464
|
+
created: false,
|
|
9465
|
+
added: 0,
|
|
9466
|
+
sealedNow: false
|
|
9467
|
+
};
|
|
9468
|
+
}
|
|
9469
|
+
const added = Math.max(0, walked.segments - held.segments);
|
|
9470
|
+
if (added === 0 && shrink === 0 && held.sealed) return {
|
|
9378
9471
|
created: false,
|
|
9379
9472
|
added: 0,
|
|
9380
9473
|
sealedNow: false
|
|
9381
9474
|
};
|
|
9382
|
-
|
|
9383
|
-
|
|
9384
|
-
|
|
9475
|
+
if (shrink > 0) this.logger.info("recorder: hour listing accounts for FEWER segments than the row did — the directory decides", {
|
|
9476
|
+
tags: { deviceId },
|
|
9477
|
+
meta: {
|
|
9478
|
+
profile,
|
|
9479
|
+
locationId,
|
|
9480
|
+
hourStartMs: hour,
|
|
9481
|
+
held: held.segments,
|
|
9482
|
+
listed: walked.segments
|
|
9483
|
+
}
|
|
9484
|
+
});
|
|
9485
|
+
await this.ledger.put(hourRowOf(identity, {
|
|
9486
|
+
...walked,
|
|
9385
9487
|
sealed: true
|
|
9386
9488
|
}));
|
|
9387
9489
|
return {
|
|
@@ -9411,9 +9513,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9411
9513
|
}
|
|
9412
9514
|
/**
|
|
9413
9515
|
* A finalize this ledger was NEVER handed — the fire-and-forget lane refused
|
|
9414
|
-
* it. The hour's row is short by that
|
|
9415
|
-
* party that knows, so the hour must not be sealed by it; a
|
|
9416
|
-
*
|
|
9516
|
+
* it. The hour's row is short by that segment and this process is the only
|
|
9517
|
+
* party that knows, so the hour must not be sealed by it; a listing corrects
|
|
9518
|
+
* it instead, which is exactly what an unsealed row is for.
|
|
9417
9519
|
*/
|
|
9418
9520
|
markRefused(row) {
|
|
9419
9521
|
this.refused.add(hourOf(row).key);
|
|
@@ -9441,7 +9543,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9441
9543
|
marked += 1;
|
|
9442
9544
|
}
|
|
9443
9545
|
if (marked === 0) return 0;
|
|
9444
|
-
this.logger.warn("recorder: segments this writer produced never reached the hour ledger — these hours stay UNSEALED and a
|
|
9546
|
+
this.logger.warn("recorder: segments this writer produced never reached the hour ledger — these hours stay UNSEALED and a listing corrects them", {
|
|
9445
9547
|
tags: { deviceId: span.deviceId },
|
|
9446
9548
|
meta: {
|
|
9447
9549
|
profile: span.profile,
|
|
@@ -9474,10 +9576,11 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9474
9576
|
graceMs: WRITER_HOUR_ENTRY_GRACE_MS
|
|
9475
9577
|
}
|
|
9476
9578
|
});
|
|
9579
|
+
this.accounted.delete(target.key);
|
|
9477
9580
|
return;
|
|
9478
9581
|
}
|
|
9479
9582
|
if (this.refused.has(target.key) || this.refusedWriterHours.has(writerHourKey(target.deviceId, target.profile, target.hourStartMs))) {
|
|
9480
|
-
this.logger.warn("recorder: hour left unsealed — a ledger write for it was refused, so its row is short and a
|
|
9583
|
+
this.logger.warn("recorder: hour left unsealed — a ledger write for it was refused, so its row is short and a listing must correct it", {
|
|
9481
9584
|
tags: { deviceId: target.deviceId },
|
|
9482
9585
|
meta: {
|
|
9483
9586
|
profile: target.profile,
|
|
@@ -9485,10 +9588,12 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9485
9588
|
hourStartMs: target.hourStartMs
|
|
9486
9589
|
}
|
|
9487
9590
|
});
|
|
9591
|
+
this.accounted.delete(target.key);
|
|
9488
9592
|
return;
|
|
9489
9593
|
}
|
|
9490
|
-
const held =
|
|
9491
|
-
|
|
9594
|
+
const held = this.ledger.get(target.key);
|
|
9595
|
+
this.accounted.delete(target.key);
|
|
9596
|
+
if (held === void 0 || held.sealed) return;
|
|
9492
9597
|
await this.ledger.put({
|
|
9493
9598
|
...held,
|
|
9494
9599
|
sealed: true
|
|
@@ -9516,11 +9621,25 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9516
9621
|
/**
|
|
9517
9622
|
* Run `fn` after every mutation already queued on `key`.
|
|
9518
9623
|
*
|
|
9519
|
-
*
|
|
9520
|
-
*
|
|
9521
|
-
*
|
|
9522
|
-
*
|
|
9523
|
-
*
|
|
9624
|
+
* It was the lost-update guard: a row was read from the STORE, folded into
|
|
9625
|
+
* and written back with awaits in between, so two such runs on one key
|
|
9626
|
+
* interleaved and the second write carried a path set that never saw the
|
|
9627
|
+
* first — the sparse rows D451 found. Since D523 there is no store read on
|
|
9628
|
+
* any of these paths (the mirror holds the whole row) and `DurableLedger.put`
|
|
9629
|
+
* sets the mirror synchronously before it awaits the store, so every
|
|
9630
|
+
* get→fold→put here is synchronous up to the mirror and the interleaving
|
|
9631
|
+
* window is gone.
|
|
9632
|
+
*
|
|
9633
|
+
* What the lane still buys is the ORDER of the durable writes per hour key —
|
|
9634
|
+
* a rollover seal fired without an await must not land over an append that
|
|
9635
|
+
* was queued before it — and a handle for {@link flush} to wait on at
|
|
9636
|
+
* shutdown. A rejected step must not poison the lane: the next step still
|
|
9637
|
+
* runs.
|
|
9638
|
+
*
|
|
9639
|
+
* Note for anyone tempted to delete it: `segment-hour-seal.spec.ts` does NOT
|
|
9640
|
+
* catch its removal any more, and says so. Pinning durable write ORDER needs
|
|
9641
|
+
* a store whose latency a test can control, which `fakeSettingsStore` has no
|
|
9642
|
+
* way to express.
|
|
9524
9643
|
*/
|
|
9525
9644
|
serialized(key, fn) {
|
|
9526
9645
|
const run = (this.lanes.get(key) ?? Promise.resolve()).then(fn, fn);
|
|
@@ -9532,116 +9651,59 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9532
9651
|
return run;
|
|
9533
9652
|
}
|
|
9534
9653
|
/**
|
|
9535
|
-
*
|
|
9654
|
+
* List the hour directories of `rows`, paced, and expand them into segment
|
|
9655
|
+
* rows.
|
|
9536
9656
|
*
|
|
9537
|
-
*
|
|
9538
|
-
*
|
|
9539
|
-
*
|
|
9540
|
-
*
|
|
9541
|
-
* walk, or refusing a write) is strictly safer than a guess.
|
|
9542
|
-
*
|
|
9543
|
-
* A row the store returns but this cannot parse is EVICTED from the mirror
|
|
9544
|
-
* rather than left as a summary that can never become resident: it re-seeds
|
|
9545
|
-
* cold on the next write, which is `DurableLedger`'s rule 4 and the only
|
|
9546
|
-
* escape from an hour permanently frozen against its own writer.
|
|
9657
|
+
* The one place the ledger touches the array. An hour whose listing FAILED
|
|
9658
|
+
* takes no part in the caller's pass and is SAID — a branch that drops work
|
|
9659
|
+
* silently reads as "there was nothing there", which for a drain or a prune
|
|
9660
|
+
* is the difference between "moved everything" and "moved what it could see".
|
|
9547
9661
|
*/
|
|
9548
|
-
async
|
|
9662
|
+
async listHours(rows, reason) {
|
|
9549
9663
|
const out = [];
|
|
9550
|
-
if (
|
|
9551
|
-
const
|
|
9552
|
-
|
|
9553
|
-
|
|
9554
|
-
|
|
9555
|
-
|
|
9556
|
-
|
|
9557
|
-
|
|
9558
|
-
|
|
9559
|
-
|
|
9560
|
-
|
|
9561
|
-
|
|
9562
|
-
|
|
9563
|
-
|
|
9564
|
-
|
|
9565
|
-
...tags,
|
|
9664
|
+
if (rows.length === 0) return out;
|
|
9665
|
+
const lister = this.listHour;
|
|
9666
|
+
if (lister === null) {
|
|
9667
|
+
this.logger.warn("recorder: no hour lister on this ledger — nothing that needs segment NAMES can run here", { meta: {
|
|
9668
|
+
hours: rows.length,
|
|
9669
|
+
reason
|
|
9670
|
+
} });
|
|
9671
|
+
return out;
|
|
9672
|
+
}
|
|
9673
|
+
let listed = 0;
|
|
9674
|
+
for (const row of rows) {
|
|
9675
|
+
const paths = await lister(row.locationId, row.deviceId, row.profile, row.hourStartMs);
|
|
9676
|
+
if (paths === null) {
|
|
9677
|
+
this.logger.warn("recorder: hour directory could not be listed — this hour takes NO part in this pass", {
|
|
9678
|
+
tags: { deviceId: row.deviceId },
|
|
9566
9679
|
meta: {
|
|
9567
|
-
|
|
9568
|
-
|
|
9569
|
-
|
|
9570
|
-
|
|
9680
|
+
profile: row.profile,
|
|
9681
|
+
locationId: row.locationId,
|
|
9682
|
+
hourStartMs: row.hourStartMs,
|
|
9683
|
+
accounted: row.segments,
|
|
9684
|
+
reason
|
|
9571
9685
|
}
|
|
9572
9686
|
});
|
|
9573
|
-
|
|
9687
|
+
continue;
|
|
9574
9688
|
}
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
|
|
9578
|
-
|
|
9579
|
-
|
|
9580
|
-
|
|
9581
|
-
|
|
9582
|
-
|
|
9583
|
-
|
|
9689
|
+
const { rows: segments, unparseable } = segmentRowsOf(row.locationId, paths);
|
|
9690
|
+
if (unparseable > 0) this.logger.warn("recorder: hour directory holds names that are not segments", {
|
|
9691
|
+
tags: { deviceId: row.deviceId },
|
|
9692
|
+
meta: {
|
|
9693
|
+
profile: row.profile,
|
|
9694
|
+
locationId: row.locationId,
|
|
9695
|
+
hourStartMs: row.hourStartMs,
|
|
9696
|
+
unparseable,
|
|
9697
|
+
reason
|
|
9584
9698
|
}
|
|
9585
|
-
|
|
9586
|
-
|
|
9587
|
-
|
|
9588
|
-
if (
|
|
9699
|
+
});
|
|
9700
|
+
out.push(...segments);
|
|
9701
|
+
listed += 1;
|
|
9702
|
+
if (listed % 8 === 0) await this.yieldBetween();
|
|
9589
9703
|
}
|
|
9590
9704
|
return out;
|
|
9591
9705
|
}
|
|
9592
9706
|
/**
|
|
9593
|
-
* Materialise every hour these devices hold, and report how many stayed
|
|
9594
|
-
* summaries.
|
|
9595
|
-
*
|
|
9596
|
-
* The reconcile is the one path that DELETES durable rows, so it may not run
|
|
9597
|
-
* against a mirror that only half knows what it holds. This is a whole-device
|
|
9598
|
-
* read and it is deliberately confined here: `reconcileFromIndex` follows an
|
|
9599
|
-
* operator-armed rescan of that same device — a recursive `readdir` over
|
|
9600
|
-
* >1M files — so a paged point read alongside it is noise, and it is the one
|
|
9601
|
-
* caller whose correctness depends on completeness.
|
|
9602
|
-
*/
|
|
9603
|
-
async materializeDevices(deviceIds) {
|
|
9604
|
-
const keys = [];
|
|
9605
|
-
for (const row of this.ledger.snapshot()) if (!row.pathsResident && deviceIds.has(row.deviceId)) keys.push(row.key);
|
|
9606
|
-
if (keys.length === 0) return;
|
|
9607
|
-
await this.materialize(keys);
|
|
9608
|
-
const stillSummary = /* @__PURE__ */ new Map();
|
|
9609
|
-
for (const row of this.ledger.snapshot()) {
|
|
9610
|
-
if (row.pathsResident || !deviceIds.has(row.deviceId)) continue;
|
|
9611
|
-
stillSummary.set(row.deviceId, (stillSummary.get(row.deviceId) ?? 0) + 1);
|
|
9612
|
-
}
|
|
9613
|
-
for (const [deviceId, unreadable] of stillSummary) this.logger.warn("recorder: hours without readable detail take NO part in this reconcile — they are neither pruned nor unioned", {
|
|
9614
|
-
tags: { deviceId },
|
|
9615
|
-
meta: {
|
|
9616
|
-
asked: keys.length,
|
|
9617
|
-
unreadable
|
|
9618
|
-
}
|
|
9619
|
-
});
|
|
9620
|
-
}
|
|
9621
|
-
/** The hour a write path may touch: current + previous. Materialised at boot
|
|
9622
|
-
* so {@link recordSegment}'s dedup gate answers from RAM (D49). */
|
|
9623
|
-
async materializeLiveTail() {
|
|
9624
|
-
const oldest = hourStartMs(this.now()) - (LIVE_TAIL_HOURS - 1) * HOUR_MS$4;
|
|
9625
|
-
const keys = [];
|
|
9626
|
-
for (const row of this.ledger.snapshot()) if (row.hourStartMs >= oldest && !row.pathsResident) keys.push(row.key);
|
|
9627
|
-
return (await this.materialize(keys)).length;
|
|
9628
|
-
}
|
|
9629
|
-
/**
|
|
9630
|
-
* Promote one HELD row to paths-resident, or answer `null`.
|
|
9631
|
-
*
|
|
9632
|
-
* `null` means "this hour is known and its detail could not be had" — the
|
|
9633
|
-
* only honest answer to a failed point read, and every caller turns it into
|
|
9634
|
-
* "do nothing". It is deliberately distinct from `this.ledger.get(key) ===
|
|
9635
|
-
* undefined`, which means "no such hour" and is what licenses a fresh row.
|
|
9636
|
-
*/
|
|
9637
|
-
async resident(key, deviceId) {
|
|
9638
|
-
const held = this.ledger.get(key);
|
|
9639
|
-
if (held !== void 0 && held.pathsResident) return held;
|
|
9640
|
-
await this.materialize([key], deviceId);
|
|
9641
|
-
const after = this.ledger.get(key);
|
|
9642
|
-
return after !== void 0 && after.pathsResident ? after : null;
|
|
9643
|
-
}
|
|
9644
|
-
/**
|
|
9645
9707
|
* Bytes / count / oldest / newest across a SET of storage locations, as ONE
|
|
9646
9708
|
* `SUM`/`COUNT`/`MIN`/`MAX` over the hour rows.
|
|
9647
9709
|
*
|
|
@@ -9682,9 +9744,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9682
9744
|
* ## Why two phases
|
|
9683
9745
|
*
|
|
9684
9746
|
* Phase one reads only the small columns and walks oldest-first until the
|
|
9685
|
-
* accumulated bytes cover the target — normally one page.
|
|
9686
|
-
*
|
|
9687
|
-
*
|
|
9747
|
+
* accumulated bytes cover the target — normally one page. Phase two LISTS the
|
|
9748
|
+
* hour directories phase one chose, which is the only thing that can name the
|
|
9749
|
+
* files eviction is about to delete (D523).
|
|
9688
9750
|
*
|
|
9689
9751
|
* `exhausted` means the stream ran out before the target was met — the
|
|
9690
9752
|
* caller's stop condition, and the one thing a bounded read must still be
|
|
@@ -9718,11 +9780,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9718
9780
|
/**
|
|
9719
9781
|
* Every camera the ledger holds an hour for on these locations.
|
|
9720
9782
|
*
|
|
9721
|
-
* Answered from
|
|
9722
|
-
*
|
|
9723
|
-
*
|
|
9724
|
-
*
|
|
9725
|
-
* would have to be turned into a verdict.
|
|
9783
|
+
* Answered from the mirror, exactly like {@link profilesWithFootage}, and for
|
|
9784
|
+
* the same reason: this is an O(writers × days) question and it must cost no
|
|
9785
|
+
* I/O. It runs on the disk-pressure path, where an I/O that fails would have
|
|
9786
|
+
* to be turned into a verdict.
|
|
9726
9787
|
*
|
|
9727
9788
|
* An EMPTY answer is "the mirror knows no hour here", which is also what a
|
|
9728
9789
|
* ledger whose load failed reports — so the caller must treat it as "decide
|
|
@@ -9737,7 +9798,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9737
9798
|
* Bytes / segments / oldest / newest per CAMERA on these locations — the
|
|
9738
9799
|
* per-camera half of the storage page, answered by the ARCHIVE (D418).
|
|
9739
9800
|
*
|
|
9740
|
-
* ## Why the
|
|
9801
|
+
* ## Why the mirror fold and not `census({ deviceId })` per camera
|
|
9741
9802
|
*
|
|
9742
9803
|
* `census` is the right authority for a SCOPE, and it cannot express this
|
|
9743
9804
|
* scope: {@link SegmentHourLedger.predicate} takes ONE `locationId`, so
|
|
@@ -9747,14 +9808,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9747
9808
|
* accounts for. Per camera per volume that is `cameras × volumes` SQL
|
|
9748
9809
|
* aggregates for one operator click, and it grows with the fleet.
|
|
9749
9810
|
*
|
|
9750
|
-
* The
|
|
9751
|
-
*
|
|
9752
|
-
*
|
|
9753
|
-
*
|
|
9754
|
-
* over the hour rows the mirror holds — O(writers × profiles × hours), ~19 k
|
|
9755
|
-
* on the live hub (see {@link REPAIR_MAX_ROWS}) against 98 445 segments,
|
|
9756
|
-
* i.e. it does NOT scale with the segment count — and it is capped for good
|
|
9757
|
-
* by {@link SEGMENT_HOUR_SUMMARY_LIMIT}, the mirror's own ceiling.
|
|
9811
|
+
* The mirror answers all of them in ONE pass with no I/O, exactly as
|
|
9812
|
+
* {@link devicesWithFootage} and {@link profilesWithFootage} already do, and
|
|
9813
|
+
* it is the same rows `accountingForLocations` sums in SQL — the mirror is
|
|
9814
|
+
* write-behind, so it is never BEHIND the store.
|
|
9758
9815
|
*
|
|
9759
9816
|
* ## What an empty answer means
|
|
9760
9817
|
*
|
|
@@ -9828,44 +9885,28 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9828
9885
|
rows: [],
|
|
9829
9886
|
exhausted: true
|
|
9830
9887
|
};
|
|
9831
|
-
const full = await this.store.query.query({
|
|
9832
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9833
|
-
filter: {
|
|
9834
|
-
whereIn: { key: chosen },
|
|
9835
|
-
limit: chosen.length
|
|
9836
|
-
}
|
|
9837
|
-
});
|
|
9838
|
-
const rows = [];
|
|
9839
|
-
for (const record of full) {
|
|
9840
|
-
const hour = recordToRow(record.id, record.data);
|
|
9841
|
-
if (hour === null) continue;
|
|
9842
|
-
this.ledger.stage(hour);
|
|
9843
|
-
for (const path of hour.paths) {
|
|
9844
|
-
const parsed = parseSegmentPath(path);
|
|
9845
|
-
if (parsed === null) continue;
|
|
9846
|
-
rows.push({
|
|
9847
|
-
deviceId: parsed.deviceId,
|
|
9848
|
-
profile: parsed.profile,
|
|
9849
|
-
startMs: parsed.startMs,
|
|
9850
|
-
durMs: parsed.durMs,
|
|
9851
|
-
bytes: parsed.bytes,
|
|
9852
|
-
path,
|
|
9853
|
-
locationId: hour.locationId
|
|
9854
|
-
});
|
|
9855
|
-
}
|
|
9856
|
-
}
|
|
9857
9888
|
return {
|
|
9858
|
-
rows:
|
|
9889
|
+
rows: (await this.listHours(this.rowsOfKeys(chosen), "oldest-segments")).toSorted((a, b) => a.startMs - b.startMs),
|
|
9859
9890
|
exhausted
|
|
9860
9891
|
};
|
|
9861
9892
|
}
|
|
9893
|
+
/** Mirror rows for keys a probe named. A key the mirror does not hold is
|
|
9894
|
+
* skipped: the probe and the mirror describe the same table, and an hour the
|
|
9895
|
+
* mirror never loaded is one this process may not act on. */
|
|
9896
|
+
rowsOfKeys(keys) {
|
|
9897
|
+
const out = [];
|
|
9898
|
+
for (const key of keys) {
|
|
9899
|
+
const row = this.ledger.get(key);
|
|
9900
|
+
if (row !== void 0) out.push(row);
|
|
9901
|
+
}
|
|
9902
|
+
return out;
|
|
9903
|
+
}
|
|
9862
9904
|
/**
|
|
9863
9905
|
* The predicate half of an {@link ArchiveScope}, as one filter.
|
|
9864
9906
|
*
|
|
9865
9907
|
* REFUSES an empty scope rather than compiling to "the whole collection".
|
|
9866
|
-
* There is no read here that may be unscoped:
|
|
9867
|
-
*
|
|
9868
|
-
* how that query comes back under a new name.
|
|
9908
|
+
* There is no read here that may be unscoped: an unscoped stream lists every
|
|
9909
|
+
* hour directory on the hub.
|
|
9869
9910
|
*/
|
|
9870
9911
|
static predicate(scope) {
|
|
9871
9912
|
const where = {};
|
|
@@ -9878,6 +9919,15 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9878
9919
|
...profiles.length > 0 ? { whereIn: { profile: [...profiles] } } : {}
|
|
9879
9920
|
};
|
|
9880
9921
|
}
|
|
9922
|
+
/** Does this row fall inside `scope`? The mirror-side twin of
|
|
9923
|
+
* {@link SegmentHourLedger.predicate}, for the passes that fold the mirror
|
|
9924
|
+
* instead of querying. */
|
|
9925
|
+
static inScope(row, scope) {
|
|
9926
|
+
if (scope.locationId !== void 0 && row.locationId !== scope.locationId) return false;
|
|
9927
|
+
if (scope.deviceId !== void 0 && row.deviceId !== scope.deviceId) return false;
|
|
9928
|
+
const profiles = scope.profiles ?? [];
|
|
9929
|
+
return profiles.length === 0 || profiles.includes(row.profile);
|
|
9930
|
+
}
|
|
9881
9931
|
/**
|
|
9882
9932
|
* What the ARCHIVE holds in this scope — segments and bytes, as ONE
|
|
9883
9933
|
* `SUM` over the hour rows.
|
|
@@ -9921,6 +9971,15 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9921
9971
|
/**
|
|
9922
9972
|
* Every segment in `scope`, oldest-first, ONE PAGE at a time.
|
|
9923
9973
|
*
|
|
9974
|
+
* ## Two phases, and which authority answers which
|
|
9975
|
+
*
|
|
9976
|
+
* Phase one asks the TABLE which hours the scope holds, oldest-first — the
|
|
9977
|
+
* ordered aggregate no directory read can answer. Phase two LISTS those hour
|
|
9978
|
+
* directories, which is the only authority for the file names the caller is
|
|
9979
|
+
* about to move or delete (D523). An hour whose listing fails is skipped and
|
|
9980
|
+
* said; the caller's own N-versus-M report is what makes the short walk
|
|
9981
|
+
* visible.
|
|
9982
|
+
*
|
|
9924
9983
|
* ## Why a keyset cursor and not `offset`
|
|
9925
9984
|
*
|
|
9926
9985
|
* Both callers MUTATE the scope while they walk it: the drain re-points each
|
|
@@ -9937,10 +9996,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9937
9996
|
*
|
|
9938
9997
|
* ## What a failed read does
|
|
9939
9998
|
*
|
|
9940
|
-
* It ENDS the stream (D49: a fallible read destroys nothing)
|
|
9941
|
-
*
|
|
9942
|
-
*
|
|
9943
|
-
* from a scope that was genuinely empty.
|
|
9999
|
+
* It ENDS the stream (D49: a fallible read destroys nothing). A stream that
|
|
10000
|
+
* swallowed the error and returned would be indistinguishable from a scope
|
|
10001
|
+
* that was genuinely empty.
|
|
9944
10002
|
*/
|
|
9945
10003
|
async *streamSegments(scope) {
|
|
9946
10004
|
const predicate = SegmentHourLedger.predicate(scope);
|
|
@@ -9993,8 +10051,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9993
10051
|
if (nextCursor !== cursor) emittedAtCursor = /* @__PURE__ */ new Set();
|
|
9994
10052
|
for (const record of fresh) if (hourOfRecord(record) === nextCursor) emittedAtCursor.add(record.id);
|
|
9995
10053
|
cursor = nextCursor;
|
|
9996
|
-
const rows = await this.
|
|
9997
|
-
if (rows.length > 0) yield rows;
|
|
10054
|
+
const rows = await this.listHours(this.rowsOfKeys(fresh.map((record) => record.id)), "archive-stream");
|
|
10055
|
+
if (rows.length > 0) yield rows.toSorted((a, b) => a.startMs - b.startMs);
|
|
9998
10056
|
if (probe.length < 128 && fresh.length === probe.length) return;
|
|
9999
10057
|
}
|
|
10000
10058
|
this.logger.warn("recorder: archive stream hit its page backstop — the rest of this scope was NOT walked", {
|
|
@@ -10008,110 +10066,50 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10008
10066
|
});
|
|
10009
10067
|
}
|
|
10010
10068
|
/**
|
|
10011
|
-
*
|
|
10012
|
-
*
|
|
10069
|
+
* Forget every hour row in `scope` — the durable half of an operator's
|
|
10070
|
+
* "discard this location's footage".
|
|
10071
|
+
*
|
|
10072
|
+
* It exists because the files are already GONE by the time it runs: the
|
|
10073
|
+
* discard wipes the class subtrees first, so every hour directory in the
|
|
10074
|
+
* scope answers ENOENT and nothing that needs a listing can name a single
|
|
10075
|
+
* segment. The rows are forgotten by their own identity instead, and the
|
|
10076
|
+
* count returned is the SEGMENTS they accounted for, which is the number the
|
|
10077
|
+
* operator is shown.
|
|
10013
10078
|
*
|
|
10014
|
-
*
|
|
10015
|
-
*
|
|
10016
|
-
*
|
|
10017
|
-
* and which `forget`s the row as it empties — so the staging is
|
|
10018
|
-
* self-cleaning and the alternative is two reads per hour.
|
|
10079
|
+
* Never reachable except from that explicitly-armed action: this is the one
|
|
10080
|
+
* surface here that deletes durable rows with no evidence from the array at
|
|
10081
|
+
* all, because the operator's own instruction is the evidence.
|
|
10019
10082
|
*/
|
|
10020
|
-
async rowsForHours(keys, scope) {
|
|
10021
|
-
if (keys.length === 0) return [];
|
|
10022
|
-
let records;
|
|
10023
|
-
try {
|
|
10024
|
-
records = await this.store.query.query({
|
|
10025
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10026
|
-
filter: {
|
|
10027
|
-
whereIn: { key: [...keys] },
|
|
10028
|
-
limit: keys.length
|
|
10029
|
-
}
|
|
10030
|
-
});
|
|
10031
|
-
} catch (err) {
|
|
10032
|
-
this.logger.warn("recorder: archive page detail unavailable — these hours take NO part in this pass", {
|
|
10033
|
-
...scope.deviceId !== void 0 ? { tags: { deviceId: scope.deviceId } } : {},
|
|
10034
|
-
meta: {
|
|
10035
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10036
|
-
hours: keys.length,
|
|
10037
|
-
error: String(err)
|
|
10038
|
-
}
|
|
10039
|
-
});
|
|
10040
|
-
return [];
|
|
10041
|
-
}
|
|
10042
|
-
const rows = [];
|
|
10043
|
-
for (const record of records) {
|
|
10044
|
-
const hour = recordToRow(record.id, record.data);
|
|
10045
|
-
if (hour === null) continue;
|
|
10046
|
-
this.ledger.stage(hour);
|
|
10047
|
-
for (const path of hour.paths) {
|
|
10048
|
-
const parsed = parseSegmentPath(path);
|
|
10049
|
-
if (parsed === null) continue;
|
|
10050
|
-
rows.push({
|
|
10051
|
-
deviceId: parsed.deviceId,
|
|
10052
|
-
profile: parsed.profile,
|
|
10053
|
-
startMs: parsed.startMs,
|
|
10054
|
-
durMs: parsed.durMs,
|
|
10055
|
-
bytes: parsed.bytes,
|
|
10056
|
-
path,
|
|
10057
|
-
locationId: hour.locationId
|
|
10058
|
-
});
|
|
10059
|
-
}
|
|
10060
|
-
}
|
|
10061
|
-
return rows.toSorted((a, b) => a.startMs - b.startMs);
|
|
10062
|
-
}
|
|
10063
10083
|
/**
|
|
10064
|
-
*
|
|
10065
|
-
* default.
|
|
10084
|
+
* Forget ONE hour row outright, and answer the segments it accounted for.
|
|
10066
10085
|
*
|
|
10067
|
-
*
|
|
10068
|
-
*
|
|
10069
|
-
*
|
|
10070
|
-
*
|
|
10071
|
-
* it selects nothing. A partial repair finishes on the next boot; until then
|
|
10072
|
-
* the affected disks UNDER-report, which prunes less and never more.
|
|
10086
|
+
* The caller must have TRUSTED evidence that the hour holds nothing — a
|
|
10087
|
+
* directory listing under a volume anchor that agreed twice (D49), or a
|
|
10088
|
+
* per-path confirmation against every root there is. This method makes no
|
|
10089
|
+
* such check itself and can make none: it is handed a verdict, not evidence.
|
|
10073
10090
|
*
|
|
10074
|
-
*
|
|
10075
|
-
*
|
|
10076
|
-
*
|
|
10077
|
-
* unbounded materialise here would be the whole-archive read this stage
|
|
10078
|
-
* removed, wearing a different name.
|
|
10091
|
+
* It exists because {@link adoptWalkedHour} deliberately refuses an EMPTY
|
|
10092
|
+
* listing — "the walk found nothing" is what an unmounted root looks like —
|
|
10093
|
+
* so the one repair a listing cannot express is the one that empties an hour.
|
|
10079
10094
|
*/
|
|
10080
|
-
async
|
|
10081
|
-
|
|
10082
|
-
|
|
10083
|
-
const
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
|
|
10087
|
-
|
|
10088
|
-
|
|
10089
|
-
|
|
10090
|
-
|
|
10091
|
-
|
|
10092
|
-
|
|
10093
|
-
|
|
10094
|
-
|
|
10095
|
-
|
|
10096
|
-
await this.materialize(pending.slice(0, REPAIR_MAX_MATERIALIZE));
|
|
10097
|
-
for (const record of stale) {
|
|
10098
|
-
const held = this.ledger.get(record.id);
|
|
10099
|
-
if (held === void 0 || !held.pathsResident || held.segments === 0) continue;
|
|
10100
|
-
await this.ledger.put(held);
|
|
10101
|
-
repaired += 1;
|
|
10102
|
-
}
|
|
10103
|
-
} catch (err) {
|
|
10104
|
-
this.logger.warn("recorder: segment-hour aggregate repair failed — disks under-report", { meta: {
|
|
10105
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10106
|
-
error: String(err)
|
|
10107
|
-
} });
|
|
10108
|
-
return repaired;
|
|
10095
|
+
async forgetHour(deviceId, profile, locationId, hour) {
|
|
10096
|
+
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
10097
|
+
return this.serialized(key, async () => {
|
|
10098
|
+
const held = this.ledger.get(key);
|
|
10099
|
+
if (held === void 0) return 0;
|
|
10100
|
+
await this.ledger.forget(key);
|
|
10101
|
+
this.accounted.delete(key);
|
|
10102
|
+
return held.segments;
|
|
10103
|
+
});
|
|
10104
|
+
}
|
|
10105
|
+
async forgetScope(scope) {
|
|
10106
|
+
let forgotten = 0;
|
|
10107
|
+
for (const row of this.ledger.snapshot()) {
|
|
10108
|
+
if (!SegmentHourLedger.inScope(row, scope)) continue;
|
|
10109
|
+
await this.ledger.forget(row.key);
|
|
10110
|
+
forgotten += row.segments;
|
|
10109
10111
|
}
|
|
10110
|
-
|
|
10111
|
-
rows: repaired,
|
|
10112
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION
|
|
10113
|
-
} });
|
|
10114
|
-
return repaired;
|
|
10112
|
+
return forgotten;
|
|
10115
10113
|
}
|
|
10116
10114
|
/**
|
|
10117
10115
|
* Write-behind: a just-finalized segment joins its hour row.
|
|
@@ -10119,8 +10117,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10119
10117
|
* THE hot path — every profile-writer lands here every `segmentSeconds`,
|
|
10120
10118
|
* ~3.4 times a second fleet-wide. The append is {@link appendSegmentPath}:
|
|
10121
10119
|
* one `parseSegmentPath` and four scalar updates, whatever the hour already
|
|
10122
|
-
* holds
|
|
10123
|
-
*
|
|
10120
|
+
* holds — and since D523 it needs no read of the store at all, because the
|
|
10121
|
+
* mirror carries the whole row.
|
|
10124
10122
|
*/
|
|
10125
10123
|
async recordSegment(row) {
|
|
10126
10124
|
const { key, hourStartMs: hour } = hourOf(row);
|
|
@@ -10133,7 +10131,12 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10133
10131
|
entryMs: row.startMs
|
|
10134
10132
|
});
|
|
10135
10133
|
await this.serialized(key, async () => {
|
|
10136
|
-
|
|
10134
|
+
const seen = this.accounted.get(key);
|
|
10135
|
+
if (seen !== void 0 && seen.has(row.path)) return;
|
|
10136
|
+
if (seen === void 0) this.accounted.set(key, new Set([row.path]));
|
|
10137
|
+
else seen.add(row.path);
|
|
10138
|
+
const held = this.ledger.get(key);
|
|
10139
|
+
if (held === void 0) {
|
|
10137
10140
|
await this.ledger.put(segmentHourRow({
|
|
10138
10141
|
key,
|
|
10139
10142
|
deviceId: row.deviceId,
|
|
@@ -10144,25 +10147,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10144
10147
|
}));
|
|
10145
10148
|
return;
|
|
10146
10149
|
}
|
|
10147
|
-
const held = await this.resident(key, row.deviceId);
|
|
10148
|
-
if (held === null) {
|
|
10149
|
-
this.refused.add(key);
|
|
10150
|
-
this.logger.warn("recorder: hour detail unavailable — this finalized segment is NOT in the durable hour row yet", {
|
|
10151
|
-
tags: { deviceId: row.deviceId },
|
|
10152
|
-
meta: {
|
|
10153
|
-
key,
|
|
10154
|
-
path: row.path
|
|
10155
|
-
}
|
|
10156
|
-
});
|
|
10157
|
-
return;
|
|
10158
|
-
}
|
|
10159
|
-
if (held.paths.includes(row.path)) return;
|
|
10160
10150
|
await this.ledger.put(appendSegmentPath(held, row.path));
|
|
10161
10151
|
});
|
|
10162
10152
|
}
|
|
10163
10153
|
/**
|
|
10164
|
-
* Eviction confirmed these
|
|
10165
|
-
* hour when
|
|
10154
|
+
* Eviction confirmed these segments are gone from disk. Take them out of the
|
|
10155
|
+
* hour's account; forget the hour when nothing is left.
|
|
10156
|
+
*
|
|
10157
|
+
* The aggregate shrinks by exactly what was removed. `minStartMs` /
|
|
10158
|
+
* `maxStartMs` are NOT re-derived — they cannot be, without a listing — so a
|
|
10159
|
+
* partly-evicted hour keeps extremes that may be older or newer than what
|
|
10160
|
+
* survives. That error is bounded by the hour itself (both values always lie
|
|
10161
|
+
* inside `[hourStartMs, hourStartMs + 1h)`), and the row is UNSEALED so the
|
|
10162
|
+
* next listing of it re-derives both exactly.
|
|
10166
10163
|
*/
|
|
10167
10164
|
async dropSegments(rows) {
|
|
10168
10165
|
const doomed = /* @__PURE__ */ new Map();
|
|
@@ -10172,70 +10169,50 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10172
10169
|
if (entry === void 0) {
|
|
10173
10170
|
doomed.set(key, {
|
|
10174
10171
|
deviceId: row.deviceId,
|
|
10175
|
-
|
|
10176
|
-
|
|
10177
|
-
continue;
|
|
10178
|
-
}
|
|
10179
|
-
entry.paths.add(row.path);
|
|
10180
|
-
}
|
|
10181
|
-
for (const [key, { deviceId, paths }] of doomed) {
|
|
10182
|
-
if (!this.ledger.has(key)) continue;
|
|
10183
|
-
const held = await this.resident(key, deviceId);
|
|
10184
|
-
if (held === null) {
|
|
10185
|
-
this.logger.warn("recorder: hour detail unavailable — these evicted paths stay in the durable hour row until a walk or the next eviction confirms them", {
|
|
10186
|
-
tags: { deviceId },
|
|
10187
|
-
meta: {
|
|
10188
|
-
key,
|
|
10189
|
-
paths: paths.size
|
|
10190
|
-
}
|
|
10172
|
+
bytes: row.bytes,
|
|
10173
|
+
segments: 1
|
|
10191
10174
|
});
|
|
10192
10175
|
continue;
|
|
10193
10176
|
}
|
|
10194
|
-
|
|
10195
|
-
|
|
10177
|
+
entry.bytes += row.bytes;
|
|
10178
|
+
entry.segments += 1;
|
|
10179
|
+
}
|
|
10180
|
+
for (const [key, dropped] of doomed) await this.serialized(key, async () => {
|
|
10181
|
+
const held = this.ledger.get(key);
|
|
10182
|
+
if (held === void 0) return;
|
|
10183
|
+
const segments = held.segments - dropped.segments;
|
|
10184
|
+
if (segments <= 0) {
|
|
10196
10185
|
await this.ledger.forget(key);
|
|
10197
|
-
|
|
10186
|
+
this.accounted.delete(key);
|
|
10187
|
+
return;
|
|
10198
10188
|
}
|
|
10199
|
-
|
|
10200
|
-
await this.ledger.put(segmentHourRow({
|
|
10189
|
+
await this.ledger.put({
|
|
10201
10190
|
...held,
|
|
10202
|
-
|
|
10203
|
-
|
|
10204
|
-
|
|
10191
|
+
sealed: false,
|
|
10192
|
+
segments,
|
|
10193
|
+
bytes: Math.max(0, held.bytes - dropped.bytes)
|
|
10194
|
+
});
|
|
10195
|
+
});
|
|
10205
10196
|
}
|
|
10206
10197
|
/**
|
|
10207
|
-
* Move `rows` from the hour
|
|
10198
|
+
* Move `rows` from the hour they are accounted under today to the SAME hour
|
|
10208
10199
|
* at `toLocationId` — the DURABLE half of a relocation.
|
|
10209
10200
|
*
|
|
10210
|
-
* Without this the mover re-points only `RecordingIndex`, which is RAM
|
|
10211
|
-
*
|
|
10212
|
-
*
|
|
10213
|
-
*
|
|
10214
|
-
* no longer holds the file. Playback does not stat (that syscall per segment
|
|
10215
|
-
* was removed deliberately), so the result is a silent black window; and once
|
|
10216
|
-
* the drained location is deleted, `collectRangeSegments` drops the rows
|
|
10217
|
-
* outright and the footage is invisible. That is the shape of the 2 131
|
|
10218
|
-
* `recordings:high` ghost segments already on this hub.
|
|
10201
|
+
* Without this the mover re-points only `RecordingIndex`, which is RAM, and
|
|
10202
|
+
* the drained location keeps the bytes on every storage page for the life of
|
|
10203
|
+
* the hub. That is the shape of the 2 131 `recordings:high` ghost rows this
|
|
10204
|
+
* hub carried.
|
|
10219
10205
|
*
|
|
10220
10206
|
* **The ordering is the invariant, not an implementation detail.** The
|
|
10221
10207
|
* destination row is written BEFORE the source row is shrunk, and the caller
|
|
10222
10208
|
* has not yet deleted the source bytes — so the only state a crash can leave
|
|
10223
|
-
* is "
|
|
10224
|
-
*
|
|
10225
|
-
*
|
|
10226
|
-
* the index `hydrated`: nothing would ever walk that hour again, and the
|
|
10227
|
-
* segment would sit on disk permanently unreachable.
|
|
10228
|
-
*
|
|
10229
|
-
* **Two path sets, not one.** A path can arrive at the target under a
|
|
10230
|
-
* DIFFERENT name than it leaves the source under: a mover that found a file
|
|
10231
|
-
* larger than its recorded size reconciles the row, and the byte count is
|
|
10232
|
-
* part of the name. So the source row is shrunk by what is LEAVING it and the
|
|
10233
|
-
* target row grows by what is ARRIVING — conflating them would leave the
|
|
10234
|
-
* source row naming a file the caller is about to delete.
|
|
10209
|
+
* is "accounted at BOTH locations, both of which hold the files", which a
|
|
10210
|
+
* listing of either hour resolves. The reverse order has a window in which
|
|
10211
|
+
* the segments are accounted NOWHERE.
|
|
10235
10212
|
*
|
|
10236
|
-
*
|
|
10237
|
-
*
|
|
10238
|
-
* the
|
|
10213
|
+
* Both rows come out UNSEALED: each carries an aggregate that arithmetic
|
|
10214
|
+
* moved rather than a listing derived, and the next walk of either hour fixes
|
|
10215
|
+
* whatever the arithmetic rounded.
|
|
10239
10216
|
*/
|
|
10240
10217
|
async repointSegments(rows, toLocationId) {
|
|
10241
10218
|
const byHour = /* @__PURE__ */ new Map();
|
|
@@ -10243,87 +10220,75 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10243
10220
|
if (row.locationId === toLocationId) continue;
|
|
10244
10221
|
const { key } = hourOf(row);
|
|
10245
10222
|
const entry = byHour.get(key);
|
|
10246
|
-
if (entry === void 0)
|
|
10247
|
-
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
entry.arriving.add(row.path);
|
|
10254
|
-
}
|
|
10255
|
-
}
|
|
10256
|
-
for (const [sourceKey, { row, leaving, arriving }] of byHour) {
|
|
10257
|
-
const held = await this.resident(sourceKey, row.deviceId);
|
|
10258
|
-
if (held === null) {
|
|
10259
|
-
this.logger.warn("recorder: hour detail unavailable — this relocation is REFUSED and nothing is deleted; the segments stay where they are", {
|
|
10260
|
-
tags: { deviceId: row.deviceId },
|
|
10261
|
-
meta: {
|
|
10262
|
-
key: sourceKey,
|
|
10263
|
-
toLocationId,
|
|
10264
|
-
paths: leaving.size
|
|
10265
|
-
}
|
|
10266
|
-
});
|
|
10267
|
-
return false;
|
|
10268
|
-
}
|
|
10269
|
-
const targetKey = segmentHourKey(row.deviceId, row.profile, toLocationId, held.hourStartMs);
|
|
10270
|
-
const target = this.ledger.has(targetKey) ? await this.resident(targetKey, row.deviceId) : void 0;
|
|
10271
|
-
if (target === null) {
|
|
10272
|
-
this.logger.warn("recorder: TARGET hour detail unavailable — this relocation is REFUSED; appending to a summary would rewrite the hour as these paths alone", {
|
|
10273
|
-
tags: { deviceId: row.deviceId },
|
|
10274
|
-
meta: {
|
|
10275
|
-
key: targetKey,
|
|
10276
|
-
fromLocationId: row.locationId,
|
|
10277
|
-
paths: arriving.size
|
|
10278
|
-
}
|
|
10223
|
+
if (entry === void 0) {
|
|
10224
|
+
byHour.set(key, {
|
|
10225
|
+
row,
|
|
10226
|
+
bytes: row.bytes,
|
|
10227
|
+
segments: 1,
|
|
10228
|
+
minStartMs: row.startMs,
|
|
10229
|
+
maxStartMs: row.startMs
|
|
10279
10230
|
});
|
|
10280
|
-
|
|
10231
|
+
continue;
|
|
10281
10232
|
}
|
|
10282
|
-
|
|
10283
|
-
|
|
10233
|
+
entry.bytes += row.bytes;
|
|
10234
|
+
entry.segments += 1;
|
|
10235
|
+
entry.minStartMs = Math.min(entry.minStartMs, row.startMs);
|
|
10236
|
+
entry.maxStartMs = Math.max(entry.maxStartMs, row.startMs);
|
|
10237
|
+
}
|
|
10238
|
+
for (const [sourceKey, moved] of byHour) {
|
|
10239
|
+
const hour = hourStartMs(moved.row.startMs);
|
|
10240
|
+
const targetKey = segmentHourKey(moved.row.deviceId, moved.row.profile, toLocationId, hour);
|
|
10241
|
+
const target = this.ledger.get(targetKey);
|
|
10242
|
+
const targetIdentity = {
|
|
10284
10243
|
key: targetKey,
|
|
10285
|
-
deviceId: row.deviceId,
|
|
10286
|
-
profile: row.profile,
|
|
10244
|
+
deviceId: moved.row.deviceId,
|
|
10245
|
+
profile: moved.row.profile,
|
|
10287
10246
|
locationId: toLocationId,
|
|
10288
|
-
hourStartMs:
|
|
10289
|
-
|
|
10247
|
+
hourStartMs: hour
|
|
10248
|
+
};
|
|
10249
|
+
const targetEmpty = target === void 0 || target.segments === 0;
|
|
10250
|
+
await this.ledger.put(hourRowOf(targetIdentity, {
|
|
10251
|
+
sealed: false,
|
|
10252
|
+
bytes: (target?.bytes ?? 0) + moved.bytes,
|
|
10253
|
+
segments: (target?.segments ?? 0) + moved.segments,
|
|
10254
|
+
minStartMs: targetEmpty ? moved.minStartMs : Math.min(target.minStartMs, moved.minStartMs),
|
|
10255
|
+
maxStartMs: targetEmpty ? moved.maxStartMs : Math.max(target.maxStartMs, moved.maxStartMs)
|
|
10290
10256
|
}));
|
|
10291
|
-
const
|
|
10292
|
-
if (
|
|
10293
|
-
|
|
10257
|
+
const held = this.ledger.get(sourceKey);
|
|
10258
|
+
if (held === void 0) continue;
|
|
10259
|
+
const segments = held.segments - moved.segments;
|
|
10260
|
+
if (segments <= 0) {
|
|
10261
|
+
await this.ledger.forget(sourceKey);
|
|
10262
|
+
continue;
|
|
10263
|
+
}
|
|
10264
|
+
await this.ledger.put({
|
|
10294
10265
|
...held,
|
|
10295
|
-
|
|
10296
|
-
|
|
10266
|
+
sealed: false,
|
|
10267
|
+
segments,
|
|
10268
|
+
bytes: Math.max(0, held.bytes - moved.bytes)
|
|
10269
|
+
});
|
|
10297
10270
|
}
|
|
10298
|
-
return true;
|
|
10299
10271
|
}
|
|
10300
10272
|
/**
|
|
10301
|
-
* After a disk walk of `deviceIds`, make the ledger match the index
|
|
10302
|
-
* two D148 exceptions:
|
|
10273
|
+
* After a disk walk of `deviceIds`, make the ledger match what the index now
|
|
10274
|
+
* holds — with two D148 exceptions:
|
|
10303
10275
|
*
|
|
10304
10276
|
* - an hour the walk did not see is dropped ONLY if it is in the past;
|
|
10305
10277
|
* the current hour may still hold a live writer whose file the readdir
|
|
10306
10278
|
* raced;
|
|
10307
|
-
* - the current hour
|
|
10308
|
-
*
|
|
10279
|
+
* - the current hour takes the LARGER of the two accounts rather than the
|
|
10280
|
+
* walk's, for the same race.
|
|
10309
10281
|
*
|
|
10310
10282
|
* Devices not in `deviceIds` are left untouched (a partial walk must not
|
|
10311
10283
|
* prune cameras it did not look at).
|
|
10312
10284
|
*
|
|
10313
10285
|
* And a walk this ledger does not TRUST prunes nothing at all — see
|
|
10314
|
-
* {@link refusedPrunes}. What such a walk FOUND is still real, so its
|
|
10315
|
-
*
|
|
10316
|
-
*
|
|
10317
|
-
* Since D287 the walked devices' hours are MATERIALISED first: every
|
|
10318
|
-
* comparison below reads `row.paths`, and a summary's empty `paths` would
|
|
10319
|
-
* read as "the ledger holds nothing here" — which is the union collapsing to
|
|
10320
|
-
* the walk's own view, i.e. the prune the refusal machinery exists to
|
|
10321
|
-
* prevent. An hour whose detail could not be read is left additive-only.
|
|
10286
|
+
* {@link refusedPrunes}. What such a walk FOUND is still real, so its larger
|
|
10287
|
+
* account is taken; what it did not find decides nothing (D49).
|
|
10322
10288
|
*/
|
|
10323
10289
|
async reconcileFromIndex(index, deviceIds, nowMs) {
|
|
10324
10290
|
const walked = new Set(deviceIds);
|
|
10325
10291
|
const currentHour = hourStartMs(nowMs);
|
|
10326
|
-
await this.materializeDevices(walked);
|
|
10327
10292
|
const fromIndex = /* @__PURE__ */ new Map();
|
|
10328
10293
|
for (const deviceId of deviceIds) for (const seg of index.segments(deviceId)) {
|
|
10329
10294
|
const { key, hourStartMs: hour } = hourOf(seg);
|
|
@@ -10340,25 +10305,26 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10340
10305
|
else fromIndex.set(key, appendSegmentPath(existing, seg.path));
|
|
10341
10306
|
}
|
|
10342
10307
|
const refused = this.refusedPrunes(walked, fromIndex);
|
|
10308
|
+
const trimmed = /* @__PURE__ */ new Set();
|
|
10309
|
+
for (const deviceId of walked) {
|
|
10310
|
+
if (!index.droppedDuringWalk(deviceId)) continue;
|
|
10311
|
+
trimmed.add(deviceId);
|
|
10312
|
+
this.logger.warn("recorder: ledger prune REFUSED — the RAM index dropped hours during this walk, so its silence is not evidence", { tags: { deviceId } });
|
|
10313
|
+
}
|
|
10343
10314
|
for (const row of this.ledger.snapshot()) {
|
|
10344
10315
|
if (!walked.has(row.deviceId)) continue;
|
|
10345
|
-
if (!row.pathsResident) continue;
|
|
10346
10316
|
const disk = fromIndex.get(row.key);
|
|
10347
|
-
const additiveOnly = row.hourStartMs === currentHour || refused.devices.has(row.deviceId) || refused.locations.has(locationKey(row.deviceId, row.locationId));
|
|
10317
|
+
const additiveOnly = row.hourStartMs === currentHour || trimmed.has(row.deviceId) || refused.devices.has(row.deviceId) || refused.locations.has(locationKey(row.deviceId, row.locationId));
|
|
10348
10318
|
if (disk === void 0) {
|
|
10349
10319
|
if (additiveOnly) continue;
|
|
10350
10320
|
await this.ledger.forget(row.key);
|
|
10351
10321
|
continue;
|
|
10352
10322
|
}
|
|
10353
10323
|
if (additiveOnly) {
|
|
10354
|
-
|
|
10355
|
-
if (!samePaths(union, row.paths)) await this.ledger.put(segmentHourRow({
|
|
10356
|
-
...row,
|
|
10357
|
-
paths: union
|
|
10358
|
-
}));
|
|
10324
|
+
if (disk.segments > row.segments) await this.ledger.put(disk);
|
|
10359
10325
|
continue;
|
|
10360
10326
|
}
|
|
10361
|
-
if (
|
|
10327
|
+
if (disk.segments !== row.segments || disk.bytes !== row.bytes) await this.ledger.put(disk);
|
|
10362
10328
|
}
|
|
10363
10329
|
for (const [key, disk] of fromIndex) {
|
|
10364
10330
|
if (this.ledger.has(key)) continue;
|
|
@@ -10477,44 +10443,45 @@ function locationKey(deviceId, locationId) {
|
|
|
10477
10443
|
* skips it forever. Under D148 as written, a ghost row can never be forgotten
|
|
10478
10444
|
* by any mechanism — which is why the live ones have stood for thirteen days.
|
|
10479
10445
|
*
|
|
10480
|
-
* The resolution is that
|
|
10481
|
-
*
|
|
10482
|
-
* (`590/high/2026/08/15/04
|
|
10483
|
-
* currently-resolvable root
|
|
10484
|
-
*
|
|
10485
|
-
*
|
|
10486
|
-
* one `readdir` at a time.** That, and nothing weaker, is what licenses the
|
|
10446
|
+
* The resolution is that the ghost row's root does not resolve but its hour
|
|
10447
|
+
* DIRECTORY is addressable anywhere: the path convention
|
|
10448
|
+
* (`590/high/2026/08/15/04/`) is a function of the row's own fields, so the same
|
|
10449
|
+
* hour can be listed under every currently-resolvable root. **A listing of the
|
|
10450
|
+
* hour directory at every candidate root is exactly the walk D148 sanctions,
|
|
10451
|
+
* done one root at a time.** That, and nothing weaker, is what licenses the
|
|
10487
10452
|
* forget. Two consequences ride with it:
|
|
10488
10453
|
*
|
|
10489
|
-
* - **
|
|
10490
|
-
*
|
|
10491
|
-
* stands.
|
|
10492
|
-
* - **An error is not an absence** (D49). A
|
|
10493
|
-
* with nothing decided — never "not
|
|
10454
|
+
* - **Every root must answer.** An hour is forgotten only when every candidate
|
|
10455
|
+
* root listed its directory and every one of them came back EMPTY. One root
|
|
10456
|
+
* that could not answer and the hour stands.
|
|
10457
|
+
* - **An error is not an absence** (D49). A listing that fails, or that reports
|
|
10458
|
+
* "I could not decide", ends the hour with nothing decided — never "not
|
|
10459
|
+
* found". Whether an absent directory means "empty" or "this disk is gone" is
|
|
10460
|
+
* the lister's question, and it answers it by reading the ROOT.
|
|
10461
|
+
*
|
|
10462
|
+
* Until D523 this probed each of the row's `paths` at each root, one
|
|
10463
|
+
* `storage.exists` at a time — up to ~1 000 syscalls to prove one hour absent.
|
|
10464
|
+
* The column is gone and the row names no file, so the unit is the directory,
|
|
10465
|
+
* which is both cheaper and the same evidence.
|
|
10494
10466
|
*
|
|
10495
10467
|
* ## Repoint versus forget
|
|
10496
10468
|
*
|
|
10497
10469
|
* The asymmetry is the whole safety argument, and it is deliberately cheap in
|
|
10498
10470
|
* the safe direction:
|
|
10499
10471
|
*
|
|
10500
|
-
* - **
|
|
10501
|
-
* repointed to that root's location
|
|
10502
|
-
*
|
|
10503
|
-
* deleting here would destroy the index of footage
|
|
10504
|
-
* failure D291 was written to prevent.
|
|
10505
|
-
* - **every
|
|
10506
|
-
*
|
|
10507
|
-
* So proving presence costs one probe and proving absence costs all of them,
|
|
10508
|
-
* which is the right way round: the expensive proof is the destructive one.
|
|
10472
|
+
* - **a non-empty listing at any root ⇒ the row is MIS-POINTED, not orphaned.**
|
|
10473
|
+
* It is repointed to that root's location, with the segments that listing
|
|
10474
|
+
* names, and the hour is finished. A renamed or re-created location produces
|
|
10475
|
+
* exactly this shape, and deleting here would destroy the index of footage
|
|
10476
|
+
* that EXISTS, which is the failure D291 was written to prevent.
|
|
10477
|
+
* - **every root lists an EMPTY hour ⇒ orphaned**, and only then forgotten.
|
|
10509
10478
|
*
|
|
10510
10479
|
* ## Pacing
|
|
10511
10480
|
*
|
|
10512
|
-
* The recordings array is
|
|
10513
|
-
*
|
|
10514
|
-
*
|
|
10515
|
-
*
|
|
10516
|
-
* Work it does not finish is left for the next pass and REPORTED, never
|
|
10517
|
-
* silently dropped.
|
|
10481
|
+
* The recordings array is the resource every live read competes for. This is a
|
|
10482
|
+
* janitor: bounded hours per pass, a hard probe ceiling, and a caller-supplied
|
|
10483
|
+
* `pace()` awaited between probe batches. Work it does not finish is left for
|
|
10484
|
+
* the next pass and REPORTED, never silently dropped.
|
|
10518
10485
|
*/
|
|
10519
10486
|
var HOUR_MS$3 = 36e5;
|
|
10520
10487
|
/**
|
|
@@ -10663,22 +10630,12 @@ var GhostLocationSweeper = class {
|
|
|
10663
10630
|
refuse(row, "live-tail", `hourStartMs=${row.hourStartMs}`);
|
|
10664
10631
|
continue;
|
|
10665
10632
|
}
|
|
10666
|
-
|
|
10667
|
-
|
|
10668
|
-
refuse(row, "detail-unavailable", "point read failed");
|
|
10669
|
-
continue;
|
|
10670
|
-
}
|
|
10671
|
-
if (paths.length === 0) {
|
|
10672
|
-
refuse(row, "no-paths", "the row materialised empty");
|
|
10673
|
-
continue;
|
|
10674
|
-
}
|
|
10675
|
-
const rows = this.rowsOf(row, paths);
|
|
10676
|
-
if (rows === null) {
|
|
10677
|
-
refuse(row, "unparseable-paths", "a path does not parse into this hour");
|
|
10633
|
+
if (row.segments === 0) {
|
|
10634
|
+
refuse(row, "no-paths", "the row accounts for no segments");
|
|
10678
10635
|
continue;
|
|
10679
10636
|
}
|
|
10680
10637
|
const before = probes;
|
|
10681
|
-
const verdict = await this.probeHour(
|
|
10638
|
+
const verdict = await this.probeHour(row, roots, maxProbes - probes, probes);
|
|
10682
10639
|
probes += verdict.probes;
|
|
10683
10640
|
tallyOf(row.deviceId).probes += probes - before;
|
|
10684
10641
|
if (verdict.outcome.kind === "refused") {
|
|
@@ -10687,10 +10644,12 @@ var GhostLocationSweeper = class {
|
|
|
10687
10644
|
}
|
|
10688
10645
|
if (verdict.outcome.kind === "found") {
|
|
10689
10646
|
const target = locationForHydratedProfile(verdict.outcome.root.aliases, row.profile);
|
|
10690
|
-
|
|
10691
|
-
|
|
10647
|
+
const rows = this.rowsOf(row, verdict.outcome.paths);
|
|
10648
|
+
if (rows === null) {
|
|
10649
|
+
refuse(row, "unparseable-paths", "a listed name does not parse into this hour");
|
|
10692
10650
|
continue;
|
|
10693
10651
|
}
|
|
10652
|
+
await this.deps.ledger.repointSegments(rows, target.id);
|
|
10694
10653
|
this.deps.applyToIndex(rows, target.id);
|
|
10695
10654
|
repointedHours += 1;
|
|
10696
10655
|
segmentsRepointed += rows.length;
|
|
@@ -10700,19 +10659,19 @@ var GhostLocationSweeper = class {
|
|
|
10700
10659
|
tally.targets.add(target.id);
|
|
10701
10660
|
continue;
|
|
10702
10661
|
}
|
|
10703
|
-
await this.deps.ledger.
|
|
10662
|
+
const forgotten = await this.deps.ledger.forgetHour(row.deviceId, row.profile, row.locationId, row.hourStartMs);
|
|
10704
10663
|
if (this.deps.ledger.knowsHour(row.deviceId, row.profile, row.locationId, row.hourStartMs)) {
|
|
10705
|
-
refuse(row, "durable-refused", "the ledger still holds the hour after the
|
|
10664
|
+
refuse(row, "durable-refused", "the ledger still holds the hour after the forget");
|
|
10706
10665
|
continue;
|
|
10707
10666
|
}
|
|
10708
|
-
this.deps.
|
|
10667
|
+
this.deps.forgetInIndex(row.deviceId, row.locationId, row.hourStartMs);
|
|
10709
10668
|
forgottenHours += 1;
|
|
10710
|
-
segmentsForgotten +=
|
|
10711
|
-
bytesForgotten +=
|
|
10669
|
+
segmentsForgotten += forgotten;
|
|
10670
|
+
bytesForgotten += row.bytes;
|
|
10712
10671
|
const tally = tallyOf(row.deviceId);
|
|
10713
10672
|
tally.forgottenHours += 1;
|
|
10714
|
-
tally.forgottenSegments +=
|
|
10715
|
-
tally.forgottenBytes +=
|
|
10673
|
+
tally.forgottenSegments += forgotten;
|
|
10674
|
+
tally.forgottenBytes += row.bytes;
|
|
10716
10675
|
}
|
|
10717
10676
|
for (const [deviceId, tally] of byDevice) this.deps.logger.info("recorder: ghost-location reconcile — camera summary", {
|
|
10718
10677
|
tags: { deviceId },
|
|
@@ -10794,60 +10753,82 @@ var GhostLocationSweeper = class {
|
|
|
10794
10753
|
return out;
|
|
10795
10754
|
}
|
|
10796
10755
|
/**
|
|
10797
|
-
*
|
|
10756
|
+
* List this hour's directory under EVERY candidate root.
|
|
10757
|
+
*
|
|
10758
|
+
* A non-empty listing at one root makes the hour mis-pointed. Two roots
|
|
10759
|
+
* holding it is a refusal, not a choice: repointing would be a guess about
|
|
10760
|
+
* which copy is the footage. Absence is only concluded when every root
|
|
10761
|
+
* answered and every answer was empty — a listing that could not decide
|
|
10762
|
+
* (`null`) or threw ends the hour with nothing decided (D49).
|
|
10798
10763
|
*
|
|
10799
|
-
*
|
|
10800
|
-
*
|
|
10801
|
-
*
|
|
10802
|
-
*
|
|
10764
|
+
* The whole loop runs even after a hit, because "found at exactly one root"
|
|
10765
|
+
* is a claim about every root and cannot be short-circuited without giving up
|
|
10766
|
+
* the multi-root refusal.
|
|
10767
|
+
*
|
|
10768
|
+
* `spent` is the listings the PASS has already made, not this hour's, and the
|
|
10769
|
+
* brake fires on that running total. Since D523 an hour costs one listing per
|
|
10770
|
+
* physical root — two or three on this hub — so a counter that reset per hour
|
|
10771
|
+
* would never reach {@link GHOST_PROBE_BATCH} and the `pace()` dep would be
|
|
10772
|
+
* dead on every real topology.
|
|
10803
10773
|
*/
|
|
10804
|
-
async probeHour(
|
|
10774
|
+
async probeHour(row, roots, budget, spent) {
|
|
10805
10775
|
let used = 0;
|
|
10806
|
-
|
|
10807
|
-
|
|
10808
|
-
|
|
10809
|
-
|
|
10776
|
+
const found = [];
|
|
10777
|
+
for (const root of roots) {
|
|
10778
|
+
if (used >= budget) return {
|
|
10779
|
+
outcome: {
|
|
10780
|
+
kind: "refused",
|
|
10781
|
+
reason: "probe-budget",
|
|
10782
|
+
detail: `stopped after ${used} listings with ${roots.length} roots to probe`
|
|
10783
|
+
},
|
|
10784
|
+
probes: used
|
|
10785
|
+
};
|
|
10786
|
+
const alias = locationForHydratedProfile(root.aliases, row.profile);
|
|
10787
|
+
used += 1;
|
|
10788
|
+
if ((spent + used) % 16 === 0) await this.deps.pace();
|
|
10789
|
+
let listing;
|
|
10790
|
+
try {
|
|
10791
|
+
listing = await this.deps.listHour(alias.id, row.deviceId, row.profile, row.hourStartMs);
|
|
10792
|
+
} catch (err) {
|
|
10793
|
+
return {
|
|
10810
10794
|
outcome: {
|
|
10811
10795
|
kind: "refused",
|
|
10812
|
-
reason: "probe-
|
|
10813
|
-
detail:
|
|
10796
|
+
reason: "probe-error",
|
|
10797
|
+
detail: `${alias.id}: ${errMsg(err)}`
|
|
10814
10798
|
},
|
|
10815
10799
|
probes: used
|
|
10816
10800
|
};
|
|
10817
|
-
const alias = locationForHydratedProfile(root.aliases, row.profile);
|
|
10818
|
-
used += 1;
|
|
10819
|
-
if (used % 16 === 0) await this.deps.pace();
|
|
10820
|
-
try {
|
|
10821
|
-
if (await this.deps.exists(alias.id, row.path)) found.push(root);
|
|
10822
|
-
} catch (err) {
|
|
10823
|
-
return {
|
|
10824
|
-
outcome: {
|
|
10825
|
-
kind: "refused",
|
|
10826
|
-
reason: "probe-error",
|
|
10827
|
-
detail: `${alias.id}: ${errMsg(err)}`
|
|
10828
|
-
},
|
|
10829
|
-
probes: used
|
|
10830
|
-
};
|
|
10831
|
-
}
|
|
10832
10801
|
}
|
|
10833
|
-
if (
|
|
10802
|
+
if (listing === null) return {
|
|
10834
10803
|
outcome: {
|
|
10835
10804
|
kind: "refused",
|
|
10836
|
-
reason: "
|
|
10837
|
-
detail:
|
|
10838
|
-
},
|
|
10839
|
-
probes: used
|
|
10840
|
-
};
|
|
10841
|
-
const only = found[0];
|
|
10842
|
-
if (only !== void 0) return {
|
|
10843
|
-
outcome: {
|
|
10844
|
-
kind: "found",
|
|
10845
|
-
root: only,
|
|
10846
|
-
path: row.path
|
|
10805
|
+
reason: "detail-unavailable",
|
|
10806
|
+
detail: `${alias.id} could not say what its copy of this hour holds`
|
|
10847
10807
|
},
|
|
10848
10808
|
probes: used
|
|
10849
10809
|
};
|
|
10810
|
+
if (listing.length > 0) found.push({
|
|
10811
|
+
root,
|
|
10812
|
+
paths: listing
|
|
10813
|
+
});
|
|
10850
10814
|
}
|
|
10815
|
+
if (found.length > 1) return {
|
|
10816
|
+
outcome: {
|
|
10817
|
+
kind: "refused",
|
|
10818
|
+
reason: "found-at-multiple-roots",
|
|
10819
|
+
detail: found.map((hit) => hit.root.key).join(", ")
|
|
10820
|
+
},
|
|
10821
|
+
probes: used
|
|
10822
|
+
};
|
|
10823
|
+
const only = found[0];
|
|
10824
|
+
if (only !== void 0) return {
|
|
10825
|
+
outcome: {
|
|
10826
|
+
kind: "found",
|
|
10827
|
+
root: only.root,
|
|
10828
|
+
paths: only.paths
|
|
10829
|
+
},
|
|
10830
|
+
probes: used
|
|
10831
|
+
};
|
|
10851
10832
|
return {
|
|
10852
10833
|
outcome: { kind: "absent" },
|
|
10853
10834
|
probes: used
|
|
@@ -11228,34 +11209,12 @@ function emptyTally() {
|
|
|
11228
11209
|
orphanFiles: 0
|
|
11229
11210
|
};
|
|
11230
11211
|
}
|
|
11231
|
-
/**
|
|
11232
|
-
function
|
|
11233
|
-
|
|
11234
|
-
|
|
11235
|
-
|
|
11236
|
-
|
|
11237
|
-
* rows (`streamSegments` expands them whole), so no hour is ever split. */
|
|
11238
|
-
function claimsOf(rows) {
|
|
11239
|
-
const byHour = /* @__PURE__ */ new Map();
|
|
11240
|
-
for (const row of rows) {
|
|
11241
|
-
const hour = hourStartMs(row.startMs);
|
|
11242
|
-
const key = `${row.deviceId}|${row.profile}|${hour}`;
|
|
11243
|
-
const held = byHour.get(key);
|
|
11244
|
-
if (held) held.push(row);
|
|
11245
|
-
else byHour.set(key, [row]);
|
|
11246
|
-
}
|
|
11247
|
-
const out = [];
|
|
11248
|
-
for (const group of byHour.values()) {
|
|
11249
|
-
const first = group[0];
|
|
11250
|
-
if (first === void 0) continue;
|
|
11251
|
-
out.push({
|
|
11252
|
-
deviceId: first.deviceId,
|
|
11253
|
-
profile: first.profile,
|
|
11254
|
-
hourStartMs: hourStartMs(first.startMs),
|
|
11255
|
-
rows: group
|
|
11256
|
-
});
|
|
11257
|
-
}
|
|
11258
|
-
return out.toSorted((a, b) => a.hourStartMs - b.hourStartMs || a.deviceId - b.deviceId);
|
|
11212
|
+
/** Does this hour row fall inside the scope the operator asked for? */
|
|
11213
|
+
function inScope(row, scope) {
|
|
11214
|
+
if (scope.locationId !== void 0 && row.locationId !== scope.locationId) return false;
|
|
11215
|
+
if (scope.deviceId !== void 0 && row.deviceId !== scope.deviceId) return false;
|
|
11216
|
+
const profiles = scope.profiles ?? [];
|
|
11217
|
+
return profiles.length === 0 || profiles.includes(row.profile);
|
|
11259
11218
|
}
|
|
11260
11219
|
/**
|
|
11261
11220
|
* One reconcile pass over one location's claimed hours.
|
|
@@ -11353,20 +11312,35 @@ var LedgerWalkReconciler = class {
|
|
|
11353
11312
|
this.reportPass(report);
|
|
11354
11313
|
return report;
|
|
11355
11314
|
}
|
|
11356
|
-
/**
|
|
11315
|
+
/**
|
|
11316
|
+
* Walk the hour ROWS of the scope, oldest first.
|
|
11317
|
+
*
|
|
11318
|
+
* From the MIRROR, not a query. Since D523 a row is ten scalars, so the whole
|
|
11319
|
+
* claim set is already in RAM and costs nothing to fold — and the pass mutates
|
|
11320
|
+
* what it walks, which is exactly the shape that makes `offset` paging skip
|
|
11321
|
+
* half a scope. A snapshot taken once cannot skip.
|
|
11322
|
+
*/
|
|
11357
11323
|
async walk(state, scope) {
|
|
11358
|
-
|
|
11324
|
+
const claims = state.ledger.snapshot().filter((row) => inScope(row, scope)).toSorted((a, b) => a.hourStartMs - b.hourStartMs || a.deviceId - b.deviceId);
|
|
11325
|
+
for (const claim of claims) {
|
|
11359
11326
|
if (state.hoursClaimed >= state.maxHours) {
|
|
11360
11327
|
state.bounded = true;
|
|
11361
11328
|
return;
|
|
11362
11329
|
}
|
|
11363
11330
|
state.hoursClaimed += 1;
|
|
11364
|
-
state.segmentsClaimed += claim.
|
|
11331
|
+
state.segmentsClaimed += claim.segments;
|
|
11365
11332
|
if (!await this.reconcileHour(state, claim)) return;
|
|
11366
11333
|
}
|
|
11367
11334
|
}
|
|
11368
11335
|
/**
|
|
11369
|
-
*
|
|
11336
|
+
* Reconcile ONE hour: what its row ACCOUNTS FOR against what its directory
|
|
11337
|
+
* HOLDS, and repair the account from the directory.
|
|
11338
|
+
*
|
|
11339
|
+
* Since D523 the row names no files, so this is a comparison of counts and
|
|
11340
|
+
* bytes — and it does not need to name them: the repair is "make the account
|
|
11341
|
+
* the listing's", which is exact, and the RESIDENT index is reset to the same
|
|
11342
|
+
* listing so a row it holds for a file the directory does not is dropped with
|
|
11343
|
+
* it.
|
|
11370
11344
|
*
|
|
11371
11345
|
* `false` ends the whole pass — the only thing that returns it is an anchor
|
|
11372
11346
|
* that stopped agreeing, and every listing taken after that point is about a
|
|
@@ -11379,10 +11353,6 @@ var LedgerWalkReconciler = class {
|
|
|
11379
11353
|
return true;
|
|
11380
11354
|
}
|
|
11381
11355
|
const relDir = hourDirRelPath(claim.deviceId, claim.profile, claim.hourStartMs);
|
|
11382
|
-
for (const row of claim.rows) if (parseSegmentPath(row.path) === null || row.path !== `${relDir}/${fileName(row.path)}`) {
|
|
11383
|
-
this.skip(state, claim, "path-mismatch", row.path);
|
|
11384
|
-
return true;
|
|
11385
|
-
}
|
|
11386
11356
|
let listing;
|
|
11387
11357
|
try {
|
|
11388
11358
|
listing = await this.deps.listHour(state.location.root, relDir);
|
|
@@ -11394,36 +11364,55 @@ var LedgerWalkReconciler = class {
|
|
|
11394
11364
|
if (state.listings % 8 === 0) await this.deps.pace();
|
|
11395
11365
|
state.hoursWalked += 1;
|
|
11396
11366
|
tally.hoursWalked += 1;
|
|
11397
|
-
const
|
|
11398
|
-
if (
|
|
11367
|
+
const missingDirectory = listing === null;
|
|
11368
|
+
if (missingDirectory) {
|
|
11399
11369
|
state.hoursMissing += 1;
|
|
11400
11370
|
tally.hoursMissing += 1;
|
|
11401
11371
|
}
|
|
11402
|
-
|
|
11403
|
-
|
|
11404
|
-
|
|
11405
|
-
|
|
11372
|
+
const present = [];
|
|
11373
|
+
for (const name of listing ?? []) {
|
|
11374
|
+
if (!name.endsWith(".m4s")) continue;
|
|
11375
|
+
const relPath = `${relDir}/${name}`;
|
|
11376
|
+
const parsed = parseSegmentPath(relPath);
|
|
11377
|
+
if (parsed === null || hourStartMs(parsed.startMs) !== claim.hourStartMs) {
|
|
11378
|
+
this.skip(state, claim, "path-mismatch", relPath);
|
|
11379
|
+
return true;
|
|
11380
|
+
}
|
|
11381
|
+
present.push(relPath);
|
|
11382
|
+
}
|
|
11383
|
+
const listed = hourAggregate(present);
|
|
11384
|
+
const orphans = Math.max(0, listed.segments - claim.segments);
|
|
11385
|
+
if (orphans > 0) {
|
|
11386
|
+
state.orphanFiles += orphans;
|
|
11387
|
+
tally.orphanFiles += orphans;
|
|
11388
|
+
for (const path of present) {
|
|
11406
11389
|
if (state.orphanSample.length >= 50) break;
|
|
11407
|
-
state.orphanSample.push(
|
|
11390
|
+
state.orphanSample.push(path);
|
|
11408
11391
|
}
|
|
11409
|
-
this.deps.logger.warn("recorder:
|
|
11392
|
+
this.deps.logger.warn("recorder: an hour directory holds more segments than its durable row accounts for", {
|
|
11410
11393
|
tags: { deviceId: claim.deviceId },
|
|
11411
11394
|
meta: {
|
|
11412
11395
|
locationId: state.location.id,
|
|
11413
11396
|
profile: claim.profile,
|
|
11414
11397
|
hourStartMs: claim.hourStartMs,
|
|
11415
|
-
|
|
11416
|
-
|
|
11398
|
+
accounted: claim.segments,
|
|
11399
|
+
listed: listed.segments,
|
|
11400
|
+
apply: state.apply
|
|
11417
11401
|
}
|
|
11418
11402
|
});
|
|
11419
11403
|
}
|
|
11420
|
-
|
|
11421
|
-
|
|
11422
|
-
|
|
11404
|
+
const ghosts = Math.max(0, claim.segments - listed.segments);
|
|
11405
|
+
if (ghosts === 0) {
|
|
11406
|
+
if (state.apply && listed.segments > 0 && listed.bytes !== claim.bytes) await state.ledger.adoptWalkedHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs, present);
|
|
11407
|
+
if (state.apply) this.resetIndex(state, claim, present);
|
|
11408
|
+
return true;
|
|
11409
|
+
}
|
|
11410
|
+
const ghostBytes = Math.max(0, claim.bytes - listed.bytes);
|
|
11411
|
+
state.ghostSegments += ghosts;
|
|
11423
11412
|
state.ghostBytes += ghostBytes;
|
|
11424
|
-
tally.ghostSegments +=
|
|
11413
|
+
tally.ghostSegments += ghosts;
|
|
11425
11414
|
tally.ghostBytes += ghostBytes;
|
|
11426
|
-
if (
|
|
11415
|
+
if (listed.segments === 0) state.ghostHoursWhole += 1;
|
|
11427
11416
|
const anchor = await this.readAnchor(state.location);
|
|
11428
11417
|
if (!anchor.ok || anchor.volumeId !== state.anchor) {
|
|
11429
11418
|
state.refused = anchor.ok ? "anchor-moved" : anchor.refusal;
|
|
@@ -11439,56 +11428,52 @@ var LedgerWalkReconciler = class {
|
|
|
11439
11428
|
});
|
|
11440
11429
|
return false;
|
|
11441
11430
|
}
|
|
11442
|
-
this.deps.logger.warn("recorder: durable hour row
|
|
11431
|
+
this.deps.logger.warn("recorder: durable hour row accounts for segments the directory does not hold", {
|
|
11443
11432
|
tags: { deviceId: claim.deviceId },
|
|
11444
11433
|
meta: {
|
|
11445
11434
|
locationId: state.location.id,
|
|
11446
11435
|
profile: claim.profile,
|
|
11447
11436
|
hourStartMs: claim.hourStartMs,
|
|
11448
|
-
|
|
11449
|
-
|
|
11437
|
+
accounted: claim.segments,
|
|
11438
|
+
listed: listed.segments,
|
|
11439
|
+
ghosts,
|
|
11450
11440
|
ghostBytes,
|
|
11451
|
-
wholeHour:
|
|
11452
|
-
missingDirectory
|
|
11441
|
+
wholeHour: listed.segments === 0,
|
|
11442
|
+
missingDirectory,
|
|
11453
11443
|
apply: state.apply
|
|
11454
11444
|
}
|
|
11455
11445
|
});
|
|
11456
11446
|
if (!state.apply) return true;
|
|
11457
|
-
await state.ledger.
|
|
11458
|
-
if (
|
|
11459
|
-
this.skip(state, claim, "durable-refused", "the ledger still
|
|
11447
|
+
const forgotten = listed.segments === 0 ? await state.ledger.forgetHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs) : await this.adoptLanded(state, claim, present, ghosts);
|
|
11448
|
+
if (forgotten === null) {
|
|
11449
|
+
this.skip(state, claim, "durable-refused", "the ledger still accounts for the ghosts");
|
|
11460
11450
|
return true;
|
|
11461
11451
|
}
|
|
11462
|
-
this.
|
|
11463
|
-
state.forgottenSegments +=
|
|
11452
|
+
this.resetIndex(state, claim, present);
|
|
11453
|
+
state.forgottenSegments += forgotten;
|
|
11464
11454
|
state.forgottenBytes += ghostBytes;
|
|
11465
|
-
tally.forgottenSegments +=
|
|
11455
|
+
tally.forgottenSegments += forgotten;
|
|
11466
11456
|
return true;
|
|
11467
11457
|
}
|
|
11468
|
-
/** Did the durable drop actually land? */
|
|
11469
|
-
async forgetLanded(state, claim, ghosts) {
|
|
11470
|
-
const { deviceId, profile, hourStartMs: hour } = claim;
|
|
11471
|
-
if (!state.ledger.knowsHour(deviceId, profile, state.location.id, hour)) return true;
|
|
11472
|
-
const held = await state.ledger.hourPaths(deviceId, profile, state.location.id, hour);
|
|
11473
|
-
if (held === null) return false;
|
|
11474
|
-
const remaining = new Set(held);
|
|
11475
|
-
return ghosts.every((row) => !remaining.has(row.path));
|
|
11476
|
-
}
|
|
11477
11458
|
/**
|
|
11478
|
-
*
|
|
11459
|
+
* Adopt the listing, then READ THE ROW BACK.
|
|
11479
11460
|
*
|
|
11480
|
-
*
|
|
11481
|
-
*
|
|
11482
|
-
*
|
|
11461
|
+
* `adoptWalkedHour` refuses a listing that would erase more than half of what
|
|
11462
|
+
* the row accounts for — the guard against a root that answered short — and it
|
|
11463
|
+
* refuses silently, which a caller that counted it as done would turn into
|
|
11464
|
+
* telling the resident index a row is gone that the archive still holds.
|
|
11465
|
+
* `null` is that refusal.
|
|
11483
11466
|
*/
|
|
11484
|
-
|
|
11485
|
-
|
|
11486
|
-
const
|
|
11487
|
-
return
|
|
11488
|
-
|
|
11489
|
-
|
|
11490
|
-
|
|
11491
|
-
|
|
11467
|
+
async adoptLanded(state, claim, present, ghosts) {
|
|
11468
|
+
await state.ledger.adoptWalkedHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs, present);
|
|
11469
|
+
const after = state.ledger.hourRow(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs);
|
|
11470
|
+
if (after === void 0) return claim.segments;
|
|
11471
|
+
return after.segments === claim.segments - ghosts ? ghosts : null;
|
|
11472
|
+
}
|
|
11473
|
+
/** Make the resident index's view of the hour the listing's, and say how many
|
|
11474
|
+
* index rows that dropped. */
|
|
11475
|
+
resetIndex(state, claim, present) {
|
|
11476
|
+
return this.deps.applyToIndex(claim.deviceId, state.location.id, claim.hourStartMs, present);
|
|
11492
11477
|
}
|
|
11493
11478
|
/** The volume anchor, or the refusal its absence produces. */
|
|
11494
11479
|
async readAnchor(location) {
|
|
@@ -17777,12 +17762,17 @@ var GHOST_LOCATION_PROBE_PAUSE_MS = 50;
|
|
|
17777
17762
|
/**
|
|
17778
17763
|
* Awaited every `WALK_HOUR_BATCH` hour listings by the ledger↔disk walk (D319).
|
|
17779
17764
|
*
|
|
17780
|
-
* Twice the ghost sweep's brake
|
|
17781
|
-
* heavier
|
|
17782
|
-
* `readdir`
|
|
17783
|
-
*
|
|
17784
|
-
*
|
|
17785
|
-
*
|
|
17765
|
+
* Twice the ghost sweep's brake. That used to be because the syscall between
|
|
17766
|
+
* brakes was far heavier — the sweep paced one `storage.exists`, this one an
|
|
17767
|
+
* hour `readdir` "measured at ~650 ms on the live share under load". Both
|
|
17768
|
+
* halves of that have changed: the sweep paces a directory listing too since
|
|
17769
|
+
* D523, and the 650 ms belonged to a FUSE mount that has been retired (an hour
|
|
17770
|
+
* of 345–349 files on `/recordings-2` lists in ~1 ms).
|
|
17771
|
+
*
|
|
17772
|
+
* It stays at twice anyway, for the reason that never depended on the number:
|
|
17773
|
+
* the walk is operator-armed and one-off, so the right trade is to be slower
|
|
17774
|
+
* and invisible rather than fast and felt — a `find` over this array hung an
|
|
17775
|
+
* SSH session on 2026-08-29.
|
|
17786
17776
|
*/
|
|
17787
17777
|
var LEDGER_WALK_PAUSE_MS = 100;
|
|
17788
17778
|
/**
|
|
@@ -18062,6 +18052,11 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18062
18052
|
}
|
|
18063
18053
|
async onInitialize() {
|
|
18064
18054
|
this.stopStallMonitor = startEventLoopStallMonitor(this.ctx.logger);
|
|
18055
|
+
this.index.armResidency({
|
|
18056
|
+
logger: this.ctx.logger,
|
|
18057
|
+
now: () => Date.now(),
|
|
18058
|
+
budget: DEFAULT_RECORDING_INDEX_BUDGET
|
|
18059
|
+
});
|
|
18065
18060
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
18066
18061
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
18067
18062
|
const configuredNode = this.config.recordingNodeId;
|
|
@@ -18079,6 +18074,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18079
18074
|
this.segmentHours = new SegmentHourLedger({
|
|
18080
18075
|
store: this.ctx.api.settingsStore,
|
|
18081
18076
|
logger: this.ctx.logger,
|
|
18077
|
+
listHour: (locationId, deviceId, profile, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18082
18078
|
isLocationUsable: (locationId) => this.placement?.isLocationUsable(locationId) ?? true
|
|
18083
18079
|
});
|
|
18084
18080
|
try {
|
|
@@ -18112,7 +18108,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18112
18108
|
await promises.unlink(fromAbs);
|
|
18113
18109
|
}
|
|
18114
18110
|
},
|
|
18115
|
-
|
|
18111
|
+
hourDirectoryPaths: (deviceId, profile, locationId, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18116
18112
|
removeDir: async (locationId, relDir) => {
|
|
18117
18113
|
const loc = requireReadableLocation(this.resolvedLocations, locationId, "recorder");
|
|
18118
18114
|
await promises.rm(path.join(loc.root, relDir), {
|
|
@@ -18597,8 +18593,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18597
18593
|
localNodeId: this.nodeId,
|
|
18598
18594
|
index: this.index,
|
|
18599
18595
|
ledger: this.segmentHours === null ? null : {
|
|
18600
|
-
|
|
18601
|
-
|
|
18596
|
+
forgetScope: (scope) => this.segmentHours.forgetScope(scope),
|
|
18597
|
+
snapshot: () => this.segmentHours.snapshot()
|
|
18602
18598
|
},
|
|
18603
18599
|
now: () => Date.now()
|
|
18604
18600
|
});
|
|
@@ -18705,9 +18701,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18705
18701
|
}
|
|
18706
18702
|
},
|
|
18707
18703
|
anchorOf: async (root) => (await readLocationMarker(root))?.volumeId ?? null,
|
|
18708
|
-
applyToIndex: (paths) =>
|
|
18709
|
-
this.index.removeSegments(paths);
|
|
18710
|
-
},
|
|
18704
|
+
applyToIndex: (deviceId, locationId, hour, paths) => this.index.resetHour(deviceId, locationId, hour, paths),
|
|
18711
18705
|
logger: this.ctx.logger,
|
|
18712
18706
|
now: () => Date.now(),
|
|
18713
18707
|
pace: () => sleep$1(LEDGER_WALK_PAUSE_MS)
|
|
@@ -18835,7 +18829,6 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18835
18829
|
});
|
|
18836
18830
|
try {
|
|
18837
18831
|
await this.segmentHours?.load();
|
|
18838
|
-
this.segmentHours?.repairAggregates();
|
|
18839
18832
|
} catch (err) {
|
|
18840
18833
|
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: errMsg(err) } });
|
|
18841
18834
|
}
|
|
@@ -18878,18 +18871,15 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18878
18871
|
const sweeper = new GhostLocationSweeper({
|
|
18879
18872
|
ledger,
|
|
18880
18873
|
locations: () => this.resolvedLocations,
|
|
18881
|
-
|
|
18882
|
-
location,
|
|
18883
|
-
relativePath
|
|
18884
|
-
}),
|
|
18874
|
+
listHour: (locationId, deviceId, profile, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18885
18875
|
applyToIndex: (rows, toLocationId) => {
|
|
18886
18876
|
this.index.removeSegments(rows.map((row) => row.path));
|
|
18887
|
-
if (toLocationId === null) return;
|
|
18888
18877
|
for (const row of rows) this.index.addSegment({
|
|
18889
18878
|
...row,
|
|
18890
18879
|
locationId: toLocationId
|
|
18891
18880
|
});
|
|
18892
18881
|
},
|
|
18882
|
+
forgetInIndex: (deviceId, locationId, hourStartMs) => this.index.resetHour(deviceId, locationId, hourStartMs, []),
|
|
18893
18883
|
logger: this.ctx.logger,
|
|
18894
18884
|
now: () => Date.now(),
|
|
18895
18885
|
pace: () => sleep$1(GHOST_LOCATION_PROBE_PAUSE_MS)
|
|
@@ -19062,9 +19052,16 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19062
19052
|
const census = this.index.sortedViewCensus();
|
|
19063
19053
|
const views = this.index.sortedViewStats();
|
|
19064
19054
|
const reuse = this.index.hydrateReuseStats();
|
|
19055
|
+
const residency = this.index.residency();
|
|
19065
19056
|
this.ctx.logger.info("recorder: index census", {
|
|
19066
19057
|
...census.largestDeviceId > 0 ? { tags: { deviceId: census.largestDeviceId } } : {},
|
|
19067
19058
|
meta: {
|
|
19059
|
+
residentRows: residency.residentRows,
|
|
19060
|
+
residentHours: residency.residentHours,
|
|
19061
|
+
rowBudget: residency.budget?.residentRowBudget ?? null,
|
|
19062
|
+
evictedRows: residency.evictedRows,
|
|
19063
|
+
evictedHours: residency.evictedHours,
|
|
19064
|
+
evictionPasses: residency.evictionPasses,
|
|
19068
19065
|
sortedViews: census.views,
|
|
19069
19066
|
rowSlots: census.rowSlots,
|
|
19070
19067
|
largestViewRows: census.largestRows,
|
|
@@ -19164,6 +19161,63 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19164
19161
|
return this.resolvedLocations;
|
|
19165
19162
|
}
|
|
19166
19163
|
/**
|
|
19164
|
+
* ONE hour bucket's segment files, location-relative — the recorder's single
|
|
19165
|
+
* implementation of {@link HourLister} (D523).
|
|
19166
|
+
*
|
|
19167
|
+
* The directory is the authority for WHICH segments exist, and the segment
|
|
19168
|
+
* NAME carries start, duration and bytes, so this is a complete description
|
|
19169
|
+
* of the hour and no file is ever opened.
|
|
19170
|
+
*
|
|
19171
|
+
* Three answers, and the difference between the last two is the whole point:
|
|
19172
|
+
*
|
|
19173
|
+
* - `null` — **undecided.** The location does not resolve, may not be read,
|
|
19174
|
+
* or the read failed for a reason that is not "this directory is not
|
|
19175
|
+
* there". Every caller turns it into "this hour takes no part in this
|
|
19176
|
+
* pass".
|
|
19177
|
+
* - `[]` — **the hour holds nothing**, and the volume confirmed it: an
|
|
19178
|
+
* ENOENT on the hour directory is only reported as empty once the
|
|
19179
|
+
* location's ROOT has answered a read. A root that has gone away answers
|
|
19180
|
+
* ENOENT for every hour on it, and reporting that as "empty" is how a pass
|
|
19181
|
+
* deletes an operator's index of real footage (D148, D393).
|
|
19182
|
+
* - the paths.
|
|
19183
|
+
*/
|
|
19184
|
+
async listHourDirectory(locationId, deviceId, profile, hourStartMs) {
|
|
19185
|
+
const location = this.resolvedLocations.find((l) => l.id === locationId);
|
|
19186
|
+
if (location === void 0 || !locationMayBeRead(location)) return null;
|
|
19187
|
+
const rel = hourDirRelPath(deviceId, profile, hourStartMs);
|
|
19188
|
+
try {
|
|
19189
|
+
return (await promises.readdir(path.join(location.root, rel))).filter((f) => f.endsWith(".m4s")).map((f) => `${rel}/${f}`);
|
|
19190
|
+
} catch (err) {
|
|
19191
|
+
if (err.code !== "ENOENT") {
|
|
19192
|
+
this.ctx.logger.warn("recorder: hour directory read failed — this hour decides nothing", {
|
|
19193
|
+
tags: { deviceId },
|
|
19194
|
+
meta: {
|
|
19195
|
+
profile,
|
|
19196
|
+
locationId,
|
|
19197
|
+
hourStartMs,
|
|
19198
|
+
error: errMsg(err)
|
|
19199
|
+
}
|
|
19200
|
+
});
|
|
19201
|
+
return null;
|
|
19202
|
+
}
|
|
19203
|
+
try {
|
|
19204
|
+
await promises.readdir(location.root);
|
|
19205
|
+
} catch {
|
|
19206
|
+
this.ctx.logger.warn("recorder: hour directory is absent AND its location root cannot be read — this hour decides nothing", {
|
|
19207
|
+
tags: { deviceId },
|
|
19208
|
+
meta: {
|
|
19209
|
+
profile,
|
|
19210
|
+
locationId,
|
|
19211
|
+
root: location.root,
|
|
19212
|
+
hourStartMs
|
|
19213
|
+
}
|
|
19214
|
+
});
|
|
19215
|
+
return null;
|
|
19216
|
+
}
|
|
19217
|
+
return [];
|
|
19218
|
+
}
|
|
19219
|
+
}
|
|
19220
|
+
/**
|
|
19167
19221
|
* Attach-boundary placement for one (device, profile). Falls back to the pure
|
|
19168
19222
|
* resolver when the service is not built (non-recording node / early boot),
|
|
19169
19223
|
* which is byte-identical to the pre-placement behaviour.
|