@camstack/addon-pipeline 1.2.256 → 1.2.257
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/{default-detection-model-A5DRjEr3.mjs → default-detection-model-BTzxxm-z.mjs} +1 -1
- package/dist/{default-detection-model-Bbw4nC11.js → default-detection-model-C8CHpxQl.js} +1 -1
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +4 -4
- package/dist/{dist-lqqUXDfs.js → dist-BMARLLlX.js} +27 -6
- package/dist/{dist-MJ4ji7um.mjs → dist-xsaGDCOu.mjs} +27 -6
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-80-uOMr8.js → node-Ca-vE0xX.js} +1 -1
- package/dist/{node-DdXmHLK4.mjs → node-Fv4ByT36.mjs} +1 -1
- package/dist/pipeline-runner/index.js +3 -3
- package/dist/pipeline-runner/index.mjs +3 -3
- package/dist/{process-memory-Cl3ubMxY.mjs → process-memory-BgtQADAY.mjs} +1 -1
- package/dist/{process-memory-Dac9TXlX.js → process-memory-C3Wjmvm0.js} +1 -1
- package/dist/recorder/index.js +778 -998
- package/dist/recorder/index.mjs +778 -998
- package/dist/{segment-demux-js-gIj82O9k.mjs → segment-demux-js-C7C6c-g2.mjs} +1 -1
- package/dist/{segment-demux-js-CWwlKQ3M.js → segment-demux-js-DHV7EOQf.js} +1 -1
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-pOLAb3B1.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CJWLoj3Z.mjs} +3 -3
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CPFjd94C.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-ReCGmeOG.mjs} +1 -1
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BXK7jbJm.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BbnIk-9T.mjs} +1 -1
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-BRnsXyTM.mjs → hostInit-BzBcvawL.mjs} +3 -3
- package/dist/stream-broker/index.js +3 -3
- package/dist/stream-broker/index.mjs +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { $t as hydrateSchema, C as OpsLogEntrySchema, Ct as recordingExportCapability, D as RECORDING_EXPORT_MAX_READ_BYTES, Dt as resolveLocationMode, H as deriveBatteryPresence, I as batteryCapability, Kt as DeviceFeature, Lt as errMsg, Mt as storageOccupancyCapability, O as RecordingConfigSchema, Pt as stripRetiredBandPreBufferSec, Q as evictionPolicyOfLocation, St as recordingCapability, W as deriveRecordingMode, Wt as BaseAddon, Xt as customAction, Y as evaluateSensorEdge, Z as evictionPolicyForMode, Zt as defineCustomActions, _n as record, bn as EventCategory, cn as _enum, ct as mayReadLocation, d as DEFAULT_EVENTS_BAND_BUFFER_SEC, dt as modeMayWrite, et as gbToBytes, g as ExportRecordSchema, gn as preprocess, hn as object, i as BatteryStatusSchema, it as loadContributionCapability, jt as storageEvictableCapability, k as RecordingSignalStatusSchema, kt as resolveRecordingProfiles, ln as array, lt as mayWriteToLocation, mn as number, on as selectAssignedProfileSlots, qt as DeviceType, rn as nodePin, rt as isFirstLevelMacroClass, sn as sleep$1, un as boolean, ut as migrateLegacyDeviceSignalConfig, vn as string, wt as recordingSignalCapability } from "../dist-
|
|
2
|
-
import { a as physicalRootOf, i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-
|
|
1
|
+
import { $t as hydrateSchema, C as OpsLogEntrySchema, Ct as recordingExportCapability, D as RECORDING_EXPORT_MAX_READ_BYTES, Dt as resolveLocationMode, H as deriveBatteryPresence, I as batteryCapability, Kt as DeviceFeature, Lt as errMsg, Mt as storageOccupancyCapability, O as RecordingConfigSchema, Pt as stripRetiredBandPreBufferSec, Q as evictionPolicyOfLocation, St as recordingCapability, W as deriveRecordingMode, Wt as BaseAddon, Xt as customAction, Y as evaluateSensorEdge, Z as evictionPolicyForMode, Zt as defineCustomActions, _n as record, bn as EventCategory, cn as _enum, ct as mayReadLocation, d as DEFAULT_EVENTS_BAND_BUFFER_SEC, dt as modeMayWrite, et as gbToBytes, g as ExportRecordSchema, gn as preprocess, hn as object, i as BatteryStatusSchema, it as loadContributionCapability, jt as storageEvictableCapability, k as RecordingSignalStatusSchema, kt as resolveRecordingProfiles, ln as array, lt as mayWriteToLocation, mn as number, on as selectAssignedProfileSlots, qt as DeviceType, rn as nodePin, rt as isFirstLevelMacroClass, sn as sleep$1, un as boolean, ut as migrateLegacyDeviceSignalConfig, vn as string, wt as recordingSignalCapability } from "../dist-xsaGDCOu.mjs";
|
|
2
|
+
import { a as physicalRootOf, i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-Fv4ByT36.mjs";
|
|
3
3
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
4
4
|
import { c as PREBUFFER_MAX_MS, r as RECORDING_PRE_ROLL_REQUEST_MS, s as withRecordingIntent } from "../restream-intent-D2VT97-U.mjs";
|
|
5
5
|
import { n as RowMapStore, t as retireRootKeys } from "../retire-root-keys-DZvlso2l.mjs";
|
|
@@ -9,7 +9,6 @@ import path from "node:path";
|
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
10
|
import { open } from "node:fs/promises";
|
|
11
11
|
import { contentTypeFor, createFileDataPlaneHandler, parseRangeHeader, startEventLoopStallMonitor } from "@camstack/system/addon-utils";
|
|
12
|
-
import { gunzipSync, gzipSync } from "node:zlib";
|
|
13
12
|
//#region src/recorder/event-capture.ts
|
|
14
13
|
/**
|
|
15
14
|
* La sottoscrizione motion del recorder: bus → `onTrigger(deviceId, atMs)`.
|
|
@@ -1118,6 +1117,37 @@ var RecordingIndex = class {
|
|
|
1118
1117
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$9);
|
|
1119
1118
|
}
|
|
1120
1119
|
/**
|
|
1120
|
+
* Make one hour of ONE location exactly `relPaths` — merge what is there and
|
|
1121
|
+
* drop what is not.
|
|
1122
|
+
*
|
|
1123
|
+
* {@link hydrateHour} merges, which is right for a reader: a listing it took
|
|
1124
|
+
* of a live hour is a lower bound, and dropping on it would delete the index
|
|
1125
|
+
* of footage the readdir merely raced. This is for the one caller that has
|
|
1126
|
+
* TRUSTED evidence the hour is over and the volume it listed is the volume it
|
|
1127
|
+
* started on — the operator-armed ledger walk — where the directory is the
|
|
1128
|
+
* whole truth about the hour and an index row naming a file that is not in it
|
|
1129
|
+
* is a ghost (D523, D148's "trusted walk" arm).
|
|
1130
|
+
*
|
|
1131
|
+
* Scoped to the (device, location, hour) triple: a path filed under another
|
|
1132
|
+
* hour, or held for another location, is none of this call's business.
|
|
1133
|
+
* @residency mutation
|
|
1134
|
+
*/
|
|
1135
|
+
resetHour(deviceId, locationId, hourStartMs, relPaths) {
|
|
1136
|
+
const m = this.mapFor(deviceId);
|
|
1137
|
+
const keep = new Set(relPaths);
|
|
1138
|
+
const dropped = [];
|
|
1139
|
+
for (const [path, row] of m) {
|
|
1140
|
+
if (row.locationId !== locationId) continue;
|
|
1141
|
+
if (Math.floor(row.startMs / HOUR_MS$9) * HOUR_MS$9 !== hourStartMs) continue;
|
|
1142
|
+
if (keep.has(path)) continue;
|
|
1143
|
+
dropped.push(path);
|
|
1144
|
+
}
|
|
1145
|
+
for (const path of dropped) m.delete(path);
|
|
1146
|
+
if (dropped.length > 0) this.invalidateSorted(deviceId);
|
|
1147
|
+
this.hydrateHour(deviceId, locationId, hourStartMs, relPaths);
|
|
1148
|
+
return dropped;
|
|
1149
|
+
}
|
|
1150
|
+
/**
|
|
1121
1151
|
* Insert/replace one finalized segment (idempotent by path).
|
|
1122
1152
|
*
|
|
1123
1153
|
* THE hot mutation: every profile-writer lands here every `segmentSeconds`,
|
|
@@ -1766,40 +1796,6 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1766
1796
|
const verify = options?.verifyClosedHours === true;
|
|
1767
1797
|
if (!verify && index.hydrationOf(deviceId, fromMs, toMs) === "hydrated") return;
|
|
1768
1798
|
const liveHour = Math.floor(Date.now() / HOUR_MS$8) * HOUR_MS$8;
|
|
1769
|
-
/** Per (locationId, profile, hour): what the LEDGER claimed, for the diff. */
|
|
1770
|
-
const claimed = /* @__PURE__ */ new Map();
|
|
1771
|
-
const claimKey = (locationId, hour) => `${locationId}:${hour}`;
|
|
1772
|
-
/**
|
|
1773
|
-
* Closed hours the ledger answered for with a row nobody SEALED (D451): what
|
|
1774
|
-
* a writer had persisted when it was last heard from, which is a legitimate
|
|
1775
|
-
* answer and not a complete one. Each is read from the array once and the
|
|
1776
|
-
* row is unioned and sealed, so the second reader pays a point read.
|
|
1777
|
-
*/
|
|
1778
|
-
const unsealed = /* @__PURE__ */ new Set();
|
|
1779
|
-
if (ledger !== void 0) {
|
|
1780
|
-
const startedMs = Date.now();
|
|
1781
|
-
const hours = await ledger.detailForWindow(deviceId, fromMs, toMs);
|
|
1782
|
-
let segments = 0;
|
|
1783
|
-
let served = 0;
|
|
1784
|
-
for (const hour of hours) {
|
|
1785
|
-
if (hour.hourStartMs >= liveHour) continue;
|
|
1786
|
-
index.hydrateHour(deviceId, hour.locationId, hour.hourStartMs, hour.paths);
|
|
1787
|
-
claimed.set(claimKey(hour.locationId, hour.hourStartMs), hour.paths.length);
|
|
1788
|
-
if (!hour.sealed) unsealed.add(hour.hourStartMs);
|
|
1789
|
-
segments += hour.paths.length;
|
|
1790
|
-
served += 1;
|
|
1791
|
-
}
|
|
1792
|
-
if (served > 0) logger.info("recorder: hydrated a window from the hour ledger", {
|
|
1793
|
-
tags: { deviceId },
|
|
1794
|
-
meta: {
|
|
1795
|
-
hours: served,
|
|
1796
|
-
segments,
|
|
1797
|
-
unsealedHours: unsealed.size,
|
|
1798
|
-
ms: Date.now() - startedMs
|
|
1799
|
-
}
|
|
1800
|
-
});
|
|
1801
|
-
if (!verify && unsealed.size === 0 && index.hydrationOf(deviceId, fromMs, toMs) === "hydrated") return;
|
|
1802
|
-
}
|
|
1803
1799
|
for (const [root, aliases] of aliasesByRoot(locations)) {
|
|
1804
1800
|
const deviceDir = path.join(root, String(deviceId));
|
|
1805
1801
|
let profiles;
|
|
@@ -1823,7 +1819,7 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1823
1819
|
let forgottenHours = 0;
|
|
1824
1820
|
let sealed = 0;
|
|
1825
1821
|
for (let hour = first; hour < toMs; hour += HOUR_MS$8) {
|
|
1826
|
-
if (!verify && hour < liveHour &&
|
|
1822
|
+
if (!verify && hour < liveHour && index.hydrationOf(deviceId, hour, hour + HOUR_MS$8) === "hydrated") continue;
|
|
1827
1823
|
for (const profile of profiles) {
|
|
1828
1824
|
const rel = hourDirRelPath(deviceId, profile, hour);
|
|
1829
1825
|
let files;
|
|
@@ -1837,11 +1833,10 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
1837
1833
|
const relPaths = files.filter((f) => f.endsWith(".m4s")).map((f) => `${rel}/${f}`);
|
|
1838
1834
|
index.hydrateHour(deviceId, locationId, hour, relPaths);
|
|
1839
1835
|
if (ledger === void 0 || relPaths.length === 0 || hour + HOUR_MS$8 > Date.now()) continue;
|
|
1840
|
-
const claimedHere = claimed.get(claimKey(locationId, hour));
|
|
1841
1836
|
const outcome = await ledger.adoptWalkedHour(deviceId, profile, locationId, hour, relPaths);
|
|
1842
1837
|
if (outcome === null) continue;
|
|
1843
1838
|
if (outcome.created) repaired += 1;
|
|
1844
|
-
else if (
|
|
1839
|
+
else if (outcome.added > 0) {
|
|
1845
1840
|
forgottenSegments += outcome.added;
|
|
1846
1841
|
forgottenHours += 1;
|
|
1847
1842
|
}
|
|
@@ -2478,10 +2473,12 @@ var RelocateEngine = class {
|
|
|
2478
2473
|
*
|
|
2479
2474
|
* The order is the whole point. Between the copy and the delete both
|
|
2480
2475
|
* locations hold every byte, so the ledger re-point in the middle can only
|
|
2481
|
-
* ever
|
|
2482
|
-
* recoverable instead of a black window.
|
|
2483
|
-
*
|
|
2484
|
-
*
|
|
2476
|
+
* ever account for a location that HAS the files — which is what makes a
|
|
2477
|
+
* crash recoverable instead of a black window.
|
|
2478
|
+
*
|
|
2479
|
+
* There is no ledger refusal to handle any more (D523): the durable half is
|
|
2480
|
+
* arithmetic over a row the mirror already holds, so nothing on that path can
|
|
2481
|
+
* fail and stop an hour dead.
|
|
2485
2482
|
*/
|
|
2486
2483
|
async moveHour(job, rows, from, to, samePhysicalRoot, sleep, bytesPerMs) {
|
|
2487
2484
|
const movable = [];
|
|
@@ -2517,17 +2514,7 @@ var RelocateEngine = class {
|
|
|
2517
2514
|
toLocationId: to.id
|
|
2518
2515
|
}
|
|
2519
2516
|
});
|
|
2520
|
-
else
|
|
2521
|
-
this.deps.logger.warn("relocate: hour NOT moved — the durable ledger refused the re-point, so the source bytes are kept", {
|
|
2522
|
-
tags: { deviceId: movable[0].deviceId },
|
|
2523
|
-
meta: {
|
|
2524
|
-
jobId: job.jobId,
|
|
2525
|
-
files: movable.length,
|
|
2526
|
-
toLocationId: to.id
|
|
2527
|
-
}
|
|
2528
|
-
});
|
|
2529
|
-
return;
|
|
2530
|
-
}
|
|
2517
|
+
else await ledger.repointSegments(movable, to.id);
|
|
2531
2518
|
if (!samePhysicalRoot) for (const row of movable) await promises.rm(path.join(from.root, row.fromPath ?? row.path), { force: true });
|
|
2532
2519
|
for (const row of movable) {
|
|
2533
2520
|
this.deps.index.removeSegments([row.fromPath ?? row.path]);
|
|
@@ -2920,23 +2907,21 @@ var SegmentStore = class SegmentStore {
|
|
|
2920
2907
|
* work now the per-bucket archive scan is gone. */
|
|
2921
2908
|
static DEFAULT_EVICT_CHUNK = 25;
|
|
2922
2909
|
/**
|
|
2923
|
-
* True when `victims` cover EVERY segment the
|
|
2924
|
-
*
|
|
2910
|
+
* True when `victims` cover EVERY segment the bucket DIRECTORY holds, AND the
|
|
2911
|
+
* bucket's hour has fully elapsed.
|
|
2925
2912
|
*
|
|
2926
2913
|
* The current-hour guard matters because `onFinalized` relocates new segments
|
|
2927
2914
|
* INTO the current bucket concurrently — an rm racing that rename would
|
|
2928
2915
|
* orphan a just-indexed segment.
|
|
2929
2916
|
*
|
|
2930
|
-
* The cover test is
|
|
2931
|
-
* `RecordingIndex
|
|
2932
|
-
*
|
|
2933
|
-
*
|
|
2934
|
-
*
|
|
2935
|
-
*
|
|
2936
|
-
* went with every file the round never named, leaving their durable rows
|
|
2937
|
-
* behind as ghosts.
|
|
2917
|
+
* The cover test is a listing of the directory the `rm -rf` is about to take,
|
|
2918
|
+
* never `RecordingIndex` and never a durable row (D523). A bucketDir is
|
|
2919
|
+
* `<dev>/<profile>/Y/M/D/H`, so one listing answers exactly the question the
|
|
2920
|
+
* delete asks. Asked of the resident index it read EQUAL whenever the hour was
|
|
2921
|
+
* partly resident — both sides short by the same rows — and the directory went
|
|
2922
|
+
* with every file the round never named.
|
|
2938
2923
|
*
|
|
2939
|
-
* An unanswerable
|
|
2924
|
+
* An unanswerable listing returns `false`: per-file deletes are slower and
|
|
2940
2925
|
* always correct, and the alternative is a recursive delete computed from a
|
|
2941
2926
|
* fraction of the truth.
|
|
2942
2927
|
*/
|
|
@@ -2945,7 +2930,7 @@ var SegmentStore = class SegmentStore {
|
|
|
2945
2930
|
if (!first) return false;
|
|
2946
2931
|
const bucketStartMs = Math.floor(first.startMs / SegmentStore.HOUR_MS) * SegmentStore.HOUR_MS;
|
|
2947
2932
|
if ((this.deps.now?.() ?? Date.now()) < bucketStartMs + SegmentStore.HOUR_MS) return false;
|
|
2948
|
-
const ask = this.deps.
|
|
2933
|
+
const ask = this.deps.hourDirectoryPaths;
|
|
2949
2934
|
if (ask === void 0) return false;
|
|
2950
2935
|
const held = await ask(first.deviceId, first.profile, location, bucketStartMs);
|
|
2951
2936
|
if (held === null || held.length === 0) return false;
|
|
@@ -4078,8 +4063,9 @@ var CalendarIndex = class {
|
|
|
4078
4063
|
*
|
|
4079
4064
|
* Also used to warm the days ADJACENT to one being viewed — changing day is a
|
|
4080
4065
|
* single click and the operator expects the summary to be there. That warm
|
|
4081
|
-
* belongs in the background:
|
|
4082
|
-
*
|
|
4066
|
+
* belongs in the background: a cold read costs whatever the array charges
|
|
4067
|
+
* that day (see the header — the ~650 ms figure is from the retired FUSE
|
|
4068
|
+
* share), and doing it inline would only move the wait onto the click.
|
|
4083
4069
|
*/
|
|
4084
4070
|
async warm(deviceId, fromMs, toMs) {
|
|
4085
4071
|
for (let day = utcDayOf(fromMs); day <= utcDayOf(toMs); day += DAY_MS$3) {
|
|
@@ -5200,22 +5186,18 @@ function createFootageDiscardJob(deps) {
|
|
|
5200
5186
|
const forgetLedger = async (location) => {
|
|
5201
5187
|
const ledger = deps.ledger;
|
|
5202
5188
|
if (ledger === null) return 0;
|
|
5203
|
-
let forgotten = 0;
|
|
5204
5189
|
const profiles = profilesForType(location.type);
|
|
5205
|
-
|
|
5190
|
+
const scope = {
|
|
5206
5191
|
locationId: location.id,
|
|
5207
5192
|
profiles
|
|
5208
|
-
}
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
});
|
|
5217
|
-
await sleep(0);
|
|
5218
|
-
}
|
|
5193
|
+
};
|
|
5194
|
+
const hours = ledger.snapshot().filter((row) => row.locationId === location.id && profiles.includes(row.profile));
|
|
5195
|
+
const forgotten = await ledger.forgetScope(scope);
|
|
5196
|
+
for (const hour of hours) deps.index.resetHour(hour.deviceId, hour.locationId, hour.hourStartMs, []);
|
|
5197
|
+
report({
|
|
5198
|
+
dbRowsDeleted: forgotten,
|
|
5199
|
+
detail: `forgot ${forgotten} ledger segments`
|
|
5200
|
+
});
|
|
5219
5201
|
return forgotten;
|
|
5220
5202
|
};
|
|
5221
5203
|
return {
|
|
@@ -8268,63 +8250,6 @@ function runFfmpeg(renderDeps, args, ext) {
|
|
|
8268
8250
|
});
|
|
8269
8251
|
}
|
|
8270
8252
|
//#endregion
|
|
8271
|
-
//#region src/recorder/addon/paths-codec.ts
|
|
8272
|
-
/**
|
|
8273
|
-
* How an hour row's `paths` becomes text, and back (D471).
|
|
8274
|
-
*
|
|
8275
|
-
* `recorder:segment-hours` is 66 MB of the settings database and **99.2 % of it
|
|
8276
|
-
* is this one column** — 15 643 bytes per row over 4 036 rows. The content is a
|
|
8277
|
-
* list of location-relative segment paths that nearly all share the same
|
|
8278
|
-
* directory prefix, repeated in full on every entry. Measured per column on a
|
|
8279
|
-
* copy of the live database, never the live file (D468).
|
|
8280
|
-
*
|
|
8281
|
-
* Compressing it is unusually safe here because of D287's two-tier mirror: boot
|
|
8282
|
-
* loads the aggregate half of every row and NOT `paths`, so this column is read
|
|
8283
|
-
* only on a point materialise, never on the hot path.
|
|
8284
|
-
*
|
|
8285
|
-
* A local copy of the same idea `pipeline-analytics` uses for `positions` —
|
|
8286
|
-
* addons never import each other (CLAUDE.md), so twenty duplicated lines are
|
|
8287
|
-
* the correct shape, not a shared dependency across an addon boundary.
|
|
8288
|
-
*/
|
|
8289
|
-
/** Marks a value this module wrote. A plain JSON array always starts `[`. */
|
|
8290
|
-
var ENCODED_PREFIX = "z1:";
|
|
8291
|
-
function encodePaths(paths) {
|
|
8292
|
-
return ENCODED_PREFIX + gzipSync(Buffer.from(JSON.stringify(paths), "utf8"), { level: 6 }).toString("base64");
|
|
8293
|
-
}
|
|
8294
|
-
/**
|
|
8295
|
-
* Decode whichever era wrote the value: the `z1:` string this writes, the plain
|
|
8296
|
-
* JSON string every pre-D471 row holds, or an array the store already parsed.
|
|
8297
|
-
*
|
|
8298
|
-
* `null` means "this row is unreadable", which the caller turns into a DISCARDED
|
|
8299
|
-
* row — the same contract the bare `JSON.parse` had. An hour whose paths cannot
|
|
8300
|
-
* be read must not silently become an hour with no segments: that would report
|
|
8301
|
-
* an empty disk, which is the failure `recordToRow` was already written to
|
|
8302
|
-
* avoid.
|
|
8303
|
-
*/
|
|
8304
|
-
function decodePaths(value) {
|
|
8305
|
-
if (Array.isArray(value)) return isPathList(value) ? value : null;
|
|
8306
|
-
if (typeof value !== "string") return null;
|
|
8307
|
-
const json = value.startsWith(ENCODED_PREFIX) ? tryGunzip(value.slice(3)) : value;
|
|
8308
|
-
if (json === null) return null;
|
|
8309
|
-
try {
|
|
8310
|
-
const parsed = JSON.parse(json);
|
|
8311
|
-
if (!Array.isArray(parsed)) return null;
|
|
8312
|
-
return isPathList(parsed) ? parsed : null;
|
|
8313
|
-
} catch {
|
|
8314
|
-
return null;
|
|
8315
|
-
}
|
|
8316
|
-
}
|
|
8317
|
-
function isPathList(v) {
|
|
8318
|
-
return v.every((p) => typeof p === "string" && p.length > 0);
|
|
8319
|
-
}
|
|
8320
|
-
function tryGunzip(b64) {
|
|
8321
|
-
try {
|
|
8322
|
-
return gunzipSync(Buffer.from(b64, "base64")).toString("utf8");
|
|
8323
|
-
} catch {
|
|
8324
|
-
return null;
|
|
8325
|
-
}
|
|
8326
|
-
}
|
|
8327
|
-
//#endregion
|
|
8328
8253
|
//#region src/durable/durable-ledger.ts
|
|
8329
8254
|
/** Default reseed cap — every current consumer's row set is installation-bounded. */
|
|
8330
8255
|
var DEFAULT_LOAD_LIMIT = 1e5;
|
|
@@ -8579,19 +8504,19 @@ var macrotask$1 = () => new Promise((resolve) => {
|
|
|
8579
8504
|
});
|
|
8580
8505
|
/**
|
|
8581
8506
|
* @durable class=mirror owner=recorder
|
|
8582
|
-
* write="write-behind on every finalized segment (hour-row upsert, serialised per hour); the writer SEALS the hour it leaves (one write per hour); a
|
|
8507
|
+
* write="write-behind on every finalized segment (hour-row upsert, serialised per hour); the writer SEALS the hour it leaves (one write per hour); a listing of a closed hour re-derives the aggregate and seals; eviction and relocation shrink it"
|
|
8583
8508
|
* retention="a row goes when the walk or eviction confirms the hour is empty — never by age, never from the ledger alone (D148)"
|
|
8584
8509
|
*/
|
|
8585
8510
|
var RECORDING_SEGMENT_HOURS_COLLECTION = "recorder:segment-hours";
|
|
8586
8511
|
/**
|
|
8587
|
-
* The four aggregate columns
|
|
8588
|
-
*
|
|
8589
|
-
*
|
|
8590
|
-
*
|
|
8591
|
-
*
|
|
8592
|
-
*
|
|
8593
|
-
*
|
|
8594
|
-
*
|
|
8512
|
+
* The four aggregate columns are what this table IS (D523).
|
|
8513
|
+
*
|
|
8514
|
+
* They exist so "how much footage is on this disk" is a `SUM`/`MIN`/`MAX` over
|
|
8515
|
+
* ~19 k hour rows instead of a walk over ~7.1 M in-RAM segment rows, and they
|
|
8516
|
+
* are derived from segment NAMES — by {@link withSegmentPath} and nothing else.
|
|
8517
|
+
* {@link hourAggregate} is that step over a whole listing and
|
|
8518
|
+
* {@link appendSegmentPath} is that step over the one path that just arrived,
|
|
8519
|
+
* so there is no second derivation to drift.
|
|
8595
8520
|
*/
|
|
8596
8521
|
var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
8597
8522
|
{
|
|
@@ -8620,11 +8545,6 @@ var RECORDING_SEGMENT_HOURS_COLUMNS = [
|
|
|
8620
8545
|
type: "INTEGER",
|
|
8621
8546
|
notNull: true
|
|
8622
8547
|
},
|
|
8623
|
-
{
|
|
8624
|
-
name: "paths",
|
|
8625
|
-
type: "TEXT",
|
|
8626
|
-
notNull: true
|
|
8627
|
-
},
|
|
8628
8548
|
{
|
|
8629
8549
|
name: "bytes",
|
|
8630
8550
|
type: "INTEGER",
|
|
@@ -8683,15 +8603,9 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
|
8683
8603
|
* `MAX(maxStartMs)` over `locationId IN (…)` — one row back, and with this
|
|
8684
8604
|
* index not one table row read to produce it (D447).
|
|
8685
8605
|
*
|
|
8686
|
-
* `
|
|
8687
|
-
*
|
|
8688
|
-
*
|
|
8689
|
-
* accounting read at the price of ~30 KB of table page per hour row. On the
|
|
8690
|
-
* live hub that was `durationMs=1001…2902` once a MINUTE on
|
|
8691
|
-
* `hub/sqlite-settings` (174 slow-call WARNs in ten hours on 2026-09-10/11),
|
|
8692
|
-
* every one of them holding the thread every configuration read in the
|
|
8693
|
-
* cluster waits on. The index is ~50 bytes a row over ~19 k rows — nothing —
|
|
8694
|
-
* and the first declare after it ships builds it in well under a second.
|
|
8606
|
+
* Since D523 dropped `paths` the table row is ten scalars, so the covering
|
|
8607
|
+
* index no longer saves ~30 KB of page per row; it still saves the row
|
|
8608
|
+
* fetch, and the accounting read is the one this hub runs on a schedule.
|
|
8695
8609
|
*/
|
|
8696
8610
|
{
|
|
8697
8611
|
name: "idx_recorder_segment_hours_loc_accounting",
|
|
@@ -8716,13 +8630,13 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [
|
|
|
8716
8630
|
* nominal `-segment_time` (ffmpeg cuts on the next GOP, never on the clock).
|
|
8717
8631
|
*
|
|
8718
8632
|
* It is a guess, and this is the shape of being wrong in each direction.
|
|
8719
|
-
* TIGHTER: a normal rollover stops sealing
|
|
8720
|
-
* once by
|
|
8721
|
-
*
|
|
8722
|
-
*
|
|
8723
|
-
*
|
|
8724
|
-
*
|
|
8725
|
-
*
|
|
8633
|
+
* TIGHTER: a normal rollover stops sealing and every hour of the archive is
|
|
8634
|
+
* listed once by the adoption sweep — at ~1 ms per hour directory on this hub's
|
|
8635
|
+
* SATA SSD, affordable but pointless. LOOSER: a runner respawn inside the
|
|
8636
|
+
* window seals an hour it entered late, which is exactly the 2026-09-17 defect;
|
|
8637
|
+
* the observed hole there was ~29 minutes, so anything under a few minutes
|
|
8638
|
+
* would have caught it and 90 s is the smallest number that does not fire on a
|
|
8639
|
+
* legitimate rollover.
|
|
8726
8640
|
*/
|
|
8727
8641
|
var WRITER_HOUR_ENTRY_GRACE_MS = 9e4;
|
|
8728
8642
|
/** The identity of the {@link hourAggregate} fold — an hour holding nothing. */
|
|
@@ -8736,12 +8650,11 @@ var EMPTY_HOUR_AGGREGATE = {
|
|
|
8736
8650
|
* ONE path folded into an aggregate — the whole derivation, expressed as its
|
|
8737
8651
|
* single step.
|
|
8738
8652
|
*
|
|
8739
|
-
* {@link hourAggregate} is this step over
|
|
8653
|
+
* {@link hourAggregate} is this step over a whole listing, and {@link
|
|
8740
8654
|
* SegmentHourLedger.recordSegment} is this step over the one path that just
|
|
8741
8655
|
* arrived. That is not two implementations that happen to agree: the full
|
|
8742
8656
|
* derivation is *defined* as this function repeated, so an append is the fold
|
|
8743
|
-
* resumed from the state the held row already carries
|
|
8744
|
-
* aggregate half IS the fold state over its `paths`.
|
|
8657
|
+
* resumed from the state the held row already carries.
|
|
8745
8658
|
*
|
|
8746
8659
|
* Why it had to become a step: `recordSegment` re-derived the whole hour on
|
|
8747
8660
|
* every finalize. Σ1..360 ≈ 65 k path parses per camera-profile-hour at 10 s
|
|
@@ -8764,49 +8677,57 @@ function withSegmentPath(agg, path) {
|
|
|
8764
8677
|
};
|
|
8765
8678
|
}
|
|
8766
8679
|
/**
|
|
8767
|
-
* Bytes / count / oldest / newest for one hour's
|
|
8768
|
-
* derivation,
|
|
8769
|
-
* by the append path.
|
|
8680
|
+
* Bytes / count / oldest / newest for one hour's LISTING — the sole FULL
|
|
8681
|
+
* derivation, and since D523 the only thing that can produce an EXACT row.
|
|
8770
8682
|
*
|
|
8771
8683
|
* Every field of a segment path encodes its own facts (`parseSegmentPath`), so
|
|
8772
|
-
* this is a derivation, not a measurement: it can never disagree with the
|
|
8773
|
-
* it was computed from.
|
|
8684
|
+
* this is a derivation, not a measurement: it can never disagree with the
|
|
8685
|
+
* listing it was computed from.
|
|
8774
8686
|
*/
|
|
8775
8687
|
function hourAggregate(paths) {
|
|
8776
8688
|
let agg = EMPTY_HOUR_AGGREGATE;
|
|
8777
8689
|
for (const path of paths) agg = withSegmentPath(agg, path);
|
|
8778
8690
|
return agg;
|
|
8779
8691
|
}
|
|
8780
|
-
/** Build the persisted row for an hour from
|
|
8692
|
+
/** Build the persisted row for an hour from a LISTING of it — the ONE
|
|
8693
|
+
* constructor that can produce an exact aggregate. */
|
|
8781
8694
|
function segmentHourRow(base) {
|
|
8695
|
+
const { paths, sealed, ...identity } = base;
|
|
8782
8696
|
return {
|
|
8783
|
-
...
|
|
8784
|
-
sealed:
|
|
8785
|
-
|
|
8786
|
-
|
|
8697
|
+
...identity,
|
|
8698
|
+
sealed: sealed ?? false,
|
|
8699
|
+
...hourAggregate(paths)
|
|
8700
|
+
};
|
|
8701
|
+
}
|
|
8702
|
+
/**
|
|
8703
|
+
* Build the persisted row for an hour from an aggregate somebody already folded
|
|
8704
|
+
* — a relocation moving one, or a listing that has already been reduced.
|
|
8705
|
+
*
|
|
8706
|
+
* `sealed` is a NAMED field and never a positional boolean, because
|
|
8707
|
+
* `scripts/check-seal-authority.ts` attributes every literal `sealed: true` in
|
|
8708
|
+
* the recorder to the method that wrote it. A seal smuggled in as a third
|
|
8709
|
+
* argument is a seal the guard cannot see.
|
|
8710
|
+
*/
|
|
8711
|
+
function hourRowOf(identity, row) {
|
|
8712
|
+
return {
|
|
8713
|
+
...identity,
|
|
8714
|
+
...row
|
|
8787
8715
|
};
|
|
8788
8716
|
}
|
|
8789
8717
|
/**
|
|
8790
8718
|
* The row an hour becomes when ONE more path joins it.
|
|
8791
8719
|
*
|
|
8792
|
-
* Identical to `segmentHourRow({
|
|
8793
|
-
* strictly cheaper: the aggregate half of `held` is already the fold over
|
|
8794
|
-
*
|
|
8720
|
+
* Identical to `segmentHourRow({ …held, paths: [...everything, path] })` and
|
|
8721
|
+
* strictly cheaper: the aggregate half of `held` is already the fold over the
|
|
8722
|
+
* hour's names, so this resumes that fold instead of restarting it. The
|
|
8795
8723
|
* equality is not an assumption — `segment-hour-append.spec.ts` pins it on
|
|
8796
8724
|
* generated hours including out-of-order arrival, duplicate paths, unparseable
|
|
8797
8725
|
* paths and storm-sized hours.
|
|
8798
|
-
*
|
|
8799
|
-
* The four fields are named rather than spread on purpose. {@link
|
|
8800
|
-
* withSegmentPath} hands its argument straight back for a path the parser
|
|
8801
|
-
* refuses, and `held` is a whole row — spreading that return would put the
|
|
8802
|
-
* PREVIOUS `paths` array back over the appended one. Which is exactly what it
|
|
8803
|
-
* did until the generated-hours test caught it.
|
|
8804
8726
|
*/
|
|
8805
8727
|
function appendSegmentPath(held, path) {
|
|
8806
8728
|
const { bytes, segments, minStartMs, maxStartMs } = withSegmentPath(held, path);
|
|
8807
8729
|
return {
|
|
8808
8730
|
...held,
|
|
8809
|
-
paths: [...held.paths, path],
|
|
8810
8731
|
bytes,
|
|
8811
8732
|
segments,
|
|
8812
8733
|
minStartMs,
|
|
@@ -8824,6 +8745,20 @@ function appendSegmentPath(held, path) {
|
|
|
8824
8745
|
* `readdir` over an unmounted root deletes all of it.
|
|
8825
8746
|
*/
|
|
8826
8747
|
var LEDGER_PRUNE_MAX_SHARE = .5;
|
|
8748
|
+
/**
|
|
8749
|
+
* The largest share of ONE hour's accounted segments a single listing may
|
|
8750
|
+
* erase (D523).
|
|
8751
|
+
*
|
|
8752
|
+
* The directory is the authority for WHICH segments exist, so a listing of a
|
|
8753
|
+
* closed hour replaces that hour's aggregate — including downwards, which is
|
|
8754
|
+
* what makes an over-counted row self-correcting. The guard is the same
|
|
8755
|
+
* argument {@link LEDGER_PRUNE_MAX_SHARE} makes one level up: a root that has
|
|
8756
|
+
* gone away, or a bucket half-deleted by a round that died, answers SHORT, and
|
|
8757
|
+
* a short answer that halves an hour is indistinguishable from one that is
|
|
8758
|
+
* simply right. Under the bar the listing wins; over it the row stands, the
|
|
8759
|
+
* hour stays unsealed, and the disagreement is named.
|
|
8760
|
+
*/
|
|
8761
|
+
var HOUR_LISTING_MAX_SHRINK_SHARE = .5;
|
|
8827
8762
|
/** Backstop on the oldest-first probe. A bounded read must stay bounded even if
|
|
8828
8763
|
* a caller asks for a target no archive can cover. */
|
|
8829
8764
|
var OLDEST_MAX_PAGES = 16;
|
|
@@ -8833,9 +8768,6 @@ var OLDEST_PROBE_COLUMNS = [
|
|
|
8833
8768
|
"bytes",
|
|
8834
8769
|
"locationId"
|
|
8835
8770
|
];
|
|
8836
|
-
/** Aggregate-repair probe: the primary key always comes back, so this asks for
|
|
8837
|
-
* the predicate column alone. */
|
|
8838
|
-
var REPAIR_PROBE_COLUMNS = ["segments"];
|
|
8839
8771
|
/**
|
|
8840
8772
|
* Backstop on {@link SegmentHourLedger.streamSegments}.
|
|
8841
8773
|
*
|
|
@@ -8874,47 +8806,8 @@ var CENSUS_FIELDS = [
|
|
|
8874
8806
|
op: "max"
|
|
8875
8807
|
}
|
|
8876
8808
|
];
|
|
8877
|
-
/** Rows repaired per boot. The live archive is ~19 k hour rows. */
|
|
8878
|
-
var REPAIR_MAX_ROWS = 5e4;
|
|
8879
|
-
/**
|
|
8880
|
-
* The boot projection (D287): everything the recorder's always-resident tier
|
|
8881
|
-
* needs, and deliberately NOT `paths`.
|
|
8882
|
-
*
|
|
8883
|
-
* `paths` is ~55 B per segment and the archive is the only thing that grows:
|
|
8884
|
-
* ~16 MB of JSON today, a projected ~460 MB at 28 TB of today's bitrate mix
|
|
8885
|
-
* and ~5.9 GB at a low-bitrate mix — one tRPC response, parsed on the main
|
|
8886
|
-
* thread, before the index exists. The summary half is O(writers × days) and
|
|
8887
|
-
* stays ~10–15 MB at any archive size.
|
|
8888
|
-
*/
|
|
8889
|
-
var SEGMENT_HOUR_SUMMARY_COLUMNS = [
|
|
8890
|
-
"deviceId",
|
|
8891
|
-
"profile",
|
|
8892
|
-
"locationId",
|
|
8893
|
-
"hourStartMs",
|
|
8894
|
-
"bytes",
|
|
8895
|
-
"segments",
|
|
8896
|
-
"minStartMs",
|
|
8897
|
-
"maxStartMs",
|
|
8898
|
-
"sealed"
|
|
8899
|
-
];
|
|
8900
8809
|
/** Summary rows folded into the mirror between yields. */
|
|
8901
8810
|
var LOAD_CHUNK = 5e3;
|
|
8902
|
-
/** Hour rows materialised per point-read page. One page covers a day of one
|
|
8903
|
-
* camera's three profiles with room to spare. */
|
|
8904
|
-
var HOUR_DETAIL_PAGE = 64;
|
|
8905
|
-
/**
|
|
8906
|
-
* The live tail materialised at boot: the current hour and the one before it.
|
|
8907
|
-
*
|
|
8908
|
-
* `recordSegment` must gate from RAM (D49), and the hour it appends to at boot
|
|
8909
|
-
* is exactly the hour the previous process was writing. Two hours × the active
|
|
8910
|
-
* writers is ~60 rows — one point-read page — and it is what makes the write
|
|
8911
|
-
* path's materialise a case that essentially never happens.
|
|
8912
|
-
*/
|
|
8913
|
-
var LIVE_TAIL_HOURS = 2;
|
|
8914
|
-
/** Rows the one-shot aggregate repair may materialise in a single boot. The
|
|
8915
|
-
* backfill is bounded so it cannot re-become the whole-archive read D287
|
|
8916
|
-
* removed; a partial pass finishes on the next boot. */
|
|
8917
|
-
var REPAIR_MAX_MATERIALIZE = 2e3;
|
|
8918
8811
|
/** The four scalars an accounting answer is made of, asked in one statement. */
|
|
8919
8812
|
var ACCOUNTING_FIELDS = [
|
|
8920
8813
|
{
|
|
@@ -8967,14 +8860,6 @@ function writerKey(deviceId, profile) {
|
|
|
8967
8860
|
function writerHourKey(deviceId, profile, hour) {
|
|
8968
8861
|
return `${deviceId}|${profile}|${hour}`;
|
|
8969
8862
|
}
|
|
8970
|
-
function detailOf(row) {
|
|
8971
|
-
return {
|
|
8972
|
-
locationId: row.locationId,
|
|
8973
|
-
hourStartMs: row.hourStartMs,
|
|
8974
|
-
paths: row.paths,
|
|
8975
|
-
sealed: row.sealed
|
|
8976
|
-
};
|
|
8977
|
-
}
|
|
8978
8863
|
/**
|
|
8979
8864
|
* The `hourStartMs` of one probe record.
|
|
8980
8865
|
*
|
|
@@ -8986,21 +8871,31 @@ function hourOfRecord(record) {
|
|
|
8986
8871
|
const raw = record.data["hourStartMs"];
|
|
8987
8872
|
return typeof raw === "number" && Number.isFinite(raw) ? raw : -1;
|
|
8988
8873
|
}
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
|
|
8992
|
-
|
|
8993
|
-
|
|
8994
|
-
|
|
8995
|
-
|
|
8874
|
+
/** One listing turned into the segment rows a caller can act on — the only
|
|
8875
|
+
* place a `SegmentRow` is built out of a directory read. */
|
|
8876
|
+
function segmentRowsOf(locationId, paths) {
|
|
8877
|
+
const rows = [];
|
|
8878
|
+
let unparseable = 0;
|
|
8879
|
+
for (const path of paths) {
|
|
8880
|
+
const parsed = parseSegmentPath(path);
|
|
8881
|
+
if (parsed === null) {
|
|
8882
|
+
unparseable += 1;
|
|
8883
|
+
continue;
|
|
8884
|
+
}
|
|
8885
|
+
rows.push({
|
|
8886
|
+
deviceId: parsed.deviceId,
|
|
8887
|
+
profile: parsed.profile,
|
|
8888
|
+
startMs: parsed.startMs,
|
|
8889
|
+
durMs: parsed.durMs,
|
|
8890
|
+
bytes: parsed.bytes,
|
|
8891
|
+
path,
|
|
8892
|
+
locationId
|
|
8893
|
+
});
|
|
8996
8894
|
}
|
|
8997
|
-
return
|
|
8998
|
-
|
|
8999
|
-
|
|
9000
|
-
|
|
9001
|
-
const set = new Set(a);
|
|
9002
|
-
for (const p of b) if (!set.has(p)) return false;
|
|
9003
|
-
return true;
|
|
8895
|
+
return {
|
|
8896
|
+
rows,
|
|
8897
|
+
unparseable
|
|
8898
|
+
};
|
|
9004
8899
|
}
|
|
9005
8900
|
function rowToValue(row) {
|
|
9006
8901
|
return {
|
|
@@ -9008,7 +8903,6 @@ function rowToValue(row) {
|
|
|
9008
8903
|
profile: row.profile,
|
|
9009
8904
|
locationId: row.locationId,
|
|
9010
8905
|
hourStartMs: row.hourStartMs,
|
|
9011
|
-
paths: encodePaths(row.paths),
|
|
9012
8906
|
bytes: row.bytes,
|
|
9013
8907
|
segments: row.segments,
|
|
9014
8908
|
minStartMs: row.minStartMs,
|
|
@@ -9021,44 +8915,21 @@ function rowToValue(row) {
|
|
|
9021
8915
|
function sealedOf(raw) {
|
|
9022
8916
|
return raw === 1 || raw === true;
|
|
9023
8917
|
}
|
|
9024
|
-
function recordToRow(key, data) {
|
|
9025
|
-
const deviceId = Number(data["deviceId"]);
|
|
9026
|
-
const profile = data["profile"];
|
|
9027
|
-
const locationId = data["locationId"];
|
|
9028
|
-
const hour = Number(data["hourStartMs"]);
|
|
9029
|
-
const decodedPaths = decodePaths(data["paths"]);
|
|
9030
|
-
if (decodedPaths === null) return null;
|
|
9031
|
-
const raw = decodedPaths;
|
|
9032
|
-
if (!Number.isFinite(deviceId) || deviceId <= 0 || typeof profile !== "string" || profile.length === 0 || typeof locationId !== "string" || locationId.length === 0 || !Number.isFinite(hour) || !Array.isArray(raw) || !raw.every((p) => typeof p === "string" && p.length > 0)) return null;
|
|
9033
|
-
return segmentHourRow({
|
|
9034
|
-
key,
|
|
9035
|
-
deviceId,
|
|
9036
|
-
profile,
|
|
9037
|
-
locationId,
|
|
9038
|
-
hourStartMs: hour,
|
|
9039
|
-
paths: raw,
|
|
9040
|
-
sealed: sealedOf(data["sealed"])
|
|
9041
|
-
});
|
|
9042
|
-
}
|
|
9043
|
-
/** The paths of a summary row: one shared empty array, never a per-row
|
|
9044
|
-
* allocation, and never mistakable for "this hour holds nothing". */
|
|
9045
|
-
var NO_PATHS = Object.freeze([]);
|
|
9046
8918
|
function finiteOr0(raw) {
|
|
9047
8919
|
const n = Number(raw);
|
|
9048
8920
|
return Number.isFinite(n) ? n : 0;
|
|
9049
8921
|
}
|
|
9050
8922
|
/**
|
|
9051
|
-
* One row
|
|
8923
|
+
* One persisted row.
|
|
9052
8924
|
*
|
|
9053
|
-
* The identity half is validated
|
|
9054
|
-
*
|
|
9055
|
-
*
|
|
9056
|
-
*
|
|
9057
|
-
*
|
|
9058
|
-
*
|
|
9059
|
-
* {@link SegmentHourLedger.repairAggregates} rewrites it.
|
|
8925
|
+
* The identity half is validated: a row that cannot say which camera, profile,
|
|
8926
|
+
* location and hour it is about is DISCARDED, because a row keyed on garbage
|
|
8927
|
+
* would refuse the write path for an hour that is perfectly fine. The aggregate
|
|
8928
|
+
* half is taken as the database wrote it — a row that predates the aggregate
|
|
8929
|
+
* columns reports zeros, which UNDER-reports its disk (prunes less, never more)
|
|
8930
|
+
* until a listing of that hour re-derives it.
|
|
9060
8931
|
*/
|
|
9061
|
-
function
|
|
8932
|
+
function recordToRow(key, data) {
|
|
9062
8933
|
const deviceId = Number(data["deviceId"]);
|
|
9063
8934
|
const profile = data["profile"];
|
|
9064
8935
|
const locationId = data["locationId"];
|
|
@@ -9070,8 +8941,6 @@ function summaryToRow(key, data) {
|
|
|
9070
8941
|
profile,
|
|
9071
8942
|
locationId,
|
|
9072
8943
|
hourStartMs: hour,
|
|
9073
|
-
paths: NO_PATHS,
|
|
9074
|
-
pathsResident: false,
|
|
9075
8944
|
sealed: sealedOf(data["sealed"]),
|
|
9076
8945
|
bytes: finiteOr0(data["bytes"]),
|
|
9077
8946
|
segments: finiteOr0(data["segments"]),
|
|
@@ -9093,11 +8962,12 @@ var SPEC = {
|
|
|
9093
8962
|
var SegmentHourLedger = class SegmentHourLedger {
|
|
9094
8963
|
ledger;
|
|
9095
8964
|
logger;
|
|
9096
|
-
/** Held for the
|
|
8965
|
+
/** Held for the ARCHIVE reads below, which are queries rather than mirror
|
|
9097
8966
|
* lookups. Every DECISION path (`recordSegment`, `dropSegments`, the prune
|
|
9098
8967
|
* refusals) still answers from the mirror alone — D49 is about gates, and an
|
|
9099
8968
|
* accounting number is not a gate. */
|
|
9100
8969
|
store;
|
|
8970
|
+
listHour;
|
|
9101
8971
|
isLocationUsable;
|
|
9102
8972
|
now;
|
|
9103
8973
|
loadChunk;
|
|
@@ -9111,6 +8981,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9111
8981
|
lanes = /* @__PURE__ */ new Map();
|
|
9112
8982
|
/** Per (device, profile): the hour its writer last appended to. */
|
|
9113
8983
|
writerHours = /* @__PURE__ */ new Map();
|
|
8984
|
+
/**
|
|
8985
|
+
* Segment paths THIS PROCESS has already accounted for, per hour key.
|
|
8986
|
+
*
|
|
8987
|
+
* The `paths` column used to make the write path idempotent for free: an
|
|
8988
|
+
* append checked `held.paths.includes(path)`. With the column gone the
|
|
8989
|
+
* aggregate cannot say whether it already counted a name, so the dedup lives
|
|
8990
|
+
* here — bounded to the hours this process actually writes (two per writer:
|
|
8991
|
+
* the current hour and the one it just left), dropped when the writer moves
|
|
8992
|
+
* on. A duplicate that survives a RESTART is not caught here and is not
|
|
8993
|
+
* silent either: the row is unsealed, and the first listing of the closed
|
|
8994
|
+
* hour re-derives the aggregate exactly ({@link adoptWalkedHour}).
|
|
8995
|
+
*/
|
|
8996
|
+
accounted = /* @__PURE__ */ new Map();
|
|
9114
8997
|
/** Hour keys a write was REFUSED for in this process — never sealed by it. */
|
|
9115
8998
|
refused = /* @__PURE__ */ new Set();
|
|
9116
8999
|
/**
|
|
@@ -9123,6 +9006,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9123
9006
|
constructor(deps) {
|
|
9124
9007
|
this.logger = deps.logger;
|
|
9125
9008
|
this.store = deps.store;
|
|
9009
|
+
this.listHour = deps.listHour ?? null;
|
|
9126
9010
|
this.isLocationUsable = deps.isLocationUsable ?? (() => true);
|
|
9127
9011
|
this.now = deps.now ?? (() => Date.now());
|
|
9128
9012
|
this.loadChunk = deps.loadPacing?.chunkSize ?? LOAD_CHUNK;
|
|
@@ -9141,22 +9025,17 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9141
9025
|
return this.ledger.declare();
|
|
9142
9026
|
}
|
|
9143
9027
|
/**
|
|
9144
|
-
* Boot load —
|
|
9145
|
-
*
|
|
9146
|
-
* One projected query over {@link SEGMENT_HOUR_SUMMARY_COLUMNS}, folded into
|
|
9147
|
-
* the mirror in chunks with a yield between them, then one point-read page
|
|
9148
|
-
* that materialises the live tail so the write path can gate from RAM.
|
|
9028
|
+
* Boot load — every hour row, in one query.
|
|
9149
9029
|
*
|
|
9150
|
-
*
|
|
9151
|
-
*
|
|
9152
|
-
*
|
|
9153
|
-
*
|
|
9030
|
+
* Since D523 the whole row is ten scalars, so there is no second tier to
|
|
9031
|
+
* materialise and no projection to choose: the mirror this returns is the
|
|
9032
|
+
* ledger, entire. It keeps `DurableLedger.load`'s two rules — a failed read
|
|
9033
|
+
* RETURNS WHAT IS ALREADY MIRRORED rather than clearing it, and a malformed
|
|
9034
|
+
* row is skipped rather than repaired.
|
|
9154
9035
|
*
|
|
9155
|
-
* It deliberately does NOT replay the archive into {@link RecordingIndex}
|
|
9156
|
-
*
|
|
9157
|
-
*
|
|
9158
|
-
* ({@link detailForWindow}), and an hour whose tail the ledger is missing is
|
|
9159
|
-
* no longer permanently invisible to playback.
|
|
9036
|
+
* It deliberately does NOT replay the archive into {@link RecordingIndex}: a
|
|
9037
|
+
* row says how MUCH an hour holds, never which files, so only a listing can
|
|
9038
|
+
* seed the index (D523).
|
|
9160
9039
|
*/
|
|
9161
9040
|
async load() {
|
|
9162
9041
|
const startedMs = this.now();
|
|
@@ -9164,7 +9043,6 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9164
9043
|
try {
|
|
9165
9044
|
records = await this.store.query.query({
|
|
9166
9045
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9167
|
-
columns: SEGMENT_HOUR_SUMMARY_COLUMNS,
|
|
9168
9046
|
filter: { limit: this.rowCap }
|
|
9169
9047
|
});
|
|
9170
9048
|
} catch (err) {
|
|
@@ -9181,7 +9059,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9181
9059
|
const end = Math.min(i + this.loadChunk, records.length);
|
|
9182
9060
|
for (let j = i; j < end; j += 1) {
|
|
9183
9061
|
const record = records[j];
|
|
9184
|
-
const row =
|
|
9062
|
+
const row = recordToRow(record.id, record.data);
|
|
9185
9063
|
if (row === null) {
|
|
9186
9064
|
skipped += 1;
|
|
9187
9065
|
continue;
|
|
@@ -9190,7 +9068,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9190
9068
|
}
|
|
9191
9069
|
if (end < records.length) await this.yieldBetween();
|
|
9192
9070
|
}
|
|
9193
|
-
if (skipped > 0) this.logger.warn("recorder: segment-hour
|
|
9071
|
+
if (skipped > 0) this.logger.warn("recorder: segment-hour rows skipped as malformed — they gate NOTHING", { meta: {
|
|
9194
9072
|
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9195
9073
|
skipped
|
|
9196
9074
|
} });
|
|
@@ -9198,11 +9076,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9198
9076
|
limit: this.rowCap,
|
|
9199
9077
|
loaded: records.length
|
|
9200
9078
|
} });
|
|
9201
|
-
const tail = await this.materializeLiveTail();
|
|
9202
9079
|
this.logger.info("recorder: segment-hour ledger loaded", { meta: {
|
|
9203
9080
|
hours: this.ledger.size,
|
|
9204
|
-
pathsLoaded: false,
|
|
9205
|
-
liveTailHours: tail,
|
|
9206
9081
|
skipped,
|
|
9207
9082
|
queryMs: queriedMs - startedMs,
|
|
9208
9083
|
ms: this.now() - startedMs,
|
|
@@ -9210,8 +9085,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9210
9085
|
} });
|
|
9211
9086
|
return this.ledger.snapshot();
|
|
9212
9087
|
}
|
|
9213
|
-
/** Every hour
|
|
9214
|
-
* `pathsResident` is false carries real aggregates and empty `paths`. */
|
|
9088
|
+
/** Every hour row held. Pure RAM. */
|
|
9215
9089
|
snapshot() {
|
|
9216
9090
|
return this.ledger.snapshot();
|
|
9217
9091
|
}
|
|
@@ -9221,10 +9095,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9221
9095
|
* The archive's answer to "what should placement plan for", and the reason it
|
|
9222
9096
|
* is here rather than a `census` loop or an index read:
|
|
9223
9097
|
*
|
|
9224
|
-
* - `RecordingIndex.accountingByProfile` is the wrong authority.
|
|
9225
|
-
*
|
|
9226
|
-
*
|
|
9227
|
-
*
|
|
9098
|
+
* - `RecordingIndex.accountingByProfile` is the wrong authority. It holds
|
|
9099
|
+
* the live tail plus whatever a reader hydrated, so a camera nobody has
|
|
9100
|
+
* viewed since boot reports NO profiles — and the placement pass then
|
|
9101
|
+
* never plans it (D295 §3a #9, D299).
|
|
9228
9102
|
* - `census({ deviceId, profiles: [p] })` is the right authority but the
|
|
9229
9103
|
* wrong shape for a 30 s tick: it is one SQL round trip per (camera,
|
|
9230
9104
|
* profile), so a 16-camera fleet pays 48 SERIAL trips over the UDS door
|
|
@@ -9232,10 +9106,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9232
9106
|
* 240003` pass (see `PLACEMENT_STORE_TIMEOUT_MS`), and it grows with the
|
|
9233
9107
|
* fleet.
|
|
9234
9108
|
*
|
|
9235
|
-
* So it is answered from
|
|
9236
|
-
* precisely so that O(writers × days) questions cost no I/O: ~10–15 MB and
|
|
9237
|
-
* ~47 k rows even at 30 TB, one fold, no query. A hydrated segment row is not
|
|
9238
|
-
* needed to know that an hour exists.
|
|
9109
|
+
* So it is answered from the mirror: one fold, no query, no I/O.
|
|
9239
9110
|
*
|
|
9240
9111
|
* An EMPTY result for a device is a real answer — "the archive knows no hour
|
|
9241
9112
|
* for this camera" — and it is also what a ledger whose load failed reports,
|
|
@@ -9252,136 +9123,111 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9252
9123
|
}
|
|
9253
9124
|
return out;
|
|
9254
9125
|
}
|
|
9255
|
-
/** Does the ledger hold a row for this hour at all? Pure RAM
|
|
9256
|
-
* storage-walk repair consults before it upserts a rebuilt hour. */
|
|
9126
|
+
/** Does the ledger hold a row for this hour at all? Pure RAM. */
|
|
9257
9127
|
knowsHour(deviceId, profile, locationId, hour) {
|
|
9258
9128
|
return this.ledger.has(segmentHourKey(deviceId, profile, locationId, hour));
|
|
9259
9129
|
}
|
|
9260
|
-
/**
|
|
9261
|
-
*
|
|
9262
|
-
|
|
9263
|
-
|
|
9264
|
-
* {@link detailForWindow} answers per (device, window) and drops the profile
|
|
9265
|
-
* on the way out, which is fine for a reader assembling a playlist and wrong
|
|
9266
|
-
* for a caller that must act on one hour ROW: the ghost-location reconcile
|
|
9267
|
-
* decides per `(deviceId, profile, locationId, hour)` and hands the resulting
|
|
9268
|
-
* paths straight back to {@link dropSegments} / {@link repointSegments},
|
|
9269
|
-
* whose bucket key is that same tuple.
|
|
9270
|
-
*
|
|
9271
|
-
* `null` is "known, and its detail could not be had" — never an empty array.
|
|
9272
|
-
* The one caller FORGETS rows on this answer, and a failed point read that
|
|
9273
|
-
* came back as `[]` would read as "this hour holds nothing", which is the
|
|
9274
|
-
* D148 prune-on-no-evidence this ledger refuses to make.
|
|
9275
|
-
*/
|
|
9276
|
-
async hourPaths(deviceId, profile, locationId, hour) {
|
|
9277
|
-
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9278
|
-
if (!this.ledger.has(key)) return null;
|
|
9279
|
-
const held = await this.resident(key, deviceId);
|
|
9280
|
-
return held === null ? null : held.paths;
|
|
9281
|
-
}
|
|
9282
|
-
/**
|
|
9283
|
-
* The hours of `[fromMs, toMs)` the ledger knows, with their paths — the
|
|
9284
|
-
* cold-hour detail read that replaces the boot replay (D287).
|
|
9285
|
-
*
|
|
9286
|
-
* One bounded point read against the settings store, never the media array:
|
|
9287
|
-
* a `readdir` of one hour directory was measured at ~650 ms under load on
|
|
9288
|
-
* the live share, and the array is the resource the scrub feeder is
|
|
9289
|
-
* competing for. Hours the ledger does NOT know are absent from the result,
|
|
9290
|
-
* and the caller walks those from disk exactly as it always did.
|
|
9291
|
-
*
|
|
9292
|
-
* **A read that fails returns what it already had** (D49). The caller sees
|
|
9293
|
-
* fewer hours, the index stays `unknown` for the rest, and the window falls
|
|
9294
|
-
* through to the walk — nothing is destroyed and nothing renders as an
|
|
9295
|
-
* empty hour that was merely unread.
|
|
9296
|
-
*/
|
|
9297
|
-
async detailForWindow(deviceId, fromMs, toMs) {
|
|
9298
|
-
const first = hourStartMs(fromMs);
|
|
9299
|
-
const out = [];
|
|
9300
|
-
const wanted = [];
|
|
9301
|
-
for (const row of this.ledger.snapshot()) {
|
|
9302
|
-
if (row.deviceId !== deviceId) continue;
|
|
9303
|
-
if (row.hourStartMs < first || row.hourStartMs >= toMs) continue;
|
|
9304
|
-
if (row.pathsResident) {
|
|
9305
|
-
out.push(detailOf(row));
|
|
9306
|
-
continue;
|
|
9307
|
-
}
|
|
9308
|
-
wanted.push(row.key);
|
|
9309
|
-
}
|
|
9310
|
-
for (const row of await this.materialize(wanted, deviceId)) out.push(detailOf(row));
|
|
9311
|
-
return out;
|
|
9130
|
+
/** One held hour row, or `undefined`. Pure RAM — since D523 a row carries
|
|
9131
|
+
* everything it has. */
|
|
9132
|
+
hourRow(deviceId, profile, locationId, hour) {
|
|
9133
|
+
return this.ledger.get(segmentHourKey(deviceId, profile, locationId, hour));
|
|
9312
9134
|
}
|
|
9313
9135
|
/**
|
|
9314
|
-
*
|
|
9315
|
-
*
|
|
9316
|
-
* The reader repairs the durable index it just reconstructed, so the next
|
|
9317
|
-
* cold read of this hour is a point read instead of another `readdir`
|
|
9318
|
-
* (`docs/design/2026-08-29-reference-nvr-recording-model.md` §4.2), and it
|
|
9319
|
-
* seals the row because a directory read of a closed hour is the one
|
|
9320
|
-
* complete answer there is. The caller guarantees the hour is CLOSED — a
|
|
9321
|
-
* live hour's directory races its writer, so a walk of it is a lower bound.
|
|
9136
|
+
* A LISTING of a CLOSED hour is adopted as that hour's account, and SEALS it.
|
|
9322
9137
|
*
|
|
9323
|
-
*
|
|
9324
|
-
*
|
|
9325
|
-
*
|
|
9326
|
-
*
|
|
9327
|
-
*
|
|
9328
|
-
*
|
|
9329
|
-
*
|
|
9138
|
+
* The directory is the authority for which segments exist (D523), so a
|
|
9139
|
+
* listing of an hour that is OVER is the one complete answer there is — it
|
|
9140
|
+
* replaces the row's aggregate rather than merging with it, which is what
|
|
9141
|
+
* makes an hour that was over-counted (a duplicate finalize across a restart)
|
|
9142
|
+
* or under-counted (a writer torn down mid-hour) self-correcting. The caller
|
|
9143
|
+
* guarantees the hour is CLOSED: a live hour's directory races its writer, so
|
|
9144
|
+
* a listing of it is a lower bound.
|
|
9330
9145
|
*
|
|
9331
|
-
*
|
|
9332
|
-
* is
|
|
9146
|
+
* A listing that would erase more than {@link HOUR_LISTING_MAX_SHRINK_SHARE}
|
|
9147
|
+
* of what the row accounts for is REFUSED and named. That is the one thing a
|
|
9148
|
+
* listing cannot be trusted about: a root that has gone away and a bucket a
|
|
9149
|
+
* dead round half-deleted both answer short, and neither is distinguishable
|
|
9150
|
+
* from an hour that is simply smaller than the row thought.
|
|
9333
9151
|
*
|
|
9334
9152
|
* @seal-authority closed-hour-listing — a directory read of an hour that is
|
|
9335
9153
|
* OVER is the one complete answer there is; the caller guarantees the hour is
|
|
9336
|
-
* closed (D451, D521).
|
|
9154
|
+
* closed (D451, D521, D523).
|
|
9337
9155
|
*/
|
|
9338
9156
|
async adoptWalkedHour(deviceId, profile, locationId, hour, paths) {
|
|
9339
9157
|
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9158
|
+
const identity = {
|
|
9159
|
+
key,
|
|
9160
|
+
deviceId,
|
|
9161
|
+
profile,
|
|
9162
|
+
locationId,
|
|
9163
|
+
hourStartMs: hour
|
|
9164
|
+
};
|
|
9165
|
+
const walked = hourAggregate(paths);
|
|
9340
9166
|
return this.serialized(key, async () => {
|
|
9341
|
-
|
|
9342
|
-
|
|
9167
|
+
const held = this.ledger.get(key);
|
|
9168
|
+
if (held === void 0) {
|
|
9169
|
+
if (walked.segments === 0) return {
|
|
9343
9170
|
created: false,
|
|
9344
9171
|
added: 0,
|
|
9345
9172
|
sealedNow: false
|
|
9346
9173
|
};
|
|
9347
|
-
|
|
9348
|
-
|
|
9349
|
-
deviceId,
|
|
9350
|
-
profile,
|
|
9351
|
-
locationId,
|
|
9352
|
-
hourStartMs: hour,
|
|
9353
|
-
paths: uniquePaths(paths),
|
|
9174
|
+
await this.ledger.put(hourRowOf(identity, {
|
|
9175
|
+
...walked,
|
|
9354
9176
|
sealed: true
|
|
9355
|
-
});
|
|
9356
|
-
|
|
9357
|
-
this.logger.info("recorder: adopted a closed hour the disk walk rebuilt and the ledger never held", {
|
|
9177
|
+
}));
|
|
9178
|
+
this.logger.info("recorder: adopted a closed hour the disk listing rebuilt and the ledger never held", {
|
|
9358
9179
|
tags: { deviceId },
|
|
9359
9180
|
meta: {
|
|
9360
9181
|
profile,
|
|
9361
9182
|
locationId,
|
|
9362
9183
|
hourStartMs: hour,
|
|
9363
|
-
segments:
|
|
9364
|
-
bytes:
|
|
9184
|
+
segments: walked.segments,
|
|
9185
|
+
bytes: walked.bytes
|
|
9365
9186
|
}
|
|
9366
9187
|
});
|
|
9367
9188
|
return {
|
|
9368
9189
|
created: true,
|
|
9369
|
-
added:
|
|
9190
|
+
added: walked.segments,
|
|
9370
9191
|
sealedNow: true
|
|
9371
9192
|
};
|
|
9372
9193
|
}
|
|
9373
|
-
const
|
|
9374
|
-
if (held
|
|
9375
|
-
|
|
9376
|
-
|
|
9377
|
-
|
|
9194
|
+
const shrink = held.segments - walked.segments;
|
|
9195
|
+
if (shrink > 0 && shrink > held.segments * .5) {
|
|
9196
|
+
this.logger.warn("recorder: hour listing REFUSED — it accounts for less than half of what this hour row holds, which is what an unmounted root looks like; the row stands and stays unsealed", {
|
|
9197
|
+
tags: { deviceId },
|
|
9198
|
+
meta: {
|
|
9199
|
+
profile,
|
|
9200
|
+
locationId,
|
|
9201
|
+
hourStartMs: hour,
|
|
9202
|
+
held: held.segments,
|
|
9203
|
+
listed: walked.segments,
|
|
9204
|
+
maxShare: HOUR_LISTING_MAX_SHRINK_SHARE
|
|
9205
|
+
}
|
|
9206
|
+
});
|
|
9207
|
+
return {
|
|
9208
|
+
created: false,
|
|
9209
|
+
added: 0,
|
|
9210
|
+
sealedNow: false
|
|
9211
|
+
};
|
|
9212
|
+
}
|
|
9213
|
+
const added = Math.max(0, walked.segments - held.segments);
|
|
9214
|
+
if (added === 0 && shrink === 0 && held.sealed) return {
|
|
9378
9215
|
created: false,
|
|
9379
9216
|
added: 0,
|
|
9380
9217
|
sealedNow: false
|
|
9381
9218
|
};
|
|
9382
|
-
|
|
9383
|
-
|
|
9384
|
-
|
|
9219
|
+
if (shrink > 0) this.logger.info("recorder: hour listing accounts for FEWER segments than the row did — the directory decides", {
|
|
9220
|
+
tags: { deviceId },
|
|
9221
|
+
meta: {
|
|
9222
|
+
profile,
|
|
9223
|
+
locationId,
|
|
9224
|
+
hourStartMs: hour,
|
|
9225
|
+
held: held.segments,
|
|
9226
|
+
listed: walked.segments
|
|
9227
|
+
}
|
|
9228
|
+
});
|
|
9229
|
+
await this.ledger.put(hourRowOf(identity, {
|
|
9230
|
+
...walked,
|
|
9385
9231
|
sealed: true
|
|
9386
9232
|
}));
|
|
9387
9233
|
return {
|
|
@@ -9411,9 +9257,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9411
9257
|
}
|
|
9412
9258
|
/**
|
|
9413
9259
|
* A finalize this ledger was NEVER handed — the fire-and-forget lane refused
|
|
9414
|
-
* it. The hour's row is short by that
|
|
9415
|
-
* party that knows, so the hour must not be sealed by it; a
|
|
9416
|
-
*
|
|
9260
|
+
* it. The hour's row is short by that segment and this process is the only
|
|
9261
|
+
* party that knows, so the hour must not be sealed by it; a listing corrects
|
|
9262
|
+
* it instead, which is exactly what an unsealed row is for.
|
|
9417
9263
|
*/
|
|
9418
9264
|
markRefused(row) {
|
|
9419
9265
|
this.refused.add(hourOf(row).key);
|
|
@@ -9441,7 +9287,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9441
9287
|
marked += 1;
|
|
9442
9288
|
}
|
|
9443
9289
|
if (marked === 0) return 0;
|
|
9444
|
-
this.logger.warn("recorder: segments this writer produced never reached the hour ledger — these hours stay UNSEALED and a
|
|
9290
|
+
this.logger.warn("recorder: segments this writer produced never reached the hour ledger — these hours stay UNSEALED and a listing corrects them", {
|
|
9445
9291
|
tags: { deviceId: span.deviceId },
|
|
9446
9292
|
meta: {
|
|
9447
9293
|
profile: span.profile,
|
|
@@ -9474,10 +9320,11 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9474
9320
|
graceMs: WRITER_HOUR_ENTRY_GRACE_MS
|
|
9475
9321
|
}
|
|
9476
9322
|
});
|
|
9323
|
+
this.accounted.delete(target.key);
|
|
9477
9324
|
return;
|
|
9478
9325
|
}
|
|
9479
9326
|
if (this.refused.has(target.key) || this.refusedWriterHours.has(writerHourKey(target.deviceId, target.profile, target.hourStartMs))) {
|
|
9480
|
-
this.logger.warn("recorder: hour left unsealed — a ledger write for it was refused, so its row is short and a
|
|
9327
|
+
this.logger.warn("recorder: hour left unsealed — a ledger write for it was refused, so its row is short and a listing must correct it", {
|
|
9481
9328
|
tags: { deviceId: target.deviceId },
|
|
9482
9329
|
meta: {
|
|
9483
9330
|
profile: target.profile,
|
|
@@ -9485,10 +9332,12 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9485
9332
|
hourStartMs: target.hourStartMs
|
|
9486
9333
|
}
|
|
9487
9334
|
});
|
|
9335
|
+
this.accounted.delete(target.key);
|
|
9488
9336
|
return;
|
|
9489
9337
|
}
|
|
9490
|
-
const held =
|
|
9491
|
-
|
|
9338
|
+
const held = this.ledger.get(target.key);
|
|
9339
|
+
this.accounted.delete(target.key);
|
|
9340
|
+
if (held === void 0 || held.sealed) return;
|
|
9492
9341
|
await this.ledger.put({
|
|
9493
9342
|
...held,
|
|
9494
9343
|
sealed: true
|
|
@@ -9516,11 +9365,25 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9516
9365
|
/**
|
|
9517
9366
|
* Run `fn` after every mutation already queued on `key`.
|
|
9518
9367
|
*
|
|
9519
|
-
*
|
|
9520
|
-
*
|
|
9521
|
-
*
|
|
9522
|
-
*
|
|
9523
|
-
*
|
|
9368
|
+
* It was the lost-update guard: a row was read from the STORE, folded into
|
|
9369
|
+
* and written back with awaits in between, so two such runs on one key
|
|
9370
|
+
* interleaved and the second write carried a path set that never saw the
|
|
9371
|
+
* first — the sparse rows D451 found. Since D523 there is no store read on
|
|
9372
|
+
* any of these paths (the mirror holds the whole row) and `DurableLedger.put`
|
|
9373
|
+
* sets the mirror synchronously before it awaits the store, so every
|
|
9374
|
+
* get→fold→put here is synchronous up to the mirror and the interleaving
|
|
9375
|
+
* window is gone.
|
|
9376
|
+
*
|
|
9377
|
+
* What the lane still buys is the ORDER of the durable writes per hour key —
|
|
9378
|
+
* a rollover seal fired without an await must not land over an append that
|
|
9379
|
+
* was queued before it — and a handle for {@link flush} to wait on at
|
|
9380
|
+
* shutdown. A rejected step must not poison the lane: the next step still
|
|
9381
|
+
* runs.
|
|
9382
|
+
*
|
|
9383
|
+
* Note for anyone tempted to delete it: `segment-hour-seal.spec.ts` does NOT
|
|
9384
|
+
* catch its removal any more, and says so. Pinning durable write ORDER needs
|
|
9385
|
+
* a store whose latency a test can control, which `fakeSettingsStore` has no
|
|
9386
|
+
* way to express.
|
|
9524
9387
|
*/
|
|
9525
9388
|
serialized(key, fn) {
|
|
9526
9389
|
const run = (this.lanes.get(key) ?? Promise.resolve()).then(fn, fn);
|
|
@@ -9532,116 +9395,59 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9532
9395
|
return run;
|
|
9533
9396
|
}
|
|
9534
9397
|
/**
|
|
9535
|
-
*
|
|
9398
|
+
* List the hour directories of `rows`, paced, and expand them into segment
|
|
9399
|
+
* rows.
|
|
9536
9400
|
*
|
|
9537
|
-
*
|
|
9538
|
-
*
|
|
9539
|
-
*
|
|
9540
|
-
*
|
|
9541
|
-
* walk, or refusing a write) is strictly safer than a guess.
|
|
9542
|
-
*
|
|
9543
|
-
* A row the store returns but this cannot parse is EVICTED from the mirror
|
|
9544
|
-
* rather than left as a summary that can never become resident: it re-seeds
|
|
9545
|
-
* cold on the next write, which is `DurableLedger`'s rule 4 and the only
|
|
9546
|
-
* escape from an hour permanently frozen against its own writer.
|
|
9401
|
+
* The one place the ledger touches the array. An hour whose listing FAILED
|
|
9402
|
+
* takes no part in the caller's pass and is SAID — a branch that drops work
|
|
9403
|
+
* silently reads as "there was nothing there", which for a drain or a prune
|
|
9404
|
+
* is the difference between "moved everything" and "moved what it could see".
|
|
9547
9405
|
*/
|
|
9548
|
-
async
|
|
9406
|
+
async listHours(rows, reason) {
|
|
9549
9407
|
const out = [];
|
|
9550
|
-
if (
|
|
9551
|
-
const
|
|
9552
|
-
|
|
9553
|
-
|
|
9554
|
-
|
|
9555
|
-
|
|
9556
|
-
|
|
9557
|
-
|
|
9558
|
-
|
|
9559
|
-
|
|
9560
|
-
|
|
9561
|
-
|
|
9562
|
-
|
|
9563
|
-
|
|
9564
|
-
|
|
9565
|
-
...tags,
|
|
9408
|
+
if (rows.length === 0) return out;
|
|
9409
|
+
const lister = this.listHour;
|
|
9410
|
+
if (lister === null) {
|
|
9411
|
+
this.logger.warn("recorder: no hour lister on this ledger — nothing that needs segment NAMES can run here", { meta: {
|
|
9412
|
+
hours: rows.length,
|
|
9413
|
+
reason
|
|
9414
|
+
} });
|
|
9415
|
+
return out;
|
|
9416
|
+
}
|
|
9417
|
+
let listed = 0;
|
|
9418
|
+
for (const row of rows) {
|
|
9419
|
+
const paths = await lister(row.locationId, row.deviceId, row.profile, row.hourStartMs);
|
|
9420
|
+
if (paths === null) {
|
|
9421
|
+
this.logger.warn("recorder: hour directory could not be listed — this hour takes NO part in this pass", {
|
|
9422
|
+
tags: { deviceId: row.deviceId },
|
|
9566
9423
|
meta: {
|
|
9567
|
-
|
|
9568
|
-
|
|
9569
|
-
|
|
9570
|
-
|
|
9424
|
+
profile: row.profile,
|
|
9425
|
+
locationId: row.locationId,
|
|
9426
|
+
hourStartMs: row.hourStartMs,
|
|
9427
|
+
accounted: row.segments,
|
|
9428
|
+
reason
|
|
9571
9429
|
}
|
|
9572
9430
|
});
|
|
9573
|
-
|
|
9431
|
+
continue;
|
|
9574
9432
|
}
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
|
|
9578
|
-
|
|
9579
|
-
|
|
9580
|
-
|
|
9581
|
-
|
|
9582
|
-
|
|
9583
|
-
|
|
9433
|
+
const { rows: segments, unparseable } = segmentRowsOf(row.locationId, paths);
|
|
9434
|
+
if (unparseable > 0) this.logger.warn("recorder: hour directory holds names that are not segments", {
|
|
9435
|
+
tags: { deviceId: row.deviceId },
|
|
9436
|
+
meta: {
|
|
9437
|
+
profile: row.profile,
|
|
9438
|
+
locationId: row.locationId,
|
|
9439
|
+
hourStartMs: row.hourStartMs,
|
|
9440
|
+
unparseable,
|
|
9441
|
+
reason
|
|
9584
9442
|
}
|
|
9585
|
-
|
|
9586
|
-
|
|
9587
|
-
|
|
9588
|
-
if (
|
|
9443
|
+
});
|
|
9444
|
+
out.push(...segments);
|
|
9445
|
+
listed += 1;
|
|
9446
|
+
if (listed % 8 === 0) await this.yieldBetween();
|
|
9589
9447
|
}
|
|
9590
9448
|
return out;
|
|
9591
9449
|
}
|
|
9592
9450
|
/**
|
|
9593
|
-
* Materialise every hour these devices hold, and report how many stayed
|
|
9594
|
-
* summaries.
|
|
9595
|
-
*
|
|
9596
|
-
* The reconcile is the one path that DELETES durable rows, so it may not run
|
|
9597
|
-
* against a mirror that only half knows what it holds. This is a whole-device
|
|
9598
|
-
* read and it is deliberately confined here: `reconcileFromIndex` follows an
|
|
9599
|
-
* operator-armed rescan of that same device — a recursive `readdir` over
|
|
9600
|
-
* >1M files — so a paged point read alongside it is noise, and it is the one
|
|
9601
|
-
* caller whose correctness depends on completeness.
|
|
9602
|
-
*/
|
|
9603
|
-
async materializeDevices(deviceIds) {
|
|
9604
|
-
const keys = [];
|
|
9605
|
-
for (const row of this.ledger.snapshot()) if (!row.pathsResident && deviceIds.has(row.deviceId)) keys.push(row.key);
|
|
9606
|
-
if (keys.length === 0) return;
|
|
9607
|
-
await this.materialize(keys);
|
|
9608
|
-
const stillSummary = /* @__PURE__ */ new Map();
|
|
9609
|
-
for (const row of this.ledger.snapshot()) {
|
|
9610
|
-
if (row.pathsResident || !deviceIds.has(row.deviceId)) continue;
|
|
9611
|
-
stillSummary.set(row.deviceId, (stillSummary.get(row.deviceId) ?? 0) + 1);
|
|
9612
|
-
}
|
|
9613
|
-
for (const [deviceId, unreadable] of stillSummary) this.logger.warn("recorder: hours without readable detail take NO part in this reconcile — they are neither pruned nor unioned", {
|
|
9614
|
-
tags: { deviceId },
|
|
9615
|
-
meta: {
|
|
9616
|
-
asked: keys.length,
|
|
9617
|
-
unreadable
|
|
9618
|
-
}
|
|
9619
|
-
});
|
|
9620
|
-
}
|
|
9621
|
-
/** The hour a write path may touch: current + previous. Materialised at boot
|
|
9622
|
-
* so {@link recordSegment}'s dedup gate answers from RAM (D49). */
|
|
9623
|
-
async materializeLiveTail() {
|
|
9624
|
-
const oldest = hourStartMs(this.now()) - (LIVE_TAIL_HOURS - 1) * HOUR_MS$4;
|
|
9625
|
-
const keys = [];
|
|
9626
|
-
for (const row of this.ledger.snapshot()) if (row.hourStartMs >= oldest && !row.pathsResident) keys.push(row.key);
|
|
9627
|
-
return (await this.materialize(keys)).length;
|
|
9628
|
-
}
|
|
9629
|
-
/**
|
|
9630
|
-
* Promote one HELD row to paths-resident, or answer `null`.
|
|
9631
|
-
*
|
|
9632
|
-
* `null` means "this hour is known and its detail could not be had" — the
|
|
9633
|
-
* only honest answer to a failed point read, and every caller turns it into
|
|
9634
|
-
* "do nothing". It is deliberately distinct from `this.ledger.get(key) ===
|
|
9635
|
-
* undefined`, which means "no such hour" and is what licenses a fresh row.
|
|
9636
|
-
*/
|
|
9637
|
-
async resident(key, deviceId) {
|
|
9638
|
-
const held = this.ledger.get(key);
|
|
9639
|
-
if (held !== void 0 && held.pathsResident) return held;
|
|
9640
|
-
await this.materialize([key], deviceId);
|
|
9641
|
-
const after = this.ledger.get(key);
|
|
9642
|
-
return after !== void 0 && after.pathsResident ? after : null;
|
|
9643
|
-
}
|
|
9644
|
-
/**
|
|
9645
9451
|
* Bytes / count / oldest / newest across a SET of storage locations, as ONE
|
|
9646
9452
|
* `SUM`/`COUNT`/`MIN`/`MAX` over the hour rows.
|
|
9647
9453
|
*
|
|
@@ -9682,9 +9488,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9682
9488
|
* ## Why two phases
|
|
9683
9489
|
*
|
|
9684
9490
|
* Phase one reads only the small columns and walks oldest-first until the
|
|
9685
|
-
* accumulated bytes cover the target — normally one page.
|
|
9686
|
-
*
|
|
9687
|
-
*
|
|
9491
|
+
* accumulated bytes cover the target — normally one page. Phase two LISTS the
|
|
9492
|
+
* hour directories phase one chose, which is the only thing that can name the
|
|
9493
|
+
* files eviction is about to delete (D523).
|
|
9688
9494
|
*
|
|
9689
9495
|
* `exhausted` means the stream ran out before the target was met — the
|
|
9690
9496
|
* caller's stop condition, and the one thing a bounded read must still be
|
|
@@ -9718,11 +9524,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9718
9524
|
/**
|
|
9719
9525
|
* Every camera the ledger holds an hour for on these locations.
|
|
9720
9526
|
*
|
|
9721
|
-
* Answered from
|
|
9722
|
-
*
|
|
9723
|
-
*
|
|
9724
|
-
*
|
|
9725
|
-
* would have to be turned into a verdict.
|
|
9527
|
+
* Answered from the mirror, exactly like {@link profilesWithFootage}, and for
|
|
9528
|
+
* the same reason: this is an O(writers × days) question and it must cost no
|
|
9529
|
+
* I/O. It runs on the disk-pressure path, where an I/O that fails would have
|
|
9530
|
+
* to be turned into a verdict.
|
|
9726
9531
|
*
|
|
9727
9532
|
* An EMPTY answer is "the mirror knows no hour here", which is also what a
|
|
9728
9533
|
* ledger whose load failed reports — so the caller must treat it as "decide
|
|
@@ -9737,7 +9542,7 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9737
9542
|
* Bytes / segments / oldest / newest per CAMERA on these locations — the
|
|
9738
9543
|
* per-camera half of the storage page, answered by the ARCHIVE (D418).
|
|
9739
9544
|
*
|
|
9740
|
-
* ## Why the
|
|
9545
|
+
* ## Why the mirror fold and not `census({ deviceId })` per camera
|
|
9741
9546
|
*
|
|
9742
9547
|
* `census` is the right authority for a SCOPE, and it cannot express this
|
|
9743
9548
|
* scope: {@link SegmentHourLedger.predicate} takes ONE `locationId`, so
|
|
@@ -9747,14 +9552,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9747
9552
|
* accounts for. Per camera per volume that is `cameras × volumes` SQL
|
|
9748
9553
|
* aggregates for one operator click, and it grows with the fleet.
|
|
9749
9554
|
*
|
|
9750
|
-
* The
|
|
9751
|
-
*
|
|
9752
|
-
*
|
|
9753
|
-
*
|
|
9754
|
-
* over the hour rows the mirror holds — O(writers × profiles × hours), ~19 k
|
|
9755
|
-
* on the live hub (see {@link REPAIR_MAX_ROWS}) against 98 445 segments,
|
|
9756
|
-
* i.e. it does NOT scale with the segment count — and it is capped for good
|
|
9757
|
-
* by {@link SEGMENT_HOUR_SUMMARY_LIMIT}, the mirror's own ceiling.
|
|
9555
|
+
* The mirror answers all of them in ONE pass with no I/O, exactly as
|
|
9556
|
+
* {@link devicesWithFootage} and {@link profilesWithFootage} already do, and
|
|
9557
|
+
* it is the same rows `accountingForLocations` sums in SQL — the mirror is
|
|
9558
|
+
* write-behind, so it is never BEHIND the store.
|
|
9758
9559
|
*
|
|
9759
9560
|
* ## What an empty answer means
|
|
9760
9561
|
*
|
|
@@ -9828,44 +9629,28 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9828
9629
|
rows: [],
|
|
9829
9630
|
exhausted: true
|
|
9830
9631
|
};
|
|
9831
|
-
const full = await this.store.query.query({
|
|
9832
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
9833
|
-
filter: {
|
|
9834
|
-
whereIn: { key: chosen },
|
|
9835
|
-
limit: chosen.length
|
|
9836
|
-
}
|
|
9837
|
-
});
|
|
9838
|
-
const rows = [];
|
|
9839
|
-
for (const record of full) {
|
|
9840
|
-
const hour = recordToRow(record.id, record.data);
|
|
9841
|
-
if (hour === null) continue;
|
|
9842
|
-
this.ledger.stage(hour);
|
|
9843
|
-
for (const path of hour.paths) {
|
|
9844
|
-
const parsed = parseSegmentPath(path);
|
|
9845
|
-
if (parsed === null) continue;
|
|
9846
|
-
rows.push({
|
|
9847
|
-
deviceId: parsed.deviceId,
|
|
9848
|
-
profile: parsed.profile,
|
|
9849
|
-
startMs: parsed.startMs,
|
|
9850
|
-
durMs: parsed.durMs,
|
|
9851
|
-
bytes: parsed.bytes,
|
|
9852
|
-
path,
|
|
9853
|
-
locationId: hour.locationId
|
|
9854
|
-
});
|
|
9855
|
-
}
|
|
9856
|
-
}
|
|
9857
9632
|
return {
|
|
9858
|
-
rows:
|
|
9633
|
+
rows: (await this.listHours(this.rowsOfKeys(chosen), "oldest-segments")).toSorted((a, b) => a.startMs - b.startMs),
|
|
9859
9634
|
exhausted
|
|
9860
9635
|
};
|
|
9861
9636
|
}
|
|
9637
|
+
/** Mirror rows for keys a probe named. A key the mirror does not hold is
|
|
9638
|
+
* skipped: the probe and the mirror describe the same table, and an hour the
|
|
9639
|
+
* mirror never loaded is one this process may not act on. */
|
|
9640
|
+
rowsOfKeys(keys) {
|
|
9641
|
+
const out = [];
|
|
9642
|
+
for (const key of keys) {
|
|
9643
|
+
const row = this.ledger.get(key);
|
|
9644
|
+
if (row !== void 0) out.push(row);
|
|
9645
|
+
}
|
|
9646
|
+
return out;
|
|
9647
|
+
}
|
|
9862
9648
|
/**
|
|
9863
9649
|
* The predicate half of an {@link ArchiveScope}, as one filter.
|
|
9864
9650
|
*
|
|
9865
9651
|
* REFUSES an empty scope rather than compiling to "the whole collection".
|
|
9866
|
-
* There is no read here that may be unscoped:
|
|
9867
|
-
*
|
|
9868
|
-
* how that query comes back under a new name.
|
|
9652
|
+
* There is no read here that may be unscoped: an unscoped stream lists every
|
|
9653
|
+
* hour directory on the hub.
|
|
9869
9654
|
*/
|
|
9870
9655
|
static predicate(scope) {
|
|
9871
9656
|
const where = {};
|
|
@@ -9878,6 +9663,15 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9878
9663
|
...profiles.length > 0 ? { whereIn: { profile: [...profiles] } } : {}
|
|
9879
9664
|
};
|
|
9880
9665
|
}
|
|
9666
|
+
/** Does this row fall inside `scope`? The mirror-side twin of
|
|
9667
|
+
* {@link SegmentHourLedger.predicate}, for the passes that fold the mirror
|
|
9668
|
+
* instead of querying. */
|
|
9669
|
+
static inScope(row, scope) {
|
|
9670
|
+
if (scope.locationId !== void 0 && row.locationId !== scope.locationId) return false;
|
|
9671
|
+
if (scope.deviceId !== void 0 && row.deviceId !== scope.deviceId) return false;
|
|
9672
|
+
const profiles = scope.profiles ?? [];
|
|
9673
|
+
return profiles.length === 0 || profiles.includes(row.profile);
|
|
9674
|
+
}
|
|
9881
9675
|
/**
|
|
9882
9676
|
* What the ARCHIVE holds in this scope — segments and bytes, as ONE
|
|
9883
9677
|
* `SUM` over the hour rows.
|
|
@@ -9921,6 +9715,15 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9921
9715
|
/**
|
|
9922
9716
|
* Every segment in `scope`, oldest-first, ONE PAGE at a time.
|
|
9923
9717
|
*
|
|
9718
|
+
* ## Two phases, and which authority answers which
|
|
9719
|
+
*
|
|
9720
|
+
* Phase one asks the TABLE which hours the scope holds, oldest-first — the
|
|
9721
|
+
* ordered aggregate no directory read can answer. Phase two LISTS those hour
|
|
9722
|
+
* directories, which is the only authority for the file names the caller is
|
|
9723
|
+
* about to move or delete (D523). An hour whose listing fails is skipped and
|
|
9724
|
+
* said; the caller's own N-versus-M report is what makes the short walk
|
|
9725
|
+
* visible.
|
|
9726
|
+
*
|
|
9924
9727
|
* ## Why a keyset cursor and not `offset`
|
|
9925
9728
|
*
|
|
9926
9729
|
* Both callers MUTATE the scope while they walk it: the drain re-points each
|
|
@@ -9937,10 +9740,9 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9937
9740
|
*
|
|
9938
9741
|
* ## What a failed read does
|
|
9939
9742
|
*
|
|
9940
|
-
* It ENDS the stream (D49: a fallible read destroys nothing)
|
|
9941
|
-
*
|
|
9942
|
-
*
|
|
9943
|
-
* from a scope that was genuinely empty.
|
|
9743
|
+
* It ENDS the stream (D49: a fallible read destroys nothing). A stream that
|
|
9744
|
+
* swallowed the error and returned would be indistinguishable from a scope
|
|
9745
|
+
* that was genuinely empty.
|
|
9944
9746
|
*/
|
|
9945
9747
|
async *streamSegments(scope) {
|
|
9946
9748
|
const predicate = SegmentHourLedger.predicate(scope);
|
|
@@ -9993,8 +9795,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9993
9795
|
if (nextCursor !== cursor) emittedAtCursor = /* @__PURE__ */ new Set();
|
|
9994
9796
|
for (const record of fresh) if (hourOfRecord(record) === nextCursor) emittedAtCursor.add(record.id);
|
|
9995
9797
|
cursor = nextCursor;
|
|
9996
|
-
const rows = await this.
|
|
9997
|
-
if (rows.length > 0) yield rows;
|
|
9798
|
+
const rows = await this.listHours(this.rowsOfKeys(fresh.map((record) => record.id)), "archive-stream");
|
|
9799
|
+
if (rows.length > 0) yield rows.toSorted((a, b) => a.startMs - b.startMs);
|
|
9998
9800
|
if (probe.length < 128 && fresh.length === probe.length) return;
|
|
9999
9801
|
}
|
|
10000
9802
|
this.logger.warn("recorder: archive stream hit its page backstop — the rest of this scope was NOT walked", {
|
|
@@ -10008,110 +9810,50 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10008
9810
|
});
|
|
10009
9811
|
}
|
|
10010
9812
|
/**
|
|
10011
|
-
*
|
|
10012
|
-
*
|
|
9813
|
+
* Forget every hour row in `scope` — the durable half of an operator's
|
|
9814
|
+
* "discard this location's footage".
|
|
9815
|
+
*
|
|
9816
|
+
* It exists because the files are already GONE by the time it runs: the
|
|
9817
|
+
* discard wipes the class subtrees first, so every hour directory in the
|
|
9818
|
+
* scope answers ENOENT and nothing that needs a listing can name a single
|
|
9819
|
+
* segment. The rows are forgotten by their own identity instead, and the
|
|
9820
|
+
* count returned is the SEGMENTS they accounted for, which is the number the
|
|
9821
|
+
* operator is shown.
|
|
10013
9822
|
*
|
|
10014
|
-
*
|
|
10015
|
-
*
|
|
10016
|
-
*
|
|
10017
|
-
* and which `forget`s the row as it empties — so the staging is
|
|
10018
|
-
* self-cleaning and the alternative is two reads per hour.
|
|
9823
|
+
* Never reachable except from that explicitly-armed action: this is the one
|
|
9824
|
+
* surface here that deletes durable rows with no evidence from the array at
|
|
9825
|
+
* all, because the operator's own instruction is the evidence.
|
|
10019
9826
|
*/
|
|
10020
|
-
async rowsForHours(keys, scope) {
|
|
10021
|
-
if (keys.length === 0) return [];
|
|
10022
|
-
let records;
|
|
10023
|
-
try {
|
|
10024
|
-
records = await this.store.query.query({
|
|
10025
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10026
|
-
filter: {
|
|
10027
|
-
whereIn: { key: [...keys] },
|
|
10028
|
-
limit: keys.length
|
|
10029
|
-
}
|
|
10030
|
-
});
|
|
10031
|
-
} catch (err) {
|
|
10032
|
-
this.logger.warn("recorder: archive page detail unavailable — these hours take NO part in this pass", {
|
|
10033
|
-
...scope.deviceId !== void 0 ? { tags: { deviceId: scope.deviceId } } : {},
|
|
10034
|
-
meta: {
|
|
10035
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10036
|
-
hours: keys.length,
|
|
10037
|
-
error: String(err)
|
|
10038
|
-
}
|
|
10039
|
-
});
|
|
10040
|
-
return [];
|
|
10041
|
-
}
|
|
10042
|
-
const rows = [];
|
|
10043
|
-
for (const record of records) {
|
|
10044
|
-
const hour = recordToRow(record.id, record.data);
|
|
10045
|
-
if (hour === null) continue;
|
|
10046
|
-
this.ledger.stage(hour);
|
|
10047
|
-
for (const path of hour.paths) {
|
|
10048
|
-
const parsed = parseSegmentPath(path);
|
|
10049
|
-
if (parsed === null) continue;
|
|
10050
|
-
rows.push({
|
|
10051
|
-
deviceId: parsed.deviceId,
|
|
10052
|
-
profile: parsed.profile,
|
|
10053
|
-
startMs: parsed.startMs,
|
|
10054
|
-
durMs: parsed.durMs,
|
|
10055
|
-
bytes: parsed.bytes,
|
|
10056
|
-
path,
|
|
10057
|
-
locationId: hour.locationId
|
|
10058
|
-
});
|
|
10059
|
-
}
|
|
10060
|
-
}
|
|
10061
|
-
return rows.toSorted((a, b) => a.startMs - b.startMs);
|
|
10062
|
-
}
|
|
10063
9827
|
/**
|
|
10064
|
-
*
|
|
10065
|
-
* default.
|
|
9828
|
+
* Forget ONE hour row outright, and answer the segments it accounted for.
|
|
10066
9829
|
*
|
|
10067
|
-
*
|
|
10068
|
-
*
|
|
10069
|
-
*
|
|
10070
|
-
*
|
|
10071
|
-
* it selects nothing. A partial repair finishes on the next boot; until then
|
|
10072
|
-
* the affected disks UNDER-report, which prunes less and never more.
|
|
9830
|
+
* The caller must have TRUSTED evidence that the hour holds nothing — a
|
|
9831
|
+
* directory listing under a volume anchor that agreed twice (D49), or a
|
|
9832
|
+
* per-path confirmation against every root there is. This method makes no
|
|
9833
|
+
* such check itself and can make none: it is handed a verdict, not evidence.
|
|
10073
9834
|
*
|
|
10074
|
-
*
|
|
10075
|
-
*
|
|
10076
|
-
*
|
|
10077
|
-
* unbounded materialise here would be the whole-archive read this stage
|
|
10078
|
-
* removed, wearing a different name.
|
|
9835
|
+
* It exists because {@link adoptWalkedHour} deliberately refuses an EMPTY
|
|
9836
|
+
* listing — "the walk found nothing" is what an unmounted root looks like —
|
|
9837
|
+
* so the one repair a listing cannot express is the one that empties an hour.
|
|
10079
9838
|
*/
|
|
10080
|
-
async
|
|
10081
|
-
|
|
10082
|
-
|
|
10083
|
-
const
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
|
|
10087
|
-
|
|
10088
|
-
|
|
10089
|
-
|
|
10090
|
-
|
|
10091
|
-
|
|
10092
|
-
|
|
10093
|
-
|
|
10094
|
-
|
|
10095
|
-
|
|
10096
|
-
await this.materialize(pending.slice(0, REPAIR_MAX_MATERIALIZE));
|
|
10097
|
-
for (const record of stale) {
|
|
10098
|
-
const held = this.ledger.get(record.id);
|
|
10099
|
-
if (held === void 0 || !held.pathsResident || held.segments === 0) continue;
|
|
10100
|
-
await this.ledger.put(held);
|
|
10101
|
-
repaired += 1;
|
|
10102
|
-
}
|
|
10103
|
-
} catch (err) {
|
|
10104
|
-
this.logger.warn("recorder: segment-hour aggregate repair failed — disks under-report", { meta: {
|
|
10105
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION,
|
|
10106
|
-
error: String(err)
|
|
10107
|
-
} });
|
|
10108
|
-
return repaired;
|
|
9839
|
+
async forgetHour(deviceId, profile, locationId, hour) {
|
|
9840
|
+
const key = segmentHourKey(deviceId, profile, locationId, hour);
|
|
9841
|
+
return this.serialized(key, async () => {
|
|
9842
|
+
const held = this.ledger.get(key);
|
|
9843
|
+
if (held === void 0) return 0;
|
|
9844
|
+
await this.ledger.forget(key);
|
|
9845
|
+
this.accounted.delete(key);
|
|
9846
|
+
return held.segments;
|
|
9847
|
+
});
|
|
9848
|
+
}
|
|
9849
|
+
async forgetScope(scope) {
|
|
9850
|
+
let forgotten = 0;
|
|
9851
|
+
for (const row of this.ledger.snapshot()) {
|
|
9852
|
+
if (!SegmentHourLedger.inScope(row, scope)) continue;
|
|
9853
|
+
await this.ledger.forget(row.key);
|
|
9854
|
+
forgotten += row.segments;
|
|
10109
9855
|
}
|
|
10110
|
-
|
|
10111
|
-
rows: repaired,
|
|
10112
|
-
collection: RECORDING_SEGMENT_HOURS_COLLECTION
|
|
10113
|
-
} });
|
|
10114
|
-
return repaired;
|
|
9856
|
+
return forgotten;
|
|
10115
9857
|
}
|
|
10116
9858
|
/**
|
|
10117
9859
|
* Write-behind: a just-finalized segment joins its hour row.
|
|
@@ -10119,8 +9861,8 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10119
9861
|
* THE hot path — every profile-writer lands here every `segmentSeconds`,
|
|
10120
9862
|
* ~3.4 times a second fleet-wide. The append is {@link appendSegmentPath}:
|
|
10121
9863
|
* one `parseSegmentPath` and four scalar updates, whatever the hour already
|
|
10122
|
-
* holds
|
|
10123
|
-
*
|
|
9864
|
+
* holds — and since D523 it needs no read of the store at all, because the
|
|
9865
|
+
* mirror carries the whole row.
|
|
10124
9866
|
*/
|
|
10125
9867
|
async recordSegment(row) {
|
|
10126
9868
|
const { key, hourStartMs: hour } = hourOf(row);
|
|
@@ -10133,7 +9875,12 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10133
9875
|
entryMs: row.startMs
|
|
10134
9876
|
});
|
|
10135
9877
|
await this.serialized(key, async () => {
|
|
10136
|
-
|
|
9878
|
+
const seen = this.accounted.get(key);
|
|
9879
|
+
if (seen !== void 0 && seen.has(row.path)) return;
|
|
9880
|
+
if (seen === void 0) this.accounted.set(key, new Set([row.path]));
|
|
9881
|
+
else seen.add(row.path);
|
|
9882
|
+
const held = this.ledger.get(key);
|
|
9883
|
+
if (held === void 0) {
|
|
10137
9884
|
await this.ledger.put(segmentHourRow({
|
|
10138
9885
|
key,
|
|
10139
9886
|
deviceId: row.deviceId,
|
|
@@ -10144,25 +9891,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10144
9891
|
}));
|
|
10145
9892
|
return;
|
|
10146
9893
|
}
|
|
10147
|
-
const held = await this.resident(key, row.deviceId);
|
|
10148
|
-
if (held === null) {
|
|
10149
|
-
this.refused.add(key);
|
|
10150
|
-
this.logger.warn("recorder: hour detail unavailable — this finalized segment is NOT in the durable hour row yet", {
|
|
10151
|
-
tags: { deviceId: row.deviceId },
|
|
10152
|
-
meta: {
|
|
10153
|
-
key,
|
|
10154
|
-
path: row.path
|
|
10155
|
-
}
|
|
10156
|
-
});
|
|
10157
|
-
return;
|
|
10158
|
-
}
|
|
10159
|
-
if (held.paths.includes(row.path)) return;
|
|
10160
9894
|
await this.ledger.put(appendSegmentPath(held, row.path));
|
|
10161
9895
|
});
|
|
10162
9896
|
}
|
|
10163
9897
|
/**
|
|
10164
|
-
* Eviction confirmed these
|
|
10165
|
-
* hour when
|
|
9898
|
+
* Eviction confirmed these segments are gone from disk. Take them out of the
|
|
9899
|
+
* hour's account; forget the hour when nothing is left.
|
|
9900
|
+
*
|
|
9901
|
+
* The aggregate shrinks by exactly what was removed. `minStartMs` /
|
|
9902
|
+
* `maxStartMs` are NOT re-derived — they cannot be, without a listing — so a
|
|
9903
|
+
* partly-evicted hour keeps extremes that may be older or newer than what
|
|
9904
|
+
* survives. That error is bounded by the hour itself (both values always lie
|
|
9905
|
+
* inside `[hourStartMs, hourStartMs + 1h)`), and the row is UNSEALED so the
|
|
9906
|
+
* next listing of it re-derives both exactly.
|
|
10166
9907
|
*/
|
|
10167
9908
|
async dropSegments(rows) {
|
|
10168
9909
|
const doomed = /* @__PURE__ */ new Map();
|
|
@@ -10172,70 +9913,50 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10172
9913
|
if (entry === void 0) {
|
|
10173
9914
|
doomed.set(key, {
|
|
10174
9915
|
deviceId: row.deviceId,
|
|
10175
|
-
|
|
10176
|
-
|
|
10177
|
-
continue;
|
|
10178
|
-
}
|
|
10179
|
-
entry.paths.add(row.path);
|
|
10180
|
-
}
|
|
10181
|
-
for (const [key, { deviceId, paths }] of doomed) {
|
|
10182
|
-
if (!this.ledger.has(key)) continue;
|
|
10183
|
-
const held = await this.resident(key, deviceId);
|
|
10184
|
-
if (held === null) {
|
|
10185
|
-
this.logger.warn("recorder: hour detail unavailable — these evicted paths stay in the durable hour row until a walk or the next eviction confirms them", {
|
|
10186
|
-
tags: { deviceId },
|
|
10187
|
-
meta: {
|
|
10188
|
-
key,
|
|
10189
|
-
paths: paths.size
|
|
10190
|
-
}
|
|
9916
|
+
bytes: row.bytes,
|
|
9917
|
+
segments: 1
|
|
10191
9918
|
});
|
|
10192
9919
|
continue;
|
|
10193
9920
|
}
|
|
10194
|
-
|
|
10195
|
-
|
|
9921
|
+
entry.bytes += row.bytes;
|
|
9922
|
+
entry.segments += 1;
|
|
9923
|
+
}
|
|
9924
|
+
for (const [key, dropped] of doomed) await this.serialized(key, async () => {
|
|
9925
|
+
const held = this.ledger.get(key);
|
|
9926
|
+
if (held === void 0) return;
|
|
9927
|
+
const segments = held.segments - dropped.segments;
|
|
9928
|
+
if (segments <= 0) {
|
|
10196
9929
|
await this.ledger.forget(key);
|
|
10197
|
-
|
|
9930
|
+
this.accounted.delete(key);
|
|
9931
|
+
return;
|
|
10198
9932
|
}
|
|
10199
|
-
|
|
10200
|
-
await this.ledger.put(segmentHourRow({
|
|
9933
|
+
await this.ledger.put({
|
|
10201
9934
|
...held,
|
|
10202
|
-
|
|
10203
|
-
|
|
10204
|
-
|
|
9935
|
+
sealed: false,
|
|
9936
|
+
segments,
|
|
9937
|
+
bytes: Math.max(0, held.bytes - dropped.bytes)
|
|
9938
|
+
});
|
|
9939
|
+
});
|
|
10205
9940
|
}
|
|
10206
9941
|
/**
|
|
10207
|
-
* Move `rows` from the hour
|
|
9942
|
+
* Move `rows` from the hour they are accounted under today to the SAME hour
|
|
10208
9943
|
* at `toLocationId` — the DURABLE half of a relocation.
|
|
10209
9944
|
*
|
|
10210
|
-
* Without this the mover re-points only `RecordingIndex`, which is RAM
|
|
10211
|
-
*
|
|
10212
|
-
*
|
|
10213
|
-
*
|
|
10214
|
-
* no longer holds the file. Playback does not stat (that syscall per segment
|
|
10215
|
-
* was removed deliberately), so the result is a silent black window; and once
|
|
10216
|
-
* the drained location is deleted, `collectRangeSegments` drops the rows
|
|
10217
|
-
* outright and the footage is invisible. That is the shape of the 2 131
|
|
10218
|
-
* `recordings:high` ghost segments already on this hub.
|
|
9945
|
+
* Without this the mover re-points only `RecordingIndex`, which is RAM, and
|
|
9946
|
+
* the drained location keeps the bytes on every storage page for the life of
|
|
9947
|
+
* the hub. That is the shape of the 2 131 `recordings:high` ghost rows this
|
|
9948
|
+
* hub carried.
|
|
10219
9949
|
*
|
|
10220
9950
|
* **The ordering is the invariant, not an implementation detail.** The
|
|
10221
9951
|
* destination row is written BEFORE the source row is shrunk, and the caller
|
|
10222
9952
|
* has not yet deleted the source bytes — so the only state a crash can leave
|
|
10223
|
-
* is "
|
|
10224
|
-
*
|
|
10225
|
-
*
|
|
10226
|
-
* the index `hydrated`: nothing would ever walk that hour again, and the
|
|
10227
|
-
* segment would sit on disk permanently unreachable.
|
|
10228
|
-
*
|
|
10229
|
-
* **Two path sets, not one.** A path can arrive at the target under a
|
|
10230
|
-
* DIFFERENT name than it leaves the source under: a mover that found a file
|
|
10231
|
-
* larger than its recorded size reconciles the row, and the byte count is
|
|
10232
|
-
* part of the name. So the source row is shrunk by what is LEAVING it and the
|
|
10233
|
-
* target row grows by what is ARRIVING — conflating them would leave the
|
|
10234
|
-
* source row naming a file the caller is about to delete.
|
|
9953
|
+
* is "accounted at BOTH locations, both of which hold the files", which a
|
|
9954
|
+
* listing of either hour resolves. The reverse order has a window in which
|
|
9955
|
+
* the segments are accounted NOWHERE.
|
|
10235
9956
|
*
|
|
10236
|
-
*
|
|
10237
|
-
*
|
|
10238
|
-
* the
|
|
9957
|
+
* Both rows come out UNSEALED: each carries an aggregate that arithmetic
|
|
9958
|
+
* moved rather than a listing derived, and the next walk of either hour fixes
|
|
9959
|
+
* whatever the arithmetic rounded.
|
|
10239
9960
|
*/
|
|
10240
9961
|
async repointSegments(rows, toLocationId) {
|
|
10241
9962
|
const byHour = /* @__PURE__ */ new Map();
|
|
@@ -10243,87 +9964,75 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10243
9964
|
if (row.locationId === toLocationId) continue;
|
|
10244
9965
|
const { key } = hourOf(row);
|
|
10245
9966
|
const entry = byHour.get(key);
|
|
10246
|
-
if (entry === void 0)
|
|
10247
|
-
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
entry.arriving.add(row.path);
|
|
10254
|
-
}
|
|
10255
|
-
}
|
|
10256
|
-
for (const [sourceKey, { row, leaving, arriving }] of byHour) {
|
|
10257
|
-
const held = await this.resident(sourceKey, row.deviceId);
|
|
10258
|
-
if (held === null) {
|
|
10259
|
-
this.logger.warn("recorder: hour detail unavailable — this relocation is REFUSED and nothing is deleted; the segments stay where they are", {
|
|
10260
|
-
tags: { deviceId: row.deviceId },
|
|
10261
|
-
meta: {
|
|
10262
|
-
key: sourceKey,
|
|
10263
|
-
toLocationId,
|
|
10264
|
-
paths: leaving.size
|
|
10265
|
-
}
|
|
10266
|
-
});
|
|
10267
|
-
return false;
|
|
10268
|
-
}
|
|
10269
|
-
const targetKey = segmentHourKey(row.deviceId, row.profile, toLocationId, held.hourStartMs);
|
|
10270
|
-
const target = this.ledger.has(targetKey) ? await this.resident(targetKey, row.deviceId) : void 0;
|
|
10271
|
-
if (target === null) {
|
|
10272
|
-
this.logger.warn("recorder: TARGET hour detail unavailable — this relocation is REFUSED; appending to a summary would rewrite the hour as these paths alone", {
|
|
10273
|
-
tags: { deviceId: row.deviceId },
|
|
10274
|
-
meta: {
|
|
10275
|
-
key: targetKey,
|
|
10276
|
-
fromLocationId: row.locationId,
|
|
10277
|
-
paths: arriving.size
|
|
10278
|
-
}
|
|
9967
|
+
if (entry === void 0) {
|
|
9968
|
+
byHour.set(key, {
|
|
9969
|
+
row,
|
|
9970
|
+
bytes: row.bytes,
|
|
9971
|
+
segments: 1,
|
|
9972
|
+
minStartMs: row.startMs,
|
|
9973
|
+
maxStartMs: row.startMs
|
|
10279
9974
|
});
|
|
10280
|
-
|
|
9975
|
+
continue;
|
|
10281
9976
|
}
|
|
10282
|
-
|
|
10283
|
-
|
|
9977
|
+
entry.bytes += row.bytes;
|
|
9978
|
+
entry.segments += 1;
|
|
9979
|
+
entry.minStartMs = Math.min(entry.minStartMs, row.startMs);
|
|
9980
|
+
entry.maxStartMs = Math.max(entry.maxStartMs, row.startMs);
|
|
9981
|
+
}
|
|
9982
|
+
for (const [sourceKey, moved] of byHour) {
|
|
9983
|
+
const hour = hourStartMs(moved.row.startMs);
|
|
9984
|
+
const targetKey = segmentHourKey(moved.row.deviceId, moved.row.profile, toLocationId, hour);
|
|
9985
|
+
const target = this.ledger.get(targetKey);
|
|
9986
|
+
const targetIdentity = {
|
|
10284
9987
|
key: targetKey,
|
|
10285
|
-
deviceId: row.deviceId,
|
|
10286
|
-
profile: row.profile,
|
|
9988
|
+
deviceId: moved.row.deviceId,
|
|
9989
|
+
profile: moved.row.profile,
|
|
10287
9990
|
locationId: toLocationId,
|
|
10288
|
-
hourStartMs:
|
|
10289
|
-
|
|
9991
|
+
hourStartMs: hour
|
|
9992
|
+
};
|
|
9993
|
+
const targetEmpty = target === void 0 || target.segments === 0;
|
|
9994
|
+
await this.ledger.put(hourRowOf(targetIdentity, {
|
|
9995
|
+
sealed: false,
|
|
9996
|
+
bytes: (target?.bytes ?? 0) + moved.bytes,
|
|
9997
|
+
segments: (target?.segments ?? 0) + moved.segments,
|
|
9998
|
+
minStartMs: targetEmpty ? moved.minStartMs : Math.min(target.minStartMs, moved.minStartMs),
|
|
9999
|
+
maxStartMs: targetEmpty ? moved.maxStartMs : Math.max(target.maxStartMs, moved.maxStartMs)
|
|
10290
10000
|
}));
|
|
10291
|
-
const
|
|
10292
|
-
if (
|
|
10293
|
-
|
|
10001
|
+
const held = this.ledger.get(sourceKey);
|
|
10002
|
+
if (held === void 0) continue;
|
|
10003
|
+
const segments = held.segments - moved.segments;
|
|
10004
|
+
if (segments <= 0) {
|
|
10005
|
+
await this.ledger.forget(sourceKey);
|
|
10006
|
+
continue;
|
|
10007
|
+
}
|
|
10008
|
+
await this.ledger.put({
|
|
10294
10009
|
...held,
|
|
10295
|
-
|
|
10296
|
-
|
|
10010
|
+
sealed: false,
|
|
10011
|
+
segments,
|
|
10012
|
+
bytes: Math.max(0, held.bytes - moved.bytes)
|
|
10013
|
+
});
|
|
10297
10014
|
}
|
|
10298
|
-
return true;
|
|
10299
10015
|
}
|
|
10300
10016
|
/**
|
|
10301
|
-
* After a disk walk of `deviceIds`, make the ledger match the index
|
|
10302
|
-
* two D148 exceptions:
|
|
10017
|
+
* After a disk walk of `deviceIds`, make the ledger match what the index now
|
|
10018
|
+
* holds — with two D148 exceptions:
|
|
10303
10019
|
*
|
|
10304
10020
|
* - an hour the walk did not see is dropped ONLY if it is in the past;
|
|
10305
10021
|
* the current hour may still hold a live writer whose file the readdir
|
|
10306
10022
|
* raced;
|
|
10307
|
-
* - the current hour
|
|
10308
|
-
*
|
|
10023
|
+
* - the current hour takes the LARGER of the two accounts rather than the
|
|
10024
|
+
* walk's, for the same race.
|
|
10309
10025
|
*
|
|
10310
10026
|
* Devices not in `deviceIds` are left untouched (a partial walk must not
|
|
10311
10027
|
* prune cameras it did not look at).
|
|
10312
10028
|
*
|
|
10313
10029
|
* And a walk this ledger does not TRUST prunes nothing at all — see
|
|
10314
|
-
* {@link refusedPrunes}. What such a walk FOUND is still real, so its
|
|
10315
|
-
*
|
|
10316
|
-
*
|
|
10317
|
-
* Since D287 the walked devices' hours are MATERIALISED first: every
|
|
10318
|
-
* comparison below reads `row.paths`, and a summary's empty `paths` would
|
|
10319
|
-
* read as "the ledger holds nothing here" — which is the union collapsing to
|
|
10320
|
-
* the walk's own view, i.e. the prune the refusal machinery exists to
|
|
10321
|
-
* prevent. An hour whose detail could not be read is left additive-only.
|
|
10030
|
+
* {@link refusedPrunes}. What such a walk FOUND is still real, so its larger
|
|
10031
|
+
* account is taken; what it did not find decides nothing (D49).
|
|
10322
10032
|
*/
|
|
10323
10033
|
async reconcileFromIndex(index, deviceIds, nowMs) {
|
|
10324
10034
|
const walked = new Set(deviceIds);
|
|
10325
10035
|
const currentHour = hourStartMs(nowMs);
|
|
10326
|
-
await this.materializeDevices(walked);
|
|
10327
10036
|
const fromIndex = /* @__PURE__ */ new Map();
|
|
10328
10037
|
for (const deviceId of deviceIds) for (const seg of index.segments(deviceId)) {
|
|
10329
10038
|
const { key, hourStartMs: hour } = hourOf(seg);
|
|
@@ -10342,7 +10051,6 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10342
10051
|
const refused = this.refusedPrunes(walked, fromIndex);
|
|
10343
10052
|
for (const row of this.ledger.snapshot()) {
|
|
10344
10053
|
if (!walked.has(row.deviceId)) continue;
|
|
10345
|
-
if (!row.pathsResident) continue;
|
|
10346
10054
|
const disk = fromIndex.get(row.key);
|
|
10347
10055
|
const additiveOnly = row.hourStartMs === currentHour || refused.devices.has(row.deviceId) || refused.locations.has(locationKey(row.deviceId, row.locationId));
|
|
10348
10056
|
if (disk === void 0) {
|
|
@@ -10351,14 +10059,10 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
10351
10059
|
continue;
|
|
10352
10060
|
}
|
|
10353
10061
|
if (additiveOnly) {
|
|
10354
|
-
|
|
10355
|
-
if (!samePaths(union, row.paths)) await this.ledger.put(segmentHourRow({
|
|
10356
|
-
...row,
|
|
10357
|
-
paths: union
|
|
10358
|
-
}));
|
|
10062
|
+
if (disk.segments > row.segments) await this.ledger.put(disk);
|
|
10359
10063
|
continue;
|
|
10360
10064
|
}
|
|
10361
|
-
if (
|
|
10065
|
+
if (disk.segments !== row.segments || disk.bytes !== row.bytes) await this.ledger.put(disk);
|
|
10362
10066
|
}
|
|
10363
10067
|
for (const [key, disk] of fromIndex) {
|
|
10364
10068
|
if (this.ledger.has(key)) continue;
|
|
@@ -10477,44 +10181,45 @@ function locationKey(deviceId, locationId) {
|
|
|
10477
10181
|
* skips it forever. Under D148 as written, a ghost row can never be forgotten
|
|
10478
10182
|
* by any mechanism — which is why the live ones have stood for thirteen days.
|
|
10479
10183
|
*
|
|
10480
|
-
* The resolution is that
|
|
10481
|
-
*
|
|
10482
|
-
* (`590/high/2026/08/15/04
|
|
10483
|
-
* currently-resolvable root
|
|
10484
|
-
*
|
|
10485
|
-
*
|
|
10486
|
-
* one `readdir` at a time.** That, and nothing weaker, is what licenses the
|
|
10184
|
+
* The resolution is that the ghost row's root does not resolve but its hour
|
|
10185
|
+
* DIRECTORY is addressable anywhere: the path convention
|
|
10186
|
+
* (`590/high/2026/08/15/04/`) is a function of the row's own fields, so the same
|
|
10187
|
+
* hour can be listed under every currently-resolvable root. **A listing of the
|
|
10188
|
+
* hour directory at every candidate root is exactly the walk D148 sanctions,
|
|
10189
|
+
* done one root at a time.** That, and nothing weaker, is what licenses the
|
|
10487
10190
|
* forget. Two consequences ride with it:
|
|
10488
10191
|
*
|
|
10489
|
-
* - **
|
|
10490
|
-
*
|
|
10491
|
-
* stands.
|
|
10492
|
-
* - **An error is not an absence** (D49). A
|
|
10493
|
-
* with nothing decided — never "not
|
|
10192
|
+
* - **Every root must answer.** An hour is forgotten only when every candidate
|
|
10193
|
+
* root listed its directory and every one of them came back EMPTY. One root
|
|
10194
|
+
* that could not answer and the hour stands.
|
|
10195
|
+
* - **An error is not an absence** (D49). A listing that fails, or that reports
|
|
10196
|
+
* "I could not decide", ends the hour with nothing decided — never "not
|
|
10197
|
+
* found". Whether an absent directory means "empty" or "this disk is gone" is
|
|
10198
|
+
* the lister's question, and it answers it by reading the ROOT.
|
|
10199
|
+
*
|
|
10200
|
+
* Until D523 this probed each of the row's `paths` at each root, one
|
|
10201
|
+
* `storage.exists` at a time — up to ~1 000 syscalls to prove one hour absent.
|
|
10202
|
+
* The column is gone and the row names no file, so the unit is the directory,
|
|
10203
|
+
* which is both cheaper and the same evidence.
|
|
10494
10204
|
*
|
|
10495
10205
|
* ## Repoint versus forget
|
|
10496
10206
|
*
|
|
10497
10207
|
* The asymmetry is the whole safety argument, and it is deliberately cheap in
|
|
10498
10208
|
* the safe direction:
|
|
10499
10209
|
*
|
|
10500
|
-
* - **
|
|
10501
|
-
* repointed to that root's location
|
|
10502
|
-
*
|
|
10503
|
-
* deleting here would destroy the index of footage
|
|
10504
|
-
* failure D291 was written to prevent.
|
|
10505
|
-
* - **every
|
|
10506
|
-
*
|
|
10507
|
-
* So proving presence costs one probe and proving absence costs all of them,
|
|
10508
|
-
* which is the right way round: the expensive proof is the destructive one.
|
|
10210
|
+
* - **a non-empty listing at any root ⇒ the row is MIS-POINTED, not orphaned.**
|
|
10211
|
+
* It is repointed to that root's location, with the segments that listing
|
|
10212
|
+
* names, and the hour is finished. A renamed or re-created location produces
|
|
10213
|
+
* exactly this shape, and deleting here would destroy the index of footage
|
|
10214
|
+
* that EXISTS, which is the failure D291 was written to prevent.
|
|
10215
|
+
* - **every root lists an EMPTY hour ⇒ orphaned**, and only then forgotten.
|
|
10509
10216
|
*
|
|
10510
10217
|
* ## Pacing
|
|
10511
10218
|
*
|
|
10512
|
-
* The recordings array is
|
|
10513
|
-
*
|
|
10514
|
-
*
|
|
10515
|
-
*
|
|
10516
|
-
* Work it does not finish is left for the next pass and REPORTED, never
|
|
10517
|
-
* silently dropped.
|
|
10219
|
+
* The recordings array is the resource every live read competes for. This is a
|
|
10220
|
+
* janitor: bounded hours per pass, a hard probe ceiling, and a caller-supplied
|
|
10221
|
+
* `pace()` awaited between probe batches. Work it does not finish is left for
|
|
10222
|
+
* the next pass and REPORTED, never silently dropped.
|
|
10518
10223
|
*/
|
|
10519
10224
|
var HOUR_MS$3 = 36e5;
|
|
10520
10225
|
/**
|
|
@@ -10663,22 +10368,12 @@ var GhostLocationSweeper = class {
|
|
|
10663
10368
|
refuse(row, "live-tail", `hourStartMs=${row.hourStartMs}`);
|
|
10664
10369
|
continue;
|
|
10665
10370
|
}
|
|
10666
|
-
|
|
10667
|
-
|
|
10668
|
-
refuse(row, "detail-unavailable", "point read failed");
|
|
10669
|
-
continue;
|
|
10670
|
-
}
|
|
10671
|
-
if (paths.length === 0) {
|
|
10672
|
-
refuse(row, "no-paths", "the row materialised empty");
|
|
10673
|
-
continue;
|
|
10674
|
-
}
|
|
10675
|
-
const rows = this.rowsOf(row, paths);
|
|
10676
|
-
if (rows === null) {
|
|
10677
|
-
refuse(row, "unparseable-paths", "a path does not parse into this hour");
|
|
10371
|
+
if (row.segments === 0) {
|
|
10372
|
+
refuse(row, "no-paths", "the row accounts for no segments");
|
|
10678
10373
|
continue;
|
|
10679
10374
|
}
|
|
10680
10375
|
const before = probes;
|
|
10681
|
-
const verdict = await this.probeHour(
|
|
10376
|
+
const verdict = await this.probeHour(row, roots, maxProbes - probes, probes);
|
|
10682
10377
|
probes += verdict.probes;
|
|
10683
10378
|
tallyOf(row.deviceId).probes += probes - before;
|
|
10684
10379
|
if (verdict.outcome.kind === "refused") {
|
|
@@ -10687,10 +10382,12 @@ var GhostLocationSweeper = class {
|
|
|
10687
10382
|
}
|
|
10688
10383
|
if (verdict.outcome.kind === "found") {
|
|
10689
10384
|
const target = locationForHydratedProfile(verdict.outcome.root.aliases, row.profile);
|
|
10690
|
-
|
|
10691
|
-
|
|
10385
|
+
const rows = this.rowsOf(row, verdict.outcome.paths);
|
|
10386
|
+
if (rows === null) {
|
|
10387
|
+
refuse(row, "unparseable-paths", "a listed name does not parse into this hour");
|
|
10692
10388
|
continue;
|
|
10693
10389
|
}
|
|
10390
|
+
await this.deps.ledger.repointSegments(rows, target.id);
|
|
10694
10391
|
this.deps.applyToIndex(rows, target.id);
|
|
10695
10392
|
repointedHours += 1;
|
|
10696
10393
|
segmentsRepointed += rows.length;
|
|
@@ -10700,19 +10397,19 @@ var GhostLocationSweeper = class {
|
|
|
10700
10397
|
tally.targets.add(target.id);
|
|
10701
10398
|
continue;
|
|
10702
10399
|
}
|
|
10703
|
-
await this.deps.ledger.
|
|
10400
|
+
const forgotten = await this.deps.ledger.forgetHour(row.deviceId, row.profile, row.locationId, row.hourStartMs);
|
|
10704
10401
|
if (this.deps.ledger.knowsHour(row.deviceId, row.profile, row.locationId, row.hourStartMs)) {
|
|
10705
|
-
refuse(row, "durable-refused", "the ledger still holds the hour after the
|
|
10402
|
+
refuse(row, "durable-refused", "the ledger still holds the hour after the forget");
|
|
10706
10403
|
continue;
|
|
10707
10404
|
}
|
|
10708
|
-
this.deps.
|
|
10405
|
+
this.deps.forgetInIndex(row.deviceId, row.locationId, row.hourStartMs);
|
|
10709
10406
|
forgottenHours += 1;
|
|
10710
|
-
segmentsForgotten +=
|
|
10711
|
-
bytesForgotten +=
|
|
10407
|
+
segmentsForgotten += forgotten;
|
|
10408
|
+
bytesForgotten += row.bytes;
|
|
10712
10409
|
const tally = tallyOf(row.deviceId);
|
|
10713
10410
|
tally.forgottenHours += 1;
|
|
10714
|
-
tally.forgottenSegments +=
|
|
10715
|
-
tally.forgottenBytes +=
|
|
10411
|
+
tally.forgottenSegments += forgotten;
|
|
10412
|
+
tally.forgottenBytes += row.bytes;
|
|
10716
10413
|
}
|
|
10717
10414
|
for (const [deviceId, tally] of byDevice) this.deps.logger.info("recorder: ghost-location reconcile — camera summary", {
|
|
10718
10415
|
tags: { deviceId },
|
|
@@ -10794,60 +10491,82 @@ var GhostLocationSweeper = class {
|
|
|
10794
10491
|
return out;
|
|
10795
10492
|
}
|
|
10796
10493
|
/**
|
|
10797
|
-
*
|
|
10494
|
+
* List this hour's directory under EVERY candidate root.
|
|
10495
|
+
*
|
|
10496
|
+
* A non-empty listing at one root makes the hour mis-pointed. Two roots
|
|
10497
|
+
* holding it is a refusal, not a choice: repointing would be a guess about
|
|
10498
|
+
* which copy is the footage. Absence is only concluded when every root
|
|
10499
|
+
* answered and every answer was empty — a listing that could not decide
|
|
10500
|
+
* (`null`) or threw ends the hour with nothing decided (D49).
|
|
10501
|
+
*
|
|
10502
|
+
* The whole loop runs even after a hit, because "found at exactly one root"
|
|
10503
|
+
* is a claim about every root and cannot be short-circuited without giving up
|
|
10504
|
+
* the multi-root refusal.
|
|
10798
10505
|
*
|
|
10799
|
-
*
|
|
10800
|
-
*
|
|
10801
|
-
*
|
|
10802
|
-
*
|
|
10506
|
+
* `spent` is the listings the PASS has already made, not this hour's, and the
|
|
10507
|
+
* brake fires on that running total. Since D523 an hour costs one listing per
|
|
10508
|
+
* physical root — two or three on this hub — so a counter that reset per hour
|
|
10509
|
+
* would never reach {@link GHOST_PROBE_BATCH} and the `pace()` dep would be
|
|
10510
|
+
* dead on every real topology.
|
|
10803
10511
|
*/
|
|
10804
|
-
async probeHour(
|
|
10512
|
+
async probeHour(row, roots, budget, spent) {
|
|
10805
10513
|
let used = 0;
|
|
10806
|
-
|
|
10807
|
-
|
|
10808
|
-
|
|
10809
|
-
|
|
10514
|
+
const found = [];
|
|
10515
|
+
for (const root of roots) {
|
|
10516
|
+
if (used >= budget) return {
|
|
10517
|
+
outcome: {
|
|
10518
|
+
kind: "refused",
|
|
10519
|
+
reason: "probe-budget",
|
|
10520
|
+
detail: `stopped after ${used} listings with ${roots.length} roots to probe`
|
|
10521
|
+
},
|
|
10522
|
+
probes: used
|
|
10523
|
+
};
|
|
10524
|
+
const alias = locationForHydratedProfile(root.aliases, row.profile);
|
|
10525
|
+
used += 1;
|
|
10526
|
+
if ((spent + used) % 16 === 0) await this.deps.pace();
|
|
10527
|
+
let listing;
|
|
10528
|
+
try {
|
|
10529
|
+
listing = await this.deps.listHour(alias.id, row.deviceId, row.profile, row.hourStartMs);
|
|
10530
|
+
} catch (err) {
|
|
10531
|
+
return {
|
|
10810
10532
|
outcome: {
|
|
10811
10533
|
kind: "refused",
|
|
10812
|
-
reason: "probe-
|
|
10813
|
-
detail:
|
|
10534
|
+
reason: "probe-error",
|
|
10535
|
+
detail: `${alias.id}: ${errMsg(err)}`
|
|
10814
10536
|
},
|
|
10815
10537
|
probes: used
|
|
10816
10538
|
};
|
|
10817
|
-
const alias = locationForHydratedProfile(root.aliases, row.profile);
|
|
10818
|
-
used += 1;
|
|
10819
|
-
if (used % 16 === 0) await this.deps.pace();
|
|
10820
|
-
try {
|
|
10821
|
-
if (await this.deps.exists(alias.id, row.path)) found.push(root);
|
|
10822
|
-
} catch (err) {
|
|
10823
|
-
return {
|
|
10824
|
-
outcome: {
|
|
10825
|
-
kind: "refused",
|
|
10826
|
-
reason: "probe-error",
|
|
10827
|
-
detail: `${alias.id}: ${errMsg(err)}`
|
|
10828
|
-
},
|
|
10829
|
-
probes: used
|
|
10830
|
-
};
|
|
10831
|
-
}
|
|
10832
10539
|
}
|
|
10833
|
-
if (
|
|
10540
|
+
if (listing === null) return {
|
|
10834
10541
|
outcome: {
|
|
10835
10542
|
kind: "refused",
|
|
10836
|
-
reason: "
|
|
10837
|
-
detail:
|
|
10838
|
-
},
|
|
10839
|
-
probes: used
|
|
10840
|
-
};
|
|
10841
|
-
const only = found[0];
|
|
10842
|
-
if (only !== void 0) return {
|
|
10843
|
-
outcome: {
|
|
10844
|
-
kind: "found",
|
|
10845
|
-
root: only,
|
|
10846
|
-
path: row.path
|
|
10543
|
+
reason: "detail-unavailable",
|
|
10544
|
+
detail: `${alias.id} could not say what its copy of this hour holds`
|
|
10847
10545
|
},
|
|
10848
10546
|
probes: used
|
|
10849
10547
|
};
|
|
10548
|
+
if (listing.length > 0) found.push({
|
|
10549
|
+
root,
|
|
10550
|
+
paths: listing
|
|
10551
|
+
});
|
|
10850
10552
|
}
|
|
10553
|
+
if (found.length > 1) return {
|
|
10554
|
+
outcome: {
|
|
10555
|
+
kind: "refused",
|
|
10556
|
+
reason: "found-at-multiple-roots",
|
|
10557
|
+
detail: found.map((hit) => hit.root.key).join(", ")
|
|
10558
|
+
},
|
|
10559
|
+
probes: used
|
|
10560
|
+
};
|
|
10561
|
+
const only = found[0];
|
|
10562
|
+
if (only !== void 0) return {
|
|
10563
|
+
outcome: {
|
|
10564
|
+
kind: "found",
|
|
10565
|
+
root: only.root,
|
|
10566
|
+
paths: only.paths
|
|
10567
|
+
},
|
|
10568
|
+
probes: used
|
|
10569
|
+
};
|
|
10851
10570
|
return {
|
|
10852
10571
|
outcome: { kind: "absent" },
|
|
10853
10572
|
probes: used
|
|
@@ -11228,34 +10947,12 @@ function emptyTally() {
|
|
|
11228
10947
|
orphanFiles: 0
|
|
11229
10948
|
};
|
|
11230
10949
|
}
|
|
11231
|
-
/**
|
|
11232
|
-
function
|
|
11233
|
-
|
|
11234
|
-
|
|
11235
|
-
|
|
11236
|
-
|
|
11237
|
-
* rows (`streamSegments` expands them whole), so no hour is ever split. */
|
|
11238
|
-
function claimsOf(rows) {
|
|
11239
|
-
const byHour = /* @__PURE__ */ new Map();
|
|
11240
|
-
for (const row of rows) {
|
|
11241
|
-
const hour = hourStartMs(row.startMs);
|
|
11242
|
-
const key = `${row.deviceId}|${row.profile}|${hour}`;
|
|
11243
|
-
const held = byHour.get(key);
|
|
11244
|
-
if (held) held.push(row);
|
|
11245
|
-
else byHour.set(key, [row]);
|
|
11246
|
-
}
|
|
11247
|
-
const out = [];
|
|
11248
|
-
for (const group of byHour.values()) {
|
|
11249
|
-
const first = group[0];
|
|
11250
|
-
if (first === void 0) continue;
|
|
11251
|
-
out.push({
|
|
11252
|
-
deviceId: first.deviceId,
|
|
11253
|
-
profile: first.profile,
|
|
11254
|
-
hourStartMs: hourStartMs(first.startMs),
|
|
11255
|
-
rows: group
|
|
11256
|
-
});
|
|
11257
|
-
}
|
|
11258
|
-
return out.toSorted((a, b) => a.hourStartMs - b.hourStartMs || a.deviceId - b.deviceId);
|
|
10950
|
+
/** Does this hour row fall inside the scope the operator asked for? */
|
|
10951
|
+
function inScope(row, scope) {
|
|
10952
|
+
if (scope.locationId !== void 0 && row.locationId !== scope.locationId) return false;
|
|
10953
|
+
if (scope.deviceId !== void 0 && row.deviceId !== scope.deviceId) return false;
|
|
10954
|
+
const profiles = scope.profiles ?? [];
|
|
10955
|
+
return profiles.length === 0 || profiles.includes(row.profile);
|
|
11259
10956
|
}
|
|
11260
10957
|
/**
|
|
11261
10958
|
* One reconcile pass over one location's claimed hours.
|
|
@@ -11353,20 +11050,35 @@ var LedgerWalkReconciler = class {
|
|
|
11353
11050
|
this.reportPass(report);
|
|
11354
11051
|
return report;
|
|
11355
11052
|
}
|
|
11356
|
-
/**
|
|
11053
|
+
/**
|
|
11054
|
+
* Walk the hour ROWS of the scope, oldest first.
|
|
11055
|
+
*
|
|
11056
|
+
* From the MIRROR, not a query. Since D523 a row is ten scalars, so the whole
|
|
11057
|
+
* claim set is already in RAM and costs nothing to fold — and the pass mutates
|
|
11058
|
+
* what it walks, which is exactly the shape that makes `offset` paging skip
|
|
11059
|
+
* half a scope. A snapshot taken once cannot skip.
|
|
11060
|
+
*/
|
|
11357
11061
|
async walk(state, scope) {
|
|
11358
|
-
|
|
11062
|
+
const claims = state.ledger.snapshot().filter((row) => inScope(row, scope)).toSorted((a, b) => a.hourStartMs - b.hourStartMs || a.deviceId - b.deviceId);
|
|
11063
|
+
for (const claim of claims) {
|
|
11359
11064
|
if (state.hoursClaimed >= state.maxHours) {
|
|
11360
11065
|
state.bounded = true;
|
|
11361
11066
|
return;
|
|
11362
11067
|
}
|
|
11363
11068
|
state.hoursClaimed += 1;
|
|
11364
|
-
state.segmentsClaimed += claim.
|
|
11069
|
+
state.segmentsClaimed += claim.segments;
|
|
11365
11070
|
if (!await this.reconcileHour(state, claim)) return;
|
|
11366
11071
|
}
|
|
11367
11072
|
}
|
|
11368
11073
|
/**
|
|
11369
|
-
*
|
|
11074
|
+
* Reconcile ONE hour: what its row ACCOUNTS FOR against what its directory
|
|
11075
|
+
* HOLDS, and repair the account from the directory.
|
|
11076
|
+
*
|
|
11077
|
+
* Since D523 the row names no files, so this is a comparison of counts and
|
|
11078
|
+
* bytes — and it does not need to name them: the repair is "make the account
|
|
11079
|
+
* the listing's", which is exact, and the RESIDENT index is reset to the same
|
|
11080
|
+
* listing so a row it holds for a file the directory does not is dropped with
|
|
11081
|
+
* it.
|
|
11370
11082
|
*
|
|
11371
11083
|
* `false` ends the whole pass — the only thing that returns it is an anchor
|
|
11372
11084
|
* that stopped agreeing, and every listing taken after that point is about a
|
|
@@ -11379,10 +11091,6 @@ var LedgerWalkReconciler = class {
|
|
|
11379
11091
|
return true;
|
|
11380
11092
|
}
|
|
11381
11093
|
const relDir = hourDirRelPath(claim.deviceId, claim.profile, claim.hourStartMs);
|
|
11382
|
-
for (const row of claim.rows) if (parseSegmentPath(row.path) === null || row.path !== `${relDir}/${fileName(row.path)}`) {
|
|
11383
|
-
this.skip(state, claim, "path-mismatch", row.path);
|
|
11384
|
-
return true;
|
|
11385
|
-
}
|
|
11386
11094
|
let listing;
|
|
11387
11095
|
try {
|
|
11388
11096
|
listing = await this.deps.listHour(state.location.root, relDir);
|
|
@@ -11394,36 +11102,55 @@ var LedgerWalkReconciler = class {
|
|
|
11394
11102
|
if (state.listings % 8 === 0) await this.deps.pace();
|
|
11395
11103
|
state.hoursWalked += 1;
|
|
11396
11104
|
tally.hoursWalked += 1;
|
|
11397
|
-
const
|
|
11398
|
-
if (
|
|
11105
|
+
const missingDirectory = listing === null;
|
|
11106
|
+
if (missingDirectory) {
|
|
11399
11107
|
state.hoursMissing += 1;
|
|
11400
11108
|
tally.hoursMissing += 1;
|
|
11401
11109
|
}
|
|
11402
|
-
|
|
11403
|
-
|
|
11404
|
-
|
|
11405
|
-
|
|
11110
|
+
const present = [];
|
|
11111
|
+
for (const name of listing ?? []) {
|
|
11112
|
+
if (!name.endsWith(".m4s")) continue;
|
|
11113
|
+
const relPath = `${relDir}/${name}`;
|
|
11114
|
+
const parsed = parseSegmentPath(relPath);
|
|
11115
|
+
if (parsed === null || hourStartMs(parsed.startMs) !== claim.hourStartMs) {
|
|
11116
|
+
this.skip(state, claim, "path-mismatch", relPath);
|
|
11117
|
+
return true;
|
|
11118
|
+
}
|
|
11119
|
+
present.push(relPath);
|
|
11120
|
+
}
|
|
11121
|
+
const listed = hourAggregate(present);
|
|
11122
|
+
const orphans = Math.max(0, listed.segments - claim.segments);
|
|
11123
|
+
if (orphans > 0) {
|
|
11124
|
+
state.orphanFiles += orphans;
|
|
11125
|
+
tally.orphanFiles += orphans;
|
|
11126
|
+
for (const path of present) {
|
|
11406
11127
|
if (state.orphanSample.length >= 50) break;
|
|
11407
|
-
state.orphanSample.push(
|
|
11128
|
+
state.orphanSample.push(path);
|
|
11408
11129
|
}
|
|
11409
|
-
this.deps.logger.warn("recorder:
|
|
11130
|
+
this.deps.logger.warn("recorder: an hour directory holds more segments than its durable row accounts for", {
|
|
11410
11131
|
tags: { deviceId: claim.deviceId },
|
|
11411
11132
|
meta: {
|
|
11412
11133
|
locationId: state.location.id,
|
|
11413
11134
|
profile: claim.profile,
|
|
11414
11135
|
hourStartMs: claim.hourStartMs,
|
|
11415
|
-
|
|
11416
|
-
|
|
11136
|
+
accounted: claim.segments,
|
|
11137
|
+
listed: listed.segments,
|
|
11138
|
+
apply: state.apply
|
|
11417
11139
|
}
|
|
11418
11140
|
});
|
|
11419
11141
|
}
|
|
11420
|
-
|
|
11421
|
-
|
|
11422
|
-
|
|
11142
|
+
const ghosts = Math.max(0, claim.segments - listed.segments);
|
|
11143
|
+
if (ghosts === 0) {
|
|
11144
|
+
if (state.apply && listed.segments > 0 && listed.bytes !== claim.bytes) await state.ledger.adoptWalkedHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs, present);
|
|
11145
|
+
if (state.apply) this.resetIndex(state, claim, present);
|
|
11146
|
+
return true;
|
|
11147
|
+
}
|
|
11148
|
+
const ghostBytes = Math.max(0, claim.bytes - listed.bytes);
|
|
11149
|
+
state.ghostSegments += ghosts;
|
|
11423
11150
|
state.ghostBytes += ghostBytes;
|
|
11424
|
-
tally.ghostSegments +=
|
|
11151
|
+
tally.ghostSegments += ghosts;
|
|
11425
11152
|
tally.ghostBytes += ghostBytes;
|
|
11426
|
-
if (
|
|
11153
|
+
if (listed.segments === 0) state.ghostHoursWhole += 1;
|
|
11427
11154
|
const anchor = await this.readAnchor(state.location);
|
|
11428
11155
|
if (!anchor.ok || anchor.volumeId !== state.anchor) {
|
|
11429
11156
|
state.refused = anchor.ok ? "anchor-moved" : anchor.refusal;
|
|
@@ -11439,56 +11166,52 @@ var LedgerWalkReconciler = class {
|
|
|
11439
11166
|
});
|
|
11440
11167
|
return false;
|
|
11441
11168
|
}
|
|
11442
|
-
this.deps.logger.warn("recorder: durable hour row
|
|
11169
|
+
this.deps.logger.warn("recorder: durable hour row accounts for segments the directory does not hold", {
|
|
11443
11170
|
tags: { deviceId: claim.deviceId },
|
|
11444
11171
|
meta: {
|
|
11445
11172
|
locationId: state.location.id,
|
|
11446
11173
|
profile: claim.profile,
|
|
11447
11174
|
hourStartMs: claim.hourStartMs,
|
|
11448
|
-
|
|
11449
|
-
|
|
11175
|
+
accounted: claim.segments,
|
|
11176
|
+
listed: listed.segments,
|
|
11177
|
+
ghosts,
|
|
11450
11178
|
ghostBytes,
|
|
11451
|
-
wholeHour:
|
|
11452
|
-
missingDirectory
|
|
11179
|
+
wholeHour: listed.segments === 0,
|
|
11180
|
+
missingDirectory,
|
|
11453
11181
|
apply: state.apply
|
|
11454
11182
|
}
|
|
11455
11183
|
});
|
|
11456
11184
|
if (!state.apply) return true;
|
|
11457
|
-
await state.ledger.
|
|
11458
|
-
if (
|
|
11459
|
-
this.skip(state, claim, "durable-refused", "the ledger still
|
|
11185
|
+
const forgotten = listed.segments === 0 ? await state.ledger.forgetHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs) : await this.adoptLanded(state, claim, present, ghosts);
|
|
11186
|
+
if (forgotten === null) {
|
|
11187
|
+
this.skip(state, claim, "durable-refused", "the ledger still accounts for the ghosts");
|
|
11460
11188
|
return true;
|
|
11461
11189
|
}
|
|
11462
|
-
this.
|
|
11463
|
-
state.forgottenSegments +=
|
|
11190
|
+
this.resetIndex(state, claim, present);
|
|
11191
|
+
state.forgottenSegments += forgotten;
|
|
11464
11192
|
state.forgottenBytes += ghostBytes;
|
|
11465
|
-
tally.forgottenSegments +=
|
|
11193
|
+
tally.forgottenSegments += forgotten;
|
|
11466
11194
|
return true;
|
|
11467
11195
|
}
|
|
11468
|
-
/** Did the durable drop actually land? */
|
|
11469
|
-
async forgetLanded(state, claim, ghosts) {
|
|
11470
|
-
const { deviceId, profile, hourStartMs: hour } = claim;
|
|
11471
|
-
if (!state.ledger.knowsHour(deviceId, profile, state.location.id, hour)) return true;
|
|
11472
|
-
const held = await state.ledger.hourPaths(deviceId, profile, state.location.id, hour);
|
|
11473
|
-
if (held === null) return false;
|
|
11474
|
-
const remaining = new Set(held);
|
|
11475
|
-
return ghosts.every((row) => !remaining.has(row.path));
|
|
11476
|
-
}
|
|
11477
11196
|
/**
|
|
11478
|
-
*
|
|
11197
|
+
* Adopt the listing, then READ THE ROW BACK.
|
|
11479
11198
|
*
|
|
11480
|
-
*
|
|
11481
|
-
*
|
|
11482
|
-
*
|
|
11199
|
+
* `adoptWalkedHour` refuses a listing that would erase more than half of what
|
|
11200
|
+
* the row accounts for — the guard against a root that answered short — and it
|
|
11201
|
+
* refuses silently, which a caller that counted it as done would turn into
|
|
11202
|
+
* telling the resident index a row is gone that the archive still holds.
|
|
11203
|
+
* `null` is that refusal.
|
|
11483
11204
|
*/
|
|
11484
|
-
|
|
11485
|
-
|
|
11486
|
-
const
|
|
11487
|
-
return
|
|
11488
|
-
|
|
11489
|
-
|
|
11490
|
-
|
|
11491
|
-
|
|
11205
|
+
async adoptLanded(state, claim, present, ghosts) {
|
|
11206
|
+
await state.ledger.adoptWalkedHour(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs, present);
|
|
11207
|
+
const after = state.ledger.hourRow(claim.deviceId, claim.profile, state.location.id, claim.hourStartMs);
|
|
11208
|
+
if (after === void 0) return claim.segments;
|
|
11209
|
+
return after.segments === claim.segments - ghosts ? ghosts : null;
|
|
11210
|
+
}
|
|
11211
|
+
/** Make the resident index's view of the hour the listing's, and say how many
|
|
11212
|
+
* index rows that dropped. */
|
|
11213
|
+
resetIndex(state, claim, present) {
|
|
11214
|
+
return this.deps.applyToIndex(claim.deviceId, state.location.id, claim.hourStartMs, present);
|
|
11492
11215
|
}
|
|
11493
11216
|
/** The volume anchor, or the refusal its absence produces. */
|
|
11494
11217
|
async readAnchor(location) {
|
|
@@ -17777,12 +17500,17 @@ var GHOST_LOCATION_PROBE_PAUSE_MS = 50;
|
|
|
17777
17500
|
/**
|
|
17778
17501
|
* Awaited every `WALK_HOUR_BATCH` hour listings by the ledger↔disk walk (D319).
|
|
17779
17502
|
*
|
|
17780
|
-
* Twice the ghost sweep's brake
|
|
17781
|
-
* heavier
|
|
17782
|
-
* `readdir`
|
|
17783
|
-
*
|
|
17784
|
-
*
|
|
17785
|
-
*
|
|
17503
|
+
* Twice the ghost sweep's brake. That used to be because the syscall between
|
|
17504
|
+
* brakes was far heavier — the sweep paced one `storage.exists`, this one an
|
|
17505
|
+
* hour `readdir` "measured at ~650 ms on the live share under load". Both
|
|
17506
|
+
* halves of that have changed: the sweep paces a directory listing too since
|
|
17507
|
+
* D523, and the 650 ms belonged to a FUSE mount that has been retired (an hour
|
|
17508
|
+
* of 345–349 files on `/recordings-2` lists in ~1 ms).
|
|
17509
|
+
*
|
|
17510
|
+
* It stays at twice anyway, for the reason that never depended on the number:
|
|
17511
|
+
* the walk is operator-armed and one-off, so the right trade is to be slower
|
|
17512
|
+
* and invisible rather than fast and felt — a `find` over this array hung an
|
|
17513
|
+
* SSH session on 2026-08-29.
|
|
17786
17514
|
*/
|
|
17787
17515
|
var LEDGER_WALK_PAUSE_MS = 100;
|
|
17788
17516
|
/**
|
|
@@ -18079,6 +17807,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18079
17807
|
this.segmentHours = new SegmentHourLedger({
|
|
18080
17808
|
store: this.ctx.api.settingsStore,
|
|
18081
17809
|
logger: this.ctx.logger,
|
|
17810
|
+
listHour: (locationId, deviceId, profile, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18082
17811
|
isLocationUsable: (locationId) => this.placement?.isLocationUsable(locationId) ?? true
|
|
18083
17812
|
});
|
|
18084
17813
|
try {
|
|
@@ -18112,7 +17841,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18112
17841
|
await promises.unlink(fromAbs);
|
|
18113
17842
|
}
|
|
18114
17843
|
},
|
|
18115
|
-
|
|
17844
|
+
hourDirectoryPaths: (deviceId, profile, locationId, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18116
17845
|
removeDir: async (locationId, relDir) => {
|
|
18117
17846
|
const loc = requireReadableLocation(this.resolvedLocations, locationId, "recorder");
|
|
18118
17847
|
await promises.rm(path.join(loc.root, relDir), {
|
|
@@ -18597,8 +18326,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18597
18326
|
localNodeId: this.nodeId,
|
|
18598
18327
|
index: this.index,
|
|
18599
18328
|
ledger: this.segmentHours === null ? null : {
|
|
18600
|
-
|
|
18601
|
-
|
|
18329
|
+
forgetScope: (scope) => this.segmentHours.forgetScope(scope),
|
|
18330
|
+
snapshot: () => this.segmentHours.snapshot()
|
|
18602
18331
|
},
|
|
18603
18332
|
now: () => Date.now()
|
|
18604
18333
|
});
|
|
@@ -18705,9 +18434,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18705
18434
|
}
|
|
18706
18435
|
},
|
|
18707
18436
|
anchorOf: async (root) => (await readLocationMarker(root))?.volumeId ?? null,
|
|
18708
|
-
applyToIndex: (paths) =>
|
|
18709
|
-
this.index.removeSegments(paths);
|
|
18710
|
-
},
|
|
18437
|
+
applyToIndex: (deviceId, locationId, hour, paths) => this.index.resetHour(deviceId, locationId, hour, paths),
|
|
18711
18438
|
logger: this.ctx.logger,
|
|
18712
18439
|
now: () => Date.now(),
|
|
18713
18440
|
pace: () => sleep$1(LEDGER_WALK_PAUSE_MS)
|
|
@@ -18835,7 +18562,6 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18835
18562
|
});
|
|
18836
18563
|
try {
|
|
18837
18564
|
await this.segmentHours?.load();
|
|
18838
|
-
this.segmentHours?.repairAggregates();
|
|
18839
18565
|
} catch (err) {
|
|
18840
18566
|
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: errMsg(err) } });
|
|
18841
18567
|
}
|
|
@@ -18878,18 +18604,15 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
18878
18604
|
const sweeper = new GhostLocationSweeper({
|
|
18879
18605
|
ledger,
|
|
18880
18606
|
locations: () => this.resolvedLocations,
|
|
18881
|
-
|
|
18882
|
-
location,
|
|
18883
|
-
relativePath
|
|
18884
|
-
}),
|
|
18607
|
+
listHour: (locationId, deviceId, profile, hourStartMs) => this.listHourDirectory(locationId, deviceId, profile, hourStartMs),
|
|
18885
18608
|
applyToIndex: (rows, toLocationId) => {
|
|
18886
18609
|
this.index.removeSegments(rows.map((row) => row.path));
|
|
18887
|
-
if (toLocationId === null) return;
|
|
18888
18610
|
for (const row of rows) this.index.addSegment({
|
|
18889
18611
|
...row,
|
|
18890
18612
|
locationId: toLocationId
|
|
18891
18613
|
});
|
|
18892
18614
|
},
|
|
18615
|
+
forgetInIndex: (deviceId, locationId, hourStartMs) => this.index.resetHour(deviceId, locationId, hourStartMs, []),
|
|
18893
18616
|
logger: this.ctx.logger,
|
|
18894
18617
|
now: () => Date.now(),
|
|
18895
18618
|
pace: () => sleep$1(GHOST_LOCATION_PROBE_PAUSE_MS)
|
|
@@ -19164,6 +18887,63 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19164
18887
|
return this.resolvedLocations;
|
|
19165
18888
|
}
|
|
19166
18889
|
/**
|
|
18890
|
+
* ONE hour bucket's segment files, location-relative — the recorder's single
|
|
18891
|
+
* implementation of {@link HourLister} (D523).
|
|
18892
|
+
*
|
|
18893
|
+
* The directory is the authority for WHICH segments exist, and the segment
|
|
18894
|
+
* NAME carries start, duration and bytes, so this is a complete description
|
|
18895
|
+
* of the hour and no file is ever opened.
|
|
18896
|
+
*
|
|
18897
|
+
* Three answers, and the difference between the last two is the whole point:
|
|
18898
|
+
*
|
|
18899
|
+
* - `null` — **undecided.** The location does not resolve, may not be read,
|
|
18900
|
+
* or the read failed for a reason that is not "this directory is not
|
|
18901
|
+
* there". Every caller turns it into "this hour takes no part in this
|
|
18902
|
+
* pass".
|
|
18903
|
+
* - `[]` — **the hour holds nothing**, and the volume confirmed it: an
|
|
18904
|
+
* ENOENT on the hour directory is only reported as empty once the
|
|
18905
|
+
* location's ROOT has answered a read. A root that has gone away answers
|
|
18906
|
+
* ENOENT for every hour on it, and reporting that as "empty" is how a pass
|
|
18907
|
+
* deletes an operator's index of real footage (D148, D393).
|
|
18908
|
+
* - the paths.
|
|
18909
|
+
*/
|
|
18910
|
+
async listHourDirectory(locationId, deviceId, profile, hourStartMs) {
|
|
18911
|
+
const location = this.resolvedLocations.find((l) => l.id === locationId);
|
|
18912
|
+
if (location === void 0 || !locationMayBeRead(location)) return null;
|
|
18913
|
+
const rel = hourDirRelPath(deviceId, profile, hourStartMs);
|
|
18914
|
+
try {
|
|
18915
|
+
return (await promises.readdir(path.join(location.root, rel))).filter((f) => f.endsWith(".m4s")).map((f) => `${rel}/${f}`);
|
|
18916
|
+
} catch (err) {
|
|
18917
|
+
if (err.code !== "ENOENT") {
|
|
18918
|
+
this.ctx.logger.warn("recorder: hour directory read failed — this hour decides nothing", {
|
|
18919
|
+
tags: { deviceId },
|
|
18920
|
+
meta: {
|
|
18921
|
+
profile,
|
|
18922
|
+
locationId,
|
|
18923
|
+
hourStartMs,
|
|
18924
|
+
error: errMsg(err)
|
|
18925
|
+
}
|
|
18926
|
+
});
|
|
18927
|
+
return null;
|
|
18928
|
+
}
|
|
18929
|
+
try {
|
|
18930
|
+
await promises.readdir(location.root);
|
|
18931
|
+
} catch {
|
|
18932
|
+
this.ctx.logger.warn("recorder: hour directory is absent AND its location root cannot be read — this hour decides nothing", {
|
|
18933
|
+
tags: { deviceId },
|
|
18934
|
+
meta: {
|
|
18935
|
+
profile,
|
|
18936
|
+
locationId,
|
|
18937
|
+
root: location.root,
|
|
18938
|
+
hourStartMs
|
|
18939
|
+
}
|
|
18940
|
+
});
|
|
18941
|
+
return null;
|
|
18942
|
+
}
|
|
18943
|
+
return [];
|
|
18944
|
+
}
|
|
18945
|
+
}
|
|
18946
|
+
/**
|
|
19167
18947
|
* Attach-boundary placement for one (device, profile). Falls back to the pure
|
|
19168
18948
|
* resolver when the service is not built (non-recording node / early boot),
|
|
19169
18949
|
* which is byte-identical to the pre-placement behaviour.
|