@camstack/addon-pipeline 1.2.302 → 1.2.303
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline-runner/index.js +211 -13
- package/dist/pipeline-runner/index.mjs +211 -13
- package/dist/recorder/index.js +1349 -70
- package/dist/recorder/index.mjs +1423 -145
- package/dist/session-decode/decode-worker-main.js +250 -57
- package/dist/session-decode/decode-worker-main.mjs +250 -57
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-DFaPx_Ot.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF7C68rV.mjs} +1 -1
- package/dist/stream-broker/{hostInit-CePd6XbV.mjs → hostInit-dac4-ofq.mjs} +1 -1
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as OpsLogEntrySchema, B as VIDEOCLIPS_MAX_OFFER_BYTES, Bt as resolveCreateExportSubject, C as ExportRecordSchema, Ct as mayWriteToLocation, Dn as array, En as _enum, F as RECORDING_SOURCE_CAMSTACK, Fn as record, Ft as recordingArchiveCapability, Ht as resolveLocationMode, I as RECORDING_SOURCE_CAMSTACK_ADDON, In as string, It as recordingCapability, Jt as storageOccupancyCapability, L as RecordingConfigSchema, Lt as recordingExportCapability, Mn as number, Nn as object, On as boolean, P as RECORDING_EXPORT_MAX_READ_BYTES, Pn as preprocess, Qt as videoclipsCapability, R as RecordingSignalStatusSchema, Rn as EventCategory, Rt as recordingSignalCapability, St as mayReadLocation, Tn as sleep$1, Tt as modeMayWrite, V as VISIT_MERGE_GAP_MS, Wt as resolveRecordingProfiles, X as clipExportFailureOf, Xt as stripRetiredBandPreBufferSec, Y as clipExportFailure, _n as hydrateSchema, _t as isFirstLevelMacroClass, a as BatteryStatusSchema, b as EVENT_OWNER_TYPES, ct as evaluateSensorEdge, dn as DeviceType, dt as evictionPolicyOfLocation, en as errMsg, ft as exportSubjectOf, g as DEFAULT_EVENTS_BAND_BUFFER_SEC, hn as defineCustomActions, mn as customAction, mt as gbToBytes, o as CLIP_BROKER_PACED_RATES, q as batteryCapability, qt as storageEvictableCapability, rt as deriveRecordingMode, sn as BaseAddon, tt as deriveBatteryPresence, un as DeviceFeature, ut as evictionPolicyForMode, vt as loadContributionCapability, wn as selectAssignedProfileSlots, wt as migrateLegacyDeviceSignalConfig, x as EVENT_PAD_MS, xn as nodePin } from "../dist-CLZmZfHT.mjs";
|
|
1
|
+
import { A as OpsLogEntrySchema, B as VIDEOCLIPS_MAX_OFFER_BYTES, Bt as resolveCreateExportSubject, C as ExportRecordSchema, Ct as mayWriteToLocation, Dn as array, En as _enum, F as RECORDING_SOURCE_CAMSTACK, Fn as record, Ft as recordingArchiveCapability, Ht as resolveLocationMode, I as RECORDING_SOURCE_CAMSTACK_ADDON, In as string, It as recordingCapability, Jt as storageOccupancyCapability, L as RecordingConfigSchema, Lt as recordingExportCapability, Mn as number, Nn as object, On as boolean, P as RECORDING_EXPORT_MAX_READ_BYTES, Pn as preprocess, Qt as videoclipsCapability, R as RecordingSignalStatusSchema, Rn as EventCategory, Rt as recordingSignalCapability, St as mayReadLocation, Tn as sleep$1, Tt as modeMayWrite, V as VISIT_MERGE_GAP_MS, Wt as resolveRecordingProfiles, X as clipExportFailureOf, Xt as stripRetiredBandPreBufferSec, Y as clipExportFailure, _n as hydrateSchema, _t as isFirstLevelMacroClass, a as BatteryStatusSchema, b as EVENT_OWNER_TYPES, ct as evaluateSensorEdge, dn as DeviceType, dt as evictionPolicyOfLocation, en as errMsg$1, ft as exportSubjectOf, g as DEFAULT_EVENTS_BAND_BUFFER_SEC, hn as defineCustomActions, mn as customAction, mt as gbToBytes, o as CLIP_BROKER_PACED_RATES, q as batteryCapability, qt as storageEvictableCapability, rt as deriveRecordingMode, sn as BaseAddon, tt as deriveBatteryPresence, un as DeviceFeature, ut as evictionPolicyForMode, vt as loadContributionCapability, wn as selectAssignedProfileSlots, wt as migrateLegacyDeviceSignalConfig, x as EVENT_PAD_MS, xn as nodePin } from "../dist-CLZmZfHT.mjs";
|
|
2
2
|
import { a as physicalRootOf, i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-C5LXX8Qo.mjs";
|
|
3
3
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
4
4
|
import { d as PREBUFFER_MAX_MS, l as withRecordingIntent, r as RECORDING_PRE_ROLL_REQUEST_MS } from "../restream-intent-DyGmLAU9.mjs";
|
|
@@ -6,9 +6,10 @@ import { n as RowMapStore, t as retireRootKeys } from "../retire-root-keys-DZvls
|
|
|
6
6
|
import { randomUUID } from "node:crypto";
|
|
7
7
|
import { createReadStream, promises, realpathSync } from "node:fs";
|
|
8
8
|
import path from "node:path";
|
|
9
|
-
import { spawn } from "node:child_process";
|
|
9
|
+
import { execFile, spawn } from "node:child_process";
|
|
10
10
|
import { open } from "node:fs/promises";
|
|
11
11
|
import { contentTypeFor, createFileDataPlaneHandler, parseRangeHeader, startEventLoopStallMonitor } from "@camstack/system/addon-utils";
|
|
12
|
+
import v8 from "node:v8";
|
|
12
13
|
//#region src/recorder/event-capture.ts
|
|
13
14
|
/**
|
|
14
15
|
* La sottoscrizione motion del recorder: bus → `onTrigger(deviceId, atMs)`.
|
|
@@ -460,6 +461,11 @@ var MFRA_RETAIN_MS = 26 * 36e5;
|
|
|
460
461
|
function isMfraTableServable(startMs, nowMs) {
|
|
461
462
|
return startMs >= nowMs - MFRA_RETAIN_MS;
|
|
462
463
|
}
|
|
464
|
+
/** A `{ timeTicks, moofOffset }` object on a 64-bit V8 without pointer
|
|
465
|
+
* compression (Node): header + two fields + the array slot pointing at it,
|
|
466
|
+
* ~48 B, plus a 16 B HeapNumber when a value passes 2³¹. An ESTIMATE, rounded
|
|
467
|
+
* up — the order of magnitude is the point, not the byte. */
|
|
468
|
+
var MFRA_SAMPLE_APPROX_BYTES = 64;
|
|
463
469
|
/**
|
|
464
470
|
* The RAM store the directory reads. Keys are `(deviceId, profile, startMs)`
|
|
465
471
|
* — exactly how a directory row identifies a segment. Insertion is
|
|
@@ -503,6 +509,20 @@ var MfraTableStore = class {
|
|
|
503
509
|
get size() {
|
|
504
510
|
return this.tables.size;
|
|
505
511
|
}
|
|
512
|
+
/**
|
|
513
|
+
* Entries and seek samples held, for the recorder census. The byte figure is
|
|
514
|
+
* an ESTIMATE — {@link MFRA_SAMPLE_APPROX_BYTES} per sample object — and is
|
|
515
|
+
* named as one; it is the order of magnitude that matters here.
|
|
516
|
+
*/
|
|
517
|
+
memoryCensus() {
|
|
518
|
+
let samples = 0;
|
|
519
|
+
for (const entry of this.tables.values()) samples += entry.table.samples.length;
|
|
520
|
+
return {
|
|
521
|
+
entries: this.tables.size,
|
|
522
|
+
samples,
|
|
523
|
+
approxBytes: samples * MFRA_SAMPLE_APPROX_BYTES
|
|
524
|
+
};
|
|
525
|
+
}
|
|
506
526
|
/** Insertion order is time order, so the expired prefix is contiguous. */
|
|
507
527
|
prune() {
|
|
508
528
|
const cutoff = this.now() - MFRA_RETAIN_MS;
|
|
@@ -1956,7 +1976,7 @@ async function resolveRecordingsLocations(api, logger) {
|
|
|
1956
1976
|
...numericConfig(l.config, "maxUsedGb", "maxUsedGb")
|
|
1957
1977
|
}, resolveLocationMode(l))));
|
|
1958
1978
|
} catch (err) {
|
|
1959
|
-
logger.warn("recorder: listLocations failed — no recordings locations resolved", { meta: { error: errMsg(err) } });
|
|
1979
|
+
logger.warn("recorder: listLocations failed — no recordings locations resolved", { meta: { error: errMsg$1(err) } });
|
|
1960
1980
|
return [];
|
|
1961
1981
|
}
|
|
1962
1982
|
}
|
|
@@ -1987,7 +2007,7 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
|
|
|
1987
2007
|
tags: { deviceId },
|
|
1988
2008
|
meta: {
|
|
1989
2009
|
locationIds: aliases.map((location) => location.id),
|
|
1990
|
-
error: errMsg(err)
|
|
2010
|
+
error: errMsg$1(err)
|
|
1991
2011
|
}
|
|
1992
2012
|
});
|
|
1993
2013
|
continue;
|
|
@@ -2062,7 +2082,7 @@ async function hydrateWindowFromStorage(index, deviceId, fromMs, toMs, locations
|
|
|
2062
2082
|
tags: { deviceId },
|
|
2063
2083
|
meta: {
|
|
2064
2084
|
locationIds: aliases.map((location) => location.id),
|
|
2065
|
-
error: errMsg(err)
|
|
2085
|
+
error: errMsg$1(err)
|
|
2066
2086
|
}
|
|
2067
2087
|
});
|
|
2068
2088
|
continue;
|
|
@@ -4213,6 +4233,24 @@ var CalendarIndex = class {
|
|
|
4213
4233
|
constructor(deps) {
|
|
4214
4234
|
this.deps = deps;
|
|
4215
4235
|
}
|
|
4236
|
+
/**
|
|
4237
|
+
* Entry counts for the recorder census. `hourSegs` is unbounded (it is
|
|
4238
|
+
* invalidated on eviction and relocation, never aged), so its segment total
|
|
4239
|
+
* is the number that says whether this cache is where the heap went.
|
|
4240
|
+
*/
|
|
4241
|
+
memoryCensus() {
|
|
4242
|
+
let days = 0;
|
|
4243
|
+
for (const perDay of this.byDevice.values()) days += perDay.size;
|
|
4244
|
+
let hourSegments = 0;
|
|
4245
|
+
for (const segments of this.hourSegs.values()) hourSegments += segments.length;
|
|
4246
|
+
return {
|
|
4247
|
+
devices: this.byDevice.size,
|
|
4248
|
+
days,
|
|
4249
|
+
hourKeys: this.hourSegs.size,
|
|
4250
|
+
hourSegments,
|
|
4251
|
+
inFlight: this.inFlight.size + this.segsInFlight.size
|
|
4252
|
+
};
|
|
4253
|
+
}
|
|
4216
4254
|
mapFor(deviceId) {
|
|
4217
4255
|
let m = this.byDevice.get(deviceId);
|
|
4218
4256
|
if (!m) {
|
|
@@ -5817,7 +5855,7 @@ function createRecordingConfigMirror(deps) {
|
|
|
5817
5855
|
if (cached === void 0) {
|
|
5818
5856
|
deps.logger.warn("recorder: config read failed and NOTHING is cached for this camera — no last-known-good to serve", {
|
|
5819
5857
|
tags: { deviceId },
|
|
5820
|
-
meta: { error: errMsg(err) }
|
|
5858
|
+
meta: { error: errMsg$1(err) }
|
|
5821
5859
|
});
|
|
5822
5860
|
throw err;
|
|
5823
5861
|
}
|
|
@@ -5826,7 +5864,7 @@ function createRecordingConfigMirror(deps) {
|
|
|
5826
5864
|
tags: { deviceId },
|
|
5827
5865
|
meta: {
|
|
5828
5866
|
ageMs: now() - cached.atMs,
|
|
5829
|
-
error: errMsg(err)
|
|
5867
|
+
error: errMsg$1(err)
|
|
5830
5868
|
}
|
|
5831
5869
|
});
|
|
5832
5870
|
return cached.config;
|
|
@@ -6575,6 +6613,14 @@ var ExportEngine = class {
|
|
|
6575
6613
|
constructor(deps) {
|
|
6576
6614
|
this.deps = deps;
|
|
6577
6615
|
}
|
|
6616
|
+
/** Entry counts, for the recorder census. */
|
|
6617
|
+
memoryCensus() {
|
|
6618
|
+
return {
|
|
6619
|
+
queued: this.queue.length,
|
|
6620
|
+
active: this.active.size,
|
|
6621
|
+
cancelled: this.cancelled.size
|
|
6622
|
+
};
|
|
6623
|
+
}
|
|
6578
6624
|
/** Queue a persisted `queued` record for rendering. */
|
|
6579
6625
|
enqueue(id) {
|
|
6580
6626
|
if (this.queue.includes(id) || this.active.has(id)) return;
|
|
@@ -8237,7 +8283,7 @@ async function evictVictims(deps, deviceId, victims) {
|
|
|
8237
8283
|
tags: { deviceId },
|
|
8238
8284
|
meta: {
|
|
8239
8285
|
locationId,
|
|
8240
|
-
error: errMsg(err)
|
|
8286
|
+
error: errMsg$1(err)
|
|
8241
8287
|
}
|
|
8242
8288
|
});
|
|
8243
8289
|
}
|
|
@@ -8699,7 +8745,7 @@ function buildRecordingProvider(deps) {
|
|
|
8699
8745
|
} catch (err) {
|
|
8700
8746
|
deps.logger.warn("recorder: re-evaluate after setDeviceConfig failed", {
|
|
8701
8747
|
tags: { deviceId },
|
|
8702
|
-
meta: { error: errMsg(err) }
|
|
8748
|
+
meta: { error: errMsg$1(err) }
|
|
8703
8749
|
});
|
|
8704
8750
|
}
|
|
8705
8751
|
return saved;
|
|
@@ -9963,6 +10009,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9963
10009
|
* the recorder happens to be placing that hour on.
|
|
9964
10010
|
*/
|
|
9965
10011
|
refusedWriterHours = /* @__PURE__ */ new Set();
|
|
10012
|
+
/** Entry counts of the in-process maps, for the recorder census. */
|
|
10013
|
+
memoryCensus() {
|
|
10014
|
+
let accountedPaths = 0;
|
|
10015
|
+
for (const paths of this.accounted.values()) accountedPaths += paths.size;
|
|
10016
|
+
return {
|
|
10017
|
+
lanes: this.lanes.size,
|
|
10018
|
+
writerHours: this.writerHours.size,
|
|
10019
|
+
accountedHours: this.accounted.size,
|
|
10020
|
+
accountedPaths,
|
|
10021
|
+
refused: this.refused.size,
|
|
10022
|
+
refusedWriterHours: this.refusedWriterHours.size
|
|
10023
|
+
};
|
|
10024
|
+
}
|
|
9966
10025
|
constructor(deps) {
|
|
9967
10026
|
this.logger = deps.logger;
|
|
9968
10027
|
this.store = deps.store;
|
|
@@ -11498,7 +11557,7 @@ var GhostLocationSweeper = class {
|
|
|
11498
11557
|
outcome: {
|
|
11499
11558
|
kind: "refused",
|
|
11500
11559
|
reason: "probe-error",
|
|
11501
|
-
detail: `${alias.id}: ${errMsg(err)}`
|
|
11560
|
+
detail: `${alias.id}: ${errMsg$1(err)}`
|
|
11502
11561
|
},
|
|
11503
11562
|
probes: used
|
|
11504
11563
|
};
|
|
@@ -11735,7 +11794,7 @@ var LedgerWalkReconciler = class {
|
|
|
11735
11794
|
try {
|
|
11736
11795
|
listing = await this.deps.listHour(state.location.root, relDir);
|
|
11737
11796
|
} catch (err) {
|
|
11738
|
-
this.skip(state, claim, "listing-error", errMsg(err));
|
|
11797
|
+
this.skip(state, claim, "listing-error", errMsg$1(err));
|
|
11739
11798
|
return true;
|
|
11740
11799
|
}
|
|
11741
11800
|
state.listings += 1;
|
|
@@ -11862,7 +11921,7 @@ var LedgerWalkReconciler = class {
|
|
|
11862
11921
|
this.deps.logger.warn("recorder: ledger walk cannot read the volume marker — an unreadable root decides nothing", { meta: {
|
|
11863
11922
|
locationId: location.id,
|
|
11864
11923
|
root: location.root,
|
|
11865
|
-
error: errMsg(err)
|
|
11924
|
+
error: errMsg$1(err)
|
|
11866
11925
|
} });
|
|
11867
11926
|
return {
|
|
11868
11927
|
ok: false,
|
|
@@ -12046,7 +12105,7 @@ var HourAdoptionSweeper = class {
|
|
|
12046
12105
|
meta: {
|
|
12047
12106
|
locationId: location.id,
|
|
12048
12107
|
rel: candidate.rel,
|
|
12049
|
-
error: errMsg(err)
|
|
12108
|
+
error: errMsg$1(err)
|
|
12050
12109
|
}
|
|
12051
12110
|
});
|
|
12052
12111
|
continue;
|
|
@@ -12150,7 +12209,7 @@ var HourAdoptionSweeper = class {
|
|
|
12150
12209
|
onError();
|
|
12151
12210
|
this.deps.logger.debug("recorder: hour adoption could not list a directory", { meta: {
|
|
12152
12211
|
dir: absDir,
|
|
12153
|
-
error: errMsg(err)
|
|
12212
|
+
error: errMsg$1(err)
|
|
12154
12213
|
} });
|
|
12155
12214
|
return [];
|
|
12156
12215
|
}
|
|
@@ -12302,7 +12361,7 @@ var LocationRefresher = class {
|
|
|
12302
12361
|
} catch (err) {
|
|
12303
12362
|
this.deps.logger.warn("recorder: could not re-resolve storage locations — the previously resolved set stands", { meta: {
|
|
12304
12363
|
trigger,
|
|
12305
|
-
error: errMsg(err)
|
|
12364
|
+
error: errMsg$1(err)
|
|
12306
12365
|
} });
|
|
12307
12366
|
} finally {
|
|
12308
12367
|
this.running = false;
|
|
@@ -12550,7 +12609,11 @@ function startPeriodicPass(deps) {
|
|
|
12550
12609
|
}
|
|
12551
12610
|
const began = now();
|
|
12552
12611
|
startedAt = began;
|
|
12553
|
-
deps.
|
|
12612
|
+
const probe = deps.heapProbe;
|
|
12613
|
+
(probe === void 0 ? deps.run() : probe.track({
|
|
12614
|
+
kind: "pass",
|
|
12615
|
+
name: deps.name
|
|
12616
|
+
}, deps.run)).catch((err) => {
|
|
12554
12617
|
deps.logger.warn("recorder: pass failed", { meta: {
|
|
12555
12618
|
pass: deps.name,
|
|
12556
12619
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -13245,6 +13308,15 @@ var PlacementService = class {
|
|
|
13245
13308
|
constructor(deps) {
|
|
13246
13309
|
this.deps = deps;
|
|
13247
13310
|
}
|
|
13311
|
+
/** Entry counts of the per-location/per-camera state, for the recorder census. */
|
|
13312
|
+
memoryCensus() {
|
|
13313
|
+
return {
|
|
13314
|
+
verified: this.verified.size,
|
|
13315
|
+
unusable: this.unusable.size,
|
|
13316
|
+
capacityFaults: this.capacityFaults.size,
|
|
13317
|
+
allUnusableReported: this.allUnusableReported.size
|
|
13318
|
+
};
|
|
13319
|
+
}
|
|
13248
13320
|
get probeDeps() {
|
|
13249
13321
|
return this.deps.probe ?? fsVolumeProbeDeps;
|
|
13250
13322
|
}
|
|
@@ -13399,7 +13471,7 @@ var PlacementService = class {
|
|
|
13399
13471
|
tags: { deviceId },
|
|
13400
13472
|
meta: {
|
|
13401
13473
|
profile,
|
|
13402
|
-
error: errMsg(err)
|
|
13474
|
+
error: errMsg$1(err)
|
|
13403
13475
|
}
|
|
13404
13476
|
});
|
|
13405
13477
|
return null;
|
|
@@ -13466,7 +13538,7 @@ var PlacementService = class {
|
|
|
13466
13538
|
blob = await this.bounded(readPlacementState(this.deps.state), "placement plan read");
|
|
13467
13539
|
} catch (err) {
|
|
13468
13540
|
this.deps.logger.warn("recorder placement: plan read failed — this tick plans NOTHING and the stored plan stands", { meta: {
|
|
13469
|
-
error: errMsg(err),
|
|
13541
|
+
error: errMsg$1(err),
|
|
13470
13542
|
timeoutMs: this.storeTimeoutMs
|
|
13471
13543
|
} });
|
|
13472
13544
|
return NO_PLAN;
|
|
@@ -13500,7 +13572,7 @@ var PlacementService = class {
|
|
|
13500
13572
|
await this.bounded(writePlacementPass(this.deps.state, blob, pass), "placement plan write");
|
|
13501
13573
|
} catch (err) {
|
|
13502
13574
|
this.deps.logger.warn("recorder placement: plan persist failed", { meta: {
|
|
13503
|
-
error: errMsg(err),
|
|
13575
|
+
error: errMsg$1(err),
|
|
13504
13576
|
timeoutMs: this.storeTimeoutMs
|
|
13505
13577
|
} });
|
|
13506
13578
|
}
|
|
@@ -13589,7 +13661,7 @@ var PlacementService = class {
|
|
|
13589
13661
|
try {
|
|
13590
13662
|
return new Map(Object.entries((await readPlacementState(this.deps.state)).assignments));
|
|
13591
13663
|
} catch (err) {
|
|
13592
|
-
this.deps.logger.warn("recorder placement: assignment read failed — reporting no plan", { meta: { error: errMsg(err) } });
|
|
13664
|
+
this.deps.logger.warn("recorder placement: assignment read failed — reporting no plan", { meta: { error: errMsg$1(err) } });
|
|
13593
13665
|
return /* @__PURE__ */ new Map();
|
|
13594
13666
|
}
|
|
13595
13667
|
}
|
|
@@ -13737,7 +13809,7 @@ var PlacementService = class {
|
|
|
13737
13809
|
tags: { deviceId },
|
|
13738
13810
|
meta: {
|
|
13739
13811
|
locationId: chosen.id,
|
|
13740
|
-
error: errMsg(err)
|
|
13812
|
+
error: errMsg$1(err)
|
|
13741
13813
|
}
|
|
13742
13814
|
});
|
|
13743
13815
|
}
|
|
@@ -13800,7 +13872,7 @@ var PlacementService = class {
|
|
|
13800
13872
|
tags: { deviceId },
|
|
13801
13873
|
meta: {
|
|
13802
13874
|
profile,
|
|
13803
|
-
error: errMsg(err)
|
|
13875
|
+
error: errMsg$1(err)
|
|
13804
13876
|
}
|
|
13805
13877
|
});
|
|
13806
13878
|
}
|
|
@@ -13825,7 +13897,7 @@ var PlacementService = class {
|
|
|
13825
13897
|
tags: { deviceId },
|
|
13826
13898
|
meta: {
|
|
13827
13899
|
profile,
|
|
13828
|
-
error: errMsg(err)
|
|
13900
|
+
error: errMsg$1(err)
|
|
13829
13901
|
}
|
|
13830
13902
|
});
|
|
13831
13903
|
}
|
|
@@ -14108,34 +14180,50 @@ function releaseTrigger(state, source, atMs, mergeGapMs) {
|
|
|
14108
14180
|
firstMs: since,
|
|
14109
14181
|
lastMs: Math.max(since, atMs)
|
|
14110
14182
|
};
|
|
14111
|
-
const
|
|
14112
|
-
const newest = windows[windows.length - 1];
|
|
14113
|
-
const merged = newest !== void 0 && span.firstMs - newest.lastMs <= mergeGapMs ? [...windows.slice(0, -1), {
|
|
14114
|
-
firstMs: Math.min(newest.firstMs, span.firstMs),
|
|
14115
|
-
lastMs: Math.max(newest.lastMs, span.lastMs)
|
|
14116
|
-
}] : [...windows, span];
|
|
14183
|
+
const merged = foldSpanInOrder(triggerWindowsFor(state, source), span, mergeGapMs);
|
|
14117
14184
|
const evicted = Math.max(0, merged.length - 32);
|
|
14118
14185
|
return withWindows(withHeld(state, held), source, evicted > 0 ? merged.slice(evicted) : merged);
|
|
14119
14186
|
}
|
|
14120
14187
|
/**
|
|
14121
|
-
* Fold one
|
|
14122
|
-
*
|
|
14123
|
-
*
|
|
14188
|
+
* Fold one span into a window list, immutably, keeping the list's INVARIANT:
|
|
14189
|
+
* sorted by `firstMs`, and no two windows within `mergeGapMs` of each other
|
|
14190
|
+
* (each is its own episode). The span EXTENDS every window it lies within the
|
|
14191
|
+
* merge gap of — possibly bridging two — and is otherwise INSERTED at its place
|
|
14192
|
+
* in time.
|
|
14193
|
+
*
|
|
14194
|
+
* Never "appended as the newest": the history is read from its end by the live
|
|
14195
|
+
* gate ({@link latestTriggerMs}) and by the evictor, so a span older than the
|
|
14196
|
+
* newest window appended after it made the newest trigger go BACKWARDS, and the
|
|
14197
|
+
* live gate tore down a writer whose episode was still open (D194, 2026-09-28).
|
|
14198
|
+
* The coalesce pass is the standard interval merge over the sorted list, so it
|
|
14199
|
+
* also heals a list that an older build left out of order.
|
|
14200
|
+
*/
|
|
14201
|
+
function foldSpanInOrder(windows, span, mergeGapMs) {
|
|
14202
|
+
return [...windows, span].toSorted((a, b) => a.firstMs - b.firstMs).reduce((acc, w) => {
|
|
14203
|
+
const prev = acc[acc.length - 1];
|
|
14204
|
+
if (prev !== void 0 && w.firstMs - prev.lastMs <= mergeGapMs) return [...acc.slice(0, -1), {
|
|
14205
|
+
firstMs: prev.firstMs,
|
|
14206
|
+
lastMs: Math.max(prev.lastMs, w.lastMs)
|
|
14207
|
+
}];
|
|
14208
|
+
return [...acc, w];
|
|
14209
|
+
}, []);
|
|
14210
|
+
}
|
|
14211
|
+
/**
|
|
14212
|
+
* Fold one qualifying trigger into the history, immutably. It EXTENDS the
|
|
14213
|
+
* window it lands within `postBufferSec` of (same episode — the recorder never
|
|
14214
|
+
* stopped in between), and opens a new one otherwise, IN TIME ORDER.
|
|
14124
14215
|
*
|
|
14125
14216
|
* An out-of-order trigger (`atMs` older than the newest window's `lastMs`, which
|
|
14126
|
-
* a bus lag spike
|
|
14127
|
-
*
|
|
14217
|
+
* a bus lag spike or a durable row older than the bus edge can produce) never
|
|
14218
|
+
* rewinds anything: it widens `firstMs` of the episode it belongs to, or opens
|
|
14219
|
+
* its own window BEFORE the newer ones. The keep span can only ever grow, and
|
|
14220
|
+
* the newest trigger can only ever move forward.
|
|
14128
14221
|
*/
|
|
14129
14222
|
function recordTrigger(state, source, atMs, mergeGapMs) {
|
|
14130
|
-
const
|
|
14131
|
-
const newest = windows[windows.length - 1];
|
|
14132
|
-
const merged = newest !== void 0 && atMs - newest.lastMs <= mergeGapMs && atMs >= newest.firstMs - mergeGapMs ? [...windows.slice(0, -1), {
|
|
14133
|
-
firstMs: Math.min(newest.firstMs, atMs),
|
|
14134
|
-
lastMs: Math.max(newest.lastMs, atMs)
|
|
14135
|
-
}] : [...windows, {
|
|
14223
|
+
const merged = foldSpanInOrder(triggerWindowsFor(state, source), {
|
|
14136
14224
|
firstMs: atMs,
|
|
14137
14225
|
lastMs: atMs
|
|
14138
|
-
}
|
|
14226
|
+
}, mergeGapMs);
|
|
14139
14227
|
const evicted = Math.max(0, merged.length - 32);
|
|
14140
14228
|
return {
|
|
14141
14229
|
state: withWindows(state, source, evicted > 0 ? merged.slice(evicted) : merged),
|
|
@@ -14159,10 +14247,18 @@ function pruneTriggerHistory(state, postMs, nowMs) {
|
|
|
14159
14247
|
}
|
|
14160
14248
|
return changed ? withHeld({ windows: next }, state.held) : state;
|
|
14161
14249
|
}
|
|
14162
|
-
/**
|
|
14250
|
+
/**
|
|
14251
|
+
* Most recent trigger for a source, or null when none is retained.
|
|
14252
|
+
*
|
|
14253
|
+
* The MAXIMUM `lastMs`, not the last element's: the list is kept in time order
|
|
14254
|
+
* ({@link recordTrigger}), but the live gate tearing a writer down is the one
|
|
14255
|
+
* outcome that destroys footage, so it does not rest on an ordering invariant
|
|
14256
|
+
* alone (D194, 2026-09-28).
|
|
14257
|
+
*/
|
|
14163
14258
|
function latestTriggerMs(state, source) {
|
|
14164
14259
|
const windows = triggerWindowsFor(state, source);
|
|
14165
|
-
|
|
14260
|
+
if (windows.length === 0) return null;
|
|
14261
|
+
return windows.reduce((max, w) => Math.max(max, w.lastMs), Number.NEGATIVE_INFINITY);
|
|
14166
14262
|
}
|
|
14167
14263
|
/**
|
|
14168
14264
|
* Is an `events` band's live demand window open at `atMs`? True iff a trigger
|
|
@@ -14327,7 +14423,7 @@ function foldDurableTriggers(state, source, timestamps, mergeGapMs) {
|
|
|
14327
14423
|
const folded = recordTrigger(next, source, atMs, mergeGapMs);
|
|
14328
14424
|
evicted += folded.evicted;
|
|
14329
14425
|
const after = triggerWindowsFor(folded.state, source);
|
|
14330
|
-
if (
|
|
14426
|
+
if (after.length !== before.length || after.some((w, i) => w.firstMs !== before[i]?.firstMs || w.lastMs !== before[i]?.lastMs)) recovered++;
|
|
14331
14427
|
next = folded.state;
|
|
14332
14428
|
}
|
|
14333
14429
|
return {
|
|
@@ -14922,6 +15018,35 @@ function planFinalizations(body, processed, forceFinal = false) {
|
|
|
14922
15018
|
rolledOffToMs: spanKnown ? front : null
|
|
14923
15019
|
};
|
|
14924
15020
|
}
|
|
15021
|
+
/** The last entry of one playlist body, as {@link DrainedPlaylist} names it;
|
|
15022
|
+
* `null` when the body has no epoch-named last entry (nothing to vouch for). */
|
|
15023
|
+
function playlistTail(body) {
|
|
15024
|
+
const entries = parseLivePlaylist(body);
|
|
15025
|
+
const last = entries[entries.length - 1];
|
|
15026
|
+
if (last === void 0) return null;
|
|
15027
|
+
const startMs = parseEpochStartMs(last.segPath);
|
|
15028
|
+
if (startMs === null) return null;
|
|
15029
|
+
return {
|
|
15030
|
+
lastSegStartMs: startMs,
|
|
15031
|
+
lastMediaSequence: parseMediaSequence(body) + entries.length - 1
|
|
15032
|
+
};
|
|
15033
|
+
}
|
|
15034
|
+
/**
|
|
15035
|
+
* Did everything a plan says rolled off belong to a predecessor run that a
|
|
15036
|
+
* watcher of this process already accounted for? Pure. All three must hold:
|
|
15037
|
+
* - a predecessor was DRAINED here, by a final pass that read its playlist;
|
|
15038
|
+
* - the body read now is NOT NEWER than the one it drained (last entry no
|
|
15039
|
+
* later, sequence no higher) — so it is that playlist, not one a foreign or
|
|
15040
|
+
* later writer produced since, whose roll-off nobody has seen;
|
|
15041
|
+
* - the body's front predates THIS writer — so none of it is this writer's.
|
|
15042
|
+
*/
|
|
15043
|
+
function rolledOffIsDrainedPredecessor(input) {
|
|
15044
|
+
const { drained, bodyTail } = input;
|
|
15045
|
+
return drained !== null && bodyTail !== null && bodyTail.lastSegStartMs <= drained.lastSegStartMs && bodyTail.lastMediaSequence <= drained.lastMediaSequence && input.rolledOffToMs !== null && classifyMissingEntry({
|
|
15046
|
+
segmentStartMs: input.rolledOffToMs,
|
|
15047
|
+
writerStartedMs: input.writerStartedMs
|
|
15048
|
+
}) === "stale-previous-run";
|
|
15049
|
+
}
|
|
14925
15050
|
/**
|
|
14926
15051
|
* Start polling `<outDir>/live.m3u8`. Returns a handle whose `stop()`
|
|
14927
15052
|
* cancels the timer. Errors during a tick are swallowed (logged at debug)
|
|
@@ -14929,6 +15054,8 @@ function planFinalizations(body, processed, forceFinal = false) {
|
|
|
14929
15054
|
*/
|
|
14930
15055
|
function startSegmentWatcher(deps) {
|
|
14931
15056
|
const playlistPath = path.join(deps.outDir, "live.m3u8");
|
|
15057
|
+
const drained = deps.drainedPredecessor ?? null;
|
|
15058
|
+
let lastReadTail = null;
|
|
14932
15059
|
const now = () => deps.now?.() ?? Date.now();
|
|
14933
15060
|
let processed = 0;
|
|
14934
15061
|
let stopped = false;
|
|
@@ -14937,17 +15064,34 @@ function startSegmentWatcher(deps) {
|
|
|
14937
15064
|
let pendingIndex = -1;
|
|
14938
15065
|
let pendingSinceMs = 0;
|
|
14939
15066
|
const tick = async (forceFinal = false) => {
|
|
14940
|
-
if (stopped || ticking) return;
|
|
15067
|
+
if (stopped || ticking) return "unreadable";
|
|
14941
15068
|
ticking = true;
|
|
14942
15069
|
try {
|
|
14943
15070
|
let body;
|
|
14944
15071
|
try {
|
|
14945
15072
|
body = await promises.readFile(playlistPath, "utf8");
|
|
14946
15073
|
} catch {
|
|
14947
|
-
return;
|
|
15074
|
+
return "unreadable";
|
|
14948
15075
|
}
|
|
15076
|
+
const bodyTail = playlistTail(body);
|
|
15077
|
+
lastReadTail = bodyTail;
|
|
14949
15078
|
const plan = planFinalizations(body, processed, forceFinal);
|
|
14950
|
-
if (plan.rolledOff > 0
|
|
15079
|
+
if (plan.rolledOff > 0 && rolledOffIsDrainedPredecessor({
|
|
15080
|
+
rolledOffToMs: plan.rolledOffToMs,
|
|
15081
|
+
writerStartedMs: deps.writerStartedMs,
|
|
15082
|
+
bodyTail,
|
|
15083
|
+
drained
|
|
15084
|
+
})) deps.logger.debug("segment playlist is the drained previous run — no roll-off to report", {
|
|
15085
|
+
tags: { deviceId: deps.deviceId },
|
|
15086
|
+
meta: {
|
|
15087
|
+
rolledOff: plan.rolledOff,
|
|
15088
|
+
processed,
|
|
15089
|
+
frontMs: plan.rolledOffToMs,
|
|
15090
|
+
writerStartedMs: deps.writerStartedMs,
|
|
15091
|
+
drainedLastSegStartMs: drained?.lastSegStartMs
|
|
15092
|
+
}
|
|
15093
|
+
});
|
|
15094
|
+
else if (plan.rolledOff > 0) {
|
|
14951
15095
|
deps.logger.warn("segment playlist rolled past entries this watcher had not handled", {
|
|
14952
15096
|
tags: { deviceId: deps.deviceId },
|
|
14953
15097
|
meta: {
|
|
@@ -15000,6 +15144,7 @@ function startSegmentWatcher(deps) {
|
|
|
15000
15144
|
pendingSinceMs = 0;
|
|
15001
15145
|
}
|
|
15002
15146
|
processed = index;
|
|
15147
|
+
return "read";
|
|
15003
15148
|
} finally {
|
|
15004
15149
|
ticking = false;
|
|
15005
15150
|
}
|
|
@@ -15026,7 +15171,7 @@ function startSegmentWatcher(deps) {
|
|
|
15026
15171
|
},
|
|
15027
15172
|
flush: async () => {
|
|
15028
15173
|
await currentTick;
|
|
15029
|
-
await startTick(true);
|
|
15174
|
+
if (await startTick(true) === "read" && lastReadTail !== null) deps.onDrained?.(lastReadTail);
|
|
15030
15175
|
}
|
|
15031
15176
|
};
|
|
15032
15177
|
}
|
|
@@ -15049,6 +15194,16 @@ function startSegmentWatcher(deps) {
|
|
|
15049
15194
|
* recorded fragment's audio can be read in-process (mp4box) and pushed straight
|
|
15050
15195
|
* to the Opus sender with no per-fragment ffmpeg spawn on playback.
|
|
15051
15196
|
*/
|
|
15197
|
+
/**
|
|
15198
|
+
* The `-segment_list` path a writer for `outDir` is given — spelled in ONE
|
|
15199
|
+
* place, because it is also how a writer is RECOGNISED in the process table.
|
|
15200
|
+
* The staging-directory guard (D667) looks for exactly this argv pair before
|
|
15201
|
+
* it lets a second ffmpeg start in the same directory; a spelling that drifted
|
|
15202
|
+
* between the two would make every predecessor invisible to it.
|
|
15203
|
+
*/
|
|
15204
|
+
function segmentListPath(outDir) {
|
|
15205
|
+
return `${outDir}/live.m3u8`;
|
|
15206
|
+
}
|
|
15052
15207
|
/** ffmpeg argv: pull RTSP (TCP), copy video / transcode audio to Opus, write
|
|
15053
15208
|
* fragmented-mp4 segments named by epoch seconds FLAT under `outDir`, plus a
|
|
15054
15209
|
* rolling `live.m3u8` playlist. The recorder segment-watcher tails the
|
|
@@ -15088,7 +15243,7 @@ function buildPassthroughArgs(a) {
|
|
|
15088
15243
|
"-strftime",
|
|
15089
15244
|
"1",
|
|
15090
15245
|
"-segment_list",
|
|
15091
|
-
|
|
15246
|
+
segmentListPath(a.outDir),
|
|
15092
15247
|
"-segment_list_type",
|
|
15093
15248
|
"m3u8",
|
|
15094
15249
|
"-segment_list_size",
|
|
@@ -15141,7 +15296,7 @@ var RESTART_WINDOW_MS = 60 * 6e4;
|
|
|
15141
15296
|
*/
|
|
15142
15297
|
var MAX_RESTARTS_IN_WINDOW = 15;
|
|
15143
15298
|
/** Grace between the SIGTERM in stop() and the escalated SIGKILL. */
|
|
15144
|
-
var KILL_GRACE_MS = 500;
|
|
15299
|
+
var KILL_GRACE_MS$1 = 500;
|
|
15145
15300
|
/** Supervises one passthrough ffmpeg for a single (camera, profile). */
|
|
15146
15301
|
var SegmentWriter = class {
|
|
15147
15302
|
cfg;
|
|
@@ -15165,10 +15320,25 @@ var SegmentWriter = class {
|
|
|
15165
15320
|
* storm cannot accumulate claims for children that are already gone.
|
|
15166
15321
|
*/
|
|
15167
15322
|
costClaim = NO_COST_CLAIM;
|
|
15323
|
+
/**
|
|
15324
|
+
* The pid of a generation `stop()` has signalled but whose `exit` has not
|
|
15325
|
+
* fired. `proc` is dropped at the signal; this keeps the pid readable until
|
|
15326
|
+
* the child is really gone, so the staging guard (D667) still knows it is
|
|
15327
|
+
* this runner's writer and not a predecessor's.
|
|
15328
|
+
*/
|
|
15329
|
+
exitingPid = void 0;
|
|
15168
15330
|
constructor(cfg, deps) {
|
|
15169
15331
|
this.cfg = cfg;
|
|
15170
15332
|
this.deps = deps;
|
|
15171
15333
|
}
|
|
15334
|
+
/**
|
|
15335
|
+
* The pid of the ffmpeg this writer is supervising right now, or undefined
|
|
15336
|
+
* between generations. Read by the staging-directory guard (D667) so a
|
|
15337
|
+
* writer the controller is TRACKING is never mistaken for a predecessor's.
|
|
15338
|
+
*/
|
|
15339
|
+
get pid() {
|
|
15340
|
+
return this.proc?.pid ?? this.exitingPid;
|
|
15341
|
+
}
|
|
15172
15342
|
start() {
|
|
15173
15343
|
if (this.stopped || this.proc) return;
|
|
15174
15344
|
const args = buildPassthroughArgs(this.cfg);
|
|
@@ -15225,6 +15395,7 @@ var SegmentWriter = class {
|
|
|
15225
15395
|
this.resolveExit?.();
|
|
15226
15396
|
this.resolveExit = null;
|
|
15227
15397
|
this.proc = null;
|
|
15398
|
+
this.exitingPid = void 0;
|
|
15228
15399
|
if (this.stopped) return;
|
|
15229
15400
|
const now = Date.now();
|
|
15230
15401
|
const ranMs = now - this.startedAt;
|
|
@@ -15338,6 +15509,7 @@ var SegmentWriter = class {
|
|
|
15338
15509
|
const exitWait = this.exitWait;
|
|
15339
15510
|
this.proc = null;
|
|
15340
15511
|
if (!proc) return;
|
|
15512
|
+
if (proc.exitCode !== void 0 || proc.signalCode !== void 0) this.exitingPid = proc.pid;
|
|
15341
15513
|
try {
|
|
15342
15514
|
proc.kill("SIGTERM");
|
|
15343
15515
|
} catch {}
|
|
@@ -15345,7 +15517,7 @@ var SegmentWriter = class {
|
|
|
15345
15517
|
if (proc.exitCode === null && proc.signalCode === null) try {
|
|
15346
15518
|
proc.kill("SIGKILL");
|
|
15347
15519
|
} catch {}
|
|
15348
|
-
}, KILL_GRACE_MS).unref?.();
|
|
15520
|
+
}, KILL_GRACE_MS$1).unref?.();
|
|
15349
15521
|
if (proc.exitCode === void 0 && proc.signalCode === void 0) return;
|
|
15350
15522
|
await exitWait;
|
|
15351
15523
|
}
|
|
@@ -15673,10 +15845,16 @@ var RecordingController = class {
|
|
|
15673
15845
|
/** Every attach admitted before a maintenance pause. The pause waits these
|
|
15674
15846
|
* promises before it snapshots/drains active writers. */
|
|
15675
15847
|
attachingTasks = /* @__PURE__ */ new Map();
|
|
15676
|
-
/**
|
|
15677
|
-
*
|
|
15678
|
-
*
|
|
15679
|
-
|
|
15848
|
+
/**
|
|
15849
|
+
* The drain ledger: per staging directory (resolved `outDir`), what the last
|
|
15850
|
+
* watcher of THIS process drained there (`segment-watcher.ts`
|
|
15851
|
+
* `DrainedPlaylist`). Consumed by the next watcher on that directory, so a
|
|
15852
|
+
* fresh watcher reading its predecessor's playlist does not report a roll-off
|
|
15853
|
+
* that predecessor already handled or reported. Per process by design: after
|
|
15854
|
+
* a runner respawn it is empty and the roll-off IS reported (D521). At most
|
|
15855
|
+
* one entry per location × device × profile directory.
|
|
15856
|
+
*/
|
|
15857
|
+
drainedPlaylists = /* @__PURE__ */ new Map();
|
|
15680
15858
|
/**
|
|
15681
15859
|
* Monotonic attach counter, the last component of the CONSUMER TAG this
|
|
15682
15860
|
* recorder puts on its RTSP dial (`recorder/<deviceId>/<profile>/<n>`).
|
|
@@ -15737,6 +15915,13 @@ var RecordingController = class {
|
|
|
15737
15915
|
* healthy device every 30s.
|
|
15738
15916
|
*/
|
|
15739
15917
|
skippedProfiles = /* @__PURE__ */ new Map();
|
|
15918
|
+
/**
|
|
15919
|
+
* Writers being torn down whose ffmpeg may not have exited yet. They left
|
|
15920
|
+
* {@link active} when the teardown began, but their pid is still ours until
|
|
15921
|
+
* the child's `exit` fires — without this set the staging guard (D667) would
|
|
15922
|
+
* reap this runner's own exiting writer and log it as a predecessor's.
|
|
15923
|
+
*/
|
|
15924
|
+
retiring = /* @__PURE__ */ new Set();
|
|
15740
15925
|
restore = null;
|
|
15741
15926
|
/**
|
|
15742
15927
|
* One pending wake-up per device, armed for the exact instant its recording
|
|
@@ -15789,7 +15974,7 @@ var RecordingController = class {
|
|
|
15789
15974
|
this.evaluateDevice(deviceId).catch((err) => {
|
|
15790
15975
|
this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
|
|
15791
15976
|
tags: { deviceId },
|
|
15792
|
-
meta: { error: errMsg(err) }
|
|
15977
|
+
meta: { error: errMsg$1(err) }
|
|
15793
15978
|
});
|
|
15794
15979
|
});
|
|
15795
15980
|
}
|
|
@@ -15839,6 +16024,10 @@ var RecordingController = class {
|
|
|
15839
16024
|
});
|
|
15840
16025
|
return out;
|
|
15841
16026
|
}
|
|
16027
|
+
/** Entries in the drain ledger — for the memory census. */
|
|
16028
|
+
get drainedLedgerSize() {
|
|
16029
|
+
return this.drainedPlaylists.size;
|
|
16030
|
+
}
|
|
15842
16031
|
/**
|
|
15843
16032
|
* Boot restore: seed the readiness-gated queue with every persisted device and
|
|
15844
16033
|
* (re)evaluate each on every `stream-broker` ready transition. Also arms the
|
|
@@ -15850,7 +16039,7 @@ var RecordingController = class {
|
|
|
15850
16039
|
try {
|
|
15851
16040
|
ids = await this.deps.enabledDeviceIds();
|
|
15852
16041
|
} catch (err) {
|
|
15853
|
-
this.deps.logger.warn("recorder controller: reading persisted devices failed", { meta: { error: errMsg(err) } });
|
|
16042
|
+
this.deps.logger.warn("recorder controller: reading persisted devices failed", { meta: { error: errMsg$1(err) } });
|
|
15854
16043
|
ids = [];
|
|
15855
16044
|
}
|
|
15856
16045
|
this.restore = new ReadinessRestore({
|
|
@@ -15861,7 +16050,7 @@ var RecordingController = class {
|
|
|
15861
16050
|
},
|
|
15862
16051
|
onDefer: (deviceId, _v, err) => this.deps.logger.warn("recorder controller: device evaluate deferred (retries on next broker ready)", {
|
|
15863
16052
|
tags: { deviceId },
|
|
15864
|
-
meta: { error: errMsg(err) }
|
|
16053
|
+
meta: { error: errMsg$1(err) }
|
|
15865
16054
|
})
|
|
15866
16055
|
});
|
|
15867
16056
|
if (this.reconcilePass === null) {
|
|
@@ -15871,7 +16060,8 @@ var RecordingController = class {
|
|
|
15871
16060
|
slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
|
|
15872
16061
|
logger: this.deps.logger,
|
|
15873
16062
|
now: () => this.now(),
|
|
15874
|
-
run: () => this.checkIdleWriters()
|
|
16063
|
+
run: () => this.checkIdleWriters(),
|
|
16064
|
+
...this.deps.heapProbe !== void 0 ? { heapProbe: this.deps.heapProbe } : {}
|
|
15875
16065
|
});
|
|
15876
16066
|
this.placementPass = startPeriodicPass({
|
|
15877
16067
|
name: "placement",
|
|
@@ -15879,7 +16069,8 @@ var RecordingController = class {
|
|
|
15879
16069
|
slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
|
|
15880
16070
|
logger: this.deps.logger,
|
|
15881
16071
|
now: () => this.now(),
|
|
15882
|
-
run: () => this.replanPlacement()
|
|
16072
|
+
run: () => this.replanPlacement(),
|
|
16073
|
+
...this.deps.heapProbe !== void 0 ? { heapProbe: this.deps.heapProbe } : {}
|
|
15883
16074
|
});
|
|
15884
16075
|
this.reconcilePass = startPeriodicPass({
|
|
15885
16076
|
name: "reconcile",
|
|
@@ -15887,7 +16078,8 @@ var RecordingController = class {
|
|
|
15887
16078
|
slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
|
|
15888
16079
|
logger: this.deps.logger,
|
|
15889
16080
|
now: () => this.now(),
|
|
15890
|
-
run: () => this.reconcile()
|
|
16081
|
+
run: () => this.reconcile(),
|
|
16082
|
+
...this.deps.heapProbe !== void 0 ? { heapProbe: this.deps.heapProbe } : {}
|
|
15891
16083
|
});
|
|
15892
16084
|
}
|
|
15893
16085
|
await this.restore.start(ids.map((id) => [id, true]));
|
|
@@ -15904,7 +16096,7 @@ var RecordingController = class {
|
|
|
15904
16096
|
for (const id of enabled) ids.add(id);
|
|
15905
16097
|
} catch (err) {
|
|
15906
16098
|
this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
|
|
15907
|
-
error: errMsg(err),
|
|
16099
|
+
error: errMsg$1(err),
|
|
15908
16100
|
activeDevices: this.active.size
|
|
15909
16101
|
} });
|
|
15910
16102
|
}
|
|
@@ -15925,7 +16117,7 @@ var RecordingController = class {
|
|
|
15925
16117
|
try {
|
|
15926
16118
|
await this.deps.onPlacementTick(ids);
|
|
15927
16119
|
} catch (err) {
|
|
15928
|
-
this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: errMsg(err) } });
|
|
16120
|
+
this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: errMsg$1(err) } });
|
|
15929
16121
|
}
|
|
15930
16122
|
const finishedAt = this.now();
|
|
15931
16123
|
if (finishedAt - startedAt > PLACEMENT_PHASE_WARN_MS) this.deps.logger.warn("recorder: placement pass ran long — phase breakdown", { meta: {
|
|
@@ -15954,7 +16146,7 @@ var RecordingController = class {
|
|
|
15954
16146
|
} catch (err) {
|
|
15955
16147
|
this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
|
|
15956
16148
|
tags: { deviceId: id },
|
|
15957
|
-
meta: { error: errMsg(err) }
|
|
16149
|
+
meta: { error: errMsg$1(err) }
|
|
15958
16150
|
});
|
|
15959
16151
|
}
|
|
15960
16152
|
});
|
|
@@ -16050,7 +16242,7 @@ var RecordingController = class {
|
|
|
16050
16242
|
} catch (err) {
|
|
16051
16243
|
this.deps.logger.warn("recorder: trigger reconcile failed — retries on the next beat", {
|
|
16052
16244
|
tags: { deviceId },
|
|
16053
|
-
meta: { error: errMsg(err) }
|
|
16245
|
+
meta: { error: errMsg$1(err) }
|
|
16054
16246
|
});
|
|
16055
16247
|
}
|
|
16056
16248
|
}
|
|
@@ -16077,7 +16269,7 @@ var RecordingController = class {
|
|
|
16077
16269
|
tags: { deviceId },
|
|
16078
16270
|
meta: {
|
|
16079
16271
|
held: since !== null,
|
|
16080
|
-
error: errMsg(err)
|
|
16272
|
+
error: errMsg$1(err)
|
|
16081
16273
|
}
|
|
16082
16274
|
});
|
|
16083
16275
|
this.releaseIfHeldPastBound(deviceId, source, since, postMs, nowMs);
|
|
@@ -16207,7 +16399,7 @@ var RecordingController = class {
|
|
|
16207
16399
|
meta: {
|
|
16208
16400
|
source: triggerKey(source),
|
|
16209
16401
|
active,
|
|
16210
|
-
error: errMsg(err)
|
|
16402
|
+
error: errMsg$1(err)
|
|
16211
16403
|
}
|
|
16212
16404
|
});
|
|
16213
16405
|
}
|
|
@@ -16254,7 +16446,7 @@ var RecordingController = class {
|
|
|
16254
16446
|
this.restore?.add(deviceId, true);
|
|
16255
16447
|
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
|
|
16256
16448
|
tags: { deviceId },
|
|
16257
|
-
meta: { error: errMsg(err) }
|
|
16449
|
+
meta: { error: errMsg$1(err) }
|
|
16258
16450
|
});
|
|
16259
16451
|
}
|
|
16260
16452
|
return;
|
|
@@ -16273,7 +16465,7 @@ var RecordingController = class {
|
|
|
16273
16465
|
this.deps.logger.warn("recorder: stalled-profile recovery deferred (retries on next broker ready/reconcile)", {
|
|
16274
16466
|
tags: { deviceId },
|
|
16275
16467
|
meta: {
|
|
16276
|
-
error: errMsg(err),
|
|
16468
|
+
error: errMsg$1(err),
|
|
16277
16469
|
profile: stalled.map((r) => r.profile)
|
|
16278
16470
|
}
|
|
16279
16471
|
});
|
|
@@ -16287,7 +16479,6 @@ var RecordingController = class {
|
|
|
16287
16479
|
for (const r of stalled) {
|
|
16288
16480
|
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
16289
16481
|
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
16290
|
-
this.firstSegmentSeen.delete(idleKey(deviceId, r.profile));
|
|
16291
16482
|
}
|
|
16292
16483
|
this.active.set(deviceId, [...keep]);
|
|
16293
16484
|
const skipped = this.skippedProfiles.get(deviceId) ?? /* @__PURE__ */ new Set();
|
|
@@ -16386,7 +16577,7 @@ var RecordingController = class {
|
|
|
16386
16577
|
meta: {
|
|
16387
16578
|
source: triggerKey(source),
|
|
16388
16579
|
atMs,
|
|
16389
|
-
error: errMsg(err)
|
|
16580
|
+
error: errMsg$1(err)
|
|
16390
16581
|
}
|
|
16391
16582
|
});
|
|
16392
16583
|
}
|
|
@@ -16420,7 +16611,7 @@ var RecordingController = class {
|
|
|
16420
16611
|
meta: {
|
|
16421
16612
|
atMs,
|
|
16422
16613
|
dbfs,
|
|
16423
|
-
error: errMsg(err)
|
|
16614
|
+
error: errMsg$1(err)
|
|
16424
16615
|
}
|
|
16425
16616
|
});
|
|
16426
16617
|
return;
|
|
@@ -16445,7 +16636,7 @@ var RecordingController = class {
|
|
|
16445
16636
|
tags: { deviceId },
|
|
16446
16637
|
meta: {
|
|
16447
16638
|
atMs,
|
|
16448
|
-
error: errMsg(err)
|
|
16639
|
+
error: errMsg$1(err)
|
|
16449
16640
|
}
|
|
16450
16641
|
});
|
|
16451
16642
|
}
|
|
@@ -16638,6 +16829,7 @@ var RecordingController = class {
|
|
|
16638
16829
|
if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
|
|
16639
16830
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
16640
16831
|
const bandAudio = bandAudioAt(config, new Date(this.now()));
|
|
16832
|
+
await this.sweepDeviceStaging(deviceId, segmentSeconds);
|
|
16641
16833
|
const recordings = [];
|
|
16642
16834
|
const failed = [];
|
|
16643
16835
|
for (const profile of profiles) try {
|
|
@@ -16648,7 +16840,7 @@ var RecordingController = class {
|
|
|
16648
16840
|
tags: { deviceId },
|
|
16649
16841
|
meta: {
|
|
16650
16842
|
profile,
|
|
16651
|
-
error: errMsg(err)
|
|
16843
|
+
error: errMsg$1(err)
|
|
16652
16844
|
}
|
|
16653
16845
|
});
|
|
16654
16846
|
}
|
|
@@ -16705,6 +16897,7 @@ var RecordingController = class {
|
|
|
16705
16897
|
const have = new Set(current.map((r) => r.profile));
|
|
16706
16898
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
16707
16899
|
const bandAudio = bandAudioAt(config, new Date(this.now()));
|
|
16900
|
+
await this.sweepDeviceStaging(deviceId, segmentSeconds);
|
|
16708
16901
|
const added = [];
|
|
16709
16902
|
for (const profile of [...skipped]) {
|
|
16710
16903
|
if (have.has(profile)) {
|
|
@@ -16719,7 +16912,7 @@ var RecordingController = class {
|
|
|
16719
16912
|
tags: { deviceId },
|
|
16720
16913
|
meta: {
|
|
16721
16914
|
profile,
|
|
16722
|
-
error: errMsg(err)
|
|
16915
|
+
error: errMsg$1(err)
|
|
16723
16916
|
}
|
|
16724
16917
|
});
|
|
16725
16918
|
}
|
|
@@ -16752,6 +16945,143 @@ var RecordingController = class {
|
|
|
16752
16945
|
return resolveRecordingProfiles(selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile), override);
|
|
16753
16946
|
}
|
|
16754
16947
|
/**
|
|
16948
|
+
* Entry counts of every long-lived container this controller holds, for the
|
|
16949
|
+
* recorder's 5-minute census line. Sizes only — no walk of anything that
|
|
16950
|
+
* grows with the archive.
|
|
16951
|
+
*/
|
|
16952
|
+
memoryCensus() {
|
|
16953
|
+
let writers = 0;
|
|
16954
|
+
for (const recordings of this.active.values()) writers += recordings.length;
|
|
16955
|
+
let skippedProfiles = 0;
|
|
16956
|
+
for (const skipped of this.skippedProfiles.values()) skippedProfiles += skipped.size;
|
|
16957
|
+
return {
|
|
16958
|
+
activeDevices: this.active.size,
|
|
16959
|
+
writers,
|
|
16960
|
+
attaching: this.attaching.size,
|
|
16961
|
+
attachingTasks: this.attachingTasks.size,
|
|
16962
|
+
drainedPlaylists: this.drainedPlaylists.size,
|
|
16963
|
+
batteryWakes: this.batteryWakes.size,
|
|
16964
|
+
triggers: this.triggers.size,
|
|
16965
|
+
signalConfirmedAt: this.signalConfirmedAt.size,
|
|
16966
|
+
signalContradictions: this.signalContradictions.size,
|
|
16967
|
+
lastSegmentAt: this.lastSegmentAt.size,
|
|
16968
|
+
attachRetained: this.attachRetained.size,
|
|
16969
|
+
unrecordableReported: this.unrecordableReported.size,
|
|
16970
|
+
skippedProfileDevices: this.skippedProfiles.size,
|
|
16971
|
+
skippedProfiles
|
|
16972
|
+
};
|
|
16973
|
+
}
|
|
16974
|
+
/**
|
|
16975
|
+
* Pids of every ffmpeg this controller started and has not seen exit —
|
|
16976
|
+
* the live writers and the ones still being torn down. Never reaped by the
|
|
16977
|
+
* staging guard (D667).
|
|
16978
|
+
*/
|
|
16979
|
+
ownedWriterPids() {
|
|
16980
|
+
const pids = /* @__PURE__ */ new Set();
|
|
16981
|
+
const add = (writer) => {
|
|
16982
|
+
const pid = writer.pid;
|
|
16983
|
+
if (pid !== void 0) pids.add(pid);
|
|
16984
|
+
};
|
|
16985
|
+
for (const recordings of this.active.values()) for (const recording of recordings) add(recording.writer);
|
|
16986
|
+
for (const writer of this.retiring) add(writer);
|
|
16987
|
+
return pids;
|
|
16988
|
+
}
|
|
16989
|
+
/**
|
|
16990
|
+
* Every staging directory of this camera, on every location, in ONE table
|
|
16991
|
+
* read and one SIGTERM → SIGKILL window (D667) — so an attach of three
|
|
16992
|
+
* profiles does not pay three reaps in series. The per-spawn claim below
|
|
16993
|
+
* still runs; after this it is one cached read.
|
|
16994
|
+
*/
|
|
16995
|
+
async sweepDeviceStaging(deviceId, segmentSeconds) {
|
|
16996
|
+
const guard = this.deps.stagingGuard;
|
|
16997
|
+
if (guard === void 0) return;
|
|
16998
|
+
const result = await guard.sweep({
|
|
16999
|
+
roots: this.deps.locations().map((location) => location.root),
|
|
17000
|
+
deviceId,
|
|
17001
|
+
ownPids: this.ownedWriterPids()
|
|
17002
|
+
});
|
|
17003
|
+
if (result.outcome !== "swept" || result.reaped.length === 0) return;
|
|
17004
|
+
const liveDirs = new Set((this.active.get(deviceId) ?? []).map((r) => r.outDir));
|
|
17005
|
+
await this.recoverReaped(result.reaped.filter((writer) => !liveDirs.has(writer.target.outDir)), segmentSeconds);
|
|
17006
|
+
}
|
|
17007
|
+
/**
|
|
17008
|
+
* Collect what each reaped writer left in its directory before a new writer
|
|
17009
|
+
* is spawned into it (D667). Logged per directory with the counts, and a
|
|
17010
|
+
* directory that could not be recovered says so — its segments stay staged
|
|
17011
|
+
* for the next boot.
|
|
17012
|
+
*/
|
|
17013
|
+
async recoverReaped(reaped, segmentSeconds) {
|
|
17014
|
+
const cutoffMs = this.now();
|
|
17015
|
+
const dirs = /* @__PURE__ */ new Map();
|
|
17016
|
+
for (const writer of reaped) dirs.set(writer.target.outDir, writer);
|
|
17017
|
+
for (const writer of dirs.values()) {
|
|
17018
|
+
const { target } = writer;
|
|
17019
|
+
const location = this.deps.locations().find((l) => l.root === target.root);
|
|
17020
|
+
const recover = this.deps.recoverStagingDirectory;
|
|
17021
|
+
if (location === void 0 || recover === void 0) {
|
|
17022
|
+
this.deps.logger.warn("recorder: a reaped writer’s segments were left in staging — recovered at the next boot", {
|
|
17023
|
+
tags: { deviceId: target.deviceId },
|
|
17024
|
+
meta: {
|
|
17025
|
+
profile: target.profile,
|
|
17026
|
+
dir: target.outDir,
|
|
17027
|
+
reason: location === void 0 ? "location-unknown" : "no-recovery-wired"
|
|
17028
|
+
}
|
|
17029
|
+
});
|
|
17030
|
+
continue;
|
|
17031
|
+
}
|
|
17032
|
+
try {
|
|
17033
|
+
const report = await recover({
|
|
17034
|
+
deviceId: target.deviceId,
|
|
17035
|
+
profile: target.profile,
|
|
17036
|
+
locationId: location.id,
|
|
17037
|
+
locationRoot: location.root,
|
|
17038
|
+
dir: target.outDir,
|
|
17039
|
+
cutoffMs,
|
|
17040
|
+
segmentSeconds
|
|
17041
|
+
});
|
|
17042
|
+
this.deps.logger.info("recorder: recovered a reaped writer’s segments before re-attaching", {
|
|
17043
|
+
tags: { deviceId: target.deviceId },
|
|
17044
|
+
meta: {
|
|
17045
|
+
profile: target.profile,
|
|
17046
|
+
dir: target.outDir,
|
|
17047
|
+
pid: writer.pid,
|
|
17048
|
+
...report
|
|
17049
|
+
}
|
|
17050
|
+
});
|
|
17051
|
+
} catch (err) {
|
|
17052
|
+
this.deps.logger.warn("recorder: recovering a reaped writer’s segments failed — left in staging for the next boot", {
|
|
17053
|
+
tags: { deviceId: target.deviceId },
|
|
17054
|
+
meta: {
|
|
17055
|
+
profile: target.profile,
|
|
17056
|
+
dir: target.outDir,
|
|
17057
|
+
error: errMsg$1(err)
|
|
17058
|
+
}
|
|
17059
|
+
});
|
|
17060
|
+
}
|
|
17061
|
+
}
|
|
17062
|
+
}
|
|
17063
|
+
/**
|
|
17064
|
+
* Make sure nobody else is writing `outDir` before a writer is spawned into
|
|
17065
|
+
* it (D667). A predecessor runner that died without a shutdown (V8 OOM,
|
|
17066
|
+
* SIGKILL) leaves its ffmpeg writers running, and a second writer in the
|
|
17067
|
+
* same directory rewrites the first one's playlist — 13 minutes and 169
|
|
17068
|
+
* writer churns on 2026-09-28. The guard reaps such a writer; one it cannot
|
|
17069
|
+
* remove is a refusal, and a refusal throws so no writer is spawned.
|
|
17070
|
+
*/
|
|
17071
|
+
async claimStagingDirectory(deviceId, profile, root, outDir, segmentSeconds) {
|
|
17072
|
+
const guard = this.deps.stagingGuard;
|
|
17073
|
+
if (guard === void 0) return;
|
|
17074
|
+
const result = await guard.claim({
|
|
17075
|
+
deviceId,
|
|
17076
|
+
profile,
|
|
17077
|
+
root,
|
|
17078
|
+
outDir,
|
|
17079
|
+
ownPids: this.ownedWriterPids()
|
|
17080
|
+
});
|
|
17081
|
+
if (result.outcome === "clear" && result.reaped.length > 0) await this.recoverReaped(result.reaped, segmentSeconds);
|
|
17082
|
+
if (result.outcome === "refused") throw new Error(`staging directory ${outDir} still has ${result.survivors.length} foreign writer(s) (pids ${result.survivors.map((proc) => proc.pid).join(",")}) — refusing to co-write`);
|
|
17083
|
+
}
|
|
17084
|
+
/**
|
|
16755
17085
|
* Attach one (device, profile): acquire the broker source, mkdir the staging
|
|
16756
17086
|
* outDir UNDER the profile's placement root (same-fs relocate — see
|
|
16757
17087
|
* {@link STAGING_DIR_NAME}), spawn the SegmentWriter, and start the watcher
|
|
@@ -16760,6 +17090,10 @@ var RecordingController = class {
|
|
|
16760
17090
|
async attachProfile(deviceId, profile, segmentSeconds, audio) {
|
|
16761
17091
|
const attachProfileStartMs = this.now();
|
|
16762
17092
|
const placement = await this.resolvePlacement(deviceId, profile);
|
|
17093
|
+
const placementDoneMs = this.now();
|
|
17094
|
+
const outDir = path.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
|
|
17095
|
+
await this.claimStagingDirectory(deviceId, profile, placement.root, outDir, segmentSeconds);
|
|
17096
|
+
const claimDoneMs = this.now();
|
|
16763
17097
|
const source = await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.getStreamWithCodec.mutate({
|
|
16764
17098
|
deviceId,
|
|
16765
17099
|
video: "copy",
|
|
@@ -16779,8 +17113,6 @@ var RecordingController = class {
|
|
|
16779
17113
|
url: source.url
|
|
16780
17114
|
}
|
|
16781
17115
|
});
|
|
16782
|
-
const placementDoneMs = this.now();
|
|
16783
|
-
const outDir = path.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
|
|
16784
17116
|
await promises.mkdir(outDir, { recursive: true });
|
|
16785
17117
|
const mkdirDoneMs = this.now();
|
|
16786
17118
|
const deviceLog = this.deps.logger.withTags({ deviceId });
|
|
@@ -16813,18 +17145,27 @@ var RecordingController = class {
|
|
|
16813
17145
|
profile,
|
|
16814
17146
|
consumer: consumerTag,
|
|
16815
17147
|
placementMs: placementDoneMs - attachProfileStartMs,
|
|
16816
|
-
|
|
17148
|
+
claimMs: claimDoneMs - placementDoneMs,
|
|
17149
|
+
getStreamMs: streamAcquiredMs - claimDoneMs,
|
|
16817
17150
|
mkdirMs: mkdirDoneMs - streamAcquiredMs,
|
|
16818
17151
|
attachProfileMs: writerStartedMs - attachProfileStartMs,
|
|
16819
17152
|
segmentSeconds
|
|
16820
17153
|
}
|
|
16821
17154
|
});
|
|
16822
17155
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
17156
|
+
let firstSegmentSeen = false;
|
|
17157
|
+
const drainKey = path.resolve(outDir);
|
|
17158
|
+
const drainedPredecessor = this.drainedPlaylists.get(drainKey) ?? null;
|
|
17159
|
+
this.drainedPlaylists.delete(drainKey);
|
|
16823
17160
|
return {
|
|
16824
17161
|
profile,
|
|
16825
17162
|
writer,
|
|
16826
17163
|
watcher: startSegmentWatcher({
|
|
16827
17164
|
outDir,
|
|
17165
|
+
drainedPredecessor,
|
|
17166
|
+
onDrained: (drained) => {
|
|
17167
|
+
this.drainedPlaylists.set(drainKey, drained);
|
|
17168
|
+
},
|
|
16828
17169
|
deviceId,
|
|
16829
17170
|
writerStartedMs,
|
|
16830
17171
|
now: () => this.now(),
|
|
@@ -16834,8 +17175,8 @@ var RecordingController = class {
|
|
|
16834
17175
|
intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
|
|
16835
17176
|
onFinalized: async (startMs, durMs, flatAbsPath) => {
|
|
16836
17177
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
16837
|
-
if (startMs >= writerStartedMs - 1e3 && !
|
|
16838
|
-
|
|
17178
|
+
if (startMs >= writerStartedMs - 1e3 && !firstSegmentSeen) {
|
|
17179
|
+
firstSegmentSeen = true;
|
|
16839
17180
|
this.deps.logger.info("recorder: first segment of this attach", {
|
|
16840
17181
|
tags: { deviceId },
|
|
16841
17182
|
meta: {
|
|
@@ -16895,7 +17236,7 @@ var RecordingController = class {
|
|
|
16895
17236
|
tags: { deviceId },
|
|
16896
17237
|
meta: {
|
|
16897
17238
|
profile,
|
|
16898
|
-
error: errMsg(err)
|
|
17239
|
+
error: errMsg$1(err)
|
|
16899
17240
|
}
|
|
16900
17241
|
});
|
|
16901
17242
|
}
|
|
@@ -16959,8 +17300,11 @@ var RecordingController = class {
|
|
|
16959
17300
|
async teardownProfile(deviceId, r) {
|
|
16960
17301
|
const teardownStartMs = Date.now();
|
|
16961
17302
|
try {
|
|
17303
|
+
this.retiring.add(r.writer);
|
|
16962
17304
|
await r.writer.stopAndWait();
|
|
16963
|
-
} catch {}
|
|
17305
|
+
} catch {} finally {
|
|
17306
|
+
this.retiring.delete(r.writer);
|
|
17307
|
+
}
|
|
16964
17308
|
const writerStopMs = Date.now() - teardownStartMs;
|
|
16965
17309
|
try {
|
|
16966
17310
|
await r.watcher.flush?.();
|
|
@@ -16980,7 +17324,7 @@ var RecordingController = class {
|
|
|
16980
17324
|
tags: { deviceId },
|
|
16981
17325
|
meta: {
|
|
16982
17326
|
profile: r.profile,
|
|
16983
|
-
error: errMsg(err)
|
|
17327
|
+
error: errMsg$1(err)
|
|
16984
17328
|
}
|
|
16985
17329
|
});
|
|
16986
17330
|
}
|
|
@@ -16995,7 +17339,7 @@ var RecordingController = class {
|
|
|
16995
17339
|
meta: {
|
|
16996
17340
|
profile: r.profile,
|
|
16997
17341
|
pipelineKey: r.pipelineKey,
|
|
16998
|
-
error: errMsg(err)
|
|
17342
|
+
error: errMsg$1(err)
|
|
16999
17343
|
}
|
|
17000
17344
|
});
|
|
17001
17345
|
}
|
|
@@ -17009,7 +17353,6 @@ var RecordingController = class {
|
|
|
17009
17353
|
for (const r of recordings) {
|
|
17010
17354
|
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
17011
17355
|
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
17012
|
-
this.firstSegmentSeen.delete(idleKey(deviceId, r.profile));
|
|
17013
17356
|
}
|
|
17014
17357
|
await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
|
|
17015
17358
|
this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
|
|
@@ -17033,7 +17376,7 @@ var RecordingController = class {
|
|
|
17033
17376
|
try {
|
|
17034
17377
|
ids = await this.deps.enabledDeviceIds();
|
|
17035
17378
|
} catch (err) {
|
|
17036
|
-
this.deps.logger.warn("recorder: resume could not enumerate configured devices", { meta: { error: errMsg(err) } });
|
|
17379
|
+
this.deps.logger.warn("recorder: resume could not enumerate configured devices", { meta: { error: errMsg$1(err) } });
|
|
17037
17380
|
}
|
|
17038
17381
|
for (const deviceId of ids) await this.evaluateDevice(deviceId);
|
|
17039
17382
|
}
|
|
@@ -17440,12 +17783,12 @@ async function ourAvailability(deps, deviceId) {
|
|
|
17440
17783
|
tags: { deviceId },
|
|
17441
17784
|
meta: {
|
|
17442
17785
|
source: RECORDING_SOURCE_CAMSTACK,
|
|
17443
|
-
error: errMsg(err)
|
|
17786
|
+
error: errMsg$1(err)
|
|
17444
17787
|
}
|
|
17445
17788
|
});
|
|
17446
17789
|
return {
|
|
17447
17790
|
state: "unreachable",
|
|
17448
|
-
reason: errMsg(err)
|
|
17791
|
+
reason: errMsg$1(err)
|
|
17449
17792
|
};
|
|
17450
17793
|
}
|
|
17451
17794
|
}
|
|
@@ -17471,7 +17814,7 @@ function buildRecordingSourceProvider(deps) {
|
|
|
17471
17814
|
tags: { deviceId },
|
|
17472
17815
|
meta: {
|
|
17473
17816
|
source: RECORDING_SOURCE_CAMSTACK,
|
|
17474
|
-
error: errMsg(err),
|
|
17817
|
+
error: errMsg$1(err),
|
|
17475
17818
|
fromMs,
|
|
17476
17819
|
toMs
|
|
17477
17820
|
}
|
|
@@ -17496,7 +17839,7 @@ function buildRecordingSourceProvider(deps) {
|
|
|
17496
17839
|
tags: { deviceId },
|
|
17497
17840
|
meta: {
|
|
17498
17841
|
source: RECORDING_SOURCE_CAMSTACK,
|
|
17499
|
-
error: errMsg(err),
|
|
17842
|
+
error: errMsg$1(err),
|
|
17500
17843
|
fromMs,
|
|
17501
17844
|
toMs
|
|
17502
17845
|
}
|
|
@@ -17524,14 +17867,14 @@ function buildRecordingSourceProvider(deps) {
|
|
|
17524
17867
|
tags: { deviceId },
|
|
17525
17868
|
meta: {
|
|
17526
17869
|
source: RECORDING_SOURCE_CAMSTACK,
|
|
17527
|
-
error: errMsg(err),
|
|
17870
|
+
error: errMsg$1(err),
|
|
17528
17871
|
fromMs,
|
|
17529
17872
|
toMs
|
|
17530
17873
|
}
|
|
17531
17874
|
});
|
|
17532
17875
|
return {
|
|
17533
17876
|
kind: "none",
|
|
17534
|
-
reason: `the recorder could not answer: ${errMsg(err)}`
|
|
17877
|
+
reason: `the recorder could not answer: ${errMsg$1(err)}`
|
|
17535
17878
|
};
|
|
17536
17879
|
}
|
|
17537
17880
|
},
|
|
@@ -17668,7 +18011,11 @@ var RedundancySweeper = class {
|
|
|
17668
18011
|
start() {
|
|
17669
18012
|
if (this.timer !== null) return;
|
|
17670
18013
|
this.timer = setInterval(() => {
|
|
17671
|
-
this.
|
|
18014
|
+
const probe = this.deps.heapProbe;
|
|
18015
|
+
probe === void 0 ? this.sweep() : probe.track({
|
|
18016
|
+
kind: "pass",
|
|
18017
|
+
name: "redundancy"
|
|
18018
|
+
}, () => this.sweep());
|
|
17672
18019
|
}, this.deps.intervalMs);
|
|
17673
18020
|
this.timer.unref?.();
|
|
17674
18021
|
}
|
|
@@ -17699,7 +18046,7 @@ var RedundancySweeper = class {
|
|
|
17699
18046
|
try {
|
|
17700
18047
|
deviceIds = await this.deps.deviceIds();
|
|
17701
18048
|
} catch (err) {
|
|
17702
|
-
this.deps.logger.warn("recorder: redundancy sweep could not read the device list", { meta: { error: errMsg(err) } });
|
|
18049
|
+
this.deps.logger.warn("recorder: redundancy sweep could not read the device list", { meta: { error: errMsg$1(err) } });
|
|
17703
18050
|
return empty;
|
|
17704
18051
|
}
|
|
17705
18052
|
const toMs = (this.deps.now ?? Date.now)();
|
|
@@ -17726,7 +18073,7 @@ var RedundancySweeper = class {
|
|
|
17726
18073
|
profile,
|
|
17727
18074
|
locationId,
|
|
17728
18075
|
count: group.length,
|
|
17729
|
-
error: errMsg(err)
|
|
18076
|
+
error: errMsg$1(err)
|
|
17730
18077
|
}
|
|
17731
18078
|
});
|
|
17732
18079
|
}
|
|
@@ -17811,7 +18158,11 @@ var RetentionSweeper = class {
|
|
|
17811
18158
|
start() {
|
|
17812
18159
|
if (this.timer !== null) return;
|
|
17813
18160
|
const tick = () => {
|
|
17814
|
-
this.
|
|
18161
|
+
const probe = this.deps.heapProbe;
|
|
18162
|
+
probe === void 0 ? this.sweep() : probe.track({
|
|
18163
|
+
kind: "pass",
|
|
18164
|
+
name: "retention"
|
|
18165
|
+
}, () => this.sweep());
|
|
17815
18166
|
};
|
|
17816
18167
|
tick();
|
|
17817
18168
|
this.timer = setInterval(tick, this.deps.intervalMs);
|
|
@@ -17841,7 +18192,7 @@ var RetentionSweeper = class {
|
|
|
17841
18192
|
try {
|
|
17842
18193
|
configs = await this.deps.readConfigs();
|
|
17843
18194
|
} catch (err) {
|
|
17844
|
-
this.deps.logger.warn("recorder: retention sweep could not read device configs", { meta: { error: errMsg(err) } });
|
|
18195
|
+
this.deps.logger.warn("recorder: retention sweep could not read device configs", { meta: { error: errMsg$1(err) } });
|
|
17845
18196
|
return;
|
|
17846
18197
|
}
|
|
17847
18198
|
const withPolicy = [...configs].filter(([, config]) => hasRetentionPolicy(config));
|
|
@@ -17871,7 +18222,7 @@ var RetentionSweeper = class {
|
|
|
17871
18222
|
} catch (err) {
|
|
17872
18223
|
this.deps.logger.warn("recorder: retention prune failed for device", {
|
|
17873
18224
|
tags: { deviceId },
|
|
17874
|
-
meta: { error: errMsg(err) }
|
|
18225
|
+
meta: { error: errMsg$1(err) }
|
|
17875
18226
|
});
|
|
17876
18227
|
}
|
|
17877
18228
|
}
|
|
@@ -17906,6 +18257,799 @@ var RECORDER_RETIRED_ROOT_KEYS = [
|
|
|
17906
18257
|
}
|
|
17907
18258
|
];
|
|
17908
18259
|
//#endregion
|
|
18260
|
+
//#region src/recorder/addon/heap-delta.ts
|
|
18261
|
+
/**
|
|
18262
|
+
* Name the unit of work that grows the recorder's heap.
|
|
18263
|
+
*
|
|
18264
|
+
* Every recorder lifetime has the same shape (findings 2026-09-28): ~50 MB for
|
|
18265
|
+
* five minutes, then one SILENT, CPU-bound phase of 4.5–7 minutes that builds
|
|
18266
|
+
* ~2.3 GB — ~910 MB of it in `large_object_space` — and holds it. The
|
|
18267
|
+
* large-object floor doubled at one boot (2026-09-03 06:56Z, 458 → 914 MB),
|
|
18268
|
+
* the first to show a second recording location. Nothing logs during the
|
|
18269
|
+
* phase, so nothing says which pass or which inbound call builds it, and a
|
|
18270
|
+
* heap snapshot is not allowed on this host.
|
|
18271
|
+
*
|
|
18272
|
+
* So every periodic pass, every boot phase and every inbound cap method is
|
|
18273
|
+
* wrapped here: two cheap heap reads (`v8.getHeapStatistics` and
|
|
18274
|
+
* `v8.getHeapSpaceStatistics` — a handful of counters, no heap walk), and ONE
|
|
18275
|
+
* line when the unit grew the heap or the large-object space by at least
|
|
18276
|
+
* `deltaBytes`, or ran for at least `durationMs`. Silent otherwise.
|
|
18277
|
+
*
|
|
18278
|
+
* Two things this line is NOT, and a reader must not take it for:
|
|
18279
|
+
*
|
|
18280
|
+
* - **Not exclusive.** The delta is over the unit's WALL window, so concurrent
|
|
18281
|
+
* work that interleaves with it is counted too. `inFlight` on the line is the
|
|
18282
|
+
* MOST other measured units that were open at any moment DURING this one —
|
|
18283
|
+
* a long pass that started alone and was overlapped by forty cap calls
|
|
18284
|
+
* reports 40, not 0. `inFlight: 0` is the only reading that pins the delta
|
|
18285
|
+
* on this unit (unmeasured work — a transport decode, a timer — can still
|
|
18286
|
+
* interleave).
|
|
18287
|
+
* - **Not retention.** `heapUsed` includes garbage no GC has collected yet, so
|
|
18288
|
+
* a unit that ALLOCATES 50 MB of transient reads is named even though it
|
|
18289
|
+
* keeps none of it. A unit named on several passes in a row while the heap
|
|
18290
|
+
* floor rises is a builder; one named once is only a candidate.
|
|
18291
|
+
*/
|
|
18292
|
+
var MIB = 1024 * 1024;
|
|
18293
|
+
var DEFAULT_HEAP_DELTA_THRESHOLDS = {
|
|
18294
|
+
deltaBytes: 50 * MIB,
|
|
18295
|
+
durationMs: 5e3
|
|
18296
|
+
};
|
|
18297
|
+
/**
|
|
18298
|
+
* Whether one unit of work is worth a line. GROWTH only: a unit that happened
|
|
18299
|
+
* to straddle a major GC shows a large negative delta that says nothing about
|
|
18300
|
+
* what it retained, and the question being asked is who BUILDS the heap.
|
|
18301
|
+
*/
|
|
18302
|
+
function judgeHeapDelta(before, after, durationMs, thresholds) {
|
|
18303
|
+
const heapDeltaBytes = before !== null && after !== null ? after.heapUsedBytes - before.heapUsedBytes : null;
|
|
18304
|
+
const losBefore = before?.largeObjectBytes ?? null;
|
|
18305
|
+
const losAfter = after?.largeObjectBytes ?? null;
|
|
18306
|
+
const largeObjectDeltaBytes = losBefore !== null && losAfter !== null ? losAfter - losBefore : null;
|
|
18307
|
+
const reasons = [];
|
|
18308
|
+
if (heapDeltaBytes !== null && heapDeltaBytes >= thresholds.deltaBytes) reasons.push("heap");
|
|
18309
|
+
if (largeObjectDeltaBytes !== null && largeObjectDeltaBytes >= thresholds.deltaBytes) reasons.push("large-object-space");
|
|
18310
|
+
if (durationMs >= thresholds.durationMs) reasons.push("duration");
|
|
18311
|
+
return {
|
|
18312
|
+
reasons,
|
|
18313
|
+
heapDeltaBytes,
|
|
18314
|
+
largeObjectDeltaBytes
|
|
18315
|
+
};
|
|
18316
|
+
}
|
|
18317
|
+
/** V8's own counters. `null` if V8 refuses — the caller logs nulls, not zeros. */
|
|
18318
|
+
function readV8HeapSample() {
|
|
18319
|
+
try {
|
|
18320
|
+
const heapUsedBytes = v8.getHeapStatistics().used_heap_size;
|
|
18321
|
+
const los = v8.getHeapSpaceStatistics().find((space) => space.space_name === "large_object_space");
|
|
18322
|
+
return {
|
|
18323
|
+
heapUsedBytes,
|
|
18324
|
+
largeObjectBytes: los === void 0 ? null : los.space_used_size
|
|
18325
|
+
};
|
|
18326
|
+
} catch {
|
|
18327
|
+
return null;
|
|
18328
|
+
}
|
|
18329
|
+
}
|
|
18330
|
+
function mb(bytes) {
|
|
18331
|
+
return bytes === null ? null : Math.round(bytes / MIB);
|
|
18332
|
+
}
|
|
18333
|
+
function isPromiseLike(value) {
|
|
18334
|
+
return typeof value === "object" && value !== null && "then" in value && typeof value.then === "function";
|
|
18335
|
+
}
|
|
18336
|
+
var HeapDeltaProbe = class {
|
|
18337
|
+
deps;
|
|
18338
|
+
/** Open units → the most OTHER units seen open at once while it ran. */
|
|
18339
|
+
peers = /* @__PURE__ */ new Map();
|
|
18340
|
+
seq = 0;
|
|
18341
|
+
sampler;
|
|
18342
|
+
now;
|
|
18343
|
+
thresholds;
|
|
18344
|
+
constructor(deps) {
|
|
18345
|
+
this.deps = deps;
|
|
18346
|
+
this.sampler = deps.sampler ?? readV8HeapSample;
|
|
18347
|
+
this.now = deps.now ?? (() => Date.now());
|
|
18348
|
+
this.thresholds = deps.thresholds ?? DEFAULT_HEAP_DELTA_THRESHOLDS;
|
|
18349
|
+
}
|
|
18350
|
+
async track(unit, run) {
|
|
18351
|
+
const open = this.begin();
|
|
18352
|
+
let failed = false;
|
|
18353
|
+
try {
|
|
18354
|
+
return await run();
|
|
18355
|
+
} catch (err) {
|
|
18356
|
+
failed = true;
|
|
18357
|
+
throw err;
|
|
18358
|
+
} finally {
|
|
18359
|
+
this.end(unit, open, failed);
|
|
18360
|
+
}
|
|
18361
|
+
}
|
|
18362
|
+
/**
|
|
18363
|
+
* The same provider, with every method measured. A `Proxy` rather than a
|
|
18364
|
+
* rebuilt object so the capability's own type passes through untouched —
|
|
18365
|
+
* no method is re-declared, and none can be forgotten when the cap grows
|
|
18366
|
+
* one. `this` stays the original object, so a class with private state
|
|
18367
|
+
* keeps working.
|
|
18368
|
+
*/
|
|
18369
|
+
wrapProvider(capability, provider) {
|
|
18370
|
+
const cache = /* @__PURE__ */ new Map();
|
|
18371
|
+
return new Proxy(provider, { get: (target, prop) => {
|
|
18372
|
+
const value = Reflect.get(target, prop, target);
|
|
18373
|
+
if (typeof value !== "function" || typeof prop === "symbol" || prop in Object.prototype) return value;
|
|
18374
|
+
const cached = cache.get(prop);
|
|
18375
|
+
if (cached !== void 0 && cached.original === value) return cached.wrapped;
|
|
18376
|
+
const unit = {
|
|
18377
|
+
kind: "cap-method",
|
|
18378
|
+
name: `${capability}.${prop}`
|
|
18379
|
+
};
|
|
18380
|
+
const wrapped = (...args) => this.call(unit, () => Reflect.apply(value, target, args));
|
|
18381
|
+
cache.set(prop, {
|
|
18382
|
+
original: value,
|
|
18383
|
+
wrapped
|
|
18384
|
+
});
|
|
18385
|
+
return wrapped;
|
|
18386
|
+
} });
|
|
18387
|
+
}
|
|
18388
|
+
call(unit, invoke) {
|
|
18389
|
+
const open = this.begin();
|
|
18390
|
+
let result;
|
|
18391
|
+
try {
|
|
18392
|
+
result = invoke();
|
|
18393
|
+
} catch (err) {
|
|
18394
|
+
this.end(unit, open, true);
|
|
18395
|
+
throw err;
|
|
18396
|
+
}
|
|
18397
|
+
if (!isPromiseLike(result)) {
|
|
18398
|
+
this.end(unit, open, false);
|
|
18399
|
+
return result;
|
|
18400
|
+
}
|
|
18401
|
+
return Promise.resolve(result).then((value) => {
|
|
18402
|
+
this.end(unit, open, false);
|
|
18403
|
+
return value;
|
|
18404
|
+
}, (err) => {
|
|
18405
|
+
this.end(unit, open, true);
|
|
18406
|
+
throw err;
|
|
18407
|
+
});
|
|
18408
|
+
}
|
|
18409
|
+
begin() {
|
|
18410
|
+
const id = ++this.seq;
|
|
18411
|
+
const others = this.peers.size;
|
|
18412
|
+
for (const [peer, most] of this.peers) this.peers.set(peer, Math.max(most, others));
|
|
18413
|
+
this.peers.set(id, others);
|
|
18414
|
+
return {
|
|
18415
|
+
id,
|
|
18416
|
+
sample: this.sampler(),
|
|
18417
|
+
at: this.now()
|
|
18418
|
+
};
|
|
18419
|
+
}
|
|
18420
|
+
end(unit, open, failed) {
|
|
18421
|
+
const inFlight = this.peers.get(open.id) ?? 0;
|
|
18422
|
+
this.peers.delete(open.id);
|
|
18423
|
+
const durationMs = this.now() - open.at;
|
|
18424
|
+
const after = this.sampler();
|
|
18425
|
+
const verdict = judgeHeapDelta(open.sample, after, durationMs, this.thresholds);
|
|
18426
|
+
if (verdict.reasons.length === 0) return;
|
|
18427
|
+
this.deps.logger.info("recorder: unit of work moved the heap", { meta: {
|
|
18428
|
+
kind: unit.kind,
|
|
18429
|
+
name: unit.name,
|
|
18430
|
+
reasons: verdict.reasons,
|
|
18431
|
+
durationMs,
|
|
18432
|
+
heapBeforeMb: mb(open.sample?.heapUsedBytes ?? null),
|
|
18433
|
+
heapAfterMb: mb(after?.heapUsedBytes ?? null),
|
|
18434
|
+
heapDeltaMb: mb(verdict.heapDeltaBytes),
|
|
18435
|
+
losBeforeMb: mb(open.sample?.largeObjectBytes ?? null),
|
|
18436
|
+
losAfterMb: mb(after?.largeObjectBytes ?? null),
|
|
18437
|
+
losDeltaMb: mb(verdict.largeObjectDeltaBytes),
|
|
18438
|
+
inFlight,
|
|
18439
|
+
failed
|
|
18440
|
+
} });
|
|
18441
|
+
}
|
|
18442
|
+
};
|
|
18443
|
+
//#endregion
|
|
18444
|
+
//#region src/recorder/addon/process-table.ts
|
|
18445
|
+
/**
|
|
18446
|
+
* Read this host's process table — just enough to recognise an ffmpeg writer
|
|
18447
|
+
* by its argv — and signal a pid.
|
|
18448
|
+
*
|
|
18449
|
+
* Both halves sit behind a small named interface so the staging-directory
|
|
18450
|
+
* guard (`staging-writer-guard.ts`, D667) can be driven by a fake in a unit
|
|
18451
|
+
* test and by the real OS in production, with no cast in between.
|
|
18452
|
+
*
|
|
18453
|
+
* Linux reads `/proc/<pid>/cmdline` (exact argv, NUL-separated) and
|
|
18454
|
+
* `/proc/<pid>/stat` (ppid, start time). macOS has no `/proc`, so it runs
|
|
18455
|
+
* `ps -axww -o pid=,ppid=,etime=,command=` — the recorder also runs on the
|
|
18456
|
+
* Mac agent. Any other platform gets no lister, and the guard says so.
|
|
18457
|
+
*/
|
|
18458
|
+
/**
|
|
18459
|
+
* Linux `USER_HZ` — the unit of `starttime` in `/proc/<pid>/stat`. It is part
|
|
18460
|
+
* of the userspace ABI and 100 on every architecture Linux ships, which is why
|
|
18461
|
+
* `procps` itself hard-codes the fallback.
|
|
18462
|
+
*/
|
|
18463
|
+
var LINUX_USER_HZ = 100;
|
|
18464
|
+
/** Field index of `starttime` counted from the field AFTER the `(comm)` one. */
|
|
18465
|
+
var STAT_STARTTIME_INDEX = 19;
|
|
18466
|
+
var STAT_PPID_INDEX = 1;
|
|
18467
|
+
/** `ps` is one short fork; a table that takes longer than this is a sick host. */
|
|
18468
|
+
var PS_TIMEOUT_MS = 5e3;
|
|
18469
|
+
var PS_MAX_BUFFER = 16 * 1024 * 1024;
|
|
18470
|
+
function errorCode(err) {
|
|
18471
|
+
if (err instanceof Error && "code" in err && typeof err.code === "string") return err.code;
|
|
18472
|
+
}
|
|
18473
|
+
/** `ppid` and `starttime` out of a `/proc/<pid>/stat` body. The `comm` field
|
|
18474
|
+
* may itself contain spaces and parentheses, so fields are counted from the
|
|
18475
|
+
* LAST `)`. */
|
|
18476
|
+
function parseLinuxStat(body) {
|
|
18477
|
+
const close = body.lastIndexOf(")");
|
|
18478
|
+
if (close < 0) return {
|
|
18479
|
+
ppid: null,
|
|
18480
|
+
startTicks: null
|
|
18481
|
+
};
|
|
18482
|
+
const fields = body.slice(close + 1).trim().split(/\s+/);
|
|
18483
|
+
const ppid = Number(fields[STAT_PPID_INDEX]);
|
|
18484
|
+
const start = Number(fields[STAT_STARTTIME_INDEX]);
|
|
18485
|
+
return {
|
|
18486
|
+
ppid: Number.isInteger(ppid) ? ppid : null,
|
|
18487
|
+
startTicks: Number.isFinite(start) ? start : null
|
|
18488
|
+
};
|
|
18489
|
+
}
|
|
18490
|
+
/** The `/proc` reader. `procRoot` is injectable for a test over a fake tree. */
|
|
18491
|
+
function createLinuxProcessLister(procRoot = "/proc") {
|
|
18492
|
+
return { async list(marker) {
|
|
18493
|
+
const entries = await promises.readdir(procRoot);
|
|
18494
|
+
const uptimeSec = await promises.readFile(`${procRoot}/uptime`, "utf8").then((body) => Number(body.split(/\s+/)[0])).catch(() => NaN);
|
|
18495
|
+
const out = [];
|
|
18496
|
+
for (const entry of entries) {
|
|
18497
|
+
const pid = Number(entry);
|
|
18498
|
+
if (!Number.isInteger(pid) || pid <= 0) continue;
|
|
18499
|
+
let raw;
|
|
18500
|
+
try {
|
|
18501
|
+
raw = await promises.readFile(`${procRoot}/${entry}/cmdline`, "utf8");
|
|
18502
|
+
} catch {
|
|
18503
|
+
continue;
|
|
18504
|
+
}
|
|
18505
|
+
if (raw.length === 0 || !raw.includes(marker)) continue;
|
|
18506
|
+
const argv = raw.split("\0");
|
|
18507
|
+
if (argv[argv.length - 1] === "") argv.pop();
|
|
18508
|
+
const stat = await promises.readFile(`${procRoot}/${entry}/stat`, "utf8").then(parseLinuxStat).catch(() => ({
|
|
18509
|
+
ppid: null,
|
|
18510
|
+
startTicks: null
|
|
18511
|
+
}));
|
|
18512
|
+
const ageMs = stat.startTicks !== null && Number.isFinite(uptimeSec) ? Math.max(0, Math.round((uptimeSec - stat.startTicks / LINUX_USER_HZ) * 1e3)) : null;
|
|
18513
|
+
out.push({
|
|
18514
|
+
pid,
|
|
18515
|
+
ppid: stat.ppid,
|
|
18516
|
+
argv,
|
|
18517
|
+
command: argv.join(" "),
|
|
18518
|
+
ageMs
|
|
18519
|
+
});
|
|
18520
|
+
}
|
|
18521
|
+
return out;
|
|
18522
|
+
} };
|
|
18523
|
+
}
|
|
18524
|
+
/** `[[dd-]hh:]mm:ss` → ms; `null` for anything else. */
|
|
18525
|
+
function parsePsElapsed(etime) {
|
|
18526
|
+
const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/.exec(etime.trim());
|
|
18527
|
+
if (match === null) return null;
|
|
18528
|
+
const [, days, hours, minutes, seconds] = match;
|
|
18529
|
+
return (Number(days ?? 0) * 86400 + Number(hours ?? 0) * 3600 + Number(minutes) * 60 + Number(seconds)) * 1e3;
|
|
18530
|
+
}
|
|
18531
|
+
/** Parse `ps -o pid=,ppid=,etime=,command=` output, keeping lines that contain `marker`. */
|
|
18532
|
+
function parsePsTable(stdout, marker) {
|
|
18533
|
+
const out = [];
|
|
18534
|
+
for (const line of stdout.split("\n")) {
|
|
18535
|
+
if (!line.includes(marker)) continue;
|
|
18536
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line);
|
|
18537
|
+
if (match === null) continue;
|
|
18538
|
+
const [, pid, ppid, etime, command] = match;
|
|
18539
|
+
out.push({
|
|
18540
|
+
pid: Number(pid),
|
|
18541
|
+
ppid: Number(ppid),
|
|
18542
|
+
argv: null,
|
|
18543
|
+
command: command ?? "",
|
|
18544
|
+
ageMs: parsePsElapsed(etime ?? "")
|
|
18545
|
+
});
|
|
18546
|
+
}
|
|
18547
|
+
return out;
|
|
18548
|
+
}
|
|
18549
|
+
/** The macOS (BSD `ps`) reader. `-ww` so a long staging path is never cut. */
|
|
18550
|
+
function createPsProcessLister(psPath = "ps") {
|
|
18551
|
+
return { list(marker) {
|
|
18552
|
+
return new Promise((resolve, reject) => {
|
|
18553
|
+
execFile(psPath, [
|
|
18554
|
+
"-axww",
|
|
18555
|
+
"-o",
|
|
18556
|
+
"pid=,ppid=,etime=,command="
|
|
18557
|
+
], {
|
|
18558
|
+
timeout: PS_TIMEOUT_MS,
|
|
18559
|
+
maxBuffer: PS_MAX_BUFFER
|
|
18560
|
+
}, (err, stdout) => {
|
|
18561
|
+
if (err) {
|
|
18562
|
+
reject(err);
|
|
18563
|
+
return;
|
|
18564
|
+
}
|
|
18565
|
+
resolve(parsePsTable(stdout, marker));
|
|
18566
|
+
});
|
|
18567
|
+
});
|
|
18568
|
+
} };
|
|
18569
|
+
}
|
|
18570
|
+
/** The lister for this platform, or `null` where there is none (Windows). */
|
|
18571
|
+
function createProcessLister(platform = process.platform) {
|
|
18572
|
+
if (platform === "linux") return createLinuxProcessLister();
|
|
18573
|
+
if (platform === "darwin" || platform === "freebsd" || platform === "openbsd") return createPsProcessLister();
|
|
18574
|
+
return null;
|
|
18575
|
+
}
|
|
18576
|
+
/** `process.kill`, with the errno turned into an answer instead of a throw. */
|
|
18577
|
+
var nodeProcessSignaller = { send(pid, signal) {
|
|
18578
|
+
try {
|
|
18579
|
+
process.kill(pid, signal);
|
|
18580
|
+
return "sent";
|
|
18581
|
+
} catch (err) {
|
|
18582
|
+
const code = errorCode(err);
|
|
18583
|
+
if (code === "ESRCH") return "gone";
|
|
18584
|
+
if (code === "EPERM") return "denied";
|
|
18585
|
+
return "failed";
|
|
18586
|
+
}
|
|
18587
|
+
} };
|
|
18588
|
+
//#endregion
|
|
18589
|
+
//#region src/recorder/addon/staging-writer-match.ts
|
|
18590
|
+
/**
|
|
18591
|
+
* Recognise a recorder writer in the process table (D667).
|
|
18592
|
+
*
|
|
18593
|
+
* A writer is recognised by the ARGV it was started with. Two parts decide it:
|
|
18594
|
+
* the executable is an ffmpeg, and `-segment_list` names a staging playlist.
|
|
18595
|
+
* The directory is what makes it a rival, not the binary: another node's
|
|
18596
|
+
* writers are in another pid namespace or on another host, and a writer of a
|
|
18597
|
+
* different directory is not competing for this one.
|
|
18598
|
+
*
|
|
18599
|
+
* Linux gives the exact argv, so every comparison there is exact. macOS `ps`
|
|
18600
|
+
* gives one space-joined string, so the match is a substring with a boundary
|
|
18601
|
+
* check on both sides. A profile name therefore may not contain a space. The
|
|
18602
|
+
* recorder's are `high`, `mid` and `low`.
|
|
18603
|
+
*/
|
|
18604
|
+
/** The argv flag every recorder writer carries, used to narrow the table read. */
|
|
18605
|
+
var SEGMENT_LIST_FLAG = "-segment_list";
|
|
18606
|
+
/**
|
|
18607
|
+
* True when the process was started from an executable whose name contains
|
|
18608
|
+
* `marker`. Without this, a `pgrep -f`, a `grep` or an `sh -c` wrapper that
|
|
18609
|
+
* merely mentions the playlist would be signalled on macOS, where the argv
|
|
18610
|
+
* cannot be told apart from the command text.
|
|
18611
|
+
*/
|
|
18612
|
+
function isWriterExecutable(proc, marker) {
|
|
18613
|
+
if (proc.argv !== null) {
|
|
18614
|
+
const exe = proc.argv[0];
|
|
18615
|
+
return exe !== void 0 && path.basename(exe).includes(marker);
|
|
18616
|
+
}
|
|
18617
|
+
const flagAt = proc.command.indexOf(" -");
|
|
18618
|
+
const exe = flagAt < 0 ? proc.command : proc.command.slice(0, flagAt);
|
|
18619
|
+
return path.basename(exe).includes(marker);
|
|
18620
|
+
}
|
|
18621
|
+
/** True when this process was started as a writer of `playlistPath`. */
|
|
18622
|
+
function writesPlaylist(proc, playlistPath) {
|
|
18623
|
+
if (proc.argv !== null) {
|
|
18624
|
+
for (let i = 0; i < proc.argv.length - 1; i++) if (proc.argv[i] === "-segment_list" && proc.argv[i + 1] === playlistPath) return true;
|
|
18625
|
+
return false;
|
|
18626
|
+
}
|
|
18627
|
+
const needle = `${SEGMENT_LIST_FLAG} ${playlistPath}`;
|
|
18628
|
+
let from = 0;
|
|
18629
|
+
for (;;) {
|
|
18630
|
+
const at = proc.command.indexOf(needle, from);
|
|
18631
|
+
if (at < 0) return false;
|
|
18632
|
+
const before = at === 0 ? " " : proc.command[at - 1];
|
|
18633
|
+
const after = proc.command[at + needle.length];
|
|
18634
|
+
if (before === " " && (after === void 0 || after === " ")) return true;
|
|
18635
|
+
from = at + 1;
|
|
18636
|
+
}
|
|
18637
|
+
}
|
|
18638
|
+
var TAIL = /^(\d+)\/([^/\s]+)\/live\.m3u8$/;
|
|
18639
|
+
var TAIL_IN_COMMAND = /^(\d+)\/([^/\s]+)\/live\.m3u8(?= |$)/;
|
|
18640
|
+
function targetFromTail(root, tail, pattern) {
|
|
18641
|
+
const match = pattern.exec(tail);
|
|
18642
|
+
if (match === null) return null;
|
|
18643
|
+
const deviceId = Number(match[1]);
|
|
18644
|
+
const profile = match[2];
|
|
18645
|
+
if (!Number.isSafeInteger(deviceId) || profile === void 0) return null;
|
|
18646
|
+
return {
|
|
18647
|
+
deviceId,
|
|
18648
|
+
profile,
|
|
18649
|
+
root,
|
|
18650
|
+
outDir: path.join(root, STAGING_DIR_NAME, String(deviceId), profile)
|
|
18651
|
+
};
|
|
18652
|
+
}
|
|
18653
|
+
/**
|
|
18654
|
+
* The staging directory this process writes, when it writes one under any of
|
|
18655
|
+
* `roots` — `<root>/.rec-tmp/<deviceId>/<profile>/live.m3u8`, nothing looser.
|
|
18656
|
+
* Used by the sweeps, which do not know in advance which directories hold an
|
|
18657
|
+
* orphan.
|
|
18658
|
+
*/
|
|
18659
|
+
function stagingTargetOf(proc, roots) {
|
|
18660
|
+
for (const root of roots) {
|
|
18661
|
+
const prefix = `${path.join(root, STAGING_DIR_NAME)}/`;
|
|
18662
|
+
if (proc.argv !== null) {
|
|
18663
|
+
for (let i = 0; i < proc.argv.length - 1; i++) {
|
|
18664
|
+
const value = proc.argv[i + 1];
|
|
18665
|
+
if (proc.argv[i] !== "-segment_list" || value === void 0) continue;
|
|
18666
|
+
if (!value.startsWith(prefix)) continue;
|
|
18667
|
+
const target = targetFromTail(root, value.slice(prefix.length), TAIL);
|
|
18668
|
+
if (target !== null) return target;
|
|
18669
|
+
}
|
|
18670
|
+
continue;
|
|
18671
|
+
}
|
|
18672
|
+
const needle = `${SEGMENT_LIST_FLAG} ${prefix}`;
|
|
18673
|
+
let from = 0;
|
|
18674
|
+
for (;;) {
|
|
18675
|
+
const at = proc.command.indexOf(needle, from);
|
|
18676
|
+
if (at < 0) break;
|
|
18677
|
+
if ((at === 0 ? " " : proc.command[at - 1]) === " ") {
|
|
18678
|
+
const target = targetFromTail(root, proc.command.slice(at + needle.length), TAIL_IN_COMMAND);
|
|
18679
|
+
if (target !== null) return target;
|
|
18680
|
+
}
|
|
18681
|
+
from = at + 1;
|
|
18682
|
+
}
|
|
18683
|
+
}
|
|
18684
|
+
return null;
|
|
18685
|
+
}
|
|
18686
|
+
/** A table read this recent is good enough for a claim. Foreign writers only
|
|
18687
|
+
* come from a predecessor, which exists before the read — and every signal
|
|
18688
|
+
* this guard sends invalidates the cache. */
|
|
18689
|
+
var CLAIM_LIST_MAX_AGE_MS = 1e3;
|
|
18690
|
+
/** Two sightings of one pid whose start times differ by more than this are two
|
|
18691
|
+
* processes: `ps` reports elapsed time in whole seconds. */
|
|
18692
|
+
var SAME_START_TOLERANCE_MS = 2e3;
|
|
18693
|
+
var POLL_MS = 100;
|
|
18694
|
+
function errMsg(err) {
|
|
18695
|
+
return err instanceof Error ? err.message : String(err);
|
|
18696
|
+
}
|
|
18697
|
+
function ageSec(ageMs) {
|
|
18698
|
+
return ageMs === null ? null : Math.round(ageMs / 1e3);
|
|
18699
|
+
}
|
|
18700
|
+
var StagingWriterReaper = class {
|
|
18701
|
+
deps;
|
|
18702
|
+
refusals = /* @__PURE__ */ new Map();
|
|
18703
|
+
cache = null;
|
|
18704
|
+
inflight = null;
|
|
18705
|
+
constructor(deps) {
|
|
18706
|
+
this.deps = deps;
|
|
18707
|
+
}
|
|
18708
|
+
async claim(request) {
|
|
18709
|
+
const target = {
|
|
18710
|
+
deviceId: request.deviceId,
|
|
18711
|
+
profile: request.profile,
|
|
18712
|
+
root: request.root,
|
|
18713
|
+
outDir: request.outDir
|
|
18714
|
+
};
|
|
18715
|
+
const held = this.heldRecord(request.outDir);
|
|
18716
|
+
if (held !== null) {
|
|
18717
|
+
const survivor = {
|
|
18718
|
+
target,
|
|
18719
|
+
pid: held.pid,
|
|
18720
|
+
ppid: null,
|
|
18721
|
+
ageMs: null,
|
|
18722
|
+
reason: "held"
|
|
18723
|
+
};
|
|
18724
|
+
this.logHeld(survivor, held);
|
|
18725
|
+
return {
|
|
18726
|
+
outcome: "refused",
|
|
18727
|
+
survivors: [survivor]
|
|
18728
|
+
};
|
|
18729
|
+
}
|
|
18730
|
+
let rows;
|
|
18731
|
+
try {
|
|
18732
|
+
rows = await this.list(this.deps.now() - CLAIM_LIST_MAX_AGE_MS);
|
|
18733
|
+
} catch (err) {
|
|
18734
|
+
this.deps.logger.warn("recorder: could not read the process table — spawning a writer WITHOUT checking its staging directory for another writer", {
|
|
18735
|
+
tags: { deviceId: request.deviceId },
|
|
18736
|
+
meta: {
|
|
18737
|
+
profile: request.profile,
|
|
18738
|
+
outDir: request.outDir,
|
|
18739
|
+
error: errMsg(err)
|
|
18740
|
+
}
|
|
18741
|
+
});
|
|
18742
|
+
return {
|
|
18743
|
+
outcome: "unverified",
|
|
18744
|
+
error: errMsg(err)
|
|
18745
|
+
};
|
|
18746
|
+
}
|
|
18747
|
+
const playlist = segmentListPath(request.outDir);
|
|
18748
|
+
const candidates = rows.filter((proc) => this.isForeignWriter(proc, request.ownPids) && writesPlaylist(proc, playlist)).map((proc) => ({
|
|
18749
|
+
proc,
|
|
18750
|
+
target
|
|
18751
|
+
}));
|
|
18752
|
+
if (candidates.length === 0) {
|
|
18753
|
+
this.forgetRefusal(target);
|
|
18754
|
+
return {
|
|
18755
|
+
outcome: "clear",
|
|
18756
|
+
reaped: []
|
|
18757
|
+
};
|
|
18758
|
+
}
|
|
18759
|
+
const outcome = await this.reap(candidates);
|
|
18760
|
+
if (outcome.refused.length > 0) return {
|
|
18761
|
+
outcome: "refused",
|
|
18762
|
+
survivors: outcome.refused
|
|
18763
|
+
};
|
|
18764
|
+
return {
|
|
18765
|
+
outcome: "clear",
|
|
18766
|
+
reaped: outcome.reaped
|
|
18767
|
+
};
|
|
18768
|
+
}
|
|
18769
|
+
async sweep(request) {
|
|
18770
|
+
const startedMs = this.deps.now();
|
|
18771
|
+
const tags = request.deviceId === null ? void 0 : { deviceId: request.deviceId };
|
|
18772
|
+
let rows;
|
|
18773
|
+
try {
|
|
18774
|
+
rows = await this.list(startedMs);
|
|
18775
|
+
} catch (err) {
|
|
18776
|
+
this.deps.logger.warn("recorder: could not read the process table — a previous runner’s writers cannot be swept", {
|
|
18777
|
+
...tags ? { tags } : {},
|
|
18778
|
+
meta: {
|
|
18779
|
+
roots: request.roots,
|
|
18780
|
+
error: errMsg(err)
|
|
18781
|
+
}
|
|
18782
|
+
});
|
|
18783
|
+
return {
|
|
18784
|
+
outcome: "unverified",
|
|
18785
|
+
error: errMsg(err)
|
|
18786
|
+
};
|
|
18787
|
+
}
|
|
18788
|
+
const candidates = [];
|
|
18789
|
+
for (const proc of rows) {
|
|
18790
|
+
if (!this.isForeignWriter(proc, request.ownPids)) continue;
|
|
18791
|
+
const target = stagingTargetOf(proc, request.roots);
|
|
18792
|
+
if (target === null) continue;
|
|
18793
|
+
if (request.deviceId !== null && target.deviceId !== request.deviceId) continue;
|
|
18794
|
+
candidates.push({
|
|
18795
|
+
proc,
|
|
18796
|
+
target
|
|
18797
|
+
});
|
|
18798
|
+
}
|
|
18799
|
+
const outcome = candidates.length === 0 ? {
|
|
18800
|
+
reaped: [],
|
|
18801
|
+
refused: []
|
|
18802
|
+
} : await this.reap(candidates);
|
|
18803
|
+
if (request.deviceId === null) this.deps.logger.info("recorder: boot sweep for a previous runner’s writers", { meta: {
|
|
18804
|
+
roots: request.roots,
|
|
18805
|
+
found: candidates.length,
|
|
18806
|
+
reaped: outcome.reaped.length,
|
|
18807
|
+
refused: outcome.refused.length,
|
|
18808
|
+
ms: this.deps.now() - startedMs
|
|
18809
|
+
} });
|
|
18810
|
+
return {
|
|
18811
|
+
outcome: "swept",
|
|
18812
|
+
reaped: outcome.reaped,
|
|
18813
|
+
refused: outcome.refused
|
|
18814
|
+
};
|
|
18815
|
+
}
|
|
18816
|
+
isForeignWriter(proc, ownPids) {
|
|
18817
|
+
return proc.pid !== this.deps.selfPid && !ownPids.has(proc.pid) && isWriterExecutable(proc, this.deps.executableMarker ?? "ffmpeg");
|
|
18818
|
+
}
|
|
18819
|
+
/**
|
|
18820
|
+
* One SIGTERM → SIGKILL window for every candidate at once. Candidates whose
|
|
18821
|
+
* directory is held are refused without a signal.
|
|
18822
|
+
*/
|
|
18823
|
+
async reap(candidates) {
|
|
18824
|
+
const started = this.deps.now();
|
|
18825
|
+
const refused = [];
|
|
18826
|
+
const active = [];
|
|
18827
|
+
for (const candidate of candidates) {
|
|
18828
|
+
const held = this.heldRecord(candidate.target.outDir);
|
|
18829
|
+
if (held === null) {
|
|
18830
|
+
active.push(candidate);
|
|
18831
|
+
continue;
|
|
18832
|
+
}
|
|
18833
|
+
const survivor = this.refusedOf(candidate, "held");
|
|
18834
|
+
this.logHeld(survivor, held);
|
|
18835
|
+
refused.push(survivor);
|
|
18836
|
+
}
|
|
18837
|
+
const terminated = this.signalAll(active, "SIGTERM", refused);
|
|
18838
|
+
let signalled = terminated;
|
|
18839
|
+
const killed = /* @__PURE__ */ new Set();
|
|
18840
|
+
if (terminated.length > 0) {
|
|
18841
|
+
const afterTerm = await this.waitGone(terminated, this.deps.termGraceMs ?? 2e3);
|
|
18842
|
+
if (afterTerm === null) {
|
|
18843
|
+
for (const candidate of terminated) refused.push(this.refusedOf(candidate, "table-unreadable"));
|
|
18844
|
+
signalled = [];
|
|
18845
|
+
} else if (afterTerm.length > 0) {
|
|
18846
|
+
const toKill = this.signalAll(afterTerm, "SIGKILL", refused);
|
|
18847
|
+
for (const candidate of toKill) killed.add(candidate.proc.pid);
|
|
18848
|
+
if (toKill.length > 0) {
|
|
18849
|
+
const afterKill = await this.waitGone(toKill, this.deps.killGraceMs ?? 1e3);
|
|
18850
|
+
const reason = afterKill === null ? "table-unreadable" : "survived";
|
|
18851
|
+
for (const candidate of afterKill ?? toKill) refused.push(this.refusedOf(candidate, reason));
|
|
18852
|
+
}
|
|
18853
|
+
}
|
|
18854
|
+
}
|
|
18855
|
+
const waitedMs = this.deps.now() - started;
|
|
18856
|
+
const refusedPids = new Set(refused.map((writer) => writer.pid));
|
|
18857
|
+
const reaped = [];
|
|
18858
|
+
for (const candidate of signalled) {
|
|
18859
|
+
if (refusedPids.has(candidate.proc.pid)) continue;
|
|
18860
|
+
const writer = {
|
|
18861
|
+
target: candidate.target,
|
|
18862
|
+
pid: candidate.proc.pid,
|
|
18863
|
+
ppid: candidate.proc.ppid,
|
|
18864
|
+
ageMs: candidate.proc.ageMs,
|
|
18865
|
+
signal: killed.has(candidate.proc.pid) ? "SIGKILL" : "SIGTERM",
|
|
18866
|
+
waitedMs
|
|
18867
|
+
};
|
|
18868
|
+
reaped.push(writer);
|
|
18869
|
+
this.logReaped(writer);
|
|
18870
|
+
}
|
|
18871
|
+
for (const writer of refused) if (writer.reason !== "held") this.recordRefusal(writer, waitedMs);
|
|
18872
|
+
return {
|
|
18873
|
+
reaped,
|
|
18874
|
+
refused
|
|
18875
|
+
};
|
|
18876
|
+
}
|
|
18877
|
+
/**
|
|
18878
|
+
* Send `signal` to each candidate. A candidate the kernel refuses (EPERM) is
|
|
18879
|
+
* refused at once — waiting out a grace for a signal that was never
|
|
18880
|
+
* delivered cannot change anything. Returns the ones worth waiting for.
|
|
18881
|
+
*/
|
|
18882
|
+
signalAll(candidates, signal, refused) {
|
|
18883
|
+
if (candidates.length === 0) return [];
|
|
18884
|
+
const pending = [];
|
|
18885
|
+
for (const candidate of candidates) if (this.deps.signaller.send(candidate.proc.pid, signal) === "denied") refused.push(this.refusedOf(candidate, "denied"));
|
|
18886
|
+
else pending.push(candidate);
|
|
18887
|
+
this.cache = null;
|
|
18888
|
+
return pending;
|
|
18889
|
+
}
|
|
18890
|
+
/**
|
|
18891
|
+
* Poll until none of `targets` is still writing its playlist, or the grace
|
|
18892
|
+
* runs out. Returns who is left, or `null` when the table could not be read
|
|
18893
|
+
* twice in a row. A zombie or a recycled pid is "gone" by this test, because
|
|
18894
|
+
* it is the ARGV that makes a writer, not the pid.
|
|
18895
|
+
*/
|
|
18896
|
+
async waitGone(targets, graceMs) {
|
|
18897
|
+
const deadline = this.deps.now() + graceMs;
|
|
18898
|
+
for (;;) {
|
|
18899
|
+
await this.deps.sleep(this.deps.pollMs ?? POLL_MS);
|
|
18900
|
+
const rows = await this.freshRowsOrNull();
|
|
18901
|
+
if (rows === null) return null;
|
|
18902
|
+
const left = targets.filter((candidate) => rows.some((proc) => proc.pid === candidate.proc.pid && isWriterExecutable(proc, this.deps.executableMarker ?? "ffmpeg") && writesPlaylist(proc, segmentListPath(candidate.target.outDir))));
|
|
18903
|
+
if (left.length === 0 || this.deps.now() >= deadline) return left;
|
|
18904
|
+
}
|
|
18905
|
+
}
|
|
18906
|
+
/** One fresh read, and ONE retry after a poll interval. */
|
|
18907
|
+
async freshRowsOrNull() {
|
|
18908
|
+
for (let attempt = 0; attempt < 2; attempt++) try {
|
|
18909
|
+
return await this.list(this.deps.now());
|
|
18910
|
+
} catch {
|
|
18911
|
+
if (attempt === 0) await this.deps.sleep(this.deps.pollMs ?? POLL_MS);
|
|
18912
|
+
}
|
|
18913
|
+
return null;
|
|
18914
|
+
}
|
|
18915
|
+
/**
|
|
18916
|
+
* The process table, read at or after `notBeforeMs`. Concurrent callers
|
|
18917
|
+
* share one read in flight, so eight attaches at boot cost one `ps`, not
|
|
18918
|
+
* eight.
|
|
18919
|
+
*/
|
|
18920
|
+
async list(notBeforeMs) {
|
|
18921
|
+
if (this.cache !== null && this.cache.at >= notBeforeMs) return this.cache.rows;
|
|
18922
|
+
if (this.inflight !== null && this.inflight.startedAt >= notBeforeMs) return (await this.inflight.promise).rows;
|
|
18923
|
+
const startedAt = this.deps.now();
|
|
18924
|
+
const promise = this.deps.lister.list(SEGMENT_LIST_FLAG).then((rows) => ({
|
|
18925
|
+
at: startedAt,
|
|
18926
|
+
rows
|
|
18927
|
+
}));
|
|
18928
|
+
const entry = {
|
|
18929
|
+
startedAt,
|
|
18930
|
+
promise
|
|
18931
|
+
};
|
|
18932
|
+
this.inflight = entry;
|
|
18933
|
+
try {
|
|
18934
|
+
const listing = await promise;
|
|
18935
|
+
if (this.cache === null || this.cache.at <= listing.at) this.cache = listing;
|
|
18936
|
+
return listing.rows;
|
|
18937
|
+
} finally {
|
|
18938
|
+
if (this.inflight === entry) this.inflight = null;
|
|
18939
|
+
}
|
|
18940
|
+
}
|
|
18941
|
+
refusedOf(candidate, reason) {
|
|
18942
|
+
return {
|
|
18943
|
+
target: candidate.target,
|
|
18944
|
+
pid: candidate.proc.pid,
|
|
18945
|
+
ppid: candidate.proc.ppid,
|
|
18946
|
+
ageMs: candidate.proc.ageMs,
|
|
18947
|
+
reason
|
|
18948
|
+
};
|
|
18949
|
+
}
|
|
18950
|
+
heldRecord(outDir) {
|
|
18951
|
+
const record = this.refusals.get(outDir);
|
|
18952
|
+
if (record === void 0) return null;
|
|
18953
|
+
const retryMs = this.deps.refusalRetryMs ?? 6e4;
|
|
18954
|
+
return this.deps.now() - record.lastAttemptMs < retryMs ? record : null;
|
|
18955
|
+
}
|
|
18956
|
+
startedAtOf(ageMs) {
|
|
18957
|
+
return ageMs === null ? null : this.deps.now() - ageMs;
|
|
18958
|
+
}
|
|
18959
|
+
/** ERROR on a new refusal state, DEBUG on a repeat of the same one. */
|
|
18960
|
+
recordRefusal(writer, waitedMs) {
|
|
18961
|
+
const now = this.deps.now();
|
|
18962
|
+
const startedAtMs = this.startedAtOf(writer.ageMs);
|
|
18963
|
+
const previous = this.refusals.get(writer.target.outDir);
|
|
18964
|
+
const same = previous !== void 0 && previous.pid === writer.pid && previous.reason === writer.reason && (previous.startedAtMs === null || startedAtMs === null || Math.abs(previous.startedAtMs - startedAtMs) <= SAME_START_TOLERANCE_MS);
|
|
18965
|
+
const attempts = same ? previous.attempts + 1 : 1;
|
|
18966
|
+
this.refusals = new Map(this.refusals).set(writer.target.outDir, {
|
|
18967
|
+
pid: writer.pid,
|
|
18968
|
+
startedAtMs,
|
|
18969
|
+
reason: writer.reason,
|
|
18970
|
+
firstMs: same ? previous.firstMs : now,
|
|
18971
|
+
lastAttemptMs: now,
|
|
18972
|
+
attempts
|
|
18973
|
+
});
|
|
18974
|
+
const extras = {
|
|
18975
|
+
tags: { deviceId: writer.target.deviceId },
|
|
18976
|
+
meta: {
|
|
18977
|
+
profile: writer.target.profile,
|
|
18978
|
+
pid: writer.pid,
|
|
18979
|
+
ppid: writer.ppid,
|
|
18980
|
+
ageSec: ageSec(writer.ageMs),
|
|
18981
|
+
reason: writer.reason,
|
|
18982
|
+
attempts,
|
|
18983
|
+
waitedMs,
|
|
18984
|
+
retryInMs: this.deps.refusalRetryMs ?? 6e4,
|
|
18985
|
+
outDir: writer.target.outDir
|
|
18986
|
+
}
|
|
18987
|
+
};
|
|
18988
|
+
if (same) {
|
|
18989
|
+
this.deps.logger.debug("recorder: staging directory still refused — same foreign writer", extras);
|
|
18990
|
+
return;
|
|
18991
|
+
}
|
|
18992
|
+
this.deps.logger.error("recorder: REFUSED to spawn a writer — another writer on this staging directory could not be removed", extras);
|
|
18993
|
+
}
|
|
18994
|
+
logHeld(writer, record) {
|
|
18995
|
+
this.deps.logger.debug("recorder: staging directory held after a refusal — not retried yet", {
|
|
18996
|
+
tags: { deviceId: writer.target.deviceId },
|
|
18997
|
+
meta: {
|
|
18998
|
+
profile: writer.target.profile,
|
|
18999
|
+
pid: record.pid,
|
|
19000
|
+
reason: record.reason,
|
|
19001
|
+
attempts: record.attempts,
|
|
19002
|
+
heldForMs: this.deps.now() - record.firstMs,
|
|
19003
|
+
outDir: writer.target.outDir
|
|
19004
|
+
}
|
|
19005
|
+
});
|
|
19006
|
+
}
|
|
19007
|
+
forgetRefusal(target) {
|
|
19008
|
+
const record = this.refusals.get(target.outDir);
|
|
19009
|
+
if (record === void 0) return;
|
|
19010
|
+
const next = new Map(this.refusals);
|
|
19011
|
+
next.delete(target.outDir);
|
|
19012
|
+
this.refusals = next;
|
|
19013
|
+
this.deps.logger.info("recorder: staging directory no longer refused — the foreign writer is gone", {
|
|
19014
|
+
tags: { deviceId: target.deviceId },
|
|
19015
|
+
meta: {
|
|
19016
|
+
profile: target.profile,
|
|
19017
|
+
pid: record.pid,
|
|
19018
|
+
attempts: record.attempts,
|
|
19019
|
+
refusedForMs: this.deps.now() - record.firstMs,
|
|
19020
|
+
outDir: target.outDir
|
|
19021
|
+
}
|
|
19022
|
+
});
|
|
19023
|
+
}
|
|
19024
|
+
logReaped(writer) {
|
|
19025
|
+
if (this.refusals.has(writer.target.outDir)) this.forgetRefusal(writer.target);
|
|
19026
|
+
this.deps.logger.warn("recorder: reaped a predecessor writer on this staging directory — one writer per directory", {
|
|
19027
|
+
tags: { deviceId: writer.target.deviceId },
|
|
19028
|
+
meta: {
|
|
19029
|
+
profile: writer.target.profile,
|
|
19030
|
+
pid: writer.pid,
|
|
19031
|
+
ppid: writer.ppid,
|
|
19032
|
+
ageSec: ageSec(writer.ageMs),
|
|
19033
|
+
signal: writer.signal,
|
|
19034
|
+
waitedMs: writer.waitedMs,
|
|
19035
|
+
outDir: writer.target.outDir
|
|
19036
|
+
}
|
|
19037
|
+
});
|
|
19038
|
+
}
|
|
19039
|
+
};
|
|
19040
|
+
//#endregion
|
|
19041
|
+
//#region src/recorder/addon/staging-boot.ts
|
|
19042
|
+
async function runStagingBoot(steps) {
|
|
19043
|
+
const stagingIsQuiescent = await steps.reapPredecessorWriters();
|
|
19044
|
+
const bootMs = steps.now();
|
|
19045
|
+
await steps.startWritePath();
|
|
19046
|
+
if (!stagingIsQuiescent) return;
|
|
19047
|
+
(async () => {
|
|
19048
|
+
await steps.recoverStagedOrphans(bootMs);
|
|
19049
|
+
await steps.sweepUnassignedStaging();
|
|
19050
|
+
})();
|
|
19051
|
+
}
|
|
19052
|
+
//#endregion
|
|
17909
19053
|
//#region src/recorder/loop-pacer.ts
|
|
17910
19054
|
var DEFAULT_BUSY_LAG_MS = 25;
|
|
17911
19055
|
var DEFAULT_BACKOFF_STEP_MS = 25;
|
|
@@ -19127,6 +20271,13 @@ var MFRA_GATE_LIMIT = 64;
|
|
|
19127
20271
|
var EXPORTS_PREFIX = "exports";
|
|
19128
20272
|
/** How often the export janitor sweeps expired exports. */
|
|
19129
20273
|
var EXPORT_SWEEP_INTERVAL_MS = 5 * 6e4;
|
|
20274
|
+
/**
|
|
20275
|
+
* The staging plan keeps anything named within 1 s before its cutoff for the
|
|
20276
|
+
* live writer (`PREVIOUS_RUN_SKEW_MS`). A reaped directory has no live writer
|
|
20277
|
+
* yet, so the cutoff is moved forward by exactly that much: everything the
|
|
20278
|
+
* reaped writer named up to the reap is collected.
|
|
20279
|
+
*/
|
|
20280
|
+
var REAPED_RECOVERY_SKEW_MS = 1e3;
|
|
19130
20281
|
/** How often the retention sweep prunes footage past each camera's per-device
|
|
19131
20282
|
* age/size policy. Retention is NOT time-critical (footage ages out over days,
|
|
19132
20283
|
* and the disk-pressure eviction path already bounds the worst case), so an
|
|
@@ -19328,6 +20479,17 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19328
20479
|
* behind scrub's `locateSegment` and expired its 8 s budget (2026-08-22).
|
|
19329
20480
|
*/
|
|
19330
20481
|
calendarInstance = null;
|
|
20482
|
+
heapProbeInstance = null;
|
|
20483
|
+
/**
|
|
20484
|
+
* Names the pass, boot phase or inbound cap method that grows the heap.
|
|
20485
|
+
* Every recorder lifetime builds ~2.3 GB in one silent phase 3–5 min after
|
|
20486
|
+
* boot and nothing has ever said which unit of work does it (see
|
|
20487
|
+
* `heap-delta.ts`). LAZY for the same reason as {@link calendar}.
|
|
20488
|
+
*/
|
|
20489
|
+
get heapProbe() {
|
|
20490
|
+
this.heapProbeInstance ??= new HeapDeltaProbe({ logger: this.ctx.logger });
|
|
20491
|
+
return this.heapProbeInstance;
|
|
20492
|
+
}
|
|
19331
20493
|
/** LAZY: a field initialiser runs before `initialize()` assigns `ctx`, and
|
|
19332
20494
|
* reading `this.ctx` there throws — the same trap that bit the playback tier. */
|
|
19333
20495
|
get calendar() {
|
|
@@ -19446,6 +20608,9 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19446
20608
|
/** The `recording` cap provider — retained so the periodic retention sweep can
|
|
19447
20609
|
* call its `pruneFootage`. Null until built in `onInitialize`. */
|
|
19448
20610
|
recordingProvider = null;
|
|
20611
|
+
/** One writer per staging directory (D667). `null` where the platform has
|
|
20612
|
+
* no process table (Windows) — said once at startup. */
|
|
20613
|
+
stagingGuard = null;
|
|
19449
20614
|
/** Periodic footage-retention sweep (owns its timer + single-flight guard).
|
|
19450
20615
|
* Constructed + started in `onHubReachable`, stopped on shutdown. */
|
|
19451
20616
|
retentionSweeper = null;
|
|
@@ -19523,7 +20688,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19523
20688
|
try {
|
|
19524
20689
|
await this.segmentHours.declare();
|
|
19525
20690
|
} catch (err) {
|
|
19526
|
-
this.ctx.logger.warn("recorder: segment-hour ledger declare failed — index stays RAM-only this boot", { meta: { error: errMsg(err) } });
|
|
20691
|
+
this.ctx.logger.warn("recorder: segment-hour ledger declare failed — index stays RAM-only this boot", { meta: { error: errMsg$1(err) } });
|
|
19527
20692
|
this.segmentHours = null;
|
|
19528
20693
|
}
|
|
19529
20694
|
const ingestOwner = await this.resolveIngestOwner();
|
|
@@ -19584,7 +20749,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19584
20749
|
tags: { deviceId: row.deviceId },
|
|
19585
20750
|
meta: {
|
|
19586
20751
|
path: row.path,
|
|
19587
|
-
error: errMsg(err)
|
|
20752
|
+
error: errMsg$1(err)
|
|
19588
20753
|
}
|
|
19589
20754
|
});
|
|
19590
20755
|
}
|
|
@@ -19629,7 +20794,20 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
19629
20794
|
usedOnLocation: async (locationId) => (await this.segmentHours?.accountingForLocations(new Set([locationId])))?.bytes ?? 0,
|
|
19630
20795
|
dailyBytesFor: (deviceId, profile) => this.dailyBytesFor(deviceId, profile)
|
|
19631
20796
|
});
|
|
20797
|
+
const processLister = createProcessLister();
|
|
20798
|
+
if (processLister === null) this.ctx.logger.warn("recorder: no process table on this platform — orphaned writers cannot be detected; boot staging mutation is disabled", { meta: { platform: process.platform } });
|
|
20799
|
+
this.stagingGuard = processLister === null ? null : new StagingWriterReaper({
|
|
20800
|
+
lister: processLister,
|
|
20801
|
+
signaller: nodeProcessSignaller,
|
|
20802
|
+
logger: this.ctx.logger,
|
|
20803
|
+
selfPid: process.pid,
|
|
20804
|
+
sleep: (ms) => sleep$1(ms),
|
|
20805
|
+
now: () => Date.now()
|
|
20806
|
+
});
|
|
19632
20807
|
this.controller = new RecordingController({
|
|
20808
|
+
...this.stagingGuard !== null ? { stagingGuard: this.stagingGuard } : {},
|
|
20809
|
+
recoverStagingDirectory: (input) => this.recoverReapedDirectory(input),
|
|
20810
|
+
heapProbe: this.heapProbe,
|
|
19633
20811
|
api: this.ctx.api,
|
|
19634
20812
|
logger: this.ctx.logger,
|
|
19635
20813
|
nodeId: this.nodeId,
|
|
@@ -20053,7 +21231,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20053
21231
|
internal: served?.baseUrl
|
|
20054
21232
|
} });
|
|
20055
21233
|
} catch (err) {
|
|
20056
|
-
this.ctx.logger.warn("recorder playback data-plane failed to serve", { meta: { error: errMsg(err) } });
|
|
21234
|
+
this.ctx.logger.warn("recorder playback data-plane failed to serve", { meta: { error: errMsg$1(err) } });
|
|
20057
21235
|
}
|
|
20058
21236
|
try {
|
|
20059
21237
|
const stillService = new StillFrameService({
|
|
@@ -20071,7 +21249,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20071
21249
|
internal: served?.baseUrl
|
|
20072
21250
|
} });
|
|
20073
21251
|
} catch (err) {
|
|
20074
|
-
this.ctx.logger.warn("still data-plane serve failed", { meta: { error: errMsg(err) } });
|
|
21252
|
+
this.ctx.logger.warn("still data-plane serve failed", { meta: { error: errMsg$1(err) } });
|
|
20075
21253
|
}
|
|
20076
21254
|
try {
|
|
20077
21255
|
const served = await this.ctx.dataPlane?.serve({
|
|
@@ -20084,7 +21262,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20084
21262
|
internal: served?.baseUrl
|
|
20085
21263
|
} });
|
|
20086
21264
|
} catch (err) {
|
|
20087
|
-
this.ctx.logger.warn("recorder exports data-plane failed to serve", { meta: { error: errMsg(err) } });
|
|
21265
|
+
this.ctx.logger.warn("recorder exports data-plane failed to serve", { meta: { error: errMsg$1(err) } });
|
|
20088
21266
|
}
|
|
20089
21267
|
this.ctx.logger.info("recorder addon started (continuous write path active)", {
|
|
20090
21268
|
tags: { nodeId: this.nodeId },
|
|
@@ -20094,35 +21272,35 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20094
21272
|
}
|
|
20095
21273
|
});
|
|
20096
21274
|
migrateLegacyDeviceSignalRows(this.configStore(), this.ctx.logger).catch((err) => {
|
|
20097
|
-
this.ctx.logger.warn("recorder: device-signal config migration failed", { meta: { error: errMsg(err) } });
|
|
21275
|
+
this.ctx.logger.warn("recorder: device-signal config migration failed", { meta: { error: errMsg$1(err) } });
|
|
20098
21276
|
}).then(() => warnLostBands(this.configStore(), this.ctx.logger));
|
|
20099
21277
|
const providers = [
|
|
20100
21278
|
{
|
|
20101
21279
|
capability: recordingArchiveCapability,
|
|
20102
|
-
provider: recordingProvider
|
|
21280
|
+
provider: this.heapProbe.wrapProvider(recordingArchiveCapability.name, recordingProvider)
|
|
20103
21281
|
},
|
|
20104
21282
|
{
|
|
20105
21283
|
capability: recordingCapability,
|
|
20106
|
-
provider: recordingSourceProvider
|
|
21284
|
+
provider: this.heapProbe.wrapProvider(recordingCapability.name, recordingSourceProvider)
|
|
20107
21285
|
},
|
|
20108
21286
|
{
|
|
20109
21287
|
capability: videoclipsCapability,
|
|
20110
|
-
provider: createRecorderVideoclipsProvider({
|
|
21288
|
+
provider: this.heapProbe.wrapProvider(videoclipsCapability.name, createRecorderVideoclipsProvider({
|
|
20111
21289
|
api: this.ctx.api,
|
|
20112
21290
|
logger: this.ctx.logger
|
|
20113
|
-
})
|
|
21291
|
+
}))
|
|
20114
21292
|
},
|
|
20115
21293
|
{
|
|
20116
21294
|
capability: storageEvictableCapability,
|
|
20117
|
-
provider: evictableProvider
|
|
21295
|
+
provider: this.heapProbe.wrapProvider(storageEvictableCapability.name, evictableProvider)
|
|
20118
21296
|
},
|
|
20119
21297
|
{
|
|
20120
21298
|
capability: storageOccupancyCapability,
|
|
20121
|
-
provider: evictableProvider
|
|
21299
|
+
provider: this.heapProbe.wrapProvider(storageOccupancyCapability.name, evictableProvider)
|
|
20122
21300
|
},
|
|
20123
21301
|
{
|
|
20124
21302
|
capability: recordingExportCapability,
|
|
20125
|
-
provider: exportProvider
|
|
21303
|
+
provider: this.heapProbe.wrapProvider(recordingExportCapability.name, exportProvider)
|
|
20126
21304
|
},
|
|
20127
21305
|
{
|
|
20128
21306
|
capability: loadContributionCapability,
|
|
@@ -20266,7 +21444,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20266
21444
|
} catch (err) {
|
|
20267
21445
|
this.ctx.logger.error("artifacts storage location is NOT available — exports, GIF and clip renders will REFUSE until it is", { meta: {
|
|
20268
21446
|
location: ARTIFACTS_MEDIA_LOCATION,
|
|
20269
|
-
error: errMsg(err),
|
|
21447
|
+
error: errMsg$1(err),
|
|
20270
21448
|
hint: "the declaration is seeded when a storage-provider registers; restart this node so the manifest is re-read, or create an artifacts location. Refusing is deliberate — the addon data dir carries no media (D341)."
|
|
20271
21449
|
} });
|
|
20272
21450
|
}
|
|
@@ -20282,14 +21460,28 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20282
21460
|
recursive: true,
|
|
20283
21461
|
force: true
|
|
20284
21462
|
}).catch(() => {});
|
|
20285
|
-
await
|
|
20286
|
-
|
|
20287
|
-
|
|
20288
|
-
|
|
20289
|
-
|
|
20290
|
-
|
|
20291
|
-
|
|
20292
|
-
|
|
21463
|
+
await runStagingBoot({
|
|
21464
|
+
reapPredecessorWriters: () => this.heapProbe.track({
|
|
21465
|
+
kind: "boot-phase",
|
|
21466
|
+
name: "staging-writer-sweep"
|
|
21467
|
+
}, () => this.reapPredecessorWriters()),
|
|
21468
|
+
sweepUnassignedStaging: () => this.heapProbe.track({
|
|
21469
|
+
kind: "boot-phase",
|
|
21470
|
+
name: "staging-unassigned-sweep"
|
|
21471
|
+
}, () => this.sweepUnassignedStaging()),
|
|
21472
|
+
startWritePath: async () => {
|
|
21473
|
+
try {
|
|
21474
|
+
await this.controller?.start();
|
|
21475
|
+
} catch (err) {
|
|
21476
|
+
this.ctx.logger.warn("recorder: controller start failed", { meta: { error: errMsg$1(err) } });
|
|
21477
|
+
}
|
|
21478
|
+
},
|
|
21479
|
+
recoverStagedOrphans: (bootMs) => this.heapProbe.track({
|
|
21480
|
+
kind: "boot-phase",
|
|
21481
|
+
name: "staging-recovery"
|
|
21482
|
+
}, () => this.recoverStagedOrphans(bootMs)),
|
|
21483
|
+
now: () => Date.now()
|
|
21484
|
+
});
|
|
20293
21485
|
if (this.locationRefresher === null) this.locationRefresher = new LocationRefresher({
|
|
20294
21486
|
logger: this.ctx.logger,
|
|
20295
21487
|
refresh: () => this.refreshLocations()
|
|
@@ -20369,32 +21561,44 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20369
21561
|
this.controller?.onSignalLevel(parsed.deviceId, { kind: "signal" }, level.data.active, atMs);
|
|
20370
21562
|
});
|
|
20371
21563
|
try {
|
|
20372
|
-
await this.
|
|
21564
|
+
await this.heapProbe.track({
|
|
21565
|
+
kind: "boot-phase",
|
|
21566
|
+
name: "ledger-load"
|
|
21567
|
+
}, async () => {
|
|
21568
|
+
await this.segmentHours?.load();
|
|
21569
|
+
});
|
|
20373
21570
|
} catch (err) {
|
|
20374
|
-
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: errMsg(err) } });
|
|
21571
|
+
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: errMsg$1(err) } });
|
|
20375
21572
|
}
|
|
20376
21573
|
let configs;
|
|
20377
21574
|
try {
|
|
20378
21575
|
configs = await readDeviceConfigs(this.configStore());
|
|
20379
21576
|
} catch (err) {
|
|
20380
|
-
this.ctx.logger.warn("recorder: reading persisted device configs failed — skipping hydrate", { meta: { error: errMsg(err) } });
|
|
21577
|
+
this.ctx.logger.warn("recorder: reading persisted device configs failed — skipping hydrate", { meta: { error: errMsg$1(err) } });
|
|
20381
21578
|
return;
|
|
20382
21579
|
}
|
|
20383
21580
|
this.applySensorTriggerBindings(configs);
|
|
20384
|
-
this.
|
|
21581
|
+
this.heapProbe.track({
|
|
21582
|
+
kind: "boot-phase",
|
|
21583
|
+
name: "warm-today"
|
|
21584
|
+
}, () => this.warmToday([...configs.keys()]));
|
|
20385
21585
|
this.ctx.logger.info("recorder: full-archive index walk skipped", { meta: {
|
|
20386
21586
|
reason: "on-demand-windows",
|
|
20387
21587
|
ledgerHours: this.segmentHours?.snapshot().length ?? 0,
|
|
20388
21588
|
deviceCount: configs.size
|
|
20389
21589
|
} });
|
|
20390
|
-
this.
|
|
21590
|
+
this.heapProbe.track({
|
|
21591
|
+
kind: "boot-phase",
|
|
21592
|
+
name: "legacy-thumbs-cleanup"
|
|
21593
|
+
}, () => this.cleanupLegacyThumbsDirs());
|
|
20391
21594
|
await this.bootstrapExports();
|
|
20392
21595
|
const provider = this.recordingProvider;
|
|
20393
21596
|
if (provider && this.retentionSweeper === null) this.retentionSweeper = new RetentionSweeper({
|
|
20394
21597
|
prune: (deviceId) => provider.pruneFootage({ deviceId }),
|
|
20395
21598
|
readConfigs: () => readDeviceConfigs(this.configStore()),
|
|
20396
21599
|
logger: this.ctx.logger,
|
|
20397
|
-
intervalMs: RETENTION_SWEEP_INTERVAL_MS
|
|
21600
|
+
intervalMs: RETENTION_SWEEP_INTERVAL_MS,
|
|
21601
|
+
heapProbe: this.heapProbe
|
|
20398
21602
|
});
|
|
20399
21603
|
this.retentionSweeper?.start();
|
|
20400
21604
|
const store = this.segmentStore;
|
|
@@ -20404,7 +21608,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20404
21608
|
evict: (locationId, rows) => store.evict(locationId, rows),
|
|
20405
21609
|
logger: this.ctx.logger,
|
|
20406
21610
|
intervalMs: REDUNDANCY_SWEEP_INTERVAL_MS,
|
|
20407
|
-
lookbackMs: REDUNDANCY_SWEEP_LOOKBACK_MS
|
|
21611
|
+
lookbackMs: REDUNDANCY_SWEEP_LOOKBACK_MS,
|
|
21612
|
+
heapProbe: this.heapProbe
|
|
20408
21613
|
});
|
|
20409
21614
|
this.redundancySweeper?.start();
|
|
20410
21615
|
const ledger = this.segmentHours;
|
|
@@ -20429,6 +21634,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20429
21634
|
name: "ghost-location-reconcile",
|
|
20430
21635
|
intervalMs: GHOST_LOCATION_SWEEP_INTERVAL_MS,
|
|
20431
21636
|
slowAfterMs: GHOST_LOCATION_SWEEP_SLOW_AFTER_MS,
|
|
21637
|
+
heapProbe: this.heapProbe,
|
|
20432
21638
|
run: async () => {
|
|
20433
21639
|
await sweeper.sweep();
|
|
20434
21640
|
},
|
|
@@ -20456,6 +21662,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20456
21662
|
intervalMs: HOUR_ADOPTION_INTERVAL_MS,
|
|
20457
21663
|
firstRunDelayMs: HOUR_ADOPTION_FIRST_RUN_DELAY_MS,
|
|
20458
21664
|
slowAfterMs: HOUR_ADOPTION_SLOW_AFTER_MS,
|
|
21665
|
+
heapProbe: this.heapProbe,
|
|
20459
21666
|
run: async () => {
|
|
20460
21667
|
if (await runAdoptionCatchUp({
|
|
20461
21668
|
locationIds: this.resolvedLocations.map((location) => location.id),
|
|
@@ -20466,7 +21673,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20466
21673
|
onError: (locationId, error) => {
|
|
20467
21674
|
this.ctx.logger.warn("recorder: hour adoption could not sweep a location", { meta: {
|
|
20468
21675
|
locationId,
|
|
20469
|
-
error: errMsg(error)
|
|
21676
|
+
error: errMsg$1(error)
|
|
20470
21677
|
} });
|
|
20471
21678
|
}
|
|
20472
21679
|
})) {
|
|
@@ -20537,7 +21744,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20537
21744
|
try {
|
|
20538
21745
|
this.applySensorTriggerBindings(await readDeviceConfigs(this.configStore()));
|
|
20539
21746
|
} catch (err) {
|
|
20540
|
-
this.ctx.logger.warn("recorder: sensor trigger index NOT refreshed — config read failed, the previous index stands", { meta: { error: errMsg(err) } });
|
|
21747
|
+
this.ctx.logger.warn("recorder: sensor trigger index NOT refreshed — config read failed, the previous index stands", { meta: { error: errMsg$1(err) } });
|
|
20541
21748
|
}
|
|
20542
21749
|
}
|
|
20543
21750
|
/** Apply an already-read config map to the sensor index. Logs only when the
|
|
@@ -20579,13 +21786,16 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20579
21786
|
logger: this.ctx.logger.child("export")
|
|
20580
21787
|
});
|
|
20581
21788
|
} catch (err) {
|
|
20582
|
-
this.ctx.logger.warn("recorder: export boot re-queue failed", { meta: { error: errMsg(err) } });
|
|
21789
|
+
this.ctx.logger.warn("recorder: export boot re-queue failed", { meta: { error: errMsg$1(err) } });
|
|
20583
21790
|
}
|
|
20584
21791
|
if (this.exportSweepTimer === null) {
|
|
20585
21792
|
const sweep = () => {
|
|
20586
21793
|
this.logIndexCensus();
|
|
20587
|
-
|
|
20588
|
-
|
|
21794
|
+
this.heapProbe.track({
|
|
21795
|
+
kind: "pass",
|
|
21796
|
+
name: "export-expiry"
|
|
21797
|
+
}, () => sweepExpiredExports(Date.now(), this.exportJanitorDeps())).catch((err) => {
|
|
21798
|
+
this.ctx.logger.warn("recorder: export expiry sweep failed", { meta: { error: errMsg$1(err) } });
|
|
20589
21799
|
});
|
|
20590
21800
|
};
|
|
20591
21801
|
sweep();
|
|
@@ -20631,7 +21841,13 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20631
21841
|
incrementalInserts: views.incrementalInserts,
|
|
20632
21842
|
rowsReused: reuse.rowsReused,
|
|
20633
21843
|
rowsAllocated: reuse.rowsAllocated,
|
|
20634
|
-
ledgerHours: this.segmentHours?.snapshot().length ?? 0
|
|
21844
|
+
ledgerHours: this.segmentHours?.snapshot().length ?? 0,
|
|
21845
|
+
calendar: this.calendarInstance?.memoryCensus() ?? null,
|
|
21846
|
+
mfra: this.mfraTables.memoryCensus(),
|
|
21847
|
+
ledger: this.segmentHours?.memoryCensus() ?? null,
|
|
21848
|
+
placement: this.placement?.memoryCensus() ?? null,
|
|
21849
|
+
exports: this.exportEngine?.memoryCensus() ?? null,
|
|
21850
|
+
controller: this.controller?.memoryCensus() ?? null
|
|
20635
21851
|
}
|
|
20636
21852
|
});
|
|
20637
21853
|
}
|
|
@@ -20655,7 +21871,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20655
21871
|
} catch (err) {
|
|
20656
21872
|
this.ctx.logger.debug("recorder: legacy thumbs cleanup skipped", { meta: {
|
|
20657
21873
|
dir,
|
|
20658
|
-
error: errMsg(err)
|
|
21874
|
+
error: errMsg$1(err)
|
|
20659
21875
|
} });
|
|
20660
21876
|
}
|
|
20661
21877
|
}
|
|
@@ -20692,7 +21908,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20692
21908
|
tags: { deviceId },
|
|
20693
21909
|
meta: {
|
|
20694
21910
|
epochMs,
|
|
20695
|
-
error: errMsg(err)
|
|
21911
|
+
error: errMsg$1(err)
|
|
20696
21912
|
}
|
|
20697
21913
|
});
|
|
20698
21914
|
res.writeHead(500).end();
|
|
@@ -20757,7 +21973,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20757
21973
|
profile,
|
|
20758
21974
|
locationId,
|
|
20759
21975
|
hourStartMs,
|
|
20760
|
-
error: errMsg(err)
|
|
21976
|
+
error: errMsg$1(err)
|
|
20761
21977
|
}
|
|
20762
21978
|
});
|
|
20763
21979
|
return null;
|
|
@@ -20939,8 +22155,68 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20939
22155
|
await replaceAssignments(store, await readPlacementState(store), /* @__PURE__ */ new Map());
|
|
20940
22156
|
this.ctx.logger.info("recorder: placement plan reset after storage migration");
|
|
20941
22157
|
} catch (err) {
|
|
20942
|
-
this.ctx.logger.warn("recorder: placement plan reset failed", { meta: { error: errMsg(err) } });
|
|
22158
|
+
this.ctx.logger.warn("recorder: placement plan reset failed", { meta: { error: errMsg$1(err) } });
|
|
22159
|
+
}
|
|
22160
|
+
}
|
|
22161
|
+
/**
|
|
22162
|
+
* Boot sweep for a previous runner's writers (D667): every foreign ffmpeg
|
|
22163
|
+
* writing under any location's `.rec-tmp`, whether or not its camera will be
|
|
22164
|
+
* attached again. On re-entry (`onHubReachable` again) the controller's own
|
|
22165
|
+
* writers are live and excluded by pid.
|
|
22166
|
+
*/
|
|
22167
|
+
async reapPredecessorWriters() {
|
|
22168
|
+
const guard = this.stagingGuard;
|
|
22169
|
+
if (guard === null) return false;
|
|
22170
|
+
const result = await guard.sweep({
|
|
22171
|
+
roots: this.resolvedLocations.map((location) => location.root),
|
|
22172
|
+
deviceId: null,
|
|
22173
|
+
ownPids: this.controller?.ownedWriterPids() ?? /* @__PURE__ */ new Set()
|
|
22174
|
+
});
|
|
22175
|
+
if (result.outcome === "unverified") {
|
|
22176
|
+
this.ctx.logger.warn("recorder: staging mutation skipped at boot — the predecessor-writer sweep was unverified", { meta: { roots: this.resolvedLocations.map((location) => location.root) } });
|
|
22177
|
+
return false;
|
|
20943
22178
|
}
|
|
22179
|
+
if (result.refused.length === 0) return true;
|
|
22180
|
+
this.ctx.logger.warn("recorder: staging mutation skipped at boot — a known predecessor writer remains", { meta: {
|
|
22181
|
+
refused: result.refused.length,
|
|
22182
|
+
roots: this.resolvedLocations.map((location) => location.root)
|
|
22183
|
+
} });
|
|
22184
|
+
return false;
|
|
22185
|
+
}
|
|
22186
|
+
/**
|
|
22187
|
+
* Collect what a reaped writer left in one staging directory, before the
|
|
22188
|
+
* controller spawns a new writer there (D667). The reaped writer's playlist
|
|
22189
|
+
* is NOT consulted: it names that writer's last entries, and nobody is left
|
|
22190
|
+
* to finalize them.
|
|
22191
|
+
*/
|
|
22192
|
+
async recoverReapedDirectory(input) {
|
|
22193
|
+
const report = await recoverStagingOrphans(input.dir, {
|
|
22194
|
+
writerStartedMs: input.cutoffMs + REAPED_RECOVERY_SKEW_MS,
|
|
22195
|
+
segmentSeconds: input.segmentSeconds
|
|
22196
|
+
}, {
|
|
22197
|
+
listDir: nodeStagingIo.listDir,
|
|
22198
|
+
readPlaylist: async () => null,
|
|
22199
|
+
finalize: async (orphan, absPath) => {
|
|
22200
|
+
const store = this.segmentStore;
|
|
22201
|
+
if (store === null) throw new Error("recorder is shutting down");
|
|
22202
|
+
await store.onFinalized({
|
|
22203
|
+
deviceId: input.deviceId,
|
|
22204
|
+
profile: input.profile,
|
|
22205
|
+
locationId: input.locationId,
|
|
22206
|
+
locationRoot: input.locationRoot,
|
|
22207
|
+
flatAbsPath: absPath,
|
|
22208
|
+
startMs: orphan.startMs,
|
|
22209
|
+
durMs: orphan.durMs
|
|
22210
|
+
});
|
|
22211
|
+
},
|
|
22212
|
+
logger: this.ctx.logger.withTags({ deviceId: input.deviceId }),
|
|
22213
|
+
shouldStop: () => this.segmentStore === null
|
|
22214
|
+
});
|
|
22215
|
+
return {
|
|
22216
|
+
found: report.found,
|
|
22217
|
+
recovered: report.recovered,
|
|
22218
|
+
failed: report.failed
|
|
22219
|
+
};
|
|
20944
22220
|
}
|
|
20945
22221
|
/**
|
|
20946
22222
|
* Boot sweep: remove the `.rec-tmp` staging tree from every recordings root
|
|
@@ -20950,8 +22226,10 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20950
22226
|
* no retention pass will ever see (`.rec-tmp` is invisible to the hydrate
|
|
20951
22227
|
* walk by design). Same shape as the `.scrub-packs` sweep above.
|
|
20952
22228
|
*
|
|
20953
|
-
* Only roots with ZERO assignments are touched, and only
|
|
20954
|
-
*
|
|
22229
|
+
* Only roots with ZERO assignments are touched, and only on the boot chain,
|
|
22230
|
+
* after orphan recovery. The placement read happens here, after the
|
|
22231
|
+
* controller has attached, so any root a new writer uses is named by an
|
|
22232
|
+
* assignment and cannot be swept.
|
|
20955
22233
|
*/
|
|
20956
22234
|
async sweepUnassignedStaging() {
|
|
20957
22235
|
let assigned;
|
|
@@ -20959,7 +22237,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20959
22237
|
const blob = await this.placementStore().get();
|
|
20960
22238
|
assigned = new Set(Object.values(blob.assignments));
|
|
20961
22239
|
} catch (err) {
|
|
20962
|
-
this.ctx.logger.warn("recorder: staging sweep skipped — placement read failed", { meta: { error: errMsg(err) } });
|
|
22240
|
+
this.ctx.logger.warn("recorder: staging sweep skipped — placement read failed", { meta: { error: errMsg$1(err) } });
|
|
20963
22241
|
return;
|
|
20964
22242
|
}
|
|
20965
22243
|
if (assigned.size === 0) return;
|
|
@@ -20975,7 +22253,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
20975
22253
|
this.ctx.logger.warn("recorder: staging sweep failed for unassigned location", { meta: {
|
|
20976
22254
|
locationId: location.id,
|
|
20977
22255
|
root: location.root,
|
|
20978
|
-
error: errMsg(err)
|
|
22256
|
+
error: errMsg$1(err)
|
|
20979
22257
|
} });
|
|
20980
22258
|
continue;
|
|
20981
22259
|
}
|
|
@@ -21022,7 +22300,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
21022
22300
|
shouldStop: () => this.segmentStore === null
|
|
21023
22301
|
});
|
|
21024
22302
|
} catch (err) {
|
|
21025
|
-
this.ctx.logger.warn("recorder: staging boot reconcile failed", { meta: { error: errMsg(err) } });
|
|
22303
|
+
this.ctx.logger.warn("recorder: staging boot reconcile failed", { meta: { error: errMsg$1(err) } });
|
|
21026
22304
|
}
|
|
21027
22305
|
}
|
|
21028
22306
|
/**
|
|
@@ -21101,7 +22379,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
21101
22379
|
try {
|
|
21102
22380
|
return await this.ctx.api.pipelineOrchestrator.getIngestOwner.query();
|
|
21103
22381
|
} catch (err) {
|
|
21104
|
-
this.ctx.logger.warn("recorder: getIngestOwner unavailable — defaulting to hub (cap not mounted yet?)", { meta: { error: errMsg(err) } });
|
|
22382
|
+
this.ctx.logger.warn("recorder: getIngestOwner unavailable — defaulting to hub (cap not mounted yet?)", { meta: { error: errMsg$1(err) } });
|
|
21105
22383
|
return { ownerNodeId: "hub" };
|
|
21106
22384
|
}
|
|
21107
22385
|
}
|
|
@@ -21210,7 +22488,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
21210
22488
|
specs: RECORDER_RETIRED_ROOT_KEYS
|
|
21211
22489
|
});
|
|
21212
22490
|
} catch (err) {
|
|
21213
|
-
this.ctx.logger.warn("recorder: retiring the flattened settings keys failed", { meta: { error: errMsg(err) } });
|
|
22491
|
+
this.ctx.logger.warn("recorder: retiring the flattened settings keys failed", { meta: { error: errMsg$1(err) } });
|
|
21214
22492
|
}
|
|
21215
22493
|
}
|
|
21216
22494
|
/**
|
|
@@ -21344,7 +22622,7 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
21344
22622
|
}, EXPORT_DELETE_GRACE_MS).catch((err) => {
|
|
21345
22623
|
this.ctx.logger.warn("recorder: markDownloaded failed", { meta: {
|
|
21346
22624
|
exportId,
|
|
21347
|
-
error: errMsg(err)
|
|
22625
|
+
error: errMsg$1(err)
|
|
21348
22626
|
} });
|
|
21349
22627
|
});
|
|
21350
22628
|
});
|