@camstack/addon-pipeline 1.2.302 → 1.2.304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline-runner/index.js +211 -13
- package/dist/pipeline-runner/index.mjs +211 -13
- package/dist/recorder/index.js +1349 -70
- package/dist/recorder/index.mjs +1423 -145
- package/dist/session-decode/decode-worker-main.js +250 -57
- package/dist/session-decode/decode-worker-main.mjs +250 -57
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-DFaPx_Ot.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF7C68rV.mjs} +1 -1
- package/dist/stream-broker/{hostInit-CePd6XbV.mjs → hostInit-dac4-ofq.mjs} +1 -1
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/recorder/index.js
CHANGED
|
@@ -15,6 +15,8 @@ node_path = require_chunk.__toESM(node_path);
|
|
|
15
15
|
let node_child_process = require("node:child_process");
|
|
16
16
|
let node_fs_promises = require("node:fs/promises");
|
|
17
17
|
let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
|
|
18
|
+
let node_v8 = require("node:v8");
|
|
19
|
+
node_v8 = require_chunk.__toESM(node_v8);
|
|
18
20
|
//#region src/recorder/event-capture.ts
|
|
19
21
|
/**
|
|
20
22
|
* La sottoscrizione motion del recorder: bus → `onTrigger(deviceId, atMs)`.
|
|
@@ -466,6 +468,11 @@ var MFRA_RETAIN_MS = 26 * 36e5;
|
|
|
466
468
|
function isMfraTableServable(startMs, nowMs) {
|
|
467
469
|
return startMs >= nowMs - MFRA_RETAIN_MS;
|
|
468
470
|
}
|
|
471
|
+
/** A `{ timeTicks, moofOffset }` object on a 64-bit V8 without pointer
|
|
472
|
+
* compression (Node): header + two fields + the array slot pointing at it,
|
|
473
|
+
* ~48 B, plus a 16 B HeapNumber when a value passes 2³¹. An ESTIMATE, rounded
|
|
474
|
+
* up — the order of magnitude is the point, not the byte. */
|
|
475
|
+
var MFRA_SAMPLE_APPROX_BYTES = 64;
|
|
469
476
|
/**
|
|
470
477
|
* The RAM store the directory reads. Keys are `(deviceId, profile, startMs)`
|
|
471
478
|
* — exactly how a directory row identifies a segment. Insertion is
|
|
@@ -509,6 +516,20 @@ var MfraTableStore = class {
|
|
|
509
516
|
get size() {
|
|
510
517
|
return this.tables.size;
|
|
511
518
|
}
|
|
519
|
+
/**
|
|
520
|
+
* Entries and seek samples held, for the recorder census. The byte figure is
|
|
521
|
+
* an ESTIMATE — {@link MFRA_SAMPLE_APPROX_BYTES} per sample object — and is
|
|
522
|
+
* named as one; it is the order of magnitude that matters here.
|
|
523
|
+
*/
|
|
524
|
+
memoryCensus() {
|
|
525
|
+
let samples = 0;
|
|
526
|
+
for (const entry of this.tables.values()) samples += entry.table.samples.length;
|
|
527
|
+
return {
|
|
528
|
+
entries: this.tables.size,
|
|
529
|
+
samples,
|
|
530
|
+
approxBytes: samples * MFRA_SAMPLE_APPROX_BYTES
|
|
531
|
+
};
|
|
532
|
+
}
|
|
512
533
|
/** Insertion order is time order, so the expired prefix is contiguous. */
|
|
513
534
|
prune() {
|
|
514
535
|
const cutoff = this.now() - MFRA_RETAIN_MS;
|
|
@@ -4219,6 +4240,24 @@ var CalendarIndex = class {
|
|
|
4219
4240
|
constructor(deps) {
|
|
4220
4241
|
this.deps = deps;
|
|
4221
4242
|
}
|
|
4243
|
+
/**
|
|
4244
|
+
* Entry counts for the recorder census. `hourSegs` is unbounded (it is
|
|
4245
|
+
* invalidated on eviction and relocation, never aged), so its segment total
|
|
4246
|
+
* is the number that says whether this cache is where the heap went.
|
|
4247
|
+
*/
|
|
4248
|
+
memoryCensus() {
|
|
4249
|
+
let days = 0;
|
|
4250
|
+
for (const perDay of this.byDevice.values()) days += perDay.size;
|
|
4251
|
+
let hourSegments = 0;
|
|
4252
|
+
for (const segments of this.hourSegs.values()) hourSegments += segments.length;
|
|
4253
|
+
return {
|
|
4254
|
+
devices: this.byDevice.size,
|
|
4255
|
+
days,
|
|
4256
|
+
hourKeys: this.hourSegs.size,
|
|
4257
|
+
hourSegments,
|
|
4258
|
+
inFlight: this.inFlight.size + this.segsInFlight.size
|
|
4259
|
+
};
|
|
4260
|
+
}
|
|
4222
4261
|
mapFor(deviceId) {
|
|
4223
4262
|
let m = this.byDevice.get(deviceId);
|
|
4224
4263
|
if (!m) {
|
|
@@ -6582,6 +6621,14 @@ var ExportEngine = class {
|
|
|
6582
6621
|
constructor(deps) {
|
|
6583
6622
|
this.deps = deps;
|
|
6584
6623
|
}
|
|
6624
|
+
/** Entry counts, for the recorder census. */
|
|
6625
|
+
memoryCensus() {
|
|
6626
|
+
return {
|
|
6627
|
+
queued: this.queue.length,
|
|
6628
|
+
active: this.active.size,
|
|
6629
|
+
cancelled: this.cancelled.size
|
|
6630
|
+
};
|
|
6631
|
+
}
|
|
6585
6632
|
/** Queue a persisted `queued` record for rendering. */
|
|
6586
6633
|
enqueue(id) {
|
|
6587
6634
|
if (this.queue.includes(id) || this.active.has(id)) return;
|
|
@@ -9970,6 +10017,19 @@ var SegmentHourLedger = class SegmentHourLedger {
|
|
|
9970
10017
|
* the recorder happens to be placing that hour on.
|
|
9971
10018
|
*/
|
|
9972
10019
|
refusedWriterHours = /* @__PURE__ */ new Set();
|
|
10020
|
+
/** Entry counts of the in-process maps, for the recorder census. */
|
|
10021
|
+
memoryCensus() {
|
|
10022
|
+
let accountedPaths = 0;
|
|
10023
|
+
for (const paths of this.accounted.values()) accountedPaths += paths.size;
|
|
10024
|
+
return {
|
|
10025
|
+
lanes: this.lanes.size,
|
|
10026
|
+
writerHours: this.writerHours.size,
|
|
10027
|
+
accountedHours: this.accounted.size,
|
|
10028
|
+
accountedPaths,
|
|
10029
|
+
refused: this.refused.size,
|
|
10030
|
+
refusedWriterHours: this.refusedWriterHours.size
|
|
10031
|
+
};
|
|
10032
|
+
}
|
|
9973
10033
|
constructor(deps) {
|
|
9974
10034
|
this.logger = deps.logger;
|
|
9975
10035
|
this.store = deps.store;
|
|
@@ -12557,7 +12617,11 @@ function startPeriodicPass(deps) {
|
|
|
12557
12617
|
}
|
|
12558
12618
|
const began = now();
|
|
12559
12619
|
startedAt = began;
|
|
12560
|
-
deps.
|
|
12620
|
+
const probe = deps.heapProbe;
|
|
12621
|
+
(probe === void 0 ? deps.run() : probe.track({
|
|
12622
|
+
kind: "pass",
|
|
12623
|
+
name: deps.name
|
|
12624
|
+
}, deps.run)).catch((err) => {
|
|
12561
12625
|
deps.logger.warn("recorder: pass failed", { meta: {
|
|
12562
12626
|
pass: deps.name,
|
|
12563
12627
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -13252,6 +13316,15 @@ var PlacementService = class {
|
|
|
13252
13316
|
constructor(deps) {
|
|
13253
13317
|
this.deps = deps;
|
|
13254
13318
|
}
|
|
13319
|
+
/** Entry counts of the per-location/per-camera state, for the recorder census. */
|
|
13320
|
+
memoryCensus() {
|
|
13321
|
+
return {
|
|
13322
|
+
verified: this.verified.size,
|
|
13323
|
+
unusable: this.unusable.size,
|
|
13324
|
+
capacityFaults: this.capacityFaults.size,
|
|
13325
|
+
allUnusableReported: this.allUnusableReported.size
|
|
13326
|
+
};
|
|
13327
|
+
}
|
|
13255
13328
|
get probeDeps() {
|
|
13256
13329
|
return this.deps.probe ?? fsVolumeProbeDeps;
|
|
13257
13330
|
}
|
|
@@ -14115,34 +14188,50 @@ function releaseTrigger(state, source, atMs, mergeGapMs) {
|
|
|
14115
14188
|
firstMs: since,
|
|
14116
14189
|
lastMs: Math.max(since, atMs)
|
|
14117
14190
|
};
|
|
14118
|
-
const
|
|
14119
|
-
const newest = windows[windows.length - 1];
|
|
14120
|
-
const merged = newest !== void 0 && span.firstMs - newest.lastMs <= mergeGapMs ? [...windows.slice(0, -1), {
|
|
14121
|
-
firstMs: Math.min(newest.firstMs, span.firstMs),
|
|
14122
|
-
lastMs: Math.max(newest.lastMs, span.lastMs)
|
|
14123
|
-
}] : [...windows, span];
|
|
14191
|
+
const merged = foldSpanInOrder(triggerWindowsFor(state, source), span, mergeGapMs);
|
|
14124
14192
|
const evicted = Math.max(0, merged.length - 32);
|
|
14125
14193
|
return withWindows(withHeld(state, held), source, evicted > 0 ? merged.slice(evicted) : merged);
|
|
14126
14194
|
}
|
|
14127
14195
|
/**
|
|
14128
|
-
* Fold one
|
|
14129
|
-
*
|
|
14130
|
-
*
|
|
14196
|
+
* Fold one span into a window list, immutably, keeping the list's INVARIANT:
|
|
14197
|
+
* sorted by `firstMs`, and no two windows within `mergeGapMs` of each other
|
|
14198
|
+
* (each is its own episode). The span EXTENDS every window it lies within the
|
|
14199
|
+
* merge gap of — possibly bridging two — and is otherwise INSERTED at its place
|
|
14200
|
+
* in time.
|
|
14201
|
+
*
|
|
14202
|
+
* Never "appended as the newest": the history is read from its end by the live
|
|
14203
|
+
* gate ({@link latestTriggerMs}) and by the evictor, so a span older than the
|
|
14204
|
+
* newest window appended after it made the newest trigger go BACKWARDS, and the
|
|
14205
|
+
* live gate tore down a writer whose episode was still open (D194, 2026-09-28).
|
|
14206
|
+
* The coalesce pass is the standard interval merge over the sorted list, so it
|
|
14207
|
+
* also heals a list that an older build left out of order.
|
|
14208
|
+
*/
|
|
14209
|
+
function foldSpanInOrder(windows, span, mergeGapMs) {
|
|
14210
|
+
return [...windows, span].toSorted((a, b) => a.firstMs - b.firstMs).reduce((acc, w) => {
|
|
14211
|
+
const prev = acc[acc.length - 1];
|
|
14212
|
+
if (prev !== void 0 && w.firstMs - prev.lastMs <= mergeGapMs) return [...acc.slice(0, -1), {
|
|
14213
|
+
firstMs: prev.firstMs,
|
|
14214
|
+
lastMs: Math.max(prev.lastMs, w.lastMs)
|
|
14215
|
+
}];
|
|
14216
|
+
return [...acc, w];
|
|
14217
|
+
}, []);
|
|
14218
|
+
}
|
|
14219
|
+
/**
|
|
14220
|
+
* Fold one qualifying trigger into the history, immutably. It EXTENDS the
|
|
14221
|
+
* window it lands within `postBufferSec` of (same episode — the recorder never
|
|
14222
|
+
* stopped in between), and opens a new one otherwise, IN TIME ORDER.
|
|
14131
14223
|
*
|
|
14132
14224
|
* An out-of-order trigger (`atMs` older than the newest window's `lastMs`, which
|
|
14133
|
-
* a bus lag spike
|
|
14134
|
-
*
|
|
14225
|
+
* a bus lag spike or a durable row older than the bus edge can produce) never
|
|
14226
|
+
* rewinds anything: it widens `firstMs` of the episode it belongs to, or opens
|
|
14227
|
+
* its own window BEFORE the newer ones. The keep span can only ever grow, and
|
|
14228
|
+
* the newest trigger can only ever move forward.
|
|
14135
14229
|
*/
|
|
14136
14230
|
function recordTrigger(state, source, atMs, mergeGapMs) {
|
|
14137
|
-
const
|
|
14138
|
-
const newest = windows[windows.length - 1];
|
|
14139
|
-
const merged = newest !== void 0 && atMs - newest.lastMs <= mergeGapMs && atMs >= newest.firstMs - mergeGapMs ? [...windows.slice(0, -1), {
|
|
14140
|
-
firstMs: Math.min(newest.firstMs, atMs),
|
|
14141
|
-
lastMs: Math.max(newest.lastMs, atMs)
|
|
14142
|
-
}] : [...windows, {
|
|
14231
|
+
const merged = foldSpanInOrder(triggerWindowsFor(state, source), {
|
|
14143
14232
|
firstMs: atMs,
|
|
14144
14233
|
lastMs: atMs
|
|
14145
|
-
}
|
|
14234
|
+
}, mergeGapMs);
|
|
14146
14235
|
const evicted = Math.max(0, merged.length - 32);
|
|
14147
14236
|
return {
|
|
14148
14237
|
state: withWindows(state, source, evicted > 0 ? merged.slice(evicted) : merged),
|
|
@@ -14166,10 +14255,18 @@ function pruneTriggerHistory(state, postMs, nowMs) {
|
|
|
14166
14255
|
}
|
|
14167
14256
|
return changed ? withHeld({ windows: next }, state.held) : state;
|
|
14168
14257
|
}
|
|
14169
|
-
/**
|
|
14258
|
+
/**
|
|
14259
|
+
* Most recent trigger for a source, or null when none is retained.
|
|
14260
|
+
*
|
|
14261
|
+
* The MAXIMUM `lastMs`, not the last element's: the list is kept in time order
|
|
14262
|
+
* ({@link recordTrigger}), but the live gate tearing a writer down is the one
|
|
14263
|
+
* outcome that destroys footage, so it does not rest on an ordering invariant
|
|
14264
|
+
* alone (D194, 2026-09-28).
|
|
14265
|
+
*/
|
|
14170
14266
|
function latestTriggerMs(state, source) {
|
|
14171
14267
|
const windows = triggerWindowsFor(state, source);
|
|
14172
|
-
|
|
14268
|
+
if (windows.length === 0) return null;
|
|
14269
|
+
return windows.reduce((max, w) => Math.max(max, w.lastMs), Number.NEGATIVE_INFINITY);
|
|
14173
14270
|
}
|
|
14174
14271
|
/**
|
|
14175
14272
|
* Is an `events` band's live demand window open at `atMs`? True iff a trigger
|
|
@@ -14334,7 +14431,7 @@ function foldDurableTriggers(state, source, timestamps, mergeGapMs) {
|
|
|
14334
14431
|
const folded = recordTrigger(next, source, atMs, mergeGapMs);
|
|
14335
14432
|
evicted += folded.evicted;
|
|
14336
14433
|
const after = triggerWindowsFor(folded.state, source);
|
|
14337
|
-
if (
|
|
14434
|
+
if (after.length !== before.length || after.some((w, i) => w.firstMs !== before[i]?.firstMs || w.lastMs !== before[i]?.lastMs)) recovered++;
|
|
14338
14435
|
next = folded.state;
|
|
14339
14436
|
}
|
|
14340
14437
|
return {
|
|
@@ -14929,6 +15026,35 @@ function planFinalizations(body, processed, forceFinal = false) {
|
|
|
14929
15026
|
rolledOffToMs: spanKnown ? front : null
|
|
14930
15027
|
};
|
|
14931
15028
|
}
|
|
15029
|
+
/** The last entry of one playlist body, as {@link DrainedPlaylist} names it;
|
|
15030
|
+
* `null` when the body has no epoch-named last entry (nothing to vouch for). */
|
|
15031
|
+
function playlistTail(body) {
|
|
15032
|
+
const entries = parseLivePlaylist(body);
|
|
15033
|
+
const last = entries[entries.length - 1];
|
|
15034
|
+
if (last === void 0) return null;
|
|
15035
|
+
const startMs = parseEpochStartMs(last.segPath);
|
|
15036
|
+
if (startMs === null) return null;
|
|
15037
|
+
return {
|
|
15038
|
+
lastSegStartMs: startMs,
|
|
15039
|
+
lastMediaSequence: parseMediaSequence(body) + entries.length - 1
|
|
15040
|
+
};
|
|
15041
|
+
}
|
|
15042
|
+
/**
|
|
15043
|
+
* Did everything a plan says rolled off belong to a predecessor run that a
|
|
15044
|
+
* watcher of this process already accounted for? Pure. All three must hold:
|
|
15045
|
+
* - a predecessor was DRAINED here, by a final pass that read its playlist;
|
|
15046
|
+
* - the body read now is NOT NEWER than the one it drained (last entry no
|
|
15047
|
+
* later, sequence no higher) — so it is that playlist, not one a foreign or
|
|
15048
|
+
* later writer produced since, whose roll-off nobody has seen;
|
|
15049
|
+
* - the body's front predates THIS writer — so none of it is this writer's.
|
|
15050
|
+
*/
|
|
15051
|
+
function rolledOffIsDrainedPredecessor(input) {
|
|
15052
|
+
const { drained, bodyTail } = input;
|
|
15053
|
+
return drained !== null && bodyTail !== null && bodyTail.lastSegStartMs <= drained.lastSegStartMs && bodyTail.lastMediaSequence <= drained.lastMediaSequence && input.rolledOffToMs !== null && classifyMissingEntry({
|
|
15054
|
+
segmentStartMs: input.rolledOffToMs,
|
|
15055
|
+
writerStartedMs: input.writerStartedMs
|
|
15056
|
+
}) === "stale-previous-run";
|
|
15057
|
+
}
|
|
14932
15058
|
/**
|
|
14933
15059
|
* Start polling `<outDir>/live.m3u8`. Returns a handle whose `stop()`
|
|
14934
15060
|
* cancels the timer. Errors during a tick are swallowed (logged at debug)
|
|
@@ -14936,6 +15062,8 @@ function planFinalizations(body, processed, forceFinal = false) {
|
|
|
14936
15062
|
*/
|
|
14937
15063
|
function startSegmentWatcher(deps) {
|
|
14938
15064
|
const playlistPath = node_path.default.join(deps.outDir, "live.m3u8");
|
|
15065
|
+
const drained = deps.drainedPredecessor ?? null;
|
|
15066
|
+
let lastReadTail = null;
|
|
14939
15067
|
const now = () => deps.now?.() ?? Date.now();
|
|
14940
15068
|
let processed = 0;
|
|
14941
15069
|
let stopped = false;
|
|
@@ -14944,17 +15072,34 @@ function startSegmentWatcher(deps) {
|
|
|
14944
15072
|
let pendingIndex = -1;
|
|
14945
15073
|
let pendingSinceMs = 0;
|
|
14946
15074
|
const tick = async (forceFinal = false) => {
|
|
14947
|
-
if (stopped || ticking) return;
|
|
15075
|
+
if (stopped || ticking) return "unreadable";
|
|
14948
15076
|
ticking = true;
|
|
14949
15077
|
try {
|
|
14950
15078
|
let body;
|
|
14951
15079
|
try {
|
|
14952
15080
|
body = await node_fs.promises.readFile(playlistPath, "utf8");
|
|
14953
15081
|
} catch {
|
|
14954
|
-
return;
|
|
15082
|
+
return "unreadable";
|
|
14955
15083
|
}
|
|
15084
|
+
const bodyTail = playlistTail(body);
|
|
15085
|
+
lastReadTail = bodyTail;
|
|
14956
15086
|
const plan = planFinalizations(body, processed, forceFinal);
|
|
14957
|
-
if (plan.rolledOff > 0
|
|
15087
|
+
if (plan.rolledOff > 0 && rolledOffIsDrainedPredecessor({
|
|
15088
|
+
rolledOffToMs: plan.rolledOffToMs,
|
|
15089
|
+
writerStartedMs: deps.writerStartedMs,
|
|
15090
|
+
bodyTail,
|
|
15091
|
+
drained
|
|
15092
|
+
})) deps.logger.debug("segment playlist is the drained previous run — no roll-off to report", {
|
|
15093
|
+
tags: { deviceId: deps.deviceId },
|
|
15094
|
+
meta: {
|
|
15095
|
+
rolledOff: plan.rolledOff,
|
|
15096
|
+
processed,
|
|
15097
|
+
frontMs: plan.rolledOffToMs,
|
|
15098
|
+
writerStartedMs: deps.writerStartedMs,
|
|
15099
|
+
drainedLastSegStartMs: drained?.lastSegStartMs
|
|
15100
|
+
}
|
|
15101
|
+
});
|
|
15102
|
+
else if (plan.rolledOff > 0) {
|
|
14958
15103
|
deps.logger.warn("segment playlist rolled past entries this watcher had not handled", {
|
|
14959
15104
|
tags: { deviceId: deps.deviceId },
|
|
14960
15105
|
meta: {
|
|
@@ -15007,6 +15152,7 @@ function startSegmentWatcher(deps) {
|
|
|
15007
15152
|
pendingSinceMs = 0;
|
|
15008
15153
|
}
|
|
15009
15154
|
processed = index;
|
|
15155
|
+
return "read";
|
|
15010
15156
|
} finally {
|
|
15011
15157
|
ticking = false;
|
|
15012
15158
|
}
|
|
@@ -15033,7 +15179,7 @@ function startSegmentWatcher(deps) {
|
|
|
15033
15179
|
},
|
|
15034
15180
|
flush: async () => {
|
|
15035
15181
|
await currentTick;
|
|
15036
|
-
await startTick(true);
|
|
15182
|
+
if (await startTick(true) === "read" && lastReadTail !== null) deps.onDrained?.(lastReadTail);
|
|
15037
15183
|
}
|
|
15038
15184
|
};
|
|
15039
15185
|
}
|
|
@@ -15056,6 +15202,16 @@ function startSegmentWatcher(deps) {
|
|
|
15056
15202
|
* recorded fragment's audio can be read in-process (mp4box) and pushed straight
|
|
15057
15203
|
* to the Opus sender with no per-fragment ffmpeg spawn on playback.
|
|
15058
15204
|
*/
|
|
15205
|
+
/**
|
|
15206
|
+
* The `-segment_list` path a writer for `outDir` is given — spelled in ONE
|
|
15207
|
+
* place, because it is also how a writer is RECOGNISED in the process table.
|
|
15208
|
+
* The staging-directory guard (D667) looks for exactly this argv pair before
|
|
15209
|
+
* it lets a second ffmpeg start in the same directory; a spelling that drifted
|
|
15210
|
+
* between the two would make every predecessor invisible to it.
|
|
15211
|
+
*/
|
|
15212
|
+
function segmentListPath(outDir) {
|
|
15213
|
+
return `${outDir}/live.m3u8`;
|
|
15214
|
+
}
|
|
15059
15215
|
/** ffmpeg argv: pull RTSP (TCP), copy video / transcode audio to Opus, write
|
|
15060
15216
|
* fragmented-mp4 segments named by epoch seconds FLAT under `outDir`, plus a
|
|
15061
15217
|
* rolling `live.m3u8` playlist. The recorder segment-watcher tails the
|
|
@@ -15095,7 +15251,7 @@ function buildPassthroughArgs(a) {
|
|
|
15095
15251
|
"-strftime",
|
|
15096
15252
|
"1",
|
|
15097
15253
|
"-segment_list",
|
|
15098
|
-
|
|
15254
|
+
segmentListPath(a.outDir),
|
|
15099
15255
|
"-segment_list_type",
|
|
15100
15256
|
"m3u8",
|
|
15101
15257
|
"-segment_list_size",
|
|
@@ -15148,7 +15304,7 @@ var RESTART_WINDOW_MS = 60 * 6e4;
|
|
|
15148
15304
|
*/
|
|
15149
15305
|
var MAX_RESTARTS_IN_WINDOW = 15;
|
|
15150
15306
|
/** Grace between the SIGTERM in stop() and the escalated SIGKILL. */
|
|
15151
|
-
var KILL_GRACE_MS = 500;
|
|
15307
|
+
var KILL_GRACE_MS$1 = 500;
|
|
15152
15308
|
/** Supervises one passthrough ffmpeg for a single (camera, profile). */
|
|
15153
15309
|
var SegmentWriter = class {
|
|
15154
15310
|
cfg;
|
|
@@ -15172,10 +15328,25 @@ var SegmentWriter = class {
|
|
|
15172
15328
|
* storm cannot accumulate claims for children that are already gone.
|
|
15173
15329
|
*/
|
|
15174
15330
|
costClaim = require_node.NO_COST_CLAIM;
|
|
15331
|
+
/**
|
|
15332
|
+
* The pid of a generation `stop()` has signalled but whose `exit` has not
|
|
15333
|
+
* fired. `proc` is dropped at the signal; this keeps the pid readable until
|
|
15334
|
+
* the child is really gone, so the staging guard (D667) still knows it is
|
|
15335
|
+
* this runner's writer and not a predecessor's.
|
|
15336
|
+
*/
|
|
15337
|
+
exitingPid = void 0;
|
|
15175
15338
|
constructor(cfg, deps) {
|
|
15176
15339
|
this.cfg = cfg;
|
|
15177
15340
|
this.deps = deps;
|
|
15178
15341
|
}
|
|
15342
|
+
/**
|
|
15343
|
+
* The pid of the ffmpeg this writer is supervising right now, or undefined
|
|
15344
|
+
* between generations. Read by the staging-directory guard (D667) so a
|
|
15345
|
+
* writer the controller is TRACKING is never mistaken for a predecessor's.
|
|
15346
|
+
*/
|
|
15347
|
+
get pid() {
|
|
15348
|
+
return this.proc?.pid ?? this.exitingPid;
|
|
15349
|
+
}
|
|
15179
15350
|
start() {
|
|
15180
15351
|
if (this.stopped || this.proc) return;
|
|
15181
15352
|
const args = buildPassthroughArgs(this.cfg);
|
|
@@ -15232,6 +15403,7 @@ var SegmentWriter = class {
|
|
|
15232
15403
|
this.resolveExit?.();
|
|
15233
15404
|
this.resolveExit = null;
|
|
15234
15405
|
this.proc = null;
|
|
15406
|
+
this.exitingPid = void 0;
|
|
15235
15407
|
if (this.stopped) return;
|
|
15236
15408
|
const now = Date.now();
|
|
15237
15409
|
const ranMs = now - this.startedAt;
|
|
@@ -15345,6 +15517,7 @@ var SegmentWriter = class {
|
|
|
15345
15517
|
const exitWait = this.exitWait;
|
|
15346
15518
|
this.proc = null;
|
|
15347
15519
|
if (!proc) return;
|
|
15520
|
+
if (proc.exitCode !== void 0 || proc.signalCode !== void 0) this.exitingPid = proc.pid;
|
|
15348
15521
|
try {
|
|
15349
15522
|
proc.kill("SIGTERM");
|
|
15350
15523
|
} catch {}
|
|
@@ -15352,7 +15525,7 @@ var SegmentWriter = class {
|
|
|
15352
15525
|
if (proc.exitCode === null && proc.signalCode === null) try {
|
|
15353
15526
|
proc.kill("SIGKILL");
|
|
15354
15527
|
} catch {}
|
|
15355
|
-
}, KILL_GRACE_MS).unref?.();
|
|
15528
|
+
}, KILL_GRACE_MS$1).unref?.();
|
|
15356
15529
|
if (proc.exitCode === void 0 && proc.signalCode === void 0) return;
|
|
15357
15530
|
await exitWait;
|
|
15358
15531
|
}
|
|
@@ -15680,10 +15853,16 @@ var RecordingController = class {
|
|
|
15680
15853
|
/** Every attach admitted before a maintenance pause. The pause waits these
|
|
15681
15854
|
* promises before it snapshots/drains active writers. */
|
|
15682
15855
|
attachingTasks = /* @__PURE__ */ new Map();
|
|
15683
|
-
/**
|
|
15684
|
-
*
|
|
15685
|
-
*
|
|
15686
|
-
|
|
15856
|
+
/**
|
|
15857
|
+
* The drain ledger: per staging directory (resolved `outDir`), what the last
|
|
15858
|
+
* watcher of THIS process drained there (`segment-watcher.ts`
|
|
15859
|
+
* `DrainedPlaylist`). Consumed by the next watcher on that directory, so a
|
|
15860
|
+
* fresh watcher reading its predecessor's playlist does not report a roll-off
|
|
15861
|
+
* that predecessor already handled or reported. Per process by design: after
|
|
15862
|
+
* a runner respawn it is empty and the roll-off IS reported (D521). At most
|
|
15863
|
+
* one entry per location × device × profile directory.
|
|
15864
|
+
*/
|
|
15865
|
+
drainedPlaylists = /* @__PURE__ */ new Map();
|
|
15687
15866
|
/**
|
|
15688
15867
|
* Monotonic attach counter, the last component of the CONSUMER TAG this
|
|
15689
15868
|
* recorder puts on its RTSP dial (`recorder/<deviceId>/<profile>/<n>`).
|
|
@@ -15744,6 +15923,13 @@ var RecordingController = class {
|
|
|
15744
15923
|
* healthy device every 30s.
|
|
15745
15924
|
*/
|
|
15746
15925
|
skippedProfiles = /* @__PURE__ */ new Map();
|
|
15926
|
+
/**
|
|
15927
|
+
* Writers being torn down whose ffmpeg may not have exited yet. They left
|
|
15928
|
+
* {@link active} when the teardown began, but their pid is still ours until
|
|
15929
|
+
* the child's `exit` fires — without this set the staging guard (D667) would
|
|
15930
|
+
* reap this runner's own exiting writer and log it as a predecessor's.
|
|
15931
|
+
*/
|
|
15932
|
+
retiring = /* @__PURE__ */ new Set();
|
|
15747
15933
|
restore = null;
|
|
15748
15934
|
/**
|
|
15749
15935
|
* One pending wake-up per device, armed for the exact instant its recording
|
|
@@ -15846,6 +16032,10 @@ var RecordingController = class {
|
|
|
15846
16032
|
});
|
|
15847
16033
|
return out;
|
|
15848
16034
|
}
|
|
16035
|
+
/** Entries in the drain ledger — for the memory census. */
|
|
16036
|
+
get drainedLedgerSize() {
|
|
16037
|
+
return this.drainedPlaylists.size;
|
|
16038
|
+
}
|
|
15849
16039
|
/**
|
|
15850
16040
|
* Boot restore: seed the readiness-gated queue with every persisted device and
|
|
15851
16041
|
* (re)evaluate each on every `stream-broker` ready transition. Also arms the
|
|
@@ -15878,7 +16068,8 @@ var RecordingController = class {
|
|
|
15878
16068
|
slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
|
|
15879
16069
|
logger: this.deps.logger,
|
|
15880
16070
|
now: () => this.now(),
|
|
15881
|
-
run: () => this.checkIdleWriters()
|
|
16071
|
+
run: () => this.checkIdleWriters(),
|
|
16072
|
+
...this.deps.heapProbe !== void 0 ? { heapProbe: this.deps.heapProbe } : {}
|
|
15882
16073
|
});
|
|
15883
16074
|
this.placementPass = startPeriodicPass({
|
|
15884
16075
|
name: "placement",
|
|
@@ -15886,7 +16077,8 @@ var RecordingController = class {
|
|
|
15886
16077
|
slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
|
|
15887
16078
|
logger: this.deps.logger,
|
|
15888
16079
|
now: () => this.now(),
|
|
15889
|
-
run: () => this.replanPlacement()
|
|
16080
|
+
run: () => this.replanPlacement(),
|
|
16081
|
+
...this.deps.heapProbe !== void 0 ? { heapProbe: this.deps.heapProbe } : {}
|
|
15890
16082
|
});
|
|
15891
16083
|
this.reconcilePass = startPeriodicPass({
|
|
15892
16084
|
name: "reconcile",
|
|
@@ -15894,7 +16086,8 @@ var RecordingController = class {
|
|
|
15894
16086
|
slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
|
|
15895
16087
|
logger: this.deps.logger,
|
|
15896
16088
|
now: () => this.now(),
|
|
15897
|
-
run: () => this.reconcile()
|
|
16089
|
+
run: () => this.reconcile(),
|
|
16090
|
+
...this.deps.heapProbe !== void 0 ? { heapProbe: this.deps.heapProbe } : {}
|
|
15898
16091
|
});
|
|
15899
16092
|
}
|
|
15900
16093
|
await this.restore.start(ids.map((id) => [id, true]));
|
|
@@ -16294,7 +16487,6 @@ var RecordingController = class {
|
|
|
16294
16487
|
for (const r of stalled) {
|
|
16295
16488
|
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
16296
16489
|
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
16297
|
-
this.firstSegmentSeen.delete(idleKey(deviceId, r.profile));
|
|
16298
16490
|
}
|
|
16299
16491
|
this.active.set(deviceId, [...keep]);
|
|
16300
16492
|
const skipped = this.skippedProfiles.get(deviceId) ?? /* @__PURE__ */ new Set();
|
|
@@ -16645,6 +16837,7 @@ var RecordingController = class {
|
|
|
16645
16837
|
if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
|
|
16646
16838
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
16647
16839
|
const bandAudio = bandAudioAt(config, new Date(this.now()));
|
|
16840
|
+
await this.sweepDeviceStaging(deviceId, segmentSeconds);
|
|
16648
16841
|
const recordings = [];
|
|
16649
16842
|
const failed = [];
|
|
16650
16843
|
for (const profile of profiles) try {
|
|
@@ -16712,6 +16905,7 @@ var RecordingController = class {
|
|
|
16712
16905
|
const have = new Set(current.map((r) => r.profile));
|
|
16713
16906
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
16714
16907
|
const bandAudio = bandAudioAt(config, new Date(this.now()));
|
|
16908
|
+
await this.sweepDeviceStaging(deviceId, segmentSeconds);
|
|
16715
16909
|
const added = [];
|
|
16716
16910
|
for (const profile of [...skipped]) {
|
|
16717
16911
|
if (have.has(profile)) {
|
|
@@ -16759,6 +16953,143 @@ var RecordingController = class {
|
|
|
16759
16953
|
return require_dist.resolveRecordingProfiles(require_dist.selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, require_dist.nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile), override);
|
|
16760
16954
|
}
|
|
16761
16955
|
/**
|
|
16956
|
+
* Entry counts of every long-lived container this controller holds, for the
|
|
16957
|
+
* recorder's 5-minute census line. Sizes only — no walk of anything that
|
|
16958
|
+
* grows with the archive.
|
|
16959
|
+
*/
|
|
16960
|
+
memoryCensus() {
|
|
16961
|
+
let writers = 0;
|
|
16962
|
+
for (const recordings of this.active.values()) writers += recordings.length;
|
|
16963
|
+
let skippedProfiles = 0;
|
|
16964
|
+
for (const skipped of this.skippedProfiles.values()) skippedProfiles += skipped.size;
|
|
16965
|
+
return {
|
|
16966
|
+
activeDevices: this.active.size,
|
|
16967
|
+
writers,
|
|
16968
|
+
attaching: this.attaching.size,
|
|
16969
|
+
attachingTasks: this.attachingTasks.size,
|
|
16970
|
+
drainedPlaylists: this.drainedPlaylists.size,
|
|
16971
|
+
batteryWakes: this.batteryWakes.size,
|
|
16972
|
+
triggers: this.triggers.size,
|
|
16973
|
+
signalConfirmedAt: this.signalConfirmedAt.size,
|
|
16974
|
+
signalContradictions: this.signalContradictions.size,
|
|
16975
|
+
lastSegmentAt: this.lastSegmentAt.size,
|
|
16976
|
+
attachRetained: this.attachRetained.size,
|
|
16977
|
+
unrecordableReported: this.unrecordableReported.size,
|
|
16978
|
+
skippedProfileDevices: this.skippedProfiles.size,
|
|
16979
|
+
skippedProfiles
|
|
16980
|
+
};
|
|
16981
|
+
}
|
|
16982
|
+
/**
|
|
16983
|
+
* Pids of every ffmpeg this controller started and has not seen exit —
|
|
16984
|
+
* the live writers and the ones still being torn down. Never reaped by the
|
|
16985
|
+
* staging guard (D667).
|
|
16986
|
+
*/
|
|
16987
|
+
ownedWriterPids() {
|
|
16988
|
+
const pids = /* @__PURE__ */ new Set();
|
|
16989
|
+
const add = (writer) => {
|
|
16990
|
+
const pid = writer.pid;
|
|
16991
|
+
if (pid !== void 0) pids.add(pid);
|
|
16992
|
+
};
|
|
16993
|
+
for (const recordings of this.active.values()) for (const recording of recordings) add(recording.writer);
|
|
16994
|
+
for (const writer of this.retiring) add(writer);
|
|
16995
|
+
return pids;
|
|
16996
|
+
}
|
|
16997
|
+
/**
|
|
16998
|
+
* Every staging directory of this camera, on every location, in ONE table
|
|
16999
|
+
* read and one SIGTERM → SIGKILL window (D667) — so an attach of three
|
|
17000
|
+
* profiles does not pay three reaps in series. The per-spawn claim below
|
|
17001
|
+
* still runs; after this it is one cached read.
|
|
17002
|
+
*/
|
|
17003
|
+
async sweepDeviceStaging(deviceId, segmentSeconds) {
|
|
17004
|
+
const guard = this.deps.stagingGuard;
|
|
17005
|
+
if (guard === void 0) return;
|
|
17006
|
+
const result = await guard.sweep({
|
|
17007
|
+
roots: this.deps.locations().map((location) => location.root),
|
|
17008
|
+
deviceId,
|
|
17009
|
+
ownPids: this.ownedWriterPids()
|
|
17010
|
+
});
|
|
17011
|
+
if (result.outcome !== "swept" || result.reaped.length === 0) return;
|
|
17012
|
+
const liveDirs = new Set((this.active.get(deviceId) ?? []).map((r) => r.outDir));
|
|
17013
|
+
await this.recoverReaped(result.reaped.filter((writer) => !liveDirs.has(writer.target.outDir)), segmentSeconds);
|
|
17014
|
+
}
|
|
17015
|
+
/**
|
|
17016
|
+
* Collect what each reaped writer left in its directory before a new writer
|
|
17017
|
+
* is spawned into it (D667). Logged per directory with the counts, and a
|
|
17018
|
+
* directory that could not be recovered says so — its segments stay staged
|
|
17019
|
+
* for the next boot.
|
|
17020
|
+
*/
|
|
17021
|
+
async recoverReaped(reaped, segmentSeconds) {
|
|
17022
|
+
const cutoffMs = this.now();
|
|
17023
|
+
const dirs = /* @__PURE__ */ new Map();
|
|
17024
|
+
for (const writer of reaped) dirs.set(writer.target.outDir, writer);
|
|
17025
|
+
for (const writer of dirs.values()) {
|
|
17026
|
+
const { target } = writer;
|
|
17027
|
+
const location = this.deps.locations().find((l) => l.root === target.root);
|
|
17028
|
+
const recover = this.deps.recoverStagingDirectory;
|
|
17029
|
+
if (location === void 0 || recover === void 0) {
|
|
17030
|
+
this.deps.logger.warn("recorder: a reaped writer’s segments were left in staging — recovered at the next boot", {
|
|
17031
|
+
tags: { deviceId: target.deviceId },
|
|
17032
|
+
meta: {
|
|
17033
|
+
profile: target.profile,
|
|
17034
|
+
dir: target.outDir,
|
|
17035
|
+
reason: location === void 0 ? "location-unknown" : "no-recovery-wired"
|
|
17036
|
+
}
|
|
17037
|
+
});
|
|
17038
|
+
continue;
|
|
17039
|
+
}
|
|
17040
|
+
try {
|
|
17041
|
+
const report = await recover({
|
|
17042
|
+
deviceId: target.deviceId,
|
|
17043
|
+
profile: target.profile,
|
|
17044
|
+
locationId: location.id,
|
|
17045
|
+
locationRoot: location.root,
|
|
17046
|
+
dir: target.outDir,
|
|
17047
|
+
cutoffMs,
|
|
17048
|
+
segmentSeconds
|
|
17049
|
+
});
|
|
17050
|
+
this.deps.logger.info("recorder: recovered a reaped writer’s segments before re-attaching", {
|
|
17051
|
+
tags: { deviceId: target.deviceId },
|
|
17052
|
+
meta: {
|
|
17053
|
+
profile: target.profile,
|
|
17054
|
+
dir: target.outDir,
|
|
17055
|
+
pid: writer.pid,
|
|
17056
|
+
...report
|
|
17057
|
+
}
|
|
17058
|
+
});
|
|
17059
|
+
} catch (err) {
|
|
17060
|
+
this.deps.logger.warn("recorder: recovering a reaped writer’s segments failed — left in staging for the next boot", {
|
|
17061
|
+
tags: { deviceId: target.deviceId },
|
|
17062
|
+
meta: {
|
|
17063
|
+
profile: target.profile,
|
|
17064
|
+
dir: target.outDir,
|
|
17065
|
+
error: require_dist.errMsg(err)
|
|
17066
|
+
}
|
|
17067
|
+
});
|
|
17068
|
+
}
|
|
17069
|
+
}
|
|
17070
|
+
}
|
|
17071
|
+
/**
|
|
17072
|
+
* Make sure nobody else is writing `outDir` before a writer is spawned into
|
|
17073
|
+
* it (D667). A predecessor runner that died without a shutdown (V8 OOM,
|
|
17074
|
+
* SIGKILL) leaves its ffmpeg writers running, and a second writer in the
|
|
17075
|
+
* same directory rewrites the first one's playlist — 13 minutes and 169
|
|
17076
|
+
* writer churns on 2026-09-28. The guard reaps such a writer; one it cannot
|
|
17077
|
+
* remove is a refusal, and a refusal throws so no writer is spawned.
|
|
17078
|
+
*/
|
|
17079
|
+
async claimStagingDirectory(deviceId, profile, root, outDir, segmentSeconds) {
|
|
17080
|
+
const guard = this.deps.stagingGuard;
|
|
17081
|
+
if (guard === void 0) return;
|
|
17082
|
+
const result = await guard.claim({
|
|
17083
|
+
deviceId,
|
|
17084
|
+
profile,
|
|
17085
|
+
root,
|
|
17086
|
+
outDir,
|
|
17087
|
+
ownPids: this.ownedWriterPids()
|
|
17088
|
+
});
|
|
17089
|
+
if (result.outcome === "clear" && result.reaped.length > 0) await this.recoverReaped(result.reaped, segmentSeconds);
|
|
17090
|
+
if (result.outcome === "refused") throw new Error(`staging directory ${outDir} still has ${result.survivors.length} foreign writer(s) (pids ${result.survivors.map((proc) => proc.pid).join(",")}) — refusing to co-write`);
|
|
17091
|
+
}
|
|
17092
|
+
/**
|
|
16762
17093
|
* Attach one (device, profile): acquire the broker source, mkdir the staging
|
|
16763
17094
|
* outDir UNDER the profile's placement root (same-fs relocate — see
|
|
16764
17095
|
* {@link STAGING_DIR_NAME}), spawn the SegmentWriter, and start the watcher
|
|
@@ -16767,6 +17098,10 @@ var RecordingController = class {
|
|
|
16767
17098
|
async attachProfile(deviceId, profile, segmentSeconds, audio) {
|
|
16768
17099
|
const attachProfileStartMs = this.now();
|
|
16769
17100
|
const placement = await this.resolvePlacement(deviceId, profile);
|
|
17101
|
+
const placementDoneMs = this.now();
|
|
17102
|
+
const outDir = node_path.default.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
|
|
17103
|
+
await this.claimStagingDirectory(deviceId, profile, placement.root, outDir, segmentSeconds);
|
|
17104
|
+
const claimDoneMs = this.now();
|
|
16770
17105
|
const source = await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.getStreamWithCodec.mutate({
|
|
16771
17106
|
deviceId,
|
|
16772
17107
|
video: "copy",
|
|
@@ -16786,8 +17121,6 @@ var RecordingController = class {
|
|
|
16786
17121
|
url: source.url
|
|
16787
17122
|
}
|
|
16788
17123
|
});
|
|
16789
|
-
const placementDoneMs = this.now();
|
|
16790
|
-
const outDir = node_path.default.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
|
|
16791
17124
|
await node_fs.promises.mkdir(outDir, { recursive: true });
|
|
16792
17125
|
const mkdirDoneMs = this.now();
|
|
16793
17126
|
const deviceLog = this.deps.logger.withTags({ deviceId });
|
|
@@ -16820,18 +17153,27 @@ var RecordingController = class {
|
|
|
16820
17153
|
profile,
|
|
16821
17154
|
consumer: consumerTag,
|
|
16822
17155
|
placementMs: placementDoneMs - attachProfileStartMs,
|
|
16823
|
-
|
|
17156
|
+
claimMs: claimDoneMs - placementDoneMs,
|
|
17157
|
+
getStreamMs: streamAcquiredMs - claimDoneMs,
|
|
16824
17158
|
mkdirMs: mkdirDoneMs - streamAcquiredMs,
|
|
16825
17159
|
attachProfileMs: writerStartedMs - attachProfileStartMs,
|
|
16826
17160
|
segmentSeconds
|
|
16827
17161
|
}
|
|
16828
17162
|
});
|
|
16829
17163
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
17164
|
+
let firstSegmentSeen = false;
|
|
17165
|
+
const drainKey = node_path.default.resolve(outDir);
|
|
17166
|
+
const drainedPredecessor = this.drainedPlaylists.get(drainKey) ?? null;
|
|
17167
|
+
this.drainedPlaylists.delete(drainKey);
|
|
16830
17168
|
return {
|
|
16831
17169
|
profile,
|
|
16832
17170
|
writer,
|
|
16833
17171
|
watcher: startSegmentWatcher({
|
|
16834
17172
|
outDir,
|
|
17173
|
+
drainedPredecessor,
|
|
17174
|
+
onDrained: (drained) => {
|
|
17175
|
+
this.drainedPlaylists.set(drainKey, drained);
|
|
17176
|
+
},
|
|
16835
17177
|
deviceId,
|
|
16836
17178
|
writerStartedMs,
|
|
16837
17179
|
now: () => this.now(),
|
|
@@ -16841,8 +17183,8 @@ var RecordingController = class {
|
|
|
16841
17183
|
intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
|
|
16842
17184
|
onFinalized: async (startMs, durMs, flatAbsPath) => {
|
|
16843
17185
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
16844
|
-
if (startMs >= writerStartedMs - 1e3 && !
|
|
16845
|
-
|
|
17186
|
+
if (startMs >= writerStartedMs - 1e3 && !firstSegmentSeen) {
|
|
17187
|
+
firstSegmentSeen = true;
|
|
16846
17188
|
this.deps.logger.info("recorder: first segment of this attach", {
|
|
16847
17189
|
tags: { deviceId },
|
|
16848
17190
|
meta: {
|
|
@@ -16966,8 +17308,11 @@ var RecordingController = class {
|
|
|
16966
17308
|
async teardownProfile(deviceId, r) {
|
|
16967
17309
|
const teardownStartMs = Date.now();
|
|
16968
17310
|
try {
|
|
17311
|
+
this.retiring.add(r.writer);
|
|
16969
17312
|
await r.writer.stopAndWait();
|
|
16970
|
-
} catch {}
|
|
17313
|
+
} catch {} finally {
|
|
17314
|
+
this.retiring.delete(r.writer);
|
|
17315
|
+
}
|
|
16971
17316
|
const writerStopMs = Date.now() - teardownStartMs;
|
|
16972
17317
|
try {
|
|
16973
17318
|
await r.watcher.flush?.();
|
|
@@ -17016,7 +17361,6 @@ var RecordingController = class {
|
|
|
17016
17361
|
for (const r of recordings) {
|
|
17017
17362
|
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
17018
17363
|
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
17019
|
-
this.firstSegmentSeen.delete(idleKey(deviceId, r.profile));
|
|
17020
17364
|
}
|
|
17021
17365
|
await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
|
|
17022
17366
|
this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
|
|
@@ -17675,7 +18019,11 @@ var RedundancySweeper = class {
|
|
|
17675
18019
|
start() {
|
|
17676
18020
|
if (this.timer !== null) return;
|
|
17677
18021
|
this.timer = setInterval(() => {
|
|
17678
|
-
this.
|
|
18022
|
+
const probe = this.deps.heapProbe;
|
|
18023
|
+
probe === void 0 ? this.sweep() : probe.track({
|
|
18024
|
+
kind: "pass",
|
|
18025
|
+
name: "redundancy"
|
|
18026
|
+
}, () => this.sweep());
|
|
17679
18027
|
}, this.deps.intervalMs);
|
|
17680
18028
|
this.timer.unref?.();
|
|
17681
18029
|
}
|
|
@@ -17818,7 +18166,11 @@ var RetentionSweeper = class {
|
|
|
17818
18166
|
start() {
|
|
17819
18167
|
if (this.timer !== null) return;
|
|
17820
18168
|
const tick = () => {
|
|
17821
|
-
this.
|
|
18169
|
+
const probe = this.deps.heapProbe;
|
|
18170
|
+
probe === void 0 ? this.sweep() : probe.track({
|
|
18171
|
+
kind: "pass",
|
|
18172
|
+
name: "retention"
|
|
18173
|
+
}, () => this.sweep());
|
|
17822
18174
|
};
|
|
17823
18175
|
tick();
|
|
17824
18176
|
this.timer = setInterval(tick, this.deps.intervalMs);
|
|
@@ -17913,6 +18265,799 @@ var RECORDER_RETIRED_ROOT_KEYS = [
|
|
|
17913
18265
|
}
|
|
17914
18266
|
];
|
|
17915
18267
|
//#endregion
|
|
18268
|
+
//#region src/recorder/addon/heap-delta.ts
|
|
18269
|
+
/**
|
|
18270
|
+
* Name the unit of work that grows the recorder's heap.
|
|
18271
|
+
*
|
|
18272
|
+
* Every recorder lifetime has the same shape (findings 2026-09-28): ~50 MB for
|
|
18273
|
+
* five minutes, then one SILENT, CPU-bound phase of 4.5–7 minutes that builds
|
|
18274
|
+
* ~2.3 GB — ~910 MB of it in `large_object_space` — and holds it. The
|
|
18275
|
+
* large-object floor doubled at one boot (2026-09-03 06:56Z, 458 → 914 MB),
|
|
18276
|
+
* the first to show a second recording location. Nothing logs during the
|
|
18277
|
+
* phase, so nothing says which pass or which inbound call builds it, and a
|
|
18278
|
+
* heap snapshot is not allowed on this host.
|
|
18279
|
+
*
|
|
18280
|
+
* So every periodic pass, every boot phase and every inbound cap method is
|
|
18281
|
+
* wrapped here: two cheap heap reads (`v8.getHeapStatistics` and
|
|
18282
|
+
* `v8.getHeapSpaceStatistics` — a handful of counters, no heap walk), and ONE
|
|
18283
|
+
* line when the unit grew the heap or the large-object space by at least
|
|
18284
|
+
* `deltaBytes`, or ran for at least `durationMs`. Silent otherwise.
|
|
18285
|
+
*
|
|
18286
|
+
* Two things this line is NOT, and a reader must not take it for:
|
|
18287
|
+
*
|
|
18288
|
+
* - **Not exclusive.** The delta is over the unit's WALL window, so concurrent
|
|
18289
|
+
* work that interleaves with it is counted too. `inFlight` on the line is the
|
|
18290
|
+
* MOST other measured units that were open at any moment DURING this one —
|
|
18291
|
+
* a long pass that started alone and was overlapped by forty cap calls
|
|
18292
|
+
* reports 40, not 0. `inFlight: 0` is the only reading that pins the delta
|
|
18293
|
+
* on this unit (unmeasured work — a transport decode, a timer — can still
|
|
18294
|
+
* interleave).
|
|
18295
|
+
* - **Not retention.** `heapUsed` includes garbage no GC has collected yet, so
|
|
18296
|
+
* a unit that ALLOCATES 50 MB of transient reads is named even though it
|
|
18297
|
+
* keeps none of it. A unit named on several passes in a row while the heap
|
|
18298
|
+
* floor rises is a builder; one named once is only a candidate.
|
|
18299
|
+
*/
|
|
18300
|
+
var MIB = 1024 * 1024;
|
|
18301
|
+
var DEFAULT_HEAP_DELTA_THRESHOLDS = {
|
|
18302
|
+
deltaBytes: 50 * MIB,
|
|
18303
|
+
durationMs: 5e3
|
|
18304
|
+
};
|
|
18305
|
+
/**
|
|
18306
|
+
* Whether one unit of work is worth a line. GROWTH only: a unit that happened
|
|
18307
|
+
* to straddle a major GC shows a large negative delta that says nothing about
|
|
18308
|
+
* what it retained, and the question being asked is who BUILDS the heap.
|
|
18309
|
+
*/
|
|
18310
|
+
function judgeHeapDelta(before, after, durationMs, thresholds) {
|
|
18311
|
+
const heapDeltaBytes = before !== null && after !== null ? after.heapUsedBytes - before.heapUsedBytes : null;
|
|
18312
|
+
const losBefore = before?.largeObjectBytes ?? null;
|
|
18313
|
+
const losAfter = after?.largeObjectBytes ?? null;
|
|
18314
|
+
const largeObjectDeltaBytes = losBefore !== null && losAfter !== null ? losAfter - losBefore : null;
|
|
18315
|
+
const reasons = [];
|
|
18316
|
+
if (heapDeltaBytes !== null && heapDeltaBytes >= thresholds.deltaBytes) reasons.push("heap");
|
|
18317
|
+
if (largeObjectDeltaBytes !== null && largeObjectDeltaBytes >= thresholds.deltaBytes) reasons.push("large-object-space");
|
|
18318
|
+
if (durationMs >= thresholds.durationMs) reasons.push("duration");
|
|
18319
|
+
return {
|
|
18320
|
+
reasons,
|
|
18321
|
+
heapDeltaBytes,
|
|
18322
|
+
largeObjectDeltaBytes
|
|
18323
|
+
};
|
|
18324
|
+
}
|
|
18325
|
+
/** V8's own counters. `null` if V8 refuses — the caller logs nulls, not zeros. */
|
|
18326
|
+
function readV8HeapSample() {
|
|
18327
|
+
try {
|
|
18328
|
+
const heapUsedBytes = node_v8.default.getHeapStatistics().used_heap_size;
|
|
18329
|
+
const los = node_v8.default.getHeapSpaceStatistics().find((space) => space.space_name === "large_object_space");
|
|
18330
|
+
return {
|
|
18331
|
+
heapUsedBytes,
|
|
18332
|
+
largeObjectBytes: los === void 0 ? null : los.space_used_size
|
|
18333
|
+
};
|
|
18334
|
+
} catch {
|
|
18335
|
+
return null;
|
|
18336
|
+
}
|
|
18337
|
+
}
|
|
18338
|
+
function mb(bytes) {
|
|
18339
|
+
return bytes === null ? null : Math.round(bytes / MIB);
|
|
18340
|
+
}
|
|
18341
|
+
function isPromiseLike(value) {
|
|
18342
|
+
return typeof value === "object" && value !== null && "then" in value && typeof value.then === "function";
|
|
18343
|
+
}
|
|
18344
|
+
var HeapDeltaProbe = class {
|
|
18345
|
+
deps;
|
|
18346
|
+
/** Open units → the most OTHER units seen open at once while it ran. */
|
|
18347
|
+
peers = /* @__PURE__ */ new Map();
|
|
18348
|
+
seq = 0;
|
|
18349
|
+
sampler;
|
|
18350
|
+
now;
|
|
18351
|
+
thresholds;
|
|
18352
|
+
constructor(deps) {
|
|
18353
|
+
this.deps = deps;
|
|
18354
|
+
this.sampler = deps.sampler ?? readV8HeapSample;
|
|
18355
|
+
this.now = deps.now ?? (() => Date.now());
|
|
18356
|
+
this.thresholds = deps.thresholds ?? DEFAULT_HEAP_DELTA_THRESHOLDS;
|
|
18357
|
+
}
|
|
18358
|
+
async track(unit, run) {
|
|
18359
|
+
const open = this.begin();
|
|
18360
|
+
let failed = false;
|
|
18361
|
+
try {
|
|
18362
|
+
return await run();
|
|
18363
|
+
} catch (err) {
|
|
18364
|
+
failed = true;
|
|
18365
|
+
throw err;
|
|
18366
|
+
} finally {
|
|
18367
|
+
this.end(unit, open, failed);
|
|
18368
|
+
}
|
|
18369
|
+
}
|
|
18370
|
+
/**
|
|
18371
|
+
* The same provider, with every method measured. A `Proxy` rather than a
|
|
18372
|
+
* rebuilt object so the capability's own type passes through untouched —
|
|
18373
|
+
* no method is re-declared, and none can be forgotten when the cap grows
|
|
18374
|
+
* one. `this` stays the original object, so a class with private state
|
|
18375
|
+
* keeps working.
|
|
18376
|
+
*/
|
|
18377
|
+
wrapProvider(capability, provider) {
|
|
18378
|
+
const cache = /* @__PURE__ */ new Map();
|
|
18379
|
+
return new Proxy(provider, { get: (target, prop) => {
|
|
18380
|
+
const value = Reflect.get(target, prop, target);
|
|
18381
|
+
if (typeof value !== "function" || typeof prop === "symbol" || prop in Object.prototype) return value;
|
|
18382
|
+
const cached = cache.get(prop);
|
|
18383
|
+
if (cached !== void 0 && cached.original === value) return cached.wrapped;
|
|
18384
|
+
const unit = {
|
|
18385
|
+
kind: "cap-method",
|
|
18386
|
+
name: `${capability}.${prop}`
|
|
18387
|
+
};
|
|
18388
|
+
const wrapped = (...args) => this.call(unit, () => Reflect.apply(value, target, args));
|
|
18389
|
+
cache.set(prop, {
|
|
18390
|
+
original: value,
|
|
18391
|
+
wrapped
|
|
18392
|
+
});
|
|
18393
|
+
return wrapped;
|
|
18394
|
+
} });
|
|
18395
|
+
}
|
|
18396
|
+
call(unit, invoke) {
|
|
18397
|
+
const open = this.begin();
|
|
18398
|
+
let result;
|
|
18399
|
+
try {
|
|
18400
|
+
result = invoke();
|
|
18401
|
+
} catch (err) {
|
|
18402
|
+
this.end(unit, open, true);
|
|
18403
|
+
throw err;
|
|
18404
|
+
}
|
|
18405
|
+
if (!isPromiseLike(result)) {
|
|
18406
|
+
this.end(unit, open, false);
|
|
18407
|
+
return result;
|
|
18408
|
+
}
|
|
18409
|
+
return Promise.resolve(result).then((value) => {
|
|
18410
|
+
this.end(unit, open, false);
|
|
18411
|
+
return value;
|
|
18412
|
+
}, (err) => {
|
|
18413
|
+
this.end(unit, open, true);
|
|
18414
|
+
throw err;
|
|
18415
|
+
});
|
|
18416
|
+
}
|
|
18417
|
+
begin() {
|
|
18418
|
+
const id = ++this.seq;
|
|
18419
|
+
const others = this.peers.size;
|
|
18420
|
+
for (const [peer, most] of this.peers) this.peers.set(peer, Math.max(most, others));
|
|
18421
|
+
this.peers.set(id, others);
|
|
18422
|
+
return {
|
|
18423
|
+
id,
|
|
18424
|
+
sample: this.sampler(),
|
|
18425
|
+
at: this.now()
|
|
18426
|
+
};
|
|
18427
|
+
}
|
|
18428
|
+
end(unit, open, failed) {
|
|
18429
|
+
const inFlight = this.peers.get(open.id) ?? 0;
|
|
18430
|
+
this.peers.delete(open.id);
|
|
18431
|
+
const durationMs = this.now() - open.at;
|
|
18432
|
+
const after = this.sampler();
|
|
18433
|
+
const verdict = judgeHeapDelta(open.sample, after, durationMs, this.thresholds);
|
|
18434
|
+
if (verdict.reasons.length === 0) return;
|
|
18435
|
+
this.deps.logger.info("recorder: unit of work moved the heap", { meta: {
|
|
18436
|
+
kind: unit.kind,
|
|
18437
|
+
name: unit.name,
|
|
18438
|
+
reasons: verdict.reasons,
|
|
18439
|
+
durationMs,
|
|
18440
|
+
heapBeforeMb: mb(open.sample?.heapUsedBytes ?? null),
|
|
18441
|
+
heapAfterMb: mb(after?.heapUsedBytes ?? null),
|
|
18442
|
+
heapDeltaMb: mb(verdict.heapDeltaBytes),
|
|
18443
|
+
losBeforeMb: mb(open.sample?.largeObjectBytes ?? null),
|
|
18444
|
+
losAfterMb: mb(after?.largeObjectBytes ?? null),
|
|
18445
|
+
losDeltaMb: mb(verdict.largeObjectDeltaBytes),
|
|
18446
|
+
inFlight,
|
|
18447
|
+
failed
|
|
18448
|
+
} });
|
|
18449
|
+
}
|
|
18450
|
+
};
|
|
18451
|
+
//#endregion
|
|
18452
|
+
//#region src/recorder/addon/process-table.ts
|
|
18453
|
+
/**
|
|
18454
|
+
* Read this host's process table — just enough to recognise an ffmpeg writer
|
|
18455
|
+
* by its argv — and signal a pid.
|
|
18456
|
+
*
|
|
18457
|
+
* Both halves sit behind a small named interface so the staging-directory
|
|
18458
|
+
* guard (`staging-writer-guard.ts`, D667) can be driven by a fake in a unit
|
|
18459
|
+
* test and by the real OS in production, with no cast in between.
|
|
18460
|
+
*
|
|
18461
|
+
* Linux reads `/proc/<pid>/cmdline` (exact argv, NUL-separated) and
|
|
18462
|
+
* `/proc/<pid>/stat` (ppid, start time). macOS has no `/proc`, so it runs
|
|
18463
|
+
* `ps -axww -o pid=,ppid=,etime=,command=` — the recorder also runs on the
|
|
18464
|
+
* Mac agent. Any other platform gets no lister, and the guard says so.
|
|
18465
|
+
*/
|
|
18466
|
+
/**
|
|
18467
|
+
* Linux `USER_HZ` — the unit of `starttime` in `/proc/<pid>/stat`. It is part
|
|
18468
|
+
* of the userspace ABI and 100 on every architecture Linux ships, which is why
|
|
18469
|
+
* `procps` itself hard-codes the fallback.
|
|
18470
|
+
*/
|
|
18471
|
+
var LINUX_USER_HZ = 100;
|
|
18472
|
+
/** Field index of `starttime` counted from the field AFTER the `(comm)` one. */
|
|
18473
|
+
var STAT_STARTTIME_INDEX = 19;
|
|
18474
|
+
var STAT_PPID_INDEX = 1;
|
|
18475
|
+
/** `ps` is one short fork; a table that takes longer than this is a sick host. */
|
|
18476
|
+
var PS_TIMEOUT_MS = 5e3;
|
|
18477
|
+
var PS_MAX_BUFFER = 16 * 1024 * 1024;
|
|
18478
|
+
function errorCode(err) {
|
|
18479
|
+
if (err instanceof Error && "code" in err && typeof err.code === "string") return err.code;
|
|
18480
|
+
}
|
|
18481
|
+
/** `ppid` and `starttime` out of a `/proc/<pid>/stat` body. The `comm` field
|
|
18482
|
+
* may itself contain spaces and parentheses, so fields are counted from the
|
|
18483
|
+
* LAST `)`. */
|
|
18484
|
+
function parseLinuxStat(body) {
|
|
18485
|
+
const close = body.lastIndexOf(")");
|
|
18486
|
+
if (close < 0) return {
|
|
18487
|
+
ppid: null,
|
|
18488
|
+
startTicks: null
|
|
18489
|
+
};
|
|
18490
|
+
const fields = body.slice(close + 1).trim().split(/\s+/);
|
|
18491
|
+
const ppid = Number(fields[STAT_PPID_INDEX]);
|
|
18492
|
+
const start = Number(fields[STAT_STARTTIME_INDEX]);
|
|
18493
|
+
return {
|
|
18494
|
+
ppid: Number.isInteger(ppid) ? ppid : null,
|
|
18495
|
+
startTicks: Number.isFinite(start) ? start : null
|
|
18496
|
+
};
|
|
18497
|
+
}
|
|
18498
|
+
/** The `/proc` reader. `procRoot` is injectable for a test over a fake tree. */
|
|
18499
|
+
function createLinuxProcessLister(procRoot = "/proc") {
|
|
18500
|
+
return { async list(marker) {
|
|
18501
|
+
const entries = await node_fs.promises.readdir(procRoot);
|
|
18502
|
+
const uptimeSec = await node_fs.promises.readFile(`${procRoot}/uptime`, "utf8").then((body) => Number(body.split(/\s+/)[0])).catch(() => NaN);
|
|
18503
|
+
const out = [];
|
|
18504
|
+
for (const entry of entries) {
|
|
18505
|
+
const pid = Number(entry);
|
|
18506
|
+
if (!Number.isInteger(pid) || pid <= 0) continue;
|
|
18507
|
+
let raw;
|
|
18508
|
+
try {
|
|
18509
|
+
raw = await node_fs.promises.readFile(`${procRoot}/${entry}/cmdline`, "utf8");
|
|
18510
|
+
} catch {
|
|
18511
|
+
continue;
|
|
18512
|
+
}
|
|
18513
|
+
if (raw.length === 0 || !raw.includes(marker)) continue;
|
|
18514
|
+
const argv = raw.split("\0");
|
|
18515
|
+
if (argv[argv.length - 1] === "") argv.pop();
|
|
18516
|
+
const stat = await node_fs.promises.readFile(`${procRoot}/${entry}/stat`, "utf8").then(parseLinuxStat).catch(() => ({
|
|
18517
|
+
ppid: null,
|
|
18518
|
+
startTicks: null
|
|
18519
|
+
}));
|
|
18520
|
+
const ageMs = stat.startTicks !== null && Number.isFinite(uptimeSec) ? Math.max(0, Math.round((uptimeSec - stat.startTicks / LINUX_USER_HZ) * 1e3)) : null;
|
|
18521
|
+
out.push({
|
|
18522
|
+
pid,
|
|
18523
|
+
ppid: stat.ppid,
|
|
18524
|
+
argv,
|
|
18525
|
+
command: argv.join(" "),
|
|
18526
|
+
ageMs
|
|
18527
|
+
});
|
|
18528
|
+
}
|
|
18529
|
+
return out;
|
|
18530
|
+
} };
|
|
18531
|
+
}
|
|
18532
|
+
/** `[[dd-]hh:]mm:ss` → ms; `null` for anything else. */
|
|
18533
|
+
function parsePsElapsed(etime) {
|
|
18534
|
+
const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/.exec(etime.trim());
|
|
18535
|
+
if (match === null) return null;
|
|
18536
|
+
const [, days, hours, minutes, seconds] = match;
|
|
18537
|
+
return (Number(days ?? 0) * 86400 + Number(hours ?? 0) * 3600 + Number(minutes) * 60 + Number(seconds)) * 1e3;
|
|
18538
|
+
}
|
|
18539
|
+
/** Parse `ps -o pid=,ppid=,etime=,command=` output, keeping lines that contain `marker`. */
|
|
18540
|
+
function parsePsTable(stdout, marker) {
|
|
18541
|
+
const out = [];
|
|
18542
|
+
for (const line of stdout.split("\n")) {
|
|
18543
|
+
if (!line.includes(marker)) continue;
|
|
18544
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line);
|
|
18545
|
+
if (match === null) continue;
|
|
18546
|
+
const [, pid, ppid, etime, command] = match;
|
|
18547
|
+
out.push({
|
|
18548
|
+
pid: Number(pid),
|
|
18549
|
+
ppid: Number(ppid),
|
|
18550
|
+
argv: null,
|
|
18551
|
+
command: command ?? "",
|
|
18552
|
+
ageMs: parsePsElapsed(etime ?? "")
|
|
18553
|
+
});
|
|
18554
|
+
}
|
|
18555
|
+
return out;
|
|
18556
|
+
}
|
|
18557
|
+
/** The macOS (BSD `ps`) reader. `-ww` so a long staging path is never cut. */
|
|
18558
|
+
function createPsProcessLister(psPath = "ps") {
|
|
18559
|
+
return { list(marker) {
|
|
18560
|
+
return new Promise((resolve, reject) => {
|
|
18561
|
+
(0, node_child_process.execFile)(psPath, [
|
|
18562
|
+
"-axww",
|
|
18563
|
+
"-o",
|
|
18564
|
+
"pid=,ppid=,etime=,command="
|
|
18565
|
+
], {
|
|
18566
|
+
timeout: PS_TIMEOUT_MS,
|
|
18567
|
+
maxBuffer: PS_MAX_BUFFER
|
|
18568
|
+
}, (err, stdout) => {
|
|
18569
|
+
if (err) {
|
|
18570
|
+
reject(err);
|
|
18571
|
+
return;
|
|
18572
|
+
}
|
|
18573
|
+
resolve(parsePsTable(stdout, marker));
|
|
18574
|
+
});
|
|
18575
|
+
});
|
|
18576
|
+
} };
|
|
18577
|
+
}
|
|
18578
|
+
/** The lister for this platform, or `null` where there is none (Windows). */
|
|
18579
|
+
function createProcessLister(platform = process.platform) {
|
|
18580
|
+
if (platform === "linux") return createLinuxProcessLister();
|
|
18581
|
+
if (platform === "darwin" || platform === "freebsd" || platform === "openbsd") return createPsProcessLister();
|
|
18582
|
+
return null;
|
|
18583
|
+
}
|
|
18584
|
+
/** `process.kill`, with the errno turned into an answer instead of a throw. */
|
|
18585
|
+
var nodeProcessSignaller = { send(pid, signal) {
|
|
18586
|
+
try {
|
|
18587
|
+
process.kill(pid, signal);
|
|
18588
|
+
return "sent";
|
|
18589
|
+
} catch (err) {
|
|
18590
|
+
const code = errorCode(err);
|
|
18591
|
+
if (code === "ESRCH") return "gone";
|
|
18592
|
+
if (code === "EPERM") return "denied";
|
|
18593
|
+
return "failed";
|
|
18594
|
+
}
|
|
18595
|
+
} };
|
|
18596
|
+
//#endregion
|
|
18597
|
+
//#region src/recorder/addon/staging-writer-match.ts
|
|
18598
|
+
/**
|
|
18599
|
+
* Recognise a recorder writer in the process table (D667).
|
|
18600
|
+
*
|
|
18601
|
+
* A writer is recognised by the ARGV it was started with. Two parts decide it:
|
|
18602
|
+
* the executable is an ffmpeg, and `-segment_list` names a staging playlist.
|
|
18603
|
+
* The directory is what makes it a rival, not the binary: another node's
|
|
18604
|
+
* writers are in another pid namespace or on another host, and a writer of a
|
|
18605
|
+
* different directory is not competing for this one.
|
|
18606
|
+
*
|
|
18607
|
+
* Linux gives the exact argv, so every comparison there is exact. macOS `ps`
|
|
18608
|
+
* gives one space-joined string, so the match is a substring with a boundary
|
|
18609
|
+
* check on both sides. A profile name therefore may not contain a space. The
|
|
18610
|
+
* recorder's are `high`, `mid` and `low`.
|
|
18611
|
+
*/
|
|
18612
|
+
/** The argv flag every recorder writer carries, used to narrow the table read. */
|
|
18613
|
+
var SEGMENT_LIST_FLAG = "-segment_list";
|
|
18614
|
+
/**
|
|
18615
|
+
* True when the process was started from an executable whose name contains
|
|
18616
|
+
* `marker`. Without this, a `pgrep -f`, a `grep` or an `sh -c` wrapper that
|
|
18617
|
+
* merely mentions the playlist would be signalled on macOS, where the argv
|
|
18618
|
+
* cannot be told apart from the command text.
|
|
18619
|
+
*/
|
|
18620
|
+
function isWriterExecutable(proc, marker) {
|
|
18621
|
+
if (proc.argv !== null) {
|
|
18622
|
+
const exe = proc.argv[0];
|
|
18623
|
+
return exe !== void 0 && node_path.default.basename(exe).includes(marker);
|
|
18624
|
+
}
|
|
18625
|
+
const flagAt = proc.command.indexOf(" -");
|
|
18626
|
+
const exe = flagAt < 0 ? proc.command : proc.command.slice(0, flagAt);
|
|
18627
|
+
return node_path.default.basename(exe).includes(marker);
|
|
18628
|
+
}
|
|
18629
|
+
/** True when this process was started as a writer of `playlistPath`. */
|
|
18630
|
+
function writesPlaylist(proc, playlistPath) {
|
|
18631
|
+
if (proc.argv !== null) {
|
|
18632
|
+
for (let i = 0; i < proc.argv.length - 1; i++) if (proc.argv[i] === "-segment_list" && proc.argv[i + 1] === playlistPath) return true;
|
|
18633
|
+
return false;
|
|
18634
|
+
}
|
|
18635
|
+
const needle = `${SEGMENT_LIST_FLAG} ${playlistPath}`;
|
|
18636
|
+
let from = 0;
|
|
18637
|
+
for (;;) {
|
|
18638
|
+
const at = proc.command.indexOf(needle, from);
|
|
18639
|
+
if (at < 0) return false;
|
|
18640
|
+
const before = at === 0 ? " " : proc.command[at - 1];
|
|
18641
|
+
const after = proc.command[at + needle.length];
|
|
18642
|
+
if (before === " " && (after === void 0 || after === " ")) return true;
|
|
18643
|
+
from = at + 1;
|
|
18644
|
+
}
|
|
18645
|
+
}
|
|
18646
|
+
var TAIL = /^(\d+)\/([^/\s]+)\/live\.m3u8$/;
|
|
18647
|
+
var TAIL_IN_COMMAND = /^(\d+)\/([^/\s]+)\/live\.m3u8(?= |$)/;
|
|
18648
|
+
function targetFromTail(root, tail, pattern) {
|
|
18649
|
+
const match = pattern.exec(tail);
|
|
18650
|
+
if (match === null) return null;
|
|
18651
|
+
const deviceId = Number(match[1]);
|
|
18652
|
+
const profile = match[2];
|
|
18653
|
+
if (!Number.isSafeInteger(deviceId) || profile === void 0) return null;
|
|
18654
|
+
return {
|
|
18655
|
+
deviceId,
|
|
18656
|
+
profile,
|
|
18657
|
+
root,
|
|
18658
|
+
outDir: node_path.default.join(root, STAGING_DIR_NAME, String(deviceId), profile)
|
|
18659
|
+
};
|
|
18660
|
+
}
|
|
18661
|
+
/**
|
|
18662
|
+
* The staging directory this process writes, when it writes one under any of
|
|
18663
|
+
* `roots` — `<root>/.rec-tmp/<deviceId>/<profile>/live.m3u8`, nothing looser.
|
|
18664
|
+
* Used by the sweeps, which do not know in advance which directories hold an
|
|
18665
|
+
* orphan.
|
|
18666
|
+
*/
|
|
18667
|
+
function stagingTargetOf(proc, roots) {
|
|
18668
|
+
for (const root of roots) {
|
|
18669
|
+
const prefix = `${node_path.default.join(root, STAGING_DIR_NAME)}/`;
|
|
18670
|
+
if (proc.argv !== null) {
|
|
18671
|
+
for (let i = 0; i < proc.argv.length - 1; i++) {
|
|
18672
|
+
const value = proc.argv[i + 1];
|
|
18673
|
+
if (proc.argv[i] !== "-segment_list" || value === void 0) continue;
|
|
18674
|
+
if (!value.startsWith(prefix)) continue;
|
|
18675
|
+
const target = targetFromTail(root, value.slice(prefix.length), TAIL);
|
|
18676
|
+
if (target !== null) return target;
|
|
18677
|
+
}
|
|
18678
|
+
continue;
|
|
18679
|
+
}
|
|
18680
|
+
const needle = `${SEGMENT_LIST_FLAG} ${prefix}`;
|
|
18681
|
+
let from = 0;
|
|
18682
|
+
for (;;) {
|
|
18683
|
+
const at = proc.command.indexOf(needle, from);
|
|
18684
|
+
if (at < 0) break;
|
|
18685
|
+
if ((at === 0 ? " " : proc.command[at - 1]) === " ") {
|
|
18686
|
+
const target = targetFromTail(root, proc.command.slice(at + needle.length), TAIL_IN_COMMAND);
|
|
18687
|
+
if (target !== null) return target;
|
|
18688
|
+
}
|
|
18689
|
+
from = at + 1;
|
|
18690
|
+
}
|
|
18691
|
+
}
|
|
18692
|
+
return null;
|
|
18693
|
+
}
|
|
18694
|
+
/** A table read this recent is good enough for a claim. Foreign writers only
|
|
18695
|
+
* come from a predecessor, which exists before the read — and every signal
|
|
18696
|
+
* this guard sends invalidates the cache. */
|
|
18697
|
+
var CLAIM_LIST_MAX_AGE_MS = 1e3;
|
|
18698
|
+
/** Two sightings of one pid whose start times differ by more than this are two
|
|
18699
|
+
* processes: `ps` reports elapsed time in whole seconds. */
|
|
18700
|
+
var SAME_START_TOLERANCE_MS = 2e3;
|
|
18701
|
+
var POLL_MS = 100;
|
|
18702
|
+
function errMsg(err) {
|
|
18703
|
+
return err instanceof Error ? err.message : String(err);
|
|
18704
|
+
}
|
|
18705
|
+
function ageSec(ageMs) {
|
|
18706
|
+
return ageMs === null ? null : Math.round(ageMs / 1e3);
|
|
18707
|
+
}
|
|
18708
|
+
var StagingWriterReaper = class {
|
|
18709
|
+
deps;
|
|
18710
|
+
refusals = /* @__PURE__ */ new Map();
|
|
18711
|
+
cache = null;
|
|
18712
|
+
inflight = null;
|
|
18713
|
+
constructor(deps) {
|
|
18714
|
+
this.deps = deps;
|
|
18715
|
+
}
|
|
18716
|
+
async claim(request) {
|
|
18717
|
+
const target = {
|
|
18718
|
+
deviceId: request.deviceId,
|
|
18719
|
+
profile: request.profile,
|
|
18720
|
+
root: request.root,
|
|
18721
|
+
outDir: request.outDir
|
|
18722
|
+
};
|
|
18723
|
+
const held = this.heldRecord(request.outDir);
|
|
18724
|
+
if (held !== null) {
|
|
18725
|
+
const survivor = {
|
|
18726
|
+
target,
|
|
18727
|
+
pid: held.pid,
|
|
18728
|
+
ppid: null,
|
|
18729
|
+
ageMs: null,
|
|
18730
|
+
reason: "held"
|
|
18731
|
+
};
|
|
18732
|
+
this.logHeld(survivor, held);
|
|
18733
|
+
return {
|
|
18734
|
+
outcome: "refused",
|
|
18735
|
+
survivors: [survivor]
|
|
18736
|
+
};
|
|
18737
|
+
}
|
|
18738
|
+
let rows;
|
|
18739
|
+
try {
|
|
18740
|
+
rows = await this.list(this.deps.now() - CLAIM_LIST_MAX_AGE_MS);
|
|
18741
|
+
} catch (err) {
|
|
18742
|
+
this.deps.logger.warn("recorder: could not read the process table — spawning a writer WITHOUT checking its staging directory for another writer", {
|
|
18743
|
+
tags: { deviceId: request.deviceId },
|
|
18744
|
+
meta: {
|
|
18745
|
+
profile: request.profile,
|
|
18746
|
+
outDir: request.outDir,
|
|
18747
|
+
error: errMsg(err)
|
|
18748
|
+
}
|
|
18749
|
+
});
|
|
18750
|
+
return {
|
|
18751
|
+
outcome: "unverified",
|
|
18752
|
+
error: errMsg(err)
|
|
18753
|
+
};
|
|
18754
|
+
}
|
|
18755
|
+
const playlist = segmentListPath(request.outDir);
|
|
18756
|
+
const candidates = rows.filter((proc) => this.isForeignWriter(proc, request.ownPids) && writesPlaylist(proc, playlist)).map((proc) => ({
|
|
18757
|
+
proc,
|
|
18758
|
+
target
|
|
18759
|
+
}));
|
|
18760
|
+
if (candidates.length === 0) {
|
|
18761
|
+
this.forgetRefusal(target);
|
|
18762
|
+
return {
|
|
18763
|
+
outcome: "clear",
|
|
18764
|
+
reaped: []
|
|
18765
|
+
};
|
|
18766
|
+
}
|
|
18767
|
+
const outcome = await this.reap(candidates);
|
|
18768
|
+
if (outcome.refused.length > 0) return {
|
|
18769
|
+
outcome: "refused",
|
|
18770
|
+
survivors: outcome.refused
|
|
18771
|
+
};
|
|
18772
|
+
return {
|
|
18773
|
+
outcome: "clear",
|
|
18774
|
+
reaped: outcome.reaped
|
|
18775
|
+
};
|
|
18776
|
+
}
|
|
18777
|
+
async sweep(request) {
|
|
18778
|
+
const startedMs = this.deps.now();
|
|
18779
|
+
const tags = request.deviceId === null ? void 0 : { deviceId: request.deviceId };
|
|
18780
|
+
let rows;
|
|
18781
|
+
try {
|
|
18782
|
+
rows = await this.list(startedMs);
|
|
18783
|
+
} catch (err) {
|
|
18784
|
+
this.deps.logger.warn("recorder: could not read the process table — a previous runner’s writers cannot be swept", {
|
|
18785
|
+
...tags ? { tags } : {},
|
|
18786
|
+
meta: {
|
|
18787
|
+
roots: request.roots,
|
|
18788
|
+
error: errMsg(err)
|
|
18789
|
+
}
|
|
18790
|
+
});
|
|
18791
|
+
return {
|
|
18792
|
+
outcome: "unverified",
|
|
18793
|
+
error: errMsg(err)
|
|
18794
|
+
};
|
|
18795
|
+
}
|
|
18796
|
+
const candidates = [];
|
|
18797
|
+
for (const proc of rows) {
|
|
18798
|
+
if (!this.isForeignWriter(proc, request.ownPids)) continue;
|
|
18799
|
+
const target = stagingTargetOf(proc, request.roots);
|
|
18800
|
+
if (target === null) continue;
|
|
18801
|
+
if (request.deviceId !== null && target.deviceId !== request.deviceId) continue;
|
|
18802
|
+
candidates.push({
|
|
18803
|
+
proc,
|
|
18804
|
+
target
|
|
18805
|
+
});
|
|
18806
|
+
}
|
|
18807
|
+
const outcome = candidates.length === 0 ? {
|
|
18808
|
+
reaped: [],
|
|
18809
|
+
refused: []
|
|
18810
|
+
} : await this.reap(candidates);
|
|
18811
|
+
if (request.deviceId === null) this.deps.logger.info("recorder: boot sweep for a previous runner’s writers", { meta: {
|
|
18812
|
+
roots: request.roots,
|
|
18813
|
+
found: candidates.length,
|
|
18814
|
+
reaped: outcome.reaped.length,
|
|
18815
|
+
refused: outcome.refused.length,
|
|
18816
|
+
ms: this.deps.now() - startedMs
|
|
18817
|
+
} });
|
|
18818
|
+
return {
|
|
18819
|
+
outcome: "swept",
|
|
18820
|
+
reaped: outcome.reaped,
|
|
18821
|
+
refused: outcome.refused
|
|
18822
|
+
};
|
|
18823
|
+
}
|
|
18824
|
+
isForeignWriter(proc, ownPids) {
|
|
18825
|
+
return proc.pid !== this.deps.selfPid && !ownPids.has(proc.pid) && isWriterExecutable(proc, this.deps.executableMarker ?? "ffmpeg");
|
|
18826
|
+
}
|
|
18827
|
+
/**
|
|
18828
|
+
* One SIGTERM → SIGKILL window for every candidate at once. Candidates whose
|
|
18829
|
+
* directory is held are refused without a signal.
|
|
18830
|
+
*/
|
|
18831
|
+
async reap(candidates) {
|
|
18832
|
+
const started = this.deps.now();
|
|
18833
|
+
const refused = [];
|
|
18834
|
+
const active = [];
|
|
18835
|
+
for (const candidate of candidates) {
|
|
18836
|
+
const held = this.heldRecord(candidate.target.outDir);
|
|
18837
|
+
if (held === null) {
|
|
18838
|
+
active.push(candidate);
|
|
18839
|
+
continue;
|
|
18840
|
+
}
|
|
18841
|
+
const survivor = this.refusedOf(candidate, "held");
|
|
18842
|
+
this.logHeld(survivor, held);
|
|
18843
|
+
refused.push(survivor);
|
|
18844
|
+
}
|
|
18845
|
+
const terminated = this.signalAll(active, "SIGTERM", refused);
|
|
18846
|
+
let signalled = terminated;
|
|
18847
|
+
const killed = /* @__PURE__ */ new Set();
|
|
18848
|
+
if (terminated.length > 0) {
|
|
18849
|
+
const afterTerm = await this.waitGone(terminated, this.deps.termGraceMs ?? 2e3);
|
|
18850
|
+
if (afterTerm === null) {
|
|
18851
|
+
for (const candidate of terminated) refused.push(this.refusedOf(candidate, "table-unreadable"));
|
|
18852
|
+
signalled = [];
|
|
18853
|
+
} else if (afterTerm.length > 0) {
|
|
18854
|
+
const toKill = this.signalAll(afterTerm, "SIGKILL", refused);
|
|
18855
|
+
for (const candidate of toKill) killed.add(candidate.proc.pid);
|
|
18856
|
+
if (toKill.length > 0) {
|
|
18857
|
+
const afterKill = await this.waitGone(toKill, this.deps.killGraceMs ?? 1e3);
|
|
18858
|
+
const reason = afterKill === null ? "table-unreadable" : "survived";
|
|
18859
|
+
for (const candidate of afterKill ?? toKill) refused.push(this.refusedOf(candidate, reason));
|
|
18860
|
+
}
|
|
18861
|
+
}
|
|
18862
|
+
}
|
|
18863
|
+
const waitedMs = this.deps.now() - started;
|
|
18864
|
+
const refusedPids = new Set(refused.map((writer) => writer.pid));
|
|
18865
|
+
const reaped = [];
|
|
18866
|
+
for (const candidate of signalled) {
|
|
18867
|
+
if (refusedPids.has(candidate.proc.pid)) continue;
|
|
18868
|
+
const writer = {
|
|
18869
|
+
target: candidate.target,
|
|
18870
|
+
pid: candidate.proc.pid,
|
|
18871
|
+
ppid: candidate.proc.ppid,
|
|
18872
|
+
ageMs: candidate.proc.ageMs,
|
|
18873
|
+
signal: killed.has(candidate.proc.pid) ? "SIGKILL" : "SIGTERM",
|
|
18874
|
+
waitedMs
|
|
18875
|
+
};
|
|
18876
|
+
reaped.push(writer);
|
|
18877
|
+
this.logReaped(writer);
|
|
18878
|
+
}
|
|
18879
|
+
for (const writer of refused) if (writer.reason !== "held") this.recordRefusal(writer, waitedMs);
|
|
18880
|
+
return {
|
|
18881
|
+
reaped,
|
|
18882
|
+
refused
|
|
18883
|
+
};
|
|
18884
|
+
}
|
|
18885
|
+
/**
|
|
18886
|
+
* Send `signal` to each candidate. A candidate the kernel refuses (EPERM) is
|
|
18887
|
+
* refused at once — waiting out a grace for a signal that was never
|
|
18888
|
+
* delivered cannot change anything. Returns the ones worth waiting for.
|
|
18889
|
+
*/
|
|
18890
|
+
signalAll(candidates, signal, refused) {
|
|
18891
|
+
if (candidates.length === 0) return [];
|
|
18892
|
+
const pending = [];
|
|
18893
|
+
for (const candidate of candidates) if (this.deps.signaller.send(candidate.proc.pid, signal) === "denied") refused.push(this.refusedOf(candidate, "denied"));
|
|
18894
|
+
else pending.push(candidate);
|
|
18895
|
+
this.cache = null;
|
|
18896
|
+
return pending;
|
|
18897
|
+
}
|
|
18898
|
+
/**
|
|
18899
|
+
* Poll until none of `targets` is still writing its playlist, or the grace
|
|
18900
|
+
* runs out. Returns who is left, or `null` when the table could not be read
|
|
18901
|
+
* twice in a row. A zombie or a recycled pid is "gone" by this test, because
|
|
18902
|
+
* it is the ARGV that makes a writer, not the pid.
|
|
18903
|
+
*/
|
|
18904
|
+
async waitGone(targets, graceMs) {
|
|
18905
|
+
const deadline = this.deps.now() + graceMs;
|
|
18906
|
+
for (;;) {
|
|
18907
|
+
await this.deps.sleep(this.deps.pollMs ?? POLL_MS);
|
|
18908
|
+
const rows = await this.freshRowsOrNull();
|
|
18909
|
+
if (rows === null) return null;
|
|
18910
|
+
const left = targets.filter((candidate) => rows.some((proc) => proc.pid === candidate.proc.pid && isWriterExecutable(proc, this.deps.executableMarker ?? "ffmpeg") && writesPlaylist(proc, segmentListPath(candidate.target.outDir))));
|
|
18911
|
+
if (left.length === 0 || this.deps.now() >= deadline) return left;
|
|
18912
|
+
}
|
|
18913
|
+
}
|
|
18914
|
+
/** One fresh read, and ONE retry after a poll interval. */
|
|
18915
|
+
async freshRowsOrNull() {
|
|
18916
|
+
for (let attempt = 0; attempt < 2; attempt++) try {
|
|
18917
|
+
return await this.list(this.deps.now());
|
|
18918
|
+
} catch {
|
|
18919
|
+
if (attempt === 0) await this.deps.sleep(this.deps.pollMs ?? POLL_MS);
|
|
18920
|
+
}
|
|
18921
|
+
return null;
|
|
18922
|
+
}
|
|
18923
|
+
/**
|
|
18924
|
+
* The process table, read at or after `notBeforeMs`. Concurrent callers
|
|
18925
|
+
* share one read in flight, so eight attaches at boot cost one `ps`, not
|
|
18926
|
+
* eight.
|
|
18927
|
+
*/
|
|
18928
|
+
async list(notBeforeMs) {
|
|
18929
|
+
if (this.cache !== null && this.cache.at >= notBeforeMs) return this.cache.rows;
|
|
18930
|
+
if (this.inflight !== null && this.inflight.startedAt >= notBeforeMs) return (await this.inflight.promise).rows;
|
|
18931
|
+
const startedAt = this.deps.now();
|
|
18932
|
+
const promise = this.deps.lister.list(SEGMENT_LIST_FLAG).then((rows) => ({
|
|
18933
|
+
at: startedAt,
|
|
18934
|
+
rows
|
|
18935
|
+
}));
|
|
18936
|
+
const entry = {
|
|
18937
|
+
startedAt,
|
|
18938
|
+
promise
|
|
18939
|
+
};
|
|
18940
|
+
this.inflight = entry;
|
|
18941
|
+
try {
|
|
18942
|
+
const listing = await promise;
|
|
18943
|
+
if (this.cache === null || this.cache.at <= listing.at) this.cache = listing;
|
|
18944
|
+
return listing.rows;
|
|
18945
|
+
} finally {
|
|
18946
|
+
if (this.inflight === entry) this.inflight = null;
|
|
18947
|
+
}
|
|
18948
|
+
}
|
|
18949
|
+
refusedOf(candidate, reason) {
|
|
18950
|
+
return {
|
|
18951
|
+
target: candidate.target,
|
|
18952
|
+
pid: candidate.proc.pid,
|
|
18953
|
+
ppid: candidate.proc.ppid,
|
|
18954
|
+
ageMs: candidate.proc.ageMs,
|
|
18955
|
+
reason
|
|
18956
|
+
};
|
|
18957
|
+
}
|
|
18958
|
+
heldRecord(outDir) {
|
|
18959
|
+
const record = this.refusals.get(outDir);
|
|
18960
|
+
if (record === void 0) return null;
|
|
18961
|
+
const retryMs = this.deps.refusalRetryMs ?? 6e4;
|
|
18962
|
+
return this.deps.now() - record.lastAttemptMs < retryMs ? record : null;
|
|
18963
|
+
}
|
|
18964
|
+
startedAtOf(ageMs) {
|
|
18965
|
+
return ageMs === null ? null : this.deps.now() - ageMs;
|
|
18966
|
+
}
|
|
18967
|
+
/** ERROR on a new refusal state, DEBUG on a repeat of the same one. */
|
|
18968
|
+
recordRefusal(writer, waitedMs) {
|
|
18969
|
+
const now = this.deps.now();
|
|
18970
|
+
const startedAtMs = this.startedAtOf(writer.ageMs);
|
|
18971
|
+
const previous = this.refusals.get(writer.target.outDir);
|
|
18972
|
+
const same = previous !== void 0 && previous.pid === writer.pid && previous.reason === writer.reason && (previous.startedAtMs === null || startedAtMs === null || Math.abs(previous.startedAtMs - startedAtMs) <= SAME_START_TOLERANCE_MS);
|
|
18973
|
+
const attempts = same ? previous.attempts + 1 : 1;
|
|
18974
|
+
this.refusals = new Map(this.refusals).set(writer.target.outDir, {
|
|
18975
|
+
pid: writer.pid,
|
|
18976
|
+
startedAtMs,
|
|
18977
|
+
reason: writer.reason,
|
|
18978
|
+
firstMs: same ? previous.firstMs : now,
|
|
18979
|
+
lastAttemptMs: now,
|
|
18980
|
+
attempts
|
|
18981
|
+
});
|
|
18982
|
+
const extras = {
|
|
18983
|
+
tags: { deviceId: writer.target.deviceId },
|
|
18984
|
+
meta: {
|
|
18985
|
+
profile: writer.target.profile,
|
|
18986
|
+
pid: writer.pid,
|
|
18987
|
+
ppid: writer.ppid,
|
|
18988
|
+
ageSec: ageSec(writer.ageMs),
|
|
18989
|
+
reason: writer.reason,
|
|
18990
|
+
attempts,
|
|
18991
|
+
waitedMs,
|
|
18992
|
+
retryInMs: this.deps.refusalRetryMs ?? 6e4,
|
|
18993
|
+
outDir: writer.target.outDir
|
|
18994
|
+
}
|
|
18995
|
+
};
|
|
18996
|
+
if (same) {
|
|
18997
|
+
this.deps.logger.debug("recorder: staging directory still refused — same foreign writer", extras);
|
|
18998
|
+
return;
|
|
18999
|
+
}
|
|
19000
|
+
this.deps.logger.error("recorder: REFUSED to spawn a writer — another writer on this staging directory could not be removed", extras);
|
|
19001
|
+
}
|
|
19002
|
+
logHeld(writer, record) {
|
|
19003
|
+
this.deps.logger.debug("recorder: staging directory held after a refusal — not retried yet", {
|
|
19004
|
+
tags: { deviceId: writer.target.deviceId },
|
|
19005
|
+
meta: {
|
|
19006
|
+
profile: writer.target.profile,
|
|
19007
|
+
pid: record.pid,
|
|
19008
|
+
reason: record.reason,
|
|
19009
|
+
attempts: record.attempts,
|
|
19010
|
+
heldForMs: this.deps.now() - record.firstMs,
|
|
19011
|
+
outDir: writer.target.outDir
|
|
19012
|
+
}
|
|
19013
|
+
});
|
|
19014
|
+
}
|
|
19015
|
+
forgetRefusal(target) {
|
|
19016
|
+
const record = this.refusals.get(target.outDir);
|
|
19017
|
+
if (record === void 0) return;
|
|
19018
|
+
const next = new Map(this.refusals);
|
|
19019
|
+
next.delete(target.outDir);
|
|
19020
|
+
this.refusals = next;
|
|
19021
|
+
this.deps.logger.info("recorder: staging directory no longer refused — the foreign writer is gone", {
|
|
19022
|
+
tags: { deviceId: target.deviceId },
|
|
19023
|
+
meta: {
|
|
19024
|
+
profile: target.profile,
|
|
19025
|
+
pid: record.pid,
|
|
19026
|
+
attempts: record.attempts,
|
|
19027
|
+
refusedForMs: this.deps.now() - record.firstMs,
|
|
19028
|
+
outDir: target.outDir
|
|
19029
|
+
}
|
|
19030
|
+
});
|
|
19031
|
+
}
|
|
19032
|
+
logReaped(writer) {
|
|
19033
|
+
if (this.refusals.has(writer.target.outDir)) this.forgetRefusal(writer.target);
|
|
19034
|
+
this.deps.logger.warn("recorder: reaped a predecessor writer on this staging directory — one writer per directory", {
|
|
19035
|
+
tags: { deviceId: writer.target.deviceId },
|
|
19036
|
+
meta: {
|
|
19037
|
+
profile: writer.target.profile,
|
|
19038
|
+
pid: writer.pid,
|
|
19039
|
+
ppid: writer.ppid,
|
|
19040
|
+
ageSec: ageSec(writer.ageMs),
|
|
19041
|
+
signal: writer.signal,
|
|
19042
|
+
waitedMs: writer.waitedMs,
|
|
19043
|
+
outDir: writer.target.outDir
|
|
19044
|
+
}
|
|
19045
|
+
});
|
|
19046
|
+
}
|
|
19047
|
+
};
|
|
19048
|
+
//#endregion
|
|
19049
|
+
//#region src/recorder/addon/staging-boot.ts
|
|
19050
|
+
async function runStagingBoot(steps) {
|
|
19051
|
+
const stagingIsQuiescent = await steps.reapPredecessorWriters();
|
|
19052
|
+
const bootMs = steps.now();
|
|
19053
|
+
await steps.startWritePath();
|
|
19054
|
+
if (!stagingIsQuiescent) return;
|
|
19055
|
+
(async () => {
|
|
19056
|
+
await steps.recoverStagedOrphans(bootMs);
|
|
19057
|
+
await steps.sweepUnassignedStaging();
|
|
19058
|
+
})();
|
|
19059
|
+
}
|
|
19060
|
+
//#endregion
|
|
17916
19061
|
//#region src/recorder/loop-pacer.ts
|
|
17917
19062
|
var DEFAULT_BUSY_LAG_MS = 25;
|
|
17918
19063
|
var DEFAULT_BACKOFF_STEP_MS = 25;
|
|
@@ -19134,6 +20279,13 @@ var MFRA_GATE_LIMIT = 64;
|
|
|
19134
20279
|
var EXPORTS_PREFIX = "exports";
|
|
19135
20280
|
/** How often the export janitor sweeps expired exports. */
|
|
19136
20281
|
var EXPORT_SWEEP_INTERVAL_MS = 5 * 6e4;
|
|
20282
|
+
/**
|
|
20283
|
+
* The staging plan keeps anything named within 1 s before its cutoff for the
|
|
20284
|
+
* live writer (`PREVIOUS_RUN_SKEW_MS`). A reaped directory has no live writer
|
|
20285
|
+
* yet, so the cutoff is moved forward by exactly that much: everything the
|
|
20286
|
+
* reaped writer named up to the reap is collected.
|
|
20287
|
+
*/
|
|
20288
|
+
var REAPED_RECOVERY_SKEW_MS = 1e3;
|
|
19137
20289
|
/** How often the retention sweep prunes footage past each camera's per-device
|
|
19138
20290
|
* age/size policy. Retention is NOT time-critical (footage ages out over days,
|
|
19139
20291
|
* and the disk-pressure eviction path already bounds the worst case), so an
|
|
@@ -19335,6 +20487,17 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
19335
20487
|
* behind scrub's `locateSegment` and expired its 8 s budget (2026-08-22).
|
|
19336
20488
|
*/
|
|
19337
20489
|
calendarInstance = null;
|
|
20490
|
+
heapProbeInstance = null;
|
|
20491
|
+
/**
|
|
20492
|
+
* Names the pass, boot phase or inbound cap method that grows the heap.
|
|
20493
|
+
* Every recorder lifetime builds ~2.3 GB in one silent phase 3–5 min after
|
|
20494
|
+
* boot and nothing has ever said which unit of work does it (see
|
|
20495
|
+
* `heap-delta.ts`). LAZY for the same reason as {@link calendar}.
|
|
20496
|
+
*/
|
|
20497
|
+
get heapProbe() {
|
|
20498
|
+
this.heapProbeInstance ??= new HeapDeltaProbe({ logger: this.ctx.logger });
|
|
20499
|
+
return this.heapProbeInstance;
|
|
20500
|
+
}
|
|
19338
20501
|
/** LAZY: a field initialiser runs before `initialize()` assigns `ctx`, and
|
|
19339
20502
|
* reading `this.ctx` there throws — the same trap that bit the playback tier. */
|
|
19340
20503
|
get calendar() {
|
|
@@ -19453,6 +20616,9 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
19453
20616
|
/** The `recording` cap provider — retained so the periodic retention sweep can
|
|
19454
20617
|
* call its `pruneFootage`. Null until built in `onInitialize`. */
|
|
19455
20618
|
recordingProvider = null;
|
|
20619
|
+
/** One writer per staging directory (D667). `null` where the platform has
|
|
20620
|
+
* no process table (Windows) — said once at startup. */
|
|
20621
|
+
stagingGuard = null;
|
|
19456
20622
|
/** Periodic footage-retention sweep (owns its timer + single-flight guard).
|
|
19457
20623
|
* Constructed + started in `onHubReachable`, stopped on shutdown. */
|
|
19458
20624
|
retentionSweeper = null;
|
|
@@ -19636,7 +20802,20 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
19636
20802
|
usedOnLocation: async (locationId) => (await this.segmentHours?.accountingForLocations(new Set([locationId])))?.bytes ?? 0,
|
|
19637
20803
|
dailyBytesFor: (deviceId, profile) => this.dailyBytesFor(deviceId, profile)
|
|
19638
20804
|
});
|
|
20805
|
+
const processLister = createProcessLister();
|
|
20806
|
+
if (processLister === null) this.ctx.logger.warn("recorder: no process table on this platform — orphaned writers cannot be detected; boot staging mutation is disabled", { meta: { platform: process.platform } });
|
|
20807
|
+
this.stagingGuard = processLister === null ? null : new StagingWriterReaper({
|
|
20808
|
+
lister: processLister,
|
|
20809
|
+
signaller: nodeProcessSignaller,
|
|
20810
|
+
logger: this.ctx.logger,
|
|
20811
|
+
selfPid: process.pid,
|
|
20812
|
+
sleep: (ms) => require_dist.sleep(ms),
|
|
20813
|
+
now: () => Date.now()
|
|
20814
|
+
});
|
|
19639
20815
|
this.controller = new RecordingController({
|
|
20816
|
+
...this.stagingGuard !== null ? { stagingGuard: this.stagingGuard } : {},
|
|
20817
|
+
recoverStagingDirectory: (input) => this.recoverReapedDirectory(input),
|
|
20818
|
+
heapProbe: this.heapProbe,
|
|
19640
20819
|
api: this.ctx.api,
|
|
19641
20820
|
logger: this.ctx.logger,
|
|
19642
20821
|
nodeId: this.nodeId,
|
|
@@ -20106,30 +21285,30 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20106
21285
|
const providers = [
|
|
20107
21286
|
{
|
|
20108
21287
|
capability: require_dist.recordingArchiveCapability,
|
|
20109
|
-
provider: recordingProvider
|
|
21288
|
+
provider: this.heapProbe.wrapProvider(require_dist.recordingArchiveCapability.name, recordingProvider)
|
|
20110
21289
|
},
|
|
20111
21290
|
{
|
|
20112
21291
|
capability: require_dist.recordingCapability,
|
|
20113
|
-
provider: recordingSourceProvider
|
|
21292
|
+
provider: this.heapProbe.wrapProvider(require_dist.recordingCapability.name, recordingSourceProvider)
|
|
20114
21293
|
},
|
|
20115
21294
|
{
|
|
20116
21295
|
capability: require_dist.videoclipsCapability,
|
|
20117
|
-
provider: createRecorderVideoclipsProvider({
|
|
21296
|
+
provider: this.heapProbe.wrapProvider(require_dist.videoclipsCapability.name, createRecorderVideoclipsProvider({
|
|
20118
21297
|
api: this.ctx.api,
|
|
20119
21298
|
logger: this.ctx.logger
|
|
20120
|
-
})
|
|
21299
|
+
}))
|
|
20121
21300
|
},
|
|
20122
21301
|
{
|
|
20123
21302
|
capability: require_dist.storageEvictableCapability,
|
|
20124
|
-
provider: evictableProvider
|
|
21303
|
+
provider: this.heapProbe.wrapProvider(require_dist.storageEvictableCapability.name, evictableProvider)
|
|
20125
21304
|
},
|
|
20126
21305
|
{
|
|
20127
21306
|
capability: require_dist.storageOccupancyCapability,
|
|
20128
|
-
provider: evictableProvider
|
|
21307
|
+
provider: this.heapProbe.wrapProvider(require_dist.storageOccupancyCapability.name, evictableProvider)
|
|
20129
21308
|
},
|
|
20130
21309
|
{
|
|
20131
21310
|
capability: require_dist.recordingExportCapability,
|
|
20132
|
-
provider: exportProvider
|
|
21311
|
+
provider: this.heapProbe.wrapProvider(require_dist.recordingExportCapability.name, exportProvider)
|
|
20133
21312
|
},
|
|
20134
21313
|
{
|
|
20135
21314
|
capability: require_dist.loadContributionCapability,
|
|
@@ -20289,14 +21468,28 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20289
21468
|
recursive: true,
|
|
20290
21469
|
force: true
|
|
20291
21470
|
}).catch(() => {});
|
|
20292
|
-
await
|
|
20293
|
-
|
|
20294
|
-
|
|
20295
|
-
|
|
20296
|
-
|
|
20297
|
-
|
|
20298
|
-
|
|
20299
|
-
|
|
21471
|
+
await runStagingBoot({
|
|
21472
|
+
reapPredecessorWriters: () => this.heapProbe.track({
|
|
21473
|
+
kind: "boot-phase",
|
|
21474
|
+
name: "staging-writer-sweep"
|
|
21475
|
+
}, () => this.reapPredecessorWriters()),
|
|
21476
|
+
sweepUnassignedStaging: () => this.heapProbe.track({
|
|
21477
|
+
kind: "boot-phase",
|
|
21478
|
+
name: "staging-unassigned-sweep"
|
|
21479
|
+
}, () => this.sweepUnassignedStaging()),
|
|
21480
|
+
startWritePath: async () => {
|
|
21481
|
+
try {
|
|
21482
|
+
await this.controller?.start();
|
|
21483
|
+
} catch (err) {
|
|
21484
|
+
this.ctx.logger.warn("recorder: controller start failed", { meta: { error: require_dist.errMsg(err) } });
|
|
21485
|
+
}
|
|
21486
|
+
},
|
|
21487
|
+
recoverStagedOrphans: (bootMs) => this.heapProbe.track({
|
|
21488
|
+
kind: "boot-phase",
|
|
21489
|
+
name: "staging-recovery"
|
|
21490
|
+
}, () => this.recoverStagedOrphans(bootMs)),
|
|
21491
|
+
now: () => Date.now()
|
|
21492
|
+
});
|
|
20300
21493
|
if (this.locationRefresher === null) this.locationRefresher = new LocationRefresher({
|
|
20301
21494
|
logger: this.ctx.logger,
|
|
20302
21495
|
refresh: () => this.refreshLocations()
|
|
@@ -20376,7 +21569,12 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20376
21569
|
this.controller?.onSignalLevel(parsed.deviceId, { kind: "signal" }, level.data.active, atMs);
|
|
20377
21570
|
});
|
|
20378
21571
|
try {
|
|
20379
|
-
await this.
|
|
21572
|
+
await this.heapProbe.track({
|
|
21573
|
+
kind: "boot-phase",
|
|
21574
|
+
name: "ledger-load"
|
|
21575
|
+
}, async () => {
|
|
21576
|
+
await this.segmentHours?.load();
|
|
21577
|
+
});
|
|
20380
21578
|
} catch (err) {
|
|
20381
21579
|
this.ctx.logger.warn("recorder: segment-hour ledger load failed — hours stay unknown until a read walks them", { meta: { error: require_dist.errMsg(err) } });
|
|
20382
21580
|
}
|
|
@@ -20388,20 +21586,27 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20388
21586
|
return;
|
|
20389
21587
|
}
|
|
20390
21588
|
this.applySensorTriggerBindings(configs);
|
|
20391
|
-
this.
|
|
21589
|
+
this.heapProbe.track({
|
|
21590
|
+
kind: "boot-phase",
|
|
21591
|
+
name: "warm-today"
|
|
21592
|
+
}, () => this.warmToday([...configs.keys()]));
|
|
20392
21593
|
this.ctx.logger.info("recorder: full-archive index walk skipped", { meta: {
|
|
20393
21594
|
reason: "on-demand-windows",
|
|
20394
21595
|
ledgerHours: this.segmentHours?.snapshot().length ?? 0,
|
|
20395
21596
|
deviceCount: configs.size
|
|
20396
21597
|
} });
|
|
20397
|
-
this.
|
|
21598
|
+
this.heapProbe.track({
|
|
21599
|
+
kind: "boot-phase",
|
|
21600
|
+
name: "legacy-thumbs-cleanup"
|
|
21601
|
+
}, () => this.cleanupLegacyThumbsDirs());
|
|
20398
21602
|
await this.bootstrapExports();
|
|
20399
21603
|
const provider = this.recordingProvider;
|
|
20400
21604
|
if (provider && this.retentionSweeper === null) this.retentionSweeper = new RetentionSweeper({
|
|
20401
21605
|
prune: (deviceId) => provider.pruneFootage({ deviceId }),
|
|
20402
21606
|
readConfigs: () => readDeviceConfigs(this.configStore()),
|
|
20403
21607
|
logger: this.ctx.logger,
|
|
20404
|
-
intervalMs: RETENTION_SWEEP_INTERVAL_MS
|
|
21608
|
+
intervalMs: RETENTION_SWEEP_INTERVAL_MS,
|
|
21609
|
+
heapProbe: this.heapProbe
|
|
20405
21610
|
});
|
|
20406
21611
|
this.retentionSweeper?.start();
|
|
20407
21612
|
const store = this.segmentStore;
|
|
@@ -20411,7 +21616,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20411
21616
|
evict: (locationId, rows) => store.evict(locationId, rows),
|
|
20412
21617
|
logger: this.ctx.logger,
|
|
20413
21618
|
intervalMs: REDUNDANCY_SWEEP_INTERVAL_MS,
|
|
20414
|
-
lookbackMs: REDUNDANCY_SWEEP_LOOKBACK_MS
|
|
21619
|
+
lookbackMs: REDUNDANCY_SWEEP_LOOKBACK_MS,
|
|
21620
|
+
heapProbe: this.heapProbe
|
|
20415
21621
|
});
|
|
20416
21622
|
this.redundancySweeper?.start();
|
|
20417
21623
|
const ledger = this.segmentHours;
|
|
@@ -20436,6 +21642,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20436
21642
|
name: "ghost-location-reconcile",
|
|
20437
21643
|
intervalMs: GHOST_LOCATION_SWEEP_INTERVAL_MS,
|
|
20438
21644
|
slowAfterMs: GHOST_LOCATION_SWEEP_SLOW_AFTER_MS,
|
|
21645
|
+
heapProbe: this.heapProbe,
|
|
20439
21646
|
run: async () => {
|
|
20440
21647
|
await sweeper.sweep();
|
|
20441
21648
|
},
|
|
@@ -20463,6 +21670,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20463
21670
|
intervalMs: HOUR_ADOPTION_INTERVAL_MS,
|
|
20464
21671
|
firstRunDelayMs: HOUR_ADOPTION_FIRST_RUN_DELAY_MS,
|
|
20465
21672
|
slowAfterMs: HOUR_ADOPTION_SLOW_AFTER_MS,
|
|
21673
|
+
heapProbe: this.heapProbe,
|
|
20466
21674
|
run: async () => {
|
|
20467
21675
|
if (await runAdoptionCatchUp({
|
|
20468
21676
|
locationIds: this.resolvedLocations.map((location) => location.id),
|
|
@@ -20591,7 +21799,10 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20591
21799
|
if (this.exportSweepTimer === null) {
|
|
20592
21800
|
const sweep = () => {
|
|
20593
21801
|
this.logIndexCensus();
|
|
20594
|
-
|
|
21802
|
+
this.heapProbe.track({
|
|
21803
|
+
kind: "pass",
|
|
21804
|
+
name: "export-expiry"
|
|
21805
|
+
}, () => sweepExpiredExports(Date.now(), this.exportJanitorDeps())).catch((err) => {
|
|
20595
21806
|
this.ctx.logger.warn("recorder: export expiry sweep failed", { meta: { error: require_dist.errMsg(err) } });
|
|
20596
21807
|
});
|
|
20597
21808
|
};
|
|
@@ -20638,7 +21849,13 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20638
21849
|
incrementalInserts: views.incrementalInserts,
|
|
20639
21850
|
rowsReused: reuse.rowsReused,
|
|
20640
21851
|
rowsAllocated: reuse.rowsAllocated,
|
|
20641
|
-
ledgerHours: this.segmentHours?.snapshot().length ?? 0
|
|
21852
|
+
ledgerHours: this.segmentHours?.snapshot().length ?? 0,
|
|
21853
|
+
calendar: this.calendarInstance?.memoryCensus() ?? null,
|
|
21854
|
+
mfra: this.mfraTables.memoryCensus(),
|
|
21855
|
+
ledger: this.segmentHours?.memoryCensus() ?? null,
|
|
21856
|
+
placement: this.placement?.memoryCensus() ?? null,
|
|
21857
|
+
exports: this.exportEngine?.memoryCensus() ?? null,
|
|
21858
|
+
controller: this.controller?.memoryCensus() ?? null
|
|
20642
21859
|
}
|
|
20643
21860
|
});
|
|
20644
21861
|
}
|
|
@@ -20950,6 +22167,66 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20950
22167
|
}
|
|
20951
22168
|
}
|
|
20952
22169
|
/**
|
|
22170
|
+
* Boot sweep for a previous runner's writers (D667): every foreign ffmpeg
|
|
22171
|
+
* writing under any location's `.rec-tmp`, whether or not its camera will be
|
|
22172
|
+
* attached again. On re-entry (`onHubReachable` again) the controller's own
|
|
22173
|
+
* writers are live and excluded by pid.
|
|
22174
|
+
*/
|
|
22175
|
+
async reapPredecessorWriters() {
|
|
22176
|
+
const guard = this.stagingGuard;
|
|
22177
|
+
if (guard === null) return false;
|
|
22178
|
+
const result = await guard.sweep({
|
|
22179
|
+
roots: this.resolvedLocations.map((location) => location.root),
|
|
22180
|
+
deviceId: null,
|
|
22181
|
+
ownPids: this.controller?.ownedWriterPids() ?? /* @__PURE__ */ new Set()
|
|
22182
|
+
});
|
|
22183
|
+
if (result.outcome === "unverified") {
|
|
22184
|
+
this.ctx.logger.warn("recorder: staging mutation skipped at boot — the predecessor-writer sweep was unverified", { meta: { roots: this.resolvedLocations.map((location) => location.root) } });
|
|
22185
|
+
return false;
|
|
22186
|
+
}
|
|
22187
|
+
if (result.refused.length === 0) return true;
|
|
22188
|
+
this.ctx.logger.warn("recorder: staging mutation skipped at boot — a known predecessor writer remains", { meta: {
|
|
22189
|
+
refused: result.refused.length,
|
|
22190
|
+
roots: this.resolvedLocations.map((location) => location.root)
|
|
22191
|
+
} });
|
|
22192
|
+
return false;
|
|
22193
|
+
}
|
|
22194
|
+
/**
|
|
22195
|
+
* Collect what a reaped writer left in one staging directory, before the
|
|
22196
|
+
* controller spawns a new writer there (D667). The reaped writer's playlist
|
|
22197
|
+
* is NOT consulted: it names that writer's last entries, and nobody is left
|
|
22198
|
+
* to finalize them.
|
|
22199
|
+
*/
|
|
22200
|
+
async recoverReapedDirectory(input) {
|
|
22201
|
+
const report = await recoverStagingOrphans(input.dir, {
|
|
22202
|
+
writerStartedMs: input.cutoffMs + REAPED_RECOVERY_SKEW_MS,
|
|
22203
|
+
segmentSeconds: input.segmentSeconds
|
|
22204
|
+
}, {
|
|
22205
|
+
listDir: nodeStagingIo.listDir,
|
|
22206
|
+
readPlaylist: async () => null,
|
|
22207
|
+
finalize: async (orphan, absPath) => {
|
|
22208
|
+
const store = this.segmentStore;
|
|
22209
|
+
if (store === null) throw new Error("recorder is shutting down");
|
|
22210
|
+
await store.onFinalized({
|
|
22211
|
+
deviceId: input.deviceId,
|
|
22212
|
+
profile: input.profile,
|
|
22213
|
+
locationId: input.locationId,
|
|
22214
|
+
locationRoot: input.locationRoot,
|
|
22215
|
+
flatAbsPath: absPath,
|
|
22216
|
+
startMs: orphan.startMs,
|
|
22217
|
+
durMs: orphan.durMs
|
|
22218
|
+
});
|
|
22219
|
+
},
|
|
22220
|
+
logger: this.ctx.logger.withTags({ deviceId: input.deviceId }),
|
|
22221
|
+
shouldStop: () => this.segmentStore === null
|
|
22222
|
+
});
|
|
22223
|
+
return {
|
|
22224
|
+
found: report.found,
|
|
22225
|
+
recovered: report.recovered,
|
|
22226
|
+
failed: report.failed
|
|
22227
|
+
};
|
|
22228
|
+
}
|
|
22229
|
+
/**
|
|
20953
22230
|
* Boot sweep: remove the `.rec-tmp` staging tree from every recordings root
|
|
20954
22231
|
* NOTHING is assigned to any more. Staging lives under the placement root, so
|
|
20955
22232
|
* a root that has just been drained, opted out or deleted from the plan keeps
|
|
@@ -20957,8 +22234,10 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
20957
22234
|
* no retention pass will ever see (`.rec-tmp` is invisible to the hydrate
|
|
20958
22235
|
* walk by design). Same shape as the `.scrub-packs` sweep above.
|
|
20959
22236
|
*
|
|
20960
|
-
* Only roots with ZERO assignments are touched, and only
|
|
20961
|
-
*
|
|
22237
|
+
* Only roots with ZERO assignments are touched, and only on the boot chain,
|
|
22238
|
+
* after orphan recovery. The placement read happens here, after the
|
|
22239
|
+
* controller has attached, so any root a new writer uses is named by an
|
|
22240
|
+
* assignment and cannot be swept.
|
|
20962
22241
|
*/
|
|
20963
22242
|
async sweepUnassignedStaging() {
|
|
20964
22243
|
let assigned;
|