@camstack/addon-pipeline 1.2.94 → 1.2.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-DnUCCZVx.js → addon-utils-B6usm-YO.js} +1 -1
- package/dist/audio-analyzer/index.js +58 -4
- package/dist/audio-analyzer/index.mjs +56 -2
- package/dist/detection-pipeline/index.js +194 -6
- package/dist/detection-pipeline/index.mjs +191 -3
- package/dist/{dist-XbrYiMV3.mjs → dist-LIWfUNC_.mjs} +324 -2
- package/dist/{dist-BbRv3bM2.js → dist-w9QWDvWn.js} +341 -1
- package/dist/{event-loop-stall-monitor-CFWrOZ6G.mjs → event-loop-stall-monitor-BDC8qURn.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-D9hqbc68.js → event-loop-stall-monitor-DndI0S2s.js} +1 -1
- package/dist/{lazy-sharp-U0EtN7_C.js → lazy-sharp-BVqBydNU.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +4 -4
- package/dist/pipeline-runner/index.mjs +3 -3
- package/dist/process-memory-C6zWIgos.js +78 -0
- package/dist/process-memory-k45KJaKH.mjs +66 -0
- package/dist/recorder/index.js +423 -47
- package/dist/recorder/index.mjs +422 -46
- package/dist/session-decode/decode-worker-child.js +2 -2
- package/dist/session-decode/decode-worker-child.mjs +1 -1
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BNOPhQ-y.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CJWMZMhQ.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-D2WOTeYi.mjs +26 -0
- package/dist/stream-broker/{hostInit-DCV2-mSi.mjs → hostInit-CGIo3d4K.mjs} +3 -3
- package/dist/stream-broker/index.js +25 -7
- package/dist/stream-broker/index.mjs +25 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-D0ewr67U.mjs → worker-protocol-DTe7Ntat.mjs} +1 -1
- package/dist/{worker-protocol-CNC-ZcU3.js → worker-protocol-DqrXmX0g.js} +1 -1
- package/package.json +12 -7
- package/python/inference_pool.py +178 -3
- package/python/test_inference_pool_backpressure.py +7 -1
- package/python/test_inference_pool_memstats.py +146 -0
- package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
- package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DMyOrQHq.mjs +0 -26
package/dist/recorder/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const require_dist = require("../dist-
|
|
1
|
+
const require_dist = require("../dist-w9QWDvWn.js");
|
|
2
2
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
3
|
-
const require_addon_utils = require("../addon-utils-
|
|
3
|
+
const require_addon_utils = require("../addon-utils-B6usm-YO.js");
|
|
4
4
|
let node_crypto = require("node:crypto");
|
|
5
5
|
let node_child_process = require("node:child_process");
|
|
6
6
|
let node_path = require("node:path");
|
|
@@ -457,6 +457,51 @@ var MfraTableStore = class {
|
|
|
457
457
|
}
|
|
458
458
|
}
|
|
459
459
|
};
|
|
460
|
+
var PendingGate = class {
|
|
461
|
+
deps;
|
|
462
|
+
inFlight = 0;
|
|
463
|
+
droppedSinceReport = 0;
|
|
464
|
+
lastReportAt = Number.NEGATIVE_INFINITY;
|
|
465
|
+
constructor(deps) {
|
|
466
|
+
this.deps = deps;
|
|
467
|
+
}
|
|
468
|
+
/** Tasks currently in flight. */
|
|
469
|
+
get pending() {
|
|
470
|
+
return this.inFlight;
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Admit `task` unless the lane is saturated.
|
|
474
|
+
*
|
|
475
|
+
* Returns `false` when the task was REFUSED — the caller must log the drop
|
|
476
|
+
* (with its `deviceId` tag when the work is about a device); a branch that
|
|
477
|
+
* drops work silently reads as "never happened".
|
|
478
|
+
*/
|
|
479
|
+
run(task) {
|
|
480
|
+
if (this.inFlight >= this.deps.limit) {
|
|
481
|
+
this.droppedSinceReport += 1;
|
|
482
|
+
const now = this.deps.now?.() ?? Date.now();
|
|
483
|
+
const intervalMs = this.deps.reportIntervalMs ?? 3e4;
|
|
484
|
+
if (now - this.lastReportAt >= intervalMs) {
|
|
485
|
+
this.lastReportAt = now;
|
|
486
|
+
const dropped = this.droppedSinceReport;
|
|
487
|
+
this.droppedSinceReport = 0;
|
|
488
|
+
this.deps.onSaturated({
|
|
489
|
+
label: this.deps.label,
|
|
490
|
+
dropped,
|
|
491
|
+
pending: this.inFlight
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
return false;
|
|
495
|
+
}
|
|
496
|
+
this.inFlight += 1;
|
|
497
|
+
Promise.resolve().then(task).catch((error) => {
|
|
498
|
+
this.deps.onTaskError?.(error);
|
|
499
|
+
}).finally(() => {
|
|
500
|
+
this.inFlight -= 1;
|
|
501
|
+
});
|
|
502
|
+
return true;
|
|
503
|
+
}
|
|
504
|
+
};
|
|
460
505
|
//#endregion
|
|
461
506
|
//#region src/recorder/merge-ranges.ts
|
|
462
507
|
function mergeRanges(segments, gapToleranceMs) {
|
|
@@ -516,12 +561,33 @@ function* hourBuckets(fromMs, toMs) {
|
|
|
516
561
|
const first = Math.floor(fromMs / HOUR_MS$5) * HOUR_MS$5;
|
|
517
562
|
for (let h = first; h < toMs; h += HOUR_MS$5) yield h;
|
|
518
563
|
}
|
|
564
|
+
/**
|
|
565
|
+
* Index of the first element whose `startMs` is GREATER than `startMs` (upper
|
|
566
|
+
* bound). Inserting there keeps equal-start rows in arrival order — exactly
|
|
567
|
+
* what a stable full sort of the backing map (insertion-ordered) produces, so
|
|
568
|
+
* an incrementally-maintained view is indistinguishable from a rebuilt one.
|
|
569
|
+
*/
|
|
570
|
+
function upperBoundByStart(rows, startMs) {
|
|
571
|
+
let lo = 0;
|
|
572
|
+
let hi = rows.length;
|
|
573
|
+
while (lo < hi) {
|
|
574
|
+
const mid = lo + hi >> 1;
|
|
575
|
+
if (rows[mid].startMs <= startMs) lo = mid + 1;
|
|
576
|
+
else hi = mid;
|
|
577
|
+
}
|
|
578
|
+
return lo;
|
|
579
|
+
}
|
|
519
580
|
var RecordingIndex = class {
|
|
520
581
|
byDevice = /* @__PURE__ */ new Map();
|
|
521
|
-
/** Start-sorted
|
|
522
|
-
*
|
|
523
|
-
*
|
|
582
|
+
/** Start-sorted views. Built lazily on first read (one copy+sort), then kept
|
|
583
|
+
* LIVE by `addSegment`'s ordered insert — rows arrive roughly ascending, so
|
|
584
|
+
* the insert is near-appending. Bulk mutations (hydrate, eviction, a re-set
|
|
585
|
+
* of an existing path) still invalidate; the steady-state finalize never
|
|
586
|
+
* does. Interactive locate/range calls reuse these instead of
|
|
587
|
+
* filtering+sorting the complete archive on every seek and scrub miss. */
|
|
524
588
|
sortedByDevice = /* @__PURE__ */ new Map();
|
|
589
|
+
fullRebuilds = 0;
|
|
590
|
+
incrementalInserts = 0;
|
|
525
591
|
/** Hour buckets this device has been walked for; `ALL_HOURS` after a full
|
|
526
592
|
* hydrate. Empty = nothing has been looked at, which is NOT the same as
|
|
527
593
|
* "there is nothing" — see {@link hydrationOf}. */
|
|
@@ -548,10 +614,34 @@ var RecordingIndex = class {
|
|
|
548
614
|
if (cached) return cached;
|
|
549
615
|
const m = this.byDevice.get(deviceId);
|
|
550
616
|
if (!m) return [];
|
|
617
|
+
this.fullRebuilds += 1;
|
|
551
618
|
const sorted = [...m.values()].filter((s) => profile == null || s.profile === profile).toSorted((a, b) => a.startMs - b.startMs);
|
|
552
619
|
deviceCache.set(key, sorted);
|
|
553
620
|
return sorted;
|
|
554
621
|
}
|
|
622
|
+
/** See {@link SortedViewStats}. Telemetry + the guard the churn test asserts on. */
|
|
623
|
+
sortedViewStats() {
|
|
624
|
+
return {
|
|
625
|
+
fullRebuilds: this.fullRebuilds,
|
|
626
|
+
incrementalInserts: this.incrementalInserts
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Ordered insert of ONE NEW row into every live view it belongs to (the
|
|
631
|
+
* all-profiles view and its own profile's view). Views not yet materialised
|
|
632
|
+
* stay unmaterialised — their first read pays the one build. PRECONDITION:
|
|
633
|
+
* the row's path was NOT already in the device map; a replace must
|
|
634
|
+
* invalidate instead, because the views hold the old row object.
|
|
635
|
+
*/
|
|
636
|
+
insertIntoSortedViews(s) {
|
|
637
|
+
const deviceCache = this.sortedByDevice.get(s.deviceId);
|
|
638
|
+
if (!deviceCache) return;
|
|
639
|
+
for (const [key, rows] of deviceCache) {
|
|
640
|
+
if (key !== "" && key !== s.profile) continue;
|
|
641
|
+
rows.splice(upperBoundByStart(rows, s.startMs), 0, s);
|
|
642
|
+
this.incrementalInserts += 1;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
555
645
|
/**
|
|
556
646
|
* Replace a device's segments FOR ONE LOCATION from a `storage.list` result (paths relative to that
|
|
557
647
|
* location root). Rows on other locations are preserved. Non-segment paths and other devices are ignored.
|
|
@@ -632,10 +722,26 @@ var RecordingIndex = class {
|
|
|
632
722
|
this.invalidateSorted(deviceId);
|
|
633
723
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$5);
|
|
634
724
|
}
|
|
635
|
-
/**
|
|
725
|
+
/**
|
|
726
|
+
* Insert/replace one finalized segment (idempotent by path).
|
|
727
|
+
*
|
|
728
|
+
* THE hot mutation: every profile-writer lands here every `segmentSeconds`,
|
|
729
|
+
* ~3.4 times a second fleet-wide. A NEW path is an ordered insert into the
|
|
730
|
+
* live sorted views (rows arrive roughly ascending, so it is near-appending
|
|
731
|
+
* — O(log n) search + a short memmove). Only the rare RE-SET of an existing
|
|
732
|
+
* path (watcher replay after a playlist reset, or a relocate that changes
|
|
733
|
+
* `locationId`) invalidates: the views hold the old row object, and a
|
|
734
|
+
* one-off rebuild is the simple correct answer for a case that is not hot.
|
|
735
|
+
*/
|
|
636
736
|
addSegment(s) {
|
|
637
|
-
this.mapFor(s.deviceId)
|
|
638
|
-
|
|
737
|
+
const m = this.mapFor(s.deviceId);
|
|
738
|
+
const replacing = m.has(s.path);
|
|
739
|
+
m.set(s.path, s);
|
|
740
|
+
if (replacing) {
|
|
741
|
+
this.invalidateSorted(s.deviceId);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
this.insertIntoSortedViews(s);
|
|
639
745
|
}
|
|
640
746
|
/** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. */
|
|
641
747
|
removeSegments(paths) {
|
|
@@ -2901,6 +3007,67 @@ async function listExportsFor(state, deviceId) {
|
|
|
2901
3007
|
return [...(await readExports(state)).values()].filter((r) => deviceId === void 0 || r.deviceId === deviceId).toSorted((a, b) => b.createdAt - a.createdAt);
|
|
2902
3008
|
}
|
|
2903
3009
|
//#endregion
|
|
3010
|
+
//#region src/recorder/addon/spawn-failure.ts
|
|
3011
|
+
/**
|
|
3012
|
+
* How the recorder reads a `child_process` `'error'` — ONE owner of the
|
|
3013
|
+
* transient-vs-permanent question.
|
|
3014
|
+
*
|
|
3015
|
+
* ── Why this file exists ───────────────────────────────────────────────────
|
|
3016
|
+
* In Node a child that cannot be spawned emits `'error'`, and an `'error'`
|
|
3017
|
+
* with no listener is re-thrown as an UNCAUGHT EXCEPTION. On 2026-08-18 three
|
|
3018
|
+
* cameras were unplugged, every writer for them re-dialled at once, the host
|
|
3019
|
+
* momentarily could not fork (`spawn ffmpeg EAGAIN`), and the `recorder`
|
|
3020
|
+
* runner died three times inside nine seconds. The D6 crash circuit-breaker
|
|
3021
|
+
* did its job and stopped respawning — leaving the whole fleet with no
|
|
3022
|
+
* recording for three hours and fifteen minutes while `/health` returned 200.
|
|
3023
|
+
*
|
|
3024
|
+
* Every ffmpeg spawn site in the recorder therefore attaches an `'error'`
|
|
3025
|
+
* listener, and every one of them classifies the failure HERE rather than
|
|
3026
|
+
* inventing its own list.
|
|
3027
|
+
*/
|
|
3028
|
+
/**
|
|
3029
|
+
* Spawn failures that RETRYING CANNOT FIX.
|
|
3030
|
+
*
|
|
3031
|
+
* The EAGAIN-vs-ENOENT split, stated once:
|
|
3032
|
+
*
|
|
3033
|
+
* - EAGAIN / EMFILE / ENOMEM are the host saying "not right now". The
|
|
3034
|
+
* resource comes back, often within one backoff. They take the ordinary
|
|
3035
|
+
* restart-with-backoff path, whose own comment already states the policy
|
|
3036
|
+
* for this shape: "only RAPID back-to-back failures count toward giving
|
|
3037
|
+
* up, so a brief blip never permanently stops a camera's recording".
|
|
3038
|
+
* - ENOENT / EACCES are the host saying "there is no ffmpeg here you may
|
|
3039
|
+
* run". Nothing about the next ten attempts differs from this one, so the
|
|
3040
|
+
* caller stops immediately and surfaces it at ERROR level: burning a
|
|
3041
|
+
* ten-attempt budget on a missing binary only delays recovery and buries
|
|
3042
|
+
* the single actionable line under ten identical warnings.
|
|
3043
|
+
*
|
|
3044
|
+
* EPERM is deliberately absent: it is reported for transient sandbox/cgroup
|
|
3045
|
+
* refusals as well as for real permission faults, and treating an ambiguous
|
|
3046
|
+
* code as permanent stops a camera that would have recovered.
|
|
3047
|
+
*/
|
|
3048
|
+
var PERMANENT_SPAWN_CODES = new Set(["ENOENT", "EACCES"]);
|
|
3049
|
+
/**
|
|
3050
|
+
* Read the `'error'` payload without a cast.
|
|
3051
|
+
*
|
|
3052
|
+
* An unknown code is treated as TRANSIENT: the cost of retrying something
|
|
3053
|
+
* permanent is a bounded restart budget, while the cost of giving up on
|
|
3054
|
+
* something transient is a camera that stops recording until an operator
|
|
3055
|
+
* notices.
|
|
3056
|
+
*/
|
|
3057
|
+
function readSpawnFailure(raw) {
|
|
3058
|
+
const code = errnoCode(raw);
|
|
3059
|
+
return {
|
|
3060
|
+
code,
|
|
3061
|
+
message: raw instanceof Error ? raw.message : String(raw),
|
|
3062
|
+
permanent: code !== void 0 && PERMANENT_SPAWN_CODES.has(code)
|
|
3063
|
+
};
|
|
3064
|
+
}
|
|
3065
|
+
function errnoCode(raw) {
|
|
3066
|
+
if (typeof raw !== "object" || raw === null || !("code" in raw)) return void 0;
|
|
3067
|
+
const code = raw.code;
|
|
3068
|
+
return typeof code === "string" ? code : void 0;
|
|
3069
|
+
}
|
|
3070
|
+
//#endregion
|
|
2904
3071
|
//#region src/recorder/addon/export-engine.ts
|
|
2905
3072
|
/** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
|
|
2906
3073
|
* segment before the decisive "Error opening input" summary, so a short tail can
|
|
@@ -3070,7 +3237,7 @@ var ExportEngine = class {
|
|
|
3070
3237
|
outPath,
|
|
3071
3238
|
options
|
|
3072
3239
|
});
|
|
3073
|
-
const ran = await this.runFfmpeg(id, args, {
|
|
3240
|
+
const ran = await this.runFfmpeg(id, rec.deviceId, args, {
|
|
3074
3241
|
totalSec: expectedOutSeconds(rec),
|
|
3075
3242
|
floor: 0,
|
|
3076
3243
|
ceil: 99
|
|
@@ -3090,7 +3257,7 @@ var ExportEngine = class {
|
|
|
3090
3257
|
inputPlaylist: playlist,
|
|
3091
3258
|
outPath: concatPath
|
|
3092
3259
|
});
|
|
3093
|
-
const concatRan = await this.runFfmpeg(id, concatArgs, {
|
|
3260
|
+
const concatRan = await this.runFfmpeg(id, rec.deviceId, concatArgs, {
|
|
3094
3261
|
totalSec: Math.max(.001, (rec.toMs - rec.fromMs) / 1e3),
|
|
3095
3262
|
floor: 0,
|
|
3096
3263
|
ceil: 49
|
|
@@ -3104,7 +3271,7 @@ var ExportEngine = class {
|
|
|
3104
3271
|
outPath,
|
|
3105
3272
|
options
|
|
3106
3273
|
});
|
|
3107
|
-
const selectRan = await this.runFfmpeg(id, selectArgs, {
|
|
3274
|
+
const selectRan = await this.runFfmpeg(id, rec.deviceId, selectArgs, {
|
|
3108
3275
|
totalSec: expectedOutSeconds(rec),
|
|
3109
3276
|
floor: 50,
|
|
3110
3277
|
ceil: 99
|
|
@@ -3116,7 +3283,7 @@ var ExportEngine = class {
|
|
|
3116
3283
|
} catch {}
|
|
3117
3284
|
}
|
|
3118
3285
|
}
|
|
3119
|
-
runFfmpeg(id, args, progress) {
|
|
3286
|
+
runFfmpeg(id, deviceId, args, progress) {
|
|
3120
3287
|
this.deps.logger.debug("export ffmpeg argv", { meta: {
|
|
3121
3288
|
exportId: id,
|
|
3122
3289
|
args
|
|
@@ -3153,6 +3320,24 @@ var ExportEngine = class {
|
|
|
3153
3320
|
stderrTail
|
|
3154
3321
|
});
|
|
3155
3322
|
});
|
|
3323
|
+
proc.on("error", (raw) => {
|
|
3324
|
+
this.activeProc = null;
|
|
3325
|
+
const failure = readSpawnFailure(raw);
|
|
3326
|
+
this.deps.logger.error("export ffmpeg could not be spawned", {
|
|
3327
|
+
tags: { deviceId },
|
|
3328
|
+
meta: {
|
|
3329
|
+
exportId: id,
|
|
3330
|
+
errorCode: failure.code,
|
|
3331
|
+
error: failure.message,
|
|
3332
|
+
permanent: failure.permanent
|
|
3333
|
+
}
|
|
3334
|
+
});
|
|
3335
|
+
stderrTail.push(`ffmpeg spawn failed: ${failure.message}` + (failure.code === void 0 ? "" : ` (${failure.code})`));
|
|
3336
|
+
resolve({
|
|
3337
|
+
code: null,
|
|
3338
|
+
stderrTail
|
|
3339
|
+
});
|
|
3340
|
+
});
|
|
3156
3341
|
});
|
|
3157
3342
|
}
|
|
3158
3343
|
async finish(id, code, outPath, stderrTail) {
|
|
@@ -6130,7 +6315,9 @@ var ReadinessRestore = class {
|
|
|
6130
6315
|
this.deps = deps;
|
|
6131
6316
|
}
|
|
6132
6317
|
get concurrency() {
|
|
6133
|
-
|
|
6318
|
+
const declared = this.deps.concurrency;
|
|
6319
|
+
const value = typeof declared === "function" ? declared() : declared;
|
|
6320
|
+
return Math.max(1, value ?? 8);
|
|
6134
6321
|
}
|
|
6135
6322
|
/** Seed the pending set, subscribe to readiness, and drain once immediately
|
|
6136
6323
|
* (covers the already-ready case). Returns when the first drain settles. */
|
|
@@ -6189,6 +6376,70 @@ var ReadinessRestore = class {
|
|
|
6189
6376
|
}
|
|
6190
6377
|
}
|
|
6191
6378
|
};
|
|
6379
|
+
/** Adaptive bound for the recorder's attach parallelism. */
|
|
6380
|
+
var SpawnPressureGovernor = class {
|
|
6381
|
+
deps;
|
|
6382
|
+
limit;
|
|
6383
|
+
min;
|
|
6384
|
+
recoverAfterMs;
|
|
6385
|
+
/** When the limit last MOVED — the clock both directions are measured from. */
|
|
6386
|
+
lastChangeAt;
|
|
6387
|
+
underPressure = false;
|
|
6388
|
+
constructor(deps) {
|
|
6389
|
+
this.deps = deps;
|
|
6390
|
+
this.min = Math.max(1, deps.minConcurrency ?? 1);
|
|
6391
|
+
this.limit = Math.max(this.min, deps.maxConcurrency);
|
|
6392
|
+
this.recoverAfterMs = deps.recoverAfterMs ?? 3e4;
|
|
6393
|
+
this.lastChangeAt = this.now();
|
|
6394
|
+
}
|
|
6395
|
+
/**
|
|
6396
|
+
* How many attaches may overlap right now.
|
|
6397
|
+
*
|
|
6398
|
+
* Reading is what drives recovery: there is no timer to leak, no interval to
|
|
6399
|
+
* lose to a runner respawn, and a recorder that stops attaching entirely
|
|
6400
|
+
* simply keeps its reduced bound until it tries again — which is correct.
|
|
6401
|
+
*/
|
|
6402
|
+
get concurrency() {
|
|
6403
|
+
const max = Math.max(this.min, this.deps.maxConcurrency);
|
|
6404
|
+
if (this.limit >= max) return max;
|
|
6405
|
+
const elapsed = this.now() - this.lastChangeAt;
|
|
6406
|
+
if (elapsed < this.recoverAfterMs) return this.limit;
|
|
6407
|
+
const lanes = Math.floor(elapsed / this.recoverAfterMs);
|
|
6408
|
+
this.limit = Math.min(max, this.limit + lanes);
|
|
6409
|
+
this.lastChangeAt += lanes * this.recoverAfterMs;
|
|
6410
|
+
if (this.limit >= max && this.underPressure) {
|
|
6411
|
+
this.underPressure = false;
|
|
6412
|
+
this.deps.logger?.info("recorder: spawn pressure cleared — attach concurrency restored", { meta: { concurrency: this.limit } });
|
|
6413
|
+
} else this.deps.logger?.info("recorder: recovering attach concurrency", { meta: {
|
|
6414
|
+
concurrency: this.limit,
|
|
6415
|
+
max
|
|
6416
|
+
} });
|
|
6417
|
+
return this.limit;
|
|
6418
|
+
}
|
|
6419
|
+
/**
|
|
6420
|
+
* A transient spawn failure happened on `deviceId`'s writer.
|
|
6421
|
+
*
|
|
6422
|
+
* Halves the CURRENT limit, not the max: two waves of pressure inside one
|
|
6423
|
+
* recovery window must compound, or a host under sustained load is asked for
|
|
6424
|
+
* the same too-large batch over and over.
|
|
6425
|
+
*/
|
|
6426
|
+
report(deviceId) {
|
|
6427
|
+
const before = this.limit;
|
|
6428
|
+
this.limit = Math.max(this.min, Math.floor(this.limit / 2));
|
|
6429
|
+
this.lastChangeAt = this.now();
|
|
6430
|
+
this.underPressure = true;
|
|
6431
|
+
this.deps.logger?.warn("recorder: spawn pressure — reducing attach concurrency", {
|
|
6432
|
+
tags: { deviceId },
|
|
6433
|
+
meta: {
|
|
6434
|
+
from: before,
|
|
6435
|
+
to: this.limit
|
|
6436
|
+
}
|
|
6437
|
+
});
|
|
6438
|
+
}
|
|
6439
|
+
now() {
|
|
6440
|
+
return this.deps.now?.() ?? Date.now();
|
|
6441
|
+
}
|
|
6442
|
+
};
|
|
6192
6443
|
//#endregion
|
|
6193
6444
|
//#region src/recorder/addon/segment-watcher.ts
|
|
6194
6445
|
/**
|
|
@@ -6574,36 +6825,87 @@ var SegmentWriter = class {
|
|
|
6574
6825
|
if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
|
|
6575
6826
|
}
|
|
6576
6827
|
});
|
|
6828
|
+
let settled = false;
|
|
6829
|
+
const terminate = (termination) => {
|
|
6830
|
+
if (settled) return;
|
|
6831
|
+
settled = true;
|
|
6832
|
+
this.onTermination(termination, stderrTail);
|
|
6833
|
+
};
|
|
6577
6834
|
proc.on("exit", (code) => {
|
|
6578
|
-
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
|
|
6835
|
+
terminate({
|
|
6836
|
+
reason: "exit",
|
|
6837
|
+
exitCode: typeof code === "number" ? code : null
|
|
6838
|
+
});
|
|
6839
|
+
});
|
|
6840
|
+
proc.on("error", (raw) => {
|
|
6841
|
+
const failure = readSpawnFailure(raw);
|
|
6842
|
+
terminate({
|
|
6843
|
+
reason: "spawn-error",
|
|
6844
|
+
errorCode: failure.code,
|
|
6845
|
+
message: failure.message,
|
|
6846
|
+
permanent: failure.permanent
|
|
6847
|
+
});
|
|
6848
|
+
});
|
|
6849
|
+
}
|
|
6850
|
+
/**
|
|
6851
|
+
* The ONE restart policy, reached from both `exit` and `error`.
|
|
6852
|
+
*
|
|
6853
|
+
* `deps.logger` is the device-scoped child the controller binds
|
|
6854
|
+
* (`logger.withTags({ deviceId })`), so every line below carries
|
|
6855
|
+
* `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
|
|
6856
|
+
* camera, and a writer that stops recording one camera must be greppable by
|
|
6857
|
+
* that camera.
|
|
6858
|
+
*/
|
|
6859
|
+
onTermination(termination, stderrTail) {
|
|
6860
|
+
this.resolveExit?.();
|
|
6861
|
+
this.resolveExit = null;
|
|
6862
|
+
this.proc = null;
|
|
6863
|
+
if (this.stopped) return;
|
|
6864
|
+
const ranMs = Date.now() - this.startedAt;
|
|
6865
|
+
if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
|
|
6866
|
+
if (termination.reason === "spawn-error") {
|
|
6867
|
+
this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
|
|
6868
|
+
outDir: this.cfg.outDir,
|
|
6869
|
+
errorCode: termination.errorCode,
|
|
6870
|
+
error: termination.message,
|
|
6871
|
+
permanent: termination.permanent
|
|
6872
|
+
} });
|
|
6873
|
+
if (termination.permanent) {
|
|
6874
|
+
this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
|
|
6586
6875
|
outDir: this.cfg.outDir,
|
|
6587
|
-
|
|
6588
|
-
|
|
6876
|
+
errorCode: termination.errorCode,
|
|
6877
|
+
error: termination.message
|
|
6589
6878
|
} });
|
|
6590
6879
|
this.stopped = true;
|
|
6591
6880
|
this.deps.onGaveUp?.();
|
|
6592
6881
|
return;
|
|
6593
6882
|
}
|
|
6594
|
-
this.
|
|
6595
|
-
|
|
6596
|
-
|
|
6883
|
+
this.deps.onResourcePressure?.();
|
|
6884
|
+
}
|
|
6885
|
+
if (this.restarts >= MAX_RESTARTS) {
|
|
6886
|
+
this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
|
|
6597
6887
|
outDir: this.cfg.outDir,
|
|
6598
|
-
|
|
6599
|
-
|
|
6600
|
-
|
|
6888
|
+
reason: termination.reason,
|
|
6889
|
+
code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
|
|
6890
|
+
stderrTail: stderrTail.join(" | ")
|
|
6601
6891
|
} });
|
|
6602
|
-
this.
|
|
6603
|
-
|
|
6604
|
-
|
|
6605
|
-
|
|
6606
|
-
|
|
6892
|
+
this.stopped = true;
|
|
6893
|
+
this.deps.onGaveUp?.();
|
|
6894
|
+
return;
|
|
6895
|
+
}
|
|
6896
|
+
this.restarts++;
|
|
6897
|
+
const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
|
|
6898
|
+
this.deps.logger.info("SegmentWriter restarting", { meta: {
|
|
6899
|
+
outDir: this.cfg.outDir,
|
|
6900
|
+
attempt: this.restarts,
|
|
6901
|
+
delayMs,
|
|
6902
|
+
ranMs,
|
|
6903
|
+
reason: termination.reason
|
|
6904
|
+
} });
|
|
6905
|
+
this.restartTimer = setTimeout(() => {
|
|
6906
|
+
this.restartTimer = null;
|
|
6907
|
+
this.start();
|
|
6908
|
+
}, delayMs);
|
|
6607
6909
|
}
|
|
6608
6910
|
stop() {
|
|
6609
6911
|
this.stopAndWait();
|
|
@@ -6895,8 +7197,20 @@ var RecordingController = class {
|
|
|
6895
7197
|
/** A reversible maintenance lease. Unlike `stopped`, it never changes
|
|
6896
7198
|
* persisted recording intent and `resume()` restarts normal convergence. */
|
|
6897
7199
|
paused = false;
|
|
7200
|
+
/**
|
|
7201
|
+
* The ADAPTIVE attach bound. A transient `spawn ffmpeg` failure (EAGAIN) is
|
|
7202
|
+
* the host saying it cannot fork right now, and the answer is to attach
|
|
7203
|
+
* FEWER cameras at once — not to serialise them, which `e2fa17a2b` removed
|
|
7204
|
+
* for good reason. Every bounded-parallel attach in this file reads it.
|
|
7205
|
+
*/
|
|
7206
|
+
spawnPressure;
|
|
6898
7207
|
constructor(deps) {
|
|
6899
7208
|
this.deps = deps;
|
|
7209
|
+
this.spawnPressure = new SpawnPressureGovernor({
|
|
7210
|
+
maxConcurrency: 8,
|
|
7211
|
+
now: () => this.now(),
|
|
7212
|
+
logger: deps.logger
|
|
7213
|
+
});
|
|
6900
7214
|
this.wakeups = new DeviceWakeups({
|
|
6901
7215
|
logger: deps.logger,
|
|
6902
7216
|
now: () => this.now(),
|
|
@@ -6964,6 +7278,7 @@ var RecordingController = class {
|
|
|
6964
7278
|
}
|
|
6965
7279
|
this.restore = new ReadinessRestore({
|
|
6966
7280
|
subscribeReady: (handler) => this.deps.onBrokerReady(handler),
|
|
7281
|
+
concurrency: () => this.spawnPressure.concurrency,
|
|
6967
7282
|
attempt: async (deviceId) => {
|
|
6968
7283
|
await this.evaluateDevice(deviceId);
|
|
6969
7284
|
},
|
|
@@ -7044,7 +7359,7 @@ var RecordingController = class {
|
|
|
7044
7359
|
*/
|
|
7045
7360
|
async reconcile() {
|
|
7046
7361
|
if (this.stopped) return;
|
|
7047
|
-
await runBounded(await this.trackedDeviceIds(),
|
|
7362
|
+
await runBounded(await this.trackedDeviceIds(), this.spawnPressure.concurrency, async (id) => {
|
|
7048
7363
|
try {
|
|
7049
7364
|
await this.evaluateDevice(id);
|
|
7050
7365
|
} catch (err) {
|
|
@@ -7403,7 +7718,8 @@ var RecordingController = class {
|
|
|
7403
7718
|
logger: deviceLog,
|
|
7404
7719
|
onGaveUp: () => {
|
|
7405
7720
|
this.recoverWriterGaveUp(deviceId);
|
|
7406
|
-
}
|
|
7721
|
+
},
|
|
7722
|
+
onResourcePressure: () => this.spawnPressure.report(deviceId)
|
|
7407
7723
|
});
|
|
7408
7724
|
const writerStartedMs = this.now();
|
|
7409
7725
|
writer.start();
|
|
@@ -8878,6 +9194,23 @@ var PLAYBACK_PREFIX = "playback";
|
|
|
8878
9194
|
/** Data-plane prefix for on-demand single stills →
|
|
8879
9195
|
* `/addon/recorder/still/<deviceId>/<epochSec>`. */
|
|
8880
9196
|
var STILL_PREFIX = "still";
|
|
9197
|
+
/**
|
|
9198
|
+
* Bound on pending hour-ledger writes (one per finalized segment, an RPC to
|
|
9199
|
+
* the hub store). Steady state completes in milliseconds and holds ~0–2; under
|
|
9200
|
+
* a store stall the fleet-wide finalize rate (~3.4/s) fills 256 in ~75s of
|
|
9201
|
+
* grace before refusals begin. Each pending write holds one hour-row copy
|
|
9202
|
+
* (≤360 paths ≈ ~25KB), so the lane's worst-case heap is single-digit MB —
|
|
9203
|
+
* a dropped write is one silent re-seed from the archive walk (D148).
|
|
9204
|
+
*/
|
|
9205
|
+
var LEDGER_GATE_LIMIT = 256;
|
|
9206
|
+
/**
|
|
9207
|
+
* Bound on pending mfra tail reads (one per finalized RECENT segment; each is
|
|
9208
|
+
* an open + two reads on the same 4-thread libuv pool the live writers use,
|
|
9209
|
+
* holding read buffers while pending). 64 bounds both the held memory and the
|
|
9210
|
+
* pool contention; a dropped capture costs that segment's client one extra
|
|
9211
|
+
* round trip to the file's tail on the first seek.
|
|
9212
|
+
*/
|
|
9213
|
+
var MFRA_GATE_LIMIT = 64;
|
|
8881
9214
|
/** Data-plane prefix for export MP4 downloads → `/addon/recorder/exports/<id>.mp4`. */
|
|
8882
9215
|
var EXPORTS_PREFIX = "exports";
|
|
8883
9216
|
/** How often the export janitor sweeps expired exports. */
|
|
@@ -8954,6 +9287,25 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8954
9287
|
/** Sync-sample tables captured at finalize, served by the directory so a
|
|
8955
9288
|
* playback seek needs no mfra tail fetch. RAM-only, TTL-bounded. */
|
|
8956
9289
|
mfraTables = new MfraTableStore();
|
|
9290
|
+
/** Bound on the per-finalize fire-and-forget lanes (item 3, 2026-08-18): a
|
|
9291
|
+
* disk/store stall must throttle this work, never balloon live heap — the
|
|
9292
|
+
* unbounded pending set is what fed the two `Ineffective mark-compacts`
|
|
9293
|
+
* fatals. Saturation is reported loudly but throttled; every individual
|
|
9294
|
+
* drop is logged with its deviceId at the call site. */
|
|
9295
|
+
ledgerWriteGate = new PendingGate({
|
|
9296
|
+
label: "hour-ledger",
|
|
9297
|
+
limit: LEDGER_GATE_LIMIT,
|
|
9298
|
+
onSaturated: (report) => {
|
|
9299
|
+
this.reportGateSaturated(report);
|
|
9300
|
+
}
|
|
9301
|
+
});
|
|
9302
|
+
mfraCaptureGate = new PendingGate({
|
|
9303
|
+
label: "mfra-capture",
|
|
9304
|
+
limit: MFRA_GATE_LIMIT,
|
|
9305
|
+
onSaturated: (report) => {
|
|
9306
|
+
this.reportGateSaturated(report);
|
|
9307
|
+
}
|
|
9308
|
+
});
|
|
8957
9309
|
/** Continuous + events band write-path controller (ffmpeg writers + watchers). */
|
|
8958
9310
|
controller = null;
|
|
8959
9311
|
/** Per-device motion/audio markers for playback (B3 events feed the controller). */
|
|
@@ -8991,6 +9343,20 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8991
9343
|
constructor() {
|
|
8992
9344
|
super({ ...DEFAULT_CONFIG });
|
|
8993
9345
|
}
|
|
9346
|
+
/**
|
|
9347
|
+
* One throttled line per saturated lane per report interval. Lane-level (no
|
|
9348
|
+
* deviceId — the lane aggregates every camera); the per-drop debug lines at
|
|
9349
|
+
* the call sites carry the tag. WARN because a saturated lane means the
|
|
9350
|
+
* store or the disk is stalling under live finalize load — the line that
|
|
9351
|
+
* explains the missing ledger hours and mfra misses an operator will see.
|
|
9352
|
+
*/
|
|
9353
|
+
reportGateSaturated(report) {
|
|
9354
|
+
this.ctx.logger.warn("recorder: fire-and-forget lane saturated — dropping over the bound", { meta: {
|
|
9355
|
+
lane: report.label,
|
|
9356
|
+
dropped: report.dropped,
|
|
9357
|
+
pending: report.pending
|
|
9358
|
+
} });
|
|
9359
|
+
}
|
|
8994
9360
|
async onInitialize() {
|
|
8995
9361
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
8996
9362
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
@@ -9054,18 +9420,28 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
9054
9420
|
this.segmentHours?.dropSegments(rows);
|
|
9055
9421
|
},
|
|
9056
9422
|
onIndexed: (row, absPath) => {
|
|
9057
|
-
this.segmentHours
|
|
9423
|
+
const ledger = this.segmentHours;
|
|
9424
|
+
if (ledger && !this.ledgerWriteGate.run(() => ledger.recordSegment(row))) this.ctx.logger.debug("recorder: hour-ledger write dropped — lane saturated; the archive walk reconciles this hour", {
|
|
9425
|
+
tags: { deviceId: row.deviceId },
|
|
9426
|
+
meta: { path: row.path }
|
|
9427
|
+
});
|
|
9058
9428
|
if (!isMfraTableServable(row.startMs, Date.now())) return;
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
|
|
9066
|
-
|
|
9067
|
-
|
|
9068
|
-
|
|
9429
|
+
if (!this.mfraCaptureGate.run(async () => {
|
|
9430
|
+
try {
|
|
9431
|
+
const table = await readMfraTable(absPath, row.bytes);
|
|
9432
|
+
this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
|
|
9433
|
+
} catch (err) {
|
|
9434
|
+
this.ctx.logger.debug("mfra capture failed", {
|
|
9435
|
+
tags: { deviceId: row.deviceId },
|
|
9436
|
+
meta: {
|
|
9437
|
+
path: row.path,
|
|
9438
|
+
error: require_dist.errMsg(err)
|
|
9439
|
+
}
|
|
9440
|
+
});
|
|
9441
|
+
}
|
|
9442
|
+
})) this.ctx.logger.debug("recorder: mfra capture dropped — lane saturated; first seek pays one tail fetch", {
|
|
9443
|
+
tags: { deviceId: row.deviceId },
|
|
9444
|
+
meta: { path: row.path }
|
|
9069
9445
|
});
|
|
9070
9446
|
},
|
|
9071
9447
|
removeDirIfEmpty: async (locationId, relDir) => {
|