@camstack/addon-pipeline 1.2.93 → 1.2.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
  2. package/dist/audio-analyzer/index.js +58 -4
  3. package/dist/audio-analyzer/index.mjs +56 -2
  4. package/dist/detection-pipeline/index.js +194 -6
  5. package/dist/detection-pipeline/index.mjs +191 -3
  6. package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
  7. package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
  8. package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
  9. package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
  10. package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
  11. package/dist/motion-wasm/index.js +2 -2
  12. package/dist/motion-wasm/index.mjs +1 -1
  13. package/dist/pipeline-runner/index.js +195 -14
  14. package/dist/pipeline-runner/index.mjs +194 -13
  15. package/dist/process-memory-B9-UD7Rp.js +78 -0
  16. package/dist/process-memory-DXOB6eHX.mjs +66 -0
  17. package/dist/recorder/index.js +943 -109
  18. package/dist/recorder/index.mjs +942 -108
  19. package/dist/session-decode/decode-worker-child.js +4 -4
  20. package/dist/session-decode/decode-worker-child.mjs +3 -3
  21. package/dist/stream-broker/_stub.js +1 -1
  22. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
  23. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
  24. package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
  25. package/dist/stream-broker/index.js +25 -7
  26. package/dist/stream-broker/index.mjs +25 -7
  27. package/dist/stream-broker/remoteEntry.js +1 -1
  28. package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
  29. package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
  30. package/package.json +13 -7
  31. package/python/inference_pool.py +178 -3
  32. package/python/test_inference_pool_backpressure.py +7 -1
  33. package/python/test_inference_pool_memstats.py +146 -0
  34. package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
  35. package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
  36. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
@@ -1,6 +1,6 @@
1
- const require_dist = require("../dist-D9To_r_d.js");
1
+ const require_dist = require("../dist-hyryByiw.js");
2
2
  const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
3
- const require_addon_utils = require("../addon-utils-CTRpEiSd.js");
3
+ const require_addon_utils = require("../addon-utils-UMWiMAxR.js");
4
4
  let node_crypto = require("node:crypto");
5
5
  let node_child_process = require("node:child_process");
6
6
  let node_path = require("node:path");
@@ -457,6 +457,51 @@ var MfraTableStore = class {
457
457
  }
458
458
  }
459
459
  };
460
+ var PendingGate = class {
461
+ deps;
462
+ inFlight = 0;
463
+ droppedSinceReport = 0;
464
+ lastReportAt = Number.NEGATIVE_INFINITY;
465
+ constructor(deps) {
466
+ this.deps = deps;
467
+ }
468
+ /** Tasks currently in flight. */
469
+ get pending() {
470
+ return this.inFlight;
471
+ }
472
+ /**
473
+ * Admit `task` unless the lane is saturated.
474
+ *
475
+ * Returns `false` when the task was REFUSED — the caller must log the drop
476
+ * (with its `deviceId` tag when the work is about a device); a branch that
477
+ * drops work silently reads as "never happened".
478
+ */
479
+ run(task) {
480
+ if (this.inFlight >= this.deps.limit) {
481
+ this.droppedSinceReport += 1;
482
+ const now = this.deps.now?.() ?? Date.now();
483
+ const intervalMs = this.deps.reportIntervalMs ?? 3e4;
484
+ if (now - this.lastReportAt >= intervalMs) {
485
+ this.lastReportAt = now;
486
+ const dropped = this.droppedSinceReport;
487
+ this.droppedSinceReport = 0;
488
+ this.deps.onSaturated({
489
+ label: this.deps.label,
490
+ dropped,
491
+ pending: this.inFlight
492
+ });
493
+ }
494
+ return false;
495
+ }
496
+ this.inFlight += 1;
497
+ Promise.resolve().then(task).catch((error) => {
498
+ this.deps.onTaskError?.(error);
499
+ }).finally(() => {
500
+ this.inFlight -= 1;
501
+ });
502
+ return true;
503
+ }
504
+ };
460
505
  //#endregion
461
506
  //#region src/recorder/merge-ranges.ts
462
507
  function mergeRanges(segments, gapToleranceMs) {
@@ -516,12 +561,33 @@ function* hourBuckets(fromMs, toMs) {
516
561
  const first = Math.floor(fromMs / HOUR_MS$5) * HOUR_MS$5;
517
562
  for (let h = first; h < toMs; h += HOUR_MS$5) yield h;
518
563
  }
564
+ /**
565
+ * Index of the first element whose `startMs` is GREATER than `startMs` (upper
566
+ * bound). Inserting there keeps equal-start rows in arrival order — exactly
567
+ * what a stable full sort of the backing map (insertion-ordered) produces, so
568
+ * an incrementally-maintained view is indistinguishable from a rebuilt one.
569
+ */
570
+ function upperBoundByStart(rows, startMs) {
571
+ let lo = 0;
572
+ let hi = rows.length;
573
+ while (lo < hi) {
574
+ const mid = lo + hi >> 1;
575
+ if (rows[mid].startMs <= startMs) lo = mid + 1;
576
+ else hi = mid;
577
+ }
578
+ return lo;
579
+ }
519
580
  var RecordingIndex = class {
520
581
  byDevice = /* @__PURE__ */ new Map();
521
- /** Start-sorted snapshots, rebuilt only after index mutations. Interactive
522
- * locate/range calls reuse them instead of filtering+sorting the complete
523
- * archive on every seek and scrub miss. */
582
+ /** Start-sorted views. Built lazily on first read (one copy+sort), then kept
583
+ * LIVE by `addSegment`'s ordered insert rows arrive roughly ascending, so
584
+ * the insert is near-appending. Bulk mutations (hydrate, eviction, a re-set
585
+ * of an existing path) still invalidate; the steady-state finalize never
586
+ * does. Interactive locate/range calls reuse these instead of
587
+ * filtering+sorting the complete archive on every seek and scrub miss. */
524
588
  sortedByDevice = /* @__PURE__ */ new Map();
589
+ fullRebuilds = 0;
590
+ incrementalInserts = 0;
525
591
  /** Hour buckets this device has been walked for; `ALL_HOURS` after a full
526
592
  * hydrate. Empty = nothing has been looked at, which is NOT the same as
527
593
  * "there is nothing" — see {@link hydrationOf}. */
@@ -548,10 +614,34 @@ var RecordingIndex = class {
548
614
  if (cached) return cached;
549
615
  const m = this.byDevice.get(deviceId);
550
616
  if (!m) return [];
617
+ this.fullRebuilds += 1;
551
618
  const sorted = [...m.values()].filter((s) => profile == null || s.profile === profile).toSorted((a, b) => a.startMs - b.startMs);
552
619
  deviceCache.set(key, sorted);
553
620
  return sorted;
554
621
  }
622
+ /** See {@link SortedViewStats}. Telemetry + the guard the churn test asserts on. */
623
+ sortedViewStats() {
624
+ return {
625
+ fullRebuilds: this.fullRebuilds,
626
+ incrementalInserts: this.incrementalInserts
627
+ };
628
+ }
629
+ /**
630
+ * Ordered insert of ONE NEW row into every live view it belongs to (the
631
+ * all-profiles view and its own profile's view). Views not yet materialised
632
+ * stay unmaterialised — their first read pays the one build. PRECONDITION:
633
+ * the row's path was NOT already in the device map; a replace must
634
+ * invalidate instead, because the views hold the old row object.
635
+ */
636
+ insertIntoSortedViews(s) {
637
+ const deviceCache = this.sortedByDevice.get(s.deviceId);
638
+ if (!deviceCache) return;
639
+ for (const [key, rows] of deviceCache) {
640
+ if (key !== "" && key !== s.profile) continue;
641
+ rows.splice(upperBoundByStart(rows, s.startMs), 0, s);
642
+ this.incrementalInserts += 1;
643
+ }
644
+ }
555
645
  /**
556
646
  * Replace a device's segments FOR ONE LOCATION from a `storage.list` result (paths relative to that
557
647
  * location root). Rows on other locations are preserved. Non-segment paths and other devices are ignored.
@@ -632,10 +722,26 @@ var RecordingIndex = class {
632
722
  this.invalidateSorted(deviceId);
633
723
  this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$5);
634
724
  }
635
- /** Insert/replace one finalized segment (idempotent by path). */
725
+ /**
726
+ * Insert/replace one finalized segment (idempotent by path).
727
+ *
728
+ * THE hot mutation: every profile-writer lands here every `segmentSeconds`,
729
+ * ~3.4 times a second fleet-wide. A NEW path is an ordered insert into the
730
+ * live sorted views (rows arrive roughly ascending, so it is near-appending
731
+ * — O(log n) search + a short memmove). Only the rare RE-SET of an existing
732
+ * path (watcher replay after a playlist reset, or a relocate that changes
733
+ * `locationId`) invalidates: the views hold the old row object, and a
734
+ * one-off rebuild is the simple correct answer for a case that is not hot.
735
+ */
636
736
  addSegment(s) {
637
- this.mapFor(s.deviceId).set(s.path, s);
638
- this.invalidateSorted(s.deviceId);
737
+ const m = this.mapFor(s.deviceId);
738
+ const replacing = m.has(s.path);
739
+ m.set(s.path, s);
740
+ if (replacing) {
741
+ this.invalidateSorted(s.deviceId);
742
+ return;
743
+ }
744
+ this.insertIntoSortedViews(s);
639
745
  }
640
746
  /** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. */
641
747
  removeSegments(paths) {
@@ -2901,6 +3007,67 @@ async function listExportsFor(state, deviceId) {
2901
3007
  return [...(await readExports(state)).values()].filter((r) => deviceId === void 0 || r.deviceId === deviceId).toSorted((a, b) => b.createdAt - a.createdAt);
2902
3008
  }
2903
3009
  //#endregion
3010
+ //#region src/recorder/addon/spawn-failure.ts
3011
+ /**
3012
+ * How the recorder reads a `child_process` `'error'` — ONE owner of the
3013
+ * transient-vs-permanent question.
3014
+ *
3015
+ * ── Why this file exists ───────────────────────────────────────────────────
3016
+ * In Node a child that cannot be spawned emits `'error'`, and an `'error'`
3017
+ * with no listener is re-thrown as an UNCAUGHT EXCEPTION. On 2026-08-18 three
3018
+ * cameras were unplugged, every writer for them re-dialled at once, the host
3019
+ * momentarily could not fork (`spawn ffmpeg EAGAIN`), and the `recorder`
3020
+ * runner died three times inside nine seconds. The D6 crash circuit-breaker
3021
+ * did its job and stopped respawning — leaving the whole fleet with no
3022
+ * recording for three hours and fifteen minutes while `/health` returned 200.
3023
+ *
3024
+ * Every ffmpeg spawn site in the recorder therefore attaches an `'error'`
3025
+ * listener, and every one of them classifies the failure HERE rather than
3026
+ * inventing its own list.
3027
+ */
3028
+ /**
3029
+ * Spawn failures that RETRYING CANNOT FIX.
3030
+ *
3031
+ * The EAGAIN-vs-ENOENT split, stated once:
3032
+ *
3033
+ * - EAGAIN / EMFILE / ENOMEM are the host saying "not right now". The
3034
+ * resource comes back, often within one backoff. They take the ordinary
3035
+ * restart-with-backoff path, whose own comment already states the policy
3036
+ * for this shape: "only RAPID back-to-back failures count toward giving
3037
+ * up, so a brief blip never permanently stops a camera's recording".
3038
+ * - ENOENT / EACCES are the host saying "there is no ffmpeg here you may
3039
+ * run". Nothing about the next ten attempts differs from this one, so the
3040
+ * caller stops immediately and surfaces it at ERROR level: burning a
3041
+ * ten-attempt budget on a missing binary only delays recovery and buries
3042
+ * the single actionable line under ten identical warnings.
3043
+ *
3044
+ * EPERM is deliberately absent: it is reported for transient sandbox/cgroup
3045
+ * refusals as well as for real permission faults, and treating an ambiguous
3046
+ * code as permanent stops a camera that would have recovered.
3047
+ */
3048
+ var PERMANENT_SPAWN_CODES = new Set(["ENOENT", "EACCES"]);
3049
+ /**
3050
+ * Read the `'error'` payload without a cast.
3051
+ *
3052
+ * An unknown code is treated as TRANSIENT: the cost of retrying something
3053
+ * permanent is a bounded restart budget, while the cost of giving up on
3054
+ * something transient is a camera that stops recording until an operator
3055
+ * notices.
3056
+ */
3057
+ function readSpawnFailure(raw) {
3058
+ const code = errnoCode(raw);
3059
+ return {
3060
+ code,
3061
+ message: raw instanceof Error ? raw.message : String(raw),
3062
+ permanent: code !== void 0 && PERMANENT_SPAWN_CODES.has(code)
3063
+ };
3064
+ }
3065
+ function errnoCode(raw) {
3066
+ if (typeof raw !== "object" || raw === null || !("code" in raw)) return void 0;
3067
+ const code = raw.code;
3068
+ return typeof code === "string" ? code : void 0;
3069
+ }
3070
+ //#endregion
2904
3071
  //#region src/recorder/addon/export-engine.ts
2905
3072
  /** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
2906
3073
  * segment before the decisive "Error opening input" summary, so a short tail can
@@ -3070,7 +3237,7 @@ var ExportEngine = class {
3070
3237
  outPath,
3071
3238
  options
3072
3239
  });
3073
- const ran = await this.runFfmpeg(id, args, {
3240
+ const ran = await this.runFfmpeg(id, rec.deviceId, args, {
3074
3241
  totalSec: expectedOutSeconds(rec),
3075
3242
  floor: 0,
3076
3243
  ceil: 99
@@ -3090,7 +3257,7 @@ var ExportEngine = class {
3090
3257
  inputPlaylist: playlist,
3091
3258
  outPath: concatPath
3092
3259
  });
3093
- const concatRan = await this.runFfmpeg(id, concatArgs, {
3260
+ const concatRan = await this.runFfmpeg(id, rec.deviceId, concatArgs, {
3094
3261
  totalSec: Math.max(.001, (rec.toMs - rec.fromMs) / 1e3),
3095
3262
  floor: 0,
3096
3263
  ceil: 49
@@ -3104,7 +3271,7 @@ var ExportEngine = class {
3104
3271
  outPath,
3105
3272
  options
3106
3273
  });
3107
- const selectRan = await this.runFfmpeg(id, selectArgs, {
3274
+ const selectRan = await this.runFfmpeg(id, rec.deviceId, selectArgs, {
3108
3275
  totalSec: expectedOutSeconds(rec),
3109
3276
  floor: 50,
3110
3277
  ceil: 99
@@ -3116,7 +3283,7 @@ var ExportEngine = class {
3116
3283
  } catch {}
3117
3284
  }
3118
3285
  }
3119
- runFfmpeg(id, args, progress) {
3286
+ runFfmpeg(id, deviceId, args, progress) {
3120
3287
  this.deps.logger.debug("export ffmpeg argv", { meta: {
3121
3288
  exportId: id,
3122
3289
  args
@@ -3153,6 +3320,24 @@ var ExportEngine = class {
3153
3320
  stderrTail
3154
3321
  });
3155
3322
  });
3323
+ proc.on("error", (raw) => {
3324
+ this.activeProc = null;
3325
+ const failure = readSpawnFailure(raw);
3326
+ this.deps.logger.error("export ffmpeg could not be spawned", {
3327
+ tags: { deviceId },
3328
+ meta: {
3329
+ exportId: id,
3330
+ errorCode: failure.code,
3331
+ error: failure.message,
3332
+ permanent: failure.permanent
3333
+ }
3334
+ });
3335
+ stderrTail.push(`ffmpeg spawn failed: ${failure.message}` + (failure.code === void 0 ? "" : ` (${failure.code})`));
3336
+ resolve({
3337
+ code: null,
3338
+ stderrTail
3339
+ });
3340
+ });
3156
3341
  });
3157
3342
  }
3158
3343
  async finish(id, code, outPath, stderrTail) {
@@ -5838,18 +6023,232 @@ function shouldRecordAt(config, triggers, at) {
5838
6023
  return eventsBandDemanded(band, triggers, at.getTime());
5839
6024
  }
5840
6025
  /**
5841
- * Should a finalized segment `[segStartMs, segEndMs]` recorded under an
5842
- * `events` band be KEPT (else discarded)? Kept iff it overlaps any qualifying
5843
- * trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]`.
5844
- * This is where `preBufferSec` retroactively retains pre-trigger footage.
6026
+ * How late the FIRST segment of an attach may be NAMED, relative to the writer
6027
+ * spawn, and still count as "the segment this attach was spawned to produce".
6028
+ *
6029
+ * ffmpeg names a segment when it OPENS it, which is after the RTSP dial and the
6030
+ * first keyframe. Measured on camera 1439 (4K, H.265) on 2026-08-18: writer
6031
+ * spawned at trigger+29.5 s, first `high` segment named at trigger+32.7 s —
6032
+ * 3.2 s. 5 s carries that with margin while staying far below the writer's own
6033
+ * "this is dead" bound (`WRITER_IDLE_MIN_MS`, 90 s), so a writer that is merely
6034
+ * slow is not mistaken for one that is producing useful footage.
5845
6035
  */
5846
- function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers) {
6036
+ var FIRST_SEGMENT_GRACE_MS = 5e3;
6037
+ /**
6038
+ * Latest segment START this attach may still produce and have kept, regardless
6039
+ * of where the operator's post-buffer ended: one full segment (nothing can be
6040
+ * finalized sooner) plus {@link FIRST_SEGMENT_GRACE_MS} of spawn→first-name
6041
+ * latency.
6042
+ */
6043
+ function attachRetainDeadlineMs(attach) {
6044
+ return attach.attachedAtMs + attach.segmentSeconds * 1e3 + FIRST_SEGMENT_GRACE_MS;
6045
+ }
6046
+ /**
6047
+ * Should a finalized segment `[segStartMs, segEndMs]` recorded under an
6048
+ * `events` band be KEPT (else discarded)? Kept iff it overlaps a qualifying
6049
+ * trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]` —
6050
+ * this is where `preBufferSec` retroactively retains pre-trigger footage — OR
6051
+ * it is the footage `attach` was spawned to produce (below).
6052
+ *
6053
+ * ## Why the attach matters here
6054
+ *
6055
+ * The LIVE gate ({@link shouldRecordAt}) is evaluated at wall clock; this KEEP
6056
+ * gate is evaluated on segment timestamps. They shared one window, and a writer
6057
+ * physically cannot finalize anything before `spawn + segmentSeconds + dial`.
6058
+ * So the last ~`segmentSeconds` of every demand window was a zone where the
6059
+ * recorder was GUARANTEED to attach, pull the camera, and delete 100% of the
6060
+ * result: on camera 1439 a trigger reached the recorder 29.5 s into its 30 s
6061
+ * post-buffer (hub overload, `busLagMs=142173`), ffmpeg ran 31 s, and all six
6062
+ * finalized segments were unlinked.
6063
+ *
6064
+ * The allowance is deliberately NOT a wider `postBufferSec`: it is anchored to
6065
+ * `attachedAtMs`, so a PUNCTUAL attach (the normal case) has a deadline far
6066
+ * inside `trigger + postMs` and retains exactly what it retained before. It
6067
+ * only ever extends the tail of a LATE attach, by at most one segment plus the
6068
+ * grace, and only for a trigger whose live window was still open AT ATTACH — the
6069
+ * precise statement of "if the live gate said yes, the keep gate honours the
6070
+ * first segment that decision produces". A stale trigger extends nothing.
6071
+ */
6072
+ function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers, attach) {
5847
6073
  const { preMs, postMs } = resolveBandBufferMs(band);
5848
- const overlaps = (lastMs) => lastMs != null && segStartMs <= lastMs + postMs && segEndMs >= lastMs - preMs;
6074
+ const deadlineMs = attachRetainDeadlineMs(attach);
6075
+ const overlaps = (lastMs) => {
6076
+ if (lastMs == null) return false;
6077
+ if (segEndMs < lastMs - preMs) return false;
6078
+ if (segStartMs <= lastMs + postMs) return true;
6079
+ return attach.attachedAtMs <= lastMs + postMs && segStartMs <= deadlineMs;
6080
+ };
5849
6081
  if (band.triggers?.motion && overlaps(triggers.lastMotionMs)) return true;
5850
6082
  if (band.triggers?.audioThresholdDbfs != null && overlaps(triggers.lastAudioMs)) return true;
5851
6083
  return false;
5852
6084
  }
6085
+ /**
6086
+ * By how many ms a discarded segment missed being kept — its start past the
6087
+ * latest start that WOULD have been retained, mirroring
6088
+ * {@link segmentRetainedForEvents}'s upper bound exactly (post-buffer end, or
6089
+ * the attach allowance for a trigger that was still live at attach). Null when
6090
+ * no qualifying trigger exists at all, which is a different fact and says so.
6091
+ *
6092
+ * Negative means the segment was dropped on the PRE-buffer side (it ended
6093
+ * before `trigger - preBufferSec`), not for being late.
6094
+ *
6095
+ * Reporting-only. The discard branch has to name a number: "outside the window"
6096
+ * with no magnitude made a 2.5 s miss and a 10-minute miss indistinguishable.
6097
+ */
6098
+ function segmentMissedByMs(segStartMs, band, triggers, attach) {
6099
+ const { postMs } = resolveBandBufferMs(band);
6100
+ const deadlineMs = attachRetainDeadlineMs(attach);
6101
+ const latestKeptStart = (lastMs) => {
6102
+ if (lastMs == null) return null;
6103
+ const windowEndMs = lastMs + postMs;
6104
+ return attach.attachedAtMs <= windowEndMs ? Math.max(windowEndMs, deadlineMs) : windowEndMs;
6105
+ };
6106
+ const bounds = [band.triggers?.motion === true ? latestKeptStart(triggers.lastMotionMs) : null, band.triggers?.audioThresholdDbfs != null ? latestKeptStart(triggers.lastAudioMs) : null].filter((ms) => ms != null);
6107
+ if (bounds.length === 0) return null;
6108
+ return segStartMs - Math.max(...bounds);
6109
+ }
6110
+ /** Local-calendar instant for `HH:MM` on the given local Y/M/D (day may overflow). */
6111
+ function localInstantMs(year, month, day, hhmm) {
6112
+ const [hours, minutes] = hhmm.split(":");
6113
+ return new Date(year, month, day, Number(hours), Number(minutes), 0, 0).getTime();
6114
+ }
6115
+ /**
6116
+ * Every instant inside the horizon at which `bandActiveAt` COULD change value:
6117
+ * each band's start and end on each local day, plus each local midnight (the
6118
+ * `days` membership boundary). A superset — cheap to compute (a few dozen
6119
+ * entries) and filtered against `activeBandAt` below.
6120
+ */
6121
+ function candidateEdgesMs(bands, nowMs) {
6122
+ const base = new Date(nowMs);
6123
+ const year = base.getFullYear();
6124
+ const month = base.getMonth();
6125
+ const day = base.getDate();
6126
+ const out = [];
6127
+ for (let offset = 0; offset <= 8; offset++) {
6128
+ out.push(new Date(year, month, day + offset, 0, 0, 0, 0).getTime());
6129
+ for (const band of bands) {
6130
+ out.push(localInstantMs(year, month, day + offset, band.start));
6131
+ out.push(localInstantMs(year, month, day + offset, band.end));
6132
+ }
6133
+ }
6134
+ return out;
6135
+ }
6136
+ /**
6137
+ * The next instant at which a DIFFERENT band (or no band) covers the clock, or
6138
+ * `null` when none exists inside {@link BAND_EDGE_HORIZON_DAYS} — the always-on
6139
+ * band (`start === end`, empty `days`) being the common case that legitimately
6140
+ * has no edge at all.
6141
+ *
6142
+ * Identity is by band REFERENCE, which is what makes this exact: two distinct
6143
+ * band objects with the same fields yield an edge that changes nothing, which
6144
+ * costs one wasted wake-up and never costs footage.
6145
+ */
6146
+ function nextBandEdgeMs(config, nowMs) {
6147
+ const bands = config.bands;
6148
+ if (!bands || bands.length === 0) return null;
6149
+ const current = activeBandAt({ bands }, new Date(nowMs));
6150
+ const candidates = [...new Set(candidateEdgesMs(bands, nowMs))].filter((ms) => ms > nowMs).toSorted((a, b) => a - b);
6151
+ for (const ms of candidates) if (activeBandAt({ bands }, new Date(ms)) !== current) return ms;
6152
+ return null;
6153
+ }
6154
+ /**
6155
+ * The first instant an `events` band's demand window is CLOSED, given the
6156
+ * triggers it listens for, or `null` when no window is open at `nowMs`.
6157
+ *
6158
+ * Demand is an OR across the listened trigger kinds, so an open window ends
6159
+ * with the LATEST of them — a motion trigger followed by an audio trigger 5 s
6160
+ * later keeps the camera attached until the audio one expires, exactly as the
6161
+ * sliding window already behaved.
6162
+ */
6163
+ function eventsWindowCloseMs(band, triggers, nowMs) {
6164
+ if (band.mode !== "events") return null;
6165
+ const { postMs } = resolveBandBufferMs(band);
6166
+ const ends = [];
6167
+ if (band.triggers?.motion === true && triggers.lastMotionMs != null) ends.push(triggers.lastMotionMs + postMs);
6168
+ if (band.triggers?.audioThresholdDbfs != null && triggers.lastAudioMs != null) ends.push(triggers.lastAudioMs + postMs);
6169
+ const open = ends.filter((ms) => ms >= nowMs);
6170
+ if (open.length === 0) return null;
6171
+ return Math.max(...open) + 1;
6172
+ }
6173
+ /**
6174
+ * THE next instant the controller must re-evaluate this device, or `null` when
6175
+ * nothing can change without an event the controller already receives (a
6176
+ * trigger or a config write).
6177
+ *
6178
+ * Whichever of the two sources comes first wins, and the reason is carried so
6179
+ * the wake-up is attributable in the log without correlating timestamps.
6180
+ */
6181
+ function nextDecisionChangeAt(input) {
6182
+ const { config, triggers, nowMs } = input;
6183
+ if (!config.enabled) return null;
6184
+ const bands = config.bands;
6185
+ if (!bands || bands.length === 0) return null;
6186
+ const band = activeBandAt({ bands }, new Date(nowMs));
6187
+ const edgeMs = nextBandEdgeMs(config, nowMs);
6188
+ const closeMs = band === null ? null : eventsWindowCloseMs(band, triggers, nowMs);
6189
+ if (closeMs === null) return edgeMs === null ? null : {
6190
+ atMs: edgeMs,
6191
+ reason: "band-edge"
6192
+ };
6193
+ if (edgeMs === null || closeMs <= edgeMs) return {
6194
+ atMs: closeMs,
6195
+ reason: "window-close"
6196
+ };
6197
+ return {
6198
+ atMs: edgeMs,
6199
+ reason: "band-edge"
6200
+ };
6201
+ }
6202
+ //#endregion
6203
+ //#region src/recorder/addon/device-wakeups.ts
6204
+ /** Node truncates a `setTimeout` delay past this to 1 ms — clamp instead. */
6205
+ var MAX_TIMER_DELAY_MS = 2147483647;
6206
+ var DeviceWakeups = class {
6207
+ deps;
6208
+ timers = /* @__PURE__ */ new Map();
6209
+ constructor(deps) {
6210
+ this.deps = deps;
6211
+ }
6212
+ /**
6213
+ * Wake this device at `atMs`, replacing any wake-up already armed for it.
6214
+ * `reason` is carried into the log line so a wake is attributable without
6215
+ * correlating it against the band model by hand.
6216
+ */
6217
+ arm(deviceId, atMs, reason) {
6218
+ this.cancel(deviceId);
6219
+ const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, atMs - this.deps.now()));
6220
+ const timer = setTimeout(() => {
6221
+ this.timers.delete(deviceId);
6222
+ this.deps.fire(deviceId);
6223
+ }, delayMs);
6224
+ timer.unref?.();
6225
+ this.timers.set(deviceId, timer);
6226
+ this.deps.logger.debug("recorder: device wake-up armed", {
6227
+ tags: { deviceId },
6228
+ meta: {
6229
+ atMs,
6230
+ delayMs,
6231
+ reason
6232
+ }
6233
+ });
6234
+ }
6235
+ /** Drop this device's pending wake-up, if any. Idempotent. */
6236
+ cancel(deviceId) {
6237
+ const timer = this.timers.get(deviceId);
6238
+ if (timer === void 0) return;
6239
+ clearTimeout(timer);
6240
+ this.timers.delete(deviceId);
6241
+ }
6242
+ /** Drop every pending wake-up — shutdown and maintenance pause. */
6243
+ cancelAll() {
6244
+ for (const timer of this.timers.values()) clearTimeout(timer);
6245
+ this.timers.clear();
6246
+ }
6247
+ /** Devices with a wake-up pending — for tests and for shutdown assertions. */
6248
+ get pendingCount() {
6249
+ return this.timers.size;
6250
+ }
6251
+ };
5853
6252
  //#endregion
5854
6253
  //#region src/recorder/addon/periodic-pass.ts
5855
6254
  function startPeriodicPass(deps) {
@@ -5916,7 +6315,9 @@ var ReadinessRestore = class {
5916
6315
  this.deps = deps;
5917
6316
  }
5918
6317
  get concurrency() {
5919
- return Math.max(1, this.deps.concurrency ?? 8);
6318
+ const declared = this.deps.concurrency;
6319
+ const value = typeof declared === "function" ? declared() : declared;
6320
+ return Math.max(1, value ?? 8);
5920
6321
  }
5921
6322
  /** Seed the pending set, subscribe to readiness, and drain once immediately
5922
6323
  * (covers the already-ready case). Returns when the first drain settles. */
@@ -5975,6 +6376,70 @@ var ReadinessRestore = class {
5975
6376
  }
5976
6377
  }
5977
6378
  };
6379
+ /** Adaptive bound for the recorder's attach parallelism. */
6380
+ var SpawnPressureGovernor = class {
6381
+ deps;
6382
+ limit;
6383
+ min;
6384
+ recoverAfterMs;
6385
+ /** When the limit last MOVED — the clock both directions are measured from. */
6386
+ lastChangeAt;
6387
+ underPressure = false;
6388
+ constructor(deps) {
6389
+ this.deps = deps;
6390
+ this.min = Math.max(1, deps.minConcurrency ?? 1);
6391
+ this.limit = Math.max(this.min, deps.maxConcurrency);
6392
+ this.recoverAfterMs = deps.recoverAfterMs ?? 3e4;
6393
+ this.lastChangeAt = this.now();
6394
+ }
6395
+ /**
6396
+ * How many attaches may overlap right now.
6397
+ *
6398
+ * Reading is what drives recovery: there is no timer to leak, no interval to
6399
+ * lose to a runner respawn, and a recorder that stops attaching entirely
6400
+ * simply keeps its reduced bound until it tries again — which is correct.
6401
+ */
6402
+ get concurrency() {
6403
+ const max = Math.max(this.min, this.deps.maxConcurrency);
6404
+ if (this.limit >= max) return max;
6405
+ const elapsed = this.now() - this.lastChangeAt;
6406
+ if (elapsed < this.recoverAfterMs) return this.limit;
6407
+ const lanes = Math.floor(elapsed / this.recoverAfterMs);
6408
+ this.limit = Math.min(max, this.limit + lanes);
6409
+ this.lastChangeAt += lanes * this.recoverAfterMs;
6410
+ if (this.limit >= max && this.underPressure) {
6411
+ this.underPressure = false;
6412
+ this.deps.logger?.info("recorder: spawn pressure cleared — attach concurrency restored", { meta: { concurrency: this.limit } });
6413
+ } else this.deps.logger?.info("recorder: recovering attach concurrency", { meta: {
6414
+ concurrency: this.limit,
6415
+ max
6416
+ } });
6417
+ return this.limit;
6418
+ }
6419
+ /**
6420
+ * A transient spawn failure happened on `deviceId`'s writer.
6421
+ *
6422
+ * Halves the CURRENT limit, not the max: two waves of pressure inside one
6423
+ * recovery window must compound, or a host under sustained load is asked for
6424
+ * the same too-large batch over and over.
6425
+ */
6426
+ report(deviceId) {
6427
+ const before = this.limit;
6428
+ this.limit = Math.max(this.min, Math.floor(this.limit / 2));
6429
+ this.lastChangeAt = this.now();
6430
+ this.underPressure = true;
6431
+ this.deps.logger?.warn("recorder: spawn pressure — reducing attach concurrency", {
6432
+ tags: { deviceId },
6433
+ meta: {
6434
+ from: before,
6435
+ to: this.limit
6436
+ }
6437
+ });
6438
+ }
6439
+ now() {
6440
+ return this.deps.now?.() ?? Date.now();
6441
+ }
6442
+ };
5978
6443
  //#endregion
5979
6444
  //#region src/recorder/addon/segment-watcher.ts
5980
6445
  /**
@@ -6360,36 +6825,87 @@ var SegmentWriter = class {
6360
6825
  if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
6361
6826
  }
6362
6827
  });
6828
+ let settled = false;
6829
+ const terminate = (termination) => {
6830
+ if (settled) return;
6831
+ settled = true;
6832
+ this.onTermination(termination, stderrTail);
6833
+ };
6363
6834
  proc.on("exit", (code) => {
6364
- this.resolveExit?.();
6365
- this.resolveExit = null;
6366
- this.proc = null;
6367
- if (this.stopped) return;
6368
- const ranMs = Date.now() - this.startedAt;
6369
- if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
6370
- if (this.restarts >= MAX_RESTARTS) {
6371
- this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
6835
+ terminate({
6836
+ reason: "exit",
6837
+ exitCode: typeof code === "number" ? code : null
6838
+ });
6839
+ });
6840
+ proc.on("error", (raw) => {
6841
+ const failure = readSpawnFailure(raw);
6842
+ terminate({
6843
+ reason: "spawn-error",
6844
+ errorCode: failure.code,
6845
+ message: failure.message,
6846
+ permanent: failure.permanent
6847
+ });
6848
+ });
6849
+ }
6850
+ /**
6851
+ * The ONE restart policy, reached from both `exit` and `error`.
6852
+ *
6853
+ * `deps.logger` is the device-scoped child the controller binds
6854
+ * (`logger.withTags({ deviceId })`), so every line below carries
6855
+ * `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
6856
+ * camera, and a writer that stops recording one camera must be greppable by
6857
+ * that camera.
6858
+ */
6859
+ onTermination(termination, stderrTail) {
6860
+ this.resolveExit?.();
6861
+ this.resolveExit = null;
6862
+ this.proc = null;
6863
+ if (this.stopped) return;
6864
+ const ranMs = Date.now() - this.startedAt;
6865
+ if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
6866
+ if (termination.reason === "spawn-error") {
6867
+ this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
6868
+ outDir: this.cfg.outDir,
6869
+ errorCode: termination.errorCode,
6870
+ error: termination.message,
6871
+ permanent: termination.permanent
6872
+ } });
6873
+ if (termination.permanent) {
6874
+ this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
6372
6875
  outDir: this.cfg.outDir,
6373
- code,
6374
- stderrTail: stderrTail.join(" | ")
6876
+ errorCode: termination.errorCode,
6877
+ error: termination.message
6375
6878
  } });
6376
6879
  this.stopped = true;
6377
6880
  this.deps.onGaveUp?.();
6378
6881
  return;
6379
6882
  }
6380
- this.restarts++;
6381
- const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
6382
- this.deps.logger.info("SegmentWriter restarting", { meta: {
6883
+ this.deps.onResourcePressure?.();
6884
+ }
6885
+ if (this.restarts >= MAX_RESTARTS) {
6886
+ this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
6383
6887
  outDir: this.cfg.outDir,
6384
- attempt: this.restarts,
6385
- delayMs,
6386
- ranMs
6888
+ reason: termination.reason,
6889
+ code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
6890
+ stderrTail: stderrTail.join(" | ")
6387
6891
  } });
6388
- this.restartTimer = setTimeout(() => {
6389
- this.restartTimer = null;
6390
- this.start();
6391
- }, delayMs);
6392
- });
6892
+ this.stopped = true;
6893
+ this.deps.onGaveUp?.();
6894
+ return;
6895
+ }
6896
+ this.restarts++;
6897
+ const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
6898
+ this.deps.logger.info("SegmentWriter restarting", { meta: {
6899
+ outDir: this.cfg.outDir,
6900
+ attempt: this.restarts,
6901
+ delayMs,
6902
+ ranMs,
6903
+ reason: termination.reason
6904
+ } });
6905
+ this.restartTimer = setTimeout(() => {
6906
+ this.restartTimer = null;
6907
+ this.start();
6908
+ }, delayMs);
6393
6909
  }
6394
6910
  stop() {
6395
6911
  this.stopAndWait();
@@ -6433,19 +6949,37 @@ var SegmentWriter = class {
6433
6949
  * - a `live.m3u8` `SegmentWatcher` that hands each finalized flat segment to
6434
6950
  * the v2 `SegmentStore.onFinalized` (stat + relocate + index).
6435
6951
  *
6436
- * Per device it decides whether a CONTINUOUS band is active right now
6437
- * (`shouldRecordContinuousAt`) and ensures the writer set is running iff so.
6438
- * It re-evaluates on three triggers:
6439
- * 1. `setDeviceConfig` (operator changed the bands / enable),
6440
- * 2. a periodic pass (catches band boundaries crossed with no config change),
6441
- * 3. a `stream-broker` ready transition (boot restore via `ReadinessRestore`).
6442
- *
6443
- * There are TWO periodic passes, on separate timers and never overlapping
6444
- * themselves (`periodic-pass.ts`): a cheap LIVENESS pass that catches writers
6445
- * pinned but producing nothing, and an expensive CONVERGENCE pass that
6446
- * re-evaluates bands and placement. They were one pass until 2026-08-13, when a
6447
- * convergence sweep wedged on unbounded broker RPCs and took the liveness
6448
- * watchdog down with it for 9-27 minutes at a time.
6952
+ * Per device it decides whether the active band demands recording right now
6953
+ * (`shouldRecordAt`) and ensures the writer set is running iff so. Convergence
6954
+ * is EVENT-DRIVEN; the periodic pass is a backstop, not the mechanism:
6955
+ * 1. `setDeviceConfig` (operator changed the bands / enable) — an RPC,
6956
+ * 2. a qualifying motion/audio trigger (`onTrigger`) opens a window,
6957
+ * 3. a per-device WAKE-UP armed for the exact instant the decision could
6958
+ * change on its own (`decision-schedule.ts` + `device-wakeups.ts`): an
6959
+ * `events` window closing at `lastTrigger + postBufferSec`, or a band edge,
6960
+ * 4. a `stream-broker` ready transition (boot restore via `ReadinessRestore`),
6961
+ * 5. the periodic RECONCILE (below), for everything the four above can lose.
6962
+ *
6963
+ * (3) is why a detach is now punctual. A trigger only ever OPENED a window — the
6964
+ * demand test is a sliding `atMs - lastMotionMs <= postMs` anchored on the LAST
6965
+ * trigger, so nothing called `evaluateDevice` at the moment it closed and the
6966
+ * detach waited for the next pass. A 4K camera stayed dialled, decoded and
6967
+ * written for up to a whole tick past the operator's `postBufferSec`.
6968
+ *
6969
+ * There are THREE periodic passes, each on its own timer and each non-overlapping
6970
+ * (`periodic-pass.ts`):
6971
+ * - LIVENESS ({@link LIVENESS_TICK_MS}) — cheap, in-memory: is a pinned writer
6972
+ * producing nothing?
6973
+ * - PLACEMENT ({@link PLACEMENT_TICK_MS}) — cheap (a statfs per location), and
6974
+ * deliberately NOT on the attach beat, so a cold broker cannot delay it.
6975
+ * - RECONCILE ({@link RECONCILE_TICK_MS}) — the only one that can touch the
6976
+ * broker, and the only backstop for a lost wake-up or a failed attach.
6977
+ *
6978
+ * They were ONE pass until 2026-08-13, when a convergence sweep wedged on
6979
+ * unbounded broker RPCs and took the liveness watchdog down with it for 9-27
6980
+ * minutes at a time; placement was split out of the second on 2026-08-18, after
6981
+ * a converge pass blocked for 60 001 ms (3 x {@link ATTACH_RPC_TIMEOUT_MS})
6982
+ * cold-dialling ~40 RTSP sources.
6449
6983
  *
6450
6984
  * EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
6451
6985
  * trigger-gating. The enable intent is persisted in durable-state by the
@@ -6458,8 +6992,45 @@ var SegmentWriter = class {
6458
6992
  * `ReadinessRegistry` is the single source of truth.
6459
6993
  */
6460
6994
  var DEFAULT_WATCH_INTERVAL_MS = 2e3;
6461
- /** Periodic re-evaluation to catch band boundaries crossed with no config change. */
6462
- var TICK_MS = 3e4;
6995
+ /**
6996
+ * Liveness cadence — "is a pinned writer producing nothing?". Unchanged: the
6997
+ * question is an in-memory scan over `active`, it costs nothing, and its bound
6998
+ * (`writerIdleBoundMs`, floor 90 s) is sized against this beat.
6999
+ */
7000
+ var LIVENESS_TICK_MS = 3e4;
7001
+ /**
7002
+ * Placement cadence — unchanged from the beat it used to share with
7003
+ * convergence, because nothing about placement wanted to be rarer. What changed
7004
+ * is that it no longer rides a pass that can block on broker RPCs: it is one
7005
+ * `statfs` per location and it re-plans only, so a plan reaches a camera at its
7006
+ * NEXT attach.
7007
+ */
7008
+ var PLACEMENT_TICK_MS = 3e4;
7009
+ /**
7010
+ * Reconcile cadence — the SAFETY NET, not the mechanism.
7011
+ *
7012
+ * It was 30 s because it WAS the mechanism: nothing else ever closed an `events`
7013
+ * window or crossed a band edge, so the loop had to keep asking. Both are now
7014
+ * armed for their exact instant (`decision-schedule.ts`), a config change is an
7015
+ * RPC, and a broker recovery is a readiness transition — so this pass exists
7016
+ * only for what those cannot cover: a wake-up lost because the event path threw
7017
+ * before it could arm one, an attach that failed with no later ready transition
7018
+ * to ride, a timer lost to a runner respawn, and clock drift.
7019
+ *
7020
+ * 2 minutes, and the number is chosen against a bound this recorder already
7021
+ * accepts. The liveness watchdog tolerates a pinned-but-silent writer for
7022
+ * `WRITER_IDLE_MIN_MS` (90 s) before it recovers it; a device that WANTS
7023
+ * recording but is detached is the same class of fault — footage is not being
7024
+ * written and only a periodic check will notice — so its recovery is put on the
7025
+ * same order rather than an order slower. Four times rarer than the beat that
7026
+ * blocked for 60 001 ms on 2026-08-18, while still bounding the worst case at
7027
+ * ~2 minutes of footage for a camera whose event path failed entirely.
7028
+ *
7029
+ * Going rarer than this trades real footage for CPU that the pass no longer
7030
+ * spends anyway: with the broker-readiness pre-check below, a reconcile over a
7031
+ * healthy fleet is N config reads and no RPC at all.
7032
+ */
7033
+ var RECONCILE_TICK_MS = 12e4;
6463
7034
  /**
6464
7035
  * Bound on the broker lease-release RPC (`releaseStreamWithCodec`) during
6465
7036
  * teardown. The UDS request default is 60s, but the runner supervisor's
@@ -6498,11 +7069,15 @@ var RELEASE_RPC_TIMEOUT_MS = 2e3;
6498
7069
  */
6499
7070
  var ATTACH_RPC_TIMEOUT_MS = 2e4;
6500
7071
  /**
6501
- * A pass longer than this is reported with its duration. Equal to the pass
6502
- * cadence: a pass that cannot finish inside its own interval is the condition
6503
- * that used to go unlogged for half an hour.
7072
+ * A pass longer than this is reported with its duration. Each pass uses its OWN
7073
+ * cadence as the budget: a pass that cannot finish inside its own interval is
7074
+ * the condition that used to go unlogged for half an hour. Keeping it per-pass
7075
+ * matters now that the three cadences differ — a 40 s reconcile is healthy
7076
+ * against a 120 s beat and was a red flag against a 30 s one.
6504
7077
  */
6505
- var PASS_SLOW_AFTER_MS = TICK_MS;
7078
+ function passSlowAfterMs(intervalMs) {
7079
+ return intervalMs;
7080
+ }
6506
7081
  /**
6507
7082
  * Race `promise` against a deadline. Handlers stay attached to the losing
6508
7083
  * promise, so a post-deadline settlement can never surface as a process-level
@@ -6585,6 +7160,14 @@ var RecordingController = class {
6585
7160
  */
6586
7161
  lastSegmentAt = /* @__PURE__ */ new Map();
6587
7162
  /**
7163
+ * Per `deviceId:profile`, the `attachedAtMs` of the attach that has retained
7164
+ * at least one `events` segment. Read only to LEVEL the discard log: an
7165
+ * attach that kept nothing is a wasted pull (warn), a later discard from one
7166
+ * that kept something is the tail of the window (info). Bounded by
7167
+ * device×profile like {@link lastSegmentAt}, and cleared with it on detach.
7168
+ */
7169
+ attachRetained = /* @__PURE__ */ new Map();
7170
+ /**
6588
7171
  * Devices already named by {@link reportUnrecordable} — one warn per device
6589
7172
  * per "enabled but can never record" episode, re-armed as soon as the device
6590
7173
  * records again.
@@ -6598,24 +7181,91 @@ var RecordingController = class {
6598
7181
  */
6599
7182
  skippedProfiles = /* @__PURE__ */ new Map();
6600
7183
  restore = null;
7184
+ /**
7185
+ * One pending wake-up per device, armed for the exact instant its recording
7186
+ * decision could change on its own — an `events` window closing or a band
7187
+ * edge. THE reason a detach is punctual instead of up to a pass late.
7188
+ */
7189
+ wakeups;
6601
7190
  /** Cheap, never-delayed: is any pinned writer producing nothing? */
6602
7191
  livenessPass = null;
6603
- /** Expensive: re-evaluate bands + placement for every configured device. */
6604
- convergePass = null;
7192
+ /** Cheap, and off the attach beat: re-plan where each (device, profile) writes. */
7193
+ placementPass = null;
7194
+ /** The backstop: re-evaluate every tracked-or-configured device. */
7195
+ reconcilePass = null;
6605
7196
  stopped = false;
6606
7197
  /** A reversible maintenance lease. Unlike `stopped`, it never changes
6607
7198
  * persisted recording intent and `resume()` restarts normal convergence. */
6608
7199
  paused = false;
7200
+ /**
7201
+ * The ADAPTIVE attach bound. A transient `spawn ffmpeg` failure (EAGAIN) is
7202
+ * the host saying it cannot fork right now, and the answer is to attach
7203
+ * FEWER cameras at once — not to serialise them, which `e2fa17a2b` removed
7204
+ * for good reason. Every bounded-parallel attach in this file reads it.
7205
+ */
7206
+ spawnPressure;
6609
7207
  constructor(deps) {
6610
7208
  this.deps = deps;
7209
+ this.spawnPressure = new SpawnPressureGovernor({
7210
+ maxConcurrency: 8,
7211
+ now: () => this.now(),
7212
+ logger: deps.logger
7213
+ });
7214
+ this.wakeups = new DeviceWakeups({
7215
+ logger: deps.logger,
7216
+ now: () => this.now(),
7217
+ fire: (deviceId) => this.onWakeup(deviceId)
7218
+ });
6611
7219
  }
6612
7220
  now() {
6613
7221
  return this.deps.now?.() ?? Date.now();
6614
7222
  }
6615
7223
  /**
7224
+ * A per-device wake-up came due: re-decide. `evaluateDevice` re-arms (or
7225
+ * cancels) the next one as part of deciding, so the chain is self-sustaining.
7226
+ *
7227
+ * A throw here breaks that chain — the device keeps whatever state it had and
7228
+ * has no timer behind it — so it is NAMED, and the reconcile is what picks it
7229
+ * up. Silence would read as "the window never closed".
7230
+ */
7231
+ onWakeup(deviceId) {
7232
+ this.evaluateDevice(deviceId).catch((err) => {
7233
+ this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
7234
+ tags: { deviceId },
7235
+ meta: { error: require_dist.errMsg(err) }
7236
+ });
7237
+ });
7238
+ }
7239
+ /**
7240
+ * Arm (or cancel) this device's next wake-up from the config it was just
7241
+ * evaluated against.
7242
+ *
7243
+ * Called from `evaluateDevice` BEFORE the attach/detach work, not after: the
7244
+ * attach chain can throw (broker not ready, stream unpublished) and the
7245
+ * schedule does not depend on whether it succeeded. Arming first means a
7246
+ * failed attach still gets its band edge, and only a failure to READ the
7247
+ * config can lose a wake-up.
7248
+ */
7249
+ scheduleNextDecision(deviceId, config) {
7250
+ if (this.stopped || this.paused) {
7251
+ this.wakeups.cancel(deviceId);
7252
+ return;
7253
+ }
7254
+ const next = nextDecisionChangeAt({
7255
+ config,
7256
+ triggers: this.triggersFor(deviceId),
7257
+ nowMs: this.now()
7258
+ });
7259
+ if (next === null) {
7260
+ this.wakeups.cancel(deviceId);
7261
+ return;
7262
+ }
7263
+ this.wakeups.arm(deviceId, next.atMs, next.reason);
7264
+ }
7265
+ /**
6616
7266
  * Boot restore: seed the readiness-gated queue with every persisted device and
6617
7267
  * (re)evaluate each on every `stream-broker` ready transition. Also arms the
6618
- * periodic tick. Idempotent — safe to call once after the index is hydrated.
7268
+ * periodic passes. Idempotent — safe to call once after the index is hydrated.
6619
7269
  */
6620
7270
  async start() {
6621
7271
  if (this.stopped) return;
@@ -6628,6 +7278,7 @@ var RecordingController = class {
6628
7278
  }
6629
7279
  this.restore = new ReadinessRestore({
6630
7280
  subscribeReady: (handler) => this.deps.onBrokerReady(handler),
7281
+ concurrency: () => this.spawnPressure.concurrency,
6631
7282
  attempt: async (deviceId) => {
6632
7283
  await this.evaluateDevice(deviceId);
6633
7284
  },
@@ -6636,45 +7287,83 @@ var RecordingController = class {
6636
7287
  meta: { error: require_dist.errMsg(err) }
6637
7288
  })
6638
7289
  });
6639
- if (this.convergePass === null) {
7290
+ if (this.reconcilePass === null) {
6640
7291
  this.livenessPass = startPeriodicPass({
6641
7292
  name: "liveness",
6642
- intervalMs: TICK_MS,
6643
- slowAfterMs: PASS_SLOW_AFTER_MS,
7293
+ intervalMs: LIVENESS_TICK_MS,
7294
+ slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
6644
7295
  logger: this.deps.logger,
6645
7296
  now: () => this.now(),
6646
7297
  run: () => this.checkIdleWriters()
6647
7298
  });
6648
- this.convergePass = startPeriodicPass({
6649
- name: "convergence",
6650
- intervalMs: TICK_MS,
6651
- slowAfterMs: PASS_SLOW_AFTER_MS,
7299
+ this.placementPass = startPeriodicPass({
7300
+ name: "placement",
7301
+ intervalMs: PLACEMENT_TICK_MS,
7302
+ slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
7303
+ logger: this.deps.logger,
7304
+ now: () => this.now(),
7305
+ run: () => this.replanPlacement()
7306
+ });
7307
+ this.reconcilePass = startPeriodicPass({
7308
+ name: "reconcile",
7309
+ intervalMs: RECONCILE_TICK_MS,
7310
+ slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
6652
7311
  logger: this.deps.logger,
6653
7312
  now: () => this.now(),
6654
- run: () => this.converge()
7313
+ run: () => this.reconcile()
6655
7314
  });
6656
7315
  }
6657
7316
  await this.restore.start(ids.map((id) => [id, true]));
6658
7317
  }
6659
- /** Re-evaluate every currently-tracked OR active device on the periodic pass. */
6660
- async converge() {
6661
- if (this.stopped) return;
7318
+ /**
7319
+ * Every device the recorder is responsible for right now: those with writers
7320
+ * attached, plus every one with a persisted config. A failure to enumerate the
7321
+ * persisted set degrades to "the active ones" rather than aborting the pass.
7322
+ */
7323
+ async trackedDeviceIds() {
6662
7324
  const ids = new Set(this.active.keys());
6663
- let persisted = [];
6664
7325
  try {
6665
- persisted = await this.deps.enabledDeviceIds();
6666
- } catch {}
6667
- for (const id of persisted) ids.add(id);
7326
+ for (const id of await this.deps.enabledDeviceIds()) ids.add(id);
7327
+ } catch (err) {
7328
+ this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
7329
+ error: require_dist.errMsg(err),
7330
+ activeDevices: this.active.size
7331
+ } });
7332
+ }
7333
+ return [...ids];
7334
+ }
7335
+ /**
7336
+ * Re-plan where each (device, profile) writes. Its OWN pass: cheap (no I/O
7337
+ * beyond a statfs per location), and it must not be starved by an attach that
7338
+ * is waiting on a 20 s broker timeout. It changes the PLAN only — a running
7339
+ * writer keeps its root until it next attaches, which is the boundary rule.
7340
+ */
7341
+ async replanPlacement() {
7342
+ if (this.stopped || this.paused) return;
7343
+ if (this.deps.onPlacementTick === void 0) return;
7344
+ const ids = await this.trackedDeviceIds();
6668
7345
  try {
6669
- await this.deps.onPlacementTick?.([...ids]);
7346
+ await this.deps.onPlacementTick(ids);
6670
7347
  } catch (err) {
6671
7348
  this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: require_dist.errMsg(err) } });
6672
7349
  }
6673
- await runBounded([...ids], 8, async (id) => {
7350
+ }
7351
+ /**
7352
+ * The backstop pass: re-evaluate every tracked-or-configured device.
7353
+ *
7354
+ * Nothing here is the normal path any more — a window close, a band edge, a
7355
+ * config change and a broker recovery all reach `evaluateDevice` on their own.
7356
+ * This exists for the four things that cannot: a wake-up the event path threw
7357
+ * before arming, an attach that failed with no later readiness transition to
7358
+ * ride, a timer lost to a runner respawn, and clock drift.
7359
+ */
7360
+ async reconcile() {
7361
+ if (this.stopped) return;
7362
+ await runBounded(await this.trackedDeviceIds(), this.spawnPressure.concurrency, async (id) => {
6674
7363
  try {
6675
7364
  await this.evaluateDevice(id);
6676
7365
  } catch (err) {
6677
- this.deps.logger.warn("recorder controller: convergence evaluate failed", {
7366
+ this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
6678
7367
  tags: { deviceId: id },
6679
7368
  meta: { error: require_dist.errMsg(err) }
6680
7369
  });
@@ -6698,8 +7387,9 @@ var RecordingController = class {
6698
7387
  * idle source can re-dial cleanly), then re-evaluate: if the band still demands
6699
7388
  * recording, `evaluateDevice` re-queues + re-attaches a fresh writer set. If
6700
7389
  * the broker is not ready, the re-attach throws and the device stays on the
6701
- * readiness queue (retried on the next broker ready) — and the periodic tick is
6702
- * the backstop. Bounded OUTER retry (TICK_MS cadence), never a tight loop.
7390
+ * readiness queue (retried on the next broker ready) — and the periodic
7391
+ * reconcile is the backstop. Bounded OUTER retry (RECONCILE_TICK_MS cadence),
7392
+ * never a tight loop.
6703
7393
  */
6704
7394
  async recoverWriterGaveUp(deviceId) {
6705
7395
  if (this.stopped) return;
@@ -6709,7 +7399,7 @@ var RecordingController = class {
6709
7399
  await this.evaluateDevice(deviceId);
6710
7400
  } catch (err) {
6711
7401
  this.restore?.add(deviceId, true);
6712
- this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/tick)", {
7402
+ this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
6713
7403
  tags: { deviceId },
6714
7404
  meta: { error: require_dist.errMsg(err) }
6715
7405
  });
@@ -6760,10 +7450,16 @@ var RecordingController = class {
6760
7450
  }
6761
7451
  /**
6762
7452
  * Record a qualifying trigger (motion/audio) for a device and converge: an
6763
- * `events` band whose window this opens attaches now; the periodic tick
6764
- * detaches it once `postBufferSec` elapses. Called from the addon's
6765
- * EventCapture subscription. The qualification test (motion detected / audio
6766
- * over threshold) is done upstream by the time we're here, it qualified.
7453
+ * `events` band whose window this opens attaches now, and the same
7454
+ * `evaluateDevice` arms the wake-up that will CLOSE it at
7455
+ * `atMs + postBufferSec`. Called from the addon's EventCapture subscription.
7456
+ * The qualification test (motion detected / audio over threshold) is done
7457
+ * upstream — by the time we're here, it qualified.
7458
+ *
7459
+ * A fresher trigger re-arms the wake-up later, which is exactly how the
7460
+ * sliding window already behaved: continuous motion stays ONE recording,
7461
+ * because `evaluateDevice` short-circuits at "already recording — converged"
7462
+ * and only the timer moves.
6767
7463
  */
6768
7464
  async onTrigger(deviceId, source, atMs) {
6769
7465
  if (this.stopped) return;
@@ -6775,7 +7471,7 @@ var RecordingController = class {
6775
7471
  try {
6776
7472
  await this.evaluateDevice(deviceId);
6777
7473
  } catch (err) {
6778
- this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/tick)", {
7474
+ this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/reconcile)", {
6779
7475
  tags: { deviceId },
6780
7476
  meta: {
6781
7477
  source,
@@ -6796,7 +7492,9 @@ var RecordingController = class {
6796
7492
  if (this.stopped || this.paused) return;
6797
7493
  const config = await this.deps.loadConfig(deviceId);
6798
7494
  if (this.stopped || this.paused) return;
6799
- if (!shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()))) {
7495
+ const wantRecording = shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()));
7496
+ this.scheduleNextDecision(deviceId, config);
7497
+ if (!wantRecording) {
6800
7498
  this.reportUnrecordable(deviceId, config);
6801
7499
  await this.detachDevice(deviceId);
6802
7500
  this.restore?.remove(deviceId);
@@ -6855,6 +7553,15 @@ var RecordingController = class {
6855
7553
  });
6856
7554
  }
6857
7555
  /**
7556
+ * Throw before spending a single broker RPC when the registry already says the
7557
+ * `stream-broker` on the owner node is not ready. See
7558
+ * {@link RecordingControllerDeps.brokerReady}; absent → assume ready.
7559
+ */
7560
+ failFastIfBrokerCold(deviceId) {
7561
+ if (this.deps.brokerReady?.() !== false) return;
7562
+ throw new Error(`recorder controller: stream-broker not ready on ${this.deps.ownerNodeId} — attach for device ${deviceId} deferred to the next ready transition`);
7563
+ }
7564
+ /**
6858
7565
  * The attach body — extracted so the {@link attaching} in-flight guard in
6859
7566
  * `evaluateDevice` wraps it cleanly. Resolves the profiles, spawns a
6860
7567
  * SegmentWriter + watcher per profile, and records the set in `active`.
@@ -6866,6 +7573,7 @@ var RecordingController = class {
6866
7573
  */
6867
7574
  async performAttach(deviceId, config) {
6868
7575
  this.restore?.add(deviceId, true);
7576
+ this.failFastIfBrokerCold(deviceId);
6869
7577
  const profiles = await this.resolveProfiles(deviceId, config.profiles);
6870
7578
  if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
6871
7579
  const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
@@ -6923,6 +7631,16 @@ var RecordingController = class {
6923
7631
  }
6924
7632
  }
6925
7633
  async runFillMissingProfiles(deviceId, config, current, skipped) {
7634
+ if (this.deps.brokerReady?.() === false) {
7635
+ this.deps.logger.info("recorder: skipped-profile retry deferred — stream-broker not ready (healthy profiles keep recording)", {
7636
+ tags: { deviceId },
7637
+ meta: {
7638
+ profiles: [...skipped],
7639
+ ownerNodeId: this.deps.ownerNodeId
7640
+ }
7641
+ });
7642
+ return;
7643
+ }
6926
7644
  const have = new Set(current.map((r) => r.profile));
6927
7645
  const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
6928
7646
  const added = [];
@@ -7000,7 +7718,8 @@ var RecordingController = class {
7000
7718
  logger: deviceLog,
7001
7719
  onGaveUp: () => {
7002
7720
  this.recoverWriterGaveUp(deviceId);
7003
- }
7721
+ },
7722
+ onResourcePressure: () => this.spawnPressure.report(deviceId)
7004
7723
  });
7005
7724
  const writerStartedMs = this.now();
7006
7725
  writer.start();
@@ -7015,7 +7734,10 @@ var RecordingController = class {
7015
7734
  intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
7016
7735
  onFinalized: async (startMs, durMs, flatAbsPath) => {
7017
7736
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
7018
- await this.handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath);
7737
+ await this.handleFinalizedSegment(deviceId, profile, placement, {
7738
+ attachedAtMs: writerStartedMs,
7739
+ segmentSeconds
7740
+ }, startMs, durMs, flatAbsPath);
7019
7741
  },
7020
7742
  logger: deviceLog
7021
7743
  }),
@@ -7032,13 +7754,16 @@ var RecordingController = class {
7032
7754
  * keep/discard gate touches `events`-mode segments only, so the continuous
7033
7755
  * path is unchanged.
7034
7756
  */
7035
- async handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath) {
7757
+ async handleFinalizedSegment(deviceId, profile, placement, attach, startMs, durMs, flatAbsPath) {
7036
7758
  try {
7037
7759
  const band = activeBandAt({ bands: (await this.deps.loadConfig(deviceId)).bands ?? [] }, new Date(startMs + durMs));
7038
- if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, this.triggersFor(deviceId))) {
7760
+ const triggers = this.triggersFor(deviceId);
7761
+ if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, triggers, attach)) {
7762
+ this.reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs);
7039
7763
  await node_fs.promises.unlink(flatAbsPath).catch(() => {});
7040
7764
  return;
7041
7765
  }
7766
+ if (band?.mode === "events") this.attachRetained.set(idleKey(deviceId, profile), attach.attachedAtMs);
7042
7767
  await this.deps.segmentStore.onFinalized({
7043
7768
  deviceId,
7044
7769
  profile,
@@ -7059,6 +7784,47 @@ var RecordingController = class {
7059
7784
  }
7060
7785
  }
7061
7786
  /**
7787
+ * Name a discarded `events`-band segment. This branch deletes recorded
7788
+ * footage and used to write NOTHING: on camera 1439 the operator saw `pinned`
7789
+ * then `unpinned` with an empty timeline in between and no line anywhere said
7790
+ * why — the recorder had spawned ffmpeg, pulled 4K RTSP for 31 s and unlinked
7791
+ * every segment it produced.
7792
+ *
7793
+ * Level is chosen, not defaulted. A discard at the TAIL of a window is the
7794
+ * feature working (`info`) — the writer keeps running until the next converge
7795
+ * tick and those trailing segments were never asked for. A discard from an
7796
+ * attach that has retained NOTHING is a fault (`warn`): the whole pull was
7797
+ * wasted, which is exactly the shape of the 1439 loss and of any future
7798
+ * regression in the attach allowance.
7799
+ */
7800
+ reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs) {
7801
+ const { preMs, postMs } = resolveBandBufferMs(band);
7802
+ const keptAnything = this.attachRetained.get(idleKey(deviceId, profile)) === attach.attachedAtMs;
7803
+ const meta = {
7804
+ profile,
7805
+ startMs,
7806
+ durMs,
7807
+ lastMotionMs: triggers.lastMotionMs,
7808
+ lastAudioMs: triggers.lastAudioMs,
7809
+ postMs,
7810
+ preMs,
7811
+ attachedAtMs: attach.attachedAtMs,
7812
+ missedByMs: segmentMissedByMs(startMs, band, triggers, attach),
7813
+ keptAnythingFromThisAttach: keptAnything
7814
+ };
7815
+ if (keptAnything) {
7816
+ this.deps.logger.info("recorder: discarding events segment outside every trigger window (tail of the window)", {
7817
+ tags: { deviceId },
7818
+ meta
7819
+ });
7820
+ return;
7821
+ }
7822
+ this.deps.logger.warn("recorder: discarding events segment — this attach has retained NOTHING, the whole pull is wasted", {
7823
+ tags: { deviceId },
7824
+ meta
7825
+ });
7826
+ }
7827
+ /**
7062
7828
  * Resolve the storage location a (device, profile) writes to, at an attach
7063
7829
  * boundary. Delegates to the injected {@link RecordingControllerDeps.placeProfile}
7064
7830
  * when the addon supplied one; otherwise to the pure `resolvePlacement` in
@@ -7100,7 +7866,10 @@ var RecordingController = class {
7100
7866
  if (!recordings) return;
7101
7867
  this.active.delete(deviceId);
7102
7868
  this.skippedProfiles.delete(deviceId);
7103
- for (const r of recordings) this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
7869
+ for (const r of recordings) {
7870
+ this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
7871
+ this.attachRetained.delete(idleKey(deviceId, r.profile));
7872
+ }
7104
7873
  await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
7105
7874
  this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
7106
7875
  tags: { deviceId },
@@ -7111,6 +7880,7 @@ var RecordingController = class {
7111
7880
  async pause() {
7112
7881
  if (this.stopped || this.paused) return;
7113
7882
  this.paused = true;
7883
+ this.wakeups.cancelAll();
7114
7884
  while (this.attachingTasks.size > 0) await Promise.allSettled([...this.attachingTasks.values()]);
7115
7885
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
7116
7886
  }
@@ -7126,13 +7896,16 @@ var RecordingController = class {
7126
7896
  }
7127
7897
  for (const deviceId of ids) await this.evaluateDevice(deviceId);
7128
7898
  }
7129
- /** Tear EVERYTHING down: timer, restore subscription, every writer/watcher/lease. */
7899
+ /** Tear EVERYTHING down: timers, restore subscription, every writer/watcher/lease. */
7130
7900
  async stop() {
7131
7901
  this.stopped = true;
7902
+ this.wakeups.cancelAll();
7132
7903
  this.livenessPass?.stop();
7133
7904
  this.livenessPass = null;
7134
- this.convergePass?.stop();
7135
- this.convergePass = null;
7905
+ this.placementPass?.stop();
7906
+ this.placementPass = null;
7907
+ this.reconcilePass?.stop();
7908
+ this.reconcilePass = null;
7136
7909
  this.restore?.stop();
7137
7910
  this.restore = null;
7138
7911
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
@@ -8421,6 +9194,23 @@ var PLAYBACK_PREFIX = "playback";
8421
9194
  /** Data-plane prefix for on-demand single stills →
8422
9195
  * `/addon/recorder/still/<deviceId>/<epochSec>`. */
8423
9196
  var STILL_PREFIX = "still";
9197
+ /**
9198
+ * Bound on pending hour-ledger writes (one per finalized segment, an RPC to
9199
+ * the hub store). Steady state completes in milliseconds and holds ~0–2; under
9200
+ * a store stall the fleet-wide finalize rate (~3.4/s) fills 256 in ~75s of
9201
+ * grace before refusals begin. Each pending write holds one hour-row copy
9202
+ * (≤360 paths ≈ ~25KB), so the lane's worst-case heap is single-digit MB —
9203
+ * a dropped write is one silent re-seed from the archive walk (D148).
9204
+ */
9205
+ var LEDGER_GATE_LIMIT = 256;
9206
+ /**
9207
+ * Bound on pending mfra tail reads (one per finalized RECENT segment; each is
9208
+ * an open + two reads on the same 4-thread libuv pool the live writers use,
9209
+ * holding read buffers while pending). 64 bounds both the held memory and the
9210
+ * pool contention; a dropped capture costs that segment's client one extra
9211
+ * round trip to the file's tail on the first seek.
9212
+ */
9213
+ var MFRA_GATE_LIMIT = 64;
8424
9214
  /** Data-plane prefix for export MP4 downloads → `/addon/recorder/exports/<id>.mp4`. */
8425
9215
  var EXPORTS_PREFIX = "exports";
8426
9216
  /** How often the export janitor sweeps expired exports. */
@@ -8497,6 +9287,25 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
8497
9287
  /** Sync-sample tables captured at finalize, served by the directory so a
8498
9288
  * playback seek needs no mfra tail fetch. RAM-only, TTL-bounded. */
8499
9289
  mfraTables = new MfraTableStore();
9290
+ /** Bound on the per-finalize fire-and-forget lanes (item 3, 2026-08-18): a
9291
+ * disk/store stall must throttle this work, never balloon live heap — the
9292
+ * unbounded pending set is what fed the two `Ineffective mark-compacts`
9293
+ * fatals. Saturation is reported loudly but throttled; every individual
9294
+ * drop is logged with its deviceId at the call site. */
9295
+ ledgerWriteGate = new PendingGate({
9296
+ label: "hour-ledger",
9297
+ limit: LEDGER_GATE_LIMIT,
9298
+ onSaturated: (report) => {
9299
+ this.reportGateSaturated(report);
9300
+ }
9301
+ });
9302
+ mfraCaptureGate = new PendingGate({
9303
+ label: "mfra-capture",
9304
+ limit: MFRA_GATE_LIMIT,
9305
+ onSaturated: (report) => {
9306
+ this.reportGateSaturated(report);
9307
+ }
9308
+ });
8500
9309
  /** Continuous + events band write-path controller (ffmpeg writers + watchers). */
8501
9310
  controller = null;
8502
9311
  /** Per-device motion/audio markers for playback (B3 events feed the controller). */
@@ -8534,6 +9343,20 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
8534
9343
  constructor() {
8535
9344
  super({ ...DEFAULT_CONFIG });
8536
9345
  }
9346
+ /**
9347
+ * One throttled line per saturated lane per report interval. Lane-level (no
9348
+ * deviceId — the lane aggregates every camera); the per-drop debug lines at
9349
+ * the call sites carry the tag. WARN because a saturated lane means the
9350
+ * store or the disk is stalling under live finalize load — the line that
9351
+ * explains the missing ledger hours and mfra misses an operator will see.
9352
+ */
9353
+ reportGateSaturated(report) {
9354
+ this.ctx.logger.warn("recorder: fire-and-forget lane saturated — dropping over the bound", { meta: {
9355
+ lane: report.label,
9356
+ dropped: report.dropped,
9357
+ pending: report.pending
9358
+ } });
9359
+ }
8537
9360
  async onInitialize() {
8538
9361
  const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
8539
9362
  this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
@@ -8597,18 +9420,28 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
8597
9420
  this.segmentHours?.dropSegments(rows);
8598
9421
  },
8599
9422
  onIndexed: (row, absPath) => {
8600
- this.segmentHours?.recordSegment(row);
9423
+ const ledger = this.segmentHours;
9424
+ if (ledger && !this.ledgerWriteGate.run(() => ledger.recordSegment(row))) this.ctx.logger.debug("recorder: hour-ledger write dropped — lane saturated; the archive walk reconciles this hour", {
9425
+ tags: { deviceId: row.deviceId },
9426
+ meta: { path: row.path }
9427
+ });
8601
9428
  if (!isMfraTableServable(row.startMs, Date.now())) return;
8602
- readMfraTable(absPath, row.bytes).then((table) => {
8603
- this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
8604
- }).catch((err) => {
8605
- this.ctx.logger.debug("mfra capture failed", {
8606
- tags: { deviceId: row.deviceId },
8607
- meta: {
8608
- path: row.path,
8609
- error: require_dist.errMsg(err)
8610
- }
8611
- });
9429
+ if (!this.mfraCaptureGate.run(async () => {
9430
+ try {
9431
+ const table = await readMfraTable(absPath, row.bytes);
9432
+ this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
9433
+ } catch (err) {
9434
+ this.ctx.logger.debug("mfra capture failed", {
9435
+ tags: { deviceId: row.deviceId },
9436
+ meta: {
9437
+ path: row.path,
9438
+ error: require_dist.errMsg(err)
9439
+ }
9440
+ });
9441
+ }
9442
+ })) this.ctx.logger.debug("recorder: mfra capture dropped — lane saturated; first seek pays one tail fetch", {
9443
+ tags: { deviceId: row.deviceId },
9444
+ meta: { path: row.path }
8612
9445
  });
8613
9446
  },
8614
9447
  removeDirIfEmpty: async (locationId, relDir) => {
@@ -8666,7 +9499,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
8666
9499
  onBrokerReady: (handler) => this.ctx.onCapabilityStateChange("stream-broker", brokerScope(ingestOwner.ownerNodeId), (state) => {
8667
9500
  if (state === "ready") handler();
8668
9501
  }),
8669
- brokerCall: makeBrokerCall(brokerHandle)
9502
+ brokerCall: makeBrokerCall(brokerHandle),
9503
+ brokerReady: () => brokerHandle.isReady
8670
9504
  });
8671
9505
  const relocateEngine = new RelocateEngine({
8672
9506
  index: this.index,