@camstack/addon-pipeline 1.2.93 → 1.2.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
  2. package/dist/audio-analyzer/index.js +58 -4
  3. package/dist/audio-analyzer/index.mjs +56 -2
  4. package/dist/detection-pipeline/index.js +194 -6
  5. package/dist/detection-pipeline/index.mjs +191 -3
  6. package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
  7. package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
  8. package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
  9. package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
  10. package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
  11. package/dist/motion-wasm/index.js +2 -2
  12. package/dist/motion-wasm/index.mjs +1 -1
  13. package/dist/pipeline-runner/index.js +195 -14
  14. package/dist/pipeline-runner/index.mjs +194 -13
  15. package/dist/process-memory-B9-UD7Rp.js +78 -0
  16. package/dist/process-memory-DXOB6eHX.mjs +66 -0
  17. package/dist/recorder/index.js +943 -109
  18. package/dist/recorder/index.mjs +942 -108
  19. package/dist/session-decode/decode-worker-child.js +4 -4
  20. package/dist/session-decode/decode-worker-child.mjs +3 -3
  21. package/dist/stream-broker/_stub.js +1 -1
  22. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
  23. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
  24. package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
  25. package/dist/stream-broker/index.js +25 -7
  26. package/dist/stream-broker/index.mjs +25 -7
  27. package/dist/stream-broker/remoteEntry.js +1 -1
  28. package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
  29. package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
  30. package/package.json +13 -7
  31. package/python/inference_pool.py +178 -3
  32. package/python/test_inference_pool_backpressure.py +7 -1
  33. package/python/test_inference_pool_memstats.py +146 -0
  34. package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
  35. package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
  36. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
@@ -1,4 +1,4 @@
1
- import { $ as resolveRecordingProfiles, Ct as selectAssignedProfileSlots, Et as array, I as deriveRecordingMode, It as EventCategory, Mt as object, Nt as record, Pt as string, S as RecordingConfigSchema, X as recordingCapability, Z as recordingExportCapability, at as errMsg, bt as nodePin, dt as BaseAddon, f as EVENT_PAD_MS, gt as hydrateSchema, jt as number, l as DEFAULT_EVENTS_BAND_BUFFER_SEC, m as ExportRecordSchema, mt as DeviceType, tt as storageEvictableCapability, v as OpsLogEntrySchema, x as RECORDING_EXPORT_MAX_READ_BYTES } from "../dist-6POVIYmC.mjs";
1
+ import { $ as recordingExportCapability, C as RecordingConfigSchema, Ct as nodePin, Et as selectAssignedProfileSlots, Ft as object, It as record, L as deriveRecordingMode, Lt as string, Pt as number, Q as recordingCapability, S as RECORDING_EXPORT_MAX_READ_BYTES, _t as DeviceType, ct as errMsg, f as EVENT_PAD_MS, it as storageEvictableCapability, kt as array, l as DEFAULT_EVENTS_BAND_BUFFER_SEC, m as ExportRecordSchema, mt as BaseAddon, nt as resolveRecordingProfiles, v as OpsLogEntrySchema, yt as hydrateSchema, zt as EventCategory } from "../dist-C49o1k7u.mjs";
2
2
  import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
3
3
  import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-A2S9D7pu.mjs";
4
4
  import { randomUUID } from "node:crypto";
@@ -456,6 +456,51 @@ var MfraTableStore = class {
456
456
  }
457
457
  }
458
458
  };
459
+ var PendingGate = class {
460
+ deps;
461
+ inFlight = 0;
462
+ droppedSinceReport = 0;
463
+ lastReportAt = Number.NEGATIVE_INFINITY;
464
+ constructor(deps) {
465
+ this.deps = deps;
466
+ }
467
+ /** Tasks currently in flight. */
468
+ get pending() {
469
+ return this.inFlight;
470
+ }
471
+ /**
472
+ * Admit `task` unless the lane is saturated.
473
+ *
474
+ * Returns `false` when the task was REFUSED — the caller must log the drop
475
+ * (with its `deviceId` tag when the work is about a device); a branch that
476
+ * drops work silently reads as "never happened".
477
+ */
478
+ run(task) {
479
+ if (this.inFlight >= this.deps.limit) {
480
+ this.droppedSinceReport += 1;
481
+ const now = this.deps.now?.() ?? Date.now();
482
+ const intervalMs = this.deps.reportIntervalMs ?? 3e4;
483
+ if (now - this.lastReportAt >= intervalMs) {
484
+ this.lastReportAt = now;
485
+ const dropped = this.droppedSinceReport;
486
+ this.droppedSinceReport = 0;
487
+ this.deps.onSaturated({
488
+ label: this.deps.label,
489
+ dropped,
490
+ pending: this.inFlight
491
+ });
492
+ }
493
+ return false;
494
+ }
495
+ this.inFlight += 1;
496
+ Promise.resolve().then(task).catch((error) => {
497
+ this.deps.onTaskError?.(error);
498
+ }).finally(() => {
499
+ this.inFlight -= 1;
500
+ });
501
+ return true;
502
+ }
503
+ };
459
504
  //#endregion
460
505
  //#region src/recorder/merge-ranges.ts
461
506
  function mergeRanges(segments, gapToleranceMs) {
@@ -515,12 +560,33 @@ function* hourBuckets(fromMs, toMs) {
515
560
  const first = Math.floor(fromMs / HOUR_MS$5) * HOUR_MS$5;
516
561
  for (let h = first; h < toMs; h += HOUR_MS$5) yield h;
517
562
  }
563
+ /**
564
+ * Index of the first element whose `startMs` is GREATER than `startMs` (upper
565
+ * bound). Inserting there keeps equal-start rows in arrival order — exactly
566
+ * what a stable full sort of the backing map (insertion-ordered) produces, so
567
+ * an incrementally-maintained view is indistinguishable from a rebuilt one.
568
+ */
569
+ function upperBoundByStart(rows, startMs) {
570
+ let lo = 0;
571
+ let hi = rows.length;
572
+ while (lo < hi) {
573
+ const mid = lo + hi >> 1;
574
+ if (rows[mid].startMs <= startMs) lo = mid + 1;
575
+ else hi = mid;
576
+ }
577
+ return lo;
578
+ }
518
579
  var RecordingIndex = class {
519
580
  byDevice = /* @__PURE__ */ new Map();
520
- /** Start-sorted snapshots, rebuilt only after index mutations. Interactive
521
- * locate/range calls reuse them instead of filtering+sorting the complete
522
- * archive on every seek and scrub miss. */
581
+ /** Start-sorted views. Built lazily on first read (one copy+sort), then kept
582
+ * LIVE by `addSegment`'s ordered insert rows arrive roughly ascending, so
583
+ * the insert is near-appending. Bulk mutations (hydrate, eviction, a re-set
584
+ * of an existing path) still invalidate; the steady-state finalize never
585
+ * does. Interactive locate/range calls reuse these instead of
586
+ * filtering+sorting the complete archive on every seek and scrub miss. */
523
587
  sortedByDevice = /* @__PURE__ */ new Map();
588
+ fullRebuilds = 0;
589
+ incrementalInserts = 0;
524
590
  /** Hour buckets this device has been walked for; `ALL_HOURS` after a full
525
591
  * hydrate. Empty = nothing has been looked at, which is NOT the same as
526
592
  * "there is nothing" — see {@link hydrationOf}. */
@@ -547,10 +613,34 @@ var RecordingIndex = class {
547
613
  if (cached) return cached;
548
614
  const m = this.byDevice.get(deviceId);
549
615
  if (!m) return [];
616
+ this.fullRebuilds += 1;
550
617
  const sorted = [...m.values()].filter((s) => profile == null || s.profile === profile).toSorted((a, b) => a.startMs - b.startMs);
551
618
  deviceCache.set(key, sorted);
552
619
  return sorted;
553
620
  }
621
+ /** See {@link SortedViewStats}. Telemetry + the guard the churn test asserts on. */
622
+ sortedViewStats() {
623
+ return {
624
+ fullRebuilds: this.fullRebuilds,
625
+ incrementalInserts: this.incrementalInserts
626
+ };
627
+ }
628
+ /**
629
+ * Ordered insert of ONE NEW row into every live view it belongs to (the
630
+ * all-profiles view and its own profile's view). Views not yet materialised
631
+ * stay unmaterialised — their first read pays the one build. PRECONDITION:
632
+ * the row's path was NOT already in the device map; a replace must
633
+ * invalidate instead, because the views hold the old row object.
634
+ */
635
+ insertIntoSortedViews(s) {
636
+ const deviceCache = this.sortedByDevice.get(s.deviceId);
637
+ if (!deviceCache) return;
638
+ for (const [key, rows] of deviceCache) {
639
+ if (key !== "" && key !== s.profile) continue;
640
+ rows.splice(upperBoundByStart(rows, s.startMs), 0, s);
641
+ this.incrementalInserts += 1;
642
+ }
643
+ }
554
644
  /**
555
645
  * Replace a device's segments FOR ONE LOCATION from a `storage.list` result (paths relative to that
556
646
  * location root). Rows on other locations are preserved. Non-segment paths and other devices are ignored.
@@ -631,10 +721,26 @@ var RecordingIndex = class {
631
721
  this.invalidateSorted(deviceId);
632
722
  this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$5);
633
723
  }
634
- /** Insert/replace one finalized segment (idempotent by path). */
724
+ /**
725
+ * Insert/replace one finalized segment (idempotent by path).
726
+ *
727
+ * THE hot mutation: every profile-writer lands here every `segmentSeconds`,
728
+ * ~3.4 times a second fleet-wide. A NEW path is an ordered insert into the
729
+ * live sorted views (rows arrive roughly ascending, so it is near-appending
730
+ * — O(log n) search + a short memmove). Only the rare RE-SET of an existing
731
+ * path (watcher replay after a playlist reset, or a relocate that changes
732
+ * `locationId`) invalidates: the views hold the old row object, and a
733
+ * one-off rebuild is the simple correct answer for a case that is not hot.
734
+ */
635
735
  addSegment(s) {
636
- this.mapFor(s.deviceId).set(s.path, s);
637
- this.invalidateSorted(s.deviceId);
736
+ const m = this.mapFor(s.deviceId);
737
+ const replacing = m.has(s.path);
738
+ m.set(s.path, s);
739
+ if (replacing) {
740
+ this.invalidateSorted(s.deviceId);
741
+ return;
742
+ }
743
+ this.insertIntoSortedViews(s);
638
744
  }
639
745
  /** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. */
640
746
  removeSegments(paths) {
@@ -2900,6 +3006,67 @@ async function listExportsFor(state, deviceId) {
2900
3006
  return [...(await readExports(state)).values()].filter((r) => deviceId === void 0 || r.deviceId === deviceId).toSorted((a, b) => b.createdAt - a.createdAt);
2901
3007
  }
2902
3008
  //#endregion
3009
+ //#region src/recorder/addon/spawn-failure.ts
3010
+ /**
3011
+ * How the recorder reads a `child_process` `'error'` — ONE owner of the
3012
+ * transient-vs-permanent question.
3013
+ *
3014
+ * ── Why this file exists ───────────────────────────────────────────────────
3015
+ * In Node a child that cannot be spawned emits `'error'`, and an `'error'`
3016
+ * with no listener is re-thrown as an UNCAUGHT EXCEPTION. On 2026-08-18 three
3017
+ * cameras were unplugged, every writer for them re-dialled at once, the host
3018
+ * momentarily could not fork (`spawn ffmpeg EAGAIN`), and the `recorder`
3019
+ * runner died three times inside nine seconds. The D6 crash circuit-breaker
3020
+ * did its job and stopped respawning — leaving the whole fleet with no
3021
+ * recording for three hours and fifteen minutes while `/health` returned 200.
3022
+ *
3023
+ * Every ffmpeg spawn site in the recorder therefore attaches an `'error'`
3024
+ * listener, and every one of them classifies the failure HERE rather than
3025
+ * inventing its own list.
3026
+ */
3027
+ /**
3028
+ * Spawn failures that RETRYING CANNOT FIX.
3029
+ *
3030
+ * The EAGAIN-vs-ENOENT split, stated once:
3031
+ *
3032
+ * - EAGAIN / EMFILE / ENOMEM are the host saying "not right now". The
3033
+ * resource comes back, often within one backoff. They take the ordinary
3034
+ * restart-with-backoff path, whose own comment already states the policy
3035
+ * for this shape: "only RAPID back-to-back failures count toward giving
3036
+ * up, so a brief blip never permanently stops a camera's recording".
3037
+ * - ENOENT / EACCES are the host saying "there is no ffmpeg here you may
3038
+ * run". Nothing about the next ten attempts differs from this one, so the
3039
+ * caller stops immediately and surfaces it at ERROR level: burning a
3040
+ * ten-attempt budget on a missing binary only delays recovery and buries
3041
+ * the single actionable line under ten identical warnings.
3042
+ *
3043
+ * EPERM is deliberately absent: it is reported for transient sandbox/cgroup
3044
+ * refusals as well as for real permission faults, and treating an ambiguous
3045
+ * code as permanent stops a camera that would have recovered.
3046
+ */
3047
+ var PERMANENT_SPAWN_CODES = new Set(["ENOENT", "EACCES"]);
3048
+ /**
3049
+ * Read the `'error'` payload without a cast.
3050
+ *
3051
+ * An unknown code is treated as TRANSIENT: the cost of retrying something
3052
+ * permanent is a bounded restart budget, while the cost of giving up on
3053
+ * something transient is a camera that stops recording until an operator
3054
+ * notices.
3055
+ */
3056
+ function readSpawnFailure(raw) {
3057
+ const code = errnoCode(raw);
3058
+ return {
3059
+ code,
3060
+ message: raw instanceof Error ? raw.message : String(raw),
3061
+ permanent: code !== void 0 && PERMANENT_SPAWN_CODES.has(code)
3062
+ };
3063
+ }
3064
+ function errnoCode(raw) {
3065
+ if (typeof raw !== "object" || raw === null || !("code" in raw)) return void 0;
3066
+ const code = raw.code;
3067
+ return typeof code === "string" ? code : void 0;
3068
+ }
3069
+ //#endregion
2903
3070
  //#region src/recorder/addon/export-engine.ts
2904
3071
  /** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
2905
3072
  * segment before the decisive "Error opening input" summary, so a short tail can
@@ -3069,7 +3236,7 @@ var ExportEngine = class {
3069
3236
  outPath,
3070
3237
  options
3071
3238
  });
3072
- const ran = await this.runFfmpeg(id, args, {
3239
+ const ran = await this.runFfmpeg(id, rec.deviceId, args, {
3073
3240
  totalSec: expectedOutSeconds(rec),
3074
3241
  floor: 0,
3075
3242
  ceil: 99
@@ -3089,7 +3256,7 @@ var ExportEngine = class {
3089
3256
  inputPlaylist: playlist,
3090
3257
  outPath: concatPath
3091
3258
  });
3092
- const concatRan = await this.runFfmpeg(id, concatArgs, {
3259
+ const concatRan = await this.runFfmpeg(id, rec.deviceId, concatArgs, {
3093
3260
  totalSec: Math.max(.001, (rec.toMs - rec.fromMs) / 1e3),
3094
3261
  floor: 0,
3095
3262
  ceil: 49
@@ -3103,7 +3270,7 @@ var ExportEngine = class {
3103
3270
  outPath,
3104
3271
  options
3105
3272
  });
3106
- const selectRan = await this.runFfmpeg(id, selectArgs, {
3273
+ const selectRan = await this.runFfmpeg(id, rec.deviceId, selectArgs, {
3107
3274
  totalSec: expectedOutSeconds(rec),
3108
3275
  floor: 50,
3109
3276
  ceil: 99
@@ -3115,7 +3282,7 @@ var ExportEngine = class {
3115
3282
  } catch {}
3116
3283
  }
3117
3284
  }
3118
- runFfmpeg(id, args, progress) {
3285
+ runFfmpeg(id, deviceId, args, progress) {
3119
3286
  this.deps.logger.debug("export ffmpeg argv", { meta: {
3120
3287
  exportId: id,
3121
3288
  args
@@ -3152,6 +3319,24 @@ var ExportEngine = class {
3152
3319
  stderrTail
3153
3320
  });
3154
3321
  });
3322
+ proc.on("error", (raw) => {
3323
+ this.activeProc = null;
3324
+ const failure = readSpawnFailure(raw);
3325
+ this.deps.logger.error("export ffmpeg could not be spawned", {
3326
+ tags: { deviceId },
3327
+ meta: {
3328
+ exportId: id,
3329
+ errorCode: failure.code,
3330
+ error: failure.message,
3331
+ permanent: failure.permanent
3332
+ }
3333
+ });
3334
+ stderrTail.push(`ffmpeg spawn failed: ${failure.message}` + (failure.code === void 0 ? "" : ` (${failure.code})`));
3335
+ resolve({
3336
+ code: null,
3337
+ stderrTail
3338
+ });
3339
+ });
3155
3340
  });
3156
3341
  }
3157
3342
  async finish(id, code, outPath, stderrTail) {
@@ -5837,18 +6022,232 @@ function shouldRecordAt(config, triggers, at) {
5837
6022
  return eventsBandDemanded(band, triggers, at.getTime());
5838
6023
  }
5839
6024
  /**
5840
- * Should a finalized segment `[segStartMs, segEndMs]` recorded under an
5841
- * `events` band be KEPT (else discarded)? Kept iff it overlaps any qualifying
5842
- * trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]`.
5843
- * This is where `preBufferSec` retroactively retains pre-trigger footage.
6025
+ * How late the FIRST segment of an attach may be NAMED, relative to the writer
6026
+ * spawn, and still count as "the segment this attach was spawned to produce".
6027
+ *
6028
+ * ffmpeg names a segment when it OPENS it, which is after the RTSP dial and the
6029
+ * first keyframe. Measured on camera 1439 (4K, H.265) on 2026-08-18: writer
6030
+ * spawned at trigger+29.5 s, first `high` segment named at trigger+32.7 s —
6031
+ * 3.2 s. 5 s carries that with margin while staying far below the writer's own
6032
+ * "this is dead" bound (`WRITER_IDLE_MIN_MS`, 90 s), so a writer that is merely
6033
+ * slow is not mistaken for one that is producing useful footage.
5844
6034
  */
5845
- function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers) {
6035
+ var FIRST_SEGMENT_GRACE_MS = 5e3;
6036
+ /**
6037
+ * Latest segment START this attach may still produce and have kept, regardless
6038
+ * of where the operator's post-buffer ended: one full segment (nothing can be
6039
+ * finalized sooner) plus {@link FIRST_SEGMENT_GRACE_MS} of spawn→first-name
6040
+ * latency.
6041
+ */
6042
+ function attachRetainDeadlineMs(attach) {
6043
+ return attach.attachedAtMs + attach.segmentSeconds * 1e3 + FIRST_SEGMENT_GRACE_MS;
6044
+ }
6045
+ /**
6046
+ * Should a finalized segment `[segStartMs, segEndMs]` recorded under an
6047
+ * `events` band be KEPT (else discarded)? Kept iff it overlaps a qualifying
6048
+ * trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]` —
6049
+ * this is where `preBufferSec` retroactively retains pre-trigger footage — OR
6050
+ * it is the footage `attach` was spawned to produce (below).
6051
+ *
6052
+ * ## Why the attach matters here
6053
+ *
6054
+ * The LIVE gate ({@link shouldRecordAt}) is evaluated at wall clock; this KEEP
6055
+ * gate is evaluated on segment timestamps. They shared one window, and a writer
6056
+ * physically cannot finalize anything before `spawn + segmentSeconds + dial`.
6057
+ * So the last ~`segmentSeconds` of every demand window was a zone where the
6058
+ * recorder was GUARANTEED to attach, pull the camera, and delete 100% of the
6059
+ * result: on camera 1439 a trigger reached the recorder 29.5 s into its 30 s
6060
+ * post-buffer (hub overload, `busLagMs=142173`), ffmpeg ran 31 s, and all six
6061
+ * finalized segments were unlinked.
6062
+ *
6063
+ * The allowance is deliberately NOT a wider `postBufferSec`: it is anchored to
6064
+ * `attachedAtMs`, so a PUNCTUAL attach (the normal case) has a deadline far
6065
+ * inside `trigger + postMs` and retains exactly what it retained before. It
6066
+ * only ever extends the tail of a LATE attach, by at most one segment plus the
6067
+ * grace, and only for a trigger whose live window was still open AT ATTACH — the
6068
+ * precise statement of "if the live gate said yes, the keep gate honours the
6069
+ * first segment that decision produces". A stale trigger extends nothing.
6070
+ */
6071
+ function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers, attach) {
5846
6072
  const { preMs, postMs } = resolveBandBufferMs(band);
5847
- const overlaps = (lastMs) => lastMs != null && segStartMs <= lastMs + postMs && segEndMs >= lastMs - preMs;
6073
+ const deadlineMs = attachRetainDeadlineMs(attach);
6074
+ const overlaps = (lastMs) => {
6075
+ if (lastMs == null) return false;
6076
+ if (segEndMs < lastMs - preMs) return false;
6077
+ if (segStartMs <= lastMs + postMs) return true;
6078
+ return attach.attachedAtMs <= lastMs + postMs && segStartMs <= deadlineMs;
6079
+ };
5848
6080
  if (band.triggers?.motion && overlaps(triggers.lastMotionMs)) return true;
5849
6081
  if (band.triggers?.audioThresholdDbfs != null && overlaps(triggers.lastAudioMs)) return true;
5850
6082
  return false;
5851
6083
  }
6084
+ /**
6085
+ * By how many ms a discarded segment missed being kept — its start past the
6086
+ * latest start that WOULD have been retained, mirroring
6087
+ * {@link segmentRetainedForEvents}'s upper bound exactly (post-buffer end, or
6088
+ * the attach allowance for a trigger that was still live at attach). Null when
6089
+ * no qualifying trigger exists at all, which is a different fact and says so.
6090
+ *
6091
+ * Negative means the segment was dropped on the PRE-buffer side (it ended
6092
+ * before `trigger - preBufferSec`), not for being late.
6093
+ *
6094
+ * Reporting-only. The discard branch has to name a number: "outside the window"
6095
+ * with no magnitude made a 2.5 s miss and a 10-minute miss indistinguishable.
6096
+ */
6097
+ function segmentMissedByMs(segStartMs, band, triggers, attach) {
6098
+ const { postMs } = resolveBandBufferMs(band);
6099
+ const deadlineMs = attachRetainDeadlineMs(attach);
6100
+ const latestKeptStart = (lastMs) => {
6101
+ if (lastMs == null) return null;
6102
+ const windowEndMs = lastMs + postMs;
6103
+ return attach.attachedAtMs <= windowEndMs ? Math.max(windowEndMs, deadlineMs) : windowEndMs;
6104
+ };
6105
+ const bounds = [band.triggers?.motion === true ? latestKeptStart(triggers.lastMotionMs) : null, band.triggers?.audioThresholdDbfs != null ? latestKeptStart(triggers.lastAudioMs) : null].filter((ms) => ms != null);
6106
+ if (bounds.length === 0) return null;
6107
+ return segStartMs - Math.max(...bounds);
6108
+ }
6109
+ /** Local-calendar instant for `HH:MM` on the given local Y/M/D (day may overflow). */
6110
+ function localInstantMs(year, month, day, hhmm) {
6111
+ const [hours, minutes] = hhmm.split(":");
6112
+ return new Date(year, month, day, Number(hours), Number(minutes), 0, 0).getTime();
6113
+ }
6114
+ /**
6115
+ * Every instant inside the horizon at which `bandActiveAt` COULD change value:
6116
+ * each band's start and end on each local day, plus each local midnight (the
6117
+ * `days` membership boundary). A superset — cheap to compute (a few dozen
6118
+ * entries) and filtered against `activeBandAt` below.
6119
+ */
6120
+ function candidateEdgesMs(bands, nowMs) {
6121
+ const base = new Date(nowMs);
6122
+ const year = base.getFullYear();
6123
+ const month = base.getMonth();
6124
+ const day = base.getDate();
6125
+ const out = [];
6126
+ for (let offset = 0; offset <= 8; offset++) {
6127
+ out.push(new Date(year, month, day + offset, 0, 0, 0, 0).getTime());
6128
+ for (const band of bands) {
6129
+ out.push(localInstantMs(year, month, day + offset, band.start));
6130
+ out.push(localInstantMs(year, month, day + offset, band.end));
6131
+ }
6132
+ }
6133
+ return out;
6134
+ }
6135
+ /**
6136
+ * The next instant at which a DIFFERENT band (or no band) covers the clock, or
6137
+ * `null` when none exists inside {@link BAND_EDGE_HORIZON_DAYS} — the always-on
6138
+ * band (`start === end`, empty `days`) being the common case that legitimately
6139
+ * has no edge at all.
6140
+ *
6141
+ * Identity is by band REFERENCE, which is what makes this exact: two distinct
6142
+ * band objects with the same fields yield an edge that changes nothing, which
6143
+ * costs one wasted wake-up and never costs footage.
6144
+ */
6145
+ function nextBandEdgeMs(config, nowMs) {
6146
+ const bands = config.bands;
6147
+ if (!bands || bands.length === 0) return null;
6148
+ const current = activeBandAt({ bands }, new Date(nowMs));
6149
+ const candidates = [...new Set(candidateEdgesMs(bands, nowMs))].filter((ms) => ms > nowMs).toSorted((a, b) => a - b);
6150
+ for (const ms of candidates) if (activeBandAt({ bands }, new Date(ms)) !== current) return ms;
6151
+ return null;
6152
+ }
6153
+ /**
6154
+ * The first instant an `events` band's demand window is CLOSED, given the
6155
+ * triggers it listens for, or `null` when no window is open at `nowMs`.
6156
+ *
6157
+ * Demand is an OR across the listened trigger kinds, so an open window ends
6158
+ * with the LATEST of them — a motion trigger followed by an audio trigger 5 s
6159
+ * later keeps the camera attached until the audio one expires, exactly as the
6160
+ * sliding window already behaved.
6161
+ */
6162
+ function eventsWindowCloseMs(band, triggers, nowMs) {
6163
+ if (band.mode !== "events") return null;
6164
+ const { postMs } = resolveBandBufferMs(band);
6165
+ const ends = [];
6166
+ if (band.triggers?.motion === true && triggers.lastMotionMs != null) ends.push(triggers.lastMotionMs + postMs);
6167
+ if (band.triggers?.audioThresholdDbfs != null && triggers.lastAudioMs != null) ends.push(triggers.lastAudioMs + postMs);
6168
+ const open = ends.filter((ms) => ms >= nowMs);
6169
+ if (open.length === 0) return null;
6170
+ return Math.max(...open) + 1;
6171
+ }
6172
+ /**
6173
+ * THE next instant the controller must re-evaluate this device, or `null` when
6174
+ * nothing can change without an event the controller already receives (a
6175
+ * trigger or a config write).
6176
+ *
6177
+ * Whichever of the two sources comes first wins, and the reason is carried so
6178
+ * the wake-up is attributable in the log without correlating timestamps.
6179
+ */
6180
+ function nextDecisionChangeAt(input) {
6181
+ const { config, triggers, nowMs } = input;
6182
+ if (!config.enabled) return null;
6183
+ const bands = config.bands;
6184
+ if (!bands || bands.length === 0) return null;
6185
+ const band = activeBandAt({ bands }, new Date(nowMs));
6186
+ const edgeMs = nextBandEdgeMs(config, nowMs);
6187
+ const closeMs = band === null ? null : eventsWindowCloseMs(band, triggers, nowMs);
6188
+ if (closeMs === null) return edgeMs === null ? null : {
6189
+ atMs: edgeMs,
6190
+ reason: "band-edge"
6191
+ };
6192
+ if (edgeMs === null || closeMs <= edgeMs) return {
6193
+ atMs: closeMs,
6194
+ reason: "window-close"
6195
+ };
6196
+ return {
6197
+ atMs: edgeMs,
6198
+ reason: "band-edge"
6199
+ };
6200
+ }
6201
+ //#endregion
6202
+ //#region src/recorder/addon/device-wakeups.ts
6203
+ /** Node truncates a `setTimeout` delay past this to 1 ms — clamp instead. */
6204
+ var MAX_TIMER_DELAY_MS = 2147483647;
6205
+ var DeviceWakeups = class {
6206
+ deps;
6207
+ timers = /* @__PURE__ */ new Map();
6208
+ constructor(deps) {
6209
+ this.deps = deps;
6210
+ }
6211
+ /**
6212
+ * Wake this device at `atMs`, replacing any wake-up already armed for it.
6213
+ * `reason` is carried into the log line so a wake is attributable without
6214
+ * correlating it against the band model by hand.
6215
+ */
6216
+ arm(deviceId, atMs, reason) {
6217
+ this.cancel(deviceId);
6218
+ const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, atMs - this.deps.now()));
6219
+ const timer = setTimeout(() => {
6220
+ this.timers.delete(deviceId);
6221
+ this.deps.fire(deviceId);
6222
+ }, delayMs);
6223
+ timer.unref?.();
6224
+ this.timers.set(deviceId, timer);
6225
+ this.deps.logger.debug("recorder: device wake-up armed", {
6226
+ tags: { deviceId },
6227
+ meta: {
6228
+ atMs,
6229
+ delayMs,
6230
+ reason
6231
+ }
6232
+ });
6233
+ }
6234
+ /** Drop this device's pending wake-up, if any. Idempotent. */
6235
+ cancel(deviceId) {
6236
+ const timer = this.timers.get(deviceId);
6237
+ if (timer === void 0) return;
6238
+ clearTimeout(timer);
6239
+ this.timers.delete(deviceId);
6240
+ }
6241
+ /** Drop every pending wake-up — shutdown and maintenance pause. */
6242
+ cancelAll() {
6243
+ for (const timer of this.timers.values()) clearTimeout(timer);
6244
+ this.timers.clear();
6245
+ }
6246
+ /** Devices with a wake-up pending — for tests and for shutdown assertions. */
6247
+ get pendingCount() {
6248
+ return this.timers.size;
6249
+ }
6250
+ };
5852
6251
  //#endregion
5853
6252
  //#region src/recorder/addon/periodic-pass.ts
5854
6253
  function startPeriodicPass(deps) {
@@ -5915,7 +6314,9 @@ var ReadinessRestore = class {
5915
6314
  this.deps = deps;
5916
6315
  }
5917
6316
  get concurrency() {
5918
- return Math.max(1, this.deps.concurrency ?? 8);
6317
+ const declared = this.deps.concurrency;
6318
+ const value = typeof declared === "function" ? declared() : declared;
6319
+ return Math.max(1, value ?? 8);
5919
6320
  }
5920
6321
  /** Seed the pending set, subscribe to readiness, and drain once immediately
5921
6322
  * (covers the already-ready case). Returns when the first drain settles. */
@@ -5974,6 +6375,70 @@ var ReadinessRestore = class {
5974
6375
  }
5975
6376
  }
5976
6377
  };
6378
+ /** Adaptive bound for the recorder's attach parallelism. */
6379
+ var SpawnPressureGovernor = class {
6380
+ deps;
6381
+ limit;
6382
+ min;
6383
+ recoverAfterMs;
6384
+ /** When the limit last MOVED — the clock both directions are measured from. */
6385
+ lastChangeAt;
6386
+ underPressure = false;
6387
+ constructor(deps) {
6388
+ this.deps = deps;
6389
+ this.min = Math.max(1, deps.minConcurrency ?? 1);
6390
+ this.limit = Math.max(this.min, deps.maxConcurrency);
6391
+ this.recoverAfterMs = deps.recoverAfterMs ?? 3e4;
6392
+ this.lastChangeAt = this.now();
6393
+ }
6394
+ /**
6395
+ * How many attaches may overlap right now.
6396
+ *
6397
+ * Reading is what drives recovery: there is no timer to leak, no interval to
6398
+ * lose to a runner respawn, and a recorder that stops attaching entirely
6399
+ * simply keeps its reduced bound until it tries again — which is correct.
6400
+ */
6401
+ get concurrency() {
6402
+ const max = Math.max(this.min, this.deps.maxConcurrency);
6403
+ if (this.limit >= max) return max;
6404
+ const elapsed = this.now() - this.lastChangeAt;
6405
+ if (elapsed < this.recoverAfterMs) return this.limit;
6406
+ const lanes = Math.floor(elapsed / this.recoverAfterMs);
6407
+ this.limit = Math.min(max, this.limit + lanes);
6408
+ this.lastChangeAt += lanes * this.recoverAfterMs;
6409
+ if (this.limit >= max && this.underPressure) {
6410
+ this.underPressure = false;
6411
+ this.deps.logger?.info("recorder: spawn pressure cleared — attach concurrency restored", { meta: { concurrency: this.limit } });
6412
+ } else this.deps.logger?.info("recorder: recovering attach concurrency", { meta: {
6413
+ concurrency: this.limit,
6414
+ max
6415
+ } });
6416
+ return this.limit;
6417
+ }
6418
+ /**
6419
+ * A transient spawn failure happened on `deviceId`'s writer.
6420
+ *
6421
+ * Halves the CURRENT limit, not the max: two waves of pressure inside one
6422
+ * recovery window must compound, or a host under sustained load is asked for
6423
+ * the same too-large batch over and over.
6424
+ */
6425
+ report(deviceId) {
6426
+ const before = this.limit;
6427
+ this.limit = Math.max(this.min, Math.floor(this.limit / 2));
6428
+ this.lastChangeAt = this.now();
6429
+ this.underPressure = true;
6430
+ this.deps.logger?.warn("recorder: spawn pressure — reducing attach concurrency", {
6431
+ tags: { deviceId },
6432
+ meta: {
6433
+ from: before,
6434
+ to: this.limit
6435
+ }
6436
+ });
6437
+ }
6438
+ now() {
6439
+ return this.deps.now?.() ?? Date.now();
6440
+ }
6441
+ };
5977
6442
  //#endregion
5978
6443
  //#region src/recorder/addon/segment-watcher.ts
5979
6444
  /**
@@ -6359,36 +6824,87 @@ var SegmentWriter = class {
6359
6824
  if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
6360
6825
  }
6361
6826
  });
6827
+ let settled = false;
6828
+ const terminate = (termination) => {
6829
+ if (settled) return;
6830
+ settled = true;
6831
+ this.onTermination(termination, stderrTail);
6832
+ };
6362
6833
  proc.on("exit", (code) => {
6363
- this.resolveExit?.();
6364
- this.resolveExit = null;
6365
- this.proc = null;
6366
- if (this.stopped) return;
6367
- const ranMs = Date.now() - this.startedAt;
6368
- if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
6369
- if (this.restarts >= MAX_RESTARTS) {
6370
- this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
6834
+ terminate({
6835
+ reason: "exit",
6836
+ exitCode: typeof code === "number" ? code : null
6837
+ });
6838
+ });
6839
+ proc.on("error", (raw) => {
6840
+ const failure = readSpawnFailure(raw);
6841
+ terminate({
6842
+ reason: "spawn-error",
6843
+ errorCode: failure.code,
6844
+ message: failure.message,
6845
+ permanent: failure.permanent
6846
+ });
6847
+ });
6848
+ }
6849
+ /**
6850
+ * The ONE restart policy, reached from both `exit` and `error`.
6851
+ *
6852
+ * `deps.logger` is the device-scoped child the controller binds
6853
+ * (`logger.withTags({ deviceId })`), so every line below carries
6854
+ * `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
6855
+ * camera, and a writer that stops recording one camera must be greppable by
6856
+ * that camera.
6857
+ */
6858
+ onTermination(termination, stderrTail) {
6859
+ this.resolveExit?.();
6860
+ this.resolveExit = null;
6861
+ this.proc = null;
6862
+ if (this.stopped) return;
6863
+ const ranMs = Date.now() - this.startedAt;
6864
+ if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
6865
+ if (termination.reason === "spawn-error") {
6866
+ this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
6867
+ outDir: this.cfg.outDir,
6868
+ errorCode: termination.errorCode,
6869
+ error: termination.message,
6870
+ permanent: termination.permanent
6871
+ } });
6872
+ if (termination.permanent) {
6873
+ this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
6371
6874
  outDir: this.cfg.outDir,
6372
- code,
6373
- stderrTail: stderrTail.join(" | ")
6875
+ errorCode: termination.errorCode,
6876
+ error: termination.message
6374
6877
  } });
6375
6878
  this.stopped = true;
6376
6879
  this.deps.onGaveUp?.();
6377
6880
  return;
6378
6881
  }
6379
- this.restarts++;
6380
- const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
6381
- this.deps.logger.info("SegmentWriter restarting", { meta: {
6882
+ this.deps.onResourcePressure?.();
6883
+ }
6884
+ if (this.restarts >= MAX_RESTARTS) {
6885
+ this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
6382
6886
  outDir: this.cfg.outDir,
6383
- attempt: this.restarts,
6384
- delayMs,
6385
- ranMs
6887
+ reason: termination.reason,
6888
+ code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
6889
+ stderrTail: stderrTail.join(" | ")
6386
6890
  } });
6387
- this.restartTimer = setTimeout(() => {
6388
- this.restartTimer = null;
6389
- this.start();
6390
- }, delayMs);
6391
- });
6891
+ this.stopped = true;
6892
+ this.deps.onGaveUp?.();
6893
+ return;
6894
+ }
6895
+ this.restarts++;
6896
+ const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
6897
+ this.deps.logger.info("SegmentWriter restarting", { meta: {
6898
+ outDir: this.cfg.outDir,
6899
+ attempt: this.restarts,
6900
+ delayMs,
6901
+ ranMs,
6902
+ reason: termination.reason
6903
+ } });
6904
+ this.restartTimer = setTimeout(() => {
6905
+ this.restartTimer = null;
6906
+ this.start();
6907
+ }, delayMs);
6392
6908
  }
6393
6909
  stop() {
6394
6910
  this.stopAndWait();
@@ -6432,19 +6948,37 @@ var SegmentWriter = class {
6432
6948
  * - a `live.m3u8` `SegmentWatcher` that hands each finalized flat segment to
6433
6949
  * the v2 `SegmentStore.onFinalized` (stat + relocate + index).
6434
6950
  *
6435
- * Per device it decides whether a CONTINUOUS band is active right now
6436
- * (`shouldRecordContinuousAt`) and ensures the writer set is running iff so.
6437
- * It re-evaluates on three triggers:
6438
- * 1. `setDeviceConfig` (operator changed the bands / enable),
6439
- * 2. a periodic pass (catches band boundaries crossed with no config change),
6440
- * 3. a `stream-broker` ready transition (boot restore via `ReadinessRestore`).
6441
- *
6442
- * There are TWO periodic passes, on separate timers and never overlapping
6443
- * themselves (`periodic-pass.ts`): a cheap LIVENESS pass that catches writers
6444
- * pinned but producing nothing, and an expensive CONVERGENCE pass that
6445
- * re-evaluates bands and placement. They were one pass until 2026-08-13, when a
6446
- * convergence sweep wedged on unbounded broker RPCs and took the liveness
6447
- * watchdog down with it for 9-27 minutes at a time.
6951
+ * Per device it decides whether the active band demands recording right now
6952
+ * (`shouldRecordAt`) and ensures the writer set is running iff so. Convergence
6953
+ * is EVENT-DRIVEN; the periodic pass is a backstop, not the mechanism:
6954
+ * 1. `setDeviceConfig` (operator changed the bands / enable) — an RPC,
6955
+ * 2. a qualifying motion/audio trigger (`onTrigger`) opens a window,
6956
+ * 3. a per-device WAKE-UP armed for the exact instant the decision could
6957
+ * change on its own (`decision-schedule.ts` + `device-wakeups.ts`): an
6958
+ * `events` window closing at `lastTrigger + postBufferSec`, or a band edge,
6959
+ * 4. a `stream-broker` ready transition (boot restore via `ReadinessRestore`),
6960
+ * 5. the periodic RECONCILE (below), for everything the four above can lose.
6961
+ *
6962
+ * (3) is why a detach is now punctual. A trigger only ever OPENED a window — the
6963
+ * demand test is a sliding `atMs - lastMotionMs <= postMs` anchored on the LAST
6964
+ * trigger, so nothing called `evaluateDevice` at the moment it closed and the
6965
+ * detach waited for the next pass. A 4K camera stayed dialled, decoded and
6966
+ * written for up to a whole tick past the operator's `postBufferSec`.
6967
+ *
6968
+ * There are THREE periodic passes, each on its own timer and each non-overlapping
6969
+ * (`periodic-pass.ts`):
6970
+ * - LIVENESS ({@link LIVENESS_TICK_MS}) — cheap, in-memory: is a pinned writer
6971
+ * producing nothing?
6972
+ * - PLACEMENT ({@link PLACEMENT_TICK_MS}) — cheap (a statfs per location), and
6973
+ * deliberately NOT on the attach beat, so a cold broker cannot delay it.
6974
+ * - RECONCILE ({@link RECONCILE_TICK_MS}) — the only one that can touch the
6975
+ * broker, and the only backstop for a lost wake-up or a failed attach.
6976
+ *
6977
+ * They were ONE pass until 2026-08-13, when a convergence sweep wedged on
6978
+ * unbounded broker RPCs and took the liveness watchdog down with it for 9-27
6979
+ * minutes at a time; placement was split out of the second on 2026-08-18, after
6980
+ * a converge pass blocked for 60 001 ms (3 x {@link ATTACH_RPC_TIMEOUT_MS})
6981
+ * cold-dialling ~40 RTSP sources.
6448
6982
  *
6449
6983
  * EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
6450
6984
  * trigger-gating. The enable intent is persisted in durable-state by the
@@ -6457,8 +6991,45 @@ var SegmentWriter = class {
6457
6991
  * `ReadinessRegistry` is the single source of truth.
6458
6992
  */
6459
6993
  var DEFAULT_WATCH_INTERVAL_MS = 2e3;
6460
- /** Periodic re-evaluation to catch band boundaries crossed with no config change. */
6461
- var TICK_MS = 3e4;
6994
+ /**
6995
+ * Liveness cadence — "is a pinned writer producing nothing?". Unchanged: the
6996
+ * question is an in-memory scan over `active`, it costs nothing, and its bound
6997
+ * (`writerIdleBoundMs`, floor 90 s) is sized against this beat.
6998
+ */
6999
+ var LIVENESS_TICK_MS = 3e4;
7000
+ /**
7001
+ * Placement cadence — unchanged from the beat it used to share with
7002
+ * convergence, because nothing about placement wanted to be rarer. What changed
7003
+ * is that it no longer rides a pass that can block on broker RPCs: it is one
7004
+ * `statfs` per location and it re-plans only, so a plan reaches a camera at its
7005
+ * NEXT attach.
7006
+ */
7007
+ var PLACEMENT_TICK_MS = 3e4;
7008
+ /**
7009
+ * Reconcile cadence — the SAFETY NET, not the mechanism.
7010
+ *
7011
+ * It was 30 s because it WAS the mechanism: nothing else ever closed an `events`
7012
+ * window or crossed a band edge, so the loop had to keep asking. Both are now
7013
+ * armed for their exact instant (`decision-schedule.ts`), a config change is an
7014
+ * RPC, and a broker recovery is a readiness transition — so this pass exists
7015
+ * only for what those cannot cover: a wake-up lost because the event path threw
7016
+ * before it could arm one, an attach that failed with no later ready transition
7017
+ * to ride, a timer lost to a runner respawn, and clock drift.
7018
+ *
7019
+ * 2 minutes, and the number is chosen against a bound this recorder already
7020
+ * accepts. The liveness watchdog tolerates a pinned-but-silent writer for
7021
+ * `WRITER_IDLE_MIN_MS` (90 s) before it recovers it; a device that WANTS
7022
+ * recording but is detached is the same class of fault — footage is not being
7023
+ * written and only a periodic check will notice — so its recovery is put on the
7024
+ * same order rather than an order slower. Four times rarer than the beat that
7025
+ * blocked for 60 001 ms on 2026-08-18, while still bounding the worst case at
7026
+ * ~2 minutes of footage for a camera whose event path failed entirely.
7027
+ *
7028
+ * Going rarer than this trades real footage for CPU that the pass no longer
7029
+ * spends anyway: with the broker-readiness pre-check below, a reconcile over a
7030
+ * healthy fleet is N config reads and no RPC at all.
7031
+ */
7032
+ var RECONCILE_TICK_MS = 12e4;
6462
7033
  /**
6463
7034
  * Bound on the broker lease-release RPC (`releaseStreamWithCodec`) during
6464
7035
  * teardown. The UDS request default is 60s, but the runner supervisor's
@@ -6497,11 +7068,15 @@ var RELEASE_RPC_TIMEOUT_MS = 2e3;
6497
7068
  */
6498
7069
  var ATTACH_RPC_TIMEOUT_MS = 2e4;
6499
7070
  /**
6500
- * A pass longer than this is reported with its duration. Equal to the pass
6501
- * cadence: a pass that cannot finish inside its own interval is the condition
6502
- * that used to go unlogged for half an hour.
7071
+ * A pass longer than this is reported with its duration. Each pass uses its OWN
7072
+ * cadence as the budget: a pass that cannot finish inside its own interval is
7073
+ * the condition that used to go unlogged for half an hour. Keeping it per-pass
7074
+ * matters now that the three cadences differ — a 40 s reconcile is healthy
7075
+ * against a 120 s beat and was a red flag against a 30 s one.
6503
7076
  */
6504
- var PASS_SLOW_AFTER_MS = TICK_MS;
7077
+ function passSlowAfterMs(intervalMs) {
7078
+ return intervalMs;
7079
+ }
6505
7080
  /**
6506
7081
  * Race `promise` against a deadline. Handlers stay attached to the losing
6507
7082
  * promise, so a post-deadline settlement can never surface as a process-level
@@ -6584,6 +7159,14 @@ var RecordingController = class {
6584
7159
  */
6585
7160
  lastSegmentAt = /* @__PURE__ */ new Map();
6586
7161
  /**
7162
+ * Per `deviceId:profile`, the `attachedAtMs` of the attach that has retained
7163
+ * at least one `events` segment. Read only to LEVEL the discard log: an
7164
+ * attach that kept nothing is a wasted pull (warn), a later discard from one
7165
+ * that kept something is the tail of the window (info). Bounded by
7166
+ * device×profile like {@link lastSegmentAt}, and cleared with it on detach.
7167
+ */
7168
+ attachRetained = /* @__PURE__ */ new Map();
7169
+ /**
6587
7170
  * Devices already named by {@link reportUnrecordable} — one warn per device
6588
7171
  * per "enabled but can never record" episode, re-armed as soon as the device
6589
7172
  * records again.
@@ -6597,24 +7180,91 @@ var RecordingController = class {
6597
7180
  */
6598
7181
  skippedProfiles = /* @__PURE__ */ new Map();
6599
7182
  restore = null;
7183
+ /**
7184
+ * One pending wake-up per device, armed for the exact instant its recording
7185
+ * decision could change on its own — an `events` window closing or a band
7186
+ * edge. THE reason a detach is punctual instead of up to a pass late.
7187
+ */
7188
+ wakeups;
6600
7189
  /** Cheap, never-delayed: is any pinned writer producing nothing? */
6601
7190
  livenessPass = null;
6602
- /** Expensive: re-evaluate bands + placement for every configured device. */
6603
- convergePass = null;
7191
+ /** Cheap, and off the attach beat: re-plan where each (device, profile) writes. */
7192
+ placementPass = null;
7193
+ /** The backstop: re-evaluate every tracked-or-configured device. */
7194
+ reconcilePass = null;
6604
7195
  stopped = false;
6605
7196
  /** A reversible maintenance lease. Unlike `stopped`, it never changes
6606
7197
  * persisted recording intent and `resume()` restarts normal convergence. */
6607
7198
  paused = false;
7199
+ /**
7200
+ * The ADAPTIVE attach bound. A transient `spawn ffmpeg` failure (EAGAIN) is
7201
+ * the host saying it cannot fork right now, and the answer is to attach
7202
+ * FEWER cameras at once — not to serialise them, which `e2fa17a2b` removed
7203
+ * for good reason. Every bounded-parallel attach in this file reads it.
7204
+ */
7205
+ spawnPressure;
6608
7206
  constructor(deps) {
6609
7207
  this.deps = deps;
7208
+ this.spawnPressure = new SpawnPressureGovernor({
7209
+ maxConcurrency: 8,
7210
+ now: () => this.now(),
7211
+ logger: deps.logger
7212
+ });
7213
+ this.wakeups = new DeviceWakeups({
7214
+ logger: deps.logger,
7215
+ now: () => this.now(),
7216
+ fire: (deviceId) => this.onWakeup(deviceId)
7217
+ });
6610
7218
  }
6611
7219
  now() {
6612
7220
  return this.deps.now?.() ?? Date.now();
6613
7221
  }
6614
7222
  /**
7223
+ * A per-device wake-up came due: re-decide. `evaluateDevice` re-arms (or
7224
+ * cancels) the next one as part of deciding, so the chain is self-sustaining.
7225
+ *
7226
+ * A throw here breaks that chain — the device keeps whatever state it had and
7227
+ * has no timer behind it — so it is NAMED, and the reconcile is what picks it
7228
+ * up. Silence would read as "the window never closed".
7229
+ */
7230
+ onWakeup(deviceId) {
7231
+ this.evaluateDevice(deviceId).catch((err) => {
7232
+ this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
7233
+ tags: { deviceId },
7234
+ meta: { error: errMsg(err) }
7235
+ });
7236
+ });
7237
+ }
7238
+ /**
7239
+ * Arm (or cancel) this device's next wake-up from the config it was just
7240
+ * evaluated against.
7241
+ *
7242
+ * Called from `evaluateDevice` BEFORE the attach/detach work, not after: the
7243
+ * attach chain can throw (broker not ready, stream unpublished) and the
7244
+ * schedule does not depend on whether it succeeded. Arming first means a
7245
+ * failed attach still gets its band edge, and only a failure to READ the
7246
+ * config can lose a wake-up.
7247
+ */
7248
+ scheduleNextDecision(deviceId, config) {
7249
+ if (this.stopped || this.paused) {
7250
+ this.wakeups.cancel(deviceId);
7251
+ return;
7252
+ }
7253
+ const next = nextDecisionChangeAt({
7254
+ config,
7255
+ triggers: this.triggersFor(deviceId),
7256
+ nowMs: this.now()
7257
+ });
7258
+ if (next === null) {
7259
+ this.wakeups.cancel(deviceId);
7260
+ return;
7261
+ }
7262
+ this.wakeups.arm(deviceId, next.atMs, next.reason);
7263
+ }
7264
+ /**
6615
7265
  * Boot restore: seed the readiness-gated queue with every persisted device and
6616
7266
  * (re)evaluate each on every `stream-broker` ready transition. Also arms the
6617
- * periodic tick. Idempotent — safe to call once after the index is hydrated.
7267
+ * periodic passes. Idempotent — safe to call once after the index is hydrated.
6618
7268
  */
6619
7269
  async start() {
6620
7270
  if (this.stopped) return;
@@ -6627,6 +7277,7 @@ var RecordingController = class {
6627
7277
  }
6628
7278
  this.restore = new ReadinessRestore({
6629
7279
  subscribeReady: (handler) => this.deps.onBrokerReady(handler),
7280
+ concurrency: () => this.spawnPressure.concurrency,
6630
7281
  attempt: async (deviceId) => {
6631
7282
  await this.evaluateDevice(deviceId);
6632
7283
  },
@@ -6635,45 +7286,83 @@ var RecordingController = class {
6635
7286
  meta: { error: errMsg(err) }
6636
7287
  })
6637
7288
  });
6638
- if (this.convergePass === null) {
7289
+ if (this.reconcilePass === null) {
6639
7290
  this.livenessPass = startPeriodicPass({
6640
7291
  name: "liveness",
6641
- intervalMs: TICK_MS,
6642
- slowAfterMs: PASS_SLOW_AFTER_MS,
7292
+ intervalMs: LIVENESS_TICK_MS,
7293
+ slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
6643
7294
  logger: this.deps.logger,
6644
7295
  now: () => this.now(),
6645
7296
  run: () => this.checkIdleWriters()
6646
7297
  });
6647
- this.convergePass = startPeriodicPass({
6648
- name: "convergence",
6649
- intervalMs: TICK_MS,
6650
- slowAfterMs: PASS_SLOW_AFTER_MS,
7298
+ this.placementPass = startPeriodicPass({
7299
+ name: "placement",
7300
+ intervalMs: PLACEMENT_TICK_MS,
7301
+ slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
7302
+ logger: this.deps.logger,
7303
+ now: () => this.now(),
7304
+ run: () => this.replanPlacement()
7305
+ });
7306
+ this.reconcilePass = startPeriodicPass({
7307
+ name: "reconcile",
7308
+ intervalMs: RECONCILE_TICK_MS,
7309
+ slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
6651
7310
  logger: this.deps.logger,
6652
7311
  now: () => this.now(),
6653
- run: () => this.converge()
7312
+ run: () => this.reconcile()
6654
7313
  });
6655
7314
  }
6656
7315
  await this.restore.start(ids.map((id) => [id, true]));
6657
7316
  }
6658
- /** Re-evaluate every currently-tracked OR active device on the periodic pass. */
6659
- async converge() {
6660
- if (this.stopped) return;
7317
+ /**
7318
+ * Every device the recorder is responsible for right now: those with writers
7319
+ * attached, plus every one with a persisted config. A failure to enumerate the
7320
+ * persisted set degrades to "the active ones" rather than aborting the pass.
7321
+ */
7322
+ async trackedDeviceIds() {
6661
7323
  const ids = new Set(this.active.keys());
6662
- let persisted = [];
6663
7324
  try {
6664
- persisted = await this.deps.enabledDeviceIds();
6665
- } catch {}
6666
- for (const id of persisted) ids.add(id);
7325
+ for (const id of await this.deps.enabledDeviceIds()) ids.add(id);
7326
+ } catch (err) {
7327
+ this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
7328
+ error: errMsg(err),
7329
+ activeDevices: this.active.size
7330
+ } });
7331
+ }
7332
+ return [...ids];
7333
+ }
7334
+ /**
7335
+ * Re-plan where each (device, profile) writes. Its OWN pass: cheap (no I/O
7336
+ * beyond a statfs per location), and it must not be starved by an attach that
7337
+ * is waiting on a 20 s broker timeout. It changes the PLAN only — a running
7338
+ * writer keeps its root until it next attaches, which is the boundary rule.
7339
+ */
7340
+ async replanPlacement() {
7341
+ if (this.stopped || this.paused) return;
7342
+ if (this.deps.onPlacementTick === void 0) return;
7343
+ const ids = await this.trackedDeviceIds();
6667
7344
  try {
6668
- await this.deps.onPlacementTick?.([...ids]);
7345
+ await this.deps.onPlacementTick(ids);
6669
7346
  } catch (err) {
6670
7347
  this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: errMsg(err) } });
6671
7348
  }
6672
- await runBounded([...ids], 8, async (id) => {
7349
+ }
7350
+ /**
7351
+ * The backstop pass: re-evaluate every tracked-or-configured device.
7352
+ *
7353
+ * Nothing here is the normal path any more — a window close, a band edge, a
7354
+ * config change and a broker recovery all reach `evaluateDevice` on their own.
7355
+ * This exists for the four things that cannot: a wake-up the event path threw
7356
+ * before arming, an attach that failed with no later readiness transition to
7357
+ * ride, a timer lost to a runner respawn, and clock drift.
7358
+ */
7359
+ async reconcile() {
7360
+ if (this.stopped) return;
7361
+ await runBounded(await this.trackedDeviceIds(), this.spawnPressure.concurrency, async (id) => {
6673
7362
  try {
6674
7363
  await this.evaluateDevice(id);
6675
7364
  } catch (err) {
6676
- this.deps.logger.warn("recorder controller: convergence evaluate failed", {
7365
+ this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
6677
7366
  tags: { deviceId: id },
6678
7367
  meta: { error: errMsg(err) }
6679
7368
  });
@@ -6697,8 +7386,9 @@ var RecordingController = class {
6697
7386
  * idle source can re-dial cleanly), then re-evaluate: if the band still demands
6698
7387
  * recording, `evaluateDevice` re-queues + re-attaches a fresh writer set. If
6699
7388
  * the broker is not ready, the re-attach throws and the device stays on the
6700
- * readiness queue (retried on the next broker ready) — and the periodic tick is
6701
- * the backstop. Bounded OUTER retry (TICK_MS cadence), never a tight loop.
7389
+ * readiness queue (retried on the next broker ready) — and the periodic
7390
+ * reconcile is the backstop. Bounded OUTER retry (RECONCILE_TICK_MS cadence),
7391
+ * never a tight loop.
6702
7392
  */
6703
7393
  async recoverWriterGaveUp(deviceId) {
6704
7394
  if (this.stopped) return;
@@ -6708,7 +7398,7 @@ var RecordingController = class {
6708
7398
  await this.evaluateDevice(deviceId);
6709
7399
  } catch (err) {
6710
7400
  this.restore?.add(deviceId, true);
6711
- this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/tick)", {
7401
+ this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
6712
7402
  tags: { deviceId },
6713
7403
  meta: { error: errMsg(err) }
6714
7404
  });
@@ -6759,10 +7449,16 @@ var RecordingController = class {
6759
7449
  }
6760
7450
  /**
6761
7451
  * Record a qualifying trigger (motion/audio) for a device and converge: an
6762
- * `events` band whose window this opens attaches now; the periodic tick
6763
- * detaches it once `postBufferSec` elapses. Called from the addon's
6764
- * EventCapture subscription. The qualification test (motion detected / audio
6765
- * over threshold) is done upstream by the time we're here, it qualified.
7452
+ * `events` band whose window this opens attaches now, and the same
7453
+ * `evaluateDevice` arms the wake-up that will CLOSE it at
7454
+ * `atMs + postBufferSec`. Called from the addon's EventCapture subscription.
7455
+ * The qualification test (motion detected / audio over threshold) is done
7456
+ * upstream — by the time we're here, it qualified.
7457
+ *
7458
+ * A fresher trigger re-arms the wake-up later, which is exactly how the
7459
+ * sliding window already behaved: continuous motion stays ONE recording,
7460
+ * because `evaluateDevice` short-circuits at "already recording — converged"
7461
+ * and only the timer moves.
6766
7462
  */
6767
7463
  async onTrigger(deviceId, source, atMs) {
6768
7464
  if (this.stopped) return;
@@ -6774,7 +7470,7 @@ var RecordingController = class {
6774
7470
  try {
6775
7471
  await this.evaluateDevice(deviceId);
6776
7472
  } catch (err) {
6777
- this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/tick)", {
7473
+ this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/reconcile)", {
6778
7474
  tags: { deviceId },
6779
7475
  meta: {
6780
7476
  source,
@@ -6795,7 +7491,9 @@ var RecordingController = class {
6795
7491
  if (this.stopped || this.paused) return;
6796
7492
  const config = await this.deps.loadConfig(deviceId);
6797
7493
  if (this.stopped || this.paused) return;
6798
- if (!shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()))) {
7494
+ const wantRecording = shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()));
7495
+ this.scheduleNextDecision(deviceId, config);
7496
+ if (!wantRecording) {
6799
7497
  this.reportUnrecordable(deviceId, config);
6800
7498
  await this.detachDevice(deviceId);
6801
7499
  this.restore?.remove(deviceId);
@@ -6854,6 +7552,15 @@ var RecordingController = class {
6854
7552
  });
6855
7553
  }
6856
7554
  /**
7555
+ * Throw before spending a single broker RPC when the registry already says the
7556
+ * `stream-broker` on the owner node is not ready. See
7557
+ * {@link RecordingControllerDeps.brokerReady}; absent → assume ready.
7558
+ */
7559
+ failFastIfBrokerCold(deviceId) {
7560
+ if (this.deps.brokerReady?.() !== false) return;
7561
+ throw new Error(`recorder controller: stream-broker not ready on ${this.deps.ownerNodeId} — attach for device ${deviceId} deferred to the next ready transition`);
7562
+ }
7563
+ /**
6857
7564
  * The attach body — extracted so the {@link attaching} in-flight guard in
6858
7565
  * `evaluateDevice` wraps it cleanly. Resolves the profiles, spawns a
6859
7566
  * SegmentWriter + watcher per profile, and records the set in `active`.
@@ -6865,6 +7572,7 @@ var RecordingController = class {
6865
7572
  */
6866
7573
  async performAttach(deviceId, config) {
6867
7574
  this.restore?.add(deviceId, true);
7575
+ this.failFastIfBrokerCold(deviceId);
6868
7576
  const profiles = await this.resolveProfiles(deviceId, config.profiles);
6869
7577
  if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
6870
7578
  const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
@@ -6922,6 +7630,16 @@ var RecordingController = class {
6922
7630
  }
6923
7631
  }
6924
7632
  async runFillMissingProfiles(deviceId, config, current, skipped) {
7633
+ if (this.deps.brokerReady?.() === false) {
7634
+ this.deps.logger.info("recorder: skipped-profile retry deferred — stream-broker not ready (healthy profiles keep recording)", {
7635
+ tags: { deviceId },
7636
+ meta: {
7637
+ profiles: [...skipped],
7638
+ ownerNodeId: this.deps.ownerNodeId
7639
+ }
7640
+ });
7641
+ return;
7642
+ }
6925
7643
  const have = new Set(current.map((r) => r.profile));
6926
7644
  const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
6927
7645
  const added = [];
@@ -6999,7 +7717,8 @@ var RecordingController = class {
6999
7717
  logger: deviceLog,
7000
7718
  onGaveUp: () => {
7001
7719
  this.recoverWriterGaveUp(deviceId);
7002
- }
7720
+ },
7721
+ onResourcePressure: () => this.spawnPressure.report(deviceId)
7003
7722
  });
7004
7723
  const writerStartedMs = this.now();
7005
7724
  writer.start();
@@ -7014,7 +7733,10 @@ var RecordingController = class {
7014
7733
  intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
7015
7734
  onFinalized: async (startMs, durMs, flatAbsPath) => {
7016
7735
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
7017
- await this.handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath);
7736
+ await this.handleFinalizedSegment(deviceId, profile, placement, {
7737
+ attachedAtMs: writerStartedMs,
7738
+ segmentSeconds
7739
+ }, startMs, durMs, flatAbsPath);
7018
7740
  },
7019
7741
  logger: deviceLog
7020
7742
  }),
@@ -7031,13 +7753,16 @@ var RecordingController = class {
7031
7753
  * keep/discard gate touches `events`-mode segments only, so the continuous
7032
7754
  * path is unchanged.
7033
7755
  */
7034
- async handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath) {
7756
+ async handleFinalizedSegment(deviceId, profile, placement, attach, startMs, durMs, flatAbsPath) {
7035
7757
  try {
7036
7758
  const band = activeBandAt({ bands: (await this.deps.loadConfig(deviceId)).bands ?? [] }, new Date(startMs + durMs));
7037
- if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, this.triggersFor(deviceId))) {
7759
+ const triggers = this.triggersFor(deviceId);
7760
+ if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, triggers, attach)) {
7761
+ this.reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs);
7038
7762
  await promises.unlink(flatAbsPath).catch(() => {});
7039
7763
  return;
7040
7764
  }
7765
+ if (band?.mode === "events") this.attachRetained.set(idleKey(deviceId, profile), attach.attachedAtMs);
7041
7766
  await this.deps.segmentStore.onFinalized({
7042
7767
  deviceId,
7043
7768
  profile,
@@ -7058,6 +7783,47 @@ var RecordingController = class {
7058
7783
  }
7059
7784
  }
7060
7785
  /**
7786
+ * Name a discarded `events`-band segment. This branch deletes recorded
7787
+ * footage and used to write NOTHING: on camera 1439 the operator saw `pinned`
7788
+ * then `unpinned` with an empty timeline in between and no line anywhere said
7789
+ * why — the recorder had spawned ffmpeg, pulled 4K RTSP for 31 s and unlinked
7790
+ * every segment it produced.
7791
+ *
7792
+ * Level is chosen, not defaulted. A discard at the TAIL of a window is the
7793
+ * feature working (`info`) — the writer keeps running until the next converge
7794
+ * tick and those trailing segments were never asked for. A discard from an
7795
+ * attach that has retained NOTHING is a fault (`warn`): the whole pull was
7796
+ * wasted, which is exactly the shape of the 1439 loss and of any future
7797
+ * regression in the attach allowance.
7798
+ */
7799
+ reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs) {
7800
+ const { preMs, postMs } = resolveBandBufferMs(band);
7801
+ const keptAnything = this.attachRetained.get(idleKey(deviceId, profile)) === attach.attachedAtMs;
7802
+ const meta = {
7803
+ profile,
7804
+ startMs,
7805
+ durMs,
7806
+ lastMotionMs: triggers.lastMotionMs,
7807
+ lastAudioMs: triggers.lastAudioMs,
7808
+ postMs,
7809
+ preMs,
7810
+ attachedAtMs: attach.attachedAtMs,
7811
+ missedByMs: segmentMissedByMs(startMs, band, triggers, attach),
7812
+ keptAnythingFromThisAttach: keptAnything
7813
+ };
7814
+ if (keptAnything) {
7815
+ this.deps.logger.info("recorder: discarding events segment outside every trigger window (tail of the window)", {
7816
+ tags: { deviceId },
7817
+ meta
7818
+ });
7819
+ return;
7820
+ }
7821
+ this.deps.logger.warn("recorder: discarding events segment — this attach has retained NOTHING, the whole pull is wasted", {
7822
+ tags: { deviceId },
7823
+ meta
7824
+ });
7825
+ }
7826
+ /**
7061
7827
  * Resolve the storage location a (device, profile) writes to, at an attach
7062
7828
  * boundary. Delegates to the injected {@link RecordingControllerDeps.placeProfile}
7063
7829
  * when the addon supplied one; otherwise to the pure `resolvePlacement` in
@@ -7099,7 +7865,10 @@ var RecordingController = class {
7099
7865
  if (!recordings) return;
7100
7866
  this.active.delete(deviceId);
7101
7867
  this.skippedProfiles.delete(deviceId);
7102
- for (const r of recordings) this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
7868
+ for (const r of recordings) {
7869
+ this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
7870
+ this.attachRetained.delete(idleKey(deviceId, r.profile));
7871
+ }
7103
7872
  await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
7104
7873
  this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
7105
7874
  tags: { deviceId },
@@ -7110,6 +7879,7 @@ var RecordingController = class {
7110
7879
  async pause() {
7111
7880
  if (this.stopped || this.paused) return;
7112
7881
  this.paused = true;
7882
+ this.wakeups.cancelAll();
7113
7883
  while (this.attachingTasks.size > 0) await Promise.allSettled([...this.attachingTasks.values()]);
7114
7884
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
7115
7885
  }
@@ -7125,13 +7895,16 @@ var RecordingController = class {
7125
7895
  }
7126
7896
  for (const deviceId of ids) await this.evaluateDevice(deviceId);
7127
7897
  }
7128
- /** Tear EVERYTHING down: timer, restore subscription, every writer/watcher/lease. */
7898
+ /** Tear EVERYTHING down: timers, restore subscription, every writer/watcher/lease. */
7129
7899
  async stop() {
7130
7900
  this.stopped = true;
7901
+ this.wakeups.cancelAll();
7131
7902
  this.livenessPass?.stop();
7132
7903
  this.livenessPass = null;
7133
- this.convergePass?.stop();
7134
- this.convergePass = null;
7904
+ this.placementPass?.stop();
7905
+ this.placementPass = null;
7906
+ this.reconcilePass?.stop();
7907
+ this.reconcilePass = null;
7135
7908
  this.restore?.stop();
7136
7909
  this.restore = null;
7137
7910
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
@@ -8420,6 +9193,23 @@ var PLAYBACK_PREFIX = "playback";
8420
9193
  /** Data-plane prefix for on-demand single stills →
8421
9194
  * `/addon/recorder/still/<deviceId>/<epochSec>`. */
8422
9195
  var STILL_PREFIX = "still";
9196
+ /**
9197
+ * Bound on pending hour-ledger writes (one per finalized segment, an RPC to
9198
+ * the hub store). Steady state completes in milliseconds and holds ~0–2; under
9199
+ * a store stall the fleet-wide finalize rate (~3.4/s) fills 256 in ~75s of
9200
+ * grace before refusals begin. Each pending write holds one hour-row copy
9201
+ * (≤360 paths ≈ ~25KB), so the lane's worst-case heap is single-digit MB —
9202
+ * a dropped write is one silent re-seed from the archive walk (D148).
9203
+ */
9204
+ var LEDGER_GATE_LIMIT = 256;
9205
+ /**
9206
+ * Bound on pending mfra tail reads (one per finalized RECENT segment; each is
9207
+ * an open + two reads on the same 4-thread libuv pool the live writers use,
9208
+ * holding read buffers while pending). 64 bounds both the held memory and the
9209
+ * pool contention; a dropped capture costs that segment's client one extra
9210
+ * round trip to the file's tail on the first seek.
9211
+ */
9212
+ var MFRA_GATE_LIMIT = 64;
8423
9213
  /** Data-plane prefix for export MP4 downloads → `/addon/recorder/exports/<id>.mp4`. */
8424
9214
  var EXPORTS_PREFIX = "exports";
8425
9215
  /** How often the export janitor sweeps expired exports. */
@@ -8496,6 +9286,25 @@ var RecorderV2Addon = class extends BaseAddon {
8496
9286
  /** Sync-sample tables captured at finalize, served by the directory so a
8497
9287
  * playback seek needs no mfra tail fetch. RAM-only, TTL-bounded. */
8498
9288
  mfraTables = new MfraTableStore();
9289
+ /** Bound on the per-finalize fire-and-forget lanes (item 3, 2026-08-18): a
9290
+ * disk/store stall must throttle this work, never balloon live heap — the
9291
+ * unbounded pending set is what fed the two `Ineffective mark-compacts`
9292
+ * fatals. Saturation is reported loudly but throttled; every individual
9293
+ * drop is logged with its deviceId at the call site. */
9294
+ ledgerWriteGate = new PendingGate({
9295
+ label: "hour-ledger",
9296
+ limit: LEDGER_GATE_LIMIT,
9297
+ onSaturated: (report) => {
9298
+ this.reportGateSaturated(report);
9299
+ }
9300
+ });
9301
+ mfraCaptureGate = new PendingGate({
9302
+ label: "mfra-capture",
9303
+ limit: MFRA_GATE_LIMIT,
9304
+ onSaturated: (report) => {
9305
+ this.reportGateSaturated(report);
9306
+ }
9307
+ });
8499
9308
  /** Continuous + events band write-path controller (ffmpeg writers + watchers). */
8500
9309
  controller = null;
8501
9310
  /** Per-device motion/audio markers for playback (B3 events feed the controller). */
@@ -8533,6 +9342,20 @@ var RecorderV2Addon = class extends BaseAddon {
8533
9342
  constructor() {
8534
9343
  super({ ...DEFAULT_CONFIG });
8535
9344
  }
9345
+ /**
9346
+ * One throttled line per saturated lane per report interval. Lane-level (no
9347
+ * deviceId — the lane aggregates every camera); the per-drop debug lines at
9348
+ * the call sites carry the tag. WARN because a saturated lane means the
9349
+ * store or the disk is stalling under live finalize load — the line that
9350
+ * explains the missing ledger hours and mfra misses an operator will see.
9351
+ */
9352
+ reportGateSaturated(report) {
9353
+ this.ctx.logger.warn("recorder: fire-and-forget lane saturated — dropping over the bound", { meta: {
9354
+ lane: report.label,
9355
+ dropped: report.dropped,
9356
+ pending: report.pending
9357
+ } });
9358
+ }
8536
9359
  async onInitialize() {
8537
9360
  const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
8538
9361
  this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
@@ -8596,18 +9419,28 @@ var RecorderV2Addon = class extends BaseAddon {
8596
9419
  this.segmentHours?.dropSegments(rows);
8597
9420
  },
8598
9421
  onIndexed: (row, absPath) => {
8599
- this.segmentHours?.recordSegment(row);
9422
+ const ledger = this.segmentHours;
9423
+ if (ledger && !this.ledgerWriteGate.run(() => ledger.recordSegment(row))) this.ctx.logger.debug("recorder: hour-ledger write dropped — lane saturated; the archive walk reconciles this hour", {
9424
+ tags: { deviceId: row.deviceId },
9425
+ meta: { path: row.path }
9426
+ });
8600
9427
  if (!isMfraTableServable(row.startMs, Date.now())) return;
8601
- readMfraTable(absPath, row.bytes).then((table) => {
8602
- this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
8603
- }).catch((err) => {
8604
- this.ctx.logger.debug("mfra capture failed", {
8605
- tags: { deviceId: row.deviceId },
8606
- meta: {
8607
- path: row.path,
8608
- error: errMsg(err)
8609
- }
8610
- });
9428
+ if (!this.mfraCaptureGate.run(async () => {
9429
+ try {
9430
+ const table = await readMfraTable(absPath, row.bytes);
9431
+ this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
9432
+ } catch (err) {
9433
+ this.ctx.logger.debug("mfra capture failed", {
9434
+ tags: { deviceId: row.deviceId },
9435
+ meta: {
9436
+ path: row.path,
9437
+ error: errMsg(err)
9438
+ }
9439
+ });
9440
+ }
9441
+ })) this.ctx.logger.debug("recorder: mfra capture dropped — lane saturated; first seek pays one tail fetch", {
9442
+ tags: { deviceId: row.deviceId },
9443
+ meta: { path: row.path }
8611
9444
  });
8612
9445
  },
8613
9446
  removeDirIfEmpty: async (locationId, relDir) => {
@@ -8665,7 +9498,8 @@ var RecorderV2Addon = class extends BaseAddon {
8665
9498
  onBrokerReady: (handler) => this.ctx.onCapabilityStateChange("stream-broker", brokerScope(ingestOwner.ownerNodeId), (state) => {
8666
9499
  if (state === "ready") handler();
8667
9500
  }),
8668
- brokerCall: makeBrokerCall(brokerHandle)
9501
+ brokerCall: makeBrokerCall(brokerHandle),
9502
+ brokerReady: () => brokerHandle.isReady
8669
9503
  });
8670
9504
  const relocateEngine = new RelocateEngine({
8671
9505
  index: this.index,