@camstack/addon-pipeline 1.2.69 → 1.2.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/audio-analyzer/index.js +1 -1
  2. package/dist/audio-analyzer/index.mjs +1 -1
  3. package/dist/detection-pipeline/index.js +32 -30
  4. package/dist/detection-pipeline/index.mjs +32 -30
  5. package/dist/{dist-tJF2BzWi.js → dist-CtOVWTCi.js} +586 -41
  6. package/dist/{dist-DGLqaonv.mjs → dist-D1Xk5Tji.mjs} +586 -41
  7. package/dist/{event-loop-stall-monitor-D9d2e49f.js → event-loop-stall-monitor-BJs7Yws7.js} +1 -1
  8. package/dist/{event-loop-stall-monitor-C6_VDupx.mjs → event-loop-stall-monitor-Cmbt664E.mjs} +1 -1
  9. package/dist/motion-wasm/index.js +1 -1
  10. package/dist/motion-wasm/index.mjs +1 -1
  11. package/dist/pipeline-runner/index.js +100 -30
  12. package/dist/pipeline-runner/index.mjs +100 -30
  13. package/dist/recorder/index.js +734 -104
  14. package/dist/recorder/index.mjs +734 -104
  15. package/dist/session-decode/decode-worker-child.js +483 -52
  16. package/dist/session-decode/decode-worker-child.mjs +483 -52
  17. package/dist/stream-broker/_stub.js +660 -556
  18. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-DCyQ7RKH.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BWf4H4Nl.mjs} +3 -3
  19. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Cu_kQpIy.mjs +26 -0
  20. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CExO0piw.mjs +26 -0
  21. package/dist/stream-broker/{hostInit-DxqtIjAe.mjs → hostInit-DD5Z1U6p.mjs} +3 -3
  22. package/dist/stream-broker/index.js +501 -32
  23. package/dist/stream-broker/index.mjs +501 -32
  24. package/dist/stream-broker/remoteEntry.js +1 -1
  25. package/dist/{worker-protocol-D6tXz9Or.js → worker-protocol-DEV7g32b.js} +26 -19
  26. package/dist/{worker-protocol-DuftUFEI.mjs → worker-protocol-Dm6LQ9Ls.mjs} +26 -19
  27. package/package.json +1 -1
  28. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-ircnzt0s.mjs +0 -26
  29. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DU7u9EJt.mjs +0 -26
@@ -1,5 +1,5 @@
1
1
  const require_chunk = require("../chunk-emK7D4bc.js");
2
- const require_dist = require("../dist-tJF2BzWi.js");
2
+ const require_dist = require("../dist-CtOVWTCi.js");
3
3
  const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
4
4
  const require_addon_utils = require("../addon-utils-CQs-AjTJ.js");
5
5
  let node_crypto = require("node:crypto");
@@ -683,6 +683,36 @@ var RecordingIndex = class {
683
683
  }
684
684
  return out;
685
685
  }
686
+ /**
687
+ * Every (device, profile, location) pile in ONE pass — bytes and file count,
688
+ * no sort and no cache rebuild.
689
+ *
690
+ * This is the input to the operator-armed rebalance (D116): it asks "where is
691
+ * each camera-profile's footage, and how much of it", which is exactly the
692
+ * unit the placement plan assigns and the relocate mover moves. Building it
693
+ * from `segmentsOnLocation` per location would sort the whole archive once
694
+ * per location instead.
695
+ */
696
+ piles() {
697
+ const out = /* @__PURE__ */ new Map();
698
+ for (const m of this.byDevice.values()) for (const s of m.values()) {
699
+ const key = `${s.deviceId}:${s.profile}:${s.locationId}`;
700
+ const prev = out.get(key);
701
+ if (prev) {
702
+ prev.bytes += s.bytes;
703
+ prev.files += 1;
704
+ continue;
705
+ }
706
+ out.set(key, {
707
+ deviceId: s.deviceId,
708
+ profile: s.profile,
709
+ locationId: s.locationId,
710
+ bytes: s.bytes,
711
+ files: 1
712
+ });
713
+ }
714
+ return [...out.values()];
715
+ }
686
716
  /** All segments on a storage location across every device, oldest-first. */
687
717
  segmentsOnLocation(locationId) {
688
718
  const out = [];
@@ -872,6 +902,19 @@ function resolveEvictionDomain(locations, locationId, realpath = node_fs.realpat
872
902
  *
873
903
  * Runs INSIDE the recorder process — container uid, same-uid files, no host
874
904
  * chown trap (see docs/history 2026-07-29).
905
+ *
906
+ * ── Phase 2 (D116, multi-location) ────────────────────────────────────────
907
+ * Two additions the OPERATOR-ARMED rebalance needs:
908
+ *
909
+ * - A move is scoped to one CAMERA (and optionally to specific profiles),
910
+ * not just to a whole disk. The rebalance's unit is a (camera, profile)
911
+ * pile, because that is the unit the placement plan assigns.
912
+ * - Concurrent starts QUEUE instead of being refused. The engine is still
913
+ * single-flight — two movers on one disk thrash both — but a rebalance
914
+ * enqueues one job per pile, and refusing the second turned a
915
+ * fifteen-camera rebalance into fifteen manual retries. Jobs run FIFO; a
916
+ * job cancelled while queued never runs, and a job that FAILS does not
917
+ * strand the queue behind it.
875
918
  */
876
919
  var DEFAULT_THROTTLE_MBPS = 40;
877
920
  /** Progress callback cadence (files) — cheap enough to fire often. */
@@ -915,22 +958,48 @@ async function moveFile(src, dst, srcBytes) {
915
958
  var RelocateEngine = class {
916
959
  deps;
917
960
  jobs = /* @__PURE__ */ new Map();
961
+ /** Job ids waiting for the mover, in enqueue order. */
962
+ queue = [];
963
+ /** Set while a job is executing — the single-flight latch. */
964
+ draining = false;
918
965
  constructor(deps) {
919
966
  this.deps = deps;
920
967
  }
921
968
  list() {
922
969
  return [...this.jobs.values()].sort((a, b) => b.startedAt - a.startedAt).map(snapshot);
923
970
  }
971
+ /**
972
+ * Cancel a running OR queued job. A running job stops after its current
973
+ * file; a queued job is terminated where it stands and never runs — which is
974
+ * why this returns true for both, and why the cancelled job still gets its
975
+ * `onFinished` audit row.
976
+ */
924
977
  cancel(jobId) {
925
978
  const job = this.jobs.get(jobId);
926
- if (!job || job.state !== "running") return false;
979
+ if (!job) return false;
980
+ if (job.state === "running") {
981
+ job.cancelRequested = true;
982
+ return true;
983
+ }
984
+ if (job.state !== "queued") return false;
927
985
  job.cancelRequested = true;
986
+ job.state = "cancelled";
987
+ job.finishedAt = this.deps.now();
988
+ const at = this.queue.indexOf(jobId);
989
+ if (at >= 0) this.queue.splice(at, 1);
990
+ this.deps.onFinished?.(snapshot(job));
928
991
  return true;
929
992
  }
930
- /** Start a relocation. Throws on invalid locations or when a job is already
931
- * running (single-flight two movers on one disk thrash both). */
993
+ /**
994
+ * Enqueue a relocation and return its job id. Throws only on an input the
995
+ * mover could never satisfy (unknown/identical/read-only locations) —
996
+ * validated SYNCHRONOUSLY so a caller arming a rebalance learns about a bad
997
+ * target before any job runs.
998
+ *
999
+ * Concurrency: the returned job may be `queued` rather than `running`. The
1000
+ * engine still moves one job at a time.
1001
+ */
932
1002
  start(input) {
933
- for (const j of this.jobs.values()) if (j.state === "running") throw new Error(`a relocation is already running (${j.jobId})`);
934
1003
  const locs = this.deps.locations();
935
1004
  const from = locs.find((l) => l.id === input.fromLocationId);
936
1005
  const to = locs.find((l) => l.id === input.toLocationId);
@@ -940,10 +1009,13 @@ var RelocateEngine = class {
940
1009
  if (to.readOnly) throw new Error(`target location is read-only: ${to.id}`);
941
1010
  const job = {
942
1011
  jobId: this.deps.newId(),
943
- state: "running",
1012
+ state: "queued",
944
1013
  fromLocationId: from.id,
945
1014
  toLocationId: to.id,
946
- deviceId: null,
1015
+ deviceId: input.deviceId ?? null,
1016
+ profiles: input.profiles ? [...input.profiles] : [],
1017
+ footageClass: input.footageClass,
1018
+ throttleMbps: input.throttleMbps ?? DEFAULT_THROTTLE_MBPS,
947
1019
  entities: input.entities ? [...input.entities] : ["segments"],
948
1020
  filesMoved: 0,
949
1021
  bytesMoved: 0,
@@ -954,15 +1026,56 @@ var RelocateEngine = class {
954
1026
  cancelRequested: false
955
1027
  };
956
1028
  this.jobs.set(job.jobId, job);
957
- this.run(job, from, to, input.throttleMbps ?? DEFAULT_THROTTLE_MBPS, input.footageClass);
1029
+ this.queue.push(job.jobId);
1030
+ this.drain();
958
1031
  return job.jobId;
959
1032
  }
960
- async run(job, from, to, throttleMbps, footageClass) {
1033
+ /**
1034
+ * Run queued jobs one at a time, in enqueue order. Re-entrant by design: the
1035
+ * `draining` latch is the single-flight guarantee, and every terminal state
1036
+ * (including `failed`) falls through to the next job — a queue stranded
1037
+ * behind one bad camera is the failure mode FIFO exists to avoid.
1038
+ */
1039
+ async drain() {
1040
+ if (this.draining) return;
1041
+ this.draining = true;
1042
+ try {
1043
+ for (;;) {
1044
+ const jobId = this.queue.shift();
1045
+ if (jobId === void 0) return;
1046
+ const job = this.jobs.get(jobId);
1047
+ if (!job || job.state !== "queued") continue;
1048
+ const locs = this.deps.locations();
1049
+ const from = locs.find((l) => l.id === job.fromLocationId);
1050
+ const to = locs.find((l) => l.id === job.toLocationId);
1051
+ if (!from || !to) {
1052
+ job.state = "failed";
1053
+ job.error = `location disappeared while queued: ${job.fromLocationId} → ${job.toLocationId}`;
1054
+ job.finishedAt = this.deps.now();
1055
+ this.deps.logger.warn("relocate job failed", {
1056
+ ...job.deviceId !== null ? { tags: { deviceId: job.deviceId } } : {},
1057
+ meta: {
1058
+ jobId: job.jobId,
1059
+ error: job.error
1060
+ }
1061
+ });
1062
+ this.deps.onFinished?.(snapshot(job));
1063
+ continue;
1064
+ }
1065
+ job.state = "running";
1066
+ job.startedAt = this.deps.now();
1067
+ await this.run(job, from, to);
1068
+ }
1069
+ } finally {
1070
+ this.draining = false;
1071
+ }
1072
+ }
1073
+ async run(job, from, to) {
961
1074
  const sleep = this.deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
962
- const bytesPerMs = throttleMbps * 1024 * 1024 / 1e3;
1075
+ const bytesPerMs = job.throttleMbps * 1024 * 1024 / 1e3;
963
1076
  try {
964
1077
  const samePhysicalRoot = await rootsPhysicallySame(from.root, to.root);
965
- const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => profileBelongsToClass(r.profile, footageClass)).sort((a, b) => a.startMs - b.startMs);
1078
+ const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => job.deviceId === null || r.deviceId === job.deviceId).filter((r) => job.profiles.length === 0 || job.profiles.includes(r.profile)).filter((r) => profileBelongsToClass(r.profile, job.footageClass)).sort((a, b) => a.startMs - b.startMs);
966
1079
  job.filesTotal = segRows.length;
967
1080
  for (const row of segRows) {
968
1081
  if (job.cancelRequested) break;
@@ -985,10 +1098,13 @@ var RelocateEngine = class {
985
1098
  } catch (err) {
986
1099
  job.state = "failed";
987
1100
  job.error = err instanceof Error ? err.message : String(err);
988
- this.deps.logger.warn("relocate job failed", { meta: {
989
- jobId: job.jobId,
990
- error: job.error
991
- } });
1101
+ this.deps.logger.warn("relocate job failed", {
1102
+ ...job.deviceId !== null ? { tags: { deviceId: job.deviceId } } : {},
1103
+ meta: {
1104
+ jobId: job.jobId,
1105
+ error: job.error
1106
+ }
1107
+ });
992
1108
  } finally {
993
1109
  job.finishedAt = this.deps.now();
994
1110
  this.deps.onFinished?.(snapshot(job));
@@ -2300,6 +2416,105 @@ function sendDirectory(res, payload) {
2300
2416
  res.end(Buffer.from(body));
2301
2417
  }
2302
2418
  //#endregion
2419
+ //#region src/recorder/addon/export-dense-map.ts
2420
+ /**
2421
+ * Shortest clip worth keeping, in ms.
2422
+ *
2423
+ * Exists only to honour `ExportDenseRangeSchema`'s `toSec > fromSec`: a range
2424
+ * clipped to a sliver of footage is still honest, but a range clipped to
2425
+ * NOTHING must not be emitted as a zero-length interval the schema refuses.
2426
+ */
2427
+ var MIN_MAPPED_MS = 1;
2428
+ /** Milliseconds of footage that precede `wallMs` in the concatenation. */
2429
+ function recordedMsBefore(segments, wallMs) {
2430
+ let acc = 0;
2431
+ for (const s of segments) if (wallMs >= s.startMs + s.durMs) acc += s.durMs;
2432
+ else if (wallMs > s.startMs) acc += wallMs - s.startMs;
2433
+ return acc;
2434
+ }
2435
+ /** Whole milliseconds → seconds, with no float tail for the filter string. */
2436
+ function msToSec(ms) {
2437
+ return Math.round(ms) / 1e3;
2438
+ }
2439
+ /**
2440
+ * Translate wall-clock dense ranges into the export's stream timeline.
2441
+ *
2442
+ * @returns the surviving ranges (in stream seconds, ordered as given) plus the
2443
+ * report the render logs.
2444
+ */
2445
+ function mapDenseRangesToStreamTime(input) {
2446
+ const segments = [...input.segments].sort((a, b) => a.startMs - b.startMs);
2447
+ const footageEndMs = segments.reduce((end, s) => Math.max(end, s.startMs + s.durMs), input.exportFromMs);
2448
+ const mapped = [];
2449
+ const dropped = [];
2450
+ for (const range of input.ranges) {
2451
+ const fromWallMs = input.exportFromMs + range.fromSec * 1e3;
2452
+ const toWallMs = input.exportFromMs + range.toSec * 1e3;
2453
+ const fromMs = Math.round(recordedMsBefore(segments, fromWallMs));
2454
+ const toMs = Math.round(recordedMsBefore(segments, toWallMs));
2455
+ if (toMs - fromMs < MIN_MAPPED_MS) {
2456
+ dropped.push({
2457
+ fromSec: range.fromSec,
2458
+ toSec: range.toSec,
2459
+ reason: fromWallMs >= footageEndMs ? "past-end" : "gap"
2460
+ });
2461
+ continue;
2462
+ }
2463
+ mapped.push({
2464
+ fromSec: msToSec(fromMs),
2465
+ toSec: msToSec(toMs)
2466
+ });
2467
+ }
2468
+ return {
2469
+ ranges: mapped,
2470
+ report: {
2471
+ requested: input.ranges.length,
2472
+ mapped: mapped.length,
2473
+ dropped
2474
+ }
2475
+ };
2476
+ }
2477
+ /**
2478
+ * One export's options with its dense ranges expressed in stream time.
2479
+ *
2480
+ * An overlay that maps to NOTHING is removed entirely rather than left with an
2481
+ * empty range list: `ExportDenseSchema` refuses `ranges: []` and
2482
+ * `selectPredicate` would emit `if(,…)`, a filter ffmpeg rejects — so a night
2483
+ * whose busy moments all fell in holes would fail the render instead of
2484
+ * delivering the uniform video it legitimately is.
2485
+ */
2486
+ function withStreamTimeDenseRanges(input) {
2487
+ const timelapse = input.options.timelapse;
2488
+ const dense = timelapse?.dense;
2489
+ if (timelapse === void 0 || dense === void 0) return {
2490
+ options: input.options,
2491
+ report: null
2492
+ };
2493
+ const { ranges, report } = mapDenseRangesToStreamTime({
2494
+ ranges: dense.ranges,
2495
+ exportFromMs: input.fromMs,
2496
+ segments: input.segments
2497
+ });
2498
+ const base = {
2499
+ everyMs: timelapse.everyMs,
2500
+ ...timelapse.outputFps !== void 0 ? { outputFps: timelapse.outputFps } : {}
2501
+ };
2502
+ const remapped = ranges.length > 0 ? {
2503
+ ...base,
2504
+ dense: {
2505
+ everyMs: dense.everyMs,
2506
+ ranges: [...ranges]
2507
+ }
2508
+ } : base;
2509
+ return {
2510
+ options: {
2511
+ ...input.options,
2512
+ timelapse: remapped
2513
+ },
2514
+ report
2515
+ };
2516
+ }
2517
+ //#endregion
2303
2518
  //#region src/recorder/addon/export-ffmpeg-args.ts
2304
2519
  /** Widest [min,max] speed a single ffmpeg `atempo` filter accepts. Outside this
2305
2520
  * band audio is dropped rather than chained (kept simple + predictable). */
@@ -2470,20 +2685,6 @@ async function listExportsFor(state, deviceId) {
2470
2685
  }
2471
2686
  //#endregion
2472
2687
  //#region src/recorder/addon/export-engine.ts
2473
- /**
2474
- * Render engine for recording exports.
2475
- *
2476
- * A FIFO queue with ONE concurrent ffmpeg render (exports are heavy; serialising
2477
- * protects the recording node). Each job: write the range's per-export source
2478
- * playlist (`resolvePlaylist`) → build argv (`buildExportArgs`) → spawn ffmpeg →
2479
- * parse `time=` from stderr for progress → on exit 0 stat the file and mark
2480
- * `ready`; on non-zero mark `failed` and remove the partial; drop the source
2481
- * playlist whatever the outcome. `cancel` SIGKILLs an in-flight render.
2482
- * State transitions patch the DurableState store and emit telemetry events.
2483
- *
2484
- * Unlike `SegmentWriter` there is NO auto-restart — a render runs once; boot
2485
- * re-queue of interrupted jobs is the provider's concern.
2486
- */
2487
2688
  /** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
2488
2689
  * segment before the decisive "Error opening input" summary, so a short tail can
2489
2690
  * push the actual cause out of the window (it very nearly did — the line that
@@ -2491,6 +2692,26 @@ async function listExportsFor(state, deviceId) {
2491
2692
  * render writes one progress line per second. */
2492
2693
  var STDERR_TAIL_LINES$1 = 40;
2493
2694
  var TIME_RE = /time=(\d+):(\d+):(\d+(?:\.\d+)?)/;
2695
+ /**
2696
+ * The floor under which an MP4 cannot hold a single decodable frame.
2697
+ *
2698
+ * **Exit 0 is not the same question as "the file holds a video".** A timelapse
2699
+ * whose frame-select predicate matched nothing produced a **262-byte** MP4 —
2700
+ * `ftyp` + `moov` + an EMPTY `mdat`, `nb_streams=0` to ffprobe — and ffmpeg
2701
+ * exited 0 while doing it. The row went `ready`, `readExportBytes` served the
2702
+ * 262 bytes, the notification centre republished them on the artifact plane
2703
+ * and a phone was handed a video no player can open (2026-08-12; the operator
2704
+ * reported it as "il video è danneggiato", and the served artifact was
2705
+ * byte-identical to the file on disk, so the render was the only suspect).
2706
+ *
2707
+ * A frame-free MP4 tops out around 300 bytes (the box headers and nothing
2708
+ * else); the smallest REAL one-frame 640×360 h264 export measured here is
2709
+ * several kilobytes. 1 KiB sits between the two with room on both sides, and
2710
+ * it is deliberately a byte floor and not an ffprobe call: the engine must be
2711
+ * able to refuse an empty render without a second process, on every node,
2712
+ * whatever ffmpeg build is installed.
2713
+ */
2714
+ var EXPORT_MIN_PLAUSIBLE_BYTES = 1024;
2494
2715
  /** Expected OUTPUT duration (seconds) for progress scaling. */
2495
2716
  function expectedOutSeconds(rec) {
2496
2717
  const rangeSec = Math.max(.001, (rec.toMs - rec.fromMs) / 1e3);
@@ -2559,35 +2780,76 @@ var ExportEngine = class {
2559
2780
  return;
2560
2781
  }
2561
2782
  try {
2562
- await this.spawnRender(id, rec, playlist);
2783
+ await this.spawnRender(id, rec, playlist.path, this.streamTimeOptions(rec, playlist.segments));
2563
2784
  } finally {
2564
2785
  if (this.deps.cleanupPlaylist) try {
2565
- await this.deps.cleanupPlaylist(playlist);
2786
+ await this.deps.cleanupPlaylist(playlist.path);
2566
2787
  } catch (err) {
2567
- this.deps.logger.warn("export: source playlist cleanup failed", { meta: {
2568
- exportId: id,
2569
- playlist,
2570
- error: err instanceof Error ? err.message : String(err)
2571
- } });
2788
+ this.deps.logger.warn("export: source playlist cleanup failed", {
2789
+ tags: { deviceId: rec.deviceId },
2790
+ meta: {
2791
+ exportId: id,
2792
+ playlist: playlist.path,
2793
+ error: err instanceof Error ? err.message : String(err)
2794
+ }
2795
+ });
2572
2796
  }
2573
2797
  }
2574
2798
  }
2799
+ /**
2800
+ * The record's options with any dense overlay moved onto ffmpeg's clock.
2801
+ *
2802
+ * The translation is reported on EVERY render that carries an overlay, not
2803
+ * only when something is dropped: `denseRangesRequested` vs
2804
+ * `denseRangesMapped` is the one line that can tell "the video is uniform
2805
+ * because the night was quiet" from "the video is uniform because every range
2806
+ * missed" — the distinction that went unnoticed for a full night because the
2807
+ * only line printed was the requested count.
2808
+ */
2809
+ streamTimeOptions(rec, segments) {
2810
+ const { options, report } = withStreamTimeDenseRanges({
2811
+ options: rec.options,
2812
+ fromMs: rec.fromMs,
2813
+ segments
2814
+ });
2815
+ if (report !== null) this.logDenseMapping(rec, report);
2816
+ return options;
2817
+ }
2818
+ /** One line, whatever the outcome — `warn` when ranges were dropped, because
2819
+ * a dropped range is footage the operator asked for and did not get. */
2820
+ logDenseMapping(rec, report) {
2821
+ const entry = {
2822
+ tags: { deviceId: rec.deviceId },
2823
+ meta: {
2824
+ exportId: rec.id,
2825
+ denseRangesRequested: report.requested,
2826
+ denseRangesMapped: report.mapped,
2827
+ ...report.dropped.length > 0 ? { denseRangesDropped: report.dropped } : {}
2828
+ }
2829
+ };
2830
+ const message = "export dense ranges mapped onto the concatenated stream";
2831
+ if (report.dropped.length > 0) this.deps.logger.warn(message, entry);
2832
+ else this.deps.logger.info(message, entry);
2833
+ }
2575
2834
  /** Spawn ffmpeg for one render and resolve when it has exited + finalized. */
2576
- async spawnRender(id, rec, playlist) {
2835
+ async spawnRender(id, rec, playlist, options) {
2577
2836
  const outPath = this.deps.outPath(rec);
2578
2837
  const args = buildExportArgs({
2579
2838
  inputPlaylist: playlist,
2580
2839
  outPath,
2581
- options: rec.options
2840
+ options
2582
2841
  });
2583
2842
  const totalSec = expectedOutSeconds(rec);
2584
- this.deps.logger.info("export render starting", { meta: {
2585
- exportId: id,
2586
- deviceId: rec.deviceId,
2587
- profile: rec.profile,
2588
- playlist,
2589
- outPath
2590
- } });
2843
+ this.deps.logger.info("export render starting", {
2844
+ tags: { deviceId: rec.deviceId },
2845
+ meta: {
2846
+ exportId: id,
2847
+ deviceId: rec.deviceId,
2848
+ profile: rec.profile,
2849
+ playlist,
2850
+ outPath
2851
+ }
2852
+ });
2591
2853
  this.deps.logger.debug("export ffmpeg argv", { meta: {
2592
2854
  exportId: id,
2593
2855
  args
@@ -2647,6 +2909,17 @@ var ExportEngine = class {
2647
2909
  await this.fail(id, err instanceof Error ? err.message : String(err));
2648
2910
  return;
2649
2911
  }
2912
+ if (bytes < 1024) {
2913
+ this.deps.logger.error("export produced no video frames", { meta: {
2914
+ exportId: id,
2915
+ outPath,
2916
+ bytes,
2917
+ minBytes: EXPORT_MIN_PLAUSIBLE_BYTES
2918
+ } });
2919
+ await this.deps.removeFile(outPath);
2920
+ await this.fail(id, `render produced no video frames (${bytes} bytes, < ${EXPORT_MIN_PLAUSIBLE_BYTES})`);
2921
+ return;
2922
+ }
2650
2923
  await patchExport(this.deps.state, id, {
2651
2924
  state: "ready",
2652
2925
  progressPct: 100,
@@ -3028,6 +3301,73 @@ function buildRecordingDeviceSchema() {
3028
3301
  }]
3029
3302
  }] };
3030
3303
  }
3304
+ /** The placement-store key for one (camera, profile). Mirrors `assignmentKey`
3305
+ * in `placement-store.ts`; kept local so this module stays dependency-free. */
3306
+ function sourceKey(source) {
3307
+ return `${source.deviceId}:${source.profile}`;
3308
+ }
3309
+ /** Biggest pile first (the overloaded disk is relieved fastest), ties broken
3310
+ * deterministically so the same input always yields the same plan. */
3311
+ function bySizeThenKey(a, b) {
3312
+ return b.bytes - a.bytes || a.deviceId - b.deviceId || a.profile.localeCompare(b.profile);
3313
+ }
3314
+ /**
3315
+ * Plan the moves that would make stored footage agree with the placement plan.
3316
+ * Pure: the same input always yields the same plan, and the input is untouched.
3317
+ */
3318
+ function planRebalance(input) {
3319
+ const minMoveBytes = input.minMoveBytes ?? 1e9;
3320
+ const remaining = new Map(input.headroom.map((h) => [h.locationId, h.headroomBytes]));
3321
+ const moves = [];
3322
+ const skipped = [];
3323
+ let bytesToMove = 0;
3324
+ for (const source of [...input.sources].sort(bySizeThenKey)) {
3325
+ const target = input.assignments.get(sourceKey(source));
3326
+ if (target === source.locationId) continue;
3327
+ const skip = (reason, toLocationId) => {
3328
+ skipped.push({
3329
+ deviceId: source.deviceId,
3330
+ profile: source.profile,
3331
+ fromLocationId: source.locationId,
3332
+ toLocationId,
3333
+ bytes: source.bytes,
3334
+ reason
3335
+ });
3336
+ };
3337
+ if (target === void 0) {
3338
+ skip("unassigned", null);
3339
+ continue;
3340
+ }
3341
+ const headroomBytes = remaining.get(target);
3342
+ if (headroomBytes === void 0) {
3343
+ skip("target-not-writable", target);
3344
+ continue;
3345
+ }
3346
+ if (source.bytes < minMoveBytes) {
3347
+ skip("below-threshold", target);
3348
+ continue;
3349
+ }
3350
+ if (source.bytes > headroomBytes) {
3351
+ skip("no-headroom", target);
3352
+ continue;
3353
+ }
3354
+ remaining.set(target, headroomBytes - source.bytes);
3355
+ bytesToMove += source.bytes;
3356
+ moves.push({
3357
+ deviceId: source.deviceId,
3358
+ profile: source.profile,
3359
+ fromLocationId: source.locationId,
3360
+ toLocationId: target,
3361
+ bytes: source.bytes,
3362
+ files: source.files
3363
+ });
3364
+ }
3365
+ return {
3366
+ moves,
3367
+ skipped,
3368
+ bytesToMove
3369
+ };
3370
+ }
3031
3371
  //#endregion
3032
3372
  //#region src/recorder/addon/recording-provider.ts
3033
3373
  /**
@@ -3305,6 +3645,75 @@ async function openRangeFromDisk(absPath, size) {
3305
3645
  close: () => handle.close()
3306
3646
  };
3307
3647
  }
3648
+ /** Bytes in one operator-facing gigabyte — the same decimal GB `maxUsedGb`
3649
+ * uses, so two knobs on the same page never mean two different things. */
3650
+ var BYTES_PER_GB = 1e9;
3651
+ /**
3652
+ * The operator-armed rebalance, both halves (D116, Phase 2).
3653
+ *
3654
+ * `arm: false` is the dry run the operator confirms; `arm: true` enqueues one
3655
+ * relocate job per planned move, FIFO behind the single-flight mover. The two
3656
+ * share this one body deliberately: what is confirmed is exactly what runs.
3657
+ *
3658
+ * Every refusal is LOGGED, per camera. A rebalance that quietly leaves half the
3659
+ * archive where it was is indistinguishable from one that had nothing to do,
3660
+ * and this repo has paid for that ambiguity twice.
3661
+ */
3662
+ async function runRebalance(deps, input, arm) {
3663
+ const relocate = deps.relocate;
3664
+ if (!deps.placementSnapshot) throw new Error("placement is not active on this node");
3665
+ if (arm && !relocate) throw new Error("relocation unavailable on this node");
3666
+ const snapshot = await deps.placementSnapshot();
3667
+ const plan = planRebalance({
3668
+ assignments: snapshot.assignments,
3669
+ sources: deps.index.piles(),
3670
+ headroom: snapshot.headroom,
3671
+ ...input.minMoveGb !== void 0 ? { minMoveBytes: input.minMoveGb * BYTES_PER_GB } : {}
3672
+ });
3673
+ for (const skip of plan.skipped) deps.logger.info("recorder rebalance: footage left where it is", {
3674
+ tags: { deviceId: skip.deviceId },
3675
+ meta: {
3676
+ profile: skip.profile,
3677
+ from: skip.fromLocationId,
3678
+ to: skip.toLocationId,
3679
+ bytes: skip.bytes,
3680
+ reason: skip.reason
3681
+ }
3682
+ });
3683
+ const result = {
3684
+ moves: [...plan.moves],
3685
+ skipped: [...plan.skipped],
3686
+ bytesToMove: plan.bytesToMove,
3687
+ jobIds: []
3688
+ };
3689
+ if (!arm || !relocate) return result;
3690
+ const jobIds = [];
3691
+ for (const move of plan.moves) {
3692
+ const jobId = relocate.start({
3693
+ fromLocationId: move.fromLocationId,
3694
+ toLocationId: move.toLocationId,
3695
+ deviceId: move.deviceId,
3696
+ profiles: [move.profile],
3697
+ ...input.throttleMbps !== void 0 ? { throttleMbps: input.throttleMbps } : {}
3698
+ });
3699
+ jobIds.push(jobId);
3700
+ deps.logger.info("recorder rebalance: relocate job enqueued", {
3701
+ tags: { deviceId: move.deviceId },
3702
+ meta: {
3703
+ jobId,
3704
+ profile: move.profile,
3705
+ from: move.fromLocationId,
3706
+ to: move.toLocationId,
3707
+ bytes: move.bytes,
3708
+ files: move.files
3709
+ }
3710
+ });
3711
+ }
3712
+ return {
3713
+ ...result,
3714
+ jobIds
3715
+ };
3716
+ }
3308
3717
  /**
3309
3718
  * Build the `IRecordingProvider`. The provider is a thin façade over the v2
3310
3719
  * core; all storage I/O flows through the injected deps.
@@ -3368,11 +3777,12 @@ function buildRecordingProvider(deps) {
3368
3777
  for (const loc of locations) for (const row of deps.index.segmentsOnLocation(loc.id)) seenDevices.add(row.deviceId);
3369
3778
  let totalUsedBytes = 0;
3370
3779
  for (const deviceId of seenDevices) {
3371
- const usedBytes = deps.index.accounting(deviceId).bytes;
3372
- totalUsedBytes += usedBytes;
3780
+ const accounting = deps.index.accounting(deviceId);
3781
+ totalUsedBytes += accounting.bytes;
3373
3782
  devices.push({
3374
3783
  deviceId,
3375
- usedBytes
3784
+ usedBytes: accounting.bytes,
3785
+ oldestMs: accounting.oldestMs
3376
3786
  });
3377
3787
  }
3378
3788
  const byPhysical = /* @__PURE__ */ new Map();
@@ -3559,6 +3969,24 @@ function buildRecordingProvider(deps) {
3559
3969
  },
3560
3970
  getStorageMigrationMoveStatus: async ({ jobId }) => deps.relocate?.list().find((job) => job.jobId === jobId) ?? null,
3561
3971
  cancelStorageMigrationMove: async ({ jobId }) => ({ cancelled: deps.relocate?.cancel(jobId) ?? false }),
3972
+ relocateFootage: async (input) => {
3973
+ if (!deps.relocate) throw new Error("relocation unavailable on this node");
3974
+ const jobId = deps.relocate.start(input);
3975
+ deps.logger.info("recorder: relocate job enqueued", {
3976
+ ...input.deviceId !== void 0 ? { tags: { deviceId: input.deviceId } } : {},
3977
+ meta: {
3978
+ jobId,
3979
+ from: input.fromLocationId,
3980
+ to: input.toLocationId,
3981
+ profiles: input.profiles ?? null
3982
+ }
3983
+ });
3984
+ return { jobId };
3985
+ },
3986
+ listRelocateJobs: async () => deps.relocate?.list() ?? [],
3987
+ cancelRelocateJob: async ({ jobId }) => ({ cancelled: deps.relocate?.cancel(jobId) ?? false }),
3988
+ planStorageRebalance: (input) => runRebalance(deps, input, false),
3989
+ startStorageRebalance: (input) => runRebalance(deps, input, true),
3562
3990
  getDeviceSettingsContribution: async ({ deviceId }) => {
3563
3991
  if (deps.isCameraDevice && !await deps.isCameraDevice(deviceId)) return null;
3564
3992
  return require_dist.hydrateSchema(buildRecordingDeviceSchema(), {});
@@ -3604,9 +4032,9 @@ function exportSourcePlaylistPath(exportsDir, deviceId, exportId) {
3604
4032
  return node_path.default.join(exportsDir, String(deviceId), `${exportId}.src.m3u8`);
3605
4033
  }
3606
4034
  /**
3607
- * Write the export's source playlist and return its absolute path, or null when
3608
- * no segment of the requested profile is present on disk for the range (the
3609
- * engine then fails the job with 'no footage for range').
4035
+ * Write the export's source playlist and return it, or null when no segment of
4036
+ * the requested profile is present on disk for the range (the engine then fails
4037
+ * the job with 'no footage for range').
3610
4038
  */
3611
4039
  async function writeExportSourcePlaylist(input) {
3612
4040
  const { segments } = await collectRangeSegments(input.deps, input.deviceId, input.profile, input.fromMs, input.toMs, "absolute");
@@ -3614,26 +4042,16 @@ async function writeExportSourcePlaylist(input) {
3614
4042
  const playlistPath = exportSourcePlaylistPath(input.exportsDir, input.deviceId, input.exportId);
3615
4043
  await node_fs.promises.mkdir(node_path.default.dirname(playlistPath), { recursive: true });
3616
4044
  await node_fs.promises.writeFile(playlistPath, buildVariantPlaylist(segments), "utf8");
3617
- return playlistPath;
4045
+ return {
4046
+ path: playlistPath,
4047
+ segments: segments.map((s) => ({
4048
+ startMs: s.startMs,
4049
+ durMs: s.durMs
4050
+ }))
4051
+ };
3618
4052
  }
3619
4053
  //#endregion
3620
4054
  //#region src/recorder/addon/footage-render.ts
3621
- /**
3622
- * One-shot media render FROM RECORDED FOOTAGE (notification attachments).
3623
- *
3624
- * Shared core behind `renderGif` and `renderClip`: both cut the same window out
3625
- * of the same `low` profile through the export machinery's per-render source
3626
- * playlist (absolute segment URIs — the ONLY playlist shape ffmpeg can consume,
3627
- * see export-source-playlist.ts), pipe it through ONE ffmpeg invocation, and
3628
- * return the bytes. Everything is transient: playlist and output are deleted on
3629
- * every outcome.
3630
- *
3631
- * Fail-closed by construction: no footage covering the window ⇒ it throws, and
3632
- * the caller (a notification rule) simply ships no attachment. A camera that is
3633
- * not recording therefore cannot produce a clip — which is exactly why the
3634
- * frame-ring plane (roadmap Phase 5) still matters for cameras with no
3635
- * recording. It is NOT a prerequisite for the cameras that do record.
3636
- */
3637
4055
  var RENDER_TIMEOUT_MS = 3e4;
3638
4056
  /**
3639
4057
  * A notification clip must sit AROUND the moment it describes. Sliding the
@@ -3731,9 +4149,9 @@ async function renderFootage(renderDeps, input, ext, buildArgs) {
3731
4149
  toMs: window.toMs
3732
4150
  });
3733
4151
  if (playlist === null) throw new Error(`no footage covers [${window.fromMs}, ${window.toMs}) for device ${input.deviceId}`);
3734
- const outPath = node_path.default.join(node_path.default.dirname(playlist), `${renderId}.${ext}`);
4152
+ const outPath = node_path.default.join(node_path.default.dirname(playlist.path), `${renderId}.${ext}`);
3735
4153
  try {
3736
- await runFfmpeg(renderDeps, buildArgs(playlist, outPath), ext);
4154
+ await runFfmpeg(renderDeps, buildArgs(playlist.path, outPath), ext);
3737
4155
  const bytes = await node_fs.promises.readFile(outPath);
3738
4156
  if (bytes.byteLength === 0) throw new Error(`${ext} render produced an empty file`);
3739
4157
  return {
@@ -3742,7 +4160,7 @@ async function renderFootage(renderDeps, input, ext, buildArgs) {
3742
4160
  toMs: window.toMs
3743
4161
  };
3744
4162
  } finally {
3745
- await node_fs.promises.rm(playlist, { force: true }).catch(() => {});
4163
+ await node_fs.promises.rm(playlist.path, { force: true }).catch(() => {});
3746
4164
  await node_fs.promises.rm(outPath, { force: true }).catch(() => {});
3747
4165
  }
3748
4166
  }
@@ -4690,6 +5108,33 @@ var PlacementService = class {
4690
5108
  return plan;
4691
5109
  }
4692
5110
  /**
5111
+ * The write targets and what each can still absorb — the SAME pool and the
5112
+ * SAME headroom `recompute` plans over.
5113
+ *
5114
+ * Exposed for the operator-armed rebalance (D116): weighing existing footage
5115
+ * against a second, independently-derived idea of "what fits" is how the two
5116
+ * halves end up disagreeing, and a rebalance that plans onto a location the
5117
+ * planner would never write to is a rebalance that undoes itself on the next
5118
+ * tick. A location absent from this list is not a write target at all.
5119
+ */
5120
+ async candidates() {
5121
+ return this.candidatesFor(this.pool());
5122
+ }
5123
+ /**
5124
+ * The plan in force: `<deviceId>:<profile>` → locationId. Empty when the
5125
+ * store cannot be read — a fallible read decides nothing (D49), and an empty
5126
+ * plan makes the rebalance report every pile as `unassigned` rather than
5127
+ * inventing moves.
5128
+ */
5129
+ async currentAssignments() {
5130
+ try {
5131
+ return new Map(Object.entries((await readPlacementState(this.deps.state)).assignments));
5132
+ } catch (err) {
5133
+ this.deps.logger.warn("recorder placement: assignment read failed — reporting no plan", { meta: { error: require_dist.errMsg(err) } });
5134
+ return /* @__PURE__ */ new Map();
5135
+ }
5136
+ }
5137
+ /**
4693
5138
  * Turn each pool location into a planner candidate. Headroom is
4694
5139
  * `min(free − minFree floor, maxUsedGb − used)`, clamped at zero: the first
4695
5140
  * term is the per-location free-space guard the storage pressure manager also
@@ -4959,6 +5404,49 @@ function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers) {
4959
5404
  return false;
4960
5405
  }
4961
5406
  //#endregion
5407
+ //#region src/recorder/addon/periodic-pass.ts
5408
+ function startPeriodicPass(deps) {
5409
+ const now = () => deps.now?.() ?? Date.now();
5410
+ let startedAt = null;
5411
+ let stopped = false;
5412
+ const fire = () => {
5413
+ if (stopped) return;
5414
+ if (startedAt !== null) {
5415
+ deps.logger.warn("recorder: pass skipped — the previous pass is still running", { meta: {
5416
+ pass: deps.name,
5417
+ inFlightMs: now() - startedAt,
5418
+ intervalMs: deps.intervalMs
5419
+ } });
5420
+ return;
5421
+ }
5422
+ const began = now();
5423
+ startedAt = began;
5424
+ deps.run().catch((err) => {
5425
+ deps.logger.warn("recorder: pass failed", { meta: {
5426
+ pass: deps.name,
5427
+ error: err instanceof Error ? err.message : String(err)
5428
+ } });
5429
+ }).finally(() => {
5430
+ const durationMs = now() - began;
5431
+ startedAt = null;
5432
+ if (durationMs > deps.slowAfterMs) deps.logger.warn("recorder: pass exceeded its budget", { meta: {
5433
+ pass: deps.name,
5434
+ durationMs,
5435
+ budgetMs: deps.slowAfterMs
5436
+ } });
5437
+ });
5438
+ };
5439
+ const timer = setInterval(fire, deps.intervalMs);
5440
+ timer.unref?.();
5441
+ return {
5442
+ stop() {
5443
+ stopped = true;
5444
+ clearInterval(timer);
5445
+ },
5446
+ isRunning: () => startedAt !== null
5447
+ };
5448
+ }
5449
+ //#endregion
4962
5450
  //#region src/recorder/addon/readiness-restore.ts
4963
5451
  var ReadinessRestore = class {
4964
5452
  deps;
@@ -5045,10 +5533,59 @@ var ReadinessRestore = class {
5045
5533
  * no subtree to classify.
5046
5534
  */
5047
5535
  var EPOCH_NAME_RE = /^(\d+)\.m4s$/;
5048
- /** How many consecutive ticks to wait for a finalized segment's flat file to
5049
- * land before giving up on it (a genuine phantom). At a 2s watch interval this
5050
- * is ~16s far longer than ffmpeg's slow-first-segment flush. */
5051
- var MAX_PENDING_TICKS = 8;
5536
+ /**
5537
+ * How long a finalized segment's flat file may take to appear before the entry
5538
+ * is given up on.
5539
+ *
5540
+ * This used to be `MAX_PENDING_TICKS = 8` — a count of TICKS, which is not a
5541
+ * measure of time. The tick loop self-excludes while a pass is running and each
5542
+ * pass does real filesystem work, so on 2026-08-10..13 eight ticks measured
5543
+ * anywhere from 8 s to 819 s. A bound has to be denominated in the thing it
5544
+ * bounds.
5545
+ *
5546
+ * 30 s is chosen from the measurement, not from taste. Over four days the
5547
+ * segments that were retried and then DID land waited p50 7.8 s and p90 11.6 s,
5548
+ * and widening the window from 16 s to 120 s moved coverage only from 92.9% to
5549
+ * 93.8%. There is no tail to chase: past this knee a segment is not slow, it is
5550
+ * lost (a writer killed mid-segment), and waiting only delays the skip and the
5551
+ * log line that reports it.
5552
+ */
5553
+ var PENDING_GRACE_MS = 3e4;
5554
+ /**
5555
+ * How far before the writer's own start a segment may be named and still be
5556
+ * treated as belonging to THIS run.
5557
+ *
5558
+ * ffmpeg names segments with whole-second `%s`, so the first segment of a run
5559
+ * carries the second that was already in progress when the process spawned —
5560
+ * up to ~1 s "before" the writer started. Without this tolerance the very first
5561
+ * segment of every recording would be classified as a previous run's leftover
5562
+ * and dropped.
5563
+ */
5564
+ var PREVIOUS_RUN_SKEW_MS = 1e3;
5565
+ /**
5566
+ * Decide whether a missing flat file is a late flush or a previous run's entry.
5567
+ *
5568
+ * ffmpeg's `-segment_list` file is not cleared between runs and a fresh watcher
5569
+ * starts at `processed = 0`, so the first passes after every attach replay the
5570
+ * PREVIOUS run's playlist. Those entries were relocated into the bucket tree
5571
+ * long ago; their absence is the system working. Over 2026-08-10..13 they were
5572
+ * 83% of all retries (median age 1 615 s, max 17.4 h) and produced 68 of the
5573
+ * 122 `never landed` warnings — a warning that reads like data loss for a file
5574
+ * that was correctly filed hours earlier.
5575
+ *
5576
+ * Note what this deliberately does NOT do: it does not skip a previous run's
5577
+ * entry whose file is STILL THERE. That case is real crash recovery — an
5578
+ * unclean stop leaves un-relocated segments in `.rec-tmp` and the playlist is
5579
+ * the only record of their durations — so the caller stats first and only asks
5580
+ * this question once the file is known to be absent.
5581
+ */
5582
+ function classifyMissingEntry(input) {
5583
+ return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS ? "stale-previous-run" : "pending";
5584
+ }
5585
+ /** Whether a pending entry has outlived its grace. Pure — the caller clocks it. */
5586
+ function shouldGiveUpOnPending(input) {
5587
+ return input.waitedMs > input.boundMs;
5588
+ }
5052
5589
  /**
5053
5590
  * Derive `startMs` (epoch milliseconds) from a flat ffmpeg segment path
5054
5591
  * (`<epochSec>.m4s`, possibly absolute). Returns null for any non-epoch path.
@@ -5123,11 +5660,22 @@ async function statSize(p) {
5123
5660
  * Unlike the old recorder, this watcher does NOT relocate or re-stat: the
5124
5661
  * SegmentStore owns the move + the authoritative byte count.
5125
5662
  */
5126
- async function handleSegmentEntry(outDir, entry, durMs, onFinalized, logger) {
5663
+ async function handleSegmentEntry(outDir, entry, durMs, onFinalized, logger, writerStartedMs) {
5127
5664
  const flatAbsPath = node_path.default.isAbsolute(entry.segPath) ? entry.segPath : node_path.default.join(outDir, entry.segPath);
5128
5665
  const startMs = parseEpochStartMs(flatAbsPath);
5129
5666
  if (startMs === null) return "skipped";
5130
5667
  if (await statSize(flatAbsPath) === null) {
5668
+ if (classifyMissingEntry({
5669
+ segmentStartMs: startMs,
5670
+ writerStartedMs
5671
+ }) === "stale-previous-run") {
5672
+ logger.debug("segment entry predates this writer — already relocated, skipping", { meta: {
5673
+ path: flatAbsPath,
5674
+ segmentStartMs: startMs,
5675
+ writerStartedMs
5676
+ } });
5677
+ return "skipped";
5678
+ }
5131
5679
  logger.debug("segment flat file not on disk yet — will retry", { meta: { path: flatAbsPath } });
5132
5680
  return "pending";
5133
5681
  }
@@ -5173,12 +5721,13 @@ function planFinalizations(body, processed, forceFinal = false) {
5173
5721
  */
5174
5722
  function startSegmentWatcher(deps) {
5175
5723
  const playlistPath = node_path.default.join(deps.outDir, "live.m3u8");
5724
+ const now = () => deps.now?.() ?? Date.now();
5176
5725
  let processed = 0;
5177
5726
  let stopped = false;
5178
5727
  let ticking = false;
5179
5728
  let currentTick = null;
5180
5729
  let pendingIndex = -1;
5181
- let pendingTicks = 0;
5730
+ let pendingSinceMs = 0;
5182
5731
  const tick = async (forceFinal = false) => {
5183
5732
  if (stopped || ticking) return;
5184
5733
  ticking = true;
@@ -5192,19 +5741,26 @@ function startSegmentWatcher(deps) {
5192
5741
  const plan = planFinalizations(body, processed, forceFinal);
5193
5742
  let index = plan.from;
5194
5743
  for (const { entry, durMs } of plan.toHandle) {
5195
- if (await handleSegmentEntry(deps.outDir, entry, durMs, deps.onFinalized, deps.logger) === "pending") {
5744
+ if (await handleSegmentEntry(deps.outDir, entry, durMs, deps.onFinalized, deps.logger, deps.writerStartedMs) === "pending") {
5196
5745
  if (pendingIndex === index) {
5197
- pendingTicks++;
5198
- if (pendingTicks >= MAX_PENDING_TICKS) {
5199
- deps.logger.warn("segment file never landed — skipping", { meta: { seg: entry.segPath } });
5746
+ const waitedMs = now() - pendingSinceMs;
5747
+ if (shouldGiveUpOnPending({
5748
+ waitedMs,
5749
+ boundMs: 3e4
5750
+ })) {
5751
+ deps.logger.warn("segment file never landed — skipping", { meta: {
5752
+ seg: entry.segPath,
5753
+ waitedMs,
5754
+ boundMs: PENDING_GRACE_MS
5755
+ } });
5200
5756
  pendingIndex = -1;
5201
- pendingTicks = 0;
5757
+ pendingSinceMs = 0;
5202
5758
  index++;
5203
5759
  continue;
5204
5760
  }
5205
5761
  } else {
5206
5762
  pendingIndex = index;
5207
- pendingTicks = 1;
5763
+ pendingSinceMs = now();
5208
5764
  }
5209
5765
  break;
5210
5766
  }
@@ -5212,7 +5768,7 @@ function startSegmentWatcher(deps) {
5212
5768
  }
5213
5769
  if (pendingIndex !== index) {
5214
5770
  pendingIndex = -1;
5215
- pendingTicks = 0;
5771
+ pendingSinceMs = 0;
5216
5772
  }
5217
5773
  processed = index;
5218
5774
  } finally {
@@ -5413,9 +5969,16 @@ var SegmentWriter = class {
5413
5969
  * (`shouldRecordContinuousAt`) and ensures the writer set is running iff so.
5414
5970
  * It re-evaluates on three triggers:
5415
5971
  * 1. `setDeviceConfig` (operator changed the bands / enable),
5416
- * 2. a periodic tick (catches band boundaries crossed with no config change),
5972
+ * 2. a periodic pass (catches band boundaries crossed with no config change),
5417
5973
  * 3. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`).
5418
5974
  *
5975
+ * There are TWO periodic passes, on separate timers and never overlapping
5976
+ * themselves (`periodic-pass.ts`): a cheap LIVENESS pass that catches writers
5977
+ * pinned but producing nothing, and an expensive CONVERGENCE pass that
5978
+ * re-evaluates bands and placement. They were one pass until 2026-08-13, when a
5979
+ * convergence sweep wedged on unbounded broker RPCs and took the liveness
5980
+ * watchdog down with it for 9-27 minutes at a time.
5981
+ *
5419
5982
  * EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
5420
5983
  * trigger-gating. The enable intent is persisted in durable-state by the
5421
5984
  * config-store, so `restoreEnabledDevices` on boot re-attaches every device
@@ -5445,6 +6008,34 @@ var TICK_MS = 3e4;
5445
6008
  */
5446
6009
  var RELEASE_RPC_TIMEOUT_MS = 2e3;
5447
6010
  /**
6011
+ * Bound on each broker RPC in the ATTACH chain (`listAllProfileSlots`,
6012
+ * `getStreamWithCodec`).
6013
+ *
6014
+ * These were the only unbounded calls left in the controller.
6015
+ * `CapabilityHandle.call` gates READINESS (10 s) and then returns `fn()`
6016
+ * untouched, so an accepted-but-unanswered request fell through to the 60 s UDS
6017
+ * default in `kernel/transport/socket-channel.ts`. On 2026-08-13 the broker was
6018
+ * degraded (516 UDS timeouts that day) and the arithmetic did the damage: one
6019
+ * device costs `listAllProfileSlots` + one `getStreamWithCodec` per profile, so
6020
+ * a 4-camera fleet on 3 profiles reached ~16 minutes inside a single recovery
6021
+ * pass — during which every device sits DETACHED (recording stopped) and in
6022
+ * `attaching` (so `evaluateDevice` returns early and the liveness watchdog,
6023
+ * which reads `active`, has nothing to report). That is the shape of the 9-27
6024
+ * minute silences and the real recording gaps behind the 80% timelapse
6025
+ * coverage.
6026
+ *
6027
+ * 20 s is generous for a real RTSP dial and far below the UDS default, so a
6028
+ * hung broker now fails the attach fast. Nothing is lost by giving up: the
6029
+ * device stays on the readiness-restore queue and the next pass retries it.
6030
+ */
6031
+ var ATTACH_RPC_TIMEOUT_MS = 2e4;
6032
+ /**
6033
+ * A pass longer than this is reported with its duration. Equal to the pass
6034
+ * cadence: a pass that cannot finish inside its own interval is the condition
6035
+ * that used to go unlogged for half an hour.
6036
+ */
6037
+ var PASS_SLOW_AFTER_MS = TICK_MS;
6038
+ /**
5448
6039
  * Race `promise` against a deadline. Handlers stay attached to the losing
5449
6040
  * promise, so a post-deadline settlement can never surface as a process-level
5450
6041
  * `unhandledRejection`.
@@ -5532,7 +6123,10 @@ var RecordingController = class {
5532
6123
  */
5533
6124
  unrecordableReported = /* @__PURE__ */ new Set();
5534
6125
  restore = null;
5535
- tickTimer = null;
6126
+ /** Cheap, never-delayed: is any pinned writer producing nothing? */
6127
+ livenessPass = null;
6128
+ /** Expensive: re-evaluate bands + placement for every configured device. */
6129
+ convergePass = null;
5536
6130
  stopped = false;
5537
6131
  /** A reversible maintenance lease. Unlike `stopped`, it never changes
5538
6132
  * persisted recording intent and `resume()` restarts normal convergence. */
@@ -5567,18 +6161,28 @@ var RecordingController = class {
5567
6161
  meta: { error: require_dist.errMsg(err) }
5568
6162
  })
5569
6163
  });
5570
- if (this.tickTimer === null) {
5571
- this.tickTimer = setInterval(() => {
5572
- this.tick();
5573
- }, TICK_MS);
5574
- this.tickTimer.unref?.();
6164
+ if (this.convergePass === null) {
6165
+ this.livenessPass = startPeriodicPass({
6166
+ name: "liveness",
6167
+ intervalMs: TICK_MS,
6168
+ slowAfterMs: PASS_SLOW_AFTER_MS,
6169
+ logger: this.deps.logger,
6170
+ now: () => this.now(),
6171
+ run: () => this.checkIdleWriters()
6172
+ });
6173
+ this.convergePass = startPeriodicPass({
6174
+ name: "convergence",
6175
+ intervalMs: TICK_MS,
6176
+ slowAfterMs: PASS_SLOW_AFTER_MS,
6177
+ logger: this.deps.logger,
6178
+ now: () => this.now(),
6179
+ run: () => this.converge()
6180
+ });
5575
6181
  }
5576
6182
  await this.restore.start(ids.map((id) => [id, true]));
5577
6183
  }
5578
- /** Re-evaluate every currently-tracked OR active device on the periodic tick. */
5579
- async tick() {
5580
- if (this.stopped) return;
5581
- await this.checkIdleWriters();
6184
+ /** Re-evaluate every currently-tracked OR active device on the periodic pass. */
6185
+ async converge() {
5582
6186
  if (this.stopped) return;
5583
6187
  const ids = new Set(this.active.keys());
5584
6188
  let persisted = [];
@@ -5594,7 +6198,7 @@ var RecordingController = class {
5594
6198
  for (const id of ids) try {
5595
6199
  await this.evaluateDevice(id);
5596
6200
  } catch (err) {
5597
- this.deps.logger.warn("recorder controller: tick evaluate failed", {
6201
+ this.deps.logger.warn("recorder controller: convergence evaluate failed", {
5598
6202
  tags: { deviceId: id },
5599
6203
  meta: { error: require_dist.errMsg(err) }
5600
6204
  });
@@ -5668,7 +6272,7 @@ var RecordingController = class {
5668
6272
  segmentSeconds: this.active.get(s.deviceId)?.find((r) => r.profile === s.profile)?.segmentSeconds
5669
6273
  }
5670
6274
  });
5671
- for (const deviceId of devices) await this.recoverWriterGaveUp(deviceId);
6275
+ await Promise.all([...devices].map((deviceId) => this.recoverWriterGaveUp(deviceId)));
5672
6276
  }
5673
6277
  /** Current trigger state for a device (cold default = no triggers seen). */
5674
6278
  triggersFor(deviceId) {
@@ -5810,7 +6414,7 @@ var RecordingController = class {
5810
6414
  * assigned sources is ignored (record every assigned source — minimum of 1).
5811
6415
  */
5812
6416
  async resolveProfiles(deviceId, override) {
5813
- const assigned = require_dist.selectAssignedProfileSlots(await this.deps.brokerCall(() => this.deps.api.streamBroker.listAllProfileSlots.query(void 0, require_dist.nodePin(this.deps.ownerNodeId))), deviceId).map((slot) => slot.profile);
6417
+ const assigned = require_dist.selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, require_dist.nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile);
5814
6418
  if (!override || override.length === 0) return assigned;
5815
6419
  const selected = assigned.filter((p) => override.includes(p));
5816
6420
  return selected.length > 0 ? selected : assigned;
@@ -5823,14 +6427,14 @@ var RecordingController = class {
5823
6427
  */
5824
6428
  async attachProfile(deviceId, profile, segmentSeconds) {
5825
6429
  const placement = await this.resolvePlacement(deviceId, profile);
5826
- const source = await this.deps.brokerCall(() => this.deps.api.streamBroker.getStreamWithCodec.mutate({
6430
+ const source = await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.getStreamWithCodec.mutate({
5827
6431
  deviceId,
5828
6432
  video: "copy",
5829
6433
  audio: "aac",
5830
6434
  profile,
5831
6435
  ...this.deps.consumerHostname !== void 0 ? { hostname: this.deps.consumerHostname } : {},
5832
6436
  tag: `recorder:${deviceId}/${profile}`
5833
- }, require_dist.nodePin(this.deps.ownerNodeId)));
6437
+ }, require_dist.nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "getStreamWithCodec"));
5834
6438
  const outDir = node_path.default.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
5835
6439
  await node_fs.promises.mkdir(outDir, { recursive: true });
5836
6440
  const deviceLog = this.deps.logger.withTags({ deviceId });
@@ -5846,6 +6450,7 @@ var RecordingController = class {
5846
6450
  this.recoverWriterGaveUp(deviceId);
5847
6451
  }
5848
6452
  });
6453
+ const writerStartedMs = this.now();
5849
6454
  writer.start();
5850
6455
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
5851
6456
  return {
@@ -5853,6 +6458,8 @@ var RecordingController = class {
5853
6458
  writer,
5854
6459
  watcher: startSegmentWatcher({
5855
6460
  outDir,
6461
+ writerStartedMs,
6462
+ now: () => this.now(),
5856
6463
  intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
5857
6464
  onFinalized: async (startMs, durMs, flatAbsPath) => {
5858
6465
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
@@ -5969,10 +6576,10 @@ var RecordingController = class {
5969
6576
  /** Tear EVERYTHING down: timer, restore subscription, every writer/watcher/lease. */
5970
6577
  async stop() {
5971
6578
  this.stopped = true;
5972
- if (this.tickTimer !== null) {
5973
- clearInterval(this.tickTimer);
5974
- this.tickTimer = null;
5975
- }
6579
+ this.livenessPass?.stop();
6580
+ this.livenessPass = null;
6581
+ this.convergePass?.stop();
6582
+ this.convergePass = null;
5976
6583
  this.restore?.stop();
5977
6584
  this.restore = null;
5978
6585
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
@@ -6575,6 +7182,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
6575
7182
  resumeForStorageMigration: (leaseId) => this.resumeForStorageMigration(leaseId),
6576
7183
  refreshStorageLocationsForMigration: (leaseId) => this.refreshStorageLocationsForMigration(leaseId),
6577
7184
  assertStorageMigrationLease: (leaseId) => this.assertStorageMigrationLease(leaseId),
7185
+ placementSnapshot: () => this.placementSnapshot(),
6578
7186
  dataDir: this.ctx.dataDir,
6579
7187
  playbackBaseUrl: () => this.playbackBaseUrl,
6580
7188
  isCameraDevice: async (deviceId) => {
@@ -7014,6 +7622,28 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
7014
7622
  await this.placement.recompute(work);
7015
7623
  }
7016
7624
  /**
7625
+ * The plan in force plus each write target's headroom, for the operator-armed
7626
+ * rebalance (D116, Phase 2). Both come from the SAME `PlacementService` the
7627
+ * 30 s tick plans with: a rebalance computed against a second idea of "what
7628
+ * fits" would plan moves the next tick undoes.
7629
+ *
7630
+ * Throws when placement is not active (a non-recording node, or early boot) —
7631
+ * the rebalance is an operator action and a silent empty plan would read as
7632
+ * "nothing to do".
7633
+ */
7634
+ async placementSnapshot() {
7635
+ const placement = this.placement;
7636
+ if (!placement) throw new Error("placement is not active on this node");
7637
+ const [assignments, candidates] = await Promise.all([placement.currentAssignments(), placement.candidates()]);
7638
+ return {
7639
+ assignments,
7640
+ headroom: candidates.map((candidate) => ({
7641
+ locationId: candidate.locationId,
7642
+ headroomBytes: candidate.headroomBytes
7643
+ }))
7644
+ };
7645
+ }
7646
+ /**
7017
7647
  * Measured bytes/day for one (camera, profile) — the demand signal the
7018
7648
  * placement planner weights against each location's headroom. Derived from
7019
7649
  * the index's own accounting (segment paths encode bytes + start), so it