@camstack/addon-pipeline 1.2.69 → 1.2.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/audio-analyzer/index.js +1 -1
  2. package/dist/audio-analyzer/index.mjs +1 -1
  3. package/dist/detection-pipeline/index.js +32 -30
  4. package/dist/detection-pipeline/index.mjs +32 -30
  5. package/dist/{dist-tJF2BzWi.js → dist-CtOVWTCi.js} +586 -41
  6. package/dist/{dist-DGLqaonv.mjs → dist-D1Xk5Tji.mjs} +586 -41
  7. package/dist/{event-loop-stall-monitor-D9d2e49f.js → event-loop-stall-monitor-BJs7Yws7.js} +1 -1
  8. package/dist/{event-loop-stall-monitor-C6_VDupx.mjs → event-loop-stall-monitor-Cmbt664E.mjs} +1 -1
  9. package/dist/motion-wasm/index.js +1 -1
  10. package/dist/motion-wasm/index.mjs +1 -1
  11. package/dist/pipeline-runner/index.js +100 -30
  12. package/dist/pipeline-runner/index.mjs +100 -30
  13. package/dist/recorder/index.js +734 -104
  14. package/dist/recorder/index.mjs +734 -104
  15. package/dist/session-decode/decode-worker-child.js +483 -52
  16. package/dist/session-decode/decode-worker-child.mjs +483 -52
  17. package/dist/stream-broker/_stub.js +660 -556
  18. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-DCyQ7RKH.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BWf4H4Nl.mjs} +3 -3
  19. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Cu_kQpIy.mjs +26 -0
  20. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CExO0piw.mjs +26 -0
  21. package/dist/stream-broker/{hostInit-DxqtIjAe.mjs → hostInit-DD5Z1U6p.mjs} +3 -3
  22. package/dist/stream-broker/index.js +501 -32
  23. package/dist/stream-broker/index.mjs +501 -32
  24. package/dist/stream-broker/remoteEntry.js +1 -1
  25. package/dist/{worker-protocol-D6tXz9Or.js → worker-protocol-DEV7g32b.js} +26 -19
  26. package/dist/{worker-protocol-DuftUFEI.mjs → worker-protocol-Dm6LQ9Ls.mjs} +26 -19
  27. package/package.json +1 -1
  28. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-ircnzt0s.mjs +0 -26
  29. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DU7u9EJt.mjs +0 -26
@@ -1,4 +1,4 @@
1
- import { At as string, Dt as number, J as recordingExportCapability, Mt as EventCategory, N as deriveRecordingMode, Ot as object, St as array, Z as storageEvictableCapability, _ as OpsLogEntrySchema, b as RECORDING_EXPORT_MAX_READ_BYTES, c as DEFAULT_EVENTS_BAND_BUFFER_SEC, d as EVENT_PAD_MS, ft as hydrateSchema, gt as nodePin, kt as record, p as ExportRecordSchema, q as recordingCapability, st as BaseAddon, tt as errMsg, ut as DeviceType, x as RecordingConfigSchema, yt as selectAssignedProfileSlots } from "../dist-DGLqaonv.mjs";
1
+ import { At as string, Dt as number, J as recordingExportCapability, Mt as EventCategory, N as deriveRecordingMode, Ot as object, St as array, Z as storageEvictableCapability, _ as OpsLogEntrySchema, b as RECORDING_EXPORT_MAX_READ_BYTES, c as DEFAULT_EVENTS_BAND_BUFFER_SEC, d as EVENT_PAD_MS, ft as hydrateSchema, gt as nodePin, kt as record, p as ExportRecordSchema, q as recordingCapability, st as BaseAddon, tt as errMsg, ut as DeviceType, x as RecordingConfigSchema, yt as selectAssignedProfileSlots } from "../dist-D1Xk5Tji.mjs";
2
2
  import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
3
3
  import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-CZ2xo67g.mjs";
4
4
  import { randomUUID } from "node:crypto";
@@ -681,6 +681,36 @@ var RecordingIndex = class {
681
681
  }
682
682
  return out;
683
683
  }
684
+ /**
685
+ * Every (device, profile, location) pile in ONE pass — bytes and file count,
686
+ * no sort and no cache rebuild.
687
+ *
688
+ * This is the input to the operator-armed rebalance (D116): it asks "where is
689
+ * each camera-profile's footage, and how much of it", which is exactly the
690
+ * unit the placement plan assigns and the relocate mover moves. Building it
691
+ * from `segmentsOnLocation` per location would sort the whole archive once
692
+ * per location instead.
693
+ */
694
+ piles() {
695
+ const out = /* @__PURE__ */ new Map();
696
+ for (const m of this.byDevice.values()) for (const s of m.values()) {
697
+ const key = `${s.deviceId}:${s.profile}:${s.locationId}`;
698
+ const prev = out.get(key);
699
+ if (prev) {
700
+ prev.bytes += s.bytes;
701
+ prev.files += 1;
702
+ continue;
703
+ }
704
+ out.set(key, {
705
+ deviceId: s.deviceId,
706
+ profile: s.profile,
707
+ locationId: s.locationId,
708
+ bytes: s.bytes,
709
+ files: 1
710
+ });
711
+ }
712
+ return [...out.values()];
713
+ }
684
714
  /** All segments on a storage location across every device, oldest-first. */
685
715
  segmentsOnLocation(locationId) {
686
716
  const out = [];
@@ -870,6 +900,19 @@ function resolveEvictionDomain(locations, locationId, realpath = realpathSync) {
870
900
  *
871
901
  * Runs INSIDE the recorder process — container uid, same-uid files, no host
872
902
  * chown trap (see docs/history 2026-07-29).
903
+ *
904
+ * ── Phase 2 (D116, multi-location) ────────────────────────────────────────
905
+ * Two additions the OPERATOR-ARMED rebalance needs:
906
+ *
907
+ * - A move is scoped to one CAMERA (and optionally to specific profiles),
908
+ * not just to a whole disk. The rebalance's unit is a (camera, profile)
909
+ * pile, because that is the unit the placement plan assigns.
910
+ * - Concurrent starts QUEUE instead of being refused. The engine is still
911
+ * single-flight — two movers on one disk thrash both — but a rebalance
912
+ * enqueues one job per pile, and refusing the second turned a
913
+ * fifteen-camera rebalance into fifteen manual retries. Jobs run FIFO; a
914
+ * job cancelled while queued never runs, and a job that FAILS does not
915
+ * strand the queue behind it.
873
916
  */
874
917
  var DEFAULT_THROTTLE_MBPS = 40;
875
918
  /** Progress callback cadence (files) — cheap enough to fire often. */
@@ -913,22 +956,48 @@ async function moveFile(src, dst, srcBytes) {
913
956
  var RelocateEngine = class {
914
957
  deps;
915
958
  jobs = /* @__PURE__ */ new Map();
959
+ /** Job ids waiting for the mover, in enqueue order. */
960
+ queue = [];
961
+ /** Set while a job is executing — the single-flight latch. */
962
+ draining = false;
916
963
  constructor(deps) {
917
964
  this.deps = deps;
918
965
  }
919
966
  list() {
920
967
  return [...this.jobs.values()].sort((a, b) => b.startedAt - a.startedAt).map(snapshot);
921
968
  }
969
+ /**
970
+ * Cancel a running OR queued job. A running job stops after its current
971
+ * file; a queued job is terminated where it stands and never runs — which is
972
+ * why this returns true for both, and why the cancelled job still gets its
973
+ * `onFinished` audit row.
974
+ */
922
975
  cancel(jobId) {
923
976
  const job = this.jobs.get(jobId);
924
- if (!job || job.state !== "running") return false;
977
+ if (!job) return false;
978
+ if (job.state === "running") {
979
+ job.cancelRequested = true;
980
+ return true;
981
+ }
982
+ if (job.state !== "queued") return false;
925
983
  job.cancelRequested = true;
984
+ job.state = "cancelled";
985
+ job.finishedAt = this.deps.now();
986
+ const at = this.queue.indexOf(jobId);
987
+ if (at >= 0) this.queue.splice(at, 1);
988
+ this.deps.onFinished?.(snapshot(job));
926
989
  return true;
927
990
  }
928
- /** Start a relocation. Throws on invalid locations or when a job is already
929
- * running (single-flight two movers on one disk thrash both). */
991
+ /**
992
+ * Enqueue a relocation and return its job id. Throws only on an input the
993
+ * mover could never satisfy (unknown/identical/read-only locations) —
994
+ * validated SYNCHRONOUSLY so a caller arming a rebalance learns about a bad
995
+ * target before any job runs.
996
+ *
997
+ * Concurrency: the returned job may be `queued` rather than `running`. The
998
+ * engine still moves one job at a time.
999
+ */
930
1000
  start(input) {
931
- for (const j of this.jobs.values()) if (j.state === "running") throw new Error(`a relocation is already running (${j.jobId})`);
932
1001
  const locs = this.deps.locations();
933
1002
  const from = locs.find((l) => l.id === input.fromLocationId);
934
1003
  const to = locs.find((l) => l.id === input.toLocationId);
@@ -938,10 +1007,13 @@ var RelocateEngine = class {
938
1007
  if (to.readOnly) throw new Error(`target location is read-only: ${to.id}`);
939
1008
  const job = {
940
1009
  jobId: this.deps.newId(),
941
- state: "running",
1010
+ state: "queued",
942
1011
  fromLocationId: from.id,
943
1012
  toLocationId: to.id,
944
- deviceId: null,
1013
+ deviceId: input.deviceId ?? null,
1014
+ profiles: input.profiles ? [...input.profiles] : [],
1015
+ footageClass: input.footageClass,
1016
+ throttleMbps: input.throttleMbps ?? DEFAULT_THROTTLE_MBPS,
945
1017
  entities: input.entities ? [...input.entities] : ["segments"],
946
1018
  filesMoved: 0,
947
1019
  bytesMoved: 0,
@@ -952,15 +1024,56 @@ var RelocateEngine = class {
952
1024
  cancelRequested: false
953
1025
  };
954
1026
  this.jobs.set(job.jobId, job);
955
- this.run(job, from, to, input.throttleMbps ?? DEFAULT_THROTTLE_MBPS, input.footageClass);
1027
+ this.queue.push(job.jobId);
1028
+ this.drain();
956
1029
  return job.jobId;
957
1030
  }
958
- async run(job, from, to, throttleMbps, footageClass) {
1031
+ /**
1032
+ * Run queued jobs one at a time, in enqueue order. Re-entrant by design: the
1033
+ * `draining` latch is the single-flight guarantee, and every terminal state
1034
+ * (including `failed`) falls through to the next job — a queue stranded
1035
+ * behind one bad camera is the failure mode FIFO exists to avoid.
1036
+ */
1037
+ async drain() {
1038
+ if (this.draining) return;
1039
+ this.draining = true;
1040
+ try {
1041
+ for (;;) {
1042
+ const jobId = this.queue.shift();
1043
+ if (jobId === void 0) return;
1044
+ const job = this.jobs.get(jobId);
1045
+ if (!job || job.state !== "queued") continue;
1046
+ const locs = this.deps.locations();
1047
+ const from = locs.find((l) => l.id === job.fromLocationId);
1048
+ const to = locs.find((l) => l.id === job.toLocationId);
1049
+ if (!from || !to) {
1050
+ job.state = "failed";
1051
+ job.error = `location disappeared while queued: ${job.fromLocationId} → ${job.toLocationId}`;
1052
+ job.finishedAt = this.deps.now();
1053
+ this.deps.logger.warn("relocate job failed", {
1054
+ ...job.deviceId !== null ? { tags: { deviceId: job.deviceId } } : {},
1055
+ meta: {
1056
+ jobId: job.jobId,
1057
+ error: job.error
1058
+ }
1059
+ });
1060
+ this.deps.onFinished?.(snapshot(job));
1061
+ continue;
1062
+ }
1063
+ job.state = "running";
1064
+ job.startedAt = this.deps.now();
1065
+ await this.run(job, from, to);
1066
+ }
1067
+ } finally {
1068
+ this.draining = false;
1069
+ }
1070
+ }
1071
+ async run(job, from, to) {
959
1072
  const sleep = this.deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
960
- const bytesPerMs = throttleMbps * 1024 * 1024 / 1e3;
1073
+ const bytesPerMs = job.throttleMbps * 1024 * 1024 / 1e3;
961
1074
  try {
962
1075
  const samePhysicalRoot = await rootsPhysicallySame(from.root, to.root);
963
- const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => profileBelongsToClass(r.profile, footageClass)).sort((a, b) => a.startMs - b.startMs);
1076
+ const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => job.deviceId === null || r.deviceId === job.deviceId).filter((r) => job.profiles.length === 0 || job.profiles.includes(r.profile)).filter((r) => profileBelongsToClass(r.profile, job.footageClass)).sort((a, b) => a.startMs - b.startMs);
964
1077
  job.filesTotal = segRows.length;
965
1078
  for (const row of segRows) {
966
1079
  if (job.cancelRequested) break;
@@ -983,10 +1096,13 @@ var RelocateEngine = class {
983
1096
  } catch (err) {
984
1097
  job.state = "failed";
985
1098
  job.error = err instanceof Error ? err.message : String(err);
986
- this.deps.logger.warn("relocate job failed", { meta: {
987
- jobId: job.jobId,
988
- error: job.error
989
- } });
1099
+ this.deps.logger.warn("relocate job failed", {
1100
+ ...job.deviceId !== null ? { tags: { deviceId: job.deviceId } } : {},
1101
+ meta: {
1102
+ jobId: job.jobId,
1103
+ error: job.error
1104
+ }
1105
+ });
990
1106
  } finally {
991
1107
  job.finishedAt = this.deps.now();
992
1108
  this.deps.onFinished?.(snapshot(job));
@@ -2298,6 +2414,105 @@ function sendDirectory(res, payload) {
2298
2414
  res.end(Buffer.from(body));
2299
2415
  }
2300
2416
  //#endregion
2417
+ //#region src/recorder/addon/export-dense-map.ts
2418
+ /**
2419
+ * Shortest clip worth keeping, in ms.
2420
+ *
2421
+ * Exists only to honour `ExportDenseRangeSchema`'s `toSec > fromSec`: a range
2422
+ * clipped to a sliver of footage is still honest, but a range clipped to
2423
+ * NOTHING must not be emitted as a zero-length interval the schema refuses.
2424
+ */
2425
+ var MIN_MAPPED_MS = 1;
2426
+ /** Milliseconds of footage that precede `wallMs` in the concatenation. */
2427
+ function recordedMsBefore(segments, wallMs) {
2428
+ let acc = 0;
2429
+ for (const s of segments) if (wallMs >= s.startMs + s.durMs) acc += s.durMs;
2430
+ else if (wallMs > s.startMs) acc += wallMs - s.startMs;
2431
+ return acc;
2432
+ }
2433
+ /** Whole milliseconds → seconds, with no float tail for the filter string. */
2434
+ function msToSec(ms) {
2435
+ return Math.round(ms) / 1e3;
2436
+ }
2437
+ /**
2438
+ * Translate wall-clock dense ranges into the export's stream timeline.
2439
+ *
2440
+ * @returns the surviving ranges (in stream seconds, ordered as given) plus the
2441
+ * report the render logs.
2442
+ */
2443
+ function mapDenseRangesToStreamTime(input) {
2444
+ const segments = [...input.segments].sort((a, b) => a.startMs - b.startMs);
2445
+ const footageEndMs = segments.reduce((end, s) => Math.max(end, s.startMs + s.durMs), input.exportFromMs);
2446
+ const mapped = [];
2447
+ const dropped = [];
2448
+ for (const range of input.ranges) {
2449
+ const fromWallMs = input.exportFromMs + range.fromSec * 1e3;
2450
+ const toWallMs = input.exportFromMs + range.toSec * 1e3;
2451
+ const fromMs = Math.round(recordedMsBefore(segments, fromWallMs));
2452
+ const toMs = Math.round(recordedMsBefore(segments, toWallMs));
2453
+ if (toMs - fromMs < MIN_MAPPED_MS) {
2454
+ dropped.push({
2455
+ fromSec: range.fromSec,
2456
+ toSec: range.toSec,
2457
+ reason: fromWallMs >= footageEndMs ? "past-end" : "gap"
2458
+ });
2459
+ continue;
2460
+ }
2461
+ mapped.push({
2462
+ fromSec: msToSec(fromMs),
2463
+ toSec: msToSec(toMs)
2464
+ });
2465
+ }
2466
+ return {
2467
+ ranges: mapped,
2468
+ report: {
2469
+ requested: input.ranges.length,
2470
+ mapped: mapped.length,
2471
+ dropped
2472
+ }
2473
+ };
2474
+ }
2475
+ /**
2476
+ * One export's options with its dense ranges expressed in stream time.
2477
+ *
2478
+ * An overlay that maps to NOTHING is removed entirely rather than left with an
2479
+ * empty range list: `ExportDenseSchema` refuses `ranges: []` and
2480
+ * `selectPredicate` would emit `if(,…)`, a filter ffmpeg rejects — so a night
2481
+ * whose busy moments all fell in holes would fail the render instead of
2482
+ * delivering the uniform video it legitimately is.
2483
+ */
2484
+ function withStreamTimeDenseRanges(input) {
2485
+ const timelapse = input.options.timelapse;
2486
+ const dense = timelapse?.dense;
2487
+ if (timelapse === void 0 || dense === void 0) return {
2488
+ options: input.options,
2489
+ report: null
2490
+ };
2491
+ const { ranges, report } = mapDenseRangesToStreamTime({
2492
+ ranges: dense.ranges,
2493
+ exportFromMs: input.fromMs,
2494
+ segments: input.segments
2495
+ });
2496
+ const base = {
2497
+ everyMs: timelapse.everyMs,
2498
+ ...timelapse.outputFps !== void 0 ? { outputFps: timelapse.outputFps } : {}
2499
+ };
2500
+ const remapped = ranges.length > 0 ? {
2501
+ ...base,
2502
+ dense: {
2503
+ everyMs: dense.everyMs,
2504
+ ranges: [...ranges]
2505
+ }
2506
+ } : base;
2507
+ return {
2508
+ options: {
2509
+ ...input.options,
2510
+ timelapse: remapped
2511
+ },
2512
+ report
2513
+ };
2514
+ }
2515
+ //#endregion
2301
2516
  //#region src/recorder/addon/export-ffmpeg-args.ts
2302
2517
  /** Widest [min,max] speed a single ffmpeg `atempo` filter accepts. Outside this
2303
2518
  * band audio is dropped rather than chained (kept simple + predictable). */
@@ -2468,20 +2683,6 @@ async function listExportsFor(state, deviceId) {
2468
2683
  }
2469
2684
  //#endregion
2470
2685
  //#region src/recorder/addon/export-engine.ts
2471
- /**
2472
- * Render engine for recording exports.
2473
- *
2474
- * A FIFO queue with ONE concurrent ffmpeg render (exports are heavy; serialising
2475
- * protects the recording node). Each job: write the range's per-export source
2476
- * playlist (`resolvePlaylist`) → build argv (`buildExportArgs`) → spawn ffmpeg →
2477
- * parse `time=` from stderr for progress → on exit 0 stat the file and mark
2478
- * `ready`; on non-zero mark `failed` and remove the partial; drop the source
2479
- * playlist whatever the outcome. `cancel` SIGKILLs an in-flight render.
2480
- * State transitions patch the DurableState store and emit telemetry events.
2481
- *
2482
- * Unlike `SegmentWriter` there is NO auto-restart — a render runs once; boot
2483
- * re-queue of interrupted jobs is the provider's concern.
2484
- */
2485
2686
  /** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
2486
2687
  * segment before the decisive "Error opening input" summary, so a short tail can
2487
2688
  * push the actual cause out of the window (it very nearly did — the line that
@@ -2489,6 +2690,26 @@ async function listExportsFor(state, deviceId) {
2489
2690
  * render writes one progress line per second. */
2490
2691
  var STDERR_TAIL_LINES$1 = 40;
2491
2692
  var TIME_RE = /time=(\d+):(\d+):(\d+(?:\.\d+)?)/;
2693
+ /**
2694
+ * The floor under which an MP4 cannot hold a single decodable frame.
2695
+ *
2696
+ * **Exit 0 is not the same question as "the file holds a video".** A timelapse
2697
+ * whose frame-select predicate matched nothing produced a **262-byte** MP4 —
2698
+ * `ftyp` + `moov` + an EMPTY `mdat`, `nb_streams=0` to ffprobe — and ffmpeg
2699
+ * exited 0 while doing it. The row went `ready`, `readExportBytes` served the
2700
+ * 262 bytes, the notification centre republished them on the artifact plane
2701
+ * and a phone was handed a video no player can open (2026-08-12; the operator
2702
+ * reported it as "il video è danneggiato", and the served artifact was
2703
+ * byte-identical to the file on disk, so the render was the only suspect).
2704
+ *
2705
+ * A frame-free MP4 tops out around 300 bytes (the box headers and nothing
2706
+ * else); the smallest REAL one-frame 640×360 h264 export measured here is
2707
+ * several kilobytes. 1 KiB sits between the two with room on both sides, and
2708
+ * it is deliberately a byte floor and not an ffprobe call: the engine must be
2709
+ * able to refuse an empty render without a second process, on every node,
2710
+ * whatever ffmpeg build is installed.
2711
+ */
2712
+ var EXPORT_MIN_PLAUSIBLE_BYTES = 1024;
2492
2713
  /** Expected OUTPUT duration (seconds) for progress scaling. */
2493
2714
  function expectedOutSeconds(rec) {
2494
2715
  const rangeSec = Math.max(.001, (rec.toMs - rec.fromMs) / 1e3);
@@ -2557,35 +2778,76 @@ var ExportEngine = class {
2557
2778
  return;
2558
2779
  }
2559
2780
  try {
2560
- await this.spawnRender(id, rec, playlist);
2781
+ await this.spawnRender(id, rec, playlist.path, this.streamTimeOptions(rec, playlist.segments));
2561
2782
  } finally {
2562
2783
  if (this.deps.cleanupPlaylist) try {
2563
- await this.deps.cleanupPlaylist(playlist);
2784
+ await this.deps.cleanupPlaylist(playlist.path);
2564
2785
  } catch (err) {
2565
- this.deps.logger.warn("export: source playlist cleanup failed", { meta: {
2566
- exportId: id,
2567
- playlist,
2568
- error: err instanceof Error ? err.message : String(err)
2569
- } });
2786
+ this.deps.logger.warn("export: source playlist cleanup failed", {
2787
+ tags: { deviceId: rec.deviceId },
2788
+ meta: {
2789
+ exportId: id,
2790
+ playlist: playlist.path,
2791
+ error: err instanceof Error ? err.message : String(err)
2792
+ }
2793
+ });
2570
2794
  }
2571
2795
  }
2572
2796
  }
2797
+ /**
2798
+ * The record's options with any dense overlay moved onto ffmpeg's clock.
2799
+ *
2800
+ * The translation is reported on EVERY render that carries an overlay, not
2801
+ * only when something is dropped: `denseRangesRequested` vs
2802
+ * `denseRangesMapped` is the one line that can tell "the video is uniform
2803
+ * because the night was quiet" from "the video is uniform because every range
2804
+ * missed" — the distinction that went unnoticed for a full night because the
2805
+ * only line printed was the requested count.
2806
+ */
2807
+ streamTimeOptions(rec, segments) {
2808
+ const { options, report } = withStreamTimeDenseRanges({
2809
+ options: rec.options,
2810
+ fromMs: rec.fromMs,
2811
+ segments
2812
+ });
2813
+ if (report !== null) this.logDenseMapping(rec, report);
2814
+ return options;
2815
+ }
2816
+ /** One line, whatever the outcome — `warn` when ranges were dropped, because
2817
+ * a dropped range is footage the operator asked for and did not get. */
2818
+ logDenseMapping(rec, report) {
2819
+ const entry = {
2820
+ tags: { deviceId: rec.deviceId },
2821
+ meta: {
2822
+ exportId: rec.id,
2823
+ denseRangesRequested: report.requested,
2824
+ denseRangesMapped: report.mapped,
2825
+ ...report.dropped.length > 0 ? { denseRangesDropped: report.dropped } : {}
2826
+ }
2827
+ };
2828
+ const message = "export dense ranges mapped onto the concatenated stream";
2829
+ if (report.dropped.length > 0) this.deps.logger.warn(message, entry);
2830
+ else this.deps.logger.info(message, entry);
2831
+ }
2573
2832
  /** Spawn ffmpeg for one render and resolve when it has exited + finalized. */
2574
- async spawnRender(id, rec, playlist) {
2833
+ async spawnRender(id, rec, playlist, options) {
2575
2834
  const outPath = this.deps.outPath(rec);
2576
2835
  const args = buildExportArgs({
2577
2836
  inputPlaylist: playlist,
2578
2837
  outPath,
2579
- options: rec.options
2838
+ options
2580
2839
  });
2581
2840
  const totalSec = expectedOutSeconds(rec);
2582
- this.deps.logger.info("export render starting", { meta: {
2583
- exportId: id,
2584
- deviceId: rec.deviceId,
2585
- profile: rec.profile,
2586
- playlist,
2587
- outPath
2588
- } });
2841
+ this.deps.logger.info("export render starting", {
2842
+ tags: { deviceId: rec.deviceId },
2843
+ meta: {
2844
+ exportId: id,
2845
+ deviceId: rec.deviceId,
2846
+ profile: rec.profile,
2847
+ playlist,
2848
+ outPath
2849
+ }
2850
+ });
2589
2851
  this.deps.logger.debug("export ffmpeg argv", { meta: {
2590
2852
  exportId: id,
2591
2853
  args
@@ -2645,6 +2907,17 @@ var ExportEngine = class {
2645
2907
  await this.fail(id, err instanceof Error ? err.message : String(err));
2646
2908
  return;
2647
2909
  }
2910
+ if (bytes < 1024) {
2911
+ this.deps.logger.error("export produced no video frames", { meta: {
2912
+ exportId: id,
2913
+ outPath,
2914
+ bytes,
2915
+ minBytes: EXPORT_MIN_PLAUSIBLE_BYTES
2916
+ } });
2917
+ await this.deps.removeFile(outPath);
2918
+ await this.fail(id, `render produced no video frames (${bytes} bytes, < ${EXPORT_MIN_PLAUSIBLE_BYTES})`);
2919
+ return;
2920
+ }
2648
2921
  await patchExport(this.deps.state, id, {
2649
2922
  state: "ready",
2650
2923
  progressPct: 100,
@@ -3026,6 +3299,73 @@ function buildRecordingDeviceSchema() {
3026
3299
  }]
3027
3300
  }] };
3028
3301
  }
3302
+ /** The placement-store key for one (camera, profile). Mirrors `assignmentKey`
3303
+ * in `placement-store.ts`; kept local so this module stays dependency-free. */
3304
+ function sourceKey(source) {
3305
+ return `${source.deviceId}:${source.profile}`;
3306
+ }
3307
+ /** Biggest pile first (the overloaded disk is relieved fastest), ties broken
3308
+ * deterministically so the same input always yields the same plan. */
3309
+ function bySizeThenKey(a, b) {
3310
+ return b.bytes - a.bytes || a.deviceId - b.deviceId || a.profile.localeCompare(b.profile);
3311
+ }
3312
+ /**
3313
+ * Plan the moves that would make stored footage agree with the placement plan.
3314
+ * Pure: the same input always yields the same plan, and the input is untouched.
3315
+ */
3316
+ function planRebalance(input) {
3317
+ const minMoveBytes = input.minMoveBytes ?? 1e9;
3318
+ const remaining = new Map(input.headroom.map((h) => [h.locationId, h.headroomBytes]));
3319
+ const moves = [];
3320
+ const skipped = [];
3321
+ let bytesToMove = 0;
3322
+ for (const source of [...input.sources].sort(bySizeThenKey)) {
3323
+ const target = input.assignments.get(sourceKey(source));
3324
+ if (target === source.locationId) continue;
3325
+ const skip = (reason, toLocationId) => {
3326
+ skipped.push({
3327
+ deviceId: source.deviceId,
3328
+ profile: source.profile,
3329
+ fromLocationId: source.locationId,
3330
+ toLocationId,
3331
+ bytes: source.bytes,
3332
+ reason
3333
+ });
3334
+ };
3335
+ if (target === void 0) {
3336
+ skip("unassigned", null);
3337
+ continue;
3338
+ }
3339
+ const headroomBytes = remaining.get(target);
3340
+ if (headroomBytes === void 0) {
3341
+ skip("target-not-writable", target);
3342
+ continue;
3343
+ }
3344
+ if (source.bytes < minMoveBytes) {
3345
+ skip("below-threshold", target);
3346
+ continue;
3347
+ }
3348
+ if (source.bytes > headroomBytes) {
3349
+ skip("no-headroom", target);
3350
+ continue;
3351
+ }
3352
+ remaining.set(target, headroomBytes - source.bytes);
3353
+ bytesToMove += source.bytes;
3354
+ moves.push({
3355
+ deviceId: source.deviceId,
3356
+ profile: source.profile,
3357
+ fromLocationId: source.locationId,
3358
+ toLocationId: target,
3359
+ bytes: source.bytes,
3360
+ files: source.files
3361
+ });
3362
+ }
3363
+ return {
3364
+ moves,
3365
+ skipped,
3366
+ bytesToMove
3367
+ };
3368
+ }
3029
3369
  //#endregion
3030
3370
  //#region src/recorder/addon/recording-provider.ts
3031
3371
  /**
@@ -3303,6 +3643,75 @@ async function openRangeFromDisk(absPath, size) {
3303
3643
  close: () => handle.close()
3304
3644
  };
3305
3645
  }
3646
+ /** Bytes in one operator-facing gigabyte — the same decimal GB `maxUsedGb`
3647
+ * uses, so two knobs on the same page never mean two different things. */
3648
+ var BYTES_PER_GB = 1e9;
3649
+ /**
3650
+ * The operator-armed rebalance, both halves (D116, Phase 2).
3651
+ *
3652
+ * `arm: false` is the dry run the operator confirms; `arm: true` enqueues one
3653
+ * relocate job per planned move, FIFO behind the single-flight mover. The two
3654
+ * share this one body deliberately: what is confirmed is exactly what runs.
3655
+ *
3656
+ * Every refusal is LOGGED, per camera. A rebalance that quietly leaves half the
3657
+ * archive where it was is indistinguishable from one that had nothing to do,
3658
+ * and this repo has paid for that ambiguity twice.
3659
+ */
3660
+ async function runRebalance(deps, input, arm) {
3661
+ const relocate = deps.relocate;
3662
+ if (!deps.placementSnapshot) throw new Error("placement is not active on this node");
3663
+ if (arm && !relocate) throw new Error("relocation unavailable on this node");
3664
+ const snapshot = await deps.placementSnapshot();
3665
+ const plan = planRebalance({
3666
+ assignments: snapshot.assignments,
3667
+ sources: deps.index.piles(),
3668
+ headroom: snapshot.headroom,
3669
+ ...input.minMoveGb !== void 0 ? { minMoveBytes: input.minMoveGb * BYTES_PER_GB } : {}
3670
+ });
3671
+ for (const skip of plan.skipped) deps.logger.info("recorder rebalance: footage left where it is", {
3672
+ tags: { deviceId: skip.deviceId },
3673
+ meta: {
3674
+ profile: skip.profile,
3675
+ from: skip.fromLocationId,
3676
+ to: skip.toLocationId,
3677
+ bytes: skip.bytes,
3678
+ reason: skip.reason
3679
+ }
3680
+ });
3681
+ const result = {
3682
+ moves: [...plan.moves],
3683
+ skipped: [...plan.skipped],
3684
+ bytesToMove: plan.bytesToMove,
3685
+ jobIds: []
3686
+ };
3687
+ if (!arm || !relocate) return result;
3688
+ const jobIds = [];
3689
+ for (const move of plan.moves) {
3690
+ const jobId = relocate.start({
3691
+ fromLocationId: move.fromLocationId,
3692
+ toLocationId: move.toLocationId,
3693
+ deviceId: move.deviceId,
3694
+ profiles: [move.profile],
3695
+ ...input.throttleMbps !== void 0 ? { throttleMbps: input.throttleMbps } : {}
3696
+ });
3697
+ jobIds.push(jobId);
3698
+ deps.logger.info("recorder rebalance: relocate job enqueued", {
3699
+ tags: { deviceId: move.deviceId },
3700
+ meta: {
3701
+ jobId,
3702
+ profile: move.profile,
3703
+ from: move.fromLocationId,
3704
+ to: move.toLocationId,
3705
+ bytes: move.bytes,
3706
+ files: move.files
3707
+ }
3708
+ });
3709
+ }
3710
+ return {
3711
+ ...result,
3712
+ jobIds
3713
+ };
3714
+ }
3306
3715
  /**
3307
3716
  * Build the `IRecordingProvider`. The provider is a thin façade over the v2
3308
3717
  * core; all storage I/O flows through the injected deps.
@@ -3366,11 +3775,12 @@ function buildRecordingProvider(deps) {
3366
3775
  for (const loc of locations) for (const row of deps.index.segmentsOnLocation(loc.id)) seenDevices.add(row.deviceId);
3367
3776
  let totalUsedBytes = 0;
3368
3777
  for (const deviceId of seenDevices) {
3369
- const usedBytes = deps.index.accounting(deviceId).bytes;
3370
- totalUsedBytes += usedBytes;
3778
+ const accounting = deps.index.accounting(deviceId);
3779
+ totalUsedBytes += accounting.bytes;
3371
3780
  devices.push({
3372
3781
  deviceId,
3373
- usedBytes
3782
+ usedBytes: accounting.bytes,
3783
+ oldestMs: accounting.oldestMs
3374
3784
  });
3375
3785
  }
3376
3786
  const byPhysical = /* @__PURE__ */ new Map();
@@ -3557,6 +3967,24 @@ function buildRecordingProvider(deps) {
3557
3967
  },
3558
3968
  getStorageMigrationMoveStatus: async ({ jobId }) => deps.relocate?.list().find((job) => job.jobId === jobId) ?? null,
3559
3969
  cancelStorageMigrationMove: async ({ jobId }) => ({ cancelled: deps.relocate?.cancel(jobId) ?? false }),
3970
+ relocateFootage: async (input) => {
3971
+ if (!deps.relocate) throw new Error("relocation unavailable on this node");
3972
+ const jobId = deps.relocate.start(input);
3973
+ deps.logger.info("recorder: relocate job enqueued", {
3974
+ ...input.deviceId !== void 0 ? { tags: { deviceId: input.deviceId } } : {},
3975
+ meta: {
3976
+ jobId,
3977
+ from: input.fromLocationId,
3978
+ to: input.toLocationId,
3979
+ profiles: input.profiles ?? null
3980
+ }
3981
+ });
3982
+ return { jobId };
3983
+ },
3984
+ listRelocateJobs: async () => deps.relocate?.list() ?? [],
3985
+ cancelRelocateJob: async ({ jobId }) => ({ cancelled: deps.relocate?.cancel(jobId) ?? false }),
3986
+ planStorageRebalance: (input) => runRebalance(deps, input, false),
3987
+ startStorageRebalance: (input) => runRebalance(deps, input, true),
3560
3988
  getDeviceSettingsContribution: async ({ deviceId }) => {
3561
3989
  if (deps.isCameraDevice && !await deps.isCameraDevice(deviceId)) return null;
3562
3990
  return hydrateSchema(buildRecordingDeviceSchema(), {});
@@ -3602,9 +4030,9 @@ function exportSourcePlaylistPath(exportsDir, deviceId, exportId) {
3602
4030
  return path.join(exportsDir, String(deviceId), `${exportId}.src.m3u8`);
3603
4031
  }
3604
4032
  /**
3605
- * Write the export's source playlist and return its absolute path, or null when
3606
- * no segment of the requested profile is present on disk for the range (the
3607
- * engine then fails the job with 'no footage for range').
4033
+ * Write the export's source playlist and return it, or null when no segment of
4034
+ * the requested profile is present on disk for the range (the engine then fails
4035
+ * the job with 'no footage for range').
3608
4036
  */
3609
4037
  async function writeExportSourcePlaylist(input) {
3610
4038
  const { segments } = await collectRangeSegments(input.deps, input.deviceId, input.profile, input.fromMs, input.toMs, "absolute");
@@ -3612,26 +4040,16 @@ async function writeExportSourcePlaylist(input) {
3612
4040
  const playlistPath = exportSourcePlaylistPath(input.exportsDir, input.deviceId, input.exportId);
3613
4041
  await promises.mkdir(path.dirname(playlistPath), { recursive: true });
3614
4042
  await promises.writeFile(playlistPath, buildVariantPlaylist(segments), "utf8");
3615
- return playlistPath;
4043
+ return {
4044
+ path: playlistPath,
4045
+ segments: segments.map((s) => ({
4046
+ startMs: s.startMs,
4047
+ durMs: s.durMs
4048
+ }))
4049
+ };
3616
4050
  }
3617
4051
  //#endregion
3618
4052
  //#region src/recorder/addon/footage-render.ts
3619
- /**
3620
- * One-shot media render FROM RECORDED FOOTAGE (notification attachments).
3621
- *
3622
- * Shared core behind `renderGif` and `renderClip`: both cut the same window out
3623
- * of the same `low` profile through the export machinery's per-render source
3624
- * playlist (absolute segment URIs — the ONLY playlist shape ffmpeg can consume,
3625
- * see export-source-playlist.ts), pipe it through ONE ffmpeg invocation, and
3626
- * return the bytes. Everything is transient: playlist and output are deleted on
3627
- * every outcome.
3628
- *
3629
- * Fail-closed by construction: no footage covering the window ⇒ it throws, and
3630
- * the caller (a notification rule) simply ships no attachment. A camera that is
3631
- * not recording therefore cannot produce a clip — which is exactly why the
3632
- * frame-ring plane (roadmap Phase 5) still matters for cameras with no
3633
- * recording. It is NOT a prerequisite for the cameras that do record.
3634
- */
3635
4053
  var RENDER_TIMEOUT_MS = 3e4;
3636
4054
  /**
3637
4055
  * A notification clip must sit AROUND the moment it describes. Sliding the
@@ -3729,9 +4147,9 @@ async function renderFootage(renderDeps, input, ext, buildArgs) {
3729
4147
  toMs: window.toMs
3730
4148
  });
3731
4149
  if (playlist === null) throw new Error(`no footage covers [${window.fromMs}, ${window.toMs}) for device ${input.deviceId}`);
3732
- const outPath = path.join(path.dirname(playlist), `${renderId}.${ext}`);
4150
+ const outPath = path.join(path.dirname(playlist.path), `${renderId}.${ext}`);
3733
4151
  try {
3734
- await runFfmpeg(renderDeps, buildArgs(playlist, outPath), ext);
4152
+ await runFfmpeg(renderDeps, buildArgs(playlist.path, outPath), ext);
3735
4153
  const bytes = await promises.readFile(outPath);
3736
4154
  if (bytes.byteLength === 0) throw new Error(`${ext} render produced an empty file`);
3737
4155
  return {
@@ -3740,7 +4158,7 @@ async function renderFootage(renderDeps, input, ext, buildArgs) {
3740
4158
  toMs: window.toMs
3741
4159
  };
3742
4160
  } finally {
3743
- await promises.rm(playlist, { force: true }).catch(() => {});
4161
+ await promises.rm(playlist.path, { force: true }).catch(() => {});
3744
4162
  await promises.rm(outPath, { force: true }).catch(() => {});
3745
4163
  }
3746
4164
  }
@@ -4688,6 +5106,33 @@ var PlacementService = class {
4688
5106
  return plan;
4689
5107
  }
4690
5108
  /**
5109
+ * The write targets and what each can still absorb — the SAME pool and the
5110
+ * SAME headroom `recompute` plans over.
5111
+ *
5112
+ * Exposed for the operator-armed rebalance (D116): weighing existing footage
5113
+ * against a second, independently-derived idea of "what fits" is how the two
5114
+ * halves end up disagreeing, and a rebalance that plans onto a location the
5115
+ * planner would never write to is a rebalance that undoes itself on the next
5116
+ * tick. A location absent from this list is not a write target at all.
5117
+ */
5118
+ async candidates() {
5119
+ return this.candidatesFor(this.pool());
5120
+ }
5121
+ /**
5122
+ * The plan in force: `<deviceId>:<profile>` → locationId. Empty when the
5123
+ * store cannot be read — a fallible read decides nothing (D49), and an empty
5124
+ * plan makes the rebalance report every pile as `unassigned` rather than
5125
+ * inventing moves.
5126
+ */
5127
+ async currentAssignments() {
5128
+ try {
5129
+ return new Map(Object.entries((await readPlacementState(this.deps.state)).assignments));
5130
+ } catch (err) {
5131
+ this.deps.logger.warn("recorder placement: assignment read failed — reporting no plan", { meta: { error: errMsg(err) } });
5132
+ return /* @__PURE__ */ new Map();
5133
+ }
5134
+ }
5135
+ /**
4691
5136
  * Turn each pool location into a planner candidate. Headroom is
4692
5137
  * `min(free − minFree floor, maxUsedGb − used)`, clamped at zero: the first
4693
5138
  * term is the per-location free-space guard the storage pressure manager also
@@ -4957,6 +5402,49 @@ function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers) {
4957
5402
  return false;
4958
5403
  }
4959
5404
  //#endregion
5405
+ //#region src/recorder/addon/periodic-pass.ts
5406
+ function startPeriodicPass(deps) {
5407
+ const now = () => deps.now?.() ?? Date.now();
5408
+ let startedAt = null;
5409
+ let stopped = false;
5410
+ const fire = () => {
5411
+ if (stopped) return;
5412
+ if (startedAt !== null) {
5413
+ deps.logger.warn("recorder: pass skipped — the previous pass is still running", { meta: {
5414
+ pass: deps.name,
5415
+ inFlightMs: now() - startedAt,
5416
+ intervalMs: deps.intervalMs
5417
+ } });
5418
+ return;
5419
+ }
5420
+ const began = now();
5421
+ startedAt = began;
5422
+ deps.run().catch((err) => {
5423
+ deps.logger.warn("recorder: pass failed", { meta: {
5424
+ pass: deps.name,
5425
+ error: err instanceof Error ? err.message : String(err)
5426
+ } });
5427
+ }).finally(() => {
5428
+ const durationMs = now() - began;
5429
+ startedAt = null;
5430
+ if (durationMs > deps.slowAfterMs) deps.logger.warn("recorder: pass exceeded its budget", { meta: {
5431
+ pass: deps.name,
5432
+ durationMs,
5433
+ budgetMs: deps.slowAfterMs
5434
+ } });
5435
+ });
5436
+ };
5437
+ const timer = setInterval(fire, deps.intervalMs);
5438
+ timer.unref?.();
5439
+ return {
5440
+ stop() {
5441
+ stopped = true;
5442
+ clearInterval(timer);
5443
+ },
5444
+ isRunning: () => startedAt !== null
5445
+ };
5446
+ }
5447
+ //#endregion
4960
5448
  //#region src/recorder/addon/readiness-restore.ts
4961
5449
  var ReadinessRestore = class {
4962
5450
  deps;
@@ -5043,10 +5531,59 @@ var ReadinessRestore = class {
5043
5531
  * no subtree to classify.
5044
5532
  */
5045
5533
  var EPOCH_NAME_RE = /^(\d+)\.m4s$/;
5046
- /** How many consecutive ticks to wait for a finalized segment's flat file to
5047
- * land before giving up on it (a genuine phantom). At a 2s watch interval this
5048
- * is ~16s far longer than ffmpeg's slow-first-segment flush. */
5049
- var MAX_PENDING_TICKS = 8;
5534
+ /**
5535
+ * How long a finalized segment's flat file may take to appear before the entry
5536
+ * is given up on.
5537
+ *
5538
+ * This used to be `MAX_PENDING_TICKS = 8` — a count of TICKS, which is not a
5539
+ * measure of time. The tick loop self-excludes while a pass is running and each
5540
+ * pass does real filesystem work, so on 2026-08-10..13 eight ticks measured
5541
+ * anywhere from 8 s to 819 s. A bound has to be denominated in the thing it
5542
+ * bounds.
5543
+ *
5544
+ * 30 s is chosen from the measurement, not from taste. Over four days the
5545
+ * segments that were retried and then DID land waited p50 7.8 s and p90 11.6 s,
5546
+ * and widening the window from 16 s to 120 s moved coverage only from 92.9% to
5547
+ * 93.8%. There is no tail to chase: past this knee a segment is not slow, it is
5548
+ * lost (a writer killed mid-segment), and waiting only delays the skip and the
5549
+ * log line that reports it.
5550
+ */
5551
+ var PENDING_GRACE_MS = 3e4;
5552
+ /**
5553
+ * How far before the writer's own start a segment may be named and still be
5554
+ * treated as belonging to THIS run.
5555
+ *
5556
+ * ffmpeg names segments with whole-second `%s`, so the first segment of a run
5557
+ * carries the second that was already in progress when the process spawned —
5558
+ * up to ~1 s "before" the writer started. Without this tolerance the very first
5559
+ * segment of every recording would be classified as a previous run's leftover
5560
+ * and dropped.
5561
+ */
5562
+ var PREVIOUS_RUN_SKEW_MS = 1e3;
5563
+ /**
5564
+ * Decide whether a missing flat file is a late flush or a previous run's entry.
5565
+ *
5566
+ * ffmpeg's `-segment_list` file is not cleared between runs and a fresh watcher
5567
+ * starts at `processed = 0`, so the first passes after every attach replay the
5568
+ * PREVIOUS run's playlist. Those entries were relocated into the bucket tree
5569
+ * long ago; their absence is the system working. Over 2026-08-10..13 they were
5570
+ * 83% of all retries (median age 1 615 s, max 17.4 h) and produced 68 of the
5571
+ * 122 `never landed` warnings — a warning that reads like data loss for a file
5572
+ * that was correctly filed hours earlier.
5573
+ *
5574
+ * Note what this deliberately does NOT do: it does not skip a previous run's
5575
+ * entry whose file is STILL THERE. That case is real crash recovery — an
5576
+ * unclean stop leaves un-relocated segments in `.rec-tmp` and the playlist is
5577
+ * the only record of their durations — so the caller stats first and only asks
5578
+ * this question once the file is known to be absent.
5579
+ */
5580
+ function classifyMissingEntry(input) {
5581
+ return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS ? "stale-previous-run" : "pending";
5582
+ }
5583
+ /** Whether a pending entry has outlived its grace. Pure — the caller clocks it. */
5584
+ function shouldGiveUpOnPending(input) {
5585
+ return input.waitedMs > input.boundMs;
5586
+ }
5050
5587
  /**
5051
5588
  * Derive `startMs` (epoch milliseconds) from a flat ffmpeg segment path
5052
5589
  * (`<epochSec>.m4s`, possibly absolute). Returns null for any non-epoch path.
@@ -5121,11 +5658,22 @@ async function statSize(p) {
5121
5658
  * Unlike the old recorder, this watcher does NOT relocate or re-stat: the
5122
5659
  * SegmentStore owns the move + the authoritative byte count.
5123
5660
  */
5124
- async function handleSegmentEntry(outDir, entry, durMs, onFinalized, logger) {
5661
+ async function handleSegmentEntry(outDir, entry, durMs, onFinalized, logger, writerStartedMs) {
5125
5662
  const flatAbsPath = path.isAbsolute(entry.segPath) ? entry.segPath : path.join(outDir, entry.segPath);
5126
5663
  const startMs = parseEpochStartMs(flatAbsPath);
5127
5664
  if (startMs === null) return "skipped";
5128
5665
  if (await statSize(flatAbsPath) === null) {
5666
+ if (classifyMissingEntry({
5667
+ segmentStartMs: startMs,
5668
+ writerStartedMs
5669
+ }) === "stale-previous-run") {
5670
+ logger.debug("segment entry predates this writer — already relocated, skipping", { meta: {
5671
+ path: flatAbsPath,
5672
+ segmentStartMs: startMs,
5673
+ writerStartedMs
5674
+ } });
5675
+ return "skipped";
5676
+ }
5129
5677
  logger.debug("segment flat file not on disk yet — will retry", { meta: { path: flatAbsPath } });
5130
5678
  return "pending";
5131
5679
  }
@@ -5171,12 +5719,13 @@ function planFinalizations(body, processed, forceFinal = false) {
5171
5719
  */
5172
5720
  function startSegmentWatcher(deps) {
5173
5721
  const playlistPath = path.join(deps.outDir, "live.m3u8");
5722
+ const now = () => deps.now?.() ?? Date.now();
5174
5723
  let processed = 0;
5175
5724
  let stopped = false;
5176
5725
  let ticking = false;
5177
5726
  let currentTick = null;
5178
5727
  let pendingIndex = -1;
5179
- let pendingTicks = 0;
5728
+ let pendingSinceMs = 0;
5180
5729
  const tick = async (forceFinal = false) => {
5181
5730
  if (stopped || ticking) return;
5182
5731
  ticking = true;
@@ -5190,19 +5739,26 @@ function startSegmentWatcher(deps) {
5190
5739
  const plan = planFinalizations(body, processed, forceFinal);
5191
5740
  let index = plan.from;
5192
5741
  for (const { entry, durMs } of plan.toHandle) {
5193
- if (await handleSegmentEntry(deps.outDir, entry, durMs, deps.onFinalized, deps.logger) === "pending") {
5742
+ if (await handleSegmentEntry(deps.outDir, entry, durMs, deps.onFinalized, deps.logger, deps.writerStartedMs) === "pending") {
5194
5743
  if (pendingIndex === index) {
5195
- pendingTicks++;
5196
- if (pendingTicks >= MAX_PENDING_TICKS) {
5197
- deps.logger.warn("segment file never landed — skipping", { meta: { seg: entry.segPath } });
5744
+ const waitedMs = now() - pendingSinceMs;
5745
+ if (shouldGiveUpOnPending({
5746
+ waitedMs,
5747
+ boundMs: 3e4
5748
+ })) {
5749
+ deps.logger.warn("segment file never landed — skipping", { meta: {
5750
+ seg: entry.segPath,
5751
+ waitedMs,
5752
+ boundMs: PENDING_GRACE_MS
5753
+ } });
5198
5754
  pendingIndex = -1;
5199
- pendingTicks = 0;
5755
+ pendingSinceMs = 0;
5200
5756
  index++;
5201
5757
  continue;
5202
5758
  }
5203
5759
  } else {
5204
5760
  pendingIndex = index;
5205
- pendingTicks = 1;
5761
+ pendingSinceMs = now();
5206
5762
  }
5207
5763
  break;
5208
5764
  }
@@ -5210,7 +5766,7 @@ function startSegmentWatcher(deps) {
5210
5766
  }
5211
5767
  if (pendingIndex !== index) {
5212
5768
  pendingIndex = -1;
5213
- pendingTicks = 0;
5769
+ pendingSinceMs = 0;
5214
5770
  }
5215
5771
  processed = index;
5216
5772
  } finally {
@@ -5411,9 +5967,16 @@ var SegmentWriter = class {
5411
5967
  * (`shouldRecordContinuousAt`) and ensures the writer set is running iff so.
5412
5968
  * It re-evaluates on three triggers:
5413
5969
  * 1. `setDeviceConfig` (operator changed the bands / enable),
5414
- * 2. a periodic tick (catches band boundaries crossed with no config change),
5970
+ * 2. a periodic pass (catches band boundaries crossed with no config change),
5415
5971
  * 3. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`).
5416
5972
  *
5973
+ * There are TWO periodic passes, on separate timers and never overlapping
5974
+ * themselves (`periodic-pass.ts`): a cheap LIVENESS pass that catches writers
5975
+ * pinned but producing nothing, and an expensive CONVERGENCE pass that
5976
+ * re-evaluates bands and placement. They were one pass until 2026-08-13, when a
5977
+ * convergence sweep wedged on unbounded broker RPCs and took the liveness
5978
+ * watchdog down with it for 9-27 minutes at a time.
5979
+ *
5417
5980
  * EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
5418
5981
  * trigger-gating. The enable intent is persisted in durable-state by the
5419
5982
  * config-store, so `restoreEnabledDevices` on boot re-attaches every device
@@ -5443,6 +6006,34 @@ var TICK_MS = 3e4;
5443
6006
  */
5444
6007
  var RELEASE_RPC_TIMEOUT_MS = 2e3;
5445
6008
  /**
6009
+ * Bound on each broker RPC in the ATTACH chain (`listAllProfileSlots`,
6010
+ * `getStreamWithCodec`).
6011
+ *
6012
+ * These were the only unbounded calls left in the controller.
6013
+ * `CapabilityHandle.call` gates READINESS (10 s) and then returns `fn()`
6014
+ * untouched, so an accepted-but-unanswered request fell through to the 60 s UDS
6015
+ * default in `kernel/transport/socket-channel.ts`. On 2026-08-13 the broker was
6016
+ * degraded (516 UDS timeouts that day) and the arithmetic did the damage: one
6017
+ * device costs `listAllProfileSlots` + one `getStreamWithCodec` per profile, so
6018
+ * a 4-camera fleet on 3 profiles reached ~16 minutes inside a single recovery
6019
+ * pass — during which every device sits DETACHED (recording stopped) and in
6020
+ * `attaching` (so `evaluateDevice` returns early and the liveness watchdog,
6021
+ * which reads `active`, has nothing to report). That is the shape of the 9-27
6022
+ * minute silences and the real recording gaps behind the 80% timelapse
6023
+ * coverage.
6024
+ *
6025
+ * 20 s is generous for a real RTSP dial and far below the UDS default, so a
6026
+ * hung broker now fails the attach fast. Nothing is lost by giving up: the
6027
+ * device stays on the readiness-restore queue and the next pass retries it.
6028
+ */
6029
+ var ATTACH_RPC_TIMEOUT_MS = 2e4;
6030
+ /**
6031
+ * A pass longer than this is reported with its duration. Equal to the pass
6032
+ * cadence: a pass that cannot finish inside its own interval is the condition
6033
+ * that used to go unlogged for half an hour.
6034
+ */
6035
+ var PASS_SLOW_AFTER_MS = TICK_MS;
6036
+ /**
5446
6037
  * Race `promise` against a deadline. Handlers stay attached to the losing
5447
6038
  * promise, so a post-deadline settlement can never surface as a process-level
5448
6039
  * `unhandledRejection`.
@@ -5530,7 +6121,10 @@ var RecordingController = class {
5530
6121
  */
5531
6122
  unrecordableReported = /* @__PURE__ */ new Set();
5532
6123
  restore = null;
5533
- tickTimer = null;
6124
+ /** Cheap, never-delayed: is any pinned writer producing nothing? */
6125
+ livenessPass = null;
6126
+ /** Expensive: re-evaluate bands + placement for every configured device. */
6127
+ convergePass = null;
5534
6128
  stopped = false;
5535
6129
  /** A reversible maintenance lease. Unlike `stopped`, it never changes
5536
6130
  * persisted recording intent and `resume()` restarts normal convergence. */
@@ -5565,18 +6159,28 @@ var RecordingController = class {
5565
6159
  meta: { error: errMsg(err) }
5566
6160
  })
5567
6161
  });
5568
- if (this.tickTimer === null) {
5569
- this.tickTimer = setInterval(() => {
5570
- this.tick();
5571
- }, TICK_MS);
5572
- this.tickTimer.unref?.();
6162
+ if (this.convergePass === null) {
6163
+ this.livenessPass = startPeriodicPass({
6164
+ name: "liveness",
6165
+ intervalMs: TICK_MS,
6166
+ slowAfterMs: PASS_SLOW_AFTER_MS,
6167
+ logger: this.deps.logger,
6168
+ now: () => this.now(),
6169
+ run: () => this.checkIdleWriters()
6170
+ });
6171
+ this.convergePass = startPeriodicPass({
6172
+ name: "convergence",
6173
+ intervalMs: TICK_MS,
6174
+ slowAfterMs: PASS_SLOW_AFTER_MS,
6175
+ logger: this.deps.logger,
6176
+ now: () => this.now(),
6177
+ run: () => this.converge()
6178
+ });
5573
6179
  }
5574
6180
  await this.restore.start(ids.map((id) => [id, true]));
5575
6181
  }
5576
- /** Re-evaluate every currently-tracked OR active device on the periodic tick. */
5577
- async tick() {
5578
- if (this.stopped) return;
5579
- await this.checkIdleWriters();
6182
+ /** Re-evaluate every currently-tracked OR active device on the periodic pass. */
6183
+ async converge() {
5580
6184
  if (this.stopped) return;
5581
6185
  const ids = new Set(this.active.keys());
5582
6186
  let persisted = [];
@@ -5592,7 +6196,7 @@ var RecordingController = class {
5592
6196
  for (const id of ids) try {
5593
6197
  await this.evaluateDevice(id);
5594
6198
  } catch (err) {
5595
- this.deps.logger.warn("recorder controller: tick evaluate failed", {
6199
+ this.deps.logger.warn("recorder controller: convergence evaluate failed", {
5596
6200
  tags: { deviceId: id },
5597
6201
  meta: { error: errMsg(err) }
5598
6202
  });
@@ -5666,7 +6270,7 @@ var RecordingController = class {
5666
6270
  segmentSeconds: this.active.get(s.deviceId)?.find((r) => r.profile === s.profile)?.segmentSeconds
5667
6271
  }
5668
6272
  });
5669
- for (const deviceId of devices) await this.recoverWriterGaveUp(deviceId);
6273
+ await Promise.all([...devices].map((deviceId) => this.recoverWriterGaveUp(deviceId)));
5670
6274
  }
5671
6275
  /** Current trigger state for a device (cold default = no triggers seen). */
5672
6276
  triggersFor(deviceId) {
@@ -5808,7 +6412,7 @@ var RecordingController = class {
5808
6412
  * assigned sources is ignored (record every assigned source — minimum of 1).
5809
6413
  */
5810
6414
  async resolveProfiles(deviceId, override) {
5811
- const assigned = selectAssignedProfileSlots(await this.deps.brokerCall(() => this.deps.api.streamBroker.listAllProfileSlots.query(void 0, nodePin(this.deps.ownerNodeId))), deviceId).map((slot) => slot.profile);
6415
+ const assigned = selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile);
5812
6416
  if (!override || override.length === 0) return assigned;
5813
6417
  const selected = assigned.filter((p) => override.includes(p));
5814
6418
  return selected.length > 0 ? selected : assigned;
@@ -5821,14 +6425,14 @@ var RecordingController = class {
5821
6425
  */
5822
6426
  async attachProfile(deviceId, profile, segmentSeconds) {
5823
6427
  const placement = await this.resolvePlacement(deviceId, profile);
5824
- const source = await this.deps.brokerCall(() => this.deps.api.streamBroker.getStreamWithCodec.mutate({
6428
+ const source = await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.getStreamWithCodec.mutate({
5825
6429
  deviceId,
5826
6430
  video: "copy",
5827
6431
  audio: "aac",
5828
6432
  profile,
5829
6433
  ...this.deps.consumerHostname !== void 0 ? { hostname: this.deps.consumerHostname } : {},
5830
6434
  tag: `recorder:${deviceId}/${profile}`
5831
- }, nodePin(this.deps.ownerNodeId)));
6435
+ }, nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "getStreamWithCodec"));
5832
6436
  const outDir = path.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
5833
6437
  await promises.mkdir(outDir, { recursive: true });
5834
6438
  const deviceLog = this.deps.logger.withTags({ deviceId });
@@ -5844,6 +6448,7 @@ var RecordingController = class {
5844
6448
  this.recoverWriterGaveUp(deviceId);
5845
6449
  }
5846
6450
  });
6451
+ const writerStartedMs = this.now();
5847
6452
  writer.start();
5848
6453
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
5849
6454
  return {
@@ -5851,6 +6456,8 @@ var RecordingController = class {
5851
6456
  writer,
5852
6457
  watcher: startSegmentWatcher({
5853
6458
  outDir,
6459
+ writerStartedMs,
6460
+ now: () => this.now(),
5854
6461
  intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
5855
6462
  onFinalized: async (startMs, durMs, flatAbsPath) => {
5856
6463
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
@@ -5967,10 +6574,10 @@ var RecordingController = class {
5967
6574
  /** Tear EVERYTHING down: timer, restore subscription, every writer/watcher/lease. */
5968
6575
  async stop() {
5969
6576
  this.stopped = true;
5970
- if (this.tickTimer !== null) {
5971
- clearInterval(this.tickTimer);
5972
- this.tickTimer = null;
5973
- }
6577
+ this.livenessPass?.stop();
6578
+ this.livenessPass = null;
6579
+ this.convergePass?.stop();
6580
+ this.convergePass = null;
5974
6581
  this.restore?.stop();
5975
6582
  this.restore = null;
5976
6583
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
@@ -6573,6 +7180,7 @@ var RecorderV2Addon = class extends BaseAddon {
6573
7180
  resumeForStorageMigration: (leaseId) => this.resumeForStorageMigration(leaseId),
6574
7181
  refreshStorageLocationsForMigration: (leaseId) => this.refreshStorageLocationsForMigration(leaseId),
6575
7182
  assertStorageMigrationLease: (leaseId) => this.assertStorageMigrationLease(leaseId),
7183
+ placementSnapshot: () => this.placementSnapshot(),
6576
7184
  dataDir: this.ctx.dataDir,
6577
7185
  playbackBaseUrl: () => this.playbackBaseUrl,
6578
7186
  isCameraDevice: async (deviceId) => {
@@ -7012,6 +7620,28 @@ var RecorderV2Addon = class extends BaseAddon {
7012
7620
  await this.placement.recompute(work);
7013
7621
  }
7014
7622
  /**
7623
+ * The plan in force plus each write target's headroom, for the operator-armed
7624
+ * rebalance (D116, Phase 2). Both come from the SAME `PlacementService` the
7625
+ * 30 s tick plans with: a rebalance computed against a second idea of "what
7626
+ * fits" would plan moves the next tick undoes.
7627
+ *
7628
+ * Throws when placement is not active (a non-recording node, or early boot) —
7629
+ * the rebalance is an operator action and a silent empty plan would read as
7630
+ * "nothing to do".
7631
+ */
7632
+ async placementSnapshot() {
7633
+ const placement = this.placement;
7634
+ if (!placement) throw new Error("placement is not active on this node");
7635
+ const [assignments, candidates] = await Promise.all([placement.currentAssignments(), placement.candidates()]);
7636
+ return {
7637
+ assignments,
7638
+ headroom: candidates.map((candidate) => ({
7639
+ locationId: candidate.locationId,
7640
+ headroomBytes: candidate.headroomBytes
7641
+ }))
7642
+ };
7643
+ }
7644
+ /**
7015
7645
  * Measured bytes/day for one (camera, profile) — the demand signal the
7016
7646
  * placement planner weights against each location's headroom. Derived from
7017
7647
  * the index's own accounting (segment paths encode bytes + start), so it