@camstack/addon-pipeline 1.2.70 → 1.2.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/audio-analyzer/index.js +1 -1
  2. package/dist/audio-analyzer/index.mjs +1 -1
  3. package/dist/detection-pipeline/index.js +2 -2
  4. package/dist/detection-pipeline/index.mjs +2 -2
  5. package/dist/{dist-BPaa4_z6.mjs → dist-CNYUiN27.mjs} +158 -43
  6. package/dist/{dist-cR-pvD_z.js → dist-DwBdGpkx.js} +158 -43
  7. package/dist/{event-loop-stall-monitor-STRdnQWm.js → event-loop-stall-monitor-1o85rEbu.js} +1 -1
  8. package/dist/{event-loop-stall-monitor-B79REtL2.mjs → event-loop-stall-monitor-EvJwrf2f.mjs} +1 -1
  9. package/dist/motion-wasm/index.js +1 -1
  10. package/dist/motion-wasm/index.mjs +1 -1
  11. package/dist/pipeline-runner/index.js +95 -28
  12. package/dist/pipeline-runner/index.mjs +95 -28
  13. package/dist/recorder/index.js +365 -73
  14. package/dist/recorder/index.mjs +365 -73
  15. package/dist/session-decode/decode-worker-child.js +483 -52
  16. package/dist/session-decode/decode-worker-child.mjs +483 -52
  17. package/dist/stream-broker/_stub.js +660 -556
  18. package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-B-HGBNad.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BEBWZAKQ.mjs} +3 -3
  19. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DkUf57A0.mjs +26 -0
  20. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CExO0piw.mjs +26 -0
  21. package/dist/stream-broker/{hostInit-B12DHOv7.mjs → hostInit-aSIXKJAL.mjs} +3 -3
  22. package/dist/stream-broker/index.js +1 -1
  23. package/dist/stream-broker/index.mjs +1 -1
  24. package/dist/stream-broker/remoteEntry.js +1 -1
  25. package/dist/{worker-protocol-CBO8nwQj.mjs → worker-protocol-C_sGeitR.mjs} +26 -19
  26. package/dist/{worker-protocol-C7V1qIIA.js → worker-protocol-D0H-aG2_.js} +26 -19
  27. package/package.json +1 -1
  28. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-D1CqbPrT.mjs +0 -26
  29. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-NjvjJWz-.mjs +0 -26
@@ -1,4 +1,4 @@
1
- import { At as string, Dt as number, J as recordingExportCapability, Mt as EventCategory, N as deriveRecordingMode, Ot as object, St as array, Z as storageEvictableCapability, _ as OpsLogEntrySchema, b as RECORDING_EXPORT_MAX_READ_BYTES, c as DEFAULT_EVENTS_BAND_BUFFER_SEC, d as EVENT_PAD_MS, ft as hydrateSchema, gt as nodePin, kt as record, p as ExportRecordSchema, q as recordingCapability, st as BaseAddon, tt as errMsg, ut as DeviceType, x as RecordingConfigSchema, yt as selectAssignedProfileSlots } from "../dist-BPaa4_z6.mjs";
1
+ import { At as string, Dt as number, J as recordingExportCapability, Mt as EventCategory, N as deriveRecordingMode, Ot as object, St as array, Z as storageEvictableCapability, _ as OpsLogEntrySchema, b as RECORDING_EXPORT_MAX_READ_BYTES, c as DEFAULT_EVENTS_BAND_BUFFER_SEC, d as EVENT_PAD_MS, ft as hydrateSchema, gt as nodePin, kt as record, p as ExportRecordSchema, q as recordingCapability, st as BaseAddon, tt as errMsg, ut as DeviceType, x as RecordingConfigSchema, yt as selectAssignedProfileSlots } from "../dist-CNYUiN27.mjs";
2
2
  import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
3
3
  import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-CZ2xo67g.mjs";
4
4
  import { randomUUID } from "node:crypto";
@@ -2414,6 +2414,105 @@ function sendDirectory(res, payload) {
2414
2414
  res.end(Buffer.from(body));
2415
2415
  }
2416
2416
  //#endregion
2417
+ //#region src/recorder/addon/export-dense-map.ts
2418
+ /**
2419
+ * Shortest clip worth keeping, in ms.
2420
+ *
2421
+ * Exists only to honour `ExportDenseRangeSchema`'s `toSec > fromSec`: a range
2422
+ * clipped to a sliver of footage is still honest, but a range clipped to
2423
+ * NOTHING must not be emitted as a zero-length interval the schema refuses.
2424
+ */
2425
+ var MIN_MAPPED_MS = 1;
2426
+ /** Milliseconds of footage that precede `wallMs` in the concatenation. */
2427
+ function recordedMsBefore(segments, wallMs) {
2428
+ let acc = 0;
2429
+ for (const s of segments) if (wallMs >= s.startMs + s.durMs) acc += s.durMs;
2430
+ else if (wallMs > s.startMs) acc += wallMs - s.startMs;
2431
+ return acc;
2432
+ }
2433
+ /** Whole milliseconds → seconds, with no float tail for the filter string. */
2434
+ function msToSec(ms) {
2435
+ return Math.round(ms) / 1e3;
2436
+ }
2437
+ /**
2438
+ * Translate wall-clock dense ranges into the export's stream timeline.
2439
+ *
2440
+ * @returns the surviving ranges (in stream seconds, ordered as given) plus the
2441
+ * report the render logs.
2442
+ */
2443
+ function mapDenseRangesToStreamTime(input) {
2444
+ const segments = [...input.segments].sort((a, b) => a.startMs - b.startMs);
2445
+ const footageEndMs = segments.reduce((end, s) => Math.max(end, s.startMs + s.durMs), input.exportFromMs);
2446
+ const mapped = [];
2447
+ const dropped = [];
2448
+ for (const range of input.ranges) {
2449
+ const fromWallMs = input.exportFromMs + range.fromSec * 1e3;
2450
+ const toWallMs = input.exportFromMs + range.toSec * 1e3;
2451
+ const fromMs = Math.round(recordedMsBefore(segments, fromWallMs));
2452
+ const toMs = Math.round(recordedMsBefore(segments, toWallMs));
2453
+ if (toMs - fromMs < MIN_MAPPED_MS) {
2454
+ dropped.push({
2455
+ fromSec: range.fromSec,
2456
+ toSec: range.toSec,
2457
+ reason: fromWallMs >= footageEndMs ? "past-end" : "gap"
2458
+ });
2459
+ continue;
2460
+ }
2461
+ mapped.push({
2462
+ fromSec: msToSec(fromMs),
2463
+ toSec: msToSec(toMs)
2464
+ });
2465
+ }
2466
+ return {
2467
+ ranges: mapped,
2468
+ report: {
2469
+ requested: input.ranges.length,
2470
+ mapped: mapped.length,
2471
+ dropped
2472
+ }
2473
+ };
2474
+ }
2475
+ /**
2476
+ * One export's options with its dense ranges expressed in stream time.
2477
+ *
2478
+ * An overlay that maps to NOTHING is removed entirely rather than left with an
2479
+ * empty range list: `ExportDenseSchema` refuses `ranges: []` and
2480
+ * `selectPredicate` would emit `if(,…)`, a filter ffmpeg rejects — so a night
2481
+ * whose busy moments all fell in holes would fail the render instead of
2482
+ * delivering the uniform video it legitimately is.
2483
+ */
2484
+ function withStreamTimeDenseRanges(input) {
2485
+ const timelapse = input.options.timelapse;
2486
+ const dense = timelapse?.dense;
2487
+ if (timelapse === void 0 || dense === void 0) return {
2488
+ options: input.options,
2489
+ report: null
2490
+ };
2491
+ const { ranges, report } = mapDenseRangesToStreamTime({
2492
+ ranges: dense.ranges,
2493
+ exportFromMs: input.fromMs,
2494
+ segments: input.segments
2495
+ });
2496
+ const base = {
2497
+ everyMs: timelapse.everyMs,
2498
+ ...timelapse.outputFps !== void 0 ? { outputFps: timelapse.outputFps } : {}
2499
+ };
2500
+ const remapped = ranges.length > 0 ? {
2501
+ ...base,
2502
+ dense: {
2503
+ everyMs: dense.everyMs,
2504
+ ranges: [...ranges]
2505
+ }
2506
+ } : base;
2507
+ return {
2508
+ options: {
2509
+ ...input.options,
2510
+ timelapse: remapped
2511
+ },
2512
+ report
2513
+ };
2514
+ }
2515
+ //#endregion
2417
2516
  //#region src/recorder/addon/export-ffmpeg-args.ts
2418
2517
  /** Widest [min,max] speed a single ffmpeg `atempo` filter accepts. Outside this
2419
2518
  * band audio is dropped rather than chained (kept simple + predictable). */
@@ -2679,35 +2778,76 @@ var ExportEngine = class {
2679
2778
  return;
2680
2779
  }
2681
2780
  try {
2682
- await this.spawnRender(id, rec, playlist);
2781
+ await this.spawnRender(id, rec, playlist.path, this.streamTimeOptions(rec, playlist.segments));
2683
2782
  } finally {
2684
2783
  if (this.deps.cleanupPlaylist) try {
2685
- await this.deps.cleanupPlaylist(playlist);
2784
+ await this.deps.cleanupPlaylist(playlist.path);
2686
2785
  } catch (err) {
2687
- this.deps.logger.warn("export: source playlist cleanup failed", { meta: {
2688
- exportId: id,
2689
- playlist,
2690
- error: err instanceof Error ? err.message : String(err)
2691
- } });
2786
+ this.deps.logger.warn("export: source playlist cleanup failed", {
2787
+ tags: { deviceId: rec.deviceId },
2788
+ meta: {
2789
+ exportId: id,
2790
+ playlist: playlist.path,
2791
+ error: err instanceof Error ? err.message : String(err)
2792
+ }
2793
+ });
2692
2794
  }
2693
2795
  }
2694
2796
  }
2797
+ /**
2798
+ * The record's options with any dense overlay moved onto ffmpeg's clock.
2799
+ *
2800
+ * The translation is reported on EVERY render that carries an overlay, not
2801
+ * only when something is dropped: `denseRangesRequested` vs
2802
+ * `denseRangesMapped` is the one line that can tell "the video is uniform
2803
+ * because the night was quiet" from "the video is uniform because every range
2804
+ * missed" — the distinction that went unnoticed for a full night because the
2805
+ * only line printed was the requested count.
2806
+ */
2807
+ streamTimeOptions(rec, segments) {
2808
+ const { options, report } = withStreamTimeDenseRanges({
2809
+ options: rec.options,
2810
+ fromMs: rec.fromMs,
2811
+ segments
2812
+ });
2813
+ if (report !== null) this.logDenseMapping(rec, report);
2814
+ return options;
2815
+ }
2816
+ /** One line, whatever the outcome — `warn` when ranges were dropped, because
2817
+ * a dropped range is footage the operator asked for and did not get. */
2818
+ logDenseMapping(rec, report) {
2819
+ const entry = {
2820
+ tags: { deviceId: rec.deviceId },
2821
+ meta: {
2822
+ exportId: rec.id,
2823
+ denseRangesRequested: report.requested,
2824
+ denseRangesMapped: report.mapped,
2825
+ ...report.dropped.length > 0 ? { denseRangesDropped: report.dropped } : {}
2826
+ }
2827
+ };
2828
+ const message = "export dense ranges mapped onto the concatenated stream";
2829
+ if (report.dropped.length > 0) this.deps.logger.warn(message, entry);
2830
+ else this.deps.logger.info(message, entry);
2831
+ }
2695
2832
  /** Spawn ffmpeg for one render and resolve when it has exited + finalized. */
2696
- async spawnRender(id, rec, playlist) {
2833
+ async spawnRender(id, rec, playlist, options) {
2697
2834
  const outPath = this.deps.outPath(rec);
2698
2835
  const args = buildExportArgs({
2699
2836
  inputPlaylist: playlist,
2700
2837
  outPath,
2701
- options: rec.options
2838
+ options
2702
2839
  });
2703
2840
  const totalSec = expectedOutSeconds(rec);
2704
- this.deps.logger.info("export render starting", { meta: {
2705
- exportId: id,
2706
- deviceId: rec.deviceId,
2707
- profile: rec.profile,
2708
- playlist,
2709
- outPath
2710
- } });
2841
+ this.deps.logger.info("export render starting", {
2842
+ tags: { deviceId: rec.deviceId },
2843
+ meta: {
2844
+ exportId: id,
2845
+ deviceId: rec.deviceId,
2846
+ profile: rec.profile,
2847
+ playlist,
2848
+ outPath
2849
+ }
2850
+ });
2711
2851
  this.deps.logger.debug("export ffmpeg argv", { meta: {
2712
2852
  exportId: id,
2713
2853
  args
@@ -3890,9 +4030,9 @@ function exportSourcePlaylistPath(exportsDir, deviceId, exportId) {
3890
4030
  return path.join(exportsDir, String(deviceId), `${exportId}.src.m3u8`);
3891
4031
  }
3892
4032
  /**
3893
- * Write the export's source playlist and return its absolute path, or null when
3894
- * no segment of the requested profile is present on disk for the range (the
3895
- * engine then fails the job with 'no footage for range').
4033
+ * Write the export's source playlist and return it, or null when no segment of
4034
+ * the requested profile is present on disk for the range (the engine then fails
4035
+ * the job with 'no footage for range').
3896
4036
  */
3897
4037
  async function writeExportSourcePlaylist(input) {
3898
4038
  const { segments } = await collectRangeSegments(input.deps, input.deviceId, input.profile, input.fromMs, input.toMs, "absolute");
@@ -3900,26 +4040,16 @@ async function writeExportSourcePlaylist(input) {
3900
4040
  const playlistPath = exportSourcePlaylistPath(input.exportsDir, input.deviceId, input.exportId);
3901
4041
  await promises.mkdir(path.dirname(playlistPath), { recursive: true });
3902
4042
  await promises.writeFile(playlistPath, buildVariantPlaylist(segments), "utf8");
3903
- return playlistPath;
4043
+ return {
4044
+ path: playlistPath,
4045
+ segments: segments.map((s) => ({
4046
+ startMs: s.startMs,
4047
+ durMs: s.durMs
4048
+ }))
4049
+ };
3904
4050
  }
3905
4051
  //#endregion
3906
4052
  //#region src/recorder/addon/footage-render.ts
3907
- /**
3908
- * One-shot media render FROM RECORDED FOOTAGE (notification attachments).
3909
- *
3910
- * Shared core behind `renderGif` and `renderClip`: both cut the same window out
3911
- * of the same `low` profile through the export machinery's per-render source
3912
- * playlist (absolute segment URIs — the ONLY playlist shape ffmpeg can consume,
3913
- * see export-source-playlist.ts), pipe it through ONE ffmpeg invocation, and
3914
- * return the bytes. Everything is transient: playlist and output are deleted on
3915
- * every outcome.
3916
- *
3917
- * Fail-closed by construction: no footage covering the window ⇒ it throws, and
3918
- * the caller (a notification rule) simply ships no attachment. A camera that is
3919
- * not recording therefore cannot produce a clip — which is exactly why the
3920
- * frame-ring plane (roadmap Phase 5) still matters for cameras with no
3921
- * recording. It is NOT a prerequisite for the cameras that do record.
3922
- */
3923
4053
  var RENDER_TIMEOUT_MS = 3e4;
3924
4054
  /**
3925
4055
  * A notification clip must sit AROUND the moment it describes. Sliding the
@@ -4017,9 +4147,9 @@ async function renderFootage(renderDeps, input, ext, buildArgs) {
4017
4147
  toMs: window.toMs
4018
4148
  });
4019
4149
  if (playlist === null) throw new Error(`no footage covers [${window.fromMs}, ${window.toMs}) for device ${input.deviceId}`);
4020
- const outPath = path.join(path.dirname(playlist), `${renderId}.${ext}`);
4150
+ const outPath = path.join(path.dirname(playlist.path), `${renderId}.${ext}`);
4021
4151
  try {
4022
- await runFfmpeg(renderDeps, buildArgs(playlist, outPath), ext);
4152
+ await runFfmpeg(renderDeps, buildArgs(playlist.path, outPath), ext);
4023
4153
  const bytes = await promises.readFile(outPath);
4024
4154
  if (bytes.byteLength === 0) throw new Error(`${ext} render produced an empty file`);
4025
4155
  return {
@@ -4028,7 +4158,7 @@ async function renderFootage(renderDeps, input, ext, buildArgs) {
4028
4158
  toMs: window.toMs
4029
4159
  };
4030
4160
  } finally {
4031
- await promises.rm(playlist, { force: true }).catch(() => {});
4161
+ await promises.rm(playlist.path, { force: true }).catch(() => {});
4032
4162
  await promises.rm(outPath, { force: true }).catch(() => {});
4033
4163
  }
4034
4164
  }
@@ -5272,6 +5402,49 @@ function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers) {
5272
5402
  return false;
5273
5403
  }
5274
5404
  //#endregion
5405
+ //#region src/recorder/addon/periodic-pass.ts
5406
+ function startPeriodicPass(deps) {
5407
+ const now = () => deps.now?.() ?? Date.now();
5408
+ let startedAt = null;
5409
+ let stopped = false;
5410
+ const fire = () => {
5411
+ if (stopped) return;
5412
+ if (startedAt !== null) {
5413
+ deps.logger.warn("recorder: pass skipped — the previous pass is still running", { meta: {
5414
+ pass: deps.name,
5415
+ inFlightMs: now() - startedAt,
5416
+ intervalMs: deps.intervalMs
5417
+ } });
5418
+ return;
5419
+ }
5420
+ const began = now();
5421
+ startedAt = began;
5422
+ deps.run().catch((err) => {
5423
+ deps.logger.warn("recorder: pass failed", { meta: {
5424
+ pass: deps.name,
5425
+ error: err instanceof Error ? err.message : String(err)
5426
+ } });
5427
+ }).finally(() => {
5428
+ const durationMs = now() - began;
5429
+ startedAt = null;
5430
+ if (durationMs > deps.slowAfterMs) deps.logger.warn("recorder: pass exceeded its budget", { meta: {
5431
+ pass: deps.name,
5432
+ durationMs,
5433
+ budgetMs: deps.slowAfterMs
5434
+ } });
5435
+ });
5436
+ };
5437
+ const timer = setInterval(fire, deps.intervalMs);
5438
+ timer.unref?.();
5439
+ return {
5440
+ stop() {
5441
+ stopped = true;
5442
+ clearInterval(timer);
5443
+ },
5444
+ isRunning: () => startedAt !== null
5445
+ };
5446
+ }
5447
+ //#endregion
5275
5448
  //#region src/recorder/addon/readiness-restore.ts
5276
5449
  var ReadinessRestore = class {
5277
5450
  deps;
@@ -5358,10 +5531,59 @@ var ReadinessRestore = class {
5358
5531
  * no subtree to classify.
5359
5532
  */
5360
5533
  var EPOCH_NAME_RE = /^(\d+)\.m4s$/;
5361
- /** How many consecutive ticks to wait for a finalized segment's flat file to
5362
- * land before giving up on it (a genuine phantom). At a 2s watch interval this
5363
- * is ~16s far longer than ffmpeg's slow-first-segment flush. */
5364
- var MAX_PENDING_TICKS = 8;
5534
+ /**
5535
+ * How long a finalized segment's flat file may take to appear before the entry
5536
+ * is given up on.
5537
+ *
5538
+ * This used to be `MAX_PENDING_TICKS = 8` — a count of TICKS, which is not a
5539
+ * measure of time. The tick loop self-excludes while a pass is running and each
5540
+ * pass does real filesystem work, so on 2026-08-10..13 eight ticks measured
5541
+ * anywhere from 8 s to 819 s. A bound has to be denominated in the thing it
5542
+ * bounds.
5543
+ *
5544
+ * 30 s is chosen from the measurement, not from taste. Over four days the
5545
+ * segments that were retried and then DID land waited p50 7.8 s and p90 11.6 s,
5546
+ * and widening the window from 16 s to 120 s moved coverage only from 92.9% to
5547
+ * 93.8%. There is no tail to chase: past this knee a segment is not slow, it is
5548
+ * lost (a writer killed mid-segment), and waiting only delays the skip and the
5549
+ * log line that reports it.
5550
+ */
5551
+ var PENDING_GRACE_MS = 3e4;
5552
+ /**
5553
+ * How far before the writer's own start a segment may be named and still be
5554
+ * treated as belonging to THIS run.
5555
+ *
5556
+ * ffmpeg names segments with whole-second `%s`, so the first segment of a run
5557
+ * carries the second that was already in progress when the process spawned —
5558
+ * up to ~1 s "before" the writer started. Without this tolerance the very first
5559
+ * segment of every recording would be classified as a previous run's leftover
5560
+ * and dropped.
5561
+ */
5562
+ var PREVIOUS_RUN_SKEW_MS = 1e3;
5563
+ /**
5564
+ * Decide whether a missing flat file is a late flush or a previous run's entry.
5565
+ *
5566
+ * ffmpeg's `-segment_list` file is not cleared between runs and a fresh watcher
5567
+ * starts at `processed = 0`, so the first passes after every attach replay the
5568
+ * PREVIOUS run's playlist. Those entries were relocated into the bucket tree
5569
+ * long ago; their absence is the system working. Over 2026-08-10..13 they were
5570
+ * 83% of all retries (median age 1 615 s, max 17.4 h) and produced 68 of the
5571
+ * 122 `never landed` warnings — a warning that reads like data loss for a file
5572
+ * that was correctly filed hours earlier.
5573
+ *
5574
+ * Note what this deliberately does NOT do: it does not skip a previous run's
5575
+ * entry whose file is STILL THERE. That case is real crash recovery — an
5576
+ * unclean stop leaves un-relocated segments in `.rec-tmp` and the playlist is
5577
+ * the only record of their durations — so the caller stats first and only asks
5578
+ * this question once the file is known to be absent.
5579
+ */
5580
+ function classifyMissingEntry(input) {
5581
+ return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS ? "stale-previous-run" : "pending";
5582
+ }
5583
+ /** Whether a pending entry has outlived its grace. Pure — the caller clocks it. */
5584
+ function shouldGiveUpOnPending(input) {
5585
+ return input.waitedMs > input.boundMs;
5586
+ }
5365
5587
  /**
5366
5588
  * Derive `startMs` (epoch milliseconds) from a flat ffmpeg segment path
5367
5589
  * (`<epochSec>.m4s`, possibly absolute). Returns null for any non-epoch path.
@@ -5436,11 +5658,22 @@ async function statSize(p) {
5436
5658
  * Unlike the old recorder, this watcher does NOT relocate or re-stat: the
5437
5659
  * SegmentStore owns the move + the authoritative byte count.
5438
5660
  */
5439
- async function handleSegmentEntry(outDir, entry, durMs, onFinalized, logger) {
5661
+ async function handleSegmentEntry(outDir, entry, durMs, onFinalized, logger, writerStartedMs) {
5440
5662
  const flatAbsPath = path.isAbsolute(entry.segPath) ? entry.segPath : path.join(outDir, entry.segPath);
5441
5663
  const startMs = parseEpochStartMs(flatAbsPath);
5442
5664
  if (startMs === null) return "skipped";
5443
5665
  if (await statSize(flatAbsPath) === null) {
5666
+ if (classifyMissingEntry({
5667
+ segmentStartMs: startMs,
5668
+ writerStartedMs
5669
+ }) === "stale-previous-run") {
5670
+ logger.debug("segment entry predates this writer — already relocated, skipping", { meta: {
5671
+ path: flatAbsPath,
5672
+ segmentStartMs: startMs,
5673
+ writerStartedMs
5674
+ } });
5675
+ return "skipped";
5676
+ }
5444
5677
  logger.debug("segment flat file not on disk yet — will retry", { meta: { path: flatAbsPath } });
5445
5678
  return "pending";
5446
5679
  }
@@ -5486,12 +5719,13 @@ function planFinalizations(body, processed, forceFinal = false) {
5486
5719
  */
5487
5720
  function startSegmentWatcher(deps) {
5488
5721
  const playlistPath = path.join(deps.outDir, "live.m3u8");
5722
+ const now = () => deps.now?.() ?? Date.now();
5489
5723
  let processed = 0;
5490
5724
  let stopped = false;
5491
5725
  let ticking = false;
5492
5726
  let currentTick = null;
5493
5727
  let pendingIndex = -1;
5494
- let pendingTicks = 0;
5728
+ let pendingSinceMs = 0;
5495
5729
  const tick = async (forceFinal = false) => {
5496
5730
  if (stopped || ticking) return;
5497
5731
  ticking = true;
@@ -5505,19 +5739,26 @@ function startSegmentWatcher(deps) {
5505
5739
  const plan = planFinalizations(body, processed, forceFinal);
5506
5740
  let index = plan.from;
5507
5741
  for (const { entry, durMs } of plan.toHandle) {
5508
- if (await handleSegmentEntry(deps.outDir, entry, durMs, deps.onFinalized, deps.logger) === "pending") {
5742
+ if (await handleSegmentEntry(deps.outDir, entry, durMs, deps.onFinalized, deps.logger, deps.writerStartedMs) === "pending") {
5509
5743
  if (pendingIndex === index) {
5510
- pendingTicks++;
5511
- if (pendingTicks >= MAX_PENDING_TICKS) {
5512
- deps.logger.warn("segment file never landed — skipping", { meta: { seg: entry.segPath } });
5744
+ const waitedMs = now() - pendingSinceMs;
5745
+ if (shouldGiveUpOnPending({
5746
+ waitedMs,
5747
+ boundMs: 3e4
5748
+ })) {
5749
+ deps.logger.warn("segment file never landed — skipping", { meta: {
5750
+ seg: entry.segPath,
5751
+ waitedMs,
5752
+ boundMs: PENDING_GRACE_MS
5753
+ } });
5513
5754
  pendingIndex = -1;
5514
- pendingTicks = 0;
5755
+ pendingSinceMs = 0;
5515
5756
  index++;
5516
5757
  continue;
5517
5758
  }
5518
5759
  } else {
5519
5760
  pendingIndex = index;
5520
- pendingTicks = 1;
5761
+ pendingSinceMs = now();
5521
5762
  }
5522
5763
  break;
5523
5764
  }
@@ -5525,7 +5766,7 @@ function startSegmentWatcher(deps) {
5525
5766
  }
5526
5767
  if (pendingIndex !== index) {
5527
5768
  pendingIndex = -1;
5528
- pendingTicks = 0;
5769
+ pendingSinceMs = 0;
5529
5770
  }
5530
5771
  processed = index;
5531
5772
  } finally {
@@ -5726,9 +5967,16 @@ var SegmentWriter = class {
5726
5967
  * (`shouldRecordContinuousAt`) and ensures the writer set is running iff so.
5727
5968
  * It re-evaluates on three triggers:
5728
5969
  * 1. `setDeviceConfig` (operator changed the bands / enable),
5729
- * 2. a periodic tick (catches band boundaries crossed with no config change),
5970
+ * 2. a periodic pass (catches band boundaries crossed with no config change),
5730
5971
  * 3. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`).
5731
5972
  *
5973
+ * There are TWO periodic passes, on separate timers and never overlapping
5974
+ * themselves (`periodic-pass.ts`): a cheap LIVENESS pass that catches writers
5975
+ * pinned but producing nothing, and an expensive CONVERGENCE pass that
5976
+ * re-evaluates bands and placement. They were one pass until 2026-08-13, when a
5977
+ * convergence sweep wedged on unbounded broker RPCs and took the liveness
5978
+ * watchdog down with it for 9-27 minutes at a time.
5979
+ *
5732
5980
  * EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
5733
5981
  * trigger-gating. The enable intent is persisted in durable-state by the
5734
5982
  * config-store, so `restoreEnabledDevices` on boot re-attaches every device
@@ -5758,6 +6006,34 @@ var TICK_MS = 3e4;
5758
6006
  */
5759
6007
  var RELEASE_RPC_TIMEOUT_MS = 2e3;
5760
6008
  /**
6009
+ * Bound on each broker RPC in the ATTACH chain (`listAllProfileSlots`,
6010
+ * `getStreamWithCodec`).
6011
+ *
6012
+ * These were the only unbounded calls left in the controller.
6013
+ * `CapabilityHandle.call` gates READINESS (10 s) and then returns `fn()`
6014
+ * untouched, so an accepted-but-unanswered request fell through to the 60 s UDS
6015
+ * default in `kernel/transport/socket-channel.ts`. On 2026-08-13 the broker was
6016
+ * degraded (516 UDS timeouts that day) and the arithmetic did the damage: one
6017
+ * device costs `listAllProfileSlots` + one `getStreamWithCodec` per profile, so
6018
+ * a 4-camera fleet on 3 profiles reached ~16 minutes inside a single recovery
6019
+ * pass — during which every device sits DETACHED (recording stopped) and in
6020
+ * `attaching` (so `evaluateDevice` returns early and the liveness watchdog,
6021
+ * which reads `active`, has nothing to report). That is the shape of the 9-27
6022
+ * minute silences and the real recording gaps behind the 80% timelapse
6023
+ * coverage.
6024
+ *
6025
+ * 20 s is generous for a real RTSP dial and far below the UDS default, so a
6026
+ * hung broker now fails the attach fast. Nothing is lost by giving up: the
6027
+ * device stays on the readiness-restore queue and the next pass retries it.
6028
+ */
6029
+ var ATTACH_RPC_TIMEOUT_MS = 2e4;
6030
+ /**
6031
+ * A pass longer than this is reported with its duration. Equal to the pass
6032
+ * cadence: a pass that cannot finish inside its own interval is the condition
6033
+ * that used to go unlogged for half an hour.
6034
+ */
6035
+ var PASS_SLOW_AFTER_MS = TICK_MS;
6036
+ /**
5761
6037
  * Race `promise` against a deadline. Handlers stay attached to the losing
5762
6038
  * promise, so a post-deadline settlement can never surface as a process-level
5763
6039
  * `unhandledRejection`.
@@ -5845,7 +6121,10 @@ var RecordingController = class {
5845
6121
  */
5846
6122
  unrecordableReported = /* @__PURE__ */ new Set();
5847
6123
  restore = null;
5848
- tickTimer = null;
6124
+ /** Cheap, never-delayed: is any pinned writer producing nothing? */
6125
+ livenessPass = null;
6126
+ /** Expensive: re-evaluate bands + placement for every configured device. */
6127
+ convergePass = null;
5849
6128
  stopped = false;
5850
6129
  /** A reversible maintenance lease. Unlike `stopped`, it never changes
5851
6130
  * persisted recording intent and `resume()` restarts normal convergence. */
@@ -5880,18 +6159,28 @@ var RecordingController = class {
5880
6159
  meta: { error: errMsg(err) }
5881
6160
  })
5882
6161
  });
5883
- if (this.tickTimer === null) {
5884
- this.tickTimer = setInterval(() => {
5885
- this.tick();
5886
- }, TICK_MS);
5887
- this.tickTimer.unref?.();
6162
+ if (this.convergePass === null) {
6163
+ this.livenessPass = startPeriodicPass({
6164
+ name: "liveness",
6165
+ intervalMs: TICK_MS,
6166
+ slowAfterMs: PASS_SLOW_AFTER_MS,
6167
+ logger: this.deps.logger,
6168
+ now: () => this.now(),
6169
+ run: () => this.checkIdleWriters()
6170
+ });
6171
+ this.convergePass = startPeriodicPass({
6172
+ name: "convergence",
6173
+ intervalMs: TICK_MS,
6174
+ slowAfterMs: PASS_SLOW_AFTER_MS,
6175
+ logger: this.deps.logger,
6176
+ now: () => this.now(),
6177
+ run: () => this.converge()
6178
+ });
5888
6179
  }
5889
6180
  await this.restore.start(ids.map((id) => [id, true]));
5890
6181
  }
5891
- /** Re-evaluate every currently-tracked OR active device on the periodic tick. */
5892
- async tick() {
5893
- if (this.stopped) return;
5894
- await this.checkIdleWriters();
6182
+ /** Re-evaluate every currently-tracked OR active device on the periodic pass. */
6183
+ async converge() {
5895
6184
  if (this.stopped) return;
5896
6185
  const ids = new Set(this.active.keys());
5897
6186
  let persisted = [];
@@ -5907,7 +6196,7 @@ var RecordingController = class {
5907
6196
  for (const id of ids) try {
5908
6197
  await this.evaluateDevice(id);
5909
6198
  } catch (err) {
5910
- this.deps.logger.warn("recorder controller: tick evaluate failed", {
6199
+ this.deps.logger.warn("recorder controller: convergence evaluate failed", {
5911
6200
  tags: { deviceId: id },
5912
6201
  meta: { error: errMsg(err) }
5913
6202
  });
@@ -5981,7 +6270,7 @@ var RecordingController = class {
5981
6270
  segmentSeconds: this.active.get(s.deviceId)?.find((r) => r.profile === s.profile)?.segmentSeconds
5982
6271
  }
5983
6272
  });
5984
- for (const deviceId of devices) await this.recoverWriterGaveUp(deviceId);
6273
+ await Promise.all([...devices].map((deviceId) => this.recoverWriterGaveUp(deviceId)));
5985
6274
  }
5986
6275
  /** Current trigger state for a device (cold default = no triggers seen). */
5987
6276
  triggersFor(deviceId) {
@@ -6123,7 +6412,7 @@ var RecordingController = class {
6123
6412
  * assigned sources is ignored (record every assigned source — minimum of 1).
6124
6413
  */
6125
6414
  async resolveProfiles(deviceId, override) {
6126
- const assigned = selectAssignedProfileSlots(await this.deps.brokerCall(() => this.deps.api.streamBroker.listAllProfileSlots.query(void 0, nodePin(this.deps.ownerNodeId))), deviceId).map((slot) => slot.profile);
6415
+ const assigned = selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile);
6127
6416
  if (!override || override.length === 0) return assigned;
6128
6417
  const selected = assigned.filter((p) => override.includes(p));
6129
6418
  return selected.length > 0 ? selected : assigned;
@@ -6136,14 +6425,14 @@ var RecordingController = class {
6136
6425
  */
6137
6426
  async attachProfile(deviceId, profile, segmentSeconds) {
6138
6427
  const placement = await this.resolvePlacement(deviceId, profile);
6139
- const source = await this.deps.brokerCall(() => this.deps.api.streamBroker.getStreamWithCodec.mutate({
6428
+ const source = await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.getStreamWithCodec.mutate({
6140
6429
  deviceId,
6141
6430
  video: "copy",
6142
6431
  audio: "aac",
6143
6432
  profile,
6144
6433
  ...this.deps.consumerHostname !== void 0 ? { hostname: this.deps.consumerHostname } : {},
6145
6434
  tag: `recorder:${deviceId}/${profile}`
6146
- }, nodePin(this.deps.ownerNodeId)));
6435
+ }, nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "getStreamWithCodec"));
6147
6436
  const outDir = path.join(placement.root, STAGING_DIR_NAME, String(deviceId), profile);
6148
6437
  await promises.mkdir(outDir, { recursive: true });
6149
6438
  const deviceLog = this.deps.logger.withTags({ deviceId });
@@ -6159,6 +6448,7 @@ var RecordingController = class {
6159
6448
  this.recoverWriterGaveUp(deviceId);
6160
6449
  }
6161
6450
  });
6451
+ const writerStartedMs = this.now();
6162
6452
  writer.start();
6163
6453
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
6164
6454
  return {
@@ -6166,6 +6456,8 @@ var RecordingController = class {
6166
6456
  writer,
6167
6457
  watcher: startSegmentWatcher({
6168
6458
  outDir,
6459
+ writerStartedMs,
6460
+ now: () => this.now(),
6169
6461
  intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
6170
6462
  onFinalized: async (startMs, durMs, flatAbsPath) => {
6171
6463
  this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
@@ -6282,10 +6574,10 @@ var RecordingController = class {
6282
6574
  /** Tear EVERYTHING down: timer, restore subscription, every writer/watcher/lease. */
6283
6575
  async stop() {
6284
6576
  this.stopped = true;
6285
- if (this.tickTimer !== null) {
6286
- clearInterval(this.tickTimer);
6287
- this.tickTimer = null;
6288
- }
6577
+ this.livenessPass?.stop();
6578
+ this.livenessPass = null;
6579
+ this.convergePass?.stop();
6580
+ this.convergePass = null;
6289
6581
  this.restore?.stop();
6290
6582
  this.restore = null;
6291
6583
  await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));