@camstack/addon-pipeline-orchestrator 1.2.50 → 1.2.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -13865,7 +13865,23 @@ var NotificationActionSchema = object({
13865
13865
  * else — see `notification-center/action-token.ts` for what that does and
13866
13866
  * does not buy.
13867
13867
  */
13868
- destructive: boolean().optional()
13868
+ destructive: boolean().optional(),
13869
+ /**
13870
+ * How the tap should REACH the url.
13871
+ *
13872
+ * `navigate` (absent, and every button authored before this field) opens it:
13873
+ * the phone leaves the notification and shows whatever the callback returns.
13874
+ * That is right for a button whose answer the operator wants to read.
13875
+ *
13876
+ * `background` fires it as a POST and stays put. It exists for the buttons
13877
+ * whose whole point is not to interrupt — "silence this for 30 minutes" is
13878
+ * an answer to the notification, and being thrown into a browser tab to
13879
+ * confirm it costs more attention than the notification did. A backend that
13880
+ * cannot do a background call renders it as an ordinary link (the adapters
13881
+ * fall back rather than dropping the button), so this is a preference, never
13882
+ * a requirement.
13883
+ */
13884
+ mode: _enum(["navigate", "background"]).optional()
13869
13885
  });
13870
13886
  /**
13871
13887
  * The canonical notification. `body` is the only hard field (Apprise model).
@@ -14866,6 +14882,9 @@ var NcSystemEventConditionSchema = object({
14866
14882
  nodeIds: array(string().min(1)).min(1).optional(),
14867
14883
  packageNames: array(string().min(1)).min(1).optional()
14868
14884
  });
14885
+ /** Hard ceiling on a window (24h). A snooze that could not expire would be an
14886
+ * outage the operator asked for once and forgot. */
14887
+ var NC_SNOOZE_MAX_MINUTES = 1440;
14869
14888
  /** Weekly schedule — OR of windows; absence on the rule = always active. */
14870
14889
  var NcScheduleSchema = object({
14871
14890
  windows: array(object({
@@ -15242,15 +15261,15 @@ var NcConditionsSchema = object({
15242
15261
  * (an `immediate` rule naming an `audio-*` class, one notification per
15243
15262
  * classified sample) stays exactly as it was for rules that already use it.
15244
15263
  *
15245
- * NOT in {@link NC_CONDITION_CATALOG} yet, and that is the sequencing rule
15246
- * rather than an oversight: the viewer mirrors the descriptor enums BY HAND
15247
- * (`camstack/src/data/notification-center.ts`, guarded by
15248
- * `scripts/check-viewer-condition-mirror.ts`) and its rule editor STRIPS the
15264
+ * In {@link NC_CONDITION_CATALOG} since P2, and the ORDER it got there is the
15265
+ * rule rather than an accident: the viewer mirrors the descriptor enums BY
15266
+ * HAND (`camstack/src/data/notification-center.ts`, guarded by
15267
+ * `scripts/check-viewer-condition-mirror.ts`) and its rule editor strips the
15249
15268
  * condition fields it does not know when a rule is saved from the phone.
15250
15269
  * Publishing an editor for a condition the app cannot round-trip is how an
15251
- * operator loses a rule's conditions by opening it — so the descriptor, the
15252
- * admin widget and the viewer mirror land together (P2 + P3), and only then
15253
- * does an audio rule become authorable.
15270
+ * operator loses a rule's conditions by opening it — so the viewer mirror
15271
+ * (P3, shipped) went FIRST, and the descriptor an editor renders from
15272
+ * follows here.
15254
15273
  */
15255
15274
  audio: NcAudioConditionSchema.optional()
15256
15275
  });
@@ -15486,6 +15505,30 @@ var NcRuleInputSchema = object({
15486
15505
  */
15487
15506
  snoozeAllowGlobal: boolean().optional(),
15488
15507
  /**
15508
+ * The snooze durations THIS rule's notification offers as buttons, in
15509
+ * minutes.
15510
+ *
15511
+ * Three states, and all three are distinct — which is exactly why this is
15512
+ * `.optional()` and never `.default()`. A Zod default does not run on the
15513
+ * addon cap path (three production failures in one day), so a schema default
15514
+ * would collapse the first two:
15515
+ *
15516
+ * | value | meaning |
15517
+ * | --- | --- |
15518
+ * | absent | the operator never said ⇒ {@link NC_DEFAULT_SNOOZE_MINUTES} |
15519
+ * | `[]` | **no snooze buttons on this rule** — the explicit override |
15520
+ * | a list | these choices, de-duplicated and sorted, at most four |
15521
+ *
15522
+ * `.max(4)` because the notifier's own action budget is small (ntfy allows
15523
+ * three buttons in total) and a rule that spent it all on snooze choices
15524
+ * would push its own tap-through actions off the notification.
15525
+ *
15526
+ * An empty list is NOT an alarm exemption: a rule the alarm is about, or
15527
+ * that arms the panel, is exempt automatically and cannot be silenced by a
15528
+ * window from anywhere (D133).
15529
+ */
15530
+ snoozeOptions: array(number().int().min(1).max(NC_SNOOZE_MAX_MINUTES)).max(4).optional(),
15531
+ /**
15489
15532
  * Devices this rule ACTUATES — arm the alarm, open a gate, turn on a light.
15490
15533
  *
15491
15534
  * This is what makes the rule set the alarm's trigger set without the alarm
@@ -15585,6 +15628,7 @@ var NcConditionDescriptorSchema = object({
15585
15628
  "device",
15586
15629
  "package",
15587
15630
  "occupancy",
15631
+ "audio",
15588
15632
  "system"
15589
15633
  ]),
15590
15634
  label: string(),
@@ -15603,6 +15647,7 @@ var NcConditionDescriptorSchema = object({
15603
15647
  "crossingSelect",
15604
15648
  "polygonDraw",
15605
15649
  "occupancy",
15650
+ "audio",
15606
15651
  "deviceState",
15607
15652
  "systemEvent"
15608
15653
  ]),
@@ -15754,7 +15799,20 @@ var NcSnoozeInputSchema = object({
15754
15799
  ruleId: string().optional(),
15755
15800
  /** Required when `scope: 'device'`. */
15756
15801
  deviceId: number().int().optional(),
15757
- durationMinutes: number().int().min(1).max(1440),
15802
+ /**
15803
+ * Narrow the window to these subject classes — "the cat, not the person".
15804
+ *
15805
+ * ORTHOGONAL to `scope`, deliberately, and absent means EVERY class: that is
15806
+ * what every window authored before this field meant, so no persisted row
15807
+ * changes meaning and no client has to learn anything to keep working.
15808
+ *
15809
+ * It is what makes the window's real key `(deviceId, classes[])` and lets it
15810
+ * cross rules (D133): the operator points at a camera and a kind of thing,
15811
+ * not at whichever of their four rules happened to produce the notification
15812
+ * they are dismissing.
15813
+ */
15814
+ classes: array(string().min(1)).min(1).optional(),
15815
+ durationMinutes: number().int().min(1).max(NC_SNOOZE_MAX_MINUTES),
15758
15816
  /**
15759
15817
  * Silence this for EVERY recipient, not just the caller. Permission is
15760
15818
  * checked server-side (the rule's `snoozeAllowGlobal`, or admin for the
@@ -15779,6 +15837,10 @@ var NcSnoozeSchema = object({
15779
15837
  scope: NcSnoozeScopeSchema,
15780
15838
  ruleId: string().optional(),
15781
15839
  deviceId: number().int().optional(),
15840
+ /** Subject classes this window covers. ABSENT = every class — see
15841
+ * {@link NcSnoozeInputSchema.shape.classes}. Lives in the JSON blob and has
15842
+ * no SQLite column: nothing queries a window by class. */
15843
+ classes: array(string().min(1)).min(1).optional(),
15782
15844
  startedAt: number(),
15783
15845
  /** Exclusive: at exactly this instant the snooze is over. Expiry is a
15784
15846
  * COMPARISON, not a job — no sweeper can leave the operator silenced. */
@@ -18031,7 +18093,14 @@ var zonesCapability = {
18031
18093
  * handle. Slice shape is `{ zones: Zone[] }` so future extensions
18032
18094
  * (e.g. zone groupings) can sit alongside the polygon list.
18033
18095
  */
18034
- runtimeState: object({ zones: array(ZoneSchema).readonly() })
18096
+ runtimeState: object({ zones: array(ZoneSchema).readonly() }),
18097
+ /**
18098
+ * Runtime-state durability: **restored** — written only on operator mutation, so a camera that never had one has nothing to re-derive from. This is the slice `zone-mirror-hydration.ts` exists to paper over.
18099
+ *
18100
+ * See `RuntimeStateDurability`. Enforced by
18101
+ * `scripts/check-runtime-state-durability.ts`.
18102
+ */
18103
+ durability: "restored"
18035
18104
  };
18036
18105
  /**
18037
18106
  * A bounding box in NORMALIZED [0,1] frame coordinates for `getNativeCrop`. The
@@ -25119,13 +25188,24 @@ method(object({
25119
25188
  /** Playback-speed multiplier for the render (1 = realtime). */
25120
25189
  var ExportSpeedSchema = number().min(.25).max(32);
25121
25190
  /**
25122
- * One dense interval, in SECONDS FROM THE EXPORT'S OWN `fromMs`.
25191
+ * One dense interval, in WALL-CLOCK SECONDS FROM THE EXPORT'S OWN `fromMs`.
25123
25192
  *
25124
- * Relative and not absolute epoch on purpose: the renderer's frame-select
25125
- * expression sees ffmpeg's `t`, which starts at 0 for the export's source
25126
- * playlist. Handing it absolute epochs would make every call site responsible
25127
- * for the same subtraction, and the one that forgot would emit a filter that
25128
- * selects nothing silently, as a uniform timelapse.
25193
+ * **Wall clock, not ffmpeg's `t`** and the recorder translates. A caller
25194
+ * derives these bounds from things that happened at a TIME (a track's
25195
+ * `firstSeen`), while `t` runs over the source playlist: the concatenation of
25196
+ * every segment present for the range, with each recording GAP removed. The
25197
+ * two agree only on a window that recorded without one interruption, and only
25198
+ * the render side knows the segments, so the translation lives there
25199
+ * (`export-dense-map.ts`, addon-pipeline).
25200
+ *
25201
+ * It was not always so. These seconds were fed to `between(t,…)` verbatim, and
25202
+ * on a 10 h window holding 29,393 s of footage every range landed late by the
25203
+ * gap accumulated before it — up to 6,607 s, well past EOF. Nothing matched,
25204
+ * the video was a uniform timelapse, and the log line reported the five ranges
25205
+ * that had been ASKED for (2026-08-13, export `57d14363`, camera 615).
25206
+ *
25207
+ * Relative and not absolute epoch, because an absolute epoch would make every
25208
+ * call site responsible for the same subtraction.
25129
25209
  */
25130
25210
  var ExportDenseRangeSchema = object({
25131
25211
  fromSec: number().nonnegative(),
@@ -26374,7 +26454,14 @@ var zoneRulesCapability = {
26374
26454
  motion: array(ZoneRuleSchema).readonly(),
26375
26455
  detection: array(ZoneRuleSchema).readonly(),
26376
26456
  package: array(ZoneRuleSchema).readonly()
26377
- })
26457
+ }),
26458
+ /**
26459
+ * Runtime-state durability: **restored** — operator intent, mutation-only, same argument as `zones`.
26460
+ *
26461
+ * See `RuntimeStateDurability`. Enforced by
26462
+ * `scripts/check-runtime-state-durability.ts`.
26463
+ */
26464
+ durability: "restored"
26378
26465
  };
26379
26466
  /**
26380
26467
  * Accessory device helpers — shared across drivers.
@@ -32055,6 +32142,7 @@ Object.freeze({
32055
32142
  "network-access": "ingress",
32056
32143
  "smtp-provider": "email"
32057
32144
  });
32145
+ new Map(AUDIO_MACRO_LABELS.flatMap((macro) => macro.icon === void 0 ? [] : [[macro.id, macro.icon]]));
32058
32146
  new Set(["devices", "classes"]);
32059
32147
  /**
32060
32148
  * TimelapseRule — the STANDALONE scheduled timelapse producer's rule model.
@@ -32367,8 +32455,8 @@ var DETAIL_CROP_PADDING_FIELD = {
32367
32455
  default: DEFAULT_DETAIL_CROP_CONVENTION.paddingRatio
32368
32456
  };
32369
32457
  /**
32370
- * THE native-frame **lease** knobs — TTL, RAM budget and demand window for the
32371
- * decode worker's native-resolution frame retention.
32458
+ * THE native-frame **lease** knobs — hold depth, RAM budget, demand window and
32459
+ * subject-tile budget for the decode worker's native-resolution retention.
32372
32460
  *
32373
32461
  * ## Why they live here and not in the addon that reads them
32374
32462
  *
@@ -32384,20 +32472,25 @@ var DETAIL_CROP_PADDING_FIELD = {
32384
32472
  * The lease is a per-decode-worker RAM window. Its purpose — the late
32385
32473
  * cross-process native crop landing on a full-resolution frame rather than the
32386
32474
  * ≤640 detection fallback — is a property of the PIPELINE, not of a node's
32387
- * hardware: a per-node TTL would mean the same camera produces different crop
32475
+ * hardware: a per-node window would mean the same camera produces different crop
32388
32476
  * quality depending on which node the balancer placed it on, and nobody could
32389
32477
  * tell that from the stored media. Node-level RAM pressure is already handled
32390
32478
  * by the per-session budget ceiling, which is itself one of these knobs.
32391
32479
  *
32392
32480
  * ## What each knob costs
32393
32481
  *
32394
- * A retained frame is a full NATIVE-resolution copy in system RAM. With the
32482
+ * A HELD frame is a full NATIVE-resolution copy in system RAM. With the
32395
32483
  * default pinned-RGB24 lease path (`CAMSTACK_SESSION_PINNED_RGB_CROP`, on):
32396
32484
  * 4K ≈ 24.9 MB/frame, 1080p ≈ 6.2 MB/frame. On the YUV420P path (flag off, and
32397
32485
  * for software-decoded sessions): 4K ≈ 12.4 MB, 1080p ≈ 3.1 MB. Worst-case
32398
- * resident RAM for ONE busy camera frameBytes × deliveredFps × ttlSeconds,
32399
- * clamped by the budget ceiling. See `docs/design/decode-path.md` "Lease
32400
- * admission" for what actually gets admitted.
32486
+ * resident RAM for ONE busy camera is now `frameBytes × holdFrames`, clamped by
32487
+ * the budget ceiling bounded by a COUNT because a held frame is waiting for
32488
+ * one specific event (its own detection result), not for a clock.
32489
+ *
32490
+ * A TILE is one subject at native resolution, JPEG-encoded: ~60-120 KB on 4K,
32491
+ * and nothing at all on a frame that detected nothing. That is the asymmetry
32492
+ * this whole shape exists for — see
32493
+ * `docs/design/2026-08-13-native-lease-two-tier-redesign.md`.
32401
32494
  */
32402
32495
  /**
32403
32496
  * Store identity of the lease knobs in `pipeline-orchestrator`'s GLOBAL
@@ -32405,10 +32498,11 @@ var DETAIL_CROP_PADDING_FIELD = {
32405
32498
  * the reader can walk every section instead of trusting the section id.
32406
32499
  */
32407
32500
  var NATIVE_LEASE_SECTION_ID = "native-lease";
32408
- var NATIVE_LEASE_TTL_KEY = "nativeLeaseTtlMs";
32501
+ var NATIVE_LEASE_HOLD_KEY = "nativeLeaseHoldFrames";
32409
32502
  var NATIVE_LEASE_BUDGET_KEY = "nativeLeaseBudgetMb";
32410
32503
  var NATIVE_LEASE_ACTIVITY_KEY = "nativeLeaseActivityMs";
32411
32504
  var NATIVE_LEASE_ADMISSION_KEY = "nativeLeaseAdmission";
32505
+ var NATIVE_LEASE_TILE_BUDGET_KEY = "nativeTileBudgetMb";
32412
32506
  /**
32413
32507
  * WHICH delivered frames the decode worker retains a native copy of.
32414
32508
  *
@@ -32430,25 +32524,32 @@ var NATIVE_LEASE_ADMISSION_KEY = "nativeLeaseAdmission";
32430
32524
  var NativeLeaseAdmissionSchema = _enum(["all", "inferred"]);
32431
32525
  object({
32432
32526
  /**
32433
- * How long a retained native frame is served before it counts as a miss.
32527
+ * How many delivered frames the worker HOLDS at once, waiting for each one's
32528
+ * detection result.
32434
32529
  *
32435
- * Must cover the FULL late-crop horizon: detection inference + the
32436
- * cross-process inference-result hop to hub post-analysis + tracking + the
32437
- * tRPC crop round-trip back. Below ~500 ms the busiest cameras' subject crops
32438
- * outrun it and fall back to the ≤640 detection frame; above ~3 s the resident
32439
- * RAM per busy camera grows linearly with no measured hit-rate gain.
32530
+ * This replaced a TTL on 2026-08-13, and the replacement is the whole point:
32531
+ * a time window was never related to the event the pixels were waiting for.
32532
+ * A held frame now lives from delivery until the runner has its `FrameResult`
32533
+ * at which moment the runner cuts the subject tiles it actually wanted and
32534
+ * releases the frame. The bound exists only so a runner that stops answering
32535
+ * cannot pin RAM: above it the OLDEST held frame is dropped and counted.
32536
+ *
32537
+ * Sizing: the steady state is `inferenceLatency × deliveredFps`, measured at
32538
+ * 40-160 ms × ≤25 fps = 1-4 frames. The default leaves headroom for a hiccup
32539
+ * without ever approaching the old resident set (43 frames × 24.9 MB at 4K).
32540
+ * Raising it does not buy hit rate — it buys tolerance for a slow runner, and
32541
+ * `holdOverflow` on the metrics line is what says you need it.
32440
32542
  */
32441
- ttlMs: number().int().min(250).max(1e4),
32543
+ holdFrames: number().int().min(1).max(64),
32442
32544
  /**
32443
32545
  * Hard per-decode-worker RAM ceiling for retained native frames, in MB.
32444
32546
  *
32445
- * Intended as a SAFETY ceiling with the TTL as the effective cap — but check
32446
- * which one is actually binding before reasoning from that. At the shipped
32447
- * 1024 MB and a 2 800 ms TTL, a 4K camera hits the CEILING first (~43 frames
32448
- * at ~24 MB each) and the TTL never gets to expire anything; `leaseMb` /
32449
- * `leaseFrames` on the metrics line say which. When the ceiling binds, a
32450
- * change that admits fewer frames buys retention WINDOW at constant RAM
32451
- * rather than giving RAM back — lower this knob if RAM is what you wanted.
32547
+ * Since 2026-08-13 this is a SAFETY ceiling and nothing else: `holdFrames`
32548
+ * is what decides how much is held, and the ceiling is the number above which
32549
+ * something is wrong. Before that it was the effective cap at 1024 MB with
32550
+ * a 2 800 ms TTL a 4K camera sat pinned at `leaseMb:1020, leaseFrames:43`
32551
+ * with the TTL expiring nothing, which is exactly the confusion the hold
32552
+ * removes. `leaseMb` / `leaseFrames` still say what is resident.
32452
32553
  * `0` DISABLES the lease entirely and falls the worker back to the tiny
32453
32554
  * leak-prone GPU surface ring (~85% crop miss; that is what the lease exists
32454
32555
  * to replace).
@@ -32474,25 +32575,47 @@ object({
32474
32575
  * there is the signal that some caller names frames outside the inference set
32475
32576
  * and that this must go back to `all`.
32476
32577
  */
32477
- admission: NativeLeaseAdmissionSchema
32578
+ admission: NativeLeaseAdmissionSchema,
32579
+ /**
32580
+ * RAM ceiling per decode worker, in MB, for the SUBJECT TILES — the
32581
+ * compressed native crops the worker cuts at the moment a frame's detection
32582
+ * result arrives, and keeps long after the frame itself is freed.
32583
+ *
32584
+ * This is the knob that replaced the old retention window, and it buys about
32585
+ * three orders of magnitude more of it: a tile is one subject at native
32586
+ * resolution, JPEG-encoded (~60-120 KB on a 4K person), against ~24.9 MB for
32587
+ * the frame it was cut from. A frame on which nothing was detected costs
32588
+ * nothing at all, which is the real change — the old lease paid per FRAME and
32589
+ * was interrogated per SUBJECT.
32590
+ *
32591
+ * `0` DISABLES tiles, leaving only the hold window and the ≤640 RAM
32592
+ * fallback — i.e. the pre-2026-08-13 miss profile. Set it there only to
32593
+ * reproduce that.
32594
+ */
32595
+ tileBudgetMb: number().int().min(0).max(1024)
32478
32596
  });
32479
32597
  /**
32480
- * The values in force when the operator has set nothing — byte-for-byte the
32481
- * constants the decode worker shipped with as env-var defaults, so making these
32482
- * settings changed no behaviour on the day it landed.
32598
+ * The values in force when the operator has set nothing.
32599
+ *
32600
+ * `budgetMb` stays at 1024 on the day the hold landed, deliberately: it stopped
32601
+ * being the retention window and became the OOM ceiling, and lowering a ceiling
32602
+ * in the same change that redefines it would make a regression and a retune
32603
+ * indistinguishable. Cut it once `tileHits` / `holdOverflow` have been read on
32604
+ * live traffic.
32483
32605
  */
32484
32606
  var DEFAULT_NATIVE_LEASE_SETTINGS = {
32485
- ttlMs: 1200,
32607
+ holdFrames: 8,
32486
32608
  budgetMb: 1024,
32487
32609
  activityMs: 15e3,
32610
+ tileBudgetMb: 64,
32488
32611
  admission: "inferred"
32489
32612
  };
32490
32613
  /** Slider bounds for the operator-facing knobs (orchestrator settings UI). */
32491
- var NATIVE_LEASE_TTL_FIELD = {
32492
- min: 250,
32493
- max: 1e4,
32494
- step: 50,
32495
- default: DEFAULT_NATIVE_LEASE_SETTINGS.ttlMs
32614
+ var NATIVE_LEASE_HOLD_FIELD = {
32615
+ min: 1,
32616
+ max: 64,
32617
+ step: 1,
32618
+ default: DEFAULT_NATIVE_LEASE_SETTINGS.holdFrames
32496
32619
  };
32497
32620
  var NATIVE_LEASE_BUDGET_FIELD = {
32498
32621
  min: 0,
@@ -32506,6 +32629,12 @@ var NATIVE_LEASE_ACTIVITY_FIELD = {
32506
32629
  step: 1e3,
32507
32630
  default: DEFAULT_NATIVE_LEASE_SETTINGS.activityMs
32508
32631
  };
32632
+ var NATIVE_LEASE_TILE_BUDGET_FIELD = {
32633
+ min: 0,
32634
+ max: 1024,
32635
+ step: 16,
32636
+ default: DEFAULT_NATIVE_LEASE_SETTINGS.tileBudgetMb
32637
+ };
32509
32638
  /** Select options for the admission knob (orchestrator settings UI). */
32510
32639
  var NATIVE_LEASE_ADMISSION_FIELD = {
32511
32640
  options: [{
@@ -32833,7 +32962,55 @@ var OrchestratorDiagnosticsSchema = object({
32833
32962
  cameraConfigCount: number().int().min(0),
32834
32963
  activeDetectionCount: number().int().min(0)
32835
32964
  });
32836
- var pipelineOrchestratorActions = defineCustomActions({ dumpState: customAction(_void(), OrchestratorDiagnosticsSchema) });
32965
+ /**
32966
+ * The node-stress long-term-statistics read surface.
32967
+ *
32968
+ * A custom action rather than a cap method, matching how the orchestrator
32969
+ * already serves `dumpState`: this is a hub-local read over a table the hub
32970
+ * owns, and it ships with one `camstack deploy` instead of a release train.
32971
+ * The MEAN is derived here and returned alongside the addable `sum`/`samples`
32972
+ * — a chart wants the first, a re-bucketing caller wants the second, and a
32973
+ * stored mean is a field that can disagree with both.
32974
+ */
32975
+ var NodeStressStatsInputSchema = object({
32976
+ /** `node-score` | `node-queue-pressure` | `node-drop-ratio` | `node-fps-deficit`. */
32977
+ series: string().optional(),
32978
+ /** A node id. Omit for every node. */
32979
+ subject: string().optional(),
32980
+ /** Inclusive bucket-start bounds, ms. */
32981
+ from: number().int().optional(),
32982
+ to: number().int().optional(),
32983
+ limit: number().int().positive().max(5e3).optional()
32984
+ });
32985
+ var NodeStressStatsRowSchema = object({
32986
+ subject: string(),
32987
+ series: string(),
32988
+ scope: string(),
32989
+ bucketStart: number(),
32990
+ samples: number(),
32991
+ sum: number(),
32992
+ mean: number(),
32993
+ min: number(),
32994
+ max: number()
32995
+ });
32996
+ var NodeStressStatsOutputSchema = object({
32997
+ rows: array(NodeStressStatsRowSchema).readonly(),
32998
+ /** Buckets still accumulating — "is it running" answerable at once, rather
32999
+ * than after five minutes of indistinguishable silence. */
33000
+ open: array(NodeStressStatsRowSchema).readonly(),
33001
+ /** The durable failover history the anti-flap guards read, newest first.
33002
+ * Exposed for the same reason the heartbeat exists: "nothing moved" has to
33003
+ * be distinguishable from "nothing is watching". */
33004
+ moves: array(object({
33005
+ deviceId: number(),
33006
+ fromNodeId: string(),
33007
+ at: number()
33008
+ })).readonly()
33009
+ });
33010
+ var pipelineOrchestratorActions = defineCustomActions({
33011
+ dumpState: customAction(_void(), OrchestratorDiagnosticsSchema),
33012
+ nodeStressStats: customAction(NodeStressStatsInputSchema, NodeStressStatsOutputSchema, { auth: "admin" })
33013
+ });
32837
33014
  /**
32838
33015
  * Sentinel returned by `buildDetectionConfig` when the profile-slot READ
32839
33016
  * itself failed transiently (e.g. `listAllProfileSlots` discovery timeout
@@ -32879,7 +33056,7 @@ var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
32879
33056
  * shape so video and audio plumbing self-heal identically.
32880
33057
  */
32881
33058
  /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
32882
- var POLL_INTERVAL_MS = 200;
33059
+ var POLL_INTERVAL_MS$1 = 200;
32883
33060
  /** How many chunks to drain per poll — a small burst absorbs jitter. */
32884
33061
  var PULL_MAX_COUNT = 8;
32885
33062
  /**
@@ -33062,7 +33239,7 @@ function startPolling(options, lifecycle) {
33062
33239
  } });
33063
33240
  if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
33064
33241
  }
33065
- if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
33242
+ if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS$1);
33066
33243
  };
33067
33244
  tick();
33068
33245
  }
@@ -35553,12 +35730,341 @@ function buildRunnerConfig(base, overrides) {
35553
35730
  };
35554
35731
  }
35555
35732
  //#endregion
35556
- //#region src/device-features-mirror.ts
35733
+ //#region src/durable/durable-ledger.ts
35734
+ /** Default reseed cap — every current consumer's row set is installation-bounded. */
35735
+ var DEFAULT_LOAD_LIMIT = 1e5;
35736
+ var DurableLedger = class DurableLedger {
35737
+ mirror = /* @__PURE__ */ new Map();
35738
+ spec;
35739
+ store;
35740
+ logger;
35741
+ constructor(deps) {
35742
+ this.spec = deps.spec;
35743
+ this.store = deps.store;
35744
+ this.logger = deps.logger;
35745
+ }
35746
+ /**
35747
+ * Register the collection. MUST run at boot, before any read or write: the
35748
+ * SQLite backend answers 412 for an undeclared collection and takes the whole
35749
+ * runner down with it (the addon-ai boot-crash lesson).
35750
+ */
35751
+ static declare(store, spec) {
35752
+ return store.declareCollection.mutate({
35753
+ collection: spec.collection,
35754
+ columns: [...spec.columns],
35755
+ ...spec.indexes !== void 0 ? { indexes: [...spec.indexes] } : {}
35756
+ });
35757
+ }
35758
+ declare() {
35759
+ return DurableLedger.declare(this.store, this.spec);
35760
+ }
35761
+ /** The collection this ledger owns — for the caller's own log lines. */
35762
+ get collection() {
35763
+ return this.spec.collection;
35764
+ }
35765
+ /**
35766
+ * Boot reseed. Replaces the mirror with what the store holds and returns the
35767
+ * rows, so a caller that must hydrate something else (a watcher, a registry)
35768
+ * gets them without a second read.
35769
+ *
35770
+ * **A failure returns what is already mirrored** rather than clearing it —
35771
+ * see contract rule 2. The count is worth logging out loud at the call site:
35772
+ * "loaded 0" after a container recreate is the one line that explains a
35773
+ * silent flood.
35774
+ */
35775
+ async load() {
35776
+ try {
35777
+ const records = await this.store.query.query({
35778
+ collection: this.spec.collection,
35779
+ filter: { limit: this.spec.loadLimit ?? DEFAULT_LOAD_LIMIT }
35780
+ });
35781
+ const next = /* @__PURE__ */ new Map();
35782
+ let skipped = 0;
35783
+ for (const record of records) {
35784
+ const row = this.spec.fromRecord(record.id, record.data);
35785
+ if (row === null) {
35786
+ skipped += 1;
35787
+ continue;
35788
+ }
35789
+ next.set(this.spec.keyOf(row), row);
35790
+ }
35791
+ this.mirror.clear();
35792
+ for (const [key, row] of next) this.mirror.set(key, row);
35793
+ if (skipped > 0) this.logger.warn("durable rows skipped as malformed — they gate NOTHING", { meta: {
35794
+ collection: this.spec.collection,
35795
+ skipped
35796
+ } });
35797
+ return [...this.mirror.values()];
35798
+ } catch (err) {
35799
+ this.logger.warn("durable load failed — keeping the state already in memory", { meta: {
35800
+ collection: this.spec.collection,
35801
+ error: String(err),
35802
+ held: this.mirror.size
35803
+ } });
35804
+ return [...this.mirror.values()];
35805
+ }
35806
+ }
35807
+ /** Every mirrored row, insertion-ordered. */
35808
+ snapshot() {
35809
+ return [...this.mirror.values()];
35810
+ }
35811
+ /** The row currently held for a key, if any. Pure RAM — never I/O. */
35812
+ get(key) {
35813
+ return this.mirror.get(key);
35814
+ }
35815
+ has(key) {
35816
+ return this.mirror.has(key);
35817
+ }
35818
+ get size() {
35819
+ return this.mirror.size;
35820
+ }
35821
+ /**
35822
+ * Judge ONE observation against what the ledger already accepted, and advance
35823
+ * it.
35824
+ *
35825
+ * Synchronous on purpose: the verdict is a function of the in-RAM mirror
35826
+ * alone, so a decision can never be gated on an I/O that might fail (D49).
35827
+ * The durable write is kicked off behind it and its failure changes no
35828
+ * verdict.
35829
+ *
35830
+ * `no-flip` leaves the held row UNTOUCHED — including any timestamp it
35831
+ * carries, which therefore means "when this key last CHANGED", not "when it
35832
+ * last spoke". That is the timestamp anyone reading the table wants.
35833
+ */
35834
+ observe(row) {
35835
+ const equalFact = this.spec.equalFact;
35836
+ if (equalFact === void 0) throw new Error(`DurableLedger(${this.spec.collection}): observe() requires the spec to declare equalFact`);
35837
+ const key = this.spec.keyOf(row);
35838
+ const held = this.mirror.get(key);
35839
+ if (held !== void 0 && equalFact(held, row)) return "no-flip";
35840
+ this.mirror.set(key, row);
35841
+ this.persist(row);
35842
+ return held === void 0 ? "seeded" : "flip";
35843
+ }
35844
+ /**
35845
+ * Upsert a row without a verdict — the write path for a ledger whose owner
35846
+ * has already decided the value changed.
35847
+ *
35848
+ * The order of the mirror advance and the durable write is the spec's
35849
+ * {@link DurableWriteMode}, not the call site's: two call sites that
35850
+ * disagreed about it would be two different durability guarantees on one
35851
+ * collection.
35852
+ */
35853
+ async put(row) {
35854
+ const key = this.spec.keyOf(row);
35855
+ if (this.spec.writeMode === "write-behind") {
35856
+ this.mirror.set(key, row);
35857
+ await this.persist(row);
35858
+ return;
35859
+ }
35860
+ await this.store.set.mutate({
35861
+ collection: this.spec.collection,
35862
+ key,
35863
+ value: this.spec.toValue(row)
35864
+ });
35865
+ this.mirror.set(key, row);
35866
+ }
35867
+ /**
35868
+ * Advance the mirror WITHOUT persisting, for an owner that deliberately
35869
+ * coalesces its writes.
35870
+ *
35871
+ * The stationary registry is the reason this exists: a parked car is
35872
+ * re-confirmed on every processed frame (5–30 Hz), and persisting each
35873
+ * confirmation would offer thousands of commits a day to the checkpoint
35874
+ * lottery (D96) to maintain a handful of rows. It stages the advance and
35875
+ * flushes on its 5-minute sweep — 288 writes a day instead of ~10⁶.
35876
+ *
35877
+ * **The cost is stated, not hidden**: a staged value that is never flushed
35878
+ * is lost on a crash. An owner may only stage a field whose staleness its
35879
+ * own TTL absorbs. Anything that GATES work must go through {@link put} or
35880
+ * {@link observe}.
35881
+ */
35882
+ stage(row) {
35883
+ this.mirror.set(this.spec.keyOf(row), row);
35884
+ }
35885
+ /**
35886
+ * Drop a key from the MIRROR only — the durable row survives.
35887
+ *
35888
+ * What a scope-unbind needs: this process stops holding the value, and a
35889
+ * rebind reloads it from the store. Deliberately distinct from
35890
+ * {@link forget}, which deletes; conflating the two is how an unbind turns
35891
+ * into a wipe.
35892
+ */
35893
+ evict(key) {
35894
+ this.mirror.delete(key);
35895
+ }
35896
+ /**
35897
+ * Drop one key, mirror and row. Best-effort on the durable half: a failed
35898
+ * delete leaves a row that the next load will re-mirror, which is a stale
35899
+ * value rather than a lost one.
35900
+ */
35901
+ async forget(key) {
35902
+ this.mirror.delete(key);
35903
+ try {
35904
+ await this.store.delete.mutate({
35905
+ collection: this.spec.collection,
35906
+ key
35907
+ });
35908
+ } catch (err) {
35909
+ this.logger.debug("durable delete failed", { meta: {
35910
+ collection: this.spec.collection,
35911
+ key,
35912
+ error: String(err)
35913
+ } });
35914
+ }
35915
+ }
35916
+ /**
35917
+ * Drop every mirrored key NOT in `activeKeys`. Returns how many rows went.
35918
+ *
35919
+ * **The caller must hold an AUTHORITATIVE active set.** A prune driven by a
35920
+ * fallible read is work destroyed on an error (D49/D130) — that is why this
35921
+ * is a method a feature opts into rather than a policy the primitive runs.
35922
+ * Best-effort per row: a failed delete keeps the key (retried next prune)
35923
+ * rather than aborting the sweep.
35924
+ */
35925
+ async pruneExcept(activeKeys) {
35926
+ let pruned = 0;
35927
+ for (const key of [...this.mirror.keys()]) {
35928
+ if (activeKeys.has(key)) continue;
35929
+ try {
35930
+ await this.store.delete.mutate({
35931
+ collection: this.spec.collection,
35932
+ key
35933
+ });
35934
+ this.mirror.delete(key);
35935
+ pruned += 1;
35936
+ } catch (err) {
35937
+ this.logger.debug("durable prune delete failed", { meta: {
35938
+ collection: this.spec.collection,
35939
+ key,
35940
+ error: String(err)
35941
+ } });
35942
+ }
35943
+ }
35944
+ return pruned;
35945
+ }
35946
+ /**
35947
+ * Write-behind durable upsert. Best-effort and logged, never thrown at the
35948
+ * decision path: the mirror already holds the truth for this process, and the
35949
+ * worst a lost write can do is one silent re-seed after the next restart.
35950
+ */
35951
+ async persist(row) {
35952
+ const deviceId = this.spec.deviceIdOf?.(row);
35953
+ try {
35954
+ await this.store.set.mutate({
35955
+ collection: this.spec.collection,
35956
+ key: this.spec.keyOf(row),
35957
+ value: this.spec.toValue(row)
35958
+ });
35959
+ } catch (err) {
35960
+ this.logger.warn("durable persist failed — this key may re-seed on boot", {
35961
+ ...deviceId !== void 0 ? { tags: { deviceId } } : {},
35962
+ meta: {
35963
+ collection: this.spec.collection,
35964
+ key: this.spec.keyOf(row),
35965
+ error: String(err)
35966
+ }
35967
+ });
35968
+ }
35969
+ }
35970
+ };
35971
+ var DEVICE_FEATURES_SPEC = {
35972
+ collection: "pipeline-orchestrator:device-features",
35973
+ columns: [
35974
+ {
35975
+ name: "deviceId",
35976
+ type: "TEXT",
35977
+ primaryKey: true,
35978
+ notNull: true
35979
+ },
35980
+ (
35981
+ /** The feature name list, verbatim. */
35982
+ {
35983
+ name: "features",
35984
+ type: "JSON",
35985
+ notNull: true
35986
+ }),
35987
+ {
35988
+ name: "updatedAt",
35989
+ type: "INTEGER",
35990
+ notNull: true
35991
+ }
35992
+ ],
35993
+ writeMode: "write-behind",
35994
+ keyOf: (row) => String(row.deviceId),
35995
+ toValue: (row) => ({
35996
+ features: [...row.features],
35997
+ updatedAt: row.updatedAt
35998
+ }),
35999
+ fromRecord: (key, data) => {
36000
+ const deviceId = Number(key);
36001
+ const raw = data["features"];
36002
+ if (!Number.isFinite(deviceId) || !Array.isArray(raw)) return null;
36003
+ const features = raw.filter((f) => typeof f === "string");
36004
+ if (features.length === 0) return null;
36005
+ const updatedAt = Number(data["updatedAt"]);
36006
+ return {
36007
+ deviceId,
36008
+ features,
36009
+ updatedAt: Number.isFinite(updatedAt) ? updatedAt : 0,
36010
+ restored: true
36011
+ };
36012
+ },
36013
+ deviceIdOf: (row) => row.deviceId
36014
+ };
35557
36015
  var DeviceFeaturesMirror = class {
36016
+ /** Process-local fallback, used only when no store was supplied. */
36017
+ local = /* @__PURE__ */ new Map();
36018
+ durable;
35558
36019
  logger;
35559
- lastKnown = /* @__PURE__ */ new Map();
35560
- constructor(logger) {
35561
- this.logger = logger;
36020
+ now;
36021
+ constructor(deps) {
36022
+ this.logger = deps.logger;
36023
+ this.now = deps.now ?? (() => Date.now());
36024
+ this.durable = deps.store === void 0 ? null : new DurableLedger({
36025
+ spec: DEVICE_FEATURES_SPEC,
36026
+ store: deps.store,
36027
+ logger: deps.logger
36028
+ });
36029
+ }
36030
+ static declare(store) {
36031
+ return DurableLedger.declare(store, DEVICE_FEATURES_SPEC);
36032
+ }
36033
+ /**
36034
+ * Seed the mirror from the last session. Call once at boot, after `declare`
36035
+ * and BEFORE the first `resolve` — the whole value of the row is that it is
36036
+ * already there when the first read fails.
36037
+ */
36038
+ async hydrate() {
36039
+ if (this.durable === null) return 0;
36040
+ const rows = await this.durable.load();
36041
+ this.logger.info("device-features mirror restored", { meta: { devices: rows.length } });
36042
+ return rows.length;
36043
+ }
36044
+ held(deviceId) {
36045
+ return this.durable === null ? this.local.get(deviceId) : this.durable.get(String(deviceId));
36046
+ }
36047
+ remember(deviceId, features) {
36048
+ const row = {
36049
+ deviceId,
36050
+ features: [...features],
36051
+ updatedAt: this.now(),
36052
+ restored: false
36053
+ };
36054
+ if (this.durable === null) {
36055
+ this.local.set(deviceId, row);
36056
+ return;
36057
+ }
36058
+ const held = this.durable.get(String(deviceId));
36059
+ if (held !== void 0 && !held.restored && sameFeatures(held.features, features)) return;
36060
+ this.durable.put(row);
36061
+ }
36062
+ drop(deviceId) {
36063
+ if (this.durable === null) {
36064
+ this.local.delete(deviceId);
36065
+ return;
36066
+ }
36067
+ this.durable.forget(String(deviceId));
35562
36068
  }
35563
36069
  /**
35564
36070
  * Resolve a device's features, preferring a fresh read but never letting a
@@ -35567,15 +36073,19 @@ var DeviceFeaturesMirror = class {
35567
36073
  async resolve(deviceId, read) {
35568
36074
  const first = await read();
35569
36075
  if (first !== null && first.length > 0) {
35570
- this.lastKnown.set(deviceId, [...first]);
36076
+ this.remember(deviceId, first);
35571
36077
  return first;
35572
36078
  }
35573
- const mirrored = this.lastKnown.get(deviceId);
36079
+ const heldRow = this.held(deviceId);
36080
+ const mirrored = heldRow?.features;
35574
36081
  if (first === null) {
35575
36082
  if (mirrored !== void 0) {
35576
36083
  this.logger.warn("device features unavailable — serving last-known mirror", {
35577
36084
  tags: { deviceId },
35578
- meta: { mirrored: mirrored.length }
36085
+ meta: {
36086
+ mirrored: mirrored.length,
36087
+ restored: heldRow?.restored === true
36088
+ }
35579
36089
  });
35580
36090
  return mirrored;
35581
36091
  }
@@ -35589,7 +36099,7 @@ var DeviceFeaturesMirror = class {
35589
36099
  tags: { deviceId },
35590
36100
  meta: { features: second.length }
35591
36101
  });
35592
- this.lastKnown.set(deviceId, [...second]);
36102
+ this.remember(deviceId, second);
35593
36103
  return second;
35594
36104
  }
35595
36105
  if (second === null) {
@@ -35600,14 +36110,20 @@ var DeviceFeaturesMirror = class {
35600
36110
  tags: { deviceId },
35601
36111
  meta: { previously: mirrored.length }
35602
36112
  });
35603
- this.lastKnown.delete(deviceId);
36113
+ this.drop(deviceId);
35604
36114
  return [];
35605
36115
  }
35606
36116
  /** Drop a device's mirror — call when the device is removed. */
35607
36117
  forget(deviceId) {
35608
- this.lastKnown.delete(deviceId);
36118
+ this.drop(deviceId);
35609
36119
  }
35610
36120
  };
36121
+ /** Order-insensitive feature-set equality — the read's order is not a fact. */
36122
+ function sameFeatures(a, b) {
36123
+ if (a.length !== b.length) return false;
36124
+ const held = new Set(a);
36125
+ return b.every((f) => held.has(f));
36126
+ }
35611
36127
  //#endregion
35612
36128
  //#region src/watchdog-camera.ts
35613
36129
  /**
@@ -35674,7 +36190,20 @@ var DetectionWiringController = class {
35674
36190
  featuresMirror;
35675
36191
  constructor(deps) {
35676
36192
  this.deps = deps;
35677
- this.featuresMirror = new DeviceFeaturesMirror(deps.logger);
36193
+ this.featuresMirror = new DeviceFeaturesMirror({
36194
+ logger: deps.logger,
36195
+ ...deps.featuresStore !== void 0 ? { store: deps.featuresStore } : {}
36196
+ });
36197
+ }
36198
+ /**
36199
+ * Declare + seed the device-features mirror. Call once at boot, BEFORE the
36200
+ * first detection start — the restored row is worth nothing after the read
36201
+ * that would have needed it.
36202
+ */
36203
+ async hydrateFeaturesMirror() {
36204
+ if (this.deps.featuresStore === void 0) return;
36205
+ await DeviceFeaturesMirror.declare(this.deps.featuresStore);
36206
+ await this.featuresMirror.hydrate();
35678
36207
  }
35679
36208
  /** `activeDetections.get(deviceId)`. */
35680
36209
  getActiveDetectionConfig(deviceId) {
@@ -36885,7 +37414,6 @@ var NodeStressController = class NodeStressController {
36885
37414
  static HEARTBEAT_MS = 60 * 6e4;
36886
37415
  samples = /* @__PURE__ */ new Map();
36887
37416
  memories = /* @__PURE__ */ new Map();
36888
- history = [];
36889
37417
  /** When the last heartbeat went out. `null` ⇒ the next sweep emits one. */
36890
37418
  lastHeartbeatAt = null;
36891
37419
  timer = null;
@@ -36929,6 +37457,18 @@ var NodeStressController = class NodeStressController {
36929
37457
  forgetDevice(deviceId) {
36930
37458
  this.samples.delete(deviceId);
36931
37459
  }
37460
+ /**
37461
+ * Recover the move history from disk BEFORE the first sweep can act.
37462
+ *
37463
+ * Best-effort by construction (the ledger keeps whatever it already has on a
37464
+ * read failure), and the count is logged out loud: a restart that recovers 0
37465
+ * moves while the operator remembers three is the line that says the budget
37466
+ * has been reset.
37467
+ */
37468
+ async hydrate() {
37469
+ const recovered = await this.deps.moves.load();
37470
+ this.deps.logger.info("node-stress move history recovered", { meta: { moves: recovered } });
37471
+ }
36932
37472
  start() {
36933
37473
  if (this.timer) return;
36934
37474
  this.timer = setInterval(() => {
@@ -36944,12 +37484,24 @@ var NodeStressController = class NodeStressController {
36944
37484
  }
36945
37485
  this.samples.clear();
36946
37486
  this.memories.clear();
36947
- this.history = [];
36948
37487
  }
36949
37488
  /** The last computed state per node — empty before the first sweep. */
36950
37489
  statesView() {
36951
37490
  return new Map([...this.memories].map(([nodeId, m]) => [nodeId, m.state]));
36952
37491
  }
37492
+ /** The durable move history, newest first — the guards' own evidence, so an
37493
+ * operator asking "why did nothing move" can read the budget. */
37494
+ historyView() {
37495
+ return this.deps.moves.entries().toSorted((a, b) => b.at - a.at);
37496
+ }
37497
+ /**
37498
+ * The per-node stress signals as of `now`, for the LTS aggregator and for
37499
+ * diagnostics. Computed from the same live samples the sweep uses, so a
37500
+ * chart can never disagree with a verdict.
37501
+ */
37502
+ signalsView(now) {
37503
+ return new Map(this.buildInputs(now).map((input) => [input.nodeId, input.signals]));
37504
+ }
36953
37505
  /** Group live samples by node and reduce each group. Exposed for tests. */
36954
37506
  buildInputs(now) {
36955
37507
  const byNode = /* @__PURE__ */ new Map();
@@ -36980,6 +37532,7 @@ var NodeStressController = class NodeStressController {
36980
37532
  const verdicts = evaluateNodeStress(this.memories, this.buildInputs(now), now, this.thresholds());
36981
37533
  for (const v of verdicts) {
36982
37534
  this.memories.set(v.nodeId, v.memory);
37535
+ this.deps.lts?.noteSignals(v.nodeId, v.signals, now);
36983
37536
  if (v.changed) this.logTransition(v);
36984
37537
  }
36985
37538
  this.maybeHeartbeat(mode, verdicts, now);
@@ -37038,11 +37591,12 @@ var NodeStressController = class NodeStressController {
37038
37591
  async actOn(verdicts, now) {
37039
37592
  if (this.moveInFlight) return;
37040
37593
  if (!verdicts.some((v) => v.state === "saturated")) return;
37041
- this.history = this.history.filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
37594
+ this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
37595
+ const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
37042
37596
  const plan = planStressFailover({
37043
37597
  verdicts,
37044
37598
  candidates: await this.collectCandidates(now),
37045
- history: this.history,
37599
+ history,
37046
37600
  nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
37047
37601
  attachedByNode: this.attachedByNode()
37048
37602
  }, now, this.guards());
@@ -37066,7 +37620,7 @@ var NodeStressController = class NodeStressController {
37066
37620
  await this.deps.detach(plan.fromNodeId, plan.deviceId);
37067
37621
  await this.deps.attach(plan.toNodeId, config);
37068
37622
  this.deps.ledger.recordAssignment(plan.deviceId, plan.toNodeId, "rebalance", false);
37069
- this.history.push({
37623
+ this.deps.moves.record({
37070
37624
  deviceId: plan.deviceId,
37071
37625
  fromNodeId: plan.fromNodeId,
37072
37626
  at: now
@@ -37138,6 +37692,513 @@ var NodeStressController = class NodeStressController {
37138
37692
  }
37139
37693
  };
37140
37694
  //#endregion
37695
+ //#region src/durable/lts-aggregator.ts
37696
+ /** Wall-clock bucket width. Five minutes, matching HA's statistics tier. */
37697
+ var LTS_BUCKET_MS = 5 * 6e4;
37698
+ /** How often the row cap is enforced. Rarely: it is a bound, not a deadline. */
37699
+ var CAP_SWEEP_INTERVAL_MS = 6 * 36e5;
37700
+ var LTS_COLUMNS = [
37701
+ (
37702
+ /** `<subject>|<series>|<scope>|<bucketStart>` — deterministic, so a bucket
37703
+ * flushed twice replaces itself rather than doubling. */
37704
+ {
37705
+ name: "id",
37706
+ type: "TEXT",
37707
+ primaryKey: true,
37708
+ notNull: true
37709
+ }),
37710
+ (
37711
+ /** The camera id, or the node id. One column, because every query is
37712
+ * "this thing over time" and the thing is one or the other. */
37713
+ {
37714
+ name: "subject",
37715
+ type: "TEXT",
37716
+ notNull: true
37717
+ }),
37718
+ (
37719
+ /** Numeric mirror of `subject` for camera rows, so a per-camera question is
37720
+ * answered by an integer index — every log line and every query about a
37721
+ * device in this repo is keyed by the numeric id. `NULL` for node rows. */
37722
+ {
37723
+ name: "deviceId",
37724
+ type: "INTEGER"
37725
+ }),
37726
+ {
37727
+ name: "series",
37728
+ type: "TEXT",
37729
+ notNull: true
37730
+ },
37731
+ (
37732
+ /** Sub-scope within a series: a zoneId for occupancy, `''` otherwise. */
37733
+ {
37734
+ name: "scope",
37735
+ type: "TEXT",
37736
+ notNull: true
37737
+ }),
37738
+ (
37739
+ /** Bucket start, wall-clock aligned to {@link LTS_BUCKET_MS}. */
37740
+ {
37741
+ name: "bucketStart",
37742
+ type: "INTEGER",
37743
+ notNull: true
37744
+ }),
37745
+ {
37746
+ name: "samples",
37747
+ type: "INTEGER",
37748
+ notNull: true
37749
+ },
37750
+ {
37751
+ name: "sum",
37752
+ type: "REAL",
37753
+ notNull: true
37754
+ },
37755
+ {
37756
+ name: "min",
37757
+ type: "REAL",
37758
+ notNull: true
37759
+ },
37760
+ {
37761
+ name: "max",
37762
+ type: "REAL",
37763
+ notNull: true
37764
+ }
37765
+ ];
37766
+ var LTS_INDEXES = [{
37767
+ name: "idx_lts_series_bucket",
37768
+ columns: ["series", "bucketStart"]
37769
+ }, {
37770
+ name: "idx_lts_device",
37771
+ columns: ["deviceId"]
37772
+ }];
37773
+ function ltsBucketStart(at, bucketMs = LTS_BUCKET_MS) {
37774
+ return Math.floor(at / bucketMs) * bucketMs;
37775
+ }
37776
+ var LtsAggregator = class {
37777
+ open = /* @__PURE__ */ new Map();
37778
+ /** Every (subject, series, scope) that has produced a row in this process —
37779
+ * the groups the cap sweep has any reason to look at. */
37780
+ groups = /* @__PURE__ */ new Set();
37781
+ lastCapSweepAt = 0;
37782
+ collection;
37783
+ store;
37784
+ logger;
37785
+ nowFn;
37786
+ bucketMs;
37787
+ maxRows;
37788
+ constructor(deps) {
37789
+ this.collection = deps.collection;
37790
+ this.store = deps.store;
37791
+ this.logger = deps.logger;
37792
+ this.nowFn = deps.now ?? (() => Date.now());
37793
+ this.bucketMs = deps.bucketMs ?? 3e5;
37794
+ this.maxRows = deps.maxRowsPerSeries ?? 105120;
37795
+ }
37796
+ static declare(store, collection) {
37797
+ return store.declareCollection.mutate({
37798
+ collection,
37799
+ columns: [...LTS_COLUMNS],
37800
+ indexes: [...LTS_INDEXES]
37801
+ });
37802
+ }
37803
+ /**
37804
+ * Record one observation. Synchronous, allocation-free after the first
37805
+ * sample of a bucket, and it cannot throw — it sits on paths that are
37806
+ * already producing the value for another reason and must not learn a new
37807
+ * failure mode.
37808
+ *
37809
+ * A non-finite value is DROPPED rather than folded in: one `NaN` would make
37810
+ * `sum`, `min` and `max` all `NaN` for the whole bucket, turning a skewed
37811
+ * row into a meaningless one.
37812
+ */
37813
+ note(input) {
37814
+ if (!Number.isFinite(input.value)) return;
37815
+ const at = input.at ?? this.nowFn();
37816
+ const scope = input.scope ?? "";
37817
+ const bucketStart = ltsBucketStart(at, this.bucketMs);
37818
+ const key = rowId(input.subject, input.series, scope, bucketStart);
37819
+ const held = this.open.get(key);
37820
+ if (held === void 0) {
37821
+ this.open.set(key, {
37822
+ subject: input.subject,
37823
+ ...input.deviceId !== void 0 ? { deviceId: input.deviceId } : {},
37824
+ series: input.series,
37825
+ scope,
37826
+ bucketStart,
37827
+ samples: 1,
37828
+ sum: input.value,
37829
+ min: input.value,
37830
+ max: input.value
37831
+ });
37832
+ return;
37833
+ }
37834
+ held.samples += 1;
37835
+ held.sum += input.value;
37836
+ if (input.value < held.min) held.min = input.value;
37837
+ if (input.value > held.max) held.max = input.value;
37838
+ }
37839
+ /** Buckets currently accumulating — diagnostics, and what a flush would write. */
37840
+ openBuckets() {
37841
+ return [...this.open.values()].map(toRow);
37842
+ }
37843
+ /**
37844
+ * Write every bucket that has CLOSED (its window ended at or before `now`)
37845
+ * and drop it from RAM. Returns the number of rows written.
37846
+ *
37847
+ * The current bucket is deliberately left alone: writing it early would mean
37848
+ * rewriting it on the next tick, which turns one row into up to sixty and
37849
+ * offers every one of them to the checkpoint lottery (D96).
37850
+ */
37851
+ async flushDue(now = this.nowFn()) {
37852
+ const currentBucket = ltsBucketStart(now, this.bucketMs);
37853
+ const due = [...this.open.values()].filter((b) => b.bucketStart < currentBucket);
37854
+ if (due.length === 0) {
37855
+ await this.maybeSweepCap(now);
37856
+ return 0;
37857
+ }
37858
+ let written = 0;
37859
+ for (const bucket of due) {
37860
+ const key = rowId(bucket.subject, bucket.series, bucket.scope, bucket.bucketStart);
37861
+ try {
37862
+ await this.store.set.mutate({
37863
+ collection: this.collection,
37864
+ key,
37865
+ value: {
37866
+ subject: bucket.subject,
37867
+ ...bucket.deviceId !== void 0 ? { deviceId: bucket.deviceId } : {},
37868
+ series: bucket.series,
37869
+ scope: bucket.scope,
37870
+ bucketStart: bucket.bucketStart,
37871
+ samples: bucket.samples,
37872
+ sum: bucket.sum,
37873
+ min: bucket.min,
37874
+ max: bucket.max
37875
+ }
37876
+ });
37877
+ written += 1;
37878
+ this.groups.add(groupKey(bucket.subject, bucket.series, bucket.scope));
37879
+ } catch (err) {
37880
+ this.logger.warn("lts bucket write failed — this interval will be missing", {
37881
+ ...bucket.deviceId !== void 0 ? { tags: { deviceId: bucket.deviceId } } : {},
37882
+ meta: {
37883
+ collection: this.collection,
37884
+ series: bucket.series,
37885
+ subject: bucket.subject,
37886
+ bucketStart: bucket.bucketStart,
37887
+ error: String(err)
37888
+ }
37889
+ });
37890
+ }
37891
+ this.open.delete(key);
37892
+ }
37893
+ await this.maybeSweepCap(now);
37894
+ return written;
37895
+ }
37896
+ /** Read closed buckets back. The read surface every chart will use. */
37897
+ async read(query = {}) {
37898
+ const where = {};
37899
+ if (query.series !== void 0) where["series"] = query.series;
37900
+ if (query.subject !== void 0) where["subject"] = query.subject;
37901
+ if (query.scope !== void 0) where["scope"] = query.scope;
37902
+ const records = await this.store.query.query({
37903
+ collection: this.collection,
37904
+ filter: {
37905
+ ...Object.keys(where).length > 0 ? { where } : {},
37906
+ ...query.from !== void 0 || query.to !== void 0 ? { whereBetween: { bucketStart: [query.from ?? 0, query.to ?? Number.MAX_SAFE_INTEGER] } } : {},
37907
+ orderBy: {
37908
+ field: "bucketStart",
37909
+ direction: "asc"
37910
+ },
37911
+ limit: query.limit ?? 5e3
37912
+ }
37913
+ });
37914
+ const rows = [];
37915
+ for (const record of records) {
37916
+ const row = recordToRow(record.data);
37917
+ if (row !== null) rows.push(row);
37918
+ }
37919
+ return rows;
37920
+ }
37921
+ /**
37922
+ * Enforce the per-(subject, series, scope) row cap, at most once every six
37923
+ * hours. Only groups this process has written to are examined: a group with
37924
+ * no new rows cannot have crossed a cap it was under.
37925
+ */
37926
+ async maybeSweepCap(now) {
37927
+ if (now - this.lastCapSweepAt < CAP_SWEEP_INTERVAL_MS) return;
37928
+ this.lastCapSweepAt = now;
37929
+ for (const group of this.groups) {
37930
+ const [subject, series, scope] = group.split("\0");
37931
+ if (subject === void 0 || series === void 0 || scope === void 0) continue;
37932
+ const where = {
37933
+ subject,
37934
+ series,
37935
+ scope
37936
+ };
37937
+ try {
37938
+ const excess = await this.store.count.query({
37939
+ collection: this.collection,
37940
+ filter: { where }
37941
+ }) - this.maxRows;
37942
+ if (excess <= 0) continue;
37943
+ const cutoff = (await this.store.query.query({
37944
+ collection: this.collection,
37945
+ filter: {
37946
+ where,
37947
+ orderBy: {
37948
+ field: "bucketStart",
37949
+ direction: "asc"
37950
+ },
37951
+ limit: excess
37952
+ }
37953
+ })).at(-1)?.data["bucketStart"];
37954
+ if (typeof cutoff !== "number") continue;
37955
+ const { deleted } = await this.store.deleteWhere.mutate({
37956
+ collection: this.collection,
37957
+ filter: {
37958
+ where,
37959
+ whereBetween: { bucketStart: [0, cutoff] }
37960
+ }
37961
+ });
37962
+ this.logger.info("lts row cap enforced", { meta: {
37963
+ collection: this.collection,
37964
+ series,
37965
+ subject,
37966
+ deleted,
37967
+ cap: this.maxRows
37968
+ } });
37969
+ } catch (err) {
37970
+ this.logger.warn("lts row cap sweep failed — the series keeps growing this cycle", { meta: {
37971
+ collection: this.collection,
37972
+ series,
37973
+ subject,
37974
+ error: String(err)
37975
+ } });
37976
+ }
37977
+ }
37978
+ }
37979
+ };
37980
+ function groupKey(subject, series, scope) {
37981
+ return `${subject}${series}${scope}`;
37982
+ }
37983
+ function rowId(subject, series, scope, bucketStart) {
37984
+ return `${subject}|${series}|${scope}|${bucketStart}`;
37985
+ }
37986
+ function toRow(bucket) {
37987
+ return {
37988
+ subject: bucket.subject,
37989
+ ...bucket.deviceId !== void 0 ? { deviceId: bucket.deviceId } : {},
37990
+ series: bucket.series,
37991
+ scope: bucket.scope,
37992
+ bucketStart: bucket.bucketStart,
37993
+ samples: bucket.samples,
37994
+ sum: bucket.sum,
37995
+ min: bucket.min,
37996
+ max: bucket.max
37997
+ };
37998
+ }
37999
+ /** Structural validation on read. A malformed row is skipped, never charted. */
38000
+ function recordToRow(data) {
38001
+ const subject = data["subject"];
38002
+ const series = data["series"];
38003
+ const scope = data["scope"];
38004
+ if (typeof subject !== "string" || typeof series !== "string") return null;
38005
+ const bucketStart = Number(data["bucketStart"]);
38006
+ const samples = Number(data["samples"]);
38007
+ const sum = Number(data["sum"]);
38008
+ const min = Number(data["min"]);
38009
+ const max = Number(data["max"]);
38010
+ if (![
38011
+ bucketStart,
38012
+ samples,
38013
+ sum,
38014
+ min,
38015
+ max
38016
+ ].every((n) => Number.isFinite(n))) return null;
38017
+ const rawDeviceId = data["deviceId"];
38018
+ const deviceId = typeof rawDeviceId === "number" && Number.isFinite(rawDeviceId) ? rawDeviceId : void 0;
38019
+ return {
38020
+ subject,
38021
+ ...deviceId !== void 0 ? { deviceId } : {},
38022
+ series,
38023
+ scope: typeof scope === "string" ? scope : "",
38024
+ bucketStart,
38025
+ samples,
38026
+ sum,
38027
+ min,
38028
+ max
38029
+ };
38030
+ }
38031
+ //#endregion
38032
+ //#region src/node-stress-lts.ts
38033
+ /**
38034
+ * @durable class=ledger owner=pipeline-orchestrator
38035
+ * write="one row per (node, series, 5-min bucket), written ONCE when the bucket closes; a node reporting no samples writes nothing"
38036
+ * retention="row CAP per (subject, series, scope) — 105,120 rows ≈ one year of 5-minute buckets. Not an age sweep: an LTS row summarises an interval and has no owner to be orphaned from."
38037
+ */
38038
+ var NODE_STRESS_LTS_COLLECTION = "pipeline-orchestrator:stats-5m";
38039
+ var NodeStressLts = class {
38040
+ lts;
38041
+ constructor(deps) {
38042
+ this.lts = new LtsAggregator({
38043
+ collection: NODE_STRESS_LTS_COLLECTION,
38044
+ store: deps.store,
38045
+ logger: deps.logger,
38046
+ ...deps.now !== void 0 ? { now: deps.now } : {}
38047
+ });
38048
+ }
38049
+ static declare(store) {
38050
+ return LtsAggregator.declare(store, NODE_STRESS_LTS_COLLECTION);
38051
+ }
38052
+ /** Fold one sweep's verdict for one node into the open buckets. */
38053
+ noteSignals(nodeId, signals, at) {
38054
+ this.lts.note({
38055
+ subject: nodeId,
38056
+ series: "node-score",
38057
+ value: signals.score,
38058
+ at
38059
+ });
38060
+ this.lts.note({
38061
+ subject: nodeId,
38062
+ series: "node-queue-pressure",
38063
+ value: signals.queuePressure,
38064
+ at
38065
+ });
38066
+ this.lts.note({
38067
+ subject: nodeId,
38068
+ series: "node-drop-ratio",
38069
+ value: signals.dropRatio,
38070
+ at
38071
+ });
38072
+ this.lts.note({
38073
+ subject: nodeId,
38074
+ series: "node-fps-deficit",
38075
+ value: signals.fpsDeficit,
38076
+ at
38077
+ });
38078
+ }
38079
+ flushDue(now) {
38080
+ return this.lts.flushDue(now);
38081
+ }
38082
+ read(query) {
38083
+ return this.lts.read(query);
38084
+ }
38085
+ openBuckets() {
38086
+ return this.lts.openBuckets();
38087
+ }
38088
+ };
38089
+ //#endregion
38090
+ //#region src/node-stress-move-ledger.ts
38091
+ /**
38092
+ * @durable class=ledger owner=pipeline-orchestrator
38093
+ * write="one row per APPLIED failover move (never per attempt); write-behind"
38094
+ * retention="pruned by the sweep once older than the longest guard window (2 h) — a move no guard can read is growth with no reader"
38095
+ */
38096
+ var NODE_STRESS_MOVES_COLLECTION = "pipeline-orchestrator:node-stress-moves";
38097
+ var NODE_STRESS_MOVES_COLUMNS = [
38098
+ (
38099
+ /** `<deviceId>:<at>` — one camera can be moved more than once, and each move
38100
+ * spends its own slice of the budget. */
38101
+ {
38102
+ name: "id",
38103
+ type: "TEXT",
38104
+ primaryKey: true,
38105
+ notNull: true
38106
+ }),
38107
+ (
38108
+ /** Indexed: every question about a move is asked per-camera. */
38109
+ {
38110
+ name: "deviceId",
38111
+ type: "INTEGER",
38112
+ notNull: true
38113
+ }),
38114
+ (
38115
+ /** The node the camera LEFT — the return ban is about coming back here. */
38116
+ {
38117
+ name: "fromNodeId",
38118
+ type: "TEXT",
38119
+ notNull: true
38120
+ }),
38121
+ {
38122
+ name: "at",
38123
+ type: "INTEGER",
38124
+ notNull: true
38125
+ }
38126
+ ];
38127
+ var NODE_STRESS_MOVES_INDEXES = [{
38128
+ name: "idx_node_stress_moves_device",
38129
+ columns: ["deviceId"]
38130
+ }];
38131
+ function moveId(entry) {
38132
+ return `${entry.deviceId}:${entry.at}`;
38133
+ }
38134
+ var NODE_STRESS_MOVES_SPEC = {
38135
+ collection: NODE_STRESS_MOVES_COLLECTION,
38136
+ columns: NODE_STRESS_MOVES_COLUMNS,
38137
+ indexes: NODE_STRESS_MOVES_INDEXES,
38138
+ writeMode: "write-behind",
38139
+ keyOf: (row) => row.id,
38140
+ toValue: (row) => ({
38141
+ deviceId: row.deviceId,
38142
+ fromNodeId: row.fromNodeId,
38143
+ at: row.at
38144
+ }),
38145
+ fromRecord: (id, data) => {
38146
+ const deviceId = Number(data["deviceId"]);
38147
+ const at = Number(data["at"]);
38148
+ const fromNodeId = data["fromNodeId"];
38149
+ if (!Number.isFinite(deviceId) || !Number.isFinite(at)) return null;
38150
+ if (typeof fromNodeId !== "string" || fromNodeId.length === 0) return null;
38151
+ return {
38152
+ id,
38153
+ deviceId,
38154
+ fromNodeId,
38155
+ at
38156
+ };
38157
+ },
38158
+ deviceIdOf: (row) => row.deviceId,
38159
+ loadLimit: 1e4
38160
+ };
38161
+ var NodeStressMoveLedger = class {
38162
+ ledger;
38163
+ constructor(deps) {
38164
+ this.ledger = new DurableLedger({
38165
+ spec: NODE_STRESS_MOVES_SPEC,
38166
+ store: deps.store,
38167
+ logger: deps.logger
38168
+ });
38169
+ }
38170
+ static declare(store) {
38171
+ return DurableLedger.declare(store, NODE_STRESS_MOVES_SPEC);
38172
+ }
38173
+ /** Boot reseed. Returns the row count so the caller can say out loud how much
38174
+ * budget it recovered — "moves recovered: 0" after a restart is the line that
38175
+ * explains a burst of relocations. */
38176
+ async load() {
38177
+ return (await this.ledger.load()).length;
38178
+ }
38179
+ /** The history the guards read. Pure RAM — a guard must never wait on I/O. */
38180
+ entries() {
38181
+ return this.ledger.snapshot().map(({ deviceId, fromNodeId, at }) => ({
38182
+ deviceId,
38183
+ fromNodeId,
38184
+ at
38185
+ }));
38186
+ }
38187
+ /** Record one APPLIED move. Mirror first; the durable write follows and its
38188
+ * failure is logged, never thrown at a relocation that already happened. */
38189
+ record(entry) {
38190
+ this.ledger.put({
38191
+ ...entry,
38192
+ id: moveId(entry)
38193
+ });
38194
+ }
38195
+ /** Drop moves older than `cutoff` — no guard can read them. Returns the count. */
38196
+ pruneOlderThan(cutoff) {
38197
+ const keep = new Set(this.ledger.snapshot().filter((row) => row.at >= cutoff).map((row) => row.id));
38198
+ return this.ledger.pruneExcept(keep);
38199
+ }
38200
+ };
38201
+ //#endregion
37141
38202
  //#region src/dispatch-reconcile.ts
37142
38203
  function runnerAttachmentKey(nodeId, deviceId) {
37143
38204
  return `${nodeId}:${deviceId}`;
@@ -41603,6 +42664,79 @@ async function migrateStepGating(deps) {
41603
42664
  } });
41604
42665
  }
41605
42666
  //#endregion
42667
+ //#region src/zone-mirror-hydration.ts
42668
+ /** How long to wait for a camera list: the hub wires `ctx.api` AFTER the addon
42669
+ * init chain resolves, and `device-manager` answers a moment later still, so a
42670
+ * task kicked off from `onInitialize` loses both races. Same shape as the
42671
+ * bindings migration's poll — one budget covering "no api yet" and "api, but
42672
+ * device-manager not answering yet", because to this sweep they are the same
42673
+ * thing: no fleet to hydrate. */
42674
+ var CAMERA_LIST_WAIT_MS = 6e3;
42675
+ var POLL_INTERVAL_MS = 200;
42676
+ /**
42677
+ * Hydrate the `zones` mirror for every camera once, at boot. Never throws: a
42678
+ * hydration sweep that cannot run must not take the orchestrator's boot with
42679
+ * it — but it is never silent either, because a skipped sweep is exactly the
42680
+ * failure this exists to end.
42681
+ */
42682
+ async function hydrateZoneMirrorsAtBoot(deps) {
42683
+ const cameraIds = await readCameraIds(deps);
42684
+ if (cameraIds === null) return;
42685
+ for (const hydrator of deps.hydrators) {
42686
+ let written = 0;
42687
+ let unchanged = 0;
42688
+ let failed = 0;
42689
+ for (const deviceId of cameraIds) try {
42690
+ const outcome = await hydrator.hydrate(deviceId);
42691
+ if (outcome === "written") written++;
42692
+ else if (outcome === "unchanged") unchanged++;
42693
+ else failed++;
42694
+ } catch (err) {
42695
+ failed++;
42696
+ deps.logger.warn("zone mirror boot hydration failed for this camera", {
42697
+ tags: { deviceId },
42698
+ meta: {
42699
+ mirror: hydrator.mirror,
42700
+ error: errMsg(err)
42701
+ }
42702
+ });
42703
+ }
42704
+ deps.logger.info("zone mirror boot hydration complete", { meta: {
42705
+ mirror: hydrator.mirror,
42706
+ cameras: cameraIds.length,
42707
+ written,
42708
+ unchanged,
42709
+ failed
42710
+ } });
42711
+ }
42712
+ }
42713
+ /**
42714
+ * The camera fleet, or `null` when it never became readable inside the budget
42715
+ * — in which case this has already said so. Retried inside one budget rather
42716
+ * than once, because at orchestrator boot "no api yet" and "device-manager not
42717
+ * answering yet" are both transient and indistinguishable from here.
42718
+ */
42719
+ async function readCameraIds(deps) {
42720
+ const deadline = Date.now() + (deps.apiWaitMs ?? CAMERA_LIST_WAIT_MS);
42721
+ let lastError = null;
42722
+ for (;;) {
42723
+ const api = deps.api();
42724
+ if (api) try {
42725
+ return (await api.deviceManager.listAll.query({
42726
+ isCamera: true,
42727
+ projection: "slim"
42728
+ })).map((camera) => camera.id);
42729
+ } catch (err) {
42730
+ lastError = err;
42731
+ }
42732
+ if (Date.now() >= deadline) {
42733
+ deps.logger.warn("zones mirror boot hydration SKIPPED — camera list never became readable; mirror-only consumers keep whatever the last write left", { meta: { error: lastError === null ? "ctx.api unavailable" : errMsg(lastError) } });
42734
+ return null;
42735
+ }
42736
+ await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
42737
+ }
42738
+ }
42739
+ //#endregion
41606
42740
  //#region src/zone-rules-provider.ts
41607
42741
  /**
41608
42742
  * `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
@@ -41619,6 +42753,20 @@ async function migrateStepGating(deps) {
41619
42753
  * before persisting — partial / corrupt writes are rejected outright
41620
42754
  * since rules drive runtime filtering and a bad payload would silently
41621
42755
  * widen the operator's intended scope.
42756
+ *
42757
+ * ── THE MIRROR IS HYDRATED AT BOOT, NOT ONLY ON MUTATION ──────────
42758
+ *
42759
+ * Same root cause as the `zones` slice (see `zones-provider.ts`): the
42760
+ * mirror used to be written only by `persist`, so a camera nobody had
42761
+ * mutated since its runtime-state row was last written had NO
42762
+ * `zone-rules` slice, and every mirror-only consumer — motion-wasm's
42763
+ * zone gate, the detection-pipeline zone gate, the admin rules editor
42764
+ * — read "no rules" until an operator happened to save one. Zones and
42765
+ * rules gate together, so hydrating one without the other still leaves
42766
+ * both runner-side gates inert. {@link ZoneRulesProvider.hydrateMirror}
42767
+ * is what the boot sweep (`zone-mirror-hydration.ts`) calls; it is an
42768
+ * RPC read, never an event replay (D8), and a stage that could not be
42769
+ * read leaves the mirror untouched (D49).
41622
42770
  */
41623
42771
  /**
41624
42772
  * Every zone-rule stage, in the declared enum order. The unified device-state
@@ -41662,6 +42810,11 @@ var ZoneRulesProvider = class {
41662
42810
  * write on one stage can never drop the other. Built lazily + memoised.
41663
42811
  */
41664
42812
  stateByDevice = /* @__PURE__ */ new Map();
42813
+ /** Per-device fingerprint of the slice this process last successfully wrote.
42814
+ * Absent ⇒ never mirrored here, so the next hydration writes. */
42815
+ mirroredFingerprint = /* @__PURE__ */ new Map();
42816
+ /** Devices already reported as unmirrorable — one warn per episode. */
42817
+ reportedMirrorFailure = /* @__PURE__ */ new Set();
41665
42818
  constructor(ctx) {
41666
42819
  this.ctx = ctx;
41667
42820
  }
@@ -41704,10 +42857,52 @@ var ZoneRulesProvider = class {
41704
42857
  if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
41705
42858
  await this.persist(deviceId, stage, parsed.data);
41706
42859
  }
42860
+ /**
42861
+ * Reconcile ONE device's `zone-rules` mirror against the durable block.
42862
+ * Called by the boot sweep for every camera. Never throws.
42863
+ *
42864
+ * A stage whose read FAILED aborts the whole hydration: the mirror is a
42865
+ * single slice carrying every stage, so writing a partially-read block would
42866
+ * publish "this stage has no rules" off a store blip — and an empty
42867
+ * `motion`/`detection` array is what makes a gate stop gating.
42868
+ */
42869
+ async hydrateMirror(deviceId) {
42870
+ const perDevice = this.stageCache(deviceId);
42871
+ for (const stage of ALL_STAGES) {
42872
+ const read = await this.readRules(deviceId, stage);
42873
+ if (!read.ok) return "unreadable";
42874
+ perDevice.set(stage, read.rules);
42875
+ }
42876
+ const slice = await this.buildSliceValue(deviceId, perDevice);
42877
+ const fingerprint = JSON.stringify(slice);
42878
+ if (this.mirroredFingerprint.get(deviceId) === fingerprint) return "unchanged";
42879
+ try {
42880
+ await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
42881
+ capName: ZONE_RULES_CAP_NAME,
42882
+ slice
42883
+ });
42884
+ } catch (err) {
42885
+ if (!this.reportedMirrorFailure.has(deviceId)) {
42886
+ this.reportedMirrorFailure.add(deviceId);
42887
+ this.ctx.logger.warn("zone-rules mirror write failed — mirror-only zone gates see NO rules for this camera until it lands", {
42888
+ tags: { deviceId },
42889
+ meta: { error: err instanceof Error ? err.message : String(err) }
42890
+ });
42891
+ }
42892
+ return "write-failed";
42893
+ }
42894
+ const first = !this.mirroredFingerprint.has(deviceId);
42895
+ this.mirroredFingerprint.set(deviceId, fingerprint);
42896
+ this.reportedMirrorFailure.delete(deviceId);
42897
+ if (first) this.ctx.logger.info("zone-rules mirror hydrated from the durable block", { tags: { deviceId } });
42898
+ return "written";
42899
+ }
41707
42900
  /** Drop a device's cache entries. Called when the device is removed. */
41708
42901
  forgetDevice(deviceId) {
41709
42902
  this.cache.delete(deviceId);
41710
42903
  this.stateByDevice.delete(deviceId);
42904
+ this.mirroredFingerprint.delete(deviceId);
42905
+ this.reportedMirrorFailure.delete(deviceId);
41711
42906
  }
41712
42907
  /** Cap-surface read: a failure folds to the empty list, as it always has. */
41713
42908
  async loadRules(deviceId, stage) {
@@ -41722,11 +42917,7 @@ var ZoneRulesProvider = class {
41722
42917
  * momentary store blip into a permanent one.
41723
42918
  */
41724
42919
  async readRules(deviceId, stage) {
41725
- let perDevice = this.cache.get(deviceId);
41726
- if (!perDevice) {
41727
- perDevice = /* @__PURE__ */ new Map();
41728
- this.cache.set(deviceId, perDevice);
41729
- }
42920
+ const perDevice = this.stageCache(deviceId);
41730
42921
  const cached = perDevice.get(stage);
41731
42922
  if (cached) return {
41732
42923
  ok: true,
@@ -41767,11 +42958,7 @@ var ZoneRulesProvider = class {
41767
42958
  };
41768
42959
  }
41769
42960
  async persist(deviceId, stage, rules) {
41770
- let perDevice = this.cache.get(deviceId);
41771
- if (!perDevice) {
41772
- perDevice = /* @__PURE__ */ new Map();
41773
- this.cache.set(deviceId, perDevice);
41774
- }
42961
+ const perDevice = this.stageCache(deviceId);
41775
42962
  perDevice.set(stage, rules);
41776
42963
  await this.rulesState(deviceId).update((prev) => ({
41777
42964
  ...prev,
@@ -41783,6 +42970,8 @@ var ZoneRulesProvider = class {
41783
42970
  capName: ZONE_RULES_CAP_NAME,
41784
42971
  slice: sliceValue
41785
42972
  });
42973
+ this.mirroredFingerprint.set(deviceId, JSON.stringify(sliceValue));
42974
+ this.reportedMirrorFailure.delete(deviceId);
41786
42975
  } catch (err) {
41787
42976
  this.ctx.logger.debug("zone-rules slice mirror failed", {
41788
42977
  tags: { deviceId },
@@ -41801,6 +42990,15 @@ var ZoneRulesProvider = class {
41801
42990
  * also warms the cache). Iterates {@link ALL_STAGES} so it stays exhaustive
41802
42991
  * over the cap's stage discriminator without a per-stage branch.
41803
42992
  */
42993
+ /** The per-stage cache map for a device, created on first use. */
42994
+ stageCache(deviceId) {
42995
+ let perDevice = this.cache.get(deviceId);
42996
+ if (!perDevice) {
42997
+ perDevice = /* @__PURE__ */ new Map();
42998
+ this.cache.set(deviceId, perDevice);
42999
+ }
43000
+ return perDevice;
43001
+ }
41804
43002
  async buildSliceValue(deviceId, perDevice) {
41805
43003
  const slice = {
41806
43004
  motion: [],
@@ -41818,11 +43016,38 @@ var ZoneRulesProvider = class {
41818
43016
  *
41819
43017
  * Per-camera CRUD over polygon detection zones. Persists to the
41820
43018
  * orchestrator's per-device settings store under the `zones` key and
41821
- * mirrors every change into the device-state `zones` slice via
43019
+ * mirrors the catalogue into the device-state `zones` slice via
41822
43020
  * `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
41823
43021
  * pipeline-executor, analytics, admin UI) read the live state with
41824
43022
  * the canonical `dev.state.zones.onChanged` channel.
41825
43023
  *
43024
+ * ── THIS PROVIDER OWNS THE MIRROR FOR ITS WHOLE LIFETIME ──────────
43025
+ *
43026
+ * The mirror used to be written in exactly ONE place: `persist`, i.e.
43027
+ * only when an operator mutates a zone. Nothing seeded it at startup,
43028
+ * so a camera whose runtime-state row had never been written (or had
43029
+ * been reset) had no `zones` slice at all, and every mirror-only
43030
+ * consumer concluded "this camera has no zones" — forever, because no
43031
+ * mutation was coming. Live cost (camera 617 'Parcheggio',
43032
+ * 2026-08-12): `zones.listZones {617}` returned 'Parcheggio papà'
43033
+ * while `deviceState.getCapSlice {617,'zones'}` returned `null`;
43034
+ * occupancy dropped all three parked cars into `unzoned`, its zone
43035
+ * rule could never fire, and the admin Zones tab — which reads the
43036
+ * same mirror — showed "No zones yet".
43037
+ *
43038
+ * So the mirror is reconciled against the durable catalogue on the
43039
+ * READ path too ({@link ZonesProvider.hydrateMirror}), and the boot
43040
+ * sweep in `zone-mirror-hydration.ts` walks every camera once at
43041
+ * startup. That is deliberately an RPC read, never an event replay:
43042
+ * events are lossy telemetry and a slice change that was dropped is
43043
+ * never re-sent (D8).
43044
+ *
43045
+ * Two rules the hydration path must keep (D49):
43046
+ * - a durable read that FAILED changes nothing — it must never be
43047
+ * mirrored, and must not be cached as "this camera has no zones";
43048
+ * - hydration is a reconcile, not a mutation: `onZonesChanged` is
43049
+ * NOT fired, so nothing downstream re-dispatches on a boot read.
43050
+ *
41826
43051
  * Onboard / firmware-reported zones are out of scope for now — every
41827
43052
  * zone is operator-drawn. The provider keeps the surface symmetric:
41828
43053
  * `addZone` rejects id collisions, `updateZone` requires an existing
@@ -41831,6 +43056,11 @@ var ZoneRulesProvider = class {
41831
43056
  var ZONES_STORE_KEY = "zones";
41832
43057
  var ZONES_CAP_NAME = "zones";
41833
43058
  var ZonesArraySchema = array(ZoneSchema);
43059
+ /** Identity of a mirrored catalogue — cheap enough to compare on every read,
43060
+ * and it changes whenever anything an operator can see changes. */
43061
+ function fingerprintZones(zones) {
43062
+ return JSON.stringify(zones);
43063
+ }
41834
43064
  var ZonesProvider = class {
41835
43065
  ctx;
41836
43066
  /** Per-device cache. Hydrated lazily on first read for a device. */
@@ -41841,6 +43071,16 @@ var ZonesProvider = class {
41841
43071
  * be dropped on persist. Built lazily + memoised per device.
41842
43072
  */
41843
43073
  stateByDevice = /* @__PURE__ */ new Map();
43074
+ /**
43075
+ * Per-device fingerprint of the catalogue this process last successfully
43076
+ * wrote to the mirror. Absent ⇒ this process has never mirrored the device,
43077
+ * so the next read hydrates; a write that FAILED leaves it absent, which is
43078
+ * what makes the retry happen on the next read rather than never.
43079
+ */
43080
+ mirroredFingerprint = /* @__PURE__ */ new Map();
43081
+ /** Devices already reported as unmirrorable — keeps the warn to one per
43082
+ * episode instead of one per read. */
43083
+ reportedMirrorFailure = /* @__PURE__ */ new Set();
41844
43084
  constructor(ctx) {
41845
43085
  this.ctx = ctx;
41846
43086
  }
@@ -41866,55 +43106,141 @@ var ZonesProvider = class {
41866
43106
  }
41867
43107
  return handle;
41868
43108
  }
43109
+ /**
43110
+ * The device's catalogue — and, on the way past, the one place a mirror-only
43111
+ * consumer's boot blindness is cured: every authoritative read reconciles the
43112
+ * device-state slice against what it just read.
43113
+ */
41869
43114
  async listZones({ deviceId }) {
41870
- return this.loadZones(deviceId);
43115
+ const read = await this.readCatalogue(deviceId);
43116
+ if (read.ok) await this.ensureMirror(deviceId, read.zones);
43117
+ return read.zones;
41871
43118
  }
41872
43119
  async addZone({ deviceId, zone }) {
41873
- const existing = await this.loadZones(deviceId);
43120
+ const existing = await this.readForMutation(deviceId);
41874
43121
  if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
41875
43122
  await this.persist(deviceId, [...existing, zone]);
41876
43123
  }
41877
43124
  async updateZone({ deviceId, zone }) {
41878
- const existing = await this.loadZones(deviceId);
43125
+ const existing = await this.readForMutation(deviceId);
41879
43126
  if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
41880
43127
  const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
41881
43128
  await this.persist(deviceId, next);
41882
43129
  }
41883
43130
  async removeZone({ deviceId, zoneId }) {
41884
- const existing = await this.loadZones(deviceId);
43131
+ const existing = await this.readForMutation(deviceId);
41885
43132
  if (!existing.some((entry) => entry.id === zoneId)) return;
41886
43133
  await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
41887
43134
  }
41888
43135
  /**
43136
+ * Reconcile ONE device's mirror against the durable catalogue. The boot
43137
+ * sweep (`zone-mirror-hydration.ts`) calls this for every camera so a
43138
+ * mirror-only consumer never starts blind; `listZones` calls it too, so a
43139
+ * camera adopted after boot is covered by its first read.
43140
+ *
43141
+ * Never throws — a hydration that cannot happen is reported, and reported
43142
+ * once (see {@link ZoneMirrorHydration}).
43143
+ */
43144
+ async hydrateMirror(deviceId) {
43145
+ const read = await this.readCatalogue(deviceId);
43146
+ if (!read.ok) return "unreadable";
43147
+ return this.ensureMirror(deviceId, read.zones);
43148
+ }
43149
+ /**
41889
43150
  * Drop a device's cache entry. Called when the device is removed so
41890
43151
  * the next attach starts from a fresh disk read.
41891
43152
  */
41892
43153
  forgetDevice(deviceId) {
41893
43154
  this.cache.delete(deviceId);
41894
43155
  this.stateByDevice.delete(deviceId);
43156
+ this.mirroredFingerprint.delete(deviceId);
43157
+ this.reportedMirrorFailure.delete(deviceId);
43158
+ }
43159
+ /**
43160
+ * The catalogue for a mutation. A mutation is read-modify-write over the
43161
+ * WHOLE array, so proceeding from a failed read would persist the operator's
43162
+ * zones away — refuse instead.
43163
+ */
43164
+ async readForMutation(deviceId) {
43165
+ const read = await this.readCatalogue(deviceId);
43166
+ if (!read.ok) throw new Error(`zones: catalogue unreadable for device ${deviceId} — refusing to write`);
43167
+ return read.zones;
41895
43168
  }
41896
- async loadZones(deviceId) {
43169
+ /**
43170
+ * Read the durable catalogue, cached per device. A FAILED read is neither
43171
+ * cached nor reported as `[]` — the caller decides what an unknown answer
43172
+ * means for it.
43173
+ */
43174
+ async readCatalogue(deviceId) {
41897
43175
  const cached = this.cache.get(deviceId);
41898
- if (cached) return cached;
41899
- let zones = [];
43176
+ if (cached) return {
43177
+ ok: true,
43178
+ zones: cached
43179
+ };
43180
+ let zones;
41900
43181
  try {
41901
43182
  zones = await this.zonesState(deviceId).get();
41902
43183
  } catch (err) {
41903
- this.ctx.logger.warn("zones store read failed — using empty list", {
43184
+ this.ctx.logger.warn("zones store read failed — catalogue UNKNOWN for this device", {
41904
43185
  tags: { deviceId },
41905
43186
  meta: { error: err instanceof Error ? err.message : String(err) }
41906
43187
  });
43188
+ return {
43189
+ ok: false,
43190
+ zones: []
43191
+ };
41907
43192
  }
41908
43193
  this.cache.set(deviceId, zones);
41909
- return zones;
43194
+ return {
43195
+ ok: true,
43196
+ zones
43197
+ };
41910
43198
  }
41911
- async persist(deviceId, zones) {
41912
- this.cache.set(deviceId, zones);
41913
- await this.zonesState(deviceId).set(zones);
43199
+ /**
43200
+ * Make the device-state mirror agree with `zones`. Idempotent per process
43201
+ * via the fingerprint; the hub itself also no-ops an identical
43202
+ * `setCapSlice`, so this is belt-and-braces against needless RPCs, not
43203
+ * against needless writes.
43204
+ */
43205
+ async ensureMirror(deviceId, zones) {
43206
+ const fingerprint = fingerprintZones(zones);
43207
+ if (this.mirroredFingerprint.get(deviceId) === fingerprint) return "unchanged";
43208
+ try {
43209
+ await this.writeMirror(deviceId, zones);
43210
+ } catch (err) {
43211
+ if (!this.reportedMirrorFailure.has(deviceId)) {
43212
+ this.reportedMirrorFailure.add(deviceId);
43213
+ this.ctx.logger.warn("zones mirror write failed — mirror-only consumers see NO zones for this camera until it lands", {
43214
+ tags: { deviceId },
43215
+ meta: {
43216
+ zones: zones.length,
43217
+ error: err instanceof Error ? err.message : String(err)
43218
+ }
43219
+ });
43220
+ }
43221
+ return "write-failed";
43222
+ }
43223
+ const first = !this.mirroredFingerprint.has(deviceId);
43224
+ this.mirroredFingerprint.set(deviceId, fingerprint);
43225
+ this.reportedMirrorFailure.delete(deviceId);
43226
+ if (first) this.ctx.logger.info("zones mirror hydrated from the durable catalogue", {
43227
+ tags: { deviceId },
43228
+ meta: { zones: zones.length }
43229
+ });
43230
+ return "written";
43231
+ }
43232
+ async writeMirror(deviceId, zones) {
41914
43233
  await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
41915
43234
  capName: ZONES_CAP_NAME,
41916
43235
  slice: { zones }
41917
43236
  });
43237
+ }
43238
+ async persist(deviceId, zones) {
43239
+ this.cache.set(deviceId, zones);
43240
+ await this.zonesState(deviceId).set(zones);
43241
+ await this.writeMirror(deviceId, zones);
43242
+ this.mirroredFingerprint.set(deviceId, fingerprintZones(zones));
43243
+ this.reportedMirrorFailure.delete(deviceId);
41918
43244
  this.ctx.onZonesChanged?.(deviceId, zones);
41919
43245
  }
41920
43246
  };
@@ -42009,8 +43335,18 @@ async function buildOrchestratorControllers(deps) {
42009
43335
  readGlobalSettings: () => globalSettings,
42010
43336
  getInitTimestamp: () => deps.initTimestamp
42011
43337
  });
43338
+ const nodeStressMoves = new NodeStressMoveLedger({
43339
+ store: deps.ctx().api.settingsStore,
43340
+ logger: deps.ctx().logger.child("node-stress")
43341
+ });
43342
+ const nodeStressLts = new NodeStressLts({
43343
+ store: deps.ctx().api.settingsStore,
43344
+ logger: deps.ctx().logger.child("node-stress-lts")
43345
+ });
42012
43346
  const nodeStress = new NodeStressController({
42013
43347
  ledger,
43348
+ moves: nodeStressMoves,
43349
+ lts: nodeStressLts,
42014
43350
  topology,
42015
43351
  settingsStore,
42016
43352
  logger: deps.ctx().logger,
@@ -42019,6 +43355,17 @@ async function buildOrchestratorControllers(deps) {
42019
43355
  readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
42020
43356
  readGlobalSettings: () => globalSettings
42021
43357
  });
43358
+ await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
43359
+ deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg(err) } });
43360
+ });
43361
+ await NodeStressLts.declare(deps.ctx().api.settingsStore).catch((err) => {
43362
+ deps.ctx().logger.warn("node-stress statistics unavailable — no baseline this boot", { meta: { error: errMsg(err) } });
43363
+ });
43364
+ const nodeStressLtsTimer = setInterval(() => {
43365
+ nodeStressLts.flushDue().catch((err) => {
43366
+ deps.ctx().logger.warn("node-stress statistics flush failed", { meta: { error: errMsg(err) } });
43367
+ });
43368
+ }, 6e4);
42022
43369
  nodeStress.start();
42023
43370
  const inferenceRotation = new RoundRobinInferenceDeviceRotation();
42024
43371
  /**
@@ -42327,6 +43674,19 @@ async function buildOrchestratorControllers(deps) {
42327
43674
  });
42328
43675
  }
42329
43676
  });
43677
+ hydrateZoneMirrorsAtBoot({
43678
+ api: () => deps.ctx().api ?? null,
43679
+ hydrators: [{
43680
+ mirror: "zones",
43681
+ hydrate: (deviceId) => zonesProvider.hydrateMirror(deviceId)
43682
+ }, {
43683
+ mirror: "zone-rules",
43684
+ hydrate: (deviceId) => zoneRulesProvider.hydrateMirror(deviceId)
43685
+ }],
43686
+ logger: deps.ctx().logger.child("zones")
43687
+ }).catch((err) => {
43688
+ deps.ctxIfReady()?.logger.warn("zones mirror boot hydration failed", { meta: { error: errMsg(err) } });
43689
+ });
42330
43690
  const unsubOrchestratorSubscriptions = wireOrchestratorSubscriptions({
42331
43691
  eventBus: deps.ctx().eventBus,
42332
43692
  logger: deps.ctx().logger,
@@ -42349,6 +43709,7 @@ async function buildOrchestratorControllers(deps) {
42349
43709
  reconcile.scheduleReconcile();
42350
43710
  const detectionWiring = new DetectionWiringController({
42351
43711
  ctx: () => deps.ctx(),
43712
+ featuresStore: deps.ctx().api.settingsStore,
42352
43713
  ledger,
42353
43714
  placement,
42354
43715
  audio,
@@ -42373,10 +43734,15 @@ async function buildOrchestratorControllers(deps) {
42373
43734
  deviceHasOnboardMotionCap: (deviceId) => deps.deviceHasOnboardMotionCap(deviceId),
42374
43735
  deviceSettingsSchema: () => deps.deviceSettingsSchema()
42375
43736
  });
43737
+ await detectionWiring.hydrateFeaturesMirror().catch((err) => {
43738
+ deps.ctx().logger.warn("device-features mirror not restored — this boot starts cold", { meta: { error: errMsg(err) } });
43739
+ });
42376
43740
  return {
42377
43741
  ledger,
42378
43742
  topology,
42379
43743
  loadService,
43744
+ nodeStressLts,
43745
+ nodeStressLtsTimer,
42380
43746
  audio,
42381
43747
  settingsStore,
42382
43748
  loadShed,
@@ -42630,25 +43996,37 @@ function buildGlobalSettingsSections(options) {
42630
43996
  id: NATIVE_LEASE_SECTION_ID,
42631
43997
  title: "Native frame lease",
42632
43998
  tab: "pipeline",
42633
- description: "How long each decode worker keeps a full-resolution copy of a delivered frame in RAM so a LATE native crop (post-analysis snapshot, face/plate detail) can still be cut from real pixels instead of the ≤640 detection frame. Cost is real: a retained 4K frame is ~24.9 MB on the default pinned-RGB24 path (~12.4 MB as YUV420P), a 1080p frame ~6.2 MB / ~3.1 MB. Worst case for ONE busy camera frameBytes × delivered fps × TTL seconds, capped by the budget below. Takes effect on the NEXT decode session for a camera, not on sessions already running.",
43999
+ description: "How a decode worker keeps native pixels available for a LATE crop (post-analysis snapshot, face/plate detail) instead of falling back to the ≤640 detection frame. Two things are kept and they cost very differently: a HELD FRAME is a full native raster (~24.9 MB at 4K on the default pinned-RGB24 path, ~6.2 MB at 1080p) and lives only until its own detection result arrives; a TILE is one subject cut from that frame at native resolution and JPEG-encoded (~60-120 KB at 4K), and lives for tens of seconds. A frame on which nothing was detected produces no tiles and costs nothing. Takes effect on the NEXT decode session for a camera, not on sessions already running.",
42634
44000
  fields: [
42635
44001
  {
42636
- key: NATIVE_LEASE_TTL_KEY,
44002
+ key: NATIVE_LEASE_HOLD_KEY,
42637
44003
  type: "slider",
42638
- label: "Lease TTL",
42639
- description: "How long a retained native frame stays claimable before it counts as a miss. It must cover the whole late-crop horizon detection inference, the cross-process hop to hub post-analysis, tracking, and the tRPC crop round-trip back. Below ~500 ms the busiest cameras outrun it and their crops silently fall back to the downscaled detection frame; every extra second multiplies resident RAM by roughly (frame bytes × delivered fps). 1200 ms is the shipped value.",
42640
- min: NATIVE_LEASE_TTL_FIELD.min,
42641
- max: NATIVE_LEASE_TTL_FIELD.max,
42642
- step: NATIVE_LEASE_TTL_FIELD.step,
42643
- default: NATIVE_LEASE_TTL_FIELD.default,
44004
+ label: "Frames held at once",
44005
+ description: "How many delivered frames a worker keeps alive while waiting for their detection results. A frame is freed as soon as its own result comes back and its subject tiles have been cut, so the steady state is inference latency × delivered fps 1 to 4 frames in practice. This number is only the bound above which the OLDEST held frame is dropped, which is what stops a runner that has stopped answering from pinning RAM. Raising it does not improve crop hit rate; it buys tolerance for a slow runner, and holdOverflow on the metrics line is what tells you that you need it.",
44006
+ min: NATIVE_LEASE_HOLD_FIELD.min,
44007
+ max: NATIVE_LEASE_HOLD_FIELD.max,
44008
+ step: NATIVE_LEASE_HOLD_FIELD.step,
44009
+ default: NATIVE_LEASE_HOLD_FIELD.default,
42644
44010
  showValue: true,
42645
- unit: "ms"
44011
+ unit: "frames"
44012
+ },
44013
+ {
44014
+ key: NATIVE_LEASE_TILE_BUDGET_KEY,
44015
+ type: "slider",
44016
+ label: "Subject tile RAM",
44017
+ description: "RAM per decode worker for the compressed SUBJECT TILES — the native-resolution crops taken at the moment a frame's detections are known, and kept long after the frame itself is gone. This is what serves a crop that arrives seconds late, which measurement says is the ordinary case (2 to 10 seconds on this cluster). At ~60-120 KB a tile, 64 MB is many hundreds of subjects. 0 turns tiles OFF and restores the old behaviour, where a late crop had nothing to fall back to but the ≤640 detection frame.",
44018
+ min: NATIVE_LEASE_TILE_BUDGET_FIELD.min,
44019
+ max: NATIVE_LEASE_TILE_BUDGET_FIELD.max,
44020
+ step: NATIVE_LEASE_TILE_BUDGET_FIELD.step,
44021
+ default: NATIVE_LEASE_TILE_BUDGET_FIELD.default,
44022
+ showValue: true,
44023
+ unit: "MB"
42646
44024
  },
42647
44025
  {
42648
44026
  key: NATIVE_LEASE_BUDGET_KEY,
42649
44027
  type: "slider",
42650
44028
  label: "Lease RAM ceiling",
42651
- description: "Hard RAM ceiling for retained frames, PER decode worker (one worker per camera per plane). This is a safety ceiling, not the working size — the TTL above is what normally reclaims frames, and at 1024 MB the ceiling is never the binding constraint on a single camera. Lower it on a small host to bound the worst case. 0 DISABLES the lease entirely and falls the worker back to the tiny GPU surface ring, which misses roughly 85% of late crops — that is the behaviour the lease exists to replace, so 0 is a diagnostic setting, not a tuning one.",
44029
+ description: "Hard RAM ceiling for held frames, PER decode worker (one worker per camera per plane). This is a safety ceiling, not the working size — the hold count above is what reclaims frames now, and the ceiling is the number above which something is wrong. Lower it on a small host to bound the worst case. 0 DISABLES the lease entirely and falls the worker back to the tiny GPU surface ring, which misses roughly 85% of late crops — that is the behaviour the lease exists to replace, so 0 is a diagnostic setting, not a tuning one.",
42652
44030
  min: NATIVE_LEASE_BUDGET_FIELD.min,
42653
44031
  max: NATIVE_LEASE_BUDGET_FIELD.max,
42654
44032
  step: NATIVE_LEASE_BUDGET_FIELD.step,
@@ -42917,6 +44295,18 @@ function deriveRuntimeSettings(config) {
42917
44295
  }
42918
44296
  //#endregion
42919
44297
  //#region src/index.ts
44298
+ /**
44299
+ * The FULL action catalog, under the name the HUB HARVESTS.
44300
+ *
44301
+ * The hub's forked-addon harvest imports this entry module and reads the
44302
+ * `customActions` NAMED export (`loadForkedCustomActionCatalog`) — returning a
44303
+ * catalog from `onInitialize` is not enough. Without this line the bridge
44304
+ * answers *"no addon 'pipeline-orchestrator' registers custom actions"* for
44305
+ * every action here, which is what `dumpState` had been doing, silently, since
44306
+ * it was written: an admin diagnostic that 404s is worse than no diagnostic,
44307
+ * because nobody discovers it is missing until they need it.
44308
+ */
44309
+ var customActions = pipelineOrchestratorActions;
42920
44310
  var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddon {
42921
44311
  /** This node's Moleculer nodeId (from this.ctx.kernel.localNodeId). */
42922
44312
  localNodeId = "hub";
@@ -43134,6 +44524,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
43134
44524
  pendingRetryTimer = null;
43135
44525
  /** Periodic auto-rebalance sweep timer (drift correction under hysteresis). */
43136
44526
  autoRebalanceTimer = null;
44527
+ /** Node-stress five-minute statistics + its flush timer (see
44528
+ * `node-stress-lts.ts`). `observe` mode's first durable output. */
44529
+ nodeStressLts = null;
44530
+ nodeStressLtsTimer = null;
43137
44531
  initTimestamp = 0;
43138
44532
  /** Storage migration maintenance lease. It only gates dispatch; it does not
43139
44533
  * change any camera wrapper or persistent pipeline configuration. */
@@ -43227,6 +44621,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
43227
44621
  this.unsubOrchestratorSubscriptions = controllers.unsubOrchestratorSubscriptions;
43228
44622
  this.pendingRetryTimer = controllers.pendingRetryTimer;
43229
44623
  this.autoRebalanceTimer = controllers.autoRebalanceTimer;
44624
+ this.nodeStressLts = controllers.nodeStressLts;
44625
+ this.nodeStressLtsTimer = controllers.nodeStressLtsTimer;
43230
44626
  return {
43231
44627
  providers: [
43232
44628
  {
@@ -43255,7 +44651,23 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
43255
44651
  }
43256
44652
  ],
43257
44653
  customActions: pipelineOrchestratorActions,
43258
- actionHandlers: { dumpState: async () => this.dumpDiagnostics() }
44654
+ actionHandlers: {
44655
+ dumpState: async () => this.dumpDiagnostics(),
44656
+ nodeStressStats: async (input) => {
44657
+ const lts = this.nodeStressLts;
44658
+ const moves = this.nodeStress?.historyView() ?? [];
44659
+ if (!lts) return {
44660
+ rows: [],
44661
+ open: [],
44662
+ moves: [...moves]
44663
+ };
44664
+ return {
44665
+ rows: (await lts.read(input)).map(withStatsMean),
44666
+ open: lts.openBuckets().map(withStatsMean),
44667
+ moves: [...moves]
44668
+ };
44669
+ }
44670
+ }
43259
44671
  };
43260
44672
  }
43261
44673
  /**
@@ -43290,6 +44702,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
43290
44702
  clearInterval(this.autoRebalanceTimer);
43291
44703
  this.autoRebalanceTimer = null;
43292
44704
  }
44705
+ if (this.nodeStressLtsTimer !== null) {
44706
+ clearInterval(this.nodeStressLtsTimer);
44707
+ this.nodeStressLtsTimer = null;
44708
+ }
44709
+ this.nodeStressLts = null;
43293
44710
  this.unsubOrchestratorSubscriptions?.();
43294
44711
  this.unsubOrchestratorSubscriptions = null;
43295
44712
  this.reconcile?.dispose();
@@ -44243,5 +45660,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
44243
45660
  return this.isSessionCamera(deviceId) && !this.session.hasActiveSession(deviceId);
44244
45661
  }
44245
45662
  };
45663
+ /** The derived mean for one statistics row — see `node-stress-lts.ts`. */
45664
+ function withStatsMean(row) {
45665
+ return {
45666
+ ...row,
45667
+ mean: row.samples > 0 ? row.sum / row.samples : 0
45668
+ };
45669
+ }
44246
45670
  //#endregion
44247
- export { balance, computeCapacityScore, PipelineOrchestratorAddon as default, pipelineOrchestratorActions };
45671
+ export { balance, computeCapacityScore, customActions, PipelineOrchestratorAddon as default, pipelineOrchestratorActions };