@camstack/addon-pipeline-orchestrator 1.2.50 → 1.2.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CmY_Q4o0.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-Cg1tRb7b.mjs} +3 -3
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Cxz75cof.mjs +26 -0
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-Eyie_s8X.mjs +26 -0
- package/dist/{hostInit-C5jtS8Jf.mjs → hostInit-JhsDp9rw.mjs} +3 -3
- package/dist/index.js +1527 -102
- package/dist/index.mjs +1527 -103
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DihZTe5Y.mjs +0 -26
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-Cu02LUjg.mjs +0 -26
package/dist/index.js
CHANGED
|
@@ -13893,7 +13893,23 @@ var NotificationActionSchema = object({
|
|
|
13893
13893
|
* else — see `notification-center/action-token.ts` for what that does and
|
|
13894
13894
|
* does not buy.
|
|
13895
13895
|
*/
|
|
13896
|
-
destructive: boolean().optional()
|
|
13896
|
+
destructive: boolean().optional(),
|
|
13897
|
+
/**
|
|
13898
|
+
* How the tap should REACH the url.
|
|
13899
|
+
*
|
|
13900
|
+
* `navigate` (absent, and every button authored before this field) opens it:
|
|
13901
|
+
* the phone leaves the notification and shows whatever the callback returns.
|
|
13902
|
+
* That is right for a button whose answer the operator wants to read.
|
|
13903
|
+
*
|
|
13904
|
+
* `background` fires it as a POST and stays put. It exists for the buttons
|
|
13905
|
+
* whose whole point is not to interrupt — "silence this for 30 minutes" is
|
|
13906
|
+
* an answer to the notification, and being thrown into a browser tab to
|
|
13907
|
+
* confirm it costs more attention than the notification did. A backend that
|
|
13908
|
+
* cannot do a background call renders it as an ordinary link (the adapters
|
|
13909
|
+
* fall back rather than dropping the button), so this is a preference, never
|
|
13910
|
+
* a requirement.
|
|
13911
|
+
*/
|
|
13912
|
+
mode: _enum(["navigate", "background"]).optional()
|
|
13897
13913
|
});
|
|
13898
13914
|
/**
|
|
13899
13915
|
* The canonical notification. `body` is the only hard field (Apprise model).
|
|
@@ -14894,6 +14910,9 @@ var NcSystemEventConditionSchema = object({
|
|
|
14894
14910
|
nodeIds: array(string().min(1)).min(1).optional(),
|
|
14895
14911
|
packageNames: array(string().min(1)).min(1).optional()
|
|
14896
14912
|
});
|
|
14913
|
+
/** Hard ceiling on a window (24h). A snooze that could not expire would be an
|
|
14914
|
+
* outage the operator asked for once and forgot. */
|
|
14915
|
+
var NC_SNOOZE_MAX_MINUTES = 1440;
|
|
14897
14916
|
/** Weekly schedule — OR of windows; absence on the rule = always active. */
|
|
14898
14917
|
var NcScheduleSchema = object({
|
|
14899
14918
|
windows: array(object({
|
|
@@ -15270,15 +15289,15 @@ var NcConditionsSchema = object({
|
|
|
15270
15289
|
* (an `immediate` rule naming an `audio-*` class, one notification per
|
|
15271
15290
|
* classified sample) stays exactly as it was for rules that already use it.
|
|
15272
15291
|
*
|
|
15273
|
-
*
|
|
15274
|
-
* rather than an
|
|
15275
|
-
* (`camstack/src/data/notification-center.ts`, guarded by
|
|
15276
|
-
* `scripts/check-viewer-condition-mirror.ts`) and its rule editor
|
|
15292
|
+
* In {@link NC_CONDITION_CATALOG} since P2, and the ORDER it got there is the
|
|
15293
|
+
* rule rather than an accident: the viewer mirrors the descriptor enums BY
|
|
15294
|
+
* HAND (`camstack/src/data/notification-center.ts`, guarded by
|
|
15295
|
+
* `scripts/check-viewer-condition-mirror.ts`) and its rule editor strips the
|
|
15277
15296
|
* condition fields it does not know when a rule is saved from the phone.
|
|
15278
15297
|
* Publishing an editor for a condition the app cannot round-trip is how an
|
|
15279
|
-
* operator loses a rule's conditions by opening it — so the
|
|
15280
|
-
*
|
|
15281
|
-
*
|
|
15298
|
+
* operator loses a rule's conditions by opening it — so the viewer mirror
|
|
15299
|
+
* (P3, shipped) went FIRST, and the descriptor an editor renders from
|
|
15300
|
+
* follows here.
|
|
15282
15301
|
*/
|
|
15283
15302
|
audio: NcAudioConditionSchema.optional()
|
|
15284
15303
|
});
|
|
@@ -15514,6 +15533,30 @@ var NcRuleInputSchema = object({
|
|
|
15514
15533
|
*/
|
|
15515
15534
|
snoozeAllowGlobal: boolean().optional(),
|
|
15516
15535
|
/**
|
|
15536
|
+
* The snooze durations THIS rule's notification offers as buttons, in
|
|
15537
|
+
* minutes.
|
|
15538
|
+
*
|
|
15539
|
+
* Three states, and all three are distinct — which is exactly why this is
|
|
15540
|
+
* `.optional()` and never `.default()`. A Zod default does not run on the
|
|
15541
|
+
* addon cap path (three production failures in one day), so a schema default
|
|
15542
|
+
* would collapse the first two:
|
|
15543
|
+
*
|
|
15544
|
+
* | value | meaning |
|
|
15545
|
+
* | --- | --- |
|
|
15546
|
+
* | absent | the operator never said ⇒ {@link NC_DEFAULT_SNOOZE_MINUTES} |
|
|
15547
|
+
* | `[]` | **no snooze buttons on this rule** — the explicit override |
|
|
15548
|
+
* | a list | these choices, de-duplicated and sorted, at most four |
|
|
15549
|
+
*
|
|
15550
|
+
* `.max(4)` because the notifier's own action budget is small (ntfy allows
|
|
15551
|
+
* three buttons in total) and a rule that spent it all on snooze choices
|
|
15552
|
+
* would push its own tap-through actions off the notification.
|
|
15553
|
+
*
|
|
15554
|
+
* An empty list is NOT an alarm exemption: a rule the alarm is about, or
|
|
15555
|
+
* that arms the panel, is exempt automatically and cannot be silenced by a
|
|
15556
|
+
* window from anywhere (D133).
|
|
15557
|
+
*/
|
|
15558
|
+
snoozeOptions: array(number().int().min(1).max(NC_SNOOZE_MAX_MINUTES)).max(4).optional(),
|
|
15559
|
+
/**
|
|
15517
15560
|
* Devices this rule ACTUATES — arm the alarm, open a gate, turn on a light.
|
|
15518
15561
|
*
|
|
15519
15562
|
* This is what makes the rule set the alarm's trigger set without the alarm
|
|
@@ -15613,6 +15656,7 @@ var NcConditionDescriptorSchema = object({
|
|
|
15613
15656
|
"device",
|
|
15614
15657
|
"package",
|
|
15615
15658
|
"occupancy",
|
|
15659
|
+
"audio",
|
|
15616
15660
|
"system"
|
|
15617
15661
|
]),
|
|
15618
15662
|
label: string(),
|
|
@@ -15631,6 +15675,7 @@ var NcConditionDescriptorSchema = object({
|
|
|
15631
15675
|
"crossingSelect",
|
|
15632
15676
|
"polygonDraw",
|
|
15633
15677
|
"occupancy",
|
|
15678
|
+
"audio",
|
|
15634
15679
|
"deviceState",
|
|
15635
15680
|
"systemEvent"
|
|
15636
15681
|
]),
|
|
@@ -15782,7 +15827,20 @@ var NcSnoozeInputSchema = object({
|
|
|
15782
15827
|
ruleId: string().optional(),
|
|
15783
15828
|
/** Required when `scope: 'device'`. */
|
|
15784
15829
|
deviceId: number().int().optional(),
|
|
15785
|
-
|
|
15830
|
+
/**
|
|
15831
|
+
* Narrow the window to these subject classes — "the cat, not the person".
|
|
15832
|
+
*
|
|
15833
|
+
* ORTHOGONAL to `scope`, deliberately, and absent means EVERY class: that is
|
|
15834
|
+
* what every window authored before this field meant, so no persisted row
|
|
15835
|
+
* changes meaning and no client has to learn anything to keep working.
|
|
15836
|
+
*
|
|
15837
|
+
* It is what makes the window's real key `(deviceId, classes[])` and lets it
|
|
15838
|
+
* cross rules (D133): the operator points at a camera and a kind of thing,
|
|
15839
|
+
* not at whichever of their four rules happened to produce the notification
|
|
15840
|
+
* they are dismissing.
|
|
15841
|
+
*/
|
|
15842
|
+
classes: array(string().min(1)).min(1).optional(),
|
|
15843
|
+
durationMinutes: number().int().min(1).max(NC_SNOOZE_MAX_MINUTES),
|
|
15786
15844
|
/**
|
|
15787
15845
|
* Silence this for EVERY recipient, not just the caller. Permission is
|
|
15788
15846
|
* checked server-side (the rule's `snoozeAllowGlobal`, or admin for the
|
|
@@ -15807,6 +15865,10 @@ var NcSnoozeSchema = object({
|
|
|
15807
15865
|
scope: NcSnoozeScopeSchema,
|
|
15808
15866
|
ruleId: string().optional(),
|
|
15809
15867
|
deviceId: number().int().optional(),
|
|
15868
|
+
/** Subject classes this window covers. ABSENT = every class — see
|
|
15869
|
+
* {@link NcSnoozeInputSchema.shape.classes}. Lives in the JSON blob and has
|
|
15870
|
+
* no SQLite column: nothing queries a window by class. */
|
|
15871
|
+
classes: array(string().min(1)).min(1).optional(),
|
|
15810
15872
|
startedAt: number(),
|
|
15811
15873
|
/** Exclusive: at exactly this instant the snooze is over. Expiry is a
|
|
15812
15874
|
* COMPARISON, not a job — no sweeper can leave the operator silenced. */
|
|
@@ -18059,7 +18121,14 @@ var zonesCapability = {
|
|
|
18059
18121
|
* handle. Slice shape is `{ zones: Zone[] }` so future extensions
|
|
18060
18122
|
* (e.g. zone groupings) can sit alongside the polygon list.
|
|
18061
18123
|
*/
|
|
18062
|
-
runtimeState: object({ zones: array(ZoneSchema).readonly() })
|
|
18124
|
+
runtimeState: object({ zones: array(ZoneSchema).readonly() }),
|
|
18125
|
+
/**
|
|
18126
|
+
* Runtime-state durability: **restored** — written only on operator mutation, so a camera that never had one has nothing to re-derive from. This is the slice `zone-mirror-hydration.ts` exists to paper over.
|
|
18127
|
+
*
|
|
18128
|
+
* See `RuntimeStateDurability`. Enforced by
|
|
18129
|
+
* `scripts/check-runtime-state-durability.ts`.
|
|
18130
|
+
*/
|
|
18131
|
+
durability: "restored"
|
|
18063
18132
|
};
|
|
18064
18133
|
/**
|
|
18065
18134
|
* A bounding box in NORMALIZED [0,1] frame coordinates for `getNativeCrop`. The
|
|
@@ -25147,13 +25216,24 @@ method(object({
|
|
|
25147
25216
|
/** Playback-speed multiplier for the render (1 = realtime). */
|
|
25148
25217
|
var ExportSpeedSchema = number().min(.25).max(32);
|
|
25149
25218
|
/**
|
|
25150
|
-
* One dense interval, in SECONDS FROM THE EXPORT'S OWN `fromMs`.
|
|
25219
|
+
* One dense interval, in WALL-CLOCK SECONDS FROM THE EXPORT'S OWN `fromMs`.
|
|
25151
25220
|
*
|
|
25152
|
-
*
|
|
25153
|
-
*
|
|
25154
|
-
*
|
|
25155
|
-
*
|
|
25156
|
-
*
|
|
25221
|
+
* **Wall clock, not ffmpeg's `t`** — and the recorder translates. A caller
|
|
25222
|
+
* derives these bounds from things that happened at a TIME (a track's
|
|
25223
|
+
* `firstSeen`), while `t` runs over the source playlist: the concatenation of
|
|
25224
|
+
* every segment present for the range, with each recording GAP removed. The
|
|
25225
|
+
* two agree only on a window that recorded without one interruption, and only
|
|
25226
|
+
* the render side knows the segments, so the translation lives there
|
|
25227
|
+
* (`export-dense-map.ts`, addon-pipeline).
|
|
25228
|
+
*
|
|
25229
|
+
* It was not always so. These seconds were fed to `between(t,…)` verbatim, and
|
|
25230
|
+
* on a 10 h window holding 29,393 s of footage every range landed late by the
|
|
25231
|
+
* gap accumulated before it — up to 6,607 s, well past EOF. Nothing matched,
|
|
25232
|
+
* the video was a uniform timelapse, and the log line reported the five ranges
|
|
25233
|
+
* that had been ASKED for (2026-08-13, export `57d14363`, camera 615).
|
|
25234
|
+
*
|
|
25235
|
+
* Relative and not absolute epoch, because an absolute epoch would make every
|
|
25236
|
+
* call site responsible for the same subtraction.
|
|
25157
25237
|
*/
|
|
25158
25238
|
var ExportDenseRangeSchema = object({
|
|
25159
25239
|
fromSec: number().nonnegative(),
|
|
@@ -26402,7 +26482,14 @@ var zoneRulesCapability = {
|
|
|
26402
26482
|
motion: array(ZoneRuleSchema).readonly(),
|
|
26403
26483
|
detection: array(ZoneRuleSchema).readonly(),
|
|
26404
26484
|
package: array(ZoneRuleSchema).readonly()
|
|
26405
|
-
})
|
|
26485
|
+
}),
|
|
26486
|
+
/**
|
|
26487
|
+
* Runtime-state durability: **restored** — operator intent, mutation-only, same argument as `zones`.
|
|
26488
|
+
*
|
|
26489
|
+
* See `RuntimeStateDurability`. Enforced by
|
|
26490
|
+
* `scripts/check-runtime-state-durability.ts`.
|
|
26491
|
+
*/
|
|
26492
|
+
durability: "restored"
|
|
26406
26493
|
};
|
|
26407
26494
|
/**
|
|
26408
26495
|
* Accessory device helpers — shared across drivers.
|
|
@@ -32083,6 +32170,7 @@ Object.freeze({
|
|
|
32083
32170
|
"network-access": "ingress",
|
|
32084
32171
|
"smtp-provider": "email"
|
|
32085
32172
|
});
|
|
32173
|
+
new Map(AUDIO_MACRO_LABELS.flatMap((macro) => macro.icon === void 0 ? [] : [[macro.id, macro.icon]]));
|
|
32086
32174
|
new Set(["devices", "classes"]);
|
|
32087
32175
|
/**
|
|
32088
32176
|
* TimelapseRule — the STANDALONE scheduled timelapse producer's rule model.
|
|
@@ -32395,8 +32483,8 @@ var DETAIL_CROP_PADDING_FIELD = {
|
|
|
32395
32483
|
default: DEFAULT_DETAIL_CROP_CONVENTION.paddingRatio
|
|
32396
32484
|
};
|
|
32397
32485
|
/**
|
|
32398
|
-
* THE native-frame **lease** knobs —
|
|
32399
|
-
* decode worker's native-resolution
|
|
32486
|
+
* THE native-frame **lease** knobs — hold depth, RAM budget, demand window and
|
|
32487
|
+
* subject-tile budget for the decode worker's native-resolution retention.
|
|
32400
32488
|
*
|
|
32401
32489
|
* ## Why they live here and not in the addon that reads them
|
|
32402
32490
|
*
|
|
@@ -32412,20 +32500,25 @@ var DETAIL_CROP_PADDING_FIELD = {
|
|
|
32412
32500
|
* The lease is a per-decode-worker RAM window. Its purpose — the late
|
|
32413
32501
|
* cross-process native crop landing on a full-resolution frame rather than the
|
|
32414
32502
|
* ≤640 detection fallback — is a property of the PIPELINE, not of a node's
|
|
32415
|
-
* hardware: a per-node
|
|
32503
|
+
* hardware: a per-node window would mean the same camera produces different crop
|
|
32416
32504
|
* quality depending on which node the balancer placed it on, and nobody could
|
|
32417
32505
|
* tell that from the stored media. Node-level RAM pressure is already handled
|
|
32418
32506
|
* by the per-session budget ceiling, which is itself one of these knobs.
|
|
32419
32507
|
*
|
|
32420
32508
|
* ## What each knob costs
|
|
32421
32509
|
*
|
|
32422
|
-
* A
|
|
32510
|
+
* A HELD frame is a full NATIVE-resolution copy in system RAM. With the
|
|
32423
32511
|
* default pinned-RGB24 lease path (`CAMSTACK_SESSION_PINNED_RGB_CROP`, on):
|
|
32424
32512
|
* 4K ≈ 24.9 MB/frame, 1080p ≈ 6.2 MB/frame. On the YUV420P path (flag off, and
|
|
32425
32513
|
* for software-decoded sessions): 4K ≈ 12.4 MB, 1080p ≈ 3.1 MB. Worst-case
|
|
32426
|
-
* resident RAM for ONE busy camera
|
|
32427
|
-
*
|
|
32428
|
-
*
|
|
32514
|
+
* resident RAM for ONE busy camera is now `frameBytes × holdFrames`, clamped by
|
|
32515
|
+
* the budget ceiling — bounded by a COUNT because a held frame is waiting for
|
|
32516
|
+
* one specific event (its own detection result), not for a clock.
|
|
32517
|
+
*
|
|
32518
|
+
* A TILE is one subject at native resolution, JPEG-encoded: ~60-120 KB on 4K,
|
|
32519
|
+
* and nothing at all on a frame that detected nothing. That is the asymmetry
|
|
32520
|
+
* this whole shape exists for — see
|
|
32521
|
+
* `docs/design/2026-08-13-native-lease-two-tier-redesign.md`.
|
|
32429
32522
|
*/
|
|
32430
32523
|
/**
|
|
32431
32524
|
* Store identity of the lease knobs in `pipeline-orchestrator`'s GLOBAL
|
|
@@ -32433,10 +32526,11 @@ var DETAIL_CROP_PADDING_FIELD = {
|
|
|
32433
32526
|
* the reader can walk every section instead of trusting the section id.
|
|
32434
32527
|
*/
|
|
32435
32528
|
var NATIVE_LEASE_SECTION_ID = "native-lease";
|
|
32436
|
-
var
|
|
32529
|
+
var NATIVE_LEASE_HOLD_KEY = "nativeLeaseHoldFrames";
|
|
32437
32530
|
var NATIVE_LEASE_BUDGET_KEY = "nativeLeaseBudgetMb";
|
|
32438
32531
|
var NATIVE_LEASE_ACTIVITY_KEY = "nativeLeaseActivityMs";
|
|
32439
32532
|
var NATIVE_LEASE_ADMISSION_KEY = "nativeLeaseAdmission";
|
|
32533
|
+
var NATIVE_LEASE_TILE_BUDGET_KEY = "nativeTileBudgetMb";
|
|
32440
32534
|
/**
|
|
32441
32535
|
* WHICH delivered frames the decode worker retains a native copy of.
|
|
32442
32536
|
*
|
|
@@ -32458,25 +32552,32 @@ var NATIVE_LEASE_ADMISSION_KEY = "nativeLeaseAdmission";
|
|
|
32458
32552
|
var NativeLeaseAdmissionSchema = _enum(["all", "inferred"]);
|
|
32459
32553
|
object({
|
|
32460
32554
|
/**
|
|
32461
|
-
* How
|
|
32555
|
+
* How many delivered frames the worker HOLDS at once, waiting for each one's
|
|
32556
|
+
* detection result.
|
|
32462
32557
|
*
|
|
32463
|
-
*
|
|
32464
|
-
*
|
|
32465
|
-
*
|
|
32466
|
-
*
|
|
32467
|
-
*
|
|
32558
|
+
* This replaced a TTL on 2026-08-13, and the replacement is the whole point:
|
|
32559
|
+
* a time window was never related to the event the pixels were waiting for.
|
|
32560
|
+
* A held frame now lives from delivery until the runner has its `FrameResult`
|
|
32561
|
+
* — at which moment the runner cuts the subject tiles it actually wanted and
|
|
32562
|
+
* releases the frame. The bound exists only so a runner that stops answering
|
|
32563
|
+
* cannot pin RAM: above it the OLDEST held frame is dropped and counted.
|
|
32564
|
+
*
|
|
32565
|
+
* Sizing: the steady state is `inferenceLatency × deliveredFps`, measured at
|
|
32566
|
+
* 40-160 ms × ≤25 fps = 1-4 frames. The default leaves headroom for a hiccup
|
|
32567
|
+
* without ever approaching the old resident set (43 frames × 24.9 MB at 4K).
|
|
32568
|
+
* Raising it does not buy hit rate — it buys tolerance for a slow runner, and
|
|
32569
|
+
* `holdOverflow` on the metrics line is what says you need it.
|
|
32468
32570
|
*/
|
|
32469
|
-
|
|
32571
|
+
holdFrames: number().int().min(1).max(64),
|
|
32470
32572
|
/**
|
|
32471
32573
|
* Hard per-decode-worker RAM ceiling for retained native frames, in MB.
|
|
32472
32574
|
*
|
|
32473
|
-
*
|
|
32474
|
-
*
|
|
32475
|
-
*
|
|
32476
|
-
*
|
|
32477
|
-
*
|
|
32478
|
-
*
|
|
32479
|
-
* rather than giving RAM back — lower this knob if RAM is what you wanted.
|
|
32575
|
+
* Since 2026-08-13 this is a SAFETY ceiling and nothing else: `holdFrames`
|
|
32576
|
+
* is what decides how much is held, and the ceiling is the number above which
|
|
32577
|
+
* something is wrong. Before that it was the effective cap — at 1024 MB with
|
|
32578
|
+
* a 2 800 ms TTL a 4K camera sat pinned at `leaseMb:1020, leaseFrames:43`
|
|
32579
|
+
* with the TTL expiring nothing, which is exactly the confusion the hold
|
|
32580
|
+
* removes. `leaseMb` / `leaseFrames` still say what is resident.
|
|
32480
32581
|
* `0` DISABLES the lease entirely and falls the worker back to the tiny
|
|
32481
32582
|
* leak-prone GPU surface ring (~85% crop miss; that is what the lease exists
|
|
32482
32583
|
* to replace).
|
|
@@ -32502,25 +32603,47 @@ object({
|
|
|
32502
32603
|
* there is the signal that some caller names frames outside the inference set
|
|
32503
32604
|
* and that this must go back to `all`.
|
|
32504
32605
|
*/
|
|
32505
|
-
admission: NativeLeaseAdmissionSchema
|
|
32606
|
+
admission: NativeLeaseAdmissionSchema,
|
|
32607
|
+
/**
|
|
32608
|
+
* RAM ceiling per decode worker, in MB, for the SUBJECT TILES — the
|
|
32609
|
+
* compressed native crops the worker cuts at the moment a frame's detection
|
|
32610
|
+
* result arrives, and keeps long after the frame itself is freed.
|
|
32611
|
+
*
|
|
32612
|
+
* This is the knob that replaced the old retention window, and it buys about
|
|
32613
|
+
* three orders of magnitude more of it: a tile is one subject at native
|
|
32614
|
+
* resolution, JPEG-encoded (~60-120 KB on a 4K person), against ~24.9 MB for
|
|
32615
|
+
* the frame it was cut from. A frame on which nothing was detected costs
|
|
32616
|
+
* nothing at all, which is the real change — the old lease paid per FRAME and
|
|
32617
|
+
* was interrogated per SUBJECT.
|
|
32618
|
+
*
|
|
32619
|
+
* `0` DISABLES tiles, leaving only the hold window and the ≤640 RAM
|
|
32620
|
+
* fallback — i.e. the pre-2026-08-13 miss profile. Set it there only to
|
|
32621
|
+
* reproduce that.
|
|
32622
|
+
*/
|
|
32623
|
+
tileBudgetMb: number().int().min(0).max(1024)
|
|
32506
32624
|
});
|
|
32507
32625
|
/**
|
|
32508
|
-
* The values in force when the operator has set nothing
|
|
32509
|
-
*
|
|
32510
|
-
*
|
|
32626
|
+
* The values in force when the operator has set nothing.
|
|
32627
|
+
*
|
|
32628
|
+
* `budgetMb` stays at 1024 on the day the hold landed, deliberately: it stopped
|
|
32629
|
+
* being the retention window and became the OOM ceiling, and lowering a ceiling
|
|
32630
|
+
* in the same change that redefines it would make a regression and a retune
|
|
32631
|
+
* indistinguishable. Cut it once `tileHits` / `holdOverflow` have been read on
|
|
32632
|
+
* live traffic.
|
|
32511
32633
|
*/
|
|
32512
32634
|
var DEFAULT_NATIVE_LEASE_SETTINGS = {
|
|
32513
|
-
|
|
32635
|
+
holdFrames: 8,
|
|
32514
32636
|
budgetMb: 1024,
|
|
32515
32637
|
activityMs: 15e3,
|
|
32638
|
+
tileBudgetMb: 64,
|
|
32516
32639
|
admission: "inferred"
|
|
32517
32640
|
};
|
|
32518
32641
|
/** Slider bounds for the operator-facing knobs (orchestrator settings UI). */
|
|
32519
|
-
var
|
|
32520
|
-
min:
|
|
32521
|
-
max:
|
|
32522
|
-
step:
|
|
32523
|
-
default: DEFAULT_NATIVE_LEASE_SETTINGS.
|
|
32642
|
+
var NATIVE_LEASE_HOLD_FIELD = {
|
|
32643
|
+
min: 1,
|
|
32644
|
+
max: 64,
|
|
32645
|
+
step: 1,
|
|
32646
|
+
default: DEFAULT_NATIVE_LEASE_SETTINGS.holdFrames
|
|
32524
32647
|
};
|
|
32525
32648
|
var NATIVE_LEASE_BUDGET_FIELD = {
|
|
32526
32649
|
min: 0,
|
|
@@ -32534,6 +32657,12 @@ var NATIVE_LEASE_ACTIVITY_FIELD = {
|
|
|
32534
32657
|
step: 1e3,
|
|
32535
32658
|
default: DEFAULT_NATIVE_LEASE_SETTINGS.activityMs
|
|
32536
32659
|
};
|
|
32660
|
+
var NATIVE_LEASE_TILE_BUDGET_FIELD = {
|
|
32661
|
+
min: 0,
|
|
32662
|
+
max: 1024,
|
|
32663
|
+
step: 16,
|
|
32664
|
+
default: DEFAULT_NATIVE_LEASE_SETTINGS.tileBudgetMb
|
|
32665
|
+
};
|
|
32537
32666
|
/** Select options for the admission knob (orchestrator settings UI). */
|
|
32538
32667
|
var NATIVE_LEASE_ADMISSION_FIELD = {
|
|
32539
32668
|
options: [{
|
|
@@ -32861,7 +32990,55 @@ var OrchestratorDiagnosticsSchema = object({
|
|
|
32861
32990
|
cameraConfigCount: number().int().min(0),
|
|
32862
32991
|
activeDetectionCount: number().int().min(0)
|
|
32863
32992
|
});
|
|
32864
|
-
|
|
32993
|
+
/**
|
|
32994
|
+
* The node-stress long-term-statistics read surface.
|
|
32995
|
+
*
|
|
32996
|
+
* A custom action rather than a cap method, matching how the orchestrator
|
|
32997
|
+
* already serves `dumpState`: this is a hub-local read over a table the hub
|
|
32998
|
+
* owns, and it ships with one `camstack deploy` instead of a release train.
|
|
32999
|
+
* The MEAN is derived here and returned alongside the addable `sum`/`samples`
|
|
33000
|
+
* — a chart wants the first, a re-bucketing caller wants the second, and a
|
|
33001
|
+
* stored mean is a field that can disagree with both.
|
|
33002
|
+
*/
|
|
33003
|
+
var NodeStressStatsInputSchema = object({
|
|
33004
|
+
/** `node-score` | `node-queue-pressure` | `node-drop-ratio` | `node-fps-deficit`. */
|
|
33005
|
+
series: string().optional(),
|
|
33006
|
+
/** A node id. Omit for every node. */
|
|
33007
|
+
subject: string().optional(),
|
|
33008
|
+
/** Inclusive bucket-start bounds, ms. */
|
|
33009
|
+
from: number().int().optional(),
|
|
33010
|
+
to: number().int().optional(),
|
|
33011
|
+
limit: number().int().positive().max(5e3).optional()
|
|
33012
|
+
});
|
|
33013
|
+
var NodeStressStatsRowSchema = object({
|
|
33014
|
+
subject: string(),
|
|
33015
|
+
series: string(),
|
|
33016
|
+
scope: string(),
|
|
33017
|
+
bucketStart: number(),
|
|
33018
|
+
samples: number(),
|
|
33019
|
+
sum: number(),
|
|
33020
|
+
mean: number(),
|
|
33021
|
+
min: number(),
|
|
33022
|
+
max: number()
|
|
33023
|
+
});
|
|
33024
|
+
var NodeStressStatsOutputSchema = object({
|
|
33025
|
+
rows: array(NodeStressStatsRowSchema).readonly(),
|
|
33026
|
+
/** Buckets still accumulating — "is it running" answerable at once, rather
|
|
33027
|
+
* than after five minutes of indistinguishable silence. */
|
|
33028
|
+
open: array(NodeStressStatsRowSchema).readonly(),
|
|
33029
|
+
/** The durable failover history the anti-flap guards read, newest first.
|
|
33030
|
+
* Exposed for the same reason the heartbeat exists: "nothing moved" has to
|
|
33031
|
+
* be distinguishable from "nothing is watching". */
|
|
33032
|
+
moves: array(object({
|
|
33033
|
+
deviceId: number(),
|
|
33034
|
+
fromNodeId: string(),
|
|
33035
|
+
at: number()
|
|
33036
|
+
})).readonly()
|
|
33037
|
+
});
|
|
33038
|
+
var pipelineOrchestratorActions = defineCustomActions({
|
|
33039
|
+
dumpState: customAction(_void(), OrchestratorDiagnosticsSchema),
|
|
33040
|
+
nodeStressStats: customAction(NodeStressStatsInputSchema, NodeStressStatsOutputSchema, { auth: "admin" })
|
|
33041
|
+
});
|
|
32865
33042
|
/**
|
|
32866
33043
|
* Sentinel returned by `buildDetectionConfig` when the profile-slot READ
|
|
32867
33044
|
* itself failed transiently (e.g. `listAllProfileSlots` discovery timeout
|
|
@@ -32907,7 +33084,7 @@ var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
|
|
|
32907
33084
|
* shape so video and audio plumbing self-heal identically.
|
|
32908
33085
|
*/
|
|
32909
33086
|
/** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
|
|
32910
|
-
var POLL_INTERVAL_MS = 200;
|
|
33087
|
+
var POLL_INTERVAL_MS$1 = 200;
|
|
32911
33088
|
/** How many chunks to drain per poll — a small burst absorbs jitter. */
|
|
32912
33089
|
var PULL_MAX_COUNT = 8;
|
|
32913
33090
|
/**
|
|
@@ -33090,7 +33267,7 @@ function startPolling(options, lifecycle) {
|
|
|
33090
33267
|
} });
|
|
33091
33268
|
if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
|
|
33092
33269
|
}
|
|
33093
|
-
if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
|
|
33270
|
+
if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS$1);
|
|
33094
33271
|
};
|
|
33095
33272
|
tick();
|
|
33096
33273
|
}
|
|
@@ -35581,12 +35758,341 @@ function buildRunnerConfig(base, overrides) {
|
|
|
35581
35758
|
};
|
|
35582
35759
|
}
|
|
35583
35760
|
//#endregion
|
|
35584
|
-
//#region src/
|
|
35761
|
+
//#region src/durable/durable-ledger.ts
|
|
35762
|
+
/** Default reseed cap — every current consumer's row set is installation-bounded. */
|
|
35763
|
+
var DEFAULT_LOAD_LIMIT = 1e5;
|
|
35764
|
+
var DurableLedger = class DurableLedger {
|
|
35765
|
+
mirror = /* @__PURE__ */ new Map();
|
|
35766
|
+
spec;
|
|
35767
|
+
store;
|
|
35768
|
+
logger;
|
|
35769
|
+
constructor(deps) {
|
|
35770
|
+
this.spec = deps.spec;
|
|
35771
|
+
this.store = deps.store;
|
|
35772
|
+
this.logger = deps.logger;
|
|
35773
|
+
}
|
|
35774
|
+
/**
|
|
35775
|
+
* Register the collection. MUST run at boot, before any read or write: the
|
|
35776
|
+
* SQLite backend answers 412 for an undeclared collection and takes the whole
|
|
35777
|
+
* runner down with it (the addon-ai boot-crash lesson).
|
|
35778
|
+
*/
|
|
35779
|
+
static declare(store, spec) {
|
|
35780
|
+
return store.declareCollection.mutate({
|
|
35781
|
+
collection: spec.collection,
|
|
35782
|
+
columns: [...spec.columns],
|
|
35783
|
+
...spec.indexes !== void 0 ? { indexes: [...spec.indexes] } : {}
|
|
35784
|
+
});
|
|
35785
|
+
}
|
|
35786
|
+
declare() {
|
|
35787
|
+
return DurableLedger.declare(this.store, this.spec);
|
|
35788
|
+
}
|
|
35789
|
+
/** The collection this ledger owns — for the caller's own log lines. */
|
|
35790
|
+
get collection() {
|
|
35791
|
+
return this.spec.collection;
|
|
35792
|
+
}
|
|
35793
|
+
/**
|
|
35794
|
+
* Boot reseed. Replaces the mirror with what the store holds and returns the
|
|
35795
|
+
* rows, so a caller that must hydrate something else (a watcher, a registry)
|
|
35796
|
+
* gets them without a second read.
|
|
35797
|
+
*
|
|
35798
|
+
* **A failure returns what is already mirrored** rather than clearing it —
|
|
35799
|
+
* see contract rule 2. The count is worth logging out loud at the call site:
|
|
35800
|
+
* "loaded 0" after a container recreate is the one line that explains a
|
|
35801
|
+
* silent flood.
|
|
35802
|
+
*/
|
|
35803
|
+
async load() {
|
|
35804
|
+
try {
|
|
35805
|
+
const records = await this.store.query.query({
|
|
35806
|
+
collection: this.spec.collection,
|
|
35807
|
+
filter: { limit: this.spec.loadLimit ?? DEFAULT_LOAD_LIMIT }
|
|
35808
|
+
});
|
|
35809
|
+
const next = /* @__PURE__ */ new Map();
|
|
35810
|
+
let skipped = 0;
|
|
35811
|
+
for (const record of records) {
|
|
35812
|
+
const row = this.spec.fromRecord(record.id, record.data);
|
|
35813
|
+
if (row === null) {
|
|
35814
|
+
skipped += 1;
|
|
35815
|
+
continue;
|
|
35816
|
+
}
|
|
35817
|
+
next.set(this.spec.keyOf(row), row);
|
|
35818
|
+
}
|
|
35819
|
+
this.mirror.clear();
|
|
35820
|
+
for (const [key, row] of next) this.mirror.set(key, row);
|
|
35821
|
+
if (skipped > 0) this.logger.warn("durable rows skipped as malformed — they gate NOTHING", { meta: {
|
|
35822
|
+
collection: this.spec.collection,
|
|
35823
|
+
skipped
|
|
35824
|
+
} });
|
|
35825
|
+
return [...this.mirror.values()];
|
|
35826
|
+
} catch (err) {
|
|
35827
|
+
this.logger.warn("durable load failed — keeping the state already in memory", { meta: {
|
|
35828
|
+
collection: this.spec.collection,
|
|
35829
|
+
error: String(err),
|
|
35830
|
+
held: this.mirror.size
|
|
35831
|
+
} });
|
|
35832
|
+
return [...this.mirror.values()];
|
|
35833
|
+
}
|
|
35834
|
+
}
|
|
35835
|
+
/** Every mirrored row, insertion-ordered. */
|
|
35836
|
+
snapshot() {
|
|
35837
|
+
return [...this.mirror.values()];
|
|
35838
|
+
}
|
|
35839
|
+
/** The row currently held for a key, if any. Pure RAM — never I/O. */
|
|
35840
|
+
get(key) {
|
|
35841
|
+
return this.mirror.get(key);
|
|
35842
|
+
}
|
|
35843
|
+
has(key) {
|
|
35844
|
+
return this.mirror.has(key);
|
|
35845
|
+
}
|
|
35846
|
+
get size() {
|
|
35847
|
+
return this.mirror.size;
|
|
35848
|
+
}
|
|
35849
|
+
/**
|
|
35850
|
+
* Judge ONE observation against what the ledger already accepted, and advance
|
|
35851
|
+
* it.
|
|
35852
|
+
*
|
|
35853
|
+
* Synchronous on purpose: the verdict is a function of the in-RAM mirror
|
|
35854
|
+
* alone, so a decision can never be gated on an I/O that might fail (D49).
|
|
35855
|
+
* The durable write is kicked off behind it and its failure changes no
|
|
35856
|
+
* verdict.
|
|
35857
|
+
*
|
|
35858
|
+
* `no-flip` leaves the held row UNTOUCHED — including any timestamp it
|
|
35859
|
+
* carries, which therefore means "when this key last CHANGED", not "when it
|
|
35860
|
+
* last spoke". That is the timestamp anyone reading the table wants.
|
|
35861
|
+
*/
|
|
35862
|
+
observe(row) {
|
|
35863
|
+
const equalFact = this.spec.equalFact;
|
|
35864
|
+
if (equalFact === void 0) throw new Error(`DurableLedger(${this.spec.collection}): observe() requires the spec to declare equalFact`);
|
|
35865
|
+
const key = this.spec.keyOf(row);
|
|
35866
|
+
const held = this.mirror.get(key);
|
|
35867
|
+
if (held !== void 0 && equalFact(held, row)) return "no-flip";
|
|
35868
|
+
this.mirror.set(key, row);
|
|
35869
|
+
this.persist(row);
|
|
35870
|
+
return held === void 0 ? "seeded" : "flip";
|
|
35871
|
+
}
|
|
35872
|
+
/**
|
|
35873
|
+
* Upsert a row without a verdict — the write path for a ledger whose owner
|
|
35874
|
+
* has already decided the value changed.
|
|
35875
|
+
*
|
|
35876
|
+
* The order of the mirror advance and the durable write is the spec's
|
|
35877
|
+
* {@link DurableWriteMode}, not the call site's: two call sites that
|
|
35878
|
+
* disagreed about it would be two different durability guarantees on one
|
|
35879
|
+
* collection.
|
|
35880
|
+
*/
|
|
35881
|
+
async put(row) {
|
|
35882
|
+
const key = this.spec.keyOf(row);
|
|
35883
|
+
if (this.spec.writeMode === "write-behind") {
|
|
35884
|
+
this.mirror.set(key, row);
|
|
35885
|
+
await this.persist(row);
|
|
35886
|
+
return;
|
|
35887
|
+
}
|
|
35888
|
+
await this.store.set.mutate({
|
|
35889
|
+
collection: this.spec.collection,
|
|
35890
|
+
key,
|
|
35891
|
+
value: this.spec.toValue(row)
|
|
35892
|
+
});
|
|
35893
|
+
this.mirror.set(key, row);
|
|
35894
|
+
}
|
|
35895
|
+
/**
|
|
35896
|
+
* Advance the mirror WITHOUT persisting, for an owner that deliberately
|
|
35897
|
+
* coalesces its writes.
|
|
35898
|
+
*
|
|
35899
|
+
* The stationary registry is the reason this exists: a parked car is
|
|
35900
|
+
* re-confirmed on every processed frame (5–30 Hz), and persisting each
|
|
35901
|
+
* confirmation would offer thousands of commits a day to the checkpoint
|
|
35902
|
+
* lottery (D96) to maintain a handful of rows. It stages the advance and
|
|
35903
|
+
* flushes on its 5-minute sweep — 288 writes a day instead of ~10⁶.
|
|
35904
|
+
*
|
|
35905
|
+
* **The cost is stated, not hidden**: a staged value that is never flushed
|
|
35906
|
+
* is lost on a crash. An owner may only stage a field whose staleness its
|
|
35907
|
+
* own TTL absorbs. Anything that GATES work must go through {@link put} or
|
|
35908
|
+
* {@link observe}.
|
|
35909
|
+
*/
|
|
35910
|
+
stage(row) {
|
|
35911
|
+
this.mirror.set(this.spec.keyOf(row), row);
|
|
35912
|
+
}
|
|
35913
|
+
/**
|
|
35914
|
+
* Drop a key from the MIRROR only — the durable row survives.
|
|
35915
|
+
*
|
|
35916
|
+
* What a scope-unbind needs: this process stops holding the value, and a
|
|
35917
|
+
* rebind reloads it from the store. Deliberately distinct from
|
|
35918
|
+
* {@link forget}, which deletes; conflating the two is how an unbind turns
|
|
35919
|
+
* into a wipe.
|
|
35920
|
+
*/
|
|
35921
|
+
evict(key) {
|
|
35922
|
+
this.mirror.delete(key);
|
|
35923
|
+
}
|
|
35924
|
+
/**
|
|
35925
|
+
* Drop one key, mirror and row. Best-effort on the durable half: a failed
|
|
35926
|
+
* delete leaves a row that the next load will re-mirror, which is a stale
|
|
35927
|
+
* value rather than a lost one.
|
|
35928
|
+
*/
|
|
35929
|
+
async forget(key) {
|
|
35930
|
+
this.mirror.delete(key);
|
|
35931
|
+
try {
|
|
35932
|
+
await this.store.delete.mutate({
|
|
35933
|
+
collection: this.spec.collection,
|
|
35934
|
+
key
|
|
35935
|
+
});
|
|
35936
|
+
} catch (err) {
|
|
35937
|
+
this.logger.debug("durable delete failed", { meta: {
|
|
35938
|
+
collection: this.spec.collection,
|
|
35939
|
+
key,
|
|
35940
|
+
error: String(err)
|
|
35941
|
+
} });
|
|
35942
|
+
}
|
|
35943
|
+
}
|
|
35944
|
+
/**
|
|
35945
|
+
* Drop every mirrored key NOT in `activeKeys`. Returns how many rows went.
|
|
35946
|
+
*
|
|
35947
|
+
* **The caller must hold an AUTHORITATIVE active set.** A prune driven by a
|
|
35948
|
+
* fallible read is work destroyed on an error (D49/D130) — that is why this
|
|
35949
|
+
* is a method a feature opts into rather than a policy the primitive runs.
|
|
35950
|
+
* Best-effort per row: a failed delete keeps the key (retried next prune)
|
|
35951
|
+
* rather than aborting the sweep.
|
|
35952
|
+
*/
|
|
35953
|
+
async pruneExcept(activeKeys) {
|
|
35954
|
+
let pruned = 0;
|
|
35955
|
+
for (const key of [...this.mirror.keys()]) {
|
|
35956
|
+
if (activeKeys.has(key)) continue;
|
|
35957
|
+
try {
|
|
35958
|
+
await this.store.delete.mutate({
|
|
35959
|
+
collection: this.spec.collection,
|
|
35960
|
+
key
|
|
35961
|
+
});
|
|
35962
|
+
this.mirror.delete(key);
|
|
35963
|
+
pruned += 1;
|
|
35964
|
+
} catch (err) {
|
|
35965
|
+
this.logger.debug("durable prune delete failed", { meta: {
|
|
35966
|
+
collection: this.spec.collection,
|
|
35967
|
+
key,
|
|
35968
|
+
error: String(err)
|
|
35969
|
+
} });
|
|
35970
|
+
}
|
|
35971
|
+
}
|
|
35972
|
+
return pruned;
|
|
35973
|
+
}
|
|
35974
|
+
/**
|
|
35975
|
+
* Write-behind durable upsert. Best-effort and logged, never thrown at the
|
|
35976
|
+
* decision path: the mirror already holds the truth for this process, and the
|
|
35977
|
+
* worst a lost write can do is one silent re-seed after the next restart.
|
|
35978
|
+
*/
|
|
35979
|
+
async persist(row) {
|
|
35980
|
+
const deviceId = this.spec.deviceIdOf?.(row);
|
|
35981
|
+
try {
|
|
35982
|
+
await this.store.set.mutate({
|
|
35983
|
+
collection: this.spec.collection,
|
|
35984
|
+
key: this.spec.keyOf(row),
|
|
35985
|
+
value: this.spec.toValue(row)
|
|
35986
|
+
});
|
|
35987
|
+
} catch (err) {
|
|
35988
|
+
this.logger.warn("durable persist failed — this key may re-seed on boot", {
|
|
35989
|
+
...deviceId !== void 0 ? { tags: { deviceId } } : {},
|
|
35990
|
+
meta: {
|
|
35991
|
+
collection: this.spec.collection,
|
|
35992
|
+
key: this.spec.keyOf(row),
|
|
35993
|
+
error: String(err)
|
|
35994
|
+
}
|
|
35995
|
+
});
|
|
35996
|
+
}
|
|
35997
|
+
}
|
|
35998
|
+
};
|
|
35999
|
+
var DEVICE_FEATURES_SPEC = {
|
|
36000
|
+
collection: "pipeline-orchestrator:device-features",
|
|
36001
|
+
columns: [
|
|
36002
|
+
{
|
|
36003
|
+
name: "deviceId",
|
|
36004
|
+
type: "TEXT",
|
|
36005
|
+
primaryKey: true,
|
|
36006
|
+
notNull: true
|
|
36007
|
+
},
|
|
36008
|
+
(
|
|
36009
|
+
/** The feature name list, verbatim. */
|
|
36010
|
+
{
|
|
36011
|
+
name: "features",
|
|
36012
|
+
type: "JSON",
|
|
36013
|
+
notNull: true
|
|
36014
|
+
}),
|
|
36015
|
+
{
|
|
36016
|
+
name: "updatedAt",
|
|
36017
|
+
type: "INTEGER",
|
|
36018
|
+
notNull: true
|
|
36019
|
+
}
|
|
36020
|
+
],
|
|
36021
|
+
writeMode: "write-behind",
|
|
36022
|
+
keyOf: (row) => String(row.deviceId),
|
|
36023
|
+
toValue: (row) => ({
|
|
36024
|
+
features: [...row.features],
|
|
36025
|
+
updatedAt: row.updatedAt
|
|
36026
|
+
}),
|
|
36027
|
+
fromRecord: (key, data) => {
|
|
36028
|
+
const deviceId = Number(key);
|
|
36029
|
+
const raw = data["features"];
|
|
36030
|
+
if (!Number.isFinite(deviceId) || !Array.isArray(raw)) return null;
|
|
36031
|
+
const features = raw.filter((f) => typeof f === "string");
|
|
36032
|
+
if (features.length === 0) return null;
|
|
36033
|
+
const updatedAt = Number(data["updatedAt"]);
|
|
36034
|
+
return {
|
|
36035
|
+
deviceId,
|
|
36036
|
+
features,
|
|
36037
|
+
updatedAt: Number.isFinite(updatedAt) ? updatedAt : 0,
|
|
36038
|
+
restored: true
|
|
36039
|
+
};
|
|
36040
|
+
},
|
|
36041
|
+
deviceIdOf: (row) => row.deviceId
|
|
36042
|
+
};
|
|
35585
36043
|
var DeviceFeaturesMirror = class {
|
|
36044
|
+
/** Process-local fallback, used only when no store was supplied. */
|
|
36045
|
+
local = /* @__PURE__ */ new Map();
|
|
36046
|
+
durable;
|
|
35586
36047
|
logger;
|
|
35587
|
-
|
|
35588
|
-
constructor(
|
|
35589
|
-
this.logger = logger;
|
|
36048
|
+
now;
|
|
36049
|
+
constructor(deps) {
|
|
36050
|
+
this.logger = deps.logger;
|
|
36051
|
+
this.now = deps.now ?? (() => Date.now());
|
|
36052
|
+
this.durable = deps.store === void 0 ? null : new DurableLedger({
|
|
36053
|
+
spec: DEVICE_FEATURES_SPEC,
|
|
36054
|
+
store: deps.store,
|
|
36055
|
+
logger: deps.logger
|
|
36056
|
+
});
|
|
36057
|
+
}
|
|
36058
|
+
static declare(store) {
|
|
36059
|
+
return DurableLedger.declare(store, DEVICE_FEATURES_SPEC);
|
|
36060
|
+
}
|
|
36061
|
+
/**
|
|
36062
|
+
* Seed the mirror from the last session. Call once at boot, after `declare`
|
|
36063
|
+
* and BEFORE the first `resolve` — the whole value of the row is that it is
|
|
36064
|
+
* already there when the first read fails.
|
|
36065
|
+
*/
|
|
36066
|
+
async hydrate() {
|
|
36067
|
+
if (this.durable === null) return 0;
|
|
36068
|
+
const rows = await this.durable.load();
|
|
36069
|
+
this.logger.info("device-features mirror restored", { meta: { devices: rows.length } });
|
|
36070
|
+
return rows.length;
|
|
36071
|
+
}
|
|
36072
|
+
held(deviceId) {
|
|
36073
|
+
return this.durable === null ? this.local.get(deviceId) : this.durable.get(String(deviceId));
|
|
36074
|
+
}
|
|
36075
|
+
remember(deviceId, features) {
|
|
36076
|
+
const row = {
|
|
36077
|
+
deviceId,
|
|
36078
|
+
features: [...features],
|
|
36079
|
+
updatedAt: this.now(),
|
|
36080
|
+
restored: false
|
|
36081
|
+
};
|
|
36082
|
+
if (this.durable === null) {
|
|
36083
|
+
this.local.set(deviceId, row);
|
|
36084
|
+
return;
|
|
36085
|
+
}
|
|
36086
|
+
const held = this.durable.get(String(deviceId));
|
|
36087
|
+
if (held !== void 0 && !held.restored && sameFeatures(held.features, features)) return;
|
|
36088
|
+
this.durable.put(row);
|
|
36089
|
+
}
|
|
36090
|
+
drop(deviceId) {
|
|
36091
|
+
if (this.durable === null) {
|
|
36092
|
+
this.local.delete(deviceId);
|
|
36093
|
+
return;
|
|
36094
|
+
}
|
|
36095
|
+
this.durable.forget(String(deviceId));
|
|
35590
36096
|
}
|
|
35591
36097
|
/**
|
|
35592
36098
|
* Resolve a device's features, preferring a fresh read but never letting a
|
|
@@ -35595,15 +36101,19 @@ var DeviceFeaturesMirror = class {
|
|
|
35595
36101
|
async resolve(deviceId, read) {
|
|
35596
36102
|
const first = await read();
|
|
35597
36103
|
if (first !== null && first.length > 0) {
|
|
35598
|
-
this.
|
|
36104
|
+
this.remember(deviceId, first);
|
|
35599
36105
|
return first;
|
|
35600
36106
|
}
|
|
35601
|
-
const
|
|
36107
|
+
const heldRow = this.held(deviceId);
|
|
36108
|
+
const mirrored = heldRow?.features;
|
|
35602
36109
|
if (first === null) {
|
|
35603
36110
|
if (mirrored !== void 0) {
|
|
35604
36111
|
this.logger.warn("device features unavailable — serving last-known mirror", {
|
|
35605
36112
|
tags: { deviceId },
|
|
35606
|
-
meta: {
|
|
36113
|
+
meta: {
|
|
36114
|
+
mirrored: mirrored.length,
|
|
36115
|
+
restored: heldRow?.restored === true
|
|
36116
|
+
}
|
|
35607
36117
|
});
|
|
35608
36118
|
return mirrored;
|
|
35609
36119
|
}
|
|
@@ -35617,7 +36127,7 @@ var DeviceFeaturesMirror = class {
|
|
|
35617
36127
|
tags: { deviceId },
|
|
35618
36128
|
meta: { features: second.length }
|
|
35619
36129
|
});
|
|
35620
|
-
this.
|
|
36130
|
+
this.remember(deviceId, second);
|
|
35621
36131
|
return second;
|
|
35622
36132
|
}
|
|
35623
36133
|
if (second === null) {
|
|
@@ -35628,14 +36138,20 @@ var DeviceFeaturesMirror = class {
|
|
|
35628
36138
|
tags: { deviceId },
|
|
35629
36139
|
meta: { previously: mirrored.length }
|
|
35630
36140
|
});
|
|
35631
|
-
this.
|
|
36141
|
+
this.drop(deviceId);
|
|
35632
36142
|
return [];
|
|
35633
36143
|
}
|
|
35634
36144
|
/** Drop a device's mirror — call when the device is removed. */
|
|
35635
36145
|
forget(deviceId) {
|
|
35636
|
-
this.
|
|
36146
|
+
this.drop(deviceId);
|
|
35637
36147
|
}
|
|
35638
36148
|
};
|
|
36149
|
+
/** Order-insensitive feature-set equality — the read's order is not a fact. */
|
|
36150
|
+
function sameFeatures(a, b) {
|
|
36151
|
+
if (a.length !== b.length) return false;
|
|
36152
|
+
const held = new Set(a);
|
|
36153
|
+
return b.every((f) => held.has(f));
|
|
36154
|
+
}
|
|
35639
36155
|
//#endregion
|
|
35640
36156
|
//#region src/watchdog-camera.ts
|
|
35641
36157
|
/**
|
|
@@ -35702,7 +36218,20 @@ var DetectionWiringController = class {
|
|
|
35702
36218
|
featuresMirror;
|
|
35703
36219
|
constructor(deps) {
|
|
35704
36220
|
this.deps = deps;
|
|
35705
|
-
this.featuresMirror = new DeviceFeaturesMirror(
|
|
36221
|
+
this.featuresMirror = new DeviceFeaturesMirror({
|
|
36222
|
+
logger: deps.logger,
|
|
36223
|
+
...deps.featuresStore !== void 0 ? { store: deps.featuresStore } : {}
|
|
36224
|
+
});
|
|
36225
|
+
}
|
|
36226
|
+
/**
|
|
36227
|
+
* Declare + seed the device-features mirror. Call once at boot, BEFORE the
|
|
36228
|
+
* first detection start — the restored row is worth nothing after the read
|
|
36229
|
+
* that would have needed it.
|
|
36230
|
+
*/
|
|
36231
|
+
async hydrateFeaturesMirror() {
|
|
36232
|
+
if (this.deps.featuresStore === void 0) return;
|
|
36233
|
+
await DeviceFeaturesMirror.declare(this.deps.featuresStore);
|
|
36234
|
+
await this.featuresMirror.hydrate();
|
|
35706
36235
|
}
|
|
35707
36236
|
/** `activeDetections.get(deviceId)`. */
|
|
35708
36237
|
getActiveDetectionConfig(deviceId) {
|
|
@@ -36913,7 +37442,6 @@ var NodeStressController = class NodeStressController {
|
|
|
36913
37442
|
static HEARTBEAT_MS = 60 * 6e4;
|
|
36914
37443
|
samples = /* @__PURE__ */ new Map();
|
|
36915
37444
|
memories = /* @__PURE__ */ new Map();
|
|
36916
|
-
history = [];
|
|
36917
37445
|
/** When the last heartbeat went out. `null` ⇒ the next sweep emits one. */
|
|
36918
37446
|
lastHeartbeatAt = null;
|
|
36919
37447
|
timer = null;
|
|
@@ -36957,6 +37485,18 @@ var NodeStressController = class NodeStressController {
|
|
|
36957
37485
|
forgetDevice(deviceId) {
|
|
36958
37486
|
this.samples.delete(deviceId);
|
|
36959
37487
|
}
|
|
37488
|
+
/**
|
|
37489
|
+
* Recover the move history from disk BEFORE the first sweep can act.
|
|
37490
|
+
*
|
|
37491
|
+
* Best-effort by construction (the ledger keeps whatever it already has on a
|
|
37492
|
+
* read failure), and the count is logged out loud: a restart that recovers 0
|
|
37493
|
+
* moves while the operator remembers three is the line that says the budget
|
|
37494
|
+
* has been reset.
|
|
37495
|
+
*/
|
|
37496
|
+
async hydrate() {
|
|
37497
|
+
const recovered = await this.deps.moves.load();
|
|
37498
|
+
this.deps.logger.info("node-stress move history recovered", { meta: { moves: recovered } });
|
|
37499
|
+
}
|
|
36960
37500
|
start() {
|
|
36961
37501
|
if (this.timer) return;
|
|
36962
37502
|
this.timer = setInterval(() => {
|
|
@@ -36972,12 +37512,24 @@ var NodeStressController = class NodeStressController {
|
|
|
36972
37512
|
}
|
|
36973
37513
|
this.samples.clear();
|
|
36974
37514
|
this.memories.clear();
|
|
36975
|
-
this.history = [];
|
|
36976
37515
|
}
|
|
36977
37516
|
/** The last computed state per node — empty before the first sweep. */
|
|
36978
37517
|
statesView() {
|
|
36979
37518
|
return new Map([...this.memories].map(([nodeId, m]) => [nodeId, m.state]));
|
|
36980
37519
|
}
|
|
37520
|
+
/** The durable move history, newest first — the guards' own evidence, so an
|
|
37521
|
+
* operator asking "why did nothing move" can read the budget. */
|
|
37522
|
+
historyView() {
|
|
37523
|
+
return this.deps.moves.entries().toSorted((a, b) => b.at - a.at);
|
|
37524
|
+
}
|
|
37525
|
+
/**
|
|
37526
|
+
* The per-node stress signals as of `now`, for the LTS aggregator and for
|
|
37527
|
+
* diagnostics. Computed from the same live samples the sweep uses, so a
|
|
37528
|
+
* chart can never disagree with a verdict.
|
|
37529
|
+
*/
|
|
37530
|
+
signalsView(now) {
|
|
37531
|
+
return new Map(this.buildInputs(now).map((input) => [input.nodeId, input.signals]));
|
|
37532
|
+
}
|
|
36981
37533
|
/** Group live samples by node and reduce each group. Exposed for tests. */
|
|
36982
37534
|
buildInputs(now) {
|
|
36983
37535
|
const byNode = /* @__PURE__ */ new Map();
|
|
@@ -37008,6 +37560,7 @@ var NodeStressController = class NodeStressController {
|
|
|
37008
37560
|
const verdicts = evaluateNodeStress(this.memories, this.buildInputs(now), now, this.thresholds());
|
|
37009
37561
|
for (const v of verdicts) {
|
|
37010
37562
|
this.memories.set(v.nodeId, v.memory);
|
|
37563
|
+
this.deps.lts?.noteSignals(v.nodeId, v.signals, now);
|
|
37011
37564
|
if (v.changed) this.logTransition(v);
|
|
37012
37565
|
}
|
|
37013
37566
|
this.maybeHeartbeat(mode, verdicts, now);
|
|
@@ -37066,11 +37619,12 @@ var NodeStressController = class NodeStressController {
|
|
|
37066
37619
|
async actOn(verdicts, now) {
|
|
37067
37620
|
if (this.moveInFlight) return;
|
|
37068
37621
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
37069
|
-
this.
|
|
37622
|
+
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
37623
|
+
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
37070
37624
|
const plan = planStressFailover({
|
|
37071
37625
|
verdicts,
|
|
37072
37626
|
candidates: await this.collectCandidates(now),
|
|
37073
|
-
history
|
|
37627
|
+
history,
|
|
37074
37628
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
37075
37629
|
attachedByNode: this.attachedByNode()
|
|
37076
37630
|
}, now, this.guards());
|
|
@@ -37094,7 +37648,7 @@ var NodeStressController = class NodeStressController {
|
|
|
37094
37648
|
await this.deps.detach(plan.fromNodeId, plan.deviceId);
|
|
37095
37649
|
await this.deps.attach(plan.toNodeId, config);
|
|
37096
37650
|
this.deps.ledger.recordAssignment(plan.deviceId, plan.toNodeId, "rebalance", false);
|
|
37097
|
-
this.
|
|
37651
|
+
this.deps.moves.record({
|
|
37098
37652
|
deviceId: plan.deviceId,
|
|
37099
37653
|
fromNodeId: plan.fromNodeId,
|
|
37100
37654
|
at: now
|
|
@@ -37166,6 +37720,513 @@ var NodeStressController = class NodeStressController {
|
|
|
37166
37720
|
}
|
|
37167
37721
|
};
|
|
37168
37722
|
//#endregion
|
|
37723
|
+
//#region src/durable/lts-aggregator.ts
|
|
37724
|
+
/** Wall-clock bucket width. Five minutes, matching HA's statistics tier. */
|
|
37725
|
+
var LTS_BUCKET_MS = 5 * 6e4;
|
|
37726
|
+
/** How often the row cap is enforced. Rarely: it is a bound, not a deadline. */
|
|
37727
|
+
var CAP_SWEEP_INTERVAL_MS = 6 * 36e5;
|
|
37728
|
+
var LTS_COLUMNS = [
|
|
37729
|
+
(
|
|
37730
|
+
/** `<subject>|<series>|<scope>|<bucketStart>` — deterministic, so a bucket
|
|
37731
|
+
* flushed twice replaces itself rather than doubling. */
|
|
37732
|
+
{
|
|
37733
|
+
name: "id",
|
|
37734
|
+
type: "TEXT",
|
|
37735
|
+
primaryKey: true,
|
|
37736
|
+
notNull: true
|
|
37737
|
+
}),
|
|
37738
|
+
(
|
|
37739
|
+
/** The camera id, or the node id. One column, because every query is
|
|
37740
|
+
* "this thing over time" and the thing is one or the other. */
|
|
37741
|
+
{
|
|
37742
|
+
name: "subject",
|
|
37743
|
+
type: "TEXT",
|
|
37744
|
+
notNull: true
|
|
37745
|
+
}),
|
|
37746
|
+
(
|
|
37747
|
+
/** Numeric mirror of `subject` for camera rows, so a per-camera question is
|
|
37748
|
+
* answered by an integer index — every log line and every query about a
|
|
37749
|
+
* device in this repo is keyed by the numeric id. `NULL` for node rows. */
|
|
37750
|
+
{
|
|
37751
|
+
name: "deviceId",
|
|
37752
|
+
type: "INTEGER"
|
|
37753
|
+
}),
|
|
37754
|
+
{
|
|
37755
|
+
name: "series",
|
|
37756
|
+
type: "TEXT",
|
|
37757
|
+
notNull: true
|
|
37758
|
+
},
|
|
37759
|
+
(
|
|
37760
|
+
/** Sub-scope within a series: a zoneId for occupancy, `''` otherwise. */
|
|
37761
|
+
{
|
|
37762
|
+
name: "scope",
|
|
37763
|
+
type: "TEXT",
|
|
37764
|
+
notNull: true
|
|
37765
|
+
}),
|
|
37766
|
+
(
|
|
37767
|
+
/** Bucket start, wall-clock aligned to {@link LTS_BUCKET_MS}. */
|
|
37768
|
+
{
|
|
37769
|
+
name: "bucketStart",
|
|
37770
|
+
type: "INTEGER",
|
|
37771
|
+
notNull: true
|
|
37772
|
+
}),
|
|
37773
|
+
{
|
|
37774
|
+
name: "samples",
|
|
37775
|
+
type: "INTEGER",
|
|
37776
|
+
notNull: true
|
|
37777
|
+
},
|
|
37778
|
+
{
|
|
37779
|
+
name: "sum",
|
|
37780
|
+
type: "REAL",
|
|
37781
|
+
notNull: true
|
|
37782
|
+
},
|
|
37783
|
+
{
|
|
37784
|
+
name: "min",
|
|
37785
|
+
type: "REAL",
|
|
37786
|
+
notNull: true
|
|
37787
|
+
},
|
|
37788
|
+
{
|
|
37789
|
+
name: "max",
|
|
37790
|
+
type: "REAL",
|
|
37791
|
+
notNull: true
|
|
37792
|
+
}
|
|
37793
|
+
];
|
|
37794
|
+
var LTS_INDEXES = [{
|
|
37795
|
+
name: "idx_lts_series_bucket",
|
|
37796
|
+
columns: ["series", "bucketStart"]
|
|
37797
|
+
}, {
|
|
37798
|
+
name: "idx_lts_device",
|
|
37799
|
+
columns: ["deviceId"]
|
|
37800
|
+
}];
|
|
37801
|
+
function ltsBucketStart(at, bucketMs = LTS_BUCKET_MS) {
|
|
37802
|
+
return Math.floor(at / bucketMs) * bucketMs;
|
|
37803
|
+
}
|
|
37804
|
+
var LtsAggregator = class {
|
|
37805
|
+
open = /* @__PURE__ */ new Map();
|
|
37806
|
+
/** Every (subject, series, scope) that has produced a row in this process —
|
|
37807
|
+
* the groups the cap sweep has any reason to look at. */
|
|
37808
|
+
groups = /* @__PURE__ */ new Set();
|
|
37809
|
+
lastCapSweepAt = 0;
|
|
37810
|
+
collection;
|
|
37811
|
+
store;
|
|
37812
|
+
logger;
|
|
37813
|
+
nowFn;
|
|
37814
|
+
bucketMs;
|
|
37815
|
+
maxRows;
|
|
37816
|
+
constructor(deps) {
|
|
37817
|
+
this.collection = deps.collection;
|
|
37818
|
+
this.store = deps.store;
|
|
37819
|
+
this.logger = deps.logger;
|
|
37820
|
+
this.nowFn = deps.now ?? (() => Date.now());
|
|
37821
|
+
this.bucketMs = deps.bucketMs ?? 3e5;
|
|
37822
|
+
this.maxRows = deps.maxRowsPerSeries ?? 105120;
|
|
37823
|
+
}
|
|
37824
|
+
static declare(store, collection) {
|
|
37825
|
+
return store.declareCollection.mutate({
|
|
37826
|
+
collection,
|
|
37827
|
+
columns: [...LTS_COLUMNS],
|
|
37828
|
+
indexes: [...LTS_INDEXES]
|
|
37829
|
+
});
|
|
37830
|
+
}
|
|
37831
|
+
/**
|
|
37832
|
+
* Record one observation. Synchronous, allocation-free after the first
|
|
37833
|
+
* sample of a bucket, and it cannot throw — it sits on paths that are
|
|
37834
|
+
* already producing the value for another reason and must not learn a new
|
|
37835
|
+
* failure mode.
|
|
37836
|
+
*
|
|
37837
|
+
* A non-finite value is DROPPED rather than folded in: one `NaN` would make
|
|
37838
|
+
* `sum`, `min` and `max` all `NaN` for the whole bucket, turning a skewed
|
|
37839
|
+
* row into a meaningless one.
|
|
37840
|
+
*/
|
|
37841
|
+
note(input) {
|
|
37842
|
+
if (!Number.isFinite(input.value)) return;
|
|
37843
|
+
const at = input.at ?? this.nowFn();
|
|
37844
|
+
const scope = input.scope ?? "";
|
|
37845
|
+
const bucketStart = ltsBucketStart(at, this.bucketMs);
|
|
37846
|
+
const key = rowId(input.subject, input.series, scope, bucketStart);
|
|
37847
|
+
const held = this.open.get(key);
|
|
37848
|
+
if (held === void 0) {
|
|
37849
|
+
this.open.set(key, {
|
|
37850
|
+
subject: input.subject,
|
|
37851
|
+
...input.deviceId !== void 0 ? { deviceId: input.deviceId } : {},
|
|
37852
|
+
series: input.series,
|
|
37853
|
+
scope,
|
|
37854
|
+
bucketStart,
|
|
37855
|
+
samples: 1,
|
|
37856
|
+
sum: input.value,
|
|
37857
|
+
min: input.value,
|
|
37858
|
+
max: input.value
|
|
37859
|
+
});
|
|
37860
|
+
return;
|
|
37861
|
+
}
|
|
37862
|
+
held.samples += 1;
|
|
37863
|
+
held.sum += input.value;
|
|
37864
|
+
if (input.value < held.min) held.min = input.value;
|
|
37865
|
+
if (input.value > held.max) held.max = input.value;
|
|
37866
|
+
}
|
|
37867
|
+
/** Buckets currently accumulating — diagnostics, and what a flush would write. */
|
|
37868
|
+
openBuckets() {
|
|
37869
|
+
return [...this.open.values()].map(toRow);
|
|
37870
|
+
}
|
|
37871
|
+
/**
|
|
37872
|
+
* Write every bucket that has CLOSED (its window ended at or before `now`)
|
|
37873
|
+
* and drop it from RAM. Returns the number of rows written.
|
|
37874
|
+
*
|
|
37875
|
+
* The current bucket is deliberately left alone: writing it early would mean
|
|
37876
|
+
* rewriting it on the next tick, which turns one row into up to sixty and
|
|
37877
|
+
* offers every one of them to the checkpoint lottery (D96).
|
|
37878
|
+
*/
|
|
37879
|
+
async flushDue(now = this.nowFn()) {
|
|
37880
|
+
const currentBucket = ltsBucketStart(now, this.bucketMs);
|
|
37881
|
+
const due = [...this.open.values()].filter((b) => b.bucketStart < currentBucket);
|
|
37882
|
+
if (due.length === 0) {
|
|
37883
|
+
await this.maybeSweepCap(now);
|
|
37884
|
+
return 0;
|
|
37885
|
+
}
|
|
37886
|
+
let written = 0;
|
|
37887
|
+
for (const bucket of due) {
|
|
37888
|
+
const key = rowId(bucket.subject, bucket.series, bucket.scope, bucket.bucketStart);
|
|
37889
|
+
try {
|
|
37890
|
+
await this.store.set.mutate({
|
|
37891
|
+
collection: this.collection,
|
|
37892
|
+
key,
|
|
37893
|
+
value: {
|
|
37894
|
+
subject: bucket.subject,
|
|
37895
|
+
...bucket.deviceId !== void 0 ? { deviceId: bucket.deviceId } : {},
|
|
37896
|
+
series: bucket.series,
|
|
37897
|
+
scope: bucket.scope,
|
|
37898
|
+
bucketStart: bucket.bucketStart,
|
|
37899
|
+
samples: bucket.samples,
|
|
37900
|
+
sum: bucket.sum,
|
|
37901
|
+
min: bucket.min,
|
|
37902
|
+
max: bucket.max
|
|
37903
|
+
}
|
|
37904
|
+
});
|
|
37905
|
+
written += 1;
|
|
37906
|
+
this.groups.add(groupKey(bucket.subject, bucket.series, bucket.scope));
|
|
37907
|
+
} catch (err) {
|
|
37908
|
+
this.logger.warn("lts bucket write failed — this interval will be missing", {
|
|
37909
|
+
...bucket.deviceId !== void 0 ? { tags: { deviceId: bucket.deviceId } } : {},
|
|
37910
|
+
meta: {
|
|
37911
|
+
collection: this.collection,
|
|
37912
|
+
series: bucket.series,
|
|
37913
|
+
subject: bucket.subject,
|
|
37914
|
+
bucketStart: bucket.bucketStart,
|
|
37915
|
+
error: String(err)
|
|
37916
|
+
}
|
|
37917
|
+
});
|
|
37918
|
+
}
|
|
37919
|
+
this.open.delete(key);
|
|
37920
|
+
}
|
|
37921
|
+
await this.maybeSweepCap(now);
|
|
37922
|
+
return written;
|
|
37923
|
+
}
|
|
37924
|
+
/** Read closed buckets back. The read surface every chart will use. */
|
|
37925
|
+
async read(query = {}) {
|
|
37926
|
+
const where = {};
|
|
37927
|
+
if (query.series !== void 0) where["series"] = query.series;
|
|
37928
|
+
if (query.subject !== void 0) where["subject"] = query.subject;
|
|
37929
|
+
if (query.scope !== void 0) where["scope"] = query.scope;
|
|
37930
|
+
const records = await this.store.query.query({
|
|
37931
|
+
collection: this.collection,
|
|
37932
|
+
filter: {
|
|
37933
|
+
...Object.keys(where).length > 0 ? { where } : {},
|
|
37934
|
+
...query.from !== void 0 || query.to !== void 0 ? { whereBetween: { bucketStart: [query.from ?? 0, query.to ?? Number.MAX_SAFE_INTEGER] } } : {},
|
|
37935
|
+
orderBy: {
|
|
37936
|
+
field: "bucketStart",
|
|
37937
|
+
direction: "asc"
|
|
37938
|
+
},
|
|
37939
|
+
limit: query.limit ?? 5e3
|
|
37940
|
+
}
|
|
37941
|
+
});
|
|
37942
|
+
const rows = [];
|
|
37943
|
+
for (const record of records) {
|
|
37944
|
+
const row = recordToRow(record.data);
|
|
37945
|
+
if (row !== null) rows.push(row);
|
|
37946
|
+
}
|
|
37947
|
+
return rows;
|
|
37948
|
+
}
|
|
37949
|
+
/**
|
|
37950
|
+
* Enforce the per-(subject, series, scope) row cap, at most once every six
|
|
37951
|
+
* hours. Only groups this process has written to are examined: a group with
|
|
37952
|
+
* no new rows cannot have crossed a cap it was under.
|
|
37953
|
+
*/
|
|
37954
|
+
async maybeSweepCap(now) {
|
|
37955
|
+
if (now - this.lastCapSweepAt < CAP_SWEEP_INTERVAL_MS) return;
|
|
37956
|
+
this.lastCapSweepAt = now;
|
|
37957
|
+
for (const group of this.groups) {
|
|
37958
|
+
const [subject, series, scope] = group.split("\0");
|
|
37959
|
+
if (subject === void 0 || series === void 0 || scope === void 0) continue;
|
|
37960
|
+
const where = {
|
|
37961
|
+
subject,
|
|
37962
|
+
series,
|
|
37963
|
+
scope
|
|
37964
|
+
};
|
|
37965
|
+
try {
|
|
37966
|
+
const excess = await this.store.count.query({
|
|
37967
|
+
collection: this.collection,
|
|
37968
|
+
filter: { where }
|
|
37969
|
+
}) - this.maxRows;
|
|
37970
|
+
if (excess <= 0) continue;
|
|
37971
|
+
const cutoff = (await this.store.query.query({
|
|
37972
|
+
collection: this.collection,
|
|
37973
|
+
filter: {
|
|
37974
|
+
where,
|
|
37975
|
+
orderBy: {
|
|
37976
|
+
field: "bucketStart",
|
|
37977
|
+
direction: "asc"
|
|
37978
|
+
},
|
|
37979
|
+
limit: excess
|
|
37980
|
+
}
|
|
37981
|
+
})).at(-1)?.data["bucketStart"];
|
|
37982
|
+
if (typeof cutoff !== "number") continue;
|
|
37983
|
+
const { deleted } = await this.store.deleteWhere.mutate({
|
|
37984
|
+
collection: this.collection,
|
|
37985
|
+
filter: {
|
|
37986
|
+
where,
|
|
37987
|
+
whereBetween: { bucketStart: [0, cutoff] }
|
|
37988
|
+
}
|
|
37989
|
+
});
|
|
37990
|
+
this.logger.info("lts row cap enforced", { meta: {
|
|
37991
|
+
collection: this.collection,
|
|
37992
|
+
series,
|
|
37993
|
+
subject,
|
|
37994
|
+
deleted,
|
|
37995
|
+
cap: this.maxRows
|
|
37996
|
+
} });
|
|
37997
|
+
} catch (err) {
|
|
37998
|
+
this.logger.warn("lts row cap sweep failed — the series keeps growing this cycle", { meta: {
|
|
37999
|
+
collection: this.collection,
|
|
38000
|
+
series,
|
|
38001
|
+
subject,
|
|
38002
|
+
error: String(err)
|
|
38003
|
+
} });
|
|
38004
|
+
}
|
|
38005
|
+
}
|
|
38006
|
+
}
|
|
38007
|
+
};
|
|
38008
|
+
function groupKey(subject, series, scope) {
|
|
38009
|
+
return `${subject}${series}${scope}`;
|
|
38010
|
+
}
|
|
38011
|
+
function rowId(subject, series, scope, bucketStart) {
|
|
38012
|
+
return `${subject}|${series}|${scope}|${bucketStart}`;
|
|
38013
|
+
}
|
|
38014
|
+
function toRow(bucket) {
|
|
38015
|
+
return {
|
|
38016
|
+
subject: bucket.subject,
|
|
38017
|
+
...bucket.deviceId !== void 0 ? { deviceId: bucket.deviceId } : {},
|
|
38018
|
+
series: bucket.series,
|
|
38019
|
+
scope: bucket.scope,
|
|
38020
|
+
bucketStart: bucket.bucketStart,
|
|
38021
|
+
samples: bucket.samples,
|
|
38022
|
+
sum: bucket.sum,
|
|
38023
|
+
min: bucket.min,
|
|
38024
|
+
max: bucket.max
|
|
38025
|
+
};
|
|
38026
|
+
}
|
|
38027
|
+
/** Structural validation on read. A malformed row is skipped, never charted. */
|
|
38028
|
+
function recordToRow(data) {
|
|
38029
|
+
const subject = data["subject"];
|
|
38030
|
+
const series = data["series"];
|
|
38031
|
+
const scope = data["scope"];
|
|
38032
|
+
if (typeof subject !== "string" || typeof series !== "string") return null;
|
|
38033
|
+
const bucketStart = Number(data["bucketStart"]);
|
|
38034
|
+
const samples = Number(data["samples"]);
|
|
38035
|
+
const sum = Number(data["sum"]);
|
|
38036
|
+
const min = Number(data["min"]);
|
|
38037
|
+
const max = Number(data["max"]);
|
|
38038
|
+
if (![
|
|
38039
|
+
bucketStart,
|
|
38040
|
+
samples,
|
|
38041
|
+
sum,
|
|
38042
|
+
min,
|
|
38043
|
+
max
|
|
38044
|
+
].every((n) => Number.isFinite(n))) return null;
|
|
38045
|
+
const rawDeviceId = data["deviceId"];
|
|
38046
|
+
const deviceId = typeof rawDeviceId === "number" && Number.isFinite(rawDeviceId) ? rawDeviceId : void 0;
|
|
38047
|
+
return {
|
|
38048
|
+
subject,
|
|
38049
|
+
...deviceId !== void 0 ? { deviceId } : {},
|
|
38050
|
+
series,
|
|
38051
|
+
scope: typeof scope === "string" ? scope : "",
|
|
38052
|
+
bucketStart,
|
|
38053
|
+
samples,
|
|
38054
|
+
sum,
|
|
38055
|
+
min,
|
|
38056
|
+
max
|
|
38057
|
+
};
|
|
38058
|
+
}
|
|
38059
|
+
//#endregion
|
|
38060
|
+
//#region src/node-stress-lts.ts
|
|
38061
|
+
/**
|
|
38062
|
+
* @durable class=ledger owner=pipeline-orchestrator
|
|
38063
|
+
* write="one row per (node, series, 5-min bucket), written ONCE when the bucket closes; a node reporting no samples writes nothing"
|
|
38064
|
+
* retention="row CAP per (subject, series, scope) — 105,120 rows ≈ one year of 5-minute buckets. Not an age sweep: an LTS row summarises an interval and has no owner to be orphaned from."
|
|
38065
|
+
*/
|
|
38066
|
+
var NODE_STRESS_LTS_COLLECTION = "pipeline-orchestrator:stats-5m";
|
|
38067
|
+
var NodeStressLts = class {
|
|
38068
|
+
lts;
|
|
38069
|
+
constructor(deps) {
|
|
38070
|
+
this.lts = new LtsAggregator({
|
|
38071
|
+
collection: NODE_STRESS_LTS_COLLECTION,
|
|
38072
|
+
store: deps.store,
|
|
38073
|
+
logger: deps.logger,
|
|
38074
|
+
...deps.now !== void 0 ? { now: deps.now } : {}
|
|
38075
|
+
});
|
|
38076
|
+
}
|
|
38077
|
+
static declare(store) {
|
|
38078
|
+
return LtsAggregator.declare(store, NODE_STRESS_LTS_COLLECTION);
|
|
38079
|
+
}
|
|
38080
|
+
/** Fold one sweep's verdict for one node into the open buckets. */
|
|
38081
|
+
noteSignals(nodeId, signals, at) {
|
|
38082
|
+
this.lts.note({
|
|
38083
|
+
subject: nodeId,
|
|
38084
|
+
series: "node-score",
|
|
38085
|
+
value: signals.score,
|
|
38086
|
+
at
|
|
38087
|
+
});
|
|
38088
|
+
this.lts.note({
|
|
38089
|
+
subject: nodeId,
|
|
38090
|
+
series: "node-queue-pressure",
|
|
38091
|
+
value: signals.queuePressure,
|
|
38092
|
+
at
|
|
38093
|
+
});
|
|
38094
|
+
this.lts.note({
|
|
38095
|
+
subject: nodeId,
|
|
38096
|
+
series: "node-drop-ratio",
|
|
38097
|
+
value: signals.dropRatio,
|
|
38098
|
+
at
|
|
38099
|
+
});
|
|
38100
|
+
this.lts.note({
|
|
38101
|
+
subject: nodeId,
|
|
38102
|
+
series: "node-fps-deficit",
|
|
38103
|
+
value: signals.fpsDeficit,
|
|
38104
|
+
at
|
|
38105
|
+
});
|
|
38106
|
+
}
|
|
38107
|
+
flushDue(now) {
|
|
38108
|
+
return this.lts.flushDue(now);
|
|
38109
|
+
}
|
|
38110
|
+
read(query) {
|
|
38111
|
+
return this.lts.read(query);
|
|
38112
|
+
}
|
|
38113
|
+
openBuckets() {
|
|
38114
|
+
return this.lts.openBuckets();
|
|
38115
|
+
}
|
|
38116
|
+
};
|
|
38117
|
+
//#endregion
|
|
38118
|
+
//#region src/node-stress-move-ledger.ts
|
|
38119
|
+
/**
|
|
38120
|
+
* @durable class=ledger owner=pipeline-orchestrator
|
|
38121
|
+
* write="one row per APPLIED failover move (never per attempt); write-behind"
|
|
38122
|
+
* retention="pruned by the sweep once older than the longest guard window (2 h) — a move no guard can read is growth with no reader"
|
|
38123
|
+
*/
|
|
38124
|
+
var NODE_STRESS_MOVES_COLLECTION = "pipeline-orchestrator:node-stress-moves";
|
|
38125
|
+
var NODE_STRESS_MOVES_COLUMNS = [
|
|
38126
|
+
(
|
|
38127
|
+
/** `<deviceId>:<at>` — one camera can be moved more than once, and each move
|
|
38128
|
+
* spends its own slice of the budget. */
|
|
38129
|
+
{
|
|
38130
|
+
name: "id",
|
|
38131
|
+
type: "TEXT",
|
|
38132
|
+
primaryKey: true,
|
|
38133
|
+
notNull: true
|
|
38134
|
+
}),
|
|
38135
|
+
(
|
|
38136
|
+
/** Indexed: every question about a move is asked per-camera. */
|
|
38137
|
+
{
|
|
38138
|
+
name: "deviceId",
|
|
38139
|
+
type: "INTEGER",
|
|
38140
|
+
notNull: true
|
|
38141
|
+
}),
|
|
38142
|
+
(
|
|
38143
|
+
/** The node the camera LEFT — the return ban is about coming back here. */
|
|
38144
|
+
{
|
|
38145
|
+
name: "fromNodeId",
|
|
38146
|
+
type: "TEXT",
|
|
38147
|
+
notNull: true
|
|
38148
|
+
}),
|
|
38149
|
+
{
|
|
38150
|
+
name: "at",
|
|
38151
|
+
type: "INTEGER",
|
|
38152
|
+
notNull: true
|
|
38153
|
+
}
|
|
38154
|
+
];
|
|
38155
|
+
var NODE_STRESS_MOVES_INDEXES = [{
|
|
38156
|
+
name: "idx_node_stress_moves_device",
|
|
38157
|
+
columns: ["deviceId"]
|
|
38158
|
+
}];
|
|
38159
|
+
function moveId(entry) {
|
|
38160
|
+
return `${entry.deviceId}:${entry.at}`;
|
|
38161
|
+
}
|
|
38162
|
+
var NODE_STRESS_MOVES_SPEC = {
|
|
38163
|
+
collection: NODE_STRESS_MOVES_COLLECTION,
|
|
38164
|
+
columns: NODE_STRESS_MOVES_COLUMNS,
|
|
38165
|
+
indexes: NODE_STRESS_MOVES_INDEXES,
|
|
38166
|
+
writeMode: "write-behind",
|
|
38167
|
+
keyOf: (row) => row.id,
|
|
38168
|
+
toValue: (row) => ({
|
|
38169
|
+
deviceId: row.deviceId,
|
|
38170
|
+
fromNodeId: row.fromNodeId,
|
|
38171
|
+
at: row.at
|
|
38172
|
+
}),
|
|
38173
|
+
fromRecord: (id, data) => {
|
|
38174
|
+
const deviceId = Number(data["deviceId"]);
|
|
38175
|
+
const at = Number(data["at"]);
|
|
38176
|
+
const fromNodeId = data["fromNodeId"];
|
|
38177
|
+
if (!Number.isFinite(deviceId) || !Number.isFinite(at)) return null;
|
|
38178
|
+
if (typeof fromNodeId !== "string" || fromNodeId.length === 0) return null;
|
|
38179
|
+
return {
|
|
38180
|
+
id,
|
|
38181
|
+
deviceId,
|
|
38182
|
+
fromNodeId,
|
|
38183
|
+
at
|
|
38184
|
+
};
|
|
38185
|
+
},
|
|
38186
|
+
deviceIdOf: (row) => row.deviceId,
|
|
38187
|
+
loadLimit: 1e4
|
|
38188
|
+
};
|
|
38189
|
+
var NodeStressMoveLedger = class {
|
|
38190
|
+
ledger;
|
|
38191
|
+
constructor(deps) {
|
|
38192
|
+
this.ledger = new DurableLedger({
|
|
38193
|
+
spec: NODE_STRESS_MOVES_SPEC,
|
|
38194
|
+
store: deps.store,
|
|
38195
|
+
logger: deps.logger
|
|
38196
|
+
});
|
|
38197
|
+
}
|
|
38198
|
+
static declare(store) {
|
|
38199
|
+
return DurableLedger.declare(store, NODE_STRESS_MOVES_SPEC);
|
|
38200
|
+
}
|
|
38201
|
+
/** Boot reseed. Returns the row count so the caller can say out loud how much
|
|
38202
|
+
* budget it recovered — "moves recovered: 0" after a restart is the line that
|
|
38203
|
+
* explains a burst of relocations. */
|
|
38204
|
+
async load() {
|
|
38205
|
+
return (await this.ledger.load()).length;
|
|
38206
|
+
}
|
|
38207
|
+
/** The history the guards read. Pure RAM — a guard must never wait on I/O. */
|
|
38208
|
+
entries() {
|
|
38209
|
+
return this.ledger.snapshot().map(({ deviceId, fromNodeId, at }) => ({
|
|
38210
|
+
deviceId,
|
|
38211
|
+
fromNodeId,
|
|
38212
|
+
at
|
|
38213
|
+
}));
|
|
38214
|
+
}
|
|
38215
|
+
/** Record one APPLIED move. Mirror first; the durable write follows and its
|
|
38216
|
+
* failure is logged, never thrown at a relocation that already happened. */
|
|
38217
|
+
record(entry) {
|
|
38218
|
+
this.ledger.put({
|
|
38219
|
+
...entry,
|
|
38220
|
+
id: moveId(entry)
|
|
38221
|
+
});
|
|
38222
|
+
}
|
|
38223
|
+
/** Drop moves older than `cutoff` — no guard can read them. Returns the count. */
|
|
38224
|
+
pruneOlderThan(cutoff) {
|
|
38225
|
+
const keep = new Set(this.ledger.snapshot().filter((row) => row.at >= cutoff).map((row) => row.id));
|
|
38226
|
+
return this.ledger.pruneExcept(keep);
|
|
38227
|
+
}
|
|
38228
|
+
};
|
|
38229
|
+
//#endregion
|
|
37169
38230
|
//#region src/dispatch-reconcile.ts
|
|
37170
38231
|
function runnerAttachmentKey(nodeId, deviceId) {
|
|
37171
38232
|
return `${nodeId}:${deviceId}`;
|
|
@@ -41631,6 +42692,79 @@ async function migrateStepGating(deps) {
|
|
|
41631
42692
|
} });
|
|
41632
42693
|
}
|
|
41633
42694
|
//#endregion
|
|
42695
|
+
//#region src/zone-mirror-hydration.ts
|
|
42696
|
+
/** How long to wait for a camera list: the hub wires `ctx.api` AFTER the addon
|
|
42697
|
+
* init chain resolves, and `device-manager` answers a moment later still, so a
|
|
42698
|
+
* task kicked off from `onInitialize` loses both races. Same shape as the
|
|
42699
|
+
* bindings migration's poll — one budget covering "no api yet" and "api, but
|
|
42700
|
+
* device-manager not answering yet", because to this sweep they are the same
|
|
42701
|
+
* thing: no fleet to hydrate. */
|
|
42702
|
+
var CAMERA_LIST_WAIT_MS = 6e3;
|
|
42703
|
+
var POLL_INTERVAL_MS = 200;
|
|
42704
|
+
/**
|
|
42705
|
+
* Hydrate the `zones` mirror for every camera once, at boot. Never throws: a
|
|
42706
|
+
* hydration sweep that cannot run must not take the orchestrator's boot with
|
|
42707
|
+
* it — but it is never silent either, because a skipped sweep is exactly the
|
|
42708
|
+
* failure this exists to end.
|
|
42709
|
+
*/
|
|
42710
|
+
async function hydrateZoneMirrorsAtBoot(deps) {
|
|
42711
|
+
const cameraIds = await readCameraIds(deps);
|
|
42712
|
+
if (cameraIds === null) return;
|
|
42713
|
+
for (const hydrator of deps.hydrators) {
|
|
42714
|
+
let written = 0;
|
|
42715
|
+
let unchanged = 0;
|
|
42716
|
+
let failed = 0;
|
|
42717
|
+
for (const deviceId of cameraIds) try {
|
|
42718
|
+
const outcome = await hydrator.hydrate(deviceId);
|
|
42719
|
+
if (outcome === "written") written++;
|
|
42720
|
+
else if (outcome === "unchanged") unchanged++;
|
|
42721
|
+
else failed++;
|
|
42722
|
+
} catch (err) {
|
|
42723
|
+
failed++;
|
|
42724
|
+
deps.logger.warn("zone mirror boot hydration failed for this camera", {
|
|
42725
|
+
tags: { deviceId },
|
|
42726
|
+
meta: {
|
|
42727
|
+
mirror: hydrator.mirror,
|
|
42728
|
+
error: errMsg(err)
|
|
42729
|
+
}
|
|
42730
|
+
});
|
|
42731
|
+
}
|
|
42732
|
+
deps.logger.info("zone mirror boot hydration complete", { meta: {
|
|
42733
|
+
mirror: hydrator.mirror,
|
|
42734
|
+
cameras: cameraIds.length,
|
|
42735
|
+
written,
|
|
42736
|
+
unchanged,
|
|
42737
|
+
failed
|
|
42738
|
+
} });
|
|
42739
|
+
}
|
|
42740
|
+
}
|
|
42741
|
+
/**
|
|
42742
|
+
* The camera fleet, or `null` when it never became readable inside the budget
|
|
42743
|
+
* — in which case this has already said so. Retried inside one budget rather
|
|
42744
|
+
* than once, because at orchestrator boot "no api yet" and "device-manager not
|
|
42745
|
+
* answering yet" are both transient and indistinguishable from here.
|
|
42746
|
+
*/
|
|
42747
|
+
async function readCameraIds(deps) {
|
|
42748
|
+
const deadline = Date.now() + (deps.apiWaitMs ?? CAMERA_LIST_WAIT_MS);
|
|
42749
|
+
let lastError = null;
|
|
42750
|
+
for (;;) {
|
|
42751
|
+
const api = deps.api();
|
|
42752
|
+
if (api) try {
|
|
42753
|
+
return (await api.deviceManager.listAll.query({
|
|
42754
|
+
isCamera: true,
|
|
42755
|
+
projection: "slim"
|
|
42756
|
+
})).map((camera) => camera.id);
|
|
42757
|
+
} catch (err) {
|
|
42758
|
+
lastError = err;
|
|
42759
|
+
}
|
|
42760
|
+
if (Date.now() >= deadline) {
|
|
42761
|
+
deps.logger.warn("zones mirror boot hydration SKIPPED — camera list never became readable; mirror-only consumers keep whatever the last write left", { meta: { error: lastError === null ? "ctx.api unavailable" : errMsg(lastError) } });
|
|
42762
|
+
return null;
|
|
42763
|
+
}
|
|
42764
|
+
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
42765
|
+
}
|
|
42766
|
+
}
|
|
42767
|
+
//#endregion
|
|
41634
42768
|
//#region src/zone-rules-provider.ts
|
|
41635
42769
|
/**
|
|
41636
42770
|
* `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
|
|
@@ -41647,6 +42781,20 @@ async function migrateStepGating(deps) {
|
|
|
41647
42781
|
* before persisting — partial / corrupt writes are rejected outright
|
|
41648
42782
|
* since rules drive runtime filtering and a bad payload would silently
|
|
41649
42783
|
* widen the operator's intended scope.
|
|
42784
|
+
*
|
|
42785
|
+
* ── THE MIRROR IS HYDRATED AT BOOT, NOT ONLY ON MUTATION ──────────
|
|
42786
|
+
*
|
|
42787
|
+
* Same root cause as the `zones` slice (see `zones-provider.ts`): the
|
|
42788
|
+
* mirror used to be written only by `persist`, so a camera nobody had
|
|
42789
|
+
* mutated since its runtime-state row was last written had NO
|
|
42790
|
+
* `zone-rules` slice, and every mirror-only consumer — motion-wasm's
|
|
42791
|
+
* zone gate, the detection-pipeline zone gate, the admin rules editor
|
|
42792
|
+
* — read "no rules" until an operator happened to save one. Zones and
|
|
42793
|
+
* rules gate together, so hydrating one without the other still leaves
|
|
42794
|
+
* both runner-side gates inert. {@link ZoneRulesProvider.hydrateMirror}
|
|
42795
|
+
* is what the boot sweep (`zone-mirror-hydration.ts`) calls; it is an
|
|
42796
|
+
* RPC read, never an event replay (D8), and a stage that could not be
|
|
42797
|
+
* read leaves the mirror untouched (D49).
|
|
41650
42798
|
*/
|
|
41651
42799
|
/**
|
|
41652
42800
|
* Every zone-rule stage, in the declared enum order. The unified device-state
|
|
@@ -41690,6 +42838,11 @@ var ZoneRulesProvider = class {
|
|
|
41690
42838
|
* write on one stage can never drop the other. Built lazily + memoised.
|
|
41691
42839
|
*/
|
|
41692
42840
|
stateByDevice = /* @__PURE__ */ new Map();
|
|
42841
|
+
/** Per-device fingerprint of the slice this process last successfully wrote.
|
|
42842
|
+
* Absent ⇒ never mirrored here, so the next hydration writes. */
|
|
42843
|
+
mirroredFingerprint = /* @__PURE__ */ new Map();
|
|
42844
|
+
/** Devices already reported as unmirrorable — one warn per episode. */
|
|
42845
|
+
reportedMirrorFailure = /* @__PURE__ */ new Set();
|
|
41693
42846
|
constructor(ctx) {
|
|
41694
42847
|
this.ctx = ctx;
|
|
41695
42848
|
}
|
|
@@ -41732,10 +42885,52 @@ var ZoneRulesProvider = class {
|
|
|
41732
42885
|
if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
|
|
41733
42886
|
await this.persist(deviceId, stage, parsed.data);
|
|
41734
42887
|
}
|
|
42888
|
+
/**
|
|
42889
|
+
* Reconcile ONE device's `zone-rules` mirror against the durable block.
|
|
42890
|
+
* Called by the boot sweep for every camera. Never throws.
|
|
42891
|
+
*
|
|
42892
|
+
* A stage whose read FAILED aborts the whole hydration: the mirror is a
|
|
42893
|
+
* single slice carrying every stage, so writing a partially-read block would
|
|
42894
|
+
* publish "this stage has no rules" off a store blip — and an empty
|
|
42895
|
+
* `motion`/`detection` array is what makes a gate stop gating.
|
|
42896
|
+
*/
|
|
42897
|
+
async hydrateMirror(deviceId) {
|
|
42898
|
+
const perDevice = this.stageCache(deviceId);
|
|
42899
|
+
for (const stage of ALL_STAGES) {
|
|
42900
|
+
const read = await this.readRules(deviceId, stage);
|
|
42901
|
+
if (!read.ok) return "unreadable";
|
|
42902
|
+
perDevice.set(stage, read.rules);
|
|
42903
|
+
}
|
|
42904
|
+
const slice = await this.buildSliceValue(deviceId, perDevice);
|
|
42905
|
+
const fingerprint = JSON.stringify(slice);
|
|
42906
|
+
if (this.mirroredFingerprint.get(deviceId) === fingerprint) return "unchanged";
|
|
42907
|
+
try {
|
|
42908
|
+
await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
|
|
42909
|
+
capName: ZONE_RULES_CAP_NAME,
|
|
42910
|
+
slice
|
|
42911
|
+
});
|
|
42912
|
+
} catch (err) {
|
|
42913
|
+
if (!this.reportedMirrorFailure.has(deviceId)) {
|
|
42914
|
+
this.reportedMirrorFailure.add(deviceId);
|
|
42915
|
+
this.ctx.logger.warn("zone-rules mirror write failed — mirror-only zone gates see NO rules for this camera until it lands", {
|
|
42916
|
+
tags: { deviceId },
|
|
42917
|
+
meta: { error: err instanceof Error ? err.message : String(err) }
|
|
42918
|
+
});
|
|
42919
|
+
}
|
|
42920
|
+
return "write-failed";
|
|
42921
|
+
}
|
|
42922
|
+
const first = !this.mirroredFingerprint.has(deviceId);
|
|
42923
|
+
this.mirroredFingerprint.set(deviceId, fingerprint);
|
|
42924
|
+
this.reportedMirrorFailure.delete(deviceId);
|
|
42925
|
+
if (first) this.ctx.logger.info("zone-rules mirror hydrated from the durable block", { tags: { deviceId } });
|
|
42926
|
+
return "written";
|
|
42927
|
+
}
|
|
41735
42928
|
/** Drop a device's cache entries. Called when the device is removed. */
|
|
41736
42929
|
forgetDevice(deviceId) {
|
|
41737
42930
|
this.cache.delete(deviceId);
|
|
41738
42931
|
this.stateByDevice.delete(deviceId);
|
|
42932
|
+
this.mirroredFingerprint.delete(deviceId);
|
|
42933
|
+
this.reportedMirrorFailure.delete(deviceId);
|
|
41739
42934
|
}
|
|
41740
42935
|
/** Cap-surface read: a failure folds to the empty list, as it always has. */
|
|
41741
42936
|
async loadRules(deviceId, stage) {
|
|
@@ -41750,11 +42945,7 @@ var ZoneRulesProvider = class {
|
|
|
41750
42945
|
* momentary store blip into a permanent one.
|
|
41751
42946
|
*/
|
|
41752
42947
|
async readRules(deviceId, stage) {
|
|
41753
|
-
|
|
41754
|
-
if (!perDevice) {
|
|
41755
|
-
perDevice = /* @__PURE__ */ new Map();
|
|
41756
|
-
this.cache.set(deviceId, perDevice);
|
|
41757
|
-
}
|
|
42948
|
+
const perDevice = this.stageCache(deviceId);
|
|
41758
42949
|
const cached = perDevice.get(stage);
|
|
41759
42950
|
if (cached) return {
|
|
41760
42951
|
ok: true,
|
|
@@ -41795,11 +42986,7 @@ var ZoneRulesProvider = class {
|
|
|
41795
42986
|
};
|
|
41796
42987
|
}
|
|
41797
42988
|
async persist(deviceId, stage, rules) {
|
|
41798
|
-
|
|
41799
|
-
if (!perDevice) {
|
|
41800
|
-
perDevice = /* @__PURE__ */ new Map();
|
|
41801
|
-
this.cache.set(deviceId, perDevice);
|
|
41802
|
-
}
|
|
42989
|
+
const perDevice = this.stageCache(deviceId);
|
|
41803
42990
|
perDevice.set(stage, rules);
|
|
41804
42991
|
await this.rulesState(deviceId).update((prev) => ({
|
|
41805
42992
|
...prev,
|
|
@@ -41811,6 +42998,8 @@ var ZoneRulesProvider = class {
|
|
|
41811
42998
|
capName: ZONE_RULES_CAP_NAME,
|
|
41812
42999
|
slice: sliceValue
|
|
41813
43000
|
});
|
|
43001
|
+
this.mirroredFingerprint.set(deviceId, JSON.stringify(sliceValue));
|
|
43002
|
+
this.reportedMirrorFailure.delete(deviceId);
|
|
41814
43003
|
} catch (err) {
|
|
41815
43004
|
this.ctx.logger.debug("zone-rules slice mirror failed", {
|
|
41816
43005
|
tags: { deviceId },
|
|
@@ -41829,6 +43018,15 @@ var ZoneRulesProvider = class {
|
|
|
41829
43018
|
* also warms the cache). Iterates {@link ALL_STAGES} so it stays exhaustive
|
|
41830
43019
|
* over the cap's stage discriminator without a per-stage branch.
|
|
41831
43020
|
*/
|
|
43021
|
+
/** The per-stage cache map for a device, created on first use. */
|
|
43022
|
+
stageCache(deviceId) {
|
|
43023
|
+
let perDevice = this.cache.get(deviceId);
|
|
43024
|
+
if (!perDevice) {
|
|
43025
|
+
perDevice = /* @__PURE__ */ new Map();
|
|
43026
|
+
this.cache.set(deviceId, perDevice);
|
|
43027
|
+
}
|
|
43028
|
+
return perDevice;
|
|
43029
|
+
}
|
|
41832
43030
|
async buildSliceValue(deviceId, perDevice) {
|
|
41833
43031
|
const slice = {
|
|
41834
43032
|
motion: [],
|
|
@@ -41846,11 +43044,38 @@ var ZoneRulesProvider = class {
|
|
|
41846
43044
|
*
|
|
41847
43045
|
* Per-camera CRUD over polygon detection zones. Persists to the
|
|
41848
43046
|
* orchestrator's per-device settings store under the `zones` key and
|
|
41849
|
-
* mirrors
|
|
43047
|
+
* mirrors the catalogue into the device-state `zones` slice via
|
|
41850
43048
|
* `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
|
|
41851
43049
|
* pipeline-executor, analytics, admin UI) read the live state with
|
|
41852
43050
|
* the canonical `dev.state.zones.onChanged` channel.
|
|
41853
43051
|
*
|
|
43052
|
+
* ── THIS PROVIDER OWNS THE MIRROR FOR ITS WHOLE LIFETIME ──────────
|
|
43053
|
+
*
|
|
43054
|
+
* The mirror used to be written in exactly ONE place: `persist`, i.e.
|
|
43055
|
+
* only when an operator mutates a zone. Nothing seeded it at startup,
|
|
43056
|
+
* so a camera whose runtime-state row had never been written (or had
|
|
43057
|
+
* been reset) had no `zones` slice at all, and every mirror-only
|
|
43058
|
+
* consumer concluded "this camera has no zones" — forever, because no
|
|
43059
|
+
* mutation was coming. Live cost (camera 617 'Parcheggio',
|
|
43060
|
+
* 2026-08-12): `zones.listZones {617}` returned 'Parcheggio papà'
|
|
43061
|
+
* while `deviceState.getCapSlice {617,'zones'}` returned `null`;
|
|
43062
|
+
* occupancy dropped all three parked cars into `unzoned`, its zone
|
|
43063
|
+
* rule could never fire, and the admin Zones tab — which reads the
|
|
43064
|
+
* same mirror — showed "No zones yet".
|
|
43065
|
+
*
|
|
43066
|
+
* So the mirror is reconciled against the durable catalogue on the
|
|
43067
|
+
* READ path too ({@link ZonesProvider.hydrateMirror}), and the boot
|
|
43068
|
+
* sweep in `zone-mirror-hydration.ts` walks every camera once at
|
|
43069
|
+
* startup. That is deliberately an RPC read, never an event replay:
|
|
43070
|
+
* events are lossy telemetry and a slice change that was dropped is
|
|
43071
|
+
* never re-sent (D8).
|
|
43072
|
+
*
|
|
43073
|
+
* Two rules the hydration path must keep (D49):
|
|
43074
|
+
* - a durable read that FAILED changes nothing — it must never be
|
|
43075
|
+
* mirrored, and must not be cached as "this camera has no zones";
|
|
43076
|
+
* - hydration is a reconcile, not a mutation: `onZonesChanged` is
|
|
43077
|
+
* NOT fired, so nothing downstream re-dispatches on a boot read.
|
|
43078
|
+
*
|
|
41854
43079
|
* Onboard / firmware-reported zones are out of scope for now — every
|
|
41855
43080
|
* zone is operator-drawn. The provider keeps the surface symmetric:
|
|
41856
43081
|
* `addZone` rejects id collisions, `updateZone` requires an existing
|
|
@@ -41859,6 +43084,11 @@ var ZoneRulesProvider = class {
|
|
|
41859
43084
|
var ZONES_STORE_KEY = "zones";
|
|
41860
43085
|
var ZONES_CAP_NAME = "zones";
|
|
41861
43086
|
var ZonesArraySchema = array(ZoneSchema);
|
|
43087
|
+
/** Identity of a mirrored catalogue — cheap enough to compare on every read,
|
|
43088
|
+
* and it changes whenever anything an operator can see changes. */
|
|
43089
|
+
function fingerprintZones(zones) {
|
|
43090
|
+
return JSON.stringify(zones);
|
|
43091
|
+
}
|
|
41862
43092
|
var ZonesProvider = class {
|
|
41863
43093
|
ctx;
|
|
41864
43094
|
/** Per-device cache. Hydrated lazily on first read for a device. */
|
|
@@ -41869,6 +43099,16 @@ var ZonesProvider = class {
|
|
|
41869
43099
|
* be dropped on persist. Built lazily + memoised per device.
|
|
41870
43100
|
*/
|
|
41871
43101
|
stateByDevice = /* @__PURE__ */ new Map();
|
|
43102
|
+
/**
|
|
43103
|
+
* Per-device fingerprint of the catalogue this process last successfully
|
|
43104
|
+
* wrote to the mirror. Absent ⇒ this process has never mirrored the device,
|
|
43105
|
+
* so the next read hydrates; a write that FAILED leaves it absent, which is
|
|
43106
|
+
* what makes the retry happen on the next read rather than never.
|
|
43107
|
+
*/
|
|
43108
|
+
mirroredFingerprint = /* @__PURE__ */ new Map();
|
|
43109
|
+
/** Devices already reported as unmirrorable — keeps the warn to one per
|
|
43110
|
+
* episode instead of one per read. */
|
|
43111
|
+
reportedMirrorFailure = /* @__PURE__ */ new Set();
|
|
41872
43112
|
constructor(ctx) {
|
|
41873
43113
|
this.ctx = ctx;
|
|
41874
43114
|
}
|
|
@@ -41894,55 +43134,141 @@ var ZonesProvider = class {
|
|
|
41894
43134
|
}
|
|
41895
43135
|
return handle;
|
|
41896
43136
|
}
|
|
43137
|
+
/**
|
|
43138
|
+
* The device's catalogue — and, on the way past, the one place a mirror-only
|
|
43139
|
+
* consumer's boot blindness is cured: every authoritative read reconciles the
|
|
43140
|
+
* device-state slice against what it just read.
|
|
43141
|
+
*/
|
|
41897
43142
|
async listZones({ deviceId }) {
|
|
41898
|
-
|
|
43143
|
+
const read = await this.readCatalogue(deviceId);
|
|
43144
|
+
if (read.ok) await this.ensureMirror(deviceId, read.zones);
|
|
43145
|
+
return read.zones;
|
|
41899
43146
|
}
|
|
41900
43147
|
async addZone({ deviceId, zone }) {
|
|
41901
|
-
const existing = await this.
|
|
43148
|
+
const existing = await this.readForMutation(deviceId);
|
|
41902
43149
|
if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
|
|
41903
43150
|
await this.persist(deviceId, [...existing, zone]);
|
|
41904
43151
|
}
|
|
41905
43152
|
async updateZone({ deviceId, zone }) {
|
|
41906
|
-
const existing = await this.
|
|
43153
|
+
const existing = await this.readForMutation(deviceId);
|
|
41907
43154
|
if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
|
|
41908
43155
|
const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
|
|
41909
43156
|
await this.persist(deviceId, next);
|
|
41910
43157
|
}
|
|
41911
43158
|
async removeZone({ deviceId, zoneId }) {
|
|
41912
|
-
const existing = await this.
|
|
43159
|
+
const existing = await this.readForMutation(deviceId);
|
|
41913
43160
|
if (!existing.some((entry) => entry.id === zoneId)) return;
|
|
41914
43161
|
await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
|
|
41915
43162
|
}
|
|
41916
43163
|
/**
|
|
43164
|
+
* Reconcile ONE device's mirror against the durable catalogue. The boot
|
|
43165
|
+
* sweep (`zone-mirror-hydration.ts`) calls this for every camera so a
|
|
43166
|
+
* mirror-only consumer never starts blind; `listZones` calls it too, so a
|
|
43167
|
+
* camera adopted after boot is covered by its first read.
|
|
43168
|
+
*
|
|
43169
|
+
* Never throws — a hydration that cannot happen is reported, and reported
|
|
43170
|
+
* once (see {@link ZoneMirrorHydration}).
|
|
43171
|
+
*/
|
|
43172
|
+
async hydrateMirror(deviceId) {
|
|
43173
|
+
const read = await this.readCatalogue(deviceId);
|
|
43174
|
+
if (!read.ok) return "unreadable";
|
|
43175
|
+
return this.ensureMirror(deviceId, read.zones);
|
|
43176
|
+
}
|
|
43177
|
+
/**
|
|
41917
43178
|
* Drop a device's cache entry. Called when the device is removed so
|
|
41918
43179
|
* the next attach starts from a fresh disk read.
|
|
41919
43180
|
*/
|
|
41920
43181
|
forgetDevice(deviceId) {
|
|
41921
43182
|
this.cache.delete(deviceId);
|
|
41922
43183
|
this.stateByDevice.delete(deviceId);
|
|
43184
|
+
this.mirroredFingerprint.delete(deviceId);
|
|
43185
|
+
this.reportedMirrorFailure.delete(deviceId);
|
|
43186
|
+
}
|
|
43187
|
+
/**
|
|
43188
|
+
* The catalogue for a mutation. A mutation is read-modify-write over the
|
|
43189
|
+
* WHOLE array, so proceeding from a failed read would persist the operator's
|
|
43190
|
+
* zones away — refuse instead.
|
|
43191
|
+
*/
|
|
43192
|
+
async readForMutation(deviceId) {
|
|
43193
|
+
const read = await this.readCatalogue(deviceId);
|
|
43194
|
+
if (!read.ok) throw new Error(`zones: catalogue unreadable for device ${deviceId} — refusing to write`);
|
|
43195
|
+
return read.zones;
|
|
41923
43196
|
}
|
|
41924
|
-
|
|
43197
|
+
/**
|
|
43198
|
+
* Read the durable catalogue, cached per device. A FAILED read is neither
|
|
43199
|
+
* cached nor reported as `[]` — the caller decides what an unknown answer
|
|
43200
|
+
* means for it.
|
|
43201
|
+
*/
|
|
43202
|
+
async readCatalogue(deviceId) {
|
|
41925
43203
|
const cached = this.cache.get(deviceId);
|
|
41926
|
-
if (cached) return
|
|
41927
|
-
|
|
43204
|
+
if (cached) return {
|
|
43205
|
+
ok: true,
|
|
43206
|
+
zones: cached
|
|
43207
|
+
};
|
|
43208
|
+
let zones;
|
|
41928
43209
|
try {
|
|
41929
43210
|
zones = await this.zonesState(deviceId).get();
|
|
41930
43211
|
} catch (err) {
|
|
41931
|
-
this.ctx.logger.warn("zones store read failed —
|
|
43212
|
+
this.ctx.logger.warn("zones store read failed — catalogue UNKNOWN for this device", {
|
|
41932
43213
|
tags: { deviceId },
|
|
41933
43214
|
meta: { error: err instanceof Error ? err.message : String(err) }
|
|
41934
43215
|
});
|
|
43216
|
+
return {
|
|
43217
|
+
ok: false,
|
|
43218
|
+
zones: []
|
|
43219
|
+
};
|
|
41935
43220
|
}
|
|
41936
43221
|
this.cache.set(deviceId, zones);
|
|
41937
|
-
return
|
|
43222
|
+
return {
|
|
43223
|
+
ok: true,
|
|
43224
|
+
zones
|
|
43225
|
+
};
|
|
41938
43226
|
}
|
|
41939
|
-
|
|
41940
|
-
|
|
41941
|
-
|
|
43227
|
+
/**
|
|
43228
|
+
* Make the device-state mirror agree with `zones`. Idempotent per process
|
|
43229
|
+
* via the fingerprint; the hub itself also no-ops an identical
|
|
43230
|
+
* `setCapSlice`, so this is belt-and-braces against needless RPCs, not
|
|
43231
|
+
* against needless writes.
|
|
43232
|
+
*/
|
|
43233
|
+
async ensureMirror(deviceId, zones) {
|
|
43234
|
+
const fingerprint = fingerprintZones(zones);
|
|
43235
|
+
if (this.mirroredFingerprint.get(deviceId) === fingerprint) return "unchanged";
|
|
43236
|
+
try {
|
|
43237
|
+
await this.writeMirror(deviceId, zones);
|
|
43238
|
+
} catch (err) {
|
|
43239
|
+
if (!this.reportedMirrorFailure.has(deviceId)) {
|
|
43240
|
+
this.reportedMirrorFailure.add(deviceId);
|
|
43241
|
+
this.ctx.logger.warn("zones mirror write failed — mirror-only consumers see NO zones for this camera until it lands", {
|
|
43242
|
+
tags: { deviceId },
|
|
43243
|
+
meta: {
|
|
43244
|
+
zones: zones.length,
|
|
43245
|
+
error: err instanceof Error ? err.message : String(err)
|
|
43246
|
+
}
|
|
43247
|
+
});
|
|
43248
|
+
}
|
|
43249
|
+
return "write-failed";
|
|
43250
|
+
}
|
|
43251
|
+
const first = !this.mirroredFingerprint.has(deviceId);
|
|
43252
|
+
this.mirroredFingerprint.set(deviceId, fingerprint);
|
|
43253
|
+
this.reportedMirrorFailure.delete(deviceId);
|
|
43254
|
+
if (first) this.ctx.logger.info("zones mirror hydrated from the durable catalogue", {
|
|
43255
|
+
tags: { deviceId },
|
|
43256
|
+
meta: { zones: zones.length }
|
|
43257
|
+
});
|
|
43258
|
+
return "written";
|
|
43259
|
+
}
|
|
43260
|
+
async writeMirror(deviceId, zones) {
|
|
41942
43261
|
await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
|
|
41943
43262
|
capName: ZONES_CAP_NAME,
|
|
41944
43263
|
slice: { zones }
|
|
41945
43264
|
});
|
|
43265
|
+
}
|
|
43266
|
+
async persist(deviceId, zones) {
|
|
43267
|
+
this.cache.set(deviceId, zones);
|
|
43268
|
+
await this.zonesState(deviceId).set(zones);
|
|
43269
|
+
await this.writeMirror(deviceId, zones);
|
|
43270
|
+
this.mirroredFingerprint.set(deviceId, fingerprintZones(zones));
|
|
43271
|
+
this.reportedMirrorFailure.delete(deviceId);
|
|
41946
43272
|
this.ctx.onZonesChanged?.(deviceId, zones);
|
|
41947
43273
|
}
|
|
41948
43274
|
};
|
|
@@ -42037,8 +43363,18 @@ async function buildOrchestratorControllers(deps) {
|
|
|
42037
43363
|
readGlobalSettings: () => globalSettings,
|
|
42038
43364
|
getInitTimestamp: () => deps.initTimestamp
|
|
42039
43365
|
});
|
|
43366
|
+
const nodeStressMoves = new NodeStressMoveLedger({
|
|
43367
|
+
store: deps.ctx().api.settingsStore,
|
|
43368
|
+
logger: deps.ctx().logger.child("node-stress")
|
|
43369
|
+
});
|
|
43370
|
+
const nodeStressLts = new NodeStressLts({
|
|
43371
|
+
store: deps.ctx().api.settingsStore,
|
|
43372
|
+
logger: deps.ctx().logger.child("node-stress-lts")
|
|
43373
|
+
});
|
|
42040
43374
|
const nodeStress = new NodeStressController({
|
|
42041
43375
|
ledger,
|
|
43376
|
+
moves: nodeStressMoves,
|
|
43377
|
+
lts: nodeStressLts,
|
|
42042
43378
|
topology,
|
|
42043
43379
|
settingsStore,
|
|
42044
43380
|
logger: deps.ctx().logger,
|
|
@@ -42047,6 +43383,17 @@ async function buildOrchestratorControllers(deps) {
|
|
|
42047
43383
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
42048
43384
|
readGlobalSettings: () => globalSettings
|
|
42049
43385
|
});
|
|
43386
|
+
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
43387
|
+
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg(err) } });
|
|
43388
|
+
});
|
|
43389
|
+
await NodeStressLts.declare(deps.ctx().api.settingsStore).catch((err) => {
|
|
43390
|
+
deps.ctx().logger.warn("node-stress statistics unavailable — no baseline this boot", { meta: { error: errMsg(err) } });
|
|
43391
|
+
});
|
|
43392
|
+
const nodeStressLtsTimer = setInterval(() => {
|
|
43393
|
+
nodeStressLts.flushDue().catch((err) => {
|
|
43394
|
+
deps.ctx().logger.warn("node-stress statistics flush failed", { meta: { error: errMsg(err) } });
|
|
43395
|
+
});
|
|
43396
|
+
}, 6e4);
|
|
42050
43397
|
nodeStress.start();
|
|
42051
43398
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
42052
43399
|
/**
|
|
@@ -42355,6 +43702,19 @@ async function buildOrchestratorControllers(deps) {
|
|
|
42355
43702
|
});
|
|
42356
43703
|
}
|
|
42357
43704
|
});
|
|
43705
|
+
hydrateZoneMirrorsAtBoot({
|
|
43706
|
+
api: () => deps.ctx().api ?? null,
|
|
43707
|
+
hydrators: [{
|
|
43708
|
+
mirror: "zones",
|
|
43709
|
+
hydrate: (deviceId) => zonesProvider.hydrateMirror(deviceId)
|
|
43710
|
+
}, {
|
|
43711
|
+
mirror: "zone-rules",
|
|
43712
|
+
hydrate: (deviceId) => zoneRulesProvider.hydrateMirror(deviceId)
|
|
43713
|
+
}],
|
|
43714
|
+
logger: deps.ctx().logger.child("zones")
|
|
43715
|
+
}).catch((err) => {
|
|
43716
|
+
deps.ctxIfReady()?.logger.warn("zones mirror boot hydration failed", { meta: { error: errMsg(err) } });
|
|
43717
|
+
});
|
|
42358
43718
|
const unsubOrchestratorSubscriptions = wireOrchestratorSubscriptions({
|
|
42359
43719
|
eventBus: deps.ctx().eventBus,
|
|
42360
43720
|
logger: deps.ctx().logger,
|
|
@@ -42377,6 +43737,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
42377
43737
|
reconcile.scheduleReconcile();
|
|
42378
43738
|
const detectionWiring = new DetectionWiringController({
|
|
42379
43739
|
ctx: () => deps.ctx(),
|
|
43740
|
+
featuresStore: deps.ctx().api.settingsStore,
|
|
42380
43741
|
ledger,
|
|
42381
43742
|
placement,
|
|
42382
43743
|
audio,
|
|
@@ -42401,10 +43762,15 @@ async function buildOrchestratorControllers(deps) {
|
|
|
42401
43762
|
deviceHasOnboardMotionCap: (deviceId) => deps.deviceHasOnboardMotionCap(deviceId),
|
|
42402
43763
|
deviceSettingsSchema: () => deps.deviceSettingsSchema()
|
|
42403
43764
|
});
|
|
43765
|
+
await detectionWiring.hydrateFeaturesMirror().catch((err) => {
|
|
43766
|
+
deps.ctx().logger.warn("device-features mirror not restored — this boot starts cold", { meta: { error: errMsg(err) } });
|
|
43767
|
+
});
|
|
42404
43768
|
return {
|
|
42405
43769
|
ledger,
|
|
42406
43770
|
topology,
|
|
42407
43771
|
loadService,
|
|
43772
|
+
nodeStressLts,
|
|
43773
|
+
nodeStressLtsTimer,
|
|
42408
43774
|
audio,
|
|
42409
43775
|
settingsStore,
|
|
42410
43776
|
loadShed,
|
|
@@ -42658,25 +44024,37 @@ function buildGlobalSettingsSections(options) {
|
|
|
42658
44024
|
id: NATIVE_LEASE_SECTION_ID,
|
|
42659
44025
|
title: "Native frame lease",
|
|
42660
44026
|
tab: "pipeline",
|
|
42661
|
-
description: "How
|
|
44027
|
+
description: "How a decode worker keeps native pixels available for a LATE crop (post-analysis snapshot, face/plate detail) instead of falling back to the ≤640 detection frame. Two things are kept and they cost very differently: a HELD FRAME is a full native raster (~24.9 MB at 4K on the default pinned-RGB24 path, ~6.2 MB at 1080p) and lives only until its own detection result arrives; a TILE is one subject cut from that frame at native resolution and JPEG-encoded (~60-120 KB at 4K), and lives for tens of seconds. A frame on which nothing was detected produces no tiles and costs nothing. Takes effect on the NEXT decode session for a camera, not on sessions already running.",
|
|
42662
44028
|
fields: [
|
|
42663
44029
|
{
|
|
42664
|
-
key:
|
|
44030
|
+
key: NATIVE_LEASE_HOLD_KEY,
|
|
42665
44031
|
type: "slider",
|
|
42666
|
-
label: "
|
|
42667
|
-
description: "How
|
|
42668
|
-
min:
|
|
42669
|
-
max:
|
|
42670
|
-
step:
|
|
42671
|
-
default:
|
|
44032
|
+
label: "Frames held at once",
|
|
44033
|
+
description: "How many delivered frames a worker keeps alive while waiting for their detection results. A frame is freed as soon as its own result comes back and its subject tiles have been cut, so the steady state is inference latency × delivered fps — 1 to 4 frames in practice. This number is only the bound above which the OLDEST held frame is dropped, which is what stops a runner that has stopped answering from pinning RAM. Raising it does not improve crop hit rate; it buys tolerance for a slow runner, and holdOverflow on the metrics line is what tells you that you need it.",
|
|
44034
|
+
min: NATIVE_LEASE_HOLD_FIELD.min,
|
|
44035
|
+
max: NATIVE_LEASE_HOLD_FIELD.max,
|
|
44036
|
+
step: NATIVE_LEASE_HOLD_FIELD.step,
|
|
44037
|
+
default: NATIVE_LEASE_HOLD_FIELD.default,
|
|
42672
44038
|
showValue: true,
|
|
42673
|
-
unit: "
|
|
44039
|
+
unit: "frames"
|
|
44040
|
+
},
|
|
44041
|
+
{
|
|
44042
|
+
key: NATIVE_LEASE_TILE_BUDGET_KEY,
|
|
44043
|
+
type: "slider",
|
|
44044
|
+
label: "Subject tile RAM",
|
|
44045
|
+
description: "RAM per decode worker for the compressed SUBJECT TILES — the native-resolution crops taken at the moment a frame's detections are known, and kept long after the frame itself is gone. This is what serves a crop that arrives seconds late, which measurement says is the ordinary case (2 to 10 seconds on this cluster). At ~60-120 KB a tile, 64 MB is many hundreds of subjects. 0 turns tiles OFF and restores the old behaviour, where a late crop had nothing to fall back to but the ≤640 detection frame.",
|
|
44046
|
+
min: NATIVE_LEASE_TILE_BUDGET_FIELD.min,
|
|
44047
|
+
max: NATIVE_LEASE_TILE_BUDGET_FIELD.max,
|
|
44048
|
+
step: NATIVE_LEASE_TILE_BUDGET_FIELD.step,
|
|
44049
|
+
default: NATIVE_LEASE_TILE_BUDGET_FIELD.default,
|
|
44050
|
+
showValue: true,
|
|
44051
|
+
unit: "MB"
|
|
42674
44052
|
},
|
|
42675
44053
|
{
|
|
42676
44054
|
key: NATIVE_LEASE_BUDGET_KEY,
|
|
42677
44055
|
type: "slider",
|
|
42678
44056
|
label: "Lease RAM ceiling",
|
|
42679
|
-
description: "Hard RAM ceiling for
|
|
44057
|
+
description: "Hard RAM ceiling for held frames, PER decode worker (one worker per camera per plane). This is a safety ceiling, not the working size — the hold count above is what reclaims frames now, and the ceiling is the number above which something is wrong. Lower it on a small host to bound the worst case. 0 DISABLES the lease entirely and falls the worker back to the tiny GPU surface ring, which misses roughly 85% of late crops — that is the behaviour the lease exists to replace, so 0 is a diagnostic setting, not a tuning one.",
|
|
42680
44058
|
min: NATIVE_LEASE_BUDGET_FIELD.min,
|
|
42681
44059
|
max: NATIVE_LEASE_BUDGET_FIELD.max,
|
|
42682
44060
|
step: NATIVE_LEASE_BUDGET_FIELD.step,
|
|
@@ -42945,6 +44323,18 @@ function deriveRuntimeSettings(config) {
|
|
|
42945
44323
|
}
|
|
42946
44324
|
//#endregion
|
|
42947
44325
|
//#region src/index.ts
|
|
44326
|
+
/**
|
|
44327
|
+
* The FULL action catalog, under the name the HUB HARVESTS.
|
|
44328
|
+
*
|
|
44329
|
+
* The hub's forked-addon harvest imports this entry module and reads the
|
|
44330
|
+
* `customActions` NAMED export (`loadForkedCustomActionCatalog`) — returning a
|
|
44331
|
+
* catalog from `onInitialize` is not enough. Without this line the bridge
|
|
44332
|
+
* answers *"no addon 'pipeline-orchestrator' registers custom actions"* for
|
|
44333
|
+
* every action here, which is what `dumpState` had been doing, silently, since
|
|
44334
|
+
* it was written: an admin diagnostic that 404s is worse than no diagnostic,
|
|
44335
|
+
* because nobody discovers it is missing until they need it.
|
|
44336
|
+
*/
|
|
44337
|
+
var customActions = pipelineOrchestratorActions;
|
|
42948
44338
|
var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddon {
|
|
42949
44339
|
/** This node's Moleculer nodeId (from this.ctx.kernel.localNodeId). */
|
|
42950
44340
|
localNodeId = "hub";
|
|
@@ -43162,6 +44552,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
43162
44552
|
pendingRetryTimer = null;
|
|
43163
44553
|
/** Periodic auto-rebalance sweep timer (drift correction under hysteresis). */
|
|
43164
44554
|
autoRebalanceTimer = null;
|
|
44555
|
+
/** Node-stress five-minute statistics + its flush timer (see
|
|
44556
|
+
* `node-stress-lts.ts`). `observe` mode's first durable output. */
|
|
44557
|
+
nodeStressLts = null;
|
|
44558
|
+
nodeStressLtsTimer = null;
|
|
43165
44559
|
initTimestamp = 0;
|
|
43166
44560
|
/** Storage migration maintenance lease. It only gates dispatch; it does not
|
|
43167
44561
|
* change any camera wrapper or persistent pipeline configuration. */
|
|
@@ -43255,6 +44649,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
43255
44649
|
this.unsubOrchestratorSubscriptions = controllers.unsubOrchestratorSubscriptions;
|
|
43256
44650
|
this.pendingRetryTimer = controllers.pendingRetryTimer;
|
|
43257
44651
|
this.autoRebalanceTimer = controllers.autoRebalanceTimer;
|
|
44652
|
+
this.nodeStressLts = controllers.nodeStressLts;
|
|
44653
|
+
this.nodeStressLtsTimer = controllers.nodeStressLtsTimer;
|
|
43258
44654
|
return {
|
|
43259
44655
|
providers: [
|
|
43260
44656
|
{
|
|
@@ -43283,7 +44679,23 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
43283
44679
|
}
|
|
43284
44680
|
],
|
|
43285
44681
|
customActions: pipelineOrchestratorActions,
|
|
43286
|
-
actionHandlers: {
|
|
44682
|
+
actionHandlers: {
|
|
44683
|
+
dumpState: async () => this.dumpDiagnostics(),
|
|
44684
|
+
nodeStressStats: async (input) => {
|
|
44685
|
+
const lts = this.nodeStressLts;
|
|
44686
|
+
const moves = this.nodeStress?.historyView() ?? [];
|
|
44687
|
+
if (!lts) return {
|
|
44688
|
+
rows: [],
|
|
44689
|
+
open: [],
|
|
44690
|
+
moves: [...moves]
|
|
44691
|
+
};
|
|
44692
|
+
return {
|
|
44693
|
+
rows: (await lts.read(input)).map(withStatsMean),
|
|
44694
|
+
open: lts.openBuckets().map(withStatsMean),
|
|
44695
|
+
moves: [...moves]
|
|
44696
|
+
};
|
|
44697
|
+
}
|
|
44698
|
+
}
|
|
43287
44699
|
};
|
|
43288
44700
|
}
|
|
43289
44701
|
/**
|
|
@@ -43318,6 +44730,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
43318
44730
|
clearInterval(this.autoRebalanceTimer);
|
|
43319
44731
|
this.autoRebalanceTimer = null;
|
|
43320
44732
|
}
|
|
44733
|
+
if (this.nodeStressLtsTimer !== null) {
|
|
44734
|
+
clearInterval(this.nodeStressLtsTimer);
|
|
44735
|
+
this.nodeStressLtsTimer = null;
|
|
44736
|
+
}
|
|
44737
|
+
this.nodeStressLts = null;
|
|
43321
44738
|
this.unsubOrchestratorSubscriptions?.();
|
|
43322
44739
|
this.unsubOrchestratorSubscriptions = null;
|
|
43323
44740
|
this.reconcile?.dispose();
|
|
@@ -44271,8 +45688,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
44271
45688
|
return this.isSessionCamera(deviceId) && !this.session.hasActiveSession(deviceId);
|
|
44272
45689
|
}
|
|
44273
45690
|
};
|
|
45691
|
+
/** The derived mean for one statistics row — see `node-stress-lts.ts`. */
|
|
45692
|
+
function withStatsMean(row) {
|
|
45693
|
+
return {
|
|
45694
|
+
...row,
|
|
45695
|
+
mean: row.samples > 0 ? row.sum / row.samples : 0
|
|
45696
|
+
};
|
|
45697
|
+
}
|
|
44274
45698
|
//#endregion
|
|
44275
45699
|
exports.balance = balance;
|
|
44276
45700
|
exports.computeCapacityScore = computeCapacityScore;
|
|
45701
|
+
exports.customActions = customActions;
|
|
44277
45702
|
exports.default = PipelineOrchestratorAddon;
|
|
44278
45703
|
exports.pipelineOrchestratorActions = pipelineOrchestratorActions;
|