@camstack/addon-pipeline-orchestrator 1.2.167 → 1.2.169

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -12904,6 +12904,9 @@ var QueryFilterSchema = object({
12904
12904
  whereBetween: record(string(), tuple([unknown(), unknown()])).optional(),
12905
12905
  /** NULL-safe exclusion: matches rows whose field is NULL OR != the value. */
12906
12906
  whereNot: record(string(), unknown()).optional(),
12907
+ /** NULL-safe exclusion of a SET — `whereNot` for more than one value. An
12908
+ * empty list excludes nothing. See `QueryFilter.whereNotIn`. */
12909
+ whereNotIn: record(string(), array(unknown())).optional(),
12907
12910
  orderBy: object({
12908
12911
  field: string(),
12909
12912
  direction: _enum(["asc", "desc"])
@@ -12924,7 +12927,8 @@ var MutationFilterSchema = object({
12924
12927
  where: record(string(), unknown()).optional(),
12925
12928
  whereIn: record(string(), array(unknown())).optional(),
12926
12929
  whereBetween: record(string(), tuple([unknown(), unknown()])).optional(),
12927
- whereNot: record(string(), unknown()).optional()
12930
+ whereNot: record(string(), unknown()).optional(),
12931
+ whereNotIn: record(string(), array(unknown())).optional()
12928
12932
  });
12929
12933
  /**
12930
12934
  * One scalar an {@link settingsStoreCapability.methods.aggregate} call asks for.
@@ -19638,10 +19642,12 @@ var TrackSourceSchema = _enum([
19638
19642
  * Terminal for the plain `markForTrain` toggle: returning it to `staging` is
19639
19643
  * a deliberate action of the retrain page, not a side effect of a checkbox.
19640
19644
  *
19641
- * There is no `null`. The state is stored `TEXT NOT NULL DEFAULT 'none'` because
19642
- * the store's filter language has only positive equality and `whereIn` no
19643
- * negation, no IS NULL so a NULL would be unselectable by ANY predicate and
19644
- * would make the entire pre-column history immortal in one deploy.
19645
+ * There is no `null`. The state is stored `TEXT NOT NULL DEFAULT 'none'` so that
19646
+ * every row is selectable by a positive predicate. (The filter language has
19647
+ * since grown the NULL-safe `whereNot` / `whereNotIn` and an `IS NULL` reading
19648
+ * of `where: { f: null }`, which is how {@link TrackSourceSchema} can be
19649
+ * filtered on a NULLABLE column — see `excludeSources`. It did not when this
19650
+ * column was designed, and a NOT NULL column is still the better shape.)
19645
19651
  */
19646
19652
  var RetrainStatusSchema = _enum([
19647
19653
  "none",
@@ -20377,7 +20383,9 @@ var RecentTracksQueryInput = object({
20377
20383
  * whose class list is unreadable are kept, and the client's rule stays the
20378
20384
  * exact one.
20379
20385
  */
20380
- classes: array(string()).optional()
20386
+ classes: array(string()).optional(),
20387
+ /** See {@link ExcludeSourcesDoc}. */
20388
+ excludeSources: array(TrackSourceSchema).optional()
20381
20389
  });
20382
20390
  /**
20383
20391
  * A Summary (D359): the per-camera session envelope that references tracks.
@@ -20805,7 +20813,9 @@ DeviceType.Camera, method(object({ deviceId: number() }), array(TrackSchema).rea
20805
20813
  * selected is an operator who has not narrowed anything, and an empty
20806
20814
  * timeline would read as a camera that saw nothing.
20807
20815
  */
20808
- classes: array(string()).optional()
20816
+ classes: array(string()).optional(),
20817
+ /** See {@link ExcludeSourcesDoc}. */
20818
+ excludeSources: array(TrackSourceSchema).optional()
20809
20819
  }), array(TrackSchema).readonly()), method(RecentTracksQueryInput, RecentTracksPageSchema), method(SummariesQueryInput, SummariesPageSchema), method(object({ id: string() }), SummaryDetailSchema.nullable()), method(object({ deviceId: number() }), _void(), {
20810
20820
  kind: "mutation",
20811
20821
  auth: "admin"
@@ -21310,8 +21320,25 @@ var occupancyRecheckFramesField = {
21310
21320
  * (analyzer attaches detected `regions[]`; onboard does not — the
21311
21321
  * camera typically only reports a binary signal plus an optional
21312
21322
  * channel/AI class which lives in dedicated event channels).
21313
- */
21314
- var MotionSourceEnum = _enum(["onboard", "analyzer"]);
21323
+ *
21324
+ * - `onboard` — the camera's firmware said something moved.
21325
+ * - `analyzer` — this runner's frame-diff said so, and attaches `regions[]`.
21326
+ * - `device-activity` — the DEVICE said it is doing its job: the
21327
+ * `recording-signal` LEVEL the same device raises for the recorder
21328
+ * ([D380](../../../../docs/decisions/adr-0380-a-device-decided-recording-is-a-mode-with-no-schedule-seeded-once.md)),
21329
+ * republished as a motion source. It attaches **nothing** — no regions, no
21330
+ * class: the only fact it carries is that the device is active, and a robot
21331
+ * vacuum that is itself the moving object has no region worth sending. It is
21332
+ * a LEVEL, so unlike `onboard` it has a real falling edge, and unlike
21333
+ * `analyzer` it must not open the frame-diff side-channel — the runner's
21334
+ * `handleOnboardMotionAnalyzer` gate is `source === 'onboard'` and stays that
21335
+ * way ([D392](../../../../docs/decisions/adr-0392-a-device-that-says-it-is-working-is-a-motion-source-of-its-own.md)).
21336
+ */
21337
+ var MotionSourceEnum = _enum([
21338
+ "onboard",
21339
+ "analyzer",
21340
+ "device-activity"
21341
+ ]);
21315
21342
  /**
21316
21343
  * List of motion sources active on a camera. Empty array is valid:
21317
21344
  * "no source" — happens for battery cams without firmware motion when
@@ -21575,13 +21602,20 @@ var RunnerCameraDeviceUIFields = [
21575
21602
  type: "multiselect",
21576
21603
  label: "Motion Sources",
21577
21604
  default: ["analyzer"],
21578
- options: [{
21579
- value: "analyzer",
21580
- label: "Frame-diff Analyzer (motion addon)"
21581
- }, {
21582
- value: "onboard",
21583
- label: "Camera Onboard Sensor"
21584
- }]
21605
+ options: [
21606
+ {
21607
+ value: "analyzer",
21608
+ label: "Frame-diff Analyzer (motion addon)"
21609
+ },
21610
+ {
21611
+ value: "onboard",
21612
+ label: "Camera Onboard Sensor"
21613
+ },
21614
+ {
21615
+ value: "device-activity",
21616
+ label: "Device activity (the device says it is working)"
21617
+ }
21618
+ ]
21585
21619
  },
21586
21620
  {
21587
21621
  key: "motionFps",
@@ -29578,8 +29612,38 @@ var RecordingSignalStatusSchema = object({
29578
29612
  /** Ms epoch of the last `active` transition. 0 if never observed. */
29579
29613
  lastChangedAt: number()
29580
29614
  });
29581
- RecordingSignalStatusSchema.extend({ lastFetchedAt: number() });
29582
- DeviceType.Camera, method(object({ deviceId: number() }), RecordingSignalStatusSchema);
29615
+ /** The runtime-state slice: the status plus the clock every slice carries. */
29616
+ var RecordingSignalRuntimeStateSchema = RecordingSignalStatusSchema.extend({ lastFetchedAt: number() });
29617
+ var recordingSignalCapability = {
29618
+ name: "recording-signal",
29619
+ scope: "device",
29620
+ deviceNative: true,
29621
+ mode: "singleton",
29622
+ deviceTypes: [DeviceType.Camera],
29623
+ methods: {
29624
+ /** The current level, straight from the slice the provider keeps fresh. */
29625
+ getStatus: method(object({ deviceId: number() }), RecordingSignalStatusSchema) },
29626
+ status: {
29627
+ schema: RecordingSignalStatusSchema,
29628
+ kind: "push",
29629
+ empty: {
29630
+ active: false,
29631
+ reason: "unknown",
29632
+ lastChangedAt: 0
29633
+ }
29634
+ },
29635
+ runtimeState: RecordingSignalRuntimeStateSchema,
29636
+ /**
29637
+ * Runtime-state durability: **session** — a restored `active: true` from
29638
+ * before a restart is exactly the stale level the recorder's reconcile bound
29639
+ * exists to end, and the provider re-derives the true level on activation
29640
+ * anyway. Nothing is lost by forgetting it; a lie is avoided.
29641
+ *
29642
+ * See `RuntimeStateDurability`. Enforced by
29643
+ * `scripts/check-runtime-state-durability.ts`.
29644
+ */
29645
+ durability: "session"
29646
+ };
29583
29647
  /**
29584
29648
  * scene-monitor — device-scoped reference-region state cap. An operator marks
29585
29649
  * a rect ROI on a camera frame and names one or more states; the engine
@@ -40055,477 +40119,278 @@ function deviceBackendToFormat(backend) {
40055
40119
  return DEVICE_BACKEND_TO_FORMAT[backend] ?? "onnx";
40056
40120
  }
40057
40121
  //#endregion
40058
- //#region src/inference-device-model.ts
40122
+ //#region src/audio-chunk-poller.ts
40059
40123
  /**
40060
- * Per-device default object-detection model + deviceKey parsing for the
40061
- * orchestrator's device-aware `getNodeInferenceDevices` view.
40124
+ * `AudioChunkPoller` the consumer-side poll loop of the decoded audio-chunk
40125
+ * plane (Phase 5 / D9).
40062
40126
  *
40063
- * This DUPLICATES the executor's per-device model resolution (P0-3:
40064
- * `resolveDeviceEngine` + `MODEL_BY_CLASS` + the object-detection step's
40065
- * `defaultModelIdByFormat` in `@camstack/addon-pipeline`). It is duplicated
40066
- * not imported because cross-addon imports are forbidden (the orchestrator
40067
- * and the detection-pipeline are separate addons; only tRPC crosses the
40068
- * boundary). Keep this in sync with `default-detection-model.ts` /
40069
- * `step-definitions.ts` if the executor's defaults change.
40127
+ * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
40128
+ * path. A live callback cannot cross a process boundary; once the `pipeline`
40129
+ * group is dissolved (Task 8) the orchestrator runs in a different process
40130
+ * from the broker, so audio delivery must go over tRPC.
40070
40131
  *
40071
- * The returned ids are honest catalog ids (verified present):
40072
- * - `yolov9m-320-int8` — Intel NPU + iGPU (yolo26 does NOT compile on the NPU)
40073
- * - `yolov9m-320` — Apple ANE (CoreML)
40074
- * - `ssd-mobilenet-v2-coco-edgetpu` — Coral USB Edge TPU (tflite)
40075
- * - `yolo26n` — CPU / CUDA (the object-detection step's universal
40076
- * nano default)
40132
+ * The consumer:
40077
40133
  *
40078
- * Do not promote the accelerated ids to 640. The evaluation in
40079
- * `docs/benchmarks/pipeline-frame-model-eval.md` failed the 640 promotion
40080
- * gates (0/3 miss recovered at the current threshold).
40134
+ * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC the broker
40135
+ * registers a per-subscription bounded FIFO queue and returns a
40136
+ * `subscriptionId`;
40137
+ * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
40138
+ * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
40139
+ * 3. feeds each chunk to its downstream audio logic;
40140
+ * 4. on teardown, `unsubscribeAudioChunks`.
40141
+ *
40142
+ * Audio is not latency-critical like video, and chunks arrive only ~every
40143
+ * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
40144
+ * a small per-poll burst keeps latency low without busy-spinning. The
40145
+ * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
40146
+ * loses a chunk.
40147
+ *
40148
+ * Boot-race tolerance: the broker for a given camStream may not be registered
40149
+ * yet when the orchestrator wires the subscription (provider addons publish
40150
+ * their cameraStreams asynchronously after their probe completes).
40151
+ * `subscribeAudioChunks` retries with exponential backoff (capped at
40152
+ * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
40153
+ * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
40154
+ * shape so video and audio plumbing self-heal identically.
40081
40155
  */
40156
+ /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
40157
+ var POLL_INTERVAL_MS$1 = 200;
40158
+ /** How many chunks to drain per poll — a small burst absorbs jitter. */
40159
+ var PULL_MAX_COUNT = 8;
40082
40160
  /**
40083
- * The always-on object-detection ROOT step id. A camera session's tracks all
40084
- * originate from this detector, so a device whose engine format can't run it
40085
- * cannot host a camera root. Mirrors the addon-pipeline step id (cross-addon
40086
- * import is forbidden this is the same duplication rationale as the model
40087
- * defaults above).
40161
+ * Consecutive pull failures before we attempt to re-subscribe. A single failed
40162
+ * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
40163
+ * sustained failure means the broker child restarted and dropped our
40164
+ * subscription, so we re-establish it.
40088
40165
  */
40089
- var OBJECT_DETECTION_STEP_ID = "object-detection";
40166
+ var RESUBSCRIBE_AFTER_FAILURES = 2;
40167
+ /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
40168
+ var RESUBSCRIBE_THROTTLE_TICKS = 5;
40169
+ /** First subscribe-retry delay, doubled on every subsequent failure. */
40170
+ var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
40090
40171
  /**
40091
- * Build the camera-root capability predicate for a node from its live catalog:
40092
- * `format canHostCameraRoot`. A format can host a camera root iff the
40093
- * catalog lists at least one object-detection model with a build for that
40094
- * format — byte-for-byte the resolver's per-device skip-gate test for the root
40095
- * step (`addonHasCompatibleModel`), so a device is deemed eligible iff the root
40096
- * would ACTUALLY provision on it.
40097
- *
40098
- * Fails OPEN when the catalog has no object-detection slot at all (never
40099
- * observed in production) so a malformed/empty catalog never strands every
40100
- * device off the balancer. Pure + deterministic.
40172
+ * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller fast
40173
+ * enough to recover within a single reconcile of the orchestrator and slow
40174
+ * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
40101
40175
  */
40102
- function makeRootCapabilityGuard(catalog) {
40103
- for (const slot of catalog.slots) {
40104
- const objDet = slot.addons.find((a) => a.id === OBJECT_DETECTION_STEP_ID);
40105
- if (objDet) return (format) => objDet.models.some((m) => Boolean(m.formats[format]));
40106
- }
40107
- return () => true;
40108
- }
40109
- /** Split a deviceKey (`<backend>:<device>`, or bare `cpu`) into its parts + format.
40110
- * Format comes from the shared {@link deviceBackendToFormat} SSOT (`@camstack/types`)
40111
- * — the previously-local `BACKEND_FORMAT` copy is gone (R3/node-F2). Used only for
40112
- * STORED-ONLY keys (a configured device the live probe didn't return); a probed
40113
- * device carries its own honest `format` from the descriptor. */
40114
- function parseDeviceKey(deviceKey) {
40115
- const colon = deviceKey.indexOf(":");
40116
- const backend = colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
40117
- return {
40118
- backend,
40119
- device: colon >= 0 ? deviceKey.slice(colon + 1) : deviceKey,
40120
- format: deviceBackendToFormat(backend)
40121
- };
40122
- }
40176
+ var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
40123
40177
  /**
40124
- * The object-detection model the executor defaults to for a deviceKey. Mirrors
40125
- * the executor's `MODEL_BY_CLASS` classification (`classifyAccelerator`) plus
40126
- * the tflite `defaultModelIdByFormat` for Coral. Never throws; unknown backends
40127
- * fall back to the universal nano default (`yolo26n`).
40178
+ * Attempts after which a still-failing subscribe escalates from the fast 5 s
40179
+ * ceiling to {@link LONG_SUBSCRIBE_RETRY_BACKOFF_MS}. ~15 attempts ≈ one
40180
+ * minute of fast retries — plenty for the boot races the 5 s ceiling exists
40181
+ * for. A broker that is STILL absent after that is a long-lived condition
40182
+ * (e.g. a DISABLED camera, whose broker the stream-broker reconcile refuses
40183
+ * to recreate — 2026-07-23) and retrying every 5 s forever is pure log/RPC
40184
+ * churn. The slow loop stays alive so audio still recovers automatically
40185
+ * (≤60 s) once the camera is re-enabled.
40128
40186
  */
40129
- function defaultModelIdForDevice(deviceKey) {
40130
- const { backend, device } = parseDeviceKey(deviceKey);
40131
- if (backend === "openvino") {
40132
- if (device === "cpu") return "yolo26n";
40133
- return "yolov9m-320-int8";
40134
- }
40135
- if (backend === "edgetpu") return "ssd-mobilenet-v2-coco-edgetpu";
40136
- if (backend === "coreml") return "yolov9m-320";
40137
- return "yolo26n";
40138
- }
40187
+ var PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS = 15;
40188
+ var LONG_SUBSCRIBE_RETRY_BACKOFF_MS = 6e4;
40139
40189
  /**
40140
- * Step-tree device jump (phase 1): validate every `steps[step].jumpDeviceKey`
40141
- * manual override in a to-be-saved `inferenceDevices` map. A jump target MUST be
40142
- * an enabled∧available device on the SAME node and DIFFERENT from the owning
40143
- * device. `enabledAvailableKeys` is the effective enabled∧available set (from
40144
- * `mergeInferenceDevices(probe, submitted)`) so an absent/unplugged/disabled
40145
- * target is rejected honestly (an operator can't route a step onto a dead pool).
40146
- * Returns the FIRST human-readable error, or `null` when every override is
40147
- * valid. Pure + deterministic.
40190
+ * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
40191
+ *
40192
+ * Always resolves to a teardown closure when the broker is not yet
40193
+ * registered the closure cancels the ongoing retry loop; when polling is
40194
+ * active it stops the loop and releases the broker subscription. Mirrors
40195
+ * `startFrameHandlePoller` so video and audio recover identically.
40148
40196
  */
40149
- function validateJumpTargets(inferenceDevices, enabledAvailableKeys) {
40150
- for (const [deviceKey, entry] of Object.entries(inferenceDevices)) for (const [stepId, step] of Object.entries(entry.steps ?? {})) {
40151
- const target = step.jumpDeviceKey;
40152
- if (target === void 0) continue;
40153
- if (target === deviceKey) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey pointing at its own device`;
40154
- if (!enabledAvailableKeys.has(target)) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey "${target}" that is not an enabled, available device on this node`;
40155
- }
40156
- return null;
40197
+ function startAudioChunkPoller(options) {
40198
+ const lifecycle = {
40199
+ stopped: false,
40200
+ retryTimer: void 0,
40201
+ pollTimer: void 0,
40202
+ activeSubscriptionId: null
40203
+ };
40204
+ const teardown = () => {
40205
+ if (lifecycle.stopped) return;
40206
+ lifecycle.stopped = true;
40207
+ if (lifecycle.retryTimer) {
40208
+ clearTimeout(lifecycle.retryTimer);
40209
+ lifecycle.retryTimer = void 0;
40210
+ }
40211
+ if (lifecycle.pollTimer) {
40212
+ clearTimeout(lifecycle.pollTimer);
40213
+ lifecycle.pollTimer = void 0;
40214
+ }
40215
+ const subId = lifecycle.activeSubscriptionId;
40216
+ if (subId) {
40217
+ lifecycle.activeSubscriptionId = null;
40218
+ options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }, nodePin(options.ownerNodeId)).catch((err) => {
40219
+ options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
40220
+ brokerId: options.brokerId,
40221
+ subscriptionId: subId,
40222
+ error: errMsg(err)
40223
+ } });
40224
+ });
40225
+ }
40226
+ };
40227
+ subscribeWithRetry(options, lifecycle);
40228
+ return teardown;
40157
40229
  }
40158
40230
  /**
40159
- * Merge a node's live-probed inference devices with its stored per-device map.
40160
- *
40161
- * The default is **AUTO = all discovered ACCELERATORS enabled** (spec C2,
40162
- * opt-OUT) with TWO deliberate exceptions, both **opt-IN** (default disabled):
40163
- *
40164
- * - **CPU**: `enumerateInferenceDevices` always emits a universal `cpu`
40165
- * floor on every platform; auto-enabling it would let the balancer
40166
- * round-robin ~1/N of sessions onto the slow CPU pool alongside the
40167
- * NPU/iGPU/ANE. CPU stays the always-available FALLBACK (a node with no
40168
- * eligible accelerator leaves `deviceKey` unset → the runner's default
40169
- * pool, which is CPU), not a balanced target — matching the spec's "no
40170
- * device eligible → fall back to CPU".
40171
- * - **Coral Edge TPU (`edgetpu`)**: the standing rule since the Coral
40172
- * executor landed is that it surfaces as selectable but is NEVER
40173
- * auto-picked — it runs a DIFFERENT, weaker model family (tflite SSD
40174
- * MobileNet, not the YOLO the other accelerators run), so silently
40175
- * enrolling a plugged-in Coral changes detection QUALITY, not just
40176
- * placement. The opt-OUT default did exactly that on 2026-08-01: a hub
40177
- * Coral nobody enabled entered the session rotation and camera 615 spent
40178
- * hours at 2.4fps failing tflite model resolution. An operator who wants
40179
- * the Coral balanced opts it in explicitly (`enabled: true`).
40180
- *
40181
- * So: an NPU/iGPU/ANE accelerator with NO stored entry is `enabled:true`; a
40182
- * CPU or edgetpu device with no stored entry is `enabled:false`; an explicit
40183
- * stored `enabled` always wins (an operator can opt CPU/Coral in, or an
40184
- * accelerator out). A stored-only key (configured but the probe did not
40185
- * return it — removed/unplugged HW) keeps its stored `enabled` and surfaces
40186
- * as `available:false`, so the UI still shows it.
40187
- *
40188
- * Pure + deterministic (sorted by key) — the single merge authority shared by
40189
- * the `getNodeInferenceDevices` view and the dispatcher's eligible-device pick.
40231
+ * Run the subscribe poll handshake with exponential backoff on subscribe
40232
+ * failures. Resolves once the subscription is acquired (and the poll loop has
40233
+ * been started) or once `lifecycle.stopped` flips, whichever comes first.
40190
40234
  */
40191
- function mergeInferenceDevices(probed, stored) {
40192
- const probedByKey = new Map(probed.map((d) => [d.key, d]));
40193
- const keys = new Set([...probedByKey.keys(), ...Object.keys(stored)]);
40194
- const out = [];
40195
- for (const key of Array.from(keys).toSorted()) {
40196
- const descriptor = probedByKey.get(key);
40197
- const opt = stored[key];
40198
- const parsed = descriptor ?? parseDeviceKey(key);
40199
- const weight = opt?.weight !== void 0 && opt.weight > 0 ? opt.weight : 1;
40200
- const autoDefault = parsed.backend !== "cpu" && parsed.backend !== "edgetpu";
40201
- out.push({
40202
- key,
40203
- backend: parsed.backend,
40204
- device: parsed.device,
40205
- format: parsed.format,
40206
- available: descriptor?.available ?? false,
40207
- enabled: opt?.enabled ?? autoDefault,
40208
- weight,
40209
- maxSessions: opt?.maxSessions ?? null,
40210
- defaultModelId: defaultModelIdForDevice(key),
40211
- ...opt?.steps && Object.keys(opt.steps).length > 0 ? { steps: { ...opt.steps } } : {}
40212
- });
40235
+ async function subscribeWithRetry(options, lifecycle) {
40236
+ const { api, brokerId, tag, ownerNodeId, logger } = options;
40237
+ const pin = nodePin(ownerNodeId);
40238
+ let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
40239
+ let attempt = 0;
40240
+ while (!lifecycle.stopped) {
40241
+ attempt += 1;
40242
+ try {
40243
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
40244
+ brokerId,
40245
+ tag
40246
+ }, pin);
40247
+ if (lifecycle.stopped) {
40248
+ await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }, pin).catch((err) => {
40249
+ logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
40250
+ brokerId,
40251
+ subscriptionId: result.subscriptionId,
40252
+ error: errMsg(err)
40253
+ } });
40254
+ });
40255
+ return;
40256
+ }
40257
+ if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
40258
+ brokerId,
40259
+ tag,
40260
+ attempt
40261
+ } });
40262
+ lifecycle.activeSubscriptionId = result.subscriptionId;
40263
+ startPolling(options, lifecycle);
40264
+ return;
40265
+ } catch (err) {
40266
+ if (lifecycle.stopped) return;
40267
+ if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
40268
+ brokerId,
40269
+ tag,
40270
+ error: errMsg(err),
40271
+ nextRetryInMs: backoffMs
40272
+ } });
40273
+ else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
40274
+ brokerId,
40275
+ tag,
40276
+ attempt,
40277
+ error: errMsg(err),
40278
+ nextRetryInMs: backoffMs
40279
+ } });
40280
+ await sleep(backoffMs, lifecycle);
40281
+ backoffMs = Math.min(attempt >= PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS ? LONG_SUBSCRIBE_RETRY_BACKOFF_MS : MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
40282
+ }
40213
40283
  }
40214
- return out;
40215
40284
  }
40216
40285
  /**
40217
- * The per-device concurrent-session caps for a node as `deviceKey → maxSessions`
40218
- * (only devices that carry an explicit cap; absent = unlimited). Fed to the
40219
- * device balancer's `nodeCaps` so a device at its cap is skipped (audit F3).
40286
+ * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
40287
+ * trigger a fresh `subscribeAudioChunks` via the recovery branch covers
40288
+ * the broker child restart case where our `subscriptionId` is silently
40289
+ * disowned.
40220
40290
  */
40221
- function inferenceDeviceCaps(stored) {
40222
- const out = {};
40223
- for (const [key, entry] of Object.entries(stored)) if (entry.maxSessions !== void 0 && entry.maxSessions > 0) out[key] = entry.maxSessions;
40224
- return out;
40225
- }
40226
- /** Is this device the CPU fallback rather than a real accelerator? */
40227
- function isCpuFallback(view) {
40228
- return view.backend === "cpu";
40229
- }
40230
- function resolveInferenceDeviceEligibility(probed, stored, canRunRoot, isPoolUsable) {
40231
- const eligible = {};
40232
- const excluded = [];
40233
- const merged = mergeInferenceDevices(probed, stored);
40234
- const acceleratorServes = merged.some((d) => !isCpuFallback(d) && d.enabled && d.available && (!canRunRoot || canRunRoot(d.format)));
40235
- for (const d of merged) {
40236
- if (!d.enabled) {
40237
- excluded.push({
40238
- key: d.key,
40239
- reason: "disabled",
40240
- format: d.format
40241
- });
40242
- continue;
40243
- }
40244
- if (!d.available) {
40245
- excluded.push({
40246
- key: d.key,
40247
- reason: "unavailable",
40248
- format: d.format
40249
- });
40250
- continue;
40251
- }
40252
- if (isPoolUsable && !isPoolUsable(d.key)) {
40253
- excluded.push({
40254
- key: d.key,
40255
- reason: "unavailable",
40256
- format: d.format
40257
- });
40258
- continue;
40259
- }
40260
- if (canRunRoot && !canRunRoot(d.format)) {
40261
- excluded.push({
40262
- key: d.key,
40263
- reason: "cannot-host-camera-root",
40264
- format: d.format
40265
- });
40266
- continue;
40291
+ function startPolling(options, lifecycle) {
40292
+ const { api, brokerId, tag, ownerNodeId, onChunk, logger } = options;
40293
+ const pin = nodePin(ownerNodeId);
40294
+ let consecutiveFailures = 0;
40295
+ const resubscribe = async () => {
40296
+ try {
40297
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
40298
+ brokerId,
40299
+ tag
40300
+ }, pin);
40301
+ lifecycle.activeSubscriptionId = result.subscriptionId;
40302
+ logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
40303
+ brokerId,
40304
+ tag,
40305
+ subscriptionId: result.subscriptionId,
40306
+ afterFailures: consecutiveFailures
40307
+ } });
40308
+ return true;
40309
+ } catch {
40310
+ return false;
40267
40311
  }
40268
- if (isCpuFallback(d) && acceleratorServes) {
40269
- excluded.push({
40270
- key: d.key,
40271
- reason: "accelerator-preferred",
40272
- format: d.format
40273
- });
40274
- continue;
40312
+ };
40313
+ const tick = async () => {
40314
+ if (lifecycle.stopped) return;
40315
+ const subId = lifecycle.activeSubscriptionId;
40316
+ if (!subId) return;
40317
+ try {
40318
+ const chunks = await api.streamBroker.pullAudioChunks.query({
40319
+ subscriptionId: subId,
40320
+ maxCount: PULL_MAX_COUNT
40321
+ }, pin);
40322
+ if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
40323
+ brokerId,
40324
+ subscriptionId: subId
40325
+ } });
40326
+ consecutiveFailures = 0;
40327
+ for (const chunk of chunks) {
40328
+ if (lifecycle.stopped) break;
40329
+ await onChunk(chunk);
40330
+ }
40331
+ } catch (err) {
40332
+ consecutiveFailures += 1;
40333
+ if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
40334
+ brokerId,
40335
+ subscriptionId: subId,
40336
+ error: errMsg(err)
40337
+ } });
40338
+ if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
40275
40339
  }
40276
- eligible[d.key] = d.weight;
40277
- }
40278
- return {
40279
- eligible,
40280
- excluded
40340
+ if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS$1);
40281
40341
  };
40342
+ tick();
40282
40343
  }
40283
40344
  /**
40284
- * Join the merged device rows with the eligibility verdict so the UI can NAME
40285
- * why an accelerator is not in play instead of leaving the operator to deduce
40286
- * it from `enabled`/`available`.
40287
- *
40288
- * Deduction is not possible for two of the four reasons — `accelerator-preferred`
40289
- * is a node-WIDE rule (a CPU row reads `enabled:true, available:true` and still
40290
- * never gets a session, D215) and `cannot-host-camera-root` needs the node's
40291
- * model catalog. Both live in {@link resolveInferenceDeviceEligibility}, so this
40292
- * function only transports its answer; it never re-derives one.
40293
- *
40294
- * Pure; preserves `merged`'s order (sorted by key) and every other field.
40345
+ * Cancellable sleep wakes early when `lifecycle.stopped` flips. We
40346
+ * keep a local wrapper around the shared {@link sleep} helper because
40347
+ * the lifecycle tracks the active retry timer for `teardown()` to
40348
+ * clear; pure `sleep()` would leak the timer if teardown fired while
40349
+ * we were waiting.
40295
40350
  */
40296
- function annotateInferenceDeviceExclusions(merged, eligibility) {
40297
- const reasonByKey = new Map(eligibility.excluded.map((e) => [e.key, e.reason]));
40298
- return merged.map((view) => ({
40299
- ...view,
40300
- exclusion: reasonByKey.get(view.key) ?? null
40301
- }));
40351
+ function sleep(ms, lifecycle) {
40352
+ return new Promise((resolve) => {
40353
+ if (lifecycle.stopped) {
40354
+ resolve();
40355
+ return;
40356
+ }
40357
+ lifecycle.retryTimer = setTimeout(() => {
40358
+ lifecycle.retryTimer = void 0;
40359
+ resolve();
40360
+ }, ms);
40361
+ });
40302
40362
  }
40303
- function resolveNodeInferenceUsability(eligibility) {
40304
- const eligibleKeys = Object.keys(eligibility.eligible).toSorted();
40305
- const unavailableKeys = eligibility.excluded.filter((e) => e.reason === "unavailable").map((e) => e.key).toSorted();
40363
+ //#endregion
40364
+ //#region src/audio-load-balancer.ts
40365
+ function balanceAudio(input) {
40366
+ if (input.nodes.length === 0) return null;
40367
+ if (input.preferredNode) {
40368
+ const pinned = input.nodes.find((n) => n.nodeId === input.preferredNode);
40369
+ if (pinned) return {
40370
+ nodeId: pinned.nodeId,
40371
+ reason: "manual"
40372
+ };
40373
+ }
40306
40374
  return {
40307
- usable: eligibleKeys.length > 0 || unavailableKeys.length === 0,
40308
- unavailableKeys,
40309
- eligibleKeys
40375
+ nodeId: input.nodes.slice().toSorted((a, b) => a.deviceCount - b.deviceCount)[0].nodeId,
40376
+ reason: "capacity"
40310
40377
  };
40311
40378
  }
40379
+ //#endregion
40380
+ //#region src/orchestrator-types.ts
40381
+ var PHASE_MODE_VALUES = new Set([
40382
+ "disabled",
40383
+ "always-on",
40384
+ "on-motion"
40385
+ ]);
40386
+ function isPipelinePhaseMode(v) {
40387
+ return PHASE_MODE_VALUES.has(v);
40388
+ }
40312
40389
  /**
40313
- * Step-tree device jump (phase 1): the attach-payload roster of a node's
40314
- * enabled∧available inference devices with the balancer knobs (`weight`,
40315
- * `maxSessions`) the runner uses to AUTO-jump an enrichment step off a device
40316
- * whose format can't run it. Built from the SAME `eligible` (deviceKey→weight)
40317
- * and `caps` (deviceKey→maxSessions) the dispatcher already computes, so the
40318
- * roster the runner sees exactly matches the balancer's candidate set. Sorted
40319
- * by key for determinism. Populated onto `RunnerCameraConfig.inferenceDevices`
40320
- * ONLY when a `deviceKey` is elected and there are ≥2 entries.
40321
- */
40322
- function buildInferenceDeviceRoster(eligible, caps) {
40323
- return Object.entries(eligible).map(([deviceKey, weight]) => ({
40324
- deviceKey,
40325
- weight: weight > 0 ? weight : 1,
40326
- maxSessions: caps[deviceKey] ?? null
40327
- })).toSorted((a, b) => a.deviceKey < b.deviceKey ? -1 : a.deviceKey > b.deviceKey ? 1 : 0);
40328
- }
40329
- //#endregion
40330
- //#region src/node-inference-usability-mirror.ts
40331
- var NodeInferenceUsabilityMirror = class {
40332
- state = /* @__PURE__ */ new Map();
40333
- /**
40334
- * Fold one observation in and report whether the caller should act.
40335
- * Never throws.
40336
- */
40337
- observe(nodeId, usable) {
40338
- const prev = this.state.get(nodeId);
40339
- if (usable) {
40340
- this.state.set(nodeId, {
40341
- usable: true,
40342
- armed: false
40343
- });
40344
- return prev !== void 0 && !prev.usable ? "recovered" : null;
40345
- }
40346
- if (prev === void 0) {
40347
- this.state.set(nodeId, {
40348
- usable: true,
40349
- armed: true
40350
- });
40351
- return null;
40352
- }
40353
- if (!prev.usable) {
40354
- this.state.set(nodeId, {
40355
- usable: false,
40356
- armed: true
40357
- });
40358
- return null;
40359
- }
40360
- if (!prev.armed) {
40361
- this.state.set(nodeId, {
40362
- usable: true,
40363
- armed: true
40364
- });
40365
- return null;
40366
- }
40367
- this.state.set(nodeId, {
40368
- usable: false,
40369
- armed: true
40370
- });
40371
- return "became-unusable";
40372
- }
40373
- /** Can this node be given cameras? Unknown nodes answer YES. */
40374
- isUsable(nodeId) {
40375
- return this.state.get(nodeId)?.usable ?? true;
40376
- }
40377
- /** Nodes currently excluded — for the placement log and diagnostics. */
40378
- unusableNodeIds() {
40379
- const out = [];
40380
- for (const [nodeId, s] of this.state) if (!s.usable) out.push(nodeId);
40381
- return out.toSorted();
40382
- }
40383
- forget(nodeId) {
40384
- this.state.delete(nodeId);
40385
- }
40386
- reset() {
40387
- this.state.clear();
40388
- }
40389
- };
40390
- //#endregion
40391
- //#region src/inference-device-usability-mirror.ts
40392
- /**
40393
- * InferenceDeviceUsabilityMirror — "can this NODE's THIS DEVICE still infer?",
40394
- * kept off the placement path.
40395
- *
40396
- * The per-`(nodeId, deviceKey)` twin of {@link NodeInferenceUsabilityMirror},
40397
- * and deliberately not a second mechanism: it composes the key and delegates
40398
- * every decision to that class, so the arm/apply reluctance D49 pinned lives in
40399
- * exactly one implementation and cannot drift between the node tier and the
40400
- * device tier.
40401
- *
40402
- * ## Why this tier had to exist
40403
- *
40404
- * The node tier already answers "does this node have ANY usable accelerator".
40405
- * On 2026-08-25 the hub's answer was, correctly, yes — its NPU was healthy the
40406
- * whole time. What died was one DEVICE, `openvino:gpu`, and nothing anywhere
40407
- * asked that question: the per-dispatch capability gate is keyed on model
40408
- * FORMAT and `openvino:gpu` and `openvino:npu` share the format `openvino`, so
40409
- * it is blind between them by construction. The balancer kept rotating cameras
40410
- * onto the dead pool — 20 sessions in 20 minutes, every one `tieBreak:
40411
- * "rotation"` — for 31 hours.
40412
- *
40413
- * ## Why a mirror and not the event
40414
- *
40415
- * D49, verbatim: *a one-shot event never gates on a fallible read*. The gate is
40416
- * this in-memory mirror, refreshed off the event path by the same
40417
- * `resolveEligibleInferenceDevices` the dispatcher already runs (plus the
40418
- * session controller's background refresher). The consequences that buys:
40419
- *
40420
- * - **A read that fails changes nothing.** The caller folds in an observation
40421
- * only when it HAS one; an unreachable node, a version-skewed executor or a
40422
- * rejected RPC never reaches {@link observe}, so the previous verdict
40423
- * stands. This is the whole reason the health read is specified as
40424
- * "synchronous over in-memory state, never throws for its own reasons": an
40425
- * empty answer must mean *nothing is refused*, not *I could not tell*.
40426
- * - **Excluding a healthy device needs a second, consecutive read.** Exclusion
40427
- * is the direction that DESTROYS work — it strands an accelerator that may
40428
- * be perfectly fine — so one bad observation only ARMS.
40429
- * - **Re-admitting is immediate and unconditional.** One good observation puts
40430
- * the device straight back. Being slow to exclude costs some wasted
40431
- * inference attempts; being slow to re-admit costs an idle accelerator and a
40432
- * node that looks broken.
40433
- */
40434
- /**
40435
- * NUL — impossible in both a Moleculer node id and a `<backend>:<device>` key,
40436
- * so the composite key can never be ambiguous. A separator that CAN occur in
40437
- * either half makes two distinct pairs collide, and a collision here silently
40438
- * excludes an accelerator nobody reported.
40439
- */
40440
- var SEPARATOR = "\0";
40441
- var InferenceDeviceUsabilityMirror = class {
40442
- /** The one implementation of the arm/apply state machine (D49). */
40443
- mirror = new NodeInferenceUsabilityMirror();
40444
- /**
40445
- * Fold one observation in and report whether the caller should act.
40446
- * Never throws.
40447
- */
40448
- observe(nodeId, deviceKey, usable) {
40449
- return this.mirror.observe(`${nodeId}${SEPARATOR}${deviceKey}`, usable);
40450
- }
40451
- /** Can the balancer put a session on this device? Unknown pairs answer YES. */
40452
- isUsable(nodeId, deviceKey) {
40453
- return this.mirror.isUsable(`${nodeId}${SEPARATOR}${deviceKey}`);
40454
- }
40455
- /** Pairs currently excluded — for the placement log and diagnostics. */
40456
- unusableDevices() {
40457
- return this.mirror.unusableNodeIds().map((composite) => {
40458
- const at = composite.indexOf(SEPARATOR);
40459
- return {
40460
- nodeId: composite.slice(0, at),
40461
- deviceKey: composite.slice(at + 1)
40462
- };
40463
- });
40464
- }
40465
- /** The excluded device keys on ONE node. */
40466
- unusableDeviceKeys(nodeId) {
40467
- return this.unusableDevices().filter((d) => d.nodeId === nodeId).map((d) => d.deviceKey);
40468
- }
40469
- forget(nodeId, deviceKey) {
40470
- this.mirror.forget(`${nodeId}${SEPARATOR}${deviceKey}`);
40471
- }
40472
- reset() {
40473
- this.mirror.reset();
40474
- }
40475
- };
40476
- /**
40477
- * Fold ONE node's health answer into the mirror and return what changed.
40478
- *
40479
- * This is the whole reading discipline, in one place, because both halves of it
40480
- * are easy to get subtly wrong and neither failure is visible in a log:
40481
- *
40482
- * - **`unhealthy === null` — the read FAILED — changes nothing.** Not a single
40483
- * entry is touched. An unreachable node, a version-skewed executor or a
40484
- * rejected RPC must be distinguishable from "asked, nothing is refused", or
40485
- * a flaky link silently re-admits a dead accelerator (D49).
40486
- * - **Every device the node HAS is observed**, not merely the refused ones.
40487
- * The first draft observed `refused ∪ already-excluded`, which omits exactly
40488
- * the devices the mirror has ARMED — so their disarming good read never
40489
- * arrived and two bad reads an HOUR apart, with a hundred healthy ones
40490
- * between them, excluded a working accelerator. "Consecutive" is only a
40491
- * property if the good observations are delivered.
40492
- *
40493
- * Pure with respect to everything except `mirror`, and never throws — it is
40494
- * called from the dispatcher's own read path.
40495
- */
40496
- function foldInferenceDeviceHealth(mirror, nodeId, unhealthy, present) {
40497
- if (unhealthy === null) return [];
40498
- const refused = new Set(unhealthy);
40499
- const observed = new Set([
40500
- ...present,
40501
- ...refused,
40502
- ...mirror.unusableDeviceKeys(nodeId)
40503
- ]);
40504
- const changes = [];
40505
- for (const deviceKey of observed) {
40506
- const transition = mirror.observe(nodeId, deviceKey, !refused.has(deviceKey));
40507
- if (transition !== null) changes.push({
40508
- deviceKey,
40509
- transition
40510
- });
40511
- }
40512
- return changes;
40513
- }
40514
- //#endregion
40515
- //#region src/orchestrator-types.ts
40516
- var PHASE_MODE_VALUES = new Set([
40517
- "disabled",
40518
- "always-on",
40519
- "on-motion"
40520
- ]);
40521
- function isPipelinePhaseMode(v) {
40522
- return PHASE_MODE_VALUES.has(v);
40523
- }
40524
- /**
40525
- * Periodic sweep interval for `retryPendingDispatches` — re-dispatches cameras
40526
- * that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
40527
- * `reconcileDispatch` is additive-only and never revisits these, so a slow
40528
- * safety-net timer + event-driven debounce triggers recover them.
40390
+ * Periodic sweep interval for `retryPendingDispatches` re-dispatches cameras
40391
+ * that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
40392
+ * `reconcileDispatch` is additive-only and never revisits these, so a slow
40393
+ * safety-net timer + event-driven debounce triggers recover them.
40529
40394
  */
40530
40395
  var PENDING_RETRY_INTERVAL_MS = 6e4;
40531
40396
  /** Debounce window for `schedulePendingRetry` — coalesces capacity/eligibility/readiness signals. */
@@ -40644,264 +40509,6 @@ var pipelineOrchestratorActions = defineCustomActions({
40644
40509
  */
40645
40510
  var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
40646
40511
  //#endregion
40647
- //#region src/audio-chunk-poller.ts
40648
- /**
40649
- * `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
40650
- * plane (Phase 5 / D9).
40651
- *
40652
- * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
40653
- * path. A live callback cannot cross a process boundary; once the `pipeline`
40654
- * group is dissolved (Task 8) the orchestrator runs in a different process
40655
- * from the broker, so audio delivery must go over tRPC.
40656
- *
40657
- * The consumer:
40658
- *
40659
- * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC — the broker
40660
- * registers a per-subscription bounded FIFO queue and returns a
40661
- * `subscriptionId`;
40662
- * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
40663
- * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
40664
- * 3. feeds each chunk to its downstream audio logic;
40665
- * 4. on teardown, `unsubscribeAudioChunks`.
40666
- *
40667
- * Audio is not latency-critical like video, and chunks arrive only ~every
40668
- * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
40669
- * a small per-poll burst keeps latency low without busy-spinning. The
40670
- * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
40671
- * loses a chunk.
40672
- *
40673
- * Boot-race tolerance: the broker for a given camStream may not be registered
40674
- * yet when the orchestrator wires the subscription (provider addons publish
40675
- * their cameraStreams asynchronously after their probe completes).
40676
- * `subscribeAudioChunks` retries with exponential backoff (capped at
40677
- * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
40678
- * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
40679
- * shape so video and audio plumbing self-heal identically.
40680
- */
40681
- /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
40682
- var POLL_INTERVAL_MS$1 = 200;
40683
- /** How many chunks to drain per poll — a small burst absorbs jitter. */
40684
- var PULL_MAX_COUNT = 8;
40685
- /**
40686
- * Consecutive pull failures before we attempt to re-subscribe. A single failed
40687
- * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
40688
- * sustained failure means the broker child restarted and dropped our
40689
- * subscription, so we re-establish it.
40690
- */
40691
- var RESUBSCRIBE_AFTER_FAILURES = 2;
40692
- /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
40693
- var RESUBSCRIBE_THROTTLE_TICKS = 5;
40694
- /** First subscribe-retry delay, doubled on every subsequent failure. */
40695
- var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
40696
- /**
40697
- * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller — fast
40698
- * enough to recover within a single reconcile of the orchestrator and slow
40699
- * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
40700
- */
40701
- var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
40702
- /**
40703
- * Attempts after which a still-failing subscribe escalates from the fast 5 s
40704
- * ceiling to {@link LONG_SUBSCRIBE_RETRY_BACKOFF_MS}. ~15 attempts ≈ one
40705
- * minute of fast retries — plenty for the boot races the 5 s ceiling exists
40706
- * for. A broker that is STILL absent after that is a long-lived condition
40707
- * (e.g. a DISABLED camera, whose broker the stream-broker reconcile refuses
40708
- * to recreate — 2026-07-23) and retrying every 5 s forever is pure log/RPC
40709
- * churn. The slow loop stays alive so audio still recovers automatically
40710
- * (≤60 s) once the camera is re-enabled.
40711
- */
40712
- var PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS = 15;
40713
- var LONG_SUBSCRIBE_RETRY_BACKOFF_MS = 6e4;
40714
- /**
40715
- * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
40716
- *
40717
- * Always resolves to a teardown closure — when the broker is not yet
40718
- * registered the closure cancels the ongoing retry loop; when polling is
40719
- * active it stops the loop and releases the broker subscription. Mirrors
40720
- * `startFrameHandlePoller` so video and audio recover identically.
40721
- */
40722
- function startAudioChunkPoller(options) {
40723
- const lifecycle = {
40724
- stopped: false,
40725
- retryTimer: void 0,
40726
- pollTimer: void 0,
40727
- activeSubscriptionId: null
40728
- };
40729
- const teardown = () => {
40730
- if (lifecycle.stopped) return;
40731
- lifecycle.stopped = true;
40732
- if (lifecycle.retryTimer) {
40733
- clearTimeout(lifecycle.retryTimer);
40734
- lifecycle.retryTimer = void 0;
40735
- }
40736
- if (lifecycle.pollTimer) {
40737
- clearTimeout(lifecycle.pollTimer);
40738
- lifecycle.pollTimer = void 0;
40739
- }
40740
- const subId = lifecycle.activeSubscriptionId;
40741
- if (subId) {
40742
- lifecycle.activeSubscriptionId = null;
40743
- options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }, nodePin(options.ownerNodeId)).catch((err) => {
40744
- options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
40745
- brokerId: options.brokerId,
40746
- subscriptionId: subId,
40747
- error: errMsg(err)
40748
- } });
40749
- });
40750
- }
40751
- };
40752
- subscribeWithRetry(options, lifecycle);
40753
- return teardown;
40754
- }
40755
- /**
40756
- * Run the subscribe → poll handshake with exponential backoff on subscribe
40757
- * failures. Resolves once the subscription is acquired (and the poll loop has
40758
- * been started) or once `lifecycle.stopped` flips, whichever comes first.
40759
- */
40760
- async function subscribeWithRetry(options, lifecycle) {
40761
- const { api, brokerId, tag, ownerNodeId, logger } = options;
40762
- const pin = nodePin(ownerNodeId);
40763
- let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
40764
- let attempt = 0;
40765
- while (!lifecycle.stopped) {
40766
- attempt += 1;
40767
- try {
40768
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
40769
- brokerId,
40770
- tag
40771
- }, pin);
40772
- if (lifecycle.stopped) {
40773
- await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }, pin).catch((err) => {
40774
- logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
40775
- brokerId,
40776
- subscriptionId: result.subscriptionId,
40777
- error: errMsg(err)
40778
- } });
40779
- });
40780
- return;
40781
- }
40782
- if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
40783
- brokerId,
40784
- tag,
40785
- attempt
40786
- } });
40787
- lifecycle.activeSubscriptionId = result.subscriptionId;
40788
- startPolling(options, lifecycle);
40789
- return;
40790
- } catch (err) {
40791
- if (lifecycle.stopped) return;
40792
- if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
40793
- brokerId,
40794
- tag,
40795
- error: errMsg(err),
40796
- nextRetryInMs: backoffMs
40797
- } });
40798
- else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
40799
- brokerId,
40800
- tag,
40801
- attempt,
40802
- error: errMsg(err),
40803
- nextRetryInMs: backoffMs
40804
- } });
40805
- await sleep(backoffMs, lifecycle);
40806
- backoffMs = Math.min(attempt >= PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS ? LONG_SUBSCRIBE_RETRY_BACKOFF_MS : MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
40807
- }
40808
- }
40809
- }
40810
- /**
40811
- * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
40812
- * trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
40813
- * the broker child restart case where our `subscriptionId` is silently
40814
- * disowned.
40815
- */
40816
- function startPolling(options, lifecycle) {
40817
- const { api, brokerId, tag, ownerNodeId, onChunk, logger } = options;
40818
- const pin = nodePin(ownerNodeId);
40819
- let consecutiveFailures = 0;
40820
- const resubscribe = async () => {
40821
- try {
40822
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
40823
- brokerId,
40824
- tag
40825
- }, pin);
40826
- lifecycle.activeSubscriptionId = result.subscriptionId;
40827
- logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
40828
- brokerId,
40829
- tag,
40830
- subscriptionId: result.subscriptionId,
40831
- afterFailures: consecutiveFailures
40832
- } });
40833
- return true;
40834
- } catch {
40835
- return false;
40836
- }
40837
- };
40838
- const tick = async () => {
40839
- if (lifecycle.stopped) return;
40840
- const subId = lifecycle.activeSubscriptionId;
40841
- if (!subId) return;
40842
- try {
40843
- const chunks = await api.streamBroker.pullAudioChunks.query({
40844
- subscriptionId: subId,
40845
- maxCount: PULL_MAX_COUNT
40846
- }, pin);
40847
- if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
40848
- brokerId,
40849
- subscriptionId: subId
40850
- } });
40851
- consecutiveFailures = 0;
40852
- for (const chunk of chunks) {
40853
- if (lifecycle.stopped) break;
40854
- await onChunk(chunk);
40855
- }
40856
- } catch (err) {
40857
- consecutiveFailures += 1;
40858
- if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
40859
- brokerId,
40860
- subscriptionId: subId,
40861
- error: errMsg(err)
40862
- } });
40863
- if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
40864
- }
40865
- if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS$1);
40866
- };
40867
- tick();
40868
- }
40869
- /**
40870
- * Cancellable sleep — wakes early when `lifecycle.stopped` flips. We
40871
- * keep a local wrapper around the shared {@link sleep} helper because
40872
- * the lifecycle tracks the active retry timer for `teardown()` to
40873
- * clear; pure `sleep()` would leak the timer if teardown fired while
40874
- * we were waiting.
40875
- */
40876
- function sleep(ms, lifecycle) {
40877
- return new Promise((resolve) => {
40878
- if (lifecycle.stopped) {
40879
- resolve();
40880
- return;
40881
- }
40882
- lifecycle.retryTimer = setTimeout(() => {
40883
- lifecycle.retryTimer = void 0;
40884
- resolve();
40885
- }, ms);
40886
- });
40887
- }
40888
- //#endregion
40889
- //#region src/audio-load-balancer.ts
40890
- function balanceAudio(input) {
40891
- if (input.nodes.length === 0) return null;
40892
- if (input.preferredNode) {
40893
- const pinned = input.nodes.find((n) => n.nodeId === input.preferredNode);
40894
- if (pinned) return {
40895
- nodeId: pinned.nodeId,
40896
- reason: "manual"
40897
- };
40898
- }
40899
- return {
40900
- nodeId: input.nodes.slice().toSorted((a, b) => a.deviceCount - b.deviceCount)[0].nodeId,
40901
- reason: "capacity"
40902
- };
40903
- }
40904
- //#endregion
40905
40512
  //#region src/audio-window-accumulator.ts
40906
40513
  var AudioWindowAccumulator = class {
40907
40514
  deviceId;
@@ -41405,234 +41012,847 @@ var AudioSubscriptionController = class {
41405
41012
  meta: { error: errMsg(err) }
41406
41013
  });
41407
41014
  });
41408
- }, windowMs);
41409
- this.motionAudioWindowTimers.set(deviceId, timer);
41410
- }
41411
- /** True while a motion-driven audio window is open for this device. */
41412
- isMotionAudioWindowOpen(deviceId) {
41413
- return this.motionAudioWindowTimers.has(deviceId);
41414
- }
41415
- /**
41416
- * Close an on-motion audio window and drop its subscription. Shared by both
41417
- * closers (quiet window elapsed / falling edge cooldown) so they can never
41418
- * disagree about what "closed" means. `reason` is logged so a camera that
41419
- * loses audio can always be told WHY from the per-device log view.
41420
- */
41421
- async closeMotionAudioWindow(deviceId, reason) {
41422
- const pendingWindow = this.motionAudioWindowTimers.get(deviceId);
41423
- if (pendingWindow) {
41424
- clearTimeout(pendingWindow);
41425
- this.motionAudioWindowTimers.delete(deviceId);
41015
+ }, windowMs);
41016
+ this.motionAudioWindowTimers.set(deviceId, timer);
41017
+ }
41018
+ /** True while a motion-driven audio window is open for this device. */
41019
+ isMotionAudioWindowOpen(deviceId) {
41020
+ return this.motionAudioWindowTimers.has(deviceId);
41021
+ }
41022
+ /**
41023
+ * Close an on-motion audio window and drop its subscription. Shared by both
41024
+ * closers (quiet window elapsed / falling edge cooldown) so they can never
41025
+ * disagree about what "closed" means. `reason` is logged so a camera that
41026
+ * loses audio can always be told WHY from the per-device log view.
41027
+ */
41028
+ async closeMotionAudioWindow(deviceId, reason) {
41029
+ const pendingWindow = this.motionAudioWindowTimers.get(deviceId);
41030
+ if (pendingWindow) {
41031
+ clearTimeout(pendingWindow);
41032
+ this.motionAudioWindowTimers.delete(deviceId);
41033
+ }
41034
+ await this.withAudioSubLock(deviceId, async () => {
41035
+ const unsub = this.audioSubscriptions.get(deviceId);
41036
+ if (!unsub) return;
41037
+ try {
41038
+ unsub();
41039
+ } catch {}
41040
+ this.audioSubscriptions.delete(deviceId);
41041
+ this.deps.logger.info("lazy audio: window closed", {
41042
+ tags: { deviceId },
41043
+ meta: { reason }
41044
+ });
41045
+ });
41046
+ }
41047
+ /**
41048
+ * Audio teardown for one device — the audio half of `stopDetection`.
41049
+ * Tears down through the per-device lock so it can't race a concurrent
41050
+ * subscribe (which would re-store a handle this teardown never sees).
41051
+ */
41052
+ async stopForDevice(deviceId) {
41053
+ await this.withAudioSubLock(deviceId, async () => {
41054
+ const unsub = this.audioSubscriptions.get(deviceId);
41055
+ if (unsub) {
41056
+ try {
41057
+ unsub();
41058
+ } catch {}
41059
+ this.audioSubscriptions.delete(deviceId);
41060
+ }
41061
+ });
41062
+ const lazyTimer = this.lazyAudioTeardownTimers.get(deviceId);
41063
+ if (lazyTimer) {
41064
+ clearTimeout(lazyTimer);
41065
+ this.lazyAudioTeardownTimers.delete(deviceId);
41066
+ }
41067
+ const windowTimer = this.motionAudioWindowTimers.get(deviceId);
41068
+ if (windowTimer) {
41069
+ clearTimeout(windowTimer);
41070
+ this.motionAudioWindowTimers.delete(deviceId);
41071
+ }
41072
+ this.audioAssignments.delete(deviceId);
41073
+ }
41074
+ /**
41075
+ * Centralized write into `audioSubscriptions`. If shutdown has begun, the
41076
+ * map has already been (or is about to be) cleared lock-free in
41077
+ * `shutdown()`; storing here would leak a zombie entry whose `unsub` is
41078
+ * never called. So when shutting down we immediately invoke `unsub`
41079
+ * (best-effort, error-swallowed) and DO NOT store. `protected` so
41080
+ * `audio-sub-lock.spec.ts`'s test subclass can assert the shutdown-guard
41081
+ * behavior without casts.
41082
+ */
41083
+ storeAudioSub(deviceId, unsub) {
41084
+ if (this.audioShuttingDown) {
41085
+ try {
41086
+ unsub();
41087
+ } catch {}
41088
+ return;
41089
+ }
41090
+ this.audioSubscriptions.set(deviceId, unsub);
41091
+ }
41092
+ /**
41093
+ * Serialize an audio-subscription critical section per device. `fn` is
41094
+ * chained onto the device's current lock tail, so concurrent calls for the
41095
+ * SAME deviceId run sequentially (FIFO); different deviceIds never block
41096
+ * each other. Thin delegate onto the `audioSubLocks` `KeyedAsyncLock`
41097
+ * instance. `protected` so `audio-sub-lock.spec.ts`'s test subclass can
41098
+ * drive the lock without casts.
41099
+ */
41100
+ withAudioSubLock(deviceId, fn) {
41101
+ return this.audioSubLocks.run(deviceId, fn);
41102
+ }
41103
+ /**
41104
+ * Subscribe to decoded audio chunks for a camera and feed them into the
41105
+ * audio-analyzer. Reads the analyzer's settings via its own
41106
+ * `resolveDeviceSettings(deviceId)` method so the orchestrator does not
41107
+ * touch the audio-analyzer schema field names directly.
41108
+ */
41109
+ async subscribeAudioStream(deviceId, config) {
41110
+ const api = this.deps.api();
41111
+ if (!api) {
41112
+ this.deps.logger.warn("this.ctx.api not available — cannot subscribe audio", { tags: { deviceId } });
41113
+ return null;
41114
+ }
41115
+ if (!await this.deps.isAudioAnalysisActive(deviceId)) return null;
41116
+ if (config.audioMode === "disabled") {
41117
+ this.deps.logger.debug("audio subscribe skipped: audioMode=disabled", { tags: { deviceId } });
41118
+ return null;
41119
+ }
41120
+ if (config.audioMode === "on-motion" && !this.isMotionAudioWindowOpen(deviceId)) {
41121
+ this.deps.logger.info("audio subscribe deferred: audioMode=on-motion, no window open", { tags: { deviceId } });
41122
+ return null;
41123
+ }
41124
+ const audioStream = config.audioStreamId ?? config.motionStreamId;
41125
+ const audioBrokerId = makeSourceBrokerId(deviceId, audioStream);
41126
+ if ((await this.deps.probeAudioTrack(deviceId, audioStream)).kind === "absent") {
41127
+ this.deps.logger.warn("audio subscription REFUSED — this stream carries no audio track", {
41128
+ tags: { deviceId },
41129
+ meta: {
41130
+ camStreamId: audioStream,
41131
+ brokerId: audioBrokerId,
41132
+ selectedBy: config.audioStreamId !== void 0 ? "audioStreamId" : "motionStreamId",
41133
+ hint: "point the camera’s audio at a stream that has an audio track — no stream is substituted automatically"
41134
+ }
41135
+ });
41136
+ return null;
41137
+ }
41138
+ const settings = await api.audioAnalysis.resolveDeviceSettings.query({ deviceId });
41139
+ if (!settings) {
41140
+ this.deps.logger.warn("audio-analysis returned no settings — audio subscription skipped", { tags: { deviceId } });
41141
+ return null;
41142
+ }
41143
+ const audioNodeId = await this.dispatch(deviceId);
41144
+ const isRemoteAudio = audioNodeId !== this.deps.localNodeId();
41145
+ this.deps.logger.info("audio subscription: resolved audio node", {
41146
+ tags: { deviceId },
41147
+ meta: {
41148
+ audioNodeId,
41149
+ isRemote: isRemoteAudio
41150
+ }
41151
+ });
41152
+ const accumulator = new AudioWindowAccumulator(deviceId);
41153
+ const teardown = startAudioChunkPoller({
41154
+ api,
41155
+ brokerId: audioBrokerId,
41156
+ tag: "audio-analyzer",
41157
+ ownerNodeId: this.deps.ingestNode(),
41158
+ logger: this.deps.logger.withTags({ deviceId }),
41159
+ onChunk: async (chunk) => {
41160
+ this.deps.watchdogNote(deviceId, "audio");
41161
+ try {
41162
+ const audioChunkInput = accumulator.push(chunk);
41163
+ if (!audioChunkInput) return;
41164
+ const result = await api.audioAnalyzer.analyseChunk.mutate({
41165
+ chunk: audioChunkInput,
41166
+ settings,
41167
+ ...isRemoteAudio ? { nodeId: audioNodeId } : {}
41168
+ });
41169
+ if (!result) return;
41170
+ const frame = buildAudioResultFrame(deviceId, result);
41171
+ this.deps.eventBus.emit({
41172
+ id: `audio-inference-${deviceId}-${Date.now()}`,
41173
+ timestamp: /* @__PURE__ */ new Date(),
41174
+ source: {
41175
+ type: "device",
41176
+ id: deviceId,
41177
+ nodeId: "hub",
41178
+ addonId: "pipeline-orchestrator",
41179
+ deviceId
41180
+ },
41181
+ category: EventCategory.PipelineAudioInferenceResult,
41182
+ data: {
41183
+ deviceId,
41184
+ frame,
41185
+ nodeId: "hub"
41186
+ }
41187
+ });
41188
+ } catch (err) {
41189
+ const msg = errMsg(err);
41190
+ this.deps.logger.error("Audio analysis failed", {
41191
+ tags: { deviceId },
41192
+ meta: { error: msg }
41193
+ });
41194
+ }
41195
+ }
41196
+ });
41197
+ this.deps.logger.info("Audio stream subscribed", { tags: { deviceId } });
41198
+ return () => {
41199
+ teardown();
41200
+ accumulator.reset();
41201
+ };
41202
+ }
41203
+ /**
41204
+ * Set true at the very start of `onShutdown`, before the audio teardown /
41205
+ * map clears below. Once set, `withAudioSubLock` turns queued/new critical
41206
+ * sections into no-ops and `storeAudioSub` refuses to store, so no
41207
+ * critical section that was in-flight (or queued) when shutdown began can
41208
+ * resurrect a zombie subscription into the cleared `audioSubscriptions`
41209
+ * map. MUST be called before anything else in `onShutdown` that could
41210
+ * race a queued audio critical section (mirrors the original
41211
+ * `this.audioShuttingDown = true` being the very first statement).
41212
+ */
41213
+ beginShutdown() {
41214
+ this.audioShuttingDown = true;
41215
+ }
41216
+ /**
41217
+ * Full audio teardown — combines the former `onShutdown`'s two separate
41218
+ * audio blocks (lazy-teardown-timer clear, then — after several unrelated
41219
+ * session/reconcile/load-shed clears — subscription teardown + lock clear
41220
+ * + assignment-map clears) into one call. Safe to combine: both blocks
41221
+ * are synchronous with no interleaved `await`, and every original
41222
+ * statement between them (`sessionRegistry.clear()`,
41223
+ * `cameraFpsMap.clear()`, `remoteHealthAttempts.clear()`,
41224
+ * `loadShedState.clear()`, `loadShedResumeTimer` cleanup) touches state
41225
+ * fully disjoint from anything audio — so their relative order to each
41226
+ * other is unaffected, and the audio-internal order (timers →
41227
+ * subscriptions → lock → assignment maps) is reproduced exactly.
41228
+ */
41229
+ shutdown() {
41230
+ for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
41231
+ this.lazyAudioTeardownTimers.clear();
41232
+ for (const t of this.motionAudioWindowTimers.values()) clearTimeout(t);
41233
+ this.motionAudioWindowTimers.clear();
41234
+ for (const unsub of this.audioSubscriptions.values()) try {
41235
+ unsub();
41236
+ } catch {}
41237
+ this.audioSubscriptions.clear();
41238
+ this.audioSubLocks.clear();
41239
+ this.audioAssignments.clear();
41240
+ this.readyAudioNodes.clear();
41241
+ }
41242
+ };
41243
+ //#endregion
41244
+ //#region src/disk-reconcile-fleet.ts
41245
+ async function reconcileFleetFromDisk(deps) {
41246
+ const deviceIds = await deps.listDeviceIds();
41247
+ const failed = [];
41248
+ let cameras = 0;
41249
+ let mediaDropped = 0;
41250
+ let tracks = 0;
41251
+ let events = 0;
41252
+ let completed = 0;
41253
+ for (const deviceId of deviceIds) {
41254
+ try {
41255
+ await deps.rescanRecordings(deviceId);
41256
+ const counts = await deps.reconcileAnalytics(deviceId);
41257
+ cameras += 1;
41258
+ mediaDropped += counts.mediaDropped;
41259
+ tracks += counts.tracks;
41260
+ events += counts.events;
41261
+ } catch {
41262
+ failed.push(deviceId);
41263
+ }
41264
+ completed += 1;
41265
+ deps.onProgress?.({
41266
+ deviceId,
41267
+ total: deviceIds.length,
41268
+ completed,
41269
+ failed: [...failed],
41270
+ mediaDropped,
41271
+ tracks,
41272
+ events
41273
+ });
41274
+ }
41275
+ return {
41276
+ cameras,
41277
+ failed,
41278
+ mediaDropped,
41279
+ tracks,
41280
+ events
41281
+ };
41282
+ }
41283
+ //#endregion
41284
+ //#region src/disk-reconcile-job.ts
41285
+ /**
41286
+ * In-memory disk-wins fleet job. The tRPC mutation starts this and returns
41287
+ * immediately; the walk runs in the addon process so a 60s UDS timeout cannot
41288
+ * abort it. Status is polled via getReconcileFromDiskStatus.
41289
+ */
41290
+ function idleDiskReconcileJob() {
41291
+ return {
41292
+ state: "idle",
41293
+ total: 0,
41294
+ completed: 0,
41295
+ currentDeviceId: null,
41296
+ failed: [],
41297
+ mediaDropped: 0,
41298
+ tracks: 0,
41299
+ events: 0,
41300
+ startedAtMs: null,
41301
+ finishedAtMs: null,
41302
+ error: null
41303
+ };
41304
+ }
41305
+ function isTimeoutError(err) {
41306
+ const message = err instanceof Error ? err.message : String(err);
41307
+ return /timed out/i.test(message);
41308
+ }
41309
+ async function withTimeoutRetry(run) {
41310
+ try {
41311
+ return await run();
41312
+ } catch (err) {
41313
+ if (!isTimeoutError(err)) throw err;
41314
+ return await run();
41315
+ }
41316
+ }
41317
+ function createDiskReconcileJobRunner(now = Date.now) {
41318
+ let job = idleDiskReconcileJob();
41319
+ let inFlight = null;
41320
+ const snapshot = () => job;
41321
+ const start = (deps) => {
41322
+ if (job.state === "running" && inFlight) return job;
41323
+ job = {
41324
+ ...idleDiskReconcileJob(),
41325
+ state: "running",
41326
+ startedAtMs: now()
41327
+ };
41328
+ deps.log?.("pipeline disk reconcile started");
41329
+ inFlight = (async () => {
41330
+ try {
41331
+ const result = await reconcileFleetFromDisk({
41332
+ listDeviceIds: deps.listDeviceIds,
41333
+ rescanRecordings: (deviceId) => withTimeoutRetry(() => deps.rescanRecordings(deviceId)),
41334
+ reconcileAnalytics: (deviceId) => withTimeoutRetry(() => deps.reconcileAnalytics(deviceId)),
41335
+ onProgress: (update) => {
41336
+ job = {
41337
+ ...job,
41338
+ total: update.total,
41339
+ completed: update.completed,
41340
+ currentDeviceId: update.deviceId,
41341
+ failed: update.failed,
41342
+ mediaDropped: update.mediaDropped,
41343
+ tracks: update.tracks,
41344
+ events: update.events
41345
+ };
41346
+ deps.onProgress?.(update);
41347
+ deps.log?.("pipeline disk reconcile camera", {
41348
+ deviceId: update.deviceId,
41349
+ completed: update.completed,
41350
+ total: update.total,
41351
+ failed: update.failed.length,
41352
+ mediaDropped: update.mediaDropped,
41353
+ tracks: update.tracks,
41354
+ events: update.events
41355
+ });
41356
+ }
41357
+ });
41358
+ job = {
41359
+ ...job,
41360
+ state: "done",
41361
+ total: result.cameras + result.failed.length,
41362
+ completed: result.cameras + result.failed.length,
41363
+ currentDeviceId: null,
41364
+ failed: result.failed,
41365
+ mediaDropped: result.mediaDropped,
41366
+ tracks: result.tracks,
41367
+ events: result.events,
41368
+ finishedAtMs: now(),
41369
+ error: null
41370
+ };
41371
+ deps.log?.("pipeline disk reconcile", {
41372
+ cameras: result.cameras,
41373
+ failed: result.failed,
41374
+ mediaDropped: result.mediaDropped,
41375
+ tracks: result.tracks,
41376
+ events: result.events
41377
+ });
41378
+ } catch (err) {
41379
+ const error = err instanceof Error ? err.message : String(err);
41380
+ job = {
41381
+ ...job,
41382
+ state: "error",
41383
+ currentDeviceId: null,
41384
+ finishedAtMs: now(),
41385
+ error
41386
+ };
41387
+ deps.log?.("pipeline disk reconcile failed", { error });
41388
+ } finally {
41389
+ inFlight = null;
41390
+ }
41391
+ })();
41392
+ return job;
41393
+ };
41394
+ return {
41395
+ snapshot,
41396
+ start
41397
+ };
41398
+ }
41399
+ //#endregion
41400
+ //#region src/inference-device-model.ts
41401
+ /**
41402
+ * Per-device default object-detection model + deviceKey parsing for the
41403
+ * orchestrator's device-aware `getNodeInferenceDevices` view.
41404
+ *
41405
+ * This DUPLICATES the executor's per-device model resolution (P0-3:
41406
+ * `resolveDeviceEngine` + `MODEL_BY_CLASS` + the object-detection step's
41407
+ * `defaultModelIdByFormat` in `@camstack/addon-pipeline`). It is duplicated —
41408
+ * not imported — because cross-addon imports are forbidden (the orchestrator
41409
+ * and the detection-pipeline are separate addons; only tRPC crosses the
41410
+ * boundary). Keep this in sync with `default-detection-model.ts` /
41411
+ * `step-definitions.ts` if the executor's defaults change.
41412
+ *
41413
+ * The returned ids are honest catalog ids (verified present):
41414
+ * - `yolov9m-320-int8` — Intel NPU + iGPU (yolo26 does NOT compile on the NPU)
41415
+ * - `yolov9m-320` — Apple ANE (CoreML)
41416
+ * - `ssd-mobilenet-v2-coco-edgetpu` — Coral USB Edge TPU (tflite)
41417
+ * - `yolo26n` — CPU / CUDA (the object-detection step's universal
41418
+ * nano default)
41419
+ *
41420
+ * Do not promote the accelerated ids to 640. The evaluation in
41421
+ * `docs/benchmarks/pipeline-frame-model-eval.md` failed the 640 promotion
41422
+ * gates (0/3 miss recovered at the current threshold).
41423
+ */
41424
+ /**
41425
+ * The always-on object-detection ROOT step id. A camera session's tracks all
41426
+ * originate from this detector, so a device whose engine format can't run it
41427
+ * cannot host a camera root. Mirrors the addon-pipeline step id (cross-addon
41428
+ * import is forbidden — this is the same duplication rationale as the model
41429
+ * defaults above).
41430
+ */
41431
+ var OBJECT_DETECTION_STEP_ID = "object-detection";
41432
+ /**
41433
+ * Build the camera-root capability predicate for a node from its live catalog:
41434
+ * `format → canHostCameraRoot`. A format can host a camera root iff the
41435
+ * catalog lists at least one object-detection model with a build for that
41436
+ * format — byte-for-byte the resolver's per-device skip-gate test for the root
41437
+ * step (`addonHasCompatibleModel`), so a device is deemed eligible iff the root
41438
+ * would ACTUALLY provision on it.
41439
+ *
41440
+ * Fails OPEN when the catalog has no object-detection slot at all (never
41441
+ * observed in production) so a malformed/empty catalog never strands every
41442
+ * device off the balancer. Pure + deterministic.
41443
+ */
41444
+ function makeRootCapabilityGuard(catalog) {
41445
+ for (const slot of catalog.slots) {
41446
+ const objDet = slot.addons.find((a) => a.id === OBJECT_DETECTION_STEP_ID);
41447
+ if (objDet) return (format) => objDet.models.some((m) => Boolean(m.formats[format]));
41448
+ }
41449
+ return () => true;
41450
+ }
41451
+ /** Split a deviceKey (`<backend>:<device>`, or bare `cpu`) into its parts + format.
41452
+ * Format comes from the shared {@link deviceBackendToFormat} SSOT (`@camstack/types`)
41453
+ * — the previously-local `BACKEND_FORMAT` copy is gone (R3/node-F2). Used only for
41454
+ * STORED-ONLY keys (a configured device the live probe didn't return); a probed
41455
+ * device carries its own honest `format` from the descriptor. */
41456
+ function parseDeviceKey(deviceKey) {
41457
+ const colon = deviceKey.indexOf(":");
41458
+ const backend = colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
41459
+ return {
41460
+ backend,
41461
+ device: colon >= 0 ? deviceKey.slice(colon + 1) : deviceKey,
41462
+ format: deviceBackendToFormat(backend)
41463
+ };
41464
+ }
41465
+ /**
41466
+ * The object-detection model the executor defaults to for a deviceKey. Mirrors
41467
+ * the executor's `MODEL_BY_CLASS` classification (`classifyAccelerator`) plus
41468
+ * the tflite `defaultModelIdByFormat` for Coral. Never throws; unknown backends
41469
+ * fall back to the universal nano default (`yolo26n`).
41470
+ */
41471
+ function defaultModelIdForDevice(deviceKey) {
41472
+ const { backend, device } = parseDeviceKey(deviceKey);
41473
+ if (backend === "openvino") {
41474
+ if (device === "cpu") return "yolo26n";
41475
+ return "yolov9m-320-int8";
41476
+ }
41477
+ if (backend === "edgetpu") return "ssd-mobilenet-v2-coco-edgetpu";
41478
+ if (backend === "coreml") return "yolov9m-320";
41479
+ return "yolo26n";
41480
+ }
41481
+ /**
41482
+ * Step-tree device jump (phase 1): validate every `steps[step].jumpDeviceKey`
41483
+ * manual override in a to-be-saved `inferenceDevices` map. A jump target MUST be
41484
+ * an enabled∧available device on the SAME node and DIFFERENT from the owning
41485
+ * device. `enabledAvailableKeys` is the effective enabled∧available set (from
41486
+ * `mergeInferenceDevices(probe, submitted)`) so an absent/unplugged/disabled
41487
+ * target is rejected honestly (an operator can't route a step onto a dead pool).
41488
+ * Returns the FIRST human-readable error, or `null` when every override is
41489
+ * valid. Pure + deterministic.
41490
+ */
41491
+ function validateJumpTargets(inferenceDevices, enabledAvailableKeys) {
41492
+ for (const [deviceKey, entry] of Object.entries(inferenceDevices)) for (const [stepId, step] of Object.entries(entry.steps ?? {})) {
41493
+ const target = step.jumpDeviceKey;
41494
+ if (target === void 0) continue;
41495
+ if (target === deviceKey) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey pointing at its own device`;
41496
+ if (!enabledAvailableKeys.has(target)) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey "${target}" that is not an enabled, available device on this node`;
41497
+ }
41498
+ return null;
41499
+ }
41500
+ /**
41501
+ * Merge a node's live-probed inference devices with its stored per-device map.
41502
+ *
41503
+ * The default is **AUTO = all discovered ACCELERATORS enabled** (spec C2,
41504
+ * opt-OUT) with TWO deliberate exceptions, both **opt-IN** (default disabled):
41505
+ *
41506
+ * - **CPU**: `enumerateInferenceDevices` always emits a universal `cpu`
41507
+ * floor on every platform; auto-enabling it would let the balancer
41508
+ * round-robin ~1/N of sessions onto the slow CPU pool alongside the
41509
+ * NPU/iGPU/ANE. CPU stays the always-available FALLBACK (a node with no
41510
+ * eligible accelerator leaves `deviceKey` unset → the runner's default
41511
+ * pool, which is CPU), not a balanced target — matching the spec's "no
41512
+ * device eligible → fall back to CPU".
41513
+ * - **Coral Edge TPU (`edgetpu`)**: the standing rule since the Coral
41514
+ * executor landed is that it surfaces as selectable but is NEVER
41515
+ * auto-picked — it runs a DIFFERENT, weaker model family (tflite SSD
41516
+ * MobileNet, not the YOLO the other accelerators run), so silently
41517
+ * enrolling a plugged-in Coral changes detection QUALITY, not just
41518
+ * placement. The opt-OUT default did exactly that on 2026-08-01: a hub
41519
+ * Coral nobody enabled entered the session rotation and camera 615 spent
41520
+ * hours at 2.4fps failing tflite model resolution. An operator who wants
41521
+ * the Coral balanced opts it in explicitly (`enabled: true`).
41522
+ *
41523
+ * So: an NPU/iGPU/ANE accelerator with NO stored entry is `enabled:true`; a
41524
+ * CPU or edgetpu device with no stored entry is `enabled:false`; an explicit
41525
+ * stored `enabled` always wins (an operator can opt CPU/Coral in, or an
41526
+ * accelerator out). A stored-only key (configured but the probe did not
41527
+ * return it — removed/unplugged HW) keeps its stored `enabled` and surfaces
41528
+ * as `available:false`, so the UI still shows it.
41529
+ *
41530
+ * Pure + deterministic (sorted by key) — the single merge authority shared by
41531
+ * the `getNodeInferenceDevices` view and the dispatcher's eligible-device pick.
41532
+ */
41533
+ function mergeInferenceDevices(probed, stored) {
41534
+ const probedByKey = new Map(probed.map((d) => [d.key, d]));
41535
+ const keys = new Set([...probedByKey.keys(), ...Object.keys(stored)]);
41536
+ const out = [];
41537
+ for (const key of Array.from(keys).toSorted()) {
41538
+ const descriptor = probedByKey.get(key);
41539
+ const opt = stored[key];
41540
+ const parsed = descriptor ?? parseDeviceKey(key);
41541
+ const weight = opt?.weight !== void 0 && opt.weight > 0 ? opt.weight : 1;
41542
+ const autoDefault = parsed.backend !== "cpu" && parsed.backend !== "edgetpu";
41543
+ out.push({
41544
+ key,
41545
+ backend: parsed.backend,
41546
+ device: parsed.device,
41547
+ format: parsed.format,
41548
+ available: descriptor?.available ?? false,
41549
+ enabled: opt?.enabled ?? autoDefault,
41550
+ weight,
41551
+ maxSessions: opt?.maxSessions ?? null,
41552
+ defaultModelId: defaultModelIdForDevice(key),
41553
+ ...opt?.steps && Object.keys(opt.steps).length > 0 ? { steps: { ...opt.steps } } : {}
41554
+ });
41555
+ }
41556
+ return out;
41557
+ }
41558
+ /**
41559
+ * The per-device concurrent-session caps for a node as `deviceKey → maxSessions`
41560
+ * (only devices that carry an explicit cap; absent = unlimited). Fed to the
41561
+ * device balancer's `nodeCaps` so a device at its cap is skipped (audit F3).
41562
+ */
41563
+ function inferenceDeviceCaps(stored) {
41564
+ const out = {};
41565
+ for (const [key, entry] of Object.entries(stored)) if (entry.maxSessions !== void 0 && entry.maxSessions > 0) out[key] = entry.maxSessions;
41566
+ return out;
41567
+ }
41568
+ /** Is this device the CPU fallback rather than a real accelerator? */
41569
+ function isCpuFallback(view) {
41570
+ return view.backend === "cpu";
41571
+ }
41572
+ function resolveInferenceDeviceEligibility(probed, stored, canRunRoot, isPoolUsable) {
41573
+ const eligible = {};
41574
+ const excluded = [];
41575
+ const merged = mergeInferenceDevices(probed, stored);
41576
+ const acceleratorServes = merged.some((d) => !isCpuFallback(d) && d.enabled && d.available && (!canRunRoot || canRunRoot(d.format)));
41577
+ for (const d of merged) {
41578
+ if (!d.enabled) {
41579
+ excluded.push({
41580
+ key: d.key,
41581
+ reason: "disabled",
41582
+ format: d.format
41583
+ });
41584
+ continue;
41585
+ }
41586
+ if (!d.available) {
41587
+ excluded.push({
41588
+ key: d.key,
41589
+ reason: "unavailable",
41590
+ format: d.format
41591
+ });
41592
+ continue;
41426
41593
  }
41427
- await this.withAudioSubLock(deviceId, async () => {
41428
- const unsub = this.audioSubscriptions.get(deviceId);
41429
- if (!unsub) return;
41430
- try {
41431
- unsub();
41432
- } catch {}
41433
- this.audioSubscriptions.delete(deviceId);
41434
- this.deps.logger.info("lazy audio: window closed", {
41435
- tags: { deviceId },
41436
- meta: { reason }
41594
+ if (isPoolUsable && !isPoolUsable(d.key)) {
41595
+ excluded.push({
41596
+ key: d.key,
41597
+ reason: "unavailable",
41598
+ format: d.format
41437
41599
  });
41438
- });
41439
- }
41440
- /**
41441
- * Audio teardown for one device — the audio half of `stopDetection`.
41442
- * Tears down through the per-device lock so it can't race a concurrent
41443
- * subscribe (which would re-store a handle this teardown never sees).
41444
- */
41445
- async stopForDevice(deviceId) {
41446
- await this.withAudioSubLock(deviceId, async () => {
41447
- const unsub = this.audioSubscriptions.get(deviceId);
41448
- if (unsub) {
41449
- try {
41450
- unsub();
41451
- } catch {}
41452
- this.audioSubscriptions.delete(deviceId);
41453
- }
41454
- });
41455
- const lazyTimer = this.lazyAudioTeardownTimers.get(deviceId);
41456
- if (lazyTimer) {
41457
- clearTimeout(lazyTimer);
41458
- this.lazyAudioTeardownTimers.delete(deviceId);
41600
+ continue;
41459
41601
  }
41460
- const windowTimer = this.motionAudioWindowTimers.get(deviceId);
41461
- if (windowTimer) {
41462
- clearTimeout(windowTimer);
41463
- this.motionAudioWindowTimers.delete(deviceId);
41602
+ if (canRunRoot && !canRunRoot(d.format)) {
41603
+ excluded.push({
41604
+ key: d.key,
41605
+ reason: "cannot-host-camera-root",
41606
+ format: d.format
41607
+ });
41608
+ continue;
41464
41609
  }
41465
- this.audioAssignments.delete(deviceId);
41466
- }
41467
- /**
41468
- * Centralized write into `audioSubscriptions`. If shutdown has begun, the
41469
- * map has already been (or is about to be) cleared lock-free in
41470
- * `shutdown()`; storing here would leak a zombie entry whose `unsub` is
41471
- * never called. So when shutting down we immediately invoke `unsub`
41472
- * (best-effort, error-swallowed) and DO NOT store. `protected` so
41473
- * `audio-sub-lock.spec.ts`'s test subclass can assert the shutdown-guard
41474
- * behavior without casts.
41475
- */
41476
- storeAudioSub(deviceId, unsub) {
41477
- if (this.audioShuttingDown) {
41478
- try {
41479
- unsub();
41480
- } catch {}
41481
- return;
41610
+ if (isCpuFallback(d) && acceleratorServes) {
41611
+ excluded.push({
41612
+ key: d.key,
41613
+ reason: "accelerator-preferred",
41614
+ format: d.format
41615
+ });
41616
+ continue;
41482
41617
  }
41483
- this.audioSubscriptions.set(deviceId, unsub);
41484
- }
41485
- /**
41486
- * Serialize an audio-subscription critical section per device. `fn` is
41487
- * chained onto the device's current lock tail, so concurrent calls for the
41488
- * SAME deviceId run sequentially (FIFO); different deviceIds never block
41489
- * each other. Thin delegate onto the `audioSubLocks` `KeyedAsyncLock`
41490
- * instance. `protected` so `audio-sub-lock.spec.ts`'s test subclass can
41491
- * drive the lock without casts.
41492
- */
41493
- withAudioSubLock(deviceId, fn) {
41494
- return this.audioSubLocks.run(deviceId, fn);
41618
+ eligible[d.key] = d.weight;
41495
41619
  }
41620
+ return {
41621
+ eligible,
41622
+ excluded
41623
+ };
41624
+ }
41625
+ /**
41626
+ * Join the merged device rows with the eligibility verdict so the UI can NAME
41627
+ * why an accelerator is not in play instead of leaving the operator to deduce
41628
+ * it from `enabled`/`available`.
41629
+ *
41630
+ * Deduction is not possible for two of the four reasons — `accelerator-preferred`
41631
+ * is a node-WIDE rule (a CPU row reads `enabled:true, available:true` and still
41632
+ * never gets a session, D215) and `cannot-host-camera-root` needs the node's
41633
+ * model catalog. Both live in {@link resolveInferenceDeviceEligibility}, so this
41634
+ * function only transports its answer; it never re-derives one.
41635
+ *
41636
+ * Pure; preserves `merged`'s order (sorted by key) and every other field.
41637
+ */
41638
+ function annotateInferenceDeviceExclusions(merged, eligibility) {
41639
+ const reasonByKey = new Map(eligibility.excluded.map((e) => [e.key, e.reason]));
41640
+ return merged.map((view) => ({
41641
+ ...view,
41642
+ exclusion: reasonByKey.get(view.key) ?? null
41643
+ }));
41644
+ }
41645
+ function resolveNodeInferenceUsability(eligibility) {
41646
+ const eligibleKeys = Object.keys(eligibility.eligible).toSorted();
41647
+ const unavailableKeys = eligibility.excluded.filter((e) => e.reason === "unavailable").map((e) => e.key).toSorted();
41648
+ return {
41649
+ usable: eligibleKeys.length > 0 || unavailableKeys.length === 0,
41650
+ unavailableKeys,
41651
+ eligibleKeys
41652
+ };
41653
+ }
41654
+ /**
41655
+ * Step-tree device jump (phase 1): the attach-payload roster of a node's
41656
+ * enabled∧available inference devices with the balancer knobs (`weight`,
41657
+ * `maxSessions`) the runner uses to AUTO-jump an enrichment step off a device
41658
+ * whose format can't run it. Built from the SAME `eligible` (deviceKey→weight)
41659
+ * and `caps` (deviceKey→maxSessions) the dispatcher already computes, so the
41660
+ * roster the runner sees exactly matches the balancer's candidate set. Sorted
41661
+ * by key for determinism. Populated onto `RunnerCameraConfig.inferenceDevices`
41662
+ * ONLY when a `deviceKey` is elected and there are ≥2 entries.
41663
+ */
41664
+ function buildInferenceDeviceRoster(eligible, caps) {
41665
+ return Object.entries(eligible).map(([deviceKey, weight]) => ({
41666
+ deviceKey,
41667
+ weight: weight > 0 ? weight : 1,
41668
+ maxSessions: caps[deviceKey] ?? null
41669
+ })).toSorted((a, b) => a.deviceKey < b.deviceKey ? -1 : a.deviceKey > b.deviceKey ? 1 : 0);
41670
+ }
41671
+ //#endregion
41672
+ //#region src/node-inference-usability-mirror.ts
41673
+ var NodeInferenceUsabilityMirror = class {
41674
+ state = /* @__PURE__ */ new Map();
41496
41675
  /**
41497
- * Subscribe to decoded audio chunks for a camera and feed them into the
41498
- * audio-analyzer. Reads the analyzer's settings via its own
41499
- * `resolveDeviceSettings(deviceId)` method so the orchestrator does not
41500
- * touch the audio-analyzer schema field names directly.
41676
+ * Fold one observation in and report whether the caller should act.
41677
+ * Never throws.
41501
41678
  */
41502
- async subscribeAudioStream(deviceId, config) {
41503
- const api = this.deps.api();
41504
- if (!api) {
41505
- this.deps.logger.warn("this.ctx.api not available — cannot subscribe audio", { tags: { deviceId } });
41506
- return null;
41507
- }
41508
- if (!await this.deps.isAudioAnalysisActive(deviceId)) return null;
41509
- if (config.audioMode === "disabled") {
41510
- this.deps.logger.debug("audio subscribe skipped: audioMode=disabled", { tags: { deviceId } });
41511
- return null;
41679
+ observe(nodeId, usable) {
41680
+ const prev = this.state.get(nodeId);
41681
+ if (usable) {
41682
+ this.state.set(nodeId, {
41683
+ usable: true,
41684
+ armed: false
41685
+ });
41686
+ return prev !== void 0 && !prev.usable ? "recovered" : null;
41512
41687
  }
41513
- if (config.audioMode === "on-motion" && !this.isMotionAudioWindowOpen(deviceId)) {
41514
- this.deps.logger.info("audio subscribe deferred: audioMode=on-motion, no window open", { tags: { deviceId } });
41688
+ if (prev === void 0) {
41689
+ this.state.set(nodeId, {
41690
+ usable: true,
41691
+ armed: true
41692
+ });
41515
41693
  return null;
41516
41694
  }
41517
- const audioStream = config.audioStreamId ?? config.motionStreamId;
41518
- const audioBrokerId = makeSourceBrokerId(deviceId, audioStream);
41519
- if ((await this.deps.probeAudioTrack(deviceId, audioStream)).kind === "absent") {
41520
- this.deps.logger.warn("audio subscription REFUSED — this stream carries no audio track", {
41521
- tags: { deviceId },
41522
- meta: {
41523
- camStreamId: audioStream,
41524
- brokerId: audioBrokerId,
41525
- selectedBy: config.audioStreamId !== void 0 ? "audioStreamId" : "motionStreamId",
41526
- hint: "point the camera’s audio at a stream that has an audio track — no stream is substituted automatically"
41527
- }
41695
+ if (!prev.usable) {
41696
+ this.state.set(nodeId, {
41697
+ usable: false,
41698
+ armed: true
41528
41699
  });
41529
41700
  return null;
41530
41701
  }
41531
- const settings = await api.audioAnalysis.resolveDeviceSettings.query({ deviceId });
41532
- if (!settings) {
41533
- this.deps.logger.warn("audio-analysis returned no settings — audio subscription skipped", { tags: { deviceId } });
41702
+ if (!prev.armed) {
41703
+ this.state.set(nodeId, {
41704
+ usable: true,
41705
+ armed: true
41706
+ });
41534
41707
  return null;
41535
41708
  }
41536
- const audioNodeId = await this.dispatch(deviceId);
41537
- const isRemoteAudio = audioNodeId !== this.deps.localNodeId();
41538
- this.deps.logger.info("audio subscription: resolved audio node", {
41539
- tags: { deviceId },
41540
- meta: {
41541
- audioNodeId,
41542
- isRemote: isRemoteAudio
41543
- }
41544
- });
41545
- const accumulator = new AudioWindowAccumulator(deviceId);
41546
- const teardown = startAudioChunkPoller({
41547
- api,
41548
- brokerId: audioBrokerId,
41549
- tag: "audio-analyzer",
41550
- ownerNodeId: this.deps.ingestNode(),
41551
- logger: this.deps.logger.withTags({ deviceId }),
41552
- onChunk: async (chunk) => {
41553
- this.deps.watchdogNote(deviceId, "audio");
41554
- try {
41555
- const audioChunkInput = accumulator.push(chunk);
41556
- if (!audioChunkInput) return;
41557
- const result = await api.audioAnalyzer.analyseChunk.mutate({
41558
- chunk: audioChunkInput,
41559
- settings,
41560
- ...isRemoteAudio ? { nodeId: audioNodeId } : {}
41561
- });
41562
- if (!result) return;
41563
- const frame = buildAudioResultFrame(deviceId, result);
41564
- this.deps.eventBus.emit({
41565
- id: `audio-inference-${deviceId}-${Date.now()}`,
41566
- timestamp: /* @__PURE__ */ new Date(),
41567
- source: {
41568
- type: "device",
41569
- id: deviceId,
41570
- nodeId: "hub",
41571
- addonId: "pipeline-orchestrator",
41572
- deviceId
41573
- },
41574
- category: EventCategory.PipelineAudioInferenceResult,
41575
- data: {
41576
- deviceId,
41577
- frame,
41578
- nodeId: "hub"
41579
- }
41580
- });
41581
- } catch (err) {
41582
- const msg = errMsg(err);
41583
- this.deps.logger.error("Audio analysis failed", {
41584
- tags: { deviceId },
41585
- meta: { error: msg }
41586
- });
41587
- }
41588
- }
41709
+ this.state.set(nodeId, {
41710
+ usable: false,
41711
+ armed: true
41589
41712
  });
41590
- this.deps.logger.info("Audio stream subscribed", { tags: { deviceId } });
41591
- return () => {
41592
- teardown();
41593
- accumulator.reset();
41594
- };
41713
+ return "became-unusable";
41595
41714
  }
41596
- /**
41597
- * Set true at the very start of `onShutdown`, before the audio teardown /
41598
- * map clears below. Once set, `withAudioSubLock` turns queued/new critical
41599
- * sections into no-ops and `storeAudioSub` refuses to store, so no
41600
- * critical section that was in-flight (or queued) when shutdown began can
41601
- * resurrect a zombie subscription into the cleared `audioSubscriptions`
41602
- * map. MUST be called before anything else in `onShutdown` that could
41603
- * race a queued audio critical section (mirrors the original
41604
- * `this.audioShuttingDown = true` being the very first statement).
41605
- */
41606
- beginShutdown() {
41607
- this.audioShuttingDown = true;
41715
+ /** Can this node be given cameras? Unknown nodes answer YES. */
41716
+ isUsable(nodeId) {
41717
+ return this.state.get(nodeId)?.usable ?? true;
41718
+ }
41719
+ /** Nodes currently excluded for the placement log and diagnostics. */
41720
+ unusableNodeIds() {
41721
+ const out = [];
41722
+ for (const [nodeId, s] of this.state) if (!s.usable) out.push(nodeId);
41723
+ return out.toSorted();
41724
+ }
41725
+ forget(nodeId) {
41726
+ this.state.delete(nodeId);
41608
41727
  }
41728
+ reset() {
41729
+ this.state.clear();
41730
+ }
41731
+ };
41732
+ //#endregion
41733
+ //#region src/inference-device-usability-mirror.ts
41734
+ /**
41735
+ * InferenceDeviceUsabilityMirror — "can this NODE's THIS DEVICE still infer?",
41736
+ * kept off the placement path.
41737
+ *
41738
+ * The per-`(nodeId, deviceKey)` twin of {@link NodeInferenceUsabilityMirror},
41739
+ * and deliberately not a second mechanism: it composes the key and delegates
41740
+ * every decision to that class, so the arm/apply reluctance D49 pinned lives in
41741
+ * exactly one implementation and cannot drift between the node tier and the
41742
+ * device tier.
41743
+ *
41744
+ * ## Why this tier had to exist
41745
+ *
41746
+ * The node tier already answers "does this node have ANY usable accelerator".
41747
+ * On 2026-08-25 the hub's answer was, correctly, yes — its NPU was healthy the
41748
+ * whole time. What died was one DEVICE, `openvino:gpu`, and nothing anywhere
41749
+ * asked that question: the per-dispatch capability gate is keyed on model
41750
+ * FORMAT and `openvino:gpu` and `openvino:npu` share the format `openvino`, so
41751
+ * it is blind between them by construction. The balancer kept rotating cameras
41752
+ * onto the dead pool — 20 sessions in 20 minutes, every one `tieBreak:
41753
+ * "rotation"` — for 31 hours.
41754
+ *
41755
+ * ## Why a mirror and not the event
41756
+ *
41757
+ * D49, verbatim: *a one-shot event never gates on a fallible read*. The gate is
41758
+ * this in-memory mirror, refreshed off the event path by the same
41759
+ * `resolveEligibleInferenceDevices` the dispatcher already runs (plus the
41760
+ * session controller's background refresher). The consequences that buys:
41761
+ *
41762
+ * - **A read that fails changes nothing.** The caller folds in an observation
41763
+ * only when it HAS one; an unreachable node, a version-skewed executor or a
41764
+ * rejected RPC never reaches {@link observe}, so the previous verdict
41765
+ * stands. This is the whole reason the health read is specified as
41766
+ * "synchronous over in-memory state, never throws for its own reasons": an
41767
+ * empty answer must mean *nothing is refused*, not *I could not tell*.
41768
+ * - **Excluding a healthy device needs a second, consecutive read.** Exclusion
41769
+ * is the direction that DESTROYS work — it strands an accelerator that may
41770
+ * be perfectly fine — so one bad observation only ARMS.
41771
+ * - **Re-admitting is immediate and unconditional.** One good observation puts
41772
+ * the device straight back. Being slow to exclude costs some wasted
41773
+ * inference attempts; being slow to re-admit costs an idle accelerator and a
41774
+ * node that looks broken.
41775
+ */
41776
+ /**
41777
+ * NUL — impossible in both a Moleculer node id and a `<backend>:<device>` key,
41778
+ * so the composite key can never be ambiguous. A separator that CAN occur in
41779
+ * either half makes two distinct pairs collide, and a collision here silently
41780
+ * excludes an accelerator nobody reported.
41781
+ */
41782
+ var SEPARATOR = "\0";
41783
+ var InferenceDeviceUsabilityMirror = class {
41784
+ /** The one implementation of the arm/apply state machine (D49). */
41785
+ mirror = new NodeInferenceUsabilityMirror();
41609
41786
  /**
41610
- * Full audio teardown combines the former `onShutdown`'s two separate
41611
- * audio blocks (lazy-teardown-timer clear, then — after several unrelated
41612
- * session/reconcile/load-shed clears — subscription teardown + lock clear
41613
- * + assignment-map clears) into one call. Safe to combine: both blocks
41614
- * are synchronous with no interleaved `await`, and every original
41615
- * statement between them (`sessionRegistry.clear()`,
41616
- * `cameraFpsMap.clear()`, `remoteHealthAttempts.clear()`,
41617
- * `loadShedState.clear()`, `loadShedResumeTimer` cleanup) touches state
41618
- * fully disjoint from anything audio — so their relative order to each
41619
- * other is unaffected, and the audio-internal order (timers →
41620
- * subscriptions → lock → assignment maps) is reproduced exactly.
41787
+ * Fold one observation in and report whether the caller should act.
41788
+ * Never throws.
41621
41789
  */
41622
- shutdown() {
41623
- for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
41624
- this.lazyAudioTeardownTimers.clear();
41625
- for (const t of this.motionAudioWindowTimers.values()) clearTimeout(t);
41626
- this.motionAudioWindowTimers.clear();
41627
- for (const unsub of this.audioSubscriptions.values()) try {
41628
- unsub();
41629
- } catch {}
41630
- this.audioSubscriptions.clear();
41631
- this.audioSubLocks.clear();
41632
- this.audioAssignments.clear();
41633
- this.readyAudioNodes.clear();
41790
+ observe(nodeId, deviceKey, usable) {
41791
+ return this.mirror.observe(`${nodeId}${SEPARATOR}${deviceKey}`, usable);
41792
+ }
41793
+ /** Can the balancer put a session on this device? Unknown pairs answer YES. */
41794
+ isUsable(nodeId, deviceKey) {
41795
+ return this.mirror.isUsable(`${nodeId}${SEPARATOR}${deviceKey}`);
41796
+ }
41797
+ /** Pairs currently excluded — for the placement log and diagnostics. */
41798
+ unusableDevices() {
41799
+ return this.mirror.unusableNodeIds().map((composite) => {
41800
+ const at = composite.indexOf(SEPARATOR);
41801
+ return {
41802
+ nodeId: composite.slice(0, at),
41803
+ deviceKey: composite.slice(at + 1)
41804
+ };
41805
+ });
41806
+ }
41807
+ /** The excluded device keys on ONE node. */
41808
+ unusableDeviceKeys(nodeId) {
41809
+ return this.unusableDevices().filter((d) => d.nodeId === nodeId).map((d) => d.deviceKey);
41810
+ }
41811
+ forget(nodeId, deviceKey) {
41812
+ this.mirror.forget(`${nodeId}${SEPARATOR}${deviceKey}`);
41813
+ }
41814
+ reset() {
41815
+ this.mirror.reset();
41634
41816
  }
41635
41817
  };
41818
+ /**
41819
+ * Fold ONE node's health answer into the mirror and return what changed.
41820
+ *
41821
+ * This is the whole reading discipline, in one place, because both halves of it
41822
+ * are easy to get subtly wrong and neither failure is visible in a log:
41823
+ *
41824
+ * - **`unhealthy === null` — the read FAILED — changes nothing.** Not a single
41825
+ * entry is touched. An unreachable node, a version-skewed executor or a
41826
+ * rejected RPC must be distinguishable from "asked, nothing is refused", or
41827
+ * a flaky link silently re-admits a dead accelerator (D49).
41828
+ * - **Every device the node HAS is observed**, not merely the refused ones.
41829
+ * The first draft observed `refused ∪ already-excluded`, which omits exactly
41830
+ * the devices the mirror has ARMED — so their disarming good read never
41831
+ * arrived and two bad reads an HOUR apart, with a hundred healthy ones
41832
+ * between them, excluded a working accelerator. "Consecutive" is only a
41833
+ * property if the good observations are delivered.
41834
+ *
41835
+ * Pure with respect to everything except `mirror`, and never throws — it is
41836
+ * called from the dispatcher's own read path.
41837
+ */
41838
+ function foldInferenceDeviceHealth(mirror, nodeId, unhealthy, present) {
41839
+ if (unhealthy === null) return [];
41840
+ const refused = new Set(unhealthy);
41841
+ const observed = new Set([
41842
+ ...present,
41843
+ ...refused,
41844
+ ...mirror.unusableDeviceKeys(nodeId)
41845
+ ]);
41846
+ const changes = [];
41847
+ for (const deviceKey of observed) {
41848
+ const transition = mirror.observe(nodeId, deviceKey, !refused.has(deviceKey));
41849
+ if (transition !== null) changes.push({
41850
+ deviceKey,
41851
+ transition
41852
+ });
41853
+ }
41854
+ return changes;
41855
+ }
41636
41856
  //#endregion
41637
41857
  //#region src/load-balancer.ts
41638
41858
  /**
@@ -43568,6 +43788,90 @@ function applyDeviceProvisioning(steps, base, override) {
43568
43788
  });
43569
43789
  }
43570
43790
  //#endregion
43791
+ //#region src/device-activity-source.ts
43792
+ /** The source's own name on the wire. One constant, used by every emit + gate. */
43793
+ var DEVICE_ACTIVITY_SOURCE = "device-activity";
43794
+ var DeviceActivitySource = class {
43795
+ #deps;
43796
+ #devices = /* @__PURE__ */ new Map();
43797
+ constructor(deps) {
43798
+ this.#deps = deps;
43799
+ }
43800
+ /**
43801
+ * A `recording-signal` level landed for a device. Idempotent in the level:
43802
+ * only a CHANGE emits an edge, and the sustain tick — not a repeated push —
43803
+ * is what keeps the session open.
43804
+ */
43805
+ onSignalLevel(deviceId, active, reason, atMs) {
43806
+ const entry = this.#devices.get(deviceId) ?? {
43807
+ active: false,
43808
+ timer: null
43809
+ };
43810
+ const wasActive = entry.active;
43811
+ entry.active = active;
43812
+ this.#devices.set(deviceId, entry);
43813
+ if (active === wasActive) return;
43814
+ if (active) {
43815
+ this.#deps.logger.info("device-activity: the device reports it is working", {
43816
+ tags: { deviceId },
43817
+ meta: { reason }
43818
+ });
43819
+ this.#emit(deviceId, true, atMs);
43820
+ this.#arm(deviceId, entry);
43821
+ return;
43822
+ }
43823
+ this.#clear(entry);
43824
+ this.#deps.logger.info("device-activity: the device reports it has stopped", {
43825
+ tags: { deviceId },
43826
+ meta: { reason }
43827
+ });
43828
+ this.#emit(deviceId, false, atMs);
43829
+ }
43830
+ /** Drop every timer (addon teardown). The mirror goes with the instance. */
43831
+ stop() {
43832
+ for (const entry of this.#devices.values()) this.#clear(entry);
43833
+ this.#devices.clear();
43834
+ }
43835
+ #arm(deviceId, entry) {
43836
+ this.#clear(entry);
43837
+ const sustainMs = this.#deps.sustainMsFor(deviceId);
43838
+ const timer = setInterval(() => {
43839
+ const current = this.#devices.get(deviceId);
43840
+ if (!current || !current.active) {
43841
+ this.#clear(current ?? entry);
43842
+ return;
43843
+ }
43844
+ this.#emit(deviceId, true, Date.now());
43845
+ }, sustainMs);
43846
+ timer.unref?.();
43847
+ entry.timer = timer;
43848
+ }
43849
+ #clear(entry) {
43850
+ if (entry.timer === null) return;
43851
+ clearInterval(entry.timer);
43852
+ entry.timer = null;
43853
+ }
43854
+ /**
43855
+ * Emit, unless this camera does not carry the source. A suppressed emit is
43856
+ * SAID — an operator who never ticked the box and an addon that is quietly
43857
+ * broken look identical from the outside otherwise.
43858
+ */
43859
+ #emit(deviceId, detected, timestamp) {
43860
+ const sources = this.#deps.motionSourcesFor(deviceId);
43861
+ if (sources === null || !sources.includes("device-activity")) {
43862
+ this.#deps.logger.debug("device-activity: level not forwarded — the camera does not list `device-activity`", {
43863
+ tags: { deviceId },
43864
+ meta: {
43865
+ detected,
43866
+ sources: sources ?? "no-active-detection"
43867
+ }
43868
+ });
43869
+ return;
43870
+ }
43871
+ this.#deps.emitMotion(deviceId, detected, timestamp);
43872
+ }
43873
+ };
43874
+ //#endregion
43571
43875
  //#region src/device-detection-settings.ts
43572
43876
  /** Read a required string leaf out of the hydrated `flat` schema values. */
43573
43877
  function mustString(flat, deviceId, key) {
@@ -43593,6 +43897,32 @@ function numberOrDefault(flat, key) {
43593
43897
  const dflt = uiField && "default" in uiField ? uiField.default : void 0;
43594
43898
  return typeof dflt === "number" ? dflt : 0;
43595
43899
  }
43900
+ /** The activity rate from the hydrated store, or the shipped default. */
43901
+ function activityFps(flat) {
43902
+ const v = flat["activityDetectionFps"];
43903
+ return typeof v === "number" && v > 0 ? v : 1;
43904
+ }
43905
+ /**
43906
+ * The detection rate a session opens at, given WHAT OPENED IT.
43907
+ *
43908
+ * A CAP, not a set: `min(cameraRate, activityRate)`. A camera already slower
43909
+ * than the activity rate stays slower, so lowering the global rate keeps
43910
+ * working. Why this lever and not the other two: a per-device `detectionFps`
43911
+ * would also slow the sessions a real motion trigger opens on the same camera,
43912
+ * and it becomes silently wrong the day the device gains a second trigger; a
43913
+ * per-SOURCE rate table would have to reach the runner and be re-applied
43914
+ * whenever the source holding the session changes, which the attach-time config
43915
+ * cannot express. Scoping it to the session's TRIGGER puts the number exactly
43916
+ * where its justification lives.
43917
+ *
43918
+ * Applies to the session the activity level OPENS. A session already open at
43919
+ * the camera rate when the level rises is not re-attached to slow it down —
43920
+ * re-attaching a live session to change one number costs a decode restart.
43921
+ */
43922
+ function detectionFpsForTrigger(config, trigger) {
43923
+ if (trigger !== "device-activity") return config.detectionFps;
43924
+ return Math.min(config.detectionFps, config.activityDetectionFps ?? 1);
43925
+ }
43596
43926
  /**
43597
43927
  * Pure decision/derivation half of the former `resolveDeviceDetectionSettings`.
43598
43928
  * Given the I/O-gathered raw materials, resolves every operator-wins →
@@ -43602,14 +43932,19 @@ function numberOrDefault(flat, key) {
43602
43932
  * the original method's "narrowing failed" catch.
43603
43933
  */
43604
43934
  function resolveDetectionSettings(input) {
43605
- const { deviceId, raw, flat, features, hasOnboardMotion, pipelineEnabled, motionDetectionEnabled } = input;
43935
+ const { deviceId, raw, flat, features, hasOnboardMotion, hasActivitySignal, pipelineEnabled, motionDetectionEnabled } = input;
43606
43936
  const profile = resolveDeviceProfile(features);
43607
43937
  const userMotionSources = raw["motionSources"];
43608
43938
  let motionSources;
43609
43939
  if (userMotionSources !== void 0) motionSources = MotionSourcesSchema.parse(userMotionSources);
43610
- else if (hasOnboardMotion) motionSources = ["onboard"];
43611
- else if (profile && features.includes(DeviceFeature.BatteryOperated)) motionSources = [];
43612
- else motionSources = MotionSourcesSchema.parse(flat["motionSources"]);
43940
+ else {
43941
+ let defaulted;
43942
+ if (hasOnboardMotion) defaulted = ["onboard"];
43943
+ else if (hasActivitySignal) defaulted = [];
43944
+ else if (profile && features.includes(DeviceFeature.BatteryOperated)) defaulted = [];
43945
+ else defaulted = MotionSourcesSchema.parse(flat["motionSources"]);
43946
+ motionSources = hasActivitySignal ? [...defaulted, DEVICE_ACTIVITY_SOURCE] : defaulted;
43947
+ }
43613
43948
  const userDetectionMode = raw["detectionMode"];
43614
43949
  const detectionMode = typeof userDetectionMode === "string" && isPipelinePhaseMode(userDetectionMode) ? userDetectionMode : profile?.defaults.detectionMode ?? "on-motion";
43615
43950
  const userAudioMode = raw["audioMode"];
@@ -43628,6 +43963,7 @@ function resolveDetectionSettings(input) {
43628
43963
  detectionStreamProfile: mustString(flat, deviceId, "detectionStreamProfile"),
43629
43964
  motionFps: numberOrDefault(flat, "motionFps"),
43630
43965
  detectionFps: numberOrDefault(flat, "detectionFps"),
43966
+ activityDetectionFps: activityFps(flat),
43631
43967
  motionCooldownMs: numberOrDefault(flat, "motionCooldownMs"),
43632
43968
  maxSessionHoldMs: numberOrDefault(flat, "maxSessionHoldMs"),
43633
43969
  audioMotionWindowMs: numberOrDefault(flat, "audioMotionWindowMs"),
@@ -43684,6 +44020,7 @@ function buildDetectionConfigFromInputs(resolved, assigned) {
43684
44020
  detectionStreamId: detectionCamStreamId,
43685
44021
  motionFps: resolved.motionFps,
43686
44022
  detectionFps: resolved.detectionFps,
44023
+ activityDetectionFps: resolved.activityDetectionFps,
43687
44024
  motionCooldownMs: resolved.motionCooldownMs,
43688
44025
  maxSessionHoldMs: resolved.maxSessionHoldMs,
43689
44026
  audioMotionWindowMs: resolved.audioMotionWindowMs,
@@ -43709,6 +44046,7 @@ function detectionConfigEquals(a, b) {
43709
44046
  if (a.detectionStreamId !== b.detectionStreamId) return false;
43710
44047
  if (a.motionFps !== b.motionFps) return false;
43711
44048
  if (a.detectionFps !== b.detectionFps) return false;
44049
+ if (a.activityDetectionFps !== b.activityDetectionFps) return false;
43712
44050
  if (a.motionCooldownMs !== b.motionCooldownMs) return false;
43713
44051
  if (a.maxSessionHoldMs !== b.maxSessionHoldMs) return false;
43714
44052
  if (a.audioMotionWindowMs !== b.audioMotionWindowMs) return false;
@@ -44335,6 +44673,7 @@ var DetectionWiringController = class {
44335
44673
  try {
44336
44674
  const features = await this.lookupDeviceFeatures(deviceId);
44337
44675
  const hasOnboardMotion = raw["motionSources"] === void 0 ? await this.deps.deviceHasOnboardMotionCap(deviceId) : false;
44676
+ const hasActivitySignal = raw["motionSources"] === void 0 ? await this.deps.deviceHasActivitySignalCap(deviceId) : false;
44338
44677
  const pipelineEnabled = await this.isDetectionPipelineActive(deviceId);
44339
44678
  const motionDetectionEnabled = await this.isMotionDetectionActive(deviceId);
44340
44679
  return resolveDetectionSettings({
@@ -44343,6 +44682,7 @@ var DetectionWiringController = class {
44343
44682
  flat,
44344
44683
  features,
44345
44684
  hasOnboardMotion,
44685
+ hasActivitySignal,
44346
44686
  pipelineEnabled,
44347
44687
  motionDetectionEnabled
44348
44688
  });
@@ -44948,9 +45288,10 @@ var DeviceConfigContributions = class {
44948
45288
  const schema = this.deps.deviceSettingsSchema();
44949
45289
  if (!schema) return null;
44950
45290
  const hasOnboardMotion = await this.deps.deviceHasOnboardMotionCap(input.deviceId);
44951
- const rawWithDefaults = raw["motionSources"] === void 0 && hasOnboardMotion ? {
45291
+ const hasActivitySignal = await this.deps.deviceHasActivitySignalCap(input.deviceId);
45292
+ const rawWithDefaults = raw["motionSources"] === void 0 && (hasOnboardMotion || hasActivitySignal) ? {
44952
45293
  ...raw,
44953
- motionSources: ["onboard"]
45294
+ motionSources: [...hasOnboardMotion ? ["onboard"] : [], ...hasActivitySignal ? ["device-activity"] : []]
44954
45295
  } : raw;
44955
45296
  const baseSections = hydrateSchema({
44956
45297
  ...schema,
@@ -47770,7 +48111,8 @@ function wireOrchestratorSubscriptions(deps) {
47770
48111
  if (!isEvent(event, EventCategory.MotionOnMotionChanged)) return;
47771
48112
  const { deviceId, detected, timestamp } = event.data;
47772
48113
  if (typeof deviceId !== "number") return;
47773
- deps.handleSessionMotion(deviceId, detected, typeof timestamp === "number" ? timestamp : void 0).catch((err) => {
48114
+ const parsedSource = MotionSourceEnum.safeParse(event.data.source);
48115
+ deps.handleSessionMotion(deviceId, detected, typeof timestamp === "number" ? timestamp : void 0, parsedSource.success ? parsedSource.data : void 0).catch((err) => {
47774
48116
  deps.logger.warn("session motion handler failed", {
47775
48117
  tags: { deviceId },
47776
48118
  meta: {
@@ -47792,8 +48134,47 @@ function wireOrchestratorSubscriptions(deps) {
47792
48134
  const deviceId = event.source.deviceId;
47793
48135
  if (typeof deviceId === "number") deps.noteWatchdogSignal(deviceId, "motion");
47794
48136
  });
48137
+ const activitySource = new DeviceActivitySource({
48138
+ logger: deps.logger,
48139
+ emitMotion: (deviceId, detected, timestamp) => {
48140
+ if (isTornDown) return;
48141
+ deps.eventBus.emit(createEvent(EventCategory.MotionOnMotionChanged, {
48142
+ type: "device",
48143
+ id: deviceId,
48144
+ deviceId
48145
+ }, {
48146
+ deviceId,
48147
+ detected,
48148
+ timestamp,
48149
+ source: DEVICE_ACTIVITY_SOURCE
48150
+ }));
48151
+ },
48152
+ motionSourcesFor: (deviceId) => deps.getActiveDetectionConfig(deviceId)?.motionSources ?? null,
48153
+ sustainMsFor: (deviceId) => {
48154
+ const cooldownMs = deps.getActiveDetectionConfig(deviceId)?.motionCooldownMs;
48155
+ return Math.max(1e3, Math.floor((cooldownMs ?? 3e4) / 2));
48156
+ }
48157
+ });
48158
+ const unsubDeviceActivity = deps.eventBus.subscribe({ category: EventCategory.DeviceStateChanged }, (event) => {
48159
+ const data = event.data;
48160
+ if (typeof data !== "object" || data === null) return;
48161
+ if (data["capName"] !== recordingSignalCapability.name) return;
48162
+ const deviceId = data["deviceId"];
48163
+ if (typeof deviceId !== "number") return;
48164
+ const level = RecordingSignalStatusSchema.safeParse(data["slice"]);
48165
+ if (!level.success) {
48166
+ deps.logger.warn("device-activity: signal slice does not parse — ignored", {
48167
+ tags: { deviceId },
48168
+ meta: { slice: data["slice"] }
48169
+ });
48170
+ return;
48171
+ }
48172
+ const atMs = event.timestamp instanceof Date ? event.timestamp.getTime() : Date.now();
48173
+ activitySource.onSignalLevel(deviceId, level.data.active, level.data.reason, atMs);
48174
+ });
47795
48175
  return () => {
47796
48176
  isTornDown = true;
48177
+ activitySource.stop();
47797
48178
  for (const t of profileSlotTimers.values()) clearTimeout(t);
47798
48179
  profileSlotTimers.clear();
47799
48180
  unsubDeviceRegistered();
@@ -47808,6 +48189,7 @@ function wireOrchestratorSubscriptions(deps) {
47808
48189
  unsubSessionMotion();
47809
48190
  unsubFrameTracked();
47810
48191
  unsubMotionAnalysis();
48192
+ unsubDeviceActivity();
47811
48193
  };
47812
48194
  }
47813
48195
  //#endregion
@@ -50567,7 +50949,7 @@ var SessionDispatchController = class {
50567
50949
  * standing attach, so `hasStandingAttach` stays true for them and
50568
50950
  * `decideSessionAction` still returns `'ignore'`.
50569
50951
  */
50570
- async handleSessionMotion(deviceId, detected, emittedAt) {
50952
+ async handleSessionMotion(deviceId, detected, emittedAt, trigger) {
50571
50953
  const receivedAt = Date.now();
50572
50954
  const busLagMs = emittedAt !== void 0 ? receivedAt - emittedAt : void 0;
50573
50955
  const config = this.deps.getActiveDetectionConfig(deviceId);
@@ -50602,7 +50984,7 @@ var SessionDispatchController = class {
50602
50984
  return;
50603
50985
  }
50604
50986
  this.activeRefireCountByDevice.delete(deviceId);
50605
- await this.dispatchDetectionSession(deviceId, cur);
50987
+ await this.dispatchDetectionSession(deviceId, cur, trigger);
50606
50988
  if (this.sessionRegistry.has(deviceId)) this.scheduleSessionTeardown(deviceId, cooldownMs);
50607
50989
  const doneAt = Date.now();
50608
50990
  this.deps.logger.info("session motion → attach latency", {
@@ -50641,7 +51023,7 @@ var SessionDispatchController = class {
50641
51023
  * `dispatchCamera`) avoids any risk of changing that already-live
50642
51024
  * standing-camera path.
50643
51025
  */
50644
- async dispatchDetectionSession(deviceId, config) {
51026
+ async dispatchDetectionSession(deviceId, config, trigger) {
50645
51027
  const log = this.deps.logger.withTags({ deviceId });
50646
51028
  await this.deps.reconcilePlacementFromRunners();
50647
51029
  const preferredAgent = await this.deps.readPipelinePin(deviceId);
@@ -50697,13 +51079,19 @@ var SessionDispatchController = class {
50697
51079
  const steps = applyDeviceProvisioning(pipelineConfig.steps, deviceBase, deviceOverride);
50698
51080
  const inferenceDevices = deviceKey && Object.keys(enabledDevices).length >= 2 ? buildInferenceDeviceRoster(enabledDevices, deviceCaps) : void 0;
50699
51081
  const zones = await this.deps.listZones(deviceId);
51082
+ const sessionDetectionFps = detectionFpsForTrigger(config, trigger);
51083
+ if (sessionDetectionFps !== config.detectionFps) log.info("session opened at the device-activity rate", { meta: {
51084
+ trigger,
51085
+ detectionFps: sessionDetectionFps,
51086
+ cameraDetectionFps: config.detectionFps
51087
+ } });
50700
51088
  const sessionConfig = {
50701
51089
  deviceId,
50702
51090
  ...deviceKey ? { deviceKey } : {},
50703
51091
  ...inferenceDevices ? { inferenceDevices } : {},
50704
51092
  motionCooldownMs: config.motionCooldownMs,
50705
51093
  motionFps: config.motionFps,
50706
- detectionFps: config.detectionFps,
51094
+ detectionFps: sessionDetectionFps,
50707
51095
  motionStreamId: config.motionStreamId,
50708
51096
  detectionStreamId: config.detectionStreamId,
50709
51097
  motionSources: [],
@@ -52008,6 +52396,7 @@ async function buildOrchestratorControllers(deps) {
52008
52396
  localNodeId: () => localNodeId,
52009
52397
  deviceSettingsSchema: () => deps.deviceSettingsSchema(),
52010
52398
  deviceHasOnboardMotionCap: (deviceId) => deps.deviceHasOnboardMotionCap(deviceId),
52399
+ deviceHasActivitySignalCap: (deviceId) => deps.deviceHasActivitySignalCap(deviceId),
52011
52400
  deviceHasNativeObjectDetectionCap: (deviceId) => deps.deviceHasNativeObjectDetectionCap(deviceId),
52012
52401
  setCameraPipelineForAgent: (input) => deps.setCameraPipelineForAgent(input),
52013
52402
  emitCameraUpdated: (deviceId, config) => deps.emitCameraUpdated(deviceId, config),
@@ -52325,6 +52714,7 @@ async function buildOrchestratorControllers(deps) {
52325
52714
  },
52326
52715
  isCapActiveForDevice: (deviceId, capName) => deps.isCapActiveForDevice(deviceId, capName),
52327
52716
  deviceHasOnboardMotionCap: (deviceId) => deps.deviceHasOnboardMotionCap(deviceId),
52717
+ deviceHasActivitySignalCap: (deviceId) => deps.deviceHasActivitySignalCap(deviceId),
52328
52718
  deviceSettingsSchema: () => deps.deviceSettingsSchema()
52329
52719
  });
52330
52720
  await detectionWiring.hydrateFeaturesMirror().catch((err) => {
@@ -52360,255 +52750,6 @@ async function buildOrchestratorControllers(deps) {
52360
52750
  };
52361
52751
  }
52362
52752
  //#endregion
52363
- //#region src/viewer-ui-provider.ts
52364
- /**
52365
- * viewer-ui provider — the pipeline-orchestrator serves the CamStack viewer web
52366
- * SPA (mirrors what the now-removed standalone addon-viewer-ui used to do).
52367
- *
52368
- * Why here: the orchestrator is a hub bootstrap addon (always installed + baked),
52369
- * so folding the viewer serving into it avoids a second addon whose only job was
52370
- * to hold static files. The viewer's Expo web export is COPIED into this addon's
52371
- * `assets/viewer/` locally by the viewer repo's `scripts/copy-dist-to-orchestrator.js`
52372
- * (run from a checkout that HAS the `camstack/` submodule + Expo toolchain — the
52373
- * publish/image CI does not, which is exactly why we copy a pre-built dist rather
52374
- * than build it here). vite emits only into `dist/`, so `assets/viewer` survives
52375
- * the addon build; `assets` is in the package `files`, so it ships on
52376
- * `camstack deploy`. The hub's `main.ts` resolves the `viewer-ui` singleton at
52377
- * boot and mounts the SPA at `/viewer/camstack`.
52378
- *
52379
- * `index.js` runs from `dist/`, so the SPA root is one level up + `assets/viewer`.
52380
- */
52381
- var __dirname$1 = node_path.default.dirname((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
52382
- /** Absolute path to the staged viewer web SPA (`<addon-root>/assets/viewer`). */
52383
- function resolveViewerDistDir() {
52384
- return node_path.default.resolve(__dirname$1, "..", "assets", "viewer");
52385
- }
52386
- /** Version of the staged viewer, written by the copy script; 'unknown' if absent. */
52387
- function readViewerVersion() {
52388
- try {
52389
- const raw = node_fs.default.readFileSync(node_path.default.join(resolveViewerDistDir(), ".viewer-version"), "utf-8").trim();
52390
- if (raw) return raw;
52391
- } catch {}
52392
- return "unknown";
52393
- }
52394
- /** Build the viewer-ui provider. Serves whatever dist was staged into
52395
- * `assets/viewer`; when nothing is staged the hub's mount cleanly 404s. */
52396
- function createViewerUiProvider() {
52397
- return {
52398
- getStaticDir: async () => ({ staticDir: resolveViewerDistDir() }),
52399
- getVersion: async () => ({ version: readViewerVersion() })
52400
- };
52401
- }
52402
- //#endregion
52403
- //#region src/disk-reconcile-fleet.ts
52404
- async function reconcileFleetFromDisk(deps) {
52405
- const deviceIds = await deps.listDeviceIds();
52406
- const failed = [];
52407
- let cameras = 0;
52408
- let mediaDropped = 0;
52409
- let tracks = 0;
52410
- let events = 0;
52411
- let completed = 0;
52412
- for (const deviceId of deviceIds) {
52413
- try {
52414
- await deps.rescanRecordings(deviceId);
52415
- const counts = await deps.reconcileAnalytics(deviceId);
52416
- cameras += 1;
52417
- mediaDropped += counts.mediaDropped;
52418
- tracks += counts.tracks;
52419
- events += counts.events;
52420
- } catch {
52421
- failed.push(deviceId);
52422
- }
52423
- completed += 1;
52424
- deps.onProgress?.({
52425
- deviceId,
52426
- total: deviceIds.length,
52427
- completed,
52428
- failed: [...failed],
52429
- mediaDropped,
52430
- tracks,
52431
- events
52432
- });
52433
- }
52434
- return {
52435
- cameras,
52436
- failed,
52437
- mediaDropped,
52438
- tracks,
52439
- events
52440
- };
52441
- }
52442
- //#endregion
52443
- //#region src/disk-reconcile-job.ts
52444
- /**
52445
- * In-memory disk-wins fleet job. The tRPC mutation starts this and returns
52446
- * immediately; the walk runs in the addon process so a 60s UDS timeout cannot
52447
- * abort it. Status is polled via getReconcileFromDiskStatus.
52448
- */
52449
- function idleDiskReconcileJob() {
52450
- return {
52451
- state: "idle",
52452
- total: 0,
52453
- completed: 0,
52454
- currentDeviceId: null,
52455
- failed: [],
52456
- mediaDropped: 0,
52457
- tracks: 0,
52458
- events: 0,
52459
- startedAtMs: null,
52460
- finishedAtMs: null,
52461
- error: null
52462
- };
52463
- }
52464
- function isTimeoutError(err) {
52465
- const message = err instanceof Error ? err.message : String(err);
52466
- return /timed out/i.test(message);
52467
- }
52468
- async function withTimeoutRetry(run) {
52469
- try {
52470
- return await run();
52471
- } catch (err) {
52472
- if (!isTimeoutError(err)) throw err;
52473
- return await run();
52474
- }
52475
- }
52476
- function createDiskReconcileJobRunner(now = Date.now) {
52477
- let job = idleDiskReconcileJob();
52478
- let inFlight = null;
52479
- const snapshot = () => job;
52480
- const start = (deps) => {
52481
- if (job.state === "running" && inFlight) return job;
52482
- job = {
52483
- ...idleDiskReconcileJob(),
52484
- state: "running",
52485
- startedAtMs: now()
52486
- };
52487
- deps.log?.("pipeline disk reconcile started");
52488
- inFlight = (async () => {
52489
- try {
52490
- const result = await reconcileFleetFromDisk({
52491
- listDeviceIds: deps.listDeviceIds,
52492
- rescanRecordings: (deviceId) => withTimeoutRetry(() => deps.rescanRecordings(deviceId)),
52493
- reconcileAnalytics: (deviceId) => withTimeoutRetry(() => deps.reconcileAnalytics(deviceId)),
52494
- onProgress: (update) => {
52495
- job = {
52496
- ...job,
52497
- total: update.total,
52498
- completed: update.completed,
52499
- currentDeviceId: update.deviceId,
52500
- failed: update.failed,
52501
- mediaDropped: update.mediaDropped,
52502
- tracks: update.tracks,
52503
- events: update.events
52504
- };
52505
- deps.onProgress?.(update);
52506
- deps.log?.("pipeline disk reconcile camera", {
52507
- deviceId: update.deviceId,
52508
- completed: update.completed,
52509
- total: update.total,
52510
- failed: update.failed.length,
52511
- mediaDropped: update.mediaDropped,
52512
- tracks: update.tracks,
52513
- events: update.events
52514
- });
52515
- }
52516
- });
52517
- job = {
52518
- ...job,
52519
- state: "done",
52520
- total: result.cameras + result.failed.length,
52521
- completed: result.cameras + result.failed.length,
52522
- currentDeviceId: null,
52523
- failed: result.failed,
52524
- mediaDropped: result.mediaDropped,
52525
- tracks: result.tracks,
52526
- events: result.events,
52527
- finishedAtMs: now(),
52528
- error: null
52529
- };
52530
- deps.log?.("pipeline disk reconcile", {
52531
- cameras: result.cameras,
52532
- failed: result.failed,
52533
- mediaDropped: result.mediaDropped,
52534
- tracks: result.tracks,
52535
- events: result.events
52536
- });
52537
- } catch (err) {
52538
- const error = err instanceof Error ? err.message : String(err);
52539
- job = {
52540
- ...job,
52541
- state: "error",
52542
- currentDeviceId: null,
52543
- finishedAtMs: now(),
52544
- error
52545
- };
52546
- deps.log?.("pipeline disk reconcile failed", { error });
52547
- } finally {
52548
- inFlight = null;
52549
- }
52550
- })();
52551
- return job;
52552
- };
52553
- return {
52554
- snapshot,
52555
- start
52556
- };
52557
- }
52558
- //#endregion
52559
- //#region src/widget-catalog.ts
52560
- var pipelineOrchestratorWidgets = [{
52561
- tab: "device-tab",
52562
- label: "Pipeline Quick Stats",
52563
- preAuth: false,
52564
- kind: "remote",
52565
- remote: {
52566
- remoteName: "addon_pipeline_orchestrator_widgets",
52567
- exposedModule: "./widgets",
52568
- componentKey: "pipeline-quick-stats"
52569
- },
52570
- stableId: "pipeline-quick-stats",
52571
- description: "Phase / Detection FPS / Inference / Active Tracks tile row.",
52572
- icon: "activity",
52573
- bundle: "remoteEntry.js",
52574
- hosts: ["device-tab", "dashboard"],
52575
- requires: {
52576
- deviceContext: true,
52577
- integrationContext: false
52578
- },
52579
- defaultSize: "md",
52580
- allowedSizes: [
52581
- "sm",
52582
- "md",
52583
- "lg"
52584
- ],
52585
- defaultColumns: 6,
52586
- defaultRows: 1
52587
- }, {
52588
- tab: "device-tab",
52589
- label: "Zone Editor",
52590
- preAuth: false,
52591
- kind: "remote",
52592
- remote: {
52593
- remoteName: "addon_pipeline_orchestrator_widgets",
52594
- exposedModule: "./widgets",
52595
- componentKey: "zone-editor"
52596
- },
52597
- stableId: "zone-editor",
52598
- description: "Polygon / tripwire CRUD + per-stage rule editor.",
52599
- icon: "shapes",
52600
- bundle: "remoteEntry.js",
52601
- hosts: ["device-tab"],
52602
- requires: {
52603
- deviceContext: true,
52604
- integrationContext: false
52605
- },
52606
- defaultSize: "xl",
52607
- allowedSizes: ["lg", "xl"],
52608
- defaultColumns: 12,
52609
- defaultRows: 4
52610
- }];
52611
- //#endregion
52612
52753
  //#region src/settings-ui-schemas.ts
52613
52754
  /** Build the addon-level schema sections (cluster roles + crop + balancer + failover). */
52614
52755
  function buildGlobalSettingsSections(options) {
@@ -52973,6 +53114,22 @@ function buildDeviceSettingsSections(nodeOptions) {
52973
53114
  field: "detectionMode",
52974
53115
  notEquals: "disabled"
52975
53116
  }
53117
+ },
53118
+ {
53119
+ key: "activityDetectionFps",
53120
+ type: "slider",
53121
+ label: "Detection FPS while the device is working",
53122
+ min: 1,
53123
+ max: 10,
53124
+ step: 1,
53125
+ default: 1,
53126
+ showValue: true,
53127
+ unit: "fps",
53128
+ description: "A device-activity session lasts as long as the job does — a cleaning run is tens of minutes. This caps the rate for that session only; a session opened by motion keeps the rate above.",
53129
+ showWhen: {
53130
+ field: "motionSources",
53131
+ includes: "device-activity"
53132
+ }
52976
53133
  }
52977
53134
  ]
52978
53135
  },
@@ -53062,6 +53219,99 @@ function deriveRuntimeSettings(config) {
53062
53219
  };
53063
53220
  }
53064
53221
  //#endregion
53222
+ //#region src/viewer-ui-provider.ts
53223
+ /**
53224
+ * viewer-ui provider — the pipeline-orchestrator serves the CamStack viewer web
53225
+ * SPA (mirrors what the now-removed standalone addon-viewer-ui used to do).
53226
+ *
53227
+ * Why here: the orchestrator is a hub bootstrap addon (always installed + baked),
53228
+ * so folding the viewer serving into it avoids a second addon whose only job was
53229
+ * to hold static files. The viewer's Expo web export is COPIED into this addon's
53230
+ * `assets/viewer/` locally by the viewer repo's `scripts/copy-dist-to-orchestrator.js`
53231
+ * (run from a checkout that HAS the `camstack/` submodule + Expo toolchain — the
53232
+ * publish/image CI does not, which is exactly why we copy a pre-built dist rather
53233
+ * than build it here). vite emits only into `dist/`, so `assets/viewer` survives
53234
+ * the addon build; `assets` is in the package `files`, so it ships on
53235
+ * `camstack deploy`. The hub's `main.ts` resolves the `viewer-ui` singleton at
53236
+ * boot and mounts the SPA at `/viewer/camstack`.
53237
+ *
53238
+ * `index.js` runs from `dist/`, so the SPA root is one level up + `assets/viewer`.
53239
+ */
53240
+ var __dirname$1 = node_path.default.dirname((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
53241
+ /** Absolute path to the staged viewer web SPA (`<addon-root>/assets/viewer`). */
53242
+ function resolveViewerDistDir() {
53243
+ return node_path.default.resolve(__dirname$1, "..", "assets", "viewer");
53244
+ }
53245
+ /** Version of the staged viewer, written by the copy script; 'unknown' if absent. */
53246
+ function readViewerVersion() {
53247
+ try {
53248
+ const raw = node_fs.default.readFileSync(node_path.default.join(resolveViewerDistDir(), ".viewer-version"), "utf-8").trim();
53249
+ if (raw) return raw;
53250
+ } catch {}
53251
+ return "unknown";
53252
+ }
53253
+ /** Build the viewer-ui provider. Serves whatever dist was staged into
53254
+ * `assets/viewer`; when nothing is staged the hub's mount cleanly 404s. */
53255
+ function createViewerUiProvider() {
53256
+ return {
53257
+ getStaticDir: async () => ({ staticDir: resolveViewerDistDir() }),
53258
+ getVersion: async () => ({ version: readViewerVersion() })
53259
+ };
53260
+ }
53261
+ //#endregion
53262
+ //#region src/widget-catalog.ts
53263
+ var pipelineOrchestratorWidgets = [{
53264
+ tab: "device-tab",
53265
+ label: "Pipeline Quick Stats",
53266
+ preAuth: false,
53267
+ kind: "remote",
53268
+ remote: {
53269
+ remoteName: "addon_pipeline_orchestrator_widgets",
53270
+ exposedModule: "./widgets",
53271
+ componentKey: "pipeline-quick-stats"
53272
+ },
53273
+ stableId: "pipeline-quick-stats",
53274
+ description: "Phase / Detection FPS / Inference / Active Tracks tile row.",
53275
+ icon: "activity",
53276
+ bundle: "remoteEntry.js",
53277
+ hosts: ["device-tab", "dashboard"],
53278
+ requires: {
53279
+ deviceContext: true,
53280
+ integrationContext: false
53281
+ },
53282
+ defaultSize: "md",
53283
+ allowedSizes: [
53284
+ "sm",
53285
+ "md",
53286
+ "lg"
53287
+ ],
53288
+ defaultColumns: 6,
53289
+ defaultRows: 1
53290
+ }, {
53291
+ tab: "device-tab",
53292
+ label: "Zone Editor",
53293
+ preAuth: false,
53294
+ kind: "remote",
53295
+ remote: {
53296
+ remoteName: "addon_pipeline_orchestrator_widgets",
53297
+ exposedModule: "./widgets",
53298
+ componentKey: "zone-editor"
53299
+ },
53300
+ stableId: "zone-editor",
53301
+ description: "Polygon / tripwire CRUD + per-stage rule editor.",
53302
+ icon: "shapes",
53303
+ bundle: "remoteEntry.js",
53304
+ hosts: ["device-tab"],
53305
+ requires: {
53306
+ deviceContext: true,
53307
+ integrationContext: false
53308
+ },
53309
+ defaultSize: "xl",
53310
+ allowedSizes: ["lg", "xl"],
53311
+ defaultColumns: 12,
53312
+ defaultRows: 4
53313
+ }];
53314
+ //#endregion
53065
53315
  //#region src/index.ts
53066
53316
  /**
53067
53317
  * addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
@@ -53376,6 +53626,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
53376
53626
  isCapActiveForDevice: (deviceId, capName) => this.isCapActiveForDevice(deviceId, capName),
53377
53627
  isAudioAnalysisActive: (deviceId) => this.isAudioAnalysisActive(deviceId),
53378
53628
  deviceHasOnboardMotionCap: (deviceId) => this.deviceHasOnboardMotionCap(deviceId),
53629
+ deviceHasActivitySignalCap: (deviceId) => this.deviceHasActivitySignalCap(deviceId),
53379
53630
  deviceHasNativeObjectDetectionCap: (deviceId) => this.deviceHasNativeObjectDetectionCap(deviceId),
53380
53631
  handleDeviceRegistered: (deviceId) => this.handleDeviceRegistered(deviceId),
53381
53632
  handleDeviceUnregistered: (deviceId) => this.handleDeviceUnregistered(deviceId),
@@ -54562,6 +54813,26 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
54562
54813
  * we err toward the analyzer (never lose detection coverage when the
54563
54814
  * binding lookup hiccups).
54564
54815
  */
54816
+ /**
54817
+ * True when the device's driver registered `recording-signal` — the cap a
54818
+ * device uses to say, itself, that it is working (a robot vacuum cleaning).
54819
+ * Drives the `device-activity` entry in the `motionSources` DEFAULT (D392),
54820
+ * so the source arrives on exactly the devices that can raise it and on no
54821
+ * other camera in the fleet.
54822
+ *
54823
+ * Same failure discipline as `deviceHasOnboardMotionCap`: a binding lookup
54824
+ * that throws answers `false`, which loses the DEFAULT and never the
54825
+ * operator's explicit choice (this is asked only when they pinned nothing).
54826
+ */
54827
+ async deviceHasActivitySignalCap(deviceId) {
54828
+ const api = this.api;
54829
+ if (!api) return false;
54830
+ try {
54831
+ return (await api.deviceManager.getBindings.query({ deviceId })).entries.some((e) => e.kind === "native" && e.capName === "recording-signal");
54832
+ } catch {
54833
+ return false;
54834
+ }
54835
+ }
54565
54836
  async deviceHasOnboardMotionCap(deviceId) {
54566
54837
  const api = this.api;
54567
54838
  if (!api) return false;