@camstack/addon-pipeline-orchestrator 1.2.167 → 1.2.169

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -12876,6 +12876,9 @@ var QueryFilterSchema = object({
12876
12876
  whereBetween: record(string(), tuple([unknown(), unknown()])).optional(),
12877
12877
  /** NULL-safe exclusion: matches rows whose field is NULL OR != the value. */
12878
12878
  whereNot: record(string(), unknown()).optional(),
12879
+ /** NULL-safe exclusion of a SET — `whereNot` for more than one value. An
12880
+ * empty list excludes nothing. See `QueryFilter.whereNotIn`. */
12881
+ whereNotIn: record(string(), array(unknown())).optional(),
12879
12882
  orderBy: object({
12880
12883
  field: string(),
12881
12884
  direction: _enum(["asc", "desc"])
@@ -12896,7 +12899,8 @@ var MutationFilterSchema = object({
12896
12899
  where: record(string(), unknown()).optional(),
12897
12900
  whereIn: record(string(), array(unknown())).optional(),
12898
12901
  whereBetween: record(string(), tuple([unknown(), unknown()])).optional(),
12899
- whereNot: record(string(), unknown()).optional()
12902
+ whereNot: record(string(), unknown()).optional(),
12903
+ whereNotIn: record(string(), array(unknown())).optional()
12900
12904
  });
12901
12905
  /**
12902
12906
  * One scalar an {@link settingsStoreCapability.methods.aggregate} call asks for.
@@ -19610,10 +19614,12 @@ var TrackSourceSchema = _enum([
19610
19614
  * Terminal for the plain `markForTrain` toggle: returning it to `staging` is
19611
19615
  * a deliberate action of the retrain page, not a side effect of a checkbox.
19612
19616
  *
19613
- * There is no `null`. The state is stored `TEXT NOT NULL DEFAULT 'none'` because
19614
- * the store's filter language has only positive equality and `whereIn` no
19615
- * negation, no IS NULL so a NULL would be unselectable by ANY predicate and
19616
- * would make the entire pre-column history immortal in one deploy.
19617
+ * There is no `null`. The state is stored `TEXT NOT NULL DEFAULT 'none'` so that
19618
+ * every row is selectable by a positive predicate. (The filter language has
19619
+ * since grown the NULL-safe `whereNot` / `whereNotIn` and an `IS NULL` reading
19620
+ * of `where: { f: null }`, which is how {@link TrackSourceSchema} can be
19621
+ * filtered on a NULLABLE column — see `excludeSources`. It did not when this
19622
+ * column was designed, and a NOT NULL column is still the better shape.)
19617
19623
  */
19618
19624
  var RetrainStatusSchema = _enum([
19619
19625
  "none",
@@ -20349,7 +20355,9 @@ var RecentTracksQueryInput = object({
20349
20355
  * whose class list is unreadable are kept, and the client's rule stays the
20350
20356
  * exact one.
20351
20357
  */
20352
- classes: array(string()).optional()
20358
+ classes: array(string()).optional(),
20359
+ /** See {@link ExcludeSourcesDoc}. */
20360
+ excludeSources: array(TrackSourceSchema).optional()
20353
20361
  });
20354
20362
  /**
20355
20363
  * A Summary (D359): the per-camera session envelope that references tracks.
@@ -20777,7 +20785,9 @@ DeviceType.Camera, method(object({ deviceId: number() }), array(TrackSchema).rea
20777
20785
  * selected is an operator who has not narrowed anything, and an empty
20778
20786
  * timeline would read as a camera that saw nothing.
20779
20787
  */
20780
- classes: array(string()).optional()
20788
+ classes: array(string()).optional(),
20789
+ /** See {@link ExcludeSourcesDoc}. */
20790
+ excludeSources: array(TrackSourceSchema).optional()
20781
20791
  }), array(TrackSchema).readonly()), method(RecentTracksQueryInput, RecentTracksPageSchema), method(SummariesQueryInput, SummariesPageSchema), method(object({ id: string() }), SummaryDetailSchema.nullable()), method(object({ deviceId: number() }), _void(), {
20782
20792
  kind: "mutation",
20783
20793
  auth: "admin"
@@ -21282,8 +21292,25 @@ var occupancyRecheckFramesField = {
21282
21292
  * (analyzer attaches detected `regions[]`; onboard does not — the
21283
21293
  * camera typically only reports a binary signal plus an optional
21284
21294
  * channel/AI class which lives in dedicated event channels).
21285
- */
21286
- var MotionSourceEnum = _enum(["onboard", "analyzer"]);
21295
+ *
21296
+ * - `onboard` — the camera's firmware said something moved.
21297
+ * - `analyzer` — this runner's frame-diff said so, and attaches `regions[]`.
21298
+ * - `device-activity` — the DEVICE said it is doing its job: the
21299
+ * `recording-signal` LEVEL the same device raises for the recorder
21300
+ * ([D380](../../../../docs/decisions/adr-0380-a-device-decided-recording-is-a-mode-with-no-schedule-seeded-once.md)),
21301
+ * republished as a motion source. It attaches **nothing** — no regions, no
21302
+ * class: the only fact it carries is that the device is active, and a robot
21303
+ * vacuum that is itself the moving object has no region worth sending. It is
21304
+ * a LEVEL, so unlike `onboard` it has a real falling edge, and unlike
21305
+ * `analyzer` it must not open the frame-diff side-channel — the runner's
21306
+ * `handleOnboardMotionAnalyzer` gate is `source === 'onboard'` and stays that
21307
+ * way ([D392](../../../../docs/decisions/adr-0392-a-device-that-says-it-is-working-is-a-motion-source-of-its-own.md)).
21308
+ */
21309
+ var MotionSourceEnum = _enum([
21310
+ "onboard",
21311
+ "analyzer",
21312
+ "device-activity"
21313
+ ]);
21287
21314
  /**
21288
21315
  * List of motion sources active on a camera. Empty array is valid:
21289
21316
  * "no source" — happens for battery cams without firmware motion when
@@ -21547,13 +21574,20 @@ var RunnerCameraDeviceUIFields = [
21547
21574
  type: "multiselect",
21548
21575
  label: "Motion Sources",
21549
21576
  default: ["analyzer"],
21550
- options: [{
21551
- value: "analyzer",
21552
- label: "Frame-diff Analyzer (motion addon)"
21553
- }, {
21554
- value: "onboard",
21555
- label: "Camera Onboard Sensor"
21556
- }]
21577
+ options: [
21578
+ {
21579
+ value: "analyzer",
21580
+ label: "Frame-diff Analyzer (motion addon)"
21581
+ },
21582
+ {
21583
+ value: "onboard",
21584
+ label: "Camera Onboard Sensor"
21585
+ },
21586
+ {
21587
+ value: "device-activity",
21588
+ label: "Device activity (the device says it is working)"
21589
+ }
21590
+ ]
21557
21591
  },
21558
21592
  {
21559
21593
  key: "motionFps",
@@ -29550,8 +29584,38 @@ var RecordingSignalStatusSchema = object({
29550
29584
  /** Ms epoch of the last `active` transition. 0 if never observed. */
29551
29585
  lastChangedAt: number()
29552
29586
  });
29553
- RecordingSignalStatusSchema.extend({ lastFetchedAt: number() });
29554
- DeviceType.Camera, method(object({ deviceId: number() }), RecordingSignalStatusSchema);
29587
+ /** The runtime-state slice: the status plus the clock every slice carries. */
29588
+ var RecordingSignalRuntimeStateSchema = RecordingSignalStatusSchema.extend({ lastFetchedAt: number() });
29589
+ var recordingSignalCapability = {
29590
+ name: "recording-signal",
29591
+ scope: "device",
29592
+ deviceNative: true,
29593
+ mode: "singleton",
29594
+ deviceTypes: [DeviceType.Camera],
29595
+ methods: {
29596
+ /** The current level, straight from the slice the provider keeps fresh. */
29597
+ getStatus: method(object({ deviceId: number() }), RecordingSignalStatusSchema) },
29598
+ status: {
29599
+ schema: RecordingSignalStatusSchema,
29600
+ kind: "push",
29601
+ empty: {
29602
+ active: false,
29603
+ reason: "unknown",
29604
+ lastChangedAt: 0
29605
+ }
29606
+ },
29607
+ runtimeState: RecordingSignalRuntimeStateSchema,
29608
+ /**
29609
+ * Runtime-state durability: **session** — a restored `active: true` from
29610
+ * before a restart is exactly the stale level the recorder's reconcile bound
29611
+ * exists to end, and the provider re-derives the true level on activation
29612
+ * anyway. Nothing is lost by forgetting it; a lie is avoided.
29613
+ *
29614
+ * See `RuntimeStateDurability`. Enforced by
29615
+ * `scripts/check-runtime-state-durability.ts`.
29616
+ */
29617
+ durability: "session"
29618
+ };
29555
29619
  /**
29556
29620
  * scene-monitor — device-scoped reference-region state cap. An operator marks
29557
29621
  * a rect ROI on a camera frame and names one or more states; the engine
@@ -40027,477 +40091,278 @@ function deviceBackendToFormat(backend) {
40027
40091
  return DEVICE_BACKEND_TO_FORMAT[backend] ?? "onnx";
40028
40092
  }
40029
40093
  //#endregion
40030
- //#region src/inference-device-model.ts
40094
+ //#region src/audio-chunk-poller.ts
40031
40095
  /**
40032
- * Per-device default object-detection model + deviceKey parsing for the
40033
- * orchestrator's device-aware `getNodeInferenceDevices` view.
40096
+ * `AudioChunkPoller` the consumer-side poll loop of the decoded audio-chunk
40097
+ * plane (Phase 5 / D9).
40034
40098
  *
40035
- * This DUPLICATES the executor's per-device model resolution (P0-3:
40036
- * `resolveDeviceEngine` + `MODEL_BY_CLASS` + the object-detection step's
40037
- * `defaultModelIdByFormat` in `@camstack/addon-pipeline`). It is duplicated
40038
- * not imported because cross-addon imports are forbidden (the orchestrator
40039
- * and the detection-pipeline are separate addons; only tRPC crosses the
40040
- * boundary). Keep this in sync with `default-detection-model.ts` /
40041
- * `step-definitions.ts` if the executor's defaults change.
40099
+ * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
40100
+ * path. A live callback cannot cross a process boundary; once the `pipeline`
40101
+ * group is dissolved (Task 8) the orchestrator runs in a different process
40102
+ * from the broker, so audio delivery must go over tRPC.
40042
40103
  *
40043
- * The returned ids are honest catalog ids (verified present):
40044
- * - `yolov9m-320-int8` — Intel NPU + iGPU (yolo26 does NOT compile on the NPU)
40045
- * - `yolov9m-320` — Apple ANE (CoreML)
40046
- * - `ssd-mobilenet-v2-coco-edgetpu` — Coral USB Edge TPU (tflite)
40047
- * - `yolo26n` — CPU / CUDA (the object-detection step's universal
40048
- * nano default)
40104
+ * The consumer:
40049
40105
  *
40050
- * Do not promote the accelerated ids to 640. The evaluation in
40051
- * `docs/benchmarks/pipeline-frame-model-eval.md` failed the 640 promotion
40052
- * gates (0/3 miss recovered at the current threshold).
40106
+ * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC the broker
40107
+ * registers a per-subscription bounded FIFO queue and returns a
40108
+ * `subscriptionId`;
40109
+ * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
40110
+ * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
40111
+ * 3. feeds each chunk to its downstream audio logic;
40112
+ * 4. on teardown, `unsubscribeAudioChunks`.
40113
+ *
40114
+ * Audio is not latency-critical like video, and chunks arrive only ~every
40115
+ * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
40116
+ * a small per-poll burst keeps latency low without busy-spinning. The
40117
+ * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
40118
+ * loses a chunk.
40119
+ *
40120
+ * Boot-race tolerance: the broker for a given camStream may not be registered
40121
+ * yet when the orchestrator wires the subscription (provider addons publish
40122
+ * their cameraStreams asynchronously after their probe completes).
40123
+ * `subscribeAudioChunks` retries with exponential backoff (capped at
40124
+ * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
40125
+ * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
40126
+ * shape so video and audio plumbing self-heal identically.
40053
40127
  */
40128
+ /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
40129
+ var POLL_INTERVAL_MS$1 = 200;
40130
+ /** How many chunks to drain per poll — a small burst absorbs jitter. */
40131
+ var PULL_MAX_COUNT = 8;
40054
40132
  /**
40055
- * The always-on object-detection ROOT step id. A camera session's tracks all
40056
- * originate from this detector, so a device whose engine format can't run it
40057
- * cannot host a camera root. Mirrors the addon-pipeline step id (cross-addon
40058
- * import is forbidden this is the same duplication rationale as the model
40059
- * defaults above).
40133
+ * Consecutive pull failures before we attempt to re-subscribe. A single failed
40134
+ * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
40135
+ * sustained failure means the broker child restarted and dropped our
40136
+ * subscription, so we re-establish it.
40060
40137
  */
40061
- var OBJECT_DETECTION_STEP_ID = "object-detection";
40138
+ var RESUBSCRIBE_AFTER_FAILURES = 2;
40139
+ /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
40140
+ var RESUBSCRIBE_THROTTLE_TICKS = 5;
40141
+ /** First subscribe-retry delay, doubled on every subsequent failure. */
40142
+ var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
40062
40143
  /**
40063
- * Build the camera-root capability predicate for a node from its live catalog:
40064
- * `format canHostCameraRoot`. A format can host a camera root iff the
40065
- * catalog lists at least one object-detection model with a build for that
40066
- * format — byte-for-byte the resolver's per-device skip-gate test for the root
40067
- * step (`addonHasCompatibleModel`), so a device is deemed eligible iff the root
40068
- * would ACTUALLY provision on it.
40069
- *
40070
- * Fails OPEN when the catalog has no object-detection slot at all (never
40071
- * observed in production) so a malformed/empty catalog never strands every
40072
- * device off the balancer. Pure + deterministic.
40144
+ * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller fast
40145
+ * enough to recover within a single reconcile of the orchestrator and slow
40146
+ * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
40073
40147
  */
40074
- function makeRootCapabilityGuard(catalog) {
40075
- for (const slot of catalog.slots) {
40076
- const objDet = slot.addons.find((a) => a.id === OBJECT_DETECTION_STEP_ID);
40077
- if (objDet) return (format) => objDet.models.some((m) => Boolean(m.formats[format]));
40078
- }
40079
- return () => true;
40080
- }
40081
- /** Split a deviceKey (`<backend>:<device>`, or bare `cpu`) into its parts + format.
40082
- * Format comes from the shared {@link deviceBackendToFormat} SSOT (`@camstack/types`)
40083
- * — the previously-local `BACKEND_FORMAT` copy is gone (R3/node-F2). Used only for
40084
- * STORED-ONLY keys (a configured device the live probe didn't return); a probed
40085
- * device carries its own honest `format` from the descriptor. */
40086
- function parseDeviceKey(deviceKey) {
40087
- const colon = deviceKey.indexOf(":");
40088
- const backend = colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
40089
- return {
40090
- backend,
40091
- device: colon >= 0 ? deviceKey.slice(colon + 1) : deviceKey,
40092
- format: deviceBackendToFormat(backend)
40093
- };
40094
- }
40148
+ var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
40095
40149
  /**
40096
- * The object-detection model the executor defaults to for a deviceKey. Mirrors
40097
- * the executor's `MODEL_BY_CLASS` classification (`classifyAccelerator`) plus
40098
- * the tflite `defaultModelIdByFormat` for Coral. Never throws; unknown backends
40099
- * fall back to the universal nano default (`yolo26n`).
40150
+ * Attempts after which a still-failing subscribe escalates from the fast 5 s
40151
+ * ceiling to {@link LONG_SUBSCRIBE_RETRY_BACKOFF_MS}. ~15 attempts ≈ one
40152
+ * minute of fast retries — plenty for the boot races the 5 s ceiling exists
40153
+ * for. A broker that is STILL absent after that is a long-lived condition
40154
+ * (e.g. a DISABLED camera, whose broker the stream-broker reconcile refuses
40155
+ * to recreate — 2026-07-23) and retrying every 5 s forever is pure log/RPC
40156
+ * churn. The slow loop stays alive so audio still recovers automatically
40157
+ * (≤60 s) once the camera is re-enabled.
40100
40158
  */
40101
- function defaultModelIdForDevice(deviceKey) {
40102
- const { backend, device } = parseDeviceKey(deviceKey);
40103
- if (backend === "openvino") {
40104
- if (device === "cpu") return "yolo26n";
40105
- return "yolov9m-320-int8";
40106
- }
40107
- if (backend === "edgetpu") return "ssd-mobilenet-v2-coco-edgetpu";
40108
- if (backend === "coreml") return "yolov9m-320";
40109
- return "yolo26n";
40110
- }
40159
+ var PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS = 15;
40160
+ var LONG_SUBSCRIBE_RETRY_BACKOFF_MS = 6e4;
40111
40161
  /**
40112
- * Step-tree device jump (phase 1): validate every `steps[step].jumpDeviceKey`
40113
- * manual override in a to-be-saved `inferenceDevices` map. A jump target MUST be
40114
- * an enabled∧available device on the SAME node and DIFFERENT from the owning
40115
- * device. `enabledAvailableKeys` is the effective enabled∧available set (from
40116
- * `mergeInferenceDevices(probe, submitted)`) so an absent/unplugged/disabled
40117
- * target is rejected honestly (an operator can't route a step onto a dead pool).
40118
- * Returns the FIRST human-readable error, or `null` when every override is
40119
- * valid. Pure + deterministic.
40162
+ * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
40163
+ *
40164
+ * Always resolves to a teardown closure when the broker is not yet
40165
+ * registered the closure cancels the ongoing retry loop; when polling is
40166
+ * active it stops the loop and releases the broker subscription. Mirrors
40167
+ * `startFrameHandlePoller` so video and audio recover identically.
40120
40168
  */
40121
- function validateJumpTargets(inferenceDevices, enabledAvailableKeys) {
40122
- for (const [deviceKey, entry] of Object.entries(inferenceDevices)) for (const [stepId, step] of Object.entries(entry.steps ?? {})) {
40123
- const target = step.jumpDeviceKey;
40124
- if (target === void 0) continue;
40125
- if (target === deviceKey) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey pointing at its own device`;
40126
- if (!enabledAvailableKeys.has(target)) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey "${target}" that is not an enabled, available device on this node`;
40127
- }
40128
- return null;
40169
+ function startAudioChunkPoller(options) {
40170
+ const lifecycle = {
40171
+ stopped: false,
40172
+ retryTimer: void 0,
40173
+ pollTimer: void 0,
40174
+ activeSubscriptionId: null
40175
+ };
40176
+ const teardown = () => {
40177
+ if (lifecycle.stopped) return;
40178
+ lifecycle.stopped = true;
40179
+ if (lifecycle.retryTimer) {
40180
+ clearTimeout(lifecycle.retryTimer);
40181
+ lifecycle.retryTimer = void 0;
40182
+ }
40183
+ if (lifecycle.pollTimer) {
40184
+ clearTimeout(lifecycle.pollTimer);
40185
+ lifecycle.pollTimer = void 0;
40186
+ }
40187
+ const subId = lifecycle.activeSubscriptionId;
40188
+ if (subId) {
40189
+ lifecycle.activeSubscriptionId = null;
40190
+ options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }, nodePin(options.ownerNodeId)).catch((err) => {
40191
+ options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
40192
+ brokerId: options.brokerId,
40193
+ subscriptionId: subId,
40194
+ error: errMsg(err)
40195
+ } });
40196
+ });
40197
+ }
40198
+ };
40199
+ subscribeWithRetry(options, lifecycle);
40200
+ return teardown;
40129
40201
  }
40130
40202
  /**
40131
- * Merge a node's live-probed inference devices with its stored per-device map.
40132
- *
40133
- * The default is **AUTO = all discovered ACCELERATORS enabled** (spec C2,
40134
- * opt-OUT) with TWO deliberate exceptions, both **opt-IN** (default disabled):
40135
- *
40136
- * - **CPU**: `enumerateInferenceDevices` always emits a universal `cpu`
40137
- * floor on every platform; auto-enabling it would let the balancer
40138
- * round-robin ~1/N of sessions onto the slow CPU pool alongside the
40139
- * NPU/iGPU/ANE. CPU stays the always-available FALLBACK (a node with no
40140
- * eligible accelerator leaves `deviceKey` unset → the runner's default
40141
- * pool, which is CPU), not a balanced target — matching the spec's "no
40142
- * device eligible → fall back to CPU".
40143
- * - **Coral Edge TPU (`edgetpu`)**: the standing rule since the Coral
40144
- * executor landed is that it surfaces as selectable but is NEVER
40145
- * auto-picked — it runs a DIFFERENT, weaker model family (tflite SSD
40146
- * MobileNet, not the YOLO the other accelerators run), so silently
40147
- * enrolling a plugged-in Coral changes detection QUALITY, not just
40148
- * placement. The opt-OUT default did exactly that on 2026-08-01: a hub
40149
- * Coral nobody enabled entered the session rotation and camera 615 spent
40150
- * hours at 2.4fps failing tflite model resolution. An operator who wants
40151
- * the Coral balanced opts it in explicitly (`enabled: true`).
40152
- *
40153
- * So: an NPU/iGPU/ANE accelerator with NO stored entry is `enabled:true`; a
40154
- * CPU or edgetpu device with no stored entry is `enabled:false`; an explicit
40155
- * stored `enabled` always wins (an operator can opt CPU/Coral in, or an
40156
- * accelerator out). A stored-only key (configured but the probe did not
40157
- * return it — removed/unplugged HW) keeps its stored `enabled` and surfaces
40158
- * as `available:false`, so the UI still shows it.
40159
- *
40160
- * Pure + deterministic (sorted by key) — the single merge authority shared by
40161
- * the `getNodeInferenceDevices` view and the dispatcher's eligible-device pick.
40203
+ * Run the subscribe poll handshake with exponential backoff on subscribe
40204
+ * failures. Resolves once the subscription is acquired (and the poll loop has
40205
+ * been started) or once `lifecycle.stopped` flips, whichever comes first.
40162
40206
  */
40163
- function mergeInferenceDevices(probed, stored) {
40164
- const probedByKey = new Map(probed.map((d) => [d.key, d]));
40165
- const keys = new Set([...probedByKey.keys(), ...Object.keys(stored)]);
40166
- const out = [];
40167
- for (const key of Array.from(keys).toSorted()) {
40168
- const descriptor = probedByKey.get(key);
40169
- const opt = stored[key];
40170
- const parsed = descriptor ?? parseDeviceKey(key);
40171
- const weight = opt?.weight !== void 0 && opt.weight > 0 ? opt.weight : 1;
40172
- const autoDefault = parsed.backend !== "cpu" && parsed.backend !== "edgetpu";
40173
- out.push({
40174
- key,
40175
- backend: parsed.backend,
40176
- device: parsed.device,
40177
- format: parsed.format,
40178
- available: descriptor?.available ?? false,
40179
- enabled: opt?.enabled ?? autoDefault,
40180
- weight,
40181
- maxSessions: opt?.maxSessions ?? null,
40182
- defaultModelId: defaultModelIdForDevice(key),
40183
- ...opt?.steps && Object.keys(opt.steps).length > 0 ? { steps: { ...opt.steps } } : {}
40184
- });
40207
+ async function subscribeWithRetry(options, lifecycle) {
40208
+ const { api, brokerId, tag, ownerNodeId, logger } = options;
40209
+ const pin = nodePin(ownerNodeId);
40210
+ let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
40211
+ let attempt = 0;
40212
+ while (!lifecycle.stopped) {
40213
+ attempt += 1;
40214
+ try {
40215
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
40216
+ brokerId,
40217
+ tag
40218
+ }, pin);
40219
+ if (lifecycle.stopped) {
40220
+ await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }, pin).catch((err) => {
40221
+ logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
40222
+ brokerId,
40223
+ subscriptionId: result.subscriptionId,
40224
+ error: errMsg(err)
40225
+ } });
40226
+ });
40227
+ return;
40228
+ }
40229
+ if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
40230
+ brokerId,
40231
+ tag,
40232
+ attempt
40233
+ } });
40234
+ lifecycle.activeSubscriptionId = result.subscriptionId;
40235
+ startPolling(options, lifecycle);
40236
+ return;
40237
+ } catch (err) {
40238
+ if (lifecycle.stopped) return;
40239
+ if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
40240
+ brokerId,
40241
+ tag,
40242
+ error: errMsg(err),
40243
+ nextRetryInMs: backoffMs
40244
+ } });
40245
+ else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
40246
+ brokerId,
40247
+ tag,
40248
+ attempt,
40249
+ error: errMsg(err),
40250
+ nextRetryInMs: backoffMs
40251
+ } });
40252
+ await sleep(backoffMs, lifecycle);
40253
+ backoffMs = Math.min(attempt >= PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS ? LONG_SUBSCRIBE_RETRY_BACKOFF_MS : MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
40254
+ }
40185
40255
  }
40186
- return out;
40187
40256
  }
40188
40257
  /**
40189
- * The per-device concurrent-session caps for a node as `deviceKey → maxSessions`
40190
- * (only devices that carry an explicit cap; absent = unlimited). Fed to the
40191
- * device balancer's `nodeCaps` so a device at its cap is skipped (audit F3).
40258
+ * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
40259
+ * trigger a fresh `subscribeAudioChunks` via the recovery branch covers
40260
+ * the broker child restart case where our `subscriptionId` is silently
40261
+ * disowned.
40192
40262
  */
40193
- function inferenceDeviceCaps(stored) {
40194
- const out = {};
40195
- for (const [key, entry] of Object.entries(stored)) if (entry.maxSessions !== void 0 && entry.maxSessions > 0) out[key] = entry.maxSessions;
40196
- return out;
40197
- }
40198
- /** Is this device the CPU fallback rather than a real accelerator? */
40199
- function isCpuFallback(view) {
40200
- return view.backend === "cpu";
40201
- }
40202
- function resolveInferenceDeviceEligibility(probed, stored, canRunRoot, isPoolUsable) {
40203
- const eligible = {};
40204
- const excluded = [];
40205
- const merged = mergeInferenceDevices(probed, stored);
40206
- const acceleratorServes = merged.some((d) => !isCpuFallback(d) && d.enabled && d.available && (!canRunRoot || canRunRoot(d.format)));
40207
- for (const d of merged) {
40208
- if (!d.enabled) {
40209
- excluded.push({
40210
- key: d.key,
40211
- reason: "disabled",
40212
- format: d.format
40213
- });
40214
- continue;
40215
- }
40216
- if (!d.available) {
40217
- excluded.push({
40218
- key: d.key,
40219
- reason: "unavailable",
40220
- format: d.format
40221
- });
40222
- continue;
40223
- }
40224
- if (isPoolUsable && !isPoolUsable(d.key)) {
40225
- excluded.push({
40226
- key: d.key,
40227
- reason: "unavailable",
40228
- format: d.format
40229
- });
40230
- continue;
40231
- }
40232
- if (canRunRoot && !canRunRoot(d.format)) {
40233
- excluded.push({
40234
- key: d.key,
40235
- reason: "cannot-host-camera-root",
40236
- format: d.format
40237
- });
40238
- continue;
40263
+ function startPolling(options, lifecycle) {
40264
+ const { api, brokerId, tag, ownerNodeId, onChunk, logger } = options;
40265
+ const pin = nodePin(ownerNodeId);
40266
+ let consecutiveFailures = 0;
40267
+ const resubscribe = async () => {
40268
+ try {
40269
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
40270
+ brokerId,
40271
+ tag
40272
+ }, pin);
40273
+ lifecycle.activeSubscriptionId = result.subscriptionId;
40274
+ logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
40275
+ brokerId,
40276
+ tag,
40277
+ subscriptionId: result.subscriptionId,
40278
+ afterFailures: consecutiveFailures
40279
+ } });
40280
+ return true;
40281
+ } catch {
40282
+ return false;
40239
40283
  }
40240
- if (isCpuFallback(d) && acceleratorServes) {
40241
- excluded.push({
40242
- key: d.key,
40243
- reason: "accelerator-preferred",
40244
- format: d.format
40245
- });
40246
- continue;
40284
+ };
40285
+ const tick = async () => {
40286
+ if (lifecycle.stopped) return;
40287
+ const subId = lifecycle.activeSubscriptionId;
40288
+ if (!subId) return;
40289
+ try {
40290
+ const chunks = await api.streamBroker.pullAudioChunks.query({
40291
+ subscriptionId: subId,
40292
+ maxCount: PULL_MAX_COUNT
40293
+ }, pin);
40294
+ if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
40295
+ brokerId,
40296
+ subscriptionId: subId
40297
+ } });
40298
+ consecutiveFailures = 0;
40299
+ for (const chunk of chunks) {
40300
+ if (lifecycle.stopped) break;
40301
+ await onChunk(chunk);
40302
+ }
40303
+ } catch (err) {
40304
+ consecutiveFailures += 1;
40305
+ if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
40306
+ brokerId,
40307
+ subscriptionId: subId,
40308
+ error: errMsg(err)
40309
+ } });
40310
+ if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
40247
40311
  }
40248
- eligible[d.key] = d.weight;
40249
- }
40250
- return {
40251
- eligible,
40252
- excluded
40312
+ if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS$1);
40253
40313
  };
40314
+ tick();
40254
40315
  }
40255
40316
  /**
40256
- * Join the merged device rows with the eligibility verdict so the UI can NAME
40257
- * why an accelerator is not in play instead of leaving the operator to deduce
40258
- * it from `enabled`/`available`.
40259
- *
40260
- * Deduction is not possible for two of the four reasons — `accelerator-preferred`
40261
- * is a node-WIDE rule (a CPU row reads `enabled:true, available:true` and still
40262
- * never gets a session, D215) and `cannot-host-camera-root` needs the node's
40263
- * model catalog. Both live in {@link resolveInferenceDeviceEligibility}, so this
40264
- * function only transports its answer; it never re-derives one.
40265
- *
40266
- * Pure; preserves `merged`'s order (sorted by key) and every other field.
40317
+ * Cancellable sleep wakes early when `lifecycle.stopped` flips. We
40318
+ * keep a local wrapper around the shared {@link sleep} helper because
40319
+ * the lifecycle tracks the active retry timer for `teardown()` to
40320
+ * clear; pure `sleep()` would leak the timer if teardown fired while
40321
+ * we were waiting.
40267
40322
  */
40268
- function annotateInferenceDeviceExclusions(merged, eligibility) {
40269
- const reasonByKey = new Map(eligibility.excluded.map((e) => [e.key, e.reason]));
40270
- return merged.map((view) => ({
40271
- ...view,
40272
- exclusion: reasonByKey.get(view.key) ?? null
40273
- }));
40323
+ function sleep(ms, lifecycle) {
40324
+ return new Promise((resolve) => {
40325
+ if (lifecycle.stopped) {
40326
+ resolve();
40327
+ return;
40328
+ }
40329
+ lifecycle.retryTimer = setTimeout(() => {
40330
+ lifecycle.retryTimer = void 0;
40331
+ resolve();
40332
+ }, ms);
40333
+ });
40274
40334
  }
40275
- function resolveNodeInferenceUsability(eligibility) {
40276
- const eligibleKeys = Object.keys(eligibility.eligible).toSorted();
40277
- const unavailableKeys = eligibility.excluded.filter((e) => e.reason === "unavailable").map((e) => e.key).toSorted();
40335
+ //#endregion
40336
+ //#region src/audio-load-balancer.ts
40337
+ function balanceAudio(input) {
40338
+ if (input.nodes.length === 0) return null;
40339
+ if (input.preferredNode) {
40340
+ const pinned = input.nodes.find((n) => n.nodeId === input.preferredNode);
40341
+ if (pinned) return {
40342
+ nodeId: pinned.nodeId,
40343
+ reason: "manual"
40344
+ };
40345
+ }
40278
40346
  return {
40279
- usable: eligibleKeys.length > 0 || unavailableKeys.length === 0,
40280
- unavailableKeys,
40281
- eligibleKeys
40347
+ nodeId: input.nodes.slice().toSorted((a, b) => a.deviceCount - b.deviceCount)[0].nodeId,
40348
+ reason: "capacity"
40282
40349
  };
40283
40350
  }
40351
+ //#endregion
40352
+ //#region src/orchestrator-types.ts
40353
+ var PHASE_MODE_VALUES = new Set([
40354
+ "disabled",
40355
+ "always-on",
40356
+ "on-motion"
40357
+ ]);
40358
+ function isPipelinePhaseMode(v) {
40359
+ return PHASE_MODE_VALUES.has(v);
40360
+ }
40284
40361
  /**
40285
- * Step-tree device jump (phase 1): the attach-payload roster of a node's
40286
- * enabled∧available inference devices with the balancer knobs (`weight`,
40287
- * `maxSessions`) the runner uses to AUTO-jump an enrichment step off a device
40288
- * whose format can't run it. Built from the SAME `eligible` (deviceKey→weight)
40289
- * and `caps` (deviceKey→maxSessions) the dispatcher already computes, so the
40290
- * roster the runner sees exactly matches the balancer's candidate set. Sorted
40291
- * by key for determinism. Populated onto `RunnerCameraConfig.inferenceDevices`
40292
- * ONLY when a `deviceKey` is elected and there are ≥2 entries.
40293
- */
40294
- function buildInferenceDeviceRoster(eligible, caps) {
40295
- return Object.entries(eligible).map(([deviceKey, weight]) => ({
40296
- deviceKey,
40297
- weight: weight > 0 ? weight : 1,
40298
- maxSessions: caps[deviceKey] ?? null
40299
- })).toSorted((a, b) => a.deviceKey < b.deviceKey ? -1 : a.deviceKey > b.deviceKey ? 1 : 0);
40300
- }
40301
- //#endregion
40302
- //#region src/node-inference-usability-mirror.ts
40303
- var NodeInferenceUsabilityMirror = class {
40304
- state = /* @__PURE__ */ new Map();
40305
- /**
40306
- * Fold one observation in and report whether the caller should act.
40307
- * Never throws.
40308
- */
40309
- observe(nodeId, usable) {
40310
- const prev = this.state.get(nodeId);
40311
- if (usable) {
40312
- this.state.set(nodeId, {
40313
- usable: true,
40314
- armed: false
40315
- });
40316
- return prev !== void 0 && !prev.usable ? "recovered" : null;
40317
- }
40318
- if (prev === void 0) {
40319
- this.state.set(nodeId, {
40320
- usable: true,
40321
- armed: true
40322
- });
40323
- return null;
40324
- }
40325
- if (!prev.usable) {
40326
- this.state.set(nodeId, {
40327
- usable: false,
40328
- armed: true
40329
- });
40330
- return null;
40331
- }
40332
- if (!prev.armed) {
40333
- this.state.set(nodeId, {
40334
- usable: true,
40335
- armed: true
40336
- });
40337
- return null;
40338
- }
40339
- this.state.set(nodeId, {
40340
- usable: false,
40341
- armed: true
40342
- });
40343
- return "became-unusable";
40344
- }
40345
- /** Can this node be given cameras? Unknown nodes answer YES. */
40346
- isUsable(nodeId) {
40347
- return this.state.get(nodeId)?.usable ?? true;
40348
- }
40349
- /** Nodes currently excluded — for the placement log and diagnostics. */
40350
- unusableNodeIds() {
40351
- const out = [];
40352
- for (const [nodeId, s] of this.state) if (!s.usable) out.push(nodeId);
40353
- return out.toSorted();
40354
- }
40355
- forget(nodeId) {
40356
- this.state.delete(nodeId);
40357
- }
40358
- reset() {
40359
- this.state.clear();
40360
- }
40361
- };
40362
- //#endregion
40363
- //#region src/inference-device-usability-mirror.ts
40364
- /**
40365
- * InferenceDeviceUsabilityMirror — "can this NODE's THIS DEVICE still infer?",
40366
- * kept off the placement path.
40367
- *
40368
- * The per-`(nodeId, deviceKey)` twin of {@link NodeInferenceUsabilityMirror},
40369
- * and deliberately not a second mechanism: it composes the key and delegates
40370
- * every decision to that class, so the arm/apply reluctance D49 pinned lives in
40371
- * exactly one implementation and cannot drift between the node tier and the
40372
- * device tier.
40373
- *
40374
- * ## Why this tier had to exist
40375
- *
40376
- * The node tier already answers "does this node have ANY usable accelerator".
40377
- * On 2026-08-25 the hub's answer was, correctly, yes — its NPU was healthy the
40378
- * whole time. What died was one DEVICE, `openvino:gpu`, and nothing anywhere
40379
- * asked that question: the per-dispatch capability gate is keyed on model
40380
- * FORMAT and `openvino:gpu` and `openvino:npu` share the format `openvino`, so
40381
- * it is blind between them by construction. The balancer kept rotating cameras
40382
- * onto the dead pool — 20 sessions in 20 minutes, every one `tieBreak:
40383
- * "rotation"` — for 31 hours.
40384
- *
40385
- * ## Why a mirror and not the event
40386
- *
40387
- * D49, verbatim: *a one-shot event never gates on a fallible read*. The gate is
40388
- * this in-memory mirror, refreshed off the event path by the same
40389
- * `resolveEligibleInferenceDevices` the dispatcher already runs (plus the
40390
- * session controller's background refresher). The consequences that buys:
40391
- *
40392
- * - **A read that fails changes nothing.** The caller folds in an observation
40393
- * only when it HAS one; an unreachable node, a version-skewed executor or a
40394
- * rejected RPC never reaches {@link observe}, so the previous verdict
40395
- * stands. This is the whole reason the health read is specified as
40396
- * "synchronous over in-memory state, never throws for its own reasons": an
40397
- * empty answer must mean *nothing is refused*, not *I could not tell*.
40398
- * - **Excluding a healthy device needs a second, consecutive read.** Exclusion
40399
- * is the direction that DESTROYS work — it strands an accelerator that may
40400
- * be perfectly fine — so one bad observation only ARMS.
40401
- * - **Re-admitting is immediate and unconditional.** One good observation puts
40402
- * the device straight back. Being slow to exclude costs some wasted
40403
- * inference attempts; being slow to re-admit costs an idle accelerator and a
40404
- * node that looks broken.
40405
- */
40406
- /**
40407
- * NUL — impossible in both a Moleculer node id and a `<backend>:<device>` key,
40408
- * so the composite key can never be ambiguous. A separator that CAN occur in
40409
- * either half makes two distinct pairs collide, and a collision here silently
40410
- * excludes an accelerator nobody reported.
40411
- */
40412
- var SEPARATOR = "\0";
40413
- var InferenceDeviceUsabilityMirror = class {
40414
- /** The one implementation of the arm/apply state machine (D49). */
40415
- mirror = new NodeInferenceUsabilityMirror();
40416
- /**
40417
- * Fold one observation in and report whether the caller should act.
40418
- * Never throws.
40419
- */
40420
- observe(nodeId, deviceKey, usable) {
40421
- return this.mirror.observe(`${nodeId}${SEPARATOR}${deviceKey}`, usable);
40422
- }
40423
- /** Can the balancer put a session on this device? Unknown pairs answer YES. */
40424
- isUsable(nodeId, deviceKey) {
40425
- return this.mirror.isUsable(`${nodeId}${SEPARATOR}${deviceKey}`);
40426
- }
40427
- /** Pairs currently excluded — for the placement log and diagnostics. */
40428
- unusableDevices() {
40429
- return this.mirror.unusableNodeIds().map((composite) => {
40430
- const at = composite.indexOf(SEPARATOR);
40431
- return {
40432
- nodeId: composite.slice(0, at),
40433
- deviceKey: composite.slice(at + 1)
40434
- };
40435
- });
40436
- }
40437
- /** The excluded device keys on ONE node. */
40438
- unusableDeviceKeys(nodeId) {
40439
- return this.unusableDevices().filter((d) => d.nodeId === nodeId).map((d) => d.deviceKey);
40440
- }
40441
- forget(nodeId, deviceKey) {
40442
- this.mirror.forget(`${nodeId}${SEPARATOR}${deviceKey}`);
40443
- }
40444
- reset() {
40445
- this.mirror.reset();
40446
- }
40447
- };
40448
- /**
40449
- * Fold ONE node's health answer into the mirror and return what changed.
40450
- *
40451
- * This is the whole reading discipline, in one place, because both halves of it
40452
- * are easy to get subtly wrong and neither failure is visible in a log:
40453
- *
40454
- * - **`unhealthy === null` — the read FAILED — changes nothing.** Not a single
40455
- * entry is touched. An unreachable node, a version-skewed executor or a
40456
- * rejected RPC must be distinguishable from "asked, nothing is refused", or
40457
- * a flaky link silently re-admits a dead accelerator (D49).
40458
- * - **Every device the node HAS is observed**, not merely the refused ones.
40459
- * The first draft observed `refused ∪ already-excluded`, which omits exactly
40460
- * the devices the mirror has ARMED — so their disarming good read never
40461
- * arrived and two bad reads an HOUR apart, with a hundred healthy ones
40462
- * between them, excluded a working accelerator. "Consecutive" is only a
40463
- * property if the good observations are delivered.
40464
- *
40465
- * Pure with respect to everything except `mirror`, and never throws — it is
40466
- * called from the dispatcher's own read path.
40467
- */
40468
- function foldInferenceDeviceHealth(mirror, nodeId, unhealthy, present) {
40469
- if (unhealthy === null) return [];
40470
- const refused = new Set(unhealthy);
40471
- const observed = new Set([
40472
- ...present,
40473
- ...refused,
40474
- ...mirror.unusableDeviceKeys(nodeId)
40475
- ]);
40476
- const changes = [];
40477
- for (const deviceKey of observed) {
40478
- const transition = mirror.observe(nodeId, deviceKey, !refused.has(deviceKey));
40479
- if (transition !== null) changes.push({
40480
- deviceKey,
40481
- transition
40482
- });
40483
- }
40484
- return changes;
40485
- }
40486
- //#endregion
40487
- //#region src/orchestrator-types.ts
40488
- var PHASE_MODE_VALUES = new Set([
40489
- "disabled",
40490
- "always-on",
40491
- "on-motion"
40492
- ]);
40493
- function isPipelinePhaseMode(v) {
40494
- return PHASE_MODE_VALUES.has(v);
40495
- }
40496
- /**
40497
- * Periodic sweep interval for `retryPendingDispatches` — re-dispatches cameras
40498
- * that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
40499
- * `reconcileDispatch` is additive-only and never revisits these, so a slow
40500
- * safety-net timer + event-driven debounce triggers recover them.
40362
+ * Periodic sweep interval for `retryPendingDispatches` re-dispatches cameras
40363
+ * that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
40364
+ * `reconcileDispatch` is additive-only and never revisits these, so a slow
40365
+ * safety-net timer + event-driven debounce triggers recover them.
40501
40366
  */
40502
40367
  var PENDING_RETRY_INTERVAL_MS = 6e4;
40503
40368
  /** Debounce window for `schedulePendingRetry` — coalesces capacity/eligibility/readiness signals. */
@@ -40616,264 +40481,6 @@ var pipelineOrchestratorActions = defineCustomActions({
40616
40481
  */
40617
40482
  var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
40618
40483
  //#endregion
40619
- //#region src/audio-chunk-poller.ts
40620
- /**
40621
- * `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
40622
- * plane (Phase 5 / D9).
40623
- *
40624
- * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
40625
- * path. A live callback cannot cross a process boundary; once the `pipeline`
40626
- * group is dissolved (Task 8) the orchestrator runs in a different process
40627
- * from the broker, so audio delivery must go over tRPC.
40628
- *
40629
- * The consumer:
40630
- *
40631
- * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC — the broker
40632
- * registers a per-subscription bounded FIFO queue and returns a
40633
- * `subscriptionId`;
40634
- * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
40635
- * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
40636
- * 3. feeds each chunk to its downstream audio logic;
40637
- * 4. on teardown, `unsubscribeAudioChunks`.
40638
- *
40639
- * Audio is not latency-critical like video, and chunks arrive only ~every
40640
- * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
40641
- * a small per-poll burst keeps latency low without busy-spinning. The
40642
- * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
40643
- * loses a chunk.
40644
- *
40645
- * Boot-race tolerance: the broker for a given camStream may not be registered
40646
- * yet when the orchestrator wires the subscription (provider addons publish
40647
- * their cameraStreams asynchronously after their probe completes).
40648
- * `subscribeAudioChunks` retries with exponential backoff (capped at
40649
- * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
40650
- * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
40651
- * shape so video and audio plumbing self-heal identically.
40652
- */
40653
- /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
40654
- var POLL_INTERVAL_MS$1 = 200;
40655
- /** How many chunks to drain per poll — a small burst absorbs jitter. */
40656
- var PULL_MAX_COUNT = 8;
40657
- /**
40658
- * Consecutive pull failures before we attempt to re-subscribe. A single failed
40659
- * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
40660
- * sustained failure means the broker child restarted and dropped our
40661
- * subscription, so we re-establish it.
40662
- */
40663
- var RESUBSCRIBE_AFTER_FAILURES = 2;
40664
- /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
40665
- var RESUBSCRIBE_THROTTLE_TICKS = 5;
40666
- /** First subscribe-retry delay, doubled on every subsequent failure. */
40667
- var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
40668
- /**
40669
- * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller — fast
40670
- * enough to recover within a single reconcile of the orchestrator and slow
40671
- * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
40672
- */
40673
- var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
40674
- /**
40675
- * Attempts after which a still-failing subscribe escalates from the fast 5 s
40676
- * ceiling to {@link LONG_SUBSCRIBE_RETRY_BACKOFF_MS}. ~15 attempts ≈ one
40677
- * minute of fast retries — plenty for the boot races the 5 s ceiling exists
40678
- * for. A broker that is STILL absent after that is a long-lived condition
40679
- * (e.g. a DISABLED camera, whose broker the stream-broker reconcile refuses
40680
- * to recreate — 2026-07-23) and retrying every 5 s forever is pure log/RPC
40681
- * churn. The slow loop stays alive so audio still recovers automatically
40682
- * (≤60 s) once the camera is re-enabled.
40683
- */
40684
- var PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS = 15;
40685
- var LONG_SUBSCRIBE_RETRY_BACKOFF_MS = 6e4;
40686
- /**
40687
- * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
40688
- *
40689
- * Always resolves to a teardown closure — when the broker is not yet
40690
- * registered the closure cancels the ongoing retry loop; when polling is
40691
- * active it stops the loop and releases the broker subscription. Mirrors
40692
- * `startFrameHandlePoller` so video and audio recover identically.
40693
- */
40694
- function startAudioChunkPoller(options) {
40695
- const lifecycle = {
40696
- stopped: false,
40697
- retryTimer: void 0,
40698
- pollTimer: void 0,
40699
- activeSubscriptionId: null
40700
- };
40701
- const teardown = () => {
40702
- if (lifecycle.stopped) return;
40703
- lifecycle.stopped = true;
40704
- if (lifecycle.retryTimer) {
40705
- clearTimeout(lifecycle.retryTimer);
40706
- lifecycle.retryTimer = void 0;
40707
- }
40708
- if (lifecycle.pollTimer) {
40709
- clearTimeout(lifecycle.pollTimer);
40710
- lifecycle.pollTimer = void 0;
40711
- }
40712
- const subId = lifecycle.activeSubscriptionId;
40713
- if (subId) {
40714
- lifecycle.activeSubscriptionId = null;
40715
- options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }, nodePin(options.ownerNodeId)).catch((err) => {
40716
- options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
40717
- brokerId: options.brokerId,
40718
- subscriptionId: subId,
40719
- error: errMsg(err)
40720
- } });
40721
- });
40722
- }
40723
- };
40724
- subscribeWithRetry(options, lifecycle);
40725
- return teardown;
40726
- }
40727
- /**
40728
- * Run the subscribe → poll handshake with exponential backoff on subscribe
40729
- * failures. Resolves once the subscription is acquired (and the poll loop has
40730
- * been started) or once `lifecycle.stopped` flips, whichever comes first.
40731
- */
40732
- async function subscribeWithRetry(options, lifecycle) {
40733
- const { api, brokerId, tag, ownerNodeId, logger } = options;
40734
- const pin = nodePin(ownerNodeId);
40735
- let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
40736
- let attempt = 0;
40737
- while (!lifecycle.stopped) {
40738
- attempt += 1;
40739
- try {
40740
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
40741
- brokerId,
40742
- tag
40743
- }, pin);
40744
- if (lifecycle.stopped) {
40745
- await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }, pin).catch((err) => {
40746
- logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
40747
- brokerId,
40748
- subscriptionId: result.subscriptionId,
40749
- error: errMsg(err)
40750
- } });
40751
- });
40752
- return;
40753
- }
40754
- if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
40755
- brokerId,
40756
- tag,
40757
- attempt
40758
- } });
40759
- lifecycle.activeSubscriptionId = result.subscriptionId;
40760
- startPolling(options, lifecycle);
40761
- return;
40762
- } catch (err) {
40763
- if (lifecycle.stopped) return;
40764
- if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
40765
- brokerId,
40766
- tag,
40767
- error: errMsg(err),
40768
- nextRetryInMs: backoffMs
40769
- } });
40770
- else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
40771
- brokerId,
40772
- tag,
40773
- attempt,
40774
- error: errMsg(err),
40775
- nextRetryInMs: backoffMs
40776
- } });
40777
- await sleep(backoffMs, lifecycle);
40778
- backoffMs = Math.min(attempt >= PERSISTENT_SUBSCRIBE_FAILURE_ATTEMPTS ? LONG_SUBSCRIBE_RETRY_BACKOFF_MS : MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
40779
- }
40780
- }
40781
- }
40782
- /**
40783
- * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
40784
- * trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
40785
- * the broker child restart case where our `subscriptionId` is silently
40786
- * disowned.
40787
- */
40788
- function startPolling(options, lifecycle) {
40789
- const { api, brokerId, tag, ownerNodeId, onChunk, logger } = options;
40790
- const pin = nodePin(ownerNodeId);
40791
- let consecutiveFailures = 0;
40792
- const resubscribe = async () => {
40793
- try {
40794
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
40795
- brokerId,
40796
- tag
40797
- }, pin);
40798
- lifecycle.activeSubscriptionId = result.subscriptionId;
40799
- logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
40800
- brokerId,
40801
- tag,
40802
- subscriptionId: result.subscriptionId,
40803
- afterFailures: consecutiveFailures
40804
- } });
40805
- return true;
40806
- } catch {
40807
- return false;
40808
- }
40809
- };
40810
- const tick = async () => {
40811
- if (lifecycle.stopped) return;
40812
- const subId = lifecycle.activeSubscriptionId;
40813
- if (!subId) return;
40814
- try {
40815
- const chunks = await api.streamBroker.pullAudioChunks.query({
40816
- subscriptionId: subId,
40817
- maxCount: PULL_MAX_COUNT
40818
- }, pin);
40819
- if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
40820
- brokerId,
40821
- subscriptionId: subId
40822
- } });
40823
- consecutiveFailures = 0;
40824
- for (const chunk of chunks) {
40825
- if (lifecycle.stopped) break;
40826
- await onChunk(chunk);
40827
- }
40828
- } catch (err) {
40829
- consecutiveFailures += 1;
40830
- if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
40831
- brokerId,
40832
- subscriptionId: subId,
40833
- error: errMsg(err)
40834
- } });
40835
- if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
40836
- }
40837
- if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS$1);
40838
- };
40839
- tick();
40840
- }
40841
- /**
40842
- * Cancellable sleep — wakes early when `lifecycle.stopped` flips. We
40843
- * keep a local wrapper around the shared {@link sleep} helper because
40844
- * the lifecycle tracks the active retry timer for `teardown()` to
40845
- * clear; pure `sleep()` would leak the timer if teardown fired while
40846
- * we were waiting.
40847
- */
40848
- function sleep(ms, lifecycle) {
40849
- return new Promise((resolve) => {
40850
- if (lifecycle.stopped) {
40851
- resolve();
40852
- return;
40853
- }
40854
- lifecycle.retryTimer = setTimeout(() => {
40855
- lifecycle.retryTimer = void 0;
40856
- resolve();
40857
- }, ms);
40858
- });
40859
- }
40860
- //#endregion
40861
- //#region src/audio-load-balancer.ts
40862
- function balanceAudio(input) {
40863
- if (input.nodes.length === 0) return null;
40864
- if (input.preferredNode) {
40865
- const pinned = input.nodes.find((n) => n.nodeId === input.preferredNode);
40866
- if (pinned) return {
40867
- nodeId: pinned.nodeId,
40868
- reason: "manual"
40869
- };
40870
- }
40871
- return {
40872
- nodeId: input.nodes.slice().toSorted((a, b) => a.deviceCount - b.deviceCount)[0].nodeId,
40873
- reason: "capacity"
40874
- };
40875
- }
40876
- //#endregion
40877
40484
  //#region src/audio-window-accumulator.ts
40878
40485
  var AudioWindowAccumulator = class {
40879
40486
  deviceId;
@@ -41377,234 +40984,847 @@ var AudioSubscriptionController = class {
41377
40984
  meta: { error: errMsg(err) }
41378
40985
  });
41379
40986
  });
41380
- }, windowMs);
41381
- this.motionAudioWindowTimers.set(deviceId, timer);
41382
- }
41383
- /** True while a motion-driven audio window is open for this device. */
41384
- isMotionAudioWindowOpen(deviceId) {
41385
- return this.motionAudioWindowTimers.has(deviceId);
41386
- }
41387
- /**
41388
- * Close an on-motion audio window and drop its subscription. Shared by both
41389
- * closers (quiet window elapsed / falling edge cooldown) so they can never
41390
- * disagree about what "closed" means. `reason` is logged so a camera that
41391
- * loses audio can always be told WHY from the per-device log view.
41392
- */
41393
- async closeMotionAudioWindow(deviceId, reason) {
41394
- const pendingWindow = this.motionAudioWindowTimers.get(deviceId);
41395
- if (pendingWindow) {
41396
- clearTimeout(pendingWindow);
41397
- this.motionAudioWindowTimers.delete(deviceId);
40987
+ }, windowMs);
40988
+ this.motionAudioWindowTimers.set(deviceId, timer);
40989
+ }
40990
+ /** True while a motion-driven audio window is open for this device. */
40991
+ isMotionAudioWindowOpen(deviceId) {
40992
+ return this.motionAudioWindowTimers.has(deviceId);
40993
+ }
40994
+ /**
40995
+ * Close an on-motion audio window and drop its subscription. Shared by both
40996
+ * closers (quiet window elapsed / falling edge cooldown) so they can never
40997
+ * disagree about what "closed" means. `reason` is logged so a camera that
40998
+ * loses audio can always be told WHY from the per-device log view.
40999
+ */
41000
+ async closeMotionAudioWindow(deviceId, reason) {
41001
+ const pendingWindow = this.motionAudioWindowTimers.get(deviceId);
41002
+ if (pendingWindow) {
41003
+ clearTimeout(pendingWindow);
41004
+ this.motionAudioWindowTimers.delete(deviceId);
41005
+ }
41006
+ await this.withAudioSubLock(deviceId, async () => {
41007
+ const unsub = this.audioSubscriptions.get(deviceId);
41008
+ if (!unsub) return;
41009
+ try {
41010
+ unsub();
41011
+ } catch {}
41012
+ this.audioSubscriptions.delete(deviceId);
41013
+ this.deps.logger.info("lazy audio: window closed", {
41014
+ tags: { deviceId },
41015
+ meta: { reason }
41016
+ });
41017
+ });
41018
+ }
41019
+ /**
41020
+ * Audio teardown for one device — the audio half of `stopDetection`.
41021
+ * Tears down through the per-device lock so it can't race a concurrent
41022
+ * subscribe (which would re-store a handle this teardown never sees).
41023
+ */
41024
+ async stopForDevice(deviceId) {
41025
+ await this.withAudioSubLock(deviceId, async () => {
41026
+ const unsub = this.audioSubscriptions.get(deviceId);
41027
+ if (unsub) {
41028
+ try {
41029
+ unsub();
41030
+ } catch {}
41031
+ this.audioSubscriptions.delete(deviceId);
41032
+ }
41033
+ });
41034
+ const lazyTimer = this.lazyAudioTeardownTimers.get(deviceId);
41035
+ if (lazyTimer) {
41036
+ clearTimeout(lazyTimer);
41037
+ this.lazyAudioTeardownTimers.delete(deviceId);
41038
+ }
41039
+ const windowTimer = this.motionAudioWindowTimers.get(deviceId);
41040
+ if (windowTimer) {
41041
+ clearTimeout(windowTimer);
41042
+ this.motionAudioWindowTimers.delete(deviceId);
41043
+ }
41044
+ this.audioAssignments.delete(deviceId);
41045
+ }
41046
+ /**
41047
+ * Centralized write into `audioSubscriptions`. If shutdown has begun, the
41048
+ * map has already been (or is about to be) cleared lock-free in
41049
+ * `shutdown()`; storing here would leak a zombie entry whose `unsub` is
41050
+ * never called. So when shutting down we immediately invoke `unsub`
41051
+ * (best-effort, error-swallowed) and DO NOT store. `protected` so
41052
+ * `audio-sub-lock.spec.ts`'s test subclass can assert the shutdown-guard
41053
+ * behavior without casts.
41054
+ */
41055
+ storeAudioSub(deviceId, unsub) {
41056
+ if (this.audioShuttingDown) {
41057
+ try {
41058
+ unsub();
41059
+ } catch {}
41060
+ return;
41061
+ }
41062
+ this.audioSubscriptions.set(deviceId, unsub);
41063
+ }
41064
+ /**
41065
+ * Serialize an audio-subscription critical section per device. `fn` is
41066
+ * chained onto the device's current lock tail, so concurrent calls for the
41067
+ * SAME deviceId run sequentially (FIFO); different deviceIds never block
41068
+ * each other. Thin delegate onto the `audioSubLocks` `KeyedAsyncLock`
41069
+ * instance. `protected` so `audio-sub-lock.spec.ts`'s test subclass can
41070
+ * drive the lock without casts.
41071
+ */
41072
+ withAudioSubLock(deviceId, fn) {
41073
+ return this.audioSubLocks.run(deviceId, fn);
41074
+ }
41075
+ /**
41076
+ * Subscribe to decoded audio chunks for a camera and feed them into the
41077
+ * audio-analyzer. Reads the analyzer's settings via its own
41078
+ * `resolveDeviceSettings(deviceId)` method so the orchestrator does not
41079
+ * touch the audio-analyzer schema field names directly.
41080
+ */
41081
+ async subscribeAudioStream(deviceId, config) {
41082
+ const api = this.deps.api();
41083
+ if (!api) {
41084
+ this.deps.logger.warn("this.ctx.api not available — cannot subscribe audio", { tags: { deviceId } });
41085
+ return null;
41086
+ }
41087
+ if (!await this.deps.isAudioAnalysisActive(deviceId)) return null;
41088
+ if (config.audioMode === "disabled") {
41089
+ this.deps.logger.debug("audio subscribe skipped: audioMode=disabled", { tags: { deviceId } });
41090
+ return null;
41091
+ }
41092
+ if (config.audioMode === "on-motion" && !this.isMotionAudioWindowOpen(deviceId)) {
41093
+ this.deps.logger.info("audio subscribe deferred: audioMode=on-motion, no window open", { tags: { deviceId } });
41094
+ return null;
41095
+ }
41096
+ const audioStream = config.audioStreamId ?? config.motionStreamId;
41097
+ const audioBrokerId = makeSourceBrokerId(deviceId, audioStream);
41098
+ if ((await this.deps.probeAudioTrack(deviceId, audioStream)).kind === "absent") {
41099
+ this.deps.logger.warn("audio subscription REFUSED — this stream carries no audio track", {
41100
+ tags: { deviceId },
41101
+ meta: {
41102
+ camStreamId: audioStream,
41103
+ brokerId: audioBrokerId,
41104
+ selectedBy: config.audioStreamId !== void 0 ? "audioStreamId" : "motionStreamId",
41105
+ hint: "point the camera’s audio at a stream that has an audio track — no stream is substituted automatically"
41106
+ }
41107
+ });
41108
+ return null;
41109
+ }
41110
+ const settings = await api.audioAnalysis.resolveDeviceSettings.query({ deviceId });
41111
+ if (!settings) {
41112
+ this.deps.logger.warn("audio-analysis returned no settings — audio subscription skipped", { tags: { deviceId } });
41113
+ return null;
41114
+ }
41115
+ const audioNodeId = await this.dispatch(deviceId);
41116
+ const isRemoteAudio = audioNodeId !== this.deps.localNodeId();
41117
+ this.deps.logger.info("audio subscription: resolved audio node", {
41118
+ tags: { deviceId },
41119
+ meta: {
41120
+ audioNodeId,
41121
+ isRemote: isRemoteAudio
41122
+ }
41123
+ });
41124
+ const accumulator = new AudioWindowAccumulator(deviceId);
41125
+ const teardown = startAudioChunkPoller({
41126
+ api,
41127
+ brokerId: audioBrokerId,
41128
+ tag: "audio-analyzer",
41129
+ ownerNodeId: this.deps.ingestNode(),
41130
+ logger: this.deps.logger.withTags({ deviceId }),
41131
+ onChunk: async (chunk) => {
41132
+ this.deps.watchdogNote(deviceId, "audio");
41133
+ try {
41134
+ const audioChunkInput = accumulator.push(chunk);
41135
+ if (!audioChunkInput) return;
41136
+ const result = await api.audioAnalyzer.analyseChunk.mutate({
41137
+ chunk: audioChunkInput,
41138
+ settings,
41139
+ ...isRemoteAudio ? { nodeId: audioNodeId } : {}
41140
+ });
41141
+ if (!result) return;
41142
+ const frame = buildAudioResultFrame(deviceId, result);
41143
+ this.deps.eventBus.emit({
41144
+ id: `audio-inference-${deviceId}-${Date.now()}`,
41145
+ timestamp: /* @__PURE__ */ new Date(),
41146
+ source: {
41147
+ type: "device",
41148
+ id: deviceId,
41149
+ nodeId: "hub",
41150
+ addonId: "pipeline-orchestrator",
41151
+ deviceId
41152
+ },
41153
+ category: EventCategory.PipelineAudioInferenceResult,
41154
+ data: {
41155
+ deviceId,
41156
+ frame,
41157
+ nodeId: "hub"
41158
+ }
41159
+ });
41160
+ } catch (err) {
41161
+ const msg = errMsg(err);
41162
+ this.deps.logger.error("Audio analysis failed", {
41163
+ tags: { deviceId },
41164
+ meta: { error: msg }
41165
+ });
41166
+ }
41167
+ }
41168
+ });
41169
+ this.deps.logger.info("Audio stream subscribed", { tags: { deviceId } });
41170
+ return () => {
41171
+ teardown();
41172
+ accumulator.reset();
41173
+ };
41174
+ }
41175
+ /**
41176
+ * Set true at the very start of `onShutdown`, before the audio teardown /
41177
+ * map clears below. Once set, `withAudioSubLock` turns queued/new critical
41178
+ * sections into no-ops and `storeAudioSub` refuses to store, so no
41179
+ * critical section that was in-flight (or queued) when shutdown began can
41180
+ * resurrect a zombie subscription into the cleared `audioSubscriptions`
41181
+ * map. MUST be called before anything else in `onShutdown` that could
41182
+ * race a queued audio critical section (mirrors the original
41183
+ * `this.audioShuttingDown = true` being the very first statement).
41184
+ */
41185
+ beginShutdown() {
41186
+ this.audioShuttingDown = true;
41187
+ }
41188
+ /**
41189
+ * Full audio teardown — combines the former `onShutdown`'s two separate
41190
+ * audio blocks (lazy-teardown-timer clear, then — after several unrelated
41191
+ * session/reconcile/load-shed clears — subscription teardown + lock clear
41192
+ * + assignment-map clears) into one call. Safe to combine: both blocks
41193
+ * are synchronous with no interleaved `await`, and every original
41194
+ * statement between them (`sessionRegistry.clear()`,
41195
+ * `cameraFpsMap.clear()`, `remoteHealthAttempts.clear()`,
41196
+ * `loadShedState.clear()`, `loadShedResumeTimer` cleanup) touches state
41197
+ * fully disjoint from anything audio — so their relative order to each
41198
+ * other is unaffected, and the audio-internal order (timers →
41199
+ * subscriptions → lock → assignment maps) is reproduced exactly.
41200
+ */
41201
+ shutdown() {
41202
+ for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
41203
+ this.lazyAudioTeardownTimers.clear();
41204
+ for (const t of this.motionAudioWindowTimers.values()) clearTimeout(t);
41205
+ this.motionAudioWindowTimers.clear();
41206
+ for (const unsub of this.audioSubscriptions.values()) try {
41207
+ unsub();
41208
+ } catch {}
41209
+ this.audioSubscriptions.clear();
41210
+ this.audioSubLocks.clear();
41211
+ this.audioAssignments.clear();
41212
+ this.readyAudioNodes.clear();
41213
+ }
41214
+ };
41215
+ //#endregion
41216
+ //#region src/disk-reconcile-fleet.ts
41217
+ async function reconcileFleetFromDisk(deps) {
41218
+ const deviceIds = await deps.listDeviceIds();
41219
+ const failed = [];
41220
+ let cameras = 0;
41221
+ let mediaDropped = 0;
41222
+ let tracks = 0;
41223
+ let events = 0;
41224
+ let completed = 0;
41225
+ for (const deviceId of deviceIds) {
41226
+ try {
41227
+ await deps.rescanRecordings(deviceId);
41228
+ const counts = await deps.reconcileAnalytics(deviceId);
41229
+ cameras += 1;
41230
+ mediaDropped += counts.mediaDropped;
41231
+ tracks += counts.tracks;
41232
+ events += counts.events;
41233
+ } catch {
41234
+ failed.push(deviceId);
41235
+ }
41236
+ completed += 1;
41237
+ deps.onProgress?.({
41238
+ deviceId,
41239
+ total: deviceIds.length,
41240
+ completed,
41241
+ failed: [...failed],
41242
+ mediaDropped,
41243
+ tracks,
41244
+ events
41245
+ });
41246
+ }
41247
+ return {
41248
+ cameras,
41249
+ failed,
41250
+ mediaDropped,
41251
+ tracks,
41252
+ events
41253
+ };
41254
+ }
41255
+ //#endregion
41256
+ //#region src/disk-reconcile-job.ts
41257
+ /**
41258
+ * In-memory disk-wins fleet job. The tRPC mutation starts this and returns
41259
+ * immediately; the walk runs in the addon process so a 60s UDS timeout cannot
41260
+ * abort it. Status is polled via getReconcileFromDiskStatus.
41261
+ */
41262
+ function idleDiskReconcileJob() {
41263
+ return {
41264
+ state: "idle",
41265
+ total: 0,
41266
+ completed: 0,
41267
+ currentDeviceId: null,
41268
+ failed: [],
41269
+ mediaDropped: 0,
41270
+ tracks: 0,
41271
+ events: 0,
41272
+ startedAtMs: null,
41273
+ finishedAtMs: null,
41274
+ error: null
41275
+ };
41276
+ }
41277
+ function isTimeoutError(err) {
41278
+ const message = err instanceof Error ? err.message : String(err);
41279
+ return /timed out/i.test(message);
41280
+ }
41281
+ async function withTimeoutRetry(run) {
41282
+ try {
41283
+ return await run();
41284
+ } catch (err) {
41285
+ if (!isTimeoutError(err)) throw err;
41286
+ return await run();
41287
+ }
41288
+ }
41289
+ function createDiskReconcileJobRunner(now = Date.now) {
41290
+ let job = idleDiskReconcileJob();
41291
+ let inFlight = null;
41292
+ const snapshot = () => job;
41293
+ const start = (deps) => {
41294
+ if (job.state === "running" && inFlight) return job;
41295
+ job = {
41296
+ ...idleDiskReconcileJob(),
41297
+ state: "running",
41298
+ startedAtMs: now()
41299
+ };
41300
+ deps.log?.("pipeline disk reconcile started");
41301
+ inFlight = (async () => {
41302
+ try {
41303
+ const result = await reconcileFleetFromDisk({
41304
+ listDeviceIds: deps.listDeviceIds,
41305
+ rescanRecordings: (deviceId) => withTimeoutRetry(() => deps.rescanRecordings(deviceId)),
41306
+ reconcileAnalytics: (deviceId) => withTimeoutRetry(() => deps.reconcileAnalytics(deviceId)),
41307
+ onProgress: (update) => {
41308
+ job = {
41309
+ ...job,
41310
+ total: update.total,
41311
+ completed: update.completed,
41312
+ currentDeviceId: update.deviceId,
41313
+ failed: update.failed,
41314
+ mediaDropped: update.mediaDropped,
41315
+ tracks: update.tracks,
41316
+ events: update.events
41317
+ };
41318
+ deps.onProgress?.(update);
41319
+ deps.log?.("pipeline disk reconcile camera", {
41320
+ deviceId: update.deviceId,
41321
+ completed: update.completed,
41322
+ total: update.total,
41323
+ failed: update.failed.length,
41324
+ mediaDropped: update.mediaDropped,
41325
+ tracks: update.tracks,
41326
+ events: update.events
41327
+ });
41328
+ }
41329
+ });
41330
+ job = {
41331
+ ...job,
41332
+ state: "done",
41333
+ total: result.cameras + result.failed.length,
41334
+ completed: result.cameras + result.failed.length,
41335
+ currentDeviceId: null,
41336
+ failed: result.failed,
41337
+ mediaDropped: result.mediaDropped,
41338
+ tracks: result.tracks,
41339
+ events: result.events,
41340
+ finishedAtMs: now(),
41341
+ error: null
41342
+ };
41343
+ deps.log?.("pipeline disk reconcile", {
41344
+ cameras: result.cameras,
41345
+ failed: result.failed,
41346
+ mediaDropped: result.mediaDropped,
41347
+ tracks: result.tracks,
41348
+ events: result.events
41349
+ });
41350
+ } catch (err) {
41351
+ const error = err instanceof Error ? err.message : String(err);
41352
+ job = {
41353
+ ...job,
41354
+ state: "error",
41355
+ currentDeviceId: null,
41356
+ finishedAtMs: now(),
41357
+ error
41358
+ };
41359
+ deps.log?.("pipeline disk reconcile failed", { error });
41360
+ } finally {
41361
+ inFlight = null;
41362
+ }
41363
+ })();
41364
+ return job;
41365
+ };
41366
+ return {
41367
+ snapshot,
41368
+ start
41369
+ };
41370
+ }
41371
+ //#endregion
41372
+ //#region src/inference-device-model.ts
41373
+ /**
41374
+ * Per-device default object-detection model + deviceKey parsing for the
41375
+ * orchestrator's device-aware `getNodeInferenceDevices` view.
41376
+ *
41377
+ * This DUPLICATES the executor's per-device model resolution (P0-3:
41378
+ * `resolveDeviceEngine` + `MODEL_BY_CLASS` + the object-detection step's
41379
+ * `defaultModelIdByFormat` in `@camstack/addon-pipeline`). It is duplicated —
41380
+ * not imported — because cross-addon imports are forbidden (the orchestrator
41381
+ * and the detection-pipeline are separate addons; only tRPC crosses the
41382
+ * boundary). Keep this in sync with `default-detection-model.ts` /
41383
+ * `step-definitions.ts` if the executor's defaults change.
41384
+ *
41385
+ * The returned ids are honest catalog ids (verified present):
41386
+ * - `yolov9m-320-int8` — Intel NPU + iGPU (yolo26 does NOT compile on the NPU)
41387
+ * - `yolov9m-320` — Apple ANE (CoreML)
41388
+ * - `ssd-mobilenet-v2-coco-edgetpu` — Coral USB Edge TPU (tflite)
41389
+ * - `yolo26n` — CPU / CUDA (the object-detection step's universal
41390
+ * nano default)
41391
+ *
41392
+ * Do not promote the accelerated ids to 640. The evaluation in
41393
+ * `docs/benchmarks/pipeline-frame-model-eval.md` failed the 640 promotion
41394
+ * gates (0/3 miss recovered at the current threshold).
41395
+ */
41396
+ /**
41397
+ * The always-on object-detection ROOT step id. A camera session's tracks all
41398
+ * originate from this detector, so a device whose engine format can't run it
41399
+ * cannot host a camera root. Mirrors the addon-pipeline step id (cross-addon
41400
+ * import is forbidden — this is the same duplication rationale as the model
41401
+ * defaults above).
41402
+ */
41403
+ var OBJECT_DETECTION_STEP_ID = "object-detection";
41404
+ /**
41405
+ * Build the camera-root capability predicate for a node from its live catalog:
41406
+ * `format → canHostCameraRoot`. A format can host a camera root iff the
41407
+ * catalog lists at least one object-detection model with a build for that
41408
+ * format — byte-for-byte the resolver's per-device skip-gate test for the root
41409
+ * step (`addonHasCompatibleModel`), so a device is deemed eligible iff the root
41410
+ * would ACTUALLY provision on it.
41411
+ *
41412
+ * Fails OPEN when the catalog has no object-detection slot at all (never
41413
+ * observed in production) so a malformed/empty catalog never strands every
41414
+ * device off the balancer. Pure + deterministic.
41415
+ */
41416
+ function makeRootCapabilityGuard(catalog) {
41417
+ for (const slot of catalog.slots) {
41418
+ const objDet = slot.addons.find((a) => a.id === OBJECT_DETECTION_STEP_ID);
41419
+ if (objDet) return (format) => objDet.models.some((m) => Boolean(m.formats[format]));
41420
+ }
41421
+ return () => true;
41422
+ }
41423
+ /** Split a deviceKey (`<backend>:<device>`, or bare `cpu`) into its parts + format.
41424
+ * Format comes from the shared {@link deviceBackendToFormat} SSOT (`@camstack/types`)
41425
+ * — the previously-local `BACKEND_FORMAT` copy is gone (R3/node-F2). Used only for
41426
+ * STORED-ONLY keys (a configured device the live probe didn't return); a probed
41427
+ * device carries its own honest `format` from the descriptor. */
41428
+ function parseDeviceKey(deviceKey) {
41429
+ const colon = deviceKey.indexOf(":");
41430
+ const backend = colon >= 0 ? deviceKey.slice(0, colon) : deviceKey;
41431
+ return {
41432
+ backend,
41433
+ device: colon >= 0 ? deviceKey.slice(colon + 1) : deviceKey,
41434
+ format: deviceBackendToFormat(backend)
41435
+ };
41436
+ }
41437
+ /**
41438
+ * The object-detection model the executor defaults to for a deviceKey. Mirrors
41439
+ * the executor's `MODEL_BY_CLASS` classification (`classifyAccelerator`) plus
41440
+ * the tflite `defaultModelIdByFormat` for Coral. Never throws; unknown backends
41441
+ * fall back to the universal nano default (`yolo26n`).
41442
+ */
41443
+ function defaultModelIdForDevice(deviceKey) {
41444
+ const { backend, device } = parseDeviceKey(deviceKey);
41445
+ if (backend === "openvino") {
41446
+ if (device === "cpu") return "yolo26n";
41447
+ return "yolov9m-320-int8";
41448
+ }
41449
+ if (backend === "edgetpu") return "ssd-mobilenet-v2-coco-edgetpu";
41450
+ if (backend === "coreml") return "yolov9m-320";
41451
+ return "yolo26n";
41452
+ }
41453
+ /**
41454
+ * Step-tree device jump (phase 1): validate every `steps[step].jumpDeviceKey`
41455
+ * manual override in a to-be-saved `inferenceDevices` map. A jump target MUST be
41456
+ * an enabled∧available device on the SAME node and DIFFERENT from the owning
41457
+ * device. `enabledAvailableKeys` is the effective enabled∧available set (from
41458
+ * `mergeInferenceDevices(probe, submitted)`) so an absent/unplugged/disabled
41459
+ * target is rejected honestly (an operator can't route a step onto a dead pool).
41460
+ * Returns the FIRST human-readable error, or `null` when every override is
41461
+ * valid. Pure + deterministic.
41462
+ */
41463
+ function validateJumpTargets(inferenceDevices, enabledAvailableKeys) {
41464
+ for (const [deviceKey, entry] of Object.entries(inferenceDevices)) for (const [stepId, step] of Object.entries(entry.steps ?? {})) {
41465
+ const target = step.jumpDeviceKey;
41466
+ if (target === void 0) continue;
41467
+ if (target === deviceKey) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey pointing at its own device`;
41468
+ if (!enabledAvailableKeys.has(target)) return `step "${stepId}" on "${deviceKey}" has a jumpDeviceKey "${target}" that is not an enabled, available device on this node`;
41469
+ }
41470
+ return null;
41471
+ }
41472
+ /**
41473
+ * Merge a node's live-probed inference devices with its stored per-device map.
41474
+ *
41475
+ * The default is **AUTO = all discovered ACCELERATORS enabled** (spec C2,
41476
+ * opt-OUT) with TWO deliberate exceptions, both **opt-IN** (default disabled):
41477
+ *
41478
+ * - **CPU**: `enumerateInferenceDevices` always emits a universal `cpu`
41479
+ * floor on every platform; auto-enabling it would let the balancer
41480
+ * round-robin ~1/N of sessions onto the slow CPU pool alongside the
41481
+ * NPU/iGPU/ANE. CPU stays the always-available FALLBACK (a node with no
41482
+ * eligible accelerator leaves `deviceKey` unset → the runner's default
41483
+ * pool, which is CPU), not a balanced target — matching the spec's "no
41484
+ * device eligible → fall back to CPU".
41485
+ * - **Coral Edge TPU (`edgetpu`)**: the standing rule since the Coral
41486
+ * executor landed is that it surfaces as selectable but is NEVER
41487
+ * auto-picked — it runs a DIFFERENT, weaker model family (tflite SSD
41488
+ * MobileNet, not the YOLO the other accelerators run), so silently
41489
+ * enrolling a plugged-in Coral changes detection QUALITY, not just
41490
+ * placement. The opt-OUT default did exactly that on 2026-08-01: a hub
41491
+ * Coral nobody enabled entered the session rotation and camera 615 spent
41492
+ * hours at 2.4fps failing tflite model resolution. An operator who wants
41493
+ * the Coral balanced opts it in explicitly (`enabled: true`).
41494
+ *
41495
+ * So: an NPU/iGPU/ANE accelerator with NO stored entry is `enabled:true`; a
41496
+ * CPU or edgetpu device with no stored entry is `enabled:false`; an explicit
41497
+ * stored `enabled` always wins (an operator can opt CPU/Coral in, or an
41498
+ * accelerator out). A stored-only key (configured but the probe did not
41499
+ * return it — removed/unplugged HW) keeps its stored `enabled` and surfaces
41500
+ * as `available:false`, so the UI still shows it.
41501
+ *
41502
+ * Pure + deterministic (sorted by key) — the single merge authority shared by
41503
+ * the `getNodeInferenceDevices` view and the dispatcher's eligible-device pick.
41504
+ */
41505
+ function mergeInferenceDevices(probed, stored) {
41506
+ const probedByKey = new Map(probed.map((d) => [d.key, d]));
41507
+ const keys = new Set([...probedByKey.keys(), ...Object.keys(stored)]);
41508
+ const out = [];
41509
+ for (const key of Array.from(keys).toSorted()) {
41510
+ const descriptor = probedByKey.get(key);
41511
+ const opt = stored[key];
41512
+ const parsed = descriptor ?? parseDeviceKey(key);
41513
+ const weight = opt?.weight !== void 0 && opt.weight > 0 ? opt.weight : 1;
41514
+ const autoDefault = parsed.backend !== "cpu" && parsed.backend !== "edgetpu";
41515
+ out.push({
41516
+ key,
41517
+ backend: parsed.backend,
41518
+ device: parsed.device,
41519
+ format: parsed.format,
41520
+ available: descriptor?.available ?? false,
41521
+ enabled: opt?.enabled ?? autoDefault,
41522
+ weight,
41523
+ maxSessions: opt?.maxSessions ?? null,
41524
+ defaultModelId: defaultModelIdForDevice(key),
41525
+ ...opt?.steps && Object.keys(opt.steps).length > 0 ? { steps: { ...opt.steps } } : {}
41526
+ });
41527
+ }
41528
+ return out;
41529
+ }
41530
+ /**
41531
+ * The per-device concurrent-session caps for a node as `deviceKey → maxSessions`
41532
+ * (only devices that carry an explicit cap; absent = unlimited). Fed to the
41533
+ * device balancer's `nodeCaps` so a device at its cap is skipped (audit F3).
41534
+ */
41535
+ function inferenceDeviceCaps(stored) {
41536
+ const out = {};
41537
+ for (const [key, entry] of Object.entries(stored)) if (entry.maxSessions !== void 0 && entry.maxSessions > 0) out[key] = entry.maxSessions;
41538
+ return out;
41539
+ }
41540
+ /** Is this device the CPU fallback rather than a real accelerator? */
41541
+ function isCpuFallback(view) {
41542
+ return view.backend === "cpu";
41543
+ }
41544
+ function resolveInferenceDeviceEligibility(probed, stored, canRunRoot, isPoolUsable) {
41545
+ const eligible = {};
41546
+ const excluded = [];
41547
+ const merged = mergeInferenceDevices(probed, stored);
41548
+ const acceleratorServes = merged.some((d) => !isCpuFallback(d) && d.enabled && d.available && (!canRunRoot || canRunRoot(d.format)));
41549
+ for (const d of merged) {
41550
+ if (!d.enabled) {
41551
+ excluded.push({
41552
+ key: d.key,
41553
+ reason: "disabled",
41554
+ format: d.format
41555
+ });
41556
+ continue;
41557
+ }
41558
+ if (!d.available) {
41559
+ excluded.push({
41560
+ key: d.key,
41561
+ reason: "unavailable",
41562
+ format: d.format
41563
+ });
41564
+ continue;
41398
41565
  }
41399
- await this.withAudioSubLock(deviceId, async () => {
41400
- const unsub = this.audioSubscriptions.get(deviceId);
41401
- if (!unsub) return;
41402
- try {
41403
- unsub();
41404
- } catch {}
41405
- this.audioSubscriptions.delete(deviceId);
41406
- this.deps.logger.info("lazy audio: window closed", {
41407
- tags: { deviceId },
41408
- meta: { reason }
41566
+ if (isPoolUsable && !isPoolUsable(d.key)) {
41567
+ excluded.push({
41568
+ key: d.key,
41569
+ reason: "unavailable",
41570
+ format: d.format
41409
41571
  });
41410
- });
41411
- }
41412
- /**
41413
- * Audio teardown for one device — the audio half of `stopDetection`.
41414
- * Tears down through the per-device lock so it can't race a concurrent
41415
- * subscribe (which would re-store a handle this teardown never sees).
41416
- */
41417
- async stopForDevice(deviceId) {
41418
- await this.withAudioSubLock(deviceId, async () => {
41419
- const unsub = this.audioSubscriptions.get(deviceId);
41420
- if (unsub) {
41421
- try {
41422
- unsub();
41423
- } catch {}
41424
- this.audioSubscriptions.delete(deviceId);
41425
- }
41426
- });
41427
- const lazyTimer = this.lazyAudioTeardownTimers.get(deviceId);
41428
- if (lazyTimer) {
41429
- clearTimeout(lazyTimer);
41430
- this.lazyAudioTeardownTimers.delete(deviceId);
41572
+ continue;
41431
41573
  }
41432
- const windowTimer = this.motionAudioWindowTimers.get(deviceId);
41433
- if (windowTimer) {
41434
- clearTimeout(windowTimer);
41435
- this.motionAudioWindowTimers.delete(deviceId);
41574
+ if (canRunRoot && !canRunRoot(d.format)) {
41575
+ excluded.push({
41576
+ key: d.key,
41577
+ reason: "cannot-host-camera-root",
41578
+ format: d.format
41579
+ });
41580
+ continue;
41436
41581
  }
41437
- this.audioAssignments.delete(deviceId);
41438
- }
41439
- /**
41440
- * Centralized write into `audioSubscriptions`. If shutdown has begun, the
41441
- * map has already been (or is about to be) cleared lock-free in
41442
- * `shutdown()`; storing here would leak a zombie entry whose `unsub` is
41443
- * never called. So when shutting down we immediately invoke `unsub`
41444
- * (best-effort, error-swallowed) and DO NOT store. `protected` so
41445
- * `audio-sub-lock.spec.ts`'s test subclass can assert the shutdown-guard
41446
- * behavior without casts.
41447
- */
41448
- storeAudioSub(deviceId, unsub) {
41449
- if (this.audioShuttingDown) {
41450
- try {
41451
- unsub();
41452
- } catch {}
41453
- return;
41582
+ if (isCpuFallback(d) && acceleratorServes) {
41583
+ excluded.push({
41584
+ key: d.key,
41585
+ reason: "accelerator-preferred",
41586
+ format: d.format
41587
+ });
41588
+ continue;
41454
41589
  }
41455
- this.audioSubscriptions.set(deviceId, unsub);
41456
- }
41457
- /**
41458
- * Serialize an audio-subscription critical section per device. `fn` is
41459
- * chained onto the device's current lock tail, so concurrent calls for the
41460
- * SAME deviceId run sequentially (FIFO); different deviceIds never block
41461
- * each other. Thin delegate onto the `audioSubLocks` `KeyedAsyncLock`
41462
- * instance. `protected` so `audio-sub-lock.spec.ts`'s test subclass can
41463
- * drive the lock without casts.
41464
- */
41465
- withAudioSubLock(deviceId, fn) {
41466
- return this.audioSubLocks.run(deviceId, fn);
41590
+ eligible[d.key] = d.weight;
41467
41591
  }
41592
+ return {
41593
+ eligible,
41594
+ excluded
41595
+ };
41596
+ }
41597
+ /**
41598
+ * Join the merged device rows with the eligibility verdict so the UI can NAME
41599
+ * why an accelerator is not in play instead of leaving the operator to deduce
41600
+ * it from `enabled`/`available`.
41601
+ *
41602
+ * Deduction is not possible for two of the four reasons — `accelerator-preferred`
41603
+ * is a node-WIDE rule (a CPU row reads `enabled:true, available:true` and still
41604
+ * never gets a session, D215) and `cannot-host-camera-root` needs the node's
41605
+ * model catalog. Both live in {@link resolveInferenceDeviceEligibility}, so this
41606
+ * function only transports its answer; it never re-derives one.
41607
+ *
41608
+ * Pure; preserves `merged`'s order (sorted by key) and every other field.
41609
+ */
41610
+ function annotateInferenceDeviceExclusions(merged, eligibility) {
41611
+ const reasonByKey = new Map(eligibility.excluded.map((e) => [e.key, e.reason]));
41612
+ return merged.map((view) => ({
41613
+ ...view,
41614
+ exclusion: reasonByKey.get(view.key) ?? null
41615
+ }));
41616
+ }
41617
+ function resolveNodeInferenceUsability(eligibility) {
41618
+ const eligibleKeys = Object.keys(eligibility.eligible).toSorted();
41619
+ const unavailableKeys = eligibility.excluded.filter((e) => e.reason === "unavailable").map((e) => e.key).toSorted();
41620
+ return {
41621
+ usable: eligibleKeys.length > 0 || unavailableKeys.length === 0,
41622
+ unavailableKeys,
41623
+ eligibleKeys
41624
+ };
41625
+ }
41626
+ /**
41627
+ * Step-tree device jump (phase 1): the attach-payload roster of a node's
41628
+ * enabled∧available inference devices with the balancer knobs (`weight`,
41629
+ * `maxSessions`) the runner uses to AUTO-jump an enrichment step off a device
41630
+ * whose format can't run it. Built from the SAME `eligible` (deviceKey→weight)
41631
+ * and `caps` (deviceKey→maxSessions) the dispatcher already computes, so the
41632
+ * roster the runner sees exactly matches the balancer's candidate set. Sorted
41633
+ * by key for determinism. Populated onto `RunnerCameraConfig.inferenceDevices`
41634
+ * ONLY when a `deviceKey` is elected and there are ≥2 entries.
41635
+ */
41636
+ function buildInferenceDeviceRoster(eligible, caps) {
41637
+ return Object.entries(eligible).map(([deviceKey, weight]) => ({
41638
+ deviceKey,
41639
+ weight: weight > 0 ? weight : 1,
41640
+ maxSessions: caps[deviceKey] ?? null
41641
+ })).toSorted((a, b) => a.deviceKey < b.deviceKey ? -1 : a.deviceKey > b.deviceKey ? 1 : 0);
41642
+ }
41643
+ //#endregion
41644
+ //#region src/node-inference-usability-mirror.ts
41645
+ var NodeInferenceUsabilityMirror = class {
41646
+ state = /* @__PURE__ */ new Map();
41468
41647
  /**
41469
- * Subscribe to decoded audio chunks for a camera and feed them into the
41470
- * audio-analyzer. Reads the analyzer's settings via its own
41471
- * `resolveDeviceSettings(deviceId)` method so the orchestrator does not
41472
- * touch the audio-analyzer schema field names directly.
41648
+ * Fold one observation in and report whether the caller should act.
41649
+ * Never throws.
41473
41650
  */
41474
- async subscribeAudioStream(deviceId, config) {
41475
- const api = this.deps.api();
41476
- if (!api) {
41477
- this.deps.logger.warn("this.ctx.api not available — cannot subscribe audio", { tags: { deviceId } });
41478
- return null;
41479
- }
41480
- if (!await this.deps.isAudioAnalysisActive(deviceId)) return null;
41481
- if (config.audioMode === "disabled") {
41482
- this.deps.logger.debug("audio subscribe skipped: audioMode=disabled", { tags: { deviceId } });
41483
- return null;
41651
+ observe(nodeId, usable) {
41652
+ const prev = this.state.get(nodeId);
41653
+ if (usable) {
41654
+ this.state.set(nodeId, {
41655
+ usable: true,
41656
+ armed: false
41657
+ });
41658
+ return prev !== void 0 && !prev.usable ? "recovered" : null;
41484
41659
  }
41485
- if (config.audioMode === "on-motion" && !this.isMotionAudioWindowOpen(deviceId)) {
41486
- this.deps.logger.info("audio subscribe deferred: audioMode=on-motion, no window open", { tags: { deviceId } });
41660
+ if (prev === void 0) {
41661
+ this.state.set(nodeId, {
41662
+ usable: true,
41663
+ armed: true
41664
+ });
41487
41665
  return null;
41488
41666
  }
41489
- const audioStream = config.audioStreamId ?? config.motionStreamId;
41490
- const audioBrokerId = makeSourceBrokerId(deviceId, audioStream);
41491
- if ((await this.deps.probeAudioTrack(deviceId, audioStream)).kind === "absent") {
41492
- this.deps.logger.warn("audio subscription REFUSED — this stream carries no audio track", {
41493
- tags: { deviceId },
41494
- meta: {
41495
- camStreamId: audioStream,
41496
- brokerId: audioBrokerId,
41497
- selectedBy: config.audioStreamId !== void 0 ? "audioStreamId" : "motionStreamId",
41498
- hint: "point the camera’s audio at a stream that has an audio track — no stream is substituted automatically"
41499
- }
41667
+ if (!prev.usable) {
41668
+ this.state.set(nodeId, {
41669
+ usable: false,
41670
+ armed: true
41500
41671
  });
41501
41672
  return null;
41502
41673
  }
41503
- const settings = await api.audioAnalysis.resolveDeviceSettings.query({ deviceId });
41504
- if (!settings) {
41505
- this.deps.logger.warn("audio-analysis returned no settings — audio subscription skipped", { tags: { deviceId } });
41674
+ if (!prev.armed) {
41675
+ this.state.set(nodeId, {
41676
+ usable: true,
41677
+ armed: true
41678
+ });
41506
41679
  return null;
41507
41680
  }
41508
- const audioNodeId = await this.dispatch(deviceId);
41509
- const isRemoteAudio = audioNodeId !== this.deps.localNodeId();
41510
- this.deps.logger.info("audio subscription: resolved audio node", {
41511
- tags: { deviceId },
41512
- meta: {
41513
- audioNodeId,
41514
- isRemote: isRemoteAudio
41515
- }
41516
- });
41517
- const accumulator = new AudioWindowAccumulator(deviceId);
41518
- const teardown = startAudioChunkPoller({
41519
- api,
41520
- brokerId: audioBrokerId,
41521
- tag: "audio-analyzer",
41522
- ownerNodeId: this.deps.ingestNode(),
41523
- logger: this.deps.logger.withTags({ deviceId }),
41524
- onChunk: async (chunk) => {
41525
- this.deps.watchdogNote(deviceId, "audio");
41526
- try {
41527
- const audioChunkInput = accumulator.push(chunk);
41528
- if (!audioChunkInput) return;
41529
- const result = await api.audioAnalyzer.analyseChunk.mutate({
41530
- chunk: audioChunkInput,
41531
- settings,
41532
- ...isRemoteAudio ? { nodeId: audioNodeId } : {}
41533
- });
41534
- if (!result) return;
41535
- const frame = buildAudioResultFrame(deviceId, result);
41536
- this.deps.eventBus.emit({
41537
- id: `audio-inference-${deviceId}-${Date.now()}`,
41538
- timestamp: /* @__PURE__ */ new Date(),
41539
- source: {
41540
- type: "device",
41541
- id: deviceId,
41542
- nodeId: "hub",
41543
- addonId: "pipeline-orchestrator",
41544
- deviceId
41545
- },
41546
- category: EventCategory.PipelineAudioInferenceResult,
41547
- data: {
41548
- deviceId,
41549
- frame,
41550
- nodeId: "hub"
41551
- }
41552
- });
41553
- } catch (err) {
41554
- const msg = errMsg(err);
41555
- this.deps.logger.error("Audio analysis failed", {
41556
- tags: { deviceId },
41557
- meta: { error: msg }
41558
- });
41559
- }
41560
- }
41681
+ this.state.set(nodeId, {
41682
+ usable: false,
41683
+ armed: true
41561
41684
  });
41562
- this.deps.logger.info("Audio stream subscribed", { tags: { deviceId } });
41563
- return () => {
41564
- teardown();
41565
- accumulator.reset();
41566
- };
41685
+ return "became-unusable";
41567
41686
  }
41568
- /**
41569
- * Set true at the very start of `onShutdown`, before the audio teardown /
41570
- * map clears below. Once set, `withAudioSubLock` turns queued/new critical
41571
- * sections into no-ops and `storeAudioSub` refuses to store, so no
41572
- * critical section that was in-flight (or queued) when shutdown began can
41573
- * resurrect a zombie subscription into the cleared `audioSubscriptions`
41574
- * map. MUST be called before anything else in `onShutdown` that could
41575
- * race a queued audio critical section (mirrors the original
41576
- * `this.audioShuttingDown = true` being the very first statement).
41577
- */
41578
- beginShutdown() {
41579
- this.audioShuttingDown = true;
41687
+ /** Can this node be given cameras? Unknown nodes answer YES. */
41688
+ isUsable(nodeId) {
41689
+ return this.state.get(nodeId)?.usable ?? true;
41690
+ }
41691
+ /** Nodes currently excluded for the placement log and diagnostics. */
41692
+ unusableNodeIds() {
41693
+ const out = [];
41694
+ for (const [nodeId, s] of this.state) if (!s.usable) out.push(nodeId);
41695
+ return out.toSorted();
41696
+ }
41697
+ forget(nodeId) {
41698
+ this.state.delete(nodeId);
41580
41699
  }
41700
+ reset() {
41701
+ this.state.clear();
41702
+ }
41703
+ };
41704
+ //#endregion
41705
+ //#region src/inference-device-usability-mirror.ts
41706
+ /**
41707
+ * InferenceDeviceUsabilityMirror — "can this NODE's THIS DEVICE still infer?",
41708
+ * kept off the placement path.
41709
+ *
41710
+ * The per-`(nodeId, deviceKey)` twin of {@link NodeInferenceUsabilityMirror},
41711
+ * and deliberately not a second mechanism: it composes the key and delegates
41712
+ * every decision to that class, so the arm/apply reluctance D49 pinned lives in
41713
+ * exactly one implementation and cannot drift between the node tier and the
41714
+ * device tier.
41715
+ *
41716
+ * ## Why this tier had to exist
41717
+ *
41718
+ * The node tier already answers "does this node have ANY usable accelerator".
41719
+ * On 2026-08-25 the hub's answer was, correctly, yes — its NPU was healthy the
41720
+ * whole time. What died was one DEVICE, `openvino:gpu`, and nothing anywhere
41721
+ * asked that question: the per-dispatch capability gate is keyed on model
41722
+ * FORMAT and `openvino:gpu` and `openvino:npu` share the format `openvino`, so
41723
+ * it is blind between them by construction. The balancer kept rotating cameras
41724
+ * onto the dead pool — 20 sessions in 20 minutes, every one `tieBreak:
41725
+ * "rotation"` — for 31 hours.
41726
+ *
41727
+ * ## Why a mirror and not the event
41728
+ *
41729
+ * D49, verbatim: *a one-shot event never gates on a fallible read*. The gate is
41730
+ * this in-memory mirror, refreshed off the event path by the same
41731
+ * `resolveEligibleInferenceDevices` the dispatcher already runs (plus the
41732
+ * session controller's background refresher). The consequences that buys:
41733
+ *
41734
+ * - **A read that fails changes nothing.** The caller folds in an observation
41735
+ * only when it HAS one; an unreachable node, a version-skewed executor or a
41736
+ * rejected RPC never reaches {@link observe}, so the previous verdict
41737
+ * stands. This is the whole reason the health read is specified as
41738
+ * "synchronous over in-memory state, never throws for its own reasons": an
41739
+ * empty answer must mean *nothing is refused*, not *I could not tell*.
41740
+ * - **Excluding a healthy device needs a second, consecutive read.** Exclusion
41741
+ * is the direction that DESTROYS work — it strands an accelerator that may
41742
+ * be perfectly fine — so one bad observation only ARMS.
41743
+ * - **Re-admitting is immediate and unconditional.** One good observation puts
41744
+ * the device straight back. Being slow to exclude costs some wasted
41745
+ * inference attempts; being slow to re-admit costs an idle accelerator and a
41746
+ * node that looks broken.
41747
+ */
41748
+ /**
41749
+ * NUL — impossible in both a Moleculer node id and a `<backend>:<device>` key,
41750
+ * so the composite key can never be ambiguous. A separator that CAN occur in
41751
+ * either half makes two distinct pairs collide, and a collision here silently
41752
+ * excludes an accelerator nobody reported.
41753
+ */
41754
+ var SEPARATOR = "\0";
41755
+ var InferenceDeviceUsabilityMirror = class {
41756
+ /** The one implementation of the arm/apply state machine (D49). */
41757
+ mirror = new NodeInferenceUsabilityMirror();
41581
41758
  /**
41582
- * Full audio teardown combines the former `onShutdown`'s two separate
41583
- * audio blocks (lazy-teardown-timer clear, then — after several unrelated
41584
- * session/reconcile/load-shed clears — subscription teardown + lock clear
41585
- * + assignment-map clears) into one call. Safe to combine: both blocks
41586
- * are synchronous with no interleaved `await`, and every original
41587
- * statement between them (`sessionRegistry.clear()`,
41588
- * `cameraFpsMap.clear()`, `remoteHealthAttempts.clear()`,
41589
- * `loadShedState.clear()`, `loadShedResumeTimer` cleanup) touches state
41590
- * fully disjoint from anything audio — so their relative order to each
41591
- * other is unaffected, and the audio-internal order (timers →
41592
- * subscriptions → lock → assignment maps) is reproduced exactly.
41759
+ * Fold one observation in and report whether the caller should act.
41760
+ * Never throws.
41593
41761
  */
41594
- shutdown() {
41595
- for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
41596
- this.lazyAudioTeardownTimers.clear();
41597
- for (const t of this.motionAudioWindowTimers.values()) clearTimeout(t);
41598
- this.motionAudioWindowTimers.clear();
41599
- for (const unsub of this.audioSubscriptions.values()) try {
41600
- unsub();
41601
- } catch {}
41602
- this.audioSubscriptions.clear();
41603
- this.audioSubLocks.clear();
41604
- this.audioAssignments.clear();
41605
- this.readyAudioNodes.clear();
41762
+ observe(nodeId, deviceKey, usable) {
41763
+ return this.mirror.observe(`${nodeId}${SEPARATOR}${deviceKey}`, usable);
41764
+ }
41765
+ /** Can the balancer put a session on this device? Unknown pairs answer YES. */
41766
+ isUsable(nodeId, deviceKey) {
41767
+ return this.mirror.isUsable(`${nodeId}${SEPARATOR}${deviceKey}`);
41768
+ }
41769
+ /** Pairs currently excluded — for the placement log and diagnostics. */
41770
+ unusableDevices() {
41771
+ return this.mirror.unusableNodeIds().map((composite) => {
41772
+ const at = composite.indexOf(SEPARATOR);
41773
+ return {
41774
+ nodeId: composite.slice(0, at),
41775
+ deviceKey: composite.slice(at + 1)
41776
+ };
41777
+ });
41778
+ }
41779
+ /** The excluded device keys on ONE node. */
41780
+ unusableDeviceKeys(nodeId) {
41781
+ return this.unusableDevices().filter((d) => d.nodeId === nodeId).map((d) => d.deviceKey);
41782
+ }
41783
+ forget(nodeId, deviceKey) {
41784
+ this.mirror.forget(`${nodeId}${SEPARATOR}${deviceKey}`);
41785
+ }
41786
+ reset() {
41787
+ this.mirror.reset();
41606
41788
  }
41607
41789
  };
41790
+ /**
41791
+ * Fold ONE node's health answer into the mirror and return what changed.
41792
+ *
41793
+ * This is the whole reading discipline, in one place, because both halves of it
41794
+ * are easy to get subtly wrong and neither failure is visible in a log:
41795
+ *
41796
+ * - **`unhealthy === null` — the read FAILED — changes nothing.** Not a single
41797
+ * entry is touched. An unreachable node, a version-skewed executor or a
41798
+ * rejected RPC must be distinguishable from "asked, nothing is refused", or
41799
+ * a flaky link silently re-admits a dead accelerator (D49).
41800
+ * - **Every device the node HAS is observed**, not merely the refused ones.
41801
+ * The first draft observed `refused ∪ already-excluded`, which omits exactly
41802
+ * the devices the mirror has ARMED — so their disarming good read never
41803
+ * arrived and two bad reads an HOUR apart, with a hundred healthy ones
41804
+ * between them, excluded a working accelerator. "Consecutive" is only a
41805
+ * property if the good observations are delivered.
41806
+ *
41807
+ * Pure with respect to everything except `mirror`, and never throws — it is
41808
+ * called from the dispatcher's own read path.
41809
+ */
41810
+ function foldInferenceDeviceHealth(mirror, nodeId, unhealthy, present) {
41811
+ if (unhealthy === null) return [];
41812
+ const refused = new Set(unhealthy);
41813
+ const observed = new Set([
41814
+ ...present,
41815
+ ...refused,
41816
+ ...mirror.unusableDeviceKeys(nodeId)
41817
+ ]);
41818
+ const changes = [];
41819
+ for (const deviceKey of observed) {
41820
+ const transition = mirror.observe(nodeId, deviceKey, !refused.has(deviceKey));
41821
+ if (transition !== null) changes.push({
41822
+ deviceKey,
41823
+ transition
41824
+ });
41825
+ }
41826
+ return changes;
41827
+ }
41608
41828
  //#endregion
41609
41829
  //#region src/load-balancer.ts
41610
41830
  /**
@@ -43540,6 +43760,90 @@ function applyDeviceProvisioning(steps, base, override) {
43540
43760
  });
43541
43761
  }
43542
43762
  //#endregion
43763
+ //#region src/device-activity-source.ts
43764
+ /** The source's own name on the wire. One constant, used by every emit + gate. */
43765
+ var DEVICE_ACTIVITY_SOURCE = "device-activity";
43766
+ var DeviceActivitySource = class {
43767
+ #deps;
43768
+ #devices = /* @__PURE__ */ new Map();
43769
+ constructor(deps) {
43770
+ this.#deps = deps;
43771
+ }
43772
+ /**
43773
+ * A `recording-signal` level landed for a device. Idempotent in the level:
43774
+ * only a CHANGE emits an edge, and the sustain tick — not a repeated push —
43775
+ * is what keeps the session open.
43776
+ */
43777
+ onSignalLevel(deviceId, active, reason, atMs) {
43778
+ const entry = this.#devices.get(deviceId) ?? {
43779
+ active: false,
43780
+ timer: null
43781
+ };
43782
+ const wasActive = entry.active;
43783
+ entry.active = active;
43784
+ this.#devices.set(deviceId, entry);
43785
+ if (active === wasActive) return;
43786
+ if (active) {
43787
+ this.#deps.logger.info("device-activity: the device reports it is working", {
43788
+ tags: { deviceId },
43789
+ meta: { reason }
43790
+ });
43791
+ this.#emit(deviceId, true, atMs);
43792
+ this.#arm(deviceId, entry);
43793
+ return;
43794
+ }
43795
+ this.#clear(entry);
43796
+ this.#deps.logger.info("device-activity: the device reports it has stopped", {
43797
+ tags: { deviceId },
43798
+ meta: { reason }
43799
+ });
43800
+ this.#emit(deviceId, false, atMs);
43801
+ }
43802
+ /** Drop every timer (addon teardown). The mirror goes with the instance. */
43803
+ stop() {
43804
+ for (const entry of this.#devices.values()) this.#clear(entry);
43805
+ this.#devices.clear();
43806
+ }
43807
+ #arm(deviceId, entry) {
43808
+ this.#clear(entry);
43809
+ const sustainMs = this.#deps.sustainMsFor(deviceId);
43810
+ const timer = setInterval(() => {
43811
+ const current = this.#devices.get(deviceId);
43812
+ if (!current || !current.active) {
43813
+ this.#clear(current ?? entry);
43814
+ return;
43815
+ }
43816
+ this.#emit(deviceId, true, Date.now());
43817
+ }, sustainMs);
43818
+ timer.unref?.();
43819
+ entry.timer = timer;
43820
+ }
43821
+ #clear(entry) {
43822
+ if (entry.timer === null) return;
43823
+ clearInterval(entry.timer);
43824
+ entry.timer = null;
43825
+ }
43826
+ /**
43827
+ * Emit, unless this camera does not carry the source. A suppressed emit is
43828
+ * SAID — an operator who never ticked the box and an addon that is quietly
43829
+ * broken look identical from the outside otherwise.
43830
+ */
43831
+ #emit(deviceId, detected, timestamp) {
43832
+ const sources = this.#deps.motionSourcesFor(deviceId);
43833
+ if (sources === null || !sources.includes("device-activity")) {
43834
+ this.#deps.logger.debug("device-activity: level not forwarded — the camera does not list `device-activity`", {
43835
+ tags: { deviceId },
43836
+ meta: {
43837
+ detected,
43838
+ sources: sources ?? "no-active-detection"
43839
+ }
43840
+ });
43841
+ return;
43842
+ }
43843
+ this.#deps.emitMotion(deviceId, detected, timestamp);
43844
+ }
43845
+ };
43846
+ //#endregion
43543
43847
  //#region src/device-detection-settings.ts
43544
43848
  /** Read a required string leaf out of the hydrated `flat` schema values. */
43545
43849
  function mustString(flat, deviceId, key) {
@@ -43565,6 +43869,32 @@ function numberOrDefault(flat, key) {
43565
43869
  const dflt = uiField && "default" in uiField ? uiField.default : void 0;
43566
43870
  return typeof dflt === "number" ? dflt : 0;
43567
43871
  }
43872
+ /** The activity rate from the hydrated store, or the shipped default. */
43873
+ function activityFps(flat) {
43874
+ const v = flat["activityDetectionFps"];
43875
+ return typeof v === "number" && v > 0 ? v : 1;
43876
+ }
43877
+ /**
43878
+ * The detection rate a session opens at, given WHAT OPENED IT.
43879
+ *
43880
+ * A CAP, not a set: `min(cameraRate, activityRate)`. A camera already slower
43881
+ * than the activity rate stays slower, so lowering the global rate keeps
43882
+ * working. Why this lever and not the other two: a per-device `detectionFps`
43883
+ * would also slow the sessions a real motion trigger opens on the same camera,
43884
+ * and it becomes silently wrong the day the device gains a second trigger; a
43885
+ * per-SOURCE rate table would have to reach the runner and be re-applied
43886
+ * whenever the source holding the session changes, which the attach-time config
43887
+ * cannot express. Scoping it to the session's TRIGGER puts the number exactly
43888
+ * where its justification lives.
43889
+ *
43890
+ * Applies to the session the activity level OPENS. A session already open at
43891
+ * the camera rate when the level rises is not re-attached to slow it down —
43892
+ * re-attaching a live session to change one number costs a decode restart.
43893
+ */
43894
+ function detectionFpsForTrigger(config, trigger) {
43895
+ if (trigger !== "device-activity") return config.detectionFps;
43896
+ return Math.min(config.detectionFps, config.activityDetectionFps ?? 1);
43897
+ }
43568
43898
  /**
43569
43899
  * Pure decision/derivation half of the former `resolveDeviceDetectionSettings`.
43570
43900
  * Given the I/O-gathered raw materials, resolves every operator-wins →
@@ -43574,14 +43904,19 @@ function numberOrDefault(flat, key) {
43574
43904
  * the original method's "narrowing failed" catch.
43575
43905
  */
43576
43906
  function resolveDetectionSettings(input) {
43577
- const { deviceId, raw, flat, features, hasOnboardMotion, pipelineEnabled, motionDetectionEnabled } = input;
43907
+ const { deviceId, raw, flat, features, hasOnboardMotion, hasActivitySignal, pipelineEnabled, motionDetectionEnabled } = input;
43578
43908
  const profile = resolveDeviceProfile(features);
43579
43909
  const userMotionSources = raw["motionSources"];
43580
43910
  let motionSources;
43581
43911
  if (userMotionSources !== void 0) motionSources = MotionSourcesSchema.parse(userMotionSources);
43582
- else if (hasOnboardMotion) motionSources = ["onboard"];
43583
- else if (profile && features.includes(DeviceFeature.BatteryOperated)) motionSources = [];
43584
- else motionSources = MotionSourcesSchema.parse(flat["motionSources"]);
43912
+ else {
43913
+ let defaulted;
43914
+ if (hasOnboardMotion) defaulted = ["onboard"];
43915
+ else if (hasActivitySignal) defaulted = [];
43916
+ else if (profile && features.includes(DeviceFeature.BatteryOperated)) defaulted = [];
43917
+ else defaulted = MotionSourcesSchema.parse(flat["motionSources"]);
43918
+ motionSources = hasActivitySignal ? [...defaulted, DEVICE_ACTIVITY_SOURCE] : defaulted;
43919
+ }
43585
43920
  const userDetectionMode = raw["detectionMode"];
43586
43921
  const detectionMode = typeof userDetectionMode === "string" && isPipelinePhaseMode(userDetectionMode) ? userDetectionMode : profile?.defaults.detectionMode ?? "on-motion";
43587
43922
  const userAudioMode = raw["audioMode"];
@@ -43600,6 +43935,7 @@ function resolveDetectionSettings(input) {
43600
43935
  detectionStreamProfile: mustString(flat, deviceId, "detectionStreamProfile"),
43601
43936
  motionFps: numberOrDefault(flat, "motionFps"),
43602
43937
  detectionFps: numberOrDefault(flat, "detectionFps"),
43938
+ activityDetectionFps: activityFps(flat),
43603
43939
  motionCooldownMs: numberOrDefault(flat, "motionCooldownMs"),
43604
43940
  maxSessionHoldMs: numberOrDefault(flat, "maxSessionHoldMs"),
43605
43941
  audioMotionWindowMs: numberOrDefault(flat, "audioMotionWindowMs"),
@@ -43656,6 +43992,7 @@ function buildDetectionConfigFromInputs(resolved, assigned) {
43656
43992
  detectionStreamId: detectionCamStreamId,
43657
43993
  motionFps: resolved.motionFps,
43658
43994
  detectionFps: resolved.detectionFps,
43995
+ activityDetectionFps: resolved.activityDetectionFps,
43659
43996
  motionCooldownMs: resolved.motionCooldownMs,
43660
43997
  maxSessionHoldMs: resolved.maxSessionHoldMs,
43661
43998
  audioMotionWindowMs: resolved.audioMotionWindowMs,
@@ -43681,6 +44018,7 @@ function detectionConfigEquals(a, b) {
43681
44018
  if (a.detectionStreamId !== b.detectionStreamId) return false;
43682
44019
  if (a.motionFps !== b.motionFps) return false;
43683
44020
  if (a.detectionFps !== b.detectionFps) return false;
44021
+ if (a.activityDetectionFps !== b.activityDetectionFps) return false;
43684
44022
  if (a.motionCooldownMs !== b.motionCooldownMs) return false;
43685
44023
  if (a.maxSessionHoldMs !== b.maxSessionHoldMs) return false;
43686
44024
  if (a.audioMotionWindowMs !== b.audioMotionWindowMs) return false;
@@ -44307,6 +44645,7 @@ var DetectionWiringController = class {
44307
44645
  try {
44308
44646
  const features = await this.lookupDeviceFeatures(deviceId);
44309
44647
  const hasOnboardMotion = raw["motionSources"] === void 0 ? await this.deps.deviceHasOnboardMotionCap(deviceId) : false;
44648
+ const hasActivitySignal = raw["motionSources"] === void 0 ? await this.deps.deviceHasActivitySignalCap(deviceId) : false;
44310
44649
  const pipelineEnabled = await this.isDetectionPipelineActive(deviceId);
44311
44650
  const motionDetectionEnabled = await this.isMotionDetectionActive(deviceId);
44312
44651
  return resolveDetectionSettings({
@@ -44315,6 +44654,7 @@ var DetectionWiringController = class {
44315
44654
  flat,
44316
44655
  features,
44317
44656
  hasOnboardMotion,
44657
+ hasActivitySignal,
44318
44658
  pipelineEnabled,
44319
44659
  motionDetectionEnabled
44320
44660
  });
@@ -44920,9 +45260,10 @@ var DeviceConfigContributions = class {
44920
45260
  const schema = this.deps.deviceSettingsSchema();
44921
45261
  if (!schema) return null;
44922
45262
  const hasOnboardMotion = await this.deps.deviceHasOnboardMotionCap(input.deviceId);
44923
- const rawWithDefaults = raw["motionSources"] === void 0 && hasOnboardMotion ? {
45263
+ const hasActivitySignal = await this.deps.deviceHasActivitySignalCap(input.deviceId);
45264
+ const rawWithDefaults = raw["motionSources"] === void 0 && (hasOnboardMotion || hasActivitySignal) ? {
44924
45265
  ...raw,
44925
- motionSources: ["onboard"]
45266
+ motionSources: [...hasOnboardMotion ? ["onboard"] : [], ...hasActivitySignal ? ["device-activity"] : []]
44926
45267
  } : raw;
44927
45268
  const baseSections = hydrateSchema({
44928
45269
  ...schema,
@@ -47742,7 +48083,8 @@ function wireOrchestratorSubscriptions(deps) {
47742
48083
  if (!isEvent(event, EventCategory.MotionOnMotionChanged)) return;
47743
48084
  const { deviceId, detected, timestamp } = event.data;
47744
48085
  if (typeof deviceId !== "number") return;
47745
- deps.handleSessionMotion(deviceId, detected, typeof timestamp === "number" ? timestamp : void 0).catch((err) => {
48086
+ const parsedSource = MotionSourceEnum.safeParse(event.data.source);
48087
+ deps.handleSessionMotion(deviceId, detected, typeof timestamp === "number" ? timestamp : void 0, parsedSource.success ? parsedSource.data : void 0).catch((err) => {
47746
48088
  deps.logger.warn("session motion handler failed", {
47747
48089
  tags: { deviceId },
47748
48090
  meta: {
@@ -47764,8 +48106,47 @@ function wireOrchestratorSubscriptions(deps) {
47764
48106
  const deviceId = event.source.deviceId;
47765
48107
  if (typeof deviceId === "number") deps.noteWatchdogSignal(deviceId, "motion");
47766
48108
  });
48109
+ const activitySource = new DeviceActivitySource({
48110
+ logger: deps.logger,
48111
+ emitMotion: (deviceId, detected, timestamp) => {
48112
+ if (isTornDown) return;
48113
+ deps.eventBus.emit(createEvent(EventCategory.MotionOnMotionChanged, {
48114
+ type: "device",
48115
+ id: deviceId,
48116
+ deviceId
48117
+ }, {
48118
+ deviceId,
48119
+ detected,
48120
+ timestamp,
48121
+ source: DEVICE_ACTIVITY_SOURCE
48122
+ }));
48123
+ },
48124
+ motionSourcesFor: (deviceId) => deps.getActiveDetectionConfig(deviceId)?.motionSources ?? null,
48125
+ sustainMsFor: (deviceId) => {
48126
+ const cooldownMs = deps.getActiveDetectionConfig(deviceId)?.motionCooldownMs;
48127
+ return Math.max(1e3, Math.floor((cooldownMs ?? 3e4) / 2));
48128
+ }
48129
+ });
48130
+ const unsubDeviceActivity = deps.eventBus.subscribe({ category: EventCategory.DeviceStateChanged }, (event) => {
48131
+ const data = event.data;
48132
+ if (typeof data !== "object" || data === null) return;
48133
+ if (data["capName"] !== recordingSignalCapability.name) return;
48134
+ const deviceId = data["deviceId"];
48135
+ if (typeof deviceId !== "number") return;
48136
+ const level = RecordingSignalStatusSchema.safeParse(data["slice"]);
48137
+ if (!level.success) {
48138
+ deps.logger.warn("device-activity: signal slice does not parse — ignored", {
48139
+ tags: { deviceId },
48140
+ meta: { slice: data["slice"] }
48141
+ });
48142
+ return;
48143
+ }
48144
+ const atMs = event.timestamp instanceof Date ? event.timestamp.getTime() : Date.now();
48145
+ activitySource.onSignalLevel(deviceId, level.data.active, level.data.reason, atMs);
48146
+ });
47767
48147
  return () => {
47768
48148
  isTornDown = true;
48149
+ activitySource.stop();
47769
48150
  for (const t of profileSlotTimers.values()) clearTimeout(t);
47770
48151
  profileSlotTimers.clear();
47771
48152
  unsubDeviceRegistered();
@@ -47780,6 +48161,7 @@ function wireOrchestratorSubscriptions(deps) {
47780
48161
  unsubSessionMotion();
47781
48162
  unsubFrameTracked();
47782
48163
  unsubMotionAnalysis();
48164
+ unsubDeviceActivity();
47783
48165
  };
47784
48166
  }
47785
48167
  //#endregion
@@ -50539,7 +50921,7 @@ var SessionDispatchController = class {
50539
50921
  * standing attach, so `hasStandingAttach` stays true for them and
50540
50922
  * `decideSessionAction` still returns `'ignore'`.
50541
50923
  */
50542
- async handleSessionMotion(deviceId, detected, emittedAt) {
50924
+ async handleSessionMotion(deviceId, detected, emittedAt, trigger) {
50543
50925
  const receivedAt = Date.now();
50544
50926
  const busLagMs = emittedAt !== void 0 ? receivedAt - emittedAt : void 0;
50545
50927
  const config = this.deps.getActiveDetectionConfig(deviceId);
@@ -50574,7 +50956,7 @@ var SessionDispatchController = class {
50574
50956
  return;
50575
50957
  }
50576
50958
  this.activeRefireCountByDevice.delete(deviceId);
50577
- await this.dispatchDetectionSession(deviceId, cur);
50959
+ await this.dispatchDetectionSession(deviceId, cur, trigger);
50578
50960
  if (this.sessionRegistry.has(deviceId)) this.scheduleSessionTeardown(deviceId, cooldownMs);
50579
50961
  const doneAt = Date.now();
50580
50962
  this.deps.logger.info("session motion → attach latency", {
@@ -50613,7 +50995,7 @@ var SessionDispatchController = class {
50613
50995
  * `dispatchCamera`) avoids any risk of changing that already-live
50614
50996
  * standing-camera path.
50615
50997
  */
50616
- async dispatchDetectionSession(deviceId, config) {
50998
+ async dispatchDetectionSession(deviceId, config, trigger) {
50617
50999
  const log = this.deps.logger.withTags({ deviceId });
50618
51000
  await this.deps.reconcilePlacementFromRunners();
50619
51001
  const preferredAgent = await this.deps.readPipelinePin(deviceId);
@@ -50669,13 +51051,19 @@ var SessionDispatchController = class {
50669
51051
  const steps = applyDeviceProvisioning(pipelineConfig.steps, deviceBase, deviceOverride);
50670
51052
  const inferenceDevices = deviceKey && Object.keys(enabledDevices).length >= 2 ? buildInferenceDeviceRoster(enabledDevices, deviceCaps) : void 0;
50671
51053
  const zones = await this.deps.listZones(deviceId);
51054
+ const sessionDetectionFps = detectionFpsForTrigger(config, trigger);
51055
+ if (sessionDetectionFps !== config.detectionFps) log.info("session opened at the device-activity rate", { meta: {
51056
+ trigger,
51057
+ detectionFps: sessionDetectionFps,
51058
+ cameraDetectionFps: config.detectionFps
51059
+ } });
50672
51060
  const sessionConfig = {
50673
51061
  deviceId,
50674
51062
  ...deviceKey ? { deviceKey } : {},
50675
51063
  ...inferenceDevices ? { inferenceDevices } : {},
50676
51064
  motionCooldownMs: config.motionCooldownMs,
50677
51065
  motionFps: config.motionFps,
50678
- detectionFps: config.detectionFps,
51066
+ detectionFps: sessionDetectionFps,
50679
51067
  motionStreamId: config.motionStreamId,
50680
51068
  detectionStreamId: config.detectionStreamId,
50681
51069
  motionSources: [],
@@ -51980,6 +52368,7 @@ async function buildOrchestratorControllers(deps) {
51980
52368
  localNodeId: () => localNodeId,
51981
52369
  deviceSettingsSchema: () => deps.deviceSettingsSchema(),
51982
52370
  deviceHasOnboardMotionCap: (deviceId) => deps.deviceHasOnboardMotionCap(deviceId),
52371
+ deviceHasActivitySignalCap: (deviceId) => deps.deviceHasActivitySignalCap(deviceId),
51983
52372
  deviceHasNativeObjectDetectionCap: (deviceId) => deps.deviceHasNativeObjectDetectionCap(deviceId),
51984
52373
  setCameraPipelineForAgent: (input) => deps.setCameraPipelineForAgent(input),
51985
52374
  emitCameraUpdated: (deviceId, config) => deps.emitCameraUpdated(deviceId, config),
@@ -52297,6 +52686,7 @@ async function buildOrchestratorControllers(deps) {
52297
52686
  },
52298
52687
  isCapActiveForDevice: (deviceId, capName) => deps.isCapActiveForDevice(deviceId, capName),
52299
52688
  deviceHasOnboardMotionCap: (deviceId) => deps.deviceHasOnboardMotionCap(deviceId),
52689
+ deviceHasActivitySignalCap: (deviceId) => deps.deviceHasActivitySignalCap(deviceId),
52300
52690
  deviceSettingsSchema: () => deps.deviceSettingsSchema()
52301
52691
  });
52302
52692
  await detectionWiring.hydrateFeaturesMirror().catch((err) => {
@@ -52332,255 +52722,6 @@ async function buildOrchestratorControllers(deps) {
52332
52722
  };
52333
52723
  }
52334
52724
  //#endregion
52335
- //#region src/viewer-ui-provider.ts
52336
- /**
52337
- * viewer-ui provider — the pipeline-orchestrator serves the CamStack viewer web
52338
- * SPA (mirrors what the now-removed standalone addon-viewer-ui used to do).
52339
- *
52340
- * Why here: the orchestrator is a hub bootstrap addon (always installed + baked),
52341
- * so folding the viewer serving into it avoids a second addon whose only job was
52342
- * to hold static files. The viewer's Expo web export is COPIED into this addon's
52343
- * `assets/viewer/` locally by the viewer repo's `scripts/copy-dist-to-orchestrator.js`
52344
- * (run from a checkout that HAS the `camstack/` submodule + Expo toolchain — the
52345
- * publish/image CI does not, which is exactly why we copy a pre-built dist rather
52346
- * than build it here). vite emits only into `dist/`, so `assets/viewer` survives
52347
- * the addon build; `assets` is in the package `files`, so it ships on
52348
- * `camstack deploy`. The hub's `main.ts` resolves the `viewer-ui` singleton at
52349
- * boot and mounts the SPA at `/viewer/camstack`.
52350
- *
52351
- * `index.js` runs from `dist/`, so the SPA root is one level up + `assets/viewer`.
52352
- */
52353
- var __dirname = path.dirname(fileURLToPath(import.meta.url));
52354
- /** Absolute path to the staged viewer web SPA (`<addon-root>/assets/viewer`). */
52355
- function resolveViewerDistDir() {
52356
- return path.resolve(__dirname, "..", "assets", "viewer");
52357
- }
52358
- /** Version of the staged viewer, written by the copy script; 'unknown' if absent. */
52359
- function readViewerVersion() {
52360
- try {
52361
- const raw = fs.readFileSync(path.join(resolveViewerDistDir(), ".viewer-version"), "utf-8").trim();
52362
- if (raw) return raw;
52363
- } catch {}
52364
- return "unknown";
52365
- }
52366
- /** Build the viewer-ui provider. Serves whatever dist was staged into
52367
- * `assets/viewer`; when nothing is staged the hub's mount cleanly 404s. */
52368
- function createViewerUiProvider() {
52369
- return {
52370
- getStaticDir: async () => ({ staticDir: resolveViewerDistDir() }),
52371
- getVersion: async () => ({ version: readViewerVersion() })
52372
- };
52373
- }
52374
- //#endregion
52375
- //#region src/disk-reconcile-fleet.ts
52376
- async function reconcileFleetFromDisk(deps) {
52377
- const deviceIds = await deps.listDeviceIds();
52378
- const failed = [];
52379
- let cameras = 0;
52380
- let mediaDropped = 0;
52381
- let tracks = 0;
52382
- let events = 0;
52383
- let completed = 0;
52384
- for (const deviceId of deviceIds) {
52385
- try {
52386
- await deps.rescanRecordings(deviceId);
52387
- const counts = await deps.reconcileAnalytics(deviceId);
52388
- cameras += 1;
52389
- mediaDropped += counts.mediaDropped;
52390
- tracks += counts.tracks;
52391
- events += counts.events;
52392
- } catch {
52393
- failed.push(deviceId);
52394
- }
52395
- completed += 1;
52396
- deps.onProgress?.({
52397
- deviceId,
52398
- total: deviceIds.length,
52399
- completed,
52400
- failed: [...failed],
52401
- mediaDropped,
52402
- tracks,
52403
- events
52404
- });
52405
- }
52406
- return {
52407
- cameras,
52408
- failed,
52409
- mediaDropped,
52410
- tracks,
52411
- events
52412
- };
52413
- }
52414
- //#endregion
52415
- //#region src/disk-reconcile-job.ts
52416
- /**
52417
- * In-memory disk-wins fleet job. The tRPC mutation starts this and returns
52418
- * immediately; the walk runs in the addon process so a 60s UDS timeout cannot
52419
- * abort it. Status is polled via getReconcileFromDiskStatus.
52420
- */
52421
- function idleDiskReconcileJob() {
52422
- return {
52423
- state: "idle",
52424
- total: 0,
52425
- completed: 0,
52426
- currentDeviceId: null,
52427
- failed: [],
52428
- mediaDropped: 0,
52429
- tracks: 0,
52430
- events: 0,
52431
- startedAtMs: null,
52432
- finishedAtMs: null,
52433
- error: null
52434
- };
52435
- }
52436
- function isTimeoutError(err) {
52437
- const message = err instanceof Error ? err.message : String(err);
52438
- return /timed out/i.test(message);
52439
- }
52440
- async function withTimeoutRetry(run) {
52441
- try {
52442
- return await run();
52443
- } catch (err) {
52444
- if (!isTimeoutError(err)) throw err;
52445
- return await run();
52446
- }
52447
- }
52448
- function createDiskReconcileJobRunner(now = Date.now) {
52449
- let job = idleDiskReconcileJob();
52450
- let inFlight = null;
52451
- const snapshot = () => job;
52452
- const start = (deps) => {
52453
- if (job.state === "running" && inFlight) return job;
52454
- job = {
52455
- ...idleDiskReconcileJob(),
52456
- state: "running",
52457
- startedAtMs: now()
52458
- };
52459
- deps.log?.("pipeline disk reconcile started");
52460
- inFlight = (async () => {
52461
- try {
52462
- const result = await reconcileFleetFromDisk({
52463
- listDeviceIds: deps.listDeviceIds,
52464
- rescanRecordings: (deviceId) => withTimeoutRetry(() => deps.rescanRecordings(deviceId)),
52465
- reconcileAnalytics: (deviceId) => withTimeoutRetry(() => deps.reconcileAnalytics(deviceId)),
52466
- onProgress: (update) => {
52467
- job = {
52468
- ...job,
52469
- total: update.total,
52470
- completed: update.completed,
52471
- currentDeviceId: update.deviceId,
52472
- failed: update.failed,
52473
- mediaDropped: update.mediaDropped,
52474
- tracks: update.tracks,
52475
- events: update.events
52476
- };
52477
- deps.onProgress?.(update);
52478
- deps.log?.("pipeline disk reconcile camera", {
52479
- deviceId: update.deviceId,
52480
- completed: update.completed,
52481
- total: update.total,
52482
- failed: update.failed.length,
52483
- mediaDropped: update.mediaDropped,
52484
- tracks: update.tracks,
52485
- events: update.events
52486
- });
52487
- }
52488
- });
52489
- job = {
52490
- ...job,
52491
- state: "done",
52492
- total: result.cameras + result.failed.length,
52493
- completed: result.cameras + result.failed.length,
52494
- currentDeviceId: null,
52495
- failed: result.failed,
52496
- mediaDropped: result.mediaDropped,
52497
- tracks: result.tracks,
52498
- events: result.events,
52499
- finishedAtMs: now(),
52500
- error: null
52501
- };
52502
- deps.log?.("pipeline disk reconcile", {
52503
- cameras: result.cameras,
52504
- failed: result.failed,
52505
- mediaDropped: result.mediaDropped,
52506
- tracks: result.tracks,
52507
- events: result.events
52508
- });
52509
- } catch (err) {
52510
- const error = err instanceof Error ? err.message : String(err);
52511
- job = {
52512
- ...job,
52513
- state: "error",
52514
- currentDeviceId: null,
52515
- finishedAtMs: now(),
52516
- error
52517
- };
52518
- deps.log?.("pipeline disk reconcile failed", { error });
52519
- } finally {
52520
- inFlight = null;
52521
- }
52522
- })();
52523
- return job;
52524
- };
52525
- return {
52526
- snapshot,
52527
- start
52528
- };
52529
- }
52530
- //#endregion
52531
- //#region src/widget-catalog.ts
52532
- var pipelineOrchestratorWidgets = [{
52533
- tab: "device-tab",
52534
- label: "Pipeline Quick Stats",
52535
- preAuth: false,
52536
- kind: "remote",
52537
- remote: {
52538
- remoteName: "addon_pipeline_orchestrator_widgets",
52539
- exposedModule: "./widgets",
52540
- componentKey: "pipeline-quick-stats"
52541
- },
52542
- stableId: "pipeline-quick-stats",
52543
- description: "Phase / Detection FPS / Inference / Active Tracks tile row.",
52544
- icon: "activity",
52545
- bundle: "remoteEntry.js",
52546
- hosts: ["device-tab", "dashboard"],
52547
- requires: {
52548
- deviceContext: true,
52549
- integrationContext: false
52550
- },
52551
- defaultSize: "md",
52552
- allowedSizes: [
52553
- "sm",
52554
- "md",
52555
- "lg"
52556
- ],
52557
- defaultColumns: 6,
52558
- defaultRows: 1
52559
- }, {
52560
- tab: "device-tab",
52561
- label: "Zone Editor",
52562
- preAuth: false,
52563
- kind: "remote",
52564
- remote: {
52565
- remoteName: "addon_pipeline_orchestrator_widgets",
52566
- exposedModule: "./widgets",
52567
- componentKey: "zone-editor"
52568
- },
52569
- stableId: "zone-editor",
52570
- description: "Polygon / tripwire CRUD + per-stage rule editor.",
52571
- icon: "shapes",
52572
- bundle: "remoteEntry.js",
52573
- hosts: ["device-tab"],
52574
- requires: {
52575
- deviceContext: true,
52576
- integrationContext: false
52577
- },
52578
- defaultSize: "xl",
52579
- allowedSizes: ["lg", "xl"],
52580
- defaultColumns: 12,
52581
- defaultRows: 4
52582
- }];
52583
- //#endregion
52584
52725
  //#region src/settings-ui-schemas.ts
52585
52726
  /** Build the addon-level schema sections (cluster roles + crop + balancer + failover). */
52586
52727
  function buildGlobalSettingsSections(options) {
@@ -52945,6 +53086,22 @@ function buildDeviceSettingsSections(nodeOptions) {
52945
53086
  field: "detectionMode",
52946
53087
  notEquals: "disabled"
52947
53088
  }
53089
+ },
53090
+ {
53091
+ key: "activityDetectionFps",
53092
+ type: "slider",
53093
+ label: "Detection FPS while the device is working",
53094
+ min: 1,
53095
+ max: 10,
53096
+ step: 1,
53097
+ default: 1,
53098
+ showValue: true,
53099
+ unit: "fps",
53100
+ description: "A device-activity session lasts as long as the job does — a cleaning run is tens of minutes. This caps the rate for that session only; a session opened by motion keeps the rate above.",
53101
+ showWhen: {
53102
+ field: "motionSources",
53103
+ includes: "device-activity"
53104
+ }
52948
53105
  }
52949
53106
  ]
52950
53107
  },
@@ -53034,6 +53191,99 @@ function deriveRuntimeSettings(config) {
53034
53191
  };
53035
53192
  }
53036
53193
  //#endregion
53194
+ //#region src/viewer-ui-provider.ts
53195
+ /**
53196
+ * viewer-ui provider — the pipeline-orchestrator serves the CamStack viewer web
53197
+ * SPA (mirrors what the now-removed standalone addon-viewer-ui used to do).
53198
+ *
53199
+ * Why here: the orchestrator is a hub bootstrap addon (always installed + baked),
53200
+ * so folding the viewer serving into it avoids a second addon whose only job was
53201
+ * to hold static files. The viewer's Expo web export is COPIED into this addon's
53202
+ * `assets/viewer/` locally by the viewer repo's `scripts/copy-dist-to-orchestrator.js`
53203
+ * (run from a checkout that HAS the `camstack/` submodule + Expo toolchain — the
53204
+ * publish/image CI does not, which is exactly why we copy a pre-built dist rather
53205
+ * than build it here). vite emits only into `dist/`, so `assets/viewer` survives
53206
+ * the addon build; `assets` is in the package `files`, so it ships on
53207
+ * `camstack deploy`. The hub's `main.ts` resolves the `viewer-ui` singleton at
53208
+ * boot and mounts the SPA at `/viewer/camstack`.
53209
+ *
53210
+ * `index.js` runs from `dist/`, so the SPA root is one level up + `assets/viewer`.
53211
+ */
53212
+ var __dirname = path.dirname(fileURLToPath(import.meta.url));
53213
+ /** Absolute path to the staged viewer web SPA (`<addon-root>/assets/viewer`). */
53214
+ function resolveViewerDistDir() {
53215
+ return path.resolve(__dirname, "..", "assets", "viewer");
53216
+ }
53217
+ /** Version of the staged viewer, written by the copy script; 'unknown' if absent. */
53218
+ function readViewerVersion() {
53219
+ try {
53220
+ const raw = fs.readFileSync(path.join(resolveViewerDistDir(), ".viewer-version"), "utf-8").trim();
53221
+ if (raw) return raw;
53222
+ } catch {}
53223
+ return "unknown";
53224
+ }
53225
+ /** Build the viewer-ui provider. Serves whatever dist was staged into
53226
+ * `assets/viewer`; when nothing is staged the hub's mount cleanly 404s. */
53227
+ function createViewerUiProvider() {
53228
+ return {
53229
+ getStaticDir: async () => ({ staticDir: resolveViewerDistDir() }),
53230
+ getVersion: async () => ({ version: readViewerVersion() })
53231
+ };
53232
+ }
53233
+ //#endregion
53234
+ //#region src/widget-catalog.ts
53235
+ var pipelineOrchestratorWidgets = [{
53236
+ tab: "device-tab",
53237
+ label: "Pipeline Quick Stats",
53238
+ preAuth: false,
53239
+ kind: "remote",
53240
+ remote: {
53241
+ remoteName: "addon_pipeline_orchestrator_widgets",
53242
+ exposedModule: "./widgets",
53243
+ componentKey: "pipeline-quick-stats"
53244
+ },
53245
+ stableId: "pipeline-quick-stats",
53246
+ description: "Phase / Detection FPS / Inference / Active Tracks tile row.",
53247
+ icon: "activity",
53248
+ bundle: "remoteEntry.js",
53249
+ hosts: ["device-tab", "dashboard"],
53250
+ requires: {
53251
+ deviceContext: true,
53252
+ integrationContext: false
53253
+ },
53254
+ defaultSize: "md",
53255
+ allowedSizes: [
53256
+ "sm",
53257
+ "md",
53258
+ "lg"
53259
+ ],
53260
+ defaultColumns: 6,
53261
+ defaultRows: 1
53262
+ }, {
53263
+ tab: "device-tab",
53264
+ label: "Zone Editor",
53265
+ preAuth: false,
53266
+ kind: "remote",
53267
+ remote: {
53268
+ remoteName: "addon_pipeline_orchestrator_widgets",
53269
+ exposedModule: "./widgets",
53270
+ componentKey: "zone-editor"
53271
+ },
53272
+ stableId: "zone-editor",
53273
+ description: "Polygon / tripwire CRUD + per-stage rule editor.",
53274
+ icon: "shapes",
53275
+ bundle: "remoteEntry.js",
53276
+ hosts: ["device-tab"],
53277
+ requires: {
53278
+ deviceContext: true,
53279
+ integrationContext: false
53280
+ },
53281
+ defaultSize: "xl",
53282
+ allowedSizes: ["lg", "xl"],
53283
+ defaultColumns: 12,
53284
+ defaultRows: 4
53285
+ }];
53286
+ //#endregion
53037
53287
  //#region src/index.ts
53038
53288
  /**
53039
53289
  * addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
@@ -53348,6 +53598,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
53348
53598
  isCapActiveForDevice: (deviceId, capName) => this.isCapActiveForDevice(deviceId, capName),
53349
53599
  isAudioAnalysisActive: (deviceId) => this.isAudioAnalysisActive(deviceId),
53350
53600
  deviceHasOnboardMotionCap: (deviceId) => this.deviceHasOnboardMotionCap(deviceId),
53601
+ deviceHasActivitySignalCap: (deviceId) => this.deviceHasActivitySignalCap(deviceId),
53351
53602
  deviceHasNativeObjectDetectionCap: (deviceId) => this.deviceHasNativeObjectDetectionCap(deviceId),
53352
53603
  handleDeviceRegistered: (deviceId) => this.handleDeviceRegistered(deviceId),
53353
53604
  handleDeviceUnregistered: (deviceId) => this.handleDeviceUnregistered(deviceId),
@@ -54534,6 +54785,26 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
54534
54785
  * we err toward the analyzer (never lose detection coverage when the
54535
54786
  * binding lookup hiccups).
54536
54787
  */
54788
+ /**
54789
+ * True when the device's driver registered `recording-signal` — the cap a
54790
+ * device uses to say, itself, that it is working (a robot vacuum cleaning).
54791
+ * Drives the `device-activity` entry in the `motionSources` DEFAULT (D392),
54792
+ * so the source arrives on exactly the devices that can raise it and on no
54793
+ * other camera in the fleet.
54794
+ *
54795
+ * Same failure discipline as `deviceHasOnboardMotionCap`: a binding lookup
54796
+ * that throws answers `false`, which loses the DEFAULT and never the
54797
+ * operator's explicit choice (this is asked only when they pinned nothing).
54798
+ */
54799
+ async deviceHasActivitySignalCap(deviceId) {
54800
+ const api = this.api;
54801
+ if (!api) return false;
54802
+ try {
54803
+ return (await api.deviceManager.getBindings.query({ deviceId })).entries.some((e) => e.kind === "native" && e.capName === "recording-signal");
54804
+ } catch {
54805
+ return false;
54806
+ }
54807
+ }
54537
54808
  async deviceHasOnboardMotionCap(deviceId) {
54538
54809
  const api = this.api;
54539
54810
  if (!api) return false;