@camstack/addon-pipeline-orchestrator 1.1.13 → 1.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -4627,7 +4627,7 @@ function _instanceof(cls, params = {}) {
4627
4627
  return inst;
4628
4628
  }
4629
4629
  //#endregion
4630
- //#region ../types/dist/sleep-C2M2zF7x.mjs
4630
+ //#region ../types/dist/sleep-MHm--th-.mjs
4631
4631
  var EventCategory = /* @__PURE__ */ function(EventCategory) {
4632
4632
  EventCategory["SystemBoot"] = "system.boot";
4633
4633
  EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -5905,6 +5905,7 @@ var CamStreamKindSchema = _enum([
5905
5905
  "pull-rtsp",
5906
5906
  "pull-rtmp",
5907
5907
  "pull-http",
5908
+ "pull-flv",
5908
5909
  "pull-rfc4571",
5909
5910
  "push-annexb",
5910
5911
  "derived"
@@ -7444,7 +7445,21 @@ var StorageLocationDeclarationSchema = object({
7444
7445
  * slots (e.g. `recordingsLow` → `recordings`) so operators only need to
7445
7446
  * configure the primary location.
7446
7447
  */
7447
- defaultsTo: string().optional()
7448
+ defaultsTo: string().optional(),
7449
+ /**
7450
+ * Which node root the seeded `<id>:default` instance is placed under on a
7451
+ * FRESH install:
7452
+ * - `'data'` (default) — the node's data dir (`CAMSTACK_DATA` / boot dir),
7453
+ * the appData volume. Right for small/durable data (backups, logs, models).
7454
+ * - `'media'` — the dedicated media volume (`CAMSTACK_MEDIA_ROOT`) when that
7455
+ * env is set, else falls back to the data root. Right for bulky, hot media
7456
+ * (recordings, event media) that should stay off the appData disk.
7457
+ *
7458
+ * Only affects the seeded default's `basePath`; operators can repoint any
7459
+ * location afterwards, and a `defaultsTo` slot inherits its parent's root
7460
+ * regardless of this field. Absent (the common case) is treated as `'data'`.
7461
+ */
7462
+ defaultRoot: _enum(["data", "media"]).optional()
7448
7463
  });
7449
7464
  var DecoderStatsSchema = object({
7450
7465
  inputFps: number(),
@@ -8322,6 +8337,10 @@ var RtspRestreamEntrySchema = object({
8322
8337
  var BrokerRtspClientSchema = object({
8323
8338
  sessionId: string(),
8324
8339
  remoteAddr: string(),
8340
+ /** Client `User-Agent` (e.g. `recorder` for the recorder's RTSP pull), or
8341
+ * null/absent when the client sent none. Lets the UI label a consumer by
8342
+ * purpose. Optional so a client built against an older schema stays valid. */
8343
+ userAgent: string().nullish(),
8325
8344
  playing: boolean(),
8326
8345
  muted: boolean(),
8327
8346
  connectedAt: number(),
@@ -13978,7 +13997,7 @@ var AddBrokerInputSchema = object({
13978
13997
  });
13979
13998
  var AddBrokerResultSchema = object({ id: string() });
13980
13999
  var IdInputSchema = object({ id: string() });
13981
- var TestResultSchema = discriminatedUnion("ok", [object({
14000
+ var TestResultSchema$1 = discriminatedUnion("ok", [object({
13982
14001
  ok: literal(true),
13983
14002
  latencyMs: number()
13984
14003
  }), object({
@@ -14001,7 +14020,7 @@ var StatusSchema = object({
14001
14020
  brokerCount: number(),
14002
14021
  embeddedRunning: boolean()
14003
14022
  });
14004
- method(_void(), array(BrokerInfoSchema)), method(IdInputSchema, BrokerConnectionDetailsSchema), method(AddBrokerInputSchema, AddBrokerResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(IdInputSchema, TestResultSchema, { kind: "mutation" }), method(StartEmbeddedInputSchema, StartEmbeddedResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(_void(), StatusSchema);
14023
+ method(_void(), array(BrokerInfoSchema)), method(IdInputSchema, BrokerConnectionDetailsSchema), method(AddBrokerInputSchema, AddBrokerResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(IdInputSchema, TestResultSchema$1, { kind: "mutation" }), method(StartEmbeddedInputSchema, StartEmbeddedResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(_void(), StatusSchema);
14005
14024
  var NetworkEndpointSchema = object({
14006
14025
  url: string(),
14007
14026
  hostname: string(),
@@ -14035,23 +14054,198 @@ var NetworkEndpointEntrySchema = NetworkEndpointSchema.extend({
14035
14054
  sourcePort: number().optional()
14036
14055
  });
14037
14056
  method(_void(), NetworkEndpointSchema, { kind: "mutation" }), method(_void(), _void(), { kind: "mutation" }), method(_void(), NetworkEndpointSchema.nullable()), method(_void(), NetworkAccessStatusSchema), method(_void(), array(NetworkEndpointEntrySchema).readonly());
14038
- method(object({
14039
- title: string(),
14057
+ /**
14058
+ * notification-output — canonical, capability-gated notification delivery.
14059
+ *
14060
+ * Apprise-derived model (see
14061
+ * `docs/superpowers/specs/2026-07-03-notification-output-notifier-matrix.md`):
14062
+ * callers emit ONE canonical `Notification`; each provider declares a
14063
+ * per-kind capability descriptor (`TargetKind`), and the pure degrade
14064
+ * engine (`@camstack/types` `prepareNotification`) transcodes / degrades the
14065
+ * message to what the kind supports — callers never special-case a service.
14066
+ *
14067
+ * DESIGN DECISIONS (locked):
14068
+ * - Target CRUD lives on THIS cap (`upsertTarget` / `deleteTarget` /
14069
+ * `setTargetEnabled`), each provider persisting via the `settings-store`
14070
+ * cap. Rationale: the admin UI needs one uniform surface across the
14071
+ * notifiers addon AND the HA addon; the addon-`globalSettingsSchema`-array
14072
+ * alternative would fork the UI per addon and cannot host the
14073
+ * discovery→adopt flow.
14074
+ * - `listTargetKinds` / `listTargets` / `discoverTargets` return arrays →
14075
+ * the generated cap-mount auto-`concatCollection`-fans them across every
14076
+ * registered provider (notifiers addon + HA addon) so one catalog is
14077
+ * routable. `send` / `testTarget` / CRUD route to ONE provider by the
14078
+ * `addonId` the generated collection router extracts from the call input.
14079
+ * - `Attachment.bytes` is `Uint8Array`. Transport-safe: superjson (the tRPC
14080
+ * transformer) + UDS MsgPack both round-trip typed arrays — already used by
14081
+ * `storage` / `storage-provider` / `recording` caps over the same path. No
14082
+ * base64 fallback needed.
14083
+ *
14084
+ * TODO (deferred, closed-set change — separate decision): add
14085
+ * `providerKind: 'notify'` so notification providers surface on the unified
14086
+ * admin "Integrations" page.
14087
+ */
14088
+ /**
14089
+ * Zentik-derived typed-media enum — the superset across every kind. Each
14090
+ * adapter picks what it supports and the degrade engine filters the rest.
14091
+ */
14092
+ var AttachmentMediaTypeSchema = _enum([
14093
+ "image",
14094
+ "video",
14095
+ "gif",
14096
+ "audio",
14097
+ "icon"
14098
+ ]);
14099
+ /**
14100
+ * A single attachment. Exactly one of `url` (remote source, most adapters
14101
+ * prefer this) or `bytes` (inline source; required for Pushover-style
14102
+ * bytes-only kinds) MUST be present — the degrade engine expresses a
14103
+ * url→bytes fetch as a `needsFetch` directive the adapter executes.
14104
+ */
14105
+ var AttachmentSchema = object({
14106
+ mediaType: AttachmentMediaTypeSchema,
14107
+ url: string().optional(),
14108
+ bytes: _instanceof(Uint8Array).optional(),
14109
+ mime: string().optional(),
14110
+ name: string().optional()
14111
+ }).refine((a) => a.url !== void 0 || a.bytes !== void 0, { message: "Attachment requires either `url` or `bytes`" });
14112
+ var NotificationFormatSchema = _enum([
14113
+ "text",
14114
+ "markdown",
14115
+ "html"
14116
+ ]);
14117
+ /** A single tap-through action button. */
14118
+ var NotificationActionSchema = object({
14119
+ id: string(),
14120
+ label: string(),
14121
+ url: string().optional()
14122
+ });
14123
+ /**
14124
+ * The canonical notification. `body` is the only hard field (Apprise model).
14125
+ * `priority` is a 5-level ORDINAL (1=lowest … 3=normal(default) … 5=urgent),
14126
+ * NOT a fixed severity enum — each kind declares its own `caps.levels` and
14127
+ * the adapter maps this ordinal onto its native level. `level?` is an
14128
+ * optional kind-native level id (`emergency`, `silent`, …) that overrides
14129
+ * `priority` for that one target.
14130
+ */
14131
+ var NotificationSchema = object({
14040
14132
  body: string(),
14041
- imageUrl: string().optional(),
14133
+ title: string().optional(),
14134
+ format: NotificationFormatSchema.default("text"),
14135
+ priority: number().int().min(1).max(5).default(3),
14136
+ level: string().optional(),
14137
+ attachments: array(AttachmentSchema).optional(),
14138
+ clickUrl: string().optional(),
14139
+ actions: array(NotificationActionSchema).optional(),
14140
+ sound: string().optional(),
14141
+ ttl: number().optional(),
14142
+ tag: string().optional(),
14042
14143
  deviceId: number().optional(),
14043
14144
  eventId: string().optional(),
14044
- priority: _enum([
14045
- "low",
14046
- "normal",
14047
- "high",
14048
- "critical"
14049
- ]).default("normal"),
14050
14145
  metadata: record(string(), unknown()).optional()
14051
- }), _void(), { kind: "mutation" }), method(_void(), object({
14146
+ });
14147
+ /** One declared native severity/priority level for a kind. */
14148
+ var TargetKindLevelSchema = object({
14149
+ id: string(),
14150
+ label: string(),
14151
+ /** Which canonical priority (1..5) this level maps to. `null` = qualitative-only. */
14152
+ ordinal: number().int().min(1).max(5).nullable(),
14153
+ flags: object({
14154
+ critical: boolean().optional(),
14155
+ silent: boolean().optional(),
14156
+ noPush: boolean().optional()
14157
+ }).optional(),
14158
+ /** e.g. Pushover `emergency` requires `retry` / `expire`. */
14159
+ requires: array(string()).optional(),
14160
+ description: string().optional()
14161
+ });
14162
+ /** The full capability block consulted before dispatch. */
14163
+ var TargetKindCapsSchema = object({
14164
+ attachments: object({
14165
+ mediaTypes: array(AttachmentMediaTypeSchema),
14166
+ mode: _enum([
14167
+ "url",
14168
+ "bytes",
14169
+ "both"
14170
+ ]),
14171
+ max: number().int().nonnegative(),
14172
+ maxBytes: number().int().positive().optional()
14173
+ }),
14174
+ /** Max action buttons (0 = none). */
14175
+ actions: number().int().nonnegative(),
14176
+ levels: array(TargetKindLevelSchema),
14177
+ format: array(NotificationFormatSchema),
14178
+ clickUrl: boolean(),
14179
+ sound: boolean(),
14180
+ ttl: boolean(),
14181
+ bodyMaxLen: number().int().positive()
14182
+ });
14183
+ /**
14184
+ * `configSchema` is a `ConfigUISchema` tree passed through to the admin
14185
+ * FormBuilder. Stored as `z.unknown()` at the cap seam (mirrors
14186
+ * `device-provider.getChildCreationSchema` `CreationSchemaOutputSchema`) —
14187
+ * the union is large and not meant for runtime validation here; the exported
14188
+ * `TargetKind` type re-tightens `configSchema` to `ConfigUISchema`.
14189
+ */
14190
+ var ConfigSchemaPassthrough = unknown();
14191
+ var TargetKindSchema = object({
14192
+ kind: string(),
14193
+ label: string(),
14194
+ icon: string(),
14195
+ /** Stamped by each provider so the concat-fanned catalog stays routable. */
14196
+ addonId: string(),
14197
+ configSchema: ConfigSchemaPassthrough,
14198
+ supportsDiscovery: boolean(),
14199
+ caps: TargetKindCapsSchema
14200
+ });
14201
+ /**
14202
+ * A persisted target. `config` holds secrets; providers REDACT secret fields
14203
+ * (return a presence marker only) when serving `listTargets` — never
14204
+ * round-trip a stored secret to the UI.
14205
+ */
14206
+ var TargetSchema = object({
14207
+ id: string(),
14208
+ name: string(),
14209
+ kind: string(),
14210
+ addonId: string(),
14211
+ enabled: boolean(),
14212
+ config: record(string(), unknown())
14213
+ });
14214
+ /** A discovery-surfaced candidate (config is partial + non-secret). */
14215
+ var DiscoveredTargetSchema = object({
14216
+ kind: string(),
14217
+ suggestedName: string(),
14218
+ config: record(string(), unknown())
14219
+ });
14220
+ /** The degrade engine's report — what was resolved / dropped / degraded. */
14221
+ var RenderedAsSchema = object({
14222
+ level: string(),
14223
+ format: NotificationFormatSchema,
14224
+ attachmentsSent: number().int().nonnegative(),
14225
+ actionsSent: number().int().nonnegative(),
14226
+ truncated: boolean(),
14227
+ dropped: array(string())
14228
+ });
14229
+ var SendResultSchema = object({
14052
14230
  success: boolean(),
14053
- error: string().optional()
14054
- }), { kind: "mutation" });
14231
+ error: string().optional(),
14232
+ renderedAs: RenderedAsSchema.optional()
14233
+ });
14234
+ /** Same shape as SendResult — kept as a distinct name for the test panel. */
14235
+ var TestResultSchema = SendResultSchema;
14236
+ method(object({}), array(TargetKindSchema)), method(object({}), array(TargetSchema)), method(object({
14237
+ kind: string(),
14238
+ config: record(string(), unknown()).optional()
14239
+ }), array(DiscoveredTargetSchema)), method(object({
14240
+ targetId: string(),
14241
+ notification: NotificationSchema
14242
+ }), SendResultSchema, { kind: "mutation" }), method(object({
14243
+ targetId: string(),
14244
+ sample: NotificationSchema.optional()
14245
+ }), TestResultSchema, { kind: "mutation" }), method(object({ target: TargetSchema }), TargetSchema, { kind: "mutation" }), method(object({ targetId: string() }), _void(), { kind: "mutation" }), method(object({
14246
+ targetId: string(),
14247
+ enabled: boolean()
14248
+ }), _void(), { kind: "mutation" });
14055
14249
  /**
14056
14250
  * Zod schemas for persisted record types.
14057
14251
  *
@@ -14841,10 +15035,11 @@ var pipelineOrchestratorCapability = {
14841
15035
  }))),
14842
15036
  /**
14843
15037
  * Get one camera's decoder placement (computed if not yet pinned).
14844
- * Consumed by `stream-broker.createBroker` so decoder provider
14845
- * selection is deterministic fixes the 2026-04-18 race where
14846
- * `capProviders[0]` silently picked ffmpeg-on-agent-0 for a
14847
- * hub-assigned camera.
15038
+ *
15039
+ * ADVISORY today: reports the orchestrator's decoder preference only.
15040
+ * Actual decode placement is broker-owned (local-node pin + frame-plane
15041
+ * co-location guard). Reserved to become the binding source/decoder-owner
15042
+ * control in the stream-LB epic (Phase 2).
14848
15043
  *
14849
15044
  * `pipelineNodeId` is the node already chosen to run inference for
14850
15045
  * this camera. When provided, the balancer prefers co-location with
@@ -20216,13 +20411,49 @@ Object.freeze({
20216
20411
  addonId: null,
20217
20412
  access: "create"
20218
20413
  },
20414
+ "notificationOutput.deleteTarget": {
20415
+ capName: "notification-output",
20416
+ capScope: "system",
20417
+ addonId: null,
20418
+ access: "delete"
20419
+ },
20420
+ "notificationOutput.discoverTargets": {
20421
+ capName: "notification-output",
20422
+ capScope: "system",
20423
+ addonId: null,
20424
+ access: "view"
20425
+ },
20426
+ "notificationOutput.listTargetKinds": {
20427
+ capName: "notification-output",
20428
+ capScope: "system",
20429
+ addonId: null,
20430
+ access: "view"
20431
+ },
20432
+ "notificationOutput.listTargets": {
20433
+ capName: "notification-output",
20434
+ capScope: "system",
20435
+ addonId: null,
20436
+ access: "view"
20437
+ },
20219
20438
  "notificationOutput.send": {
20220
20439
  capName: "notification-output",
20221
20440
  capScope: "system",
20222
20441
  addonId: null,
20223
20442
  access: "create"
20224
20443
  },
20225
- "notificationOutput.sendTest": {
20444
+ "notificationOutput.setTargetEnabled": {
20445
+ capName: "notification-output",
20446
+ capScope: "system",
20447
+ addonId: null,
20448
+ access: "create"
20449
+ },
20450
+ "notificationOutput.testTarget": {
20451
+ capName: "notification-output",
20452
+ capScope: "system",
20453
+ addonId: null,
20454
+ access: "create"
20455
+ },
20456
+ "notificationOutput.upsertTarget": {
20226
20457
  capName: "notification-output",
20227
20458
  capScope: "system",
20228
20459
  addonId: null,
@@ -22398,470 +22629,232 @@ function buildTreeFromAddons(enabled, catalog) {
22398
22629
  return roots;
22399
22630
  }
22400
22631
  //#endregion
22401
- //#region src/zones-provider.ts
22632
+ //#region src/audio-chunk-poller.ts
22402
22633
  /**
22403
- * `zones-provider.ts` — implements `zonesCapability` for the orchestrator.
22634
+ * `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
22635
+ * plane (Phase 5 / D9).
22404
22636
  *
22405
- * Per-camera CRUD over polygon detection zones. Persists to the
22406
- * orchestrator's per-device settings store under the `zones` key and
22407
- * mirrors every change into the device-state `zones` slice via
22408
- * `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
22409
- * pipeline-executor, analytics, admin UI) read the live state with
22410
- * the canonical `dev.state.zones.onChanged` channel.
22637
+ * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
22638
+ * path. A live callback cannot cross a process boundary; once the `pipeline`
22639
+ * group is dissolved (Task 8) the orchestrator runs in a different process
22640
+ * from the broker, so audio delivery must go over tRPC.
22411
22641
  *
22412
- * Onboard / firmware-reported zones are out of scope for now — every
22413
- * zone is operator-drawn. The provider keeps the surface symmetric:
22414
- * `addZone` rejects id collisions, `updateZone` requires an existing
22415
- * id, `removeZone` is idempotent.
22416
- */
22417
- var ZONES_STORE_KEY = "zones";
22418
- var ZONES_CAP_NAME = "zones";
22419
- var ZonesArraySchema = array(ZoneSchema);
22420
- var ZonesProvider = class {
22421
- ctx;
22422
- /** Per-device cache. Hydrated lazily on first read for a device. */
22423
- cache = /* @__PURE__ */ new Map();
22424
- /**
22425
- * Per-device durable handle over the `zones` store key. The WHOLE
22426
- * validated zone array round-trips on every read/write so no field can
22427
- * be dropped on persist. Built lazily + memoised per device.
22428
- */
22429
- stateByDevice = /* @__PURE__ */ new Map();
22430
- constructor(ctx) {
22431
- this.ctx = ctx;
22432
- }
22433
- /** Lazily build (and memoise) the durable `zones` handle for a device. */
22434
- zonesState(deviceId) {
22435
- let handle = this.stateByDevice.get(deviceId);
22436
- if (!handle) {
22437
- handle = createDurableState({
22438
- key: ZONES_STORE_KEY,
22439
- schema: ZonesArraySchema,
22440
- fallback: [],
22441
- read: () => this.ctx.settings.readDeviceStore(deviceId),
22442
- write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
22443
- onParseError: (key, error) => this.ctx.logger.warn("zones store entry failed to parse — resetting", {
22444
- tags: { deviceId },
22445
- meta: {
22446
- key,
22447
- error: error instanceof Error ? error.message : String(error)
22448
- }
22449
- })
22450
- });
22451
- this.stateByDevice.set(deviceId, handle);
22452
- }
22453
- return handle;
22454
- }
22455
- async listZones({ deviceId }) {
22456
- return this.loadZones(deviceId);
22457
- }
22458
- async addZone({ deviceId, zone }) {
22459
- const existing = await this.loadZones(deviceId);
22460
- if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
22461
- await this.persist(deviceId, [...existing, zone]);
22462
- }
22463
- async updateZone({ deviceId, zone }) {
22464
- const existing = await this.loadZones(deviceId);
22465
- if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
22466
- const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
22467
- await this.persist(deviceId, next);
22468
- }
22469
- async removeZone({ deviceId, zoneId }) {
22470
- const existing = await this.loadZones(deviceId);
22471
- if (!existing.some((entry) => entry.id === zoneId)) return;
22472
- await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
22473
- }
22474
- /**
22475
- * Drop a device's cache entry. Called when the device is removed so
22476
- * the next attach starts from a fresh disk read.
22477
- */
22478
- forgetDevice(deviceId) {
22479
- this.cache.delete(deviceId);
22480
- this.stateByDevice.delete(deviceId);
22481
- }
22482
- async loadZones(deviceId) {
22483
- const cached = this.cache.get(deviceId);
22484
- if (cached) return cached;
22485
- let zones = [];
22486
- try {
22487
- zones = await this.zonesState(deviceId).get();
22488
- } catch (err) {
22489
- this.ctx.logger.warn("zones store read failed — using empty list", {
22490
- tags: { deviceId },
22491
- meta: { error: err instanceof Error ? err.message : String(err) }
22492
- });
22493
- }
22494
- this.cache.set(deviceId, zones);
22495
- return zones;
22496
- }
22497
- async persist(deviceId, zones) {
22498
- this.cache.set(deviceId, zones);
22499
- await this.zonesState(deviceId).set(zones);
22500
- await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
22501
- capName: ZONES_CAP_NAME,
22502
- slice: { zones }
22503
- });
22504
- this.ctx.onZonesChanged?.(deviceId, zones);
22505
- }
22506
- };
22507
- //#endregion
22508
- //#region src/zone-rules-provider.ts
22509
- /**
22510
- * `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
22511
- * orchestrator.
22642
+ * The consumer:
22512
22643
  *
22513
- * Per-stage rule arrays (motion / detection) live next to zones in the
22514
- * orchestrator's per-device store under `zoneRules.<stage>` keys.
22515
- * Every mutation mirrors to a stage-specific device-state slice
22516
- * (`motion-zone-rules`, `detection-zone-rules`) so consumer addons
22517
- * subscribe independently and pick up the new gating without an extra
22518
- * cap round-trip.
22644
+ * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC the broker
22645
+ * registers a per-subscription bounded FIFO queue and returns a
22646
+ * `subscriptionId`;
22647
+ * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
22648
+ * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
22649
+ * 3. feeds each chunk to its downstream audio logic;
22650
+ * 4. on teardown, `unsubscribeAudioChunks`.
22519
22651
  *
22520
- * The provider validates each rule against {@link ZoneRuleSchema}
22521
- * before persisting partial / corrupt writes are rejected outright
22522
- * since rules drive runtime filtering and a bad payload would silently
22523
- * widen the operator's intended scope.
22652
+ * Audio is not latency-critical like video, and chunks arrive only ~every
22653
+ * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
22654
+ * a small per-poll burst keeps latency low without busy-spinning. The
22655
+ * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
22656
+ * loses a chunk.
22657
+ *
22658
+ * Boot-race tolerance: the broker for a given camStream may not be registered
22659
+ * yet when the orchestrator wires the subscription (provider addons publish
22660
+ * their cameraStreams asynchronously after their probe completes).
22661
+ * `subscribeAudioChunks` retries with exponential backoff (capped at
22662
+ * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
22663
+ * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
22664
+ * shape so video and audio plumbing self-heal identically.
22524
22665
  */
22525
- /** Settings store key for stage rules. Kept under a single nested
22526
- * object so a future stage just adds another property without
22527
- * reshuffling the schema. */
22528
- var RULES_STORE_KEY = "zoneRules";
22529
- /** Cap name for the unified runtime-state slice. Matches the cap's
22530
- * declared `name` so the codegen DeviceProxy auto-wires
22531
- * `device.state.zoneRules`. The slice value is the full
22532
- * `{motion, detection}` object — both stages travel together so a
22533
- * single reactive handle covers every consumer. */
22534
- var ZONE_RULES_CAP_NAME = "zone-rules";
22535
- var RulesArraySchema = array(ZoneRuleSchema);
22666
+ /** Poll period audio chunks arrive ~every 500ms; 200ms keeps latency low. */
22667
+ var POLL_INTERVAL_MS = 200;
22668
+ /** How many chunks to drain per poll — a small burst absorbs jitter. */
22669
+ var PULL_MAX_COUNT = 8;
22536
22670
  /**
22537
- * Whole-blob schema for the per-device `zoneRules` store key. Both stages
22538
- * travel together under one key. Deliberately lenient each stage is
22539
- * `unknown` so a corrupt single stage can NOT drop the sibling on load
22540
- * (durable-state `get()` falls back to `{}` only on a whole-blob parse
22541
- * failure). Strict per-stage validation, with its own reset-on-corrupt
22542
- * warn, still happens in `loadRules` exactly as before the migration.
22671
+ * Consecutive pull failures before we attempt to re-subscribe. A single failed
22672
+ * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
22673
+ * sustained failure means the broker child restarted and dropped our
22674
+ * subscription, so we re-establish it.
22543
22675
  */
22544
- var ZoneRulesBlockSchema = object({
22545
- motion: unknown().optional(),
22546
- detection: unknown().optional()
22547
- }).passthrough();
22548
- var ZoneRulesProvider = class {
22549
- ctx;
22550
- /** Per-device per-stage cache. Hydrated lazily on first read. */
22551
- cache = /* @__PURE__ */ new Map();
22552
- /**
22553
- * Per-device durable handle over the `zoneRules` store key. The WHOLE
22554
- * `{motion?, detection?}` block round-trips on every read/write so a
22555
- * write on one stage can never drop the other. Built lazily + memoised.
22556
- */
22557
- stateByDevice = /* @__PURE__ */ new Map();
22558
- constructor(ctx) {
22559
- this.ctx = ctx;
22560
- }
22561
- /** Lazily build (and memoise) the durable `zoneRules` handle for a device. */
22562
- rulesState(deviceId) {
22563
- let handle = this.stateByDevice.get(deviceId);
22564
- if (!handle) {
22565
- handle = createDurableState({
22566
- key: RULES_STORE_KEY,
22567
- schema: ZoneRulesBlockSchema,
22568
- fallback: {},
22569
- read: () => this.ctx.settings.readDeviceStore(deviceId),
22570
- write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
22571
- onParseError: (key, error) => this.ctx.logger.warn("zone-rules store block failed to parse — resetting", {
22572
- tags: { deviceId },
22573
- meta: {
22574
- key,
22575
- error: error instanceof Error ? error.message : String(error)
22576
- }
22577
- })
22578
- });
22579
- this.stateByDevice.set(deviceId, handle);
22676
+ var RESUBSCRIBE_AFTER_FAILURES = 2;
22677
+ /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
22678
+ var RESUBSCRIBE_THROTTLE_TICKS = 5;
22679
+ /** First subscribe-retry delay, doubled on every subsequent failure. */
22680
+ var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
22681
+ /**
22682
+ * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller — fast
22683
+ * enough to recover within a single reconcile of the orchestrator and slow
22684
+ * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
22685
+ */
22686
+ var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
22687
+ /**
22688
+ * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
22689
+ *
22690
+ * Always resolves to a teardown closure — when the broker is not yet
22691
+ * registered the closure cancels the ongoing retry loop; when polling is
22692
+ * active it stops the loop and releases the broker subscription. Mirrors
22693
+ * `startFrameHandlePoller` so video and audio recover identically.
22694
+ */
22695
+ function startAudioChunkPoller(options) {
22696
+ const lifecycle = {
22697
+ stopped: false,
22698
+ retryTimer: void 0,
22699
+ pollTimer: void 0,
22700
+ activeSubscriptionId: null
22701
+ };
22702
+ const teardown = () => {
22703
+ if (lifecycle.stopped) return;
22704
+ lifecycle.stopped = true;
22705
+ if (lifecycle.retryTimer) {
22706
+ clearTimeout(lifecycle.retryTimer);
22707
+ lifecycle.retryTimer = void 0;
22580
22708
  }
22581
- return handle;
22582
- }
22583
- async listRules({ deviceId, stage }) {
22584
- return this.loadRules(deviceId, stage);
22585
- }
22586
- async setRules({ deviceId, stage, rules }) {
22587
- const parsed = RulesArraySchema.safeParse(rules);
22588
- if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
22589
- await this.persist(deviceId, stage, parsed.data);
22590
- }
22591
- /** Drop a device's cache entries. Called when the device is removed. */
22592
- forgetDevice(deviceId) {
22593
- this.cache.delete(deviceId);
22594
- this.stateByDevice.delete(deviceId);
22595
- }
22596
- async loadRules(deviceId, stage) {
22597
- let perDevice = this.cache.get(deviceId);
22598
- if (!perDevice) {
22599
- perDevice = /* @__PURE__ */ new Map();
22600
- this.cache.set(deviceId, perDevice);
22709
+ if (lifecycle.pollTimer) {
22710
+ clearTimeout(lifecycle.pollTimer);
22711
+ lifecycle.pollTimer = void 0;
22601
22712
  }
22602
- const cached = perDevice.get(stage);
22603
- if (cached) return cached;
22604
- let rules = [];
22713
+ const subId = lifecycle.activeSubscriptionId;
22714
+ if (subId) {
22715
+ lifecycle.activeSubscriptionId = null;
22716
+ options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }).catch((err) => {
22717
+ options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
22718
+ brokerId: options.brokerId,
22719
+ subscriptionId: subId,
22720
+ error: errMsg(err)
22721
+ } });
22722
+ });
22723
+ }
22724
+ };
22725
+ subscribeWithRetry(options, lifecycle);
22726
+ return teardown;
22727
+ }
22728
+ /**
22729
+ * Run the subscribe → poll handshake with exponential backoff on subscribe
22730
+ * failures. Resolves once the subscription is acquired (and the poll loop has
22731
+ * been started) or once `lifecycle.stopped` flips, whichever comes first.
22732
+ */
22733
+ async function subscribeWithRetry(options, lifecycle) {
22734
+ const { api, brokerId, tag, logger } = options;
22735
+ let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
22736
+ let attempt = 0;
22737
+ while (!lifecycle.stopped) {
22738
+ attempt += 1;
22605
22739
  try {
22606
- const raw = (await this.rulesState(deviceId).get())[stage];
22607
- if (raw !== void 0) {
22608
- const parsed = RulesArraySchema.safeParse(raw);
22609
- if (parsed.success) rules = parsed.data;
22610
- else this.ctx.logger.warn("zone-rules store entry failed to parse — resetting", {
22611
- tags: { deviceId },
22612
- meta: {
22613
- stage,
22614
- issues: parsed.error.issues
22615
- }
22740
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
22741
+ brokerId,
22742
+ tag
22743
+ });
22744
+ if (lifecycle.stopped) {
22745
+ await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
22746
+ logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
22747
+ brokerId,
22748
+ subscriptionId: result.subscriptionId,
22749
+ error: errMsg(err)
22750
+ } });
22616
22751
  });
22752
+ return;
22617
22753
  }
22754
+ if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
22755
+ brokerId,
22756
+ tag,
22757
+ attempt
22758
+ } });
22759
+ lifecycle.activeSubscriptionId = result.subscriptionId;
22760
+ startPolling(options, lifecycle);
22761
+ return;
22618
22762
  } catch (err) {
22619
- this.ctx.logger.warn("zone-rules store read failed — using empty list", {
22620
- tags: { deviceId },
22621
- meta: {
22622
- stage,
22623
- error: err instanceof Error ? err.message : String(err)
22624
- }
22625
- });
22763
+ if (lifecycle.stopped) return;
22764
+ if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
22765
+ brokerId,
22766
+ tag,
22767
+ error: errMsg(err),
22768
+ nextRetryInMs: backoffMs
22769
+ } });
22770
+ else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
22771
+ brokerId,
22772
+ tag,
22773
+ attempt,
22774
+ error: errMsg(err),
22775
+ nextRetryInMs: backoffMs
22776
+ } });
22777
+ await sleep(backoffMs, lifecycle);
22778
+ backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
22626
22779
  }
22627
- perDevice.set(stage, rules);
22628
- return rules;
22629
22780
  }
22630
- async persist(deviceId, stage, rules) {
22631
- let perDevice = this.cache.get(deviceId);
22632
- if (!perDevice) {
22633
- perDevice = /* @__PURE__ */ new Map();
22634
- this.cache.set(deviceId, perDevice);
22781
+ }
22782
+ /**
22783
+ * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
22784
+ * trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
22785
+ * the broker child restart case where our `subscriptionId` is silently
22786
+ * disowned.
22787
+ */
22788
+ function startPolling(options, lifecycle) {
22789
+ const { api, brokerId, tag, onChunk, logger } = options;
22790
+ let consecutiveFailures = 0;
22791
+ const resubscribe = async () => {
22792
+ try {
22793
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
22794
+ brokerId,
22795
+ tag
22796
+ });
22797
+ lifecycle.activeSubscriptionId = result.subscriptionId;
22798
+ logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
22799
+ brokerId,
22800
+ tag,
22801
+ subscriptionId: result.subscriptionId,
22802
+ afterFailures: consecutiveFailures
22803
+ } });
22804
+ return true;
22805
+ } catch {
22806
+ return false;
22635
22807
  }
22636
- perDevice.set(stage, rules);
22637
- await this.rulesState(deviceId).update((prev) => ({
22638
- ...prev,
22639
- [stage]: rules
22640
- }));
22641
- const otherStage = stage === "motion" ? "detection" : "motion";
22642
- const otherRules = perDevice.get(otherStage) ?? await this.loadRules(deviceId, otherStage);
22643
- const sliceValue = stage === "motion" ? {
22644
- motion: rules,
22645
- detection: otherRules
22646
- } : {
22647
- motion: otherRules,
22648
- detection: rules
22649
- };
22808
+ };
22809
+ const tick = async () => {
22810
+ if (lifecycle.stopped) return;
22811
+ const subId = lifecycle.activeSubscriptionId;
22812
+ if (!subId) return;
22650
22813
  try {
22651
- await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
22652
- capName: ZONE_RULES_CAP_NAME,
22653
- slice: sliceValue
22814
+ const chunks = await api.streamBroker.pullAudioChunks.query({
22815
+ subscriptionId: subId,
22816
+ maxCount: PULL_MAX_COUNT
22654
22817
  });
22818
+ if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
22819
+ brokerId,
22820
+ subscriptionId: subId
22821
+ } });
22822
+ consecutiveFailures = 0;
22823
+ for (const chunk of chunks) {
22824
+ if (lifecycle.stopped) break;
22825
+ await onChunk(chunk);
22826
+ }
22655
22827
  } catch (err) {
22656
- this.ctx.logger.debug("zone-rules slice mirror failed", {
22657
- tags: { deviceId },
22658
- meta: {
22659
- stage,
22660
- error: err instanceof Error ? err.message : String(err)
22661
- }
22662
- });
22663
- }
22664
- this.ctx.onRulesChanged?.(deviceId, stage, rules);
22665
- }
22666
- };
22667
- //#endregion
22668
- //#region src/orchestrator-store-schemas.ts
22669
- /**
22670
- * `orchestrator-store-schemas.ts` — whole-blob Zod schemas for the
22671
- * orchestrator's addon-store keys that persist an ENTIRE collection
22672
- * under a single key (`nodeBindings`, `templates`, `agentSettings`,
22673
- * `cameraSettings`).
22674
- *
22675
- * These back the durable-state primitive (`this.state(KEY, Schema, {})`)
22676
- * so a hand-written serializer can never silently drop a field on save:
22677
- * the whole validated value round-trips on every read and write.
22678
- *
22679
- * Each schema mirrors the SAME shape the orchestrator already persisted
22680
- * (the `pipeline-orchestrator.cap.ts` storage contract). Every field that
22681
- * is only conditionally written is `.optional()`, so an existing blob
22682
- * that predates a newer field still loads — durable-state `get()` only
22683
- * falls back to the empty map on a whole-blob parse failure, so the
22684
- * schema is kept faithful to the on-disk shape rather than overly strict.
22685
- */
22686
- var EngineChoiceSchema = object({
22687
- runtime: _enum(["node", "python"]),
22688
- backend: string(),
22689
- format: string(),
22690
- device: string().optional()
22691
- });
22692
- var NodeBindingsSchema = record(string(), record(string(), string()));
22693
- var StoredPipelineConfigSchema = object({
22694
- engine: EngineChoiceSchema,
22695
- steps: array(PipelineStepInputSchema).readonly(),
22696
- audio: object({
22697
- engine: EngineChoiceSchema,
22698
- modelId: string(),
22699
- enabled: boolean(),
22700
- settings: record(string(), unknown()).readonly().optional()
22701
- }).nullable().optional()
22702
- });
22703
- var StoredPipelineTemplateSchema = object({
22704
- id: string(),
22705
- name: string(),
22706
- description: string().optional(),
22707
- config: StoredPipelineConfigSchema,
22708
- createdAt: string(),
22709
- updatedAt: string()
22710
- });
22711
- var TemplatesSchema = record(string(), StoredPipelineTemplateSchema);
22712
- var StoredAgentAddonConfigSchema = object({
22713
- enabled: boolean(),
22714
- modelId: string(),
22715
- settings: record(string(), unknown()).readonly()
22716
- });
22717
- var StoredAgentPipelineSettingsSchema = object({
22718
- addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
22719
- maxCameras: number().int().nonnegative().nullable().default(null)
22720
- });
22721
- var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
22722
- var StoredCameraStepOverridePatchSchema = object({
22723
- enabled: boolean().optional(),
22724
- modelId: string().optional(),
22725
- settings: record(string(), unknown()).readonly().optional()
22726
- });
22727
- var StoredCameraPipelineForAgentSchema = object({
22728
- steps: array(PipelineStepInputSchema).readonly(),
22729
- audio: object({
22730
- modelId: string(),
22731
- enabled: boolean()
22732
- }).nullable()
22733
- });
22734
- var StoredCameraPipelineSettingsSchema = object({
22735
- pinnedAgentNodeId: string().optional(),
22736
- stepToggles: record(string(), boolean()).optional(),
22737
- stepOverridesByAgent: record(string(), record(string(), StoredCameraStepOverridePatchSchema)).optional(),
22738
- pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
22739
- /**
22740
- * Legacy "nuke inference" flag. Superseded by the detection-pipeline
22741
- * wrapper binding, but the one-shot boot migration
22742
- * (`migrateLegacyFlagsToBindings`) still reads it off the persisted
22743
- * blob to flip the binding off. Kept here so the durable round-trip
22744
- * does NOT strip it before the migration runs.
22745
- */
22746
- disableInference: boolean().optional()
22747
- });
22748
- var CameraSettingsMapSchema = record(string(), StoredCameraPipelineSettingsSchema);
22749
- //#endregion
22750
- //#region src/load-balancer.ts
22751
- /**
22752
- * Compute the L2 capacity score for a runner node. Lower is better.
22753
- * The score is a weighted sum of the runner's active workload so the balancer
22754
- * prefers agents that are serving fewer cameras OR draining queues quickly.
22755
- *
22756
- * Rationale:
22757
- * - `attachedCameras * avgInferenceFps` approximates the total inference rate
22758
- * the agent is currently sustaining (not just how many cameras are assigned).
22759
- * - `queueDepthTotal` penalises agents that are falling behind the frame feed.
22760
- */
22761
- function computeCapacityScore(load) {
22762
- return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
22763
- }
22764
- /**
22765
- * Returns true when the node has remaining capacity.
22766
- * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
22767
- * its `attachedCameras` count is strictly less than the cap.
22768
- * Pins count toward the cap via `attachedCameras`.
22769
- */
22770
- function isEligible(node, caps) {
22771
- const cap = caps?.[node.nodeId];
22772
- if (cap === null || cap === void 0 || cap <= 0) return true;
22773
- return node.attachedCameras < cap;
22774
- }
22775
- /**
22776
- * Run the two-level camera balancer.
22777
- *
22778
- * L1 (manual affinity): if `preferredAgent` names an online node AND that
22779
- * node is under its `maxCameras` cap, return it. If the node is online but at
22780
- * or over cap, return `{kind:'pending'}` — a pinned camera is never silently
22781
- * over-assigned.
22782
- *
22783
- * L2 (capacity): filter to eligible nodes and pick the lowest capacity score.
22784
- * If all nodes are at/over cap, return `{kind:'pending'}`.
22785
- *
22786
- * Returns `null` when no runners are online. The orchestrator decides how to
22787
- * react — typically by logging and deferring the assignment until a runner
22788
- * comes online.
22789
- */
22790
- function balance(input) {
22791
- const online = input.nodes.filter((n) => n.nodeId.length > 0);
22792
- if (online.length === 0) return null;
22793
- const eligible = online.filter((n) => isEligible(n, input.nodeCaps));
22794
- if (input.preferredAgent) {
22795
- const pinnedOnline = online.find((n) => n.nodeId === input.preferredAgent);
22796
- if (pinnedOnline) {
22797
- if (eligible.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
22798
- kind: "assigned",
22799
- agentNodeId: pinnedOnline.nodeId,
22800
- reason: "manual",
22801
- score: computeCapacityScore(pinnedOnline)
22802
- };
22803
- return {
22804
- kind: "pending",
22805
- reason: "over-cap"
22806
- };
22828
+ consecutiveFailures += 1;
22829
+ if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
22830
+ brokerId,
22831
+ subscriptionId: subId,
22832
+ error: errMsg(err)
22833
+ } });
22834
+ if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
22807
22835
  }
22808
- }
22809
- if (eligible.length === 0) return {
22810
- kind: "pending",
22811
- reason: "over-cap"
22812
- };
22813
- const best = eligible.map((node) => ({
22814
- node,
22815
- score: computeCapacityScore(node)
22816
- })).toSorted((a, b) => a.score - b.score)[0];
22817
- return {
22818
- kind: "assigned",
22819
- agentNodeId: best.node.nodeId,
22820
- reason: "capacity",
22821
- score: best.score
22836
+ if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
22822
22837
  };
22838
+ tick();
22823
22839
  }
22824
22840
  /**
22825
- * Decide whether a manual per-device decoder pin may be honored.
22826
- *
22827
- * A pin to a node OUTSIDE `enabledDecoderNodes` must NOT be honored blindly —
22828
- * that is how video decode reaches a node that is not eligible to decode (e.g.
22829
- * a node whose shm frame ring the broker cannot read, or one an operator
22830
- * disabled). A pin to an ENABLED node is honored; anything else falls through
22831
- * to the auto-balance path (which filters by `enabledDecoderNodes`).
22832
- */
22833
- function isDecoderPinHonored(pinnedNodeId, enabledDecoderNodes) {
22834
- return enabledDecoderNodes.includes(pinnedNodeId);
22835
- }
22836
- /**
22837
- * Choose decoder node. Priority: manual pin → co-located with pipeline → capacity.
22841
+ * Cancellable sleep wakes early when `lifecycle.stopped` flips. We
22842
+ * keep a local wrapper around the shared {@link sleep} helper because
22843
+ * the lifecycle tracks the active retry timer for `teardown()` to
22844
+ * clear; pure `sleep()` would leak the timer if teardown fired while
22845
+ * we were waiting.
22838
22846
  */
22839
- function balanceDecoder(input) {
22840
- const { decoderNodes, pipelineNodeId, preferredDecoderNode } = input;
22841
- if (decoderNodes.length === 0) return null;
22842
- if (preferredDecoderNode) {
22843
- const match = decoderNodes.find((n) => n.nodeId === preferredDecoderNode);
22844
- if (match) return {
22845
- decoderNodeId: match.nodeId,
22846
- reason: "manual",
22847
- score: computeCapacityScore(match)
22848
- };
22849
- }
22850
- const colocated = decoderNodes.find((n) => n.nodeId === pipelineNodeId);
22851
- if (colocated) return {
22852
- decoderNodeId: colocated.nodeId,
22853
- reason: "co-located",
22854
- score: computeCapacityScore(colocated)
22855
- };
22856
- const best = decoderNodes.map((node) => ({
22857
- node,
22858
- score: computeCapacityScore(node)
22859
- })).toSorted((a, b) => a.score - b.score)[0];
22860
- return {
22861
- decoderNodeId: best.node.nodeId,
22862
- reason: "capacity",
22863
- score: best.score
22864
- };
22847
+ function sleep(ms, lifecycle) {
22848
+ return new Promise((resolve) => {
22849
+ if (lifecycle.stopped) {
22850
+ resolve();
22851
+ return;
22852
+ }
22853
+ lifecycle.retryTimer = setTimeout(() => {
22854
+ lifecycle.retryTimer = void 0;
22855
+ resolve();
22856
+ }, ms);
22857
+ });
22865
22858
  }
22866
22859
  //#endregion
22867
22860
  //#region src/audio-load-balancer.ts
@@ -22880,259 +22873,393 @@ function balanceAudio(input) {
22880
22873
  };
22881
22874
  }
22882
22875
  //#endregion
22883
- //#region src/audio-chunk-poller.ts
22884
- /**
22885
- * `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
22886
- * plane (Phase 5 / D9).
22887
- *
22888
- * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
22889
- * path. A live callback cannot cross a process boundary; once the `pipeline`
22890
- * group is dissolved (Task 8) the orchestrator runs in a different process
22891
- * from the broker, so audio delivery must go over tRPC.
22892
- *
22893
- * The consumer:
22894
- *
22895
- * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC — the broker
22896
- * registers a per-subscription bounded FIFO queue and returns a
22897
- * `subscriptionId`;
22898
- * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
22899
- * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
22900
- * 3. feeds each chunk to its downstream audio logic;
22901
- * 4. on teardown, `unsubscribeAudioChunks`.
22902
- *
22903
- * Audio is not latency-critical like video, and chunks arrive only ~every
22904
- * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
22905
- * a small per-poll burst keeps latency low without busy-spinning. The
22906
- * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
22907
- * loses a chunk.
22908
- *
22909
- * Boot-race tolerance: the broker for a given camStream may not be registered
22910
- * yet when the orchestrator wires the subscription (provider addons publish
22911
- * their cameraStreams asynchronously after their probe completes).
22912
- * `subscribeAudioChunks` retries with exponential backoff (capped at
22913
- * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
22914
- * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
22915
- * shape so video and audio plumbing self-heal identically.
22916
- */
22917
- /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
22918
- var POLL_INTERVAL_MS = 200;
22919
- /** How many chunks to drain per poll — a small burst absorbs jitter. */
22920
- var PULL_MAX_COUNT = 8;
22876
+ //#region src/camera-status/compose-camera-status.ts
22877
+ function mapAssignment(input) {
22878
+ return {
22879
+ detectionNodeId: input.detectionNodeId,
22880
+ decoderNodeId: input.decoderNodeId,
22881
+ audioNodeId: input.audioNodeId,
22882
+ pinned: {
22883
+ detection: input.pinned.detection,
22884
+ decoder: input.pinned.decoder,
22885
+ audio: input.pinned.audio
22886
+ },
22887
+ reasons: {
22888
+ detection: input.reasons.detection,
22889
+ decoder: input.reasons.decoder,
22890
+ audio: input.reasons.audio
22891
+ }
22892
+ };
22893
+ }
22894
+ function mapSource(sourceResult) {
22895
+ if (sourceResult === null) return { streams: [] };
22896
+ return { streams: sourceResult.streams.map((s) => ({
22897
+ camStreamId: s.camStreamId,
22898
+ codec: s.codec,
22899
+ width: s.width,
22900
+ height: s.height,
22901
+ fps: s.fps,
22902
+ kind: s.kind
22903
+ })) };
22904
+ }
22905
+ function mapBroker(brokerResult) {
22906
+ if (brokerResult === null) return null;
22907
+ return {
22908
+ profiles: brokerResult.profiles.map((p) => ({
22909
+ profile: p.profile,
22910
+ status: p.status,
22911
+ codec: p.codec,
22912
+ width: p.width,
22913
+ height: p.height,
22914
+ subscribers: p.subscribers,
22915
+ inFps: p.inFps,
22916
+ outFps: p.outFps
22917
+ })),
22918
+ webrtcSessions: brokerResult.webrtcSessions,
22919
+ rtspRestream: brokerResult.rtspRestream
22920
+ };
22921
+ }
22922
+ function mapDecoderShm(shm) {
22923
+ return {
22924
+ framesWritten: shm.framesWritten,
22925
+ getFrameHits: shm.getFrameHits,
22926
+ getFrameMisses: shm.getFrameMisses,
22927
+ budgetMb: shm.budgetMb
22928
+ };
22929
+ }
22930
+ function mapDecoder(decoderResult) {
22931
+ if (decoderResult === null) return null;
22932
+ return {
22933
+ nodeId: decoderResult.nodeId,
22934
+ formats: [...decoderResult.formats],
22935
+ sessionCount: decoderResult.sessionCount,
22936
+ shm: mapDecoderShm(decoderResult.shm)
22937
+ };
22938
+ }
22939
+ function mapMotion(motionResult) {
22940
+ if (motionResult === null) return null;
22941
+ return {
22942
+ enabled: motionResult.enabled,
22943
+ fps: motionResult.fps
22944
+ };
22945
+ }
22946
+ function mapProvisioning(p) {
22947
+ if (p.error !== void 0) return {
22948
+ state: p.state,
22949
+ error: p.error
22950
+ };
22951
+ return { state: p.state };
22952
+ }
22953
+ function mapDetection(detectionResult) {
22954
+ if (detectionResult === null) return null;
22955
+ const phase = detectionResult.phase;
22956
+ return {
22957
+ nodeId: detectionResult.nodeId,
22958
+ engine: {
22959
+ backend: detectionResult.engine.backend,
22960
+ device: detectionResult.engine.device
22961
+ },
22962
+ phase,
22963
+ configuredFps: detectionResult.configuredFps,
22964
+ actualFps: detectionResult.actualFps,
22965
+ queueDepth: detectionResult.queueDepth,
22966
+ avgInferenceMs: detectionResult.avgInferenceMs,
22967
+ provisioning: mapProvisioning(detectionResult.provisioning)
22968
+ };
22969
+ }
22970
+ function mapAudio(audioResult) {
22971
+ if (audioResult === null) return null;
22972
+ return {
22973
+ nodeId: audioResult.nodeId,
22974
+ enabled: audioResult.enabled
22975
+ };
22976
+ }
22977
+ function mapRecording(recordingResult) {
22978
+ if (recordingResult === null) return null;
22979
+ return {
22980
+ mode: recordingResult.mode,
22981
+ active: recordingResult.active,
22982
+ storageBytes: recordingResult.storageBytes
22983
+ };
22984
+ }
22921
22985
  /**
22922
- * Consecutive pull failures before we attempt to re-subscribe. A single failed
22923
- * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
22924
- * sustained failure means the broker child restarted and dropped our
22925
- * subscription, so we re-establish it.
22986
+ * Pure function that composes a `CameraStatus` from per-stage fetch results.
22987
+ *
22988
+ * - `assignment` is always built from orchestrator-local data (never null).
22989
+ * - `source` always present: defaults to `{ streams: [] }` when sourceResult is null.
22990
+ * - Every other block is null when its stage result is null (graceful degradation).
22991
+ * - `fetchedAt` is stamped exactly as provided — never calls `Date.now()`.
22992
+ * - No mutation of the input.
22926
22993
  */
22927
- var RESUBSCRIBE_AFTER_FAILURES = 2;
22928
- /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
22929
- var RESUBSCRIBE_THROTTLE_TICKS = 5;
22930
- /** First subscribe-retry delay, doubled on every subsequent failure. */
22931
- var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
22994
+ function composeCameraStatus(input) {
22995
+ return {
22996
+ deviceId: input.deviceId,
22997
+ assignment: mapAssignment(input),
22998
+ source: mapSource(input.sourceResult),
22999
+ broker: mapBroker(input.brokerResult),
23000
+ decoder: mapDecoder(input.decoderResult),
23001
+ motion: mapMotion(input.motionResult),
23002
+ detection: mapDetection(input.detectionResult),
23003
+ audio: mapAudio(input.audioResult),
23004
+ recording: mapRecording(input.recordingResult),
23005
+ fetchedAt: input.fetchedAt
23006
+ };
23007
+ }
23008
+ //#endregion
23009
+ //#region src/dispatch-reconcile.ts
22932
23010
  /**
22933
- * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller fast
22934
- * enough to recover within a single reconcile of the orchestrator and slow
22935
- * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
23011
+ * Derives the set of deviceIds that currently have at least one broker
23012
+ * profile slot that is assigned (status !== 'unassigned') AND has a
23013
+ * non-null sourceCamStreamId.
23014
+ *
23015
+ * This is the "desired" fleet — cameras the orchestrator should have
23016
+ * dispatched. Used by `reconcileDispatch` to compute the gap against
23017
+ * `cameraConfigs`.
22936
23018
  */
22937
- var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
23019
+ function desiredDeviceIdsFromSlots(slots) {
23020
+ const result = /* @__PURE__ */ new Set();
23021
+ for (const slot of slots) if (slot.status !== "unassigned" && slot.sourceCamStreamId !== null) result.add(slot.deviceId);
23022
+ return result;
23023
+ }
22938
23024
  /**
22939
- * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
23025
+ * Returns the deviceIds that are present in `desired` but absent from
23026
+ * `known`. These are the cameras the orchestrator has not yet dispatched
23027
+ * and needs to process via `handleDeviceRegistered`.
22940
23028
  *
22941
- * Always resolves to a teardown closure when the broker is not yet
22942
- * registered the closure cancels the ongoing retry loop; when polling is
22943
- * active it stops the loop and releases the broker subscription. Mirrors
22944
- * `startFrameHandlePoller` so video and audio recover identically.
23029
+ * Additive only cameras present in `known` are never included, even
23030
+ * if they fall out of `desired`, to avoid fighting the live event path.
22945
23031
  */
22946
- function startAudioChunkPoller(options) {
22947
- const lifecycle = {
22948
- stopped: false,
22949
- retryTimer: void 0,
22950
- pollTimer: void 0,
22951
- activeSubscriptionId: null
22952
- };
22953
- const teardown = () => {
22954
- if (lifecycle.stopped) return;
22955
- lifecycle.stopped = true;
22956
- if (lifecycle.retryTimer) {
22957
- clearTimeout(lifecycle.retryTimer);
22958
- lifecycle.retryTimer = void 0;
22959
- }
22960
- if (lifecycle.pollTimer) {
22961
- clearTimeout(lifecycle.pollTimer);
22962
- lifecycle.pollTimer = void 0;
22963
- }
22964
- const subId = lifecycle.activeSubscriptionId;
22965
- if (subId) {
22966
- lifecycle.activeSubscriptionId = null;
22967
- options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }).catch((err) => {
22968
- options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
22969
- brokerId: options.brokerId,
22970
- subscriptionId: subId,
22971
- error: errMsg(err)
22972
- } });
22973
- });
22974
- }
22975
- };
22976
- subscribeWithRetry(options, lifecycle);
22977
- return teardown;
23032
+ function computeDispatchGap(desired, known) {
23033
+ return [...desired].filter((id) => !known.has(id));
22978
23034
  }
23035
+ //#endregion
23036
+ //#region src/load-balancer.ts
22979
23037
  /**
22980
- * Run the subscribe poll handshake with exponential backoff on subscribe
22981
- * failures. Resolves once the subscription is acquired (and the poll loop has
22982
- * been started) or once `lifecycle.stopped` flips, whichever comes first.
23038
+ * Compute the L2 capacity score for a runner node. Lower is better.
23039
+ * The score is a weighted sum of the runner's active workload so the balancer
23040
+ * prefers agents that are serving fewer cameras OR draining queues quickly.
23041
+ *
23042
+ * Rationale:
23043
+ * - `attachedCameras * avgInferenceFps` approximates the total inference rate
23044
+ * the agent is currently sustaining (not just how many cameras are assigned).
23045
+ * - `queueDepthTotal` penalises agents that are falling behind the frame feed.
22983
23046
  */
22984
- async function subscribeWithRetry(options, lifecycle) {
22985
- const { api, brokerId, tag, logger } = options;
22986
- let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
22987
- let attempt = 0;
22988
- while (!lifecycle.stopped) {
22989
- attempt += 1;
22990
- try {
22991
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
22992
- brokerId,
22993
- tag
22994
- });
22995
- if (lifecycle.stopped) {
22996
- await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
22997
- logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
22998
- brokerId,
22999
- subscriptionId: result.subscriptionId,
23000
- error: errMsg(err)
23001
- } });
23002
- });
23003
- return;
23004
- }
23005
- if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
23006
- brokerId,
23007
- tag,
23008
- attempt
23009
- } });
23010
- lifecycle.activeSubscriptionId = result.subscriptionId;
23011
- startPolling(options, lifecycle);
23012
- return;
23013
- } catch (err) {
23014
- if (lifecycle.stopped) return;
23015
- if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
23016
- brokerId,
23017
- tag,
23018
- error: errMsg(err),
23019
- nextRetryInMs: backoffMs
23020
- } });
23021
- else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
23022
- brokerId,
23023
- tag,
23024
- attempt,
23025
- error: errMsg(err),
23026
- nextRetryInMs: backoffMs
23027
- } });
23028
- await sleep(backoffMs, lifecycle);
23029
- backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
23030
- }
23031
- }
23047
+ function computeCapacityScore(load) {
23048
+ return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
23032
23049
  }
23033
23050
  /**
23034
- * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
23035
- * trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
23036
- * the broker child restart case where our `subscriptionId` is silently
23037
- * disowned.
23051
+ * Returns true when the node has remaining capacity.
23052
+ * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
23053
+ * its `attachedCameras` count is strictly less than the cap.
23054
+ * Pins count toward the cap via `attachedCameras`.
23038
23055
  */
23039
- function startPolling(options, lifecycle) {
23040
- const { api, brokerId, tag, onChunk, logger } = options;
23041
- let consecutiveFailures = 0;
23042
- const resubscribe = async () => {
23043
- try {
23044
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
23045
- brokerId,
23046
- tag
23047
- });
23048
- lifecycle.activeSubscriptionId = result.subscriptionId;
23049
- logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
23050
- brokerId,
23051
- tag,
23052
- subscriptionId: result.subscriptionId,
23053
- afterFailures: consecutiveFailures
23054
- } });
23055
- return true;
23056
- } catch {
23057
- return false;
23056
+ function isEligible(node, caps) {
23057
+ const cap = caps?.[node.nodeId];
23058
+ if (cap === null || cap === void 0 || cap <= 0) return true;
23059
+ return node.attachedCameras < cap;
23060
+ }
23061
+ /**
23062
+ * Run the two-level camera balancer.
23063
+ *
23064
+ * Frame-source constraint: when `eligibleNodes` is set, only nodes in that set
23065
+ * can obtain this camera's decoded frames. Such a node is a prerequisite for
23066
+ * BOTH L1 and L2 — a node outside `eligibleNodes` is never assignable, and a
23067
+ * pin to an online-but-ineligible node returns `{kind:'pending',
23068
+ * reason:'no-frame-source'}` (mirrors the over-cap-pin behaviour: a pinned
23069
+ * camera is never silently placed on a different node).
23070
+ *
23071
+ * L1 (manual affinity): if `preferredAgent` names an online + frame-sourceable
23072
+ * node AND that node is under its `maxCameras` cap, return it. If the node is
23073
+ * online but at/over cap, return `{kind:'pending', reason:'over-cap'}`; if it
23074
+ * is online but not frame-sourceable, `{kind:'pending', reason:'no-frame-source'}`
23075
+ * — a pinned camera is never silently over-assigned or re-homed.
23076
+ *
23077
+ * L2 (capacity): among the frame-sourceable nodes, filter to those under cap
23078
+ * and pick the lowest capacity score. If no frame-sourceable node exists (but
23079
+ * online nodes do), return `no-frame-source`; if they exist but are all at/over
23080
+ * cap, return `over-cap`.
23081
+ *
23082
+ * Returns `null` when no runners are online. The orchestrator decides how to
23083
+ * react — typically by logging and deferring the assignment until a runner
23084
+ * comes online.
23085
+ */
23086
+ function balance(input) {
23087
+ const online = input.nodes.filter((n) => n.nodeId.length > 0);
23088
+ if (online.length === 0) return null;
23089
+ const sourceable = input.eligibleNodes ? online.filter((n) => input.eligibleNodes.includes(n.nodeId)) : online;
23090
+ const eligible = sourceable.filter((n) => isEligible(n, input.nodeCaps));
23091
+ if (input.preferredAgent) {
23092
+ const pinnedOnline = online.find((n) => n.nodeId === input.preferredAgent);
23093
+ if (pinnedOnline) {
23094
+ if (!sourceable.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
23095
+ kind: "pending",
23096
+ reason: "no-frame-source"
23097
+ };
23098
+ if (eligible.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
23099
+ kind: "assigned",
23100
+ agentNodeId: pinnedOnline.nodeId,
23101
+ reason: "manual",
23102
+ score: computeCapacityScore(pinnedOnline)
23103
+ };
23104
+ return {
23105
+ kind: "pending",
23106
+ reason: "over-cap"
23107
+ };
23058
23108
  }
23109
+ }
23110
+ if (eligible.length === 0) return {
23111
+ kind: "pending",
23112
+ reason: sourceable.length === 0 ? "no-frame-source" : "over-cap"
23059
23113
  };
23060
- const tick = async () => {
23061
- if (lifecycle.stopped) return;
23062
- const subId = lifecycle.activeSubscriptionId;
23063
- if (!subId) return;
23064
- try {
23065
- const chunks = await api.streamBroker.pullAudioChunks.query({
23066
- subscriptionId: subId,
23067
- maxCount: PULL_MAX_COUNT
23068
- });
23069
- if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
23070
- brokerId,
23071
- subscriptionId: subId
23072
- } });
23073
- consecutiveFailures = 0;
23074
- for (const chunk of chunks) {
23075
- if (lifecycle.stopped) break;
23076
- await onChunk(chunk);
23077
- }
23078
- } catch (err) {
23079
- consecutiveFailures += 1;
23080
- if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
23081
- brokerId,
23082
- subscriptionId: subId,
23083
- error: errMsg(err)
23084
- } });
23085
- if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
23086
- }
23087
- if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
23114
+ const best = eligible.map((node) => ({
23115
+ node,
23116
+ score: computeCapacityScore(node)
23117
+ })).toSorted((a, b) => a.score - b.score)[0];
23118
+ return {
23119
+ kind: "assigned",
23120
+ agentNodeId: best.node.nodeId,
23121
+ reason: "capacity",
23122
+ score: best.score
23088
23123
  };
23089
- tick();
23090
23124
  }
23091
23125
  /**
23092
- * Cancellable sleep wakes early when `lifecycle.stopped` flips. We
23093
- * keep a local wrapper around the shared {@link sleep} helper because
23094
- * the lifecycle tracks the active retry timer for `teardown()` to
23095
- * clear; pure `sleep()` would leak the timer if teardown fired while
23096
- * we were waiting.
23126
+ * Decide whether a manual per-device decoder pin may be honored.
23127
+ *
23128
+ * A pin to a node OUTSIDE `enabledDecoderNodes` must NOT be honored blindly —
23129
+ * that is how video decode reaches a node that is not eligible to decode (e.g.
23130
+ * a node whose shm frame ring the broker cannot read, or one an operator
23131
+ * disabled). A pin to an ENABLED node is honored; anything else falls through
23132
+ * to the auto-balance path (which filters by `enabledDecoderNodes`).
23097
23133
  */
23098
- function sleep(ms, lifecycle) {
23099
- return new Promise((resolve) => {
23100
- if (lifecycle.stopped) {
23101
- resolve();
23102
- return;
23103
- }
23104
- lifecycle.retryTimer = setTimeout(() => {
23105
- lifecycle.retryTimer = void 0;
23106
- resolve();
23107
- }, ms);
23108
- });
23134
+ function isDecoderPinHonored(pinnedNodeId, enabledDecoderNodes) {
23135
+ return enabledDecoderNodes.includes(pinnedNodeId);
23136
+ }
23137
+ /**
23138
+ * Choose decoder node. Priority: manual pin → co-located with pipeline → capacity.
23139
+ */
23140
+ function balanceDecoder(input) {
23141
+ const { decoderNodes, pipelineNodeId, preferredDecoderNode } = input;
23142
+ if (decoderNodes.length === 0) return null;
23143
+ if (preferredDecoderNode) {
23144
+ const match = decoderNodes.find((n) => n.nodeId === preferredDecoderNode);
23145
+ if (match) return {
23146
+ decoderNodeId: match.nodeId,
23147
+ reason: "manual",
23148
+ score: computeCapacityScore(match)
23149
+ };
23150
+ }
23151
+ const colocated = decoderNodes.find((n) => n.nodeId === pipelineNodeId);
23152
+ if (colocated) return {
23153
+ decoderNodeId: colocated.nodeId,
23154
+ reason: "co-located",
23155
+ score: computeCapacityScore(colocated)
23156
+ };
23157
+ const best = decoderNodes.map((node) => ({
23158
+ node,
23159
+ score: computeCapacityScore(node)
23160
+ })).toSorted((a, b) => a.score - b.score)[0];
23161
+ return {
23162
+ decoderNodeId: best.node.nodeId,
23163
+ reason: "capacity",
23164
+ score: best.score
23165
+ };
23109
23166
  }
23110
23167
  //#endregion
23111
- //#region src/dispatch-reconcile.ts
23168
+ //#region src/orchestrator-store-schemas.ts
23112
23169
  /**
23113
- * Derives the set of deviceIds that currently have at least one broker
23114
- * profile slot that is assigned (status !== 'unassigned') AND has a
23115
- * non-null sourceCamStreamId.
23170
+ * `orchestrator-store-schemas.ts` whole-blob Zod schemas for the
23171
+ * orchestrator's addon-store keys that persist an ENTIRE collection
23172
+ * under a single key (`nodeBindings`, `templates`, `agentSettings`,
23173
+ * `cameraSettings`).
23116
23174
  *
23117
- * This is the "desired" fleet cameras the orchestrator should have
23118
- * dispatched. Used by `reconcileDispatch` to compute the gap against
23119
- * `cameraConfigs`.
23175
+ * These back the durable-state primitive (`this.state(KEY, Schema, {})`)
23176
+ * so a hand-written serializer can never silently drop a field on save:
23177
+ * the whole validated value round-trips on every read and write.
23178
+ *
23179
+ * Each schema mirrors the SAME shape the orchestrator already persisted
23180
+ * (the `pipeline-orchestrator.cap.ts` storage contract). Every field that
23181
+ * is only conditionally written is `.optional()`, so an existing blob
23182
+ * that predates a newer field still loads — durable-state `get()` only
23183
+ * falls back to the empty map on a whole-blob parse failure, so the
23184
+ * schema is kept faithful to the on-disk shape rather than overly strict.
23120
23185
  */
23121
- function desiredDeviceIdsFromSlots(slots) {
23122
- const result = /* @__PURE__ */ new Set();
23123
- for (const slot of slots) if (slot.status !== "unassigned" && slot.sourceCamStreamId !== null) result.add(slot.deviceId);
23124
- return result;
23125
- }
23186
+ var EngineChoiceSchema = object({
23187
+ runtime: _enum(["node", "python"]),
23188
+ backend: string(),
23189
+ format: string(),
23190
+ device: string().optional()
23191
+ });
23192
+ var NodeBindingsSchema = record(string(), record(string(), string()));
23193
+ var StoredPipelineConfigSchema = object({
23194
+ engine: EngineChoiceSchema,
23195
+ steps: array(PipelineStepInputSchema).readonly(),
23196
+ audio: object({
23197
+ engine: EngineChoiceSchema,
23198
+ modelId: string(),
23199
+ enabled: boolean(),
23200
+ settings: record(string(), unknown()).readonly().optional()
23201
+ }).nullable().optional()
23202
+ });
23203
+ var StoredPipelineTemplateSchema = object({
23204
+ id: string(),
23205
+ name: string(),
23206
+ description: string().optional(),
23207
+ config: StoredPipelineConfigSchema,
23208
+ createdAt: string(),
23209
+ updatedAt: string()
23210
+ });
23211
+ var TemplatesSchema = record(string(), StoredPipelineTemplateSchema);
23212
+ var StoredAgentAddonConfigSchema = object({
23213
+ enabled: boolean(),
23214
+ modelId: string(),
23215
+ settings: record(string(), unknown()).readonly()
23216
+ });
23217
+ var StoredAgentPipelineSettingsSchema = object({
23218
+ addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
23219
+ maxCameras: number().int().nonnegative().nullable().default(null)
23220
+ });
23221
+ var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
23222
+ var StoredCameraStepOverridePatchSchema = object({
23223
+ enabled: boolean().optional(),
23224
+ modelId: string().optional(),
23225
+ settings: record(string(), unknown()).readonly().optional()
23226
+ });
23227
+ var StoredCameraPipelineForAgentSchema = object({
23228
+ steps: array(PipelineStepInputSchema).readonly(),
23229
+ audio: object({
23230
+ modelId: string(),
23231
+ enabled: boolean()
23232
+ }).nullable()
23233
+ });
23234
+ var StoredCameraPipelineSettingsSchema = object({
23235
+ pinnedAgentNodeId: string().optional(),
23236
+ stepToggles: record(string(), boolean()).optional(),
23237
+ stepOverridesByAgent: record(string(), record(string(), StoredCameraStepOverridePatchSchema)).optional(),
23238
+ pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
23239
+ /**
23240
+ * Legacy "nuke inference" flag. Superseded by the detection-pipeline
23241
+ * wrapper binding, but the one-shot boot migration
23242
+ * (`migrateLegacyFlagsToBindings`) still reads it off the persisted
23243
+ * blob to flip the binding off. Kept here so the durable round-trip
23244
+ * does NOT strip it before the migration runs.
23245
+ */
23246
+ disableInference: boolean().optional()
23247
+ });
23248
+ var CameraSettingsMapSchema = record(string(), StoredCameraPipelineSettingsSchema);
23249
+ //#endregion
23250
+ //#region src/pending-retry.ts
23126
23251
  /**
23127
- * Returns the deviceIds that are present in `desired` but absent from
23128
- * `known`. These are the cameras the orchestrator has not yet dispatched
23129
- * and needs to process via `handleDeviceRegistered`.
23252
+ * Returns the deviceIds that are KNOWN (present in `known`) but currently
23253
+ * UNASSIGNED (absent from `assigned`). These are the "stranded" cameras the
23254
+ * orchestrator has a runner config for them but no live pipeline assignment
23255
+ * (pending / over-cap / no-frame-source / load-shed).
23130
23256
  *
23131
- * Additive only cameras present in `known` are never included, even
23132
- * if they fall out of `desired`, to avoid fighting the live event path.
23257
+ * Mirror of `computeDispatchGap` in `dispatch-reconcile.ts`: a plain set
23258
+ * difference (`known \ assigned`). Pure no side effects, deterministic
23259
+ * ordering (iteration order of `known`).
23133
23260
  */
23134
- function computeDispatchGap(desired, known) {
23135
- return [...desired].filter((id) => !known.has(id));
23261
+ function computeStrandedDevices(known, assigned) {
23262
+ return [...known].filter((id) => !assigned.has(id));
23136
23263
  }
23137
23264
  //#endregion
23138
23265
  //#region src/pipeline-watchdog.ts
@@ -23165,219 +23292,407 @@ var PipelineWatchdog = class {
23165
23292
  });
23166
23293
  this.state.set(cam.deviceId, stages);
23167
23294
  }
23168
- unregister(deviceId) {
23169
- this.cameras.delete(deviceId);
23170
- this.state.delete(deviceId);
23295
+ unregister(deviceId) {
23296
+ this.cameras.delete(deviceId);
23297
+ this.state.delete(deviceId);
23298
+ }
23299
+ /** Called when a stage's live signal arrives (audio chunk / motion frame / inference). */
23300
+ noteSignal(deviceId, stage) {
23301
+ const rt = this.state.get(deviceId)?.get(stage);
23302
+ if (!rt) return;
23303
+ rt.lastSeenMs = this.deps.now();
23304
+ rt.attempts = 0;
23305
+ }
23306
+ tick() {
23307
+ const now = this.deps.now();
23308
+ for (const cam of this.cameras.values()) {
23309
+ const { line, stalled, recoveries } = this.evaluate(cam, now);
23310
+ if (stalled) this.deps.logger.warn(line);
23311
+ else this.deps.logger.info(line);
23312
+ for (const r of recoveries) {
23313
+ const rt = this.state.get(cam.deviceId)?.get(r.stage);
23314
+ if (rt) rt.attempts += 1;
23315
+ this.deps.recover(cam.deviceId, r.stage, r.streamId);
23316
+ }
23317
+ }
23318
+ }
23319
+ start(intervalMs) {
23320
+ if (this.timer) return;
23321
+ this.timer = setInterval(() => this.tick(), intervalMs);
23322
+ }
23323
+ stop() {
23324
+ if (this.timer) clearInterval(this.timer);
23325
+ this.timer = null;
23326
+ }
23327
+ evaluate(cam, now) {
23328
+ const parts = [];
23329
+ const recoveries = [];
23330
+ let stalled = false;
23331
+ const stageOrder = [
23332
+ "audio",
23333
+ "motion",
23334
+ "detection"
23335
+ ];
23336
+ const stageMode = {
23337
+ audio: cam.audioMode,
23338
+ motion: cam.motionSources.includes("analyzer") ? "analyzer" : cam.motionSources.join("+"),
23339
+ detection: cam.detectionMode
23340
+ };
23341
+ for (const stage of stageOrder) {
23342
+ const streamId = cam.continuousStages.get(stage);
23343
+ if (streamId === void 0) {
23344
+ parts.push(`${stage}=${stageMode[stage]}(idle)`);
23345
+ continue;
23346
+ }
23347
+ const rt = this.state.get(cam.deviceId)?.get(stage);
23348
+ if (!rt) {
23349
+ parts.push(`${stage}=${stageMode[stage]}(unknown)`);
23350
+ continue;
23351
+ }
23352
+ const staleness = now - rt.lastSeenMs;
23353
+ const thresholdMs = this.deps.thresholds[STAGE_THRESHOLD_KEY[stage]];
23354
+ const sSec = Math.round(staleness / 1e3);
23355
+ if (staleness <= thresholdMs) parts.push(`${stage}=${stageMode[stage]}@${streamId}(OK ${sSec}s)`);
23356
+ else if (rt.attempts >= this.deps.thresholds.maxRecoveryAttempts) {
23357
+ stalled = true;
23358
+ parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · exhausted ${rt.attempts})`);
23359
+ } else {
23360
+ stalled = true;
23361
+ recoveries.push({
23362
+ stage,
23363
+ streamId
23364
+ });
23365
+ parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · recovering ${rt.attempts + 1}/${this.deps.thresholds.maxRecoveryAttempts})`);
23366
+ }
23367
+ }
23368
+ return {
23369
+ line: `[watchdog] cam${cam.deviceId} ${cam.deviceName} · ${parts.join(" · ")} → ${stalled ? "STALLED" : "OK"}`,
23370
+ stalled,
23371
+ recoveries
23372
+ };
23373
+ }
23374
+ };
23375
+ //#endregion
23376
+ //#region src/remote-health.ts
23377
+ /**
23378
+ * Default thresholds: 3-min grace (cold start), 2-min stale window (metrics
23379
+ * stopped entirely), 0.5 fps floor. A degraded foreign-handle camera runs
23380
+ * ~1 fps (above the bar); with T2's frame-source eligibility a remote
23381
+ * assignment is only legal on a frame-source node, so sustained sub-0.5 fps
23382
+ * means genuinely broken.
23383
+ */
23384
+ var DEFAULT_REMOTE_HEALTH_OPTS = {
23385
+ graceMs: 3 * 6e4,
23386
+ staleMs: 2 * 6e4,
23387
+ minFps: .5
23388
+ };
23389
+ /**
23390
+ * Evaluate every remote pipeline assignment and return the cameras that must be
23391
+ * re-placed. Rules (each assignment evaluated independently):
23392
+ * 1. LOCAL assignments (`agentNodeId === localNodeId`) are skipped — the hub
23393
+ * watchdog owns them.
23394
+ * 2. Cameras not in `activeDeviceIds` are skipped — detection isn't expected.
23395
+ * 3. Assignments younger than `graceMs` are skipped — cold-start grace.
23396
+ * 4. No fps entry, or `now - lastSeen > staleMs` → `stale-metrics`.
23397
+ * 5. Otherwise `fps < minFps` → `zero-fps`.
23398
+ * 6. Otherwise healthy → no action.
23399
+ *
23400
+ * Deterministic ordering: iteration order of `assignments`.
23401
+ */
23402
+ function evaluateRemoteHealth(input) {
23403
+ const { assignments, fpsMap, activeDeviceIds, localNodeId, now, opts } = input;
23404
+ const actions = [];
23405
+ for (const [deviceId, assignment] of assignments) {
23406
+ if (assignment.agentNodeId === localNodeId) continue;
23407
+ if (!activeDeviceIds.has(deviceId)) continue;
23408
+ if (now - assignment.assignedAt < opts.graceMs) continue;
23409
+ const entry = fpsMap.get(deviceId);
23410
+ if (!entry || now - entry.lastSeen > opts.staleMs) {
23411
+ actions.push({
23412
+ deviceId,
23413
+ kind: "replace",
23414
+ why: "stale-metrics"
23415
+ });
23416
+ continue;
23417
+ }
23418
+ if (entry.fps < opts.minFps) {
23419
+ actions.push({
23420
+ deviceId,
23421
+ kind: "replace",
23422
+ why: "zero-fps"
23423
+ });
23424
+ continue;
23425
+ }
23426
+ }
23427
+ return actions;
23428
+ }
23429
+ //#endregion
23430
+ //#region src/zone-rules-provider.ts
23431
+ /**
23432
+ * `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
23433
+ * orchestrator.
23434
+ *
23435
+ * Per-stage rule arrays (motion / detection) live next to zones in the
23436
+ * orchestrator's per-device store under `zoneRules.<stage>` keys.
23437
+ * Every mutation mirrors to a stage-specific device-state slice
23438
+ * (`motion-zone-rules`, `detection-zone-rules`) so consumer addons
23439
+ * subscribe independently and pick up the new gating without an extra
23440
+ * cap round-trip.
23441
+ *
23442
+ * The provider validates each rule against {@link ZoneRuleSchema}
23443
+ * before persisting — partial / corrupt writes are rejected outright
23444
+ * since rules drive runtime filtering and a bad payload would silently
23445
+ * widen the operator's intended scope.
23446
+ */
23447
+ /** Settings store key for stage rules. Kept under a single nested
23448
+ * object so a future stage just adds another property without
23449
+ * reshuffling the schema. */
23450
+ var RULES_STORE_KEY = "zoneRules";
23451
+ /** Cap name for the unified runtime-state slice. Matches the cap's
23452
+ * declared `name` so the codegen DeviceProxy auto-wires
23453
+ * `device.state.zoneRules`. The slice value is the full
23454
+ * `{motion, detection}` object — both stages travel together so a
23455
+ * single reactive handle covers every consumer. */
23456
+ var ZONE_RULES_CAP_NAME = "zone-rules";
23457
+ var RulesArraySchema = array(ZoneRuleSchema);
23458
+ /**
23459
+ * Whole-blob schema for the per-device `zoneRules` store key. Both stages
23460
+ * travel together under one key. Deliberately lenient — each stage is
23461
+ * `unknown` so a corrupt single stage can NOT drop the sibling on load
23462
+ * (durable-state `get()` falls back to `{}` only on a whole-blob parse
23463
+ * failure). Strict per-stage validation, with its own reset-on-corrupt
23464
+ * warn, still happens in `loadRules` exactly as before the migration.
23465
+ */
23466
+ var ZoneRulesBlockSchema = object({
23467
+ motion: unknown().optional(),
23468
+ detection: unknown().optional()
23469
+ }).passthrough();
23470
+ var ZoneRulesProvider = class {
23471
+ ctx;
23472
+ /** Per-device per-stage cache. Hydrated lazily on first read. */
23473
+ cache = /* @__PURE__ */ new Map();
23474
+ /**
23475
+ * Per-device durable handle over the `zoneRules` store key. The WHOLE
23476
+ * `{motion?, detection?}` block round-trips on every read/write so a
23477
+ * write on one stage can never drop the other. Built lazily + memoised.
23478
+ */
23479
+ stateByDevice = /* @__PURE__ */ new Map();
23480
+ constructor(ctx) {
23481
+ this.ctx = ctx;
23482
+ }
23483
+ /** Lazily build (and memoise) the durable `zoneRules` handle for a device. */
23484
+ rulesState(deviceId) {
23485
+ let handle = this.stateByDevice.get(deviceId);
23486
+ if (!handle) {
23487
+ handle = createDurableState({
23488
+ key: RULES_STORE_KEY,
23489
+ schema: ZoneRulesBlockSchema,
23490
+ fallback: {},
23491
+ read: () => this.ctx.settings.readDeviceStore(deviceId),
23492
+ write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
23493
+ onParseError: (key, error) => this.ctx.logger.warn("zone-rules store block failed to parse — resetting", {
23494
+ tags: { deviceId },
23495
+ meta: {
23496
+ key,
23497
+ error: error instanceof Error ? error.message : String(error)
23498
+ }
23499
+ })
23500
+ });
23501
+ this.stateByDevice.set(deviceId, handle);
23502
+ }
23503
+ return handle;
23504
+ }
23505
+ async listRules({ deviceId, stage }) {
23506
+ return this.loadRules(deviceId, stage);
23507
+ }
23508
+ async setRules({ deviceId, stage, rules }) {
23509
+ const parsed = RulesArraySchema.safeParse(rules);
23510
+ if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
23511
+ await this.persist(deviceId, stage, parsed.data);
23171
23512
  }
23172
- /** Called when a stage's live signal arrives (audio chunk / motion frame / inference). */
23173
- noteSignal(deviceId, stage) {
23174
- const rt = this.state.get(deviceId)?.get(stage);
23175
- if (!rt) return;
23176
- rt.lastSeenMs = this.deps.now();
23177
- rt.attempts = 0;
23513
+ /** Drop a device's cache entries. Called when the device is removed. */
23514
+ forgetDevice(deviceId) {
23515
+ this.cache.delete(deviceId);
23516
+ this.stateByDevice.delete(deviceId);
23178
23517
  }
23179
- tick() {
23180
- const now = this.deps.now();
23181
- for (const cam of this.cameras.values()) {
23182
- const { line, stalled, recoveries } = this.evaluate(cam, now);
23183
- if (stalled) this.deps.logger.warn(line);
23184
- else this.deps.logger.info(line);
23185
- for (const r of recoveries) {
23186
- const rt = this.state.get(cam.deviceId)?.get(r.stage);
23187
- if (rt) rt.attempts += 1;
23188
- this.deps.recover(cam.deviceId, r.stage, r.streamId);
23518
+ async loadRules(deviceId, stage) {
23519
+ let perDevice = this.cache.get(deviceId);
23520
+ if (!perDevice) {
23521
+ perDevice = /* @__PURE__ */ new Map();
23522
+ this.cache.set(deviceId, perDevice);
23523
+ }
23524
+ const cached = perDevice.get(stage);
23525
+ if (cached) return cached;
23526
+ let rules = [];
23527
+ try {
23528
+ const raw = (await this.rulesState(deviceId).get())[stage];
23529
+ if (raw !== void 0) {
23530
+ const parsed = RulesArraySchema.safeParse(raw);
23531
+ if (parsed.success) rules = parsed.data;
23532
+ else this.ctx.logger.warn("zone-rules store entry failed to parse — resetting", {
23533
+ tags: { deviceId },
23534
+ meta: {
23535
+ stage,
23536
+ issues: parsed.error.issues
23537
+ }
23538
+ });
23189
23539
  }
23540
+ } catch (err) {
23541
+ this.ctx.logger.warn("zone-rules store read failed — using empty list", {
23542
+ tags: { deviceId },
23543
+ meta: {
23544
+ stage,
23545
+ error: err instanceof Error ? err.message : String(err)
23546
+ }
23547
+ });
23190
23548
  }
23549
+ perDevice.set(stage, rules);
23550
+ return rules;
23191
23551
  }
23192
- start(intervalMs) {
23193
- if (this.timer) return;
23194
- this.timer = setInterval(() => this.tick(), intervalMs);
23195
- }
23196
- stop() {
23197
- if (this.timer) clearInterval(this.timer);
23198
- this.timer = null;
23199
- }
23200
- evaluate(cam, now) {
23201
- const parts = [];
23202
- const recoveries = [];
23203
- let stalled = false;
23204
- const stageOrder = [
23205
- "audio",
23206
- "motion",
23207
- "detection"
23208
- ];
23209
- const stageMode = {
23210
- audio: cam.audioMode,
23211
- motion: cam.motionSources.includes("analyzer") ? "analyzer" : cam.motionSources.join("+"),
23212
- detection: cam.detectionMode
23552
+ async persist(deviceId, stage, rules) {
23553
+ let perDevice = this.cache.get(deviceId);
23554
+ if (!perDevice) {
23555
+ perDevice = /* @__PURE__ */ new Map();
23556
+ this.cache.set(deviceId, perDevice);
23557
+ }
23558
+ perDevice.set(stage, rules);
23559
+ await this.rulesState(deviceId).update((prev) => ({
23560
+ ...prev,
23561
+ [stage]: rules
23562
+ }));
23563
+ const otherStage = stage === "motion" ? "detection" : "motion";
23564
+ const otherRules = perDevice.get(otherStage) ?? await this.loadRules(deviceId, otherStage);
23565
+ const sliceValue = stage === "motion" ? {
23566
+ motion: rules,
23567
+ detection: otherRules
23568
+ } : {
23569
+ motion: otherRules,
23570
+ detection: rules
23213
23571
  };
23214
- for (const stage of stageOrder) {
23215
- const streamId = cam.continuousStages.get(stage);
23216
- if (streamId === void 0) {
23217
- parts.push(`${stage}=${stageMode[stage]}(idle)`);
23218
- continue;
23219
- }
23220
- const rt = this.state.get(cam.deviceId)?.get(stage);
23221
- if (!rt) {
23222
- parts.push(`${stage}=${stageMode[stage]}(unknown)`);
23223
- continue;
23224
- }
23225
- const staleness = now - rt.lastSeenMs;
23226
- const thresholdMs = this.deps.thresholds[STAGE_THRESHOLD_KEY[stage]];
23227
- const sSec = Math.round(staleness / 1e3);
23228
- if (staleness <= thresholdMs) parts.push(`${stage}=${stageMode[stage]}@${streamId}(OK ${sSec}s)`);
23229
- else if (rt.attempts >= this.deps.thresholds.maxRecoveryAttempts) {
23230
- stalled = true;
23231
- parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · exhausted ${rt.attempts})`);
23232
- } else {
23233
- stalled = true;
23234
- recoveries.push({
23572
+ try {
23573
+ await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
23574
+ capName: ZONE_RULES_CAP_NAME,
23575
+ slice: sliceValue
23576
+ });
23577
+ } catch (err) {
23578
+ this.ctx.logger.debug("zone-rules slice mirror failed", {
23579
+ tags: { deviceId },
23580
+ meta: {
23235
23581
  stage,
23236
- streamId
23237
- });
23238
- parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · recovering ${rt.attempts + 1}/${this.deps.thresholds.maxRecoveryAttempts})`);
23239
- }
23582
+ error: err instanceof Error ? err.message : String(err)
23583
+ }
23584
+ });
23240
23585
  }
23241
- return {
23242
- line: `[watchdog] cam${cam.deviceId} ${cam.deviceName} · ${parts.join(" · ")} → ${stalled ? "STALLED" : "OK"}`,
23243
- stalled,
23244
- recoveries
23245
- };
23586
+ this.ctx.onRulesChanged?.(deviceId, stage, rules);
23246
23587
  }
23247
23588
  };
23248
23589
  //#endregion
23249
- //#region src/camera-status/compose-camera-status.ts
23250
- function mapAssignment(input) {
23251
- return {
23252
- detectionNodeId: input.detectionNodeId,
23253
- decoderNodeId: input.decoderNodeId,
23254
- audioNodeId: input.audioNodeId,
23255
- pinned: {
23256
- detection: input.pinned.detection,
23257
- decoder: input.pinned.decoder,
23258
- audio: input.pinned.audio
23259
- },
23260
- reasons: {
23261
- detection: input.reasons.detection,
23262
- decoder: input.reasons.decoder,
23263
- audio: input.reasons.audio
23264
- }
23265
- };
23266
- }
23267
- function mapSource(sourceResult) {
23268
- if (sourceResult === null) return { streams: [] };
23269
- return { streams: sourceResult.streams.map((s) => ({
23270
- camStreamId: s.camStreamId,
23271
- codec: s.codec,
23272
- width: s.width,
23273
- height: s.height,
23274
- fps: s.fps,
23275
- kind: s.kind
23276
- })) };
23277
- }
23278
- function mapBroker(brokerResult) {
23279
- if (brokerResult === null) return null;
23280
- return {
23281
- profiles: brokerResult.profiles.map((p) => ({
23282
- profile: p.profile,
23283
- status: p.status,
23284
- codec: p.codec,
23285
- width: p.width,
23286
- height: p.height,
23287
- subscribers: p.subscribers,
23288
- inFps: p.inFps,
23289
- outFps: p.outFps
23290
- })),
23291
- webrtcSessions: brokerResult.webrtcSessions,
23292
- rtspRestream: brokerResult.rtspRestream
23293
- };
23294
- }
23295
- function mapDecoderShm(shm) {
23296
- return {
23297
- framesWritten: shm.framesWritten,
23298
- getFrameHits: shm.getFrameHits,
23299
- getFrameMisses: shm.getFrameMisses,
23300
- budgetMb: shm.budgetMb
23301
- };
23302
- }
23303
- function mapDecoder(decoderResult) {
23304
- if (decoderResult === null) return null;
23305
- return {
23306
- nodeId: decoderResult.nodeId,
23307
- formats: [...decoderResult.formats],
23308
- sessionCount: decoderResult.sessionCount,
23309
- shm: mapDecoderShm(decoderResult.shm)
23310
- };
23311
- }
23312
- function mapMotion(motionResult) {
23313
- if (motionResult === null) return null;
23314
- return {
23315
- enabled: motionResult.enabled,
23316
- fps: motionResult.fps
23317
- };
23318
- }
23319
- function mapProvisioning(p) {
23320
- if (p.error !== void 0) return {
23321
- state: p.state,
23322
- error: p.error
23323
- };
23324
- return { state: p.state };
23325
- }
23326
- function mapDetection(detectionResult) {
23327
- if (detectionResult === null) return null;
23328
- const phase = detectionResult.phase;
23329
- return {
23330
- nodeId: detectionResult.nodeId,
23331
- engine: {
23332
- backend: detectionResult.engine.backend,
23333
- device: detectionResult.engine.device
23334
- },
23335
- phase,
23336
- configuredFps: detectionResult.configuredFps,
23337
- actualFps: detectionResult.actualFps,
23338
- queueDepth: detectionResult.queueDepth,
23339
- avgInferenceMs: detectionResult.avgInferenceMs,
23340
- provisioning: mapProvisioning(detectionResult.provisioning)
23341
- };
23342
- }
23343
- function mapAudio(audioResult) {
23344
- if (audioResult === null) return null;
23345
- return {
23346
- nodeId: audioResult.nodeId,
23347
- enabled: audioResult.enabled
23348
- };
23349
- }
23350
- function mapRecording(recordingResult) {
23351
- if (recordingResult === null) return null;
23352
- return {
23353
- mode: recordingResult.mode,
23354
- active: recordingResult.active,
23355
- storageBytes: recordingResult.storageBytes
23356
- };
23357
- }
23590
+ //#region src/zones-provider.ts
23358
23591
  /**
23359
- * Pure function that composes a `CameraStatus` from per-stage fetch results.
23592
+ * `zones-provider.ts` implements `zonesCapability` for the orchestrator.
23593
+ *
23594
+ * Per-camera CRUD over polygon detection zones. Persists to the
23595
+ * orchestrator's per-device settings store under the `zones` key and
23596
+ * mirrors every change into the device-state `zones` slice via
23597
+ * `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
23598
+ * pipeline-executor, analytics, admin UI) read the live state with
23599
+ * the canonical `dev.state.zones.onChanged` channel.
23360
23600
  *
23361
- * - `assignment` is always built from orchestrator-local data (never null).
23362
- * - `source` always present: defaults to `{ streams: [] }` when sourceResult is null.
23363
- * - Every other block is null when its stage result is null (graceful degradation).
23364
- * - `fetchedAt` is stamped exactly as provided — never calls `Date.now()`.
23365
- * - No mutation of the input.
23601
+ * Onboard / firmware-reported zones are out of scope for now every
23602
+ * zone is operator-drawn. The provider keeps the surface symmetric:
23603
+ * `addZone` rejects id collisions, `updateZone` requires an existing
23604
+ * id, `removeZone` is idempotent.
23366
23605
  */
23367
- function composeCameraStatus(input) {
23368
- return {
23369
- deviceId: input.deviceId,
23370
- assignment: mapAssignment(input),
23371
- source: mapSource(input.sourceResult),
23372
- broker: mapBroker(input.brokerResult),
23373
- decoder: mapDecoder(input.decoderResult),
23374
- motion: mapMotion(input.motionResult),
23375
- detection: mapDetection(input.detectionResult),
23376
- audio: mapAudio(input.audioResult),
23377
- recording: mapRecording(input.recordingResult),
23378
- fetchedAt: input.fetchedAt
23379
- };
23380
- }
23606
+ var ZONES_STORE_KEY = "zones";
23607
+ var ZONES_CAP_NAME = "zones";
23608
+ var ZonesArraySchema = array(ZoneSchema);
23609
+ var ZonesProvider = class {
23610
+ ctx;
23611
+ /** Per-device cache. Hydrated lazily on first read for a device. */
23612
+ cache = /* @__PURE__ */ new Map();
23613
+ /**
23614
+ * Per-device durable handle over the `zones` store key. The WHOLE
23615
+ * validated zone array round-trips on every read/write so no field can
23616
+ * be dropped on persist. Built lazily + memoised per device.
23617
+ */
23618
+ stateByDevice = /* @__PURE__ */ new Map();
23619
+ constructor(ctx) {
23620
+ this.ctx = ctx;
23621
+ }
23622
+ /** Lazily build (and memoise) the durable `zones` handle for a device. */
23623
+ zonesState(deviceId) {
23624
+ let handle = this.stateByDevice.get(deviceId);
23625
+ if (!handle) {
23626
+ handle = createDurableState({
23627
+ key: ZONES_STORE_KEY,
23628
+ schema: ZonesArraySchema,
23629
+ fallback: [],
23630
+ read: () => this.ctx.settings.readDeviceStore(deviceId),
23631
+ write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
23632
+ onParseError: (key, error) => this.ctx.logger.warn("zones store entry failed to parse — resetting", {
23633
+ tags: { deviceId },
23634
+ meta: {
23635
+ key,
23636
+ error: error instanceof Error ? error.message : String(error)
23637
+ }
23638
+ })
23639
+ });
23640
+ this.stateByDevice.set(deviceId, handle);
23641
+ }
23642
+ return handle;
23643
+ }
23644
+ async listZones({ deviceId }) {
23645
+ return this.loadZones(deviceId);
23646
+ }
23647
+ async addZone({ deviceId, zone }) {
23648
+ const existing = await this.loadZones(deviceId);
23649
+ if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
23650
+ await this.persist(deviceId, [...existing, zone]);
23651
+ }
23652
+ async updateZone({ deviceId, zone }) {
23653
+ const existing = await this.loadZones(deviceId);
23654
+ if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
23655
+ const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
23656
+ await this.persist(deviceId, next);
23657
+ }
23658
+ async removeZone({ deviceId, zoneId }) {
23659
+ const existing = await this.loadZones(deviceId);
23660
+ if (!existing.some((entry) => entry.id === zoneId)) return;
23661
+ await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
23662
+ }
23663
+ /**
23664
+ * Drop a device's cache entry. Called when the device is removed so
23665
+ * the next attach starts from a fresh disk read.
23666
+ */
23667
+ forgetDevice(deviceId) {
23668
+ this.cache.delete(deviceId);
23669
+ this.stateByDevice.delete(deviceId);
23670
+ }
23671
+ async loadZones(deviceId) {
23672
+ const cached = this.cache.get(deviceId);
23673
+ if (cached) return cached;
23674
+ let zones = [];
23675
+ try {
23676
+ zones = await this.zonesState(deviceId).get();
23677
+ } catch (err) {
23678
+ this.ctx.logger.warn("zones store read failed — using empty list", {
23679
+ tags: { deviceId },
23680
+ meta: { error: err instanceof Error ? err.message : String(err) }
23681
+ });
23682
+ }
23683
+ this.cache.set(deviceId, zones);
23684
+ return zones;
23685
+ }
23686
+ async persist(deviceId, zones) {
23687
+ this.cache.set(deviceId, zones);
23688
+ await this.zonesState(deviceId).set(zones);
23689
+ await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
23690
+ capName: ZONES_CAP_NAME,
23691
+ slice: { zones }
23692
+ });
23693
+ this.ctx.onZonesChanged?.(deviceId, zones);
23694
+ }
23695
+ };
23381
23696
  //#endregion
23382
23697
  //#region src/index.ts
23383
23698
  var PHASE_MODE_VALUES = new Set([
@@ -23396,6 +23711,24 @@ var DEFAULT_BROKER_CALL_TIMEOUT_MS = 5e3;
23396
23711
  /** Debounce window for `scheduleReconcile` — coalesces bursts of topology/slot-change signals. */
23397
23712
  var RECONCILE_DEBOUNCE_MS = 200;
23398
23713
  /**
23714
+ * Periodic sweep interval for `retryPendingDispatches` — re-dispatches cameras
23715
+ * that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
23716
+ * `reconcileDispatch` is additive-only and never revisits these, so a slow
23717
+ * safety-net timer + event-driven debounce triggers recover them.
23718
+ */
23719
+ var PENDING_RETRY_INTERVAL_MS = 6e4;
23720
+ /** Debounce window for `schedulePendingRetry` — coalesces capacity/eligibility/readiness signals. */
23721
+ var PENDING_RETRY_DEBOUNCE_MS = 2e3;
23722
+ /**
23723
+ * Remote-assignment health loop (T7) bounded backoff. A remote camera flagged
23724
+ * as unhealthy (0-fps / metrics-stale) is re-placed at most
23725
+ * `REMOTE_HEALTH_MAX_ATTEMPTS` times inside a rolling `REMOTE_HEALTH_WINDOW_MS`
23726
+ * window; once exhausted the camera is left visibly pending (`'unhealthy'`) for
23727
+ * the operator instead of churning forever.
23728
+ */
23729
+ var REMOTE_HEALTH_MAX_ATTEMPTS = 3;
23730
+ var REMOTE_HEALTH_WINDOW_MS = 60 * 6e4;
23731
+ /**
23399
23732
  * Device-details keys routed through the orchestrator's pipeline
23400
23733
  * settings writer instead of the device orchestration store. The
23401
23734
  * `cameraPipeline` key carries the full `CameraPipelineConfig`
@@ -23484,6 +23817,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
23484
23817
  zoneRulesProvider = null;
23485
23818
  /** In-memory assignment map — mirrored into events + per-device settings for the pin. */
23486
23819
  assignments = /* @__PURE__ */ new Map();
23820
+ /**
23821
+ * Why a camera is currently unassigned (pending), keyed by deviceId. Set on
23822
+ * every pending placement path (over-cap, no-frame-source, load-shed) and
23823
+ * cleared when the camera is (re)assigned or released. Consumed by T3's
23824
+ * pending-retry sweep + `getCameraStatus` surface — nothing reads it yet in
23825
+ * this commit; the map is populated so T3 can wire the read side without
23826
+ * re-touching every placement path.
23827
+ *
23828
+ * `protected` (not `private`) to match the existing test-seam convention in
23829
+ * this class (`audioSubscriptions`, `audioSubLocks`) — the frame-source
23830
+ * eligibility spec subclasses the addon to assert the recorded reason.
23831
+ */
23832
+ pendingReasons = /* @__PURE__ */ new Map();
23833
+ /**
23834
+ * Remote-assignment health loop (T7) per-device backoff ledger. Each entry
23835
+ * tracks how many times a remote camera has been re-placed inside the current
23836
+ * rolling window (`REMOTE_HEALTH_WINDOW_MS`). Bounds churn: once the count
23837
+ * hits `REMOTE_HEALTH_MAX_ATTEMPTS`, the camera is left pending (`'unhealthy'`)
23838
+ * for the operator instead of being re-placed again. `protected` so the
23839
+ * remote-health spec can assert the ledger without casts.
23840
+ */
23841
+ remoteHealthAttempts = /* @__PURE__ */ new Map();
23487
23842
  /** Per-device audio node assignment — nodeId of the audio-analyzer handling this device's chunks. */
23488
23843
  audioNodeByDevice = /* @__PURE__ */ new Map();
23489
23844
  /** Assignments with metadata — replaces plain audioNodeByDevice values as the source of truth. */
@@ -23603,6 +23958,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
23603
23958
  reconcileInFlight = false;
23604
23959
  /** Set to true when a reconcile is requested while one is already in-flight; triggers a follow-up pass. */
23605
23960
  reconcileRerunRequested = false;
23961
+ /** Periodic `retryPendingDispatches` safety-net timer (T3). */
23962
+ pendingRetryTimer = null;
23963
+ /** Pending `schedulePendingRetry` debounce timer (T3). */
23964
+ pendingRetryDebounceTimer = null;
23965
+ /** True while `retryPendingDispatches` is running. */
23966
+ pendingRetryInFlight = false;
23967
+ /** Set when a pending-retry is requested while one is already in-flight; triggers a follow-up pass. */
23968
+ pendingRetryRerunRequested = false;
23606
23969
  initTimestamp = 0;
23607
23970
  constructor() {
23608
23971
  super({});
@@ -23819,6 +24182,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
23819
24182
  thresholds: DEFAULT_WATCHDOG_THRESHOLDS
23820
24183
  });
23821
24184
  this.pipelineWatchdog.start(PipelineOrchestratorAddon.WATCHDOG_INTERVAL_MS);
24185
+ this.pendingRetryTimer = setInterval(() => void this.retryPendingDispatches(), PENDING_RETRY_INTERVAL_MS);
23822
24186
  this.ctx.logger.info("Pipeline orchestrator detection-wiring subscriptions installed");
23823
24187
  this.migrateLegacyFlagsToBindings().catch((err) => {
23824
24188
  this.ctx.logger.warn("bindings migration failed — will retry on next boot", { meta: { error: errMsg(err) } });
@@ -24048,6 +24412,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24048
24412
  clearTimeout(this.reconcileTimer);
24049
24413
  this.reconcileTimer = null;
24050
24414
  }
24415
+ if (this.pendingRetryTimer !== null) {
24416
+ clearInterval(this.pendingRetryTimer);
24417
+ this.pendingRetryTimer = null;
24418
+ }
24419
+ if (this.pendingRetryDebounceTimer !== null) {
24420
+ clearTimeout(this.pendingRetryDebounceTimer);
24421
+ this.pendingRetryDebounceTimer = null;
24422
+ }
24051
24423
  this.unsubDeviceRegistered?.();
24052
24424
  this.unsubDeviceRegistered = null;
24053
24425
  this.unsubDeviceUnregistered?.();
@@ -24073,6 +24445,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24073
24445
  for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
24074
24446
  this.lazyAudioTeardownTimers.clear();
24075
24447
  this.cameraFpsMap.clear();
24448
+ this.remoteHealthAttempts.clear();
24076
24449
  this.loadShedState.clear();
24077
24450
  if (this.loadShedResumeTimer) {
24078
24451
  clearInterval(this.loadShedResumeTimer);
@@ -24134,6 +24507,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24134
24507
  windowMs
24135
24508
  }
24136
24509
  });
24510
+ this.pendingReasons.set(runnerConfig.deviceId, "load-shed");
24137
24511
  return {
24138
24512
  success: true,
24139
24513
  kind: "pending"
@@ -24147,11 +24521,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24147
24521
  const decision = balance({
24148
24522
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
24149
24523
  preferredAgent,
24150
- nodeCaps: this.buildNodeCaps()
24524
+ nodeCaps: await this.buildNodeCaps(),
24525
+ eligibleNodes: this.detectionEligibleNodes()
24151
24526
  });
24152
24527
  if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
24153
24528
  if (decision.kind === "pending") {
24154
- this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId: runnerConfig.deviceId } });
24529
+ this.pendingReasons.set(runnerConfig.deviceId, decision.reason);
24530
+ this.ctx.logger.warn("camera left pending — no eligible node", {
24531
+ tags: { deviceId: runnerConfig.deviceId },
24532
+ meta: { reason: decision.reason }
24533
+ });
24155
24534
  return {
24156
24535
  success: true,
24157
24536
  kind: "pending"
@@ -24202,11 +24581,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24202
24581
  }
24203
24582
  this.assignments.delete(input.deviceId);
24204
24583
  this.cameraConfigs.delete(input.deviceId);
24584
+ this.pendingReasons.delete(input.deviceId);
24205
24585
  this.pipelineWatchdog?.unregister(input.deviceId);
24206
24586
  return { success: true };
24207
24587
  }
24208
24588
  async assignPipeline(input) {
24209
24589
  if (!this.ctx) throw new Error("PipelineOrchestrator: assignPipeline called before initialize");
24590
+ const eligible = this.detectionEligibleNodes();
24591
+ if (!eligible.includes(input.agentNodeId)) throw new Error(`Cannot pin camera ${input.deviceId} detection to '${input.agentNodeId}': the node cannot obtain this camera's decoded frames (frame-source nodes: ${eligible.join(", ") || "none"}). Add it to Enabled Decoder Nodes first.`);
24210
24592
  await this.ctx.settings?.writeDeviceStore(input.deviceId, { [PREFERRED_AGENT_SETTING]: input.agentNodeId }).catch((err) => {
24211
24593
  const msg = errMsg(err);
24212
24594
  this.ctx.logger.warn("assignPipeline: failed to persist pin", {
@@ -24252,11 +24634,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24252
24634
  const decision = balance({
24253
24635
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
24254
24636
  preferredAgent: null,
24255
- nodeCaps: this.buildNodeCaps()
24637
+ nodeCaps: await this.buildNodeCaps(),
24638
+ eligibleNodes: this.detectionEligibleNodes()
24256
24639
  });
24257
24640
  if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
24258
- else if (decision.kind === "pending") this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId: input.deviceId } });
24259
- else {
24641
+ else if (decision.kind === "pending") {
24642
+ this.pendingReasons.set(input.deviceId, decision.reason);
24643
+ this.ctx.logger.warn("camera left pending — no eligible node", {
24644
+ tags: { deviceId: input.deviceId },
24645
+ meta: { reason: decision.reason }
24646
+ });
24647
+ } else {
24260
24648
  const targetNodeId = decision.agentNodeId;
24261
24649
  if (current && current.agentNodeId !== targetNodeId) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
24262
24650
  const msg = errMsg(err);
@@ -24294,7 +24682,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24294
24682
  async rebalance() {
24295
24683
  if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
24296
24684
  const loads = await this.collectAgentLoad({ onlyEnabled: true });
24297
- const nodeCaps = this.buildNodeCaps();
24685
+ const nodeCaps = await this.buildNodeCaps();
24298
24686
  let migrated = 0;
24299
24687
  for (const [deviceId, config] of this.cameraConfigs) {
24300
24688
  const current = this.assignments.get(deviceId);
@@ -24302,11 +24690,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24302
24690
  const decision = balance({
24303
24691
  nodes: loads,
24304
24692
  preferredAgent: await this.readPreferredAgent(deviceId),
24305
- nodeCaps
24693
+ nodeCaps,
24694
+ eligibleNodes: this.detectionEligibleNodes()
24306
24695
  });
24307
24696
  if (!decision) continue;
24308
24697
  if (decision.kind === "pending") {
24309
- this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId } });
24698
+ this.pendingReasons.set(deviceId, decision.reason);
24699
+ this.ctx.logger.warn("camera left pending — no eligible node", {
24700
+ tags: { deviceId },
24701
+ meta: { reason: decision.reason }
24702
+ });
24310
24703
  continue;
24311
24704
  }
24312
24705
  if (current && current.agentNodeId === decision.agentNodeId) continue;
@@ -24513,6 +24906,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24513
24906
  return this.enabledNodes.includes(nodeId);
24514
24907
  }
24515
24908
  /**
24909
+ * The set of nodes that can OBTAIN a camera's decoded frames.
24910
+ *
24911
+ * **Phase 1 (today):** detection HARD-requires that the runner is itself a
24912
+ * decoder node — a node can only run detection where it can read the shm
24913
+ * frame ring locally. So the frame-source set is exactly
24914
+ * `enabledDecoderNodes`.
24915
+ *
24916
+ * **Phase 2 (T10):** this becomes per-camera source/decoder ownership — a
24917
+ * detection node will be able to source a camera's frames from a co-located
24918
+ * OR a remote restream/decoder leg. Replace this body then; every predicate
24919
+ * change edits this ONE method (and `detectionEligibleNodes`).
24920
+ */
24921
+ frameSourceNodes() {
24922
+ return this.enabledDecoderNodes;
24923
+ }
24924
+ /**
24925
+ * Nodes eligible to run a camera's detection pipeline: the whitelist of
24926
+ * detection-enabled nodes (`enabledNodes`) intersected with the nodes that
24927
+ * can obtain the camera's frames (`frameSourceNodes()`). A node that is
24928
+ * detection-enabled but cannot source frames (or vice-versa) is NEVER a
24929
+ * valid placement — the balancer receives this as `eligibleNodes` and treats
24930
+ * anything outside it as unassignable (`no-frame-source`).
24931
+ *
24932
+ * Single choke point for T2/T10: passed to every `balance()` call site and
24933
+ * enforced by `assignPipeline` before persisting a pin.
24934
+ */
24935
+ detectionEligibleNodes() {
24936
+ const frameSource = this.frameSourceNodes();
24937
+ return this.enabledNodes.filter((nodeId) => frameSource.includes(nodeId));
24938
+ }
24939
+ /**
24516
24940
  * Query every online runner in the cluster for its current load, plus the
24517
24941
  * local runner if co-located. Refreshes the `cachedAgentLoad` snapshot so
24518
24942
  * `getAgentLoad()` / `getGlobalMetrics()` can return fresh data.
@@ -24613,8 +25037,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24613
25037
  * Used by every `balance()` call so the balancer can honour operator-set
24614
25038
  * per-node maximums without polling the store on each decision.
24615
25039
  */
24616
- buildNodeCaps() {
24617
- const blob = this.agentSettingsState.get();
25040
+ async buildNodeCaps() {
25041
+ const blob = await this.agentSettingsState.get();
24618
25042
  const caps = {};
24619
25043
  for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
24620
25044
  return caps;
@@ -24678,6 +25102,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24678
25102
  assignedAt: Date.now()
24679
25103
  };
24680
25104
  this.assignments.set(deviceId, assignment);
25105
+ this.pendingReasons.delete(deviceId);
24681
25106
  if (!this.ctx?.eventBus) return;
24682
25107
  const payload = {
24683
25108
  deviceId,
@@ -24762,7 +25187,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24762
25187
  const decision = balance({
24763
25188
  nodes: loads,
24764
25189
  preferredAgent: null,
24765
- nodeCaps: this.buildNodeCaps()
25190
+ nodeCaps: await this.buildNodeCaps(),
25191
+ eligibleNodes: this.detectionEligibleNodes()
24766
25192
  });
24767
25193
  if (!decision) {
24768
25194
  this.ctx.logger.error("Failover: no online runner", { tags: { deviceId } });
@@ -24770,7 +25196,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24770
25196
  continue;
24771
25197
  }
24772
25198
  if (decision.kind === "pending") {
24773
- this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId } });
25199
+ this.pendingReasons.set(deviceId, decision.reason);
25200
+ this.ctx.logger.warn("Failover: camera left pending — no eligible node", {
25201
+ tags: { deviceId },
25202
+ meta: { reason: decision.reason }
25203
+ });
24774
25204
  this.assignments.delete(deviceId);
24775
25205
  continue;
24776
25206
  }
@@ -24809,6 +25239,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24809
25239
  tags: { nodeId },
24810
25240
  meta: { policy: this.failoverPolicy.onReconnect }
24811
25241
  });
25242
+ this.schedulePendingRetry();
24812
25243
  if (this.failoverPolicy.onReconnect === "rebalance") {
24813
25244
  await this.rebalance().catch((err) => {
24814
25245
  const msg = errMsg(err);
@@ -24816,6 +25247,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24816
25247
  });
24817
25248
  return;
24818
25249
  }
25250
+ if (!this.detectionEligibleNodes().includes(nodeId)) {
25251
+ this.ctx.logger.warn("restore skipped — node not frame-source eligible", {
25252
+ tags: { nodeId },
25253
+ meta: { eligibleNodes: this.detectionEligibleNodes().join(",") }
25254
+ });
25255
+ for (const [deviceId, assignment] of this.assignments) {
25256
+ if (assignment.agentNodeId !== nodeId) continue;
25257
+ this.pendingReasons.set(deviceId, "no-frame-source");
25258
+ }
25259
+ return;
25260
+ }
24819
25261
  let restored = 0;
24820
25262
  for (const [deviceId, config] of this.cameraConfigs) {
24821
25263
  const current = this.assignments.get(deviceId);
@@ -24903,34 +25345,43 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24903
25345
  async reattachAudioForNode(nodeId) {
24904
25346
  for (const [deviceId, audioNode] of this.audioNodeByDevice) {
24905
25347
  if (audioNode !== nodeId) continue;
24906
- const config = this.cameraConfigs.get(deviceId);
24907
- if (!config) continue;
24908
- const audioCfg = {
24909
- ...config,
24910
- enabled: config.pipelineEnabled
24911
- };
24912
- try {
24913
- await this.withAudioSubLock(deviceId, async () => {
24914
- const prior = this.audioSubscriptions.get(deviceId);
24915
- if (prior) {
24916
- try {
24917
- prior();
24918
- } catch {}
24919
- this.audioSubscriptions.delete(deviceId);
24920
- }
24921
- const unsub = await this.subscribeAudioStream(deviceId, audioCfg);
24922
- if (unsub) if (this.activeDetections.has(deviceId)) this.storeAudioSub(deviceId, unsub);
24923
- else unsub();
24924
- });
24925
- } catch (err) {
24926
- this.ctx.logger.error("audio re-attach on analyzer readiness failed", {
24927
- tags: {
24928
- deviceId,
24929
- nodeId
24930
- },
24931
- meta: { error: errMsg(err) }
24932
- });
24933
- }
25348
+ await this.reattachAudioForDevice(deviceId);
25349
+ }
25350
+ }
25351
+ /**
25352
+ * Re-establish the audio subscription for ONE camera: tear down any prior
25353
+ * sub and re-subscribe under the current pin/balance, keeping the new handle
25354
+ * only while detection is active. The whole get→teardown→subscribe→store
25355
+ * sequence runs inside `withAudioSubLock` so a prior handle is never orphaned
25356
+ * (the same invariant `reattachAudioForNode` relied on before it was
25357
+ * extracted here). No-op when the device has no runner config or audio is
25358
+ * disabled/lazy (`subscribeAudioStream` self-gates).
25359
+ */
25360
+ async reattachAudioForDevice(deviceId) {
25361
+ const config = this.cameraConfigs.get(deviceId);
25362
+ if (!config) return;
25363
+ const audioCfg = {
25364
+ ...config,
25365
+ enabled: config.pipelineEnabled
25366
+ };
25367
+ try {
25368
+ await this.withAudioSubLock(deviceId, async () => {
25369
+ const prior = this.audioSubscriptions.get(deviceId);
25370
+ if (prior) {
25371
+ try {
25372
+ prior();
25373
+ } catch {}
25374
+ this.audioSubscriptions.delete(deviceId);
25375
+ }
25376
+ const unsub = await this.subscribeAudioStream(deviceId, audioCfg);
25377
+ if (unsub) if (this.activeDetections.has(deviceId)) this.storeAudioSub(deviceId, unsub);
25378
+ else unsub();
25379
+ });
25380
+ } catch (err) {
25381
+ this.ctx.logger.error("audio re-attach failed", {
25382
+ tags: { deviceId },
25383
+ meta: { error: errMsg(err) }
25384
+ });
24934
25385
  }
24935
25386
  }
24936
25387
  /**
@@ -25005,6 +25456,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25005
25456
  });
25006
25457
  }, 3e3);
25007
25458
  this.scheduleReconcile();
25459
+ this.schedulePendingRetry();
25008
25460
  } catch (err) {
25009
25461
  this.ctx.logger.debug("readiness seed+redispatch failed", {
25010
25462
  tags: { nodeId },
@@ -25119,6 +25571,175 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25119
25571
  }
25120
25572
  }
25121
25573
  }
25574
+ /**
25575
+ * Coalesce bursts of capacity/eligibility/readiness signals into a single
25576
+ * `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
25577
+ * dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
25578
+ * hammer `dispatchCamera`.
25579
+ */
25580
+ schedulePendingRetry() {
25581
+ if (this.pendingRetryDebounceTimer !== null) clearTimeout(this.pendingRetryDebounceTimer);
25582
+ this.pendingRetryDebounceTimer = setTimeout(() => {
25583
+ this.pendingRetryDebounceTimer = null;
25584
+ this.retryPendingDispatches();
25585
+ }, PENDING_RETRY_DEBOUNCE_MS);
25586
+ }
25587
+ /**
25588
+ * Re-dispatch every KNOWN-but-UNASSIGNED camera. `reconcileDispatch` is
25589
+ * additive-only over `cameraConfigs` and never revisits a camera that is
25590
+ * tracked but has no live assignment (left pending by over-cap /
25591
+ * no-frame-source / load-shed), so those cameras would otherwise stay
25592
+ * stranded until an unrelated live event happens to touch them. This sweep
25593
+ * closes that gap.
25594
+ *
25595
+ * `dispatchCamera` re-reads pins, re-balances with the frame-source
25596
+ * predicate, and re-returns pending harmlessly when nothing changed —
25597
+ * so a no-op sweep is cheap and side-effect-free. Serialized with an
25598
+ * in-flight flag + trailing-rerun bit (mirrors `reconcileDispatch`).
25599
+ */
25600
+ async retryPendingDispatches() {
25601
+ if (this.pendingRetryInFlight) {
25602
+ this.pendingRetryRerunRequested = true;
25603
+ return;
25604
+ }
25605
+ if (this.reconcileInFlight) return;
25606
+ if (!this.api) return;
25607
+ this.pendingRetryInFlight = true;
25608
+ try {
25609
+ const stranded = computeStrandedDevices(new Set(this.cameraConfigs.keys()), new Set(this.assignments.keys()));
25610
+ if (stranded.length > 0) {
25611
+ const loadShedActive = this.isAnyNodeLoadShed();
25612
+ let retried = 0;
25613
+ let stillPending = 0;
25614
+ for (const deviceId of stranded) {
25615
+ if (this.pendingReasons.get(deviceId) === "load-shed" && loadShedActive) {
25616
+ stillPending++;
25617
+ continue;
25618
+ }
25619
+ const cfg = this.cameraConfigs.get(deviceId);
25620
+ if (!cfg) continue;
25621
+ try {
25622
+ const result = await this.dispatchCamera(cfg);
25623
+ retried++;
25624
+ if (result.kind === "pending") stillPending++;
25625
+ } catch (err) {
25626
+ this.ctx.logger.warn("pending retry: dispatchCamera failed", {
25627
+ tags: { deviceId },
25628
+ meta: { error: errMsg(err) }
25629
+ });
25630
+ stillPending++;
25631
+ }
25632
+ }
25633
+ this.ctx.logger.info("pending retry sweep", { meta: {
25634
+ stranded: stranded.length,
25635
+ retried,
25636
+ stillPending
25637
+ } });
25638
+ }
25639
+ await this.evaluateRemoteAssignmentHealth();
25640
+ } finally {
25641
+ this.pendingRetryInFlight = false;
25642
+ if (this.pendingRetryRerunRequested) {
25643
+ this.pendingRetryRerunRequested = false;
25644
+ this.schedulePendingRetry();
25645
+ }
25646
+ }
25647
+ }
25648
+ /** True when any node is currently load-shed paused (used to skip churny retries). */
25649
+ isAnyNodeLoadShed() {
25650
+ for (const state of this.loadShedState.values()) if (state.pausedAt !== null) return true;
25651
+ return false;
25652
+ }
25653
+ /**
25654
+ * Evaluate the health of every REMOTE pipeline assignment and act on the
25655
+ * cameras the pure `evaluateRemoteHealth` flags. The hub watchdog covers
25656
+ * local cameras; this is its remote-node equivalent (gap i). Called from the
25657
+ * T3 sweep tick after retrying pending dispatches.
25658
+ *
25659
+ * `protected` so the remote-health orchestrator spec can drive one evaluation
25660
+ * pass deterministically (mirrors the `pendingReasons`/`audioSubLocks` test-seam
25661
+ * convention in this class) without waiting on the periodic timer.
25662
+ */
25663
+ async evaluateRemoteAssignmentHealth() {
25664
+ if (!this.api) return;
25665
+ const actions = evaluateRemoteHealth({
25666
+ assignments: this.assignments,
25667
+ fpsMap: this.cameraFpsMap,
25668
+ activeDeviceIds: new Set(this.activeDetections.keys()),
25669
+ localNodeId: this.localNodeId,
25670
+ now: Date.now(),
25671
+ opts: DEFAULT_REMOTE_HEALTH_OPTS
25672
+ });
25673
+ for (const action of actions) await this.handleRemoteHealthReplace(action);
25674
+ }
25675
+ /**
25676
+ * Re-place a single unhealthy remote camera with bounded per-device backoff.
25677
+ * Under budget: detach from the unhealthy node + re-dispatch (inherits the
25678
+ * T2 frame-source predicate). Over budget: log an error, drop the dead
25679
+ * assignment, and mark the camera `'unhealthy'`-pending for the operator so
25680
+ * `getCameraStatus` surfaces it (R2 "bounded-retry ... state visible").
25681
+ */
25682
+ async handleRemoteHealthReplace(action) {
25683
+ const { deviceId, why } = action;
25684
+ const assignment = this.assignments.get(deviceId);
25685
+ if (!assignment) return;
25686
+ const cfg = this.cameraConfigs.get(deviceId);
25687
+ if (!cfg) return;
25688
+ const now = Date.now();
25689
+ const prior = this.remoteHealthAttempts.get(deviceId);
25690
+ const windowFresh = prior !== void 0 && now - prior.windowStart < REMOTE_HEALTH_WINDOW_MS;
25691
+ const count = windowFresh ? prior.count : 0;
25692
+ if (count >= REMOTE_HEALTH_MAX_ATTEMPTS) {
25693
+ this.ctx.logger.error("remote assignment unhealthy — retries exhausted, leaving pending", {
25694
+ tags: {
25695
+ deviceId,
25696
+ nodeId: assignment.agentNodeId
25697
+ },
25698
+ meta: {
25699
+ why,
25700
+ attempts: count,
25701
+ windowMs: REMOTE_HEALTH_WINDOW_MS
25702
+ }
25703
+ });
25704
+ this.assignments.delete(deviceId);
25705
+ this.pendingReasons.set(deviceId, "unhealthy");
25706
+ await this.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
25707
+ this.ctx.logger.debug("remote-health exhausted-detach failed", {
25708
+ tags: { deviceId },
25709
+ meta: { error: errMsg(err) }
25710
+ });
25711
+ });
25712
+ return;
25713
+ }
25714
+ this.remoteHealthAttempts.set(deviceId, {
25715
+ count: count + 1,
25716
+ windowStart: windowFresh ? prior.windowStart : now
25717
+ });
25718
+ this.ctx.logger.warn("remote assignment unhealthy — re-placing camera", {
25719
+ tags: {
25720
+ deviceId,
25721
+ nodeId: assignment.agentNodeId
25722
+ },
25723
+ meta: {
25724
+ why,
25725
+ attempt: count + 1,
25726
+ maxAttempts: REMOTE_HEALTH_MAX_ATTEMPTS
25727
+ }
25728
+ });
25729
+ await this.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
25730
+ this.ctx.logger.warn("remote-health detach failed", {
25731
+ tags: { deviceId },
25732
+ meta: { error: errMsg(err) }
25733
+ });
25734
+ });
25735
+ this.assignments.delete(deviceId);
25736
+ await this.dispatchCamera(cfg).catch((err) => {
25737
+ this.ctx.logger.warn("remote-health re-dispatch failed", {
25738
+ tags: { deviceId },
25739
+ meta: { error: errMsg(err) }
25740
+ });
25741
+ });
25742
+ }
25122
25743
  async getCapabilityBindings(input) {
25123
25744
  if (!this.ctx?.settings) return {};
25124
25745
  const perNodeRaw = (await this.nodeBindingsState.get().catch(() => ({})))[input.nodeId];
@@ -25212,12 +25833,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25212
25833
  }
25213
25834
  async assignAudio(input) {
25214
25835
  await this.ctx.settings?.writeDeviceStore(input.deviceId, { [AUDIO_NODE_SETTING]: input.nodeId });
25836
+ if (!this.enabledAudioNodes.includes(input.nodeId)) this.ctx.logger.warn("audio pin stored but node is not in enabledAudioNodes — pin will not take effect until enabled", { tags: {
25837
+ deviceId: input.deviceId,
25838
+ nodeId: input.nodeId
25839
+ } });
25215
25840
  this.audioAssignments.delete(input.deviceId);
25216
25841
  this.audioNodeByDevice.delete(input.deviceId);
25217
25842
  this.ctx.logger.info("Audio node pinned", { tags: {
25218
25843
  deviceId: input.deviceId,
25219
25844
  nodeId: input.nodeId
25220
25845
  } });
25846
+ await this.reattachAudioForDevice(input.deviceId);
25221
25847
  return { success: true };
25222
25848
  }
25223
25849
  async unassignAudio(input) {
@@ -25225,6 +25851,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25225
25851
  this.audioAssignments.delete(input.deviceId);
25226
25852
  this.audioNodeByDevice.delete(input.deviceId);
25227
25853
  this.ctx.logger.info("Audio node unpinned", { tags: { deviceId: input.deviceId } });
25854
+ await this.reattachAudioForDevice(input.deviceId);
25228
25855
  return { success: true };
25229
25856
  }
25230
25857
  async getAudioAssignment(input) {
@@ -25308,6 +25935,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25308
25935
  tags: { nodeId: input.agentNodeId },
25309
25936
  meta: { maxCameras: input.maxCameras }
25310
25937
  });
25938
+ this.schedulePendingRetry();
25311
25939
  return { success: true };
25312
25940
  }
25313
25941
  async getCameraSettings(input) {
@@ -25471,7 +26099,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25471
26099
  const detectionNodeId = pipelineAssignment?.agentNodeId ?? null;
25472
26100
  const decoderPinRaw = (api ? await this.ctx.settings?.readDeviceStore(deviceId).catch(() => ({})) ?? {} : {})["decoderNodeId"];
25473
26101
  const decoderPinned = typeof decoderPinRaw === "string" && decoderPinRaw !== "auto";
25474
- const decoderNodeId = detectionNodeId ? await this.resolveDecoderNode(deviceId, detectionNodeId).catch(() => null) : null;
26102
+ const advisoryDecoderNodeId = detectionNodeId ? await this.resolveDecoderNode(deviceId, detectionNodeId).catch(() => null) : null;
25475
26103
  const audioAssignment = this.audioAssignments.get(deviceId) ?? null;
25476
26104
  const audioNodeId = audioAssignment?.nodeId ?? null;
25477
26105
  const audioPinned = audioAssignment?.pinned ?? false;
@@ -25480,11 +26108,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25480
26108
  decoder: decoderPinned,
25481
26109
  audio: audioPinned
25482
26110
  };
25483
- const reasons = {
25484
- detection: pipelineAssignment?.reason,
25485
- decoder: decoderPinned ? "manual" : "co-located",
25486
- audio: audioPinned ? "manual" : void 0
25487
- };
26111
+ const detectionReason = pipelineAssignment !== null ? pipelineAssignment.reason : this.cameraConfigs.has(deviceId) ? `pending:${this.pendingReasons.get(deviceId) ?? "pending"}` : void 0;
26112
+ const liveDecoder = { nodeId: null };
25488
26113
  const allSlotsFetch = api ? api.streamBroker.listAllProfileSlots.query() : null;
25489
26114
  const sourceFetch = api && allSlotsFetch ? this.boundedStage(allSlotsFetch.then((slots) => {
25490
26115
  return { streams: slots.filter((s) => s.deviceId === deviceId).map((s) => ({
@@ -25515,17 +26140,19 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25515
26140
  clients: await api.streamBroker.listClients.query({ brokerId: slot.brokerId }).catch(() => null)
25516
26141
  };
25517
26142
  })), api.streamBroker.getAllRtspEntries.query({}).catch(() => null)]);
26143
+ const profileDetails = statsAndClients.map(({ slot, stats, clients }) => ({
26144
+ profile: slot.profile,
26145
+ status: slot.status,
26146
+ codec: stats?.codec ?? slot.codec ?? "",
26147
+ width: slot.resolution?.width ?? 0,
26148
+ height: slot.resolution?.height ?? 0,
26149
+ subscribers: clients?.encodedSubscribers ?? 0,
26150
+ inFps: stats?.inputFps ?? 0,
26151
+ outFps: stats?.decodeFps ?? 0
26152
+ }));
26153
+ for (const { stats } of statsAndClients) if (liveDecoder.nodeId === null && typeof stats?.decoderNodeId === "string") liveDecoder.nodeId = stats.decoderNodeId;
25518
26154
  return {
25519
- profiles: statsAndClients.map(({ slot, stats, clients }) => ({
25520
- profile: slot.profile,
25521
- status: slot.status,
25522
- codec: stats?.codec ?? slot.codec ?? "",
25523
- width: slot.resolution?.width ?? 0,
25524
- height: slot.resolution?.height ?? 0,
25525
- subscribers: clients?.encodedSubscribers ?? 0,
25526
- inFps: stats?.inputFps ?? 0,
25527
- outFps: stats?.decodeFps ?? 0
25528
- })),
26155
+ profiles: profileDetails,
25529
26156
  webrtcSessions: statsAndClients.reduce((total, { clients }) => {
25530
26157
  if (!clients) return total;
25531
26158
  return total + clients.encoded.filter((c) => WEBRTC_KINDS.has(c.attribution.kind)).length;
@@ -25535,8 +26162,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25535
26162
  }) ?? false
25536
26163
  };
25537
26164
  }), STAGE_TIMEOUT_MS) : Promise.resolve(null);
25538
- const decoderFetch = decoderNodeId ? Promise.resolve({
25539
- nodeId: decoderNodeId,
26165
+ const decoderFetch = advisoryDecoderNodeId ? Promise.resolve({
26166
+ nodeId: advisoryDecoderNodeId,
25540
26167
  formats: [],
25541
26168
  sessionCount: 0,
25542
26169
  shm: {
@@ -25604,6 +26231,13 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25604
26231
  detectionFetch,
25605
26232
  recordingFetch
25606
26233
  ]);
26234
+ const liveDecoderNodeId = brokerResult !== null ? liveDecoder.nodeId : null;
26235
+ const decoderNodeId = liveDecoderNodeId ?? advisoryDecoderNodeId;
26236
+ const reasons = {
26237
+ detection: detectionReason,
26238
+ decoder: decoderPinned ? "manual" : liveDecoderNodeId !== null ? "session" : decoderNodeId !== null ? "advisory" : void 0,
26239
+ audio: audioPinned ? "manual" : void 0
26240
+ };
25607
26241
  return composeCameraStatus({
25608
26242
  deviceId,
25609
26243
  fetchedAt: Date.now(),
@@ -25959,7 +26593,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25959
26593
  id: "cluster",
25960
26594
  title: "Cluster",
25961
26595
  tab: "pipeline",
25962
- description: "Which cluster nodes are eligible to run the detection pipeline. Leave at least one node enabled an empty list falls back to all online nodes so the pipeline never becomes completely unassignable.",
26596
+ description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
25963
26597
  fields: [
25964
26598
  {
25965
26599
  key: "enabledNodes",
@@ -26491,6 +27125,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26491
27125
  this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
26492
27126
  const rawEnabledAudio = config["enabledAudioNodes"];
26493
27127
  this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
27128
+ this.schedulePendingRetry();
26494
27129
  }
26495
27130
  get api() {
26496
27131
  return this.ctx.api ?? null;