@camstack/addon-pipeline-orchestrator 1.1.13 → 1.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -4631,7 +4631,7 @@ function _instanceof(cls, params = {}) {
4631
4631
  return inst;
4632
4632
  }
4633
4633
  //#endregion
4634
- //#region ../types/dist/sleep-C2M2zF7x.mjs
4634
+ //#region ../types/dist/sleep-MHm--th-.mjs
4635
4635
  var EventCategory = /* @__PURE__ */ function(EventCategory) {
4636
4636
  EventCategory["SystemBoot"] = "system.boot";
4637
4637
  EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -5909,6 +5909,7 @@ var CamStreamKindSchema = _enum([
5909
5909
  "pull-rtsp",
5910
5910
  "pull-rtmp",
5911
5911
  "pull-http",
5912
+ "pull-flv",
5912
5913
  "pull-rfc4571",
5913
5914
  "push-annexb",
5914
5915
  "derived"
@@ -7448,7 +7449,21 @@ var StorageLocationDeclarationSchema = object({
7448
7449
  * slots (e.g. `recordingsLow` → `recordings`) so operators only need to
7449
7450
  * configure the primary location.
7450
7451
  */
7451
- defaultsTo: string().optional()
7452
+ defaultsTo: string().optional(),
7453
+ /**
7454
+ * Which node root the seeded `<id>:default` instance is placed under on a
7455
+ * FRESH install:
7456
+ * - `'data'` (default) — the node's data dir (`CAMSTACK_DATA` / boot dir),
7457
+ * the appData volume. Right for small/durable data (backups, logs, models).
7458
+ * - `'media'` — the dedicated media volume (`CAMSTACK_MEDIA_ROOT`) when that
7459
+ * env is set, else falls back to the data root. Right for bulky, hot media
7460
+ * (recordings, event media) that should stay off the appData disk.
7461
+ *
7462
+ * Only affects the seeded default's `basePath`; operators can repoint any
7463
+ * location afterwards, and a `defaultsTo` slot inherits its parent's root
7464
+ * regardless of this field. Absent (the common case) is treated as `'data'`.
7465
+ */
7466
+ defaultRoot: _enum(["data", "media"]).optional()
7452
7467
  });
7453
7468
  var DecoderStatsSchema = object({
7454
7469
  inputFps: number(),
@@ -8326,6 +8341,10 @@ var RtspRestreamEntrySchema = object({
8326
8341
  var BrokerRtspClientSchema = object({
8327
8342
  sessionId: string(),
8328
8343
  remoteAddr: string(),
8344
+ /** Client `User-Agent` (e.g. `recorder` for the recorder's RTSP pull), or
8345
+ * null/absent when the client sent none. Lets the UI label a consumer by
8346
+ * purpose. Optional so a client built against an older schema stays valid. */
8347
+ userAgent: string().nullish(),
8329
8348
  playing: boolean(),
8330
8349
  muted: boolean(),
8331
8350
  connectedAt: number(),
@@ -13982,7 +14001,7 @@ var AddBrokerInputSchema = object({
13982
14001
  });
13983
14002
  var AddBrokerResultSchema = object({ id: string() });
13984
14003
  var IdInputSchema = object({ id: string() });
13985
- var TestResultSchema = discriminatedUnion("ok", [object({
14004
+ var TestResultSchema$1 = discriminatedUnion("ok", [object({
13986
14005
  ok: literal(true),
13987
14006
  latencyMs: number()
13988
14007
  }), object({
@@ -14005,7 +14024,7 @@ var StatusSchema = object({
14005
14024
  brokerCount: number(),
14006
14025
  embeddedRunning: boolean()
14007
14026
  });
14008
- method(_void(), array(BrokerInfoSchema)), method(IdInputSchema, BrokerConnectionDetailsSchema), method(AddBrokerInputSchema, AddBrokerResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(IdInputSchema, TestResultSchema, { kind: "mutation" }), method(StartEmbeddedInputSchema, StartEmbeddedResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(_void(), StatusSchema);
14027
+ method(_void(), array(BrokerInfoSchema)), method(IdInputSchema, BrokerConnectionDetailsSchema), method(AddBrokerInputSchema, AddBrokerResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(IdInputSchema, TestResultSchema$1, { kind: "mutation" }), method(StartEmbeddedInputSchema, StartEmbeddedResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(_void(), StatusSchema);
14009
14028
  var NetworkEndpointSchema = object({
14010
14029
  url: string(),
14011
14030
  hostname: string(),
@@ -14039,23 +14058,198 @@ var NetworkEndpointEntrySchema = NetworkEndpointSchema.extend({
14039
14058
  sourcePort: number().optional()
14040
14059
  });
14041
14060
  method(_void(), NetworkEndpointSchema, { kind: "mutation" }), method(_void(), _void(), { kind: "mutation" }), method(_void(), NetworkEndpointSchema.nullable()), method(_void(), NetworkAccessStatusSchema), method(_void(), array(NetworkEndpointEntrySchema).readonly());
14042
- method(object({
14043
- title: string(),
14061
+ /**
14062
+ * notification-output — canonical, capability-gated notification delivery.
14063
+ *
14064
+ * Apprise-derived model (see
14065
+ * `docs/superpowers/specs/2026-07-03-notification-output-notifier-matrix.md`):
14066
+ * callers emit ONE canonical `Notification`; each provider declares a
14067
+ * per-kind capability descriptor (`TargetKind`), and the pure degrade
14068
+ * engine (`@camstack/types` `prepareNotification`) transcodes / degrades the
14069
+ * message to what the kind supports — callers never special-case a service.
14070
+ *
14071
+ * DESIGN DECISIONS (locked):
14072
+ * - Target CRUD lives on THIS cap (`upsertTarget` / `deleteTarget` /
14073
+ * `setTargetEnabled`), each provider persisting via the `settings-store`
14074
+ * cap. Rationale: the admin UI needs one uniform surface across the
14075
+ * notifiers addon AND the HA addon; the addon-`globalSettingsSchema`-array
14076
+ * alternative would fork the UI per addon and cannot host the
14077
+ * discovery→adopt flow.
14078
+ * - `listTargetKinds` / `listTargets` / `discoverTargets` return arrays →
14079
+ * the generated cap-mount auto-`concatCollection`-fans them across every
14080
+ * registered provider (notifiers addon + HA addon) so one catalog is
14081
+ * routable. `send` / `testTarget` / CRUD route to ONE provider by the
14082
+ * `addonId` the generated collection router extracts from the call input.
14083
+ * - `Attachment.bytes` is `Uint8Array`. Transport-safe: superjson (the tRPC
14084
+ * transformer) + UDS MsgPack both round-trip typed arrays — already used by
14085
+ * `storage` / `storage-provider` / `recording` caps over the same path. No
14086
+ * base64 fallback needed.
14087
+ *
14088
+ * TODO (deferred, closed-set change — separate decision): add
14089
+ * `providerKind: 'notify'` so notification providers surface on the unified
14090
+ * admin "Integrations" page.
14091
+ */
14092
+ /**
14093
+ * Zentik-derived typed-media enum — the superset across every kind. Each
14094
+ * adapter picks what it supports and the degrade engine filters the rest.
14095
+ */
14096
+ var AttachmentMediaTypeSchema = _enum([
14097
+ "image",
14098
+ "video",
14099
+ "gif",
14100
+ "audio",
14101
+ "icon"
14102
+ ]);
14103
+ /**
14104
+ * A single attachment. Exactly one of `url` (remote source, most adapters
14105
+ * prefer this) or `bytes` (inline source; required for Pushover-style
14106
+ * bytes-only kinds) MUST be present — the degrade engine expresses a
14107
+ * url→bytes fetch as a `needsFetch` directive the adapter executes.
14108
+ */
14109
+ var AttachmentSchema = object({
14110
+ mediaType: AttachmentMediaTypeSchema,
14111
+ url: string().optional(),
14112
+ bytes: _instanceof(Uint8Array).optional(),
14113
+ mime: string().optional(),
14114
+ name: string().optional()
14115
+ }).refine((a) => a.url !== void 0 || a.bytes !== void 0, { message: "Attachment requires either `url` or `bytes`" });
14116
+ var NotificationFormatSchema = _enum([
14117
+ "text",
14118
+ "markdown",
14119
+ "html"
14120
+ ]);
14121
+ /** A single tap-through action button. */
14122
+ var NotificationActionSchema = object({
14123
+ id: string(),
14124
+ label: string(),
14125
+ url: string().optional()
14126
+ });
14127
+ /**
14128
+ * The canonical notification. `body` is the only hard field (Apprise model).
14129
+ * `priority` is a 5-level ORDINAL (1=lowest … 3=normal(default) … 5=urgent),
14130
+ * NOT a fixed severity enum — each kind declares its own `caps.levels` and
14131
+ * the adapter maps this ordinal onto its native level. `level?` is an
14132
+ * optional kind-native level id (`emergency`, `silent`, …) that overrides
14133
+ * `priority` for that one target.
14134
+ */
14135
+ var NotificationSchema = object({
14044
14136
  body: string(),
14045
- imageUrl: string().optional(),
14137
+ title: string().optional(),
14138
+ format: NotificationFormatSchema.default("text"),
14139
+ priority: number().int().min(1).max(5).default(3),
14140
+ level: string().optional(),
14141
+ attachments: array(AttachmentSchema).optional(),
14142
+ clickUrl: string().optional(),
14143
+ actions: array(NotificationActionSchema).optional(),
14144
+ sound: string().optional(),
14145
+ ttl: number().optional(),
14146
+ tag: string().optional(),
14046
14147
  deviceId: number().optional(),
14047
14148
  eventId: string().optional(),
14048
- priority: _enum([
14049
- "low",
14050
- "normal",
14051
- "high",
14052
- "critical"
14053
- ]).default("normal"),
14054
14149
  metadata: record(string(), unknown()).optional()
14055
- }), _void(), { kind: "mutation" }), method(_void(), object({
14150
+ });
14151
+ /** One declared native severity/priority level for a kind. */
14152
+ var TargetKindLevelSchema = object({
14153
+ id: string(),
14154
+ label: string(),
14155
+ /** Which canonical priority (1..5) this level maps to. `null` = qualitative-only. */
14156
+ ordinal: number().int().min(1).max(5).nullable(),
14157
+ flags: object({
14158
+ critical: boolean().optional(),
14159
+ silent: boolean().optional(),
14160
+ noPush: boolean().optional()
14161
+ }).optional(),
14162
+ /** e.g. Pushover `emergency` requires `retry` / `expire`. */
14163
+ requires: array(string()).optional(),
14164
+ description: string().optional()
14165
+ });
14166
+ /** The full capability block consulted before dispatch. */
14167
+ var TargetKindCapsSchema = object({
14168
+ attachments: object({
14169
+ mediaTypes: array(AttachmentMediaTypeSchema),
14170
+ mode: _enum([
14171
+ "url",
14172
+ "bytes",
14173
+ "both"
14174
+ ]),
14175
+ max: number().int().nonnegative(),
14176
+ maxBytes: number().int().positive().optional()
14177
+ }),
14178
+ /** Max action buttons (0 = none). */
14179
+ actions: number().int().nonnegative(),
14180
+ levels: array(TargetKindLevelSchema),
14181
+ format: array(NotificationFormatSchema),
14182
+ clickUrl: boolean(),
14183
+ sound: boolean(),
14184
+ ttl: boolean(),
14185
+ bodyMaxLen: number().int().positive()
14186
+ });
14187
+ /**
14188
+ * `configSchema` is a `ConfigUISchema` tree passed through to the admin
14189
+ * FormBuilder. Stored as `z.unknown()` at the cap seam (mirrors
14190
+ * `device-provider.getChildCreationSchema` `CreationSchemaOutputSchema`) —
14191
+ * the union is large and not meant for runtime validation here; the exported
14192
+ * `TargetKind` type re-tightens `configSchema` to `ConfigUISchema`.
14193
+ */
14194
+ var ConfigSchemaPassthrough = unknown();
14195
+ var TargetKindSchema = object({
14196
+ kind: string(),
14197
+ label: string(),
14198
+ icon: string(),
14199
+ /** Stamped by each provider so the concat-fanned catalog stays routable. */
14200
+ addonId: string(),
14201
+ configSchema: ConfigSchemaPassthrough,
14202
+ supportsDiscovery: boolean(),
14203
+ caps: TargetKindCapsSchema
14204
+ });
14205
+ /**
14206
+ * A persisted target. `config` holds secrets; providers REDACT secret fields
14207
+ * (return a presence marker only) when serving `listTargets` — never
14208
+ * round-trip a stored secret to the UI.
14209
+ */
14210
+ var TargetSchema = object({
14211
+ id: string(),
14212
+ name: string(),
14213
+ kind: string(),
14214
+ addonId: string(),
14215
+ enabled: boolean(),
14216
+ config: record(string(), unknown())
14217
+ });
14218
+ /** A discovery-surfaced candidate (config is partial + non-secret). */
14219
+ var DiscoveredTargetSchema = object({
14220
+ kind: string(),
14221
+ suggestedName: string(),
14222
+ config: record(string(), unknown())
14223
+ });
14224
+ /** The degrade engine's report — what was resolved / dropped / degraded. */
14225
+ var RenderedAsSchema = object({
14226
+ level: string(),
14227
+ format: NotificationFormatSchema,
14228
+ attachmentsSent: number().int().nonnegative(),
14229
+ actionsSent: number().int().nonnegative(),
14230
+ truncated: boolean(),
14231
+ dropped: array(string())
14232
+ });
14233
+ var SendResultSchema = object({
14056
14234
  success: boolean(),
14057
- error: string().optional()
14058
- }), { kind: "mutation" });
14235
+ error: string().optional(),
14236
+ renderedAs: RenderedAsSchema.optional()
14237
+ });
14238
+ /** Same shape as SendResult — kept as a distinct name for the test panel. */
14239
+ var TestResultSchema = SendResultSchema;
14240
+ method(object({}), array(TargetKindSchema)), method(object({}), array(TargetSchema)), method(object({
14241
+ kind: string(),
14242
+ config: record(string(), unknown()).optional()
14243
+ }), array(DiscoveredTargetSchema)), method(object({
14244
+ targetId: string(),
14245
+ notification: NotificationSchema
14246
+ }), SendResultSchema, { kind: "mutation" }), method(object({
14247
+ targetId: string(),
14248
+ sample: NotificationSchema.optional()
14249
+ }), TestResultSchema, { kind: "mutation" }), method(object({ target: TargetSchema }), TargetSchema, { kind: "mutation" }), method(object({ targetId: string() }), _void(), { kind: "mutation" }), method(object({
14250
+ targetId: string(),
14251
+ enabled: boolean()
14252
+ }), _void(), { kind: "mutation" });
14059
14253
  /**
14060
14254
  * Zod schemas for persisted record types.
14061
14255
  *
@@ -14845,10 +15039,11 @@ var pipelineOrchestratorCapability = {
14845
15039
  }))),
14846
15040
  /**
14847
15041
  * Get one camera's decoder placement (computed if not yet pinned).
14848
- * Consumed by `stream-broker.createBroker` so decoder provider
14849
- * selection is deterministic fixes the 2026-04-18 race where
14850
- * `capProviders[0]` silently picked ffmpeg-on-agent-0 for a
14851
- * hub-assigned camera.
15042
+ *
15043
+ * ADVISORY today: reports the orchestrator's decoder preference only.
15044
+ * Actual decode placement is broker-owned (local-node pin + frame-plane
15045
+ * co-location guard). Reserved to become the binding source/decoder-owner
15046
+ * control in the stream-LB epic (Phase 2).
14852
15047
  *
14853
15048
  * `pipelineNodeId` is the node already chosen to run inference for
14854
15049
  * this camera. When provided, the balancer prefers co-location with
@@ -20220,13 +20415,49 @@ Object.freeze({
20220
20415
  addonId: null,
20221
20416
  access: "create"
20222
20417
  },
20418
+ "notificationOutput.deleteTarget": {
20419
+ capName: "notification-output",
20420
+ capScope: "system",
20421
+ addonId: null,
20422
+ access: "delete"
20423
+ },
20424
+ "notificationOutput.discoverTargets": {
20425
+ capName: "notification-output",
20426
+ capScope: "system",
20427
+ addonId: null,
20428
+ access: "view"
20429
+ },
20430
+ "notificationOutput.listTargetKinds": {
20431
+ capName: "notification-output",
20432
+ capScope: "system",
20433
+ addonId: null,
20434
+ access: "view"
20435
+ },
20436
+ "notificationOutput.listTargets": {
20437
+ capName: "notification-output",
20438
+ capScope: "system",
20439
+ addonId: null,
20440
+ access: "view"
20441
+ },
20223
20442
  "notificationOutput.send": {
20224
20443
  capName: "notification-output",
20225
20444
  capScope: "system",
20226
20445
  addonId: null,
20227
20446
  access: "create"
20228
20447
  },
20229
- "notificationOutput.sendTest": {
20448
+ "notificationOutput.setTargetEnabled": {
20449
+ capName: "notification-output",
20450
+ capScope: "system",
20451
+ addonId: null,
20452
+ access: "create"
20453
+ },
20454
+ "notificationOutput.testTarget": {
20455
+ capName: "notification-output",
20456
+ capScope: "system",
20457
+ addonId: null,
20458
+ access: "create"
20459
+ },
20460
+ "notificationOutput.upsertTarget": {
20230
20461
  capName: "notification-output",
20231
20462
  capScope: "system",
20232
20463
  addonId: null,
@@ -22402,470 +22633,232 @@ function buildTreeFromAddons(enabled, catalog) {
22402
22633
  return roots;
22403
22634
  }
22404
22635
  //#endregion
22405
- //#region src/zones-provider.ts
22636
+ //#region src/audio-chunk-poller.ts
22406
22637
  /**
22407
- * `zones-provider.ts` — implements `zonesCapability` for the orchestrator.
22638
+ * `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
22639
+ * plane (Phase 5 / D9).
22408
22640
  *
22409
- * Per-camera CRUD over polygon detection zones. Persists to the
22410
- * orchestrator's per-device settings store under the `zones` key and
22411
- * mirrors every change into the device-state `zones` slice via
22412
- * `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
22413
- * pipeline-executor, analytics, admin UI) read the live state with
22414
- * the canonical `dev.state.zones.onChanged` channel.
22641
+ * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
22642
+ * path. A live callback cannot cross a process boundary; once the `pipeline`
22643
+ * group is dissolved (Task 8) the orchestrator runs in a different process
22644
+ * from the broker, so audio delivery must go over tRPC.
22415
22645
  *
22416
- * Onboard / firmware-reported zones are out of scope for now — every
22417
- * zone is operator-drawn. The provider keeps the surface symmetric:
22418
- * `addZone` rejects id collisions, `updateZone` requires an existing
22419
- * id, `removeZone` is idempotent.
22420
- */
22421
- var ZONES_STORE_KEY = "zones";
22422
- var ZONES_CAP_NAME = "zones";
22423
- var ZonesArraySchema = array(ZoneSchema);
22424
- var ZonesProvider = class {
22425
- ctx;
22426
- /** Per-device cache. Hydrated lazily on first read for a device. */
22427
- cache = /* @__PURE__ */ new Map();
22428
- /**
22429
- * Per-device durable handle over the `zones` store key. The WHOLE
22430
- * validated zone array round-trips on every read/write so no field can
22431
- * be dropped on persist. Built lazily + memoised per device.
22432
- */
22433
- stateByDevice = /* @__PURE__ */ new Map();
22434
- constructor(ctx) {
22435
- this.ctx = ctx;
22436
- }
22437
- /** Lazily build (and memoise) the durable `zones` handle for a device. */
22438
- zonesState(deviceId) {
22439
- let handle = this.stateByDevice.get(deviceId);
22440
- if (!handle) {
22441
- handle = createDurableState({
22442
- key: ZONES_STORE_KEY,
22443
- schema: ZonesArraySchema,
22444
- fallback: [],
22445
- read: () => this.ctx.settings.readDeviceStore(deviceId),
22446
- write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
22447
- onParseError: (key, error) => this.ctx.logger.warn("zones store entry failed to parse — resetting", {
22448
- tags: { deviceId },
22449
- meta: {
22450
- key,
22451
- error: error instanceof Error ? error.message : String(error)
22452
- }
22453
- })
22454
- });
22455
- this.stateByDevice.set(deviceId, handle);
22456
- }
22457
- return handle;
22458
- }
22459
- async listZones({ deviceId }) {
22460
- return this.loadZones(deviceId);
22461
- }
22462
- async addZone({ deviceId, zone }) {
22463
- const existing = await this.loadZones(deviceId);
22464
- if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
22465
- await this.persist(deviceId, [...existing, zone]);
22466
- }
22467
- async updateZone({ deviceId, zone }) {
22468
- const existing = await this.loadZones(deviceId);
22469
- if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
22470
- const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
22471
- await this.persist(deviceId, next);
22472
- }
22473
- async removeZone({ deviceId, zoneId }) {
22474
- const existing = await this.loadZones(deviceId);
22475
- if (!existing.some((entry) => entry.id === zoneId)) return;
22476
- await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
22477
- }
22478
- /**
22479
- * Drop a device's cache entry. Called when the device is removed so
22480
- * the next attach starts from a fresh disk read.
22481
- */
22482
- forgetDevice(deviceId) {
22483
- this.cache.delete(deviceId);
22484
- this.stateByDevice.delete(deviceId);
22485
- }
22486
- async loadZones(deviceId) {
22487
- const cached = this.cache.get(deviceId);
22488
- if (cached) return cached;
22489
- let zones = [];
22490
- try {
22491
- zones = await this.zonesState(deviceId).get();
22492
- } catch (err) {
22493
- this.ctx.logger.warn("zones store read failed — using empty list", {
22494
- tags: { deviceId },
22495
- meta: { error: err instanceof Error ? err.message : String(err) }
22496
- });
22497
- }
22498
- this.cache.set(deviceId, zones);
22499
- return zones;
22500
- }
22501
- async persist(deviceId, zones) {
22502
- this.cache.set(deviceId, zones);
22503
- await this.zonesState(deviceId).set(zones);
22504
- await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
22505
- capName: ZONES_CAP_NAME,
22506
- slice: { zones }
22507
- });
22508
- this.ctx.onZonesChanged?.(deviceId, zones);
22509
- }
22510
- };
22511
- //#endregion
22512
- //#region src/zone-rules-provider.ts
22513
- /**
22514
- * `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
22515
- * orchestrator.
22646
+ * The consumer:
22516
22647
  *
22517
- * Per-stage rule arrays (motion / detection) live next to zones in the
22518
- * orchestrator's per-device store under `zoneRules.<stage>` keys.
22519
- * Every mutation mirrors to a stage-specific device-state slice
22520
- * (`motion-zone-rules`, `detection-zone-rules`) so consumer addons
22521
- * subscribe independently and pick up the new gating without an extra
22522
- * cap round-trip.
22648
+ * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC the broker
22649
+ * registers a per-subscription bounded FIFO queue and returns a
22650
+ * `subscriptionId`;
22651
+ * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
22652
+ * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
22653
+ * 3. feeds each chunk to its downstream audio logic;
22654
+ * 4. on teardown, `unsubscribeAudioChunks`.
22523
22655
  *
22524
- * The provider validates each rule against {@link ZoneRuleSchema}
22525
- * before persisting partial / corrupt writes are rejected outright
22526
- * since rules drive runtime filtering and a bad payload would silently
22527
- * widen the operator's intended scope.
22656
+ * Audio is not latency-critical like video, and chunks arrive only ~every
22657
+ * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
22658
+ * a small per-poll burst keeps latency low without busy-spinning. The
22659
+ * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
22660
+ * loses a chunk.
22661
+ *
22662
+ * Boot-race tolerance: the broker for a given camStream may not be registered
22663
+ * yet when the orchestrator wires the subscription (provider addons publish
22664
+ * their cameraStreams asynchronously after their probe completes).
22665
+ * `subscribeAudioChunks` retries with exponential backoff (capped at
22666
+ * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
22667
+ * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
22668
+ * shape so video and audio plumbing self-heal identically.
22528
22669
  */
22529
- /** Settings store key for stage rules. Kept under a single nested
22530
- * object so a future stage just adds another property without
22531
- * reshuffling the schema. */
22532
- var RULES_STORE_KEY = "zoneRules";
22533
- /** Cap name for the unified runtime-state slice. Matches the cap's
22534
- * declared `name` so the codegen DeviceProxy auto-wires
22535
- * `device.state.zoneRules`. The slice value is the full
22536
- * `{motion, detection}` object — both stages travel together so a
22537
- * single reactive handle covers every consumer. */
22538
- var ZONE_RULES_CAP_NAME = "zone-rules";
22539
- var RulesArraySchema = array(ZoneRuleSchema);
22670
+ /** Poll period audio chunks arrive ~every 500ms; 200ms keeps latency low. */
22671
+ var POLL_INTERVAL_MS = 200;
22672
+ /** How many chunks to drain per poll — a small burst absorbs jitter. */
22673
+ var PULL_MAX_COUNT = 8;
22540
22674
  /**
22541
- * Whole-blob schema for the per-device `zoneRules` store key. Both stages
22542
- * travel together under one key. Deliberately lenient each stage is
22543
- * `unknown` so a corrupt single stage can NOT drop the sibling on load
22544
- * (durable-state `get()` falls back to `{}` only on a whole-blob parse
22545
- * failure). Strict per-stage validation, with its own reset-on-corrupt
22546
- * warn, still happens in `loadRules` exactly as before the migration.
22675
+ * Consecutive pull failures before we attempt to re-subscribe. A single failed
22676
+ * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
22677
+ * sustained failure means the broker child restarted and dropped our
22678
+ * subscription, so we re-establish it.
22547
22679
  */
22548
- var ZoneRulesBlockSchema = object({
22549
- motion: unknown().optional(),
22550
- detection: unknown().optional()
22551
- }).passthrough();
22552
- var ZoneRulesProvider = class {
22553
- ctx;
22554
- /** Per-device per-stage cache. Hydrated lazily on first read. */
22555
- cache = /* @__PURE__ */ new Map();
22556
- /**
22557
- * Per-device durable handle over the `zoneRules` store key. The WHOLE
22558
- * `{motion?, detection?}` block round-trips on every read/write so a
22559
- * write on one stage can never drop the other. Built lazily + memoised.
22560
- */
22561
- stateByDevice = /* @__PURE__ */ new Map();
22562
- constructor(ctx) {
22563
- this.ctx = ctx;
22564
- }
22565
- /** Lazily build (and memoise) the durable `zoneRules` handle for a device. */
22566
- rulesState(deviceId) {
22567
- let handle = this.stateByDevice.get(deviceId);
22568
- if (!handle) {
22569
- handle = createDurableState({
22570
- key: RULES_STORE_KEY,
22571
- schema: ZoneRulesBlockSchema,
22572
- fallback: {},
22573
- read: () => this.ctx.settings.readDeviceStore(deviceId),
22574
- write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
22575
- onParseError: (key, error) => this.ctx.logger.warn("zone-rules store block failed to parse — resetting", {
22576
- tags: { deviceId },
22577
- meta: {
22578
- key,
22579
- error: error instanceof Error ? error.message : String(error)
22580
- }
22581
- })
22582
- });
22583
- this.stateByDevice.set(deviceId, handle);
22680
+ var RESUBSCRIBE_AFTER_FAILURES = 2;
22681
+ /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
22682
+ var RESUBSCRIBE_THROTTLE_TICKS = 5;
22683
+ /** First subscribe-retry delay, doubled on every subsequent failure. */
22684
+ var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
22685
+ /**
22686
+ * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller — fast
22687
+ * enough to recover within a single reconcile of the orchestrator and slow
22688
+ * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
22689
+ */
22690
+ var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
22691
+ /**
22692
+ * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
22693
+ *
22694
+ * Always resolves to a teardown closure — when the broker is not yet
22695
+ * registered the closure cancels the ongoing retry loop; when polling is
22696
+ * active it stops the loop and releases the broker subscription. Mirrors
22697
+ * `startFrameHandlePoller` so video and audio recover identically.
22698
+ */
22699
+ function startAudioChunkPoller(options) {
22700
+ const lifecycle = {
22701
+ stopped: false,
22702
+ retryTimer: void 0,
22703
+ pollTimer: void 0,
22704
+ activeSubscriptionId: null
22705
+ };
22706
+ const teardown = () => {
22707
+ if (lifecycle.stopped) return;
22708
+ lifecycle.stopped = true;
22709
+ if (lifecycle.retryTimer) {
22710
+ clearTimeout(lifecycle.retryTimer);
22711
+ lifecycle.retryTimer = void 0;
22584
22712
  }
22585
- return handle;
22586
- }
22587
- async listRules({ deviceId, stage }) {
22588
- return this.loadRules(deviceId, stage);
22589
- }
22590
- async setRules({ deviceId, stage, rules }) {
22591
- const parsed = RulesArraySchema.safeParse(rules);
22592
- if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
22593
- await this.persist(deviceId, stage, parsed.data);
22594
- }
22595
- /** Drop a device's cache entries. Called when the device is removed. */
22596
- forgetDevice(deviceId) {
22597
- this.cache.delete(deviceId);
22598
- this.stateByDevice.delete(deviceId);
22599
- }
22600
- async loadRules(deviceId, stage) {
22601
- let perDevice = this.cache.get(deviceId);
22602
- if (!perDevice) {
22603
- perDevice = /* @__PURE__ */ new Map();
22604
- this.cache.set(deviceId, perDevice);
22713
+ if (lifecycle.pollTimer) {
22714
+ clearTimeout(lifecycle.pollTimer);
22715
+ lifecycle.pollTimer = void 0;
22605
22716
  }
22606
- const cached = perDevice.get(stage);
22607
- if (cached) return cached;
22608
- let rules = [];
22717
+ const subId = lifecycle.activeSubscriptionId;
22718
+ if (subId) {
22719
+ lifecycle.activeSubscriptionId = null;
22720
+ options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }).catch((err) => {
22721
+ options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
22722
+ brokerId: options.brokerId,
22723
+ subscriptionId: subId,
22724
+ error: errMsg(err)
22725
+ } });
22726
+ });
22727
+ }
22728
+ };
22729
+ subscribeWithRetry(options, lifecycle);
22730
+ return teardown;
22731
+ }
22732
+ /**
22733
+ * Run the subscribe → poll handshake with exponential backoff on subscribe
22734
+ * failures. Resolves once the subscription is acquired (and the poll loop has
22735
+ * been started) or once `lifecycle.stopped` flips, whichever comes first.
22736
+ */
22737
+ async function subscribeWithRetry(options, lifecycle) {
22738
+ const { api, brokerId, tag, logger } = options;
22739
+ let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
22740
+ let attempt = 0;
22741
+ while (!lifecycle.stopped) {
22742
+ attempt += 1;
22609
22743
  try {
22610
- const raw = (await this.rulesState(deviceId).get())[stage];
22611
- if (raw !== void 0) {
22612
- const parsed = RulesArraySchema.safeParse(raw);
22613
- if (parsed.success) rules = parsed.data;
22614
- else this.ctx.logger.warn("zone-rules store entry failed to parse — resetting", {
22615
- tags: { deviceId },
22616
- meta: {
22617
- stage,
22618
- issues: parsed.error.issues
22619
- }
22744
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
22745
+ brokerId,
22746
+ tag
22747
+ });
22748
+ if (lifecycle.stopped) {
22749
+ await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
22750
+ logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
22751
+ brokerId,
22752
+ subscriptionId: result.subscriptionId,
22753
+ error: errMsg(err)
22754
+ } });
22620
22755
  });
22756
+ return;
22621
22757
  }
22758
+ if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
22759
+ brokerId,
22760
+ tag,
22761
+ attempt
22762
+ } });
22763
+ lifecycle.activeSubscriptionId = result.subscriptionId;
22764
+ startPolling(options, lifecycle);
22765
+ return;
22622
22766
  } catch (err) {
22623
- this.ctx.logger.warn("zone-rules store read failed — using empty list", {
22624
- tags: { deviceId },
22625
- meta: {
22626
- stage,
22627
- error: err instanceof Error ? err.message : String(err)
22628
- }
22629
- });
22767
+ if (lifecycle.stopped) return;
22768
+ if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
22769
+ brokerId,
22770
+ tag,
22771
+ error: errMsg(err),
22772
+ nextRetryInMs: backoffMs
22773
+ } });
22774
+ else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
22775
+ brokerId,
22776
+ tag,
22777
+ attempt,
22778
+ error: errMsg(err),
22779
+ nextRetryInMs: backoffMs
22780
+ } });
22781
+ await sleep(backoffMs, lifecycle);
22782
+ backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
22630
22783
  }
22631
- perDevice.set(stage, rules);
22632
- return rules;
22633
22784
  }
22634
- async persist(deviceId, stage, rules) {
22635
- let perDevice = this.cache.get(deviceId);
22636
- if (!perDevice) {
22637
- perDevice = /* @__PURE__ */ new Map();
22638
- this.cache.set(deviceId, perDevice);
22785
+ }
22786
+ /**
22787
+ * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
22788
+ * trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
22789
+ * the broker child restart case where our `subscriptionId` is silently
22790
+ * disowned.
22791
+ */
22792
+ function startPolling(options, lifecycle) {
22793
+ const { api, brokerId, tag, onChunk, logger } = options;
22794
+ let consecutiveFailures = 0;
22795
+ const resubscribe = async () => {
22796
+ try {
22797
+ const result = await api.streamBroker.subscribeAudioChunks.mutate({
22798
+ brokerId,
22799
+ tag
22800
+ });
22801
+ lifecycle.activeSubscriptionId = result.subscriptionId;
22802
+ logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
22803
+ brokerId,
22804
+ tag,
22805
+ subscriptionId: result.subscriptionId,
22806
+ afterFailures: consecutiveFailures
22807
+ } });
22808
+ return true;
22809
+ } catch {
22810
+ return false;
22639
22811
  }
22640
- perDevice.set(stage, rules);
22641
- await this.rulesState(deviceId).update((prev) => ({
22642
- ...prev,
22643
- [stage]: rules
22644
- }));
22645
- const otherStage = stage === "motion" ? "detection" : "motion";
22646
- const otherRules = perDevice.get(otherStage) ?? await this.loadRules(deviceId, otherStage);
22647
- const sliceValue = stage === "motion" ? {
22648
- motion: rules,
22649
- detection: otherRules
22650
- } : {
22651
- motion: otherRules,
22652
- detection: rules
22653
- };
22812
+ };
22813
+ const tick = async () => {
22814
+ if (lifecycle.stopped) return;
22815
+ const subId = lifecycle.activeSubscriptionId;
22816
+ if (!subId) return;
22654
22817
  try {
22655
- await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
22656
- capName: ZONE_RULES_CAP_NAME,
22657
- slice: sliceValue
22818
+ const chunks = await api.streamBroker.pullAudioChunks.query({
22819
+ subscriptionId: subId,
22820
+ maxCount: PULL_MAX_COUNT
22658
22821
  });
22822
+ if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
22823
+ brokerId,
22824
+ subscriptionId: subId
22825
+ } });
22826
+ consecutiveFailures = 0;
22827
+ for (const chunk of chunks) {
22828
+ if (lifecycle.stopped) break;
22829
+ await onChunk(chunk);
22830
+ }
22659
22831
  } catch (err) {
22660
- this.ctx.logger.debug("zone-rules slice mirror failed", {
22661
- tags: { deviceId },
22662
- meta: {
22663
- stage,
22664
- error: err instanceof Error ? err.message : String(err)
22665
- }
22666
- });
22667
- }
22668
- this.ctx.onRulesChanged?.(deviceId, stage, rules);
22669
- }
22670
- };
22671
- //#endregion
22672
- //#region src/orchestrator-store-schemas.ts
22673
- /**
22674
- * `orchestrator-store-schemas.ts` — whole-blob Zod schemas for the
22675
- * orchestrator's addon-store keys that persist an ENTIRE collection
22676
- * under a single key (`nodeBindings`, `templates`, `agentSettings`,
22677
- * `cameraSettings`).
22678
- *
22679
- * These back the durable-state primitive (`this.state(KEY, Schema, {})`)
22680
- * so a hand-written serializer can never silently drop a field on save:
22681
- * the whole validated value round-trips on every read and write.
22682
- *
22683
- * Each schema mirrors the SAME shape the orchestrator already persisted
22684
- * (the `pipeline-orchestrator.cap.ts` storage contract). Every field that
22685
- * is only conditionally written is `.optional()`, so an existing blob
22686
- * that predates a newer field still loads — durable-state `get()` only
22687
- * falls back to the empty map on a whole-blob parse failure, so the
22688
- * schema is kept faithful to the on-disk shape rather than overly strict.
22689
- */
22690
- var EngineChoiceSchema = object({
22691
- runtime: _enum(["node", "python"]),
22692
- backend: string(),
22693
- format: string(),
22694
- device: string().optional()
22695
- });
22696
- var NodeBindingsSchema = record(string(), record(string(), string()));
22697
- var StoredPipelineConfigSchema = object({
22698
- engine: EngineChoiceSchema,
22699
- steps: array(PipelineStepInputSchema).readonly(),
22700
- audio: object({
22701
- engine: EngineChoiceSchema,
22702
- modelId: string(),
22703
- enabled: boolean(),
22704
- settings: record(string(), unknown()).readonly().optional()
22705
- }).nullable().optional()
22706
- });
22707
- var StoredPipelineTemplateSchema = object({
22708
- id: string(),
22709
- name: string(),
22710
- description: string().optional(),
22711
- config: StoredPipelineConfigSchema,
22712
- createdAt: string(),
22713
- updatedAt: string()
22714
- });
22715
- var TemplatesSchema = record(string(), StoredPipelineTemplateSchema);
22716
- var StoredAgentAddonConfigSchema = object({
22717
- enabled: boolean(),
22718
- modelId: string(),
22719
- settings: record(string(), unknown()).readonly()
22720
- });
22721
- var StoredAgentPipelineSettingsSchema = object({
22722
- addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
22723
- maxCameras: number().int().nonnegative().nullable().default(null)
22724
- });
22725
- var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
22726
- var StoredCameraStepOverridePatchSchema = object({
22727
- enabled: boolean().optional(),
22728
- modelId: string().optional(),
22729
- settings: record(string(), unknown()).readonly().optional()
22730
- });
22731
- var StoredCameraPipelineForAgentSchema = object({
22732
- steps: array(PipelineStepInputSchema).readonly(),
22733
- audio: object({
22734
- modelId: string(),
22735
- enabled: boolean()
22736
- }).nullable()
22737
- });
22738
- var StoredCameraPipelineSettingsSchema = object({
22739
- pinnedAgentNodeId: string().optional(),
22740
- stepToggles: record(string(), boolean()).optional(),
22741
- stepOverridesByAgent: record(string(), record(string(), StoredCameraStepOverridePatchSchema)).optional(),
22742
- pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
22743
- /**
22744
- * Legacy "nuke inference" flag. Superseded by the detection-pipeline
22745
- * wrapper binding, but the one-shot boot migration
22746
- * (`migrateLegacyFlagsToBindings`) still reads it off the persisted
22747
- * blob to flip the binding off. Kept here so the durable round-trip
22748
- * does NOT strip it before the migration runs.
22749
- */
22750
- disableInference: boolean().optional()
22751
- });
22752
- var CameraSettingsMapSchema = record(string(), StoredCameraPipelineSettingsSchema);
22753
- //#endregion
22754
- //#region src/load-balancer.ts
22755
- /**
22756
- * Compute the L2 capacity score for a runner node. Lower is better.
22757
- * The score is a weighted sum of the runner's active workload so the balancer
22758
- * prefers agents that are serving fewer cameras OR draining queues quickly.
22759
- *
22760
- * Rationale:
22761
- * - `attachedCameras * avgInferenceFps` approximates the total inference rate
22762
- * the agent is currently sustaining (not just how many cameras are assigned).
22763
- * - `queueDepthTotal` penalises agents that are falling behind the frame feed.
22764
- */
22765
- function computeCapacityScore(load) {
22766
- return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
22767
- }
22768
- /**
22769
- * Returns true when the node has remaining capacity.
22770
- * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
22771
- * its `attachedCameras` count is strictly less than the cap.
22772
- * Pins count toward the cap via `attachedCameras`.
22773
- */
22774
- function isEligible(node, caps) {
22775
- const cap = caps?.[node.nodeId];
22776
- if (cap === null || cap === void 0 || cap <= 0) return true;
22777
- return node.attachedCameras < cap;
22778
- }
22779
- /**
22780
- * Run the two-level camera balancer.
22781
- *
22782
- * L1 (manual affinity): if `preferredAgent` names an online node AND that
22783
- * node is under its `maxCameras` cap, return it. If the node is online but at
22784
- * or over cap, return `{kind:'pending'}` — a pinned camera is never silently
22785
- * over-assigned.
22786
- *
22787
- * L2 (capacity): filter to eligible nodes and pick the lowest capacity score.
22788
- * If all nodes are at/over cap, return `{kind:'pending'}`.
22789
- *
22790
- * Returns `null` when no runners are online. The orchestrator decides how to
22791
- * react — typically by logging and deferring the assignment until a runner
22792
- * comes online.
22793
- */
22794
- function balance(input) {
22795
- const online = input.nodes.filter((n) => n.nodeId.length > 0);
22796
- if (online.length === 0) return null;
22797
- const eligible = online.filter((n) => isEligible(n, input.nodeCaps));
22798
- if (input.preferredAgent) {
22799
- const pinnedOnline = online.find((n) => n.nodeId === input.preferredAgent);
22800
- if (pinnedOnline) {
22801
- if (eligible.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
22802
- kind: "assigned",
22803
- agentNodeId: pinnedOnline.nodeId,
22804
- reason: "manual",
22805
- score: computeCapacityScore(pinnedOnline)
22806
- };
22807
- return {
22808
- kind: "pending",
22809
- reason: "over-cap"
22810
- };
22832
+ consecutiveFailures += 1;
22833
+ if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
22834
+ brokerId,
22835
+ subscriptionId: subId,
22836
+ error: errMsg(err)
22837
+ } });
22838
+ if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
22811
22839
  }
22812
- }
22813
- if (eligible.length === 0) return {
22814
- kind: "pending",
22815
- reason: "over-cap"
22816
- };
22817
- const best = eligible.map((node) => ({
22818
- node,
22819
- score: computeCapacityScore(node)
22820
- })).toSorted((a, b) => a.score - b.score)[0];
22821
- return {
22822
- kind: "assigned",
22823
- agentNodeId: best.node.nodeId,
22824
- reason: "capacity",
22825
- score: best.score
22840
+ if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
22826
22841
  };
22842
+ tick();
22827
22843
  }
22828
22844
  /**
22829
- * Decide whether a manual per-device decoder pin may be honored.
22830
- *
22831
- * A pin to a node OUTSIDE `enabledDecoderNodes` must NOT be honored blindly —
22832
- * that is how video decode reaches a node that is not eligible to decode (e.g.
22833
- * a node whose shm frame ring the broker cannot read, or one an operator
22834
- * disabled). A pin to an ENABLED node is honored; anything else falls through
22835
- * to the auto-balance path (which filters by `enabledDecoderNodes`).
22836
- */
22837
- function isDecoderPinHonored(pinnedNodeId, enabledDecoderNodes) {
22838
- return enabledDecoderNodes.includes(pinnedNodeId);
22839
- }
22840
- /**
22841
- * Choose decoder node. Priority: manual pin → co-located with pipeline → capacity.
22845
+ * Cancellable sleep wakes early when `lifecycle.stopped` flips. We
22846
+ * keep a local wrapper around the shared {@link sleep} helper because
22847
+ * the lifecycle tracks the active retry timer for `teardown()` to
22848
+ * clear; pure `sleep()` would leak the timer if teardown fired while
22849
+ * we were waiting.
22842
22850
  */
22843
- function balanceDecoder(input) {
22844
- const { decoderNodes, pipelineNodeId, preferredDecoderNode } = input;
22845
- if (decoderNodes.length === 0) return null;
22846
- if (preferredDecoderNode) {
22847
- const match = decoderNodes.find((n) => n.nodeId === preferredDecoderNode);
22848
- if (match) return {
22849
- decoderNodeId: match.nodeId,
22850
- reason: "manual",
22851
- score: computeCapacityScore(match)
22852
- };
22853
- }
22854
- const colocated = decoderNodes.find((n) => n.nodeId === pipelineNodeId);
22855
- if (colocated) return {
22856
- decoderNodeId: colocated.nodeId,
22857
- reason: "co-located",
22858
- score: computeCapacityScore(colocated)
22859
- };
22860
- const best = decoderNodes.map((node) => ({
22861
- node,
22862
- score: computeCapacityScore(node)
22863
- })).toSorted((a, b) => a.score - b.score)[0];
22864
- return {
22865
- decoderNodeId: best.node.nodeId,
22866
- reason: "capacity",
22867
- score: best.score
22868
- };
22851
+ function sleep(ms, lifecycle) {
22852
+ return new Promise((resolve) => {
22853
+ if (lifecycle.stopped) {
22854
+ resolve();
22855
+ return;
22856
+ }
22857
+ lifecycle.retryTimer = setTimeout(() => {
22858
+ lifecycle.retryTimer = void 0;
22859
+ resolve();
22860
+ }, ms);
22861
+ });
22869
22862
  }
22870
22863
  //#endregion
22871
22864
  //#region src/audio-load-balancer.ts
@@ -22884,259 +22877,393 @@ function balanceAudio(input) {
22884
22877
  };
22885
22878
  }
22886
22879
  //#endregion
22887
- //#region src/audio-chunk-poller.ts
22888
- /**
22889
- * `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
22890
- * plane (Phase 5 / D9).
22891
- *
22892
- * Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
22893
- * path. A live callback cannot cross a process boundary; once the `pipeline`
22894
- * group is dissolved (Task 8) the orchestrator runs in a different process
22895
- * from the broker, so audio delivery must go over tRPC.
22896
- *
22897
- * The consumer:
22898
- *
22899
- * 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC — the broker
22900
- * registers a per-subscription bounded FIFO queue and returns a
22901
- * `subscriptionId`;
22902
- * 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
22903
- * interval, draining `DecodedAudioChunk`s in FIFO arrival order;
22904
- * 3. feeds each chunk to its downstream audio logic;
22905
- * 4. on teardown, `unsubscribeAudioChunks`.
22906
- *
22907
- * Audio is not latency-critical like video, and chunks arrive only ~every
22908
- * 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
22909
- * a small per-poll burst keeps latency low without busy-spinning. The
22910
- * broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
22911
- * loses a chunk.
22912
- *
22913
- * Boot-race tolerance: the broker for a given camStream may not be registered
22914
- * yet when the orchestrator wires the subscription (provider addons publish
22915
- * their cameraStreams asynchronously after their probe completes).
22916
- * `subscribeAudioChunks` retries with exponential backoff (capped at
22917
- * `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
22918
- * teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
22919
- * shape so video and audio plumbing self-heal identically.
22920
- */
22921
- /** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
22922
- var POLL_INTERVAL_MS = 200;
22923
- /** How many chunks to drain per poll — a small burst absorbs jitter. */
22924
- var PULL_MAX_COUNT = 8;
22880
+ //#region src/camera-status/compose-camera-status.ts
22881
+ function mapAssignment(input) {
22882
+ return {
22883
+ detectionNodeId: input.detectionNodeId,
22884
+ decoderNodeId: input.decoderNodeId,
22885
+ audioNodeId: input.audioNodeId,
22886
+ pinned: {
22887
+ detection: input.pinned.detection,
22888
+ decoder: input.pinned.decoder,
22889
+ audio: input.pinned.audio
22890
+ },
22891
+ reasons: {
22892
+ detection: input.reasons.detection,
22893
+ decoder: input.reasons.decoder,
22894
+ audio: input.reasons.audio
22895
+ }
22896
+ };
22897
+ }
22898
+ function mapSource(sourceResult) {
22899
+ if (sourceResult === null) return { streams: [] };
22900
+ return { streams: sourceResult.streams.map((s) => ({
22901
+ camStreamId: s.camStreamId,
22902
+ codec: s.codec,
22903
+ width: s.width,
22904
+ height: s.height,
22905
+ fps: s.fps,
22906
+ kind: s.kind
22907
+ })) };
22908
+ }
22909
+ function mapBroker(brokerResult) {
22910
+ if (brokerResult === null) return null;
22911
+ return {
22912
+ profiles: brokerResult.profiles.map((p) => ({
22913
+ profile: p.profile,
22914
+ status: p.status,
22915
+ codec: p.codec,
22916
+ width: p.width,
22917
+ height: p.height,
22918
+ subscribers: p.subscribers,
22919
+ inFps: p.inFps,
22920
+ outFps: p.outFps
22921
+ })),
22922
+ webrtcSessions: brokerResult.webrtcSessions,
22923
+ rtspRestream: brokerResult.rtspRestream
22924
+ };
22925
+ }
22926
+ function mapDecoderShm(shm) {
22927
+ return {
22928
+ framesWritten: shm.framesWritten,
22929
+ getFrameHits: shm.getFrameHits,
22930
+ getFrameMisses: shm.getFrameMisses,
22931
+ budgetMb: shm.budgetMb
22932
+ };
22933
+ }
22934
+ function mapDecoder(decoderResult) {
22935
+ if (decoderResult === null) return null;
22936
+ return {
22937
+ nodeId: decoderResult.nodeId,
22938
+ formats: [...decoderResult.formats],
22939
+ sessionCount: decoderResult.sessionCount,
22940
+ shm: mapDecoderShm(decoderResult.shm)
22941
+ };
22942
+ }
22943
+ function mapMotion(motionResult) {
22944
+ if (motionResult === null) return null;
22945
+ return {
22946
+ enabled: motionResult.enabled,
22947
+ fps: motionResult.fps
22948
+ };
22949
+ }
22950
+ function mapProvisioning(p) {
22951
+ if (p.error !== void 0) return {
22952
+ state: p.state,
22953
+ error: p.error
22954
+ };
22955
+ return { state: p.state };
22956
+ }
22957
+ function mapDetection(detectionResult) {
22958
+ if (detectionResult === null) return null;
22959
+ const phase = detectionResult.phase;
22960
+ return {
22961
+ nodeId: detectionResult.nodeId,
22962
+ engine: {
22963
+ backend: detectionResult.engine.backend,
22964
+ device: detectionResult.engine.device
22965
+ },
22966
+ phase,
22967
+ configuredFps: detectionResult.configuredFps,
22968
+ actualFps: detectionResult.actualFps,
22969
+ queueDepth: detectionResult.queueDepth,
22970
+ avgInferenceMs: detectionResult.avgInferenceMs,
22971
+ provisioning: mapProvisioning(detectionResult.provisioning)
22972
+ };
22973
+ }
22974
+ function mapAudio(audioResult) {
22975
+ if (audioResult === null) return null;
22976
+ return {
22977
+ nodeId: audioResult.nodeId,
22978
+ enabled: audioResult.enabled
22979
+ };
22980
+ }
22981
+ function mapRecording(recordingResult) {
22982
+ if (recordingResult === null) return null;
22983
+ return {
22984
+ mode: recordingResult.mode,
22985
+ active: recordingResult.active,
22986
+ storageBytes: recordingResult.storageBytes
22987
+ };
22988
+ }
22925
22989
  /**
22926
- * Consecutive pull failures before we attempt to re-subscribe. A single failed
22927
- * poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
22928
- * sustained failure means the broker child restarted and dropped our
22929
- * subscription, so we re-establish it.
22990
+ * Pure function that composes a `CameraStatus` from per-stage fetch results.
22991
+ *
22992
+ * - `assignment` is always built from orchestrator-local data (never null).
22993
+ * - `source` always present: defaults to `{ streams: [] }` when sourceResult is null.
22994
+ * - Every other block is null when its stage result is null (graceful degradation).
22995
+ * - `fetchedAt` is stamped exactly as provided — never calls `Date.now()`.
22996
+ * - No mutation of the input.
22930
22997
  */
22931
- var RESUBSCRIBE_AFTER_FAILURES = 2;
22932
- /** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
22933
- var RESUBSCRIBE_THROTTLE_TICKS = 5;
22934
- /** First subscribe-retry delay, doubled on every subsequent failure. */
22935
- var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
22998
+ function composeCameraStatus(input) {
22999
+ return {
23000
+ deviceId: input.deviceId,
23001
+ assignment: mapAssignment(input),
23002
+ source: mapSource(input.sourceResult),
23003
+ broker: mapBroker(input.brokerResult),
23004
+ decoder: mapDecoder(input.decoderResult),
23005
+ motion: mapMotion(input.motionResult),
23006
+ detection: mapDetection(input.detectionResult),
23007
+ audio: mapAudio(input.audioResult),
23008
+ recording: mapRecording(input.recordingResult),
23009
+ fetchedAt: input.fetchedAt
23010
+ };
23011
+ }
23012
+ //#endregion
23013
+ //#region src/dispatch-reconcile.ts
22936
23014
  /**
22937
- * Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller fast
22938
- * enough to recover within a single reconcile of the orchestrator and slow
22939
- * enough that a misconfigured camStream costs ~12 op/min, not 5/s.
23015
+ * Derives the set of deviceIds that currently have at least one broker
23016
+ * profile slot that is assigned (status !== 'unassigned') AND has a
23017
+ * non-null sourceCamStreamId.
23018
+ *
23019
+ * This is the "desired" fleet — cameras the orchestrator should have
23020
+ * dispatched. Used by `reconcileDispatch` to compute the gap against
23021
+ * `cameraConfigs`.
22940
23022
  */
22941
- var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
23023
+ function desiredDeviceIdsFromSlots(slots) {
23024
+ const result = /* @__PURE__ */ new Set();
23025
+ for (const slot of slots) if (slot.status !== "unassigned" && slot.sourceCamStreamId !== null) result.add(slot.deviceId);
23026
+ return result;
23027
+ }
22942
23028
  /**
22943
- * Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
23029
+ * Returns the deviceIds that are present in `desired` but absent from
23030
+ * `known`. These are the cameras the orchestrator has not yet dispatched
23031
+ * and needs to process via `handleDeviceRegistered`.
22944
23032
  *
22945
- * Always resolves to a teardown closure when the broker is not yet
22946
- * registered the closure cancels the ongoing retry loop; when polling is
22947
- * active it stops the loop and releases the broker subscription. Mirrors
22948
- * `startFrameHandlePoller` so video and audio recover identically.
23033
+ * Additive only cameras present in `known` are never included, even
23034
+ * if they fall out of `desired`, to avoid fighting the live event path.
22949
23035
  */
22950
- function startAudioChunkPoller(options) {
22951
- const lifecycle = {
22952
- stopped: false,
22953
- retryTimer: void 0,
22954
- pollTimer: void 0,
22955
- activeSubscriptionId: null
22956
- };
22957
- const teardown = () => {
22958
- if (lifecycle.stopped) return;
22959
- lifecycle.stopped = true;
22960
- if (lifecycle.retryTimer) {
22961
- clearTimeout(lifecycle.retryTimer);
22962
- lifecycle.retryTimer = void 0;
22963
- }
22964
- if (lifecycle.pollTimer) {
22965
- clearTimeout(lifecycle.pollTimer);
22966
- lifecycle.pollTimer = void 0;
22967
- }
22968
- const subId = lifecycle.activeSubscriptionId;
22969
- if (subId) {
22970
- lifecycle.activeSubscriptionId = null;
22971
- options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }).catch((err) => {
22972
- options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
22973
- brokerId: options.brokerId,
22974
- subscriptionId: subId,
22975
- error: errMsg(err)
22976
- } });
22977
- });
22978
- }
22979
- };
22980
- subscribeWithRetry(options, lifecycle);
22981
- return teardown;
23036
+ function computeDispatchGap(desired, known) {
23037
+ return [...desired].filter((id) => !known.has(id));
22982
23038
  }
23039
+ //#endregion
23040
+ //#region src/load-balancer.ts
22983
23041
  /**
22984
- * Run the subscribe poll handshake with exponential backoff on subscribe
22985
- * failures. Resolves once the subscription is acquired (and the poll loop has
22986
- * been started) or once `lifecycle.stopped` flips, whichever comes first.
23042
+ * Compute the L2 capacity score for a runner node. Lower is better.
23043
+ * The score is a weighted sum of the runner's active workload so the balancer
23044
+ * prefers agents that are serving fewer cameras OR draining queues quickly.
23045
+ *
23046
+ * Rationale:
23047
+ * - `attachedCameras * avgInferenceFps` approximates the total inference rate
23048
+ * the agent is currently sustaining (not just how many cameras are assigned).
23049
+ * - `queueDepthTotal` penalises agents that are falling behind the frame feed.
22987
23050
  */
22988
- async function subscribeWithRetry(options, lifecycle) {
22989
- const { api, brokerId, tag, logger } = options;
22990
- let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
22991
- let attempt = 0;
22992
- while (!lifecycle.stopped) {
22993
- attempt += 1;
22994
- try {
22995
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
22996
- brokerId,
22997
- tag
22998
- });
22999
- if (lifecycle.stopped) {
23000
- await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
23001
- logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
23002
- brokerId,
23003
- subscriptionId: result.subscriptionId,
23004
- error: errMsg(err)
23005
- } });
23006
- });
23007
- return;
23008
- }
23009
- if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
23010
- brokerId,
23011
- tag,
23012
- attempt
23013
- } });
23014
- lifecycle.activeSubscriptionId = result.subscriptionId;
23015
- startPolling(options, lifecycle);
23016
- return;
23017
- } catch (err) {
23018
- if (lifecycle.stopped) return;
23019
- if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
23020
- brokerId,
23021
- tag,
23022
- error: errMsg(err),
23023
- nextRetryInMs: backoffMs
23024
- } });
23025
- else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
23026
- brokerId,
23027
- tag,
23028
- attempt,
23029
- error: errMsg(err),
23030
- nextRetryInMs: backoffMs
23031
- } });
23032
- await sleep(backoffMs, lifecycle);
23033
- backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
23034
- }
23035
- }
23051
+ function computeCapacityScore(load) {
23052
+ return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
23036
23053
  }
23037
23054
  /**
23038
- * Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
23039
- * trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
23040
- * the broker child restart case where our `subscriptionId` is silently
23041
- * disowned.
23055
+ * Returns true when the node has remaining capacity.
23056
+ * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
23057
+ * its `attachedCameras` count is strictly less than the cap.
23058
+ * Pins count toward the cap via `attachedCameras`.
23042
23059
  */
23043
- function startPolling(options, lifecycle) {
23044
- const { api, brokerId, tag, onChunk, logger } = options;
23045
- let consecutiveFailures = 0;
23046
- const resubscribe = async () => {
23047
- try {
23048
- const result = await api.streamBroker.subscribeAudioChunks.mutate({
23049
- brokerId,
23050
- tag
23051
- });
23052
- lifecycle.activeSubscriptionId = result.subscriptionId;
23053
- logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
23054
- brokerId,
23055
- tag,
23056
- subscriptionId: result.subscriptionId,
23057
- afterFailures: consecutiveFailures
23058
- } });
23059
- return true;
23060
- } catch {
23061
- return false;
23060
+ function isEligible(node, caps) {
23061
+ const cap = caps?.[node.nodeId];
23062
+ if (cap === null || cap === void 0 || cap <= 0) return true;
23063
+ return node.attachedCameras < cap;
23064
+ }
23065
+ /**
23066
+ * Run the two-level camera balancer.
23067
+ *
23068
+ * Frame-source constraint: when `eligibleNodes` is set, only nodes in that set
23069
+ * can obtain this camera's decoded frames. Such a node is a prerequisite for
23070
+ * BOTH L1 and L2 — a node outside `eligibleNodes` is never assignable, and a
23071
+ * pin to an online-but-ineligible node returns `{kind:'pending',
23072
+ * reason:'no-frame-source'}` (mirrors the over-cap-pin behaviour: a pinned
23073
+ * camera is never silently placed on a different node).
23074
+ *
23075
+ * L1 (manual affinity): if `preferredAgent` names an online + frame-sourceable
23076
+ * node AND that node is under its `maxCameras` cap, return it. If the node is
23077
+ * online but at/over cap, return `{kind:'pending', reason:'over-cap'}`; if it
23078
+ * is online but not frame-sourceable, `{kind:'pending', reason:'no-frame-source'}`
23079
+ * — a pinned camera is never silently over-assigned or re-homed.
23080
+ *
23081
+ * L2 (capacity): among the frame-sourceable nodes, filter to those under cap
23082
+ * and pick the lowest capacity score. If no frame-sourceable node exists (but
23083
+ * online nodes do), return `no-frame-source`; if they exist but are all at/over
23084
+ * cap, return `over-cap`.
23085
+ *
23086
+ * Returns `null` when no runners are online. The orchestrator decides how to
23087
+ * react — typically by logging and deferring the assignment until a runner
23088
+ * comes online.
23089
+ */
23090
+ function balance(input) {
23091
+ const online = input.nodes.filter((n) => n.nodeId.length > 0);
23092
+ if (online.length === 0) return null;
23093
+ const sourceable = input.eligibleNodes ? online.filter((n) => input.eligibleNodes.includes(n.nodeId)) : online;
23094
+ const eligible = sourceable.filter((n) => isEligible(n, input.nodeCaps));
23095
+ if (input.preferredAgent) {
23096
+ const pinnedOnline = online.find((n) => n.nodeId === input.preferredAgent);
23097
+ if (pinnedOnline) {
23098
+ if (!sourceable.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
23099
+ kind: "pending",
23100
+ reason: "no-frame-source"
23101
+ };
23102
+ if (eligible.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
23103
+ kind: "assigned",
23104
+ agentNodeId: pinnedOnline.nodeId,
23105
+ reason: "manual",
23106
+ score: computeCapacityScore(pinnedOnline)
23107
+ };
23108
+ return {
23109
+ kind: "pending",
23110
+ reason: "over-cap"
23111
+ };
23062
23112
  }
23113
+ }
23114
+ if (eligible.length === 0) return {
23115
+ kind: "pending",
23116
+ reason: sourceable.length === 0 ? "no-frame-source" : "over-cap"
23063
23117
  };
23064
- const tick = async () => {
23065
- if (lifecycle.stopped) return;
23066
- const subId = lifecycle.activeSubscriptionId;
23067
- if (!subId) return;
23068
- try {
23069
- const chunks = await api.streamBroker.pullAudioChunks.query({
23070
- subscriptionId: subId,
23071
- maxCount: PULL_MAX_COUNT
23072
- });
23073
- if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
23074
- brokerId,
23075
- subscriptionId: subId
23076
- } });
23077
- consecutiveFailures = 0;
23078
- for (const chunk of chunks) {
23079
- if (lifecycle.stopped) break;
23080
- await onChunk(chunk);
23081
- }
23082
- } catch (err) {
23083
- consecutiveFailures += 1;
23084
- if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
23085
- brokerId,
23086
- subscriptionId: subId,
23087
- error: errMsg(err)
23088
- } });
23089
- if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
23090
- }
23091
- if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
23118
+ const best = eligible.map((node) => ({
23119
+ node,
23120
+ score: computeCapacityScore(node)
23121
+ })).toSorted((a, b) => a.score - b.score)[0];
23122
+ return {
23123
+ kind: "assigned",
23124
+ agentNodeId: best.node.nodeId,
23125
+ reason: "capacity",
23126
+ score: best.score
23092
23127
  };
23093
- tick();
23094
23128
  }
23095
23129
  /**
23096
- * Cancellable sleep wakes early when `lifecycle.stopped` flips. We
23097
- * keep a local wrapper around the shared {@link sleep} helper because
23098
- * the lifecycle tracks the active retry timer for `teardown()` to
23099
- * clear; pure `sleep()` would leak the timer if teardown fired while
23100
- * we were waiting.
23130
+ * Decide whether a manual per-device decoder pin may be honored.
23131
+ *
23132
+ * A pin to a node OUTSIDE `enabledDecoderNodes` must NOT be honored blindly —
23133
+ * that is how video decode reaches a node that is not eligible to decode (e.g.
23134
+ * a node whose shm frame ring the broker cannot read, or one an operator
23135
+ * disabled). A pin to an ENABLED node is honored; anything else falls through
23136
+ * to the auto-balance path (which filters by `enabledDecoderNodes`).
23101
23137
  */
23102
- function sleep(ms, lifecycle) {
23103
- return new Promise((resolve) => {
23104
- if (lifecycle.stopped) {
23105
- resolve();
23106
- return;
23107
- }
23108
- lifecycle.retryTimer = setTimeout(() => {
23109
- lifecycle.retryTimer = void 0;
23110
- resolve();
23111
- }, ms);
23112
- });
23138
+ function isDecoderPinHonored(pinnedNodeId, enabledDecoderNodes) {
23139
+ return enabledDecoderNodes.includes(pinnedNodeId);
23140
+ }
23141
+ /**
23142
+ * Choose decoder node. Priority: manual pin → co-located with pipeline → capacity.
23143
+ */
23144
+ function balanceDecoder(input) {
23145
+ const { decoderNodes, pipelineNodeId, preferredDecoderNode } = input;
23146
+ if (decoderNodes.length === 0) return null;
23147
+ if (preferredDecoderNode) {
23148
+ const match = decoderNodes.find((n) => n.nodeId === preferredDecoderNode);
23149
+ if (match) return {
23150
+ decoderNodeId: match.nodeId,
23151
+ reason: "manual",
23152
+ score: computeCapacityScore(match)
23153
+ };
23154
+ }
23155
+ const colocated = decoderNodes.find((n) => n.nodeId === pipelineNodeId);
23156
+ if (colocated) return {
23157
+ decoderNodeId: colocated.nodeId,
23158
+ reason: "co-located",
23159
+ score: computeCapacityScore(colocated)
23160
+ };
23161
+ const best = decoderNodes.map((node) => ({
23162
+ node,
23163
+ score: computeCapacityScore(node)
23164
+ })).toSorted((a, b) => a.score - b.score)[0];
23165
+ return {
23166
+ decoderNodeId: best.node.nodeId,
23167
+ reason: "capacity",
23168
+ score: best.score
23169
+ };
23113
23170
  }
23114
23171
  //#endregion
23115
- //#region src/dispatch-reconcile.ts
23172
+ //#region src/orchestrator-store-schemas.ts
23116
23173
  /**
23117
- * Derives the set of deviceIds that currently have at least one broker
23118
- * profile slot that is assigned (status !== 'unassigned') AND has a
23119
- * non-null sourceCamStreamId.
23174
+ * `orchestrator-store-schemas.ts` whole-blob Zod schemas for the
23175
+ * orchestrator's addon-store keys that persist an ENTIRE collection
23176
+ * under a single key (`nodeBindings`, `templates`, `agentSettings`,
23177
+ * `cameraSettings`).
23120
23178
  *
23121
- * This is the "desired" fleet cameras the orchestrator should have
23122
- * dispatched. Used by `reconcileDispatch` to compute the gap against
23123
- * `cameraConfigs`.
23179
+ * These back the durable-state primitive (`this.state(KEY, Schema, {})`)
23180
+ * so a hand-written serializer can never silently drop a field on save:
23181
+ * the whole validated value round-trips on every read and write.
23182
+ *
23183
+ * Each schema mirrors the SAME shape the orchestrator already persisted
23184
+ * (the `pipeline-orchestrator.cap.ts` storage contract). Every field that
23185
+ * is only conditionally written is `.optional()`, so an existing blob
23186
+ * that predates a newer field still loads — durable-state `get()` only
23187
+ * falls back to the empty map on a whole-blob parse failure, so the
23188
+ * schema is kept faithful to the on-disk shape rather than overly strict.
23124
23189
  */
23125
- function desiredDeviceIdsFromSlots(slots) {
23126
- const result = /* @__PURE__ */ new Set();
23127
- for (const slot of slots) if (slot.status !== "unassigned" && slot.sourceCamStreamId !== null) result.add(slot.deviceId);
23128
- return result;
23129
- }
23190
+ var EngineChoiceSchema = object({
23191
+ runtime: _enum(["node", "python"]),
23192
+ backend: string(),
23193
+ format: string(),
23194
+ device: string().optional()
23195
+ });
23196
+ var NodeBindingsSchema = record(string(), record(string(), string()));
23197
+ var StoredPipelineConfigSchema = object({
23198
+ engine: EngineChoiceSchema,
23199
+ steps: array(PipelineStepInputSchema).readonly(),
23200
+ audio: object({
23201
+ engine: EngineChoiceSchema,
23202
+ modelId: string(),
23203
+ enabled: boolean(),
23204
+ settings: record(string(), unknown()).readonly().optional()
23205
+ }).nullable().optional()
23206
+ });
23207
+ var StoredPipelineTemplateSchema = object({
23208
+ id: string(),
23209
+ name: string(),
23210
+ description: string().optional(),
23211
+ config: StoredPipelineConfigSchema,
23212
+ createdAt: string(),
23213
+ updatedAt: string()
23214
+ });
23215
+ var TemplatesSchema = record(string(), StoredPipelineTemplateSchema);
23216
+ var StoredAgentAddonConfigSchema = object({
23217
+ enabled: boolean(),
23218
+ modelId: string(),
23219
+ settings: record(string(), unknown()).readonly()
23220
+ });
23221
+ var StoredAgentPipelineSettingsSchema = object({
23222
+ addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
23223
+ maxCameras: number().int().nonnegative().nullable().default(null)
23224
+ });
23225
+ var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
23226
+ var StoredCameraStepOverridePatchSchema = object({
23227
+ enabled: boolean().optional(),
23228
+ modelId: string().optional(),
23229
+ settings: record(string(), unknown()).readonly().optional()
23230
+ });
23231
+ var StoredCameraPipelineForAgentSchema = object({
23232
+ steps: array(PipelineStepInputSchema).readonly(),
23233
+ audio: object({
23234
+ modelId: string(),
23235
+ enabled: boolean()
23236
+ }).nullable()
23237
+ });
23238
+ var StoredCameraPipelineSettingsSchema = object({
23239
+ pinnedAgentNodeId: string().optional(),
23240
+ stepToggles: record(string(), boolean()).optional(),
23241
+ stepOverridesByAgent: record(string(), record(string(), StoredCameraStepOverridePatchSchema)).optional(),
23242
+ pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
23243
+ /**
23244
+ * Legacy "nuke inference" flag. Superseded by the detection-pipeline
23245
+ * wrapper binding, but the one-shot boot migration
23246
+ * (`migrateLegacyFlagsToBindings`) still reads it off the persisted
23247
+ * blob to flip the binding off. Kept here so the durable round-trip
23248
+ * does NOT strip it before the migration runs.
23249
+ */
23250
+ disableInference: boolean().optional()
23251
+ });
23252
+ var CameraSettingsMapSchema = record(string(), StoredCameraPipelineSettingsSchema);
23253
+ //#endregion
23254
+ //#region src/pending-retry.ts
23130
23255
  /**
23131
- * Returns the deviceIds that are present in `desired` but absent from
23132
- * `known`. These are the cameras the orchestrator has not yet dispatched
23133
- * and needs to process via `handleDeviceRegistered`.
23256
+ * Returns the deviceIds that are KNOWN (present in `known`) but currently
23257
+ * UNASSIGNED (absent from `assigned`). These are the "stranded" cameras the
23258
+ * orchestrator has a runner config for them but no live pipeline assignment
23259
+ * (pending / over-cap / no-frame-source / load-shed).
23134
23260
  *
23135
- * Additive only cameras present in `known` are never included, even
23136
- * if they fall out of `desired`, to avoid fighting the live event path.
23261
+ * Mirror of `computeDispatchGap` in `dispatch-reconcile.ts`: a plain set
23262
+ * difference (`known \ assigned`). Pure no side effects, deterministic
23263
+ * ordering (iteration order of `known`).
23137
23264
  */
23138
- function computeDispatchGap(desired, known) {
23139
- return [...desired].filter((id) => !known.has(id));
23265
+ function computeStrandedDevices(known, assigned) {
23266
+ return [...known].filter((id) => !assigned.has(id));
23140
23267
  }
23141
23268
  //#endregion
23142
23269
  //#region src/pipeline-watchdog.ts
@@ -23169,219 +23296,407 @@ var PipelineWatchdog = class {
23169
23296
  });
23170
23297
  this.state.set(cam.deviceId, stages);
23171
23298
  }
23172
- unregister(deviceId) {
23173
- this.cameras.delete(deviceId);
23174
- this.state.delete(deviceId);
23299
+ unregister(deviceId) {
23300
+ this.cameras.delete(deviceId);
23301
+ this.state.delete(deviceId);
23302
+ }
23303
+ /** Called when a stage's live signal arrives (audio chunk / motion frame / inference). */
23304
+ noteSignal(deviceId, stage) {
23305
+ const rt = this.state.get(deviceId)?.get(stage);
23306
+ if (!rt) return;
23307
+ rt.lastSeenMs = this.deps.now();
23308
+ rt.attempts = 0;
23309
+ }
23310
+ tick() {
23311
+ const now = this.deps.now();
23312
+ for (const cam of this.cameras.values()) {
23313
+ const { line, stalled, recoveries } = this.evaluate(cam, now);
23314
+ if (stalled) this.deps.logger.warn(line);
23315
+ else this.deps.logger.info(line);
23316
+ for (const r of recoveries) {
23317
+ const rt = this.state.get(cam.deviceId)?.get(r.stage);
23318
+ if (rt) rt.attempts += 1;
23319
+ this.deps.recover(cam.deviceId, r.stage, r.streamId);
23320
+ }
23321
+ }
23322
+ }
23323
+ start(intervalMs) {
23324
+ if (this.timer) return;
23325
+ this.timer = setInterval(() => this.tick(), intervalMs);
23326
+ }
23327
+ stop() {
23328
+ if (this.timer) clearInterval(this.timer);
23329
+ this.timer = null;
23330
+ }
23331
+ evaluate(cam, now) {
23332
+ const parts = [];
23333
+ const recoveries = [];
23334
+ let stalled = false;
23335
+ const stageOrder = [
23336
+ "audio",
23337
+ "motion",
23338
+ "detection"
23339
+ ];
23340
+ const stageMode = {
23341
+ audio: cam.audioMode,
23342
+ motion: cam.motionSources.includes("analyzer") ? "analyzer" : cam.motionSources.join("+"),
23343
+ detection: cam.detectionMode
23344
+ };
23345
+ for (const stage of stageOrder) {
23346
+ const streamId = cam.continuousStages.get(stage);
23347
+ if (streamId === void 0) {
23348
+ parts.push(`${stage}=${stageMode[stage]}(idle)`);
23349
+ continue;
23350
+ }
23351
+ const rt = this.state.get(cam.deviceId)?.get(stage);
23352
+ if (!rt) {
23353
+ parts.push(`${stage}=${stageMode[stage]}(unknown)`);
23354
+ continue;
23355
+ }
23356
+ const staleness = now - rt.lastSeenMs;
23357
+ const thresholdMs = this.deps.thresholds[STAGE_THRESHOLD_KEY[stage]];
23358
+ const sSec = Math.round(staleness / 1e3);
23359
+ if (staleness <= thresholdMs) parts.push(`${stage}=${stageMode[stage]}@${streamId}(OK ${sSec}s)`);
23360
+ else if (rt.attempts >= this.deps.thresholds.maxRecoveryAttempts) {
23361
+ stalled = true;
23362
+ parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · exhausted ${rt.attempts})`);
23363
+ } else {
23364
+ stalled = true;
23365
+ recoveries.push({
23366
+ stage,
23367
+ streamId
23368
+ });
23369
+ parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · recovering ${rt.attempts + 1}/${this.deps.thresholds.maxRecoveryAttempts})`);
23370
+ }
23371
+ }
23372
+ return {
23373
+ line: `[watchdog] cam${cam.deviceId} ${cam.deviceName} · ${parts.join(" · ")} → ${stalled ? "STALLED" : "OK"}`,
23374
+ stalled,
23375
+ recoveries
23376
+ };
23377
+ }
23378
+ };
23379
+ //#endregion
23380
+ //#region src/remote-health.ts
23381
+ /**
23382
+ * Default thresholds: 3-min grace (cold start), 2-min stale window (metrics
23383
+ * stopped entirely), 0.5 fps floor. A degraded foreign-handle camera runs
23384
+ * ~1 fps (above the bar); with T2's frame-source eligibility a remote
23385
+ * assignment is only legal on a frame-source node, so sustained sub-0.5 fps
23386
+ * means genuinely broken.
23387
+ */
23388
+ var DEFAULT_REMOTE_HEALTH_OPTS = {
23389
+ graceMs: 3 * 6e4,
23390
+ staleMs: 2 * 6e4,
23391
+ minFps: .5
23392
+ };
23393
+ /**
23394
+ * Evaluate every remote pipeline assignment and return the cameras that must be
23395
+ * re-placed. Rules (each assignment evaluated independently):
23396
+ * 1. LOCAL assignments (`agentNodeId === localNodeId`) are skipped — the hub
23397
+ * watchdog owns them.
23398
+ * 2. Cameras not in `activeDeviceIds` are skipped — detection isn't expected.
23399
+ * 3. Assignments younger than `graceMs` are skipped — cold-start grace.
23400
+ * 4. No fps entry, or `now - lastSeen > staleMs` → `stale-metrics`.
23401
+ * 5. Otherwise `fps < minFps` → `zero-fps`.
23402
+ * 6. Otherwise healthy → no action.
23403
+ *
23404
+ * Deterministic ordering: iteration order of `assignments`.
23405
+ */
23406
+ function evaluateRemoteHealth(input) {
23407
+ const { assignments, fpsMap, activeDeviceIds, localNodeId, now, opts } = input;
23408
+ const actions = [];
23409
+ for (const [deviceId, assignment] of assignments) {
23410
+ if (assignment.agentNodeId === localNodeId) continue;
23411
+ if (!activeDeviceIds.has(deviceId)) continue;
23412
+ if (now - assignment.assignedAt < opts.graceMs) continue;
23413
+ const entry = fpsMap.get(deviceId);
23414
+ if (!entry || now - entry.lastSeen > opts.staleMs) {
23415
+ actions.push({
23416
+ deviceId,
23417
+ kind: "replace",
23418
+ why: "stale-metrics"
23419
+ });
23420
+ continue;
23421
+ }
23422
+ if (entry.fps < opts.minFps) {
23423
+ actions.push({
23424
+ deviceId,
23425
+ kind: "replace",
23426
+ why: "zero-fps"
23427
+ });
23428
+ continue;
23429
+ }
23430
+ }
23431
+ return actions;
23432
+ }
23433
+ //#endregion
23434
+ //#region src/zone-rules-provider.ts
23435
+ /**
23436
+ * `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
23437
+ * orchestrator.
23438
+ *
23439
+ * Per-stage rule arrays (motion / detection) live next to zones in the
23440
+ * orchestrator's per-device store under `zoneRules.<stage>` keys.
23441
+ * Every mutation mirrors to a stage-specific device-state slice
23442
+ * (`motion-zone-rules`, `detection-zone-rules`) so consumer addons
23443
+ * subscribe independently and pick up the new gating without an extra
23444
+ * cap round-trip.
23445
+ *
23446
+ * The provider validates each rule against {@link ZoneRuleSchema}
23447
+ * before persisting — partial / corrupt writes are rejected outright
23448
+ * since rules drive runtime filtering and a bad payload would silently
23449
+ * widen the operator's intended scope.
23450
+ */
23451
+ /** Settings store key for stage rules. Kept under a single nested
23452
+ * object so a future stage just adds another property without
23453
+ * reshuffling the schema. */
23454
+ var RULES_STORE_KEY = "zoneRules";
23455
+ /** Cap name for the unified runtime-state slice. Matches the cap's
23456
+ * declared `name` so the codegen DeviceProxy auto-wires
23457
+ * `device.state.zoneRules`. The slice value is the full
23458
+ * `{motion, detection}` object — both stages travel together so a
23459
+ * single reactive handle covers every consumer. */
23460
+ var ZONE_RULES_CAP_NAME = "zone-rules";
23461
+ var RulesArraySchema = array(ZoneRuleSchema);
23462
+ /**
23463
+ * Whole-blob schema for the per-device `zoneRules` store key. Both stages
23464
+ * travel together under one key. Deliberately lenient — each stage is
23465
+ * `unknown` so a corrupt single stage can NOT drop the sibling on load
23466
+ * (durable-state `get()` falls back to `{}` only on a whole-blob parse
23467
+ * failure). Strict per-stage validation, with its own reset-on-corrupt
23468
+ * warn, still happens in `loadRules` exactly as before the migration.
23469
+ */
23470
+ var ZoneRulesBlockSchema = object({
23471
+ motion: unknown().optional(),
23472
+ detection: unknown().optional()
23473
+ }).passthrough();
23474
+ var ZoneRulesProvider = class {
23475
+ ctx;
23476
+ /** Per-device per-stage cache. Hydrated lazily on first read. */
23477
+ cache = /* @__PURE__ */ new Map();
23478
+ /**
23479
+ * Per-device durable handle over the `zoneRules` store key. The WHOLE
23480
+ * `{motion?, detection?}` block round-trips on every read/write so a
23481
+ * write on one stage can never drop the other. Built lazily + memoised.
23482
+ */
23483
+ stateByDevice = /* @__PURE__ */ new Map();
23484
+ constructor(ctx) {
23485
+ this.ctx = ctx;
23486
+ }
23487
+ /** Lazily build (and memoise) the durable `zoneRules` handle for a device. */
23488
+ rulesState(deviceId) {
23489
+ let handle = this.stateByDevice.get(deviceId);
23490
+ if (!handle) {
23491
+ handle = createDurableState({
23492
+ key: RULES_STORE_KEY,
23493
+ schema: ZoneRulesBlockSchema,
23494
+ fallback: {},
23495
+ read: () => this.ctx.settings.readDeviceStore(deviceId),
23496
+ write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
23497
+ onParseError: (key, error) => this.ctx.logger.warn("zone-rules store block failed to parse — resetting", {
23498
+ tags: { deviceId },
23499
+ meta: {
23500
+ key,
23501
+ error: error instanceof Error ? error.message : String(error)
23502
+ }
23503
+ })
23504
+ });
23505
+ this.stateByDevice.set(deviceId, handle);
23506
+ }
23507
+ return handle;
23508
+ }
23509
+ async listRules({ deviceId, stage }) {
23510
+ return this.loadRules(deviceId, stage);
23511
+ }
23512
+ async setRules({ deviceId, stage, rules }) {
23513
+ const parsed = RulesArraySchema.safeParse(rules);
23514
+ if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
23515
+ await this.persist(deviceId, stage, parsed.data);
23175
23516
  }
23176
- /** Called when a stage's live signal arrives (audio chunk / motion frame / inference). */
23177
- noteSignal(deviceId, stage) {
23178
- const rt = this.state.get(deviceId)?.get(stage);
23179
- if (!rt) return;
23180
- rt.lastSeenMs = this.deps.now();
23181
- rt.attempts = 0;
23517
+ /** Drop a device's cache entries. Called when the device is removed. */
23518
+ forgetDevice(deviceId) {
23519
+ this.cache.delete(deviceId);
23520
+ this.stateByDevice.delete(deviceId);
23182
23521
  }
23183
- tick() {
23184
- const now = this.deps.now();
23185
- for (const cam of this.cameras.values()) {
23186
- const { line, stalled, recoveries } = this.evaluate(cam, now);
23187
- if (stalled) this.deps.logger.warn(line);
23188
- else this.deps.logger.info(line);
23189
- for (const r of recoveries) {
23190
- const rt = this.state.get(cam.deviceId)?.get(r.stage);
23191
- if (rt) rt.attempts += 1;
23192
- this.deps.recover(cam.deviceId, r.stage, r.streamId);
23522
+ async loadRules(deviceId, stage) {
23523
+ let perDevice = this.cache.get(deviceId);
23524
+ if (!perDevice) {
23525
+ perDevice = /* @__PURE__ */ new Map();
23526
+ this.cache.set(deviceId, perDevice);
23527
+ }
23528
+ const cached = perDevice.get(stage);
23529
+ if (cached) return cached;
23530
+ let rules = [];
23531
+ try {
23532
+ const raw = (await this.rulesState(deviceId).get())[stage];
23533
+ if (raw !== void 0) {
23534
+ const parsed = RulesArraySchema.safeParse(raw);
23535
+ if (parsed.success) rules = parsed.data;
23536
+ else this.ctx.logger.warn("zone-rules store entry failed to parse — resetting", {
23537
+ tags: { deviceId },
23538
+ meta: {
23539
+ stage,
23540
+ issues: parsed.error.issues
23541
+ }
23542
+ });
23193
23543
  }
23544
+ } catch (err) {
23545
+ this.ctx.logger.warn("zone-rules store read failed — using empty list", {
23546
+ tags: { deviceId },
23547
+ meta: {
23548
+ stage,
23549
+ error: err instanceof Error ? err.message : String(err)
23550
+ }
23551
+ });
23194
23552
  }
23553
+ perDevice.set(stage, rules);
23554
+ return rules;
23195
23555
  }
23196
- start(intervalMs) {
23197
- if (this.timer) return;
23198
- this.timer = setInterval(() => this.tick(), intervalMs);
23199
- }
23200
- stop() {
23201
- if (this.timer) clearInterval(this.timer);
23202
- this.timer = null;
23203
- }
23204
- evaluate(cam, now) {
23205
- const parts = [];
23206
- const recoveries = [];
23207
- let stalled = false;
23208
- const stageOrder = [
23209
- "audio",
23210
- "motion",
23211
- "detection"
23212
- ];
23213
- const stageMode = {
23214
- audio: cam.audioMode,
23215
- motion: cam.motionSources.includes("analyzer") ? "analyzer" : cam.motionSources.join("+"),
23216
- detection: cam.detectionMode
23556
+ async persist(deviceId, stage, rules) {
23557
+ let perDevice = this.cache.get(deviceId);
23558
+ if (!perDevice) {
23559
+ perDevice = /* @__PURE__ */ new Map();
23560
+ this.cache.set(deviceId, perDevice);
23561
+ }
23562
+ perDevice.set(stage, rules);
23563
+ await this.rulesState(deviceId).update((prev) => ({
23564
+ ...prev,
23565
+ [stage]: rules
23566
+ }));
23567
+ const otherStage = stage === "motion" ? "detection" : "motion";
23568
+ const otherRules = perDevice.get(otherStage) ?? await this.loadRules(deviceId, otherStage);
23569
+ const sliceValue = stage === "motion" ? {
23570
+ motion: rules,
23571
+ detection: otherRules
23572
+ } : {
23573
+ motion: otherRules,
23574
+ detection: rules
23217
23575
  };
23218
- for (const stage of stageOrder) {
23219
- const streamId = cam.continuousStages.get(stage);
23220
- if (streamId === void 0) {
23221
- parts.push(`${stage}=${stageMode[stage]}(idle)`);
23222
- continue;
23223
- }
23224
- const rt = this.state.get(cam.deviceId)?.get(stage);
23225
- if (!rt) {
23226
- parts.push(`${stage}=${stageMode[stage]}(unknown)`);
23227
- continue;
23228
- }
23229
- const staleness = now - rt.lastSeenMs;
23230
- const thresholdMs = this.deps.thresholds[STAGE_THRESHOLD_KEY[stage]];
23231
- const sSec = Math.round(staleness / 1e3);
23232
- if (staleness <= thresholdMs) parts.push(`${stage}=${stageMode[stage]}@${streamId}(OK ${sSec}s)`);
23233
- else if (rt.attempts >= this.deps.thresholds.maxRecoveryAttempts) {
23234
- stalled = true;
23235
- parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · exhausted ${rt.attempts})`);
23236
- } else {
23237
- stalled = true;
23238
- recoveries.push({
23576
+ try {
23577
+ await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
23578
+ capName: ZONE_RULES_CAP_NAME,
23579
+ slice: sliceValue
23580
+ });
23581
+ } catch (err) {
23582
+ this.ctx.logger.debug("zone-rules slice mirror failed", {
23583
+ tags: { deviceId },
23584
+ meta: {
23239
23585
  stage,
23240
- streamId
23241
- });
23242
- parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · recovering ${rt.attempts + 1}/${this.deps.thresholds.maxRecoveryAttempts})`);
23243
- }
23586
+ error: err instanceof Error ? err.message : String(err)
23587
+ }
23588
+ });
23244
23589
  }
23245
- return {
23246
- line: `[watchdog] cam${cam.deviceId} ${cam.deviceName} · ${parts.join(" · ")} → ${stalled ? "STALLED" : "OK"}`,
23247
- stalled,
23248
- recoveries
23249
- };
23590
+ this.ctx.onRulesChanged?.(deviceId, stage, rules);
23250
23591
  }
23251
23592
  };
23252
23593
  //#endregion
23253
- //#region src/camera-status/compose-camera-status.ts
23254
- function mapAssignment(input) {
23255
- return {
23256
- detectionNodeId: input.detectionNodeId,
23257
- decoderNodeId: input.decoderNodeId,
23258
- audioNodeId: input.audioNodeId,
23259
- pinned: {
23260
- detection: input.pinned.detection,
23261
- decoder: input.pinned.decoder,
23262
- audio: input.pinned.audio
23263
- },
23264
- reasons: {
23265
- detection: input.reasons.detection,
23266
- decoder: input.reasons.decoder,
23267
- audio: input.reasons.audio
23268
- }
23269
- };
23270
- }
23271
- function mapSource(sourceResult) {
23272
- if (sourceResult === null) return { streams: [] };
23273
- return { streams: sourceResult.streams.map((s) => ({
23274
- camStreamId: s.camStreamId,
23275
- codec: s.codec,
23276
- width: s.width,
23277
- height: s.height,
23278
- fps: s.fps,
23279
- kind: s.kind
23280
- })) };
23281
- }
23282
- function mapBroker(brokerResult) {
23283
- if (brokerResult === null) return null;
23284
- return {
23285
- profiles: brokerResult.profiles.map((p) => ({
23286
- profile: p.profile,
23287
- status: p.status,
23288
- codec: p.codec,
23289
- width: p.width,
23290
- height: p.height,
23291
- subscribers: p.subscribers,
23292
- inFps: p.inFps,
23293
- outFps: p.outFps
23294
- })),
23295
- webrtcSessions: brokerResult.webrtcSessions,
23296
- rtspRestream: brokerResult.rtspRestream
23297
- };
23298
- }
23299
- function mapDecoderShm(shm) {
23300
- return {
23301
- framesWritten: shm.framesWritten,
23302
- getFrameHits: shm.getFrameHits,
23303
- getFrameMisses: shm.getFrameMisses,
23304
- budgetMb: shm.budgetMb
23305
- };
23306
- }
23307
- function mapDecoder(decoderResult) {
23308
- if (decoderResult === null) return null;
23309
- return {
23310
- nodeId: decoderResult.nodeId,
23311
- formats: [...decoderResult.formats],
23312
- sessionCount: decoderResult.sessionCount,
23313
- shm: mapDecoderShm(decoderResult.shm)
23314
- };
23315
- }
23316
- function mapMotion(motionResult) {
23317
- if (motionResult === null) return null;
23318
- return {
23319
- enabled: motionResult.enabled,
23320
- fps: motionResult.fps
23321
- };
23322
- }
23323
- function mapProvisioning(p) {
23324
- if (p.error !== void 0) return {
23325
- state: p.state,
23326
- error: p.error
23327
- };
23328
- return { state: p.state };
23329
- }
23330
- function mapDetection(detectionResult) {
23331
- if (detectionResult === null) return null;
23332
- const phase = detectionResult.phase;
23333
- return {
23334
- nodeId: detectionResult.nodeId,
23335
- engine: {
23336
- backend: detectionResult.engine.backend,
23337
- device: detectionResult.engine.device
23338
- },
23339
- phase,
23340
- configuredFps: detectionResult.configuredFps,
23341
- actualFps: detectionResult.actualFps,
23342
- queueDepth: detectionResult.queueDepth,
23343
- avgInferenceMs: detectionResult.avgInferenceMs,
23344
- provisioning: mapProvisioning(detectionResult.provisioning)
23345
- };
23346
- }
23347
- function mapAudio(audioResult) {
23348
- if (audioResult === null) return null;
23349
- return {
23350
- nodeId: audioResult.nodeId,
23351
- enabled: audioResult.enabled
23352
- };
23353
- }
23354
- function mapRecording(recordingResult) {
23355
- if (recordingResult === null) return null;
23356
- return {
23357
- mode: recordingResult.mode,
23358
- active: recordingResult.active,
23359
- storageBytes: recordingResult.storageBytes
23360
- };
23361
- }
23594
+ //#region src/zones-provider.ts
23362
23595
  /**
23363
- * Pure function that composes a `CameraStatus` from per-stage fetch results.
23596
+ * `zones-provider.ts` implements `zonesCapability` for the orchestrator.
23597
+ *
23598
+ * Per-camera CRUD over polygon detection zones. Persists to the
23599
+ * orchestrator's per-device settings store under the `zones` key and
23600
+ * mirrors every change into the device-state `zones` slice via
23601
+ * `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
23602
+ * pipeline-executor, analytics, admin UI) read the live state with
23603
+ * the canonical `dev.state.zones.onChanged` channel.
23364
23604
  *
23365
- * - `assignment` is always built from orchestrator-local data (never null).
23366
- * - `source` always present: defaults to `{ streams: [] }` when sourceResult is null.
23367
- * - Every other block is null when its stage result is null (graceful degradation).
23368
- * - `fetchedAt` is stamped exactly as provided — never calls `Date.now()`.
23369
- * - No mutation of the input.
23605
+ * Onboard / firmware-reported zones are out of scope for now every
23606
+ * zone is operator-drawn. The provider keeps the surface symmetric:
23607
+ * `addZone` rejects id collisions, `updateZone` requires an existing
23608
+ * id, `removeZone` is idempotent.
23370
23609
  */
23371
- function composeCameraStatus(input) {
23372
- return {
23373
- deviceId: input.deviceId,
23374
- assignment: mapAssignment(input),
23375
- source: mapSource(input.sourceResult),
23376
- broker: mapBroker(input.brokerResult),
23377
- decoder: mapDecoder(input.decoderResult),
23378
- motion: mapMotion(input.motionResult),
23379
- detection: mapDetection(input.detectionResult),
23380
- audio: mapAudio(input.audioResult),
23381
- recording: mapRecording(input.recordingResult),
23382
- fetchedAt: input.fetchedAt
23383
- };
23384
- }
23610
+ var ZONES_STORE_KEY = "zones";
23611
+ var ZONES_CAP_NAME = "zones";
23612
+ var ZonesArraySchema = array(ZoneSchema);
23613
+ var ZonesProvider = class {
23614
+ ctx;
23615
+ /** Per-device cache. Hydrated lazily on first read for a device. */
23616
+ cache = /* @__PURE__ */ new Map();
23617
+ /**
23618
+ * Per-device durable handle over the `zones` store key. The WHOLE
23619
+ * validated zone array round-trips on every read/write so no field can
23620
+ * be dropped on persist. Built lazily + memoised per device.
23621
+ */
23622
+ stateByDevice = /* @__PURE__ */ new Map();
23623
+ constructor(ctx) {
23624
+ this.ctx = ctx;
23625
+ }
23626
+ /** Lazily build (and memoise) the durable `zones` handle for a device. */
23627
+ zonesState(deviceId) {
23628
+ let handle = this.stateByDevice.get(deviceId);
23629
+ if (!handle) {
23630
+ handle = createDurableState({
23631
+ key: ZONES_STORE_KEY,
23632
+ schema: ZonesArraySchema,
23633
+ fallback: [],
23634
+ read: () => this.ctx.settings.readDeviceStore(deviceId),
23635
+ write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
23636
+ onParseError: (key, error) => this.ctx.logger.warn("zones store entry failed to parse — resetting", {
23637
+ tags: { deviceId },
23638
+ meta: {
23639
+ key,
23640
+ error: error instanceof Error ? error.message : String(error)
23641
+ }
23642
+ })
23643
+ });
23644
+ this.stateByDevice.set(deviceId, handle);
23645
+ }
23646
+ return handle;
23647
+ }
23648
+ async listZones({ deviceId }) {
23649
+ return this.loadZones(deviceId);
23650
+ }
23651
+ async addZone({ deviceId, zone }) {
23652
+ const existing = await this.loadZones(deviceId);
23653
+ if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
23654
+ await this.persist(deviceId, [...existing, zone]);
23655
+ }
23656
+ async updateZone({ deviceId, zone }) {
23657
+ const existing = await this.loadZones(deviceId);
23658
+ if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
23659
+ const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
23660
+ await this.persist(deviceId, next);
23661
+ }
23662
+ async removeZone({ deviceId, zoneId }) {
23663
+ const existing = await this.loadZones(deviceId);
23664
+ if (!existing.some((entry) => entry.id === zoneId)) return;
23665
+ await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
23666
+ }
23667
+ /**
23668
+ * Drop a device's cache entry. Called when the device is removed so
23669
+ * the next attach starts from a fresh disk read.
23670
+ */
23671
+ forgetDevice(deviceId) {
23672
+ this.cache.delete(deviceId);
23673
+ this.stateByDevice.delete(deviceId);
23674
+ }
23675
+ async loadZones(deviceId) {
23676
+ const cached = this.cache.get(deviceId);
23677
+ if (cached) return cached;
23678
+ let zones = [];
23679
+ try {
23680
+ zones = await this.zonesState(deviceId).get();
23681
+ } catch (err) {
23682
+ this.ctx.logger.warn("zones store read failed — using empty list", {
23683
+ tags: { deviceId },
23684
+ meta: { error: err instanceof Error ? err.message : String(err) }
23685
+ });
23686
+ }
23687
+ this.cache.set(deviceId, zones);
23688
+ return zones;
23689
+ }
23690
+ async persist(deviceId, zones) {
23691
+ this.cache.set(deviceId, zones);
23692
+ await this.zonesState(deviceId).set(zones);
23693
+ await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
23694
+ capName: ZONES_CAP_NAME,
23695
+ slice: { zones }
23696
+ });
23697
+ this.ctx.onZonesChanged?.(deviceId, zones);
23698
+ }
23699
+ };
23385
23700
  //#endregion
23386
23701
  //#region src/index.ts
23387
23702
  var PHASE_MODE_VALUES = new Set([
@@ -23400,6 +23715,24 @@ var DEFAULT_BROKER_CALL_TIMEOUT_MS = 5e3;
23400
23715
  /** Debounce window for `scheduleReconcile` — coalesces bursts of topology/slot-change signals. */
23401
23716
  var RECONCILE_DEBOUNCE_MS = 200;
23402
23717
  /**
23718
+ * Periodic sweep interval for `retryPendingDispatches` — re-dispatches cameras
23719
+ * that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
23720
+ * `reconcileDispatch` is additive-only and never revisits these, so a slow
23721
+ * safety-net timer + event-driven debounce triggers recover them.
23722
+ */
23723
+ var PENDING_RETRY_INTERVAL_MS = 6e4;
23724
+ /** Debounce window for `schedulePendingRetry` — coalesces capacity/eligibility/readiness signals. */
23725
+ var PENDING_RETRY_DEBOUNCE_MS = 2e3;
23726
+ /**
23727
+ * Remote-assignment health loop (T7) bounded backoff. A remote camera flagged
23728
+ * as unhealthy (0-fps / metrics-stale) is re-placed at most
23729
+ * `REMOTE_HEALTH_MAX_ATTEMPTS` times inside a rolling `REMOTE_HEALTH_WINDOW_MS`
23730
+ * window; once exhausted the camera is left visibly pending (`'unhealthy'`) for
23731
+ * the operator instead of churning forever.
23732
+ */
23733
+ var REMOTE_HEALTH_MAX_ATTEMPTS = 3;
23734
+ var REMOTE_HEALTH_WINDOW_MS = 60 * 6e4;
23735
+ /**
23403
23736
  * Device-details keys routed through the orchestrator's pipeline
23404
23737
  * settings writer instead of the device orchestration store. The
23405
23738
  * `cameraPipeline` key carries the full `CameraPipelineConfig`
@@ -23488,6 +23821,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
23488
23821
  zoneRulesProvider = null;
23489
23822
  /** In-memory assignment map — mirrored into events + per-device settings for the pin. */
23490
23823
  assignments = /* @__PURE__ */ new Map();
23824
+ /**
23825
+ * Why a camera is currently unassigned (pending), keyed by deviceId. Set on
23826
+ * every pending placement path (over-cap, no-frame-source, load-shed) and
23827
+ * cleared when the camera is (re)assigned or released. Consumed by T3's
23828
+ * pending-retry sweep + `getCameraStatus` surface — nothing reads it yet in
23829
+ * this commit; the map is populated so T3 can wire the read side without
23830
+ * re-touching every placement path.
23831
+ *
23832
+ * `protected` (not `private`) to match the existing test-seam convention in
23833
+ * this class (`audioSubscriptions`, `audioSubLocks`) — the frame-source
23834
+ * eligibility spec subclasses the addon to assert the recorded reason.
23835
+ */
23836
+ pendingReasons = /* @__PURE__ */ new Map();
23837
+ /**
23838
+ * Remote-assignment health loop (T7) per-device backoff ledger. Each entry
23839
+ * tracks how many times a remote camera has been re-placed inside the current
23840
+ * rolling window (`REMOTE_HEALTH_WINDOW_MS`). Bounds churn: once the count
23841
+ * hits `REMOTE_HEALTH_MAX_ATTEMPTS`, the camera is left pending (`'unhealthy'`)
23842
+ * for the operator instead of being re-placed again. `protected` so the
23843
+ * remote-health spec can assert the ledger without casts.
23844
+ */
23845
+ remoteHealthAttempts = /* @__PURE__ */ new Map();
23491
23846
  /** Per-device audio node assignment — nodeId of the audio-analyzer handling this device's chunks. */
23492
23847
  audioNodeByDevice = /* @__PURE__ */ new Map();
23493
23848
  /** Assignments with metadata — replaces plain audioNodeByDevice values as the source of truth. */
@@ -23607,6 +23962,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
23607
23962
  reconcileInFlight = false;
23608
23963
  /** Set to true when a reconcile is requested while one is already in-flight; triggers a follow-up pass. */
23609
23964
  reconcileRerunRequested = false;
23965
+ /** Periodic `retryPendingDispatches` safety-net timer (T3). */
23966
+ pendingRetryTimer = null;
23967
+ /** Pending `schedulePendingRetry` debounce timer (T3). */
23968
+ pendingRetryDebounceTimer = null;
23969
+ /** True while `retryPendingDispatches` is running. */
23970
+ pendingRetryInFlight = false;
23971
+ /** Set when a pending-retry is requested while one is already in-flight; triggers a follow-up pass. */
23972
+ pendingRetryRerunRequested = false;
23610
23973
  initTimestamp = 0;
23611
23974
  constructor() {
23612
23975
  super({});
@@ -23823,6 +24186,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
23823
24186
  thresholds: DEFAULT_WATCHDOG_THRESHOLDS
23824
24187
  });
23825
24188
  this.pipelineWatchdog.start(PipelineOrchestratorAddon.WATCHDOG_INTERVAL_MS);
24189
+ this.pendingRetryTimer = setInterval(() => void this.retryPendingDispatches(), PENDING_RETRY_INTERVAL_MS);
23826
24190
  this.ctx.logger.info("Pipeline orchestrator detection-wiring subscriptions installed");
23827
24191
  this.migrateLegacyFlagsToBindings().catch((err) => {
23828
24192
  this.ctx.logger.warn("bindings migration failed — will retry on next boot", { meta: { error: errMsg(err) } });
@@ -24052,6 +24416,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24052
24416
  clearTimeout(this.reconcileTimer);
24053
24417
  this.reconcileTimer = null;
24054
24418
  }
24419
+ if (this.pendingRetryTimer !== null) {
24420
+ clearInterval(this.pendingRetryTimer);
24421
+ this.pendingRetryTimer = null;
24422
+ }
24423
+ if (this.pendingRetryDebounceTimer !== null) {
24424
+ clearTimeout(this.pendingRetryDebounceTimer);
24425
+ this.pendingRetryDebounceTimer = null;
24426
+ }
24055
24427
  this.unsubDeviceRegistered?.();
24056
24428
  this.unsubDeviceRegistered = null;
24057
24429
  this.unsubDeviceUnregistered?.();
@@ -24077,6 +24449,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24077
24449
  for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
24078
24450
  this.lazyAudioTeardownTimers.clear();
24079
24451
  this.cameraFpsMap.clear();
24452
+ this.remoteHealthAttempts.clear();
24080
24453
  this.loadShedState.clear();
24081
24454
  if (this.loadShedResumeTimer) {
24082
24455
  clearInterval(this.loadShedResumeTimer);
@@ -24138,6 +24511,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24138
24511
  windowMs
24139
24512
  }
24140
24513
  });
24514
+ this.pendingReasons.set(runnerConfig.deviceId, "load-shed");
24141
24515
  return {
24142
24516
  success: true,
24143
24517
  kind: "pending"
@@ -24151,11 +24525,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24151
24525
  const decision = balance({
24152
24526
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
24153
24527
  preferredAgent,
24154
- nodeCaps: this.buildNodeCaps()
24528
+ nodeCaps: await this.buildNodeCaps(),
24529
+ eligibleNodes: this.detectionEligibleNodes()
24155
24530
  });
24156
24531
  if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
24157
24532
  if (decision.kind === "pending") {
24158
- this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId: runnerConfig.deviceId } });
24533
+ this.pendingReasons.set(runnerConfig.deviceId, decision.reason);
24534
+ this.ctx.logger.warn("camera left pending — no eligible node", {
24535
+ tags: { deviceId: runnerConfig.deviceId },
24536
+ meta: { reason: decision.reason }
24537
+ });
24159
24538
  return {
24160
24539
  success: true,
24161
24540
  kind: "pending"
@@ -24206,11 +24585,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24206
24585
  }
24207
24586
  this.assignments.delete(input.deviceId);
24208
24587
  this.cameraConfigs.delete(input.deviceId);
24588
+ this.pendingReasons.delete(input.deviceId);
24209
24589
  this.pipelineWatchdog?.unregister(input.deviceId);
24210
24590
  return { success: true };
24211
24591
  }
24212
24592
  async assignPipeline(input) {
24213
24593
  if (!this.ctx) throw new Error("PipelineOrchestrator: assignPipeline called before initialize");
24594
+ const eligible = this.detectionEligibleNodes();
24595
+ if (!eligible.includes(input.agentNodeId)) throw new Error(`Cannot pin camera ${input.deviceId} detection to '${input.agentNodeId}': the node cannot obtain this camera's decoded frames (frame-source nodes: ${eligible.join(", ") || "none"}). Add it to Enabled Decoder Nodes first.`);
24214
24596
  await this.ctx.settings?.writeDeviceStore(input.deviceId, { [PREFERRED_AGENT_SETTING]: input.agentNodeId }).catch((err) => {
24215
24597
  const msg = errMsg(err);
24216
24598
  this.ctx.logger.warn("assignPipeline: failed to persist pin", {
@@ -24256,11 +24638,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24256
24638
  const decision = balance({
24257
24639
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
24258
24640
  preferredAgent: null,
24259
- nodeCaps: this.buildNodeCaps()
24641
+ nodeCaps: await this.buildNodeCaps(),
24642
+ eligibleNodes: this.detectionEligibleNodes()
24260
24643
  });
24261
24644
  if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
24262
- else if (decision.kind === "pending") this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId: input.deviceId } });
24263
- else {
24645
+ else if (decision.kind === "pending") {
24646
+ this.pendingReasons.set(input.deviceId, decision.reason);
24647
+ this.ctx.logger.warn("camera left pending — no eligible node", {
24648
+ tags: { deviceId: input.deviceId },
24649
+ meta: { reason: decision.reason }
24650
+ });
24651
+ } else {
24264
24652
  const targetNodeId = decision.agentNodeId;
24265
24653
  if (current && current.agentNodeId !== targetNodeId) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
24266
24654
  const msg = errMsg(err);
@@ -24298,7 +24686,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24298
24686
  async rebalance() {
24299
24687
  if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
24300
24688
  const loads = await this.collectAgentLoad({ onlyEnabled: true });
24301
- const nodeCaps = this.buildNodeCaps();
24689
+ const nodeCaps = await this.buildNodeCaps();
24302
24690
  let migrated = 0;
24303
24691
  for (const [deviceId, config] of this.cameraConfigs) {
24304
24692
  const current = this.assignments.get(deviceId);
@@ -24306,11 +24694,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24306
24694
  const decision = balance({
24307
24695
  nodes: loads,
24308
24696
  preferredAgent: await this.readPreferredAgent(deviceId),
24309
- nodeCaps
24697
+ nodeCaps,
24698
+ eligibleNodes: this.detectionEligibleNodes()
24310
24699
  });
24311
24700
  if (!decision) continue;
24312
24701
  if (decision.kind === "pending") {
24313
- this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId } });
24702
+ this.pendingReasons.set(deviceId, decision.reason);
24703
+ this.ctx.logger.warn("camera left pending — no eligible node", {
24704
+ tags: { deviceId },
24705
+ meta: { reason: decision.reason }
24706
+ });
24314
24707
  continue;
24315
24708
  }
24316
24709
  if (current && current.agentNodeId === decision.agentNodeId) continue;
@@ -24517,6 +24910,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24517
24910
  return this.enabledNodes.includes(nodeId);
24518
24911
  }
24519
24912
  /**
24913
+ * The set of nodes that can OBTAIN a camera's decoded frames.
24914
+ *
24915
+ * **Phase 1 (today):** detection HARD-requires that the runner is itself a
24916
+ * decoder node — a node can only run detection where it can read the shm
24917
+ * frame ring locally. So the frame-source set is exactly
24918
+ * `enabledDecoderNodes`.
24919
+ *
24920
+ * **Phase 2 (T10):** this becomes per-camera source/decoder ownership — a
24921
+ * detection node will be able to source a camera's frames from a co-located
24922
+ * OR a remote restream/decoder leg. Replace this body then; every predicate
24923
+ * change edits this ONE method (and `detectionEligibleNodes`).
24924
+ */
24925
+ frameSourceNodes() {
24926
+ return this.enabledDecoderNodes;
24927
+ }
24928
+ /**
24929
+ * Nodes eligible to run a camera's detection pipeline: the whitelist of
24930
+ * detection-enabled nodes (`enabledNodes`) intersected with the nodes that
24931
+ * can obtain the camera's frames (`frameSourceNodes()`). A node that is
24932
+ * detection-enabled but cannot source frames (or vice-versa) is NEVER a
24933
+ * valid placement — the balancer receives this as `eligibleNodes` and treats
24934
+ * anything outside it as unassignable (`no-frame-source`).
24935
+ *
24936
+ * Single choke point for T2/T10: passed to every `balance()` call site and
24937
+ * enforced by `assignPipeline` before persisting a pin.
24938
+ */
24939
+ detectionEligibleNodes() {
24940
+ const frameSource = this.frameSourceNodes();
24941
+ return this.enabledNodes.filter((nodeId) => frameSource.includes(nodeId));
24942
+ }
24943
+ /**
24520
24944
  * Query every online runner in the cluster for its current load, plus the
24521
24945
  * local runner if co-located. Refreshes the `cachedAgentLoad` snapshot so
24522
24946
  * `getAgentLoad()` / `getGlobalMetrics()` can return fresh data.
@@ -24617,8 +25041,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24617
25041
  * Used by every `balance()` call so the balancer can honour operator-set
24618
25042
  * per-node maximums without polling the store on each decision.
24619
25043
  */
24620
- buildNodeCaps() {
24621
- const blob = this.agentSettingsState.get();
25044
+ async buildNodeCaps() {
25045
+ const blob = await this.agentSettingsState.get();
24622
25046
  const caps = {};
24623
25047
  for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
24624
25048
  return caps;
@@ -24682,6 +25106,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24682
25106
  assignedAt: Date.now()
24683
25107
  };
24684
25108
  this.assignments.set(deviceId, assignment);
25109
+ this.pendingReasons.delete(deviceId);
24685
25110
  if (!this.ctx?.eventBus) return;
24686
25111
  const payload = {
24687
25112
  deviceId,
@@ -24766,7 +25191,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24766
25191
  const decision = balance({
24767
25192
  nodes: loads,
24768
25193
  preferredAgent: null,
24769
- nodeCaps: this.buildNodeCaps()
25194
+ nodeCaps: await this.buildNodeCaps(),
25195
+ eligibleNodes: this.detectionEligibleNodes()
24770
25196
  });
24771
25197
  if (!decision) {
24772
25198
  this.ctx.logger.error("Failover: no online runner", { tags: { deviceId } });
@@ -24774,7 +25200,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24774
25200
  continue;
24775
25201
  }
24776
25202
  if (decision.kind === "pending") {
24777
- this.ctx.logger.warn("camera left pending — all nodes at maxCameras", { tags: { deviceId } });
25203
+ this.pendingReasons.set(deviceId, decision.reason);
25204
+ this.ctx.logger.warn("Failover: camera left pending — no eligible node", {
25205
+ tags: { deviceId },
25206
+ meta: { reason: decision.reason }
25207
+ });
24778
25208
  this.assignments.delete(deviceId);
24779
25209
  continue;
24780
25210
  }
@@ -24813,6 +25243,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24813
25243
  tags: { nodeId },
24814
25244
  meta: { policy: this.failoverPolicy.onReconnect }
24815
25245
  });
25246
+ this.schedulePendingRetry();
24816
25247
  if (this.failoverPolicy.onReconnect === "rebalance") {
24817
25248
  await this.rebalance().catch((err) => {
24818
25249
  const msg = errMsg(err);
@@ -24820,6 +25251,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24820
25251
  });
24821
25252
  return;
24822
25253
  }
25254
+ if (!this.detectionEligibleNodes().includes(nodeId)) {
25255
+ this.ctx.logger.warn("restore skipped — node not frame-source eligible", {
25256
+ tags: { nodeId },
25257
+ meta: { eligibleNodes: this.detectionEligibleNodes().join(",") }
25258
+ });
25259
+ for (const [deviceId, assignment] of this.assignments) {
25260
+ if (assignment.agentNodeId !== nodeId) continue;
25261
+ this.pendingReasons.set(deviceId, "no-frame-source");
25262
+ }
25263
+ return;
25264
+ }
24823
25265
  let restored = 0;
24824
25266
  for (const [deviceId, config] of this.cameraConfigs) {
24825
25267
  const current = this.assignments.get(deviceId);
@@ -24907,34 +25349,43 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
24907
25349
  async reattachAudioForNode(nodeId) {
24908
25350
  for (const [deviceId, audioNode] of this.audioNodeByDevice) {
24909
25351
  if (audioNode !== nodeId) continue;
24910
- const config = this.cameraConfigs.get(deviceId);
24911
- if (!config) continue;
24912
- const audioCfg = {
24913
- ...config,
24914
- enabled: config.pipelineEnabled
24915
- };
24916
- try {
24917
- await this.withAudioSubLock(deviceId, async () => {
24918
- const prior = this.audioSubscriptions.get(deviceId);
24919
- if (prior) {
24920
- try {
24921
- prior();
24922
- } catch {}
24923
- this.audioSubscriptions.delete(deviceId);
24924
- }
24925
- const unsub = await this.subscribeAudioStream(deviceId, audioCfg);
24926
- if (unsub) if (this.activeDetections.has(deviceId)) this.storeAudioSub(deviceId, unsub);
24927
- else unsub();
24928
- });
24929
- } catch (err) {
24930
- this.ctx.logger.error("audio re-attach on analyzer readiness failed", {
24931
- tags: {
24932
- deviceId,
24933
- nodeId
24934
- },
24935
- meta: { error: errMsg(err) }
24936
- });
24937
- }
25352
+ await this.reattachAudioForDevice(deviceId);
25353
+ }
25354
+ }
25355
+ /**
25356
+ * Re-establish the audio subscription for ONE camera: tear down any prior
25357
+ * sub and re-subscribe under the current pin/balance, keeping the new handle
25358
+ * only while detection is active. The whole get→teardown→subscribe→store
25359
+ * sequence runs inside `withAudioSubLock` so a prior handle is never orphaned
25360
+ * (the same invariant `reattachAudioForNode` relied on before it was
25361
+ * extracted here). No-op when the device has no runner config or audio is
25362
+ * disabled/lazy (`subscribeAudioStream` self-gates).
25363
+ */
25364
+ async reattachAudioForDevice(deviceId) {
25365
+ const config = this.cameraConfigs.get(deviceId);
25366
+ if (!config) return;
25367
+ const audioCfg = {
25368
+ ...config,
25369
+ enabled: config.pipelineEnabled
25370
+ };
25371
+ try {
25372
+ await this.withAudioSubLock(deviceId, async () => {
25373
+ const prior = this.audioSubscriptions.get(deviceId);
25374
+ if (prior) {
25375
+ try {
25376
+ prior();
25377
+ } catch {}
25378
+ this.audioSubscriptions.delete(deviceId);
25379
+ }
25380
+ const unsub = await this.subscribeAudioStream(deviceId, audioCfg);
25381
+ if (unsub) if (this.activeDetections.has(deviceId)) this.storeAudioSub(deviceId, unsub);
25382
+ else unsub();
25383
+ });
25384
+ } catch (err) {
25385
+ this.ctx.logger.error("audio re-attach failed", {
25386
+ tags: { deviceId },
25387
+ meta: { error: errMsg(err) }
25388
+ });
24938
25389
  }
24939
25390
  }
24940
25391
  /**
@@ -25009,6 +25460,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25009
25460
  });
25010
25461
  }, 3e3);
25011
25462
  this.scheduleReconcile();
25463
+ this.schedulePendingRetry();
25012
25464
  } catch (err) {
25013
25465
  this.ctx.logger.debug("readiness seed+redispatch failed", {
25014
25466
  tags: { nodeId },
@@ -25123,6 +25575,175 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25123
25575
  }
25124
25576
  }
25125
25577
  }
25578
+ /**
25579
+ * Coalesce bursts of capacity/eligibility/readiness signals into a single
25580
+ * `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
25581
+ * dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
25582
+ * hammer `dispatchCamera`.
25583
+ */
25584
+ schedulePendingRetry() {
25585
+ if (this.pendingRetryDebounceTimer !== null) clearTimeout(this.pendingRetryDebounceTimer);
25586
+ this.pendingRetryDebounceTimer = setTimeout(() => {
25587
+ this.pendingRetryDebounceTimer = null;
25588
+ this.retryPendingDispatches();
25589
+ }, PENDING_RETRY_DEBOUNCE_MS);
25590
+ }
25591
+ /**
25592
+ * Re-dispatch every KNOWN-but-UNASSIGNED camera. `reconcileDispatch` is
25593
+ * additive-only over `cameraConfigs` and never revisits a camera that is
25594
+ * tracked but has no live assignment (left pending by over-cap /
25595
+ * no-frame-source / load-shed), so those cameras would otherwise stay
25596
+ * stranded until an unrelated live event happens to touch them. This sweep
25597
+ * closes that gap.
25598
+ *
25599
+ * `dispatchCamera` re-reads pins, re-balances with the frame-source
25600
+ * predicate, and re-returns pending harmlessly when nothing changed —
25601
+ * so a no-op sweep is cheap and side-effect-free. Serialized with an
25602
+ * in-flight flag + trailing-rerun bit (mirrors `reconcileDispatch`).
25603
+ */
25604
+ async retryPendingDispatches() {
25605
+ if (this.pendingRetryInFlight) {
25606
+ this.pendingRetryRerunRequested = true;
25607
+ return;
25608
+ }
25609
+ if (this.reconcileInFlight) return;
25610
+ if (!this.api) return;
25611
+ this.pendingRetryInFlight = true;
25612
+ try {
25613
+ const stranded = computeStrandedDevices(new Set(this.cameraConfigs.keys()), new Set(this.assignments.keys()));
25614
+ if (stranded.length > 0) {
25615
+ const loadShedActive = this.isAnyNodeLoadShed();
25616
+ let retried = 0;
25617
+ let stillPending = 0;
25618
+ for (const deviceId of stranded) {
25619
+ if (this.pendingReasons.get(deviceId) === "load-shed" && loadShedActive) {
25620
+ stillPending++;
25621
+ continue;
25622
+ }
25623
+ const cfg = this.cameraConfigs.get(deviceId);
25624
+ if (!cfg) continue;
25625
+ try {
25626
+ const result = await this.dispatchCamera(cfg);
25627
+ retried++;
25628
+ if (result.kind === "pending") stillPending++;
25629
+ } catch (err) {
25630
+ this.ctx.logger.warn("pending retry: dispatchCamera failed", {
25631
+ tags: { deviceId },
25632
+ meta: { error: errMsg(err) }
25633
+ });
25634
+ stillPending++;
25635
+ }
25636
+ }
25637
+ this.ctx.logger.info("pending retry sweep", { meta: {
25638
+ stranded: stranded.length,
25639
+ retried,
25640
+ stillPending
25641
+ } });
25642
+ }
25643
+ await this.evaluateRemoteAssignmentHealth();
25644
+ } finally {
25645
+ this.pendingRetryInFlight = false;
25646
+ if (this.pendingRetryRerunRequested) {
25647
+ this.pendingRetryRerunRequested = false;
25648
+ this.schedulePendingRetry();
25649
+ }
25650
+ }
25651
+ }
25652
+ /** True when any node is currently load-shed paused (used to skip churny retries). */
25653
+ isAnyNodeLoadShed() {
25654
+ for (const state of this.loadShedState.values()) if (state.pausedAt !== null) return true;
25655
+ return false;
25656
+ }
25657
+ /**
25658
+ * Evaluate the health of every REMOTE pipeline assignment and act on the
25659
+ * cameras the pure `evaluateRemoteHealth` flags. The hub watchdog covers
25660
+ * local cameras; this is its remote-node equivalent (gap i). Called from the
25661
+ * T3 sweep tick after retrying pending dispatches.
25662
+ *
25663
+ * `protected` so the remote-health orchestrator spec can drive one evaluation
25664
+ * pass deterministically (mirrors the `pendingReasons`/`audioSubLocks` test-seam
25665
+ * convention in this class) without waiting on the periodic timer.
25666
+ */
25667
+ async evaluateRemoteAssignmentHealth() {
25668
+ if (!this.api) return;
25669
+ const actions = evaluateRemoteHealth({
25670
+ assignments: this.assignments,
25671
+ fpsMap: this.cameraFpsMap,
25672
+ activeDeviceIds: new Set(this.activeDetections.keys()),
25673
+ localNodeId: this.localNodeId,
25674
+ now: Date.now(),
25675
+ opts: DEFAULT_REMOTE_HEALTH_OPTS
25676
+ });
25677
+ for (const action of actions) await this.handleRemoteHealthReplace(action);
25678
+ }
25679
+ /**
25680
+ * Re-place a single unhealthy remote camera with bounded per-device backoff.
25681
+ * Under budget: detach from the unhealthy node + re-dispatch (inherits the
25682
+ * T2 frame-source predicate). Over budget: log an error, drop the dead
25683
+ * assignment, and mark the camera `'unhealthy'`-pending for the operator so
25684
+ * `getCameraStatus` surfaces it (R2 "bounded-retry ... state visible").
25685
+ */
25686
+ async handleRemoteHealthReplace(action) {
25687
+ const { deviceId, why } = action;
25688
+ const assignment = this.assignments.get(deviceId);
25689
+ if (!assignment) return;
25690
+ const cfg = this.cameraConfigs.get(deviceId);
25691
+ if (!cfg) return;
25692
+ const now = Date.now();
25693
+ const prior = this.remoteHealthAttempts.get(deviceId);
25694
+ const windowFresh = prior !== void 0 && now - prior.windowStart < REMOTE_HEALTH_WINDOW_MS;
25695
+ const count = windowFresh ? prior.count : 0;
25696
+ if (count >= REMOTE_HEALTH_MAX_ATTEMPTS) {
25697
+ this.ctx.logger.error("remote assignment unhealthy — retries exhausted, leaving pending", {
25698
+ tags: {
25699
+ deviceId,
25700
+ nodeId: assignment.agentNodeId
25701
+ },
25702
+ meta: {
25703
+ why,
25704
+ attempts: count,
25705
+ windowMs: REMOTE_HEALTH_WINDOW_MS
25706
+ }
25707
+ });
25708
+ this.assignments.delete(deviceId);
25709
+ this.pendingReasons.set(deviceId, "unhealthy");
25710
+ await this.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
25711
+ this.ctx.logger.debug("remote-health exhausted-detach failed", {
25712
+ tags: { deviceId },
25713
+ meta: { error: errMsg(err) }
25714
+ });
25715
+ });
25716
+ return;
25717
+ }
25718
+ this.remoteHealthAttempts.set(deviceId, {
25719
+ count: count + 1,
25720
+ windowStart: windowFresh ? prior.windowStart : now
25721
+ });
25722
+ this.ctx.logger.warn("remote assignment unhealthy — re-placing camera", {
25723
+ tags: {
25724
+ deviceId,
25725
+ nodeId: assignment.agentNodeId
25726
+ },
25727
+ meta: {
25728
+ why,
25729
+ attempt: count + 1,
25730
+ maxAttempts: REMOTE_HEALTH_MAX_ATTEMPTS
25731
+ }
25732
+ });
25733
+ await this.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
25734
+ this.ctx.logger.warn("remote-health detach failed", {
25735
+ tags: { deviceId },
25736
+ meta: { error: errMsg(err) }
25737
+ });
25738
+ });
25739
+ this.assignments.delete(deviceId);
25740
+ await this.dispatchCamera(cfg).catch((err) => {
25741
+ this.ctx.logger.warn("remote-health re-dispatch failed", {
25742
+ tags: { deviceId },
25743
+ meta: { error: errMsg(err) }
25744
+ });
25745
+ });
25746
+ }
25126
25747
  async getCapabilityBindings(input) {
25127
25748
  if (!this.ctx?.settings) return {};
25128
25749
  const perNodeRaw = (await this.nodeBindingsState.get().catch(() => ({})))[input.nodeId];
@@ -25216,12 +25837,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25216
25837
  }
25217
25838
  async assignAudio(input) {
25218
25839
  await this.ctx.settings?.writeDeviceStore(input.deviceId, { [AUDIO_NODE_SETTING]: input.nodeId });
25840
+ if (!this.enabledAudioNodes.includes(input.nodeId)) this.ctx.logger.warn("audio pin stored but node is not in enabledAudioNodes — pin will not take effect until enabled", { tags: {
25841
+ deviceId: input.deviceId,
25842
+ nodeId: input.nodeId
25843
+ } });
25219
25844
  this.audioAssignments.delete(input.deviceId);
25220
25845
  this.audioNodeByDevice.delete(input.deviceId);
25221
25846
  this.ctx.logger.info("Audio node pinned", { tags: {
25222
25847
  deviceId: input.deviceId,
25223
25848
  nodeId: input.nodeId
25224
25849
  } });
25850
+ await this.reattachAudioForDevice(input.deviceId);
25225
25851
  return { success: true };
25226
25852
  }
25227
25853
  async unassignAudio(input) {
@@ -25229,6 +25855,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25229
25855
  this.audioAssignments.delete(input.deviceId);
25230
25856
  this.audioNodeByDevice.delete(input.deviceId);
25231
25857
  this.ctx.logger.info("Audio node unpinned", { tags: { deviceId: input.deviceId } });
25858
+ await this.reattachAudioForDevice(input.deviceId);
25232
25859
  return { success: true };
25233
25860
  }
25234
25861
  async getAudioAssignment(input) {
@@ -25312,6 +25939,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25312
25939
  tags: { nodeId: input.agentNodeId },
25313
25940
  meta: { maxCameras: input.maxCameras }
25314
25941
  });
25942
+ this.schedulePendingRetry();
25315
25943
  return { success: true };
25316
25944
  }
25317
25945
  async getCameraSettings(input) {
@@ -25475,7 +26103,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25475
26103
  const detectionNodeId = pipelineAssignment?.agentNodeId ?? null;
25476
26104
  const decoderPinRaw = (api ? await this.ctx.settings?.readDeviceStore(deviceId).catch(() => ({})) ?? {} : {})["decoderNodeId"];
25477
26105
  const decoderPinned = typeof decoderPinRaw === "string" && decoderPinRaw !== "auto";
25478
- const decoderNodeId = detectionNodeId ? await this.resolveDecoderNode(deviceId, detectionNodeId).catch(() => null) : null;
26106
+ const advisoryDecoderNodeId = detectionNodeId ? await this.resolveDecoderNode(deviceId, detectionNodeId).catch(() => null) : null;
25479
26107
  const audioAssignment = this.audioAssignments.get(deviceId) ?? null;
25480
26108
  const audioNodeId = audioAssignment?.nodeId ?? null;
25481
26109
  const audioPinned = audioAssignment?.pinned ?? false;
@@ -25484,11 +26112,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25484
26112
  decoder: decoderPinned,
25485
26113
  audio: audioPinned
25486
26114
  };
25487
- const reasons = {
25488
- detection: pipelineAssignment?.reason,
25489
- decoder: decoderPinned ? "manual" : "co-located",
25490
- audio: audioPinned ? "manual" : void 0
25491
- };
26115
+ const detectionReason = pipelineAssignment !== null ? pipelineAssignment.reason : this.cameraConfigs.has(deviceId) ? `pending:${this.pendingReasons.get(deviceId) ?? "pending"}` : void 0;
26116
+ const liveDecoder = { nodeId: null };
25492
26117
  const allSlotsFetch = api ? api.streamBroker.listAllProfileSlots.query() : null;
25493
26118
  const sourceFetch = api && allSlotsFetch ? this.boundedStage(allSlotsFetch.then((slots) => {
25494
26119
  return { streams: slots.filter((s) => s.deviceId === deviceId).map((s) => ({
@@ -25519,17 +26144,19 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25519
26144
  clients: await api.streamBroker.listClients.query({ brokerId: slot.brokerId }).catch(() => null)
25520
26145
  };
25521
26146
  })), api.streamBroker.getAllRtspEntries.query({}).catch(() => null)]);
26147
+ const profileDetails = statsAndClients.map(({ slot, stats, clients }) => ({
26148
+ profile: slot.profile,
26149
+ status: slot.status,
26150
+ codec: stats?.codec ?? slot.codec ?? "",
26151
+ width: slot.resolution?.width ?? 0,
26152
+ height: slot.resolution?.height ?? 0,
26153
+ subscribers: clients?.encodedSubscribers ?? 0,
26154
+ inFps: stats?.inputFps ?? 0,
26155
+ outFps: stats?.decodeFps ?? 0
26156
+ }));
26157
+ for (const { stats } of statsAndClients) if (liveDecoder.nodeId === null && typeof stats?.decoderNodeId === "string") liveDecoder.nodeId = stats.decoderNodeId;
25522
26158
  return {
25523
- profiles: statsAndClients.map(({ slot, stats, clients }) => ({
25524
- profile: slot.profile,
25525
- status: slot.status,
25526
- codec: stats?.codec ?? slot.codec ?? "",
25527
- width: slot.resolution?.width ?? 0,
25528
- height: slot.resolution?.height ?? 0,
25529
- subscribers: clients?.encodedSubscribers ?? 0,
25530
- inFps: stats?.inputFps ?? 0,
25531
- outFps: stats?.decodeFps ?? 0
25532
- })),
26159
+ profiles: profileDetails,
25533
26160
  webrtcSessions: statsAndClients.reduce((total, { clients }) => {
25534
26161
  if (!clients) return total;
25535
26162
  return total + clients.encoded.filter((c) => WEBRTC_KINDS.has(c.attribution.kind)).length;
@@ -25539,8 +26166,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25539
26166
  }) ?? false
25540
26167
  };
25541
26168
  }), STAGE_TIMEOUT_MS) : Promise.resolve(null);
25542
- const decoderFetch = decoderNodeId ? Promise.resolve({
25543
- nodeId: decoderNodeId,
26169
+ const decoderFetch = advisoryDecoderNodeId ? Promise.resolve({
26170
+ nodeId: advisoryDecoderNodeId,
25544
26171
  formats: [],
25545
26172
  sessionCount: 0,
25546
26173
  shm: {
@@ -25608,6 +26235,13 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25608
26235
  detectionFetch,
25609
26236
  recordingFetch
25610
26237
  ]);
26238
+ const liveDecoderNodeId = brokerResult !== null ? liveDecoder.nodeId : null;
26239
+ const decoderNodeId = liveDecoderNodeId ?? advisoryDecoderNodeId;
26240
+ const reasons = {
26241
+ detection: detectionReason,
26242
+ decoder: decoderPinned ? "manual" : liveDecoderNodeId !== null ? "session" : decoderNodeId !== null ? "advisory" : void 0,
26243
+ audio: audioPinned ? "manual" : void 0
26244
+ };
25611
26245
  return composeCameraStatus({
25612
26246
  deviceId,
25613
26247
  fetchedAt: Date.now(),
@@ -25963,7 +26597,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25963
26597
  id: "cluster",
25964
26598
  title: "Cluster",
25965
26599
  tab: "pipeline",
25966
- description: "Which cluster nodes are eligible to run the detection pipeline. Leave at least one node enabled an empty list falls back to all online nodes so the pipeline never becomes completely unassignable.",
26600
+ description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
25967
26601
  fields: [
25968
26602
  {
25969
26603
  key: "enabledNodes",
@@ -26495,6 +27129,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26495
27129
  this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
26496
27130
  const rawEnabledAudio = config["enabledAudioNodes"];
26497
27131
  this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
27132
+ this.schedulePendingRetry();
26498
27133
  }
26499
27134
  get api() {
26500
27135
  return this.ctx.api ?? null;