@camstack/addon-pipeline-orchestrator 1.1.14 → 1.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-wXpruvsb.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-C2Ftp01d.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CEVTusPn.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-D5WB-hDc.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-sANx-U6O.mjs} +1 -1
- package/dist/{hostInit-CjgkznUo.mjs → hostInit-DuphCRk3.mjs} +2 -2
- package/dist/index.js +1596 -940
- package/dist/index.mjs +1596 -940
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-yz7hJdXe.mjs +0 -26
package/dist/index.mjs
CHANGED
|
@@ -4627,7 +4627,7 @@ function _instanceof(cls, params = {}) {
|
|
|
4627
4627
|
return inst;
|
|
4628
4628
|
}
|
|
4629
4629
|
//#endregion
|
|
4630
|
-
//#region ../types/dist/sleep-
|
|
4630
|
+
//#region ../types/dist/sleep-MHm--th-.mjs
|
|
4631
4631
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
4632
4632
|
EventCategory["SystemBoot"] = "system.boot";
|
|
4633
4633
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -5905,6 +5905,7 @@ var CamStreamKindSchema = _enum([
|
|
|
5905
5905
|
"pull-rtsp",
|
|
5906
5906
|
"pull-rtmp",
|
|
5907
5907
|
"pull-http",
|
|
5908
|
+
"pull-flv",
|
|
5908
5909
|
"pull-rfc4571",
|
|
5909
5910
|
"push-annexb",
|
|
5910
5911
|
"derived"
|
|
@@ -13996,7 +13997,7 @@ var AddBrokerInputSchema = object({
|
|
|
13996
13997
|
});
|
|
13997
13998
|
var AddBrokerResultSchema = object({ id: string() });
|
|
13998
13999
|
var IdInputSchema = object({ id: string() });
|
|
13999
|
-
var TestResultSchema = discriminatedUnion("ok", [object({
|
|
14000
|
+
var TestResultSchema$1 = discriminatedUnion("ok", [object({
|
|
14000
14001
|
ok: literal(true),
|
|
14001
14002
|
latencyMs: number()
|
|
14002
14003
|
}), object({
|
|
@@ -14019,7 +14020,7 @@ var StatusSchema = object({
|
|
|
14019
14020
|
brokerCount: number(),
|
|
14020
14021
|
embeddedRunning: boolean()
|
|
14021
14022
|
});
|
|
14022
|
-
method(_void(), array(BrokerInfoSchema)), method(IdInputSchema, BrokerConnectionDetailsSchema), method(AddBrokerInputSchema, AddBrokerResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(IdInputSchema, TestResultSchema, { kind: "mutation" }), method(StartEmbeddedInputSchema, StartEmbeddedResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(_void(), StatusSchema);
|
|
14023
|
+
method(_void(), array(BrokerInfoSchema)), method(IdInputSchema, BrokerConnectionDetailsSchema), method(AddBrokerInputSchema, AddBrokerResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(IdInputSchema, TestResultSchema$1, { kind: "mutation" }), method(StartEmbeddedInputSchema, StartEmbeddedResultSchema, { kind: "mutation" }), method(IdInputSchema, _void(), { kind: "mutation" }), method(_void(), StatusSchema);
|
|
14023
14024
|
var NetworkEndpointSchema = object({
|
|
14024
14025
|
url: string(),
|
|
14025
14026
|
hostname: string(),
|
|
@@ -14053,23 +14054,198 @@ var NetworkEndpointEntrySchema = NetworkEndpointSchema.extend({
|
|
|
14053
14054
|
sourcePort: number().optional()
|
|
14054
14055
|
});
|
|
14055
14056
|
method(_void(), NetworkEndpointSchema, { kind: "mutation" }), method(_void(), _void(), { kind: "mutation" }), method(_void(), NetworkEndpointSchema.nullable()), method(_void(), NetworkAccessStatusSchema), method(_void(), array(NetworkEndpointEntrySchema).readonly());
|
|
14056
|
-
|
|
14057
|
-
|
|
14057
|
+
/**
|
|
14058
|
+
* notification-output — canonical, capability-gated notification delivery.
|
|
14059
|
+
*
|
|
14060
|
+
* Apprise-derived model (see
|
|
14061
|
+
* `docs/superpowers/specs/2026-07-03-notification-output-notifier-matrix.md`):
|
|
14062
|
+
* callers emit ONE canonical `Notification`; each provider declares a
|
|
14063
|
+
* per-kind capability descriptor (`TargetKind`), and the pure degrade
|
|
14064
|
+
* engine (`@camstack/types` `prepareNotification`) transcodes / degrades the
|
|
14065
|
+
* message to what the kind supports — callers never special-case a service.
|
|
14066
|
+
*
|
|
14067
|
+
* DESIGN DECISIONS (locked):
|
|
14068
|
+
* - Target CRUD lives on THIS cap (`upsertTarget` / `deleteTarget` /
|
|
14069
|
+
* `setTargetEnabled`), each provider persisting via the `settings-store`
|
|
14070
|
+
* cap. Rationale: the admin UI needs one uniform surface across the
|
|
14071
|
+
* notifiers addon AND the HA addon; the addon-`globalSettingsSchema`-array
|
|
14072
|
+
* alternative would fork the UI per addon and cannot host the
|
|
14073
|
+
* discovery→adopt flow.
|
|
14074
|
+
* - `listTargetKinds` / `listTargets` / `discoverTargets` return arrays →
|
|
14075
|
+
* the generated cap-mount auto-`concatCollection`-fans them across every
|
|
14076
|
+
* registered provider (notifiers addon + HA addon) so one catalog is
|
|
14077
|
+
* routable. `send` / `testTarget` / CRUD route to ONE provider by the
|
|
14078
|
+
* `addonId` the generated collection router extracts from the call input.
|
|
14079
|
+
* - `Attachment.bytes` is `Uint8Array`. Transport-safe: superjson (the tRPC
|
|
14080
|
+
* transformer) + UDS MsgPack both round-trip typed arrays — already used by
|
|
14081
|
+
* `storage` / `storage-provider` / `recording` caps over the same path. No
|
|
14082
|
+
* base64 fallback needed.
|
|
14083
|
+
*
|
|
14084
|
+
* TODO (deferred, closed-set change — separate decision): add
|
|
14085
|
+
* `providerKind: 'notify'` so notification providers surface on the unified
|
|
14086
|
+
* admin "Integrations" page.
|
|
14087
|
+
*/
|
|
14088
|
+
/**
|
|
14089
|
+
* Zentik-derived typed-media enum — the superset across every kind. Each
|
|
14090
|
+
* adapter picks what it supports and the degrade engine filters the rest.
|
|
14091
|
+
*/
|
|
14092
|
+
var AttachmentMediaTypeSchema = _enum([
|
|
14093
|
+
"image",
|
|
14094
|
+
"video",
|
|
14095
|
+
"gif",
|
|
14096
|
+
"audio",
|
|
14097
|
+
"icon"
|
|
14098
|
+
]);
|
|
14099
|
+
/**
|
|
14100
|
+
* A single attachment. Exactly one of `url` (remote source, most adapters
|
|
14101
|
+
* prefer this) or `bytes` (inline source; required for Pushover-style
|
|
14102
|
+
* bytes-only kinds) MUST be present — the degrade engine expresses a
|
|
14103
|
+
* url→bytes fetch as a `needsFetch` directive the adapter executes.
|
|
14104
|
+
*/
|
|
14105
|
+
var AttachmentSchema = object({
|
|
14106
|
+
mediaType: AttachmentMediaTypeSchema,
|
|
14107
|
+
url: string().optional(),
|
|
14108
|
+
bytes: _instanceof(Uint8Array).optional(),
|
|
14109
|
+
mime: string().optional(),
|
|
14110
|
+
name: string().optional()
|
|
14111
|
+
}).refine((a) => a.url !== void 0 || a.bytes !== void 0, { message: "Attachment requires either `url` or `bytes`" });
|
|
14112
|
+
var NotificationFormatSchema = _enum([
|
|
14113
|
+
"text",
|
|
14114
|
+
"markdown",
|
|
14115
|
+
"html"
|
|
14116
|
+
]);
|
|
14117
|
+
/** A single tap-through action button. */
|
|
14118
|
+
var NotificationActionSchema = object({
|
|
14119
|
+
id: string(),
|
|
14120
|
+
label: string(),
|
|
14121
|
+
url: string().optional()
|
|
14122
|
+
});
|
|
14123
|
+
/**
|
|
14124
|
+
* The canonical notification. `body` is the only hard field (Apprise model).
|
|
14125
|
+
* `priority` is a 5-level ORDINAL (1=lowest … 3=normal(default) … 5=urgent),
|
|
14126
|
+
* NOT a fixed severity enum — each kind declares its own `caps.levels` and
|
|
14127
|
+
* the adapter maps this ordinal onto its native level. `level?` is an
|
|
14128
|
+
* optional kind-native level id (`emergency`, `silent`, …) that overrides
|
|
14129
|
+
* `priority` for that one target.
|
|
14130
|
+
*/
|
|
14131
|
+
var NotificationSchema = object({
|
|
14058
14132
|
body: string(),
|
|
14059
|
-
|
|
14133
|
+
title: string().optional(),
|
|
14134
|
+
format: NotificationFormatSchema.default("text"),
|
|
14135
|
+
priority: number().int().min(1).max(5).default(3),
|
|
14136
|
+
level: string().optional(),
|
|
14137
|
+
attachments: array(AttachmentSchema).optional(),
|
|
14138
|
+
clickUrl: string().optional(),
|
|
14139
|
+
actions: array(NotificationActionSchema).optional(),
|
|
14140
|
+
sound: string().optional(),
|
|
14141
|
+
ttl: number().optional(),
|
|
14142
|
+
tag: string().optional(),
|
|
14060
14143
|
deviceId: number().optional(),
|
|
14061
14144
|
eventId: string().optional(),
|
|
14062
|
-
priority: _enum([
|
|
14063
|
-
"low",
|
|
14064
|
-
"normal",
|
|
14065
|
-
"high",
|
|
14066
|
-
"critical"
|
|
14067
|
-
]).default("normal"),
|
|
14068
14145
|
metadata: record(string(), unknown()).optional()
|
|
14069
|
-
})
|
|
14146
|
+
});
|
|
14147
|
+
/** One declared native severity/priority level for a kind. */
|
|
14148
|
+
var TargetKindLevelSchema = object({
|
|
14149
|
+
id: string(),
|
|
14150
|
+
label: string(),
|
|
14151
|
+
/** Which canonical priority (1..5) this level maps to. `null` = qualitative-only. */
|
|
14152
|
+
ordinal: number().int().min(1).max(5).nullable(),
|
|
14153
|
+
flags: object({
|
|
14154
|
+
critical: boolean().optional(),
|
|
14155
|
+
silent: boolean().optional(),
|
|
14156
|
+
noPush: boolean().optional()
|
|
14157
|
+
}).optional(),
|
|
14158
|
+
/** e.g. Pushover `emergency` requires `retry` / `expire`. */
|
|
14159
|
+
requires: array(string()).optional(),
|
|
14160
|
+
description: string().optional()
|
|
14161
|
+
});
|
|
14162
|
+
/** The full capability block consulted before dispatch. */
|
|
14163
|
+
var TargetKindCapsSchema = object({
|
|
14164
|
+
attachments: object({
|
|
14165
|
+
mediaTypes: array(AttachmentMediaTypeSchema),
|
|
14166
|
+
mode: _enum([
|
|
14167
|
+
"url",
|
|
14168
|
+
"bytes",
|
|
14169
|
+
"both"
|
|
14170
|
+
]),
|
|
14171
|
+
max: number().int().nonnegative(),
|
|
14172
|
+
maxBytes: number().int().positive().optional()
|
|
14173
|
+
}),
|
|
14174
|
+
/** Max action buttons (0 = none). */
|
|
14175
|
+
actions: number().int().nonnegative(),
|
|
14176
|
+
levels: array(TargetKindLevelSchema),
|
|
14177
|
+
format: array(NotificationFormatSchema),
|
|
14178
|
+
clickUrl: boolean(),
|
|
14179
|
+
sound: boolean(),
|
|
14180
|
+
ttl: boolean(),
|
|
14181
|
+
bodyMaxLen: number().int().positive()
|
|
14182
|
+
});
|
|
14183
|
+
/**
|
|
14184
|
+
* `configSchema` is a `ConfigUISchema` tree passed through to the admin
|
|
14185
|
+
* FormBuilder. Stored as `z.unknown()` at the cap seam (mirrors
|
|
14186
|
+
* `device-provider.getChildCreationSchema` `CreationSchemaOutputSchema`) —
|
|
14187
|
+
* the union is large and not meant for runtime validation here; the exported
|
|
14188
|
+
* `TargetKind` type re-tightens `configSchema` to `ConfigUISchema`.
|
|
14189
|
+
*/
|
|
14190
|
+
var ConfigSchemaPassthrough = unknown();
|
|
14191
|
+
var TargetKindSchema = object({
|
|
14192
|
+
kind: string(),
|
|
14193
|
+
label: string(),
|
|
14194
|
+
icon: string(),
|
|
14195
|
+
/** Stamped by each provider so the concat-fanned catalog stays routable. */
|
|
14196
|
+
addonId: string(),
|
|
14197
|
+
configSchema: ConfigSchemaPassthrough,
|
|
14198
|
+
supportsDiscovery: boolean(),
|
|
14199
|
+
caps: TargetKindCapsSchema
|
|
14200
|
+
});
|
|
14201
|
+
/**
|
|
14202
|
+
* A persisted target. `config` holds secrets; providers REDACT secret fields
|
|
14203
|
+
* (return a presence marker only) when serving `listTargets` — never
|
|
14204
|
+
* round-trip a stored secret to the UI.
|
|
14205
|
+
*/
|
|
14206
|
+
var TargetSchema = object({
|
|
14207
|
+
id: string(),
|
|
14208
|
+
name: string(),
|
|
14209
|
+
kind: string(),
|
|
14210
|
+
addonId: string(),
|
|
14211
|
+
enabled: boolean(),
|
|
14212
|
+
config: record(string(), unknown())
|
|
14213
|
+
});
|
|
14214
|
+
/** A discovery-surfaced candidate (config is partial + non-secret). */
|
|
14215
|
+
var DiscoveredTargetSchema = object({
|
|
14216
|
+
kind: string(),
|
|
14217
|
+
suggestedName: string(),
|
|
14218
|
+
config: record(string(), unknown())
|
|
14219
|
+
});
|
|
14220
|
+
/** The degrade engine's report — what was resolved / dropped / degraded. */
|
|
14221
|
+
var RenderedAsSchema = object({
|
|
14222
|
+
level: string(),
|
|
14223
|
+
format: NotificationFormatSchema,
|
|
14224
|
+
attachmentsSent: number().int().nonnegative(),
|
|
14225
|
+
actionsSent: number().int().nonnegative(),
|
|
14226
|
+
truncated: boolean(),
|
|
14227
|
+
dropped: array(string())
|
|
14228
|
+
});
|
|
14229
|
+
var SendResultSchema = object({
|
|
14070
14230
|
success: boolean(),
|
|
14071
|
-
error: string().optional()
|
|
14072
|
-
|
|
14231
|
+
error: string().optional(),
|
|
14232
|
+
renderedAs: RenderedAsSchema.optional()
|
|
14233
|
+
});
|
|
14234
|
+
/** Same shape as SendResult — kept as a distinct name for the test panel. */
|
|
14235
|
+
var TestResultSchema = SendResultSchema;
|
|
14236
|
+
method(object({}), array(TargetKindSchema)), method(object({}), array(TargetSchema)), method(object({
|
|
14237
|
+
kind: string(),
|
|
14238
|
+
config: record(string(), unknown()).optional()
|
|
14239
|
+
}), array(DiscoveredTargetSchema)), method(object({
|
|
14240
|
+
targetId: string(),
|
|
14241
|
+
notification: NotificationSchema
|
|
14242
|
+
}), SendResultSchema, { kind: "mutation" }), method(object({
|
|
14243
|
+
targetId: string(),
|
|
14244
|
+
sample: NotificationSchema.optional()
|
|
14245
|
+
}), TestResultSchema, { kind: "mutation" }), method(object({ target: TargetSchema }), TargetSchema, { kind: "mutation" }), method(object({ targetId: string() }), _void(), { kind: "mutation" }), method(object({
|
|
14246
|
+
targetId: string(),
|
|
14247
|
+
enabled: boolean()
|
|
14248
|
+
}), _void(), { kind: "mutation" });
|
|
14073
14249
|
/**
|
|
14074
14250
|
* Zod schemas for persisted record types.
|
|
14075
14251
|
*
|
|
@@ -14859,10 +15035,11 @@ var pipelineOrchestratorCapability = {
|
|
|
14859
15035
|
}))),
|
|
14860
15036
|
/**
|
|
14861
15037
|
* Get one camera's decoder placement (computed if not yet pinned).
|
|
14862
|
-
*
|
|
14863
|
-
*
|
|
14864
|
-
*
|
|
14865
|
-
*
|
|
15038
|
+
*
|
|
15039
|
+
* ADVISORY today: reports the orchestrator's decoder preference only.
|
|
15040
|
+
* Actual decode placement is broker-owned (local-node pin + frame-plane
|
|
15041
|
+
* co-location guard). Reserved to become the binding source/decoder-owner
|
|
15042
|
+
* control in the stream-LB epic (Phase 2).
|
|
14866
15043
|
*
|
|
14867
15044
|
* `pipelineNodeId` is the node already chosen to run inference for
|
|
14868
15045
|
* this camera. When provided, the balancer prefers co-location with
|
|
@@ -20234,13 +20411,49 @@ Object.freeze({
|
|
|
20234
20411
|
addonId: null,
|
|
20235
20412
|
access: "create"
|
|
20236
20413
|
},
|
|
20414
|
+
"notificationOutput.deleteTarget": {
|
|
20415
|
+
capName: "notification-output",
|
|
20416
|
+
capScope: "system",
|
|
20417
|
+
addonId: null,
|
|
20418
|
+
access: "delete"
|
|
20419
|
+
},
|
|
20420
|
+
"notificationOutput.discoverTargets": {
|
|
20421
|
+
capName: "notification-output",
|
|
20422
|
+
capScope: "system",
|
|
20423
|
+
addonId: null,
|
|
20424
|
+
access: "view"
|
|
20425
|
+
},
|
|
20426
|
+
"notificationOutput.listTargetKinds": {
|
|
20427
|
+
capName: "notification-output",
|
|
20428
|
+
capScope: "system",
|
|
20429
|
+
addonId: null,
|
|
20430
|
+
access: "view"
|
|
20431
|
+
},
|
|
20432
|
+
"notificationOutput.listTargets": {
|
|
20433
|
+
capName: "notification-output",
|
|
20434
|
+
capScope: "system",
|
|
20435
|
+
addonId: null,
|
|
20436
|
+
access: "view"
|
|
20437
|
+
},
|
|
20237
20438
|
"notificationOutput.send": {
|
|
20238
20439
|
capName: "notification-output",
|
|
20239
20440
|
capScope: "system",
|
|
20240
20441
|
addonId: null,
|
|
20241
20442
|
access: "create"
|
|
20242
20443
|
},
|
|
20243
|
-
"notificationOutput.
|
|
20444
|
+
"notificationOutput.setTargetEnabled": {
|
|
20445
|
+
capName: "notification-output",
|
|
20446
|
+
capScope: "system",
|
|
20447
|
+
addonId: null,
|
|
20448
|
+
access: "create"
|
|
20449
|
+
},
|
|
20450
|
+
"notificationOutput.testTarget": {
|
|
20451
|
+
capName: "notification-output",
|
|
20452
|
+
capScope: "system",
|
|
20453
|
+
addonId: null,
|
|
20454
|
+
access: "create"
|
|
20455
|
+
},
|
|
20456
|
+
"notificationOutput.upsertTarget": {
|
|
20244
20457
|
capName: "notification-output",
|
|
20245
20458
|
capScope: "system",
|
|
20246
20459
|
addonId: null,
|
|
@@ -22416,470 +22629,232 @@ function buildTreeFromAddons(enabled, catalog) {
|
|
|
22416
22629
|
return roots;
|
|
22417
22630
|
}
|
|
22418
22631
|
//#endregion
|
|
22419
|
-
//#region src/
|
|
22632
|
+
//#region src/audio-chunk-poller.ts
|
|
22420
22633
|
/**
|
|
22421
|
-
* `
|
|
22634
|
+
* `AudioChunkPoller` — the consumer-side poll loop of the decoded audio-chunk
|
|
22635
|
+
* plane (Phase 5 / D9).
|
|
22422
22636
|
*
|
|
22423
|
-
*
|
|
22424
|
-
*
|
|
22425
|
-
*
|
|
22426
|
-
*
|
|
22427
|
-
* pipeline-executor, analytics, admin UI) read the live state with
|
|
22428
|
-
* the canonical `dev.state.zones.onChanged` channel.
|
|
22637
|
+
* Replaces the live-object `IStreamBroker.onDecodedAudioChunk(cb)` callback
|
|
22638
|
+
* path. A live callback cannot cross a process boundary; once the `pipeline`
|
|
22639
|
+
* group is dissolved (Task 8) the orchestrator runs in a different process
|
|
22640
|
+
* from the broker, so audio delivery must go over tRPC.
|
|
22429
22641
|
*
|
|
22430
|
-
*
|
|
22431
|
-
* zone is operator-drawn. The provider keeps the surface symmetric:
|
|
22432
|
-
* `addZone` rejects id collisions, `updateZone` requires an existing
|
|
22433
|
-
* id, `removeZone` is idempotent.
|
|
22434
|
-
*/
|
|
22435
|
-
var ZONES_STORE_KEY = "zones";
|
|
22436
|
-
var ZONES_CAP_NAME = "zones";
|
|
22437
|
-
var ZonesArraySchema = array(ZoneSchema);
|
|
22438
|
-
var ZonesProvider = class {
|
|
22439
|
-
ctx;
|
|
22440
|
-
/** Per-device cache. Hydrated lazily on first read for a device. */
|
|
22441
|
-
cache = /* @__PURE__ */ new Map();
|
|
22442
|
-
/**
|
|
22443
|
-
* Per-device durable handle over the `zones` store key. The WHOLE
|
|
22444
|
-
* validated zone array round-trips on every read/write so no field can
|
|
22445
|
-
* be dropped on persist. Built lazily + memoised per device.
|
|
22446
|
-
*/
|
|
22447
|
-
stateByDevice = /* @__PURE__ */ new Map();
|
|
22448
|
-
constructor(ctx) {
|
|
22449
|
-
this.ctx = ctx;
|
|
22450
|
-
}
|
|
22451
|
-
/** Lazily build (and memoise) the durable `zones` handle for a device. */
|
|
22452
|
-
zonesState(deviceId) {
|
|
22453
|
-
let handle = this.stateByDevice.get(deviceId);
|
|
22454
|
-
if (!handle) {
|
|
22455
|
-
handle = createDurableState({
|
|
22456
|
-
key: ZONES_STORE_KEY,
|
|
22457
|
-
schema: ZonesArraySchema,
|
|
22458
|
-
fallback: [],
|
|
22459
|
-
read: () => this.ctx.settings.readDeviceStore(deviceId),
|
|
22460
|
-
write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
|
|
22461
|
-
onParseError: (key, error) => this.ctx.logger.warn("zones store entry failed to parse — resetting", {
|
|
22462
|
-
tags: { deviceId },
|
|
22463
|
-
meta: {
|
|
22464
|
-
key,
|
|
22465
|
-
error: error instanceof Error ? error.message : String(error)
|
|
22466
|
-
}
|
|
22467
|
-
})
|
|
22468
|
-
});
|
|
22469
|
-
this.stateByDevice.set(deviceId, handle);
|
|
22470
|
-
}
|
|
22471
|
-
return handle;
|
|
22472
|
-
}
|
|
22473
|
-
async listZones({ deviceId }) {
|
|
22474
|
-
return this.loadZones(deviceId);
|
|
22475
|
-
}
|
|
22476
|
-
async addZone({ deviceId, zone }) {
|
|
22477
|
-
const existing = await this.loadZones(deviceId);
|
|
22478
|
-
if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
|
|
22479
|
-
await this.persist(deviceId, [...existing, zone]);
|
|
22480
|
-
}
|
|
22481
|
-
async updateZone({ deviceId, zone }) {
|
|
22482
|
-
const existing = await this.loadZones(deviceId);
|
|
22483
|
-
if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
|
|
22484
|
-
const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
|
|
22485
|
-
await this.persist(deviceId, next);
|
|
22486
|
-
}
|
|
22487
|
-
async removeZone({ deviceId, zoneId }) {
|
|
22488
|
-
const existing = await this.loadZones(deviceId);
|
|
22489
|
-
if (!existing.some((entry) => entry.id === zoneId)) return;
|
|
22490
|
-
await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
|
|
22491
|
-
}
|
|
22492
|
-
/**
|
|
22493
|
-
* Drop a device's cache entry. Called when the device is removed so
|
|
22494
|
-
* the next attach starts from a fresh disk read.
|
|
22495
|
-
*/
|
|
22496
|
-
forgetDevice(deviceId) {
|
|
22497
|
-
this.cache.delete(deviceId);
|
|
22498
|
-
this.stateByDevice.delete(deviceId);
|
|
22499
|
-
}
|
|
22500
|
-
async loadZones(deviceId) {
|
|
22501
|
-
const cached = this.cache.get(deviceId);
|
|
22502
|
-
if (cached) return cached;
|
|
22503
|
-
let zones = [];
|
|
22504
|
-
try {
|
|
22505
|
-
zones = await this.zonesState(deviceId).get();
|
|
22506
|
-
} catch (err) {
|
|
22507
|
-
this.ctx.logger.warn("zones store read failed — using empty list", {
|
|
22508
|
-
tags: { deviceId },
|
|
22509
|
-
meta: { error: err instanceof Error ? err.message : String(err) }
|
|
22510
|
-
});
|
|
22511
|
-
}
|
|
22512
|
-
this.cache.set(deviceId, zones);
|
|
22513
|
-
return zones;
|
|
22514
|
-
}
|
|
22515
|
-
async persist(deviceId, zones) {
|
|
22516
|
-
this.cache.set(deviceId, zones);
|
|
22517
|
-
await this.zonesState(deviceId).set(zones);
|
|
22518
|
-
await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
|
|
22519
|
-
capName: ZONES_CAP_NAME,
|
|
22520
|
-
slice: { zones }
|
|
22521
|
-
});
|
|
22522
|
-
this.ctx.onZonesChanged?.(deviceId, zones);
|
|
22523
|
-
}
|
|
22524
|
-
};
|
|
22525
|
-
//#endregion
|
|
22526
|
-
//#region src/zone-rules-provider.ts
|
|
22527
|
-
/**
|
|
22528
|
-
* `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
|
|
22529
|
-
* orchestrator.
|
|
22642
|
+
* The consumer:
|
|
22530
22643
|
*
|
|
22531
|
-
*
|
|
22532
|
-
*
|
|
22533
|
-
*
|
|
22534
|
-
* (
|
|
22535
|
-
*
|
|
22536
|
-
*
|
|
22644
|
+
* 1. `subscribeAudioChunks({ brokerId, tag })` over tRPC — the broker
|
|
22645
|
+
* registers a per-subscription bounded FIFO queue and returns a
|
|
22646
|
+
* `subscriptionId`;
|
|
22647
|
+
* 2. polls `pullAudioChunks({ subscriptionId, maxCount })` on a modest
|
|
22648
|
+
* interval, draining `DecodedAudioChunk`s in FIFO arrival order;
|
|
22649
|
+
* 3. feeds each chunk to its downstream audio logic;
|
|
22650
|
+
* 4. on teardown, `unsubscribeAudioChunks`.
|
|
22537
22651
|
*
|
|
22538
|
-
*
|
|
22539
|
-
*
|
|
22540
|
-
*
|
|
22541
|
-
*
|
|
22652
|
+
* Audio is not latency-critical like video, and chunks arrive only ~every
|
|
22653
|
+
* 500ms (the broker's `AudioCodecSession` window). A modest poll period plus
|
|
22654
|
+
* a small per-poll burst keeps latency low without busy-spinning. The
|
|
22655
|
+
* broker's queue is FIFO/no-drop-sized so a poll-period of jitter never
|
|
22656
|
+
* loses a chunk.
|
|
22657
|
+
*
|
|
22658
|
+
* Boot-race tolerance: the broker for a given camStream may not be registered
|
|
22659
|
+
* yet when the orchestrator wires the subscription (provider addons publish
|
|
22660
|
+
* their cameraStreams asynchronously after their probe completes).
|
|
22661
|
+
* `subscribeAudioChunks` retries with exponential backoff (capped at
|
|
22662
|
+
* `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
|
|
22663
|
+
* teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
|
|
22664
|
+
* shape so video and audio plumbing self-heal identically.
|
|
22542
22665
|
*/
|
|
22543
|
-
/**
|
|
22544
|
-
|
|
22545
|
-
|
|
22546
|
-
var
|
|
22547
|
-
/** Cap name for the unified runtime-state slice. Matches the cap's
|
|
22548
|
-
* declared `name` so the codegen DeviceProxy auto-wires
|
|
22549
|
-
* `device.state.zoneRules`. The slice value is the full
|
|
22550
|
-
* `{motion, detection}` object — both stages travel together so a
|
|
22551
|
-
* single reactive handle covers every consumer. */
|
|
22552
|
-
var ZONE_RULES_CAP_NAME = "zone-rules";
|
|
22553
|
-
var RulesArraySchema = array(ZoneRuleSchema);
|
|
22554
|
-
/**
|
|
22555
|
-
* Whole-blob schema for the per-device `zoneRules` store key. Both stages
|
|
22556
|
-
* travel together under one key. Deliberately lenient — each stage is
|
|
22557
|
-
* `unknown` so a corrupt single stage can NOT drop the sibling on load
|
|
22558
|
-
* (durable-state `get()` falls back to `{}` only on a whole-blob parse
|
|
22559
|
-
* failure). Strict per-stage validation, with its own reset-on-corrupt
|
|
22560
|
-
* warn, still happens in `loadRules` exactly as before the migration.
|
|
22561
|
-
*/
|
|
22562
|
-
var ZoneRulesBlockSchema = object({
|
|
22563
|
-
motion: unknown().optional(),
|
|
22564
|
-
detection: unknown().optional()
|
|
22565
|
-
}).passthrough();
|
|
22566
|
-
var ZoneRulesProvider = class {
|
|
22567
|
-
ctx;
|
|
22568
|
-
/** Per-device per-stage cache. Hydrated lazily on first read. */
|
|
22569
|
-
cache = /* @__PURE__ */ new Map();
|
|
22570
|
-
/**
|
|
22571
|
-
* Per-device durable handle over the `zoneRules` store key. The WHOLE
|
|
22572
|
-
* `{motion?, detection?}` block round-trips on every read/write so a
|
|
22573
|
-
* write on one stage can never drop the other. Built lazily + memoised.
|
|
22574
|
-
*/
|
|
22575
|
-
stateByDevice = /* @__PURE__ */ new Map();
|
|
22576
|
-
constructor(ctx) {
|
|
22577
|
-
this.ctx = ctx;
|
|
22578
|
-
}
|
|
22579
|
-
/** Lazily build (and memoise) the durable `zoneRules` handle for a device. */
|
|
22580
|
-
rulesState(deviceId) {
|
|
22581
|
-
let handle = this.stateByDevice.get(deviceId);
|
|
22582
|
-
if (!handle) {
|
|
22583
|
-
handle = createDurableState({
|
|
22584
|
-
key: RULES_STORE_KEY,
|
|
22585
|
-
schema: ZoneRulesBlockSchema,
|
|
22586
|
-
fallback: {},
|
|
22587
|
-
read: () => this.ctx.settings.readDeviceStore(deviceId),
|
|
22588
|
-
write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
|
|
22589
|
-
onParseError: (key, error) => this.ctx.logger.warn("zone-rules store block failed to parse — resetting", {
|
|
22590
|
-
tags: { deviceId },
|
|
22591
|
-
meta: {
|
|
22592
|
-
key,
|
|
22593
|
-
error: error instanceof Error ? error.message : String(error)
|
|
22594
|
-
}
|
|
22595
|
-
})
|
|
22596
|
-
});
|
|
22597
|
-
this.stateByDevice.set(deviceId, handle);
|
|
22598
|
-
}
|
|
22599
|
-
return handle;
|
|
22600
|
-
}
|
|
22601
|
-
async listRules({ deviceId, stage }) {
|
|
22602
|
-
return this.loadRules(deviceId, stage);
|
|
22603
|
-
}
|
|
22604
|
-
async setRules({ deviceId, stage, rules }) {
|
|
22605
|
-
const parsed = RulesArraySchema.safeParse(rules);
|
|
22606
|
-
if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
|
|
22607
|
-
await this.persist(deviceId, stage, parsed.data);
|
|
22608
|
-
}
|
|
22609
|
-
/** Drop a device's cache entries. Called when the device is removed. */
|
|
22610
|
-
forgetDevice(deviceId) {
|
|
22611
|
-
this.cache.delete(deviceId);
|
|
22612
|
-
this.stateByDevice.delete(deviceId);
|
|
22613
|
-
}
|
|
22614
|
-
async loadRules(deviceId, stage) {
|
|
22615
|
-
let perDevice = this.cache.get(deviceId);
|
|
22616
|
-
if (!perDevice) {
|
|
22617
|
-
perDevice = /* @__PURE__ */ new Map();
|
|
22618
|
-
this.cache.set(deviceId, perDevice);
|
|
22619
|
-
}
|
|
22620
|
-
const cached = perDevice.get(stage);
|
|
22621
|
-
if (cached) return cached;
|
|
22622
|
-
let rules = [];
|
|
22623
|
-
try {
|
|
22624
|
-
const raw = (await this.rulesState(deviceId).get())[stage];
|
|
22625
|
-
if (raw !== void 0) {
|
|
22626
|
-
const parsed = RulesArraySchema.safeParse(raw);
|
|
22627
|
-
if (parsed.success) rules = parsed.data;
|
|
22628
|
-
else this.ctx.logger.warn("zone-rules store entry failed to parse — resetting", {
|
|
22629
|
-
tags: { deviceId },
|
|
22630
|
-
meta: {
|
|
22631
|
-
stage,
|
|
22632
|
-
issues: parsed.error.issues
|
|
22633
|
-
}
|
|
22634
|
-
});
|
|
22635
|
-
}
|
|
22636
|
-
} catch (err) {
|
|
22637
|
-
this.ctx.logger.warn("zone-rules store read failed — using empty list", {
|
|
22638
|
-
tags: { deviceId },
|
|
22639
|
-
meta: {
|
|
22640
|
-
stage,
|
|
22641
|
-
error: err instanceof Error ? err.message : String(err)
|
|
22642
|
-
}
|
|
22643
|
-
});
|
|
22644
|
-
}
|
|
22645
|
-
perDevice.set(stage, rules);
|
|
22646
|
-
return rules;
|
|
22647
|
-
}
|
|
22648
|
-
async persist(deviceId, stage, rules) {
|
|
22649
|
-
let perDevice = this.cache.get(deviceId);
|
|
22650
|
-
if (!perDevice) {
|
|
22651
|
-
perDevice = /* @__PURE__ */ new Map();
|
|
22652
|
-
this.cache.set(deviceId, perDevice);
|
|
22653
|
-
}
|
|
22654
|
-
perDevice.set(stage, rules);
|
|
22655
|
-
await this.rulesState(deviceId).update((prev) => ({
|
|
22656
|
-
...prev,
|
|
22657
|
-
[stage]: rules
|
|
22658
|
-
}));
|
|
22659
|
-
const otherStage = stage === "motion" ? "detection" : "motion";
|
|
22660
|
-
const otherRules = perDevice.get(otherStage) ?? await this.loadRules(deviceId, otherStage);
|
|
22661
|
-
const sliceValue = stage === "motion" ? {
|
|
22662
|
-
motion: rules,
|
|
22663
|
-
detection: otherRules
|
|
22664
|
-
} : {
|
|
22665
|
-
motion: otherRules,
|
|
22666
|
-
detection: rules
|
|
22667
|
-
};
|
|
22668
|
-
try {
|
|
22669
|
-
await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
|
|
22670
|
-
capName: ZONE_RULES_CAP_NAME,
|
|
22671
|
-
slice: sliceValue
|
|
22672
|
-
});
|
|
22673
|
-
} catch (err) {
|
|
22674
|
-
this.ctx.logger.debug("zone-rules slice mirror failed", {
|
|
22675
|
-
tags: { deviceId },
|
|
22676
|
-
meta: {
|
|
22677
|
-
stage,
|
|
22678
|
-
error: err instanceof Error ? err.message : String(err)
|
|
22679
|
-
}
|
|
22680
|
-
});
|
|
22681
|
-
}
|
|
22682
|
-
this.ctx.onRulesChanged?.(deviceId, stage, rules);
|
|
22683
|
-
}
|
|
22684
|
-
};
|
|
22685
|
-
//#endregion
|
|
22686
|
-
//#region src/orchestrator-store-schemas.ts
|
|
22687
|
-
/**
|
|
22688
|
-
* `orchestrator-store-schemas.ts` — whole-blob Zod schemas for the
|
|
22689
|
-
* orchestrator's addon-store keys that persist an ENTIRE collection
|
|
22690
|
-
* under a single key (`nodeBindings`, `templates`, `agentSettings`,
|
|
22691
|
-
* `cameraSettings`).
|
|
22692
|
-
*
|
|
22693
|
-
* These back the durable-state primitive (`this.state(KEY, Schema, {})`)
|
|
22694
|
-
* so a hand-written serializer can never silently drop a field on save:
|
|
22695
|
-
* the whole validated value round-trips on every read and write.
|
|
22696
|
-
*
|
|
22697
|
-
* Each schema mirrors the SAME shape the orchestrator already persisted
|
|
22698
|
-
* (the `pipeline-orchestrator.cap.ts` storage contract). Every field that
|
|
22699
|
-
* is only conditionally written is `.optional()`, so an existing blob
|
|
22700
|
-
* that predates a newer field still loads — durable-state `get()` only
|
|
22701
|
-
* falls back to the empty map on a whole-blob parse failure, so the
|
|
22702
|
-
* schema is kept faithful to the on-disk shape rather than overly strict.
|
|
22703
|
-
*/
|
|
22704
|
-
var EngineChoiceSchema = object({
|
|
22705
|
-
runtime: _enum(["node", "python"]),
|
|
22706
|
-
backend: string(),
|
|
22707
|
-
format: string(),
|
|
22708
|
-
device: string().optional()
|
|
22709
|
-
});
|
|
22710
|
-
var NodeBindingsSchema = record(string(), record(string(), string()));
|
|
22711
|
-
var StoredPipelineConfigSchema = object({
|
|
22712
|
-
engine: EngineChoiceSchema,
|
|
22713
|
-
steps: array(PipelineStepInputSchema).readonly(),
|
|
22714
|
-
audio: object({
|
|
22715
|
-
engine: EngineChoiceSchema,
|
|
22716
|
-
modelId: string(),
|
|
22717
|
-
enabled: boolean(),
|
|
22718
|
-
settings: record(string(), unknown()).readonly().optional()
|
|
22719
|
-
}).nullable().optional()
|
|
22720
|
-
});
|
|
22721
|
-
var StoredPipelineTemplateSchema = object({
|
|
22722
|
-
id: string(),
|
|
22723
|
-
name: string(),
|
|
22724
|
-
description: string().optional(),
|
|
22725
|
-
config: StoredPipelineConfigSchema,
|
|
22726
|
-
createdAt: string(),
|
|
22727
|
-
updatedAt: string()
|
|
22728
|
-
});
|
|
22729
|
-
var TemplatesSchema = record(string(), StoredPipelineTemplateSchema);
|
|
22730
|
-
var StoredAgentAddonConfigSchema = object({
|
|
22731
|
-
enabled: boolean(),
|
|
22732
|
-
modelId: string(),
|
|
22733
|
-
settings: record(string(), unknown()).readonly()
|
|
22734
|
-
});
|
|
22735
|
-
var StoredAgentPipelineSettingsSchema = object({
|
|
22736
|
-
addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
|
|
22737
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
22738
|
-
});
|
|
22739
|
-
var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
|
|
22740
|
-
var StoredCameraStepOverridePatchSchema = object({
|
|
22741
|
-
enabled: boolean().optional(),
|
|
22742
|
-
modelId: string().optional(),
|
|
22743
|
-
settings: record(string(), unknown()).readonly().optional()
|
|
22744
|
-
});
|
|
22745
|
-
var StoredCameraPipelineForAgentSchema = object({
|
|
22746
|
-
steps: array(PipelineStepInputSchema).readonly(),
|
|
22747
|
-
audio: object({
|
|
22748
|
-
modelId: string(),
|
|
22749
|
-
enabled: boolean()
|
|
22750
|
-
}).nullable()
|
|
22751
|
-
});
|
|
22752
|
-
var StoredCameraPipelineSettingsSchema = object({
|
|
22753
|
-
pinnedAgentNodeId: string().optional(),
|
|
22754
|
-
stepToggles: record(string(), boolean()).optional(),
|
|
22755
|
-
stepOverridesByAgent: record(string(), record(string(), StoredCameraStepOverridePatchSchema)).optional(),
|
|
22756
|
-
pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
|
|
22757
|
-
/**
|
|
22758
|
-
* Legacy "nuke inference" flag. Superseded by the detection-pipeline
|
|
22759
|
-
* wrapper binding, but the one-shot boot migration
|
|
22760
|
-
* (`migrateLegacyFlagsToBindings`) still reads it off the persisted
|
|
22761
|
-
* blob to flip the binding off. Kept here so the durable round-trip
|
|
22762
|
-
* does NOT strip it before the migration runs.
|
|
22763
|
-
*/
|
|
22764
|
-
disableInference: boolean().optional()
|
|
22765
|
-
});
|
|
22766
|
-
var CameraSettingsMapSchema = record(string(), StoredCameraPipelineSettingsSchema);
|
|
22767
|
-
//#endregion
|
|
22768
|
-
//#region src/load-balancer.ts
|
|
22666
|
+
/** Poll period — audio chunks arrive ~every 500ms; 200ms keeps latency low. */
|
|
22667
|
+
var POLL_INTERVAL_MS = 200;
|
|
22668
|
+
/** How many chunks to drain per poll — a small burst absorbs jitter. */
|
|
22669
|
+
var PULL_MAX_COUNT = 8;
|
|
22769
22670
|
/**
|
|
22770
|
-
*
|
|
22771
|
-
*
|
|
22772
|
-
*
|
|
22773
|
-
*
|
|
22774
|
-
* Rationale:
|
|
22775
|
-
* - `attachedCameras * avgInferenceFps` approximates the total inference rate
|
|
22776
|
-
* the agent is currently sustaining (not just how many cameras are assigned).
|
|
22777
|
-
* - `queueDepthTotal` penalises agents that are falling behind the frame feed.
|
|
22671
|
+
* Consecutive pull failures before we attempt to re-subscribe. A single failed
|
|
22672
|
+
* poll is treated as a blip (re-subscribing would leak a broker-side FIFO); a
|
|
22673
|
+
* sustained failure means the broker child restarted and dropped our
|
|
22674
|
+
* subscription, so we re-establish it.
|
|
22778
22675
|
*/
|
|
22779
|
-
|
|
22780
|
-
|
|
22781
|
-
|
|
22676
|
+
var RESUBSCRIBE_AFTER_FAILURES = 2;
|
|
22677
|
+
/** Re-subscribe at most once every N ticks while failing (~1s at 200ms). */
|
|
22678
|
+
var RESUBSCRIBE_THROTTLE_TICKS = 5;
|
|
22679
|
+
/** First subscribe-retry delay, doubled on every subsequent failure. */
|
|
22680
|
+
var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
|
|
22782
22681
|
/**
|
|
22783
|
-
*
|
|
22784
|
-
*
|
|
22785
|
-
*
|
|
22786
|
-
* Pins count toward the cap via `attachedCameras`.
|
|
22682
|
+
* Subscribe-retry backoff ceiling. 5 s matches the frame-handle poller — fast
|
|
22683
|
+
* enough to recover within a single reconcile of the orchestrator and slow
|
|
22684
|
+
* enough that a misconfigured camStream costs ~12 op/min, not 5/s.
|
|
22787
22685
|
*/
|
|
22788
|
-
|
|
22789
|
-
const cap = caps?.[node.nodeId];
|
|
22790
|
-
if (cap === null || cap === void 0 || cap <= 0) return true;
|
|
22791
|
-
return node.attachedCameras < cap;
|
|
22792
|
-
}
|
|
22686
|
+
var MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
|
|
22793
22687
|
/**
|
|
22794
|
-
*
|
|
22795
|
-
*
|
|
22796
|
-
* L1 (manual affinity): if `preferredAgent` names an online node AND that
|
|
22797
|
-
* node is under its `maxCameras` cap, return it. If the node is online but at
|
|
22798
|
-
* or over cap, return `{kind:'pending'}` — a pinned camera is never silently
|
|
22799
|
-
* over-assigned.
|
|
22800
|
-
*
|
|
22801
|
-
* L2 (capacity): filter to eligible nodes and pick the lowest capacity score.
|
|
22802
|
-
* If all nodes are at/over cap, return `{kind:'pending'}`.
|
|
22688
|
+
* Subscribe to a broker's decoded audio-chunk stream and start the poll loop.
|
|
22803
22689
|
*
|
|
22804
|
-
*
|
|
22805
|
-
*
|
|
22806
|
-
*
|
|
22690
|
+
* Always resolves to a teardown closure — when the broker is not yet
|
|
22691
|
+
* registered the closure cancels the ongoing retry loop; when polling is
|
|
22692
|
+
* active it stops the loop and releases the broker subscription. Mirrors
|
|
22693
|
+
* `startFrameHandlePoller` so video and audio recover identically.
|
|
22807
22694
|
*/
|
|
22808
|
-
function
|
|
22809
|
-
const
|
|
22810
|
-
|
|
22811
|
-
|
|
22812
|
-
|
|
22813
|
-
|
|
22814
|
-
if (pinnedOnline) {
|
|
22815
|
-
if (eligible.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
|
|
22816
|
-
kind: "assigned",
|
|
22817
|
-
agentNodeId: pinnedOnline.nodeId,
|
|
22818
|
-
reason: "manual",
|
|
22819
|
-
score: computeCapacityScore(pinnedOnline)
|
|
22820
|
-
};
|
|
22821
|
-
return {
|
|
22822
|
-
kind: "pending",
|
|
22823
|
-
reason: "over-cap"
|
|
22824
|
-
};
|
|
22825
|
-
}
|
|
22826
|
-
}
|
|
22827
|
-
if (eligible.length === 0) return {
|
|
22828
|
-
kind: "pending",
|
|
22829
|
-
reason: "over-cap"
|
|
22695
|
+
function startAudioChunkPoller(options) {
|
|
22696
|
+
const lifecycle = {
|
|
22697
|
+
stopped: false,
|
|
22698
|
+
retryTimer: void 0,
|
|
22699
|
+
pollTimer: void 0,
|
|
22700
|
+
activeSubscriptionId: null
|
|
22830
22701
|
};
|
|
22831
|
-
const
|
|
22832
|
-
|
|
22833
|
-
|
|
22834
|
-
|
|
22835
|
-
|
|
22836
|
-
|
|
22837
|
-
|
|
22838
|
-
|
|
22839
|
-
|
|
22702
|
+
const teardown = () => {
|
|
22703
|
+
if (lifecycle.stopped) return;
|
|
22704
|
+
lifecycle.stopped = true;
|
|
22705
|
+
if (lifecycle.retryTimer) {
|
|
22706
|
+
clearTimeout(lifecycle.retryTimer);
|
|
22707
|
+
lifecycle.retryTimer = void 0;
|
|
22708
|
+
}
|
|
22709
|
+
if (lifecycle.pollTimer) {
|
|
22710
|
+
clearTimeout(lifecycle.pollTimer);
|
|
22711
|
+
lifecycle.pollTimer = void 0;
|
|
22712
|
+
}
|
|
22713
|
+
const subId = lifecycle.activeSubscriptionId;
|
|
22714
|
+
if (subId) {
|
|
22715
|
+
lifecycle.activeSubscriptionId = null;
|
|
22716
|
+
options.api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: subId }).catch((err) => {
|
|
22717
|
+
options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
|
|
22718
|
+
brokerId: options.brokerId,
|
|
22719
|
+
subscriptionId: subId,
|
|
22720
|
+
error: errMsg(err)
|
|
22721
|
+
} });
|
|
22722
|
+
});
|
|
22723
|
+
}
|
|
22840
22724
|
};
|
|
22725
|
+
subscribeWithRetry(options, lifecycle);
|
|
22726
|
+
return teardown;
|
|
22841
22727
|
}
|
|
22842
22728
|
/**
|
|
22843
|
-
*
|
|
22844
|
-
*
|
|
22845
|
-
*
|
|
22846
|
-
* that is how video decode reaches a node that is not eligible to decode (e.g.
|
|
22847
|
-
* a node whose shm frame ring the broker cannot read, or one an operator
|
|
22848
|
-
* disabled). A pin to an ENABLED node is honored; anything else falls through
|
|
22849
|
-
* to the auto-balance path (which filters by `enabledDecoderNodes`).
|
|
22729
|
+
* Run the subscribe → poll handshake with exponential backoff on subscribe
|
|
22730
|
+
* failures. Resolves once the subscription is acquired (and the poll loop has
|
|
22731
|
+
* been started) or once `lifecycle.stopped` flips, whichever comes first.
|
|
22850
22732
|
*/
|
|
22851
|
-
function
|
|
22852
|
-
|
|
22733
|
+
async function subscribeWithRetry(options, lifecycle) {
|
|
22734
|
+
const { api, brokerId, tag, logger } = options;
|
|
22735
|
+
let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
|
|
22736
|
+
let attempt = 0;
|
|
22737
|
+
while (!lifecycle.stopped) {
|
|
22738
|
+
attempt += 1;
|
|
22739
|
+
try {
|
|
22740
|
+
const result = await api.streamBroker.subscribeAudioChunks.mutate({
|
|
22741
|
+
brokerId,
|
|
22742
|
+
tag
|
|
22743
|
+
});
|
|
22744
|
+
if (lifecycle.stopped) {
|
|
22745
|
+
await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
|
|
22746
|
+
logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
|
|
22747
|
+
brokerId,
|
|
22748
|
+
subscriptionId: result.subscriptionId,
|
|
22749
|
+
error: errMsg(err)
|
|
22750
|
+
} });
|
|
22751
|
+
});
|
|
22752
|
+
return;
|
|
22753
|
+
}
|
|
22754
|
+
if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
|
|
22755
|
+
brokerId,
|
|
22756
|
+
tag,
|
|
22757
|
+
attempt
|
|
22758
|
+
} });
|
|
22759
|
+
lifecycle.activeSubscriptionId = result.subscriptionId;
|
|
22760
|
+
startPolling(options, lifecycle);
|
|
22761
|
+
return;
|
|
22762
|
+
} catch (err) {
|
|
22763
|
+
if (lifecycle.stopped) return;
|
|
22764
|
+
if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
|
|
22765
|
+
brokerId,
|
|
22766
|
+
tag,
|
|
22767
|
+
error: errMsg(err),
|
|
22768
|
+
nextRetryInMs: backoffMs
|
|
22769
|
+
} });
|
|
22770
|
+
else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
|
|
22771
|
+
brokerId,
|
|
22772
|
+
tag,
|
|
22773
|
+
attempt,
|
|
22774
|
+
error: errMsg(err),
|
|
22775
|
+
nextRetryInMs: backoffMs
|
|
22776
|
+
} });
|
|
22777
|
+
await sleep(backoffMs, lifecycle);
|
|
22778
|
+
backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
|
|
22779
|
+
}
|
|
22780
|
+
}
|
|
22853
22781
|
}
|
|
22854
22782
|
/**
|
|
22855
|
-
*
|
|
22783
|
+
* Drive the steady-state poll loop. Mid-stream `pullAudioChunks` failures
|
|
22784
|
+
* trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
|
|
22785
|
+
* the broker child restart case where our `subscriptionId` is silently
|
|
22786
|
+
* disowned.
|
|
22856
22787
|
*/
|
|
22857
|
-
function
|
|
22858
|
-
const {
|
|
22859
|
-
|
|
22860
|
-
|
|
22861
|
-
|
|
22862
|
-
|
|
22863
|
-
|
|
22864
|
-
|
|
22865
|
-
|
|
22866
|
-
|
|
22867
|
-
|
|
22868
|
-
|
|
22869
|
-
|
|
22870
|
-
|
|
22871
|
-
|
|
22872
|
-
|
|
22788
|
+
function startPolling(options, lifecycle) {
|
|
22789
|
+
const { api, brokerId, tag, onChunk, logger } = options;
|
|
22790
|
+
let consecutiveFailures = 0;
|
|
22791
|
+
const resubscribe = async () => {
|
|
22792
|
+
try {
|
|
22793
|
+
const result = await api.streamBroker.subscribeAudioChunks.mutate({
|
|
22794
|
+
brokerId,
|
|
22795
|
+
tag
|
|
22796
|
+
});
|
|
22797
|
+
lifecycle.activeSubscriptionId = result.subscriptionId;
|
|
22798
|
+
logger.info("audio-chunk poller: re-subscribed after broker outage", { meta: {
|
|
22799
|
+
brokerId,
|
|
22800
|
+
tag,
|
|
22801
|
+
subscriptionId: result.subscriptionId,
|
|
22802
|
+
afterFailures: consecutiveFailures
|
|
22803
|
+
} });
|
|
22804
|
+
return true;
|
|
22805
|
+
} catch {
|
|
22806
|
+
return false;
|
|
22807
|
+
}
|
|
22873
22808
|
};
|
|
22874
|
-
const
|
|
22875
|
-
|
|
22876
|
-
|
|
22877
|
-
|
|
22878
|
-
|
|
22879
|
-
|
|
22880
|
-
|
|
22881
|
-
|
|
22809
|
+
const tick = async () => {
|
|
22810
|
+
if (lifecycle.stopped) return;
|
|
22811
|
+
const subId = lifecycle.activeSubscriptionId;
|
|
22812
|
+
if (!subId) return;
|
|
22813
|
+
try {
|
|
22814
|
+
const chunks = await api.streamBroker.pullAudioChunks.query({
|
|
22815
|
+
subscriptionId: subId,
|
|
22816
|
+
maxCount: PULL_MAX_COUNT
|
|
22817
|
+
});
|
|
22818
|
+
if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
|
|
22819
|
+
brokerId,
|
|
22820
|
+
subscriptionId: subId
|
|
22821
|
+
} });
|
|
22822
|
+
consecutiveFailures = 0;
|
|
22823
|
+
for (const chunk of chunks) {
|
|
22824
|
+
if (lifecycle.stopped) break;
|
|
22825
|
+
await onChunk(chunk);
|
|
22826
|
+
}
|
|
22827
|
+
} catch (err) {
|
|
22828
|
+
consecutiveFailures += 1;
|
|
22829
|
+
if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
|
|
22830
|
+
brokerId,
|
|
22831
|
+
subscriptionId: subId,
|
|
22832
|
+
error: errMsg(err)
|
|
22833
|
+
} });
|
|
22834
|
+
if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
|
|
22835
|
+
}
|
|
22836
|
+
if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
|
|
22882
22837
|
};
|
|
22838
|
+
tick();
|
|
22839
|
+
}
|
|
22840
|
+
/**
|
|
22841
|
+
* Cancellable sleep — wakes early when `lifecycle.stopped` flips. We
|
|
22842
|
+
* keep a local wrapper around the shared {@link sleep} helper because
|
|
22843
|
+
* the lifecycle tracks the active retry timer for `teardown()` to
|
|
22844
|
+
* clear; pure `sleep()` would leak the timer if teardown fired while
|
|
22845
|
+
* we were waiting.
|
|
22846
|
+
*/
|
|
22847
|
+
function sleep(ms, lifecycle) {
|
|
22848
|
+
return new Promise((resolve) => {
|
|
22849
|
+
if (lifecycle.stopped) {
|
|
22850
|
+
resolve();
|
|
22851
|
+
return;
|
|
22852
|
+
}
|
|
22853
|
+
lifecycle.retryTimer = setTimeout(() => {
|
|
22854
|
+
lifecycle.retryTimer = void 0;
|
|
22855
|
+
resolve();
|
|
22856
|
+
}, ms);
|
|
22857
|
+
});
|
|
22883
22858
|
}
|
|
22884
22859
|
//#endregion
|
|
22885
22860
|
//#region src/audio-load-balancer.ts
|
|
@@ -22898,259 +22873,412 @@ function balanceAudio(input) {
|
|
|
22898
22873
|
};
|
|
22899
22874
|
}
|
|
22900
22875
|
//#endregion
|
|
22901
|
-
//#region src/
|
|
22876
|
+
//#region src/camera-status/compose-camera-status.ts
|
|
22877
|
+
function mapAssignment(input) {
|
|
22878
|
+
return {
|
|
22879
|
+
detectionNodeId: input.detectionNodeId,
|
|
22880
|
+
decoderNodeId: input.decoderNodeId,
|
|
22881
|
+
audioNodeId: input.audioNodeId,
|
|
22882
|
+
pinned: {
|
|
22883
|
+
detection: input.pinned.detection,
|
|
22884
|
+
decoder: input.pinned.decoder,
|
|
22885
|
+
audio: input.pinned.audio
|
|
22886
|
+
},
|
|
22887
|
+
reasons: {
|
|
22888
|
+
detection: input.reasons.detection,
|
|
22889
|
+
decoder: input.reasons.decoder,
|
|
22890
|
+
audio: input.reasons.audio
|
|
22891
|
+
}
|
|
22892
|
+
};
|
|
22893
|
+
}
|
|
22894
|
+
function mapSource(sourceResult) {
|
|
22895
|
+
if (sourceResult === null) return { streams: [] };
|
|
22896
|
+
return { streams: sourceResult.streams.map((s) => ({
|
|
22897
|
+
camStreamId: s.camStreamId,
|
|
22898
|
+
codec: s.codec,
|
|
22899
|
+
width: s.width,
|
|
22900
|
+
height: s.height,
|
|
22901
|
+
fps: s.fps,
|
|
22902
|
+
kind: s.kind
|
|
22903
|
+
})) };
|
|
22904
|
+
}
|
|
22905
|
+
function mapBroker(brokerResult) {
|
|
22906
|
+
if (brokerResult === null) return null;
|
|
22907
|
+
return {
|
|
22908
|
+
profiles: brokerResult.profiles.map((p) => ({
|
|
22909
|
+
profile: p.profile,
|
|
22910
|
+
status: p.status,
|
|
22911
|
+
codec: p.codec,
|
|
22912
|
+
width: p.width,
|
|
22913
|
+
height: p.height,
|
|
22914
|
+
subscribers: p.subscribers,
|
|
22915
|
+
inFps: p.inFps,
|
|
22916
|
+
outFps: p.outFps
|
|
22917
|
+
})),
|
|
22918
|
+
webrtcSessions: brokerResult.webrtcSessions,
|
|
22919
|
+
rtspRestream: brokerResult.rtspRestream
|
|
22920
|
+
};
|
|
22921
|
+
}
|
|
22922
|
+
function mapDecoderShm(shm) {
|
|
22923
|
+
return {
|
|
22924
|
+
framesWritten: shm.framesWritten,
|
|
22925
|
+
getFrameHits: shm.getFrameHits,
|
|
22926
|
+
getFrameMisses: shm.getFrameMisses,
|
|
22927
|
+
budgetMb: shm.budgetMb
|
|
22928
|
+
};
|
|
22929
|
+
}
|
|
22930
|
+
function mapDecoder(decoderResult) {
|
|
22931
|
+
if (decoderResult === null) return null;
|
|
22932
|
+
return {
|
|
22933
|
+
nodeId: decoderResult.nodeId,
|
|
22934
|
+
formats: [...decoderResult.formats],
|
|
22935
|
+
sessionCount: decoderResult.sessionCount,
|
|
22936
|
+
shm: mapDecoderShm(decoderResult.shm)
|
|
22937
|
+
};
|
|
22938
|
+
}
|
|
22939
|
+
function mapMotion(motionResult) {
|
|
22940
|
+
if (motionResult === null) return null;
|
|
22941
|
+
return {
|
|
22942
|
+
enabled: motionResult.enabled,
|
|
22943
|
+
fps: motionResult.fps
|
|
22944
|
+
};
|
|
22945
|
+
}
|
|
22946
|
+
function mapProvisioning(p) {
|
|
22947
|
+
if (p.error !== void 0) return {
|
|
22948
|
+
state: p.state,
|
|
22949
|
+
error: p.error
|
|
22950
|
+
};
|
|
22951
|
+
return { state: p.state };
|
|
22952
|
+
}
|
|
22953
|
+
function mapDetection(detectionResult) {
|
|
22954
|
+
if (detectionResult === null) return null;
|
|
22955
|
+
const phase = detectionResult.phase;
|
|
22956
|
+
return {
|
|
22957
|
+
nodeId: detectionResult.nodeId,
|
|
22958
|
+
engine: {
|
|
22959
|
+
backend: detectionResult.engine.backend,
|
|
22960
|
+
device: detectionResult.engine.device
|
|
22961
|
+
},
|
|
22962
|
+
phase,
|
|
22963
|
+
configuredFps: detectionResult.configuredFps,
|
|
22964
|
+
actualFps: detectionResult.actualFps,
|
|
22965
|
+
queueDepth: detectionResult.queueDepth,
|
|
22966
|
+
avgInferenceMs: detectionResult.avgInferenceMs,
|
|
22967
|
+
provisioning: mapProvisioning(detectionResult.provisioning)
|
|
22968
|
+
};
|
|
22969
|
+
}
|
|
22970
|
+
function mapAudio(audioResult) {
|
|
22971
|
+
if (audioResult === null) return null;
|
|
22972
|
+
return {
|
|
22973
|
+
nodeId: audioResult.nodeId,
|
|
22974
|
+
enabled: audioResult.enabled
|
|
22975
|
+
};
|
|
22976
|
+
}
|
|
22977
|
+
function mapRecording(recordingResult) {
|
|
22978
|
+
if (recordingResult === null) return null;
|
|
22979
|
+
return {
|
|
22980
|
+
mode: recordingResult.mode,
|
|
22981
|
+
active: recordingResult.active,
|
|
22982
|
+
storageBytes: recordingResult.storageBytes
|
|
22983
|
+
};
|
|
22984
|
+
}
|
|
22902
22985
|
/**
|
|
22903
|
-
*
|
|
22904
|
-
* plane (Phase 5 / D9).
|
|
22986
|
+
* Pure function that composes a `CameraStatus` from per-stage fetch results.
|
|
22905
22987
|
*
|
|
22906
|
-
*
|
|
22907
|
-
*
|
|
22908
|
-
*
|
|
22909
|
-
*
|
|
22988
|
+
* - `assignment` is always built from orchestrator-local data (never null).
|
|
22989
|
+
* - `source` always present: defaults to `{ streams: [] }` when sourceResult is null.
|
|
22990
|
+
* - Every other block is null when its stage result is null (graceful degradation).
|
|
22991
|
+
* - `fetchedAt` is stamped exactly as provided — never calls `Date.now()`.
|
|
22992
|
+
* - No mutation of the input.
|
|
22993
|
+
*/
|
|
22994
|
+
function composeCameraStatus(input) {
|
|
22995
|
+
return {
|
|
22996
|
+
deviceId: input.deviceId,
|
|
22997
|
+
assignment: mapAssignment(input),
|
|
22998
|
+
source: mapSource(input.sourceResult),
|
|
22999
|
+
broker: mapBroker(input.brokerResult),
|
|
23000
|
+
decoder: mapDecoder(input.decoderResult),
|
|
23001
|
+
motion: mapMotion(input.motionResult),
|
|
23002
|
+
detection: mapDetection(input.detectionResult),
|
|
23003
|
+
audio: mapAudio(input.audioResult),
|
|
23004
|
+
recording: mapRecording(input.recordingResult),
|
|
23005
|
+
fetchedAt: input.fetchedAt
|
|
23006
|
+
};
|
|
23007
|
+
}
|
|
23008
|
+
//#endregion
|
|
23009
|
+
//#region src/dispatch-reconcile.ts
|
|
23010
|
+
/**
|
|
23011
|
+
* Derives the set of deviceIds that currently have at least one broker
|
|
23012
|
+
* profile slot that is assigned (status !== 'unassigned') AND has a
|
|
23013
|
+
* non-null sourceCamStreamId.
|
|
22910
23014
|
*
|
|
22911
|
-
*
|
|
23015
|
+
* This is the "desired" fleet — cameras the orchestrator should have
|
|
23016
|
+
* dispatched. Used by `reconcileDispatch` to compute the gap against
|
|
23017
|
+
* `cameraConfigs`.
|
|
23018
|
+
*/
|
|
23019
|
+
function desiredDeviceIdsFromSlots(slots) {
|
|
23020
|
+
const result = /* @__PURE__ */ new Set();
|
|
23021
|
+
for (const slot of slots) if (slot.status !== "unassigned" && slot.sourceCamStreamId !== null) result.add(slot.deviceId);
|
|
23022
|
+
return result;
|
|
23023
|
+
}
|
|
23024
|
+
/**
|
|
23025
|
+
* Returns the deviceIds that are present in `desired` but absent from
|
|
23026
|
+
* `known`. These are the cameras the orchestrator has not yet dispatched
|
|
23027
|
+
* and needs to process via `handleDeviceRegistered`.
|
|
22912
23028
|
*
|
|
22913
|
-
*
|
|
22914
|
-
*
|
|
22915
|
-
|
|
22916
|
-
|
|
22917
|
-
|
|
22918
|
-
|
|
22919
|
-
|
|
23029
|
+
* Additive only — cameras present in `known` are never included, even
|
|
23030
|
+
* if they fall out of `desired`, to avoid fighting the live event path.
|
|
23031
|
+
*/
|
|
23032
|
+
function computeDispatchGap(desired, known) {
|
|
23033
|
+
return [...desired].filter((id) => !known.has(id));
|
|
23034
|
+
}
|
|
23035
|
+
//#endregion
|
|
23036
|
+
//#region src/frame-source-eligibility.ts
|
|
23037
|
+
/**
|
|
23038
|
+
* The set of nodes that can obtain a given camera's decoded frames.
|
|
22920
23039
|
*
|
|
22921
|
-
*
|
|
22922
|
-
*
|
|
22923
|
-
*
|
|
22924
|
-
*
|
|
22925
|
-
* loses a chunk.
|
|
23040
|
+
* - No `sourceOwnerNodeId` → `enabledDecoderNodes` unchanged (today's
|
|
23041
|
+
* semantics; `remoteSourcingNodes` cannot restrict anything when no owner
|
|
23042
|
+
* restricts first — it only ever WIDENS an owner-restricted set).
|
|
23043
|
+
* - With an owner → `enabledDecoderNodes ∩ ({owner} ∪ remoteSourcingNodes)`.
|
|
22926
23044
|
*
|
|
22927
|
-
*
|
|
22928
|
-
*
|
|
22929
|
-
* their cameraStreams asynchronously after their probe completes).
|
|
22930
|
-
* `subscribeAudioChunks` retries with exponential backoff (capped at
|
|
22931
|
-
* `MAX_SUBSCRIBE_RETRY_BACKOFF_MS`) until it succeeds or the returned
|
|
22932
|
-
* teardown closure is invoked. Mirrors the `FrameHandlePoller` recovery
|
|
22933
|
-
* shape so video and audio plumbing self-heal identically.
|
|
23045
|
+
* Pure: never mutates its inputs; result preserves `enabledDecoderNodes`
|
|
23046
|
+
* order.
|
|
22934
23047
|
*/
|
|
22935
|
-
|
|
22936
|
-
|
|
22937
|
-
|
|
22938
|
-
|
|
23048
|
+
function computeFrameSourceNodes(input) {
|
|
23049
|
+
const { enabledDecoderNodes, sourceOwnerNodeId, remoteSourcingNodes } = input;
|
|
23050
|
+
if (sourceOwnerNodeId === void 0) return enabledDecoderNodes;
|
|
23051
|
+
const remote = remoteSourcingNodes ?? [];
|
|
23052
|
+
return enabledDecoderNodes.filter((nodeId) => nodeId === sourceOwnerNodeId || remote.includes(nodeId));
|
|
23053
|
+
}
|
|
23054
|
+
//#endregion
|
|
23055
|
+
//#region src/load-balancer.ts
|
|
22939
23056
|
/**
|
|
22940
|
-
*
|
|
22941
|
-
*
|
|
22942
|
-
*
|
|
22943
|
-
*
|
|
23057
|
+
* Compute the L2 capacity score for a runner node. Lower is better.
|
|
23058
|
+
* The score is a weighted sum of the runner's active workload so the balancer
|
|
23059
|
+
* prefers agents that are serving fewer cameras OR draining queues quickly.
|
|
23060
|
+
*
|
|
23061
|
+
* Rationale:
|
|
23062
|
+
* - `attachedCameras * avgInferenceFps` approximates the total inference rate
|
|
23063
|
+
* the agent is currently sustaining (not just how many cameras are assigned).
|
|
23064
|
+
* - `queueDepthTotal` penalises agents that are falling behind the frame feed.
|
|
22944
23065
|
*/
|
|
22945
|
-
|
|
22946
|
-
|
|
22947
|
-
|
|
22948
|
-
/** First subscribe-retry delay, doubled on every subsequent failure. */
|
|
22949
|
-
var INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
|
|
23066
|
+
function computeCapacityScore(load) {
|
|
23067
|
+
return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
|
|
23068
|
+
}
|
|
22950
23069
|
/**
|
|
22951
|
-
*
|
|
22952
|
-
*
|
|
22953
|
-
*
|
|
23070
|
+
* Returns true when the node has remaining capacity.
|
|
23071
|
+
* A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
|
|
23072
|
+
* its `attachedCameras` count is strictly less than the cap.
|
|
23073
|
+
* Pins count toward the cap via `attachedCameras`.
|
|
22954
23074
|
*/
|
|
22955
|
-
|
|
23075
|
+
function isEligible(node, caps) {
|
|
23076
|
+
const cap = caps?.[node.nodeId];
|
|
23077
|
+
if (cap === null || cap === void 0 || cap <= 0) return true;
|
|
23078
|
+
return node.attachedCameras < cap;
|
|
23079
|
+
}
|
|
22956
23080
|
/**
|
|
22957
|
-
*
|
|
23081
|
+
* Run the two-level camera balancer.
|
|
22958
23082
|
*
|
|
22959
|
-
*
|
|
22960
|
-
*
|
|
22961
|
-
*
|
|
22962
|
-
*
|
|
23083
|
+
* Frame-source constraint: when `eligibleNodes` is set, only nodes in that set
|
|
23084
|
+
* can obtain this camera's decoded frames. Such a node is a prerequisite for
|
|
23085
|
+
* BOTH L1 and L2 — a node outside `eligibleNodes` is never assignable, and a
|
|
23086
|
+
* pin to an online-but-ineligible node returns `{kind:'pending',
|
|
23087
|
+
* reason:'no-frame-source'}` (mirrors the over-cap-pin behaviour: a pinned
|
|
23088
|
+
* camera is never silently placed on a different node).
|
|
23089
|
+
*
|
|
23090
|
+
* L1 (manual affinity): if `preferredAgent` names an online + frame-sourceable
|
|
23091
|
+
* node AND that node is under its `maxCameras` cap, return it. If the node is
|
|
23092
|
+
* online but at/over cap, return `{kind:'pending', reason:'over-cap'}`; if it
|
|
23093
|
+
* is online but not frame-sourceable, `{kind:'pending', reason:'no-frame-source'}`
|
|
23094
|
+
* — a pinned camera is never silently over-assigned or re-homed.
|
|
23095
|
+
*
|
|
23096
|
+
* L2 (capacity): among the frame-sourceable nodes, filter to those under cap
|
|
23097
|
+
* and pick the lowest capacity score. If no frame-sourceable node exists (but
|
|
23098
|
+
* online nodes do), return `no-frame-source`; if they exist but are all at/over
|
|
23099
|
+
* cap, return `over-cap`.
|
|
23100
|
+
*
|
|
23101
|
+
* Returns `null` when no runners are online. The orchestrator decides how to
|
|
23102
|
+
* react — typically by logging and deferring the assignment until a runner
|
|
23103
|
+
* comes online.
|
|
22963
23104
|
*/
|
|
22964
|
-
function
|
|
22965
|
-
const
|
|
22966
|
-
|
|
22967
|
-
|
|
22968
|
-
|
|
22969
|
-
|
|
22970
|
-
|
|
22971
|
-
|
|
22972
|
-
|
|
22973
|
-
|
|
22974
|
-
|
|
22975
|
-
|
|
22976
|
-
|
|
22977
|
-
|
|
22978
|
-
|
|
22979
|
-
|
|
22980
|
-
|
|
22981
|
-
|
|
22982
|
-
|
|
22983
|
-
|
|
22984
|
-
|
|
22985
|
-
|
|
22986
|
-
options.logger.warn("audio-chunk poller: unsubscribeAudioChunks failed", { meta: {
|
|
22987
|
-
brokerId: options.brokerId,
|
|
22988
|
-
subscriptionId: subId,
|
|
22989
|
-
error: errMsg(err)
|
|
22990
|
-
} });
|
|
22991
|
-
});
|
|
23105
|
+
function balance(input) {
|
|
23106
|
+
const online = input.nodes.filter((n) => n.nodeId.length > 0);
|
|
23107
|
+
if (online.length === 0) return null;
|
|
23108
|
+
const sourceable = input.eligibleNodes ? online.filter((n) => input.eligibleNodes.includes(n.nodeId)) : online;
|
|
23109
|
+
const eligible = sourceable.filter((n) => isEligible(n, input.nodeCaps));
|
|
23110
|
+
if (input.preferredAgent) {
|
|
23111
|
+
const pinnedOnline = online.find((n) => n.nodeId === input.preferredAgent);
|
|
23112
|
+
if (pinnedOnline) {
|
|
23113
|
+
if (!sourceable.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
|
|
23114
|
+
kind: "pending",
|
|
23115
|
+
reason: "no-frame-source"
|
|
23116
|
+
};
|
|
23117
|
+
if (eligible.some((n) => n.nodeId === pinnedOnline.nodeId)) return {
|
|
23118
|
+
kind: "assigned",
|
|
23119
|
+
agentNodeId: pinnedOnline.nodeId,
|
|
23120
|
+
reason: "manual",
|
|
23121
|
+
score: computeCapacityScore(pinnedOnline)
|
|
23122
|
+
};
|
|
23123
|
+
return {
|
|
23124
|
+
kind: "pending",
|
|
23125
|
+
reason: "over-cap"
|
|
23126
|
+
};
|
|
22992
23127
|
}
|
|
23128
|
+
}
|
|
23129
|
+
if (eligible.length === 0) return {
|
|
23130
|
+
kind: "pending",
|
|
23131
|
+
reason: sourceable.length === 0 ? "no-frame-source" : "over-cap"
|
|
23132
|
+
};
|
|
23133
|
+
const best = eligible.map((node) => ({
|
|
23134
|
+
node,
|
|
23135
|
+
score: computeCapacityScore(node)
|
|
23136
|
+
})).toSorted((a, b) => a.score - b.score)[0];
|
|
23137
|
+
return {
|
|
23138
|
+
kind: "assigned",
|
|
23139
|
+
agentNodeId: best.node.nodeId,
|
|
23140
|
+
reason: "capacity",
|
|
23141
|
+
score: best.score
|
|
22993
23142
|
};
|
|
22994
|
-
subscribeWithRetry(options, lifecycle);
|
|
22995
|
-
return teardown;
|
|
22996
23143
|
}
|
|
22997
23144
|
/**
|
|
22998
|
-
*
|
|
22999
|
-
*
|
|
23000
|
-
*
|
|
23145
|
+
* Decide whether a manual per-device decoder pin may be honored.
|
|
23146
|
+
*
|
|
23147
|
+
* A pin to a node OUTSIDE `enabledDecoderNodes` must NOT be honored blindly —
|
|
23148
|
+
* that is how video decode reaches a node that is not eligible to decode (e.g.
|
|
23149
|
+
* a node whose shm frame ring the broker cannot read, or one an operator
|
|
23150
|
+
* disabled). A pin to an ENABLED node is honored; anything else falls through
|
|
23151
|
+
* to the auto-balance path (which filters by `enabledDecoderNodes`).
|
|
23001
23152
|
*/
|
|
23002
|
-
|
|
23003
|
-
|
|
23004
|
-
let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
|
|
23005
|
-
let attempt = 0;
|
|
23006
|
-
while (!lifecycle.stopped) {
|
|
23007
|
-
attempt += 1;
|
|
23008
|
-
try {
|
|
23009
|
-
const result = await api.streamBroker.subscribeAudioChunks.mutate({
|
|
23010
|
-
brokerId,
|
|
23011
|
-
tag
|
|
23012
|
-
});
|
|
23013
|
-
if (lifecycle.stopped) {
|
|
23014
|
-
await api.streamBroker.unsubscribeAudioChunks.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
|
|
23015
|
-
logger.warn("audio-chunk poller: late unsubscribe failed", { meta: {
|
|
23016
|
-
brokerId,
|
|
23017
|
-
subscriptionId: result.subscriptionId,
|
|
23018
|
-
error: errMsg(err)
|
|
23019
|
-
} });
|
|
23020
|
-
});
|
|
23021
|
-
return;
|
|
23022
|
-
}
|
|
23023
|
-
if (attempt > 1) logger.info("audio-chunk poller: subscribeAudioChunks recovered", { meta: {
|
|
23024
|
-
brokerId,
|
|
23025
|
-
tag,
|
|
23026
|
-
attempt
|
|
23027
|
-
} });
|
|
23028
|
-
lifecycle.activeSubscriptionId = result.subscriptionId;
|
|
23029
|
-
startPolling(options, lifecycle);
|
|
23030
|
-
return;
|
|
23031
|
-
} catch (err) {
|
|
23032
|
-
if (lifecycle.stopped) return;
|
|
23033
|
-
if (attempt === 1) logger.warn("audio-chunk poller: subscribeAudioChunks failed, retrying", { meta: {
|
|
23034
|
-
brokerId,
|
|
23035
|
-
tag,
|
|
23036
|
-
error: errMsg(err),
|
|
23037
|
-
nextRetryInMs: backoffMs
|
|
23038
|
-
} });
|
|
23039
|
-
else logger.debug("audio-chunk poller: subscribeAudioChunks still failing", { meta: {
|
|
23040
|
-
brokerId,
|
|
23041
|
-
tag,
|
|
23042
|
-
attempt,
|
|
23043
|
-
error: errMsg(err),
|
|
23044
|
-
nextRetryInMs: backoffMs
|
|
23045
|
-
} });
|
|
23046
|
-
await sleep(backoffMs, lifecycle);
|
|
23047
|
-
backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
|
|
23048
|
-
}
|
|
23049
|
-
}
|
|
23153
|
+
function isDecoderPinHonored(pinnedNodeId, enabledDecoderNodes) {
|
|
23154
|
+
return enabledDecoderNodes.includes(pinnedNodeId);
|
|
23050
23155
|
}
|
|
23051
23156
|
/**
|
|
23052
|
-
*
|
|
23053
|
-
* trigger a fresh `subscribeAudioChunks` via the recovery branch — covers
|
|
23054
|
-
* the broker child restart case where our `subscriptionId` is silently
|
|
23055
|
-
* disowned.
|
|
23157
|
+
* Choose decoder node. Priority: manual pin → co-located with pipeline → capacity.
|
|
23056
23158
|
*/
|
|
23057
|
-
function
|
|
23058
|
-
const {
|
|
23059
|
-
|
|
23060
|
-
|
|
23061
|
-
|
|
23062
|
-
|
|
23063
|
-
|
|
23064
|
-
|
|
23065
|
-
|
|
23066
|
-
|
|
23067
|
-
|
|
23068
|
-
|
|
23069
|
-
|
|
23070
|
-
|
|
23071
|
-
|
|
23072
|
-
|
|
23073
|
-
return true;
|
|
23074
|
-
} catch {
|
|
23075
|
-
return false;
|
|
23076
|
-
}
|
|
23159
|
+
function balanceDecoder(input) {
|
|
23160
|
+
const { decoderNodes, pipelineNodeId, preferredDecoderNode } = input;
|
|
23161
|
+
if (decoderNodes.length === 0) return null;
|
|
23162
|
+
if (preferredDecoderNode) {
|
|
23163
|
+
const match = decoderNodes.find((n) => n.nodeId === preferredDecoderNode);
|
|
23164
|
+
if (match) return {
|
|
23165
|
+
decoderNodeId: match.nodeId,
|
|
23166
|
+
reason: "manual",
|
|
23167
|
+
score: computeCapacityScore(match)
|
|
23168
|
+
};
|
|
23169
|
+
}
|
|
23170
|
+
const colocated = decoderNodes.find((n) => n.nodeId === pipelineNodeId);
|
|
23171
|
+
if (colocated) return {
|
|
23172
|
+
decoderNodeId: colocated.nodeId,
|
|
23173
|
+
reason: "co-located",
|
|
23174
|
+
score: computeCapacityScore(colocated)
|
|
23077
23175
|
};
|
|
23078
|
-
const
|
|
23079
|
-
|
|
23080
|
-
|
|
23081
|
-
|
|
23082
|
-
|
|
23083
|
-
|
|
23084
|
-
|
|
23085
|
-
|
|
23086
|
-
});
|
|
23087
|
-
if (consecutiveFailures > 0) logger.info("audio-chunk poller: stream recovered", { meta: {
|
|
23088
|
-
brokerId,
|
|
23089
|
-
subscriptionId: subId
|
|
23090
|
-
} });
|
|
23091
|
-
consecutiveFailures = 0;
|
|
23092
|
-
for (const chunk of chunks) {
|
|
23093
|
-
if (lifecycle.stopped) break;
|
|
23094
|
-
await onChunk(chunk);
|
|
23095
|
-
}
|
|
23096
|
-
} catch (err) {
|
|
23097
|
-
consecutiveFailures += 1;
|
|
23098
|
-
if (consecutiveFailures === 1) logger.warn("audio-chunk poller: pullAudioChunks failed — attempting recovery", { meta: {
|
|
23099
|
-
brokerId,
|
|
23100
|
-
subscriptionId: subId,
|
|
23101
|
-
error: errMsg(err)
|
|
23102
|
-
} });
|
|
23103
|
-
if (!lifecycle.stopped && consecutiveFailures >= RESUBSCRIBE_AFTER_FAILURES && (consecutiveFailures - RESUBSCRIBE_AFTER_FAILURES) % RESUBSCRIBE_THROTTLE_TICKS === 0) await resubscribe();
|
|
23104
|
-
}
|
|
23105
|
-
if (!lifecycle.stopped) lifecycle.pollTimer = setTimeout(() => void tick(), POLL_INTERVAL_MS);
|
|
23176
|
+
const best = decoderNodes.map((node) => ({
|
|
23177
|
+
node,
|
|
23178
|
+
score: computeCapacityScore(node)
|
|
23179
|
+
})).toSorted((a, b) => a.score - b.score)[0];
|
|
23180
|
+
return {
|
|
23181
|
+
decoderNodeId: best.node.nodeId,
|
|
23182
|
+
reason: "capacity",
|
|
23183
|
+
score: best.score
|
|
23106
23184
|
};
|
|
23107
|
-
tick();
|
|
23108
|
-
}
|
|
23109
|
-
/**
|
|
23110
|
-
* Cancellable sleep — wakes early when `lifecycle.stopped` flips. We
|
|
23111
|
-
* keep a local wrapper around the shared {@link sleep} helper because
|
|
23112
|
-
* the lifecycle tracks the active retry timer for `teardown()` to
|
|
23113
|
-
* clear; pure `sleep()` would leak the timer if teardown fired while
|
|
23114
|
-
* we were waiting.
|
|
23115
|
-
*/
|
|
23116
|
-
function sleep(ms, lifecycle) {
|
|
23117
|
-
return new Promise((resolve) => {
|
|
23118
|
-
if (lifecycle.stopped) {
|
|
23119
|
-
resolve();
|
|
23120
|
-
return;
|
|
23121
|
-
}
|
|
23122
|
-
lifecycle.retryTimer = setTimeout(() => {
|
|
23123
|
-
lifecycle.retryTimer = void 0;
|
|
23124
|
-
resolve();
|
|
23125
|
-
}, ms);
|
|
23126
|
-
});
|
|
23127
23185
|
}
|
|
23128
23186
|
//#endregion
|
|
23129
|
-
//#region src/
|
|
23187
|
+
//#region src/orchestrator-store-schemas.ts
|
|
23130
23188
|
/**
|
|
23131
|
-
*
|
|
23132
|
-
*
|
|
23133
|
-
*
|
|
23189
|
+
* `orchestrator-store-schemas.ts` — whole-blob Zod schemas for the
|
|
23190
|
+
* orchestrator's addon-store keys that persist an ENTIRE collection
|
|
23191
|
+
* under a single key (`nodeBindings`, `templates`, `agentSettings`,
|
|
23192
|
+
* `cameraSettings`).
|
|
23134
23193
|
*
|
|
23135
|
-
*
|
|
23136
|
-
*
|
|
23137
|
-
*
|
|
23194
|
+
* These back the durable-state primitive (`this.state(KEY, Schema, {})`)
|
|
23195
|
+
* so a hand-written serializer can never silently drop a field on save:
|
|
23196
|
+
* the whole validated value round-trips on every read and write.
|
|
23197
|
+
*
|
|
23198
|
+
* Each schema mirrors the SAME shape the orchestrator already persisted
|
|
23199
|
+
* (the `pipeline-orchestrator.cap.ts` storage contract). Every field that
|
|
23200
|
+
* is only conditionally written is `.optional()`, so an existing blob
|
|
23201
|
+
* that predates a newer field still loads — durable-state `get()` only
|
|
23202
|
+
* falls back to the empty map on a whole-blob parse failure, so the
|
|
23203
|
+
* schema is kept faithful to the on-disk shape rather than overly strict.
|
|
23138
23204
|
*/
|
|
23139
|
-
|
|
23140
|
-
|
|
23141
|
-
|
|
23142
|
-
|
|
23143
|
-
|
|
23205
|
+
var EngineChoiceSchema = object({
|
|
23206
|
+
runtime: _enum(["node", "python"]),
|
|
23207
|
+
backend: string(),
|
|
23208
|
+
format: string(),
|
|
23209
|
+
device: string().optional()
|
|
23210
|
+
});
|
|
23211
|
+
var NodeBindingsSchema = record(string(), record(string(), string()));
|
|
23212
|
+
var StoredPipelineConfigSchema = object({
|
|
23213
|
+
engine: EngineChoiceSchema,
|
|
23214
|
+
steps: array(PipelineStepInputSchema).readonly(),
|
|
23215
|
+
audio: object({
|
|
23216
|
+
engine: EngineChoiceSchema,
|
|
23217
|
+
modelId: string(),
|
|
23218
|
+
enabled: boolean(),
|
|
23219
|
+
settings: record(string(), unknown()).readonly().optional()
|
|
23220
|
+
}).nullable().optional()
|
|
23221
|
+
});
|
|
23222
|
+
var StoredPipelineTemplateSchema = object({
|
|
23223
|
+
id: string(),
|
|
23224
|
+
name: string(),
|
|
23225
|
+
description: string().optional(),
|
|
23226
|
+
config: StoredPipelineConfigSchema,
|
|
23227
|
+
createdAt: string(),
|
|
23228
|
+
updatedAt: string()
|
|
23229
|
+
});
|
|
23230
|
+
var TemplatesSchema = record(string(), StoredPipelineTemplateSchema);
|
|
23231
|
+
var StoredAgentAddonConfigSchema = object({
|
|
23232
|
+
enabled: boolean(),
|
|
23233
|
+
modelId: string(),
|
|
23234
|
+
settings: record(string(), unknown()).readonly()
|
|
23235
|
+
});
|
|
23236
|
+
var StoredAgentPipelineSettingsSchema = object({
|
|
23237
|
+
addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
|
|
23238
|
+
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
23239
|
+
});
|
|
23240
|
+
var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
|
|
23241
|
+
var StoredCameraStepOverridePatchSchema = object({
|
|
23242
|
+
enabled: boolean().optional(),
|
|
23243
|
+
modelId: string().optional(),
|
|
23244
|
+
settings: record(string(), unknown()).readonly().optional()
|
|
23245
|
+
});
|
|
23246
|
+
var StoredCameraPipelineForAgentSchema = object({
|
|
23247
|
+
steps: array(PipelineStepInputSchema).readonly(),
|
|
23248
|
+
audio: object({
|
|
23249
|
+
modelId: string(),
|
|
23250
|
+
enabled: boolean()
|
|
23251
|
+
}).nullable()
|
|
23252
|
+
});
|
|
23253
|
+
var StoredCameraPipelineSettingsSchema = object({
|
|
23254
|
+
pinnedAgentNodeId: string().optional(),
|
|
23255
|
+
stepToggles: record(string(), boolean()).optional(),
|
|
23256
|
+
stepOverridesByAgent: record(string(), record(string(), StoredCameraStepOverridePatchSchema)).optional(),
|
|
23257
|
+
pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
|
|
23258
|
+
/**
|
|
23259
|
+
* Legacy "nuke inference" flag. Superseded by the detection-pipeline
|
|
23260
|
+
* wrapper binding, but the one-shot boot migration
|
|
23261
|
+
* (`migrateLegacyFlagsToBindings`) still reads it off the persisted
|
|
23262
|
+
* blob to flip the binding off. Kept here so the durable round-trip
|
|
23263
|
+
* does NOT strip it before the migration runs.
|
|
23264
|
+
*/
|
|
23265
|
+
disableInference: boolean().optional()
|
|
23266
|
+
});
|
|
23267
|
+
var CameraSettingsMapSchema = record(string(), StoredCameraPipelineSettingsSchema);
|
|
23268
|
+
//#endregion
|
|
23269
|
+
//#region src/pending-retry.ts
|
|
23144
23270
|
/**
|
|
23145
|
-
* Returns the deviceIds that are present in `
|
|
23146
|
-
* `
|
|
23147
|
-
*
|
|
23271
|
+
* Returns the deviceIds that are KNOWN (present in `known`) but currently
|
|
23272
|
+
* UNASSIGNED (absent from `assigned`). These are the "stranded" cameras — the
|
|
23273
|
+
* orchestrator has a runner config for them but no live pipeline assignment
|
|
23274
|
+
* (pending / over-cap / no-frame-source / load-shed).
|
|
23148
23275
|
*
|
|
23149
|
-
*
|
|
23150
|
-
*
|
|
23276
|
+
* Mirror of `computeDispatchGap` in `dispatch-reconcile.ts`: a plain set
|
|
23277
|
+
* difference (`known \ assigned`). Pure — no side effects, deterministic
|
|
23278
|
+
* ordering (iteration order of `known`).
|
|
23151
23279
|
*/
|
|
23152
|
-
function
|
|
23153
|
-
return [...
|
|
23280
|
+
function computeStrandedDevices(known, assigned) {
|
|
23281
|
+
return [...known].filter((id) => !assigned.has(id));
|
|
23154
23282
|
}
|
|
23155
23283
|
//#endregion
|
|
23156
23284
|
//#region src/pipeline-watchdog.ts
|
|
@@ -23183,219 +23311,407 @@ var PipelineWatchdog = class {
|
|
|
23183
23311
|
});
|
|
23184
23312
|
this.state.set(cam.deviceId, stages);
|
|
23185
23313
|
}
|
|
23186
|
-
unregister(deviceId) {
|
|
23187
|
-
this.cameras.delete(deviceId);
|
|
23188
|
-
this.state.delete(deviceId);
|
|
23314
|
+
unregister(deviceId) {
|
|
23315
|
+
this.cameras.delete(deviceId);
|
|
23316
|
+
this.state.delete(deviceId);
|
|
23317
|
+
}
|
|
23318
|
+
/** Called when a stage's live signal arrives (audio chunk / motion frame / inference). */
|
|
23319
|
+
noteSignal(deviceId, stage) {
|
|
23320
|
+
const rt = this.state.get(deviceId)?.get(stage);
|
|
23321
|
+
if (!rt) return;
|
|
23322
|
+
rt.lastSeenMs = this.deps.now();
|
|
23323
|
+
rt.attempts = 0;
|
|
23324
|
+
}
|
|
23325
|
+
tick() {
|
|
23326
|
+
const now = this.deps.now();
|
|
23327
|
+
for (const cam of this.cameras.values()) {
|
|
23328
|
+
const { line, stalled, recoveries } = this.evaluate(cam, now);
|
|
23329
|
+
if (stalled) this.deps.logger.warn(line);
|
|
23330
|
+
else this.deps.logger.info(line);
|
|
23331
|
+
for (const r of recoveries) {
|
|
23332
|
+
const rt = this.state.get(cam.deviceId)?.get(r.stage);
|
|
23333
|
+
if (rt) rt.attempts += 1;
|
|
23334
|
+
this.deps.recover(cam.deviceId, r.stage, r.streamId);
|
|
23335
|
+
}
|
|
23336
|
+
}
|
|
23337
|
+
}
|
|
23338
|
+
start(intervalMs) {
|
|
23339
|
+
if (this.timer) return;
|
|
23340
|
+
this.timer = setInterval(() => this.tick(), intervalMs);
|
|
23341
|
+
}
|
|
23342
|
+
stop() {
|
|
23343
|
+
if (this.timer) clearInterval(this.timer);
|
|
23344
|
+
this.timer = null;
|
|
23345
|
+
}
|
|
23346
|
+
evaluate(cam, now) {
|
|
23347
|
+
const parts = [];
|
|
23348
|
+
const recoveries = [];
|
|
23349
|
+
let stalled = false;
|
|
23350
|
+
const stageOrder = [
|
|
23351
|
+
"audio",
|
|
23352
|
+
"motion",
|
|
23353
|
+
"detection"
|
|
23354
|
+
];
|
|
23355
|
+
const stageMode = {
|
|
23356
|
+
audio: cam.audioMode,
|
|
23357
|
+
motion: cam.motionSources.includes("analyzer") ? "analyzer" : cam.motionSources.join("+"),
|
|
23358
|
+
detection: cam.detectionMode
|
|
23359
|
+
};
|
|
23360
|
+
for (const stage of stageOrder) {
|
|
23361
|
+
const streamId = cam.continuousStages.get(stage);
|
|
23362
|
+
if (streamId === void 0) {
|
|
23363
|
+
parts.push(`${stage}=${stageMode[stage]}(idle)`);
|
|
23364
|
+
continue;
|
|
23365
|
+
}
|
|
23366
|
+
const rt = this.state.get(cam.deviceId)?.get(stage);
|
|
23367
|
+
if (!rt) {
|
|
23368
|
+
parts.push(`${stage}=${stageMode[stage]}(unknown)`);
|
|
23369
|
+
continue;
|
|
23370
|
+
}
|
|
23371
|
+
const staleness = now - rt.lastSeenMs;
|
|
23372
|
+
const thresholdMs = this.deps.thresholds[STAGE_THRESHOLD_KEY[stage]];
|
|
23373
|
+
const sSec = Math.round(staleness / 1e3);
|
|
23374
|
+
if (staleness <= thresholdMs) parts.push(`${stage}=${stageMode[stage]}@${streamId}(OK ${sSec}s)`);
|
|
23375
|
+
else if (rt.attempts >= this.deps.thresholds.maxRecoveryAttempts) {
|
|
23376
|
+
stalled = true;
|
|
23377
|
+
parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · exhausted ${rt.attempts})`);
|
|
23378
|
+
} else {
|
|
23379
|
+
stalled = true;
|
|
23380
|
+
recoveries.push({
|
|
23381
|
+
stage,
|
|
23382
|
+
streamId
|
|
23383
|
+
});
|
|
23384
|
+
parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · recovering ${rt.attempts + 1}/${this.deps.thresholds.maxRecoveryAttempts})`);
|
|
23385
|
+
}
|
|
23386
|
+
}
|
|
23387
|
+
return {
|
|
23388
|
+
line: `[watchdog] cam${cam.deviceId} ${cam.deviceName} · ${parts.join(" · ")} → ${stalled ? "STALLED" : "OK"}`,
|
|
23389
|
+
stalled,
|
|
23390
|
+
recoveries
|
|
23391
|
+
};
|
|
23392
|
+
}
|
|
23393
|
+
};
|
|
23394
|
+
//#endregion
|
|
23395
|
+
//#region src/remote-health.ts
|
|
23396
|
+
/**
|
|
23397
|
+
* Default thresholds: 3-min grace (cold start), 2-min stale window (metrics
|
|
23398
|
+
* stopped entirely), 0.5 fps floor. A degraded foreign-handle camera runs
|
|
23399
|
+
* ~1 fps (above the bar); with T2's frame-source eligibility a remote
|
|
23400
|
+
* assignment is only legal on a frame-source node, so sustained sub-0.5 fps
|
|
23401
|
+
* means genuinely broken.
|
|
23402
|
+
*/
|
|
23403
|
+
var DEFAULT_REMOTE_HEALTH_OPTS = {
|
|
23404
|
+
graceMs: 3 * 6e4,
|
|
23405
|
+
staleMs: 2 * 6e4,
|
|
23406
|
+
minFps: .5
|
|
23407
|
+
};
|
|
23408
|
+
/**
|
|
23409
|
+
* Evaluate every remote pipeline assignment and return the cameras that must be
|
|
23410
|
+
* re-placed. Rules (each assignment evaluated independently):
|
|
23411
|
+
* 1. LOCAL assignments (`agentNodeId === localNodeId`) are skipped — the hub
|
|
23412
|
+
* watchdog owns them.
|
|
23413
|
+
* 2. Cameras not in `activeDeviceIds` are skipped — detection isn't expected.
|
|
23414
|
+
* 3. Assignments younger than `graceMs` are skipped — cold-start grace.
|
|
23415
|
+
* 4. No fps entry, or `now - lastSeen > staleMs` → `stale-metrics`.
|
|
23416
|
+
* 5. Otherwise `fps < minFps` → `zero-fps`.
|
|
23417
|
+
* 6. Otherwise healthy → no action.
|
|
23418
|
+
*
|
|
23419
|
+
* Deterministic ordering: iteration order of `assignments`.
|
|
23420
|
+
*/
|
|
23421
|
+
function evaluateRemoteHealth(input) {
|
|
23422
|
+
const { assignments, fpsMap, activeDeviceIds, localNodeId, now, opts } = input;
|
|
23423
|
+
const actions = [];
|
|
23424
|
+
for (const [deviceId, assignment] of assignments) {
|
|
23425
|
+
if (assignment.agentNodeId === localNodeId) continue;
|
|
23426
|
+
if (!activeDeviceIds.has(deviceId)) continue;
|
|
23427
|
+
if (now - assignment.assignedAt < opts.graceMs) continue;
|
|
23428
|
+
const entry = fpsMap.get(deviceId);
|
|
23429
|
+
if (!entry || now - entry.lastSeen > opts.staleMs) {
|
|
23430
|
+
actions.push({
|
|
23431
|
+
deviceId,
|
|
23432
|
+
kind: "replace",
|
|
23433
|
+
why: "stale-metrics"
|
|
23434
|
+
});
|
|
23435
|
+
continue;
|
|
23436
|
+
}
|
|
23437
|
+
if (entry.fps < opts.minFps) {
|
|
23438
|
+
actions.push({
|
|
23439
|
+
deviceId,
|
|
23440
|
+
kind: "replace",
|
|
23441
|
+
why: "zero-fps"
|
|
23442
|
+
});
|
|
23443
|
+
continue;
|
|
23444
|
+
}
|
|
23445
|
+
}
|
|
23446
|
+
return actions;
|
|
23447
|
+
}
|
|
23448
|
+
//#endregion
|
|
23449
|
+
//#region src/zone-rules-provider.ts
|
|
23450
|
+
/**
|
|
23451
|
+
* `zone-rules-provider.ts` — implements `zoneRulesCapability` for the
|
|
23452
|
+
* orchestrator.
|
|
23453
|
+
*
|
|
23454
|
+
* Per-stage rule arrays (motion / detection) live next to zones in the
|
|
23455
|
+
* orchestrator's per-device store under `zoneRules.<stage>` keys.
|
|
23456
|
+
* Every mutation mirrors to a stage-specific device-state slice
|
|
23457
|
+
* (`motion-zone-rules`, `detection-zone-rules`) so consumer addons
|
|
23458
|
+
* subscribe independently and pick up the new gating without an extra
|
|
23459
|
+
* cap round-trip.
|
|
23460
|
+
*
|
|
23461
|
+
* The provider validates each rule against {@link ZoneRuleSchema}
|
|
23462
|
+
* before persisting — partial / corrupt writes are rejected outright
|
|
23463
|
+
* since rules drive runtime filtering and a bad payload would silently
|
|
23464
|
+
* widen the operator's intended scope.
|
|
23465
|
+
*/
|
|
23466
|
+
/** Settings store key for stage rules. Kept under a single nested
|
|
23467
|
+
* object so a future stage just adds another property without
|
|
23468
|
+
* reshuffling the schema. */
|
|
23469
|
+
var RULES_STORE_KEY = "zoneRules";
|
|
23470
|
+
/** Cap name for the unified runtime-state slice. Matches the cap's
|
|
23471
|
+
* declared `name` so the codegen DeviceProxy auto-wires
|
|
23472
|
+
* `device.state.zoneRules`. The slice value is the full
|
|
23473
|
+
* `{motion, detection}` object — both stages travel together so a
|
|
23474
|
+
* single reactive handle covers every consumer. */
|
|
23475
|
+
var ZONE_RULES_CAP_NAME = "zone-rules";
|
|
23476
|
+
var RulesArraySchema = array(ZoneRuleSchema);
|
|
23477
|
+
/**
|
|
23478
|
+
* Whole-blob schema for the per-device `zoneRules` store key. Both stages
|
|
23479
|
+
* travel together under one key. Deliberately lenient — each stage is
|
|
23480
|
+
* `unknown` so a corrupt single stage can NOT drop the sibling on load
|
|
23481
|
+
* (durable-state `get()` falls back to `{}` only on a whole-blob parse
|
|
23482
|
+
* failure). Strict per-stage validation, with its own reset-on-corrupt
|
|
23483
|
+
* warn, still happens in `loadRules` exactly as before the migration.
|
|
23484
|
+
*/
|
|
23485
|
+
var ZoneRulesBlockSchema = object({
|
|
23486
|
+
motion: unknown().optional(),
|
|
23487
|
+
detection: unknown().optional()
|
|
23488
|
+
}).passthrough();
|
|
23489
|
+
var ZoneRulesProvider = class {
|
|
23490
|
+
ctx;
|
|
23491
|
+
/** Per-device per-stage cache. Hydrated lazily on first read. */
|
|
23492
|
+
cache = /* @__PURE__ */ new Map();
|
|
23493
|
+
/**
|
|
23494
|
+
* Per-device durable handle over the `zoneRules` store key. The WHOLE
|
|
23495
|
+
* `{motion?, detection?}` block round-trips on every read/write so a
|
|
23496
|
+
* write on one stage can never drop the other. Built lazily + memoised.
|
|
23497
|
+
*/
|
|
23498
|
+
stateByDevice = /* @__PURE__ */ new Map();
|
|
23499
|
+
constructor(ctx) {
|
|
23500
|
+
this.ctx = ctx;
|
|
23501
|
+
}
|
|
23502
|
+
/** Lazily build (and memoise) the durable `zoneRules` handle for a device. */
|
|
23503
|
+
rulesState(deviceId) {
|
|
23504
|
+
let handle = this.stateByDevice.get(deviceId);
|
|
23505
|
+
if (!handle) {
|
|
23506
|
+
handle = createDurableState({
|
|
23507
|
+
key: RULES_STORE_KEY,
|
|
23508
|
+
schema: ZoneRulesBlockSchema,
|
|
23509
|
+
fallback: {},
|
|
23510
|
+
read: () => this.ctx.settings.readDeviceStore(deviceId),
|
|
23511
|
+
write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
|
|
23512
|
+
onParseError: (key, error) => this.ctx.logger.warn("zone-rules store block failed to parse — resetting", {
|
|
23513
|
+
tags: { deviceId },
|
|
23514
|
+
meta: {
|
|
23515
|
+
key,
|
|
23516
|
+
error: error instanceof Error ? error.message : String(error)
|
|
23517
|
+
}
|
|
23518
|
+
})
|
|
23519
|
+
});
|
|
23520
|
+
this.stateByDevice.set(deviceId, handle);
|
|
23521
|
+
}
|
|
23522
|
+
return handle;
|
|
23523
|
+
}
|
|
23524
|
+
async listRules({ deviceId, stage }) {
|
|
23525
|
+
return this.loadRules(deviceId, stage);
|
|
23526
|
+
}
|
|
23527
|
+
async setRules({ deviceId, stage, rules }) {
|
|
23528
|
+
const parsed = RulesArraySchema.safeParse(rules);
|
|
23529
|
+
if (!parsed.success) throw new Error(`zone-rules.setRules: invalid rule payload — ${parsed.error.issues.map((i) => i.message).join("; ")}`);
|
|
23530
|
+
await this.persist(deviceId, stage, parsed.data);
|
|
23189
23531
|
}
|
|
23190
|
-
/**
|
|
23191
|
-
|
|
23192
|
-
|
|
23193
|
-
|
|
23194
|
-
rt.lastSeenMs = this.deps.now();
|
|
23195
|
-
rt.attempts = 0;
|
|
23532
|
+
/** Drop a device's cache entries. Called when the device is removed. */
|
|
23533
|
+
forgetDevice(deviceId) {
|
|
23534
|
+
this.cache.delete(deviceId);
|
|
23535
|
+
this.stateByDevice.delete(deviceId);
|
|
23196
23536
|
}
|
|
23197
|
-
|
|
23198
|
-
|
|
23199
|
-
|
|
23200
|
-
|
|
23201
|
-
|
|
23202
|
-
|
|
23203
|
-
|
|
23204
|
-
|
|
23205
|
-
|
|
23206
|
-
|
|
23537
|
+
async loadRules(deviceId, stage) {
|
|
23538
|
+
let perDevice = this.cache.get(deviceId);
|
|
23539
|
+
if (!perDevice) {
|
|
23540
|
+
perDevice = /* @__PURE__ */ new Map();
|
|
23541
|
+
this.cache.set(deviceId, perDevice);
|
|
23542
|
+
}
|
|
23543
|
+
const cached = perDevice.get(stage);
|
|
23544
|
+
if (cached) return cached;
|
|
23545
|
+
let rules = [];
|
|
23546
|
+
try {
|
|
23547
|
+
const raw = (await this.rulesState(deviceId).get())[stage];
|
|
23548
|
+
if (raw !== void 0) {
|
|
23549
|
+
const parsed = RulesArraySchema.safeParse(raw);
|
|
23550
|
+
if (parsed.success) rules = parsed.data;
|
|
23551
|
+
else this.ctx.logger.warn("zone-rules store entry failed to parse — resetting", {
|
|
23552
|
+
tags: { deviceId },
|
|
23553
|
+
meta: {
|
|
23554
|
+
stage,
|
|
23555
|
+
issues: parsed.error.issues
|
|
23556
|
+
}
|
|
23557
|
+
});
|
|
23207
23558
|
}
|
|
23559
|
+
} catch (err) {
|
|
23560
|
+
this.ctx.logger.warn("zone-rules store read failed — using empty list", {
|
|
23561
|
+
tags: { deviceId },
|
|
23562
|
+
meta: {
|
|
23563
|
+
stage,
|
|
23564
|
+
error: err instanceof Error ? err.message : String(err)
|
|
23565
|
+
}
|
|
23566
|
+
});
|
|
23208
23567
|
}
|
|
23568
|
+
perDevice.set(stage, rules);
|
|
23569
|
+
return rules;
|
|
23209
23570
|
}
|
|
23210
|
-
|
|
23211
|
-
|
|
23212
|
-
|
|
23213
|
-
|
|
23214
|
-
|
|
23215
|
-
|
|
23216
|
-
|
|
23217
|
-
|
|
23218
|
-
|
|
23219
|
-
|
|
23220
|
-
|
|
23221
|
-
|
|
23222
|
-
const
|
|
23223
|
-
|
|
23224
|
-
|
|
23225
|
-
|
|
23226
|
-
|
|
23227
|
-
|
|
23228
|
-
|
|
23229
|
-
motion: cam.motionSources.includes("analyzer") ? "analyzer" : cam.motionSources.join("+"),
|
|
23230
|
-
detection: cam.detectionMode
|
|
23571
|
+
async persist(deviceId, stage, rules) {
|
|
23572
|
+
let perDevice = this.cache.get(deviceId);
|
|
23573
|
+
if (!perDevice) {
|
|
23574
|
+
perDevice = /* @__PURE__ */ new Map();
|
|
23575
|
+
this.cache.set(deviceId, perDevice);
|
|
23576
|
+
}
|
|
23577
|
+
perDevice.set(stage, rules);
|
|
23578
|
+
await this.rulesState(deviceId).update((prev) => ({
|
|
23579
|
+
...prev,
|
|
23580
|
+
[stage]: rules
|
|
23581
|
+
}));
|
|
23582
|
+
const otherStage = stage === "motion" ? "detection" : "motion";
|
|
23583
|
+
const otherRules = perDevice.get(otherStage) ?? await this.loadRules(deviceId, otherStage);
|
|
23584
|
+
const sliceValue = stage === "motion" ? {
|
|
23585
|
+
motion: rules,
|
|
23586
|
+
detection: otherRules
|
|
23587
|
+
} : {
|
|
23588
|
+
motion: otherRules,
|
|
23589
|
+
detection: rules
|
|
23231
23590
|
};
|
|
23232
|
-
|
|
23233
|
-
|
|
23234
|
-
|
|
23235
|
-
|
|
23236
|
-
|
|
23237
|
-
|
|
23238
|
-
|
|
23239
|
-
|
|
23240
|
-
|
|
23241
|
-
continue;
|
|
23242
|
-
}
|
|
23243
|
-
const staleness = now - rt.lastSeenMs;
|
|
23244
|
-
const thresholdMs = this.deps.thresholds[STAGE_THRESHOLD_KEY[stage]];
|
|
23245
|
-
const sSec = Math.round(staleness / 1e3);
|
|
23246
|
-
if (staleness <= thresholdMs) parts.push(`${stage}=${stageMode[stage]}@${streamId}(OK ${sSec}s)`);
|
|
23247
|
-
else if (rt.attempts >= this.deps.thresholds.maxRecoveryAttempts) {
|
|
23248
|
-
stalled = true;
|
|
23249
|
-
parts.push(`${stage}=${stageMode[stage]}@${streamId}(STALLED ${sSec}s · exhausted ${rt.attempts})`);
|
|
23250
|
-
} else {
|
|
23251
|
-
stalled = true;
|
|
23252
|
-
recoveries.push({
|
|
23591
|
+
try {
|
|
23592
|
+
await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
|
|
23593
|
+
capName: ZONE_RULES_CAP_NAME,
|
|
23594
|
+
slice: sliceValue
|
|
23595
|
+
});
|
|
23596
|
+
} catch (err) {
|
|
23597
|
+
this.ctx.logger.debug("zone-rules slice mirror failed", {
|
|
23598
|
+
tags: { deviceId },
|
|
23599
|
+
meta: {
|
|
23253
23600
|
stage,
|
|
23254
|
-
|
|
23255
|
-
}
|
|
23256
|
-
|
|
23257
|
-
}
|
|
23601
|
+
error: err instanceof Error ? err.message : String(err)
|
|
23602
|
+
}
|
|
23603
|
+
});
|
|
23258
23604
|
}
|
|
23259
|
-
|
|
23260
|
-
line: `[watchdog] cam${cam.deviceId} ${cam.deviceName} · ${parts.join(" · ")} → ${stalled ? "STALLED" : "OK"}`,
|
|
23261
|
-
stalled,
|
|
23262
|
-
recoveries
|
|
23263
|
-
};
|
|
23605
|
+
this.ctx.onRulesChanged?.(deviceId, stage, rules);
|
|
23264
23606
|
}
|
|
23265
23607
|
};
|
|
23266
23608
|
//#endregion
|
|
23267
|
-
//#region src/
|
|
23268
|
-
function mapAssignment(input) {
|
|
23269
|
-
return {
|
|
23270
|
-
detectionNodeId: input.detectionNodeId,
|
|
23271
|
-
decoderNodeId: input.decoderNodeId,
|
|
23272
|
-
audioNodeId: input.audioNodeId,
|
|
23273
|
-
pinned: {
|
|
23274
|
-
detection: input.pinned.detection,
|
|
23275
|
-
decoder: input.pinned.decoder,
|
|
23276
|
-
audio: input.pinned.audio
|
|
23277
|
-
},
|
|
23278
|
-
reasons: {
|
|
23279
|
-
detection: input.reasons.detection,
|
|
23280
|
-
decoder: input.reasons.decoder,
|
|
23281
|
-
audio: input.reasons.audio
|
|
23282
|
-
}
|
|
23283
|
-
};
|
|
23284
|
-
}
|
|
23285
|
-
function mapSource(sourceResult) {
|
|
23286
|
-
if (sourceResult === null) return { streams: [] };
|
|
23287
|
-
return { streams: sourceResult.streams.map((s) => ({
|
|
23288
|
-
camStreamId: s.camStreamId,
|
|
23289
|
-
codec: s.codec,
|
|
23290
|
-
width: s.width,
|
|
23291
|
-
height: s.height,
|
|
23292
|
-
fps: s.fps,
|
|
23293
|
-
kind: s.kind
|
|
23294
|
-
})) };
|
|
23295
|
-
}
|
|
23296
|
-
function mapBroker(brokerResult) {
|
|
23297
|
-
if (brokerResult === null) return null;
|
|
23298
|
-
return {
|
|
23299
|
-
profiles: brokerResult.profiles.map((p) => ({
|
|
23300
|
-
profile: p.profile,
|
|
23301
|
-
status: p.status,
|
|
23302
|
-
codec: p.codec,
|
|
23303
|
-
width: p.width,
|
|
23304
|
-
height: p.height,
|
|
23305
|
-
subscribers: p.subscribers,
|
|
23306
|
-
inFps: p.inFps,
|
|
23307
|
-
outFps: p.outFps
|
|
23308
|
-
})),
|
|
23309
|
-
webrtcSessions: brokerResult.webrtcSessions,
|
|
23310
|
-
rtspRestream: brokerResult.rtspRestream
|
|
23311
|
-
};
|
|
23312
|
-
}
|
|
23313
|
-
function mapDecoderShm(shm) {
|
|
23314
|
-
return {
|
|
23315
|
-
framesWritten: shm.framesWritten,
|
|
23316
|
-
getFrameHits: shm.getFrameHits,
|
|
23317
|
-
getFrameMisses: shm.getFrameMisses,
|
|
23318
|
-
budgetMb: shm.budgetMb
|
|
23319
|
-
};
|
|
23320
|
-
}
|
|
23321
|
-
function mapDecoder(decoderResult) {
|
|
23322
|
-
if (decoderResult === null) return null;
|
|
23323
|
-
return {
|
|
23324
|
-
nodeId: decoderResult.nodeId,
|
|
23325
|
-
formats: [...decoderResult.formats],
|
|
23326
|
-
sessionCount: decoderResult.sessionCount,
|
|
23327
|
-
shm: mapDecoderShm(decoderResult.shm)
|
|
23328
|
-
};
|
|
23329
|
-
}
|
|
23330
|
-
function mapMotion(motionResult) {
|
|
23331
|
-
if (motionResult === null) return null;
|
|
23332
|
-
return {
|
|
23333
|
-
enabled: motionResult.enabled,
|
|
23334
|
-
fps: motionResult.fps
|
|
23335
|
-
};
|
|
23336
|
-
}
|
|
23337
|
-
function mapProvisioning(p) {
|
|
23338
|
-
if (p.error !== void 0) return {
|
|
23339
|
-
state: p.state,
|
|
23340
|
-
error: p.error
|
|
23341
|
-
};
|
|
23342
|
-
return { state: p.state };
|
|
23343
|
-
}
|
|
23344
|
-
function mapDetection(detectionResult) {
|
|
23345
|
-
if (detectionResult === null) return null;
|
|
23346
|
-
const phase = detectionResult.phase;
|
|
23347
|
-
return {
|
|
23348
|
-
nodeId: detectionResult.nodeId,
|
|
23349
|
-
engine: {
|
|
23350
|
-
backend: detectionResult.engine.backend,
|
|
23351
|
-
device: detectionResult.engine.device
|
|
23352
|
-
},
|
|
23353
|
-
phase,
|
|
23354
|
-
configuredFps: detectionResult.configuredFps,
|
|
23355
|
-
actualFps: detectionResult.actualFps,
|
|
23356
|
-
queueDepth: detectionResult.queueDepth,
|
|
23357
|
-
avgInferenceMs: detectionResult.avgInferenceMs,
|
|
23358
|
-
provisioning: mapProvisioning(detectionResult.provisioning)
|
|
23359
|
-
};
|
|
23360
|
-
}
|
|
23361
|
-
function mapAudio(audioResult) {
|
|
23362
|
-
if (audioResult === null) return null;
|
|
23363
|
-
return {
|
|
23364
|
-
nodeId: audioResult.nodeId,
|
|
23365
|
-
enabled: audioResult.enabled
|
|
23366
|
-
};
|
|
23367
|
-
}
|
|
23368
|
-
function mapRecording(recordingResult) {
|
|
23369
|
-
if (recordingResult === null) return null;
|
|
23370
|
-
return {
|
|
23371
|
-
mode: recordingResult.mode,
|
|
23372
|
-
active: recordingResult.active,
|
|
23373
|
-
storageBytes: recordingResult.storageBytes
|
|
23374
|
-
};
|
|
23375
|
-
}
|
|
23609
|
+
//#region src/zones-provider.ts
|
|
23376
23610
|
/**
|
|
23377
|
-
*
|
|
23611
|
+
* `zones-provider.ts` — implements `zonesCapability` for the orchestrator.
|
|
23612
|
+
*
|
|
23613
|
+
* Per-camera CRUD over polygon detection zones. Persists to the
|
|
23614
|
+
* orchestrator's per-device settings store under the `zones` key and
|
|
23615
|
+
* mirrors every change into the device-state `zones` slice via
|
|
23616
|
+
* `api.deviceState.setCapSlice`. Downstream consumers (motion-wasm,
|
|
23617
|
+
* pipeline-executor, analytics, admin UI) read the live state with
|
|
23618
|
+
* the canonical `dev.state.zones.onChanged` channel.
|
|
23378
23619
|
*
|
|
23379
|
-
* -
|
|
23380
|
-
* -
|
|
23381
|
-
*
|
|
23382
|
-
*
|
|
23383
|
-
* - No mutation of the input.
|
|
23620
|
+
* Onboard / firmware-reported zones are out of scope for now — every
|
|
23621
|
+
* zone is operator-drawn. The provider keeps the surface symmetric:
|
|
23622
|
+
* `addZone` rejects id collisions, `updateZone` requires an existing
|
|
23623
|
+
* id, `removeZone` is idempotent.
|
|
23384
23624
|
*/
|
|
23385
|
-
|
|
23386
|
-
|
|
23387
|
-
|
|
23388
|
-
|
|
23389
|
-
|
|
23390
|
-
|
|
23391
|
-
|
|
23392
|
-
|
|
23393
|
-
|
|
23394
|
-
|
|
23395
|
-
|
|
23396
|
-
|
|
23397
|
-
|
|
23398
|
-
|
|
23625
|
+
var ZONES_STORE_KEY = "zones";
|
|
23626
|
+
var ZONES_CAP_NAME = "zones";
|
|
23627
|
+
var ZonesArraySchema = array(ZoneSchema);
|
|
23628
|
+
var ZonesProvider = class {
|
|
23629
|
+
ctx;
|
|
23630
|
+
/** Per-device cache. Hydrated lazily on first read for a device. */
|
|
23631
|
+
cache = /* @__PURE__ */ new Map();
|
|
23632
|
+
/**
|
|
23633
|
+
* Per-device durable handle over the `zones` store key. The WHOLE
|
|
23634
|
+
* validated zone array round-trips on every read/write so no field can
|
|
23635
|
+
* be dropped on persist. Built lazily + memoised per device.
|
|
23636
|
+
*/
|
|
23637
|
+
stateByDevice = /* @__PURE__ */ new Map();
|
|
23638
|
+
constructor(ctx) {
|
|
23639
|
+
this.ctx = ctx;
|
|
23640
|
+
}
|
|
23641
|
+
/** Lazily build (and memoise) the durable `zones` handle for a device. */
|
|
23642
|
+
zonesState(deviceId) {
|
|
23643
|
+
let handle = this.stateByDevice.get(deviceId);
|
|
23644
|
+
if (!handle) {
|
|
23645
|
+
handle = createDurableState({
|
|
23646
|
+
key: ZONES_STORE_KEY,
|
|
23647
|
+
schema: ZonesArraySchema,
|
|
23648
|
+
fallback: [],
|
|
23649
|
+
read: () => this.ctx.settings.readDeviceStore(deviceId),
|
|
23650
|
+
write: (patch) => this.ctx.settings.writeDeviceStore(deviceId, patch),
|
|
23651
|
+
onParseError: (key, error) => this.ctx.logger.warn("zones store entry failed to parse — resetting", {
|
|
23652
|
+
tags: { deviceId },
|
|
23653
|
+
meta: {
|
|
23654
|
+
key,
|
|
23655
|
+
error: error instanceof Error ? error.message : String(error)
|
|
23656
|
+
}
|
|
23657
|
+
})
|
|
23658
|
+
});
|
|
23659
|
+
this.stateByDevice.set(deviceId, handle);
|
|
23660
|
+
}
|
|
23661
|
+
return handle;
|
|
23662
|
+
}
|
|
23663
|
+
async listZones({ deviceId }) {
|
|
23664
|
+
return this.loadZones(deviceId);
|
|
23665
|
+
}
|
|
23666
|
+
async addZone({ deviceId, zone }) {
|
|
23667
|
+
const existing = await this.loadZones(deviceId);
|
|
23668
|
+
if (existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.addZone: id ${zone.id} already exists`);
|
|
23669
|
+
await this.persist(deviceId, [...existing, zone]);
|
|
23670
|
+
}
|
|
23671
|
+
async updateZone({ deviceId, zone }) {
|
|
23672
|
+
const existing = await this.loadZones(deviceId);
|
|
23673
|
+
if (!existing.some((entry) => entry.id === zone.id)) throw new Error(`zones.updateZone: id ${zone.id} not found`);
|
|
23674
|
+
const next = existing.map((entry) => entry.id === zone.id ? zone : entry);
|
|
23675
|
+
await this.persist(deviceId, next);
|
|
23676
|
+
}
|
|
23677
|
+
async removeZone({ deviceId, zoneId }) {
|
|
23678
|
+
const existing = await this.loadZones(deviceId);
|
|
23679
|
+
if (!existing.some((entry) => entry.id === zoneId)) return;
|
|
23680
|
+
await this.persist(deviceId, existing.filter((entry) => entry.id !== zoneId));
|
|
23681
|
+
}
|
|
23682
|
+
/**
|
|
23683
|
+
* Drop a device's cache entry. Called when the device is removed so
|
|
23684
|
+
* the next attach starts from a fresh disk read.
|
|
23685
|
+
*/
|
|
23686
|
+
forgetDevice(deviceId) {
|
|
23687
|
+
this.cache.delete(deviceId);
|
|
23688
|
+
this.stateByDevice.delete(deviceId);
|
|
23689
|
+
}
|
|
23690
|
+
async loadZones(deviceId) {
|
|
23691
|
+
const cached = this.cache.get(deviceId);
|
|
23692
|
+
if (cached) return cached;
|
|
23693
|
+
let zones = [];
|
|
23694
|
+
try {
|
|
23695
|
+
zones = await this.zonesState(deviceId).get();
|
|
23696
|
+
} catch (err) {
|
|
23697
|
+
this.ctx.logger.warn("zones store read failed — using empty list", {
|
|
23698
|
+
tags: { deviceId },
|
|
23699
|
+
meta: { error: err instanceof Error ? err.message : String(err) }
|
|
23700
|
+
});
|
|
23701
|
+
}
|
|
23702
|
+
this.cache.set(deviceId, zones);
|
|
23703
|
+
return zones;
|
|
23704
|
+
}
|
|
23705
|
+
async persist(deviceId, zones) {
|
|
23706
|
+
this.cache.set(deviceId, zones);
|
|
23707
|
+
await this.zonesState(deviceId).set(zones);
|
|
23708
|
+
await (await this.ctx.fetchDevice(deviceId)).deviceState.setCapSlice({
|
|
23709
|
+
capName: ZONES_CAP_NAME,
|
|
23710
|
+
slice: { zones }
|
|
23711
|
+
});
|
|
23712
|
+
this.ctx.onZonesChanged?.(deviceId, zones);
|
|
23713
|
+
}
|
|
23714
|
+
};
|
|
23399
23715
|
//#endregion
|
|
23400
23716
|
//#region src/index.ts
|
|
23401
23717
|
var PHASE_MODE_VALUES = new Set([
|
|
@@ -23414,6 +23730,24 @@ var DEFAULT_BROKER_CALL_TIMEOUT_MS = 5e3;
|
|
|
23414
23730
|
/** Debounce window for `scheduleReconcile` — coalesces bursts of topology/slot-change signals. */
|
|
23415
23731
|
var RECONCILE_DEBOUNCE_MS = 200;
|
|
23416
23732
|
/**
|
|
23733
|
+
* Periodic sweep interval for `retryPendingDispatches` — re-dispatches cameras
|
|
23734
|
+
* that are KNOWN but UNASSIGNED (pending/over-cap/no-frame-source/load-shed).
|
|
23735
|
+
* `reconcileDispatch` is additive-only and never revisits these, so a slow
|
|
23736
|
+
* safety-net timer + event-driven debounce triggers recover them.
|
|
23737
|
+
*/
|
|
23738
|
+
var PENDING_RETRY_INTERVAL_MS = 6e4;
|
|
23739
|
+
/** Debounce window for `schedulePendingRetry` — coalesces capacity/eligibility/readiness signals. */
|
|
23740
|
+
var PENDING_RETRY_DEBOUNCE_MS = 2e3;
|
|
23741
|
+
/**
|
|
23742
|
+
* Remote-assignment health loop (T7) bounded backoff. A remote camera flagged
|
|
23743
|
+
* as unhealthy (0-fps / metrics-stale) is re-placed at most
|
|
23744
|
+
* `REMOTE_HEALTH_MAX_ATTEMPTS` times inside a rolling `REMOTE_HEALTH_WINDOW_MS`
|
|
23745
|
+
* window; once exhausted the camera is left visibly pending (`'unhealthy'`) for
|
|
23746
|
+
* the operator instead of churning forever.
|
|
23747
|
+
*/
|
|
23748
|
+
var REMOTE_HEALTH_MAX_ATTEMPTS = 3;
|
|
23749
|
+
var REMOTE_HEALTH_WINDOW_MS = 60 * 6e4;
|
|
23750
|
+
/**
|
|
23417
23751
|
* Device-details keys routed through the orchestrator's pipeline
|
|
23418
23752
|
* settings writer instead of the device orchestration store. The
|
|
23419
23753
|
* `cameraPipeline` key carries the full `CameraPipelineConfig`
|
|
@@ -23502,6 +23836,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
23502
23836
|
zoneRulesProvider = null;
|
|
23503
23837
|
/** In-memory assignment map — mirrored into events + per-device settings for the pin. */
|
|
23504
23838
|
assignments = /* @__PURE__ */ new Map();
|
|
23839
|
+
/**
|
|
23840
|
+
* Why a camera is currently unassigned (pending), keyed by deviceId. Set on
|
|
23841
|
+
* every pending placement path (over-cap, no-frame-source, load-shed) and
|
|
23842
|
+
* cleared when the camera is (re)assigned or released. Consumed by T3's
|
|
23843
|
+
* pending-retry sweep + `getCameraStatus` surface — nothing reads it yet in
|
|
23844
|
+
* this commit; the map is populated so T3 can wire the read side without
|
|
23845
|
+
* re-touching every placement path.
|
|
23846
|
+
*
|
|
23847
|
+
* `protected` (not `private`) to match the existing test-seam convention in
|
|
23848
|
+
* this class (`audioSubscriptions`, `audioSubLocks`) — the frame-source
|
|
23849
|
+
* eligibility spec subclasses the addon to assert the recorded reason.
|
|
23850
|
+
*/
|
|
23851
|
+
pendingReasons = /* @__PURE__ */ new Map();
|
|
23852
|
+
/**
|
|
23853
|
+
* Remote-assignment health loop (T7) per-device backoff ledger. Each entry
|
|
23854
|
+
* tracks how many times a remote camera has been re-placed inside the current
|
|
23855
|
+
* rolling window (`REMOTE_HEALTH_WINDOW_MS`). Bounds churn: once the count
|
|
23856
|
+
* hits `REMOTE_HEALTH_MAX_ATTEMPTS`, the camera is left pending (`'unhealthy'`)
|
|
23857
|
+
* for the operator instead of being re-placed again. `protected` so the
|
|
23858
|
+
* remote-health spec can assert the ledger without casts.
|
|
23859
|
+
*/
|
|
23860
|
+
remoteHealthAttempts = /* @__PURE__ */ new Map();
|
|
23505
23861
|
/** Per-device audio node assignment — nodeId of the audio-analyzer handling this device's chunks. */
|
|
23506
23862
|
audioNodeByDevice = /* @__PURE__ */ new Map();
|
|
23507
23863
|
/** Assignments with metadata — replaces plain audioNodeByDevice values as the source of truth. */
|
|
@@ -23621,6 +23977,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
23621
23977
|
reconcileInFlight = false;
|
|
23622
23978
|
/** Set to true when a reconcile is requested while one is already in-flight; triggers a follow-up pass. */
|
|
23623
23979
|
reconcileRerunRequested = false;
|
|
23980
|
+
/** Periodic `retryPendingDispatches` safety-net timer (T3). */
|
|
23981
|
+
pendingRetryTimer = null;
|
|
23982
|
+
/** Pending `schedulePendingRetry` debounce timer (T3). */
|
|
23983
|
+
pendingRetryDebounceTimer = null;
|
|
23984
|
+
/** True while `retryPendingDispatches` is running. */
|
|
23985
|
+
pendingRetryInFlight = false;
|
|
23986
|
+
/** Set when a pending-retry is requested while one is already in-flight; triggers a follow-up pass. */
|
|
23987
|
+
pendingRetryRerunRequested = false;
|
|
23624
23988
|
initTimestamp = 0;
|
|
23625
23989
|
constructor() {
|
|
23626
23990
|
super({});
|
|
@@ -23837,6 +24201,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
23837
24201
|
thresholds: DEFAULT_WATCHDOG_THRESHOLDS
|
|
23838
24202
|
});
|
|
23839
24203
|
this.pipelineWatchdog.start(PipelineOrchestratorAddon.WATCHDOG_INTERVAL_MS);
|
|
24204
|
+
this.pendingRetryTimer = setInterval(() => void this.retryPendingDispatches(), PENDING_RETRY_INTERVAL_MS);
|
|
23840
24205
|
this.ctx.logger.info("Pipeline orchestrator detection-wiring subscriptions installed");
|
|
23841
24206
|
this.migrateLegacyFlagsToBindings().catch((err) => {
|
|
23842
24207
|
this.ctx.logger.warn("bindings migration failed — will retry on next boot", { meta: { error: errMsg(err) } });
|
|
@@ -24066,6 +24431,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24066
24431
|
clearTimeout(this.reconcileTimer);
|
|
24067
24432
|
this.reconcileTimer = null;
|
|
24068
24433
|
}
|
|
24434
|
+
if (this.pendingRetryTimer !== null) {
|
|
24435
|
+
clearInterval(this.pendingRetryTimer);
|
|
24436
|
+
this.pendingRetryTimer = null;
|
|
24437
|
+
}
|
|
24438
|
+
if (this.pendingRetryDebounceTimer !== null) {
|
|
24439
|
+
clearTimeout(this.pendingRetryDebounceTimer);
|
|
24440
|
+
this.pendingRetryDebounceTimer = null;
|
|
24441
|
+
}
|
|
24069
24442
|
this.unsubDeviceRegistered?.();
|
|
24070
24443
|
this.unsubDeviceRegistered = null;
|
|
24071
24444
|
this.unsubDeviceUnregistered?.();
|
|
@@ -24091,6 +24464,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24091
24464
|
for (const t of this.lazyAudioTeardownTimers.values()) clearTimeout(t);
|
|
24092
24465
|
this.lazyAudioTeardownTimers.clear();
|
|
24093
24466
|
this.cameraFpsMap.clear();
|
|
24467
|
+
this.remoteHealthAttempts.clear();
|
|
24094
24468
|
this.loadShedState.clear();
|
|
24095
24469
|
if (this.loadShedResumeTimer) {
|
|
24096
24470
|
clearInterval(this.loadShedResumeTimer);
|
|
@@ -24152,6 +24526,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24152
24526
|
windowMs
|
|
24153
24527
|
}
|
|
24154
24528
|
});
|
|
24529
|
+
this.pendingReasons.set(runnerConfig.deviceId, "load-shed");
|
|
24155
24530
|
return {
|
|
24156
24531
|
success: true,
|
|
24157
24532
|
kind: "pending"
|
|
@@ -24165,11 +24540,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24165
24540
|
const decision = balance({
|
|
24166
24541
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
24167
24542
|
preferredAgent,
|
|
24168
|
-
nodeCaps: this.buildNodeCaps()
|
|
24543
|
+
nodeCaps: await this.buildNodeCaps(),
|
|
24544
|
+
eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
|
|
24169
24545
|
});
|
|
24170
24546
|
if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
|
|
24171
24547
|
if (decision.kind === "pending") {
|
|
24172
|
-
this.
|
|
24548
|
+
this.pendingReasons.set(runnerConfig.deviceId, decision.reason);
|
|
24549
|
+
this.ctx.logger.warn("camera left pending — no eligible node", {
|
|
24550
|
+
tags: { deviceId: runnerConfig.deviceId },
|
|
24551
|
+
meta: { reason: decision.reason }
|
|
24552
|
+
});
|
|
24173
24553
|
return {
|
|
24174
24554
|
success: true,
|
|
24175
24555
|
kind: "pending"
|
|
@@ -24220,11 +24600,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24220
24600
|
}
|
|
24221
24601
|
this.assignments.delete(input.deviceId);
|
|
24222
24602
|
this.cameraConfigs.delete(input.deviceId);
|
|
24603
|
+
this.pendingReasons.delete(input.deviceId);
|
|
24223
24604
|
this.pipelineWatchdog?.unregister(input.deviceId);
|
|
24224
24605
|
return { success: true };
|
|
24225
24606
|
}
|
|
24226
24607
|
async assignPipeline(input) {
|
|
24227
24608
|
if (!this.ctx) throw new Error("PipelineOrchestrator: assignPipeline called before initialize");
|
|
24609
|
+
const eligible = this.detectionEligibleNodes(input.deviceId);
|
|
24610
|
+
if (!eligible.includes(input.agentNodeId)) throw new Error(`Cannot pin camera ${input.deviceId} detection to '${input.agentNodeId}': the node cannot obtain this camera's decoded frames (frame-source nodes: ${eligible.join(", ") || "none"}). Add it to Enabled Decoder Nodes first.`);
|
|
24228
24611
|
await this.ctx.settings?.writeDeviceStore(input.deviceId, { [PREFERRED_AGENT_SETTING]: input.agentNodeId }).catch((err) => {
|
|
24229
24612
|
const msg = errMsg(err);
|
|
24230
24613
|
this.ctx.logger.warn("assignPipeline: failed to persist pin", {
|
|
@@ -24270,11 +24653,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24270
24653
|
const decision = balance({
|
|
24271
24654
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
24272
24655
|
preferredAgent: null,
|
|
24273
|
-
nodeCaps: this.buildNodeCaps()
|
|
24656
|
+
nodeCaps: await this.buildNodeCaps(),
|
|
24657
|
+
eligibleNodes: this.detectionEligibleNodes(input.deviceId)
|
|
24274
24658
|
});
|
|
24275
24659
|
if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
|
|
24276
|
-
else if (decision.kind === "pending")
|
|
24277
|
-
|
|
24660
|
+
else if (decision.kind === "pending") {
|
|
24661
|
+
this.pendingReasons.set(input.deviceId, decision.reason);
|
|
24662
|
+
this.ctx.logger.warn("camera left pending — no eligible node", {
|
|
24663
|
+
tags: { deviceId: input.deviceId },
|
|
24664
|
+
meta: { reason: decision.reason }
|
|
24665
|
+
});
|
|
24666
|
+
} else {
|
|
24278
24667
|
const targetNodeId = decision.agentNodeId;
|
|
24279
24668
|
if (current && current.agentNodeId !== targetNodeId) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
|
|
24280
24669
|
const msg = errMsg(err);
|
|
@@ -24312,7 +24701,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24312
24701
|
async rebalance() {
|
|
24313
24702
|
if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
|
|
24314
24703
|
const loads = await this.collectAgentLoad({ onlyEnabled: true });
|
|
24315
|
-
const nodeCaps = this.buildNodeCaps();
|
|
24704
|
+
const nodeCaps = await this.buildNodeCaps();
|
|
24316
24705
|
let migrated = 0;
|
|
24317
24706
|
for (const [deviceId, config] of this.cameraConfigs) {
|
|
24318
24707
|
const current = this.assignments.get(deviceId);
|
|
@@ -24320,11 +24709,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24320
24709
|
const decision = balance({
|
|
24321
24710
|
nodes: loads,
|
|
24322
24711
|
preferredAgent: await this.readPreferredAgent(deviceId),
|
|
24323
|
-
nodeCaps
|
|
24712
|
+
nodeCaps,
|
|
24713
|
+
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
24324
24714
|
});
|
|
24325
24715
|
if (!decision) continue;
|
|
24326
24716
|
if (decision.kind === "pending") {
|
|
24327
|
-
this.
|
|
24717
|
+
this.pendingReasons.set(deviceId, decision.reason);
|
|
24718
|
+
this.ctx.logger.warn("camera left pending — no eligible node", {
|
|
24719
|
+
tags: { deviceId },
|
|
24720
|
+
meta: { reason: decision.reason }
|
|
24721
|
+
});
|
|
24328
24722
|
continue;
|
|
24329
24723
|
}
|
|
24330
24724
|
if (current && current.agentNodeId === decision.agentNodeId) continue;
|
|
@@ -24531,6 +24925,57 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24531
24925
|
return this.enabledNodes.includes(nodeId);
|
|
24532
24926
|
}
|
|
24533
24927
|
/**
|
|
24928
|
+
* The node that owns `deviceId`'s source pull (dials the real RTSP, hosts
|
|
24929
|
+
* the broker). **P2a (today): intentionally `undefined`** — per-camera
|
|
24930
|
+
* source ownership is not modeled yet, and an absent owner makes
|
|
24931
|
+
* `computeFrameSourceNodes` collapse to the global `enabledDecoderNodes`
|
|
24932
|
+
* (bit-identical to pre-P2a behavior, the safe-rollout invariant).
|
|
24933
|
+
*
|
|
24934
|
+
* Deliberately NOT `'hub'` yet: with an owner set and no
|
|
24935
|
+
* `remoteSourcingNodes`, the predicate would restrict eligibility to the
|
|
24936
|
+
* hub even when agents are decoder-enabled — a behavior change reserved for
|
|
24937
|
+
* P2d, which backs this seam with the `assignSource`/`getSourceAssignment`
|
|
24938
|
+
* cap surface (restream-owner design §2.3).
|
|
24939
|
+
*/
|
|
24940
|
+
sourceOwner(_deviceId) {}
|
|
24941
|
+
/**
|
|
24942
|
+
* The set of nodes that can OBTAIN a camera's decoded frames — PER-CAMERA
|
|
24943
|
+
* since P2a (restream-owner design §2.3), delegating to the pure
|
|
24944
|
+
* `computeFrameSourceNodes` predicate: a node qualifies iff it can decode
|
|
24945
|
+
* locally (`enabledDecoderNodes`) AND (it owns the camera's source pull OR
|
|
24946
|
+
* it may dial the owner's restream — `remoteSourcingNodes`, a P2c/P2d
|
|
24947
|
+
* rollout knob not wired yet).
|
|
24948
|
+
*
|
|
24949
|
+
* **Today's effective value:** `sourceOwner()` returns `undefined` for
|
|
24950
|
+
* every camera, so this is exactly `enabledDecoderNodes` — identical to the
|
|
24951
|
+
* pre-P2a global predicate. Every predicate change edits the pure module
|
|
24952
|
+
* (and this ONE method); `deviceId` is threaded from all placement sites so
|
|
24953
|
+
* later phases turn per-camera without re-touching call sites.
|
|
24954
|
+
*/
|
|
24955
|
+
frameSourceNodes(deviceId) {
|
|
24956
|
+
return computeFrameSourceNodes({
|
|
24957
|
+
enabledDecoderNodes: this.enabledDecoderNodes,
|
|
24958
|
+
sourceOwnerNodeId: this.sourceOwner(deviceId)
|
|
24959
|
+
});
|
|
24960
|
+
}
|
|
24961
|
+
/**
|
|
24962
|
+
* Nodes eligible to run a camera's detection pipeline: the whitelist of
|
|
24963
|
+
* detection-enabled nodes (`enabledNodes`) intersected with the nodes that
|
|
24964
|
+
* can obtain the camera's frames (`frameSourceNodes(deviceId)`). A node
|
|
24965
|
+
* that is detection-enabled but cannot source frames (or vice-versa) is
|
|
24966
|
+
* NEVER a valid placement — the balancer receives this as `eligibleNodes`
|
|
24967
|
+
* and treats anything outside it as unassignable (`no-frame-source`).
|
|
24968
|
+
*
|
|
24969
|
+
* Single choke point for T2/T10: passed to every `balance()` call site and
|
|
24970
|
+
* enforced by `assignPipeline` before persisting a pin. `deviceId` is
|
|
24971
|
+
* optional only for node-scoped checks (reconnect-restore guard); camera
|
|
24972
|
+
* placement sites always pass it.
|
|
24973
|
+
*/
|
|
24974
|
+
detectionEligibleNodes(deviceId) {
|
|
24975
|
+
const frameSource = this.frameSourceNodes(deviceId);
|
|
24976
|
+
return this.enabledNodes.filter((nodeId) => frameSource.includes(nodeId));
|
|
24977
|
+
}
|
|
24978
|
+
/**
|
|
24534
24979
|
* Query every online runner in the cluster for its current load, plus the
|
|
24535
24980
|
* local runner if co-located. Refreshes the `cachedAgentLoad` snapshot so
|
|
24536
24981
|
* `getAgentLoad()` / `getGlobalMetrics()` can return fresh data.
|
|
@@ -24631,8 +25076,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24631
25076
|
* Used by every `balance()` call so the balancer can honour operator-set
|
|
24632
25077
|
* per-node maximums without polling the store on each decision.
|
|
24633
25078
|
*/
|
|
24634
|
-
buildNodeCaps() {
|
|
24635
|
-
const blob = this.agentSettingsState.get();
|
|
25079
|
+
async buildNodeCaps() {
|
|
25080
|
+
const blob = await this.agentSettingsState.get();
|
|
24636
25081
|
const caps = {};
|
|
24637
25082
|
for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
|
|
24638
25083
|
return caps;
|
|
@@ -24696,6 +25141,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24696
25141
|
assignedAt: Date.now()
|
|
24697
25142
|
};
|
|
24698
25143
|
this.assignments.set(deviceId, assignment);
|
|
25144
|
+
this.pendingReasons.delete(deviceId);
|
|
24699
25145
|
if (!this.ctx?.eventBus) return;
|
|
24700
25146
|
const payload = {
|
|
24701
25147
|
deviceId,
|
|
@@ -24780,7 +25226,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24780
25226
|
const decision = balance({
|
|
24781
25227
|
nodes: loads,
|
|
24782
25228
|
preferredAgent: null,
|
|
24783
|
-
nodeCaps: this.buildNodeCaps()
|
|
25229
|
+
nodeCaps: await this.buildNodeCaps(),
|
|
25230
|
+
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
24784
25231
|
});
|
|
24785
25232
|
if (!decision) {
|
|
24786
25233
|
this.ctx.logger.error("Failover: no online runner", { tags: { deviceId } });
|
|
@@ -24788,7 +25235,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24788
25235
|
continue;
|
|
24789
25236
|
}
|
|
24790
25237
|
if (decision.kind === "pending") {
|
|
24791
|
-
this.
|
|
25238
|
+
this.pendingReasons.set(deviceId, decision.reason);
|
|
25239
|
+
this.ctx.logger.warn("Failover: camera left pending — no eligible node", {
|
|
25240
|
+
tags: { deviceId },
|
|
25241
|
+
meta: { reason: decision.reason }
|
|
25242
|
+
});
|
|
24792
25243
|
this.assignments.delete(deviceId);
|
|
24793
25244
|
continue;
|
|
24794
25245
|
}
|
|
@@ -24827,6 +25278,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24827
25278
|
tags: { nodeId },
|
|
24828
25279
|
meta: { policy: this.failoverPolicy.onReconnect }
|
|
24829
25280
|
});
|
|
25281
|
+
this.schedulePendingRetry();
|
|
24830
25282
|
if (this.failoverPolicy.onReconnect === "rebalance") {
|
|
24831
25283
|
await this.rebalance().catch((err) => {
|
|
24832
25284
|
const msg = errMsg(err);
|
|
@@ -24834,6 +25286,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24834
25286
|
});
|
|
24835
25287
|
return;
|
|
24836
25288
|
}
|
|
25289
|
+
if (!this.detectionEligibleNodes().includes(nodeId)) {
|
|
25290
|
+
this.ctx.logger.warn("restore skipped — node not frame-source eligible", {
|
|
25291
|
+
tags: { nodeId },
|
|
25292
|
+
meta: { eligibleNodes: this.detectionEligibleNodes().join(",") }
|
|
25293
|
+
});
|
|
25294
|
+
for (const [deviceId, assignment] of this.assignments) {
|
|
25295
|
+
if (assignment.agentNodeId !== nodeId) continue;
|
|
25296
|
+
this.pendingReasons.set(deviceId, "no-frame-source");
|
|
25297
|
+
}
|
|
25298
|
+
return;
|
|
25299
|
+
}
|
|
24837
25300
|
let restored = 0;
|
|
24838
25301
|
for (const [deviceId, config] of this.cameraConfigs) {
|
|
24839
25302
|
const current = this.assignments.get(deviceId);
|
|
@@ -24921,34 +25384,43 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
24921
25384
|
async reattachAudioForNode(nodeId) {
|
|
24922
25385
|
for (const [deviceId, audioNode] of this.audioNodeByDevice) {
|
|
24923
25386
|
if (audioNode !== nodeId) continue;
|
|
24924
|
-
|
|
24925
|
-
|
|
24926
|
-
|
|
24927
|
-
|
|
24928
|
-
|
|
24929
|
-
|
|
24930
|
-
|
|
24931
|
-
|
|
24932
|
-
|
|
24933
|
-
|
|
24934
|
-
|
|
24935
|
-
|
|
24936
|
-
|
|
24937
|
-
|
|
24938
|
-
|
|
24939
|
-
|
|
24940
|
-
|
|
24941
|
-
|
|
24942
|
-
|
|
24943
|
-
|
|
24944
|
-
|
|
24945
|
-
|
|
24946
|
-
|
|
24947
|
-
|
|
24948
|
-
|
|
24949
|
-
|
|
24950
|
-
|
|
24951
|
-
|
|
25387
|
+
await this.reattachAudioForDevice(deviceId);
|
|
25388
|
+
}
|
|
25389
|
+
}
|
|
25390
|
+
/**
|
|
25391
|
+
* Re-establish the audio subscription for ONE camera: tear down any prior
|
|
25392
|
+
* sub and re-subscribe under the current pin/balance, keeping the new handle
|
|
25393
|
+
* only while detection is active. The whole get→teardown→subscribe→store
|
|
25394
|
+
* sequence runs inside `withAudioSubLock` so a prior handle is never orphaned
|
|
25395
|
+
* (the same invariant `reattachAudioForNode` relied on before it was
|
|
25396
|
+
* extracted here). No-op when the device has no runner config or audio is
|
|
25397
|
+
* disabled/lazy (`subscribeAudioStream` self-gates).
|
|
25398
|
+
*/
|
|
25399
|
+
async reattachAudioForDevice(deviceId) {
|
|
25400
|
+
const config = this.cameraConfigs.get(deviceId);
|
|
25401
|
+
if (!config) return;
|
|
25402
|
+
const audioCfg = {
|
|
25403
|
+
...config,
|
|
25404
|
+
enabled: config.pipelineEnabled
|
|
25405
|
+
};
|
|
25406
|
+
try {
|
|
25407
|
+
await this.withAudioSubLock(deviceId, async () => {
|
|
25408
|
+
const prior = this.audioSubscriptions.get(deviceId);
|
|
25409
|
+
if (prior) {
|
|
25410
|
+
try {
|
|
25411
|
+
prior();
|
|
25412
|
+
} catch {}
|
|
25413
|
+
this.audioSubscriptions.delete(deviceId);
|
|
25414
|
+
}
|
|
25415
|
+
const unsub = await this.subscribeAudioStream(deviceId, audioCfg);
|
|
25416
|
+
if (unsub) if (this.activeDetections.has(deviceId)) this.storeAudioSub(deviceId, unsub);
|
|
25417
|
+
else unsub();
|
|
25418
|
+
});
|
|
25419
|
+
} catch (err) {
|
|
25420
|
+
this.ctx.logger.error("audio re-attach failed", {
|
|
25421
|
+
tags: { deviceId },
|
|
25422
|
+
meta: { error: errMsg(err) }
|
|
25423
|
+
});
|
|
24952
25424
|
}
|
|
24953
25425
|
}
|
|
24954
25426
|
/**
|
|
@@ -25023,6 +25495,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25023
25495
|
});
|
|
25024
25496
|
}, 3e3);
|
|
25025
25497
|
this.scheduleReconcile();
|
|
25498
|
+
this.schedulePendingRetry();
|
|
25026
25499
|
} catch (err) {
|
|
25027
25500
|
this.ctx.logger.debug("readiness seed+redispatch failed", {
|
|
25028
25501
|
tags: { nodeId },
|
|
@@ -25137,6 +25610,175 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25137
25610
|
}
|
|
25138
25611
|
}
|
|
25139
25612
|
}
|
|
25613
|
+
/**
|
|
25614
|
+
* Coalesce bursts of capacity/eligibility/readiness signals into a single
|
|
25615
|
+
* `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
|
|
25616
|
+
* dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
|
|
25617
|
+
* hammer `dispatchCamera`.
|
|
25618
|
+
*/
|
|
25619
|
+
schedulePendingRetry() {
|
|
25620
|
+
if (this.pendingRetryDebounceTimer !== null) clearTimeout(this.pendingRetryDebounceTimer);
|
|
25621
|
+
this.pendingRetryDebounceTimer = setTimeout(() => {
|
|
25622
|
+
this.pendingRetryDebounceTimer = null;
|
|
25623
|
+
this.retryPendingDispatches();
|
|
25624
|
+
}, PENDING_RETRY_DEBOUNCE_MS);
|
|
25625
|
+
}
|
|
25626
|
+
/**
|
|
25627
|
+
* Re-dispatch every KNOWN-but-UNASSIGNED camera. `reconcileDispatch` is
|
|
25628
|
+
* additive-only over `cameraConfigs` and never revisits a camera that is
|
|
25629
|
+
* tracked but has no live assignment (left pending by over-cap /
|
|
25630
|
+
* no-frame-source / load-shed), so those cameras would otherwise stay
|
|
25631
|
+
* stranded until an unrelated live event happens to touch them. This sweep
|
|
25632
|
+
* closes that gap.
|
|
25633
|
+
*
|
|
25634
|
+
* `dispatchCamera` re-reads pins, re-balances with the frame-source
|
|
25635
|
+
* predicate, and re-returns pending harmlessly when nothing changed —
|
|
25636
|
+
* so a no-op sweep is cheap and side-effect-free. Serialized with an
|
|
25637
|
+
* in-flight flag + trailing-rerun bit (mirrors `reconcileDispatch`).
|
|
25638
|
+
*/
|
|
25639
|
+
async retryPendingDispatches() {
|
|
25640
|
+
if (this.pendingRetryInFlight) {
|
|
25641
|
+
this.pendingRetryRerunRequested = true;
|
|
25642
|
+
return;
|
|
25643
|
+
}
|
|
25644
|
+
if (this.reconcileInFlight) return;
|
|
25645
|
+
if (!this.api) return;
|
|
25646
|
+
this.pendingRetryInFlight = true;
|
|
25647
|
+
try {
|
|
25648
|
+
const stranded = computeStrandedDevices(new Set(this.cameraConfigs.keys()), new Set(this.assignments.keys()));
|
|
25649
|
+
if (stranded.length > 0) {
|
|
25650
|
+
const loadShedActive = this.isAnyNodeLoadShed();
|
|
25651
|
+
let retried = 0;
|
|
25652
|
+
let stillPending = 0;
|
|
25653
|
+
for (const deviceId of stranded) {
|
|
25654
|
+
if (this.pendingReasons.get(deviceId) === "load-shed" && loadShedActive) {
|
|
25655
|
+
stillPending++;
|
|
25656
|
+
continue;
|
|
25657
|
+
}
|
|
25658
|
+
const cfg = this.cameraConfigs.get(deviceId);
|
|
25659
|
+
if (!cfg) continue;
|
|
25660
|
+
try {
|
|
25661
|
+
const result = await this.dispatchCamera(cfg);
|
|
25662
|
+
retried++;
|
|
25663
|
+
if (result.kind === "pending") stillPending++;
|
|
25664
|
+
} catch (err) {
|
|
25665
|
+
this.ctx.logger.warn("pending retry: dispatchCamera failed", {
|
|
25666
|
+
tags: { deviceId },
|
|
25667
|
+
meta: { error: errMsg(err) }
|
|
25668
|
+
});
|
|
25669
|
+
stillPending++;
|
|
25670
|
+
}
|
|
25671
|
+
}
|
|
25672
|
+
this.ctx.logger.info("pending retry sweep", { meta: {
|
|
25673
|
+
stranded: stranded.length,
|
|
25674
|
+
retried,
|
|
25675
|
+
stillPending
|
|
25676
|
+
} });
|
|
25677
|
+
}
|
|
25678
|
+
await this.evaluateRemoteAssignmentHealth();
|
|
25679
|
+
} finally {
|
|
25680
|
+
this.pendingRetryInFlight = false;
|
|
25681
|
+
if (this.pendingRetryRerunRequested) {
|
|
25682
|
+
this.pendingRetryRerunRequested = false;
|
|
25683
|
+
this.schedulePendingRetry();
|
|
25684
|
+
}
|
|
25685
|
+
}
|
|
25686
|
+
}
|
|
25687
|
+
/** True when any node is currently load-shed paused (used to skip churny retries). */
|
|
25688
|
+
isAnyNodeLoadShed() {
|
|
25689
|
+
for (const state of this.loadShedState.values()) if (state.pausedAt !== null) return true;
|
|
25690
|
+
return false;
|
|
25691
|
+
}
|
|
25692
|
+
/**
|
|
25693
|
+
* Evaluate the health of every REMOTE pipeline assignment and act on the
|
|
25694
|
+
* cameras the pure `evaluateRemoteHealth` flags. The hub watchdog covers
|
|
25695
|
+
* local cameras; this is its remote-node equivalent (gap i). Called from the
|
|
25696
|
+
* T3 sweep tick after retrying pending dispatches.
|
|
25697
|
+
*
|
|
25698
|
+
* `protected` so the remote-health orchestrator spec can drive one evaluation
|
|
25699
|
+
* pass deterministically (mirrors the `pendingReasons`/`audioSubLocks` test-seam
|
|
25700
|
+
* convention in this class) without waiting on the periodic timer.
|
|
25701
|
+
*/
|
|
25702
|
+
async evaluateRemoteAssignmentHealth() {
|
|
25703
|
+
if (!this.api) return;
|
|
25704
|
+
const actions = evaluateRemoteHealth({
|
|
25705
|
+
assignments: this.assignments,
|
|
25706
|
+
fpsMap: this.cameraFpsMap,
|
|
25707
|
+
activeDeviceIds: new Set(this.activeDetections.keys()),
|
|
25708
|
+
localNodeId: this.localNodeId,
|
|
25709
|
+
now: Date.now(),
|
|
25710
|
+
opts: DEFAULT_REMOTE_HEALTH_OPTS
|
|
25711
|
+
});
|
|
25712
|
+
for (const action of actions) await this.handleRemoteHealthReplace(action);
|
|
25713
|
+
}
|
|
25714
|
+
/**
|
|
25715
|
+
* Re-place a single unhealthy remote camera with bounded per-device backoff.
|
|
25716
|
+
* Under budget: detach from the unhealthy node + re-dispatch (inherits the
|
|
25717
|
+
* T2 frame-source predicate). Over budget: log an error, drop the dead
|
|
25718
|
+
* assignment, and mark the camera `'unhealthy'`-pending for the operator so
|
|
25719
|
+
* `getCameraStatus` surfaces it (R2 "bounded-retry ... state visible").
|
|
25720
|
+
*/
|
|
25721
|
+
async handleRemoteHealthReplace(action) {
|
|
25722
|
+
const { deviceId, why } = action;
|
|
25723
|
+
const assignment = this.assignments.get(deviceId);
|
|
25724
|
+
if (!assignment) return;
|
|
25725
|
+
const cfg = this.cameraConfigs.get(deviceId);
|
|
25726
|
+
if (!cfg) return;
|
|
25727
|
+
const now = Date.now();
|
|
25728
|
+
const prior = this.remoteHealthAttempts.get(deviceId);
|
|
25729
|
+
const windowFresh = prior !== void 0 && now - prior.windowStart < REMOTE_HEALTH_WINDOW_MS;
|
|
25730
|
+
const count = windowFresh ? prior.count : 0;
|
|
25731
|
+
if (count >= REMOTE_HEALTH_MAX_ATTEMPTS) {
|
|
25732
|
+
this.ctx.logger.error("remote assignment unhealthy — retries exhausted, leaving pending", {
|
|
25733
|
+
tags: {
|
|
25734
|
+
deviceId,
|
|
25735
|
+
nodeId: assignment.agentNodeId
|
|
25736
|
+
},
|
|
25737
|
+
meta: {
|
|
25738
|
+
why,
|
|
25739
|
+
attempts: count,
|
|
25740
|
+
windowMs: REMOTE_HEALTH_WINDOW_MS
|
|
25741
|
+
}
|
|
25742
|
+
});
|
|
25743
|
+
this.assignments.delete(deviceId);
|
|
25744
|
+
this.pendingReasons.set(deviceId, "unhealthy");
|
|
25745
|
+
await this.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
|
|
25746
|
+
this.ctx.logger.debug("remote-health exhausted-detach failed", {
|
|
25747
|
+
tags: { deviceId },
|
|
25748
|
+
meta: { error: errMsg(err) }
|
|
25749
|
+
});
|
|
25750
|
+
});
|
|
25751
|
+
return;
|
|
25752
|
+
}
|
|
25753
|
+
this.remoteHealthAttempts.set(deviceId, {
|
|
25754
|
+
count: count + 1,
|
|
25755
|
+
windowStart: windowFresh ? prior.windowStart : now
|
|
25756
|
+
});
|
|
25757
|
+
this.ctx.logger.warn("remote assignment unhealthy — re-placing camera", {
|
|
25758
|
+
tags: {
|
|
25759
|
+
deviceId,
|
|
25760
|
+
nodeId: assignment.agentNodeId
|
|
25761
|
+
},
|
|
25762
|
+
meta: {
|
|
25763
|
+
why,
|
|
25764
|
+
attempt: count + 1,
|
|
25765
|
+
maxAttempts: REMOTE_HEALTH_MAX_ATTEMPTS
|
|
25766
|
+
}
|
|
25767
|
+
});
|
|
25768
|
+
await this.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
|
|
25769
|
+
this.ctx.logger.warn("remote-health detach failed", {
|
|
25770
|
+
tags: { deviceId },
|
|
25771
|
+
meta: { error: errMsg(err) }
|
|
25772
|
+
});
|
|
25773
|
+
});
|
|
25774
|
+
this.assignments.delete(deviceId);
|
|
25775
|
+
await this.dispatchCamera(cfg).catch((err) => {
|
|
25776
|
+
this.ctx.logger.warn("remote-health re-dispatch failed", {
|
|
25777
|
+
tags: { deviceId },
|
|
25778
|
+
meta: { error: errMsg(err) }
|
|
25779
|
+
});
|
|
25780
|
+
});
|
|
25781
|
+
}
|
|
25140
25782
|
async getCapabilityBindings(input) {
|
|
25141
25783
|
if (!this.ctx?.settings) return {};
|
|
25142
25784
|
const perNodeRaw = (await this.nodeBindingsState.get().catch(() => ({})))[input.nodeId];
|
|
@@ -25230,12 +25872,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25230
25872
|
}
|
|
25231
25873
|
async assignAudio(input) {
|
|
25232
25874
|
await this.ctx.settings?.writeDeviceStore(input.deviceId, { [AUDIO_NODE_SETTING]: input.nodeId });
|
|
25875
|
+
if (!this.enabledAudioNodes.includes(input.nodeId)) this.ctx.logger.warn("audio pin stored but node is not in enabledAudioNodes — pin will not take effect until enabled", { tags: {
|
|
25876
|
+
deviceId: input.deviceId,
|
|
25877
|
+
nodeId: input.nodeId
|
|
25878
|
+
} });
|
|
25233
25879
|
this.audioAssignments.delete(input.deviceId);
|
|
25234
25880
|
this.audioNodeByDevice.delete(input.deviceId);
|
|
25235
25881
|
this.ctx.logger.info("Audio node pinned", { tags: {
|
|
25236
25882
|
deviceId: input.deviceId,
|
|
25237
25883
|
nodeId: input.nodeId
|
|
25238
25884
|
} });
|
|
25885
|
+
await this.reattachAudioForDevice(input.deviceId);
|
|
25239
25886
|
return { success: true };
|
|
25240
25887
|
}
|
|
25241
25888
|
async unassignAudio(input) {
|
|
@@ -25243,6 +25890,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25243
25890
|
this.audioAssignments.delete(input.deviceId);
|
|
25244
25891
|
this.audioNodeByDevice.delete(input.deviceId);
|
|
25245
25892
|
this.ctx.logger.info("Audio node unpinned", { tags: { deviceId: input.deviceId } });
|
|
25893
|
+
await this.reattachAudioForDevice(input.deviceId);
|
|
25246
25894
|
return { success: true };
|
|
25247
25895
|
}
|
|
25248
25896
|
async getAudioAssignment(input) {
|
|
@@ -25326,6 +25974,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25326
25974
|
tags: { nodeId: input.agentNodeId },
|
|
25327
25975
|
meta: { maxCameras: input.maxCameras }
|
|
25328
25976
|
});
|
|
25977
|
+
this.schedulePendingRetry();
|
|
25329
25978
|
return { success: true };
|
|
25330
25979
|
}
|
|
25331
25980
|
async getCameraSettings(input) {
|
|
@@ -25489,7 +26138,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25489
26138
|
const detectionNodeId = pipelineAssignment?.agentNodeId ?? null;
|
|
25490
26139
|
const decoderPinRaw = (api ? await this.ctx.settings?.readDeviceStore(deviceId).catch(() => ({})) ?? {} : {})["decoderNodeId"];
|
|
25491
26140
|
const decoderPinned = typeof decoderPinRaw === "string" && decoderPinRaw !== "auto";
|
|
25492
|
-
const
|
|
26141
|
+
const advisoryDecoderNodeId = detectionNodeId ? await this.resolveDecoderNode(deviceId, detectionNodeId).catch(() => null) : null;
|
|
25493
26142
|
const audioAssignment = this.audioAssignments.get(deviceId) ?? null;
|
|
25494
26143
|
const audioNodeId = audioAssignment?.nodeId ?? null;
|
|
25495
26144
|
const audioPinned = audioAssignment?.pinned ?? false;
|
|
@@ -25498,11 +26147,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25498
26147
|
decoder: decoderPinned,
|
|
25499
26148
|
audio: audioPinned
|
|
25500
26149
|
};
|
|
25501
|
-
const
|
|
25502
|
-
|
|
25503
|
-
decoder: decoderPinned ? "manual" : "co-located",
|
|
25504
|
-
audio: audioPinned ? "manual" : void 0
|
|
25505
|
-
};
|
|
26150
|
+
const detectionReason = pipelineAssignment !== null ? pipelineAssignment.reason : this.cameraConfigs.has(deviceId) ? `pending:${this.pendingReasons.get(deviceId) ?? "pending"}` : void 0;
|
|
26151
|
+
const liveDecoder = { nodeId: null };
|
|
25506
26152
|
const allSlotsFetch = api ? api.streamBroker.listAllProfileSlots.query() : null;
|
|
25507
26153
|
const sourceFetch = api && allSlotsFetch ? this.boundedStage(allSlotsFetch.then((slots) => {
|
|
25508
26154
|
return { streams: slots.filter((s) => s.deviceId === deviceId).map((s) => ({
|
|
@@ -25533,17 +26179,19 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25533
26179
|
clients: await api.streamBroker.listClients.query({ brokerId: slot.brokerId }).catch(() => null)
|
|
25534
26180
|
};
|
|
25535
26181
|
})), api.streamBroker.getAllRtspEntries.query({}).catch(() => null)]);
|
|
26182
|
+
const profileDetails = statsAndClients.map(({ slot, stats, clients }) => ({
|
|
26183
|
+
profile: slot.profile,
|
|
26184
|
+
status: slot.status,
|
|
26185
|
+
codec: stats?.codec ?? slot.codec ?? "",
|
|
26186
|
+
width: slot.resolution?.width ?? 0,
|
|
26187
|
+
height: slot.resolution?.height ?? 0,
|
|
26188
|
+
subscribers: clients?.encodedSubscribers ?? 0,
|
|
26189
|
+
inFps: stats?.inputFps ?? 0,
|
|
26190
|
+
outFps: stats?.decodeFps ?? 0
|
|
26191
|
+
}));
|
|
26192
|
+
for (const { stats } of statsAndClients) if (liveDecoder.nodeId === null && typeof stats?.decoderNodeId === "string") liveDecoder.nodeId = stats.decoderNodeId;
|
|
25536
26193
|
return {
|
|
25537
|
-
profiles:
|
|
25538
|
-
profile: slot.profile,
|
|
25539
|
-
status: slot.status,
|
|
25540
|
-
codec: stats?.codec ?? slot.codec ?? "",
|
|
25541
|
-
width: slot.resolution?.width ?? 0,
|
|
25542
|
-
height: slot.resolution?.height ?? 0,
|
|
25543
|
-
subscribers: clients?.encodedSubscribers ?? 0,
|
|
25544
|
-
inFps: stats?.inputFps ?? 0,
|
|
25545
|
-
outFps: stats?.decodeFps ?? 0
|
|
25546
|
-
})),
|
|
26194
|
+
profiles: profileDetails,
|
|
25547
26195
|
webrtcSessions: statsAndClients.reduce((total, { clients }) => {
|
|
25548
26196
|
if (!clients) return total;
|
|
25549
26197
|
return total + clients.encoded.filter((c) => WEBRTC_KINDS.has(c.attribution.kind)).length;
|
|
@@ -25553,8 +26201,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25553
26201
|
}) ?? false
|
|
25554
26202
|
};
|
|
25555
26203
|
}), STAGE_TIMEOUT_MS) : Promise.resolve(null);
|
|
25556
|
-
const decoderFetch =
|
|
25557
|
-
nodeId:
|
|
26204
|
+
const decoderFetch = advisoryDecoderNodeId ? Promise.resolve({
|
|
26205
|
+
nodeId: advisoryDecoderNodeId,
|
|
25558
26206
|
formats: [],
|
|
25559
26207
|
sessionCount: 0,
|
|
25560
26208
|
shm: {
|
|
@@ -25622,6 +26270,13 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25622
26270
|
detectionFetch,
|
|
25623
26271
|
recordingFetch
|
|
25624
26272
|
]);
|
|
26273
|
+
const liveDecoderNodeId = brokerResult !== null ? liveDecoder.nodeId : null;
|
|
26274
|
+
const decoderNodeId = liveDecoderNodeId ?? advisoryDecoderNodeId;
|
|
26275
|
+
const reasons = {
|
|
26276
|
+
detection: detectionReason,
|
|
26277
|
+
decoder: decoderPinned ? "manual" : liveDecoderNodeId !== null ? "session" : decoderNodeId !== null ? "advisory" : void 0,
|
|
26278
|
+
audio: audioPinned ? "manual" : void 0
|
|
26279
|
+
};
|
|
25625
26280
|
return composeCameraStatus({
|
|
25626
26281
|
deviceId,
|
|
25627
26282
|
fetchedAt: Date.now(),
|
|
@@ -25977,7 +26632,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25977
26632
|
id: "cluster",
|
|
25978
26633
|
title: "Cluster",
|
|
25979
26634
|
tab: "pipeline",
|
|
25980
|
-
description: "Which cluster nodes are eligible to run the detection pipeline.
|
|
26635
|
+
description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
|
|
25981
26636
|
fields: [
|
|
25982
26637
|
{
|
|
25983
26638
|
key: "enabledNodes",
|
|
@@ -26509,6 +27164,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26509
27164
|
this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
|
|
26510
27165
|
const rawEnabledAudio = config["enabledAudioNodes"];
|
|
26511
27166
|
this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
|
|
27167
|
+
this.schedulePendingRetry();
|
|
26512
27168
|
}
|
|
26513
27169
|
get api() {
|
|
26514
27170
|
return this.ctx.api ?? null;
|