@camstack/addon-pipeline-orchestrator 1.1.20 → 1.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -15703,7 +15703,17 @@ var AgentAddonConfigSchema = object({
15703
15703
  });
15704
15704
  var AgentPipelineSettingsSchema = object({
15705
15705
  addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
15706
- maxCameras: number().int().nonnegative().nullable().default(null)
15706
+ maxCameras: number().int().nonnegative().nullable().default(null),
15707
+ /** Per-node detection weight (relative share for the quota balancer). */
15708
+ detectWeight: number().positive().optional(),
15709
+ /** Node is eligible to run the detection pipeline (decode + inference). */
15710
+ detect: boolean().optional(),
15711
+ /** Node is eligible to host decoder sessions. */
15712
+ decode: boolean().optional(),
15713
+ /** Node is eligible to run audio-analyzer sessions. */
15714
+ audio: boolean().optional(),
15715
+ /** Node is eligible to be the ingest / source-owner (serve the restream). */
15716
+ ingest: boolean().optional()
15707
15717
  });
15708
15718
  var CameraPipelineForAgentSchema = object({
15709
15719
  steps: array(PipelineStepInputSchema).readonly(),
@@ -15778,7 +15788,10 @@ var AgentLoadSummarySchema = object({
15778
15788
  online: boolean(),
15779
15789
  load: RunnerLocalLoadSchema,
15780
15790
  /** Computed score used by the L2 capacity balancer (lower = less loaded). */
15781
- score: number()
15791
+ score: number(),
15792
+ /** This node's decode hwaccel backend (per-node `probedBestHwaccel`), or null
15793
+ * when not yet probed — for the cluster Pipeline table UI (P0.2). */
15794
+ decodeHwaccel: string().nullable()
15782
15795
  });
15783
15796
  /**
15784
15797
  * Aggregate metrics across the whole detection cluster. Replaces the legacy
@@ -16134,6 +16147,38 @@ var pipelineOrchestratorCapability = {
16134
16147
  kind: "mutation",
16135
16148
  auth: "admin"
16136
16149
  }),
16150
+ /**
16151
+ * Set a node's detection WEIGHT (relative share for the quota balancer).
16152
+ * `null` clears it (back to the implicit weight 1). Unpinned cameras
16153
+ * distribute proportionally to weight; `maxCameras` still clamps on top.
16154
+ */
16155
+ setAgentDetectWeight: method(object({
16156
+ agentNodeId: string(),
16157
+ detectWeight: number().positive().nullable()
16158
+ }), object({ success: literal(true) }), {
16159
+ kind: "mutation",
16160
+ auth: "admin"
16161
+ }),
16162
+ /**
16163
+ * Set one node's placement CAPABILITIES — the per-node record that
16164
+ * replaced the four flat orchestrator lists (`enabledNodes`,
16165
+ * `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
16166
+ * Each flag is optional in the patch: omit a flag to leave it unchanged,
16167
+ * pass `null` to reset it to the node default (`hub` → capable, every
16168
+ * other node → not). Detection/decode/audio eligibility is derived from
16169
+ * these flags across the cluster; changing them re-derives the enabled
16170
+ * sets and reconciles dispatch immediately.
16171
+ */
16172
+ setAgentCapabilities: method(object({
16173
+ agentNodeId: string(),
16174
+ detect: boolean().nullable().optional(),
16175
+ decode: boolean().nullable().optional(),
16176
+ audio: boolean().nullable().optional(),
16177
+ ingest: boolean().nullable().optional()
16178
+ }), object({ success: literal(true) }), {
16179
+ kind: "mutation",
16180
+ auth: "admin"
16181
+ }),
16137
16182
  /** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
16138
16183
  getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
16139
16184
  /** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
@@ -18489,7 +18534,7 @@ var HardwareEncoderIdSchema = _enum([
18489
18534
  "libx264",
18490
18535
  "libx265"
18491
18536
  ]);
18492
- var HardwareEncodersSchema = object({
18537
+ object({
18493
18538
  encoders: array(object({
18494
18539
  encoder: HardwareEncoderIdSchema,
18495
18540
  codec: _enum(["H264", "H265"]),
@@ -18508,15 +18553,7 @@ var HardwareEncodersSchema = object({
18508
18553
  defaultH265: HardwareEncoderIdSchema,
18509
18554
  probedAt: number()
18510
18555
  });
18511
- /**
18512
- * Decode-side companion to {@link HardwareEncodersSchema}: the `-hwaccel`
18513
- * methods the configured ffmpeg binary actually supports (parsed from
18514
- * `ffmpeg -hwaccels`). Used to gate decode-hwaccel attempts so the transcode
18515
- * egress never spends a spawn on a backend this build cannot offer. Per-stream
18516
- * decode fragility (e.g. VideoToolbox HEVC) is still handled by the egress's
18517
- * software fallback — this only filters out wholly-unsupported backends.
18518
- */
18519
- var HardwareDecodeAccelsSchema = object({
18556
+ object({
18520
18557
  methods: array(string()).readonly(),
18521
18558
  probedAt: number()
18522
18559
  });
@@ -18579,13 +18616,7 @@ var ResolvedInferenceConfigSchema = object({
18579
18616
  format: ModelFormatSchema,
18580
18617
  reason: string()
18581
18618
  });
18582
- method(_void(), PlatformCapabilitiesSchema), method(_void(), HardwareInfoSchema), method(object({ requirements: array(ModelRequirementSchema).readonly() }), ResolvedInferenceConfigSchema), method(object({ prefer: HwAccelBackendInputSchema }), HwAccelResolutionSchema), method(_void(), HardwareEncodersSchema), method(_void(), HardwareEncodersSchema, {
18583
- kind: "mutation",
18584
- auth: "admin"
18585
- }), method(_void(), HardwareDecodeAccelsSchema), method(_void(), HardwareDecodeAccelsSchema, {
18586
- kind: "mutation",
18587
- auth: "admin"
18588
- });
18619
+ method(_void(), PlatformCapabilitiesSchema), method(_void(), HardwareInfoSchema), method(object({ requirements: array(ModelRequirementSchema).readonly() }), ResolvedInferenceConfigSchema), method(object({ prefer: HwAccelBackendInputSchema }), HwAccelResolutionSchema);
18589
18620
  var PtzPresetSchema = object({
18590
18621
  id: string(),
18591
18622
  name: string()
@@ -22054,6 +22085,18 @@ Object.freeze({
22054
22085
  addonId: null,
22055
22086
  access: "create"
22056
22087
  },
22088
+ "pipelineOrchestrator.setAgentCapabilities": {
22089
+ capName: "pipeline-orchestrator",
22090
+ capScope: "system",
22091
+ addonId: null,
22092
+ access: "create"
22093
+ },
22094
+ "pipelineOrchestrator.setAgentDetectWeight": {
22095
+ capName: "pipeline-orchestrator",
22096
+ capScope: "system",
22097
+ addonId: null,
22098
+ access: "create"
22099
+ },
22057
22100
  "pipelineOrchestrator.setAgentMaxCameras": {
22058
22101
  capName: "pipeline-orchestrator",
22059
22102
  capScope: "system",
@@ -22210,30 +22253,6 @@ Object.freeze({
22210
22253
  addonId: null,
22211
22254
  access: "view"
22212
22255
  },
22213
- "platformProbe.getHardwareDecodeAccels": {
22214
- capName: "platform-probe",
22215
- capScope: "system",
22216
- addonId: null,
22217
- access: "view"
22218
- },
22219
- "platformProbe.getHardwareEncoders": {
22220
- capName: "platform-probe",
22221
- capScope: "system",
22222
- addonId: null,
22223
- access: "view"
22224
- },
22225
- "platformProbe.refreshHardwareDecodeAccels": {
22226
- capName: "platform-probe",
22227
- capScope: "system",
22228
- addonId: null,
22229
- access: "create"
22230
- },
22231
- "platformProbe.refreshHardwareEncoders": {
22232
- capName: "platform-probe",
22233
- capScope: "system",
22234
- addonId: null,
22235
- access: "create"
22236
- },
22237
22256
  "platformProbe.resolveHwAccel": {
22238
22257
  capName: "platform-probe",
22239
22258
  capScope: "system",
@@ -24221,6 +24240,23 @@ function computeCapacityScore(load) {
24221
24240
  return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
24222
24241
  }
24223
24242
  /**
24243
+ * Node detection weight, defaulting to 1 when absent/invalid. A higher weight
24244
+ * makes a node preferred proportionally more (see {@link BalancerInput.weights}).
24245
+ */
24246
+ function nodeWeight(nodeId, weights) {
24247
+ const w = weights?.[nodeId];
24248
+ return w !== void 0 && w > 0 ? w : 1;
24249
+ }
24250
+ /**
24251
+ * Weight-adjusted capacity score — the value the balancer minimises. Dividing
24252
+ * the raw capacity score by the node's weight makes a heavier node accumulate
24253
+ * proportionally more load before it loses preference, yielding weight-
24254
+ * proportional distribution while still preferring drained/idle nodes.
24255
+ */
24256
+ function weightedScore(load, weights) {
24257
+ return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
24258
+ }
24259
+ /**
24224
24260
  * Returns true when the node has remaining capacity.
24225
24261
  * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
24226
24262
  * its `attachedCameras` count is strictly less than the cap.
@@ -24286,8 +24322,9 @@ function balance(input) {
24286
24322
  };
24287
24323
  const best = eligible.map((node) => ({
24288
24324
  node,
24289
- score: computeCapacityScore(node)
24290
- })).toSorted((a, b) => a.score - b.score)[0];
24325
+ score: computeCapacityScore(node),
24326
+ sortKey: weightedScore(node, input.weights)
24327
+ })).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
24291
24328
  return {
24292
24329
  kind: "assigned",
24293
24330
  agentNodeId: best.node.nodeId,
@@ -24389,7 +24426,12 @@ var StoredAgentAddonConfigSchema = object({
24389
24426
  });
24390
24427
  var StoredAgentPipelineSettingsSchema = object({
24391
24428
  addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
24392
- maxCameras: number().int().nonnegative().nullable().default(null)
24429
+ maxCameras: number().int().nonnegative().nullable().default(null),
24430
+ detectWeight: number().positive().optional(),
24431
+ detect: boolean().optional(),
24432
+ decode: boolean().optional(),
24433
+ audio: boolean().optional(),
24434
+ ingest: boolean().optional()
24393
24435
  });
24394
24436
  var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
24395
24437
  var StoredCameraStepOverridePatchSchema = object({
@@ -24928,6 +24970,13 @@ var OrchestratorDiagnosticsSchema = object({
24928
24970
  knownRunnerNodes: array(string()),
24929
24971
  cachedAgentLoadNodeIds: array(string()),
24930
24972
  enabledNodes: array(string()),
24973
+ enabledDecoderNodes: array(string()),
24974
+ enabledAudioNodes: array(string()),
24975
+ clusterRoles: object({
24976
+ ingestNode: string(),
24977
+ recordingNode: string(),
24978
+ audioNode: string()
24979
+ }),
24931
24980
  assignedDeviceCount: number().int().min(0),
24932
24981
  cameraConfigCount: number().int().min(0),
24933
24982
  activeDetectionCount: number().int().min(0)
@@ -25044,36 +25093,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25044
25093
  failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
25045
25094
  /**
25046
25095
  * Hub-wide allow-list of node ids eligible to run the detection
25047
- * pipeline. Driven by the `enabledNodes` multiselect in the addon
25048
- * schema. Hydrated from the schema default (`['hub']`) on first
25049
- * boot; every other node must be explicitly added by the operator.
25050
- * The list is a strict whitelist — disabled nodes stay connected
25051
- * for other capabilities (metrics, logs, cluster-wide events) but
25052
- * never receive camera assignments.
25096
+ * pipeline. DERIVED (placement-model redesign) from the per-node
25097
+ * capability store (`agentSettings[node].detect`) no longer a flat
25098
+ * `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
25099
+ * on boot and on every capability write. The list is a strict whitelist —
25100
+ * disabled nodes stay connected for other capabilities (metrics, logs,
25101
+ * cluster-wide events) but never receive camera assignments.
25053
25102
  */
25054
25103
  enabledNodes = ["hub"];
25055
25104
  /**
25056
25105
  * Hub-wide allow-list of node ids eligible to run decoder sessions.
25057
- * Driven by the `enabledDecoderNodes` multiselect in the addon schema.
25106
+ * DERIVED from the per-node capability store (`agentSettings[node].decode`).
25058
25107
  * Defaults to `['hub']` — the hub always has a decoder available.
25059
25108
  */
25060
25109
  enabledDecoderNodes = ["hub"];
25061
25110
  /**
25062
- * Phase-2 rollout knob (P2d): nodes allowed to run the REMOTE-SOURCE leg —
25063
- * dial a camera's source-owner restream over the LAN and decode it locally.
25064
- * Driven by the `remoteSourcingNodes` multiselect in the addon schema.
25065
- * Default EMPTY = Phase-2 transport OFF: source ownership stays unmodeled
25066
- * and every behavior (eligibility, assignment, attach payload) is
25067
- * bit-identical to pre-Phase-2. See `source-owner.ts`.
25111
+ * Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
25112
+ * restream over the LAN and decode it locally. Held for the source-owner
25113
+ * plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
25114
+ * frame transport is a separate activation phase (placement-model §9 P1).
25115
+ * With this empty, `resolveSourceOwner` returns `undefined` and every
25116
+ * `selectRunnerFrameSource` emits `local-broker` bit-identical to
25117
+ * pre-Phase-2 behavior.
25068
25118
  */
25069
25119
  remoteSourcingNodes = [];
25070
25120
  /**
25071
25121
  * Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
25072
- * Driven by the `enabledAudioNodes` multiselect in the addon schema.
25122
+ * DERIVED from the per-node capability store (`agentSettings[node].audio`).
25073
25123
  * Defaults to `['hub']`.
25074
25124
  */
25075
25125
  enabledAudioNodes = ["hub"];
25076
25126
  /**
25127
+ * Cluster-wide singleton role assignments (placement-model §6). Each role
25128
+ * is the ONE node that serves it for every camera. Driven by the
25129
+ * `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
25130
+ * global schema; defaults to the hub for all three.
25131
+ */
25132
+ clusterRoles = {
25133
+ ingestNode: "hub",
25134
+ recordingNode: "hub",
25135
+ audioNode: "hub"
25136
+ };
25137
+ /**
25077
25138
  * Full global settings snapshot kept in memory so synchronous cap methods
25078
25139
  * (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
25079
25140
  * backing store. Refreshed in `initialize` and `onConfigChange`.
@@ -25142,6 +25203,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25142
25203
  loadShedState = /* @__PURE__ */ new Map();
25143
25204
  /** Timer for the auto-resume sweep. */
25144
25205
  loadShedResumeTimer = null;
25206
+ /** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
25207
+ shedInFlight = /* @__PURE__ */ new Set();
25145
25208
  /** Pending `scheduleReconcile` debounce timer. */
25146
25209
  reconcileTimer = null;
25147
25210
  /** True while `reconcileDispatch` is awaiting the RPC round-trip. */
@@ -25166,6 +25229,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25166
25229
  const stored = await this.resolveGlobalStore();
25167
25230
  this.globalSettings = { ...stored };
25168
25231
  this.applyRuntimeSettings(this.globalSettings);
25232
+ await this.refreshNodeCapabilities();
25169
25233
  } catch (err) {
25170
25234
  const msg = errMsg(err);
25171
25235
  this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
@@ -25581,6 +25645,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25581
25645
  knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
25582
25646
  cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
25583
25647
  enabledNodes: [...this.enabledNodes],
25648
+ enabledDecoderNodes: [...this.enabledDecoderNodes],
25649
+ enabledAudioNodes: [...this.enabledAudioNodes],
25650
+ clusterRoles: { ...this.clusterRoles },
25584
25651
  assignedDeviceCount: this.assignments.size,
25585
25652
  cameraConfigCount: this.cameraConfigs.size,
25586
25653
  activeDetectionCount: this.activeDetections.size
@@ -25705,6 +25772,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25705
25772
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25706
25773
  preferredAgent,
25707
25774
  nodeCaps: await this.buildNodeCaps(),
25775
+ weights: await this.buildNodeWeights(),
25708
25776
  eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
25709
25777
  });
25710
25778
  if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
@@ -25821,6 +25889,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25821
25889
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25822
25890
  preferredAgent: null,
25823
25891
  nodeCaps: await this.buildNodeCaps(),
25892
+ weights: await this.buildNodeWeights(),
25824
25893
  eligibleNodes: this.detectionEligibleNodes(input.deviceId)
25825
25894
  });
25826
25895
  if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
@@ -25869,6 +25938,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25869
25938
  if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
25870
25939
  const loads = await this.collectAgentLoad({ onlyEnabled: true });
25871
25940
  const nodeCaps = await this.buildNodeCaps();
25941
+ const nodeWeights = await this.buildNodeWeights();
25872
25942
  const attachedDelta = /* @__PURE__ */ new Map();
25873
25943
  const bumpAttached = (nodeId, by) => {
25874
25944
  attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
@@ -25890,6 +25960,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25890
25960
  nodes: talliedLoads(),
25891
25961
  preferredAgent,
25892
25962
  nodeCaps,
25963
+ weights: nodeWeights,
25893
25964
  eligibleNodes: this.detectionEligibleNodes(deviceId)
25894
25965
  });
25895
25966
  if (!decision) continue;
@@ -25966,6 +26037,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25966
26037
  * re-dispatching).
25967
26038
  */
25968
26039
  async getAgentLoad() {
26040
+ await this.refreshDecodeHwaccels();
25969
26041
  await this.collectAgentLoad();
25970
26042
  return [...this.cachedAgentLoad.values()];
25971
26043
  }
@@ -26231,11 +26303,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26231
26303
  nodeId: load.nodeId,
26232
26304
  online: true,
26233
26305
  load,
26234
- score: computeCapacityScore(load)
26306
+ score: computeCapacityScore(load),
26307
+ decodeHwaccel: this.decodeHwaccelByNode.get(load.nodeId) ?? null
26235
26308
  });
26236
26309
  this.cachedAgentLoad = next;
26237
26310
  }
26238
26311
  /**
26312
+ * Per-node decode hwaccel cache (`nodeId` → backend), refreshed by
26313
+ * {@link refreshDecodeHwaccels} before a UI-facing {@link getAgentLoad}. Read
26314
+ * from the decoder addon's per-node `probedBestHwaccel@<node>` setting — a
26315
+ * node-LOCAL probe published to a per-node setting — via `addonSettings`,
26316
+ * NEVER the `platform-probe` singleton (whose pin answers with hub hardware).
26317
+ */
26318
+ decodeHwaccelByNode = /* @__PURE__ */ new Map();
26319
+ /** Refresh {@link decodeHwaccelByNode} for every known runner node. */
26320
+ async refreshDecodeHwaccels() {
26321
+ await Promise.all([...this.knownRunnerNodes].map(async (nodeId) => {
26322
+ const hwaccel = await this.readNodeDecodeHwaccel(nodeId);
26323
+ if (hwaccel !== void 0) this.decodeHwaccelByNode.set(nodeId, hwaccel);
26324
+ }));
26325
+ }
26326
+ /**
26327
+ * Read a node's `probedBestHwaccel` from the decoder-ffmpeg addon's per-node
26328
+ * settings via the hub-routed `addonSettings` cap. Returns `null` when the
26329
+ * value is empty/unset, `undefined` on a read failure (keep the last known).
26330
+ * Mirrors {@link readDetectionPipelineEngine}.
26331
+ */
26332
+ async readNodeDecodeHwaccel(nodeId) {
26333
+ const api = this.ctx.api;
26334
+ if (!api?.addonSettings) return void 0;
26335
+ try {
26336
+ const schema = await api.addonSettings.getGlobalSettings.query({
26337
+ addonId: "decoder-ffmpeg",
26338
+ nodeId
26339
+ });
26340
+ if (!schema) return void 0;
26341
+ for (const s of schema.sections) for (const f of s.fields) if (f.key === "probedBestHwaccel") return typeof f.value === "string" && f.value.length > 0 ? f.value : null;
26342
+ return null;
26343
+ } catch {
26344
+ return;
26345
+ }
26346
+ }
26347
+ /**
26239
26348
  * Per-node `getLocalLoad` budget (ms). A WEDGED runner — transport up
26240
26349
  * enough to stay in the service registry, but whose `broker.call` neither
26241
26350
  * resolves nor rejects (observed live as
@@ -26307,6 +26416,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26307
26416
  for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
26308
26417
  return caps;
26309
26418
  }
26419
+ /**
26420
+ * Build a per-node detection WEIGHT map from the persisted agent settings —
26421
+ * the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
26422
+ * are simply absent (the balancer treats absent as weight 1). Passed to every
26423
+ * `balance()` call so unpinned cameras distribute proportionally to weight.
26424
+ */
26425
+ async buildNodeWeights() {
26426
+ const blob = await this.agentSettingsState.get();
26427
+ const weights = {};
26428
+ for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
26429
+ return weights;
26430
+ }
26310
26431
  async readAudioNodePin(deviceId) {
26311
26432
  if (!this.ctx?.settings) return null;
26312
26433
  try {
@@ -26455,6 +26576,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26455
26576
  nodes: loads,
26456
26577
  preferredAgent: null,
26457
26578
  nodeCaps: await this.buildNodeCaps(),
26579
+ weights: await this.buildNodeWeights(),
26458
26580
  eligibleNodes: this.detectionEligibleNodes(deviceId)
26459
26581
  });
26460
26582
  if (!decision) {
@@ -27182,6 +27304,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27182
27304
  };
27183
27305
  const { [input.agentNodeId]: _drop, ...rest } = all;
27184
27306
  await this.agentSettingsState.set(rest);
27307
+ await this.refreshNodeCapabilities();
27185
27308
  this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
27186
27309
  return {
27187
27310
  success: true,
@@ -27205,6 +27328,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27205
27328
  this.schedulePendingRetry();
27206
27329
  return { success: true };
27207
27330
  }
27331
+ async setAgentDetectWeight(input) {
27332
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
27333
+ const weight = input.detectWeight ?? 1;
27334
+ const next = existing ? {
27335
+ ...existing,
27336
+ detectWeight: weight
27337
+ } : {
27338
+ addonDefaults: {},
27339
+ maxCameras: null,
27340
+ detectWeight: weight
27341
+ };
27342
+ await this.writeAgentSettings(input.agentNodeId, next);
27343
+ this.ctx.logger.info("agentSettings.detectWeight updated", {
27344
+ tags: { nodeId: input.agentNodeId },
27345
+ meta: { detectWeight: weight ?? null }
27346
+ });
27347
+ this.rebalance().catch((err) => {
27348
+ this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
27349
+ });
27350
+ return { success: true };
27351
+ }
27352
+ /**
27353
+ * Set one node's placement capabilities — the per-node record that replaced
27354
+ * the four flat orchestrator lists. Each flag is a tri-state patch: omit to
27355
+ * leave unchanged, `null` to reset to the node default (persisted as an
27356
+ * absent flag), `true`/`false` to force. After persisting, re-derive the
27357
+ * enabled-node sets and reconcile dispatch so the change takes effect
27358
+ * immediately (reuses the existing balancer / retry, no custom logic).
27359
+ */
27360
+ async setAgentCapabilities(input) {
27361
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
27362
+ addonDefaults: {},
27363
+ maxCameras: null
27364
+ };
27365
+ const apply = (key) => {
27366
+ const patch = input[key];
27367
+ if (patch === void 0) return existing[key];
27368
+ if (patch === null) return void 0;
27369
+ return patch;
27370
+ };
27371
+ const next = {
27372
+ ...existing,
27373
+ detect: apply("detect"),
27374
+ decode: apply("decode"),
27375
+ audio: apply("audio"),
27376
+ ingest: apply("ingest")
27377
+ };
27378
+ await this.writeAgentSettings(input.agentNodeId, next);
27379
+ this.ctx.logger.info("agentSettings.capabilities updated", {
27380
+ tags: { nodeId: input.agentNodeId },
27381
+ meta: {
27382
+ detect: next.detect,
27383
+ decode: next.decode,
27384
+ audio: next.audio,
27385
+ ingest: next.ingest
27386
+ }
27387
+ });
27388
+ await this.refreshNodeCapabilities();
27389
+ this.knownRunnerNodes.add(input.agentNodeId);
27390
+ this.schedulePendingRetry();
27391
+ return { success: true };
27392
+ }
27208
27393
  async getCameraSettings(input) {
27209
27394
  return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
27210
27395
  }
@@ -27570,7 +27755,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27570
27755
  const all = await this.readAgentSettingsMap();
27571
27756
  const existing = all[nodeId];
27572
27757
  all[nodeId] = {
27573
- addonDefaults: settings.addonDefaults,
27758
+ ...existing,
27759
+ ...settings,
27574
27760
  maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
27575
27761
  };
27576
27762
  await this.agentSettingsState.set(all);
@@ -27853,45 +28039,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27853
28039
  const rand = Math.random().toString(36).slice(2, 8);
27854
28040
  return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
27855
28041
  }
27856
- /** Build the addon-level schema (cluster-wide tunables: balancer + failover). */
28042
+ /** Build the addon-level schema (cluster roles + balancer + failover). */
27857
28043
  globalSettingsSchema() {
27858
28044
  return this.schema({ sections: [
27859
28045
  {
27860
28046
  id: "cluster",
27861
- title: "Cluster",
28047
+ title: "Cluster Roles",
27862
28048
  tab: "pipeline",
27863
- description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
28049
+ description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
27864
28050
  fields: [
27865
28051
  {
27866
- key: "enabledNodes",
27867
- type: "node-multiselect",
27868
- label: "Enabled Detection Nodes",
27869
- description: "Only the selected nodes are passed to the load balancer. Disabled nodes remain connected for other capabilities (metrics, logs, cluster-wide events) but will never receive camera assignments.",
27870
- default: ["hub"],
27871
- showOffline: true
27872
- },
27873
- {
27874
- key: "enabledDecoderNodes",
27875
- type: "node-multiselect",
27876
- label: "Enabled Decoder Nodes",
27877
- description: "Nodes eligible to run decoder sessions.",
27878
- default: ["hub"],
28052
+ key: "ingestNode",
28053
+ type: "node-select",
28054
+ label: "Ingest Node",
28055
+ description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest hub is a later phase; kept here so the role is modeled.)",
28056
+ default: "hub",
27879
28057
  showOffline: true
27880
28058
  },
27881
28059
  {
27882
- key: "remoteSourcingNodes",
27883
- type: "node-multiselect",
27884
- label: "Remote Frame-Sourcing Nodes",
27885
- description: "Cross-node detection rollout (Phase 2): nodes allowed to pull a camera's compressed restream from its source-owner (the hub) and decode it locally for detection. Empty (default) = off — detection stays co-located with the frame source. A node must ALSO be in Enabled Decoder Nodes to qualify.",
27886
- default: [],
28060
+ key: "recordingNode",
28061
+ type: "node-select",
28062
+ label: "Recording Node",
28063
+ description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
28064
+ default: "hub",
27887
28065
  showOffline: true
27888
28066
  },
27889
28067
  {
27890
- key: "enabledAudioNodes",
27891
- type: "node-multiselect",
27892
- label: "Enabled Audio Nodes",
27893
- description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
27894
- default: ["hub"],
28068
+ key: "audioNode",
28069
+ type: "node-select",
28070
+ label: "Audio Node",
28071
+ description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
28072
+ default: "hub",
27895
28073
  showOffline: true
27896
28074
  }
27897
28075
  ]
@@ -28394,16 +28572,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28394
28572
  pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
28395
28573
  onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
28396
28574
  };
28397
- const rawEnabled = config["enabledNodes"];
28398
- this.enabledNodes = rawEnabled === void 0 ? ["hub"] : Array.isArray(rawEnabled) ? rawEnabled.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28399
- const rawEnabledDecoder = config["enabledDecoderNodes"];
28400
- this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28401
- const rawEnabledAudio = config["enabledAudioNodes"];
28402
- this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28403
- const rawRemoteSourcing = config["remoteSourcingNodes"];
28404
- this.remoteSourcingNodes = Array.isArray(rawRemoteSourcing) ? rawRemoteSourcing.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28575
+ const readRole = (key) => {
28576
+ const raw = config[key];
28577
+ return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
28578
+ };
28579
+ this.clusterRoles = {
28580
+ ingestNode: readRole("ingestNode"),
28581
+ recordingNode: readRole("recordingNode"),
28582
+ audioNode: readRole("audioNode")
28583
+ };
28405
28584
  this.schedulePendingRetry();
28406
28585
  }
28586
+ /**
28587
+ * Default placement capability for a node when its stored flag is unset:
28588
+ * the hub is capable of every role out of the box; every other node must be
28589
+ * explicitly enabled. This reproduces the classic `['hub']` defaults of the
28590
+ * four removed flat lists against an empty capability store.
28591
+ */
28592
+ static nodeCapabilityDefault(nodeId) {
28593
+ return nodeId === "hub";
28594
+ }
28595
+ /**
28596
+ * Recompute the derived enabled-node sets (`enabledNodes` /
28597
+ * `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
28598
+ * store. A node is eligible for a concern iff its stored flag is `true`, or
28599
+ * the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
28600
+ * Forked child nodeIds (`hub/classifier`) are never dispatchable and are
28601
+ * excluded. `hub` is always considered even when absent from the store, so a
28602
+ * fresh install keeps the hub-only cluster working.
28603
+ */
28604
+ async refreshNodeCapabilities() {
28605
+ let blob = {};
28606
+ try {
28607
+ blob = await this.agentSettingsState.get();
28608
+ } catch (err) {
28609
+ this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
28610
+ }
28611
+ const nodeIds = new Set(["hub"]);
28612
+ for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
28613
+ const detect = [];
28614
+ const decode = [];
28615
+ const audio = [];
28616
+ const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
28617
+ for (const nodeId of nodeIds) {
28618
+ const settings = blob[nodeId];
28619
+ if (capable(settings?.detect, nodeId)) detect.push(nodeId);
28620
+ if (capable(settings?.decode, nodeId)) decode.push(nodeId);
28621
+ if (capable(settings?.audio, nodeId)) audio.push(nodeId);
28622
+ }
28623
+ this.enabledNodes = detect.toSorted();
28624
+ this.enabledDecoderNodes = decode.toSorted();
28625
+ this.enabledAudioNodes = audio.toSorted();
28626
+ }
28407
28627
  get api() {
28408
28628
  return this.ctx.api ?? null;
28409
28629
  }
@@ -29000,7 +29220,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29000
29220
  });
29001
29221
  return;
29002
29222
  }
29003
- this.ctx.logger.warn("Load management: pausing camera detection (sustained low fps)", {
29223
+ if (this.shedInFlight.has(deviceId)) return;
29224
+ this.shedInFlight.add(deviceId);
29225
+ this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
29004
29226
  tags: { deviceId },
29005
29227
  meta: {
29006
29228
  actualFps: metrics.actualFps,
@@ -29009,7 +29231,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29009
29231
  windowMs
29010
29232
  }
29011
29233
  });
29012
- state.pausedAt = now;
29234
+ this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
29235
+ }
29236
+ /**
29237
+ * Cluster-wide load-shed action: relocate the sustained-slow camera to a
29238
+ * less-loaded node if the balancer (ceiling + weight aware) finds one,
29239
+ * otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
29240
+ * placement logic.
29241
+ */
29242
+ async shedOrRelocate(deviceId, state) {
29243
+ if (state.pausedAt !== null) return;
29244
+ if (await this.tryClusterRelocate(deviceId).catch((err) => {
29245
+ this.ctx.logger.warn("Load management: cluster relocate failed", {
29246
+ tags: { deviceId },
29247
+ meta: { error: errMsg(err) }
29248
+ });
29249
+ return false;
29250
+ })) {
29251
+ state.lowSinceTs = null;
29252
+ return;
29253
+ }
29254
+ this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
29255
+ state.pausedAt = Date.now();
29013
29256
  state.lowSinceTs = null;
29014
29257
  this.pipelineWatchdog?.unregister(deviceId);
29015
29258
  this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
@@ -29020,6 +29263,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29020
29263
  });
29021
29264
  this.ensureLoadShedResumeTimer();
29022
29265
  }
29266
+ /**
29267
+ * Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
29268
+ * Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
29269
+ * frame-source eligibility); the camera's current node counts it in its own
29270
+ * load, so a genuinely less-loaded node with spare capacity wins. Returns
29271
+ * `true` iff the camera was moved to a DIFFERENT node. A manually pinned
29272
+ * camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
29273
+ */
29274
+ async tryClusterRelocate(deviceId) {
29275
+ const current = this.assignments.get(deviceId);
29276
+ const cached = this.cameraConfigs.get(deviceId);
29277
+ if (!current || !cached) return false;
29278
+ const currentNode = current.agentNodeId;
29279
+ if (await this.readPipelinePin(deviceId)) return false;
29280
+ const decision = balance({
29281
+ nodes: await this.collectAgentLoad({ onlyEnabled: true }),
29282
+ preferredAgent: null,
29283
+ nodeCaps: await this.buildNodeCaps(),
29284
+ weights: await this.buildNodeWeights(),
29285
+ eligibleNodes: this.detectionEligibleNodes(deviceId)
29286
+ });
29287
+ if (!decision || decision.kind !== "assigned") return false;
29288
+ const target = decision.agentNodeId;
29289
+ if (target === currentNode) return false;
29290
+ await this.detachOn(currentNode, deviceId).catch((err) => {
29291
+ this.ctx.logger.debug("Load management: relocate detach-old failed", {
29292
+ tags: { deviceId },
29293
+ meta: {
29294
+ from: currentNode,
29295
+ error: errMsg(err)
29296
+ }
29297
+ });
29298
+ });
29299
+ await this.attachOn(target, cached);
29300
+ this.recordAssignment(deviceId, target, "rebalance", false);
29301
+ this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
29302
+ tags: { deviceId },
29303
+ meta: {
29304
+ from: currentNode,
29305
+ to: target,
29306
+ score: decision.score
29307
+ }
29308
+ });
29309
+ return true;
29310
+ }
29023
29311
  /** Base cooldown before first auto-resume attempt. */
29024
29312
  static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
29025
29313
  /** Maximum backoff cap. */