@camstack/addon-pipeline-orchestrator 1.1.20 → 1.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -15699,7 +15699,17 @@ var AgentAddonConfigSchema = object({
15699
15699
  });
15700
15700
  var AgentPipelineSettingsSchema = object({
15701
15701
  addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
15702
- maxCameras: number().int().nonnegative().nullable().default(null)
15702
+ maxCameras: number().int().nonnegative().nullable().default(null),
15703
+ /** Per-node detection weight (relative share for the quota balancer). */
15704
+ detectWeight: number().positive().optional(),
15705
+ /** Node is eligible to run the detection pipeline (decode + inference). */
15706
+ detect: boolean().optional(),
15707
+ /** Node is eligible to host decoder sessions. */
15708
+ decode: boolean().optional(),
15709
+ /** Node is eligible to run audio-analyzer sessions. */
15710
+ audio: boolean().optional(),
15711
+ /** Node is eligible to be the ingest / source-owner (serve the restream). */
15712
+ ingest: boolean().optional()
15703
15713
  });
15704
15714
  var CameraPipelineForAgentSchema = object({
15705
15715
  steps: array(PipelineStepInputSchema).readonly(),
@@ -15774,7 +15784,10 @@ var AgentLoadSummarySchema = object({
15774
15784
  online: boolean(),
15775
15785
  load: RunnerLocalLoadSchema,
15776
15786
  /** Computed score used by the L2 capacity balancer (lower = less loaded). */
15777
- score: number()
15787
+ score: number(),
15788
+ /** This node's decode hwaccel backend (per-node `probedBestHwaccel`), or null
15789
+ * when not yet probed — for the cluster Pipeline table UI (P0.2). */
15790
+ decodeHwaccel: string().nullable()
15778
15791
  });
15779
15792
  /**
15780
15793
  * Aggregate metrics across the whole detection cluster. Replaces the legacy
@@ -16130,6 +16143,38 @@ var pipelineOrchestratorCapability = {
16130
16143
  kind: "mutation",
16131
16144
  auth: "admin"
16132
16145
  }),
16146
+ /**
16147
+ * Set a node's detection WEIGHT (relative share for the quota balancer).
16148
+ * `null` clears it (back to the implicit weight 1). Unpinned cameras
16149
+ * distribute proportionally to weight; `maxCameras` still clamps on top.
16150
+ */
16151
+ setAgentDetectWeight: method(object({
16152
+ agentNodeId: string(),
16153
+ detectWeight: number().positive().nullable()
16154
+ }), object({ success: literal(true) }), {
16155
+ kind: "mutation",
16156
+ auth: "admin"
16157
+ }),
16158
+ /**
16159
+ * Set one node's placement CAPABILITIES — the per-node record that
16160
+ * replaced the four flat orchestrator lists (`enabledNodes`,
16161
+ * `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
16162
+ * Each flag is optional in the patch: omit a flag to leave it unchanged,
16163
+ * pass `null` to reset it to the node default (`hub` → capable, every
16164
+ * other node → not). Detection/decode/audio eligibility is derived from
16165
+ * these flags across the cluster; changing them re-derives the enabled
16166
+ * sets and reconciles dispatch immediately.
16167
+ */
16168
+ setAgentCapabilities: method(object({
16169
+ agentNodeId: string(),
16170
+ detect: boolean().nullable().optional(),
16171
+ decode: boolean().nullable().optional(),
16172
+ audio: boolean().nullable().optional(),
16173
+ ingest: boolean().nullable().optional()
16174
+ }), object({ success: literal(true) }), {
16175
+ kind: "mutation",
16176
+ auth: "admin"
16177
+ }),
16133
16178
  /** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
16134
16179
  getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
16135
16180
  /** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
@@ -18485,7 +18530,7 @@ var HardwareEncoderIdSchema = _enum([
18485
18530
  "libx264",
18486
18531
  "libx265"
18487
18532
  ]);
18488
- var HardwareEncodersSchema = object({
18533
+ object({
18489
18534
  encoders: array(object({
18490
18535
  encoder: HardwareEncoderIdSchema,
18491
18536
  codec: _enum(["H264", "H265"]),
@@ -18504,15 +18549,7 @@ var HardwareEncodersSchema = object({
18504
18549
  defaultH265: HardwareEncoderIdSchema,
18505
18550
  probedAt: number()
18506
18551
  });
18507
- /**
18508
- * Decode-side companion to {@link HardwareEncodersSchema}: the `-hwaccel`
18509
- * methods the configured ffmpeg binary actually supports (parsed from
18510
- * `ffmpeg -hwaccels`). Used to gate decode-hwaccel attempts so the transcode
18511
- * egress never spends a spawn on a backend this build cannot offer. Per-stream
18512
- * decode fragility (e.g. VideoToolbox HEVC) is still handled by the egress's
18513
- * software fallback — this only filters out wholly-unsupported backends.
18514
- */
18515
- var HardwareDecodeAccelsSchema = object({
18552
+ object({
18516
18553
  methods: array(string()).readonly(),
18517
18554
  probedAt: number()
18518
18555
  });
@@ -18575,13 +18612,7 @@ var ResolvedInferenceConfigSchema = object({
18575
18612
  format: ModelFormatSchema,
18576
18613
  reason: string()
18577
18614
  });
18578
- method(_void(), PlatformCapabilitiesSchema), method(_void(), HardwareInfoSchema), method(object({ requirements: array(ModelRequirementSchema).readonly() }), ResolvedInferenceConfigSchema), method(object({ prefer: HwAccelBackendInputSchema }), HwAccelResolutionSchema), method(_void(), HardwareEncodersSchema), method(_void(), HardwareEncodersSchema, {
18579
- kind: "mutation",
18580
- auth: "admin"
18581
- }), method(_void(), HardwareDecodeAccelsSchema), method(_void(), HardwareDecodeAccelsSchema, {
18582
- kind: "mutation",
18583
- auth: "admin"
18584
- });
18615
+ method(_void(), PlatformCapabilitiesSchema), method(_void(), HardwareInfoSchema), method(object({ requirements: array(ModelRequirementSchema).readonly() }), ResolvedInferenceConfigSchema), method(object({ prefer: HwAccelBackendInputSchema }), HwAccelResolutionSchema);
18585
18616
  var PtzPresetSchema = object({
18586
18617
  id: string(),
18587
18618
  name: string()
@@ -22050,6 +22081,18 @@ Object.freeze({
22050
22081
  addonId: null,
22051
22082
  access: "create"
22052
22083
  },
22084
+ "pipelineOrchestrator.setAgentCapabilities": {
22085
+ capName: "pipeline-orchestrator",
22086
+ capScope: "system",
22087
+ addonId: null,
22088
+ access: "create"
22089
+ },
22090
+ "pipelineOrchestrator.setAgentDetectWeight": {
22091
+ capName: "pipeline-orchestrator",
22092
+ capScope: "system",
22093
+ addonId: null,
22094
+ access: "create"
22095
+ },
22053
22096
  "pipelineOrchestrator.setAgentMaxCameras": {
22054
22097
  capName: "pipeline-orchestrator",
22055
22098
  capScope: "system",
@@ -22206,30 +22249,6 @@ Object.freeze({
22206
22249
  addonId: null,
22207
22250
  access: "view"
22208
22251
  },
22209
- "platformProbe.getHardwareDecodeAccels": {
22210
- capName: "platform-probe",
22211
- capScope: "system",
22212
- addonId: null,
22213
- access: "view"
22214
- },
22215
- "platformProbe.getHardwareEncoders": {
22216
- capName: "platform-probe",
22217
- capScope: "system",
22218
- addonId: null,
22219
- access: "view"
22220
- },
22221
- "platformProbe.refreshHardwareDecodeAccels": {
22222
- capName: "platform-probe",
22223
- capScope: "system",
22224
- addonId: null,
22225
- access: "create"
22226
- },
22227
- "platformProbe.refreshHardwareEncoders": {
22228
- capName: "platform-probe",
22229
- capScope: "system",
22230
- addonId: null,
22231
- access: "create"
22232
- },
22233
22252
  "platformProbe.resolveHwAccel": {
22234
22253
  capName: "platform-probe",
22235
22254
  capScope: "system",
@@ -24217,6 +24236,23 @@ function computeCapacityScore(load) {
24217
24236
  return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
24218
24237
  }
24219
24238
  /**
24239
+ * Node detection weight, defaulting to 1 when absent/invalid. A higher weight
24240
+ * makes a node preferred proportionally more (see {@link BalancerInput.weights}).
24241
+ */
24242
+ function nodeWeight(nodeId, weights) {
24243
+ const w = weights?.[nodeId];
24244
+ return w !== void 0 && w > 0 ? w : 1;
24245
+ }
24246
+ /**
24247
+ * Weight-adjusted capacity score — the value the balancer minimises. Dividing
24248
+ * the raw capacity score by the node's weight makes a heavier node accumulate
24249
+ * proportionally more load before it loses preference, yielding weight-
24250
+ * proportional distribution while still preferring drained/idle nodes.
24251
+ */
24252
+ function weightedScore(load, weights) {
24253
+ return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
24254
+ }
24255
+ /**
24220
24256
  * Returns true when the node has remaining capacity.
24221
24257
  * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
24222
24258
  * its `attachedCameras` count is strictly less than the cap.
@@ -24282,8 +24318,9 @@ function balance(input) {
24282
24318
  };
24283
24319
  const best = eligible.map((node) => ({
24284
24320
  node,
24285
- score: computeCapacityScore(node)
24286
- })).toSorted((a, b) => a.score - b.score)[0];
24321
+ score: computeCapacityScore(node),
24322
+ sortKey: weightedScore(node, input.weights)
24323
+ })).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
24287
24324
  return {
24288
24325
  kind: "assigned",
24289
24326
  agentNodeId: best.node.nodeId,
@@ -24385,7 +24422,12 @@ var StoredAgentAddonConfigSchema = object({
24385
24422
  });
24386
24423
  var StoredAgentPipelineSettingsSchema = object({
24387
24424
  addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
24388
- maxCameras: number().int().nonnegative().nullable().default(null)
24425
+ maxCameras: number().int().nonnegative().nullable().default(null),
24426
+ detectWeight: number().positive().optional(),
24427
+ detect: boolean().optional(),
24428
+ decode: boolean().optional(),
24429
+ audio: boolean().optional(),
24430
+ ingest: boolean().optional()
24389
24431
  });
24390
24432
  var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
24391
24433
  var StoredCameraStepOverridePatchSchema = object({
@@ -24924,6 +24966,13 @@ var OrchestratorDiagnosticsSchema = object({
24924
24966
  knownRunnerNodes: array(string()),
24925
24967
  cachedAgentLoadNodeIds: array(string()),
24926
24968
  enabledNodes: array(string()),
24969
+ enabledDecoderNodes: array(string()),
24970
+ enabledAudioNodes: array(string()),
24971
+ clusterRoles: object({
24972
+ ingestNode: string(),
24973
+ recordingNode: string(),
24974
+ audioNode: string()
24975
+ }),
24927
24976
  assignedDeviceCount: number().int().min(0),
24928
24977
  cameraConfigCount: number().int().min(0),
24929
24978
  activeDetectionCount: number().int().min(0)
@@ -25040,36 +25089,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25040
25089
  failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
25041
25090
  /**
25042
25091
  * Hub-wide allow-list of node ids eligible to run the detection
25043
- * pipeline. Driven by the `enabledNodes` multiselect in the addon
25044
- * schema. Hydrated from the schema default (`['hub']`) on first
25045
- * boot; every other node must be explicitly added by the operator.
25046
- * The list is a strict whitelist — disabled nodes stay connected
25047
- * for other capabilities (metrics, logs, cluster-wide events) but
25048
- * never receive camera assignments.
25092
+ * pipeline. DERIVED (placement-model redesign) from the per-node
25093
+ * capability store (`agentSettings[node].detect`) no longer a flat
25094
+ * `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
25095
+ * on boot and on every capability write. The list is a strict whitelist —
25096
+ * disabled nodes stay connected for other capabilities (metrics, logs,
25097
+ * cluster-wide events) but never receive camera assignments.
25049
25098
  */
25050
25099
  enabledNodes = ["hub"];
25051
25100
  /**
25052
25101
  * Hub-wide allow-list of node ids eligible to run decoder sessions.
25053
- * Driven by the `enabledDecoderNodes` multiselect in the addon schema.
25102
+ * DERIVED from the per-node capability store (`agentSettings[node].decode`).
25054
25103
  * Defaults to `['hub']` — the hub always has a decoder available.
25055
25104
  */
25056
25105
  enabledDecoderNodes = ["hub"];
25057
25106
  /**
25058
- * Phase-2 rollout knob (P2d): nodes allowed to run the REMOTE-SOURCE leg —
25059
- * dial a camera's source-owner restream over the LAN and decode it locally.
25060
- * Driven by the `remoteSourcingNodes` multiselect in the addon schema.
25061
- * Default EMPTY = Phase-2 transport OFF: source ownership stays unmodeled
25062
- * and every behavior (eligibility, assignment, attach payload) is
25063
- * bit-identical to pre-Phase-2. See `source-owner.ts`.
25107
+ * Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
25108
+ * restream over the LAN and decode it locally. Held for the source-owner
25109
+ * plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
25110
+ * frame transport is a separate activation phase (placement-model §9 P1).
25111
+ * With this empty, `resolveSourceOwner` returns `undefined` and every
25112
+ * `selectRunnerFrameSource` emits `local-broker` bit-identical to
25113
+ * pre-Phase-2 behavior.
25064
25114
  */
25065
25115
  remoteSourcingNodes = [];
25066
25116
  /**
25067
25117
  * Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
25068
- * Driven by the `enabledAudioNodes` multiselect in the addon schema.
25118
+ * DERIVED from the per-node capability store (`agentSettings[node].audio`).
25069
25119
  * Defaults to `['hub']`.
25070
25120
  */
25071
25121
  enabledAudioNodes = ["hub"];
25072
25122
  /**
25123
+ * Cluster-wide singleton role assignments (placement-model §6). Each role
25124
+ * is the ONE node that serves it for every camera. Driven by the
25125
+ * `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
25126
+ * global schema; defaults to the hub for all three.
25127
+ */
25128
+ clusterRoles = {
25129
+ ingestNode: "hub",
25130
+ recordingNode: "hub",
25131
+ audioNode: "hub"
25132
+ };
25133
+ /**
25073
25134
  * Full global settings snapshot kept in memory so synchronous cap methods
25074
25135
  * (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
25075
25136
  * backing store. Refreshed in `initialize` and `onConfigChange`.
@@ -25138,6 +25199,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25138
25199
  loadShedState = /* @__PURE__ */ new Map();
25139
25200
  /** Timer for the auto-resume sweep. */
25140
25201
  loadShedResumeTimer = null;
25202
+ /** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
25203
+ shedInFlight = /* @__PURE__ */ new Set();
25141
25204
  /** Pending `scheduleReconcile` debounce timer. */
25142
25205
  reconcileTimer = null;
25143
25206
  /** True while `reconcileDispatch` is awaiting the RPC round-trip. */
@@ -25162,6 +25225,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25162
25225
  const stored = await this.resolveGlobalStore();
25163
25226
  this.globalSettings = { ...stored };
25164
25227
  this.applyRuntimeSettings(this.globalSettings);
25228
+ await this.refreshNodeCapabilities();
25165
25229
  } catch (err) {
25166
25230
  const msg = errMsg(err);
25167
25231
  this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
@@ -25577,6 +25641,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25577
25641
  knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
25578
25642
  cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
25579
25643
  enabledNodes: [...this.enabledNodes],
25644
+ enabledDecoderNodes: [...this.enabledDecoderNodes],
25645
+ enabledAudioNodes: [...this.enabledAudioNodes],
25646
+ clusterRoles: { ...this.clusterRoles },
25580
25647
  assignedDeviceCount: this.assignments.size,
25581
25648
  cameraConfigCount: this.cameraConfigs.size,
25582
25649
  activeDetectionCount: this.activeDetections.size
@@ -25701,6 +25768,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25701
25768
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25702
25769
  preferredAgent,
25703
25770
  nodeCaps: await this.buildNodeCaps(),
25771
+ weights: await this.buildNodeWeights(),
25704
25772
  eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
25705
25773
  });
25706
25774
  if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
@@ -25817,6 +25885,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25817
25885
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25818
25886
  preferredAgent: null,
25819
25887
  nodeCaps: await this.buildNodeCaps(),
25888
+ weights: await this.buildNodeWeights(),
25820
25889
  eligibleNodes: this.detectionEligibleNodes(input.deviceId)
25821
25890
  });
25822
25891
  if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
@@ -25865,6 +25934,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25865
25934
  if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
25866
25935
  const loads = await this.collectAgentLoad({ onlyEnabled: true });
25867
25936
  const nodeCaps = await this.buildNodeCaps();
25937
+ const nodeWeights = await this.buildNodeWeights();
25868
25938
  const attachedDelta = /* @__PURE__ */ new Map();
25869
25939
  const bumpAttached = (nodeId, by) => {
25870
25940
  attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
@@ -25886,6 +25956,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25886
25956
  nodes: talliedLoads(),
25887
25957
  preferredAgent,
25888
25958
  nodeCaps,
25959
+ weights: nodeWeights,
25889
25960
  eligibleNodes: this.detectionEligibleNodes(deviceId)
25890
25961
  });
25891
25962
  if (!decision) continue;
@@ -25962,6 +26033,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25962
26033
  * re-dispatching).
25963
26034
  */
25964
26035
  async getAgentLoad() {
26036
+ await this.refreshDecodeHwaccels();
25965
26037
  await this.collectAgentLoad();
25966
26038
  return [...this.cachedAgentLoad.values()];
25967
26039
  }
@@ -26227,11 +26299,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26227
26299
  nodeId: load.nodeId,
26228
26300
  online: true,
26229
26301
  load,
26230
- score: computeCapacityScore(load)
26302
+ score: computeCapacityScore(load),
26303
+ decodeHwaccel: this.decodeHwaccelByNode.get(load.nodeId) ?? null
26231
26304
  });
26232
26305
  this.cachedAgentLoad = next;
26233
26306
  }
26234
26307
  /**
26308
+ * Per-node decode hwaccel cache (`nodeId` → backend), refreshed by
26309
+ * {@link refreshDecodeHwaccels} before a UI-facing {@link getAgentLoad}. Read
26310
+ * from the decoder addon's per-node `probedBestHwaccel@<node>` setting — a
26311
+ * node-LOCAL probe published to a per-node setting — via `addonSettings`,
26312
+ * NEVER the `platform-probe` singleton (whose pin answers with hub hardware).
26313
+ */
26314
+ decodeHwaccelByNode = /* @__PURE__ */ new Map();
26315
+ /** Refresh {@link decodeHwaccelByNode} for every known runner node. */
26316
+ async refreshDecodeHwaccels() {
26317
+ await Promise.all([...this.knownRunnerNodes].map(async (nodeId) => {
26318
+ const hwaccel = await this.readNodeDecodeHwaccel(nodeId);
26319
+ if (hwaccel !== void 0) this.decodeHwaccelByNode.set(nodeId, hwaccel);
26320
+ }));
26321
+ }
26322
+ /**
26323
+ * Read a node's `probedBestHwaccel` from the decoder-ffmpeg addon's per-node
26324
+ * settings via the hub-routed `addonSettings` cap. Returns `null` when the
26325
+ * value is empty/unset, `undefined` on a read failure (keep the last known).
26326
+ * Mirrors {@link readDetectionPipelineEngine}.
26327
+ */
26328
+ async readNodeDecodeHwaccel(nodeId) {
26329
+ const api = this.ctx.api;
26330
+ if (!api?.addonSettings) return void 0;
26331
+ try {
26332
+ const schema = await api.addonSettings.getGlobalSettings.query({
26333
+ addonId: "decoder-ffmpeg",
26334
+ nodeId
26335
+ });
26336
+ if (!schema) return void 0;
26337
+ for (const s of schema.sections) for (const f of s.fields) if (f.key === "probedBestHwaccel") return typeof f.value === "string" && f.value.length > 0 ? f.value : null;
26338
+ return null;
26339
+ } catch {
26340
+ return;
26341
+ }
26342
+ }
26343
+ /**
26235
26344
  * Per-node `getLocalLoad` budget (ms). A WEDGED runner — transport up
26236
26345
  * enough to stay in the service registry, but whose `broker.call` neither
26237
26346
  * resolves nor rejects (observed live as
@@ -26303,6 +26412,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26303
26412
  for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
26304
26413
  return caps;
26305
26414
  }
26415
+ /**
26416
+ * Build a per-node detection WEIGHT map from the persisted agent settings —
26417
+ * the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
26418
+ * are simply absent (the balancer treats absent as weight 1). Passed to every
26419
+ * `balance()` call so unpinned cameras distribute proportionally to weight.
26420
+ */
26421
+ async buildNodeWeights() {
26422
+ const blob = await this.agentSettingsState.get();
26423
+ const weights = {};
26424
+ for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
26425
+ return weights;
26426
+ }
26306
26427
  async readAudioNodePin(deviceId) {
26307
26428
  if (!this.ctx?.settings) return null;
26308
26429
  try {
@@ -26451,6 +26572,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26451
26572
  nodes: loads,
26452
26573
  preferredAgent: null,
26453
26574
  nodeCaps: await this.buildNodeCaps(),
26575
+ weights: await this.buildNodeWeights(),
26454
26576
  eligibleNodes: this.detectionEligibleNodes(deviceId)
26455
26577
  });
26456
26578
  if (!decision) {
@@ -27178,6 +27300,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27178
27300
  };
27179
27301
  const { [input.agentNodeId]: _drop, ...rest } = all;
27180
27302
  await this.agentSettingsState.set(rest);
27303
+ await this.refreshNodeCapabilities();
27181
27304
  this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
27182
27305
  return {
27183
27306
  success: true,
@@ -27201,6 +27324,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27201
27324
  this.schedulePendingRetry();
27202
27325
  return { success: true };
27203
27326
  }
27327
+ async setAgentDetectWeight(input) {
27328
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
27329
+ const weight = input.detectWeight ?? 1;
27330
+ const next = existing ? {
27331
+ ...existing,
27332
+ detectWeight: weight
27333
+ } : {
27334
+ addonDefaults: {},
27335
+ maxCameras: null,
27336
+ detectWeight: weight
27337
+ };
27338
+ await this.writeAgentSettings(input.agentNodeId, next);
27339
+ this.ctx.logger.info("agentSettings.detectWeight updated", {
27340
+ tags: { nodeId: input.agentNodeId },
27341
+ meta: { detectWeight: weight ?? null }
27342
+ });
27343
+ this.rebalance().catch((err) => {
27344
+ this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
27345
+ });
27346
+ return { success: true };
27347
+ }
27348
+ /**
27349
+ * Set one node's placement capabilities — the per-node record that replaced
27350
+ * the four flat orchestrator lists. Each flag is a tri-state patch: omit to
27351
+ * leave unchanged, `null` to reset to the node default (persisted as an
27352
+ * absent flag), `true`/`false` to force. After persisting, re-derive the
27353
+ * enabled-node sets and reconcile dispatch so the change takes effect
27354
+ * immediately (reuses the existing balancer / retry, no custom logic).
27355
+ */
27356
+ async setAgentCapabilities(input) {
27357
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
27358
+ addonDefaults: {},
27359
+ maxCameras: null
27360
+ };
27361
+ const apply = (key) => {
27362
+ const patch = input[key];
27363
+ if (patch === void 0) return existing[key];
27364
+ if (patch === null) return void 0;
27365
+ return patch;
27366
+ };
27367
+ const next = {
27368
+ ...existing,
27369
+ detect: apply("detect"),
27370
+ decode: apply("decode"),
27371
+ audio: apply("audio"),
27372
+ ingest: apply("ingest")
27373
+ };
27374
+ await this.writeAgentSettings(input.agentNodeId, next);
27375
+ this.ctx.logger.info("agentSettings.capabilities updated", {
27376
+ tags: { nodeId: input.agentNodeId },
27377
+ meta: {
27378
+ detect: next.detect,
27379
+ decode: next.decode,
27380
+ audio: next.audio,
27381
+ ingest: next.ingest
27382
+ }
27383
+ });
27384
+ await this.refreshNodeCapabilities();
27385
+ this.knownRunnerNodes.add(input.agentNodeId);
27386
+ this.schedulePendingRetry();
27387
+ return { success: true };
27388
+ }
27204
27389
  async getCameraSettings(input) {
27205
27390
  return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
27206
27391
  }
@@ -27566,7 +27751,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27566
27751
  const all = await this.readAgentSettingsMap();
27567
27752
  const existing = all[nodeId];
27568
27753
  all[nodeId] = {
27569
- addonDefaults: settings.addonDefaults,
27754
+ ...existing,
27755
+ ...settings,
27570
27756
  maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
27571
27757
  };
27572
27758
  await this.agentSettingsState.set(all);
@@ -27849,45 +28035,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27849
28035
  const rand = Math.random().toString(36).slice(2, 8);
27850
28036
  return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
27851
28037
  }
27852
- /** Build the addon-level schema (cluster-wide tunables: balancer + failover). */
28038
+ /** Build the addon-level schema (cluster roles + balancer + failover). */
27853
28039
  globalSettingsSchema() {
27854
28040
  return this.schema({ sections: [
27855
28041
  {
27856
28042
  id: "cluster",
27857
- title: "Cluster",
28043
+ title: "Cluster Roles",
27858
28044
  tab: "pipeline",
27859
- description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
28045
+ description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
27860
28046
  fields: [
27861
28047
  {
27862
- key: "enabledNodes",
27863
- type: "node-multiselect",
27864
- label: "Enabled Detection Nodes",
27865
- description: "Only the selected nodes are passed to the load balancer. Disabled nodes remain connected for other capabilities (metrics, logs, cluster-wide events) but will never receive camera assignments.",
27866
- default: ["hub"],
27867
- showOffline: true
27868
- },
27869
- {
27870
- key: "enabledDecoderNodes",
27871
- type: "node-multiselect",
27872
- label: "Enabled Decoder Nodes",
27873
- description: "Nodes eligible to run decoder sessions.",
27874
- default: ["hub"],
28048
+ key: "ingestNode",
28049
+ type: "node-select",
28050
+ label: "Ingest Node",
28051
+ description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest hub is a later phase; kept here so the role is modeled.)",
28052
+ default: "hub",
27875
28053
  showOffline: true
27876
28054
  },
27877
28055
  {
27878
- key: "remoteSourcingNodes",
27879
- type: "node-multiselect",
27880
- label: "Remote Frame-Sourcing Nodes",
27881
- description: "Cross-node detection rollout (Phase 2): nodes allowed to pull a camera's compressed restream from its source-owner (the hub) and decode it locally for detection. Empty (default) = off — detection stays co-located with the frame source. A node must ALSO be in Enabled Decoder Nodes to qualify.",
27882
- default: [],
28056
+ key: "recordingNode",
28057
+ type: "node-select",
28058
+ label: "Recording Node",
28059
+ description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
28060
+ default: "hub",
27883
28061
  showOffline: true
27884
28062
  },
27885
28063
  {
27886
- key: "enabledAudioNodes",
27887
- type: "node-multiselect",
27888
- label: "Enabled Audio Nodes",
27889
- description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
27890
- default: ["hub"],
28064
+ key: "audioNode",
28065
+ type: "node-select",
28066
+ label: "Audio Node",
28067
+ description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
28068
+ default: "hub",
27891
28069
  showOffline: true
27892
28070
  }
27893
28071
  ]
@@ -28390,16 +28568,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28390
28568
  pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
28391
28569
  onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
28392
28570
  };
28393
- const rawEnabled = config["enabledNodes"];
28394
- this.enabledNodes = rawEnabled === void 0 ? ["hub"] : Array.isArray(rawEnabled) ? rawEnabled.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28395
- const rawEnabledDecoder = config["enabledDecoderNodes"];
28396
- this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28397
- const rawEnabledAudio = config["enabledAudioNodes"];
28398
- this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28399
- const rawRemoteSourcing = config["remoteSourcingNodes"];
28400
- this.remoteSourcingNodes = Array.isArray(rawRemoteSourcing) ? rawRemoteSourcing.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28571
+ const readRole = (key) => {
28572
+ const raw = config[key];
28573
+ return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
28574
+ };
28575
+ this.clusterRoles = {
28576
+ ingestNode: readRole("ingestNode"),
28577
+ recordingNode: readRole("recordingNode"),
28578
+ audioNode: readRole("audioNode")
28579
+ };
28401
28580
  this.schedulePendingRetry();
28402
28581
  }
28582
+ /**
28583
+ * Default placement capability for a node when its stored flag is unset:
28584
+ * the hub is capable of every role out of the box; every other node must be
28585
+ * explicitly enabled. This reproduces the classic `['hub']` defaults of the
28586
+ * four removed flat lists against an empty capability store.
28587
+ */
28588
+ static nodeCapabilityDefault(nodeId) {
28589
+ return nodeId === "hub";
28590
+ }
28591
+ /**
28592
+ * Recompute the derived enabled-node sets (`enabledNodes` /
28593
+ * `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
28594
+ * store. A node is eligible for a concern iff its stored flag is `true`, or
28595
+ * the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
28596
+ * Forked child nodeIds (`hub/classifier`) are never dispatchable and are
28597
+ * excluded. `hub` is always considered even when absent from the store, so a
28598
+ * fresh install keeps the hub-only cluster working.
28599
+ */
28600
+ async refreshNodeCapabilities() {
28601
+ let blob = {};
28602
+ try {
28603
+ blob = await this.agentSettingsState.get();
28604
+ } catch (err) {
28605
+ this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
28606
+ }
28607
+ const nodeIds = new Set(["hub"]);
28608
+ for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
28609
+ const detect = [];
28610
+ const decode = [];
28611
+ const audio = [];
28612
+ const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
28613
+ for (const nodeId of nodeIds) {
28614
+ const settings = blob[nodeId];
28615
+ if (capable(settings?.detect, nodeId)) detect.push(nodeId);
28616
+ if (capable(settings?.decode, nodeId)) decode.push(nodeId);
28617
+ if (capable(settings?.audio, nodeId)) audio.push(nodeId);
28618
+ }
28619
+ this.enabledNodes = detect.toSorted();
28620
+ this.enabledDecoderNodes = decode.toSorted();
28621
+ this.enabledAudioNodes = audio.toSorted();
28622
+ }
28403
28623
  get api() {
28404
28624
  return this.ctx.api ?? null;
28405
28625
  }
@@ -28996,7 +29216,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28996
29216
  });
28997
29217
  return;
28998
29218
  }
28999
- this.ctx.logger.warn("Load management: pausing camera detection (sustained low fps)", {
29219
+ if (this.shedInFlight.has(deviceId)) return;
29220
+ this.shedInFlight.add(deviceId);
29221
+ this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
29000
29222
  tags: { deviceId },
29001
29223
  meta: {
29002
29224
  actualFps: metrics.actualFps,
@@ -29005,7 +29227,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29005
29227
  windowMs
29006
29228
  }
29007
29229
  });
29008
- state.pausedAt = now;
29230
+ this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
29231
+ }
29232
+ /**
29233
+ * Cluster-wide load-shed action: relocate the sustained-slow camera to a
29234
+ * less-loaded node if the balancer (ceiling + weight aware) finds one,
29235
+ * otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
29236
+ * placement logic.
29237
+ */
29238
+ async shedOrRelocate(deviceId, state) {
29239
+ if (state.pausedAt !== null) return;
29240
+ if (await this.tryClusterRelocate(deviceId).catch((err) => {
29241
+ this.ctx.logger.warn("Load management: cluster relocate failed", {
29242
+ tags: { deviceId },
29243
+ meta: { error: errMsg(err) }
29244
+ });
29245
+ return false;
29246
+ })) {
29247
+ state.lowSinceTs = null;
29248
+ return;
29249
+ }
29250
+ this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
29251
+ state.pausedAt = Date.now();
29009
29252
  state.lowSinceTs = null;
29010
29253
  this.pipelineWatchdog?.unregister(deviceId);
29011
29254
  this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
@@ -29016,6 +29259,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29016
29259
  });
29017
29260
  this.ensureLoadShedResumeTimer();
29018
29261
  }
29262
+ /**
29263
+ * Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
29264
+ * Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
29265
+ * frame-source eligibility); the camera's current node counts it in its own
29266
+ * load, so a genuinely less-loaded node with spare capacity wins. Returns
29267
+ * `true` iff the camera was moved to a DIFFERENT node. A manually pinned
29268
+ * camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
29269
+ */
29270
+ async tryClusterRelocate(deviceId) {
29271
+ const current = this.assignments.get(deviceId);
29272
+ const cached = this.cameraConfigs.get(deviceId);
29273
+ if (!current || !cached) return false;
29274
+ const currentNode = current.agentNodeId;
29275
+ if (await this.readPipelinePin(deviceId)) return false;
29276
+ const decision = balance({
29277
+ nodes: await this.collectAgentLoad({ onlyEnabled: true }),
29278
+ preferredAgent: null,
29279
+ nodeCaps: await this.buildNodeCaps(),
29280
+ weights: await this.buildNodeWeights(),
29281
+ eligibleNodes: this.detectionEligibleNodes(deviceId)
29282
+ });
29283
+ if (!decision || decision.kind !== "assigned") return false;
29284
+ const target = decision.agentNodeId;
29285
+ if (target === currentNode) return false;
29286
+ await this.detachOn(currentNode, deviceId).catch((err) => {
29287
+ this.ctx.logger.debug("Load management: relocate detach-old failed", {
29288
+ tags: { deviceId },
29289
+ meta: {
29290
+ from: currentNode,
29291
+ error: errMsg(err)
29292
+ }
29293
+ });
29294
+ });
29295
+ await this.attachOn(target, cached);
29296
+ this.recordAssignment(deviceId, target, "rebalance", false);
29297
+ this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
29298
+ tags: { deviceId },
29299
+ meta: {
29300
+ from: currentNode,
29301
+ to: target,
29302
+ score: decision.score
29303
+ }
29304
+ });
29305
+ return true;
29306
+ }
29019
29307
  /** Base cooldown before first auto-resume attempt. */
29020
29308
  static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
29021
29309
  /** Maximum backoff cap. */