@camstack/addon-pipeline-orchestrator 1.1.21 → 1.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -15699,7 +15699,17 @@ var AgentAddonConfigSchema = object({
15699
15699
  });
15700
15700
  var AgentPipelineSettingsSchema = object({
15701
15701
  addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
15702
- maxCameras: number().int().nonnegative().nullable().default(null)
15702
+ maxCameras: number().int().nonnegative().nullable().default(null),
15703
+ /** Per-node detection weight (relative share for the quota balancer). */
15704
+ detectWeight: number().positive().optional(),
15705
+ /** Node is eligible to run the detection pipeline (decode + inference). */
15706
+ detect: boolean().optional(),
15707
+ /** Node is eligible to host decoder sessions. */
15708
+ decode: boolean().optional(),
15709
+ /** Node is eligible to run audio-analyzer sessions. */
15710
+ audio: boolean().optional(),
15711
+ /** Node is eligible to be the ingest / source-owner (serve the restream). */
15712
+ ingest: boolean().optional()
15703
15713
  });
15704
15714
  var CameraPipelineForAgentSchema = object({
15705
15715
  steps: array(PipelineStepInputSchema).readonly(),
@@ -16133,6 +16143,38 @@ var pipelineOrchestratorCapability = {
16133
16143
  kind: "mutation",
16134
16144
  auth: "admin"
16135
16145
  }),
16146
+ /**
16147
+ * Set a node's detection WEIGHT (relative share for the quota balancer).
16148
+ * `null` clears it (back to the implicit weight 1). Unpinned cameras
16149
+ * distribute proportionally to weight; `maxCameras` still clamps on top.
16150
+ */
16151
+ setAgentDetectWeight: method(object({
16152
+ agentNodeId: string(),
16153
+ detectWeight: number().positive().nullable()
16154
+ }), object({ success: literal(true) }), {
16155
+ kind: "mutation",
16156
+ auth: "admin"
16157
+ }),
16158
+ /**
16159
+ * Set one node's placement CAPABILITIES — the per-node record that
16160
+ * replaced the four flat orchestrator lists (`enabledNodes`,
16161
+ * `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
16162
+ * Each flag is optional in the patch: omit a flag to leave it unchanged,
16163
+ * pass `null` to reset it to the node default (`hub` → capable, every
16164
+ * other node → not). Detection/decode/audio eligibility is derived from
16165
+ * these flags across the cluster; changing them re-derives the enabled
16166
+ * sets and reconciles dispatch immediately.
16167
+ */
16168
+ setAgentCapabilities: method(object({
16169
+ agentNodeId: string(),
16170
+ detect: boolean().nullable().optional(),
16171
+ decode: boolean().nullable().optional(),
16172
+ audio: boolean().nullable().optional(),
16173
+ ingest: boolean().nullable().optional()
16174
+ }), object({ success: literal(true) }), {
16175
+ kind: "mutation",
16176
+ auth: "admin"
16177
+ }),
16136
16178
  /** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
16137
16179
  getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
16138
16180
  /** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
@@ -22039,6 +22081,18 @@ Object.freeze({
22039
22081
  addonId: null,
22040
22082
  access: "create"
22041
22083
  },
22084
+ "pipelineOrchestrator.setAgentCapabilities": {
22085
+ capName: "pipeline-orchestrator",
22086
+ capScope: "system",
22087
+ addonId: null,
22088
+ access: "create"
22089
+ },
22090
+ "pipelineOrchestrator.setAgentDetectWeight": {
22091
+ capName: "pipeline-orchestrator",
22092
+ capScope: "system",
22093
+ addonId: null,
22094
+ access: "create"
22095
+ },
22042
22096
  "pipelineOrchestrator.setAgentMaxCameras": {
22043
22097
  capName: "pipeline-orchestrator",
22044
22098
  capScope: "system",
@@ -24182,6 +24236,23 @@ function computeCapacityScore(load) {
24182
24236
  return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
24183
24237
  }
24184
24238
  /**
24239
+ * Node detection weight, defaulting to 1 when absent/invalid. A higher weight
24240
+ * makes a node preferred proportionally more (see {@link BalancerInput.weights}).
24241
+ */
24242
+ function nodeWeight(nodeId, weights) {
24243
+ const w = weights?.[nodeId];
24244
+ return w !== void 0 && w > 0 ? w : 1;
24245
+ }
24246
+ /**
24247
+ * Weight-adjusted capacity score — the value the balancer minimises. Dividing
24248
+ * the raw capacity score by the node's weight makes a heavier node accumulate
24249
+ * proportionally more load before it loses preference, yielding weight-
24250
+ * proportional distribution while still preferring drained/idle nodes.
24251
+ */
24252
+ function weightedScore(load, weights) {
24253
+ return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
24254
+ }
24255
+ /**
24185
24256
  * Returns true when the node has remaining capacity.
24186
24257
  * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
24187
24258
  * its `attachedCameras` count is strictly less than the cap.
@@ -24247,8 +24318,9 @@ function balance(input) {
24247
24318
  };
24248
24319
  const best = eligible.map((node) => ({
24249
24320
  node,
24250
- score: computeCapacityScore(node)
24251
- })).toSorted((a, b) => a.score - b.score)[0];
24321
+ score: computeCapacityScore(node),
24322
+ sortKey: weightedScore(node, input.weights)
24323
+ })).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
24252
24324
  return {
24253
24325
  kind: "assigned",
24254
24326
  agentNodeId: best.node.nodeId,
@@ -24350,7 +24422,12 @@ var StoredAgentAddonConfigSchema = object({
24350
24422
  });
24351
24423
  var StoredAgentPipelineSettingsSchema = object({
24352
24424
  addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
24353
- maxCameras: number().int().nonnegative().nullable().default(null)
24425
+ maxCameras: number().int().nonnegative().nullable().default(null),
24426
+ detectWeight: number().positive().optional(),
24427
+ detect: boolean().optional(),
24428
+ decode: boolean().optional(),
24429
+ audio: boolean().optional(),
24430
+ ingest: boolean().optional()
24354
24431
  });
24355
24432
  var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
24356
24433
  var StoredCameraStepOverridePatchSchema = object({
@@ -24889,11 +24966,27 @@ var OrchestratorDiagnosticsSchema = object({
24889
24966
  knownRunnerNodes: array(string()),
24890
24967
  cachedAgentLoadNodeIds: array(string()),
24891
24968
  enabledNodes: array(string()),
24969
+ enabledDecoderNodes: array(string()),
24970
+ enabledAudioNodes: array(string()),
24971
+ clusterRoles: object({
24972
+ ingestNode: string(),
24973
+ recordingNode: string(),
24974
+ audioNode: string()
24975
+ }),
24892
24976
  assignedDeviceCount: number().int().min(0),
24893
24977
  cameraConfigCount: number().int().min(0),
24894
24978
  activeDetectionCount: number().int().min(0)
24895
24979
  });
24896
24980
  var pipelineOrchestratorActions = defineCustomActions({ dumpState: customAction(_void(), OrchestratorDiagnosticsSchema) });
24981
+ /**
24982
+ * Sentinel returned by `buildDetectionConfig` when the profile-slot READ
24983
+ * itself failed transiently (e.g. `listAllProfileSlots` discovery timeout
24984
+ * while the stream-broker is (re)starting) — as opposed to `null`, which
24985
+ * means "genuinely no assigned slot / not configured". Callers MUST treat
24986
+ * this differently from `null`: never stop active detection on a transient
24987
+ * read failure (the slots almost certainly still exist), and schedule a retry.
24988
+ */
24989
+ var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
24897
24990
  var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddon {
24898
24991
  /** This node's Moleculer nodeId (from this.ctx.kernel.localNodeId). */
24899
24992
  localNodeId = "hub";
@@ -25005,36 +25098,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25005
25098
  failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
25006
25099
  /**
25007
25100
  * Hub-wide allow-list of node ids eligible to run the detection
25008
- * pipeline. Driven by the `enabledNodes` multiselect in the addon
25009
- * schema. Hydrated from the schema default (`['hub']`) on first
25010
- * boot; every other node must be explicitly added by the operator.
25011
- * The list is a strict whitelist — disabled nodes stay connected
25012
- * for other capabilities (metrics, logs, cluster-wide events) but
25013
- * never receive camera assignments.
25101
+ * pipeline. DERIVED (placement-model redesign) from the per-node
25102
+ * capability store (`agentSettings[node].detect`) no longer a flat
25103
+ * `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
25104
+ * on boot and on every capability write. The list is a strict whitelist —
25105
+ * disabled nodes stay connected for other capabilities (metrics, logs,
25106
+ * cluster-wide events) but never receive camera assignments.
25014
25107
  */
25015
25108
  enabledNodes = ["hub"];
25016
25109
  /**
25017
25110
  * Hub-wide allow-list of node ids eligible to run decoder sessions.
25018
- * Driven by the `enabledDecoderNodes` multiselect in the addon schema.
25111
+ * DERIVED from the per-node capability store (`agentSettings[node].decode`).
25019
25112
  * Defaults to `['hub']` — the hub always has a decoder available.
25020
25113
  */
25021
25114
  enabledDecoderNodes = ["hub"];
25022
25115
  /**
25023
- * Phase-2 rollout knob (P2d): nodes allowed to run the REMOTE-SOURCE leg —
25024
- * dial a camera's source-owner restream over the LAN and decode it locally.
25025
- * Driven by the `remoteSourcingNodes` multiselect in the addon schema.
25026
- * Default EMPTY = Phase-2 transport OFF: source ownership stays unmodeled
25027
- * and every behavior (eligibility, assignment, attach payload) is
25028
- * bit-identical to pre-Phase-2. See `source-owner.ts`.
25116
+ * Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
25117
+ * restream over the LAN and decode it locally. Held for the source-owner
25118
+ * plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
25119
+ * frame transport is a separate activation phase (placement-model §9 P1).
25120
+ * With this empty, `resolveSourceOwner` returns `undefined` and every
25121
+ * `selectRunnerFrameSource` emits `local-broker` bit-identical to
25122
+ * pre-Phase-2 behavior.
25029
25123
  */
25030
25124
  remoteSourcingNodes = [];
25031
25125
  /**
25032
25126
  * Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
25033
- * Driven by the `enabledAudioNodes` multiselect in the addon schema.
25127
+ * DERIVED from the per-node capability store (`agentSettings[node].audio`).
25034
25128
  * Defaults to `['hub']`.
25035
25129
  */
25036
25130
  enabledAudioNodes = ["hub"];
25037
25131
  /**
25132
+ * Cluster-wide singleton role assignments (placement-model §6). Each role
25133
+ * is the ONE node that serves it for every camera. Driven by the
25134
+ * `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
25135
+ * global schema; defaults to the hub for all three.
25136
+ */
25137
+ clusterRoles = {
25138
+ ingestNode: "hub",
25139
+ recordingNode: "hub",
25140
+ audioNode: "hub"
25141
+ };
25142
+ /**
25038
25143
  * Full global settings snapshot kept in memory so synchronous cap methods
25039
25144
  * (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
25040
25145
  * backing store. Refreshed in `initialize` and `onConfigChange`.
@@ -25103,6 +25208,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25103
25208
  loadShedState = /* @__PURE__ */ new Map();
25104
25209
  /** Timer for the auto-resume sweep. */
25105
25210
  loadShedResumeTimer = null;
25211
+ /** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
25212
+ shedInFlight = /* @__PURE__ */ new Set();
25213
+ /**
25214
+ * Per-device backoff for retrying {@link handleDeviceRegistered} after a
25215
+ * TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). Timers
25216
+ * are `unref`'d and cleared on shutdown / device removal.
25217
+ */
25218
+ slotReadRetryTimers = /* @__PURE__ */ new Map();
25219
+ slotReadRetryAttempts = /* @__PURE__ */ new Map();
25220
+ static SLOT_READ_MAX_RETRIES = 6;
25221
+ static SLOT_READ_RETRY_BASE_MS = 500;
25106
25222
  /** Pending `scheduleReconcile` debounce timer. */
25107
25223
  reconcileTimer = null;
25108
25224
  /** True while `reconcileDispatch` is awaiting the RPC round-trip. */
@@ -25127,6 +25243,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25127
25243
  const stored = await this.resolveGlobalStore();
25128
25244
  this.globalSettings = { ...stored };
25129
25245
  this.applyRuntimeSettings(this.globalSettings);
25246
+ await this.refreshNodeCapabilities();
25130
25247
  } catch (err) {
25131
25248
  const msg = errMsg(err);
25132
25249
  this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
@@ -25542,6 +25659,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25542
25659
  knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
25543
25660
  cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
25544
25661
  enabledNodes: [...this.enabledNodes],
25662
+ enabledDecoderNodes: [...this.enabledDecoderNodes],
25663
+ enabledAudioNodes: [...this.enabledAudioNodes],
25664
+ clusterRoles: { ...this.clusterRoles },
25545
25665
  assignedDeviceCount: this.assignments.size,
25546
25666
  cameraConfigCount: this.cameraConfigs.size,
25547
25667
  activeDetectionCount: this.activeDetections.size
@@ -25566,6 +25686,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25566
25686
  clearTimeout(this.pendingRetryDebounceTimer);
25567
25687
  this.pendingRetryDebounceTimer = null;
25568
25688
  }
25689
+ for (const t of this.slotReadRetryTimers.values()) clearTimeout(t);
25690
+ this.slotReadRetryTimers.clear();
25691
+ this.slotReadRetryAttempts.clear();
25569
25692
  this.unsubDeviceRegistered?.();
25570
25693
  this.unsubDeviceRegistered = null;
25571
25694
  this.unsubDeviceUnregistered?.();
@@ -25666,6 +25789,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25666
25789
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25667
25790
  preferredAgent,
25668
25791
  nodeCaps: await this.buildNodeCaps(),
25792
+ weights: await this.buildNodeWeights(),
25669
25793
  eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
25670
25794
  });
25671
25795
  if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
@@ -25782,6 +25906,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25782
25906
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25783
25907
  preferredAgent: null,
25784
25908
  nodeCaps: await this.buildNodeCaps(),
25909
+ weights: await this.buildNodeWeights(),
25785
25910
  eligibleNodes: this.detectionEligibleNodes(input.deviceId)
25786
25911
  });
25787
25912
  if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
@@ -25830,6 +25955,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25830
25955
  if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
25831
25956
  const loads = await this.collectAgentLoad({ onlyEnabled: true });
25832
25957
  const nodeCaps = await this.buildNodeCaps();
25958
+ const nodeWeights = await this.buildNodeWeights();
25833
25959
  const attachedDelta = /* @__PURE__ */ new Map();
25834
25960
  const bumpAttached = (nodeId, by) => {
25835
25961
  attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
@@ -25851,6 +25977,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25851
25977
  nodes: talliedLoads(),
25852
25978
  preferredAgent,
25853
25979
  nodeCaps,
25980
+ weights: nodeWeights,
25854
25981
  eligibleNodes: this.detectionEligibleNodes(deviceId)
25855
25982
  });
25856
25983
  if (!decision) continue;
@@ -26306,6 +26433,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26306
26433
  for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
26307
26434
  return caps;
26308
26435
  }
26436
+ /**
26437
+ * Build a per-node detection WEIGHT map from the persisted agent settings —
26438
+ * the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
26439
+ * are simply absent (the balancer treats absent as weight 1). Passed to every
26440
+ * `balance()` call so unpinned cameras distribute proportionally to weight.
26441
+ */
26442
+ async buildNodeWeights() {
26443
+ const blob = await this.agentSettingsState.get();
26444
+ const weights = {};
26445
+ for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
26446
+ return weights;
26447
+ }
26309
26448
  async readAudioNodePin(deviceId) {
26310
26449
  if (!this.ctx?.settings) return null;
26311
26450
  try {
@@ -26454,6 +26593,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26454
26593
  nodes: loads,
26455
26594
  preferredAgent: null,
26456
26595
  nodeCaps: await this.buildNodeCaps(),
26596
+ weights: await this.buildNodeWeights(),
26457
26597
  eligibleNodes: this.detectionEligibleNodes(deviceId)
26458
26598
  });
26459
26599
  if (!decision) {
@@ -26838,6 +26978,47 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26838
26978
  }
26839
26979
  }
26840
26980
  /**
26981
+ * Bounded per-device retry of {@link handleDeviceRegistered} after a
26982
+ * TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). The
26983
+ * read (`listAllProfileSlots`) can time out during a broker (re)start; we
26984
+ * must neither stop a running camera nor leave a not-yet-started camera
26985
+ * stranded — so re-run the registration handler with exponential backoff
26986
+ * until the read succeeds or the budget is exhausted. Reset on any
26987
+ * successful read ({@link clearSlotReadRetry}).
26988
+ */
26989
+ scheduleSlotReadRetry(deviceId) {
26990
+ const attempts = this.slotReadRetryAttempts.get(deviceId) ?? 0;
26991
+ if (attempts >= PipelineOrchestratorAddon.SLOT_READ_MAX_RETRIES) {
26992
+ this.ctx.logger.warn("slot-read retry budget exhausted — leaving detection as-is", { tags: { deviceId } });
26993
+ this.slotReadRetryAttempts.delete(deviceId);
26994
+ return;
26995
+ }
26996
+ const existing = this.slotReadRetryTimers.get(deviceId);
26997
+ if (existing) clearTimeout(existing);
26998
+ const delay = PipelineOrchestratorAddon.SLOT_READ_RETRY_BASE_MS * 2 ** attempts;
26999
+ this.slotReadRetryAttempts.set(deviceId, attempts + 1);
27000
+ const timer = setTimeout(() => {
27001
+ this.slotReadRetryTimers.delete(deviceId);
27002
+ this.handleDeviceRegistered(deviceId).catch((err) => {
27003
+ this.ctx.logger.debug("slot-read retry: handleDeviceRegistered failed", {
27004
+ tags: { deviceId },
27005
+ meta: { error: errMsg(err) }
27006
+ });
27007
+ });
27008
+ }, delay);
27009
+ if (typeof timer === "object" && timer !== null && "unref" in timer) timer.unref();
27010
+ this.slotReadRetryTimers.set(deviceId, timer);
27011
+ }
27012
+ /** Cancel any pending slot-read retry + reset the budget for a device. */
27013
+ clearSlotReadRetry(deviceId) {
27014
+ const existing = this.slotReadRetryTimers.get(deviceId);
27015
+ if (existing) {
27016
+ clearTimeout(existing);
27017
+ this.slotReadRetryTimers.delete(deviceId);
27018
+ }
27019
+ this.slotReadRetryAttempts.delete(deviceId);
27020
+ }
27021
+ /**
26841
27022
  * Coalesce bursts of capacity/eligibility/readiness signals into a single
26842
27023
  * `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
26843
27024
  * dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
@@ -27181,6 +27362,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27181
27362
  };
27182
27363
  const { [input.agentNodeId]: _drop, ...rest } = all;
27183
27364
  await this.agentSettingsState.set(rest);
27365
+ await this.refreshNodeCapabilities();
27184
27366
  this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
27185
27367
  return {
27186
27368
  success: true,
@@ -27204,6 +27386,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27204
27386
  this.schedulePendingRetry();
27205
27387
  return { success: true };
27206
27388
  }
27389
+ async setAgentDetectWeight(input) {
27390
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
27391
+ const weight = input.detectWeight ?? 1;
27392
+ const next = existing ? {
27393
+ ...existing,
27394
+ detectWeight: weight
27395
+ } : {
27396
+ addonDefaults: {},
27397
+ maxCameras: null,
27398
+ detectWeight: weight
27399
+ };
27400
+ await this.writeAgentSettings(input.agentNodeId, next);
27401
+ this.ctx.logger.info("agentSettings.detectWeight updated", {
27402
+ tags: { nodeId: input.agentNodeId },
27403
+ meta: { detectWeight: weight ?? null }
27404
+ });
27405
+ this.rebalance().catch((err) => {
27406
+ this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
27407
+ });
27408
+ return { success: true };
27409
+ }
27410
+ /**
27411
+ * Set one node's placement capabilities — the per-node record that replaced
27412
+ * the four flat orchestrator lists. Each flag is a tri-state patch: omit to
27413
+ * leave unchanged, `null` to reset to the node default (persisted as an
27414
+ * absent flag), `true`/`false` to force. After persisting, re-derive the
27415
+ * enabled-node sets and reconcile dispatch so the change takes effect
27416
+ * immediately (reuses the existing balancer / retry, no custom logic).
27417
+ */
27418
+ async setAgentCapabilities(input) {
27419
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
27420
+ addonDefaults: {},
27421
+ maxCameras: null
27422
+ };
27423
+ const apply = (key) => {
27424
+ const patch = input[key];
27425
+ if (patch === void 0) return existing[key];
27426
+ if (patch === null) return void 0;
27427
+ return patch;
27428
+ };
27429
+ const next = {
27430
+ ...existing,
27431
+ detect: apply("detect"),
27432
+ decode: apply("decode"),
27433
+ audio: apply("audio"),
27434
+ ingest: apply("ingest")
27435
+ };
27436
+ await this.writeAgentSettings(input.agentNodeId, next);
27437
+ this.ctx.logger.info("agentSettings.capabilities updated", {
27438
+ tags: { nodeId: input.agentNodeId },
27439
+ meta: {
27440
+ detect: next.detect,
27441
+ decode: next.decode,
27442
+ audio: next.audio,
27443
+ ingest: next.ingest
27444
+ }
27445
+ });
27446
+ await this.refreshNodeCapabilities();
27447
+ this.knownRunnerNodes.add(input.agentNodeId);
27448
+ this.schedulePendingRetry();
27449
+ return { success: true };
27450
+ }
27207
27451
  async getCameraSettings(input) {
27208
27452
  return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
27209
27453
  }
@@ -27569,7 +27813,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27569
27813
  const all = await this.readAgentSettingsMap();
27570
27814
  const existing = all[nodeId];
27571
27815
  all[nodeId] = {
27572
- addonDefaults: settings.addonDefaults,
27816
+ ...existing,
27817
+ ...settings,
27573
27818
  maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
27574
27819
  };
27575
27820
  await this.agentSettingsState.set(all);
@@ -27852,45 +28097,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27852
28097
  const rand = Math.random().toString(36).slice(2, 8);
27853
28098
  return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
27854
28099
  }
27855
- /** Build the addon-level schema (cluster-wide tunables: balancer + failover). */
28100
+ /** Build the addon-level schema (cluster roles + balancer + failover). */
27856
28101
  globalSettingsSchema() {
27857
28102
  return this.schema({ sections: [
27858
28103
  {
27859
28104
  id: "cluster",
27860
- title: "Cluster",
28105
+ title: "Cluster Roles",
27861
28106
  tab: "pipeline",
27862
- description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
28107
+ description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
27863
28108
  fields: [
27864
28109
  {
27865
- key: "enabledNodes",
27866
- type: "node-multiselect",
27867
- label: "Enabled Detection Nodes",
27868
- description: "Only the selected nodes are passed to the load balancer. Disabled nodes remain connected for other capabilities (metrics, logs, cluster-wide events) but will never receive camera assignments.",
27869
- default: ["hub"],
28110
+ key: "ingestNode",
28111
+ type: "node-select",
28112
+ label: "Ingest Node",
28113
+ description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest hub is a later phase; kept here so the role is modeled.)",
28114
+ default: "hub",
27870
28115
  showOffline: true
27871
28116
  },
27872
28117
  {
27873
- key: "enabledDecoderNodes",
27874
- type: "node-multiselect",
27875
- label: "Enabled Decoder Nodes",
27876
- description: "Nodes eligible to run decoder sessions.",
27877
- default: ["hub"],
28118
+ key: "recordingNode",
28119
+ type: "node-select",
28120
+ label: "Recording Node",
28121
+ description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
28122
+ default: "hub",
27878
28123
  showOffline: true
27879
28124
  },
27880
28125
  {
27881
- key: "remoteSourcingNodes",
27882
- type: "node-multiselect",
27883
- label: "Remote Frame-Sourcing Nodes",
27884
- description: "Cross-node detection rollout (Phase 2): nodes allowed to pull a camera's compressed restream from its source-owner (the hub) and decode it locally for detection. Empty (default) = off — detection stays co-located with the frame source. A node must ALSO be in Enabled Decoder Nodes to qualify.",
27885
- default: [],
27886
- showOffline: true
27887
- },
27888
- {
27889
- key: "enabledAudioNodes",
27890
- type: "node-multiselect",
27891
- label: "Enabled Audio Nodes",
27892
- description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
27893
- default: ["hub"],
28126
+ key: "audioNode",
28127
+ type: "node-select",
28128
+ label: "Audio Node",
28129
+ description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
28130
+ default: "hub",
27894
28131
  showOffline: true
27895
28132
  }
27896
28133
  ]
@@ -28393,16 +28630,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28393
28630
  pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
28394
28631
  onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
28395
28632
  };
28396
- const rawEnabled = config["enabledNodes"];
28397
- this.enabledNodes = rawEnabled === void 0 ? ["hub"] : Array.isArray(rawEnabled) ? rawEnabled.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28398
- const rawEnabledDecoder = config["enabledDecoderNodes"];
28399
- this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28400
- const rawEnabledAudio = config["enabledAudioNodes"];
28401
- this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28402
- const rawRemoteSourcing = config["remoteSourcingNodes"];
28403
- this.remoteSourcingNodes = Array.isArray(rawRemoteSourcing) ? rawRemoteSourcing.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28633
+ const readRole = (key) => {
28634
+ const raw = config[key];
28635
+ return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
28636
+ };
28637
+ this.clusterRoles = {
28638
+ ingestNode: readRole("ingestNode"),
28639
+ recordingNode: readRole("recordingNode"),
28640
+ audioNode: readRole("audioNode")
28641
+ };
28404
28642
  this.schedulePendingRetry();
28405
28643
  }
28644
+ /**
28645
+ * Default placement capability for a node when its stored flag is unset:
28646
+ * the hub is capable of every role out of the box; every other node must be
28647
+ * explicitly enabled. This reproduces the classic `['hub']` defaults of the
28648
+ * four removed flat lists against an empty capability store.
28649
+ */
28650
+ static nodeCapabilityDefault(nodeId) {
28651
+ return nodeId === "hub";
28652
+ }
28653
+ /**
28654
+ * Recompute the derived enabled-node sets (`enabledNodes` /
28655
+ * `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
28656
+ * store. A node is eligible for a concern iff its stored flag is `true`, or
28657
+ * the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
28658
+ * Forked child nodeIds (`hub/classifier`) are never dispatchable and are
28659
+ * excluded. `hub` is always considered even when absent from the store, so a
28660
+ * fresh install keeps the hub-only cluster working.
28661
+ */
28662
+ async refreshNodeCapabilities() {
28663
+ let blob = {};
28664
+ try {
28665
+ blob = await this.agentSettingsState.get();
28666
+ } catch (err) {
28667
+ this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
28668
+ }
28669
+ const nodeIds = new Set(["hub"]);
28670
+ for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
28671
+ const detect = [];
28672
+ const decode = [];
28673
+ const audio = [];
28674
+ const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
28675
+ for (const nodeId of nodeIds) {
28676
+ const settings = blob[nodeId];
28677
+ if (capable(settings?.detect, nodeId)) detect.push(nodeId);
28678
+ if (capable(settings?.decode, nodeId)) decode.push(nodeId);
28679
+ if (capable(settings?.audio, nodeId)) audio.push(nodeId);
28680
+ }
28681
+ this.enabledNodes = detect.toSorted();
28682
+ this.enabledDecoderNodes = decode.toSorted();
28683
+ this.enabledAudioNodes = audio.toSorted();
28684
+ }
28406
28685
  get api() {
28407
28686
  return this.ctx.api ?? null;
28408
28687
  }
@@ -28616,7 +28895,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28616
28895
  */
28617
28896
  async fetchAssignedProfiles(deviceId) {
28618
28897
  const api = this.api;
28619
- if (!api) return /* @__PURE__ */ new Map();
28898
+ if (!api) return null;
28620
28899
  try {
28621
28900
  const slots = await api.streamBroker.listAllProfileSlots.query();
28622
28901
  const out = /* @__PURE__ */ new Map();
@@ -28626,8 +28905,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28626
28905
  out.set(slot.profile, slot.sourceCamStreamId);
28627
28906
  }
28628
28907
  return out;
28629
- } catch {
28630
- return /* @__PURE__ */ new Map();
28908
+ } catch (err) {
28909
+ this.ctx.logger.debug("fetchAssignedProfiles: slot read failed (transient) — treating as unknown", {
28910
+ tags: { deviceId },
28911
+ meta: { error: errMsg(err) }
28912
+ });
28913
+ return null;
28631
28914
  }
28632
28915
  }
28633
28916
  /**
@@ -28646,6 +28929,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28646
28929
  */
28647
28930
  async buildDetectionConfig(deviceId) {
28648
28931
  const assigned = await this.fetchAssignedProfiles(deviceId);
28932
+ if (assigned === null) return TRANSIENT_SLOT_READ;
28649
28933
  if (assigned.size === 0) return null;
28650
28934
  const resolved = await this.resolveDeviceDetectionSettings(deviceId);
28651
28935
  if (!resolved) return null;
@@ -28789,6 +29073,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28789
29073
  }
28790
29074
  /** Stop detection and purge all persisted config for a removed device. */
28791
29075
  async handleDeviceUnregistered(deviceId) {
29076
+ this.clearSlotReadRetry(deviceId);
28792
29077
  await this.stopDetection(deviceId);
28793
29078
  this.zonesProvider?.forgetDevice(deviceId);
28794
29079
  this.zoneRulesProvider?.forgetDevice(deviceId);
@@ -28810,6 +29095,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28810
29095
  const log = this.ctx.logger.withTags({ deviceId });
28811
29096
  log.info("handleDeviceRegistered", { meta: { phase: "handleDeviceRegistered" } });
28812
29097
  const config = await this.buildDetectionConfig(deviceId);
29098
+ if (config === TRANSIENT_SLOT_READ) {
29099
+ log.warn("Profile-slot read failed transiently — keeping detection, scheduling retry");
29100
+ this.scheduleSlotReadRetry(deviceId);
29101
+ return;
29102
+ }
29103
+ this.clearSlotReadRetry(deviceId);
28813
29104
  log.info("[pipeline-orchestrator] buildDetectionConfig", config ? { meta: {
28814
29105
  enabled: config.enabled,
28815
29106
  motionStreamId: config.motionStreamId,
@@ -28875,6 +29166,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28875
29166
  const log = this.ctx.logger.withTags({ deviceId });
28876
29167
  if (this.activeDetections.has(deviceId)) await this.stopDetection(deviceId);
28877
29168
  const config = await this.buildDetectionConfig(deviceId);
29169
+ if (config === TRANSIENT_SLOT_READ) {
29170
+ log.warn("Settings changed — slot read failed transiently, scheduling retry");
29171
+ this.scheduleSlotReadRetry(deviceId);
29172
+ return;
29173
+ }
29174
+ this.clearSlotReadRetry(deviceId);
28878
29175
  if (!config) {
28879
29176
  log.info("Settings changed — no assigned slot, detection stopped");
28880
29177
  return;
@@ -28999,7 +29296,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28999
29296
  });
29000
29297
  return;
29001
29298
  }
29002
- this.ctx.logger.warn("Load management: pausing camera detection (sustained low fps)", {
29299
+ if (this.shedInFlight.has(deviceId)) return;
29300
+ this.shedInFlight.add(deviceId);
29301
+ this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
29003
29302
  tags: { deviceId },
29004
29303
  meta: {
29005
29304
  actualFps: metrics.actualFps,
@@ -29008,7 +29307,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29008
29307
  windowMs
29009
29308
  }
29010
29309
  });
29011
- state.pausedAt = now;
29310
+ this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
29311
+ }
29312
+ /**
29313
+ * Cluster-wide load-shed action: relocate the sustained-slow camera to a
29314
+ * less-loaded node if the balancer (ceiling + weight aware) finds one,
29315
+ * otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
29316
+ * placement logic.
29317
+ */
29318
+ async shedOrRelocate(deviceId, state) {
29319
+ if (state.pausedAt !== null) return;
29320
+ if (await this.tryClusterRelocate(deviceId).catch((err) => {
29321
+ this.ctx.logger.warn("Load management: cluster relocate failed", {
29322
+ tags: { deviceId },
29323
+ meta: { error: errMsg(err) }
29324
+ });
29325
+ return false;
29326
+ })) {
29327
+ state.lowSinceTs = null;
29328
+ return;
29329
+ }
29330
+ this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
29331
+ state.pausedAt = Date.now();
29012
29332
  state.lowSinceTs = null;
29013
29333
  this.pipelineWatchdog?.unregister(deviceId);
29014
29334
  this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
@@ -29019,6 +29339,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29019
29339
  });
29020
29340
  this.ensureLoadShedResumeTimer();
29021
29341
  }
29342
+ /**
29343
+ * Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
29344
+ * Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
29345
+ * frame-source eligibility); the camera's current node counts it in its own
29346
+ * load, so a genuinely less-loaded node with spare capacity wins. Returns
29347
+ * `true` iff the camera was moved to a DIFFERENT node. A manually pinned
29348
+ * camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
29349
+ */
29350
+ async tryClusterRelocate(deviceId) {
29351
+ const current = this.assignments.get(deviceId);
29352
+ const cached = this.cameraConfigs.get(deviceId);
29353
+ if (!current || !cached) return false;
29354
+ const currentNode = current.agentNodeId;
29355
+ if (await this.readPipelinePin(deviceId)) return false;
29356
+ const decision = balance({
29357
+ nodes: await this.collectAgentLoad({ onlyEnabled: true }),
29358
+ preferredAgent: null,
29359
+ nodeCaps: await this.buildNodeCaps(),
29360
+ weights: await this.buildNodeWeights(),
29361
+ eligibleNodes: this.detectionEligibleNodes(deviceId)
29362
+ });
29363
+ if (!decision || decision.kind !== "assigned") return false;
29364
+ const target = decision.agentNodeId;
29365
+ if (target === currentNode) return false;
29366
+ await this.detachOn(currentNode, deviceId).catch((err) => {
29367
+ this.ctx.logger.debug("Load management: relocate detach-old failed", {
29368
+ tags: { deviceId },
29369
+ meta: {
29370
+ from: currentNode,
29371
+ error: errMsg(err)
29372
+ }
29373
+ });
29374
+ });
29375
+ await this.attachOn(target, cached);
29376
+ this.recordAssignment(deviceId, target, "rebalance", false);
29377
+ this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
29378
+ tags: { deviceId },
29379
+ meta: {
29380
+ from: currentNode,
29381
+ to: target,
29382
+ score: decision.score
29383
+ }
29384
+ });
29385
+ return true;
29386
+ }
29022
29387
  /** Base cooldown before first auto-resume attempt. */
29023
29388
  static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
29024
29389
  /** Maximum backoff cap. */