@camstack/addon-pipeline-orchestrator 1.1.21 → 1.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -15703,7 +15703,17 @@ var AgentAddonConfigSchema = object({
15703
15703
  });
15704
15704
  var AgentPipelineSettingsSchema = object({
15705
15705
  addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
15706
- maxCameras: number().int().nonnegative().nullable().default(null)
15706
+ maxCameras: number().int().nonnegative().nullable().default(null),
15707
+ /** Per-node detection weight (relative share for the quota balancer). */
15708
+ detectWeight: number().positive().optional(),
15709
+ /** Node is eligible to run the detection pipeline (decode + inference). */
15710
+ detect: boolean().optional(),
15711
+ /** Node is eligible to host decoder sessions. */
15712
+ decode: boolean().optional(),
15713
+ /** Node is eligible to run audio-analyzer sessions. */
15714
+ audio: boolean().optional(),
15715
+ /** Node is eligible to be the ingest / source-owner (serve the restream). */
15716
+ ingest: boolean().optional()
15707
15717
  });
15708
15718
  var CameraPipelineForAgentSchema = object({
15709
15719
  steps: array(PipelineStepInputSchema).readonly(),
@@ -16137,6 +16147,38 @@ var pipelineOrchestratorCapability = {
16137
16147
  kind: "mutation",
16138
16148
  auth: "admin"
16139
16149
  }),
16150
+ /**
16151
+ * Set a node's detection WEIGHT (relative share for the quota balancer).
16152
+ * `null` clears it (back to the implicit weight 1). Unpinned cameras
16153
+ * distribute proportionally to weight; `maxCameras` still clamps on top.
16154
+ */
16155
+ setAgentDetectWeight: method(object({
16156
+ agentNodeId: string(),
16157
+ detectWeight: number().positive().nullable()
16158
+ }), object({ success: literal(true) }), {
16159
+ kind: "mutation",
16160
+ auth: "admin"
16161
+ }),
16162
+ /**
16163
+ * Set one node's placement CAPABILITIES — the per-node record that
16164
+ * replaced the four flat orchestrator lists (`enabledNodes`,
16165
+ * `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
16166
+ * Each flag is optional in the patch: omit a flag to leave it unchanged,
16167
+ * pass `null` to reset it to the node default (`hub` → capable, every
16168
+ * other node → not). Detection/decode/audio eligibility is derived from
16169
+ * these flags across the cluster; changing them re-derives the enabled
16170
+ * sets and reconciles dispatch immediately.
16171
+ */
16172
+ setAgentCapabilities: method(object({
16173
+ agentNodeId: string(),
16174
+ detect: boolean().nullable().optional(),
16175
+ decode: boolean().nullable().optional(),
16176
+ audio: boolean().nullable().optional(),
16177
+ ingest: boolean().nullable().optional()
16178
+ }), object({ success: literal(true) }), {
16179
+ kind: "mutation",
16180
+ auth: "admin"
16181
+ }),
16140
16182
  /** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
16141
16183
  getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
16142
16184
  /** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
@@ -22043,6 +22085,18 @@ Object.freeze({
22043
22085
  addonId: null,
22044
22086
  access: "create"
22045
22087
  },
22088
+ "pipelineOrchestrator.setAgentCapabilities": {
22089
+ capName: "pipeline-orchestrator",
22090
+ capScope: "system",
22091
+ addonId: null,
22092
+ access: "create"
22093
+ },
22094
+ "pipelineOrchestrator.setAgentDetectWeight": {
22095
+ capName: "pipeline-orchestrator",
22096
+ capScope: "system",
22097
+ addonId: null,
22098
+ access: "create"
22099
+ },
22046
22100
  "pipelineOrchestrator.setAgentMaxCameras": {
22047
22101
  capName: "pipeline-orchestrator",
22048
22102
  capScope: "system",
@@ -24186,6 +24240,23 @@ function computeCapacityScore(load) {
24186
24240
  return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
24187
24241
  }
24188
24242
  /**
24243
+ * Node detection weight, defaulting to 1 when absent/invalid. A higher weight
24244
+ * makes a node preferred proportionally more (see {@link BalancerInput.weights}).
24245
+ */
24246
+ function nodeWeight(nodeId, weights) {
24247
+ const w = weights?.[nodeId];
24248
+ return w !== void 0 && w > 0 ? w : 1;
24249
+ }
24250
+ /**
24251
+ * Weight-adjusted capacity score — the value the balancer minimises. Dividing
24252
+ * the raw capacity score by the node's weight makes a heavier node accumulate
24253
+ * proportionally more load before it loses preference, yielding weight-
24254
+ * proportional distribution while still preferring drained/idle nodes.
24255
+ */
24256
+ function weightedScore(load, weights) {
24257
+ return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
24258
+ }
24259
+ /**
24189
24260
  * Returns true when the node has remaining capacity.
24190
24261
  * A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
24191
24262
  * its `attachedCameras` count is strictly less than the cap.
@@ -24251,8 +24322,9 @@ function balance(input) {
24251
24322
  };
24252
24323
  const best = eligible.map((node) => ({
24253
24324
  node,
24254
- score: computeCapacityScore(node)
24255
- })).toSorted((a, b) => a.score - b.score)[0];
24325
+ score: computeCapacityScore(node),
24326
+ sortKey: weightedScore(node, input.weights)
24327
+ })).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
24256
24328
  return {
24257
24329
  kind: "assigned",
24258
24330
  agentNodeId: best.node.nodeId,
@@ -24354,7 +24426,12 @@ var StoredAgentAddonConfigSchema = object({
24354
24426
  });
24355
24427
  var StoredAgentPipelineSettingsSchema = object({
24356
24428
  addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
24357
- maxCameras: number().int().nonnegative().nullable().default(null)
24429
+ maxCameras: number().int().nonnegative().nullable().default(null),
24430
+ detectWeight: number().positive().optional(),
24431
+ detect: boolean().optional(),
24432
+ decode: boolean().optional(),
24433
+ audio: boolean().optional(),
24434
+ ingest: boolean().optional()
24358
24435
  });
24359
24436
  var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
24360
24437
  var StoredCameraStepOverridePatchSchema = object({
@@ -24893,11 +24970,27 @@ var OrchestratorDiagnosticsSchema = object({
24893
24970
  knownRunnerNodes: array(string()),
24894
24971
  cachedAgentLoadNodeIds: array(string()),
24895
24972
  enabledNodes: array(string()),
24973
+ enabledDecoderNodes: array(string()),
24974
+ enabledAudioNodes: array(string()),
24975
+ clusterRoles: object({
24976
+ ingestNode: string(),
24977
+ recordingNode: string(),
24978
+ audioNode: string()
24979
+ }),
24896
24980
  assignedDeviceCount: number().int().min(0),
24897
24981
  cameraConfigCount: number().int().min(0),
24898
24982
  activeDetectionCount: number().int().min(0)
24899
24983
  });
24900
24984
  var pipelineOrchestratorActions = defineCustomActions({ dumpState: customAction(_void(), OrchestratorDiagnosticsSchema) });
24985
+ /**
24986
+ * Sentinel returned by `buildDetectionConfig` when the profile-slot READ
24987
+ * itself failed transiently (e.g. `listAllProfileSlots` discovery timeout
24988
+ * while the stream-broker is (re)starting) — as opposed to `null`, which
24989
+ * means "genuinely no assigned slot / not configured". Callers MUST treat
24990
+ * this differently from `null`: never stop active detection on a transient
24991
+ * read failure (the slots almost certainly still exist), and schedule a retry.
24992
+ */
24993
+ var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
24901
24994
  var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddon {
24902
24995
  /** This node's Moleculer nodeId (from this.ctx.kernel.localNodeId). */
24903
24996
  localNodeId = "hub";
@@ -25009,36 +25102,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25009
25102
  failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
25010
25103
  /**
25011
25104
  * Hub-wide allow-list of node ids eligible to run the detection
25012
- * pipeline. Driven by the `enabledNodes` multiselect in the addon
25013
- * schema. Hydrated from the schema default (`['hub']`) on first
25014
- * boot; every other node must be explicitly added by the operator.
25015
- * The list is a strict whitelist — disabled nodes stay connected
25016
- * for other capabilities (metrics, logs, cluster-wide events) but
25017
- * never receive camera assignments.
25105
+ * pipeline. DERIVED (placement-model redesign) from the per-node
25106
+ * capability store (`agentSettings[node].detect`) no longer a flat
25107
+ * `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
25108
+ * on boot and on every capability write. The list is a strict whitelist —
25109
+ * disabled nodes stay connected for other capabilities (metrics, logs,
25110
+ * cluster-wide events) but never receive camera assignments.
25018
25111
  */
25019
25112
  enabledNodes = ["hub"];
25020
25113
  /**
25021
25114
  * Hub-wide allow-list of node ids eligible to run decoder sessions.
25022
- * Driven by the `enabledDecoderNodes` multiselect in the addon schema.
25115
+ * DERIVED from the per-node capability store (`agentSettings[node].decode`).
25023
25116
  * Defaults to `['hub']` — the hub always has a decoder available.
25024
25117
  */
25025
25118
  enabledDecoderNodes = ["hub"];
25026
25119
  /**
25027
- * Phase-2 rollout knob (P2d): nodes allowed to run the REMOTE-SOURCE leg —
25028
- * dial a camera's source-owner restream over the LAN and decode it locally.
25029
- * Driven by the `remoteSourcingNodes` multiselect in the addon schema.
25030
- * Default EMPTY = Phase-2 transport OFF: source ownership stays unmodeled
25031
- * and every behavior (eligibility, assignment, attach payload) is
25032
- * bit-identical to pre-Phase-2. See `source-owner.ts`.
25120
+ * Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
25121
+ * restream over the LAN and decode it locally. Held for the source-owner
25122
+ * plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
25123
+ * frame transport is a separate activation phase (placement-model §9 P1).
25124
+ * With this empty, `resolveSourceOwner` returns `undefined` and every
25125
+ * `selectRunnerFrameSource` emits `local-broker` bit-identical to
25126
+ * pre-Phase-2 behavior.
25033
25127
  */
25034
25128
  remoteSourcingNodes = [];
25035
25129
  /**
25036
25130
  * Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
25037
- * Driven by the `enabledAudioNodes` multiselect in the addon schema.
25131
+ * DERIVED from the per-node capability store (`agentSettings[node].audio`).
25038
25132
  * Defaults to `['hub']`.
25039
25133
  */
25040
25134
  enabledAudioNodes = ["hub"];
25041
25135
  /**
25136
+ * Cluster-wide singleton role assignments (placement-model §6). Each role
25137
+ * is the ONE node that serves it for every camera. Driven by the
25138
+ * `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
25139
+ * global schema; defaults to the hub for all three.
25140
+ */
25141
+ clusterRoles = {
25142
+ ingestNode: "hub",
25143
+ recordingNode: "hub",
25144
+ audioNode: "hub"
25145
+ };
25146
+ /**
25042
25147
  * Full global settings snapshot kept in memory so synchronous cap methods
25043
25148
  * (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
25044
25149
  * backing store. Refreshed in `initialize` and `onConfigChange`.
@@ -25107,6 +25212,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25107
25212
  loadShedState = /* @__PURE__ */ new Map();
25108
25213
  /** Timer for the auto-resume sweep. */
25109
25214
  loadShedResumeTimer = null;
25215
+ /** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
25216
+ shedInFlight = /* @__PURE__ */ new Set();
25217
+ /**
25218
+ * Per-device backoff for retrying {@link handleDeviceRegistered} after a
25219
+ * TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). Timers
25220
+ * are `unref`'d and cleared on shutdown / device removal.
25221
+ */
25222
+ slotReadRetryTimers = /* @__PURE__ */ new Map();
25223
+ slotReadRetryAttempts = /* @__PURE__ */ new Map();
25224
+ static SLOT_READ_MAX_RETRIES = 6;
25225
+ static SLOT_READ_RETRY_BASE_MS = 500;
25110
25226
  /** Pending `scheduleReconcile` debounce timer. */
25111
25227
  reconcileTimer = null;
25112
25228
  /** True while `reconcileDispatch` is awaiting the RPC round-trip. */
@@ -25131,6 +25247,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25131
25247
  const stored = await this.resolveGlobalStore();
25132
25248
  this.globalSettings = { ...stored };
25133
25249
  this.applyRuntimeSettings(this.globalSettings);
25250
+ await this.refreshNodeCapabilities();
25134
25251
  } catch (err) {
25135
25252
  const msg = errMsg(err);
25136
25253
  this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
@@ -25546,6 +25663,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25546
25663
  knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
25547
25664
  cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
25548
25665
  enabledNodes: [...this.enabledNodes],
25666
+ enabledDecoderNodes: [...this.enabledDecoderNodes],
25667
+ enabledAudioNodes: [...this.enabledAudioNodes],
25668
+ clusterRoles: { ...this.clusterRoles },
25549
25669
  assignedDeviceCount: this.assignments.size,
25550
25670
  cameraConfigCount: this.cameraConfigs.size,
25551
25671
  activeDetectionCount: this.activeDetections.size
@@ -25570,6 +25690,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25570
25690
  clearTimeout(this.pendingRetryDebounceTimer);
25571
25691
  this.pendingRetryDebounceTimer = null;
25572
25692
  }
25693
+ for (const t of this.slotReadRetryTimers.values()) clearTimeout(t);
25694
+ this.slotReadRetryTimers.clear();
25695
+ this.slotReadRetryAttempts.clear();
25573
25696
  this.unsubDeviceRegistered?.();
25574
25697
  this.unsubDeviceRegistered = null;
25575
25698
  this.unsubDeviceUnregistered?.();
@@ -25670,6 +25793,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25670
25793
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25671
25794
  preferredAgent,
25672
25795
  nodeCaps: await this.buildNodeCaps(),
25796
+ weights: await this.buildNodeWeights(),
25673
25797
  eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
25674
25798
  });
25675
25799
  if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
@@ -25786,6 +25910,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25786
25910
  nodes: await this.collectAgentLoad({ onlyEnabled: true }),
25787
25911
  preferredAgent: null,
25788
25912
  nodeCaps: await this.buildNodeCaps(),
25913
+ weights: await this.buildNodeWeights(),
25789
25914
  eligibleNodes: this.detectionEligibleNodes(input.deviceId)
25790
25915
  });
25791
25916
  if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
@@ -25834,6 +25959,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25834
25959
  if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
25835
25960
  const loads = await this.collectAgentLoad({ onlyEnabled: true });
25836
25961
  const nodeCaps = await this.buildNodeCaps();
25962
+ const nodeWeights = await this.buildNodeWeights();
25837
25963
  const attachedDelta = /* @__PURE__ */ new Map();
25838
25964
  const bumpAttached = (nodeId, by) => {
25839
25965
  attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
@@ -25855,6 +25981,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
25855
25981
  nodes: talliedLoads(),
25856
25982
  preferredAgent,
25857
25983
  nodeCaps,
25984
+ weights: nodeWeights,
25858
25985
  eligibleNodes: this.detectionEligibleNodes(deviceId)
25859
25986
  });
25860
25987
  if (!decision) continue;
@@ -26310,6 +26437,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26310
26437
  for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
26311
26438
  return caps;
26312
26439
  }
26440
+ /**
26441
+ * Build a per-node detection WEIGHT map from the persisted agent settings —
26442
+ * the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
26443
+ * are simply absent (the balancer treats absent as weight 1). Passed to every
26444
+ * `balance()` call so unpinned cameras distribute proportionally to weight.
26445
+ */
26446
+ async buildNodeWeights() {
26447
+ const blob = await this.agentSettingsState.get();
26448
+ const weights = {};
26449
+ for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
26450
+ return weights;
26451
+ }
26313
26452
  async readAudioNodePin(deviceId) {
26314
26453
  if (!this.ctx?.settings) return null;
26315
26454
  try {
@@ -26458,6 +26597,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26458
26597
  nodes: loads,
26459
26598
  preferredAgent: null,
26460
26599
  nodeCaps: await this.buildNodeCaps(),
26600
+ weights: await this.buildNodeWeights(),
26461
26601
  eligibleNodes: this.detectionEligibleNodes(deviceId)
26462
26602
  });
26463
26603
  if (!decision) {
@@ -26842,6 +26982,47 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
26842
26982
  }
26843
26983
  }
26844
26984
  /**
26985
+ * Bounded per-device retry of {@link handleDeviceRegistered} after a
26986
+ * TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). The
26987
+ * read (`listAllProfileSlots`) can time out during a broker (re)start; we
26988
+ * must neither stop a running camera nor leave a not-yet-started camera
26989
+ * stranded — so re-run the registration handler with exponential backoff
26990
+ * until the read succeeds or the budget is exhausted. Reset on any
26991
+ * successful read ({@link clearSlotReadRetry}).
26992
+ */
26993
+ scheduleSlotReadRetry(deviceId) {
26994
+ const attempts = this.slotReadRetryAttempts.get(deviceId) ?? 0;
26995
+ if (attempts >= PipelineOrchestratorAddon.SLOT_READ_MAX_RETRIES) {
26996
+ this.ctx.logger.warn("slot-read retry budget exhausted — leaving detection as-is", { tags: { deviceId } });
26997
+ this.slotReadRetryAttempts.delete(deviceId);
26998
+ return;
26999
+ }
27000
+ const existing = this.slotReadRetryTimers.get(deviceId);
27001
+ if (existing) clearTimeout(existing);
27002
+ const delay = PipelineOrchestratorAddon.SLOT_READ_RETRY_BASE_MS * 2 ** attempts;
27003
+ this.slotReadRetryAttempts.set(deviceId, attempts + 1);
27004
+ const timer = setTimeout(() => {
27005
+ this.slotReadRetryTimers.delete(deviceId);
27006
+ this.handleDeviceRegistered(deviceId).catch((err) => {
27007
+ this.ctx.logger.debug("slot-read retry: handleDeviceRegistered failed", {
27008
+ tags: { deviceId },
27009
+ meta: { error: errMsg(err) }
27010
+ });
27011
+ });
27012
+ }, delay);
27013
+ if (typeof timer === "object" && timer !== null && "unref" in timer) timer.unref();
27014
+ this.slotReadRetryTimers.set(deviceId, timer);
27015
+ }
27016
+ /** Cancel any pending slot-read retry + reset the budget for a device. */
27017
+ clearSlotReadRetry(deviceId) {
27018
+ const existing = this.slotReadRetryTimers.get(deviceId);
27019
+ if (existing) {
27020
+ clearTimeout(existing);
27021
+ this.slotReadRetryTimers.delete(deviceId);
27022
+ }
27023
+ this.slotReadRetryAttempts.delete(deviceId);
27024
+ }
27025
+ /**
26845
27026
  * Coalesce bursts of capacity/eligibility/readiness signals into a single
26846
27027
  * `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
26847
27028
  * dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
@@ -27185,6 +27366,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27185
27366
  };
27186
27367
  const { [input.agentNodeId]: _drop, ...rest } = all;
27187
27368
  await this.agentSettingsState.set(rest);
27369
+ await this.refreshNodeCapabilities();
27188
27370
  this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
27189
27371
  return {
27190
27372
  success: true,
@@ -27208,6 +27390,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27208
27390
  this.schedulePendingRetry();
27209
27391
  return { success: true };
27210
27392
  }
27393
+ async setAgentDetectWeight(input) {
27394
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
27395
+ const weight = input.detectWeight ?? 1;
27396
+ const next = existing ? {
27397
+ ...existing,
27398
+ detectWeight: weight
27399
+ } : {
27400
+ addonDefaults: {},
27401
+ maxCameras: null,
27402
+ detectWeight: weight
27403
+ };
27404
+ await this.writeAgentSettings(input.agentNodeId, next);
27405
+ this.ctx.logger.info("agentSettings.detectWeight updated", {
27406
+ tags: { nodeId: input.agentNodeId },
27407
+ meta: { detectWeight: weight ?? null }
27408
+ });
27409
+ this.rebalance().catch((err) => {
27410
+ this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
27411
+ });
27412
+ return { success: true };
27413
+ }
27414
+ /**
27415
+ * Set one node's placement capabilities — the per-node record that replaced
27416
+ * the four flat orchestrator lists. Each flag is a tri-state patch: omit to
27417
+ * leave unchanged, `null` to reset to the node default (persisted as an
27418
+ * absent flag), `true`/`false` to force. After persisting, re-derive the
27419
+ * enabled-node sets and reconcile dispatch so the change takes effect
27420
+ * immediately (reuses the existing balancer / retry, no custom logic).
27421
+ */
27422
+ async setAgentCapabilities(input) {
27423
+ const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
27424
+ addonDefaults: {},
27425
+ maxCameras: null
27426
+ };
27427
+ const apply = (key) => {
27428
+ const patch = input[key];
27429
+ if (patch === void 0) return existing[key];
27430
+ if (patch === null) return void 0;
27431
+ return patch;
27432
+ };
27433
+ const next = {
27434
+ ...existing,
27435
+ detect: apply("detect"),
27436
+ decode: apply("decode"),
27437
+ audio: apply("audio"),
27438
+ ingest: apply("ingest")
27439
+ };
27440
+ await this.writeAgentSettings(input.agentNodeId, next);
27441
+ this.ctx.logger.info("agentSettings.capabilities updated", {
27442
+ tags: { nodeId: input.agentNodeId },
27443
+ meta: {
27444
+ detect: next.detect,
27445
+ decode: next.decode,
27446
+ audio: next.audio,
27447
+ ingest: next.ingest
27448
+ }
27449
+ });
27450
+ await this.refreshNodeCapabilities();
27451
+ this.knownRunnerNodes.add(input.agentNodeId);
27452
+ this.schedulePendingRetry();
27453
+ return { success: true };
27454
+ }
27211
27455
  async getCameraSettings(input) {
27212
27456
  return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
27213
27457
  }
@@ -27573,7 +27817,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27573
27817
  const all = await this.readAgentSettingsMap();
27574
27818
  const existing = all[nodeId];
27575
27819
  all[nodeId] = {
27576
- addonDefaults: settings.addonDefaults,
27820
+ ...existing,
27821
+ ...settings,
27577
27822
  maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
27578
27823
  };
27579
27824
  await this.agentSettingsState.set(all);
@@ -27856,45 +28101,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
27856
28101
  const rand = Math.random().toString(36).slice(2, 8);
27857
28102
  return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
27858
28103
  }
27859
- /** Build the addon-level schema (cluster-wide tunables: balancer + failover). */
28104
+ /** Build the addon-level schema (cluster roles + balancer + failover). */
27860
28105
  globalSettingsSchema() {
27861
28106
  return this.schema({ sections: [
27862
28107
  {
27863
28108
  id: "cluster",
27864
- title: "Cluster",
28109
+ title: "Cluster Roles",
27865
28110
  tab: "pipeline",
27866
- description: "Which cluster nodes are eligible to run the detection pipeline. Strict whitelist: an empty list disables dispatch everywhere. Fresh installs default to ['hub']. Video detection additionally requires the node to be frame-source capable (Enabled Decoder Nodes) until cross-node frame transport ships.",
28111
+ description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
27867
28112
  fields: [
27868
28113
  {
27869
- key: "enabledNodes",
27870
- type: "node-multiselect",
27871
- label: "Enabled Detection Nodes",
27872
- description: "Only the selected nodes are passed to the load balancer. Disabled nodes remain connected for other capabilities (metrics, logs, cluster-wide events) but will never receive camera assignments.",
27873
- default: ["hub"],
28114
+ key: "ingestNode",
28115
+ type: "node-select",
28116
+ label: "Ingest Node",
28117
+ description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest hub is a later phase; kept here so the role is modeled.)",
28118
+ default: "hub",
27874
28119
  showOffline: true
27875
28120
  },
27876
28121
  {
27877
- key: "enabledDecoderNodes",
27878
- type: "node-multiselect",
27879
- label: "Enabled Decoder Nodes",
27880
- description: "Nodes eligible to run decoder sessions.",
27881
- default: ["hub"],
28122
+ key: "recordingNode",
28123
+ type: "node-select",
28124
+ label: "Recording Node",
28125
+ description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
28126
+ default: "hub",
27882
28127
  showOffline: true
27883
28128
  },
27884
28129
  {
27885
- key: "remoteSourcingNodes",
27886
- type: "node-multiselect",
27887
- label: "Remote Frame-Sourcing Nodes",
27888
- description: "Cross-node detection rollout (Phase 2): nodes allowed to pull a camera's compressed restream from its source-owner (the hub) and decode it locally for detection. Empty (default) = off — detection stays co-located with the frame source. A node must ALSO be in Enabled Decoder Nodes to qualify.",
27889
- default: [],
27890
- showOffline: true
27891
- },
27892
- {
27893
- key: "enabledAudioNodes",
27894
- type: "node-multiselect",
27895
- label: "Enabled Audio Nodes",
27896
- description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
27897
- default: ["hub"],
28130
+ key: "audioNode",
28131
+ type: "node-select",
28132
+ label: "Audio Node",
28133
+ description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
28134
+ default: "hub",
27898
28135
  showOffline: true
27899
28136
  }
27900
28137
  ]
@@ -28397,16 +28634,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28397
28634
  pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
28398
28635
  onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
28399
28636
  };
28400
- const rawEnabled = config["enabledNodes"];
28401
- this.enabledNodes = rawEnabled === void 0 ? ["hub"] : Array.isArray(rawEnabled) ? rawEnabled.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28402
- const rawEnabledDecoder = config["enabledDecoderNodes"];
28403
- this.enabledDecoderNodes = rawEnabledDecoder === void 0 ? ["hub"] : Array.isArray(rawEnabledDecoder) ? rawEnabledDecoder.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28404
- const rawEnabledAudio = config["enabledAudioNodes"];
28405
- this.enabledAudioNodes = rawEnabledAudio === void 0 ? ["hub"] : Array.isArray(rawEnabledAudio) ? rawEnabledAudio.filter((v) => typeof v === "string" && !v.includes("/")) : ["hub"];
28406
- const rawRemoteSourcing = config["remoteSourcingNodes"];
28407
- this.remoteSourcingNodes = Array.isArray(rawRemoteSourcing) ? rawRemoteSourcing.filter((v) => typeof v === "string" && !v.includes("/")) : [];
28637
+ const readRole = (key) => {
28638
+ const raw = config[key];
28639
+ return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
28640
+ };
28641
+ this.clusterRoles = {
28642
+ ingestNode: readRole("ingestNode"),
28643
+ recordingNode: readRole("recordingNode"),
28644
+ audioNode: readRole("audioNode")
28645
+ };
28408
28646
  this.schedulePendingRetry();
28409
28647
  }
28648
+ /**
28649
+ * Default placement capability for a node when its stored flag is unset:
28650
+ * the hub is capable of every role out of the box; every other node must be
28651
+ * explicitly enabled. This reproduces the classic `['hub']` defaults of the
28652
+ * four removed flat lists against an empty capability store.
28653
+ */
28654
+ static nodeCapabilityDefault(nodeId) {
28655
+ return nodeId === "hub";
28656
+ }
28657
+ /**
28658
+ * Recompute the derived enabled-node sets (`enabledNodes` /
28659
+ * `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
28660
+ * store. A node is eligible for a concern iff its stored flag is `true`, or
28661
+ * the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
28662
+ * Forked child nodeIds (`hub/classifier`) are never dispatchable and are
28663
+ * excluded. `hub` is always considered even when absent from the store, so a
28664
+ * fresh install keeps the hub-only cluster working.
28665
+ */
28666
+ async refreshNodeCapabilities() {
28667
+ let blob = {};
28668
+ try {
28669
+ blob = await this.agentSettingsState.get();
28670
+ } catch (err) {
28671
+ this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
28672
+ }
28673
+ const nodeIds = new Set(["hub"]);
28674
+ for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
28675
+ const detect = [];
28676
+ const decode = [];
28677
+ const audio = [];
28678
+ const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
28679
+ for (const nodeId of nodeIds) {
28680
+ const settings = blob[nodeId];
28681
+ if (capable(settings?.detect, nodeId)) detect.push(nodeId);
28682
+ if (capable(settings?.decode, nodeId)) decode.push(nodeId);
28683
+ if (capable(settings?.audio, nodeId)) audio.push(nodeId);
28684
+ }
28685
+ this.enabledNodes = detect.toSorted();
28686
+ this.enabledDecoderNodes = decode.toSorted();
28687
+ this.enabledAudioNodes = audio.toSorted();
28688
+ }
28410
28689
  get api() {
28411
28690
  return this.ctx.api ?? null;
28412
28691
  }
@@ -28620,7 +28899,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28620
28899
  */
28621
28900
  async fetchAssignedProfiles(deviceId) {
28622
28901
  const api = this.api;
28623
- if (!api) return /* @__PURE__ */ new Map();
28902
+ if (!api) return null;
28624
28903
  try {
28625
28904
  const slots = await api.streamBroker.listAllProfileSlots.query();
28626
28905
  const out = /* @__PURE__ */ new Map();
@@ -28630,8 +28909,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28630
28909
  out.set(slot.profile, slot.sourceCamStreamId);
28631
28910
  }
28632
28911
  return out;
28633
- } catch {
28634
- return /* @__PURE__ */ new Map();
28912
+ } catch (err) {
28913
+ this.ctx.logger.debug("fetchAssignedProfiles: slot read failed (transient) — treating as unknown", {
28914
+ tags: { deviceId },
28915
+ meta: { error: errMsg(err) }
28916
+ });
28917
+ return null;
28635
28918
  }
28636
28919
  }
28637
28920
  /**
@@ -28650,6 +28933,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28650
28933
  */
28651
28934
  async buildDetectionConfig(deviceId) {
28652
28935
  const assigned = await this.fetchAssignedProfiles(deviceId);
28936
+ if (assigned === null) return TRANSIENT_SLOT_READ;
28653
28937
  if (assigned.size === 0) return null;
28654
28938
  const resolved = await this.resolveDeviceDetectionSettings(deviceId);
28655
28939
  if (!resolved) return null;
@@ -28793,6 +29077,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28793
29077
  }
28794
29078
  /** Stop detection and purge all persisted config for a removed device. */
28795
29079
  async handleDeviceUnregistered(deviceId) {
29080
+ this.clearSlotReadRetry(deviceId);
28796
29081
  await this.stopDetection(deviceId);
28797
29082
  this.zonesProvider?.forgetDevice(deviceId);
28798
29083
  this.zoneRulesProvider?.forgetDevice(deviceId);
@@ -28814,6 +29099,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28814
29099
  const log = this.ctx.logger.withTags({ deviceId });
28815
29100
  log.info("handleDeviceRegistered", { meta: { phase: "handleDeviceRegistered" } });
28816
29101
  const config = await this.buildDetectionConfig(deviceId);
29102
+ if (config === TRANSIENT_SLOT_READ) {
29103
+ log.warn("Profile-slot read failed transiently — keeping detection, scheduling retry");
29104
+ this.scheduleSlotReadRetry(deviceId);
29105
+ return;
29106
+ }
29107
+ this.clearSlotReadRetry(deviceId);
28817
29108
  log.info("[pipeline-orchestrator] buildDetectionConfig", config ? { meta: {
28818
29109
  enabled: config.enabled,
28819
29110
  motionStreamId: config.motionStreamId,
@@ -28879,6 +29170,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
28879
29170
  const log = this.ctx.logger.withTags({ deviceId });
28880
29171
  if (this.activeDetections.has(deviceId)) await this.stopDetection(deviceId);
28881
29172
  const config = await this.buildDetectionConfig(deviceId);
29173
+ if (config === TRANSIENT_SLOT_READ) {
29174
+ log.warn("Settings changed — slot read failed transiently, scheduling retry");
29175
+ this.scheduleSlotReadRetry(deviceId);
29176
+ return;
29177
+ }
29178
+ this.clearSlotReadRetry(deviceId);
28882
29179
  if (!config) {
28883
29180
  log.info("Settings changed — no assigned slot, detection stopped");
28884
29181
  return;
@@ -29003,7 +29300,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29003
29300
  });
29004
29301
  return;
29005
29302
  }
29006
- this.ctx.logger.warn("Load management: pausing camera detection (sustained low fps)", {
29303
+ if (this.shedInFlight.has(deviceId)) return;
29304
+ this.shedInFlight.add(deviceId);
29305
+ this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
29007
29306
  tags: { deviceId },
29008
29307
  meta: {
29009
29308
  actualFps: metrics.actualFps,
@@ -29012,7 +29311,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29012
29311
  windowMs
29013
29312
  }
29014
29313
  });
29015
- state.pausedAt = now;
29314
+ this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
29315
+ }
29316
+ /**
29317
+ * Cluster-wide load-shed action: relocate the sustained-slow camera to a
29318
+ * less-loaded node if the balancer (ceiling + weight aware) finds one,
29319
+ * otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
29320
+ * placement logic.
29321
+ */
29322
+ async shedOrRelocate(deviceId, state) {
29323
+ if (state.pausedAt !== null) return;
29324
+ if (await this.tryClusterRelocate(deviceId).catch((err) => {
29325
+ this.ctx.logger.warn("Load management: cluster relocate failed", {
29326
+ tags: { deviceId },
29327
+ meta: { error: errMsg(err) }
29328
+ });
29329
+ return false;
29330
+ })) {
29331
+ state.lowSinceTs = null;
29332
+ return;
29333
+ }
29334
+ this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
29335
+ state.pausedAt = Date.now();
29016
29336
  state.lowSinceTs = null;
29017
29337
  this.pipelineWatchdog?.unregister(deviceId);
29018
29338
  this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
@@ -29023,6 +29343,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
29023
29343
  });
29024
29344
  this.ensureLoadShedResumeTimer();
29025
29345
  }
29346
+ /**
29347
+ * Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
29348
+ * Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
29349
+ * frame-source eligibility); the camera's current node counts it in its own
29350
+ * load, so a genuinely less-loaded node with spare capacity wins. Returns
29351
+ * `true` iff the camera was moved to a DIFFERENT node. A manually pinned
29352
+ * camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
29353
+ */
29354
+ async tryClusterRelocate(deviceId) {
29355
+ const current = this.assignments.get(deviceId);
29356
+ const cached = this.cameraConfigs.get(deviceId);
29357
+ if (!current || !cached) return false;
29358
+ const currentNode = current.agentNodeId;
29359
+ if (await this.readPipelinePin(deviceId)) return false;
29360
+ const decision = balance({
29361
+ nodes: await this.collectAgentLoad({ onlyEnabled: true }),
29362
+ preferredAgent: null,
29363
+ nodeCaps: await this.buildNodeCaps(),
29364
+ weights: await this.buildNodeWeights(),
29365
+ eligibleNodes: this.detectionEligibleNodes(deviceId)
29366
+ });
29367
+ if (!decision || decision.kind !== "assigned") return false;
29368
+ const target = decision.agentNodeId;
29369
+ if (target === currentNode) return false;
29370
+ await this.detachOn(currentNode, deviceId).catch((err) => {
29371
+ this.ctx.logger.debug("Load management: relocate detach-old failed", {
29372
+ tags: { deviceId },
29373
+ meta: {
29374
+ from: currentNode,
29375
+ error: errMsg(err)
29376
+ }
29377
+ });
29378
+ });
29379
+ await this.attachOn(target, cached);
29380
+ this.recordAssignment(deviceId, target, "rebalance", false);
29381
+ this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
29382
+ tags: { deviceId },
29383
+ meta: {
29384
+ from: currentNode,
29385
+ to: target,
29386
+ score: decision.score
29387
+ }
29388
+ });
29389
+ return true;
29390
+ }
29026
29391
  /** Base cooldown before first auto-resume attempt. */
29027
29392
  static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
29028
29393
  /** Maximum backoff cap. */