@camstack/addon-pipeline-orchestrator 1.1.21 → 1.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +1 -1
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DVzgRExi.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CQmCy3VK.mjs} +3 -3
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DdDNCPJ5.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BA0gpKU7.mjs} +1 -1
- package/dist/{hostInit-CEqZfaom.mjs → hostInit-ibn4yaBr.mjs} +3 -3
- package/dist/index.js +423 -58
- package/dist/index.mjs +423 -58
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -15699,7 +15699,17 @@ var AgentAddonConfigSchema = object({
|
|
|
15699
15699
|
});
|
|
15700
15700
|
var AgentPipelineSettingsSchema = object({
|
|
15701
15701
|
addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
|
|
15702
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
15702
|
+
maxCameras: number().int().nonnegative().nullable().default(null),
|
|
15703
|
+
/** Per-node detection weight (relative share for the quota balancer). */
|
|
15704
|
+
detectWeight: number().positive().optional(),
|
|
15705
|
+
/** Node is eligible to run the detection pipeline (decode + inference). */
|
|
15706
|
+
detect: boolean().optional(),
|
|
15707
|
+
/** Node is eligible to host decoder sessions. */
|
|
15708
|
+
decode: boolean().optional(),
|
|
15709
|
+
/** Node is eligible to run audio-analyzer sessions. */
|
|
15710
|
+
audio: boolean().optional(),
|
|
15711
|
+
/** Node is eligible to be the ingest / source-owner (serve the restream). */
|
|
15712
|
+
ingest: boolean().optional()
|
|
15703
15713
|
});
|
|
15704
15714
|
var CameraPipelineForAgentSchema = object({
|
|
15705
15715
|
steps: array(PipelineStepInputSchema).readonly(),
|
|
@@ -16133,6 +16143,38 @@ var pipelineOrchestratorCapability = {
|
|
|
16133
16143
|
kind: "mutation",
|
|
16134
16144
|
auth: "admin"
|
|
16135
16145
|
}),
|
|
16146
|
+
/**
|
|
16147
|
+
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
16148
|
+
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
16149
|
+
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
16150
|
+
*/
|
|
16151
|
+
setAgentDetectWeight: method(object({
|
|
16152
|
+
agentNodeId: string(),
|
|
16153
|
+
detectWeight: number().positive().nullable()
|
|
16154
|
+
}), object({ success: literal(true) }), {
|
|
16155
|
+
kind: "mutation",
|
|
16156
|
+
auth: "admin"
|
|
16157
|
+
}),
|
|
16158
|
+
/**
|
|
16159
|
+
* Set one node's placement CAPABILITIES — the per-node record that
|
|
16160
|
+
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
16161
|
+
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
16162
|
+
* Each flag is optional in the patch: omit a flag to leave it unchanged,
|
|
16163
|
+
* pass `null` to reset it to the node default (`hub` → capable, every
|
|
16164
|
+
* other node → not). Detection/decode/audio eligibility is derived from
|
|
16165
|
+
* these flags across the cluster; changing them re-derives the enabled
|
|
16166
|
+
* sets and reconciles dispatch immediately.
|
|
16167
|
+
*/
|
|
16168
|
+
setAgentCapabilities: method(object({
|
|
16169
|
+
agentNodeId: string(),
|
|
16170
|
+
detect: boolean().nullable().optional(),
|
|
16171
|
+
decode: boolean().nullable().optional(),
|
|
16172
|
+
audio: boolean().nullable().optional(),
|
|
16173
|
+
ingest: boolean().nullable().optional()
|
|
16174
|
+
}), object({ success: literal(true) }), {
|
|
16175
|
+
kind: "mutation",
|
|
16176
|
+
auth: "admin"
|
|
16177
|
+
}),
|
|
16136
16178
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
16137
16179
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
16138
16180
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -22039,6 +22081,18 @@ Object.freeze({
|
|
|
22039
22081
|
addonId: null,
|
|
22040
22082
|
access: "create"
|
|
22041
22083
|
},
|
|
22084
|
+
"pipelineOrchestrator.setAgentCapabilities": {
|
|
22085
|
+
capName: "pipeline-orchestrator",
|
|
22086
|
+
capScope: "system",
|
|
22087
|
+
addonId: null,
|
|
22088
|
+
access: "create"
|
|
22089
|
+
},
|
|
22090
|
+
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
22091
|
+
capName: "pipeline-orchestrator",
|
|
22092
|
+
capScope: "system",
|
|
22093
|
+
addonId: null,
|
|
22094
|
+
access: "create"
|
|
22095
|
+
},
|
|
22042
22096
|
"pipelineOrchestrator.setAgentMaxCameras": {
|
|
22043
22097
|
capName: "pipeline-orchestrator",
|
|
22044
22098
|
capScope: "system",
|
|
@@ -24182,6 +24236,23 @@ function computeCapacityScore(load) {
|
|
|
24182
24236
|
return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
|
|
24183
24237
|
}
|
|
24184
24238
|
/**
|
|
24239
|
+
* Node detection weight, defaulting to 1 when absent/invalid. A higher weight
|
|
24240
|
+
* makes a node preferred proportionally more (see {@link BalancerInput.weights}).
|
|
24241
|
+
*/
|
|
24242
|
+
function nodeWeight(nodeId, weights) {
|
|
24243
|
+
const w = weights?.[nodeId];
|
|
24244
|
+
return w !== void 0 && w > 0 ? w : 1;
|
|
24245
|
+
}
|
|
24246
|
+
/**
|
|
24247
|
+
* Weight-adjusted capacity score — the value the balancer minimises. Dividing
|
|
24248
|
+
* the raw capacity score by the node's weight makes a heavier node accumulate
|
|
24249
|
+
* proportionally more load before it loses preference, yielding weight-
|
|
24250
|
+
* proportional distribution while still preferring drained/idle nodes.
|
|
24251
|
+
*/
|
|
24252
|
+
function weightedScore(load, weights) {
|
|
24253
|
+
return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
|
|
24254
|
+
}
|
|
24255
|
+
/**
|
|
24185
24256
|
* Returns true when the node has remaining capacity.
|
|
24186
24257
|
* A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
|
|
24187
24258
|
* its `attachedCameras` count is strictly less than the cap.
|
|
@@ -24247,8 +24318,9 @@ function balance(input) {
|
|
|
24247
24318
|
};
|
|
24248
24319
|
const best = eligible.map((node) => ({
|
|
24249
24320
|
node,
|
|
24250
|
-
score: computeCapacityScore(node)
|
|
24251
|
-
|
|
24321
|
+
score: computeCapacityScore(node),
|
|
24322
|
+
sortKey: weightedScore(node, input.weights)
|
|
24323
|
+
})).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
|
|
24252
24324
|
return {
|
|
24253
24325
|
kind: "assigned",
|
|
24254
24326
|
agentNodeId: best.node.nodeId,
|
|
@@ -24350,7 +24422,12 @@ var StoredAgentAddonConfigSchema = object({
|
|
|
24350
24422
|
});
|
|
24351
24423
|
var StoredAgentPipelineSettingsSchema = object({
|
|
24352
24424
|
addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
|
|
24353
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
24425
|
+
maxCameras: number().int().nonnegative().nullable().default(null),
|
|
24426
|
+
detectWeight: number().positive().optional(),
|
|
24427
|
+
detect: boolean().optional(),
|
|
24428
|
+
decode: boolean().optional(),
|
|
24429
|
+
audio: boolean().optional(),
|
|
24430
|
+
ingest: boolean().optional()
|
|
24354
24431
|
});
|
|
24355
24432
|
var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
|
|
24356
24433
|
var StoredCameraStepOverridePatchSchema = object({
|
|
@@ -24889,11 +24966,27 @@ var OrchestratorDiagnosticsSchema = object({
|
|
|
24889
24966
|
knownRunnerNodes: array(string()),
|
|
24890
24967
|
cachedAgentLoadNodeIds: array(string()),
|
|
24891
24968
|
enabledNodes: array(string()),
|
|
24969
|
+
enabledDecoderNodes: array(string()),
|
|
24970
|
+
enabledAudioNodes: array(string()),
|
|
24971
|
+
clusterRoles: object({
|
|
24972
|
+
ingestNode: string(),
|
|
24973
|
+
recordingNode: string(),
|
|
24974
|
+
audioNode: string()
|
|
24975
|
+
}),
|
|
24892
24976
|
assignedDeviceCount: number().int().min(0),
|
|
24893
24977
|
cameraConfigCount: number().int().min(0),
|
|
24894
24978
|
activeDetectionCount: number().int().min(0)
|
|
24895
24979
|
});
|
|
24896
24980
|
var pipelineOrchestratorActions = defineCustomActions({ dumpState: customAction(_void(), OrchestratorDiagnosticsSchema) });
|
|
24981
|
+
/**
|
|
24982
|
+
* Sentinel returned by `buildDetectionConfig` when the profile-slot READ
|
|
24983
|
+
* itself failed transiently (e.g. `listAllProfileSlots` discovery timeout
|
|
24984
|
+
* while the stream-broker is (re)starting) — as opposed to `null`, which
|
|
24985
|
+
* means "genuinely no assigned slot / not configured". Callers MUST treat
|
|
24986
|
+
* this differently from `null`: never stop active detection on a transient
|
|
24987
|
+
* read failure (the slots almost certainly still exist), and schedule a retry.
|
|
24988
|
+
*/
|
|
24989
|
+
var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
|
|
24897
24990
|
var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddon {
|
|
24898
24991
|
/** This node's Moleculer nodeId (from this.ctx.kernel.localNodeId). */
|
|
24899
24992
|
localNodeId = "hub";
|
|
@@ -25005,36 +25098,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25005
25098
|
failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
|
|
25006
25099
|
/**
|
|
25007
25100
|
* Hub-wide allow-list of node ids eligible to run the detection
|
|
25008
|
-
* pipeline.
|
|
25009
|
-
*
|
|
25010
|
-
*
|
|
25011
|
-
* The list is a strict whitelist —
|
|
25012
|
-
* for other capabilities (metrics, logs,
|
|
25013
|
-
* never receive camera assignments.
|
|
25101
|
+
* pipeline. DERIVED (placement-model redesign) from the per-node
|
|
25102
|
+
* capability store (`agentSettings[node].detect`) — no longer a flat
|
|
25103
|
+
* `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
|
|
25104
|
+
* on boot and on every capability write. The list is a strict whitelist —
|
|
25105
|
+
* disabled nodes stay connected for other capabilities (metrics, logs,
|
|
25106
|
+
* cluster-wide events) but never receive camera assignments.
|
|
25014
25107
|
*/
|
|
25015
25108
|
enabledNodes = ["hub"];
|
|
25016
25109
|
/**
|
|
25017
25110
|
* Hub-wide allow-list of node ids eligible to run decoder sessions.
|
|
25018
|
-
*
|
|
25111
|
+
* DERIVED from the per-node capability store (`agentSettings[node].decode`).
|
|
25019
25112
|
* Defaults to `['hub']` — the hub always has a decoder available.
|
|
25020
25113
|
*/
|
|
25021
25114
|
enabledDecoderNodes = ["hub"];
|
|
25022
25115
|
/**
|
|
25023
|
-
*
|
|
25024
|
-
*
|
|
25025
|
-
*
|
|
25026
|
-
*
|
|
25027
|
-
*
|
|
25028
|
-
*
|
|
25116
|
+
* Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
|
|
25117
|
+
* restream over the LAN and decode it locally. Held for the source-owner
|
|
25118
|
+
* plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
|
|
25119
|
+
* frame transport is a separate activation phase (placement-model §9 P1).
|
|
25120
|
+
* With this empty, `resolveSourceOwner` returns `undefined` and every
|
|
25121
|
+
* `selectRunnerFrameSource` emits `local-broker` — bit-identical to
|
|
25122
|
+
* pre-Phase-2 behavior.
|
|
25029
25123
|
*/
|
|
25030
25124
|
remoteSourcingNodes = [];
|
|
25031
25125
|
/**
|
|
25032
25126
|
* Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
|
|
25033
|
-
*
|
|
25127
|
+
* DERIVED from the per-node capability store (`agentSettings[node].audio`).
|
|
25034
25128
|
* Defaults to `['hub']`.
|
|
25035
25129
|
*/
|
|
25036
25130
|
enabledAudioNodes = ["hub"];
|
|
25037
25131
|
/**
|
|
25132
|
+
* Cluster-wide singleton role assignments (placement-model §6). Each role
|
|
25133
|
+
* is the ONE node that serves it for every camera. Driven by the
|
|
25134
|
+
* `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
|
|
25135
|
+
* global schema; defaults to the hub for all three.
|
|
25136
|
+
*/
|
|
25137
|
+
clusterRoles = {
|
|
25138
|
+
ingestNode: "hub",
|
|
25139
|
+
recordingNode: "hub",
|
|
25140
|
+
audioNode: "hub"
|
|
25141
|
+
};
|
|
25142
|
+
/**
|
|
25038
25143
|
* Full global settings snapshot kept in memory so synchronous cap methods
|
|
25039
25144
|
* (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
|
|
25040
25145
|
* backing store. Refreshed in `initialize` and `onConfigChange`.
|
|
@@ -25103,6 +25208,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25103
25208
|
loadShedState = /* @__PURE__ */ new Map();
|
|
25104
25209
|
/** Timer for the auto-resume sweep. */
|
|
25105
25210
|
loadShedResumeTimer = null;
|
|
25211
|
+
/** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
|
|
25212
|
+
shedInFlight = /* @__PURE__ */ new Set();
|
|
25213
|
+
/**
|
|
25214
|
+
* Per-device backoff for retrying {@link handleDeviceRegistered} after a
|
|
25215
|
+
* TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). Timers
|
|
25216
|
+
* are `unref`'d and cleared on shutdown / device removal.
|
|
25217
|
+
*/
|
|
25218
|
+
slotReadRetryTimers = /* @__PURE__ */ new Map();
|
|
25219
|
+
slotReadRetryAttempts = /* @__PURE__ */ new Map();
|
|
25220
|
+
static SLOT_READ_MAX_RETRIES = 6;
|
|
25221
|
+
static SLOT_READ_RETRY_BASE_MS = 500;
|
|
25106
25222
|
/** Pending `scheduleReconcile` debounce timer. */
|
|
25107
25223
|
reconcileTimer = null;
|
|
25108
25224
|
/** True while `reconcileDispatch` is awaiting the RPC round-trip. */
|
|
@@ -25127,6 +25243,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25127
25243
|
const stored = await this.resolveGlobalStore();
|
|
25128
25244
|
this.globalSettings = { ...stored };
|
|
25129
25245
|
this.applyRuntimeSettings(this.globalSettings);
|
|
25246
|
+
await this.refreshNodeCapabilities();
|
|
25130
25247
|
} catch (err) {
|
|
25131
25248
|
const msg = errMsg(err);
|
|
25132
25249
|
this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
|
|
@@ -25542,6 +25659,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25542
25659
|
knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
|
|
25543
25660
|
cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
|
|
25544
25661
|
enabledNodes: [...this.enabledNodes],
|
|
25662
|
+
enabledDecoderNodes: [...this.enabledDecoderNodes],
|
|
25663
|
+
enabledAudioNodes: [...this.enabledAudioNodes],
|
|
25664
|
+
clusterRoles: { ...this.clusterRoles },
|
|
25545
25665
|
assignedDeviceCount: this.assignments.size,
|
|
25546
25666
|
cameraConfigCount: this.cameraConfigs.size,
|
|
25547
25667
|
activeDetectionCount: this.activeDetections.size
|
|
@@ -25566,6 +25686,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25566
25686
|
clearTimeout(this.pendingRetryDebounceTimer);
|
|
25567
25687
|
this.pendingRetryDebounceTimer = null;
|
|
25568
25688
|
}
|
|
25689
|
+
for (const t of this.slotReadRetryTimers.values()) clearTimeout(t);
|
|
25690
|
+
this.slotReadRetryTimers.clear();
|
|
25691
|
+
this.slotReadRetryAttempts.clear();
|
|
25569
25692
|
this.unsubDeviceRegistered?.();
|
|
25570
25693
|
this.unsubDeviceRegistered = null;
|
|
25571
25694
|
this.unsubDeviceUnregistered?.();
|
|
@@ -25666,6 +25789,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25666
25789
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
25667
25790
|
preferredAgent,
|
|
25668
25791
|
nodeCaps: await this.buildNodeCaps(),
|
|
25792
|
+
weights: await this.buildNodeWeights(),
|
|
25669
25793
|
eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
|
|
25670
25794
|
});
|
|
25671
25795
|
if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
|
|
@@ -25782,6 +25906,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25782
25906
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
25783
25907
|
preferredAgent: null,
|
|
25784
25908
|
nodeCaps: await this.buildNodeCaps(),
|
|
25909
|
+
weights: await this.buildNodeWeights(),
|
|
25785
25910
|
eligibleNodes: this.detectionEligibleNodes(input.deviceId)
|
|
25786
25911
|
});
|
|
25787
25912
|
if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
|
|
@@ -25830,6 +25955,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25830
25955
|
if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
|
|
25831
25956
|
const loads = await this.collectAgentLoad({ onlyEnabled: true });
|
|
25832
25957
|
const nodeCaps = await this.buildNodeCaps();
|
|
25958
|
+
const nodeWeights = await this.buildNodeWeights();
|
|
25833
25959
|
const attachedDelta = /* @__PURE__ */ new Map();
|
|
25834
25960
|
const bumpAttached = (nodeId, by) => {
|
|
25835
25961
|
attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
|
|
@@ -25851,6 +25977,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25851
25977
|
nodes: talliedLoads(),
|
|
25852
25978
|
preferredAgent,
|
|
25853
25979
|
nodeCaps,
|
|
25980
|
+
weights: nodeWeights,
|
|
25854
25981
|
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
25855
25982
|
});
|
|
25856
25983
|
if (!decision) continue;
|
|
@@ -26306,6 +26433,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26306
26433
|
for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
|
|
26307
26434
|
return caps;
|
|
26308
26435
|
}
|
|
26436
|
+
/**
|
|
26437
|
+
* Build a per-node detection WEIGHT map from the persisted agent settings —
|
|
26438
|
+
* the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
|
|
26439
|
+
* are simply absent (the balancer treats absent as weight 1). Passed to every
|
|
26440
|
+
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
26441
|
+
*/
|
|
26442
|
+
async buildNodeWeights() {
|
|
26443
|
+
const blob = await this.agentSettingsState.get();
|
|
26444
|
+
const weights = {};
|
|
26445
|
+
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
26446
|
+
return weights;
|
|
26447
|
+
}
|
|
26309
26448
|
async readAudioNodePin(deviceId) {
|
|
26310
26449
|
if (!this.ctx?.settings) return null;
|
|
26311
26450
|
try {
|
|
@@ -26454,6 +26593,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26454
26593
|
nodes: loads,
|
|
26455
26594
|
preferredAgent: null,
|
|
26456
26595
|
nodeCaps: await this.buildNodeCaps(),
|
|
26596
|
+
weights: await this.buildNodeWeights(),
|
|
26457
26597
|
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
26458
26598
|
});
|
|
26459
26599
|
if (!decision) {
|
|
@@ -26838,6 +26978,47 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26838
26978
|
}
|
|
26839
26979
|
}
|
|
26840
26980
|
/**
|
|
26981
|
+
* Bounded per-device retry of {@link handleDeviceRegistered} after a
|
|
26982
|
+
* TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). The
|
|
26983
|
+
* read (`listAllProfileSlots`) can time out during a broker (re)start; we
|
|
26984
|
+
* must neither stop a running camera nor leave a not-yet-started camera
|
|
26985
|
+
* stranded — so re-run the registration handler with exponential backoff
|
|
26986
|
+
* until the read succeeds or the budget is exhausted. Reset on any
|
|
26987
|
+
* successful read ({@link clearSlotReadRetry}).
|
|
26988
|
+
*/
|
|
26989
|
+
scheduleSlotReadRetry(deviceId) {
|
|
26990
|
+
const attempts = this.slotReadRetryAttempts.get(deviceId) ?? 0;
|
|
26991
|
+
if (attempts >= PipelineOrchestratorAddon.SLOT_READ_MAX_RETRIES) {
|
|
26992
|
+
this.ctx.logger.warn("slot-read retry budget exhausted — leaving detection as-is", { tags: { deviceId } });
|
|
26993
|
+
this.slotReadRetryAttempts.delete(deviceId);
|
|
26994
|
+
return;
|
|
26995
|
+
}
|
|
26996
|
+
const existing = this.slotReadRetryTimers.get(deviceId);
|
|
26997
|
+
if (existing) clearTimeout(existing);
|
|
26998
|
+
const delay = PipelineOrchestratorAddon.SLOT_READ_RETRY_BASE_MS * 2 ** attempts;
|
|
26999
|
+
this.slotReadRetryAttempts.set(deviceId, attempts + 1);
|
|
27000
|
+
const timer = setTimeout(() => {
|
|
27001
|
+
this.slotReadRetryTimers.delete(deviceId);
|
|
27002
|
+
this.handleDeviceRegistered(deviceId).catch((err) => {
|
|
27003
|
+
this.ctx.logger.debug("slot-read retry: handleDeviceRegistered failed", {
|
|
27004
|
+
tags: { deviceId },
|
|
27005
|
+
meta: { error: errMsg(err) }
|
|
27006
|
+
});
|
|
27007
|
+
});
|
|
27008
|
+
}, delay);
|
|
27009
|
+
if (typeof timer === "object" && timer !== null && "unref" in timer) timer.unref();
|
|
27010
|
+
this.slotReadRetryTimers.set(deviceId, timer);
|
|
27011
|
+
}
|
|
27012
|
+
/** Cancel any pending slot-read retry + reset the budget for a device. */
|
|
27013
|
+
clearSlotReadRetry(deviceId) {
|
|
27014
|
+
const existing = this.slotReadRetryTimers.get(deviceId);
|
|
27015
|
+
if (existing) {
|
|
27016
|
+
clearTimeout(existing);
|
|
27017
|
+
this.slotReadRetryTimers.delete(deviceId);
|
|
27018
|
+
}
|
|
27019
|
+
this.slotReadRetryAttempts.delete(deviceId);
|
|
27020
|
+
}
|
|
27021
|
+
/**
|
|
26841
27022
|
* Coalesce bursts of capacity/eligibility/readiness signals into a single
|
|
26842
27023
|
* `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
|
|
26843
27024
|
* dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
|
|
@@ -27181,6 +27362,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27181
27362
|
};
|
|
27182
27363
|
const { [input.agentNodeId]: _drop, ...rest } = all;
|
|
27183
27364
|
await this.agentSettingsState.set(rest);
|
|
27365
|
+
await this.refreshNodeCapabilities();
|
|
27184
27366
|
this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
|
|
27185
27367
|
return {
|
|
27186
27368
|
success: true,
|
|
@@ -27204,6 +27386,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27204
27386
|
this.schedulePendingRetry();
|
|
27205
27387
|
return { success: true };
|
|
27206
27388
|
}
|
|
27389
|
+
async setAgentDetectWeight(input) {
|
|
27390
|
+
const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
|
|
27391
|
+
const weight = input.detectWeight ?? 1;
|
|
27392
|
+
const next = existing ? {
|
|
27393
|
+
...existing,
|
|
27394
|
+
detectWeight: weight
|
|
27395
|
+
} : {
|
|
27396
|
+
addonDefaults: {},
|
|
27397
|
+
maxCameras: null,
|
|
27398
|
+
detectWeight: weight
|
|
27399
|
+
};
|
|
27400
|
+
await this.writeAgentSettings(input.agentNodeId, next);
|
|
27401
|
+
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
27402
|
+
tags: { nodeId: input.agentNodeId },
|
|
27403
|
+
meta: { detectWeight: weight ?? null }
|
|
27404
|
+
});
|
|
27405
|
+
this.rebalance().catch((err) => {
|
|
27406
|
+
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
27407
|
+
});
|
|
27408
|
+
return { success: true };
|
|
27409
|
+
}
|
|
27410
|
+
/**
|
|
27411
|
+
* Set one node's placement capabilities — the per-node record that replaced
|
|
27412
|
+
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
27413
|
+
* leave unchanged, `null` to reset to the node default (persisted as an
|
|
27414
|
+
* absent flag), `true`/`false` to force. After persisting, re-derive the
|
|
27415
|
+
* enabled-node sets and reconcile dispatch so the change takes effect
|
|
27416
|
+
* immediately (reuses the existing balancer / retry, no custom logic).
|
|
27417
|
+
*/
|
|
27418
|
+
async setAgentCapabilities(input) {
|
|
27419
|
+
const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
|
|
27420
|
+
addonDefaults: {},
|
|
27421
|
+
maxCameras: null
|
|
27422
|
+
};
|
|
27423
|
+
const apply = (key) => {
|
|
27424
|
+
const patch = input[key];
|
|
27425
|
+
if (patch === void 0) return existing[key];
|
|
27426
|
+
if (patch === null) return void 0;
|
|
27427
|
+
return patch;
|
|
27428
|
+
};
|
|
27429
|
+
const next = {
|
|
27430
|
+
...existing,
|
|
27431
|
+
detect: apply("detect"),
|
|
27432
|
+
decode: apply("decode"),
|
|
27433
|
+
audio: apply("audio"),
|
|
27434
|
+
ingest: apply("ingest")
|
|
27435
|
+
};
|
|
27436
|
+
await this.writeAgentSettings(input.agentNodeId, next);
|
|
27437
|
+
this.ctx.logger.info("agentSettings.capabilities updated", {
|
|
27438
|
+
tags: { nodeId: input.agentNodeId },
|
|
27439
|
+
meta: {
|
|
27440
|
+
detect: next.detect,
|
|
27441
|
+
decode: next.decode,
|
|
27442
|
+
audio: next.audio,
|
|
27443
|
+
ingest: next.ingest
|
|
27444
|
+
}
|
|
27445
|
+
});
|
|
27446
|
+
await this.refreshNodeCapabilities();
|
|
27447
|
+
this.knownRunnerNodes.add(input.agentNodeId);
|
|
27448
|
+
this.schedulePendingRetry();
|
|
27449
|
+
return { success: true };
|
|
27450
|
+
}
|
|
27207
27451
|
async getCameraSettings(input) {
|
|
27208
27452
|
return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|
|
27209
27453
|
}
|
|
@@ -27569,7 +27813,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27569
27813
|
const all = await this.readAgentSettingsMap();
|
|
27570
27814
|
const existing = all[nodeId];
|
|
27571
27815
|
all[nodeId] = {
|
|
27572
|
-
|
|
27816
|
+
...existing,
|
|
27817
|
+
...settings,
|
|
27573
27818
|
maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
|
|
27574
27819
|
};
|
|
27575
27820
|
await this.agentSettingsState.set(all);
|
|
@@ -27852,45 +28097,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27852
28097
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
27853
28098
|
return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
|
|
27854
28099
|
}
|
|
27855
|
-
/** Build the addon-level schema (cluster
|
|
28100
|
+
/** Build the addon-level schema (cluster roles + balancer + failover). */
|
|
27856
28101
|
globalSettingsSchema() {
|
|
27857
28102
|
return this.schema({ sections: [
|
|
27858
28103
|
{
|
|
27859
28104
|
id: "cluster",
|
|
27860
|
-
title: "Cluster",
|
|
28105
|
+
title: "Cluster Roles",
|
|
27861
28106
|
tab: "pipeline",
|
|
27862
|
-
description: "
|
|
28107
|
+
description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
|
|
27863
28108
|
fields: [
|
|
27864
28109
|
{
|
|
27865
|
-
key: "
|
|
27866
|
-
type: "node-
|
|
27867
|
-
label: "
|
|
27868
|
-
description: "
|
|
27869
|
-
default:
|
|
28110
|
+
key: "ingestNode",
|
|
28111
|
+
type: "node-select",
|
|
28112
|
+
label: "Ingest Node",
|
|
28113
|
+
description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest ≠ hub is a later phase; kept here so the role is modeled.)",
|
|
28114
|
+
default: "hub",
|
|
27870
28115
|
showOffline: true
|
|
27871
28116
|
},
|
|
27872
28117
|
{
|
|
27873
|
-
key: "
|
|
27874
|
-
type: "node-
|
|
27875
|
-
label: "
|
|
27876
|
-
description: "
|
|
27877
|
-
default:
|
|
28118
|
+
key: "recordingNode",
|
|
28119
|
+
type: "node-select",
|
|
28120
|
+
label: "Recording Node",
|
|
28121
|
+
description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
|
|
28122
|
+
default: "hub",
|
|
27878
28123
|
showOffline: true
|
|
27879
28124
|
},
|
|
27880
28125
|
{
|
|
27881
|
-
key: "
|
|
27882
|
-
type: "node-
|
|
27883
|
-
label: "
|
|
27884
|
-
description: "
|
|
27885
|
-
default:
|
|
27886
|
-
showOffline: true
|
|
27887
|
-
},
|
|
27888
|
-
{
|
|
27889
|
-
key: "enabledAudioNodes",
|
|
27890
|
-
type: "node-multiselect",
|
|
27891
|
-
label: "Enabled Audio Nodes",
|
|
27892
|
-
description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
|
|
27893
|
-
default: ["hub"],
|
|
28126
|
+
key: "audioNode",
|
|
28127
|
+
type: "node-select",
|
|
28128
|
+
label: "Audio Node",
|
|
28129
|
+
description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
|
|
28130
|
+
default: "hub",
|
|
27894
28131
|
showOffline: true
|
|
27895
28132
|
}
|
|
27896
28133
|
]
|
|
@@ -28393,16 +28630,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28393
28630
|
pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
|
|
28394
28631
|
onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
|
|
28395
28632
|
};
|
|
28396
|
-
const
|
|
28397
|
-
|
|
28398
|
-
|
|
28399
|
-
|
|
28400
|
-
|
|
28401
|
-
|
|
28402
|
-
|
|
28403
|
-
|
|
28633
|
+
const readRole = (key) => {
|
|
28634
|
+
const raw = config[key];
|
|
28635
|
+
return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
|
|
28636
|
+
};
|
|
28637
|
+
this.clusterRoles = {
|
|
28638
|
+
ingestNode: readRole("ingestNode"),
|
|
28639
|
+
recordingNode: readRole("recordingNode"),
|
|
28640
|
+
audioNode: readRole("audioNode")
|
|
28641
|
+
};
|
|
28404
28642
|
this.schedulePendingRetry();
|
|
28405
28643
|
}
|
|
28644
|
+
/**
|
|
28645
|
+
* Default placement capability for a node when its stored flag is unset:
|
|
28646
|
+
* the hub is capable of every role out of the box; every other node must be
|
|
28647
|
+
* explicitly enabled. This reproduces the classic `['hub']` defaults of the
|
|
28648
|
+
* four removed flat lists against an empty capability store.
|
|
28649
|
+
*/
|
|
28650
|
+
static nodeCapabilityDefault(nodeId) {
|
|
28651
|
+
return nodeId === "hub";
|
|
28652
|
+
}
|
|
28653
|
+
/**
|
|
28654
|
+
* Recompute the derived enabled-node sets (`enabledNodes` /
|
|
28655
|
+
* `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
|
|
28656
|
+
* store. A node is eligible for a concern iff its stored flag is `true`, or
|
|
28657
|
+
* the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
|
|
28658
|
+
* Forked child nodeIds (`hub/classifier`) are never dispatchable and are
|
|
28659
|
+
* excluded. `hub` is always considered even when absent from the store, so a
|
|
28660
|
+
* fresh install keeps the hub-only cluster working.
|
|
28661
|
+
*/
|
|
28662
|
+
async refreshNodeCapabilities() {
|
|
28663
|
+
let blob = {};
|
|
28664
|
+
try {
|
|
28665
|
+
blob = await this.agentSettingsState.get();
|
|
28666
|
+
} catch (err) {
|
|
28667
|
+
this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
|
|
28668
|
+
}
|
|
28669
|
+
const nodeIds = new Set(["hub"]);
|
|
28670
|
+
for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
|
|
28671
|
+
const detect = [];
|
|
28672
|
+
const decode = [];
|
|
28673
|
+
const audio = [];
|
|
28674
|
+
const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
|
|
28675
|
+
for (const nodeId of nodeIds) {
|
|
28676
|
+
const settings = blob[nodeId];
|
|
28677
|
+
if (capable(settings?.detect, nodeId)) detect.push(nodeId);
|
|
28678
|
+
if (capable(settings?.decode, nodeId)) decode.push(nodeId);
|
|
28679
|
+
if (capable(settings?.audio, nodeId)) audio.push(nodeId);
|
|
28680
|
+
}
|
|
28681
|
+
this.enabledNodes = detect.toSorted();
|
|
28682
|
+
this.enabledDecoderNodes = decode.toSorted();
|
|
28683
|
+
this.enabledAudioNodes = audio.toSorted();
|
|
28684
|
+
}
|
|
28406
28685
|
get api() {
|
|
28407
28686
|
return this.ctx.api ?? null;
|
|
28408
28687
|
}
|
|
@@ -28616,7 +28895,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28616
28895
|
*/
|
|
28617
28896
|
async fetchAssignedProfiles(deviceId) {
|
|
28618
28897
|
const api = this.api;
|
|
28619
|
-
if (!api) return
|
|
28898
|
+
if (!api) return null;
|
|
28620
28899
|
try {
|
|
28621
28900
|
const slots = await api.streamBroker.listAllProfileSlots.query();
|
|
28622
28901
|
const out = /* @__PURE__ */ new Map();
|
|
@@ -28626,8 +28905,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28626
28905
|
out.set(slot.profile, slot.sourceCamStreamId);
|
|
28627
28906
|
}
|
|
28628
28907
|
return out;
|
|
28629
|
-
} catch {
|
|
28630
|
-
|
|
28908
|
+
} catch (err) {
|
|
28909
|
+
this.ctx.logger.debug("fetchAssignedProfiles: slot read failed (transient) — treating as unknown", {
|
|
28910
|
+
tags: { deviceId },
|
|
28911
|
+
meta: { error: errMsg(err) }
|
|
28912
|
+
});
|
|
28913
|
+
return null;
|
|
28631
28914
|
}
|
|
28632
28915
|
}
|
|
28633
28916
|
/**
|
|
@@ -28646,6 +28929,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28646
28929
|
*/
|
|
28647
28930
|
async buildDetectionConfig(deviceId) {
|
|
28648
28931
|
const assigned = await this.fetchAssignedProfiles(deviceId);
|
|
28932
|
+
if (assigned === null) return TRANSIENT_SLOT_READ;
|
|
28649
28933
|
if (assigned.size === 0) return null;
|
|
28650
28934
|
const resolved = await this.resolveDeviceDetectionSettings(deviceId);
|
|
28651
28935
|
if (!resolved) return null;
|
|
@@ -28789,6 +29073,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28789
29073
|
}
|
|
28790
29074
|
/** Stop detection and purge all persisted config for a removed device. */
|
|
28791
29075
|
async handleDeviceUnregistered(deviceId) {
|
|
29076
|
+
this.clearSlotReadRetry(deviceId);
|
|
28792
29077
|
await this.stopDetection(deviceId);
|
|
28793
29078
|
this.zonesProvider?.forgetDevice(deviceId);
|
|
28794
29079
|
this.zoneRulesProvider?.forgetDevice(deviceId);
|
|
@@ -28810,6 +29095,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28810
29095
|
const log = this.ctx.logger.withTags({ deviceId });
|
|
28811
29096
|
log.info("handleDeviceRegistered", { meta: { phase: "handleDeviceRegistered" } });
|
|
28812
29097
|
const config = await this.buildDetectionConfig(deviceId);
|
|
29098
|
+
if (config === TRANSIENT_SLOT_READ) {
|
|
29099
|
+
log.warn("Profile-slot read failed transiently — keeping detection, scheduling retry");
|
|
29100
|
+
this.scheduleSlotReadRetry(deviceId);
|
|
29101
|
+
return;
|
|
29102
|
+
}
|
|
29103
|
+
this.clearSlotReadRetry(deviceId);
|
|
28813
29104
|
log.info("[pipeline-orchestrator] buildDetectionConfig", config ? { meta: {
|
|
28814
29105
|
enabled: config.enabled,
|
|
28815
29106
|
motionStreamId: config.motionStreamId,
|
|
@@ -28875,6 +29166,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28875
29166
|
const log = this.ctx.logger.withTags({ deviceId });
|
|
28876
29167
|
if (this.activeDetections.has(deviceId)) await this.stopDetection(deviceId);
|
|
28877
29168
|
const config = await this.buildDetectionConfig(deviceId);
|
|
29169
|
+
if (config === TRANSIENT_SLOT_READ) {
|
|
29170
|
+
log.warn("Settings changed — slot read failed transiently, scheduling retry");
|
|
29171
|
+
this.scheduleSlotReadRetry(deviceId);
|
|
29172
|
+
return;
|
|
29173
|
+
}
|
|
29174
|
+
this.clearSlotReadRetry(deviceId);
|
|
28878
29175
|
if (!config) {
|
|
28879
29176
|
log.info("Settings changed — no assigned slot, detection stopped");
|
|
28880
29177
|
return;
|
|
@@ -28999,7 +29296,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28999
29296
|
});
|
|
29000
29297
|
return;
|
|
29001
29298
|
}
|
|
29002
|
-
this.
|
|
29299
|
+
if (this.shedInFlight.has(deviceId)) return;
|
|
29300
|
+
this.shedInFlight.add(deviceId);
|
|
29301
|
+
this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
|
|
29003
29302
|
tags: { deviceId },
|
|
29004
29303
|
meta: {
|
|
29005
29304
|
actualFps: metrics.actualFps,
|
|
@@ -29008,7 +29307,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29008
29307
|
windowMs
|
|
29009
29308
|
}
|
|
29010
29309
|
});
|
|
29011
|
-
state.
|
|
29310
|
+
this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
|
|
29311
|
+
}
|
|
29312
|
+
/**
|
|
29313
|
+
* Cluster-wide load-shed action: relocate the sustained-slow camera to a
|
|
29314
|
+
* less-loaded node if the balancer (ceiling + weight aware) finds one,
|
|
29315
|
+
* otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
|
|
29316
|
+
* placement logic.
|
|
29317
|
+
*/
|
|
29318
|
+
async shedOrRelocate(deviceId, state) {
|
|
29319
|
+
if (state.pausedAt !== null) return;
|
|
29320
|
+
if (await this.tryClusterRelocate(deviceId).catch((err) => {
|
|
29321
|
+
this.ctx.logger.warn("Load management: cluster relocate failed", {
|
|
29322
|
+
tags: { deviceId },
|
|
29323
|
+
meta: { error: errMsg(err) }
|
|
29324
|
+
});
|
|
29325
|
+
return false;
|
|
29326
|
+
})) {
|
|
29327
|
+
state.lowSinceTs = null;
|
|
29328
|
+
return;
|
|
29329
|
+
}
|
|
29330
|
+
this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
|
|
29331
|
+
state.pausedAt = Date.now();
|
|
29012
29332
|
state.lowSinceTs = null;
|
|
29013
29333
|
this.pipelineWatchdog?.unregister(deviceId);
|
|
29014
29334
|
this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
|
|
@@ -29019,6 +29339,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29019
29339
|
});
|
|
29020
29340
|
this.ensureLoadShedResumeTimer();
|
|
29021
29341
|
}
|
|
29342
|
+
/**
|
|
29343
|
+
* Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
|
|
29344
|
+
* Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
|
|
29345
|
+
* frame-source eligibility); the camera's current node counts it in its own
|
|
29346
|
+
* load, so a genuinely less-loaded node with spare capacity wins. Returns
|
|
29347
|
+
* `true` iff the camera was moved to a DIFFERENT node. A manually pinned
|
|
29348
|
+
* camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
|
|
29349
|
+
*/
|
|
29350
|
+
async tryClusterRelocate(deviceId) {
|
|
29351
|
+
const current = this.assignments.get(deviceId);
|
|
29352
|
+
const cached = this.cameraConfigs.get(deviceId);
|
|
29353
|
+
if (!current || !cached) return false;
|
|
29354
|
+
const currentNode = current.agentNodeId;
|
|
29355
|
+
if (await this.readPipelinePin(deviceId)) return false;
|
|
29356
|
+
const decision = balance({
|
|
29357
|
+
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
29358
|
+
preferredAgent: null,
|
|
29359
|
+
nodeCaps: await this.buildNodeCaps(),
|
|
29360
|
+
weights: await this.buildNodeWeights(),
|
|
29361
|
+
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
29362
|
+
});
|
|
29363
|
+
if (!decision || decision.kind !== "assigned") return false;
|
|
29364
|
+
const target = decision.agentNodeId;
|
|
29365
|
+
if (target === currentNode) return false;
|
|
29366
|
+
await this.detachOn(currentNode, deviceId).catch((err) => {
|
|
29367
|
+
this.ctx.logger.debug("Load management: relocate detach-old failed", {
|
|
29368
|
+
tags: { deviceId },
|
|
29369
|
+
meta: {
|
|
29370
|
+
from: currentNode,
|
|
29371
|
+
error: errMsg(err)
|
|
29372
|
+
}
|
|
29373
|
+
});
|
|
29374
|
+
});
|
|
29375
|
+
await this.attachOn(target, cached);
|
|
29376
|
+
this.recordAssignment(deviceId, target, "rebalance", false);
|
|
29377
|
+
this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
|
|
29378
|
+
tags: { deviceId },
|
|
29379
|
+
meta: {
|
|
29380
|
+
from: currentNode,
|
|
29381
|
+
to: target,
|
|
29382
|
+
score: decision.score
|
|
29383
|
+
}
|
|
29384
|
+
});
|
|
29385
|
+
return true;
|
|
29386
|
+
}
|
|
29022
29387
|
/** Base cooldown before first auto-resume attempt. */
|
|
29023
29388
|
static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
|
|
29024
29389
|
/** Maximum backoff cap. */
|