@camstack/addon-pipeline-orchestrator 1.1.21 → 1.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +1 -1
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DVzgRExi.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CQmCy3VK.mjs} +3 -3
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DdDNCPJ5.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BA0gpKU7.mjs} +1 -1
- package/dist/{hostInit-CEqZfaom.mjs → hostInit-ibn4yaBr.mjs} +3 -3
- package/dist/index.js +423 -58
- package/dist/index.mjs +423 -58
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -15703,7 +15703,17 @@ var AgentAddonConfigSchema = object({
|
|
|
15703
15703
|
});
|
|
15704
15704
|
var AgentPipelineSettingsSchema = object({
|
|
15705
15705
|
addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
|
|
15706
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
15706
|
+
maxCameras: number().int().nonnegative().nullable().default(null),
|
|
15707
|
+
/** Per-node detection weight (relative share for the quota balancer). */
|
|
15708
|
+
detectWeight: number().positive().optional(),
|
|
15709
|
+
/** Node is eligible to run the detection pipeline (decode + inference). */
|
|
15710
|
+
detect: boolean().optional(),
|
|
15711
|
+
/** Node is eligible to host decoder sessions. */
|
|
15712
|
+
decode: boolean().optional(),
|
|
15713
|
+
/** Node is eligible to run audio-analyzer sessions. */
|
|
15714
|
+
audio: boolean().optional(),
|
|
15715
|
+
/** Node is eligible to be the ingest / source-owner (serve the restream). */
|
|
15716
|
+
ingest: boolean().optional()
|
|
15707
15717
|
});
|
|
15708
15718
|
var CameraPipelineForAgentSchema = object({
|
|
15709
15719
|
steps: array(PipelineStepInputSchema).readonly(),
|
|
@@ -16137,6 +16147,38 @@ var pipelineOrchestratorCapability = {
|
|
|
16137
16147
|
kind: "mutation",
|
|
16138
16148
|
auth: "admin"
|
|
16139
16149
|
}),
|
|
16150
|
+
/**
|
|
16151
|
+
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
16152
|
+
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
16153
|
+
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
16154
|
+
*/
|
|
16155
|
+
setAgentDetectWeight: method(object({
|
|
16156
|
+
agentNodeId: string(),
|
|
16157
|
+
detectWeight: number().positive().nullable()
|
|
16158
|
+
}), object({ success: literal(true) }), {
|
|
16159
|
+
kind: "mutation",
|
|
16160
|
+
auth: "admin"
|
|
16161
|
+
}),
|
|
16162
|
+
/**
|
|
16163
|
+
* Set one node's placement CAPABILITIES — the per-node record that
|
|
16164
|
+
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
16165
|
+
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
16166
|
+
* Each flag is optional in the patch: omit a flag to leave it unchanged,
|
|
16167
|
+
* pass `null` to reset it to the node default (`hub` → capable, every
|
|
16168
|
+
* other node → not). Detection/decode/audio eligibility is derived from
|
|
16169
|
+
* these flags across the cluster; changing them re-derives the enabled
|
|
16170
|
+
* sets and reconciles dispatch immediately.
|
|
16171
|
+
*/
|
|
16172
|
+
setAgentCapabilities: method(object({
|
|
16173
|
+
agentNodeId: string(),
|
|
16174
|
+
detect: boolean().nullable().optional(),
|
|
16175
|
+
decode: boolean().nullable().optional(),
|
|
16176
|
+
audio: boolean().nullable().optional(),
|
|
16177
|
+
ingest: boolean().nullable().optional()
|
|
16178
|
+
}), object({ success: literal(true) }), {
|
|
16179
|
+
kind: "mutation",
|
|
16180
|
+
auth: "admin"
|
|
16181
|
+
}),
|
|
16140
16182
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
16141
16183
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
16142
16184
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -22043,6 +22085,18 @@ Object.freeze({
|
|
|
22043
22085
|
addonId: null,
|
|
22044
22086
|
access: "create"
|
|
22045
22087
|
},
|
|
22088
|
+
"pipelineOrchestrator.setAgentCapabilities": {
|
|
22089
|
+
capName: "pipeline-orchestrator",
|
|
22090
|
+
capScope: "system",
|
|
22091
|
+
addonId: null,
|
|
22092
|
+
access: "create"
|
|
22093
|
+
},
|
|
22094
|
+
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
22095
|
+
capName: "pipeline-orchestrator",
|
|
22096
|
+
capScope: "system",
|
|
22097
|
+
addonId: null,
|
|
22098
|
+
access: "create"
|
|
22099
|
+
},
|
|
22046
22100
|
"pipelineOrchestrator.setAgentMaxCameras": {
|
|
22047
22101
|
capName: "pipeline-orchestrator",
|
|
22048
22102
|
capScope: "system",
|
|
@@ -24186,6 +24240,23 @@ function computeCapacityScore(load) {
|
|
|
24186
24240
|
return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
|
|
24187
24241
|
}
|
|
24188
24242
|
/**
|
|
24243
|
+
* Node detection weight, defaulting to 1 when absent/invalid. A higher weight
|
|
24244
|
+
* makes a node preferred proportionally more (see {@link BalancerInput.weights}).
|
|
24245
|
+
*/
|
|
24246
|
+
function nodeWeight(nodeId, weights) {
|
|
24247
|
+
const w = weights?.[nodeId];
|
|
24248
|
+
return w !== void 0 && w > 0 ? w : 1;
|
|
24249
|
+
}
|
|
24250
|
+
/**
|
|
24251
|
+
* Weight-adjusted capacity score — the value the balancer minimises. Dividing
|
|
24252
|
+
* the raw capacity score by the node's weight makes a heavier node accumulate
|
|
24253
|
+
* proportionally more load before it loses preference, yielding weight-
|
|
24254
|
+
* proportional distribution while still preferring drained/idle nodes.
|
|
24255
|
+
*/
|
|
24256
|
+
function weightedScore(load, weights) {
|
|
24257
|
+
return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
|
|
24258
|
+
}
|
|
24259
|
+
/**
|
|
24189
24260
|
* Returns true when the node has remaining capacity.
|
|
24190
24261
|
* A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
|
|
24191
24262
|
* its `attachedCameras` count is strictly less than the cap.
|
|
@@ -24251,8 +24322,9 @@ function balance(input) {
|
|
|
24251
24322
|
};
|
|
24252
24323
|
const best = eligible.map((node) => ({
|
|
24253
24324
|
node,
|
|
24254
|
-
score: computeCapacityScore(node)
|
|
24255
|
-
|
|
24325
|
+
score: computeCapacityScore(node),
|
|
24326
|
+
sortKey: weightedScore(node, input.weights)
|
|
24327
|
+
})).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
|
|
24256
24328
|
return {
|
|
24257
24329
|
kind: "assigned",
|
|
24258
24330
|
agentNodeId: best.node.nodeId,
|
|
@@ -24354,7 +24426,12 @@ var StoredAgentAddonConfigSchema = object({
|
|
|
24354
24426
|
});
|
|
24355
24427
|
var StoredAgentPipelineSettingsSchema = object({
|
|
24356
24428
|
addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
|
|
24357
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
24429
|
+
maxCameras: number().int().nonnegative().nullable().default(null),
|
|
24430
|
+
detectWeight: number().positive().optional(),
|
|
24431
|
+
detect: boolean().optional(),
|
|
24432
|
+
decode: boolean().optional(),
|
|
24433
|
+
audio: boolean().optional(),
|
|
24434
|
+
ingest: boolean().optional()
|
|
24358
24435
|
});
|
|
24359
24436
|
var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
|
|
24360
24437
|
var StoredCameraStepOverridePatchSchema = object({
|
|
@@ -24893,11 +24970,27 @@ var OrchestratorDiagnosticsSchema = object({
|
|
|
24893
24970
|
knownRunnerNodes: array(string()),
|
|
24894
24971
|
cachedAgentLoadNodeIds: array(string()),
|
|
24895
24972
|
enabledNodes: array(string()),
|
|
24973
|
+
enabledDecoderNodes: array(string()),
|
|
24974
|
+
enabledAudioNodes: array(string()),
|
|
24975
|
+
clusterRoles: object({
|
|
24976
|
+
ingestNode: string(),
|
|
24977
|
+
recordingNode: string(),
|
|
24978
|
+
audioNode: string()
|
|
24979
|
+
}),
|
|
24896
24980
|
assignedDeviceCount: number().int().min(0),
|
|
24897
24981
|
cameraConfigCount: number().int().min(0),
|
|
24898
24982
|
activeDetectionCount: number().int().min(0)
|
|
24899
24983
|
});
|
|
24900
24984
|
var pipelineOrchestratorActions = defineCustomActions({ dumpState: customAction(_void(), OrchestratorDiagnosticsSchema) });
|
|
24985
|
+
/**
|
|
24986
|
+
* Sentinel returned by `buildDetectionConfig` when the profile-slot READ
|
|
24987
|
+
* itself failed transiently (e.g. `listAllProfileSlots` discovery timeout
|
|
24988
|
+
* while the stream-broker is (re)starting) — as opposed to `null`, which
|
|
24989
|
+
* means "genuinely no assigned slot / not configured". Callers MUST treat
|
|
24990
|
+
* this differently from `null`: never stop active detection on a transient
|
|
24991
|
+
* read failure (the slots almost certainly still exist), and schedule a retry.
|
|
24992
|
+
*/
|
|
24993
|
+
var TRANSIENT_SLOT_READ = Symbol("transient-slot-read");
|
|
24901
24994
|
var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddon {
|
|
24902
24995
|
/** This node's Moleculer nodeId (from this.ctx.kernel.localNodeId). */
|
|
24903
24996
|
localNodeId = "hub";
|
|
@@ -25009,36 +25102,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25009
25102
|
failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
|
|
25010
25103
|
/**
|
|
25011
25104
|
* Hub-wide allow-list of node ids eligible to run the detection
|
|
25012
|
-
* pipeline.
|
|
25013
|
-
*
|
|
25014
|
-
*
|
|
25015
|
-
* The list is a strict whitelist —
|
|
25016
|
-
* for other capabilities (metrics, logs,
|
|
25017
|
-
* never receive camera assignments.
|
|
25105
|
+
* pipeline. DERIVED (placement-model redesign) from the per-node
|
|
25106
|
+
* capability store (`agentSettings[node].detect`) — no longer a flat
|
|
25107
|
+
* `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
|
|
25108
|
+
* on boot and on every capability write. The list is a strict whitelist —
|
|
25109
|
+
* disabled nodes stay connected for other capabilities (metrics, logs,
|
|
25110
|
+
* cluster-wide events) but never receive camera assignments.
|
|
25018
25111
|
*/
|
|
25019
25112
|
enabledNodes = ["hub"];
|
|
25020
25113
|
/**
|
|
25021
25114
|
* Hub-wide allow-list of node ids eligible to run decoder sessions.
|
|
25022
|
-
*
|
|
25115
|
+
* DERIVED from the per-node capability store (`agentSettings[node].decode`).
|
|
25023
25116
|
* Defaults to `['hub']` — the hub always has a decoder available.
|
|
25024
25117
|
*/
|
|
25025
25118
|
enabledDecoderNodes = ["hub"];
|
|
25026
25119
|
/**
|
|
25027
|
-
*
|
|
25028
|
-
*
|
|
25029
|
-
*
|
|
25030
|
-
*
|
|
25031
|
-
*
|
|
25032
|
-
*
|
|
25120
|
+
* Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
|
|
25121
|
+
* restream over the LAN and decode it locally. Held for the source-owner
|
|
25122
|
+
* plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
|
|
25123
|
+
* frame transport is a separate activation phase (placement-model §9 P1).
|
|
25124
|
+
* With this empty, `resolveSourceOwner` returns `undefined` and every
|
|
25125
|
+
* `selectRunnerFrameSource` emits `local-broker` — bit-identical to
|
|
25126
|
+
* pre-Phase-2 behavior.
|
|
25033
25127
|
*/
|
|
25034
25128
|
remoteSourcingNodes = [];
|
|
25035
25129
|
/**
|
|
25036
25130
|
* Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
|
|
25037
|
-
*
|
|
25131
|
+
* DERIVED from the per-node capability store (`agentSettings[node].audio`).
|
|
25038
25132
|
* Defaults to `['hub']`.
|
|
25039
25133
|
*/
|
|
25040
25134
|
enabledAudioNodes = ["hub"];
|
|
25041
25135
|
/**
|
|
25136
|
+
* Cluster-wide singleton role assignments (placement-model §6). Each role
|
|
25137
|
+
* is the ONE node that serves it for every camera. Driven by the
|
|
25138
|
+
* `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
|
|
25139
|
+
* global schema; defaults to the hub for all three.
|
|
25140
|
+
*/
|
|
25141
|
+
clusterRoles = {
|
|
25142
|
+
ingestNode: "hub",
|
|
25143
|
+
recordingNode: "hub",
|
|
25144
|
+
audioNode: "hub"
|
|
25145
|
+
};
|
|
25146
|
+
/**
|
|
25042
25147
|
* Full global settings snapshot kept in memory so synchronous cap methods
|
|
25043
25148
|
* (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
|
|
25044
25149
|
* backing store. Refreshed in `initialize` and `onConfigChange`.
|
|
@@ -25107,6 +25212,17 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25107
25212
|
loadShedState = /* @__PURE__ */ new Map();
|
|
25108
25213
|
/** Timer for the auto-resume sweep. */
|
|
25109
25214
|
loadShedResumeTimer = null;
|
|
25215
|
+
/** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
|
|
25216
|
+
shedInFlight = /* @__PURE__ */ new Set();
|
|
25217
|
+
/**
|
|
25218
|
+
* Per-device backoff for retrying {@link handleDeviceRegistered} after a
|
|
25219
|
+
* TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). Timers
|
|
25220
|
+
* are `unref`'d and cleared on shutdown / device removal.
|
|
25221
|
+
*/
|
|
25222
|
+
slotReadRetryTimers = /* @__PURE__ */ new Map();
|
|
25223
|
+
slotReadRetryAttempts = /* @__PURE__ */ new Map();
|
|
25224
|
+
static SLOT_READ_MAX_RETRIES = 6;
|
|
25225
|
+
static SLOT_READ_RETRY_BASE_MS = 500;
|
|
25110
25226
|
/** Pending `scheduleReconcile` debounce timer. */
|
|
25111
25227
|
reconcileTimer = null;
|
|
25112
25228
|
/** True while `reconcileDispatch` is awaiting the RPC round-trip. */
|
|
@@ -25131,6 +25247,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25131
25247
|
const stored = await this.resolveGlobalStore();
|
|
25132
25248
|
this.globalSettings = { ...stored };
|
|
25133
25249
|
this.applyRuntimeSettings(this.globalSettings);
|
|
25250
|
+
await this.refreshNodeCapabilities();
|
|
25134
25251
|
} catch (err) {
|
|
25135
25252
|
const msg = errMsg(err);
|
|
25136
25253
|
this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
|
|
@@ -25546,6 +25663,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25546
25663
|
knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
|
|
25547
25664
|
cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
|
|
25548
25665
|
enabledNodes: [...this.enabledNodes],
|
|
25666
|
+
enabledDecoderNodes: [...this.enabledDecoderNodes],
|
|
25667
|
+
enabledAudioNodes: [...this.enabledAudioNodes],
|
|
25668
|
+
clusterRoles: { ...this.clusterRoles },
|
|
25549
25669
|
assignedDeviceCount: this.assignments.size,
|
|
25550
25670
|
cameraConfigCount: this.cameraConfigs.size,
|
|
25551
25671
|
activeDetectionCount: this.activeDetections.size
|
|
@@ -25570,6 +25690,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25570
25690
|
clearTimeout(this.pendingRetryDebounceTimer);
|
|
25571
25691
|
this.pendingRetryDebounceTimer = null;
|
|
25572
25692
|
}
|
|
25693
|
+
for (const t of this.slotReadRetryTimers.values()) clearTimeout(t);
|
|
25694
|
+
this.slotReadRetryTimers.clear();
|
|
25695
|
+
this.slotReadRetryAttempts.clear();
|
|
25573
25696
|
this.unsubDeviceRegistered?.();
|
|
25574
25697
|
this.unsubDeviceRegistered = null;
|
|
25575
25698
|
this.unsubDeviceUnregistered?.();
|
|
@@ -25670,6 +25793,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25670
25793
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
25671
25794
|
preferredAgent,
|
|
25672
25795
|
nodeCaps: await this.buildNodeCaps(),
|
|
25796
|
+
weights: await this.buildNodeWeights(),
|
|
25673
25797
|
eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
|
|
25674
25798
|
});
|
|
25675
25799
|
if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
|
|
@@ -25786,6 +25910,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25786
25910
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
25787
25911
|
preferredAgent: null,
|
|
25788
25912
|
nodeCaps: await this.buildNodeCaps(),
|
|
25913
|
+
weights: await this.buildNodeWeights(),
|
|
25789
25914
|
eligibleNodes: this.detectionEligibleNodes(input.deviceId)
|
|
25790
25915
|
});
|
|
25791
25916
|
if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
|
|
@@ -25834,6 +25959,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25834
25959
|
if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
|
|
25835
25960
|
const loads = await this.collectAgentLoad({ onlyEnabled: true });
|
|
25836
25961
|
const nodeCaps = await this.buildNodeCaps();
|
|
25962
|
+
const nodeWeights = await this.buildNodeWeights();
|
|
25837
25963
|
const attachedDelta = /* @__PURE__ */ new Map();
|
|
25838
25964
|
const bumpAttached = (nodeId, by) => {
|
|
25839
25965
|
attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
|
|
@@ -25855,6 +25981,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25855
25981
|
nodes: talliedLoads(),
|
|
25856
25982
|
preferredAgent,
|
|
25857
25983
|
nodeCaps,
|
|
25984
|
+
weights: nodeWeights,
|
|
25858
25985
|
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
25859
25986
|
});
|
|
25860
25987
|
if (!decision) continue;
|
|
@@ -26310,6 +26437,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26310
26437
|
for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
|
|
26311
26438
|
return caps;
|
|
26312
26439
|
}
|
|
26440
|
+
/**
|
|
26441
|
+
* Build a per-node detection WEIGHT map from the persisted agent settings —
|
|
26442
|
+
* the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
|
|
26443
|
+
* are simply absent (the balancer treats absent as weight 1). Passed to every
|
|
26444
|
+
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
26445
|
+
*/
|
|
26446
|
+
async buildNodeWeights() {
|
|
26447
|
+
const blob = await this.agentSettingsState.get();
|
|
26448
|
+
const weights = {};
|
|
26449
|
+
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
26450
|
+
return weights;
|
|
26451
|
+
}
|
|
26313
26452
|
async readAudioNodePin(deviceId) {
|
|
26314
26453
|
if (!this.ctx?.settings) return null;
|
|
26315
26454
|
try {
|
|
@@ -26458,6 +26597,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26458
26597
|
nodes: loads,
|
|
26459
26598
|
preferredAgent: null,
|
|
26460
26599
|
nodeCaps: await this.buildNodeCaps(),
|
|
26600
|
+
weights: await this.buildNodeWeights(),
|
|
26461
26601
|
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
26462
26602
|
});
|
|
26463
26603
|
if (!decision) {
|
|
@@ -26842,6 +26982,47 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26842
26982
|
}
|
|
26843
26983
|
}
|
|
26844
26984
|
/**
|
|
26985
|
+
* Bounded per-device retry of {@link handleDeviceRegistered} after a
|
|
26986
|
+
* TRANSIENT profile-slot read failure ({@link TRANSIENT_SLOT_READ}). The
|
|
26987
|
+
* read (`listAllProfileSlots`) can time out during a broker (re)start; we
|
|
26988
|
+
* must neither stop a running camera nor leave a not-yet-started camera
|
|
26989
|
+
* stranded — so re-run the registration handler with exponential backoff
|
|
26990
|
+
* until the read succeeds or the budget is exhausted. Reset on any
|
|
26991
|
+
* successful read ({@link clearSlotReadRetry}).
|
|
26992
|
+
*/
|
|
26993
|
+
scheduleSlotReadRetry(deviceId) {
|
|
26994
|
+
const attempts = this.slotReadRetryAttempts.get(deviceId) ?? 0;
|
|
26995
|
+
if (attempts >= PipelineOrchestratorAddon.SLOT_READ_MAX_RETRIES) {
|
|
26996
|
+
this.ctx.logger.warn("slot-read retry budget exhausted — leaving detection as-is", { tags: { deviceId } });
|
|
26997
|
+
this.slotReadRetryAttempts.delete(deviceId);
|
|
26998
|
+
return;
|
|
26999
|
+
}
|
|
27000
|
+
const existing = this.slotReadRetryTimers.get(deviceId);
|
|
27001
|
+
if (existing) clearTimeout(existing);
|
|
27002
|
+
const delay = PipelineOrchestratorAddon.SLOT_READ_RETRY_BASE_MS * 2 ** attempts;
|
|
27003
|
+
this.slotReadRetryAttempts.set(deviceId, attempts + 1);
|
|
27004
|
+
const timer = setTimeout(() => {
|
|
27005
|
+
this.slotReadRetryTimers.delete(deviceId);
|
|
27006
|
+
this.handleDeviceRegistered(deviceId).catch((err) => {
|
|
27007
|
+
this.ctx.logger.debug("slot-read retry: handleDeviceRegistered failed", {
|
|
27008
|
+
tags: { deviceId },
|
|
27009
|
+
meta: { error: errMsg(err) }
|
|
27010
|
+
});
|
|
27011
|
+
});
|
|
27012
|
+
}, delay);
|
|
27013
|
+
if (typeof timer === "object" && timer !== null && "unref" in timer) timer.unref();
|
|
27014
|
+
this.slotReadRetryTimers.set(deviceId, timer);
|
|
27015
|
+
}
|
|
27016
|
+
/** Cancel any pending slot-read retry + reset the budget for a device. */
|
|
27017
|
+
clearSlotReadRetry(deviceId) {
|
|
27018
|
+
const existing = this.slotReadRetryTimers.get(deviceId);
|
|
27019
|
+
if (existing) {
|
|
27020
|
+
clearTimeout(existing);
|
|
27021
|
+
this.slotReadRetryTimers.delete(deviceId);
|
|
27022
|
+
}
|
|
27023
|
+
this.slotReadRetryAttempts.delete(deviceId);
|
|
27024
|
+
}
|
|
27025
|
+
/**
|
|
26845
27026
|
* Coalesce bursts of capacity/eligibility/readiness signals into a single
|
|
26846
27027
|
* `retryPendingDispatches` pass. Mirrors `scheduleReconcile`, but on a
|
|
26847
27028
|
* dedicated (longer) debounce so a raise-cap / node-connect flurry doesn't
|
|
@@ -27185,6 +27366,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27185
27366
|
};
|
|
27186
27367
|
const { [input.agentNodeId]: _drop, ...rest } = all;
|
|
27187
27368
|
await this.agentSettingsState.set(rest);
|
|
27369
|
+
await this.refreshNodeCapabilities();
|
|
27188
27370
|
this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
|
|
27189
27371
|
return {
|
|
27190
27372
|
success: true,
|
|
@@ -27208,6 +27390,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27208
27390
|
this.schedulePendingRetry();
|
|
27209
27391
|
return { success: true };
|
|
27210
27392
|
}
|
|
27393
|
+
async setAgentDetectWeight(input) {
|
|
27394
|
+
const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
|
|
27395
|
+
const weight = input.detectWeight ?? 1;
|
|
27396
|
+
const next = existing ? {
|
|
27397
|
+
...existing,
|
|
27398
|
+
detectWeight: weight
|
|
27399
|
+
} : {
|
|
27400
|
+
addonDefaults: {},
|
|
27401
|
+
maxCameras: null,
|
|
27402
|
+
detectWeight: weight
|
|
27403
|
+
};
|
|
27404
|
+
await this.writeAgentSettings(input.agentNodeId, next);
|
|
27405
|
+
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
27406
|
+
tags: { nodeId: input.agentNodeId },
|
|
27407
|
+
meta: { detectWeight: weight ?? null }
|
|
27408
|
+
});
|
|
27409
|
+
this.rebalance().catch((err) => {
|
|
27410
|
+
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
27411
|
+
});
|
|
27412
|
+
return { success: true };
|
|
27413
|
+
}
|
|
27414
|
+
/**
|
|
27415
|
+
* Set one node's placement capabilities — the per-node record that replaced
|
|
27416
|
+
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
27417
|
+
* leave unchanged, `null` to reset to the node default (persisted as an
|
|
27418
|
+
* absent flag), `true`/`false` to force. After persisting, re-derive the
|
|
27419
|
+
* enabled-node sets and reconcile dispatch so the change takes effect
|
|
27420
|
+
* immediately (reuses the existing balancer / retry, no custom logic).
|
|
27421
|
+
*/
|
|
27422
|
+
async setAgentCapabilities(input) {
|
|
27423
|
+
const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
|
|
27424
|
+
addonDefaults: {},
|
|
27425
|
+
maxCameras: null
|
|
27426
|
+
};
|
|
27427
|
+
const apply = (key) => {
|
|
27428
|
+
const patch = input[key];
|
|
27429
|
+
if (patch === void 0) return existing[key];
|
|
27430
|
+
if (patch === null) return void 0;
|
|
27431
|
+
return patch;
|
|
27432
|
+
};
|
|
27433
|
+
const next = {
|
|
27434
|
+
...existing,
|
|
27435
|
+
detect: apply("detect"),
|
|
27436
|
+
decode: apply("decode"),
|
|
27437
|
+
audio: apply("audio"),
|
|
27438
|
+
ingest: apply("ingest")
|
|
27439
|
+
};
|
|
27440
|
+
await this.writeAgentSettings(input.agentNodeId, next);
|
|
27441
|
+
this.ctx.logger.info("agentSettings.capabilities updated", {
|
|
27442
|
+
tags: { nodeId: input.agentNodeId },
|
|
27443
|
+
meta: {
|
|
27444
|
+
detect: next.detect,
|
|
27445
|
+
decode: next.decode,
|
|
27446
|
+
audio: next.audio,
|
|
27447
|
+
ingest: next.ingest
|
|
27448
|
+
}
|
|
27449
|
+
});
|
|
27450
|
+
await this.refreshNodeCapabilities();
|
|
27451
|
+
this.knownRunnerNodes.add(input.agentNodeId);
|
|
27452
|
+
this.schedulePendingRetry();
|
|
27453
|
+
return { success: true };
|
|
27454
|
+
}
|
|
27211
27455
|
async getCameraSettings(input) {
|
|
27212
27456
|
return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|
|
27213
27457
|
}
|
|
@@ -27573,7 +27817,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27573
27817
|
const all = await this.readAgentSettingsMap();
|
|
27574
27818
|
const existing = all[nodeId];
|
|
27575
27819
|
all[nodeId] = {
|
|
27576
|
-
|
|
27820
|
+
...existing,
|
|
27821
|
+
...settings,
|
|
27577
27822
|
maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
|
|
27578
27823
|
};
|
|
27579
27824
|
await this.agentSettingsState.set(all);
|
|
@@ -27856,45 +28101,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27856
28101
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
27857
28102
|
return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
|
|
27858
28103
|
}
|
|
27859
|
-
/** Build the addon-level schema (cluster
|
|
28104
|
+
/** Build the addon-level schema (cluster roles + balancer + failover). */
|
|
27860
28105
|
globalSettingsSchema() {
|
|
27861
28106
|
return this.schema({ sections: [
|
|
27862
28107
|
{
|
|
27863
28108
|
id: "cluster",
|
|
27864
|
-
title: "Cluster",
|
|
28109
|
+
title: "Cluster Roles",
|
|
27865
28110
|
tab: "pipeline",
|
|
27866
|
-
description: "
|
|
28111
|
+
description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
|
|
27867
28112
|
fields: [
|
|
27868
28113
|
{
|
|
27869
|
-
key: "
|
|
27870
|
-
type: "node-
|
|
27871
|
-
label: "
|
|
27872
|
-
description: "
|
|
27873
|
-
default:
|
|
28114
|
+
key: "ingestNode",
|
|
28115
|
+
type: "node-select",
|
|
28116
|
+
label: "Ingest Node",
|
|
28117
|
+
description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest ≠ hub is a later phase; kept here so the role is modeled.)",
|
|
28118
|
+
default: "hub",
|
|
27874
28119
|
showOffline: true
|
|
27875
28120
|
},
|
|
27876
28121
|
{
|
|
27877
|
-
key: "
|
|
27878
|
-
type: "node-
|
|
27879
|
-
label: "
|
|
27880
|
-
description: "
|
|
27881
|
-
default:
|
|
28122
|
+
key: "recordingNode",
|
|
28123
|
+
type: "node-select",
|
|
28124
|
+
label: "Recording Node",
|
|
28125
|
+
description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
|
|
28126
|
+
default: "hub",
|
|
27882
28127
|
showOffline: true
|
|
27883
28128
|
},
|
|
27884
28129
|
{
|
|
27885
|
-
key: "
|
|
27886
|
-
type: "node-
|
|
27887
|
-
label: "
|
|
27888
|
-
description: "
|
|
27889
|
-
default:
|
|
27890
|
-
showOffline: true
|
|
27891
|
-
},
|
|
27892
|
-
{
|
|
27893
|
-
key: "enabledAudioNodes",
|
|
27894
|
-
type: "node-multiselect",
|
|
27895
|
-
label: "Enabled Audio Nodes",
|
|
27896
|
-
description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
|
|
27897
|
-
default: ["hub"],
|
|
28130
|
+
key: "audioNode",
|
|
28131
|
+
type: "node-select",
|
|
28132
|
+
label: "Audio Node",
|
|
28133
|
+
description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
|
|
28134
|
+
default: "hub",
|
|
27898
28135
|
showOffline: true
|
|
27899
28136
|
}
|
|
27900
28137
|
]
|
|
@@ -28397,16 +28634,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28397
28634
|
pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
|
|
28398
28635
|
onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
|
|
28399
28636
|
};
|
|
28400
|
-
const
|
|
28401
|
-
|
|
28402
|
-
|
|
28403
|
-
|
|
28404
|
-
|
|
28405
|
-
|
|
28406
|
-
|
|
28407
|
-
|
|
28637
|
+
const readRole = (key) => {
|
|
28638
|
+
const raw = config[key];
|
|
28639
|
+
return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
|
|
28640
|
+
};
|
|
28641
|
+
this.clusterRoles = {
|
|
28642
|
+
ingestNode: readRole("ingestNode"),
|
|
28643
|
+
recordingNode: readRole("recordingNode"),
|
|
28644
|
+
audioNode: readRole("audioNode")
|
|
28645
|
+
};
|
|
28408
28646
|
this.schedulePendingRetry();
|
|
28409
28647
|
}
|
|
28648
|
+
/**
|
|
28649
|
+
* Default placement capability for a node when its stored flag is unset:
|
|
28650
|
+
* the hub is capable of every role out of the box; every other node must be
|
|
28651
|
+
* explicitly enabled. This reproduces the classic `['hub']` defaults of the
|
|
28652
|
+
* four removed flat lists against an empty capability store.
|
|
28653
|
+
*/
|
|
28654
|
+
static nodeCapabilityDefault(nodeId) {
|
|
28655
|
+
return nodeId === "hub";
|
|
28656
|
+
}
|
|
28657
|
+
/**
|
|
28658
|
+
* Recompute the derived enabled-node sets (`enabledNodes` /
|
|
28659
|
+
* `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
|
|
28660
|
+
* store. A node is eligible for a concern iff its stored flag is `true`, or
|
|
28661
|
+
* the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
|
|
28662
|
+
* Forked child nodeIds (`hub/classifier`) are never dispatchable and are
|
|
28663
|
+
* excluded. `hub` is always considered even when absent from the store, so a
|
|
28664
|
+
* fresh install keeps the hub-only cluster working.
|
|
28665
|
+
*/
|
|
28666
|
+
async refreshNodeCapabilities() {
|
|
28667
|
+
let blob = {};
|
|
28668
|
+
try {
|
|
28669
|
+
blob = await this.agentSettingsState.get();
|
|
28670
|
+
} catch (err) {
|
|
28671
|
+
this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
|
|
28672
|
+
}
|
|
28673
|
+
const nodeIds = new Set(["hub"]);
|
|
28674
|
+
for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
|
|
28675
|
+
const detect = [];
|
|
28676
|
+
const decode = [];
|
|
28677
|
+
const audio = [];
|
|
28678
|
+
const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
|
|
28679
|
+
for (const nodeId of nodeIds) {
|
|
28680
|
+
const settings = blob[nodeId];
|
|
28681
|
+
if (capable(settings?.detect, nodeId)) detect.push(nodeId);
|
|
28682
|
+
if (capable(settings?.decode, nodeId)) decode.push(nodeId);
|
|
28683
|
+
if (capable(settings?.audio, nodeId)) audio.push(nodeId);
|
|
28684
|
+
}
|
|
28685
|
+
this.enabledNodes = detect.toSorted();
|
|
28686
|
+
this.enabledDecoderNodes = decode.toSorted();
|
|
28687
|
+
this.enabledAudioNodes = audio.toSorted();
|
|
28688
|
+
}
|
|
28410
28689
|
get api() {
|
|
28411
28690
|
return this.ctx.api ?? null;
|
|
28412
28691
|
}
|
|
@@ -28620,7 +28899,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28620
28899
|
*/
|
|
28621
28900
|
async fetchAssignedProfiles(deviceId) {
|
|
28622
28901
|
const api = this.api;
|
|
28623
|
-
if (!api) return
|
|
28902
|
+
if (!api) return null;
|
|
28624
28903
|
try {
|
|
28625
28904
|
const slots = await api.streamBroker.listAllProfileSlots.query();
|
|
28626
28905
|
const out = /* @__PURE__ */ new Map();
|
|
@@ -28630,8 +28909,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28630
28909
|
out.set(slot.profile, slot.sourceCamStreamId);
|
|
28631
28910
|
}
|
|
28632
28911
|
return out;
|
|
28633
|
-
} catch {
|
|
28634
|
-
|
|
28912
|
+
} catch (err) {
|
|
28913
|
+
this.ctx.logger.debug("fetchAssignedProfiles: slot read failed (transient) — treating as unknown", {
|
|
28914
|
+
tags: { deviceId },
|
|
28915
|
+
meta: { error: errMsg(err) }
|
|
28916
|
+
});
|
|
28917
|
+
return null;
|
|
28635
28918
|
}
|
|
28636
28919
|
}
|
|
28637
28920
|
/**
|
|
@@ -28650,6 +28933,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28650
28933
|
*/
|
|
28651
28934
|
async buildDetectionConfig(deviceId) {
|
|
28652
28935
|
const assigned = await this.fetchAssignedProfiles(deviceId);
|
|
28936
|
+
if (assigned === null) return TRANSIENT_SLOT_READ;
|
|
28653
28937
|
if (assigned.size === 0) return null;
|
|
28654
28938
|
const resolved = await this.resolveDeviceDetectionSettings(deviceId);
|
|
28655
28939
|
if (!resolved) return null;
|
|
@@ -28793,6 +29077,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28793
29077
|
}
|
|
28794
29078
|
/** Stop detection and purge all persisted config for a removed device. */
|
|
28795
29079
|
async handleDeviceUnregistered(deviceId) {
|
|
29080
|
+
this.clearSlotReadRetry(deviceId);
|
|
28796
29081
|
await this.stopDetection(deviceId);
|
|
28797
29082
|
this.zonesProvider?.forgetDevice(deviceId);
|
|
28798
29083
|
this.zoneRulesProvider?.forgetDevice(deviceId);
|
|
@@ -28814,6 +29099,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28814
29099
|
const log = this.ctx.logger.withTags({ deviceId });
|
|
28815
29100
|
log.info("handleDeviceRegistered", { meta: { phase: "handleDeviceRegistered" } });
|
|
28816
29101
|
const config = await this.buildDetectionConfig(deviceId);
|
|
29102
|
+
if (config === TRANSIENT_SLOT_READ) {
|
|
29103
|
+
log.warn("Profile-slot read failed transiently — keeping detection, scheduling retry");
|
|
29104
|
+
this.scheduleSlotReadRetry(deviceId);
|
|
29105
|
+
return;
|
|
29106
|
+
}
|
|
29107
|
+
this.clearSlotReadRetry(deviceId);
|
|
28817
29108
|
log.info("[pipeline-orchestrator] buildDetectionConfig", config ? { meta: {
|
|
28818
29109
|
enabled: config.enabled,
|
|
28819
29110
|
motionStreamId: config.motionStreamId,
|
|
@@ -28879,6 +29170,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28879
29170
|
const log = this.ctx.logger.withTags({ deviceId });
|
|
28880
29171
|
if (this.activeDetections.has(deviceId)) await this.stopDetection(deviceId);
|
|
28881
29172
|
const config = await this.buildDetectionConfig(deviceId);
|
|
29173
|
+
if (config === TRANSIENT_SLOT_READ) {
|
|
29174
|
+
log.warn("Settings changed — slot read failed transiently, scheduling retry");
|
|
29175
|
+
this.scheduleSlotReadRetry(deviceId);
|
|
29176
|
+
return;
|
|
29177
|
+
}
|
|
29178
|
+
this.clearSlotReadRetry(deviceId);
|
|
28882
29179
|
if (!config) {
|
|
28883
29180
|
log.info("Settings changed — no assigned slot, detection stopped");
|
|
28884
29181
|
return;
|
|
@@ -29003,7 +29300,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29003
29300
|
});
|
|
29004
29301
|
return;
|
|
29005
29302
|
}
|
|
29006
|
-
this.
|
|
29303
|
+
if (this.shedInFlight.has(deviceId)) return;
|
|
29304
|
+
this.shedInFlight.add(deviceId);
|
|
29305
|
+
this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
|
|
29007
29306
|
tags: { deviceId },
|
|
29008
29307
|
meta: {
|
|
29009
29308
|
actualFps: metrics.actualFps,
|
|
@@ -29012,7 +29311,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29012
29311
|
windowMs
|
|
29013
29312
|
}
|
|
29014
29313
|
});
|
|
29015
|
-
state.
|
|
29314
|
+
this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
|
|
29315
|
+
}
|
|
29316
|
+
/**
|
|
29317
|
+
* Cluster-wide load-shed action: relocate the sustained-slow camera to a
|
|
29318
|
+
* less-loaded node if the balancer (ceiling + weight aware) finds one,
|
|
29319
|
+
* otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
|
|
29320
|
+
* placement logic.
|
|
29321
|
+
*/
|
|
29322
|
+
async shedOrRelocate(deviceId, state) {
|
|
29323
|
+
if (state.pausedAt !== null) return;
|
|
29324
|
+
if (await this.tryClusterRelocate(deviceId).catch((err) => {
|
|
29325
|
+
this.ctx.logger.warn("Load management: cluster relocate failed", {
|
|
29326
|
+
tags: { deviceId },
|
|
29327
|
+
meta: { error: errMsg(err) }
|
|
29328
|
+
});
|
|
29329
|
+
return false;
|
|
29330
|
+
})) {
|
|
29331
|
+
state.lowSinceTs = null;
|
|
29332
|
+
return;
|
|
29333
|
+
}
|
|
29334
|
+
this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
|
|
29335
|
+
state.pausedAt = Date.now();
|
|
29016
29336
|
state.lowSinceTs = null;
|
|
29017
29337
|
this.pipelineWatchdog?.unregister(deviceId);
|
|
29018
29338
|
this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
|
|
@@ -29023,6 +29343,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29023
29343
|
});
|
|
29024
29344
|
this.ensureLoadShedResumeTimer();
|
|
29025
29345
|
}
|
|
29346
|
+
/**
|
|
29347
|
+
* Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
|
|
29348
|
+
* Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
|
|
29349
|
+
* frame-source eligibility); the camera's current node counts it in its own
|
|
29350
|
+
* load, so a genuinely less-loaded node with spare capacity wins. Returns
|
|
29351
|
+
* `true` iff the camera was moved to a DIFFERENT node. A manually pinned
|
|
29352
|
+
* camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
|
|
29353
|
+
*/
|
|
29354
|
+
async tryClusterRelocate(deviceId) {
|
|
29355
|
+
const current = this.assignments.get(deviceId);
|
|
29356
|
+
const cached = this.cameraConfigs.get(deviceId);
|
|
29357
|
+
if (!current || !cached) return false;
|
|
29358
|
+
const currentNode = current.agentNodeId;
|
|
29359
|
+
if (await this.readPipelinePin(deviceId)) return false;
|
|
29360
|
+
const decision = balance({
|
|
29361
|
+
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
29362
|
+
preferredAgent: null,
|
|
29363
|
+
nodeCaps: await this.buildNodeCaps(),
|
|
29364
|
+
weights: await this.buildNodeWeights(),
|
|
29365
|
+
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
29366
|
+
});
|
|
29367
|
+
if (!decision || decision.kind !== "assigned") return false;
|
|
29368
|
+
const target = decision.agentNodeId;
|
|
29369
|
+
if (target === currentNode) return false;
|
|
29370
|
+
await this.detachOn(currentNode, deviceId).catch((err) => {
|
|
29371
|
+
this.ctx.logger.debug("Load management: relocate detach-old failed", {
|
|
29372
|
+
tags: { deviceId },
|
|
29373
|
+
meta: {
|
|
29374
|
+
from: currentNode,
|
|
29375
|
+
error: errMsg(err)
|
|
29376
|
+
}
|
|
29377
|
+
});
|
|
29378
|
+
});
|
|
29379
|
+
await this.attachOn(target, cached);
|
|
29380
|
+
this.recordAssignment(deviceId, target, "rebalance", false);
|
|
29381
|
+
this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
|
|
29382
|
+
tags: { deviceId },
|
|
29383
|
+
meta: {
|
|
29384
|
+
from: currentNode,
|
|
29385
|
+
to: target,
|
|
29386
|
+
score: decision.score
|
|
29387
|
+
}
|
|
29388
|
+
});
|
|
29389
|
+
return true;
|
|
29390
|
+
}
|
|
29026
29391
|
/** Base cooldown before first auto-resume attempt. */
|
|
29027
29392
|
static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
|
|
29028
29393
|
/** Maximum backoff cap. */
|