@camstack/addon-pipeline-orchestrator 1.1.21 → 1.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +1 -1
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DVzgRExi.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CQmCy3VK.mjs} +3 -3
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DdDNCPJ5.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-BA0gpKU7.mjs} +1 -1
- package/dist/{hostInit-CEqZfaom.mjs → hostInit-ibn4yaBr.mjs} +3 -3
- package/dist/index.js +340 -55
- package/dist/index.mjs +340 -55
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -15703,7 +15703,17 @@ var AgentAddonConfigSchema = object({
|
|
|
15703
15703
|
});
|
|
15704
15704
|
var AgentPipelineSettingsSchema = object({
|
|
15705
15705
|
addonDefaults: record(string(), AgentAddonConfigSchema).readonly(),
|
|
15706
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
15706
|
+
maxCameras: number().int().nonnegative().nullable().default(null),
|
|
15707
|
+
/** Per-node detection weight (relative share for the quota balancer). */
|
|
15708
|
+
detectWeight: number().positive().optional(),
|
|
15709
|
+
/** Node is eligible to run the detection pipeline (decode + inference). */
|
|
15710
|
+
detect: boolean().optional(),
|
|
15711
|
+
/** Node is eligible to host decoder sessions. */
|
|
15712
|
+
decode: boolean().optional(),
|
|
15713
|
+
/** Node is eligible to run audio-analyzer sessions. */
|
|
15714
|
+
audio: boolean().optional(),
|
|
15715
|
+
/** Node is eligible to be the ingest / source-owner (serve the restream). */
|
|
15716
|
+
ingest: boolean().optional()
|
|
15707
15717
|
});
|
|
15708
15718
|
var CameraPipelineForAgentSchema = object({
|
|
15709
15719
|
steps: array(PipelineStepInputSchema).readonly(),
|
|
@@ -16137,6 +16147,38 @@ var pipelineOrchestratorCapability = {
|
|
|
16137
16147
|
kind: "mutation",
|
|
16138
16148
|
auth: "admin"
|
|
16139
16149
|
}),
|
|
16150
|
+
/**
|
|
16151
|
+
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
16152
|
+
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
16153
|
+
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
16154
|
+
*/
|
|
16155
|
+
setAgentDetectWeight: method(object({
|
|
16156
|
+
agentNodeId: string(),
|
|
16157
|
+
detectWeight: number().positive().nullable()
|
|
16158
|
+
}), object({ success: literal(true) }), {
|
|
16159
|
+
kind: "mutation",
|
|
16160
|
+
auth: "admin"
|
|
16161
|
+
}),
|
|
16162
|
+
/**
|
|
16163
|
+
* Set one node's placement CAPABILITIES — the per-node record that
|
|
16164
|
+
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
16165
|
+
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
16166
|
+
* Each flag is optional in the patch: omit a flag to leave it unchanged,
|
|
16167
|
+
* pass `null` to reset it to the node default (`hub` → capable, every
|
|
16168
|
+
* other node → not). Detection/decode/audio eligibility is derived from
|
|
16169
|
+
* these flags across the cluster; changing them re-derives the enabled
|
|
16170
|
+
* sets and reconciles dispatch immediately.
|
|
16171
|
+
*/
|
|
16172
|
+
setAgentCapabilities: method(object({
|
|
16173
|
+
agentNodeId: string(),
|
|
16174
|
+
detect: boolean().nullable().optional(),
|
|
16175
|
+
decode: boolean().nullable().optional(),
|
|
16176
|
+
audio: boolean().nullable().optional(),
|
|
16177
|
+
ingest: boolean().nullable().optional()
|
|
16178
|
+
}), object({ success: literal(true) }), {
|
|
16179
|
+
kind: "mutation",
|
|
16180
|
+
auth: "admin"
|
|
16181
|
+
}),
|
|
16140
16182
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
16141
16183
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
16142
16184
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -22043,6 +22085,18 @@ Object.freeze({
|
|
|
22043
22085
|
addonId: null,
|
|
22044
22086
|
access: "create"
|
|
22045
22087
|
},
|
|
22088
|
+
"pipelineOrchestrator.setAgentCapabilities": {
|
|
22089
|
+
capName: "pipeline-orchestrator",
|
|
22090
|
+
capScope: "system",
|
|
22091
|
+
addonId: null,
|
|
22092
|
+
access: "create"
|
|
22093
|
+
},
|
|
22094
|
+
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
22095
|
+
capName: "pipeline-orchestrator",
|
|
22096
|
+
capScope: "system",
|
|
22097
|
+
addonId: null,
|
|
22098
|
+
access: "create"
|
|
22099
|
+
},
|
|
22046
22100
|
"pipelineOrchestrator.setAgentMaxCameras": {
|
|
22047
22101
|
capName: "pipeline-orchestrator",
|
|
22048
22102
|
capScope: "system",
|
|
@@ -24186,6 +24240,23 @@ function computeCapacityScore(load) {
|
|
|
24186
24240
|
return load.attachedCameras * Math.max(load.avgInferenceFps, 0) + Math.max(load.queueDepthTotal, 0);
|
|
24187
24241
|
}
|
|
24188
24242
|
/**
|
|
24243
|
+
* Node detection weight, defaulting to 1 when absent/invalid. A higher weight
|
|
24244
|
+
* makes a node preferred proportionally more (see {@link BalancerInput.weights}).
|
|
24245
|
+
*/
|
|
24246
|
+
function nodeWeight(nodeId, weights) {
|
|
24247
|
+
const w = weights?.[nodeId];
|
|
24248
|
+
return w !== void 0 && w > 0 ? w : 1;
|
|
24249
|
+
}
|
|
24250
|
+
/**
|
|
24251
|
+
* Weight-adjusted capacity score — the value the balancer minimises. Dividing
|
|
24252
|
+
* the raw capacity score by the node's weight makes a heavier node accumulate
|
|
24253
|
+
* proportionally more load before it loses preference, yielding weight-
|
|
24254
|
+
* proportional distribution while still preferring drained/idle nodes.
|
|
24255
|
+
*/
|
|
24256
|
+
function weightedScore(load, weights) {
|
|
24257
|
+
return computeCapacityScore(load) / nodeWeight(load.nodeId, weights);
|
|
24258
|
+
}
|
|
24259
|
+
/**
|
|
24189
24260
|
* Returns true when the node has remaining capacity.
|
|
24190
24261
|
* A node is eligible iff `cap` is unlimited (null/absent/<=0) OR
|
|
24191
24262
|
* its `attachedCameras` count is strictly less than the cap.
|
|
@@ -24251,8 +24322,9 @@ function balance(input) {
|
|
|
24251
24322
|
};
|
|
24252
24323
|
const best = eligible.map((node) => ({
|
|
24253
24324
|
node,
|
|
24254
|
-
score: computeCapacityScore(node)
|
|
24255
|
-
|
|
24325
|
+
score: computeCapacityScore(node),
|
|
24326
|
+
sortKey: weightedScore(node, input.weights)
|
|
24327
|
+
})).toSorted((a, b) => a.sortKey - b.sortKey || a.node.nodeId.localeCompare(b.node.nodeId))[0];
|
|
24256
24328
|
return {
|
|
24257
24329
|
kind: "assigned",
|
|
24258
24330
|
agentNodeId: best.node.nodeId,
|
|
@@ -24354,7 +24426,12 @@ var StoredAgentAddonConfigSchema = object({
|
|
|
24354
24426
|
});
|
|
24355
24427
|
var StoredAgentPipelineSettingsSchema = object({
|
|
24356
24428
|
addonDefaults: record(string(), StoredAgentAddonConfigSchema).readonly(),
|
|
24357
|
-
maxCameras: number().int().nonnegative().nullable().default(null)
|
|
24429
|
+
maxCameras: number().int().nonnegative().nullable().default(null),
|
|
24430
|
+
detectWeight: number().positive().optional(),
|
|
24431
|
+
detect: boolean().optional(),
|
|
24432
|
+
decode: boolean().optional(),
|
|
24433
|
+
audio: boolean().optional(),
|
|
24434
|
+
ingest: boolean().optional()
|
|
24358
24435
|
});
|
|
24359
24436
|
var AgentSettingsMapSchema = record(string(), StoredAgentPipelineSettingsSchema);
|
|
24360
24437
|
var StoredCameraStepOverridePatchSchema = object({
|
|
@@ -24893,6 +24970,13 @@ var OrchestratorDiagnosticsSchema = object({
|
|
|
24893
24970
|
knownRunnerNodes: array(string()),
|
|
24894
24971
|
cachedAgentLoadNodeIds: array(string()),
|
|
24895
24972
|
enabledNodes: array(string()),
|
|
24973
|
+
enabledDecoderNodes: array(string()),
|
|
24974
|
+
enabledAudioNodes: array(string()),
|
|
24975
|
+
clusterRoles: object({
|
|
24976
|
+
ingestNode: string(),
|
|
24977
|
+
recordingNode: string(),
|
|
24978
|
+
audioNode: string()
|
|
24979
|
+
}),
|
|
24896
24980
|
assignedDeviceCount: number().int().min(0),
|
|
24897
24981
|
cameraConfigCount: number().int().min(0),
|
|
24898
24982
|
activeDetectionCount: number().int().min(0)
|
|
@@ -25009,36 +25093,48 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25009
25093
|
failoverPolicy = { ...DEFAULT_FAILOVER_POLICY };
|
|
25010
25094
|
/**
|
|
25011
25095
|
* Hub-wide allow-list of node ids eligible to run the detection
|
|
25012
|
-
* pipeline.
|
|
25013
|
-
*
|
|
25014
|
-
*
|
|
25015
|
-
* The list is a strict whitelist —
|
|
25016
|
-
* for other capabilities (metrics, logs,
|
|
25017
|
-
* never receive camera assignments.
|
|
25096
|
+
* pipeline. DERIVED (placement-model redesign) from the per-node
|
|
25097
|
+
* capability store (`agentSettings[node].detect`) — no longer a flat
|
|
25098
|
+
* `enabledNodes` multiselect. Recomputed by {@link refreshNodeCapabilities}
|
|
25099
|
+
* on boot and on every capability write. The list is a strict whitelist —
|
|
25100
|
+
* disabled nodes stay connected for other capabilities (metrics, logs,
|
|
25101
|
+
* cluster-wide events) but never receive camera assignments.
|
|
25018
25102
|
*/
|
|
25019
25103
|
enabledNodes = ["hub"];
|
|
25020
25104
|
/**
|
|
25021
25105
|
* Hub-wide allow-list of node ids eligible to run decoder sessions.
|
|
25022
|
-
*
|
|
25106
|
+
* DERIVED from the per-node capability store (`agentSettings[node].decode`).
|
|
25023
25107
|
* Defaults to `['hub']` — the hub always has a decoder available.
|
|
25024
25108
|
*/
|
|
25025
25109
|
enabledDecoderNodes = ["hub"];
|
|
25026
25110
|
/**
|
|
25027
|
-
*
|
|
25028
|
-
*
|
|
25029
|
-
*
|
|
25030
|
-
*
|
|
25031
|
-
*
|
|
25032
|
-
*
|
|
25111
|
+
* Nodes allowed to run the REMOTE-SOURCE leg — dial a camera's source-owner
|
|
25112
|
+
* restream over the LAN and decode it locally. Held for the source-owner
|
|
25113
|
+
* plumbing (`source-owner.ts`) but currently ALWAYS EMPTY: the cross-node
|
|
25114
|
+
* frame transport is a separate activation phase (placement-model §9 P1).
|
|
25115
|
+
* With this empty, `resolveSourceOwner` returns `undefined` and every
|
|
25116
|
+
* `selectRunnerFrameSource` emits `local-broker` — bit-identical to
|
|
25117
|
+
* pre-Phase-2 behavior.
|
|
25033
25118
|
*/
|
|
25034
25119
|
remoteSourcingNodes = [];
|
|
25035
25120
|
/**
|
|
25036
25121
|
* Hub-wide allow-list of node ids eligible to run audio-analyzer sessions.
|
|
25037
|
-
*
|
|
25122
|
+
* DERIVED from the per-node capability store (`agentSettings[node].audio`).
|
|
25038
25123
|
* Defaults to `['hub']`.
|
|
25039
25124
|
*/
|
|
25040
25125
|
enabledAudioNodes = ["hub"];
|
|
25041
25126
|
/**
|
|
25127
|
+
* Cluster-wide singleton role assignments (placement-model §6). Each role
|
|
25128
|
+
* is the ONE node that serves it for every camera. Driven by the
|
|
25129
|
+
* `ingestNode` / `recordingNode` / `audioNode` node-selects in the addon
|
|
25130
|
+
* global schema; defaults to the hub for all three.
|
|
25131
|
+
*/
|
|
25132
|
+
clusterRoles = {
|
|
25133
|
+
ingestNode: "hub",
|
|
25134
|
+
recordingNode: "hub",
|
|
25135
|
+
audioNode: "hub"
|
|
25136
|
+
};
|
|
25137
|
+
/**
|
|
25042
25138
|
* Full global settings snapshot kept in memory so synchronous cap methods
|
|
25043
25139
|
* (e.g. `getGlobalOrchestrationSettings`) can resolve without awaiting the
|
|
25044
25140
|
* backing store. Refreshed in `initialize` and `onConfigChange`.
|
|
@@ -25107,6 +25203,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25107
25203
|
loadShedState = /* @__PURE__ */ new Map();
|
|
25108
25204
|
/** Timer for the auto-resume sweep. */
|
|
25109
25205
|
loadShedResumeTimer = null;
|
|
25206
|
+
/** Per-camera guard so repeated low-fps snapshots don't stack relocate/pause. */
|
|
25207
|
+
shedInFlight = /* @__PURE__ */ new Set();
|
|
25110
25208
|
/** Pending `scheduleReconcile` debounce timer. */
|
|
25111
25209
|
reconcileTimer = null;
|
|
25112
25210
|
/** True while `reconcileDispatch` is awaiting the RPC round-trip. */
|
|
@@ -25131,6 +25229,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25131
25229
|
const stored = await this.resolveGlobalStore();
|
|
25132
25230
|
this.globalSettings = { ...stored };
|
|
25133
25231
|
this.applyRuntimeSettings(this.globalSettings);
|
|
25232
|
+
await this.refreshNodeCapabilities();
|
|
25134
25233
|
} catch (err) {
|
|
25135
25234
|
const msg = errMsg(err);
|
|
25136
25235
|
this.ctx.logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
|
|
@@ -25546,6 +25645,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25546
25645
|
knownRunnerNodes: [...this.knownRunnerNodes].toSorted(),
|
|
25547
25646
|
cachedAgentLoadNodeIds: [...this.cachedAgentLoad.keys()].toSorted(),
|
|
25548
25647
|
enabledNodes: [...this.enabledNodes],
|
|
25648
|
+
enabledDecoderNodes: [...this.enabledDecoderNodes],
|
|
25649
|
+
enabledAudioNodes: [...this.enabledAudioNodes],
|
|
25650
|
+
clusterRoles: { ...this.clusterRoles },
|
|
25549
25651
|
assignedDeviceCount: this.assignments.size,
|
|
25550
25652
|
cameraConfigCount: this.cameraConfigs.size,
|
|
25551
25653
|
activeDetectionCount: this.activeDetections.size
|
|
@@ -25670,6 +25772,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25670
25772
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
25671
25773
|
preferredAgent,
|
|
25672
25774
|
nodeCaps: await this.buildNodeCaps(),
|
|
25775
|
+
weights: await this.buildNodeWeights(),
|
|
25673
25776
|
eligibleNodes: this.detectionEligibleNodes(runnerConfig.deviceId)
|
|
25674
25777
|
});
|
|
25675
25778
|
if (!decision) throw new Error(`dispatchCamera: no runner available for ${runnerConfig.deviceId}`);
|
|
@@ -25786,6 +25889,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25786
25889
|
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
25787
25890
|
preferredAgent: null,
|
|
25788
25891
|
nodeCaps: await this.buildNodeCaps(),
|
|
25892
|
+
weights: await this.buildNodeWeights(),
|
|
25789
25893
|
eligibleNodes: this.detectionEligibleNodes(input.deviceId)
|
|
25790
25894
|
});
|
|
25791
25895
|
if (!decision) this.ctx.logger.warn("unassignPipeline: no runner available, leaving unassigned", { tags: { deviceId: input.deviceId } });
|
|
@@ -25834,6 +25938,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25834
25938
|
if (!this.ctx) throw new Error("PipelineOrchestrator: rebalance called before initialize");
|
|
25835
25939
|
const loads = await this.collectAgentLoad({ onlyEnabled: true });
|
|
25836
25940
|
const nodeCaps = await this.buildNodeCaps();
|
|
25941
|
+
const nodeWeights = await this.buildNodeWeights();
|
|
25837
25942
|
const attachedDelta = /* @__PURE__ */ new Map();
|
|
25838
25943
|
const bumpAttached = (nodeId, by) => {
|
|
25839
25944
|
attachedDelta.set(nodeId, (attachedDelta.get(nodeId) ?? 0) + by);
|
|
@@ -25855,6 +25960,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
25855
25960
|
nodes: talliedLoads(),
|
|
25856
25961
|
preferredAgent,
|
|
25857
25962
|
nodeCaps,
|
|
25963
|
+
weights: nodeWeights,
|
|
25858
25964
|
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
25859
25965
|
});
|
|
25860
25966
|
if (!decision) continue;
|
|
@@ -26310,6 +26416,18 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26310
26416
|
for (const [nodeId, settings] of Object.entries(blob)) caps[nodeId] = settings.maxCameras ?? null;
|
|
26311
26417
|
return caps;
|
|
26312
26418
|
}
|
|
26419
|
+
/**
|
|
26420
|
+
* Build a per-node detection WEIGHT map from the persisted agent settings —
|
|
26421
|
+
* the sibling of {@link buildNodeCaps}. Nodes without a stored `detectWeight`
|
|
26422
|
+
* are simply absent (the balancer treats absent as weight 1). Passed to every
|
|
26423
|
+
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
26424
|
+
*/
|
|
26425
|
+
async buildNodeWeights() {
|
|
26426
|
+
const blob = await this.agentSettingsState.get();
|
|
26427
|
+
const weights = {};
|
|
26428
|
+
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
26429
|
+
return weights;
|
|
26430
|
+
}
|
|
26313
26431
|
async readAudioNodePin(deviceId) {
|
|
26314
26432
|
if (!this.ctx?.settings) return null;
|
|
26315
26433
|
try {
|
|
@@ -26458,6 +26576,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
26458
26576
|
nodes: loads,
|
|
26459
26577
|
preferredAgent: null,
|
|
26460
26578
|
nodeCaps: await this.buildNodeCaps(),
|
|
26579
|
+
weights: await this.buildNodeWeights(),
|
|
26461
26580
|
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
26462
26581
|
});
|
|
26463
26582
|
if (!decision) {
|
|
@@ -27185,6 +27304,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27185
27304
|
};
|
|
27186
27305
|
const { [input.agentNodeId]: _drop, ...rest } = all;
|
|
27187
27306
|
await this.agentSettingsState.set(rest);
|
|
27307
|
+
await this.refreshNodeCapabilities();
|
|
27188
27308
|
this.ctx.logger.info("agentSettings entry removed", { tags: { nodeId: input.agentNodeId } });
|
|
27189
27309
|
return {
|
|
27190
27310
|
success: true,
|
|
@@ -27208,6 +27328,68 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27208
27328
|
this.schedulePendingRetry();
|
|
27209
27329
|
return { success: true };
|
|
27210
27330
|
}
|
|
27331
|
+
async setAgentDetectWeight(input) {
|
|
27332
|
+
const existing = (await this.readAgentSettingsMap())[input.agentNodeId];
|
|
27333
|
+
const weight = input.detectWeight ?? 1;
|
|
27334
|
+
const next = existing ? {
|
|
27335
|
+
...existing,
|
|
27336
|
+
detectWeight: weight
|
|
27337
|
+
} : {
|
|
27338
|
+
addonDefaults: {},
|
|
27339
|
+
maxCameras: null,
|
|
27340
|
+
detectWeight: weight
|
|
27341
|
+
};
|
|
27342
|
+
await this.writeAgentSettings(input.agentNodeId, next);
|
|
27343
|
+
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
27344
|
+
tags: { nodeId: input.agentNodeId },
|
|
27345
|
+
meta: { detectWeight: weight ?? null }
|
|
27346
|
+
});
|
|
27347
|
+
this.rebalance().catch((err) => {
|
|
27348
|
+
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
27349
|
+
});
|
|
27350
|
+
return { success: true };
|
|
27351
|
+
}
|
|
27352
|
+
/**
|
|
27353
|
+
* Set one node's placement capabilities — the per-node record that replaced
|
|
27354
|
+
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
27355
|
+
* leave unchanged, `null` to reset to the node default (persisted as an
|
|
27356
|
+
* absent flag), `true`/`false` to force. After persisting, re-derive the
|
|
27357
|
+
* enabled-node sets and reconcile dispatch so the change takes effect
|
|
27358
|
+
* immediately (reuses the existing balancer / retry, no custom logic).
|
|
27359
|
+
*/
|
|
27360
|
+
async setAgentCapabilities(input) {
|
|
27361
|
+
const existing = (await this.readAgentSettingsMap())[input.agentNodeId] ?? {
|
|
27362
|
+
addonDefaults: {},
|
|
27363
|
+
maxCameras: null
|
|
27364
|
+
};
|
|
27365
|
+
const apply = (key) => {
|
|
27366
|
+
const patch = input[key];
|
|
27367
|
+
if (patch === void 0) return existing[key];
|
|
27368
|
+
if (patch === null) return void 0;
|
|
27369
|
+
return patch;
|
|
27370
|
+
};
|
|
27371
|
+
const next = {
|
|
27372
|
+
...existing,
|
|
27373
|
+
detect: apply("detect"),
|
|
27374
|
+
decode: apply("decode"),
|
|
27375
|
+
audio: apply("audio"),
|
|
27376
|
+
ingest: apply("ingest")
|
|
27377
|
+
};
|
|
27378
|
+
await this.writeAgentSettings(input.agentNodeId, next);
|
|
27379
|
+
this.ctx.logger.info("agentSettings.capabilities updated", {
|
|
27380
|
+
tags: { nodeId: input.agentNodeId },
|
|
27381
|
+
meta: {
|
|
27382
|
+
detect: next.detect,
|
|
27383
|
+
decode: next.decode,
|
|
27384
|
+
audio: next.audio,
|
|
27385
|
+
ingest: next.ingest
|
|
27386
|
+
}
|
|
27387
|
+
});
|
|
27388
|
+
await this.refreshNodeCapabilities();
|
|
27389
|
+
this.knownRunnerNodes.add(input.agentNodeId);
|
|
27390
|
+
this.schedulePendingRetry();
|
|
27391
|
+
return { success: true };
|
|
27392
|
+
}
|
|
27211
27393
|
async getCameraSettings(input) {
|
|
27212
27394
|
return (await this.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|
|
27213
27395
|
}
|
|
@@ -27573,7 +27755,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27573
27755
|
const all = await this.readAgentSettingsMap();
|
|
27574
27756
|
const existing = all[nodeId];
|
|
27575
27757
|
all[nodeId] = {
|
|
27576
|
-
|
|
27758
|
+
...existing,
|
|
27759
|
+
...settings,
|
|
27577
27760
|
maxCameras: settings.maxCameras !== void 0 ? settings.maxCameras ?? null : existing?.maxCameras ?? null
|
|
27578
27761
|
};
|
|
27579
27762
|
await this.agentSettingsState.set(all);
|
|
@@ -27856,45 +28039,37 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
27856
28039
|
const rand = Math.random().toString(36).slice(2, 8);
|
|
27857
28040
|
return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
|
|
27858
28041
|
}
|
|
27859
|
-
/** Build the addon-level schema (cluster
|
|
28042
|
+
/** Build the addon-level schema (cluster roles + balancer + failover). */
|
|
27860
28043
|
globalSettingsSchema() {
|
|
27861
28044
|
return this.schema({ sections: [
|
|
27862
28045
|
{
|
|
27863
28046
|
id: "cluster",
|
|
27864
|
-
title: "Cluster",
|
|
28047
|
+
title: "Cluster Roles",
|
|
27865
28048
|
tab: "pipeline",
|
|
27866
|
-
description: "
|
|
28049
|
+
description: "Cluster-wide singleton roles: the single node that serves each role for every camera. Per-node detection/decode/audio CAPABILITIES + weights + camera caps are edited in the per-node capability table (agent settings), not here.",
|
|
27867
28050
|
fields: [
|
|
27868
28051
|
{
|
|
27869
|
-
key: "
|
|
27870
|
-
type: "node-
|
|
27871
|
-
label: "
|
|
27872
|
-
description: "
|
|
27873
|
-
default:
|
|
28052
|
+
key: "ingestNode",
|
|
28053
|
+
type: "node-select",
|
|
28054
|
+
label: "Ingest Node",
|
|
28055
|
+
description: "The node that connects every camera and serves the compressed restream. Defaults to the hub. (Arbitrary ingest ≠ hub is a later phase; kept here so the role is modeled.)",
|
|
28056
|
+
default: "hub",
|
|
27874
28057
|
showOffline: true
|
|
27875
28058
|
},
|
|
27876
28059
|
{
|
|
27877
|
-
key: "
|
|
27878
|
-
type: "node-
|
|
27879
|
-
label: "
|
|
27880
|
-
description: "
|
|
27881
|
-
default:
|
|
28060
|
+
key: "recordingNode",
|
|
28061
|
+
type: "node-select",
|
|
28062
|
+
label: "Recording Node",
|
|
28063
|
+
description: "The node that records every camera (passthrough copy to its storage location). Defaults to the hub.",
|
|
28064
|
+
default: "hub",
|
|
27882
28065
|
showOffline: true
|
|
27883
28066
|
},
|
|
27884
28067
|
{
|
|
27885
|
-
key: "
|
|
27886
|
-
type: "node-
|
|
27887
|
-
label: "
|
|
27888
|
-
description: "
|
|
27889
|
-
default:
|
|
27890
|
-
showOffline: true
|
|
27891
|
-
},
|
|
27892
|
-
{
|
|
27893
|
-
key: "enabledAudioNodes",
|
|
27894
|
-
type: "node-multiselect",
|
|
27895
|
-
label: "Enabled Audio Nodes",
|
|
27896
|
-
description: "Nodes eligible to run audio-analyzer sessions. Disabled nodes remain connected for other capabilities but will never receive audio assignments.",
|
|
27897
|
-
default: ["hub"],
|
|
28068
|
+
key: "audioNode",
|
|
28069
|
+
type: "node-select",
|
|
28070
|
+
label: "Audio Node",
|
|
28071
|
+
description: "The node that runs all audio analysis. Defaults to the hub. Must be an audio-capable node.",
|
|
28072
|
+
default: "hub",
|
|
27898
28073
|
showOffline: true
|
|
27899
28074
|
}
|
|
27900
28075
|
]
|
|
@@ -28397,16 +28572,58 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
28397
28572
|
pinnedOnDisconnect: config["pinnedOnDisconnect"] === "unpin-and-migrate" ? "unpin-and-migrate" : "leave-pinned",
|
|
28398
28573
|
onReconnect: config["onReconnect"] === "rebalance" ? "rebalance" : "restore"
|
|
28399
28574
|
};
|
|
28400
|
-
const
|
|
28401
|
-
|
|
28402
|
-
|
|
28403
|
-
|
|
28404
|
-
|
|
28405
|
-
|
|
28406
|
-
|
|
28407
|
-
|
|
28575
|
+
const readRole = (key) => {
|
|
28576
|
+
const raw = config[key];
|
|
28577
|
+
return typeof raw === "string" && raw.length > 0 && !raw.includes("/") ? raw : "hub";
|
|
28578
|
+
};
|
|
28579
|
+
this.clusterRoles = {
|
|
28580
|
+
ingestNode: readRole("ingestNode"),
|
|
28581
|
+
recordingNode: readRole("recordingNode"),
|
|
28582
|
+
audioNode: readRole("audioNode")
|
|
28583
|
+
};
|
|
28408
28584
|
this.schedulePendingRetry();
|
|
28409
28585
|
}
|
|
28586
|
+
/**
|
|
28587
|
+
* Default placement capability for a node when its stored flag is unset:
|
|
28588
|
+
* the hub is capable of every role out of the box; every other node must be
|
|
28589
|
+
* explicitly enabled. This reproduces the classic `['hub']` defaults of the
|
|
28590
|
+
* four removed flat lists against an empty capability store.
|
|
28591
|
+
*/
|
|
28592
|
+
static nodeCapabilityDefault(nodeId) {
|
|
28593
|
+
return nodeId === "hub";
|
|
28594
|
+
}
|
|
28595
|
+
/**
|
|
28596
|
+
* Recompute the derived enabled-node sets (`enabledNodes` /
|
|
28597
|
+
* `enabledDecoderNodes` / `enabledAudioNodes`) from the per-node capability
|
|
28598
|
+
* store. A node is eligible for a concern iff its stored flag is `true`, or
|
|
28599
|
+
* the flag is unset AND the node defaults capable ({@link nodeCapabilityDefault}).
|
|
28600
|
+
* Forked child nodeIds (`hub/classifier`) are never dispatchable and are
|
|
28601
|
+
* excluded. `hub` is always considered even when absent from the store, so a
|
|
28602
|
+
* fresh install keeps the hub-only cluster working.
|
|
28603
|
+
*/
|
|
28604
|
+
async refreshNodeCapabilities() {
|
|
28605
|
+
let blob = {};
|
|
28606
|
+
try {
|
|
28607
|
+
blob = await this.agentSettingsState.get();
|
|
28608
|
+
} catch (err) {
|
|
28609
|
+
this.ctx.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg(err) } });
|
|
28610
|
+
}
|
|
28611
|
+
const nodeIds = new Set(["hub"]);
|
|
28612
|
+
for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
|
|
28613
|
+
const detect = [];
|
|
28614
|
+
const decode = [];
|
|
28615
|
+
const audio = [];
|
|
28616
|
+
const capable = (flag, nodeId) => typeof flag === "boolean" ? flag : PipelineOrchestratorAddon.nodeCapabilityDefault(nodeId);
|
|
28617
|
+
for (const nodeId of nodeIds) {
|
|
28618
|
+
const settings = blob[nodeId];
|
|
28619
|
+
if (capable(settings?.detect, nodeId)) detect.push(nodeId);
|
|
28620
|
+
if (capable(settings?.decode, nodeId)) decode.push(nodeId);
|
|
28621
|
+
if (capable(settings?.audio, nodeId)) audio.push(nodeId);
|
|
28622
|
+
}
|
|
28623
|
+
this.enabledNodes = detect.toSorted();
|
|
28624
|
+
this.enabledDecoderNodes = decode.toSorted();
|
|
28625
|
+
this.enabledAudioNodes = audio.toSorted();
|
|
28626
|
+
}
|
|
28410
28627
|
get api() {
|
|
28411
28628
|
return this.ctx.api ?? null;
|
|
28412
28629
|
}
|
|
@@ -29003,7 +29220,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29003
29220
|
});
|
|
29004
29221
|
return;
|
|
29005
29222
|
}
|
|
29006
|
-
this.
|
|
29223
|
+
if (this.shedInFlight.has(deviceId)) return;
|
|
29224
|
+
this.shedInFlight.add(deviceId);
|
|
29225
|
+
this.ctx.logger.warn("Load management: sustained low fps — relocating or pausing", {
|
|
29007
29226
|
tags: { deviceId },
|
|
29008
29227
|
meta: {
|
|
29009
29228
|
actualFps: metrics.actualFps,
|
|
@@ -29012,7 +29231,28 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29012
29231
|
windowMs
|
|
29013
29232
|
}
|
|
29014
29233
|
});
|
|
29015
|
-
state.
|
|
29234
|
+
this.shedOrRelocate(deviceId, state).finally(() => this.shedInFlight.delete(deviceId));
|
|
29235
|
+
}
|
|
29236
|
+
/**
|
|
29237
|
+
* Cluster-wide load-shed action: relocate the sustained-slow camera to a
|
|
29238
|
+
* less-loaded node if the balancer (ceiling + weight aware) finds one,
|
|
29239
|
+
* otherwise pause it. Reuses `balance()` / `attachOn` / `detachOn` — no custom
|
|
29240
|
+
* placement logic.
|
|
29241
|
+
*/
|
|
29242
|
+
async shedOrRelocate(deviceId, state) {
|
|
29243
|
+
if (state.pausedAt !== null) return;
|
|
29244
|
+
if (await this.tryClusterRelocate(deviceId).catch((err) => {
|
|
29245
|
+
this.ctx.logger.warn("Load management: cluster relocate failed", {
|
|
29246
|
+
tags: { deviceId },
|
|
29247
|
+
meta: { error: errMsg(err) }
|
|
29248
|
+
});
|
|
29249
|
+
return false;
|
|
29250
|
+
})) {
|
|
29251
|
+
state.lowSinceTs = null;
|
|
29252
|
+
return;
|
|
29253
|
+
}
|
|
29254
|
+
this.ctx.logger.warn("Load management: pausing camera detection (no spare cluster capacity)", { tags: { deviceId } });
|
|
29255
|
+
state.pausedAt = Date.now();
|
|
29016
29256
|
state.lowSinceTs = null;
|
|
29017
29257
|
this.pipelineWatchdog?.unregister(deviceId);
|
|
29018
29258
|
this.ctx.api?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
|
|
@@ -29023,6 +29263,51 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
29023
29263
|
});
|
|
29024
29264
|
this.ensureLoadShedResumeTimer();
|
|
29025
29265
|
}
|
|
29266
|
+
/**
|
|
29267
|
+
* Try to move a sustained-slow camera to a less-loaded node CLUSTER-WIDE.
|
|
29268
|
+
* Runs the same `balance()` the dispatcher uses (per-node ceiling + weight +
|
|
29269
|
+
* frame-source eligibility); the camera's current node counts it in its own
|
|
29270
|
+
* load, so a genuinely less-loaded node with spare capacity wins. Returns
|
|
29271
|
+
* `true` iff the camera was moved to a DIFFERENT node. A manually pinned
|
|
29272
|
+
* camera never relocates. Reuses `detachOn`/`attachOn`/`recordAssignment`.
|
|
29273
|
+
*/
|
|
29274
|
+
async tryClusterRelocate(deviceId) {
|
|
29275
|
+
const current = this.assignments.get(deviceId);
|
|
29276
|
+
const cached = this.cameraConfigs.get(deviceId);
|
|
29277
|
+
if (!current || !cached) return false;
|
|
29278
|
+
const currentNode = current.agentNodeId;
|
|
29279
|
+
if (await this.readPipelinePin(deviceId)) return false;
|
|
29280
|
+
const decision = balance({
|
|
29281
|
+
nodes: await this.collectAgentLoad({ onlyEnabled: true }),
|
|
29282
|
+
preferredAgent: null,
|
|
29283
|
+
nodeCaps: await this.buildNodeCaps(),
|
|
29284
|
+
weights: await this.buildNodeWeights(),
|
|
29285
|
+
eligibleNodes: this.detectionEligibleNodes(deviceId)
|
|
29286
|
+
});
|
|
29287
|
+
if (!decision || decision.kind !== "assigned") return false;
|
|
29288
|
+
const target = decision.agentNodeId;
|
|
29289
|
+
if (target === currentNode) return false;
|
|
29290
|
+
await this.detachOn(currentNode, deviceId).catch((err) => {
|
|
29291
|
+
this.ctx.logger.debug("Load management: relocate detach-old failed", {
|
|
29292
|
+
tags: { deviceId },
|
|
29293
|
+
meta: {
|
|
29294
|
+
from: currentNode,
|
|
29295
|
+
error: errMsg(err)
|
|
29296
|
+
}
|
|
29297
|
+
});
|
|
29298
|
+
});
|
|
29299
|
+
await this.attachOn(target, cached);
|
|
29300
|
+
this.recordAssignment(deviceId, target, "rebalance", false);
|
|
29301
|
+
this.ctx.logger.info("Load management: relocated camera to spare-capacity node (cluster-wide)", {
|
|
29302
|
+
tags: { deviceId },
|
|
29303
|
+
meta: {
|
|
29304
|
+
from: currentNode,
|
|
29305
|
+
to: target,
|
|
29306
|
+
score: decision.score
|
|
29307
|
+
}
|
|
29308
|
+
});
|
|
29309
|
+
return true;
|
|
29310
|
+
}
|
|
29026
29311
|
/** Base cooldown before first auto-resume attempt. */
|
|
29027
29312
|
static LOAD_SHED_BASE_COOLDOWN_MS = 3e4;
|
|
29028
29313
|
/** Maximum backoff cap. */
|