@camstack/addon-pipeline-orchestrator 1.2.290 → 1.2.291
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DHY2k_nC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CdkZH5kw.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs} +1 -1
- package/dist/{hostInit-CRtAczui.mjs → hostInit-BpNmAa0Q.mjs} +2 -2
- package/dist/index.js +813 -71
- package/dist/index.mjs +813 -71
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DWLgJB4I.mjs +0 -26
package/dist/index.js
CHANGED
|
@@ -25033,6 +25033,105 @@ var pipelineExecutorCapability = {
|
|
|
25033
25033
|
}
|
|
25034
25034
|
};
|
|
25035
25035
|
/**
|
|
25036
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25037
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25038
|
+
* hardware share suggestion and the stale device pins.
|
|
25039
|
+
*/
|
|
25040
|
+
/**
|
|
25041
|
+
* How a new detection session picks its node and device.
|
|
25042
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25043
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25044
|
+
* weight only bites once a node already holds a session.
|
|
25045
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25046
|
+
* full or drowning node yields to a free one (D748).
|
|
25047
|
+
*/
|
|
25048
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25049
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "least-loaded";
|
|
25050
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25051
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25052
|
+
/** Parse a stored value; anything unknown is the default (nothing changes on deploy). */
|
|
25053
|
+
function parseSessionPlacementPolicy(value) {
|
|
25054
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25055
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25056
|
+
}
|
|
25057
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25058
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25059
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25060
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25061
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25062
|
+
});
|
|
25063
|
+
var PlacementStatsInputSchema = object({
|
|
25064
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25065
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25066
|
+
var PlacementDeviceStatsSchema = object({
|
|
25067
|
+
deviceKey: string(),
|
|
25068
|
+
/** Sessions running on this device now. */
|
|
25069
|
+
sessionsNow: number(),
|
|
25070
|
+
/** Sessions started on this device inside the window. */
|
|
25071
|
+
sessionsStarted: number(),
|
|
25072
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25073
|
+
observedShare: number().nullable()
|
|
25074
|
+
});
|
|
25075
|
+
var PlacementNodeStatsSchema = object({
|
|
25076
|
+
nodeId: string(),
|
|
25077
|
+
sessionsNow: number(),
|
|
25078
|
+
sessionsStarted: number(),
|
|
25079
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25080
|
+
observedShare: number().nullable(),
|
|
25081
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25082
|
+
peakSessions: number(),
|
|
25083
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25084
|
+
yieldsAway: number(),
|
|
25085
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25086
|
+
});
|
|
25087
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25088
|
+
var StaleDevicePinSchema = object({
|
|
25089
|
+
deviceId: number(),
|
|
25090
|
+
deviceKey: string(),
|
|
25091
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25092
|
+
reason: _enum(["disabled", "absent"])
|
|
25093
|
+
});
|
|
25094
|
+
var PlacementStatsSchema = object({
|
|
25095
|
+
policy: SessionPlacementPolicySchema,
|
|
25096
|
+
windowMs: number(),
|
|
25097
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25098
|
+
observedForMs: number(),
|
|
25099
|
+
sessionsNow: number(),
|
|
25100
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25101
|
+
clusterPeakSessions: number(),
|
|
25102
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25103
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25104
|
+
});
|
|
25105
|
+
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25106
|
+
var SuggestedDeviceShareSchema = object({
|
|
25107
|
+
deviceKey: string(),
|
|
25108
|
+
/** Share of the node (0–100, rounded). */
|
|
25109
|
+
share: number(),
|
|
25110
|
+
modelId: string().nullable(),
|
|
25111
|
+
/** Inferences per second at that model, from the measured table. */
|
|
25112
|
+
fps: number().nullable(),
|
|
25113
|
+
measured: boolean(),
|
|
25114
|
+
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25115
|
+
suggestedMaxSessions: number().nullable()
|
|
25116
|
+
});
|
|
25117
|
+
var SuggestedNodeShareSchema = object({
|
|
25118
|
+
nodeId: string(),
|
|
25119
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25120
|
+
share: number().nullable(),
|
|
25121
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25122
|
+
capacitySessions: number(),
|
|
25123
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25124
|
+
skippedReason: string().nullable()
|
|
25125
|
+
});
|
|
25126
|
+
var SuggestSharesResultSchema = object({
|
|
25127
|
+
headroom: number(),
|
|
25128
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25129
|
+
cameraFps: number(),
|
|
25130
|
+
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25131
|
+
iterations: number(),
|
|
25132
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25133
|
+
});
|
|
25134
|
+
/**
|
|
25036
25135
|
* The wire shapes of the Best settings plan and apply (D745), used by the
|
|
25037
25136
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25038
25137
|
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
@@ -25109,7 +25208,9 @@ var DeviceDemandSchema = object({
|
|
|
25109
25208
|
"estimate",
|
|
25110
25209
|
"none"
|
|
25111
25210
|
]),
|
|
25112
|
-
observedForMs: number().nullable()
|
|
25211
|
+
observedForMs: number().nullable(),
|
|
25212
|
+
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25213
|
+
placement: SessionPlacementPolicySchema
|
|
25113
25214
|
});
|
|
25114
25215
|
var DeviceModelChoiceSchema = object({
|
|
25115
25216
|
modelId: string(),
|
|
@@ -25181,14 +25282,24 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25181
25282
|
deviceKey: string()
|
|
25182
25283
|
})
|
|
25183
25284
|
]);
|
|
25184
|
-
var BestSettingsOptionsSchema = object({
|
|
25185
|
-
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25186
|
-
headroom: number().min(.3).max(.5).default(.4)
|
|
25285
|
+
var BestSettingsOptionsSchema = object({
|
|
25286
|
+
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25287
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25288
|
+
/**
|
|
25289
|
+
* What-if shares (D748): plan as if these node/device shares were stored
|
|
25290
|
+
* and the policy were `share`. Absent = the stored shares under the stored
|
|
25291
|
+
* policy. A what-if plan is for display; apply always re-plans from what
|
|
25292
|
+
* is stored.
|
|
25293
|
+
*/
|
|
25294
|
+
shares: LoadBalancingSharesInputSchema.optional()
|
|
25295
|
+
});
|
|
25187
25296
|
var BestSettingsPlanSchema = object({
|
|
25188
25297
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25189
25298
|
planHash: string(),
|
|
25190
25299
|
family: literal("yolov9"),
|
|
25191
25300
|
headroom: number(),
|
|
25301
|
+
/** Planned under what-if shares (D748): for display — apply refuses it as stale. */
|
|
25302
|
+
whatIf: boolean(),
|
|
25192
25303
|
generatedAt: number(),
|
|
25193
25304
|
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25194
25305
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
@@ -26638,7 +26749,6 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26638
26749
|
settings: record(string(), unknown()).readonly().optional()
|
|
26639
26750
|
});
|
|
26640
26751
|
var CameraPipelineSettingsSchema = object({
|
|
26641
|
-
pinnedAgentNodeId: string().optional(),
|
|
26642
26752
|
stepToggles: record(string(), boolean()).optional(),
|
|
26643
26753
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26644
26754
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
@@ -27286,18 +27396,6 @@ var pipelineOrchestratorCapability = {
|
|
|
27286
27396
|
auth: "admin"
|
|
27287
27397
|
}),
|
|
27288
27398
|
/**
|
|
27289
|
-
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
27290
|
-
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
27291
|
-
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
27292
|
-
*/
|
|
27293
|
-
setAgentDetectWeight: method(object({
|
|
27294
|
-
agentNodeId: string(),
|
|
27295
|
-
detectWeight: number().positive().nullable()
|
|
27296
|
-
}), object({ success: literal(true) }), {
|
|
27297
|
-
kind: "mutation",
|
|
27298
|
-
auth: "admin"
|
|
27299
|
-
}),
|
|
27300
|
-
/**
|
|
27301
27399
|
* Set one node's placement CAPABILITIES — the per-node record that
|
|
27302
27400
|
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
27303
27401
|
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
@@ -27437,6 +27535,33 @@ var pipelineOrchestratorCapability = {
|
|
|
27437
27535
|
kind: "mutation",
|
|
27438
27536
|
auth: "admin"
|
|
27439
27537
|
}),
|
|
27538
|
+
/**
|
|
27539
|
+
* Load balancing (D748) — write node shares (`detectWeight`) and per-node
|
|
27540
|
+
* device shares (inference-device `weight`) in ONE atomic read-modify-write
|
|
27541
|
+
* of the agent settings — the only writer of node shares (the per-node
|
|
27542
|
+
* `setAgentDetectWeight` it replaced raced its siblings, one RMW each).
|
|
27543
|
+
* A node or device left out keeps its weight. Never changes the
|
|
27544
|
+
* `sessionPlacement` policy.
|
|
27545
|
+
*/
|
|
27546
|
+
setLoadBalancingShares: method(LoadBalancingSharesInputSchema, object({ success: literal(true) }), {
|
|
27547
|
+
kind: "mutation",
|
|
27548
|
+
auth: "admin"
|
|
27549
|
+
}),
|
|
27550
|
+
/**
|
|
27551
|
+
* Load balancing (D748) — what placement actually did: per node and
|
|
27552
|
+
* device, sessions now, sessions started and the time-weighted share of
|
|
27553
|
+
* PARALLEL sessions over `windowMs`; the cluster's parallel peak; yields
|
|
27554
|
+
* a share election made; and every device pin the session path cannot
|
|
27555
|
+
* honour. In memory on the hub — a restart starts a new window.
|
|
27556
|
+
*/
|
|
27557
|
+
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27558
|
+
/**
|
|
27559
|
+
* Load balancing (D748) — shares proportional to each node's measured
|
|
27560
|
+
* capacity at the models Best settings would pick, iterated (≤ 3 rounds)
|
|
27561
|
+
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27562
|
+
* Writes nothing.
|
|
27563
|
+
*/
|
|
27564
|
+
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27440
27565
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27441
27566
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27442
27567
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -43616,6 +43741,12 @@ Object.freeze({
|
|
|
43616
43741
|
addonId: null,
|
|
43617
43742
|
access: "view"
|
|
43618
43743
|
},
|
|
43744
|
+
"pipelineOrchestrator.getPlacementStats": {
|
|
43745
|
+
capName: "pipeline-orchestrator",
|
|
43746
|
+
capScope: "system",
|
|
43747
|
+
addonId: null,
|
|
43748
|
+
access: "view"
|
|
43749
|
+
},
|
|
43619
43750
|
"pipelineOrchestrator.getReconcileFromDiskStatus": {
|
|
43620
43751
|
capName: "pipeline-orchestrator",
|
|
43621
43752
|
capScope: "system",
|
|
@@ -43706,12 +43837,6 @@ Object.freeze({
|
|
|
43706
43837
|
addonId: null,
|
|
43707
43838
|
access: "create"
|
|
43708
43839
|
},
|
|
43709
|
-
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
43710
|
-
capName: "pipeline-orchestrator",
|
|
43711
|
-
capScope: "system",
|
|
43712
|
-
addonId: null,
|
|
43713
|
-
access: "create"
|
|
43714
|
-
},
|
|
43715
43840
|
"pipelineOrchestrator.setAgentInferenceDevices": {
|
|
43716
43841
|
capName: "pipeline-orchestrator",
|
|
43717
43842
|
capScope: "system",
|
|
@@ -43766,12 +43891,24 @@ Object.freeze({
|
|
|
43766
43891
|
addonId: null,
|
|
43767
43892
|
access: "create"
|
|
43768
43893
|
},
|
|
43894
|
+
"pipelineOrchestrator.setLoadBalancingShares": {
|
|
43895
|
+
capName: "pipeline-orchestrator",
|
|
43896
|
+
capScope: "system",
|
|
43897
|
+
addonId: null,
|
|
43898
|
+
access: "create"
|
|
43899
|
+
},
|
|
43769
43900
|
"pipelineOrchestrator.setPipelineDevicePin": {
|
|
43770
43901
|
capName: "pipeline-orchestrator",
|
|
43771
43902
|
capScope: "system",
|
|
43772
43903
|
addonId: null,
|
|
43773
43904
|
access: "create"
|
|
43774
43905
|
},
|
|
43906
|
+
"pipelineOrchestrator.suggestShares": {
|
|
43907
|
+
capName: "pipeline-orchestrator",
|
|
43908
|
+
capScope: "system",
|
|
43909
|
+
addonId: null,
|
|
43910
|
+
access: "view"
|
|
43911
|
+
},
|
|
43775
43912
|
"pipelineOrchestrator.unassignAudio": {
|
|
43776
43913
|
capName: "pipeline-orchestrator",
|
|
43777
43914
|
capScope: "system",
|
|
@@ -51521,6 +51658,7 @@ function estimatedConcurrentSessions(cameraCount, options) {
|
|
|
51521
51658
|
*/
|
|
51522
51659
|
function estimateDeviceDemand(input) {
|
|
51523
51660
|
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51661
|
+
const placement = input.shares ? "share" : "least-loaded";
|
|
51524
51662
|
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51525
51663
|
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51526
51664
|
const weightByNode = /* @__PURE__ */ new Map();
|
|
@@ -51540,19 +51678,28 @@ function estimateDeviceDemand(input) {
|
|
|
51540
51678
|
const noteMax = (key, fps) => {
|
|
51541
51679
|
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51542
51680
|
};
|
|
51681
|
+
const pinNodeOf = (pin) => {
|
|
51682
|
+
if (pin.nodeId !== null) return pin.nodeId;
|
|
51683
|
+
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51684
|
+
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51685
|
+
};
|
|
51686
|
+
let alwaysOnTotal = 0;
|
|
51543
51687
|
for (const cam of input.cameras) {
|
|
51544
51688
|
if (!charges(cam)) continue;
|
|
51545
|
-
|
|
51546
|
-
|
|
51689
|
+
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51690
|
+
if (cam.devicePin !== null && pinNode !== null) {
|
|
51691
|
+
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51547
51692
|
if (!eligibleById.has(key)) continue;
|
|
51548
51693
|
bump(pinnedFps, key, cam.fps);
|
|
51549
51694
|
noteMax(key, cam.fps);
|
|
51550
51695
|
continue;
|
|
51551
51696
|
}
|
|
51552
|
-
|
|
51553
|
-
|
|
51697
|
+
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51698
|
+
if (standing !== null) {
|
|
51699
|
+
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51700
|
+
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51554
51701
|
for (const d of input.eligible) {
|
|
51555
|
-
if (d.nodeId !==
|
|
51702
|
+
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51556
51703
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51557
51704
|
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51558
51705
|
bump(alwaysOnFps, key, cam.fps * share);
|
|
@@ -51566,13 +51713,23 @@ function estimateDeviceDemand(input) {
|
|
|
51566
51713
|
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51567
51714
|
const nodeCount = weightByNode.size;
|
|
51568
51715
|
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51716
|
+
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51717
|
+
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51718
|
+
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51719
|
+
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51569
51720
|
return input.eligible.map((d) => {
|
|
51570
51721
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51571
51722
|
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51572
51723
|
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51573
51724
|
let onMotionSessions = 0;
|
|
51574
51725
|
let onMotionSource = "none";
|
|
51575
|
-
if (onMotionRates.length > 0
|
|
51726
|
+
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51727
|
+
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51728
|
+
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51729
|
+
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51730
|
+
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51731
|
+
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51732
|
+
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51576
51733
|
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51577
51734
|
onMotionSource = "observed-24h";
|
|
51578
51735
|
} else {
|
|
@@ -51590,7 +51747,8 @@ function estimateDeviceDemand(input) {
|
|
|
51590
51747
|
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51591
51748
|
onMotionSessions,
|
|
51592
51749
|
onMotionSource,
|
|
51593
|
-
observedForMs: peak?.observedForMs ?? null
|
|
51750
|
+
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51751
|
+
placement
|
|
51594
51752
|
};
|
|
51595
51753
|
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51596
51754
|
}
|
|
@@ -51819,6 +51977,46 @@ function recommendNode(input) {
|
|
|
51819
51977
|
reasons: nodeReasons
|
|
51820
51978
|
};
|
|
51821
51979
|
}
|
|
51980
|
+
/** Deficits equal up to float noise (0.7 × 2 − 1 is 0.3999…, not 0.4). */
|
|
51981
|
+
var DEFICIT_EPSILON = 1e-9;
|
|
51982
|
+
function byDeficit(a, b) {
|
|
51983
|
+
const diff = b.deficit - a.deficit;
|
|
51984
|
+
return (Math.abs(diff) < DEFICIT_EPSILON ? 0 : diff) || b.candidate.share - a.candidate.share || a.candidate.key.localeCompare(b.candidate.key);
|
|
51985
|
+
}
|
|
51986
|
+
function spareOf(c) {
|
|
51987
|
+
return c.spare ?? Number.POSITIVE_INFINITY;
|
|
51988
|
+
}
|
|
51989
|
+
function electByShare(candidates) {
|
|
51990
|
+
if (candidates.length === 0) return { kind: "none" };
|
|
51991
|
+
const shareSum = candidates.reduce((s, c) => s + Math.max(c.share, 0), 0);
|
|
51992
|
+
const running = candidates.reduce((s, c) => s + Math.max(c.running, 0), 0);
|
|
51993
|
+
const scored = candidates.map((candidate) => ({
|
|
51994
|
+
candidate,
|
|
51995
|
+
deficit: (shareSum > 0 ? Math.max(candidate.share, 0) / shareSum : 1 / candidates.length) * (running + 1) - candidate.running
|
|
51996
|
+
})).toSorted(byDeficit);
|
|
51997
|
+
const turn = scored[0].candidate;
|
|
51998
|
+
if (!turn.capped && !turn.saturated) return {
|
|
51999
|
+
kind: "elected",
|
|
52000
|
+
key: turn.key,
|
|
52001
|
+
reason: "share"
|
|
52002
|
+
};
|
|
52003
|
+
const free = scored.filter((s) => !s.candidate.capped && !s.candidate.saturated).toSorted((a, b) => spareOf(b.candidate) - spareOf(a.candidate) || byDeficit(a, b));
|
|
52004
|
+
const yieldReason = turn.capped ? "at-cap" : "saturated";
|
|
52005
|
+
if (free.length > 0) return {
|
|
52006
|
+
kind: "elected",
|
|
52007
|
+
key: free[0].candidate.key,
|
|
52008
|
+
reason: "yield",
|
|
52009
|
+
turnKey: turn.key,
|
|
52010
|
+
yieldReason
|
|
52011
|
+
};
|
|
52012
|
+
const uncapped = scored.find((s) => !s.candidate.capped);
|
|
52013
|
+
if (uncapped === void 0) return { kind: "none" };
|
|
52014
|
+
return {
|
|
52015
|
+
kind: "elected",
|
|
52016
|
+
key: uncapped.candidate.key,
|
|
52017
|
+
reason: "share"
|
|
52018
|
+
};
|
|
52019
|
+
}
|
|
51822
52020
|
/**
|
|
51823
52021
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
51824
52022
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -59881,6 +60079,35 @@ function pickInferenceDevice(input) {
|
|
|
59881
60079
|
const loadByKey = new Map((input.loads ?? []).map((l) => [l.deviceKey, l]));
|
|
59882
60080
|
const healthyKeys = eligibleKeys.filter((k) => !isDeviceSaturated(loadByKey.get(k)));
|
|
59883
60081
|
const consideredKeys = pin !== null || healthyKeys.length === 0 || healthyKeys.length === eligibleKeys.length ? eligibleKeys : healthyKeys;
|
|
60082
|
+
if (input.policy === "share" && pin === null) {
|
|
60083
|
+
const election = electByShare(eligibleKeys.map((key) => {
|
|
60084
|
+
const load = loadByKey.get(key);
|
|
60085
|
+
const attached = load?.attachedCameras ?? 0;
|
|
60086
|
+
const cap = input.caps?.[key];
|
|
60087
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
60088
|
+
return {
|
|
60089
|
+
key,
|
|
60090
|
+
share: input.eligible[key] ?? 1,
|
|
60091
|
+
running: attached,
|
|
60092
|
+
capped: hasCap && attached >= cap,
|
|
60093
|
+
saturated: isDeviceSaturated(load),
|
|
60094
|
+
...hasCap ? { spare: Math.max(0, cap - attached) } : {}
|
|
60095
|
+
};
|
|
60096
|
+
}));
|
|
60097
|
+
if (election.kind === "none") return {
|
|
60098
|
+
kind: "default-pool",
|
|
60099
|
+
reason: "all-devices-at-cap",
|
|
60100
|
+
eligibleKeys
|
|
60101
|
+
};
|
|
60102
|
+
return {
|
|
60103
|
+
kind: "selected",
|
|
60104
|
+
deviceKey: election.key,
|
|
60105
|
+
reason: "share",
|
|
60106
|
+
tieBreak: null,
|
|
60107
|
+
eligibleKeys,
|
|
60108
|
+
tiedKeys: [election.key]
|
|
60109
|
+
};
|
|
60110
|
+
}
|
|
59884
60111
|
const balancerInput = {
|
|
59885
60112
|
nodes: consideredKeys.map((k) => deviceAsLoad(k, loadByKey.get(k))),
|
|
59886
60113
|
preferredAgent: pin,
|
|
@@ -64204,7 +64431,6 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
64204
64431
|
}).nullable()
|
|
64205
64432
|
});
|
|
64206
64433
|
var StoredCameraPipelineSettingsSchema = object({
|
|
64207
|
-
pinnedAgentNodeId: string().optional(),
|
|
64208
64434
|
stepToggles: record(string(), boolean()).optional(),
|
|
64209
64435
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
64210
64436
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
@@ -66708,7 +66934,15 @@ var TtlMemo = class {
|
|
|
66708
66934
|
//#endregion
|
|
66709
66935
|
//#region src/detection-session-registry.ts
|
|
66710
66936
|
var DetectionSessionRegistry = class {
|
|
66937
|
+
onChange;
|
|
66711
66938
|
sessions = /* @__PURE__ */ new Map();
|
|
66939
|
+
/**
|
|
66940
|
+
* @param onChange called after every register / deregister / clear with the
|
|
66941
|
+
* running set — feeds the placement statistics (D748). Never on a no-op.
|
|
66942
|
+
*/
|
|
66943
|
+
constructor(onChange = () => {}) {
|
|
66944
|
+
this.onChange = onChange;
|
|
66945
|
+
}
|
|
66712
66946
|
register(deviceId, nodeId, startedAt, triggeredBy, deviceKey) {
|
|
66713
66947
|
const existing = this.sessions.get(deviceId);
|
|
66714
66948
|
if (existing?.teardownTimer) clearTimeout(existing.teardownTimer);
|
|
@@ -66721,6 +66955,7 @@ var DetectionSessionRegistry = class {
|
|
|
66721
66955
|
...deviceKey ? { deviceKey } : {}
|
|
66722
66956
|
};
|
|
66723
66957
|
this.sessions.set(deviceId, session);
|
|
66958
|
+
this.onChange(this.list());
|
|
66724
66959
|
return session;
|
|
66725
66960
|
}
|
|
66726
66961
|
get(deviceId) {
|
|
@@ -66734,6 +66969,7 @@ var DetectionSessionRegistry = class {
|
|
|
66734
66969
|
if (!session) return void 0;
|
|
66735
66970
|
if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66736
66971
|
this.sessions.delete(deviceId);
|
|
66972
|
+
this.onChange(this.list());
|
|
66737
66973
|
return session;
|
|
66738
66974
|
}
|
|
66739
66975
|
setTeardownTimer(deviceId, timer) {
|
|
@@ -66756,9 +66992,39 @@ var DetectionSessionRegistry = class {
|
|
|
66756
66992
|
}
|
|
66757
66993
|
clear() {
|
|
66758
66994
|
for (const session of this.sessions.values()) if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66995
|
+
const hadSessions = this.sessions.size > 0;
|
|
66759
66996
|
this.sessions.clear();
|
|
66997
|
+
if (hadSessions) this.onChange([]);
|
|
66760
66998
|
}
|
|
66761
66999
|
};
|
|
67000
|
+
//#endregion
|
|
67001
|
+
//#region src/session-share-placement.ts
|
|
67002
|
+
function nodeSaturated(load) {
|
|
67003
|
+
return load.devices.length > 0 && load.devices.every((d) => isDeviceSaturated(d));
|
|
67004
|
+
}
|
|
67005
|
+
function electNodeByShare(input) {
|
|
67006
|
+
return electByShare(input.loads.filter((l) => l.nodeId.length > 0 && input.eligible.includes(l.nodeId)).map((load) => {
|
|
67007
|
+
const cap = input.caps[load.nodeId];
|
|
67008
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
67009
|
+
const weight = input.weights[load.nodeId];
|
|
67010
|
+
return {
|
|
67011
|
+
key: load.nodeId,
|
|
67012
|
+
share: weight !== void 0 && weight > 0 ? weight : 1,
|
|
67013
|
+
running: input.runningByNode.get(load.nodeId) ?? 0,
|
|
67014
|
+
capped: hasCap && load.attachedCameras >= cap,
|
|
67015
|
+
saturated: nodeSaturated(load),
|
|
67016
|
+
...hasCap ? { spare: Math.max(0, cap - load.attachedCameras) } : {}
|
|
67017
|
+
};
|
|
67018
|
+
}));
|
|
67019
|
+
}
|
|
67020
|
+
/** Parallel sessions per node: sessions and standing cameras, each camera once. */
|
|
67021
|
+
function countRunningByNode(placements) {
|
|
67022
|
+
const seen = /* @__PURE__ */ new Map();
|
|
67023
|
+
for (const p of placements) if (!seen.has(p.deviceId)) seen.set(p.deviceId, p.nodeId);
|
|
67024
|
+
const out = /* @__PURE__ */ new Map();
|
|
67025
|
+
for (const nodeId of seen.values()) out.set(nodeId, (out.get(nodeId) ?? 0) + 1);
|
|
67026
|
+
return out;
|
|
67027
|
+
}
|
|
66762
67028
|
/**
|
|
66763
67029
|
* Minimum time a detection session stays open, whatever closes it.
|
|
66764
67030
|
*
|
|
@@ -66871,7 +67137,15 @@ var SessionDispatchController = class {
|
|
|
66871
67137
|
* (not `private`) — see the class doc for the two production/test reasons
|
|
66872
67138
|
* `index.ts` reaches through it directly.
|
|
66873
67139
|
*/
|
|
66874
|
-
sessionRegistry = new DetectionSessionRegistry()
|
|
67140
|
+
sessionRegistry = new DetectionSessionRegistry((sessions) => this.deps.placementStats?.noteSessions(sessions.map((s) => ({
|
|
67141
|
+
nodeId: s.nodeId,
|
|
67142
|
+
deviceKey: s.deviceKey ?? ""
|
|
67143
|
+
}))));
|
|
67144
|
+
/**
|
|
67145
|
+
* `deviceId|deviceKey` of device pins the session path has already said it
|
|
67146
|
+
* ignored — one line per stale pin, not one per session (D748).
|
|
67147
|
+
*/
|
|
67148
|
+
reportedStalePins = /* @__PURE__ */ new Set();
|
|
66875
67149
|
/**
|
|
66876
67150
|
* Per-device async mutex for session dispatch/teardown critical sections —
|
|
66877
67151
|
* backed by the generic `KeyedAsyncLock` (mirrors the audio controller's
|
|
@@ -67332,6 +67606,76 @@ var SessionDispatchController = class {
|
|
|
67332
67606
|
});
|
|
67333
67607
|
}
|
|
67334
67608
|
/**
|
|
67609
|
+
* The session's node. `least-loaded` (and any node pin, under either
|
|
67610
|
+
* policy) is `balance()` exactly as before. `share` holds the node shares
|
|
67611
|
+
* of PARALLEL sessions (D748): sessions now running and standing cameras
|
|
67612
|
+
* count toward their node, and a node at its cap or drowning yields —
|
|
67613
|
+
* logged with the reason, and counted against the node in the stats.
|
|
67614
|
+
*/
|
|
67615
|
+
electSessionNode(input) {
|
|
67616
|
+
const balanced = () => balance({
|
|
67617
|
+
nodes: input.loads,
|
|
67618
|
+
preferredAgent: input.preferredAgent,
|
|
67619
|
+
nodeCaps: input.nodeCaps,
|
|
67620
|
+
weights: input.weights,
|
|
67621
|
+
eligibleNodes: input.eligible
|
|
67622
|
+
});
|
|
67623
|
+
if (input.policy !== "share" || input.preferredAgent) return balanced();
|
|
67624
|
+
const running = countRunningByNode([...this.sessionRegistry.list().map((s) => ({
|
|
67625
|
+
deviceId: s.deviceId,
|
|
67626
|
+
nodeId: s.nodeId
|
|
67627
|
+
})), ...this.deps.ledger.listAssignments().map((a) => ({
|
|
67628
|
+
deviceId: a.deviceId,
|
|
67629
|
+
nodeId: a.agentNodeId
|
|
67630
|
+
}))]);
|
|
67631
|
+
const election = electNodeByShare({
|
|
67632
|
+
loads: input.loads,
|
|
67633
|
+
eligible: input.eligible,
|
|
67634
|
+
weights: input.weights,
|
|
67635
|
+
caps: input.nodeCaps,
|
|
67636
|
+
runningByNode: running
|
|
67637
|
+
});
|
|
67638
|
+
if (election.kind === "none") return balanced();
|
|
67639
|
+
if (election.reason === "yield") {
|
|
67640
|
+
this.deps.placementStats?.noteYield(election.turnKey);
|
|
67641
|
+
this.deps.logger.info("detection session: share turn yielded — the node had no room", {
|
|
67642
|
+
tags: {
|
|
67643
|
+
deviceId: input.deviceId,
|
|
67644
|
+
nodeId: election.key
|
|
67645
|
+
},
|
|
67646
|
+
meta: {
|
|
67647
|
+
turnNodeId: election.turnKey,
|
|
67648
|
+
reason: election.yieldReason
|
|
67649
|
+
}
|
|
67650
|
+
});
|
|
67651
|
+
}
|
|
67652
|
+
const load = input.loads.find((l) => l.nodeId === election.key);
|
|
67653
|
+
return {
|
|
67654
|
+
kind: "assigned",
|
|
67655
|
+
agentNodeId: election.key,
|
|
67656
|
+
reason: "capacity",
|
|
67657
|
+
score: load ? load.attachedCameras + load.queueDepthTotal : 0
|
|
67658
|
+
};
|
|
67659
|
+
}
|
|
67660
|
+
/**
|
|
67661
|
+
* Say ONCE that a camera's device pin names a device this node does not
|
|
67662
|
+
* offer (disabled, absent): the session path ignores it, and the Load
|
|
67663
|
+
* balancing tab offers to clear it. Never cleared here (D748).
|
|
67664
|
+
*/
|
|
67665
|
+
noteStalePin(deviceId, pinnedDeviceKey, enabledDevices) {
|
|
67666
|
+
if (pinnedDeviceKey === null || pinnedDeviceKey in enabledDevices) return;
|
|
67667
|
+
const key = `${deviceId}|${pinnedDeviceKey}`;
|
|
67668
|
+
if (this.reportedStalePins.has(key)) return;
|
|
67669
|
+
this.reportedStalePins.add(key);
|
|
67670
|
+
this.deps.logger.warn("device pin ignored — the pinned device is not enabled on the elected node", {
|
|
67671
|
+
tags: { deviceId },
|
|
67672
|
+
meta: {
|
|
67673
|
+
pin: pinnedDeviceKey,
|
|
67674
|
+
enabled: Object.keys(enabledDevices).join(",")
|
|
67675
|
+
}
|
|
67676
|
+
});
|
|
67677
|
+
}
|
|
67678
|
+
/**
|
|
67335
67679
|
* Dispatch a session-time detection attach for an on-motion camera whose
|
|
67336
67680
|
* `decideSessionAction` returned `'dispatch'`. Builds a session
|
|
67337
67681
|
* `RunnerCameraConfig` the same way `startDetection` does, but forces
|
|
@@ -67382,12 +67726,17 @@ var SessionDispatchController = class {
|
|
|
67382
67726
|
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
67383
67727
|
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
67384
67728
|
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
67385
|
-
const
|
|
67386
|
-
|
|
67729
|
+
const nodeCaps = await this.nodeCaps.get(CLUSTER_KEY);
|
|
67730
|
+
const weights = await this.nodeWeights.get(CLUSTER_KEY);
|
|
67731
|
+
const policy = this.deps.sessionPlacement?.() ?? "least-loaded";
|
|
67732
|
+
const decision = this.electSessionNode({
|
|
67733
|
+
deviceId,
|
|
67734
|
+
policy,
|
|
67735
|
+
loads,
|
|
67387
67736
|
preferredAgent,
|
|
67388
|
-
nodeCaps
|
|
67389
|
-
weights
|
|
67390
|
-
|
|
67737
|
+
nodeCaps,
|
|
67738
|
+
weights,
|
|
67739
|
+
eligible
|
|
67391
67740
|
});
|
|
67392
67741
|
if (!decision || decision.kind !== "assigned") {
|
|
67393
67742
|
log.warn("detection session: no eligible node — will retry on next motion", { meta: { reason: decision?.kind === "pending" ? decision.reason : "no-runners-online" } });
|
|
@@ -67400,12 +67749,14 @@ var SessionDispatchController = class {
|
|
|
67400
67749
|
const targetLoad = loads.find((n) => n.nodeId === targetNodeId);
|
|
67401
67750
|
const deviceCaps = await this.inferenceDeviceCaps.get(targetNodeId);
|
|
67402
67751
|
const pinnedDeviceKey = await this.deps.readPipelineDevicePin(deviceId);
|
|
67752
|
+
this.noteStalePin(deviceId, pinnedDeviceKey, enabledDevices);
|
|
67403
67753
|
const selection = pickInferenceDevice({
|
|
67404
67754
|
eligible: enabledDevices,
|
|
67405
67755
|
loads: targetLoad?.devices ?? [],
|
|
67406
67756
|
caps: deviceCaps,
|
|
67407
|
-
pinnedDeviceKey,
|
|
67408
|
-
rotation: this.deps.inferenceRotation
|
|
67757
|
+
pinnedDeviceKey: pinnedDeviceKey !== null && pinnedDeviceKey in enabledDevices ? pinnedDeviceKey : null,
|
|
67758
|
+
rotation: this.deps.inferenceRotation,
|
|
67759
|
+
policy
|
|
67409
67760
|
});
|
|
67410
67761
|
const deviceKey = selectedDeviceKey(selection);
|
|
67411
67762
|
log.info("inference device selection", {
|
|
@@ -67493,6 +67844,7 @@ var SessionDispatchController = class {
|
|
|
67493
67844
|
attachedAt = Date.now();
|
|
67494
67845
|
this.deps.recordAssignment(deviceId, targetNodeId, decision.reason, decision.reason === "manual");
|
|
67495
67846
|
this.sessionRegistry.register(deviceId, targetNodeId, Date.now(), triggeredBy, deviceKey);
|
|
67847
|
+
this.deps.placementStats?.noteStart(targetNodeId, deviceKey ?? "");
|
|
67496
67848
|
if (this.sessionRefusals.delete(deviceId)) log.info("detection session refusal cleared — a session opened", {
|
|
67497
67849
|
tags: {
|
|
67498
67850
|
deviceId,
|
|
@@ -68762,6 +69114,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68762
69114
|
}, 6e4);
|
|
68763
69115
|
nodeStress.start();
|
|
68764
69116
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69117
|
+
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
68765
69118
|
/**
|
|
68766
69119
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
68767
69120
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -68845,6 +69198,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68845
69198
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
68846
69199
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
68847
69200
|
inferenceRotation,
|
|
69201
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69202
|
+
placementStats: deps.placementStats,
|
|
68848
69203
|
logger: deps.ctx().logger
|
|
68849
69204
|
});
|
|
68850
69205
|
const deviceConfig = new DeviceConfigContributions({
|
|
@@ -69378,7 +69733,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69378
69733
|
unsubAgentOffline,
|
|
69379
69734
|
unsubOrchestratorSubscriptions,
|
|
69380
69735
|
pendingRetryTimer,
|
|
69381
|
-
autoRebalanceTimer
|
|
69736
|
+
autoRebalanceTimer,
|
|
69737
|
+
sessionPlacement: () => readSessionPlacement()
|
|
69382
69738
|
};
|
|
69383
69739
|
}
|
|
69384
69740
|
//#endregion
|
|
@@ -69423,6 +69779,20 @@ function buildGlobalSettingsSections(options) {
|
|
|
69423
69779
|
title: "Load Balancer",
|
|
69424
69780
|
tab: "pipeline",
|
|
69425
69781
|
fields: [
|
|
69782
|
+
{
|
|
69783
|
+
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69784
|
+
type: "select",
|
|
69785
|
+
label: "Session placement",
|
|
69786
|
+
description: "How a new detection session picks its node and device. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy. Honour shares = the share of PARALLEL sessions is held to the node and device weights, and a node at its cap yields to a free one. Moving a weight slider never changes this policy.",
|
|
69787
|
+
options: [{
|
|
69788
|
+
value: "least-loaded",
|
|
69789
|
+
label: "Least-loaded"
|
|
69790
|
+
}, {
|
|
69791
|
+
value: "share",
|
|
69792
|
+
label: "Honour shares"
|
|
69793
|
+
}],
|
|
69794
|
+
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69795
|
+
},
|
|
69426
69796
|
{
|
|
69427
69797
|
key: "fpsMinThreshold",
|
|
69428
69798
|
type: "slider",
|
|
@@ -70002,7 +70372,7 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70002
70372
|
* settings: the attach provisions the node pin over the snapshot
|
|
70003
70373
|
* (`applyDeviceProvisioning`: override ?? base), so once the snapshot pin
|
|
70004
70374
|
* is gone the node's recommended model runs.
|
|
70005
|
-
* - never: `stepToggles`,
|
|
70375
|
+
* - never: `stepToggles`, devices' enablement, pins of
|
|
70006
70376
|
* embedding steps (cluster-scoped, changing one forces a re-index).
|
|
70007
70377
|
*/
|
|
70008
70378
|
/** The root detector step. */
|
|
@@ -70264,6 +70634,29 @@ function boundKeysOf(catalogs) {
|
|
|
70264
70634
|
}
|
|
70265
70635
|
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70266
70636
|
}
|
|
70637
|
+
/** Bounded fixed point of the hardware share suggestion (D748). */
|
|
70638
|
+
var SUGGEST_MAX_ROUNDS = 3;
|
|
70639
|
+
/** Integer shares summing to 100 (largest remainder). */
|
|
70640
|
+
function toPercent(weights) {
|
|
70641
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70642
|
+
const out = /* @__PURE__ */ new Map();
|
|
70643
|
+
if (total <= 0) return out;
|
|
70644
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
70645
|
+
k,
|
|
70646
|
+
exact: w / total * 100
|
|
70647
|
+
}));
|
|
70648
|
+
let assigned = 0;
|
|
70649
|
+
for (const r of raw) {
|
|
70650
|
+
out.set(r.k, Math.floor(r.exact));
|
|
70651
|
+
assigned += Math.floor(r.exact);
|
|
70652
|
+
}
|
|
70653
|
+
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70654
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
70655
|
+
const r = byRemainder[i % byRemainder.length];
|
|
70656
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70657
|
+
}
|
|
70658
|
+
return out;
|
|
70659
|
+
}
|
|
70267
70660
|
var BestSettingsService = class {
|
|
70268
70661
|
deps;
|
|
70269
70662
|
applying = false;
|
|
@@ -70271,9 +70664,87 @@ var BestSettingsService = class {
|
|
|
70271
70664
|
this.deps = deps;
|
|
70272
70665
|
}
|
|
70273
70666
|
async plan(input) {
|
|
70274
|
-
return (await this.computePlan(input.headroom)).plan;
|
|
70667
|
+
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70668
|
+
}
|
|
70669
|
+
/**
|
|
70670
|
+
* Shares proportional to each node's capacity at the models Best settings
|
|
70671
|
+
* would pick under those very shares (D748). Model and load depend on each
|
|
70672
|
+
* other, so it iterates from equal shares until the rounded shares stop
|
|
70673
|
+
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
70674
|
+
* ADVISORY session cap — never written, never enforced.
|
|
70675
|
+
*/
|
|
70676
|
+
async suggestShares(input) {
|
|
70677
|
+
let shares = {
|
|
70678
|
+
nodes: {},
|
|
70679
|
+
devices: {}
|
|
70680
|
+
};
|
|
70681
|
+
let result = [];
|
|
70682
|
+
let cameraFps = 10;
|
|
70683
|
+
let rounds = 0;
|
|
70684
|
+
for (let round = 1; round <= SUGGEST_MAX_ROUNDS; round++) {
|
|
70685
|
+
rounds = round;
|
|
70686
|
+
const rec = await this.recommend(input.headroom, shares);
|
|
70687
|
+
cameraFps = rec.cameraFps;
|
|
70688
|
+
result = this.sharesFrom(rec, input.headroom);
|
|
70689
|
+
const next = {
|
|
70690
|
+
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70691
|
+
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70692
|
+
};
|
|
70693
|
+
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70694
|
+
shares = next;
|
|
70695
|
+
if (settled) break;
|
|
70696
|
+
}
|
|
70697
|
+
return {
|
|
70698
|
+
headroom: input.headroom,
|
|
70699
|
+
cameraFps,
|
|
70700
|
+
iterations: rounds,
|
|
70701
|
+
nodes: [...result]
|
|
70702
|
+
};
|
|
70275
70703
|
}
|
|
70276
|
-
|
|
70704
|
+
sharesFrom(rec, headroom) {
|
|
70705
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70706
|
+
const perNode = rec.recommendations.map((r) => {
|
|
70707
|
+
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70708
|
+
const fps = d.chosen?.fps ?? 0;
|
|
70709
|
+
return {
|
|
70710
|
+
d,
|
|
70711
|
+
fps,
|
|
70712
|
+
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70713
|
+
};
|
|
70714
|
+
});
|
|
70715
|
+
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70716
|
+
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70717
|
+
return {
|
|
70718
|
+
r,
|
|
70719
|
+
devices,
|
|
70720
|
+
capacity
|
|
70721
|
+
};
|
|
70722
|
+
});
|
|
70723
|
+
const nodePct = toPercent(capacityByNode);
|
|
70724
|
+
return perNode.map(({ r, devices, capacity }) => {
|
|
70725
|
+
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70726
|
+
return {
|
|
70727
|
+
nodeId: r.nodeId,
|
|
70728
|
+
share: nodePct.get(r.nodeId) ?? null,
|
|
70729
|
+
capacitySessions: capacity,
|
|
70730
|
+
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70731
|
+
devices: devices.map((x) => ({
|
|
70732
|
+
deviceKey: x.d.deviceKey,
|
|
70733
|
+
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70734
|
+
modelId: x.d.chosen?.modelId ?? null,
|
|
70735
|
+
fps: x.d.chosen?.fps ?? null,
|
|
70736
|
+
measured: x.d.chosen?.measured ?? false,
|
|
70737
|
+
suggestedMaxSessions: x.sessions
|
|
70738
|
+
}))
|
|
70739
|
+
};
|
|
70740
|
+
});
|
|
70741
|
+
}
|
|
70742
|
+
/**
|
|
70743
|
+
* Gather every node and recommend. `shares` (a what-if or the suggestion's
|
|
70744
|
+
* round) forces share placement with those shares overlaid on the stored
|
|
70745
|
+
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70746
|
+
*/
|
|
70747
|
+
async recommend(headroom, shares) {
|
|
70277
70748
|
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70278
70749
|
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70279
70750
|
const [devices, hardware, catalog] = await Promise.all([
|
|
@@ -70288,6 +70759,7 @@ var BestSettingsService = class {
|
|
|
70288
70759
|
catalog
|
|
70289
70760
|
};
|
|
70290
70761
|
}));
|
|
70762
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? stored;
|
|
70291
70763
|
const nodes = gathered.map((g) => ({
|
|
70292
70764
|
nodeId: g.nodeId,
|
|
70293
70765
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70303,18 +70775,37 @@ var BestSettingsService = class {
|
|
|
70303
70775
|
const eligible = gathered.flatMap((g) => g.devices.reachable ? g.devices.devices.filter((d) => d.enabled && d.available).map((d) => ({
|
|
70304
70776
|
nodeId: g.nodeId,
|
|
70305
70777
|
deviceKey: d.key,
|
|
70306
|
-
weight: d.weight
|
|
70778
|
+
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70307
70779
|
})) : []);
|
|
70780
|
+
const cameras = await this.deps.cameraLoads();
|
|
70781
|
+
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70308
70782
|
const demand = estimateDeviceDemand({
|
|
70309
|
-
cameras
|
|
70783
|
+
cameras,
|
|
70310
70784
|
eligible,
|
|
70311
|
-
peaks: this.deps.peaks()
|
|
70785
|
+
peaks: this.deps.peaks(),
|
|
70786
|
+
...policy === "share" ? { shares: {
|
|
70787
|
+
nodes: {
|
|
70788
|
+
...await this.deps.readNodeShares(),
|
|
70789
|
+
...shares?.nodes
|
|
70790
|
+
},
|
|
70791
|
+
clusterPeak: this.deps.clusterPeak()
|
|
70792
|
+
} } : {}
|
|
70312
70793
|
});
|
|
70313
70794
|
const recommendations = nodes.map((node) => recommendNode({
|
|
70314
70795
|
node,
|
|
70315
70796
|
demand,
|
|
70316
70797
|
headroom
|
|
70317
70798
|
}));
|
|
70799
|
+
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
70800
|
+
return {
|
|
70801
|
+
gathered,
|
|
70802
|
+
nodes,
|
|
70803
|
+
recommendations,
|
|
70804
|
+
cameraFps: cameraFps > 0 ? cameraFps : 10
|
|
70805
|
+
};
|
|
70806
|
+
}
|
|
70807
|
+
async computePlan(headroom, shares) {
|
|
70808
|
+
const { gathered, nodes, recommendations } = await this.recommend(headroom, shares);
|
|
70318
70809
|
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70319
70810
|
const targets = /* @__PURE__ */ new Map();
|
|
70320
70811
|
const formats = /* @__PURE__ */ new Map();
|
|
@@ -70376,6 +70867,7 @@ var BestSettingsService = class {
|
|
|
70376
70867
|
planHash: (0, node_crypto.createHash)("sha256").update(canonical({
|
|
70377
70868
|
version: 1,
|
|
70378
70869
|
headroom,
|
|
70870
|
+
whatIf: shares ?? null,
|
|
70379
70871
|
picks: recommendations.map((r) => ({
|
|
70380
70872
|
nodeId: r.nodeId,
|
|
70381
70873
|
skipped: r.skipped,
|
|
@@ -70387,6 +70879,7 @@ var BestSettingsService = class {
|
|
|
70387
70879
|
})).digest("hex"),
|
|
70388
70880
|
family: "yolov9",
|
|
70389
70881
|
headroom,
|
|
70882
|
+
whatIf: shares !== void 0,
|
|
70390
70883
|
generatedAt: this.deps.now(),
|
|
70391
70884
|
nodes: recommendations,
|
|
70392
70885
|
nodeChanges: [...nodeChanges],
|
|
@@ -70559,6 +71052,7 @@ var BUCKET_MS = 3600 * 1e3;
|
|
|
70559
71052
|
var DevicePeakTracker = class DevicePeakTracker {
|
|
70560
71053
|
now;
|
|
70561
71054
|
series = /* @__PURE__ */ new Map();
|
|
71055
|
+
cluster = null;
|
|
70562
71056
|
constructor(now = () => Date.now()) {
|
|
70563
71057
|
this.now = now;
|
|
70564
71058
|
}
|
|
@@ -70569,12 +71063,14 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70569
71063
|
noteCensus(loads) {
|
|
70570
71064
|
const now = this.now();
|
|
70571
71065
|
const bucket = now - now % BUCKET_MS;
|
|
71066
|
+
let clusterTotal = 0;
|
|
70572
71067
|
for (const load of loads) {
|
|
70573
71068
|
const rows = load.devices.length > 0 ? load.devices : [{
|
|
70574
71069
|
deviceKey: "",
|
|
70575
71070
|
attachedCameras: load.attachedCameras
|
|
70576
71071
|
}];
|
|
70577
71072
|
for (const device of rows) {
|
|
71073
|
+
clusterTotal += device.attachedCameras;
|
|
70578
71074
|
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
70579
71075
|
let s = this.series.get(key);
|
|
70580
71076
|
if (s === void 0) {
|
|
@@ -70588,6 +71084,26 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70588
71084
|
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
70589
71085
|
}
|
|
70590
71086
|
}
|
|
71087
|
+
if (loads.length === 0) return;
|
|
71088
|
+
const c = this.cluster ?? {
|
|
71089
|
+
firstSeenAt: now,
|
|
71090
|
+
buckets: /* @__PURE__ */ new Map()
|
|
71091
|
+
};
|
|
71092
|
+
this.cluster = c;
|
|
71093
|
+
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71094
|
+
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71095
|
+
}
|
|
71096
|
+
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71097
|
+
clusterPeak() {
|
|
71098
|
+
const c = this.cluster;
|
|
71099
|
+
if (c === null) return null;
|
|
71100
|
+
const now = this.now();
|
|
71101
|
+
let peak = 0;
|
|
71102
|
+
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71103
|
+
return {
|
|
71104
|
+
peakSessions: peak,
|
|
71105
|
+
observedForMs: now - c.firstSeenAt
|
|
71106
|
+
};
|
|
70591
71107
|
}
|
|
70592
71108
|
/** Every observed device's peak over the window and how long it has been watched. */
|
|
70593
71109
|
peaks() {
|
|
@@ -70608,6 +71124,186 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70608
71124
|
}
|
|
70609
71125
|
};
|
|
70610
71126
|
//#endregion
|
|
71127
|
+
//#region src/session-placement-stats.ts
|
|
71128
|
+
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71129
|
+
function keyOf(nodeId, deviceKey) {
|
|
71130
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
71131
|
+
}
|
|
71132
|
+
function split(key) {
|
|
71133
|
+
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71134
|
+
return {
|
|
71135
|
+
nodeId,
|
|
71136
|
+
deviceKey
|
|
71137
|
+
};
|
|
71138
|
+
}
|
|
71139
|
+
var SessionPlacementStats = class {
|
|
71140
|
+
now;
|
|
71141
|
+
segments = [];
|
|
71142
|
+
starts = [];
|
|
71143
|
+
yields = [];
|
|
71144
|
+
startedAt;
|
|
71145
|
+
constructor(now = () => Date.now()) {
|
|
71146
|
+
this.now = now;
|
|
71147
|
+
this.startedAt = now();
|
|
71148
|
+
}
|
|
71149
|
+
/** The running-session set changed: open a new segment with these counts. */
|
|
71150
|
+
noteSessions(sessions) {
|
|
71151
|
+
const counts = /* @__PURE__ */ new Map();
|
|
71152
|
+
for (const s of sessions) {
|
|
71153
|
+
const k = keyOf(s.nodeId, s.deviceKey);
|
|
71154
|
+
counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
71155
|
+
}
|
|
71156
|
+
this.segments.push({
|
|
71157
|
+
at: this.now(),
|
|
71158
|
+
counts
|
|
71159
|
+
});
|
|
71160
|
+
this.prune();
|
|
71161
|
+
}
|
|
71162
|
+
noteStart(nodeId, deviceKey) {
|
|
71163
|
+
this.starts.push({
|
|
71164
|
+
at: this.now(),
|
|
71165
|
+
nodeId,
|
|
71166
|
+
deviceKey
|
|
71167
|
+
});
|
|
71168
|
+
this.prune();
|
|
71169
|
+
}
|
|
71170
|
+
/** A share election handed `nodeId`'s turn to another node. */
|
|
71171
|
+
noteYield(nodeId) {
|
|
71172
|
+
this.yields.push({
|
|
71173
|
+
at: this.now(),
|
|
71174
|
+
nodeId
|
|
71175
|
+
});
|
|
71176
|
+
this.prune();
|
|
71177
|
+
}
|
|
71178
|
+
/** For tests: how many segments are retained. */
|
|
71179
|
+
segmentCount() {
|
|
71180
|
+
return this.segments.length;
|
|
71181
|
+
}
|
|
71182
|
+
prune() {
|
|
71183
|
+
const floor = this.now() - RETAIN_MS;
|
|
71184
|
+
while (this.segments.length > 1 && this.segments[1].at <= floor) this.segments.shift();
|
|
71185
|
+
while (this.starts.length > 0 && this.starts[0].at < floor) this.starts.shift();
|
|
71186
|
+
while (this.yields.length > 0 && this.yields[0].at < floor) this.yields.shift();
|
|
71187
|
+
}
|
|
71188
|
+
snapshot(windowMs) {
|
|
71189
|
+
const now = this.now();
|
|
71190
|
+
const from = now - windowMs;
|
|
71191
|
+
const integral = /* @__PURE__ */ new Map();
|
|
71192
|
+
let clusterPeak = 0;
|
|
71193
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71194
|
+
const seg = this.segments[i];
|
|
71195
|
+
const end = this.segments[i + 1]?.at ?? now;
|
|
71196
|
+
const start = Math.max(seg.at, from);
|
|
71197
|
+
if (end <= start) continue;
|
|
71198
|
+
let total = 0;
|
|
71199
|
+
for (const [k, n] of seg.counts) {
|
|
71200
|
+
integral.set(k, (integral.get(k) ?? 0) + n * (end - start));
|
|
71201
|
+
total += n;
|
|
71202
|
+
}
|
|
71203
|
+
clusterPeak = Math.max(clusterPeak, total);
|
|
71204
|
+
}
|
|
71205
|
+
const current = this.segments.at(-1)?.counts ?? /* @__PURE__ */ new Map();
|
|
71206
|
+
const nodeIds = /* @__PURE__ */ new Set();
|
|
71207
|
+
for (const k of [...integral.keys(), ...current.keys()]) nodeIds.add(split(k).nodeId);
|
|
71208
|
+
for (const s of this.starts) if (s.at >= from) nodeIds.add(s.nodeId);
|
|
71209
|
+
const clusterIntegral = [...integral.values()].reduce((a, b) => a + b, 0);
|
|
71210
|
+
const nodes = [...nodeIds].toSorted().map((nodeId) => {
|
|
71211
|
+
const deviceKeys = /* @__PURE__ */ new Set();
|
|
71212
|
+
for (const k of [...integral.keys(), ...current.keys()]) {
|
|
71213
|
+
const s = split(k);
|
|
71214
|
+
if (s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71215
|
+
}
|
|
71216
|
+
for (const s of this.starts) if (s.at >= from && s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71217
|
+
const nodeIntegral = [...deviceKeys].reduce((sum, d) => sum + (integral.get(keyOf(nodeId, d)) ?? 0), 0);
|
|
71218
|
+
const devices = [...deviceKeys].toSorted().map((deviceKey) => ({
|
|
71219
|
+
deviceKey,
|
|
71220
|
+
sessionsNow: current.get(keyOf(nodeId, deviceKey)) ?? 0,
|
|
71221
|
+
sessionsStarted: this.starts.filter((s) => s.at >= from && s.nodeId === nodeId && s.deviceKey === deviceKey).length,
|
|
71222
|
+
observedShare: nodeIntegral > 0 ? (integral.get(keyOf(nodeId, deviceKey)) ?? 0) / nodeIntegral : null
|
|
71223
|
+
}));
|
|
71224
|
+
let peak = 0;
|
|
71225
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71226
|
+
const seg = this.segments[i];
|
|
71227
|
+
if ((this.segments[i + 1]?.at ?? now) <= Math.max(seg.at, from)) continue;
|
|
71228
|
+
let n = 0;
|
|
71229
|
+
for (const [k, c] of seg.counts) if (split(k).nodeId === nodeId) n += c;
|
|
71230
|
+
peak = Math.max(peak, n);
|
|
71231
|
+
}
|
|
71232
|
+
return {
|
|
71233
|
+
nodeId,
|
|
71234
|
+
sessionsNow: devices.reduce((s, d) => s + d.sessionsNow, 0),
|
|
71235
|
+
sessionsStarted: devices.reduce((s, d) => s + d.sessionsStarted, 0),
|
|
71236
|
+
observedShare: clusterIntegral > 0 && nodeIntegral > 0 ? nodeIntegral / clusterIntegral : null,
|
|
71237
|
+
peakSessions: peak,
|
|
71238
|
+
yieldsAway: this.yields.filter((y) => y.at >= from && y.nodeId === nodeId).length,
|
|
71239
|
+
devices
|
|
71240
|
+
};
|
|
71241
|
+
});
|
|
71242
|
+
return {
|
|
71243
|
+
observedForMs: Math.min(windowMs, now - this.startedAt),
|
|
71244
|
+
sessionsNow: [...current.values()].reduce((a, b) => a + b, 0),
|
|
71245
|
+
clusterPeakSessions: clusterPeak,
|
|
71246
|
+
nodes
|
|
71247
|
+
};
|
|
71248
|
+
}
|
|
71249
|
+
};
|
|
71250
|
+
//#endregion
|
|
71251
|
+
//#region src/load-balancing-shares.ts
|
|
71252
|
+
function nextDevices(stored, shares, effective) {
|
|
71253
|
+
let changed = false;
|
|
71254
|
+
const out = { ...stored };
|
|
71255
|
+
for (const [key, weight] of Object.entries(shares)) {
|
|
71256
|
+
const existing = stored?.[key];
|
|
71257
|
+
if (existing !== void 0) {
|
|
71258
|
+
out[key] = {
|
|
71259
|
+
...existing,
|
|
71260
|
+
weight
|
|
71261
|
+
};
|
|
71262
|
+
changed = true;
|
|
71263
|
+
continue;
|
|
71264
|
+
}
|
|
71265
|
+
const enabled = effective[key];
|
|
71266
|
+
if (enabled === void 0) continue;
|
|
71267
|
+
out[key] = {
|
|
71268
|
+
enabled,
|
|
71269
|
+
weight
|
|
71270
|
+
};
|
|
71271
|
+
changed = true;
|
|
71272
|
+
}
|
|
71273
|
+
return changed ? out : stored;
|
|
71274
|
+
}
|
|
71275
|
+
function applyLoadBalancingShares(prev, write) {
|
|
71276
|
+
const next = { ...prev };
|
|
71277
|
+
const nodeIds = new Set([...Object.keys(write.nodes), ...Object.keys(write.devices)]);
|
|
71278
|
+
for (const nodeId of nodeIds) {
|
|
71279
|
+
const existing = prev[nodeId];
|
|
71280
|
+
const nodeShare = write.nodes[nodeId];
|
|
71281
|
+
const devices = nextDevices(existing?.inferenceDevices, write.devices[nodeId] ?? {}, write.effectiveEnabled[nodeId] ?? {});
|
|
71282
|
+
if (nodeShare === void 0 && devices === existing?.inferenceDevices) continue;
|
|
71283
|
+
next[nodeId] = {
|
|
71284
|
+
...existing ?? { maxCameras: null },
|
|
71285
|
+
...nodeShare !== void 0 ? { detectWeight: nodeShare } : {},
|
|
71286
|
+
...devices !== void 0 ? { inferenceDevices: devices } : {}
|
|
71287
|
+
};
|
|
71288
|
+
}
|
|
71289
|
+
return next;
|
|
71290
|
+
}
|
|
71291
|
+
//#endregion
|
|
71292
|
+
//#region src/stale-device-pins.ts
|
|
71293
|
+
function findStaleDevicePins(pins, nodes) {
|
|
71294
|
+
const usable = /* @__PURE__ */ new Map();
|
|
71295
|
+
for (const node of nodes) for (const d of node.devices) usable.set(d.key, (usable.get(d.key) ?? false) || d.enabled && d.available);
|
|
71296
|
+
return pins.flatMap((pin) => {
|
|
71297
|
+
const state = usable.get(pin.deviceKey);
|
|
71298
|
+
if (state === true) return [];
|
|
71299
|
+
return [{
|
|
71300
|
+
deviceId: pin.deviceId,
|
|
71301
|
+
deviceKey: pin.deviceKey,
|
|
71302
|
+
reason: state === false ? "disabled" : "absent"
|
|
71303
|
+
}];
|
|
71304
|
+
}).toSorted((a, b) => a.deviceId - b.deviceId);
|
|
71305
|
+
}
|
|
71306
|
+
//#endregion
|
|
70611
71307
|
//#region src/index.ts
|
|
70612
71308
|
/**
|
|
70613
71309
|
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
@@ -70677,6 +71373,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70677
71373
|
clusterModelChoiceReady = false;
|
|
70678
71374
|
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
70679
71375
|
devicePeaks = new DevicePeakTracker();
|
|
71376
|
+
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71377
|
+
placementStats = new SessionPlacementStats();
|
|
71378
|
+
/** The stored session placement policy (D748); `least-loaded` before init. */
|
|
71379
|
+
readSessionPlacement = () => "least-loaded";
|
|
70680
71380
|
bestSettingsService = null;
|
|
70681
71381
|
/**
|
|
70682
71382
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
@@ -70955,6 +71655,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70955
71655
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
70956
71656
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
70957
71657
|
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
71658
|
+
placementStats: this.placementStats,
|
|
70958
71659
|
setTopology: (topology) => {
|
|
70959
71660
|
this.topology = topology;
|
|
70960
71661
|
},
|
|
@@ -70981,6 +71682,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70981
71682
|
this.nodeStress = controllers.nodeStress;
|
|
70982
71683
|
this.placement = controllers.placement;
|
|
70983
71684
|
this.session = controllers.session;
|
|
71685
|
+
this.readSessionPlacement = controllers.sessionPlacement;
|
|
70984
71686
|
this.deviceConfig = controllers.deviceConfig;
|
|
70985
71687
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
70986
71688
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71473,26 +72175,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71473
72175
|
this.reconcile.schedulePendingRetry();
|
|
71474
72176
|
return { success: true };
|
|
71475
72177
|
}
|
|
71476
|
-
async setAgentDetectWeight(input) {
|
|
71477
|
-
const existing = (await this.settingsStore.readAgentSettingsMap())[input.agentNodeId];
|
|
71478
|
-
const weight = input.detectWeight ?? 1;
|
|
71479
|
-
const next = existing ? {
|
|
71480
|
-
...existing,
|
|
71481
|
-
detectWeight: weight
|
|
71482
|
-
} : {
|
|
71483
|
-
maxCameras: null,
|
|
71484
|
-
detectWeight: weight
|
|
71485
|
-
};
|
|
71486
|
-
await this.settingsStore.writeAgentSettings(input.agentNodeId, next);
|
|
71487
|
-
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
71488
|
-
tags: { nodeId: input.agentNodeId },
|
|
71489
|
-
meta: { detectWeight: weight ?? null }
|
|
71490
|
-
});
|
|
71491
|
-
this.rebalance().catch((err) => {
|
|
71492
|
-
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
71493
|
-
});
|
|
71494
|
-
return { success: true };
|
|
71495
|
-
}
|
|
71496
72178
|
/**
|
|
71497
72179
|
* Set one node's placement capabilities — the per-node record that replaced
|
|
71498
72180
|
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
@@ -72090,6 +72772,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72090
72772
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72091
72773
|
cameraLoads: () => this.bestSettingsCameraLoads(),
|
|
72092
72774
|
peaks: () => this.devicePeaks.peaks(),
|
|
72775
|
+
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72776
|
+
sessionPlacement: () => this.readSessionPlacement(),
|
|
72777
|
+
readNodeShares: () => store.buildNodeWeights(),
|
|
72093
72778
|
detectorStore: async (input) => {
|
|
72094
72779
|
const api = this.ctx.api;
|
|
72095
72780
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72117,19 +72802,20 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72117
72802
|
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72118
72803
|
async bestSettingsCameraLoads() {
|
|
72119
72804
|
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72120
|
-
const cameraMap = await this.settingsStore.readCameraSettingsMap();
|
|
72121
72805
|
const loads = [];
|
|
72122
72806
|
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
72123
|
-
const
|
|
72807
|
+
const nodePin = await this.readPipelinePin(deviceId);
|
|
72124
72808
|
const pinnedKey = await this.readPipelineDevicePin(deviceId);
|
|
72809
|
+
const standingNode = config.detectionMode === "always-on" ? standing.get(deviceId) ?? null : null;
|
|
72125
72810
|
loads.push({
|
|
72126
72811
|
deviceId,
|
|
72127
72812
|
mode: config.detectionMode,
|
|
72128
72813
|
fps: config.detectionFps,
|
|
72129
72814
|
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72130
|
-
standingNodeId:
|
|
72131
|
-
|
|
72132
|
-
|
|
72815
|
+
standingNodeId: standingNode,
|
|
72816
|
+
nodePinId: nodePin,
|
|
72817
|
+
devicePin: pinnedKey !== null ? {
|
|
72818
|
+
nodeId: nodePin ?? standingNode,
|
|
72133
72819
|
deviceKey: pinnedKey
|
|
72134
72820
|
} : null
|
|
72135
72821
|
});
|
|
@@ -72139,6 +72825,62 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72139
72825
|
async planBestSettings(input) {
|
|
72140
72826
|
return this.bestSettings().plan(input);
|
|
72141
72827
|
}
|
|
72828
|
+
/**
|
|
72829
|
+
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
72830
|
+
* device with no stored entry is made explicit with its effective
|
|
72831
|
+
* enablement, read from the probe-aware device list. Never touches the
|
|
72832
|
+
* `sessionPlacement` policy.
|
|
72833
|
+
*/
|
|
72834
|
+
async setLoadBalancingShares(input) {
|
|
72835
|
+
const effectiveEnabled = {};
|
|
72836
|
+
for (const nodeId of Object.keys(input.devices)) {
|
|
72837
|
+
const read = await this.getNodeInferenceDevices({ nodeId });
|
|
72838
|
+
effectiveEnabled[nodeId] = Object.fromEntries(read.devices.map((d) => [d.key, d.enabled]));
|
|
72839
|
+
}
|
|
72840
|
+
await this.settingsStore.rewriteAgentSettingsMap((prev) => applyLoadBalancingShares(prev, {
|
|
72841
|
+
nodes: input.nodes,
|
|
72842
|
+
devices: input.devices,
|
|
72843
|
+
effectiveEnabled
|
|
72844
|
+
}));
|
|
72845
|
+
this.ctx.logger.info("load balancing shares updated", { meta: {
|
|
72846
|
+
nodes: JSON.stringify(input.nodes),
|
|
72847
|
+
devices: JSON.stringify(input.devices),
|
|
72848
|
+
policy: this.readSessionPlacement()
|
|
72849
|
+
} });
|
|
72850
|
+
if (Object.keys(input.nodes).length > 0) this.rebalance().catch((err) => {
|
|
72851
|
+
this.ctx.logger.warn("setLoadBalancingShares: rebalance failed", { meta: { error: errMsg$2(err) } });
|
|
72852
|
+
});
|
|
72853
|
+
return { success: true };
|
|
72854
|
+
}
|
|
72855
|
+
async getPlacementStats(input) {
|
|
72856
|
+
const snap = this.placementStats.snapshot(input.windowMs);
|
|
72857
|
+
return {
|
|
72858
|
+
policy: this.readSessionPlacement(),
|
|
72859
|
+
windowMs: input.windowMs,
|
|
72860
|
+
observedForMs: snap.observedForMs,
|
|
72861
|
+
sessionsNow: snap.sessionsNow,
|
|
72862
|
+
clusterPeakSessions: snap.clusterPeakSessions,
|
|
72863
|
+
nodes: snap.nodes,
|
|
72864
|
+
stalePins: await this.listStaleDevicePins()
|
|
72865
|
+
};
|
|
72866
|
+
}
|
|
72867
|
+
/** Device pins the session path cannot honour — see `stale-device-pins.ts`. */
|
|
72868
|
+
async listStaleDevicePins() {
|
|
72869
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])];
|
|
72870
|
+
const offered = await Promise.all(nodeIds.map(async (nodeId) => ({ devices: (await this.getNodeInferenceDevices({ nodeId }).catch(() => null))?.devices ?? [] })));
|
|
72871
|
+
const pins = [];
|
|
72872
|
+
for (const deviceId of this.detectionWiring.activeDetections.keys()) {
|
|
72873
|
+
const deviceKey = await this.readPipelineDevicePin(deviceId);
|
|
72874
|
+
if (deviceKey !== null) pins.push({
|
|
72875
|
+
deviceId,
|
|
72876
|
+
deviceKey
|
|
72877
|
+
});
|
|
72878
|
+
}
|
|
72879
|
+
return findStaleDevicePins(pins, offered);
|
|
72880
|
+
}
|
|
72881
|
+
async suggestShares(input) {
|
|
72882
|
+
return this.bestSettings().suggestShares(input);
|
|
72883
|
+
}
|
|
72142
72884
|
async applyBestSettings(input) {
|
|
72143
72885
|
return this.bestSettings().apply(input);
|
|
72144
72886
|
}
|