@camstack/addon-pipeline-orchestrator 1.2.290 → 1.2.291
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DHY2k_nC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CdkZH5kw.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs} +1 -1
- package/dist/{hostInit-CRtAczui.mjs → hostInit-BpNmAa0Q.mjs} +2 -2
- package/dist/index.js +813 -71
- package/dist/index.mjs +813 -71
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DWLgJB4I.mjs +0 -26
package/dist/index.mjs
CHANGED
|
@@ -25005,6 +25005,105 @@ var pipelineExecutorCapability = {
|
|
|
25005
25005
|
}
|
|
25006
25006
|
};
|
|
25007
25007
|
/**
|
|
25008
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25009
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25010
|
+
* hardware share suggestion and the stale device pins.
|
|
25011
|
+
*/
|
|
25012
|
+
/**
|
|
25013
|
+
* How a new detection session picks its node and device.
|
|
25014
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25015
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25016
|
+
* weight only bites once a node already holds a session.
|
|
25017
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25018
|
+
* full or drowning node yields to a free one (D748).
|
|
25019
|
+
*/
|
|
25020
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25021
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "least-loaded";
|
|
25022
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25023
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25024
|
+
/** Parse a stored value; anything unknown is the default (nothing changes on deploy). */
|
|
25025
|
+
function parseSessionPlacementPolicy(value) {
|
|
25026
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25027
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25028
|
+
}
|
|
25029
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25030
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25031
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25032
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25033
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25034
|
+
});
|
|
25035
|
+
var PlacementStatsInputSchema = object({
|
|
25036
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25037
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25038
|
+
var PlacementDeviceStatsSchema = object({
|
|
25039
|
+
deviceKey: string(),
|
|
25040
|
+
/** Sessions running on this device now. */
|
|
25041
|
+
sessionsNow: number(),
|
|
25042
|
+
/** Sessions started on this device inside the window. */
|
|
25043
|
+
sessionsStarted: number(),
|
|
25044
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25045
|
+
observedShare: number().nullable()
|
|
25046
|
+
});
|
|
25047
|
+
var PlacementNodeStatsSchema = object({
|
|
25048
|
+
nodeId: string(),
|
|
25049
|
+
sessionsNow: number(),
|
|
25050
|
+
sessionsStarted: number(),
|
|
25051
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25052
|
+
observedShare: number().nullable(),
|
|
25053
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25054
|
+
peakSessions: number(),
|
|
25055
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25056
|
+
yieldsAway: number(),
|
|
25057
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25058
|
+
});
|
|
25059
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25060
|
+
var StaleDevicePinSchema = object({
|
|
25061
|
+
deviceId: number(),
|
|
25062
|
+
deviceKey: string(),
|
|
25063
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25064
|
+
reason: _enum(["disabled", "absent"])
|
|
25065
|
+
});
|
|
25066
|
+
var PlacementStatsSchema = object({
|
|
25067
|
+
policy: SessionPlacementPolicySchema,
|
|
25068
|
+
windowMs: number(),
|
|
25069
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25070
|
+
observedForMs: number(),
|
|
25071
|
+
sessionsNow: number(),
|
|
25072
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25073
|
+
clusterPeakSessions: number(),
|
|
25074
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25075
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25076
|
+
});
|
|
25077
|
+
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25078
|
+
var SuggestedDeviceShareSchema = object({
|
|
25079
|
+
deviceKey: string(),
|
|
25080
|
+
/** Share of the node (0–100, rounded). */
|
|
25081
|
+
share: number(),
|
|
25082
|
+
modelId: string().nullable(),
|
|
25083
|
+
/** Inferences per second at that model, from the measured table. */
|
|
25084
|
+
fps: number().nullable(),
|
|
25085
|
+
measured: boolean(),
|
|
25086
|
+
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25087
|
+
suggestedMaxSessions: number().nullable()
|
|
25088
|
+
});
|
|
25089
|
+
var SuggestedNodeShareSchema = object({
|
|
25090
|
+
nodeId: string(),
|
|
25091
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25092
|
+
share: number().nullable(),
|
|
25093
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25094
|
+
capacitySessions: number(),
|
|
25095
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25096
|
+
skippedReason: string().nullable()
|
|
25097
|
+
});
|
|
25098
|
+
var SuggestSharesResultSchema = object({
|
|
25099
|
+
headroom: number(),
|
|
25100
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25101
|
+
cameraFps: number(),
|
|
25102
|
+
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25103
|
+
iterations: number(),
|
|
25104
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25105
|
+
});
|
|
25106
|
+
/**
|
|
25008
25107
|
* The wire shapes of the Best settings plan and apply (D745), used by the
|
|
25009
25108
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25010
25109
|
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
@@ -25081,7 +25180,9 @@ var DeviceDemandSchema = object({
|
|
|
25081
25180
|
"estimate",
|
|
25082
25181
|
"none"
|
|
25083
25182
|
]),
|
|
25084
|
-
observedForMs: number().nullable()
|
|
25183
|
+
observedForMs: number().nullable(),
|
|
25184
|
+
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25185
|
+
placement: SessionPlacementPolicySchema
|
|
25085
25186
|
});
|
|
25086
25187
|
var DeviceModelChoiceSchema = object({
|
|
25087
25188
|
modelId: string(),
|
|
@@ -25153,14 +25254,24 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25153
25254
|
deviceKey: string()
|
|
25154
25255
|
})
|
|
25155
25256
|
]);
|
|
25156
|
-
var BestSettingsOptionsSchema = object({
|
|
25157
|
-
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25158
|
-
headroom: number().min(.3).max(.5).default(.4)
|
|
25257
|
+
var BestSettingsOptionsSchema = object({
|
|
25258
|
+
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25259
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25260
|
+
/**
|
|
25261
|
+
* What-if shares (D748): plan as if these node/device shares were stored
|
|
25262
|
+
* and the policy were `share`. Absent = the stored shares under the stored
|
|
25263
|
+
* policy. A what-if plan is for display; apply always re-plans from what
|
|
25264
|
+
* is stored.
|
|
25265
|
+
*/
|
|
25266
|
+
shares: LoadBalancingSharesInputSchema.optional()
|
|
25267
|
+
});
|
|
25159
25268
|
var BestSettingsPlanSchema = object({
|
|
25160
25269
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25161
25270
|
planHash: string(),
|
|
25162
25271
|
family: literal("yolov9"),
|
|
25163
25272
|
headroom: number(),
|
|
25273
|
+
/** Planned under what-if shares (D748): for display — apply refuses it as stale. */
|
|
25274
|
+
whatIf: boolean(),
|
|
25164
25275
|
generatedAt: number(),
|
|
25165
25276
|
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25166
25277
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
@@ -26610,7 +26721,6 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26610
26721
|
settings: record(string(), unknown()).readonly().optional()
|
|
26611
26722
|
});
|
|
26612
26723
|
var CameraPipelineSettingsSchema = object({
|
|
26613
|
-
pinnedAgentNodeId: string().optional(),
|
|
26614
26724
|
stepToggles: record(string(), boolean()).optional(),
|
|
26615
26725
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26616
26726
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
@@ -27258,18 +27368,6 @@ var pipelineOrchestratorCapability = {
|
|
|
27258
27368
|
auth: "admin"
|
|
27259
27369
|
}),
|
|
27260
27370
|
/**
|
|
27261
|
-
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
27262
|
-
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
27263
|
-
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
27264
|
-
*/
|
|
27265
|
-
setAgentDetectWeight: method(object({
|
|
27266
|
-
agentNodeId: string(),
|
|
27267
|
-
detectWeight: number().positive().nullable()
|
|
27268
|
-
}), object({ success: literal(true) }), {
|
|
27269
|
-
kind: "mutation",
|
|
27270
|
-
auth: "admin"
|
|
27271
|
-
}),
|
|
27272
|
-
/**
|
|
27273
27371
|
* Set one node's placement CAPABILITIES — the per-node record that
|
|
27274
27372
|
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
27275
27373
|
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
@@ -27409,6 +27507,33 @@ var pipelineOrchestratorCapability = {
|
|
|
27409
27507
|
kind: "mutation",
|
|
27410
27508
|
auth: "admin"
|
|
27411
27509
|
}),
|
|
27510
|
+
/**
|
|
27511
|
+
* Load balancing (D748) — write node shares (`detectWeight`) and per-node
|
|
27512
|
+
* device shares (inference-device `weight`) in ONE atomic read-modify-write
|
|
27513
|
+
* of the agent settings — the only writer of node shares (the per-node
|
|
27514
|
+
* `setAgentDetectWeight` it replaced raced its siblings, one RMW each).
|
|
27515
|
+
* A node or device left out keeps its weight. Never changes the
|
|
27516
|
+
* `sessionPlacement` policy.
|
|
27517
|
+
*/
|
|
27518
|
+
setLoadBalancingShares: method(LoadBalancingSharesInputSchema, object({ success: literal(true) }), {
|
|
27519
|
+
kind: "mutation",
|
|
27520
|
+
auth: "admin"
|
|
27521
|
+
}),
|
|
27522
|
+
/**
|
|
27523
|
+
* Load balancing (D748) — what placement actually did: per node and
|
|
27524
|
+
* device, sessions now, sessions started and the time-weighted share of
|
|
27525
|
+
* PARALLEL sessions over `windowMs`; the cluster's parallel peak; yields
|
|
27526
|
+
* a share election made; and every device pin the session path cannot
|
|
27527
|
+
* honour. In memory on the hub — a restart starts a new window.
|
|
27528
|
+
*/
|
|
27529
|
+
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27530
|
+
/**
|
|
27531
|
+
* Load balancing (D748) — shares proportional to each node's measured
|
|
27532
|
+
* capacity at the models Best settings would pick, iterated (≤ 3 rounds)
|
|
27533
|
+
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27534
|
+
* Writes nothing.
|
|
27535
|
+
*/
|
|
27536
|
+
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27412
27537
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27413
27538
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27414
27539
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -43588,6 +43713,12 @@ Object.freeze({
|
|
|
43588
43713
|
addonId: null,
|
|
43589
43714
|
access: "view"
|
|
43590
43715
|
},
|
|
43716
|
+
"pipelineOrchestrator.getPlacementStats": {
|
|
43717
|
+
capName: "pipeline-orchestrator",
|
|
43718
|
+
capScope: "system",
|
|
43719
|
+
addonId: null,
|
|
43720
|
+
access: "view"
|
|
43721
|
+
},
|
|
43591
43722
|
"pipelineOrchestrator.getReconcileFromDiskStatus": {
|
|
43592
43723
|
capName: "pipeline-orchestrator",
|
|
43593
43724
|
capScope: "system",
|
|
@@ -43678,12 +43809,6 @@ Object.freeze({
|
|
|
43678
43809
|
addonId: null,
|
|
43679
43810
|
access: "create"
|
|
43680
43811
|
},
|
|
43681
|
-
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
43682
|
-
capName: "pipeline-orchestrator",
|
|
43683
|
-
capScope: "system",
|
|
43684
|
-
addonId: null,
|
|
43685
|
-
access: "create"
|
|
43686
|
-
},
|
|
43687
43812
|
"pipelineOrchestrator.setAgentInferenceDevices": {
|
|
43688
43813
|
capName: "pipeline-orchestrator",
|
|
43689
43814
|
capScope: "system",
|
|
@@ -43738,12 +43863,24 @@ Object.freeze({
|
|
|
43738
43863
|
addonId: null,
|
|
43739
43864
|
access: "create"
|
|
43740
43865
|
},
|
|
43866
|
+
"pipelineOrchestrator.setLoadBalancingShares": {
|
|
43867
|
+
capName: "pipeline-orchestrator",
|
|
43868
|
+
capScope: "system",
|
|
43869
|
+
addonId: null,
|
|
43870
|
+
access: "create"
|
|
43871
|
+
},
|
|
43741
43872
|
"pipelineOrchestrator.setPipelineDevicePin": {
|
|
43742
43873
|
capName: "pipeline-orchestrator",
|
|
43743
43874
|
capScope: "system",
|
|
43744
43875
|
addonId: null,
|
|
43745
43876
|
access: "create"
|
|
43746
43877
|
},
|
|
43878
|
+
"pipelineOrchestrator.suggestShares": {
|
|
43879
|
+
capName: "pipeline-orchestrator",
|
|
43880
|
+
capScope: "system",
|
|
43881
|
+
addonId: null,
|
|
43882
|
+
access: "view"
|
|
43883
|
+
},
|
|
43747
43884
|
"pipelineOrchestrator.unassignAudio": {
|
|
43748
43885
|
capName: "pipeline-orchestrator",
|
|
43749
43886
|
capScope: "system",
|
|
@@ -51493,6 +51630,7 @@ function estimatedConcurrentSessions(cameraCount, options) {
|
|
|
51493
51630
|
*/
|
|
51494
51631
|
function estimateDeviceDemand(input) {
|
|
51495
51632
|
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51633
|
+
const placement = input.shares ? "share" : "least-loaded";
|
|
51496
51634
|
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51497
51635
|
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51498
51636
|
const weightByNode = /* @__PURE__ */ new Map();
|
|
@@ -51512,19 +51650,28 @@ function estimateDeviceDemand(input) {
|
|
|
51512
51650
|
const noteMax = (key, fps) => {
|
|
51513
51651
|
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51514
51652
|
};
|
|
51653
|
+
const pinNodeOf = (pin) => {
|
|
51654
|
+
if (pin.nodeId !== null) return pin.nodeId;
|
|
51655
|
+
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51656
|
+
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51657
|
+
};
|
|
51658
|
+
let alwaysOnTotal = 0;
|
|
51515
51659
|
for (const cam of input.cameras) {
|
|
51516
51660
|
if (!charges(cam)) continue;
|
|
51517
|
-
|
|
51518
|
-
|
|
51661
|
+
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51662
|
+
if (cam.devicePin !== null && pinNode !== null) {
|
|
51663
|
+
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51519
51664
|
if (!eligibleById.has(key)) continue;
|
|
51520
51665
|
bump(pinnedFps, key, cam.fps);
|
|
51521
51666
|
noteMax(key, cam.fps);
|
|
51522
51667
|
continue;
|
|
51523
51668
|
}
|
|
51524
|
-
|
|
51525
|
-
|
|
51669
|
+
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51670
|
+
if (standing !== null) {
|
|
51671
|
+
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51672
|
+
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51526
51673
|
for (const d of input.eligible) {
|
|
51527
|
-
if (d.nodeId !==
|
|
51674
|
+
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51528
51675
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51529
51676
|
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51530
51677
|
bump(alwaysOnFps, key, cam.fps * share);
|
|
@@ -51538,13 +51685,23 @@ function estimateDeviceDemand(input) {
|
|
|
51538
51685
|
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51539
51686
|
const nodeCount = weightByNode.size;
|
|
51540
51687
|
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51688
|
+
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51689
|
+
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51690
|
+
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51691
|
+
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51541
51692
|
return input.eligible.map((d) => {
|
|
51542
51693
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51543
51694
|
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51544
51695
|
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51545
51696
|
let onMotionSessions = 0;
|
|
51546
51697
|
let onMotionSource = "none";
|
|
51547
|
-
if (onMotionRates.length > 0
|
|
51698
|
+
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51699
|
+
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51700
|
+
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51701
|
+
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51702
|
+
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51703
|
+
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51704
|
+
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51548
51705
|
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51549
51706
|
onMotionSource = "observed-24h";
|
|
51550
51707
|
} else {
|
|
@@ -51562,7 +51719,8 @@ function estimateDeviceDemand(input) {
|
|
|
51562
51719
|
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51563
51720
|
onMotionSessions,
|
|
51564
51721
|
onMotionSource,
|
|
51565
|
-
observedForMs: peak?.observedForMs ?? null
|
|
51722
|
+
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51723
|
+
placement
|
|
51566
51724
|
};
|
|
51567
51725
|
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51568
51726
|
}
|
|
@@ -51791,6 +51949,46 @@ function recommendNode(input) {
|
|
|
51791
51949
|
reasons: nodeReasons
|
|
51792
51950
|
};
|
|
51793
51951
|
}
|
|
51952
|
+
/** Deficits equal up to float noise (0.7 × 2 − 1 is 0.3999…, not 0.4). */
|
|
51953
|
+
var DEFICIT_EPSILON = 1e-9;
|
|
51954
|
+
function byDeficit(a, b) {
|
|
51955
|
+
const diff = b.deficit - a.deficit;
|
|
51956
|
+
return (Math.abs(diff) < DEFICIT_EPSILON ? 0 : diff) || b.candidate.share - a.candidate.share || a.candidate.key.localeCompare(b.candidate.key);
|
|
51957
|
+
}
|
|
51958
|
+
function spareOf(c) {
|
|
51959
|
+
return c.spare ?? Number.POSITIVE_INFINITY;
|
|
51960
|
+
}
|
|
51961
|
+
function electByShare(candidates) {
|
|
51962
|
+
if (candidates.length === 0) return { kind: "none" };
|
|
51963
|
+
const shareSum = candidates.reduce((s, c) => s + Math.max(c.share, 0), 0);
|
|
51964
|
+
const running = candidates.reduce((s, c) => s + Math.max(c.running, 0), 0);
|
|
51965
|
+
const scored = candidates.map((candidate) => ({
|
|
51966
|
+
candidate,
|
|
51967
|
+
deficit: (shareSum > 0 ? Math.max(candidate.share, 0) / shareSum : 1 / candidates.length) * (running + 1) - candidate.running
|
|
51968
|
+
})).toSorted(byDeficit);
|
|
51969
|
+
const turn = scored[0].candidate;
|
|
51970
|
+
if (!turn.capped && !turn.saturated) return {
|
|
51971
|
+
kind: "elected",
|
|
51972
|
+
key: turn.key,
|
|
51973
|
+
reason: "share"
|
|
51974
|
+
};
|
|
51975
|
+
const free = scored.filter((s) => !s.candidate.capped && !s.candidate.saturated).toSorted((a, b) => spareOf(b.candidate) - spareOf(a.candidate) || byDeficit(a, b));
|
|
51976
|
+
const yieldReason = turn.capped ? "at-cap" : "saturated";
|
|
51977
|
+
if (free.length > 0) return {
|
|
51978
|
+
kind: "elected",
|
|
51979
|
+
key: free[0].candidate.key,
|
|
51980
|
+
reason: "yield",
|
|
51981
|
+
turnKey: turn.key,
|
|
51982
|
+
yieldReason
|
|
51983
|
+
};
|
|
51984
|
+
const uncapped = scored.find((s) => !s.candidate.capped);
|
|
51985
|
+
if (uncapped === void 0) return { kind: "none" };
|
|
51986
|
+
return {
|
|
51987
|
+
kind: "elected",
|
|
51988
|
+
key: uncapped.candidate.key,
|
|
51989
|
+
reason: "share"
|
|
51990
|
+
};
|
|
51991
|
+
}
|
|
51794
51992
|
/**
|
|
51795
51993
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
51796
51994
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -59853,6 +60051,35 @@ function pickInferenceDevice(input) {
|
|
|
59853
60051
|
const loadByKey = new Map((input.loads ?? []).map((l) => [l.deviceKey, l]));
|
|
59854
60052
|
const healthyKeys = eligibleKeys.filter((k) => !isDeviceSaturated(loadByKey.get(k)));
|
|
59855
60053
|
const consideredKeys = pin !== null || healthyKeys.length === 0 || healthyKeys.length === eligibleKeys.length ? eligibleKeys : healthyKeys;
|
|
60054
|
+
if (input.policy === "share" && pin === null) {
|
|
60055
|
+
const election = electByShare(eligibleKeys.map((key) => {
|
|
60056
|
+
const load = loadByKey.get(key);
|
|
60057
|
+
const attached = load?.attachedCameras ?? 0;
|
|
60058
|
+
const cap = input.caps?.[key];
|
|
60059
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
60060
|
+
return {
|
|
60061
|
+
key,
|
|
60062
|
+
share: input.eligible[key] ?? 1,
|
|
60063
|
+
running: attached,
|
|
60064
|
+
capped: hasCap && attached >= cap,
|
|
60065
|
+
saturated: isDeviceSaturated(load),
|
|
60066
|
+
...hasCap ? { spare: Math.max(0, cap - attached) } : {}
|
|
60067
|
+
};
|
|
60068
|
+
}));
|
|
60069
|
+
if (election.kind === "none") return {
|
|
60070
|
+
kind: "default-pool",
|
|
60071
|
+
reason: "all-devices-at-cap",
|
|
60072
|
+
eligibleKeys
|
|
60073
|
+
};
|
|
60074
|
+
return {
|
|
60075
|
+
kind: "selected",
|
|
60076
|
+
deviceKey: election.key,
|
|
60077
|
+
reason: "share",
|
|
60078
|
+
tieBreak: null,
|
|
60079
|
+
eligibleKeys,
|
|
60080
|
+
tiedKeys: [election.key]
|
|
60081
|
+
};
|
|
60082
|
+
}
|
|
59856
60083
|
const balancerInput = {
|
|
59857
60084
|
nodes: consideredKeys.map((k) => deviceAsLoad(k, loadByKey.get(k))),
|
|
59858
60085
|
preferredAgent: pin,
|
|
@@ -64176,7 +64403,6 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
64176
64403
|
}).nullable()
|
|
64177
64404
|
});
|
|
64178
64405
|
var StoredCameraPipelineSettingsSchema = object({
|
|
64179
|
-
pinnedAgentNodeId: string().optional(),
|
|
64180
64406
|
stepToggles: record(string(), boolean()).optional(),
|
|
64181
64407
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
64182
64408
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
@@ -66680,7 +66906,15 @@ var TtlMemo = class {
|
|
|
66680
66906
|
//#endregion
|
|
66681
66907
|
//#region src/detection-session-registry.ts
|
|
66682
66908
|
var DetectionSessionRegistry = class {
|
|
66909
|
+
onChange;
|
|
66683
66910
|
sessions = /* @__PURE__ */ new Map();
|
|
66911
|
+
/**
|
|
66912
|
+
* @param onChange called after every register / deregister / clear with the
|
|
66913
|
+
* running set — feeds the placement statistics (D748). Never on a no-op.
|
|
66914
|
+
*/
|
|
66915
|
+
constructor(onChange = () => {}) {
|
|
66916
|
+
this.onChange = onChange;
|
|
66917
|
+
}
|
|
66684
66918
|
register(deviceId, nodeId, startedAt, triggeredBy, deviceKey) {
|
|
66685
66919
|
const existing = this.sessions.get(deviceId);
|
|
66686
66920
|
if (existing?.teardownTimer) clearTimeout(existing.teardownTimer);
|
|
@@ -66693,6 +66927,7 @@ var DetectionSessionRegistry = class {
|
|
|
66693
66927
|
...deviceKey ? { deviceKey } : {}
|
|
66694
66928
|
};
|
|
66695
66929
|
this.sessions.set(deviceId, session);
|
|
66930
|
+
this.onChange(this.list());
|
|
66696
66931
|
return session;
|
|
66697
66932
|
}
|
|
66698
66933
|
get(deviceId) {
|
|
@@ -66706,6 +66941,7 @@ var DetectionSessionRegistry = class {
|
|
|
66706
66941
|
if (!session) return void 0;
|
|
66707
66942
|
if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66708
66943
|
this.sessions.delete(deviceId);
|
|
66944
|
+
this.onChange(this.list());
|
|
66709
66945
|
return session;
|
|
66710
66946
|
}
|
|
66711
66947
|
setTeardownTimer(deviceId, timer) {
|
|
@@ -66728,9 +66964,39 @@ var DetectionSessionRegistry = class {
|
|
|
66728
66964
|
}
|
|
66729
66965
|
clear() {
|
|
66730
66966
|
for (const session of this.sessions.values()) if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66967
|
+
const hadSessions = this.sessions.size > 0;
|
|
66731
66968
|
this.sessions.clear();
|
|
66969
|
+
if (hadSessions) this.onChange([]);
|
|
66732
66970
|
}
|
|
66733
66971
|
};
|
|
66972
|
+
//#endregion
|
|
66973
|
+
//#region src/session-share-placement.ts
|
|
66974
|
+
function nodeSaturated(load) {
|
|
66975
|
+
return load.devices.length > 0 && load.devices.every((d) => isDeviceSaturated(d));
|
|
66976
|
+
}
|
|
66977
|
+
function electNodeByShare(input) {
|
|
66978
|
+
return electByShare(input.loads.filter((l) => l.nodeId.length > 0 && input.eligible.includes(l.nodeId)).map((load) => {
|
|
66979
|
+
const cap = input.caps[load.nodeId];
|
|
66980
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
66981
|
+
const weight = input.weights[load.nodeId];
|
|
66982
|
+
return {
|
|
66983
|
+
key: load.nodeId,
|
|
66984
|
+
share: weight !== void 0 && weight > 0 ? weight : 1,
|
|
66985
|
+
running: input.runningByNode.get(load.nodeId) ?? 0,
|
|
66986
|
+
capped: hasCap && load.attachedCameras >= cap,
|
|
66987
|
+
saturated: nodeSaturated(load),
|
|
66988
|
+
...hasCap ? { spare: Math.max(0, cap - load.attachedCameras) } : {}
|
|
66989
|
+
};
|
|
66990
|
+
}));
|
|
66991
|
+
}
|
|
66992
|
+
/** Parallel sessions per node: sessions and standing cameras, each camera once. */
|
|
66993
|
+
function countRunningByNode(placements) {
|
|
66994
|
+
const seen = /* @__PURE__ */ new Map();
|
|
66995
|
+
for (const p of placements) if (!seen.has(p.deviceId)) seen.set(p.deviceId, p.nodeId);
|
|
66996
|
+
const out = /* @__PURE__ */ new Map();
|
|
66997
|
+
for (const nodeId of seen.values()) out.set(nodeId, (out.get(nodeId) ?? 0) + 1);
|
|
66998
|
+
return out;
|
|
66999
|
+
}
|
|
66734
67000
|
/**
|
|
66735
67001
|
* Minimum time a detection session stays open, whatever closes it.
|
|
66736
67002
|
*
|
|
@@ -66843,7 +67109,15 @@ var SessionDispatchController = class {
|
|
|
66843
67109
|
* (not `private`) — see the class doc for the two production/test reasons
|
|
66844
67110
|
* `index.ts` reaches through it directly.
|
|
66845
67111
|
*/
|
|
66846
|
-
sessionRegistry = new DetectionSessionRegistry()
|
|
67112
|
+
sessionRegistry = new DetectionSessionRegistry((sessions) => this.deps.placementStats?.noteSessions(sessions.map((s) => ({
|
|
67113
|
+
nodeId: s.nodeId,
|
|
67114
|
+
deviceKey: s.deviceKey ?? ""
|
|
67115
|
+
}))));
|
|
67116
|
+
/**
|
|
67117
|
+
* `deviceId|deviceKey` of device pins the session path has already said it
|
|
67118
|
+
* ignored — one line per stale pin, not one per session (D748).
|
|
67119
|
+
*/
|
|
67120
|
+
reportedStalePins = /* @__PURE__ */ new Set();
|
|
66847
67121
|
/**
|
|
66848
67122
|
* Per-device async mutex for session dispatch/teardown critical sections —
|
|
66849
67123
|
* backed by the generic `KeyedAsyncLock` (mirrors the audio controller's
|
|
@@ -67304,6 +67578,76 @@ var SessionDispatchController = class {
|
|
|
67304
67578
|
});
|
|
67305
67579
|
}
|
|
67306
67580
|
/**
|
|
67581
|
+
* The session's node. `least-loaded` (and any node pin, under either
|
|
67582
|
+
* policy) is `balance()` exactly as before. `share` holds the node shares
|
|
67583
|
+
* of PARALLEL sessions (D748): sessions now running and standing cameras
|
|
67584
|
+
* count toward their node, and a node at its cap or drowning yields —
|
|
67585
|
+
* logged with the reason, and counted against the node in the stats.
|
|
67586
|
+
*/
|
|
67587
|
+
electSessionNode(input) {
|
|
67588
|
+
const balanced = () => balance({
|
|
67589
|
+
nodes: input.loads,
|
|
67590
|
+
preferredAgent: input.preferredAgent,
|
|
67591
|
+
nodeCaps: input.nodeCaps,
|
|
67592
|
+
weights: input.weights,
|
|
67593
|
+
eligibleNodes: input.eligible
|
|
67594
|
+
});
|
|
67595
|
+
if (input.policy !== "share" || input.preferredAgent) return balanced();
|
|
67596
|
+
const running = countRunningByNode([...this.sessionRegistry.list().map((s) => ({
|
|
67597
|
+
deviceId: s.deviceId,
|
|
67598
|
+
nodeId: s.nodeId
|
|
67599
|
+
})), ...this.deps.ledger.listAssignments().map((a) => ({
|
|
67600
|
+
deviceId: a.deviceId,
|
|
67601
|
+
nodeId: a.agentNodeId
|
|
67602
|
+
}))]);
|
|
67603
|
+
const election = electNodeByShare({
|
|
67604
|
+
loads: input.loads,
|
|
67605
|
+
eligible: input.eligible,
|
|
67606
|
+
weights: input.weights,
|
|
67607
|
+
caps: input.nodeCaps,
|
|
67608
|
+
runningByNode: running
|
|
67609
|
+
});
|
|
67610
|
+
if (election.kind === "none") return balanced();
|
|
67611
|
+
if (election.reason === "yield") {
|
|
67612
|
+
this.deps.placementStats?.noteYield(election.turnKey);
|
|
67613
|
+
this.deps.logger.info("detection session: share turn yielded — the node had no room", {
|
|
67614
|
+
tags: {
|
|
67615
|
+
deviceId: input.deviceId,
|
|
67616
|
+
nodeId: election.key
|
|
67617
|
+
},
|
|
67618
|
+
meta: {
|
|
67619
|
+
turnNodeId: election.turnKey,
|
|
67620
|
+
reason: election.yieldReason
|
|
67621
|
+
}
|
|
67622
|
+
});
|
|
67623
|
+
}
|
|
67624
|
+
const load = input.loads.find((l) => l.nodeId === election.key);
|
|
67625
|
+
return {
|
|
67626
|
+
kind: "assigned",
|
|
67627
|
+
agentNodeId: election.key,
|
|
67628
|
+
reason: "capacity",
|
|
67629
|
+
score: load ? load.attachedCameras + load.queueDepthTotal : 0
|
|
67630
|
+
};
|
|
67631
|
+
}
|
|
67632
|
+
/**
|
|
67633
|
+
* Say ONCE that a camera's device pin names a device this node does not
|
|
67634
|
+
* offer (disabled, absent): the session path ignores it, and the Load
|
|
67635
|
+
* balancing tab offers to clear it. Never cleared here (D748).
|
|
67636
|
+
*/
|
|
67637
|
+
noteStalePin(deviceId, pinnedDeviceKey, enabledDevices) {
|
|
67638
|
+
if (pinnedDeviceKey === null || pinnedDeviceKey in enabledDevices) return;
|
|
67639
|
+
const key = `${deviceId}|${pinnedDeviceKey}`;
|
|
67640
|
+
if (this.reportedStalePins.has(key)) return;
|
|
67641
|
+
this.reportedStalePins.add(key);
|
|
67642
|
+
this.deps.logger.warn("device pin ignored — the pinned device is not enabled on the elected node", {
|
|
67643
|
+
tags: { deviceId },
|
|
67644
|
+
meta: {
|
|
67645
|
+
pin: pinnedDeviceKey,
|
|
67646
|
+
enabled: Object.keys(enabledDevices).join(",")
|
|
67647
|
+
}
|
|
67648
|
+
});
|
|
67649
|
+
}
|
|
67650
|
+
/**
|
|
67307
67651
|
* Dispatch a session-time detection attach for an on-motion camera whose
|
|
67308
67652
|
* `decideSessionAction` returned `'dispatch'`. Builds a session
|
|
67309
67653
|
* `RunnerCameraConfig` the same way `startDetection` does, but forces
|
|
@@ -67354,12 +67698,17 @@ var SessionDispatchController = class {
|
|
|
67354
67698
|
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
67355
67699
|
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
67356
67700
|
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
67357
|
-
const
|
|
67358
|
-
|
|
67701
|
+
const nodeCaps = await this.nodeCaps.get(CLUSTER_KEY);
|
|
67702
|
+
const weights = await this.nodeWeights.get(CLUSTER_KEY);
|
|
67703
|
+
const policy = this.deps.sessionPlacement?.() ?? "least-loaded";
|
|
67704
|
+
const decision = this.electSessionNode({
|
|
67705
|
+
deviceId,
|
|
67706
|
+
policy,
|
|
67707
|
+
loads,
|
|
67359
67708
|
preferredAgent,
|
|
67360
|
-
nodeCaps
|
|
67361
|
-
weights
|
|
67362
|
-
|
|
67709
|
+
nodeCaps,
|
|
67710
|
+
weights,
|
|
67711
|
+
eligible
|
|
67363
67712
|
});
|
|
67364
67713
|
if (!decision || decision.kind !== "assigned") {
|
|
67365
67714
|
log.warn("detection session: no eligible node — will retry on next motion", { meta: { reason: decision?.kind === "pending" ? decision.reason : "no-runners-online" } });
|
|
@@ -67372,12 +67721,14 @@ var SessionDispatchController = class {
|
|
|
67372
67721
|
const targetLoad = loads.find((n) => n.nodeId === targetNodeId);
|
|
67373
67722
|
const deviceCaps = await this.inferenceDeviceCaps.get(targetNodeId);
|
|
67374
67723
|
const pinnedDeviceKey = await this.deps.readPipelineDevicePin(deviceId);
|
|
67724
|
+
this.noteStalePin(deviceId, pinnedDeviceKey, enabledDevices);
|
|
67375
67725
|
const selection = pickInferenceDevice({
|
|
67376
67726
|
eligible: enabledDevices,
|
|
67377
67727
|
loads: targetLoad?.devices ?? [],
|
|
67378
67728
|
caps: deviceCaps,
|
|
67379
|
-
pinnedDeviceKey,
|
|
67380
|
-
rotation: this.deps.inferenceRotation
|
|
67729
|
+
pinnedDeviceKey: pinnedDeviceKey !== null && pinnedDeviceKey in enabledDevices ? pinnedDeviceKey : null,
|
|
67730
|
+
rotation: this.deps.inferenceRotation,
|
|
67731
|
+
policy
|
|
67381
67732
|
});
|
|
67382
67733
|
const deviceKey = selectedDeviceKey(selection);
|
|
67383
67734
|
log.info("inference device selection", {
|
|
@@ -67465,6 +67816,7 @@ var SessionDispatchController = class {
|
|
|
67465
67816
|
attachedAt = Date.now();
|
|
67466
67817
|
this.deps.recordAssignment(deviceId, targetNodeId, decision.reason, decision.reason === "manual");
|
|
67467
67818
|
this.sessionRegistry.register(deviceId, targetNodeId, Date.now(), triggeredBy, deviceKey);
|
|
67819
|
+
this.deps.placementStats?.noteStart(targetNodeId, deviceKey ?? "");
|
|
67468
67820
|
if (this.sessionRefusals.delete(deviceId)) log.info("detection session refusal cleared — a session opened", {
|
|
67469
67821
|
tags: {
|
|
67470
67822
|
deviceId,
|
|
@@ -68734,6 +69086,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68734
69086
|
}, 6e4);
|
|
68735
69087
|
nodeStress.start();
|
|
68736
69088
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69089
|
+
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
68737
69090
|
/**
|
|
68738
69091
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
68739
69092
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -68817,6 +69170,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68817
69170
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
68818
69171
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
68819
69172
|
inferenceRotation,
|
|
69173
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69174
|
+
placementStats: deps.placementStats,
|
|
68820
69175
|
logger: deps.ctx().logger
|
|
68821
69176
|
});
|
|
68822
69177
|
const deviceConfig = new DeviceConfigContributions({
|
|
@@ -69350,7 +69705,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69350
69705
|
unsubAgentOffline,
|
|
69351
69706
|
unsubOrchestratorSubscriptions,
|
|
69352
69707
|
pendingRetryTimer,
|
|
69353
|
-
autoRebalanceTimer
|
|
69708
|
+
autoRebalanceTimer,
|
|
69709
|
+
sessionPlacement: () => readSessionPlacement()
|
|
69354
69710
|
};
|
|
69355
69711
|
}
|
|
69356
69712
|
//#endregion
|
|
@@ -69395,6 +69751,20 @@ function buildGlobalSettingsSections(options) {
|
|
|
69395
69751
|
title: "Load Balancer",
|
|
69396
69752
|
tab: "pipeline",
|
|
69397
69753
|
fields: [
|
|
69754
|
+
{
|
|
69755
|
+
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69756
|
+
type: "select",
|
|
69757
|
+
label: "Session placement",
|
|
69758
|
+
description: "How a new detection session picks its node and device. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy. Honour shares = the share of PARALLEL sessions is held to the node and device weights, and a node at its cap yields to a free one. Moving a weight slider never changes this policy.",
|
|
69759
|
+
options: [{
|
|
69760
|
+
value: "least-loaded",
|
|
69761
|
+
label: "Least-loaded"
|
|
69762
|
+
}, {
|
|
69763
|
+
value: "share",
|
|
69764
|
+
label: "Honour shares"
|
|
69765
|
+
}],
|
|
69766
|
+
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69767
|
+
},
|
|
69398
69768
|
{
|
|
69399
69769
|
key: "fpsMinThreshold",
|
|
69400
69770
|
type: "slider",
|
|
@@ -69974,7 +70344,7 @@ var pipelineOrchestratorWidgets = [{
|
|
|
69974
70344
|
* settings: the attach provisions the node pin over the snapshot
|
|
69975
70345
|
* (`applyDeviceProvisioning`: override ?? base), so once the snapshot pin
|
|
69976
70346
|
* is gone the node's recommended model runs.
|
|
69977
|
-
* - never: `stepToggles`,
|
|
70347
|
+
* - never: `stepToggles`, devices' enablement, pins of
|
|
69978
70348
|
* embedding steps (cluster-scoped, changing one forces a re-index).
|
|
69979
70349
|
*/
|
|
69980
70350
|
/** The root detector step. */
|
|
@@ -70236,6 +70606,29 @@ function boundKeysOf(catalogs) {
|
|
|
70236
70606
|
}
|
|
70237
70607
|
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70238
70608
|
}
|
|
70609
|
+
/** Bounded fixed point of the hardware share suggestion (D748). */
|
|
70610
|
+
var SUGGEST_MAX_ROUNDS = 3;
|
|
70611
|
+
/** Integer shares summing to 100 (largest remainder). */
|
|
70612
|
+
function toPercent(weights) {
|
|
70613
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70614
|
+
const out = /* @__PURE__ */ new Map();
|
|
70615
|
+
if (total <= 0) return out;
|
|
70616
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
70617
|
+
k,
|
|
70618
|
+
exact: w / total * 100
|
|
70619
|
+
}));
|
|
70620
|
+
let assigned = 0;
|
|
70621
|
+
for (const r of raw) {
|
|
70622
|
+
out.set(r.k, Math.floor(r.exact));
|
|
70623
|
+
assigned += Math.floor(r.exact);
|
|
70624
|
+
}
|
|
70625
|
+
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70626
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
70627
|
+
const r = byRemainder[i % byRemainder.length];
|
|
70628
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70629
|
+
}
|
|
70630
|
+
return out;
|
|
70631
|
+
}
|
|
70239
70632
|
var BestSettingsService = class {
|
|
70240
70633
|
deps;
|
|
70241
70634
|
applying = false;
|
|
@@ -70243,9 +70636,87 @@ var BestSettingsService = class {
|
|
|
70243
70636
|
this.deps = deps;
|
|
70244
70637
|
}
|
|
70245
70638
|
async plan(input) {
|
|
70246
|
-
return (await this.computePlan(input.headroom)).plan;
|
|
70639
|
+
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70640
|
+
}
|
|
70641
|
+
/**
|
|
70642
|
+
* Shares proportional to each node's capacity at the models Best settings
|
|
70643
|
+
* would pick under those very shares (D748). Model and load depend on each
|
|
70644
|
+
* other, so it iterates from equal shares until the rounded shares stop
|
|
70645
|
+
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
70646
|
+
* ADVISORY session cap — never written, never enforced.
|
|
70647
|
+
*/
|
|
70648
|
+
async suggestShares(input) {
|
|
70649
|
+
let shares = {
|
|
70650
|
+
nodes: {},
|
|
70651
|
+
devices: {}
|
|
70652
|
+
};
|
|
70653
|
+
let result = [];
|
|
70654
|
+
let cameraFps = 10;
|
|
70655
|
+
let rounds = 0;
|
|
70656
|
+
for (let round = 1; round <= SUGGEST_MAX_ROUNDS; round++) {
|
|
70657
|
+
rounds = round;
|
|
70658
|
+
const rec = await this.recommend(input.headroom, shares);
|
|
70659
|
+
cameraFps = rec.cameraFps;
|
|
70660
|
+
result = this.sharesFrom(rec, input.headroom);
|
|
70661
|
+
const next = {
|
|
70662
|
+
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70663
|
+
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70664
|
+
};
|
|
70665
|
+
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70666
|
+
shares = next;
|
|
70667
|
+
if (settled) break;
|
|
70668
|
+
}
|
|
70669
|
+
return {
|
|
70670
|
+
headroom: input.headroom,
|
|
70671
|
+
cameraFps,
|
|
70672
|
+
iterations: rounds,
|
|
70673
|
+
nodes: [...result]
|
|
70674
|
+
};
|
|
70247
70675
|
}
|
|
70248
|
-
|
|
70676
|
+
sharesFrom(rec, headroom) {
|
|
70677
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70678
|
+
const perNode = rec.recommendations.map((r) => {
|
|
70679
|
+
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70680
|
+
const fps = d.chosen?.fps ?? 0;
|
|
70681
|
+
return {
|
|
70682
|
+
d,
|
|
70683
|
+
fps,
|
|
70684
|
+
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70685
|
+
};
|
|
70686
|
+
});
|
|
70687
|
+
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70688
|
+
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70689
|
+
return {
|
|
70690
|
+
r,
|
|
70691
|
+
devices,
|
|
70692
|
+
capacity
|
|
70693
|
+
};
|
|
70694
|
+
});
|
|
70695
|
+
const nodePct = toPercent(capacityByNode);
|
|
70696
|
+
return perNode.map(({ r, devices, capacity }) => {
|
|
70697
|
+
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70698
|
+
return {
|
|
70699
|
+
nodeId: r.nodeId,
|
|
70700
|
+
share: nodePct.get(r.nodeId) ?? null,
|
|
70701
|
+
capacitySessions: capacity,
|
|
70702
|
+
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70703
|
+
devices: devices.map((x) => ({
|
|
70704
|
+
deviceKey: x.d.deviceKey,
|
|
70705
|
+
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70706
|
+
modelId: x.d.chosen?.modelId ?? null,
|
|
70707
|
+
fps: x.d.chosen?.fps ?? null,
|
|
70708
|
+
measured: x.d.chosen?.measured ?? false,
|
|
70709
|
+
suggestedMaxSessions: x.sessions
|
|
70710
|
+
}))
|
|
70711
|
+
};
|
|
70712
|
+
});
|
|
70713
|
+
}
|
|
70714
|
+
/**
|
|
70715
|
+
* Gather every node and recommend. `shares` (a what-if or the suggestion's
|
|
70716
|
+
* round) forces share placement with those shares overlaid on the stored
|
|
70717
|
+
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70718
|
+
*/
|
|
70719
|
+
async recommend(headroom, shares) {
|
|
70249
70720
|
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70250
70721
|
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70251
70722
|
const [devices, hardware, catalog] = await Promise.all([
|
|
@@ -70260,6 +70731,7 @@ var BestSettingsService = class {
|
|
|
70260
70731
|
catalog
|
|
70261
70732
|
};
|
|
70262
70733
|
}));
|
|
70734
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? stored;
|
|
70263
70735
|
const nodes = gathered.map((g) => ({
|
|
70264
70736
|
nodeId: g.nodeId,
|
|
70265
70737
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70275,18 +70747,37 @@ var BestSettingsService = class {
|
|
|
70275
70747
|
const eligible = gathered.flatMap((g) => g.devices.reachable ? g.devices.devices.filter((d) => d.enabled && d.available).map((d) => ({
|
|
70276
70748
|
nodeId: g.nodeId,
|
|
70277
70749
|
deviceKey: d.key,
|
|
70278
|
-
weight: d.weight
|
|
70750
|
+
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70279
70751
|
})) : []);
|
|
70752
|
+
const cameras = await this.deps.cameraLoads();
|
|
70753
|
+
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70280
70754
|
const demand = estimateDeviceDemand({
|
|
70281
|
-
cameras
|
|
70755
|
+
cameras,
|
|
70282
70756
|
eligible,
|
|
70283
|
-
peaks: this.deps.peaks()
|
|
70757
|
+
peaks: this.deps.peaks(),
|
|
70758
|
+
...policy === "share" ? { shares: {
|
|
70759
|
+
nodes: {
|
|
70760
|
+
...await this.deps.readNodeShares(),
|
|
70761
|
+
...shares?.nodes
|
|
70762
|
+
},
|
|
70763
|
+
clusterPeak: this.deps.clusterPeak()
|
|
70764
|
+
} } : {}
|
|
70284
70765
|
});
|
|
70285
70766
|
const recommendations = nodes.map((node) => recommendNode({
|
|
70286
70767
|
node,
|
|
70287
70768
|
demand,
|
|
70288
70769
|
headroom
|
|
70289
70770
|
}));
|
|
70771
|
+
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
70772
|
+
return {
|
|
70773
|
+
gathered,
|
|
70774
|
+
nodes,
|
|
70775
|
+
recommendations,
|
|
70776
|
+
cameraFps: cameraFps > 0 ? cameraFps : 10
|
|
70777
|
+
};
|
|
70778
|
+
}
|
|
70779
|
+
async computePlan(headroom, shares) {
|
|
70780
|
+
const { gathered, nodes, recommendations } = await this.recommend(headroom, shares);
|
|
70290
70781
|
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70291
70782
|
const targets = /* @__PURE__ */ new Map();
|
|
70292
70783
|
const formats = /* @__PURE__ */ new Map();
|
|
@@ -70348,6 +70839,7 @@ var BestSettingsService = class {
|
|
|
70348
70839
|
planHash: createHash("sha256").update(canonical({
|
|
70349
70840
|
version: 1,
|
|
70350
70841
|
headroom,
|
|
70842
|
+
whatIf: shares ?? null,
|
|
70351
70843
|
picks: recommendations.map((r) => ({
|
|
70352
70844
|
nodeId: r.nodeId,
|
|
70353
70845
|
skipped: r.skipped,
|
|
@@ -70359,6 +70851,7 @@ var BestSettingsService = class {
|
|
|
70359
70851
|
})).digest("hex"),
|
|
70360
70852
|
family: "yolov9",
|
|
70361
70853
|
headroom,
|
|
70854
|
+
whatIf: shares !== void 0,
|
|
70362
70855
|
generatedAt: this.deps.now(),
|
|
70363
70856
|
nodes: recommendations,
|
|
70364
70857
|
nodeChanges: [...nodeChanges],
|
|
@@ -70531,6 +71024,7 @@ var BUCKET_MS = 3600 * 1e3;
|
|
|
70531
71024
|
var DevicePeakTracker = class DevicePeakTracker {
|
|
70532
71025
|
now;
|
|
70533
71026
|
series = /* @__PURE__ */ new Map();
|
|
71027
|
+
cluster = null;
|
|
70534
71028
|
constructor(now = () => Date.now()) {
|
|
70535
71029
|
this.now = now;
|
|
70536
71030
|
}
|
|
@@ -70541,12 +71035,14 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70541
71035
|
noteCensus(loads) {
|
|
70542
71036
|
const now = this.now();
|
|
70543
71037
|
const bucket = now - now % BUCKET_MS;
|
|
71038
|
+
let clusterTotal = 0;
|
|
70544
71039
|
for (const load of loads) {
|
|
70545
71040
|
const rows = load.devices.length > 0 ? load.devices : [{
|
|
70546
71041
|
deviceKey: "",
|
|
70547
71042
|
attachedCameras: load.attachedCameras
|
|
70548
71043
|
}];
|
|
70549
71044
|
for (const device of rows) {
|
|
71045
|
+
clusterTotal += device.attachedCameras;
|
|
70550
71046
|
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
70551
71047
|
let s = this.series.get(key);
|
|
70552
71048
|
if (s === void 0) {
|
|
@@ -70560,6 +71056,26 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70560
71056
|
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
70561
71057
|
}
|
|
70562
71058
|
}
|
|
71059
|
+
if (loads.length === 0) return;
|
|
71060
|
+
const c = this.cluster ?? {
|
|
71061
|
+
firstSeenAt: now,
|
|
71062
|
+
buckets: /* @__PURE__ */ new Map()
|
|
71063
|
+
};
|
|
71064
|
+
this.cluster = c;
|
|
71065
|
+
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71066
|
+
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71067
|
+
}
|
|
71068
|
+
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71069
|
+
clusterPeak() {
|
|
71070
|
+
const c = this.cluster;
|
|
71071
|
+
if (c === null) return null;
|
|
71072
|
+
const now = this.now();
|
|
71073
|
+
let peak = 0;
|
|
71074
|
+
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71075
|
+
return {
|
|
71076
|
+
peakSessions: peak,
|
|
71077
|
+
observedForMs: now - c.firstSeenAt
|
|
71078
|
+
};
|
|
70563
71079
|
}
|
|
70564
71080
|
/** Every observed device's peak over the window and how long it has been watched. */
|
|
70565
71081
|
peaks() {
|
|
@@ -70580,6 +71096,186 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70580
71096
|
}
|
|
70581
71097
|
};
|
|
70582
71098
|
//#endregion
|
|
71099
|
+
//#region src/session-placement-stats.ts
|
|
71100
|
+
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71101
|
+
function keyOf(nodeId, deviceKey) {
|
|
71102
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
71103
|
+
}
|
|
71104
|
+
function split(key) {
|
|
71105
|
+
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71106
|
+
return {
|
|
71107
|
+
nodeId,
|
|
71108
|
+
deviceKey
|
|
71109
|
+
};
|
|
71110
|
+
}
|
|
71111
|
+
var SessionPlacementStats = class {
|
|
71112
|
+
now;
|
|
71113
|
+
segments = [];
|
|
71114
|
+
starts = [];
|
|
71115
|
+
yields = [];
|
|
71116
|
+
startedAt;
|
|
71117
|
+
constructor(now = () => Date.now()) {
|
|
71118
|
+
this.now = now;
|
|
71119
|
+
this.startedAt = now();
|
|
71120
|
+
}
|
|
71121
|
+
/** The running-session set changed: open a new segment with these counts. */
|
|
71122
|
+
noteSessions(sessions) {
|
|
71123
|
+
const counts = /* @__PURE__ */ new Map();
|
|
71124
|
+
for (const s of sessions) {
|
|
71125
|
+
const k = keyOf(s.nodeId, s.deviceKey);
|
|
71126
|
+
counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
71127
|
+
}
|
|
71128
|
+
this.segments.push({
|
|
71129
|
+
at: this.now(),
|
|
71130
|
+
counts
|
|
71131
|
+
});
|
|
71132
|
+
this.prune();
|
|
71133
|
+
}
|
|
71134
|
+
noteStart(nodeId, deviceKey) {
|
|
71135
|
+
this.starts.push({
|
|
71136
|
+
at: this.now(),
|
|
71137
|
+
nodeId,
|
|
71138
|
+
deviceKey
|
|
71139
|
+
});
|
|
71140
|
+
this.prune();
|
|
71141
|
+
}
|
|
71142
|
+
/** A share election handed `nodeId`'s turn to another node. */
|
|
71143
|
+
noteYield(nodeId) {
|
|
71144
|
+
this.yields.push({
|
|
71145
|
+
at: this.now(),
|
|
71146
|
+
nodeId
|
|
71147
|
+
});
|
|
71148
|
+
this.prune();
|
|
71149
|
+
}
|
|
71150
|
+
/** For tests: how many segments are retained. */
|
|
71151
|
+
segmentCount() {
|
|
71152
|
+
return this.segments.length;
|
|
71153
|
+
}
|
|
71154
|
+
prune() {
|
|
71155
|
+
const floor = this.now() - RETAIN_MS;
|
|
71156
|
+
while (this.segments.length > 1 && this.segments[1].at <= floor) this.segments.shift();
|
|
71157
|
+
while (this.starts.length > 0 && this.starts[0].at < floor) this.starts.shift();
|
|
71158
|
+
while (this.yields.length > 0 && this.yields[0].at < floor) this.yields.shift();
|
|
71159
|
+
}
|
|
71160
|
+
snapshot(windowMs) {
|
|
71161
|
+
const now = this.now();
|
|
71162
|
+
const from = now - windowMs;
|
|
71163
|
+
const integral = /* @__PURE__ */ new Map();
|
|
71164
|
+
let clusterPeak = 0;
|
|
71165
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71166
|
+
const seg = this.segments[i];
|
|
71167
|
+
const end = this.segments[i + 1]?.at ?? now;
|
|
71168
|
+
const start = Math.max(seg.at, from);
|
|
71169
|
+
if (end <= start) continue;
|
|
71170
|
+
let total = 0;
|
|
71171
|
+
for (const [k, n] of seg.counts) {
|
|
71172
|
+
integral.set(k, (integral.get(k) ?? 0) + n * (end - start));
|
|
71173
|
+
total += n;
|
|
71174
|
+
}
|
|
71175
|
+
clusterPeak = Math.max(clusterPeak, total);
|
|
71176
|
+
}
|
|
71177
|
+
const current = this.segments.at(-1)?.counts ?? /* @__PURE__ */ new Map();
|
|
71178
|
+
const nodeIds = /* @__PURE__ */ new Set();
|
|
71179
|
+
for (const k of [...integral.keys(), ...current.keys()]) nodeIds.add(split(k).nodeId);
|
|
71180
|
+
for (const s of this.starts) if (s.at >= from) nodeIds.add(s.nodeId);
|
|
71181
|
+
const clusterIntegral = [...integral.values()].reduce((a, b) => a + b, 0);
|
|
71182
|
+
const nodes = [...nodeIds].toSorted().map((nodeId) => {
|
|
71183
|
+
const deviceKeys = /* @__PURE__ */ new Set();
|
|
71184
|
+
for (const k of [...integral.keys(), ...current.keys()]) {
|
|
71185
|
+
const s = split(k);
|
|
71186
|
+
if (s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71187
|
+
}
|
|
71188
|
+
for (const s of this.starts) if (s.at >= from && s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71189
|
+
const nodeIntegral = [...deviceKeys].reduce((sum, d) => sum + (integral.get(keyOf(nodeId, d)) ?? 0), 0);
|
|
71190
|
+
const devices = [...deviceKeys].toSorted().map((deviceKey) => ({
|
|
71191
|
+
deviceKey,
|
|
71192
|
+
sessionsNow: current.get(keyOf(nodeId, deviceKey)) ?? 0,
|
|
71193
|
+
sessionsStarted: this.starts.filter((s) => s.at >= from && s.nodeId === nodeId && s.deviceKey === deviceKey).length,
|
|
71194
|
+
observedShare: nodeIntegral > 0 ? (integral.get(keyOf(nodeId, deviceKey)) ?? 0) / nodeIntegral : null
|
|
71195
|
+
}));
|
|
71196
|
+
let peak = 0;
|
|
71197
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71198
|
+
const seg = this.segments[i];
|
|
71199
|
+
if ((this.segments[i + 1]?.at ?? now) <= Math.max(seg.at, from)) continue;
|
|
71200
|
+
let n = 0;
|
|
71201
|
+
for (const [k, c] of seg.counts) if (split(k).nodeId === nodeId) n += c;
|
|
71202
|
+
peak = Math.max(peak, n);
|
|
71203
|
+
}
|
|
71204
|
+
return {
|
|
71205
|
+
nodeId,
|
|
71206
|
+
sessionsNow: devices.reduce((s, d) => s + d.sessionsNow, 0),
|
|
71207
|
+
sessionsStarted: devices.reduce((s, d) => s + d.sessionsStarted, 0),
|
|
71208
|
+
observedShare: clusterIntegral > 0 && nodeIntegral > 0 ? nodeIntegral / clusterIntegral : null,
|
|
71209
|
+
peakSessions: peak,
|
|
71210
|
+
yieldsAway: this.yields.filter((y) => y.at >= from && y.nodeId === nodeId).length,
|
|
71211
|
+
devices
|
|
71212
|
+
};
|
|
71213
|
+
});
|
|
71214
|
+
return {
|
|
71215
|
+
observedForMs: Math.min(windowMs, now - this.startedAt),
|
|
71216
|
+
sessionsNow: [...current.values()].reduce((a, b) => a + b, 0),
|
|
71217
|
+
clusterPeakSessions: clusterPeak,
|
|
71218
|
+
nodes
|
|
71219
|
+
};
|
|
71220
|
+
}
|
|
71221
|
+
};
|
|
71222
|
+
//#endregion
|
|
71223
|
+
//#region src/load-balancing-shares.ts
|
|
71224
|
+
function nextDevices(stored, shares, effective) {
|
|
71225
|
+
let changed = false;
|
|
71226
|
+
const out = { ...stored };
|
|
71227
|
+
for (const [key, weight] of Object.entries(shares)) {
|
|
71228
|
+
const existing = stored?.[key];
|
|
71229
|
+
if (existing !== void 0) {
|
|
71230
|
+
out[key] = {
|
|
71231
|
+
...existing,
|
|
71232
|
+
weight
|
|
71233
|
+
};
|
|
71234
|
+
changed = true;
|
|
71235
|
+
continue;
|
|
71236
|
+
}
|
|
71237
|
+
const enabled = effective[key];
|
|
71238
|
+
if (enabled === void 0) continue;
|
|
71239
|
+
out[key] = {
|
|
71240
|
+
enabled,
|
|
71241
|
+
weight
|
|
71242
|
+
};
|
|
71243
|
+
changed = true;
|
|
71244
|
+
}
|
|
71245
|
+
return changed ? out : stored;
|
|
71246
|
+
}
|
|
71247
|
+
function applyLoadBalancingShares(prev, write) {
|
|
71248
|
+
const next = { ...prev };
|
|
71249
|
+
const nodeIds = new Set([...Object.keys(write.nodes), ...Object.keys(write.devices)]);
|
|
71250
|
+
for (const nodeId of nodeIds) {
|
|
71251
|
+
const existing = prev[nodeId];
|
|
71252
|
+
const nodeShare = write.nodes[nodeId];
|
|
71253
|
+
const devices = nextDevices(existing?.inferenceDevices, write.devices[nodeId] ?? {}, write.effectiveEnabled[nodeId] ?? {});
|
|
71254
|
+
if (nodeShare === void 0 && devices === existing?.inferenceDevices) continue;
|
|
71255
|
+
next[nodeId] = {
|
|
71256
|
+
...existing ?? { maxCameras: null },
|
|
71257
|
+
...nodeShare !== void 0 ? { detectWeight: nodeShare } : {},
|
|
71258
|
+
...devices !== void 0 ? { inferenceDevices: devices } : {}
|
|
71259
|
+
};
|
|
71260
|
+
}
|
|
71261
|
+
return next;
|
|
71262
|
+
}
|
|
71263
|
+
//#endregion
|
|
71264
|
+
//#region src/stale-device-pins.ts
|
|
71265
|
+
function findStaleDevicePins(pins, nodes) {
|
|
71266
|
+
const usable = /* @__PURE__ */ new Map();
|
|
71267
|
+
for (const node of nodes) for (const d of node.devices) usable.set(d.key, (usable.get(d.key) ?? false) || d.enabled && d.available);
|
|
71268
|
+
return pins.flatMap((pin) => {
|
|
71269
|
+
const state = usable.get(pin.deviceKey);
|
|
71270
|
+
if (state === true) return [];
|
|
71271
|
+
return [{
|
|
71272
|
+
deviceId: pin.deviceId,
|
|
71273
|
+
deviceKey: pin.deviceKey,
|
|
71274
|
+
reason: state === false ? "disabled" : "absent"
|
|
71275
|
+
}];
|
|
71276
|
+
}).toSorted((a, b) => a.deviceId - b.deviceId);
|
|
71277
|
+
}
|
|
71278
|
+
//#endregion
|
|
70583
71279
|
//#region src/index.ts
|
|
70584
71280
|
/**
|
|
70585
71281
|
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
@@ -70649,6 +71345,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70649
71345
|
clusterModelChoiceReady = false;
|
|
70650
71346
|
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
70651
71347
|
devicePeaks = new DevicePeakTracker();
|
|
71348
|
+
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71349
|
+
placementStats = new SessionPlacementStats();
|
|
71350
|
+
/** The stored session placement policy (D748); `least-loaded` before init. */
|
|
71351
|
+
readSessionPlacement = () => "least-loaded";
|
|
70652
71352
|
bestSettingsService = null;
|
|
70653
71353
|
/**
|
|
70654
71354
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
@@ -70927,6 +71627,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70927
71627
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
70928
71628
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
70929
71629
|
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
71630
|
+
placementStats: this.placementStats,
|
|
70930
71631
|
setTopology: (topology) => {
|
|
70931
71632
|
this.topology = topology;
|
|
70932
71633
|
},
|
|
@@ -70953,6 +71654,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70953
71654
|
this.nodeStress = controllers.nodeStress;
|
|
70954
71655
|
this.placement = controllers.placement;
|
|
70955
71656
|
this.session = controllers.session;
|
|
71657
|
+
this.readSessionPlacement = controllers.sessionPlacement;
|
|
70956
71658
|
this.deviceConfig = controllers.deviceConfig;
|
|
70957
71659
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
70958
71660
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71445,26 +72147,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71445
72147
|
this.reconcile.schedulePendingRetry();
|
|
71446
72148
|
return { success: true };
|
|
71447
72149
|
}
|
|
71448
|
-
async setAgentDetectWeight(input) {
|
|
71449
|
-
const existing = (await this.settingsStore.readAgentSettingsMap())[input.agentNodeId];
|
|
71450
|
-
const weight = input.detectWeight ?? 1;
|
|
71451
|
-
const next = existing ? {
|
|
71452
|
-
...existing,
|
|
71453
|
-
detectWeight: weight
|
|
71454
|
-
} : {
|
|
71455
|
-
maxCameras: null,
|
|
71456
|
-
detectWeight: weight
|
|
71457
|
-
};
|
|
71458
|
-
await this.settingsStore.writeAgentSettings(input.agentNodeId, next);
|
|
71459
|
-
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
71460
|
-
tags: { nodeId: input.agentNodeId },
|
|
71461
|
-
meta: { detectWeight: weight ?? null }
|
|
71462
|
-
});
|
|
71463
|
-
this.rebalance().catch((err) => {
|
|
71464
|
-
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
71465
|
-
});
|
|
71466
|
-
return { success: true };
|
|
71467
|
-
}
|
|
71468
72150
|
/**
|
|
71469
72151
|
* Set one node's placement capabilities — the per-node record that replaced
|
|
71470
72152
|
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
@@ -72062,6 +72744,9 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72062
72744
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72063
72745
|
cameraLoads: () => this.bestSettingsCameraLoads(),
|
|
72064
72746
|
peaks: () => this.devicePeaks.peaks(),
|
|
72747
|
+
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72748
|
+
sessionPlacement: () => this.readSessionPlacement(),
|
|
72749
|
+
readNodeShares: () => store.buildNodeWeights(),
|
|
72065
72750
|
detectorStore: async (input) => {
|
|
72066
72751
|
const api = this.ctx.api;
|
|
72067
72752
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72089,19 +72774,20 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72089
72774
|
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72090
72775
|
async bestSettingsCameraLoads() {
|
|
72091
72776
|
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72092
|
-
const cameraMap = await this.settingsStore.readCameraSettingsMap();
|
|
72093
72777
|
const loads = [];
|
|
72094
72778
|
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
72095
|
-
const
|
|
72779
|
+
const nodePin = await this.readPipelinePin(deviceId);
|
|
72096
72780
|
const pinnedKey = await this.readPipelineDevicePin(deviceId);
|
|
72781
|
+
const standingNode = config.detectionMode === "always-on" ? standing.get(deviceId) ?? null : null;
|
|
72097
72782
|
loads.push({
|
|
72098
72783
|
deviceId,
|
|
72099
72784
|
mode: config.detectionMode,
|
|
72100
72785
|
fps: config.detectionFps,
|
|
72101
72786
|
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72102
|
-
standingNodeId:
|
|
72103
|
-
|
|
72104
|
-
|
|
72787
|
+
standingNodeId: standingNode,
|
|
72788
|
+
nodePinId: nodePin,
|
|
72789
|
+
devicePin: pinnedKey !== null ? {
|
|
72790
|
+
nodeId: nodePin ?? standingNode,
|
|
72105
72791
|
deviceKey: pinnedKey
|
|
72106
72792
|
} : null
|
|
72107
72793
|
});
|
|
@@ -72111,6 +72797,62 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72111
72797
|
async planBestSettings(input) {
|
|
72112
72798
|
return this.bestSettings().plan(input);
|
|
72113
72799
|
}
|
|
72800
|
+
/**
|
|
72801
|
+
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
72802
|
+
* device with no stored entry is made explicit with its effective
|
|
72803
|
+
* enablement, read from the probe-aware device list. Never touches the
|
|
72804
|
+
* `sessionPlacement` policy.
|
|
72805
|
+
*/
|
|
72806
|
+
async setLoadBalancingShares(input) {
|
|
72807
|
+
const effectiveEnabled = {};
|
|
72808
|
+
for (const nodeId of Object.keys(input.devices)) {
|
|
72809
|
+
const read = await this.getNodeInferenceDevices({ nodeId });
|
|
72810
|
+
effectiveEnabled[nodeId] = Object.fromEntries(read.devices.map((d) => [d.key, d.enabled]));
|
|
72811
|
+
}
|
|
72812
|
+
await this.settingsStore.rewriteAgentSettingsMap((prev) => applyLoadBalancingShares(prev, {
|
|
72813
|
+
nodes: input.nodes,
|
|
72814
|
+
devices: input.devices,
|
|
72815
|
+
effectiveEnabled
|
|
72816
|
+
}));
|
|
72817
|
+
this.ctx.logger.info("load balancing shares updated", { meta: {
|
|
72818
|
+
nodes: JSON.stringify(input.nodes),
|
|
72819
|
+
devices: JSON.stringify(input.devices),
|
|
72820
|
+
policy: this.readSessionPlacement()
|
|
72821
|
+
} });
|
|
72822
|
+
if (Object.keys(input.nodes).length > 0) this.rebalance().catch((err) => {
|
|
72823
|
+
this.ctx.logger.warn("setLoadBalancingShares: rebalance failed", { meta: { error: errMsg$2(err) } });
|
|
72824
|
+
});
|
|
72825
|
+
return { success: true };
|
|
72826
|
+
}
|
|
72827
|
+
async getPlacementStats(input) {
|
|
72828
|
+
const snap = this.placementStats.snapshot(input.windowMs);
|
|
72829
|
+
return {
|
|
72830
|
+
policy: this.readSessionPlacement(),
|
|
72831
|
+
windowMs: input.windowMs,
|
|
72832
|
+
observedForMs: snap.observedForMs,
|
|
72833
|
+
sessionsNow: snap.sessionsNow,
|
|
72834
|
+
clusterPeakSessions: snap.clusterPeakSessions,
|
|
72835
|
+
nodes: snap.nodes,
|
|
72836
|
+
stalePins: await this.listStaleDevicePins()
|
|
72837
|
+
};
|
|
72838
|
+
}
|
|
72839
|
+
/** Device pins the session path cannot honour — see `stale-device-pins.ts`. */
|
|
72840
|
+
async listStaleDevicePins() {
|
|
72841
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])];
|
|
72842
|
+
const offered = await Promise.all(nodeIds.map(async (nodeId) => ({ devices: (await this.getNodeInferenceDevices({ nodeId }).catch(() => null))?.devices ?? [] })));
|
|
72843
|
+
const pins = [];
|
|
72844
|
+
for (const deviceId of this.detectionWiring.activeDetections.keys()) {
|
|
72845
|
+
const deviceKey = await this.readPipelineDevicePin(deviceId);
|
|
72846
|
+
if (deviceKey !== null) pins.push({
|
|
72847
|
+
deviceId,
|
|
72848
|
+
deviceKey
|
|
72849
|
+
});
|
|
72850
|
+
}
|
|
72851
|
+
return findStaleDevicePins(pins, offered);
|
|
72852
|
+
}
|
|
72853
|
+
async suggestShares(input) {
|
|
72854
|
+
return this.bestSettings().suggestShares(input);
|
|
72855
|
+
}
|
|
72114
72856
|
async applyBestSettings(input) {
|
|
72115
72857
|
return this.bestSettings().apply(input);
|
|
72116
72858
|
}
|