@camstack/addon-pipeline-orchestrator 1.2.290 → 1.2.292
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DHY2k_nC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets--PHyrJOr.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-tJspAxws.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CdkZH5kw.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CcezrkWV.mjs} +1 -1
- package/dist/{hostInit-CRtAczui.mjs → hostInit-D7n994Lm.mjs} +2 -2
- package/dist/index.js +1337 -129
- package/dist/index.mjs +1337 -129
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DWLgJB4I.mjs +0 -26
package/dist/index.js
CHANGED
|
@@ -24661,7 +24661,7 @@ var pipelineExecutorCapability = {
|
|
|
24661
24661
|
auth: "admin"
|
|
24662
24662
|
}),
|
|
24663
24663
|
/**
|
|
24664
|
-
*
|
|
24664
|
+
* Auto tune (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24665
24665
|
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24666
24666
|
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24667
24667
|
* running model's own defaults apply. Every other key of a camera's store
|
|
@@ -25033,7 +25033,165 @@ var pipelineExecutorCapability = {
|
|
|
25033
25033
|
}
|
|
25034
25034
|
};
|
|
25035
25035
|
/**
|
|
25036
|
-
*
|
|
25036
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25037
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25038
|
+
* hardware share suggestion and the stale device pins.
|
|
25039
|
+
*/
|
|
25040
|
+
/**
|
|
25041
|
+
* How a new detection session picks its node and device.
|
|
25042
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25043
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25044
|
+
* weight only bites once a node already holds a session.
|
|
25045
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25046
|
+
* full or drowning node yields to a free one (D748).
|
|
25047
|
+
*/
|
|
25048
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25049
|
+
/**
|
|
25050
|
+
* `share` since D750: the percentages are held by default. `least-loaded` stays
|
|
25051
|
+
* selectable.
|
|
25052
|
+
*/
|
|
25053
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "share";
|
|
25054
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25055
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25056
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25057
|
+
function parseSessionPlacementPolicy(value) {
|
|
25058
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25059
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25060
|
+
}
|
|
25061
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25062
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25063
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25064
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25065
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25066
|
+
});
|
|
25067
|
+
var PlacementStatsInputSchema = object({
|
|
25068
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25069
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25070
|
+
var PlacementDeviceStatsSchema = object({
|
|
25071
|
+
deviceKey: string(),
|
|
25072
|
+
/** Sessions running on this device now. */
|
|
25073
|
+
sessionsNow: number(),
|
|
25074
|
+
/** Sessions started on this device inside the window. */
|
|
25075
|
+
sessionsStarted: number(),
|
|
25076
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25077
|
+
observedShare: number().nullable()
|
|
25078
|
+
});
|
|
25079
|
+
var PlacementNodeStatsSchema = object({
|
|
25080
|
+
nodeId: string(),
|
|
25081
|
+
sessionsNow: number(),
|
|
25082
|
+
sessionsStarted: number(),
|
|
25083
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25084
|
+
observedShare: number().nullable(),
|
|
25085
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25086
|
+
peakSessions: number(),
|
|
25087
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25088
|
+
yieldsAway: number(),
|
|
25089
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25090
|
+
});
|
|
25091
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25092
|
+
var StaleDevicePinSchema = object({
|
|
25093
|
+
deviceId: number(),
|
|
25094
|
+
deviceKey: string(),
|
|
25095
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25096
|
+
reason: _enum(["disabled", "absent"])
|
|
25097
|
+
});
|
|
25098
|
+
var PlacementStatsSchema = object({
|
|
25099
|
+
policy: SessionPlacementPolicySchema,
|
|
25100
|
+
windowMs: number(),
|
|
25101
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25102
|
+
observedForMs: number(),
|
|
25103
|
+
sessionsNow: number(),
|
|
25104
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25105
|
+
clusterPeakSessions: number(),
|
|
25106
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25107
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25108
|
+
});
|
|
25109
|
+
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25110
|
+
var SuggestedDeviceShareSchema = object({
|
|
25111
|
+
deviceKey: string(),
|
|
25112
|
+
/** Share of the node (0–100, rounded). */
|
|
25113
|
+
share: number(),
|
|
25114
|
+
modelId: string().nullable(),
|
|
25115
|
+
/** Inferences per second at that model, from the measured table. */
|
|
25116
|
+
fps: number().nullable(),
|
|
25117
|
+
measured: boolean(),
|
|
25118
|
+
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25119
|
+
suggestedMaxSessions: number().nullable()
|
|
25120
|
+
});
|
|
25121
|
+
var SuggestedNodeShareSchema = object({
|
|
25122
|
+
nodeId: string(),
|
|
25123
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25124
|
+
share: number().nullable(),
|
|
25125
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25126
|
+
capacitySessions: number(),
|
|
25127
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25128
|
+
skippedReason: string().nullable()
|
|
25129
|
+
});
|
|
25130
|
+
var SuggestSharesResultSchema = object({
|
|
25131
|
+
headroom: number(),
|
|
25132
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25133
|
+
cameraFps: number(),
|
|
25134
|
+
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25135
|
+
iterations: number(),
|
|
25136
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25137
|
+
});
|
|
25138
|
+
/**
|
|
25139
|
+
* Where the node and device shares come from (D750).
|
|
25140
|
+
* - `auto` — computed by the system from the cluster's measured capacity at
|
|
25141
|
+
* the Auto tune models, and recomputed whenever a node or device joins,
|
|
25142
|
+
* leaves, is switched on or off, or changes model.
|
|
25143
|
+
* - `manual` — the operator's stored sliders. A node or device that cannot
|
|
25144
|
+
* take sessions gives its share to the others in proportion; the stored
|
|
25145
|
+
* values are never rewritten for that.
|
|
25146
|
+
*/
|
|
25147
|
+
var ShareSourceSchema = _enum(["auto", "manual"]);
|
|
25148
|
+
var DEFAULT_SHARE_SOURCE = "auto";
|
|
25149
|
+
/** Headroom the automatic shares are computed at (Auto tune's default). */
|
|
25150
|
+
var AUTO_SHARE_HEADROOM = .4;
|
|
25151
|
+
/** The global-settings key the share source is stored under (orchestrator `load` section). */
|
|
25152
|
+
var SHARE_SOURCE_SETTING_KEY = "shareSource";
|
|
25153
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25154
|
+
function parseShareSource(value) {
|
|
25155
|
+
const parsed = ShareSourceSchema.safeParse(value);
|
|
25156
|
+
return parsed.success ? parsed.data : DEFAULT_SHARE_SOURCE;
|
|
25157
|
+
}
|
|
25158
|
+
var SetShareSourceInputSchema = object({ source: ShareSourceSchema });
|
|
25159
|
+
object({
|
|
25160
|
+
nodes: record(string(), number()),
|
|
25161
|
+
devices: record(string(), record(string(), number()))
|
|
25162
|
+
});
|
|
25163
|
+
var EffectiveDeviceShareSchema = object({
|
|
25164
|
+
deviceKey: string(),
|
|
25165
|
+
/** Can take sessions now (enabled, available, can run the root). */
|
|
25166
|
+
eligible: boolean(),
|
|
25167
|
+
/** The share the source assigns, as a percentage of the node's devices. */
|
|
25168
|
+
configuredPct: number(),
|
|
25169
|
+
/** The share actually used, after an ineligible device's share went to the others. */
|
|
25170
|
+
effectivePct: number()
|
|
25171
|
+
});
|
|
25172
|
+
var EffectiveNodeShareSchema = object({
|
|
25173
|
+
nodeId: string(),
|
|
25174
|
+
eligible: boolean(),
|
|
25175
|
+
configuredPct: number(),
|
|
25176
|
+
effectivePct: number(),
|
|
25177
|
+
devices: array(EffectiveDeviceShareSchema).readonly()
|
|
25178
|
+
});
|
|
25179
|
+
var EffectiveSharesSchema = object({
|
|
25180
|
+
source: ShareSourceSchema,
|
|
25181
|
+
policy: SessionPlacementPolicySchema,
|
|
25182
|
+
/** `auto` with no computed answer yet: the stored sliders are used meanwhile. */
|
|
25183
|
+
autoPending: boolean(),
|
|
25184
|
+
/** When the automatic shares were last computed (ms epoch); null = never. */
|
|
25185
|
+
computedAt: number().nullable(),
|
|
25186
|
+
/** What made the system recompute them last. */
|
|
25187
|
+
trigger: string().nullable(),
|
|
25188
|
+
nodes: array(EffectiveNodeShareSchema).readonly(),
|
|
25189
|
+
/** The relative weights the election reads (before eligibility). */
|
|
25190
|
+
nodeWeights: record(string(), number()),
|
|
25191
|
+
deviceWeights: record(string(), record(string(), number()))
|
|
25192
|
+
});
|
|
25193
|
+
/**
|
|
25194
|
+
* The wire shapes of the Auto tune plan and apply (D745), used by the
|
|
25037
25195
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25038
25196
|
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
25039
25197
|
* schemas are what crosses the cap boundary.
|
|
@@ -25109,7 +25267,9 @@ var DeviceDemandSchema = object({
|
|
|
25109
25267
|
"estimate",
|
|
25110
25268
|
"none"
|
|
25111
25269
|
]),
|
|
25112
|
-
observedForMs: number().nullable()
|
|
25270
|
+
observedForMs: number().nullable(),
|
|
25271
|
+
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25272
|
+
placement: SessionPlacementPolicySchema
|
|
25113
25273
|
});
|
|
25114
25274
|
var DeviceModelChoiceSchema = object({
|
|
25115
25275
|
modelId: string(),
|
|
@@ -25166,7 +25326,7 @@ var DetectorStoreClearSchema = object({
|
|
|
25166
25326
|
deviceId: number(),
|
|
25167
25327
|
keys: array(string()).readonly()
|
|
25168
25328
|
});
|
|
25169
|
-
var
|
|
25329
|
+
var AutoTuneWarningSchema = discriminatedUnion("kind", [
|
|
25170
25330
|
object({
|
|
25171
25331
|
kind: literal("detector-store-cluster-wide"),
|
|
25172
25332
|
skippedNodes: array(string()).readonly()
|
|
@@ -25181,26 +25341,36 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25181
25341
|
deviceKey: string()
|
|
25182
25342
|
})
|
|
25183
25343
|
]);
|
|
25184
|
-
var
|
|
25185
|
-
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25186
|
-
headroom: number().min(.3).max(.5).default(.4)
|
|
25187
|
-
|
|
25344
|
+
var AutoTuneOptionsSchema = object({
|
|
25345
|
+
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25346
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25347
|
+
/**
|
|
25348
|
+
* What-if shares (D748): plan as if these node/device shares were stored
|
|
25349
|
+
* and the policy were `share`. Absent = the stored shares under the stored
|
|
25350
|
+
* policy. A what-if plan is for display; apply always re-plans from what
|
|
25351
|
+
* is stored.
|
|
25352
|
+
*/
|
|
25353
|
+
shares: LoadBalancingSharesInputSchema.optional()
|
|
25354
|
+
});
|
|
25355
|
+
var AutoTunePlanSchema = object({
|
|
25188
25356
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25189
25357
|
planHash: string(),
|
|
25190
25358
|
family: literal("yolov9"),
|
|
25191
25359
|
headroom: number(),
|
|
25360
|
+
/** Planned under what-if shares (D748): for display — apply refuses it as stale. */
|
|
25361
|
+
whatIf: boolean(),
|
|
25192
25362
|
generatedAt: number(),
|
|
25193
25363
|
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25194
25364
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25195
25365
|
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25196
25366
|
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25197
|
-
warnings: array(
|
|
25367
|
+
warnings: array(AutoTuneWarningSchema).readonly()
|
|
25198
25368
|
});
|
|
25199
|
-
var
|
|
25369
|
+
var AutoTuneApplyInputSchema = object({
|
|
25200
25370
|
planHash: string(),
|
|
25201
25371
|
headroom: number().min(.3).max(.5).default(.4)
|
|
25202
25372
|
});
|
|
25203
|
-
var
|
|
25373
|
+
var AutoTunePhaseResultSchema = object({
|
|
25204
25374
|
phase: _enum([
|
|
25205
25375
|
"download",
|
|
25206
25376
|
"camera-overrides",
|
|
@@ -25211,19 +25381,19 @@ var BestSettingsPhaseResultSchema = object({
|
|
|
25211
25381
|
ok: boolean(),
|
|
25212
25382
|
detail: string()
|
|
25213
25383
|
});
|
|
25214
|
-
var
|
|
25384
|
+
var AutoTuneApplyResultSchema = discriminatedUnion("kind", [
|
|
25215
25385
|
object({
|
|
25216
25386
|
kind: literal("applied"),
|
|
25217
25387
|
planHash: string(),
|
|
25218
|
-
phases: array(
|
|
25388
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25219
25389
|
}),
|
|
25220
25390
|
object({
|
|
25221
25391
|
kind: literal("aborted"),
|
|
25222
|
-
phases: array(
|
|
25392
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25223
25393
|
}),
|
|
25224
25394
|
object({
|
|
25225
25395
|
kind: literal("stale"),
|
|
25226
|
-
plan:
|
|
25396
|
+
plan: AutoTunePlanSchema
|
|
25227
25397
|
}),
|
|
25228
25398
|
object({
|
|
25229
25399
|
kind: literal("refused"),
|
|
@@ -26638,7 +26808,6 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26638
26808
|
settings: record(string(), unknown()).readonly().optional()
|
|
26639
26809
|
});
|
|
26640
26810
|
var CameraPipelineSettingsSchema = object({
|
|
26641
|
-
pinnedAgentNodeId: string().optional(),
|
|
26642
26811
|
stepToggles: record(string(), boolean()).optional(),
|
|
26643
26812
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26644
26813
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
@@ -27286,18 +27455,6 @@ var pipelineOrchestratorCapability = {
|
|
|
27286
27455
|
auth: "admin"
|
|
27287
27456
|
}),
|
|
27288
27457
|
/**
|
|
27289
|
-
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
27290
|
-
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
27291
|
-
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
27292
|
-
*/
|
|
27293
|
-
setAgentDetectWeight: method(object({
|
|
27294
|
-
agentNodeId: string(),
|
|
27295
|
-
detectWeight: number().positive().nullable()
|
|
27296
|
-
}), object({ success: literal(true) }), {
|
|
27297
|
-
kind: "mutation",
|
|
27298
|
-
auth: "admin"
|
|
27299
|
-
}),
|
|
27300
|
-
/**
|
|
27301
27458
|
* Set one node's placement CAPABILITIES — the per-node record that
|
|
27302
27459
|
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
27303
27460
|
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
@@ -27417,7 +27574,7 @@ var pipelineOrchestratorCapability = {
|
|
|
27417
27574
|
auth: "admin"
|
|
27418
27575
|
}),
|
|
27419
27576
|
/**
|
|
27420
|
-
*
|
|
27577
|
+
* Auto tune (D745) — DRY RUN. Recommends ONE detector family (YOLOv9)
|
|
27421
27578
|
* for the whole cluster from each node's static hardware class, the
|
|
27422
27579
|
* measured latency table and the detection demand, and lists every write
|
|
27423
27580
|
* the apply would make: the node model pins set, the enrichment pins and
|
|
@@ -27426,14 +27583,57 @@ var pipelineOrchestratorCapability = {
|
|
|
27426
27583
|
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27427
27584
|
* rules are never part of it. `planHash` names exactly this diff.
|
|
27428
27585
|
*/
|
|
27429
|
-
|
|
27586
|
+
planAutoTune: method(AutoTuneOptionsSchema, AutoTunePlanSchema, { auth: "admin" }),
|
|
27430
27587
|
/**
|
|
27431
|
-
*
|
|
27588
|
+
* Auto tune (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27432
27589
|
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27433
27590
|
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27434
27591
|
* nodes are skipped; there is no undo.
|
|
27435
27592
|
*/
|
|
27436
|
-
|
|
27593
|
+
applyAutoTune: method(AutoTuneApplyInputSchema, AutoTuneApplyResultSchema, {
|
|
27594
|
+
kind: "mutation",
|
|
27595
|
+
auth: "admin"
|
|
27596
|
+
}),
|
|
27597
|
+
/**
|
|
27598
|
+
* Load balancing (D748) — write node shares (`detectWeight`) and per-node
|
|
27599
|
+
* device shares (inference-device `weight`) in ONE atomic read-modify-write
|
|
27600
|
+
* of the agent settings — the only writer of node shares (the per-node
|
|
27601
|
+
* `setAgentDetectWeight` it replaced raced its siblings, one RMW each).
|
|
27602
|
+
* A node or device left out keeps its weight. Never changes the
|
|
27603
|
+
* `sessionPlacement` policy.
|
|
27604
|
+
*/
|
|
27605
|
+
setLoadBalancingShares: method(LoadBalancingSharesInputSchema, object({ success: literal(true) }), {
|
|
27606
|
+
kind: "mutation",
|
|
27607
|
+
auth: "admin"
|
|
27608
|
+
}),
|
|
27609
|
+
/**
|
|
27610
|
+
* Load balancing (D748) — what placement actually did: per node and
|
|
27611
|
+
* device, sessions now, sessions started and the time-weighted share of
|
|
27612
|
+
* PARALLEL sessions over `windowMs`; the cluster's parallel peak; yields
|
|
27613
|
+
* a share election made; and every device pin the session path cannot
|
|
27614
|
+
* honour. In memory on the hub — a restart starts a new window.
|
|
27615
|
+
*/
|
|
27616
|
+
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27617
|
+
/**
|
|
27618
|
+
* Load balancing (D748) — shares proportional to each node's measured
|
|
27619
|
+
* capacity at the models Auto tune would pick, iterated (≤ 3 rounds)
|
|
27620
|
+
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27621
|
+
* Writes nothing.
|
|
27622
|
+
*/
|
|
27623
|
+
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27624
|
+
/**
|
|
27625
|
+
* Load balancing (D750) — the shares placement reads now: the source
|
|
27626
|
+
* (automatic capacity shares or the operator's sliders), each node's and
|
|
27627
|
+
* device's configured share, and the effective share after a node or
|
|
27628
|
+
* device that cannot take sessions gave its share to the others.
|
|
27629
|
+
*/
|
|
27630
|
+
getEffectiveShares: method(_void(), EffectiveSharesSchema, { auth: "admin" }),
|
|
27631
|
+
/**
|
|
27632
|
+
* Load balancing (D750) — switch between automatic and manual shares.
|
|
27633
|
+
* Going manual seeds the sliders with the current automatic shares (one
|
|
27634
|
+
* atomic write), so nothing jumps; going automatic recomputes at once.
|
|
27635
|
+
*/
|
|
27636
|
+
setShareSource: method(SetShareSourceInputSchema, object({ success: literal(true) }), {
|
|
27437
27637
|
kind: "mutation",
|
|
27438
27638
|
auth: "admin"
|
|
27439
27639
|
}),
|
|
@@ -43460,7 +43660,7 @@ Object.freeze({
|
|
|
43460
43660
|
addonId: null,
|
|
43461
43661
|
access: "view"
|
|
43462
43662
|
},
|
|
43463
|
-
"pipelineOrchestrator.
|
|
43663
|
+
"pipelineOrchestrator.applyAutoTune": {
|
|
43464
43664
|
capName: "pipeline-orchestrator",
|
|
43465
43665
|
capScope: "system",
|
|
43466
43666
|
addonId: null,
|
|
@@ -43580,6 +43780,12 @@ Object.freeze({
|
|
|
43580
43780
|
addonId: null,
|
|
43581
43781
|
access: "view"
|
|
43582
43782
|
},
|
|
43783
|
+
"pipelineOrchestrator.getEffectiveShares": {
|
|
43784
|
+
capName: "pipeline-orchestrator",
|
|
43785
|
+
capScope: "system",
|
|
43786
|
+
addonId: null,
|
|
43787
|
+
access: "view"
|
|
43788
|
+
},
|
|
43583
43789
|
"pipelineOrchestrator.getGlobalMetrics": {
|
|
43584
43790
|
capName: "pipeline-orchestrator",
|
|
43585
43791
|
capScope: "system",
|
|
@@ -43616,6 +43822,12 @@ Object.freeze({
|
|
|
43616
43822
|
addonId: null,
|
|
43617
43823
|
access: "view"
|
|
43618
43824
|
},
|
|
43825
|
+
"pipelineOrchestrator.getPlacementStats": {
|
|
43826
|
+
capName: "pipeline-orchestrator",
|
|
43827
|
+
capScope: "system",
|
|
43828
|
+
addonId: null,
|
|
43829
|
+
access: "view"
|
|
43830
|
+
},
|
|
43619
43831
|
"pipelineOrchestrator.getReconcileFromDiskStatus": {
|
|
43620
43832
|
capName: "pipeline-orchestrator",
|
|
43621
43833
|
capScope: "system",
|
|
@@ -43640,7 +43852,7 @@ Object.freeze({
|
|
|
43640
43852
|
addonId: null,
|
|
43641
43853
|
access: "create"
|
|
43642
43854
|
},
|
|
43643
|
-
"pipelineOrchestrator.
|
|
43855
|
+
"pipelineOrchestrator.planAutoTune": {
|
|
43644
43856
|
capName: "pipeline-orchestrator",
|
|
43645
43857
|
capScope: "system",
|
|
43646
43858
|
addonId: null,
|
|
@@ -43706,12 +43918,6 @@ Object.freeze({
|
|
|
43706
43918
|
addonId: null,
|
|
43707
43919
|
access: "create"
|
|
43708
43920
|
},
|
|
43709
|
-
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
43710
|
-
capName: "pipeline-orchestrator",
|
|
43711
|
-
capScope: "system",
|
|
43712
|
-
addonId: null,
|
|
43713
|
-
access: "create"
|
|
43714
|
-
},
|
|
43715
43921
|
"pipelineOrchestrator.setAgentInferenceDevices": {
|
|
43716
43922
|
capName: "pipeline-orchestrator",
|
|
43717
43923
|
capScope: "system",
|
|
@@ -43766,12 +43972,30 @@ Object.freeze({
|
|
|
43766
43972
|
addonId: null,
|
|
43767
43973
|
access: "create"
|
|
43768
43974
|
},
|
|
43975
|
+
"pipelineOrchestrator.setLoadBalancingShares": {
|
|
43976
|
+
capName: "pipeline-orchestrator",
|
|
43977
|
+
capScope: "system",
|
|
43978
|
+
addonId: null,
|
|
43979
|
+
access: "create"
|
|
43980
|
+
},
|
|
43769
43981
|
"pipelineOrchestrator.setPipelineDevicePin": {
|
|
43770
43982
|
capName: "pipeline-orchestrator",
|
|
43771
43983
|
capScope: "system",
|
|
43772
43984
|
addonId: null,
|
|
43773
43985
|
access: "create"
|
|
43774
43986
|
},
|
|
43987
|
+
"pipelineOrchestrator.setShareSource": {
|
|
43988
|
+
capName: "pipeline-orchestrator",
|
|
43989
|
+
capScope: "system",
|
|
43990
|
+
addonId: null,
|
|
43991
|
+
access: "create"
|
|
43992
|
+
},
|
|
43993
|
+
"pipelineOrchestrator.suggestShares": {
|
|
43994
|
+
capName: "pipeline-orchestrator",
|
|
43995
|
+
capScope: "system",
|
|
43996
|
+
addonId: null,
|
|
43997
|
+
access: "view"
|
|
43998
|
+
},
|
|
43775
43999
|
"pipelineOrchestrator.unassignAudio": {
|
|
43776
44000
|
capName: "pipeline-orchestrator",
|
|
43777
44001
|
capScope: "system",
|
|
@@ -51004,7 +51228,7 @@ function modelBindingKey(key) {
|
|
|
51004
51228
|
/**
|
|
51005
51229
|
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
51006
51230
|
* flattened (D738). One definition for the runner (which applies the binding)
|
|
51007
|
-
* and the orchestrator's
|
|
51231
|
+
* and the orchestrator's Auto tune (which clears those overrides, D745).
|
|
51008
51232
|
*/
|
|
51009
51233
|
function collectModelBoundKeys(fields) {
|
|
51010
51234
|
return fields.flatMap((field) => {
|
|
@@ -51521,6 +51745,7 @@ function estimatedConcurrentSessions(cameraCount, options) {
|
|
|
51521
51745
|
*/
|
|
51522
51746
|
function estimateDeviceDemand(input) {
|
|
51523
51747
|
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51748
|
+
const placement = input.shares ? "share" : "least-loaded";
|
|
51524
51749
|
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51525
51750
|
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51526
51751
|
const weightByNode = /* @__PURE__ */ new Map();
|
|
@@ -51540,19 +51765,28 @@ function estimateDeviceDemand(input) {
|
|
|
51540
51765
|
const noteMax = (key, fps) => {
|
|
51541
51766
|
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51542
51767
|
};
|
|
51768
|
+
const pinNodeOf = (pin) => {
|
|
51769
|
+
if (pin.nodeId !== null) return pin.nodeId;
|
|
51770
|
+
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51771
|
+
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51772
|
+
};
|
|
51773
|
+
let alwaysOnTotal = 0;
|
|
51543
51774
|
for (const cam of input.cameras) {
|
|
51544
51775
|
if (!charges(cam)) continue;
|
|
51545
|
-
|
|
51546
|
-
|
|
51776
|
+
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51777
|
+
if (cam.devicePin !== null && pinNode !== null) {
|
|
51778
|
+
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51547
51779
|
if (!eligibleById.has(key)) continue;
|
|
51548
51780
|
bump(pinnedFps, key, cam.fps);
|
|
51549
51781
|
noteMax(key, cam.fps);
|
|
51550
51782
|
continue;
|
|
51551
51783
|
}
|
|
51552
|
-
|
|
51553
|
-
|
|
51784
|
+
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51785
|
+
if (standing !== null) {
|
|
51786
|
+
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51787
|
+
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51554
51788
|
for (const d of input.eligible) {
|
|
51555
|
-
if (d.nodeId !==
|
|
51789
|
+
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51556
51790
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51557
51791
|
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51558
51792
|
bump(alwaysOnFps, key, cam.fps * share);
|
|
@@ -51566,13 +51800,23 @@ function estimateDeviceDemand(input) {
|
|
|
51566
51800
|
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51567
51801
|
const nodeCount = weightByNode.size;
|
|
51568
51802
|
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51803
|
+
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51804
|
+
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51805
|
+
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51806
|
+
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51569
51807
|
return input.eligible.map((d) => {
|
|
51570
51808
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51571
51809
|
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51572
51810
|
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51573
51811
|
let onMotionSessions = 0;
|
|
51574
51812
|
let onMotionSource = "none";
|
|
51575
|
-
if (onMotionRates.length > 0
|
|
51813
|
+
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51814
|
+
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51815
|
+
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51816
|
+
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51817
|
+
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51818
|
+
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51819
|
+
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51576
51820
|
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51577
51821
|
onMotionSource = "observed-24h";
|
|
51578
51822
|
} else {
|
|
@@ -51590,12 +51834,13 @@ function estimateDeviceDemand(input) {
|
|
|
51590
51834
|
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51591
51835
|
onMotionSessions,
|
|
51592
51836
|
onMotionSource,
|
|
51593
|
-
observedForMs: peak?.observedForMs ?? null
|
|
51837
|
+
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51838
|
+
placement
|
|
51594
51839
|
};
|
|
51595
51840
|
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51596
51841
|
}
|
|
51597
51842
|
/** The family the cluster runs. */
|
|
51598
|
-
var
|
|
51843
|
+
var AUTO_TUNE_FAMILY = "yolov9";
|
|
51599
51844
|
function round(value, digits = 3) {
|
|
51600
51845
|
const f = 10 ** digits;
|
|
51601
51846
|
return Math.round(value * f) / f;
|
|
@@ -51608,7 +51853,7 @@ function fitsOn(cell, demand, headroom) {
|
|
|
51608
51853
|
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51609
51854
|
}
|
|
51610
51855
|
function candidatesFor(contexts, formatsByModel) {
|
|
51611
|
-
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(
|
|
51856
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(AUTO_TUNE_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51612
51857
|
modelId: id,
|
|
51613
51858
|
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51614
51859
|
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
@@ -51819,6 +52064,97 @@ function recommendNode(input) {
|
|
|
51819
52064
|
reasons: nodeReasons
|
|
51820
52065
|
};
|
|
51821
52066
|
}
|
|
52067
|
+
/** Deficits equal up to float noise (0.7 × 2 − 1 is 0.3999…, not 0.4). */
|
|
52068
|
+
var DEFICIT_EPSILON = 1e-9;
|
|
52069
|
+
function byDeficit(a, b) {
|
|
52070
|
+
const diff = b.deficit - a.deficit;
|
|
52071
|
+
return (Math.abs(diff) < DEFICIT_EPSILON ? 0 : diff) || b.candidate.share - a.candidate.share || a.candidate.key.localeCompare(b.candidate.key);
|
|
52072
|
+
}
|
|
52073
|
+
function spareOf(c) {
|
|
52074
|
+
return c.spare ?? Number.POSITIVE_INFINITY;
|
|
52075
|
+
}
|
|
52076
|
+
function electByShare(candidates) {
|
|
52077
|
+
if (candidates.length === 0) return { kind: "none" };
|
|
52078
|
+
const shareSum = candidates.reduce((s, c) => s + Math.max(c.share, 0), 0);
|
|
52079
|
+
const running = candidates.reduce((s, c) => s + Math.max(c.running, 0), 0);
|
|
52080
|
+
const scored = candidates.map((candidate) => ({
|
|
52081
|
+
candidate,
|
|
52082
|
+
deficit: (shareSum > 0 ? Math.max(candidate.share, 0) / shareSum : 1 / candidates.length) * (running + 1) - candidate.running
|
|
52083
|
+
})).toSorted(byDeficit);
|
|
52084
|
+
const turn = scored[0].candidate;
|
|
52085
|
+
if (!turn.capped && !turn.saturated) return {
|
|
52086
|
+
kind: "elected",
|
|
52087
|
+
key: turn.key,
|
|
52088
|
+
reason: "share"
|
|
52089
|
+
};
|
|
52090
|
+
const free = scored.filter((s) => !s.candidate.capped && !s.candidate.saturated).toSorted((a, b) => spareOf(b.candidate) - spareOf(a.candidate) || byDeficit(a, b));
|
|
52091
|
+
const yieldReason = turn.capped ? "at-cap" : "saturated";
|
|
52092
|
+
if (free.length > 0) return {
|
|
52093
|
+
kind: "elected",
|
|
52094
|
+
key: free[0].candidate.key,
|
|
52095
|
+
reason: "yield",
|
|
52096
|
+
turnKey: turn.key,
|
|
52097
|
+
yieldReason
|
|
52098
|
+
};
|
|
52099
|
+
const uncapped = scored.find((s) => !s.candidate.capped);
|
|
52100
|
+
if (uncapped === void 0) return { kind: "none" };
|
|
52101
|
+
return {
|
|
52102
|
+
kind: "elected",
|
|
52103
|
+
key: uncapped.candidate.key,
|
|
52104
|
+
reason: "share"
|
|
52105
|
+
};
|
|
52106
|
+
}
|
|
52107
|
+
/** Weight a node or device has when its source names none (the balancer's own default). */
|
|
52108
|
+
var DEFAULT_WEIGHT = 1;
|
|
52109
|
+
function positive(value) {
|
|
52110
|
+
return value !== void 0 && value > 0 ? value : DEFAULT_WEIGHT;
|
|
52111
|
+
}
|
|
52112
|
+
function percentages(keys, weightOf, counts) {
|
|
52113
|
+
const total = keys.reduce((sum, k) => sum + (counts(k) ? weightOf(k) : 0), 0);
|
|
52114
|
+
return new Map(keys.map((k) => [k, total > 0 && counts(k) ? Math.round(weightOf(k) / total * 1e3) / 10 : 0]));
|
|
52115
|
+
}
|
|
52116
|
+
function resolveEffectiveShares(input) {
|
|
52117
|
+
const weights = input.source === "auto" && input.auto !== null && input.auto !== null ? input.auto : input.stored;
|
|
52118
|
+
const nodeWeights = {};
|
|
52119
|
+
for (const nodeId of input.nodeIds) nodeWeights[nodeId] = positive(weights.nodes[nodeId]);
|
|
52120
|
+
const eligibleNodes = new Set(input.eligibleNodes);
|
|
52121
|
+
const configured = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52122
|
+
const effective = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(k));
|
|
52123
|
+
const deviceWeights = {};
|
|
52124
|
+
const nodes = input.nodeIds.map((nodeId) => {
|
|
52125
|
+
const eligibleKeys = new Set(input.eligibleDevices[nodeId] ?? []);
|
|
52126
|
+
const named = weights.devices[nodeId] ?? {};
|
|
52127
|
+
const keys = [...new Set([...Object.keys(named), ...eligibleKeys])].toSorted();
|
|
52128
|
+
const perDevice = {};
|
|
52129
|
+
for (const key of keys) perDevice[key] = positive(named[key]);
|
|
52130
|
+
if (keys.length > 0) deviceWeights[nodeId] = perDevice;
|
|
52131
|
+
const devConfigured = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52132
|
+
const devEffective = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(nodeId) && eligibleKeys.has(k));
|
|
52133
|
+
const devices = keys.map((deviceKey) => ({
|
|
52134
|
+
deviceKey,
|
|
52135
|
+
eligible: eligibleKeys.has(deviceKey),
|
|
52136
|
+
configuredPct: devConfigured.get(deviceKey) ?? 0,
|
|
52137
|
+
effectivePct: devEffective.get(deviceKey) ?? 0
|
|
52138
|
+
}));
|
|
52139
|
+
return {
|
|
52140
|
+
nodeId,
|
|
52141
|
+
eligible: eligibleNodes.has(nodeId),
|
|
52142
|
+
configuredPct: configured.get(nodeId) ?? 0,
|
|
52143
|
+
effectivePct: effective.get(nodeId) ?? 0,
|
|
52144
|
+
devices
|
|
52145
|
+
};
|
|
52146
|
+
});
|
|
52147
|
+
return {
|
|
52148
|
+
source: input.source,
|
|
52149
|
+
policy: input.policy ?? "share",
|
|
52150
|
+
autoPending: input.source === "auto" && input.auto === null,
|
|
52151
|
+
computedAt: input.computedAt ?? null,
|
|
52152
|
+
trigger: input.trigger ?? null,
|
|
52153
|
+
nodes,
|
|
52154
|
+
nodeWeights,
|
|
52155
|
+
deviceWeights
|
|
52156
|
+
};
|
|
52157
|
+
}
|
|
51822
52158
|
/**
|
|
51823
52159
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
51824
52160
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -59881,6 +60217,35 @@ function pickInferenceDevice(input) {
|
|
|
59881
60217
|
const loadByKey = new Map((input.loads ?? []).map((l) => [l.deviceKey, l]));
|
|
59882
60218
|
const healthyKeys = eligibleKeys.filter((k) => !isDeviceSaturated(loadByKey.get(k)));
|
|
59883
60219
|
const consideredKeys = pin !== null || healthyKeys.length === 0 || healthyKeys.length === eligibleKeys.length ? eligibleKeys : healthyKeys;
|
|
60220
|
+
if (input.policy === "share" && pin === null) {
|
|
60221
|
+
const election = electByShare(eligibleKeys.map((key) => {
|
|
60222
|
+
const load = loadByKey.get(key);
|
|
60223
|
+
const attached = load?.attachedCameras ?? 0;
|
|
60224
|
+
const cap = input.caps?.[key];
|
|
60225
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
60226
|
+
return {
|
|
60227
|
+
key,
|
|
60228
|
+
share: input.eligible[key] ?? 1,
|
|
60229
|
+
running: attached,
|
|
60230
|
+
capped: hasCap && attached >= cap,
|
|
60231
|
+
saturated: isDeviceSaturated(load),
|
|
60232
|
+
...hasCap ? { spare: Math.max(0, cap - attached) } : {}
|
|
60233
|
+
};
|
|
60234
|
+
}));
|
|
60235
|
+
if (election.kind === "none") return {
|
|
60236
|
+
kind: "default-pool",
|
|
60237
|
+
reason: "all-devices-at-cap",
|
|
60238
|
+
eligibleKeys
|
|
60239
|
+
};
|
|
60240
|
+
return {
|
|
60241
|
+
kind: "selected",
|
|
60242
|
+
deviceKey: election.key,
|
|
60243
|
+
reason: "share",
|
|
60244
|
+
tieBreak: null,
|
|
60245
|
+
eligibleKeys,
|
|
60246
|
+
tiedKeys: [election.key]
|
|
60247
|
+
};
|
|
60248
|
+
}
|
|
59884
60249
|
const balancerInput = {
|
|
59885
60250
|
nodes: consideredKeys.map((k) => deviceAsLoad(k, loadByKey.get(k))),
|
|
59886
60251
|
preferredAgent: pin,
|
|
@@ -62184,6 +62549,7 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62184
62549
|
};
|
|
62185
62550
|
const okByNode = new Map(input.verdicts.filter((v) => v.state === "ok").map((v) => [v.nodeId, v]));
|
|
62186
62551
|
const sourceAttached = input.attachedByNode[saturated.nodeId] ?? 0;
|
|
62552
|
+
if (input.shares !== void 0) return planShareTarget(input, input.shares, saturated, movable, okByNode, now, guards);
|
|
62187
62553
|
for (const camera of movable) {
|
|
62188
62554
|
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62189
62555
|
const target = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => {
|
|
@@ -62208,6 +62574,52 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62208
62574
|
nodeId: saturated.nodeId
|
|
62209
62575
|
};
|
|
62210
62576
|
}
|
|
62577
|
+
/** Deficits equal up to float noise. */
|
|
62578
|
+
var SHARE_EPSILON = 1e-9;
|
|
62579
|
+
function underCap(input, nodeId) {
|
|
62580
|
+
const cap = input.nodeCaps?.[nodeId];
|
|
62581
|
+
const attached = input.attachedByNode[nodeId] ?? 0;
|
|
62582
|
+
return cap === null || cap === void 0 || cap <= 0 || attached < cap;
|
|
62583
|
+
}
|
|
62584
|
+
/**
|
|
62585
|
+
* Share placement's failover target (D750): among the camera's eligible,
|
|
62586
|
+
* `ok`, under-cap nodes, the one furthest below its share of the cameras
|
|
62587
|
+
* attached across the source and those nodes — the same deficit rule a new
|
|
62588
|
+
* session is placed by. A node at or over its share is no target, so a move
|
|
62589
|
+
* never undoes the balance. Nothing below its share ⇒ hold, naming the camera.
|
|
62590
|
+
*/
|
|
62591
|
+
function planShareTarget(input, shares, saturated, movable, okByNode, now, guards) {
|
|
62592
|
+
const shareOf = (nodeId) => {
|
|
62593
|
+
const share = shares[nodeId];
|
|
62594
|
+
return share !== void 0 && share > 0 ? share : 0;
|
|
62595
|
+
};
|
|
62596
|
+
for (const camera of movable) {
|
|
62597
|
+
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62598
|
+
const healthy = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => underCap(input, v.nodeId));
|
|
62599
|
+
const set = [saturated.nodeId, ...healthy.map((v) => v.nodeId)];
|
|
62600
|
+
const shareSum = set.reduce((sum, nodeId) => sum + shareOf(nodeId), 0);
|
|
62601
|
+
const attachedSum = set.reduce((sum, nodeId) => sum + (input.attachedByNode[nodeId] ?? 0), 0);
|
|
62602
|
+
if (shareSum <= 0) continue;
|
|
62603
|
+
const deficit = (nodeId) => shareOf(nodeId) / shareSum * attachedSum - (input.attachedByNode[nodeId] ?? 0);
|
|
62604
|
+
const target = healthy.filter((v) => deficit(v.nodeId) > SHARE_EPSILON).toSorted((a, b) => deficit(b.nodeId) - deficit(a.nodeId) || a.signals.score - b.signals.score || a.nodeId.localeCompare(b.nodeId))[0];
|
|
62605
|
+
if (!target) continue;
|
|
62606
|
+
return {
|
|
62607
|
+
kind: "move",
|
|
62608
|
+
deviceId: camera.deviceId,
|
|
62609
|
+
fromNodeId: saturated.nodeId,
|
|
62610
|
+
toNodeId: target.nodeId,
|
|
62611
|
+
fromScore: saturated.signals.score,
|
|
62612
|
+
toScore: target.signals.score
|
|
62613
|
+
};
|
|
62614
|
+
}
|
|
62615
|
+
const first = movable[0];
|
|
62616
|
+
return {
|
|
62617
|
+
kind: "hold",
|
|
62618
|
+
reason: "no-target",
|
|
62619
|
+
nodeId: saturated.nodeId,
|
|
62620
|
+
...first !== void 0 ? { deviceId: first.deviceId } : {}
|
|
62621
|
+
};
|
|
62622
|
+
}
|
|
62211
62623
|
//#endregion
|
|
62212
62624
|
//#region src/node-stress-controller.ts
|
|
62213
62625
|
/** Narrow an unknown settings value to a {@link NodeStressMode}. */
|
|
@@ -62416,17 +62828,27 @@ var NodeStressController = class NodeStressController {
|
|
|
62416
62828
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
62417
62829
|
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
62418
62830
|
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
62831
|
+
const candidates = await this.collectCandidates(now);
|
|
62832
|
+
const shares = await this.deps.readFailoverShares?.() ?? null;
|
|
62419
62833
|
const plan = planStressFailover({
|
|
62420
62834
|
verdicts,
|
|
62421
|
-
candidates
|
|
62835
|
+
candidates,
|
|
62422
62836
|
history,
|
|
62423
62837
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
62424
|
-
attachedByNode: this.attachedByNode()
|
|
62838
|
+
attachedByNode: this.attachedByNode(),
|
|
62839
|
+
...shares !== null ? { shares } : {}
|
|
62425
62840
|
}, now, this.guards());
|
|
62426
62841
|
if (plan.kind === "hold") {
|
|
62427
62842
|
this.deps.logger.warn("node saturated — no failover applied", {
|
|
62428
|
-
tags: {
|
|
62429
|
-
|
|
62843
|
+
tags: {
|
|
62844
|
+
nodeId: plan.nodeId ?? "",
|
|
62845
|
+
...plan.deviceId !== void 0 ? { deviceId: plan.deviceId } : {}
|
|
62846
|
+
},
|
|
62847
|
+
meta: {
|
|
62848
|
+
reason: plan.reason,
|
|
62849
|
+
rule: shares !== null ? "share-deficit" : "fewest-cameras",
|
|
62850
|
+
...plan.reason === "no-target" && shares !== null ? { why: "no healthy node below its share can take it" } : {}
|
|
62851
|
+
}
|
|
62430
62852
|
});
|
|
62431
62853
|
return;
|
|
62432
62854
|
}
|
|
@@ -64204,7 +64626,6 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
64204
64626
|
}).nullable()
|
|
64205
64627
|
});
|
|
64206
64628
|
var StoredCameraPipelineSettingsSchema = object({
|
|
64207
|
-
pinnedAgentNodeId: string().optional(),
|
|
64208
64629
|
stepToggles: record(string(), boolean()).optional(),
|
|
64209
64630
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
64210
64631
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
@@ -65039,12 +65460,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65039
65460
|
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
65040
65461
|
*/
|
|
65041
65462
|
async buildNodeWeights() {
|
|
65463
|
+
if (this.effectiveNodeWeights !== null) return this.effectiveNodeWeights();
|
|
65464
|
+
return this.buildStoredNodeWeights();
|
|
65465
|
+
}
|
|
65466
|
+
/**
|
|
65467
|
+
* The operator's stored node sliders, whatever the share source — what
|
|
65468
|
+
* `manual` applies and what `auto` falls back to before its first answer.
|
|
65469
|
+
*/
|
|
65470
|
+
async buildStoredNodeWeights() {
|
|
65042
65471
|
const blob = await this.agentSettingsState.get();
|
|
65043
65472
|
const weights = {};
|
|
65044
65473
|
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
65045
65474
|
return weights;
|
|
65046
65475
|
}
|
|
65047
65476
|
/**
|
|
65477
|
+
* Where {@link buildNodeWeights} reads from once the share source is wired
|
|
65478
|
+
* (D750): the effective shares (automatic or manual). Every balancer, the
|
|
65479
|
+
* share election and the stress failover read through it, so they can never
|
|
65480
|
+
* disagree on what a node's share is. Unset (tests, early boot) ⇒ stored.
|
|
65481
|
+
*/
|
|
65482
|
+
effectiveNodeWeights = null;
|
|
65483
|
+
setEffectiveNodeWeights(read) {
|
|
65484
|
+
this.effectiveNodeWeights = read;
|
|
65485
|
+
}
|
|
65486
|
+
/**
|
|
65048
65487
|
* The per-(node,device) BASE provisioning (C7.2/C7.4) for one `(node,
|
|
65049
65488
|
* deviceKey)` — the effective `stepId → {modelId?, settings?}` map the
|
|
65050
65489
|
* dispatch provisioning stage attaches onto the resolved step tree before
|
|
@@ -66708,7 +67147,15 @@ var TtlMemo = class {
|
|
|
66708
67147
|
//#endregion
|
|
66709
67148
|
//#region src/detection-session-registry.ts
|
|
66710
67149
|
var DetectionSessionRegistry = class {
|
|
67150
|
+
onChange;
|
|
66711
67151
|
sessions = /* @__PURE__ */ new Map();
|
|
67152
|
+
/**
|
|
67153
|
+
* @param onChange called after every register / deregister / clear with the
|
|
67154
|
+
* running set — feeds the placement statistics (D748). Never on a no-op.
|
|
67155
|
+
*/
|
|
67156
|
+
constructor(onChange = () => {}) {
|
|
67157
|
+
this.onChange = onChange;
|
|
67158
|
+
}
|
|
66712
67159
|
register(deviceId, nodeId, startedAt, triggeredBy, deviceKey) {
|
|
66713
67160
|
const existing = this.sessions.get(deviceId);
|
|
66714
67161
|
if (existing?.teardownTimer) clearTimeout(existing.teardownTimer);
|
|
@@ -66721,6 +67168,7 @@ var DetectionSessionRegistry = class {
|
|
|
66721
67168
|
...deviceKey ? { deviceKey } : {}
|
|
66722
67169
|
};
|
|
66723
67170
|
this.sessions.set(deviceId, session);
|
|
67171
|
+
this.onChange(this.list());
|
|
66724
67172
|
return session;
|
|
66725
67173
|
}
|
|
66726
67174
|
get(deviceId) {
|
|
@@ -66734,6 +67182,7 @@ var DetectionSessionRegistry = class {
|
|
|
66734
67182
|
if (!session) return void 0;
|
|
66735
67183
|
if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66736
67184
|
this.sessions.delete(deviceId);
|
|
67185
|
+
this.onChange(this.list());
|
|
66737
67186
|
return session;
|
|
66738
67187
|
}
|
|
66739
67188
|
setTeardownTimer(deviceId, timer) {
|
|
@@ -66756,9 +67205,39 @@ var DetectionSessionRegistry = class {
|
|
|
66756
67205
|
}
|
|
66757
67206
|
clear() {
|
|
66758
67207
|
for (const session of this.sessions.values()) if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
67208
|
+
const hadSessions = this.sessions.size > 0;
|
|
66759
67209
|
this.sessions.clear();
|
|
67210
|
+
if (hadSessions) this.onChange([]);
|
|
66760
67211
|
}
|
|
66761
67212
|
};
|
|
67213
|
+
//#endregion
|
|
67214
|
+
//#region src/session-share-placement.ts
|
|
67215
|
+
function nodeSaturated(load) {
|
|
67216
|
+
return load.devices.length > 0 && load.devices.every((d) => isDeviceSaturated(d));
|
|
67217
|
+
}
|
|
67218
|
+
function electNodeByShare(input) {
|
|
67219
|
+
return electByShare(input.loads.filter((l) => l.nodeId.length > 0 && input.eligible.includes(l.nodeId)).map((load) => {
|
|
67220
|
+
const cap = input.caps[load.nodeId];
|
|
67221
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
67222
|
+
const weight = input.weights[load.nodeId];
|
|
67223
|
+
return {
|
|
67224
|
+
key: load.nodeId,
|
|
67225
|
+
share: weight !== void 0 && weight > 0 ? weight : 1,
|
|
67226
|
+
running: input.runningByNode.get(load.nodeId) ?? 0,
|
|
67227
|
+
capped: hasCap && load.attachedCameras >= cap,
|
|
67228
|
+
saturated: nodeSaturated(load),
|
|
67229
|
+
...hasCap ? { spare: Math.max(0, cap - load.attachedCameras) } : {}
|
|
67230
|
+
};
|
|
67231
|
+
}));
|
|
67232
|
+
}
|
|
67233
|
+
/** Parallel sessions per node: sessions and standing cameras, each camera once. */
|
|
67234
|
+
function countRunningByNode(placements) {
|
|
67235
|
+
const seen = /* @__PURE__ */ new Map();
|
|
67236
|
+
for (const p of placements) if (!seen.has(p.deviceId)) seen.set(p.deviceId, p.nodeId);
|
|
67237
|
+
const out = /* @__PURE__ */ new Map();
|
|
67238
|
+
for (const nodeId of seen.values()) out.set(nodeId, (out.get(nodeId) ?? 0) + 1);
|
|
67239
|
+
return out;
|
|
67240
|
+
}
|
|
66762
67241
|
/**
|
|
66763
67242
|
* Minimum time a detection session stays open, whatever closes it.
|
|
66764
67243
|
*
|
|
@@ -66871,7 +67350,15 @@ var SessionDispatchController = class {
|
|
|
66871
67350
|
* (not `private`) — see the class doc for the two production/test reasons
|
|
66872
67351
|
* `index.ts` reaches through it directly.
|
|
66873
67352
|
*/
|
|
66874
|
-
sessionRegistry = new DetectionSessionRegistry()
|
|
67353
|
+
sessionRegistry = new DetectionSessionRegistry((sessions) => this.deps.placementStats?.noteSessions(sessions.map((s) => ({
|
|
67354
|
+
nodeId: s.nodeId,
|
|
67355
|
+
deviceKey: s.deviceKey ?? ""
|
|
67356
|
+
}))));
|
|
67357
|
+
/**
|
|
67358
|
+
* `deviceId|deviceKey` of device pins the session path has already said it
|
|
67359
|
+
* ignored — one line per stale pin, not one per session (D748).
|
|
67360
|
+
*/
|
|
67361
|
+
reportedStalePins = /* @__PURE__ */ new Set();
|
|
66875
67362
|
/**
|
|
66876
67363
|
* Per-device async mutex for session dispatch/teardown critical sections —
|
|
66877
67364
|
* backed by the generic `KeyedAsyncLock` (mirrors the audio controller's
|
|
@@ -67332,6 +67819,77 @@ var SessionDispatchController = class {
|
|
|
67332
67819
|
});
|
|
67333
67820
|
}
|
|
67334
67821
|
/**
|
|
67822
|
+
* The session's node. `least-loaded` (and any node pin, under either
|
|
67823
|
+
* policy) is `balance()` exactly as before. `share` holds the node shares
|
|
67824
|
+
* of PARALLEL sessions (D748): sessions now running and standing cameras
|
|
67825
|
+
* count toward their node, and a node at its cap or drowning yields —
|
|
67826
|
+
* logged with the reason, and counted against the node in the stats.
|
|
67827
|
+
*/
|
|
67828
|
+
electSessionNode(input) {
|
|
67829
|
+
const balanced = () => balance({
|
|
67830
|
+
nodes: input.loads,
|
|
67831
|
+
preferredAgent: input.preferredAgent,
|
|
67832
|
+
nodeCaps: input.nodeCaps,
|
|
67833
|
+
weights: input.weights,
|
|
67834
|
+
eligibleNodes: input.eligible
|
|
67835
|
+
});
|
|
67836
|
+
if (input.policy !== "share" || input.preferredAgent) return balanced();
|
|
67837
|
+
const running = countRunningByNode([...this.sessionRegistry.list().map((s) => ({
|
|
67838
|
+
deviceId: s.deviceId,
|
|
67839
|
+
nodeId: s.nodeId
|
|
67840
|
+
})), ...this.deps.ledger.listAssignments().map((a) => ({
|
|
67841
|
+
deviceId: a.deviceId,
|
|
67842
|
+
nodeId: a.agentNodeId
|
|
67843
|
+
}))]);
|
|
67844
|
+
const present = new Set(this.deps.topology.knownRunnerNodeIds());
|
|
67845
|
+
const election = electNodeByShare({
|
|
67846
|
+
loads: input.loads,
|
|
67847
|
+
eligible: input.eligible.filter((nodeId) => present.has(nodeId)),
|
|
67848
|
+
weights: input.weights,
|
|
67849
|
+
caps: input.nodeCaps,
|
|
67850
|
+
runningByNode: running
|
|
67851
|
+
});
|
|
67852
|
+
if (election.kind === "none") return balanced();
|
|
67853
|
+
if (election.reason === "yield") {
|
|
67854
|
+
this.deps.placementStats?.noteYield(election.turnKey);
|
|
67855
|
+
this.deps.logger.info("detection session: share turn yielded — the node had no room", {
|
|
67856
|
+
tags: {
|
|
67857
|
+
deviceId: input.deviceId,
|
|
67858
|
+
nodeId: election.key
|
|
67859
|
+
},
|
|
67860
|
+
meta: {
|
|
67861
|
+
turnNodeId: election.turnKey,
|
|
67862
|
+
reason: election.yieldReason
|
|
67863
|
+
}
|
|
67864
|
+
});
|
|
67865
|
+
}
|
|
67866
|
+
const load = input.loads.find((l) => l.nodeId === election.key);
|
|
67867
|
+
return {
|
|
67868
|
+
kind: "assigned",
|
|
67869
|
+
agentNodeId: election.key,
|
|
67870
|
+
reason: "capacity",
|
|
67871
|
+
score: load ? load.attachedCameras + load.queueDepthTotal : 0
|
|
67872
|
+
};
|
|
67873
|
+
}
|
|
67874
|
+
/**
|
|
67875
|
+
* Say ONCE that a camera's device pin names a device this node does not
|
|
67876
|
+
* offer (disabled, absent): the session path ignores it, and the Load
|
|
67877
|
+
* balancing tab offers to clear it. Never cleared here (D748).
|
|
67878
|
+
*/
|
|
67879
|
+
noteStalePin(deviceId, pinnedDeviceKey, enabledDevices) {
|
|
67880
|
+
if (pinnedDeviceKey === null || pinnedDeviceKey in enabledDevices) return;
|
|
67881
|
+
const key = `${deviceId}|${pinnedDeviceKey}`;
|
|
67882
|
+
if (this.reportedStalePins.has(key)) return;
|
|
67883
|
+
this.reportedStalePins.add(key);
|
|
67884
|
+
this.deps.logger.warn("device pin ignored — the pinned device is not enabled on the elected node", {
|
|
67885
|
+
tags: { deviceId },
|
|
67886
|
+
meta: {
|
|
67887
|
+
pin: pinnedDeviceKey,
|
|
67888
|
+
enabled: Object.keys(enabledDevices).join(",")
|
|
67889
|
+
}
|
|
67890
|
+
});
|
|
67891
|
+
}
|
|
67892
|
+
/**
|
|
67335
67893
|
* Dispatch a session-time detection attach for an on-motion camera whose
|
|
67336
67894
|
* `decideSessionAction` returned `'dispatch'`. Builds a session
|
|
67337
67895
|
* `RunnerCameraConfig` the same way `startDetection` does, but forces
|
|
@@ -67382,12 +67940,17 @@ var SessionDispatchController = class {
|
|
|
67382
67940
|
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
67383
67941
|
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
67384
67942
|
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
67385
|
-
const
|
|
67386
|
-
|
|
67943
|
+
const nodeCaps = await this.nodeCaps.get(CLUSTER_KEY);
|
|
67944
|
+
const weights = await this.nodeWeights.get(CLUSTER_KEY);
|
|
67945
|
+
const policy = this.deps.sessionPlacement?.() ?? "least-loaded";
|
|
67946
|
+
const decision = this.electSessionNode({
|
|
67947
|
+
deviceId,
|
|
67948
|
+
policy,
|
|
67949
|
+
loads,
|
|
67387
67950
|
preferredAgent,
|
|
67388
|
-
nodeCaps
|
|
67389
|
-
weights
|
|
67390
|
-
|
|
67951
|
+
nodeCaps,
|
|
67952
|
+
weights,
|
|
67953
|
+
eligible
|
|
67391
67954
|
});
|
|
67392
67955
|
if (!decision || decision.kind !== "assigned") {
|
|
67393
67956
|
log.warn("detection session: no eligible node — will retry on next motion", { meta: { reason: decision?.kind === "pending" ? decision.reason : "no-runners-online" } });
|
|
@@ -67400,12 +67963,14 @@ var SessionDispatchController = class {
|
|
|
67400
67963
|
const targetLoad = loads.find((n) => n.nodeId === targetNodeId);
|
|
67401
67964
|
const deviceCaps = await this.inferenceDeviceCaps.get(targetNodeId);
|
|
67402
67965
|
const pinnedDeviceKey = await this.deps.readPipelineDevicePin(deviceId);
|
|
67966
|
+
this.noteStalePin(deviceId, pinnedDeviceKey, enabledDevices);
|
|
67403
67967
|
const selection = pickInferenceDevice({
|
|
67404
67968
|
eligible: enabledDevices,
|
|
67405
67969
|
loads: targetLoad?.devices ?? [],
|
|
67406
67970
|
caps: deviceCaps,
|
|
67407
|
-
pinnedDeviceKey,
|
|
67408
|
-
rotation: this.deps.inferenceRotation
|
|
67971
|
+
pinnedDeviceKey: pinnedDeviceKey !== null && pinnedDeviceKey in enabledDevices ? pinnedDeviceKey : null,
|
|
67972
|
+
rotation: this.deps.inferenceRotation,
|
|
67973
|
+
policy
|
|
67409
67974
|
});
|
|
67410
67975
|
const deviceKey = selectedDeviceKey(selection);
|
|
67411
67976
|
log.info("inference device selection", {
|
|
@@ -67493,6 +68058,7 @@ var SessionDispatchController = class {
|
|
|
67493
68058
|
attachedAt = Date.now();
|
|
67494
68059
|
this.deps.recordAssignment(deviceId, targetNodeId, decision.reason, decision.reason === "manual");
|
|
67495
68060
|
this.sessionRegistry.register(deviceId, targetNodeId, Date.now(), triggeredBy, deviceKey);
|
|
68061
|
+
this.deps.placementStats?.noteStart(targetNodeId, deviceKey ?? "");
|
|
67496
68062
|
if (this.sessionRefusals.delete(deviceId)) log.info("detection session refusal cleared — a session opened", {
|
|
67497
68063
|
tags: {
|
|
67498
68064
|
deviceId,
|
|
@@ -68747,7 +69313,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68747
69313
|
attach: (nodeId, config) => placement.attachOn(nodeId, config),
|
|
68748
69314
|
detach: (nodeId, deviceId) => placement.detachOn(nodeId, deviceId),
|
|
68749
69315
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
68750
|
-
readGlobalSettings: () => globalSettings
|
|
69316
|
+
readGlobalSettings: () => globalSettings,
|
|
69317
|
+
readFailoverShares: async () => readSessionPlacement() === "share" ? settingsStore.buildNodeWeights() : null
|
|
68751
69318
|
});
|
|
68752
69319
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
68753
69320
|
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
@@ -68762,6 +69329,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68762
69329
|
}, 6e4);
|
|
68763
69330
|
nodeStress.start();
|
|
68764
69331
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69332
|
+
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
69333
|
+
const readShareSource = () => parseShareSource(globalSettings?.[SHARE_SOURCE_SETTING_KEY]);
|
|
68765
69334
|
/**
|
|
68766
69335
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
68767
69336
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -68845,6 +69414,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68845
69414
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
68846
69415
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
68847
69416
|
inferenceRotation,
|
|
69417
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69418
|
+
placementStats: deps.placementStats,
|
|
68848
69419
|
logger: deps.ctx().logger
|
|
68849
69420
|
});
|
|
68850
69421
|
const deviceConfig = new DeviceConfigContributions({
|
|
@@ -69096,6 +69667,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69096
69667
|
nodeLifecycle.unsubscribeDetectionPipelineReadiness(nodeId);
|
|
69097
69668
|
audio.markNodeOffline(nodeId);
|
|
69098
69669
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
69670
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-offline:${nodeId}`);
|
|
69099
69671
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
69100
69672
|
const msg = errMsg$2(err);
|
|
69101
69673
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
@@ -69139,6 +69711,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69139
69711
|
});
|
|
69140
69712
|
});
|
|
69141
69713
|
reconcile.scheduleReconcile();
|
|
69714
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-online:${nodeId}`);
|
|
69142
69715
|
});
|
|
69143
69716
|
const unsubAgentOnline = () => {
|
|
69144
69717
|
unsubAgentOnlineEvent();
|
|
@@ -69378,7 +69951,9 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69378
69951
|
unsubAgentOffline,
|
|
69379
69952
|
unsubOrchestratorSubscriptions,
|
|
69380
69953
|
pendingRetryTimer,
|
|
69381
|
-
autoRebalanceTimer
|
|
69954
|
+
autoRebalanceTimer,
|
|
69955
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69956
|
+
shareSource: () => readShareSource()
|
|
69382
69957
|
};
|
|
69383
69958
|
}
|
|
69384
69959
|
//#endregion
|
|
@@ -69423,6 +69998,20 @@ function buildGlobalSettingsSections(options) {
|
|
|
69423
69998
|
title: "Load Balancer",
|
|
69424
69999
|
tab: "pipeline",
|
|
69425
70000
|
fields: [
|
|
70001
|
+
{
|
|
70002
|
+
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
70003
|
+
type: "select",
|
|
70004
|
+
label: "Session placement",
|
|
70005
|
+
description: "How a new detection session picks its node and device. Honour shares (default) = the share of PARALLEL sessions is held to the node and device shares, and a node at its cap yields to a free one. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy.",
|
|
70006
|
+
options: [{
|
|
70007
|
+
value: "share",
|
|
70008
|
+
label: "Honour shares"
|
|
70009
|
+
}, {
|
|
70010
|
+
value: "least-loaded",
|
|
70011
|
+
label: "Least-loaded"
|
|
70012
|
+
}],
|
|
70013
|
+
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
70014
|
+
},
|
|
69426
70015
|
{
|
|
69427
70016
|
key: "fpsMinThreshold",
|
|
69428
70017
|
type: "slider",
|
|
@@ -69981,9 +70570,9 @@ var pipelineOrchestratorWidgets = [{
|
|
|
69981
70570
|
defaultRows: 4
|
|
69982
70571
|
}];
|
|
69983
70572
|
//#endregion
|
|
69984
|
-
//#region src/
|
|
70573
|
+
//#region src/auto-tune/plan-diff.ts
|
|
69985
70574
|
/**
|
|
69986
|
-
* The
|
|
70575
|
+
* The Auto tune writes, as pure transforms of the orchestrator's two
|
|
69987
70576
|
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
69988
70577
|
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
69989
70578
|
*
|
|
@@ -70002,12 +70591,12 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70002
70591
|
* settings: the attach provisions the node pin over the snapshot
|
|
70003
70592
|
* (`applyDeviceProvisioning`: override ?? base), so once the snapshot pin
|
|
70004
70593
|
* is gone the node's recommended model runs.
|
|
70005
|
-
* - never: `stepToggles`,
|
|
70594
|
+
* - never: `stepToggles`, devices' enablement, pins of
|
|
70006
70595
|
* embedding steps (cluster-scoped, changing one forces a re-index).
|
|
70007
70596
|
*/
|
|
70008
70597
|
/** The root detector step. */
|
|
70009
70598
|
var DETECTOR_STEP_ID = "object-detection";
|
|
70010
|
-
/** Steps whose `modelId` pin
|
|
70599
|
+
/** Steps whose `modelId` pin Auto tune never touches. */
|
|
70011
70600
|
function isModelPinKept(stepId) {
|
|
70012
70601
|
return stepId.endsWith("-embedding");
|
|
70013
70602
|
}
|
|
@@ -70059,7 +70648,7 @@ function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
|
70059
70648
|
};
|
|
70060
70649
|
}
|
|
70061
70650
|
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70062
|
-
function
|
|
70651
|
+
function applyNodeAutoTune(blob, input) {
|
|
70063
70652
|
const next = { ...blob };
|
|
70064
70653
|
const changes = [];
|
|
70065
70654
|
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
@@ -70129,7 +70718,7 @@ function scrubTree(steps, input, onClear) {
|
|
|
70129
70718
|
});
|
|
70130
70719
|
}
|
|
70131
70720
|
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70132
|
-
function
|
|
70721
|
+
function applyCameraAutoTune(blob, input) {
|
|
70133
70722
|
const next = {};
|
|
70134
70723
|
const clears = [];
|
|
70135
70724
|
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
@@ -70213,12 +70802,12 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70213
70802
|
};
|
|
70214
70803
|
}
|
|
70215
70804
|
//#endregion
|
|
70216
|
-
//#region src/
|
|
70805
|
+
//#region src/auto-tune/auto-tune-service.ts
|
|
70217
70806
|
/**
|
|
70218
|
-
*
|
|
70807
|
+
* Auto tune (D745): gather the inputs, compute the plan, and apply exactly
|
|
70219
70808
|
* the plan the operator confirmed.
|
|
70220
70809
|
*
|
|
70221
|
-
* The plan is the recommender (`@camstack/types`
|
|
70810
|
+
* The plan is the recommender (`@camstack/types` auto-tune) over every
|
|
70222
70811
|
* runner node, plus the diff of the writes (`plan-diff.ts`) and the
|
|
70223
70812
|
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70224
70813
|
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
@@ -70264,16 +70853,117 @@ function boundKeysOf(catalogs) {
|
|
|
70264
70853
|
}
|
|
70265
70854
|
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70266
70855
|
}
|
|
70267
|
-
|
|
70856
|
+
/** Bounded fixed point of the hardware share suggestion (D748). */
|
|
70857
|
+
var SUGGEST_MAX_ROUNDS = 3;
|
|
70858
|
+
/** Integer shares summing to 100 (largest remainder). */
|
|
70859
|
+
function toPercent(weights) {
|
|
70860
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70861
|
+
const out = /* @__PURE__ */ new Map();
|
|
70862
|
+
if (total <= 0) return out;
|
|
70863
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
70864
|
+
k,
|
|
70865
|
+
exact: w / total * 100
|
|
70866
|
+
}));
|
|
70867
|
+
let assigned = 0;
|
|
70868
|
+
for (const r of raw) {
|
|
70869
|
+
out.set(r.k, Math.floor(r.exact));
|
|
70870
|
+
assigned += Math.floor(r.exact);
|
|
70871
|
+
}
|
|
70872
|
+
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70873
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
70874
|
+
const r = byRemainder[i % byRemainder.length];
|
|
70875
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70876
|
+
}
|
|
70877
|
+
return out;
|
|
70878
|
+
}
|
|
70879
|
+
var AutoTuneService = class {
|
|
70268
70880
|
deps;
|
|
70269
70881
|
applying = false;
|
|
70270
70882
|
constructor(deps) {
|
|
70271
70883
|
this.deps = deps;
|
|
70272
70884
|
}
|
|
70273
70885
|
async plan(input) {
|
|
70274
|
-
return (await this.computePlan(input.headroom)).plan;
|
|
70886
|
+
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70275
70887
|
}
|
|
70276
|
-
|
|
70888
|
+
/**
|
|
70889
|
+
* Shares proportional to each node's capacity at the models Auto tune
|
|
70890
|
+
* would pick under those very shares (D748). Model and load depend on each
|
|
70891
|
+
* other, so it iterates from equal shares until the rounded shares stop
|
|
70892
|
+
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
70893
|
+
* ADVISORY session cap — never written, never enforced.
|
|
70894
|
+
*/
|
|
70895
|
+
async suggestShares(input) {
|
|
70896
|
+
let shares = {
|
|
70897
|
+
nodes: {},
|
|
70898
|
+
devices: {}
|
|
70899
|
+
};
|
|
70900
|
+
let result = [];
|
|
70901
|
+
let cameraFps = 10;
|
|
70902
|
+
let rounds = 0;
|
|
70903
|
+
for (let round = 1; round <= SUGGEST_MAX_ROUNDS; round++) {
|
|
70904
|
+
rounds = round;
|
|
70905
|
+
const rec = await this.recommend(input.headroom, shares);
|
|
70906
|
+
cameraFps = rec.cameraFps;
|
|
70907
|
+
result = this.sharesFrom(rec, input.headroom);
|
|
70908
|
+
const next = {
|
|
70909
|
+
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70910
|
+
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70911
|
+
};
|
|
70912
|
+
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70913
|
+
shares = next;
|
|
70914
|
+
if (settled) break;
|
|
70915
|
+
}
|
|
70916
|
+
return {
|
|
70917
|
+
headroom: input.headroom,
|
|
70918
|
+
cameraFps,
|
|
70919
|
+
iterations: rounds,
|
|
70920
|
+
nodes: [...result]
|
|
70921
|
+
};
|
|
70922
|
+
}
|
|
70923
|
+
sharesFrom(rec, headroom) {
|
|
70924
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70925
|
+
const perNode = rec.recommendations.map((r) => {
|
|
70926
|
+
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70927
|
+
const fps = d.chosen?.fps ?? 0;
|
|
70928
|
+
return {
|
|
70929
|
+
d,
|
|
70930
|
+
fps,
|
|
70931
|
+
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70932
|
+
};
|
|
70933
|
+
});
|
|
70934
|
+
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70935
|
+
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70936
|
+
return {
|
|
70937
|
+
r,
|
|
70938
|
+
devices,
|
|
70939
|
+
capacity
|
|
70940
|
+
};
|
|
70941
|
+
});
|
|
70942
|
+
const nodePct = toPercent(capacityByNode);
|
|
70943
|
+
return perNode.map(({ r, devices, capacity }) => {
|
|
70944
|
+
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70945
|
+
return {
|
|
70946
|
+
nodeId: r.nodeId,
|
|
70947
|
+
share: nodePct.get(r.nodeId) ?? null,
|
|
70948
|
+
capacitySessions: capacity,
|
|
70949
|
+
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70950
|
+
devices: devices.map((x) => ({
|
|
70951
|
+
deviceKey: x.d.deviceKey,
|
|
70952
|
+
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70953
|
+
modelId: x.d.chosen?.modelId ?? null,
|
|
70954
|
+
fps: x.d.chosen?.fps ?? null,
|
|
70955
|
+
measured: x.d.chosen?.measured ?? false,
|
|
70956
|
+
suggestedMaxSessions: x.sessions
|
|
70957
|
+
}))
|
|
70958
|
+
};
|
|
70959
|
+
});
|
|
70960
|
+
}
|
|
70961
|
+
/**
|
|
70962
|
+
* Gather every node and recommend. `shares` (a what-if or the suggestion's
|
|
70963
|
+
* round) forces share placement with those shares overlaid on the stored
|
|
70964
|
+
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70965
|
+
*/
|
|
70966
|
+
async recommend(headroom, shares) {
|
|
70277
70967
|
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70278
70968
|
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70279
70969
|
const [devices, hardware, catalog] = await Promise.all([
|
|
@@ -70288,6 +70978,8 @@ var BestSettingsService = class {
|
|
|
70288
70978
|
catalog
|
|
70289
70979
|
};
|
|
70290
70980
|
}));
|
|
70981
|
+
const effective = await this.deps.readEffectiveWeights();
|
|
70982
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? effective.devices[nodeId]?.[key] ?? stored;
|
|
70291
70983
|
const nodes = gathered.map((g) => ({
|
|
70292
70984
|
nodeId: g.nodeId,
|
|
70293
70985
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70303,18 +70995,37 @@ var BestSettingsService = class {
|
|
|
70303
70995
|
const eligible = gathered.flatMap((g) => g.devices.reachable ? g.devices.devices.filter((d) => d.enabled && d.available).map((d) => ({
|
|
70304
70996
|
nodeId: g.nodeId,
|
|
70305
70997
|
deviceKey: d.key,
|
|
70306
|
-
weight: d.weight
|
|
70998
|
+
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70307
70999
|
})) : []);
|
|
71000
|
+
const cameras = await this.deps.cameraLoads();
|
|
71001
|
+
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70308
71002
|
const demand = estimateDeviceDemand({
|
|
70309
|
-
cameras
|
|
71003
|
+
cameras,
|
|
70310
71004
|
eligible,
|
|
70311
|
-
peaks: this.deps.peaks()
|
|
71005
|
+
peaks: this.deps.peaks(),
|
|
71006
|
+
...policy === "share" ? { shares: {
|
|
71007
|
+
nodes: {
|
|
71008
|
+
...effective.nodes,
|
|
71009
|
+
...shares?.nodes
|
|
71010
|
+
},
|
|
71011
|
+
clusterPeak: this.deps.clusterPeak()
|
|
71012
|
+
} } : {}
|
|
70312
71013
|
});
|
|
70313
71014
|
const recommendations = nodes.map((node) => recommendNode({
|
|
70314
71015
|
node,
|
|
70315
71016
|
demand,
|
|
70316
71017
|
headroom
|
|
70317
71018
|
}));
|
|
71019
|
+
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
71020
|
+
return {
|
|
71021
|
+
gathered,
|
|
71022
|
+
nodes,
|
|
71023
|
+
recommendations,
|
|
71024
|
+
cameraFps: cameraFps > 0 ? cameraFps : 10
|
|
71025
|
+
};
|
|
71026
|
+
}
|
|
71027
|
+
async computePlan(headroom, shares) {
|
|
71028
|
+
const { gathered, nodes, recommendations } = await this.recommend(headroom, shares);
|
|
70318
71029
|
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70319
71030
|
const targets = /* @__PURE__ */ new Map();
|
|
70320
71031
|
const formats = /* @__PURE__ */ new Map();
|
|
@@ -70341,8 +71052,8 @@ var BestSettingsService = class {
|
|
|
70341
71052
|
targets,
|
|
70342
71053
|
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70343
71054
|
};
|
|
70344
|
-
const nodeChanges =
|
|
70345
|
-
const cameraClears =
|
|
71055
|
+
const nodeChanges = applyNodeAutoTune(await this.deps.readAgentSettings(), diffInput).changes;
|
|
71056
|
+
const cameraClears = applyCameraAutoTune(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70346
71057
|
const warnings = [];
|
|
70347
71058
|
let detectorStoreClears = [];
|
|
70348
71059
|
if (touchedNodes.size > 0) try {
|
|
@@ -70376,6 +71087,7 @@ var BestSettingsService = class {
|
|
|
70376
71087
|
planHash: (0, node_crypto.createHash)("sha256").update(canonical({
|
|
70377
71088
|
version: 1,
|
|
70378
71089
|
headroom,
|
|
71090
|
+
whatIf: shares ?? null,
|
|
70379
71091
|
picks: recommendations.map((r) => ({
|
|
70380
71092
|
nodeId: r.nodeId,
|
|
70381
71093
|
skipped: r.skipped,
|
|
@@ -70387,6 +71099,7 @@ var BestSettingsService = class {
|
|
|
70387
71099
|
})).digest("hex"),
|
|
70388
71100
|
family: "yolov9",
|
|
70389
71101
|
headroom,
|
|
71102
|
+
whatIf: shares !== void 0,
|
|
70390
71103
|
generatedAt: this.deps.now(),
|
|
70391
71104
|
nodes: recommendations,
|
|
70392
71105
|
nodeChanges: [...nodeChanges],
|
|
@@ -70413,7 +71126,7 @@ var BestSettingsService = class {
|
|
|
70413
71126
|
async applyLocked(input) {
|
|
70414
71127
|
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
70415
71128
|
if (plan.planHash !== input.planHash) {
|
|
70416
|
-
this.deps.logger.warn("
|
|
71129
|
+
this.deps.logger.warn("auto tune: stale plan refused", { meta: {
|
|
70417
71130
|
confirmed: input.planHash,
|
|
70418
71131
|
current: plan.planHash
|
|
70419
71132
|
} });
|
|
@@ -70445,15 +71158,15 @@ var BestSettingsService = class {
|
|
|
70445
71158
|
ok: false,
|
|
70446
71159
|
detail: errMsg(err)
|
|
70447
71160
|
});
|
|
70448
|
-
this.deps.logger.warn("
|
|
71161
|
+
this.deps.logger.warn("auto tune: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70449
71162
|
return {
|
|
70450
71163
|
kind: "aborted",
|
|
70451
71164
|
phases
|
|
70452
71165
|
};
|
|
70453
71166
|
}
|
|
70454
71167
|
try {
|
|
70455
|
-
await this.deps.rewriteCameraSettings((prev) =>
|
|
70456
|
-
for (const c of plan.cameraClears) this.deps.logger.info("
|
|
71168
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraAutoTune(prev, diffInput).next);
|
|
71169
|
+
for (const c of plan.cameraClears) this.deps.logger.info("auto tune: camera override cleared", {
|
|
70457
71170
|
tags: {
|
|
70458
71171
|
deviceId: c.deviceId,
|
|
70459
71172
|
nodeId: c.nodeId
|
|
@@ -70497,8 +71210,8 @@ var BestSettingsService = class {
|
|
|
70497
71210
|
});
|
|
70498
71211
|
}
|
|
70499
71212
|
try {
|
|
70500
|
-
await this.deps.rewriteAgentSettings((prev) =>
|
|
70501
|
-
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("
|
|
71213
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeAutoTune(prev, diffInput).next);
|
|
71214
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("auto tune: node model set", {
|
|
70502
71215
|
tags: { nodeId: c.nodeId },
|
|
70503
71216
|
meta: {
|
|
70504
71217
|
nodeId: c.nodeId,
|
|
@@ -70532,14 +71245,14 @@ var BestSettingsService = class {
|
|
|
70532
71245
|
ok: failed.length === 0,
|
|
70533
71246
|
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
70534
71247
|
});
|
|
70535
|
-
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("
|
|
71248
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("auto tune: node skipped", {
|
|
70536
71249
|
tags: { nodeId: rec.nodeId },
|
|
70537
71250
|
meta: {
|
|
70538
71251
|
nodeId: rec.nodeId,
|
|
70539
71252
|
reasons: rec.reasons
|
|
70540
71253
|
}
|
|
70541
71254
|
});
|
|
70542
|
-
this.deps.logger.info("
|
|
71255
|
+
this.deps.logger.info("auto tune: applied", { meta: {
|
|
70543
71256
|
planHash: plan.planHash,
|
|
70544
71257
|
nodeChanges: plan.nodeChanges.length,
|
|
70545
71258
|
cameraClears: plan.cameraClears.length,
|
|
@@ -70554,11 +71267,12 @@ var BestSettingsService = class {
|
|
|
70554
71267
|
}
|
|
70555
71268
|
};
|
|
70556
71269
|
//#endregion
|
|
70557
|
-
//#region src/
|
|
71270
|
+
//#region src/auto-tune/device-peak-tracker.ts
|
|
70558
71271
|
var BUCKET_MS = 3600 * 1e3;
|
|
70559
71272
|
var DevicePeakTracker = class DevicePeakTracker {
|
|
70560
71273
|
now;
|
|
70561
71274
|
series = /* @__PURE__ */ new Map();
|
|
71275
|
+
cluster = null;
|
|
70562
71276
|
constructor(now = () => Date.now()) {
|
|
70563
71277
|
this.now = now;
|
|
70564
71278
|
}
|
|
@@ -70569,12 +71283,14 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70569
71283
|
noteCensus(loads) {
|
|
70570
71284
|
const now = this.now();
|
|
70571
71285
|
const bucket = now - now % BUCKET_MS;
|
|
71286
|
+
let clusterTotal = 0;
|
|
70572
71287
|
for (const load of loads) {
|
|
70573
71288
|
const rows = load.devices.length > 0 ? load.devices : [{
|
|
70574
71289
|
deviceKey: "",
|
|
70575
71290
|
attachedCameras: load.attachedCameras
|
|
70576
71291
|
}];
|
|
70577
71292
|
for (const device of rows) {
|
|
71293
|
+
clusterTotal += device.attachedCameras;
|
|
70578
71294
|
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
70579
71295
|
let s = this.series.get(key);
|
|
70580
71296
|
if (s === void 0) {
|
|
@@ -70588,6 +71304,26 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70588
71304
|
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
70589
71305
|
}
|
|
70590
71306
|
}
|
|
71307
|
+
if (loads.length === 0) return;
|
|
71308
|
+
const c = this.cluster ?? {
|
|
71309
|
+
firstSeenAt: now,
|
|
71310
|
+
buckets: /* @__PURE__ */ new Map()
|
|
71311
|
+
};
|
|
71312
|
+
this.cluster = c;
|
|
71313
|
+
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71314
|
+
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71315
|
+
}
|
|
71316
|
+
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71317
|
+
clusterPeak() {
|
|
71318
|
+
const c = this.cluster;
|
|
71319
|
+
if (c === null) return null;
|
|
71320
|
+
const now = this.now();
|
|
71321
|
+
let peak = 0;
|
|
71322
|
+
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71323
|
+
return {
|
|
71324
|
+
peakSessions: peak,
|
|
71325
|
+
observedForMs: now - c.firstSeenAt
|
|
71326
|
+
};
|
|
70591
71327
|
}
|
|
70592
71328
|
/** Every observed device's peak over the window and how long it has been watched. */
|
|
70593
71329
|
peaks() {
|
|
@@ -70608,6 +71344,356 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70608
71344
|
}
|
|
70609
71345
|
};
|
|
70610
71346
|
//#endregion
|
|
71347
|
+
//#region src/session-placement-stats.ts
|
|
71348
|
+
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71349
|
+
function keyOf(nodeId, deviceKey) {
|
|
71350
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
71351
|
+
}
|
|
71352
|
+
function split(key) {
|
|
71353
|
+
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71354
|
+
return {
|
|
71355
|
+
nodeId,
|
|
71356
|
+
deviceKey
|
|
71357
|
+
};
|
|
71358
|
+
}
|
|
71359
|
+
var SessionPlacementStats = class {
|
|
71360
|
+
now;
|
|
71361
|
+
segments = [];
|
|
71362
|
+
starts = [];
|
|
71363
|
+
yields = [];
|
|
71364
|
+
startedAt;
|
|
71365
|
+
constructor(now = () => Date.now()) {
|
|
71366
|
+
this.now = now;
|
|
71367
|
+
this.startedAt = now();
|
|
71368
|
+
}
|
|
71369
|
+
/** The running-session set changed: open a new segment with these counts. */
|
|
71370
|
+
noteSessions(sessions) {
|
|
71371
|
+
const counts = /* @__PURE__ */ new Map();
|
|
71372
|
+
for (const s of sessions) {
|
|
71373
|
+
const k = keyOf(s.nodeId, s.deviceKey);
|
|
71374
|
+
counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
71375
|
+
}
|
|
71376
|
+
this.segments.push({
|
|
71377
|
+
at: this.now(),
|
|
71378
|
+
counts
|
|
71379
|
+
});
|
|
71380
|
+
this.prune();
|
|
71381
|
+
}
|
|
71382
|
+
noteStart(nodeId, deviceKey) {
|
|
71383
|
+
this.starts.push({
|
|
71384
|
+
at: this.now(),
|
|
71385
|
+
nodeId,
|
|
71386
|
+
deviceKey
|
|
71387
|
+
});
|
|
71388
|
+
this.prune();
|
|
71389
|
+
}
|
|
71390
|
+
/** A share election handed `nodeId`'s turn to another node. */
|
|
71391
|
+
noteYield(nodeId) {
|
|
71392
|
+
this.yields.push({
|
|
71393
|
+
at: this.now(),
|
|
71394
|
+
nodeId
|
|
71395
|
+
});
|
|
71396
|
+
this.prune();
|
|
71397
|
+
}
|
|
71398
|
+
/** For tests: how many segments are retained. */
|
|
71399
|
+
segmentCount() {
|
|
71400
|
+
return this.segments.length;
|
|
71401
|
+
}
|
|
71402
|
+
prune() {
|
|
71403
|
+
const floor = this.now() - RETAIN_MS;
|
|
71404
|
+
while (this.segments.length > 1 && this.segments[1].at <= floor) this.segments.shift();
|
|
71405
|
+
while (this.starts.length > 0 && this.starts[0].at < floor) this.starts.shift();
|
|
71406
|
+
while (this.yields.length > 0 && this.yields[0].at < floor) this.yields.shift();
|
|
71407
|
+
}
|
|
71408
|
+
snapshot(windowMs) {
|
|
71409
|
+
const now = this.now();
|
|
71410
|
+
const from = now - windowMs;
|
|
71411
|
+
const integral = /* @__PURE__ */ new Map();
|
|
71412
|
+
let clusterPeak = 0;
|
|
71413
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71414
|
+
const seg = this.segments[i];
|
|
71415
|
+
const end = this.segments[i + 1]?.at ?? now;
|
|
71416
|
+
const start = Math.max(seg.at, from);
|
|
71417
|
+
if (end <= start) continue;
|
|
71418
|
+
let total = 0;
|
|
71419
|
+
for (const [k, n] of seg.counts) {
|
|
71420
|
+
integral.set(k, (integral.get(k) ?? 0) + n * (end - start));
|
|
71421
|
+
total += n;
|
|
71422
|
+
}
|
|
71423
|
+
clusterPeak = Math.max(clusterPeak, total);
|
|
71424
|
+
}
|
|
71425
|
+
const current = this.segments.at(-1)?.counts ?? /* @__PURE__ */ new Map();
|
|
71426
|
+
const nodeIds = /* @__PURE__ */ new Set();
|
|
71427
|
+
for (const k of [...integral.keys(), ...current.keys()]) nodeIds.add(split(k).nodeId);
|
|
71428
|
+
for (const s of this.starts) if (s.at >= from) nodeIds.add(s.nodeId);
|
|
71429
|
+
const clusterIntegral = [...integral.values()].reduce((a, b) => a + b, 0);
|
|
71430
|
+
const nodes = [...nodeIds].toSorted().map((nodeId) => {
|
|
71431
|
+
const deviceKeys = /* @__PURE__ */ new Set();
|
|
71432
|
+
for (const k of [...integral.keys(), ...current.keys()]) {
|
|
71433
|
+
const s = split(k);
|
|
71434
|
+
if (s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71435
|
+
}
|
|
71436
|
+
for (const s of this.starts) if (s.at >= from && s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71437
|
+
const nodeIntegral = [...deviceKeys].reduce((sum, d) => sum + (integral.get(keyOf(nodeId, d)) ?? 0), 0);
|
|
71438
|
+
const devices = [...deviceKeys].toSorted().map((deviceKey) => ({
|
|
71439
|
+
deviceKey,
|
|
71440
|
+
sessionsNow: current.get(keyOf(nodeId, deviceKey)) ?? 0,
|
|
71441
|
+
sessionsStarted: this.starts.filter((s) => s.at >= from && s.nodeId === nodeId && s.deviceKey === deviceKey).length,
|
|
71442
|
+
observedShare: nodeIntegral > 0 ? (integral.get(keyOf(nodeId, deviceKey)) ?? 0) / nodeIntegral : null
|
|
71443
|
+
}));
|
|
71444
|
+
let peak = 0;
|
|
71445
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71446
|
+
const seg = this.segments[i];
|
|
71447
|
+
if ((this.segments[i + 1]?.at ?? now) <= Math.max(seg.at, from)) continue;
|
|
71448
|
+
let n = 0;
|
|
71449
|
+
for (const [k, c] of seg.counts) if (split(k).nodeId === nodeId) n += c;
|
|
71450
|
+
peak = Math.max(peak, n);
|
|
71451
|
+
}
|
|
71452
|
+
return {
|
|
71453
|
+
nodeId,
|
|
71454
|
+
sessionsNow: devices.reduce((s, d) => s + d.sessionsNow, 0),
|
|
71455
|
+
sessionsStarted: devices.reduce((s, d) => s + d.sessionsStarted, 0),
|
|
71456
|
+
observedShare: clusterIntegral > 0 && nodeIntegral > 0 ? nodeIntegral / clusterIntegral : null,
|
|
71457
|
+
peakSessions: peak,
|
|
71458
|
+
yieldsAway: this.yields.filter((y) => y.at >= from && y.nodeId === nodeId).length,
|
|
71459
|
+
devices
|
|
71460
|
+
};
|
|
71461
|
+
});
|
|
71462
|
+
return {
|
|
71463
|
+
observedForMs: Math.min(windowMs, now - this.startedAt),
|
|
71464
|
+
sessionsNow: [...current.values()].reduce((a, b) => a + b, 0),
|
|
71465
|
+
clusterPeakSessions: clusterPeak,
|
|
71466
|
+
nodes
|
|
71467
|
+
};
|
|
71468
|
+
}
|
|
71469
|
+
};
|
|
71470
|
+
//#endregion
|
|
71471
|
+
//#region src/load-balancing-shares.ts
|
|
71472
|
+
function nextDevices(stored, shares, effective) {
|
|
71473
|
+
let changed = false;
|
|
71474
|
+
const out = { ...stored };
|
|
71475
|
+
for (const [key, weight] of Object.entries(shares)) {
|
|
71476
|
+
const existing = stored?.[key];
|
|
71477
|
+
if (existing !== void 0) {
|
|
71478
|
+
out[key] = {
|
|
71479
|
+
...existing,
|
|
71480
|
+
weight
|
|
71481
|
+
};
|
|
71482
|
+
changed = true;
|
|
71483
|
+
continue;
|
|
71484
|
+
}
|
|
71485
|
+
const enabled = effective[key];
|
|
71486
|
+
if (enabled === void 0) continue;
|
|
71487
|
+
out[key] = {
|
|
71488
|
+
enabled,
|
|
71489
|
+
weight
|
|
71490
|
+
};
|
|
71491
|
+
changed = true;
|
|
71492
|
+
}
|
|
71493
|
+
return changed ? out : stored;
|
|
71494
|
+
}
|
|
71495
|
+
function applyLoadBalancingShares(prev, write) {
|
|
71496
|
+
const next = { ...prev };
|
|
71497
|
+
const nodeIds = new Set([...Object.keys(write.nodes), ...Object.keys(write.devices)]);
|
|
71498
|
+
for (const nodeId of nodeIds) {
|
|
71499
|
+
const existing = prev[nodeId];
|
|
71500
|
+
const nodeShare = write.nodes[nodeId];
|
|
71501
|
+
const devices = nextDevices(existing?.inferenceDevices, write.devices[nodeId] ?? {}, write.effectiveEnabled[nodeId] ?? {});
|
|
71502
|
+
if (nodeShare === void 0 && devices === existing?.inferenceDevices) continue;
|
|
71503
|
+
next[nodeId] = {
|
|
71504
|
+
...existing ?? { maxCameras: null },
|
|
71505
|
+
...nodeShare !== void 0 ? { detectWeight: nodeShare } : {},
|
|
71506
|
+
...devices !== void 0 ? { inferenceDevices: devices } : {}
|
|
71507
|
+
};
|
|
71508
|
+
}
|
|
71509
|
+
return next;
|
|
71510
|
+
}
|
|
71511
|
+
//#endregion
|
|
71512
|
+
//#region src/load-balancing/effective-shares-service.ts
|
|
71513
|
+
function weightsFrom(result) {
|
|
71514
|
+
const nodes = {};
|
|
71515
|
+
const devices = {};
|
|
71516
|
+
for (const node of result.nodes) {
|
|
71517
|
+
if (node.share !== null && node.share > 0) nodes[node.nodeId] = node.share;
|
|
71518
|
+
const perDevice = {};
|
|
71519
|
+
for (const d of node.devices) if (d.share > 0) perDevice[d.deviceKey] = d.share;
|
|
71520
|
+
if (Object.keys(perDevice).length > 0) devices[node.nodeId] = perDevice;
|
|
71521
|
+
}
|
|
71522
|
+
return {
|
|
71523
|
+
nodes,
|
|
71524
|
+
devices
|
|
71525
|
+
};
|
|
71526
|
+
}
|
|
71527
|
+
var EffectiveSharesService = class EffectiveSharesService {
|
|
71528
|
+
deps;
|
|
71529
|
+
/** An automatic answer older than this is recomputed by the reconcile. */
|
|
71530
|
+
static MAX_AGE_MS = 15 * 6e4;
|
|
71531
|
+
/** Reconcile cadence. */
|
|
71532
|
+
static RECONCILE_MS = 6e4;
|
|
71533
|
+
answer = null;
|
|
71534
|
+
inFlight = null;
|
|
71535
|
+
pendingTrigger = null;
|
|
71536
|
+
timer = null;
|
|
71537
|
+
constructor(deps) {
|
|
71538
|
+
this.deps = deps;
|
|
71539
|
+
}
|
|
71540
|
+
now() {
|
|
71541
|
+
return this.deps.now?.() ?? Date.now();
|
|
71542
|
+
}
|
|
71543
|
+
/** The last automatic answer; null = never computed. */
|
|
71544
|
+
current() {
|
|
71545
|
+
return this.answer;
|
|
71546
|
+
}
|
|
71547
|
+
/** Node weights the election reads. */
|
|
71548
|
+
async nodeWeights() {
|
|
71549
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return { ...this.answer.weights.nodes };
|
|
71550
|
+
return { ...(await this.deps.readStored()).nodes };
|
|
71551
|
+
}
|
|
71552
|
+
/** The weights of a node's ELIGIBLE devices (`eligible` carries their stored weights). */
|
|
71553
|
+
deviceWeights(nodeId, eligible) {
|
|
71554
|
+
if (this.deps.readSource() !== "auto" || this.answer === null) return { ...eligible };
|
|
71555
|
+
const auto = this.answer.weights.devices[nodeId] ?? {};
|
|
71556
|
+
const out = {};
|
|
71557
|
+
for (const [key, stored] of Object.entries(eligible)) {
|
|
71558
|
+
const share = auto[key];
|
|
71559
|
+
out[key] = share !== void 0 && share > 0 ? share : stored;
|
|
71560
|
+
}
|
|
71561
|
+
return out;
|
|
71562
|
+
}
|
|
71563
|
+
/** Both maps, as the Auto tune recommender charges load by them. */
|
|
71564
|
+
async effectiveWeights() {
|
|
71565
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return this.answer.weights;
|
|
71566
|
+
return this.deps.readStored();
|
|
71567
|
+
}
|
|
71568
|
+
/**
|
|
71569
|
+
* Recompute now (auto only). Concurrent calls coalesce: one runs, and one
|
|
71570
|
+
* trailing run picks up whatever was reported meanwhile.
|
|
71571
|
+
*/
|
|
71572
|
+
async recompute(trigger) {
|
|
71573
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71574
|
+
if (this.inFlight !== null) {
|
|
71575
|
+
this.pendingTrigger = trigger;
|
|
71576
|
+
return this.inFlight;
|
|
71577
|
+
}
|
|
71578
|
+
this.inFlight = this.run(trigger).finally(() => {
|
|
71579
|
+
this.inFlight = null;
|
|
71580
|
+
});
|
|
71581
|
+
await this.inFlight;
|
|
71582
|
+
const trailing = this.pendingTrigger;
|
|
71583
|
+
if (trailing !== null) {
|
|
71584
|
+
this.pendingTrigger = null;
|
|
71585
|
+
await this.recompute(trailing);
|
|
71586
|
+
}
|
|
71587
|
+
}
|
|
71588
|
+
async run(trigger) {
|
|
71589
|
+
let signature;
|
|
71590
|
+
let result;
|
|
71591
|
+
try {
|
|
71592
|
+
signature = await this.deps.topologySignature();
|
|
71593
|
+
result = await this.deps.suggest();
|
|
71594
|
+
} catch (err) {
|
|
71595
|
+
this.deps.logger.warn("load balancing: automatic shares not recomputed — last answer kept", { meta: {
|
|
71596
|
+
trigger,
|
|
71597
|
+
error: errMsg$2(err),
|
|
71598
|
+
hasAnswer: this.answer !== null
|
|
71599
|
+
} });
|
|
71600
|
+
return;
|
|
71601
|
+
}
|
|
71602
|
+
const before = this.answer?.weights ?? null;
|
|
71603
|
+
const after = weightsFrom(result);
|
|
71604
|
+
this.answer = {
|
|
71605
|
+
weights: after,
|
|
71606
|
+
computedAt: this.now(),
|
|
71607
|
+
trigger,
|
|
71608
|
+
signature
|
|
71609
|
+
};
|
|
71610
|
+
const nodeIds = [...new Set([...Object.keys(before?.nodes ?? {}), ...result.nodes.map((n) => n.nodeId)])];
|
|
71611
|
+
for (const nodeId of nodeIds.toSorted()) this.deps.logger.info("load balancing: automatic shares recomputed", {
|
|
71612
|
+
tags: { nodeId },
|
|
71613
|
+
meta: {
|
|
71614
|
+
trigger,
|
|
71615
|
+
before: before?.nodes[nodeId] ?? null,
|
|
71616
|
+
after: after.nodes[nodeId] ?? null,
|
|
71617
|
+
devicesBefore: JSON.stringify(before?.devices[nodeId] ?? {}),
|
|
71618
|
+
devicesAfter: JSON.stringify(after.devices[nodeId] ?? {})
|
|
71619
|
+
}
|
|
71620
|
+
});
|
|
71621
|
+
}
|
|
71622
|
+
/** Recompute if the topology changed or the answer has grown old (auto only). */
|
|
71623
|
+
async reconcile() {
|
|
71624
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71625
|
+
if (this.answer === null) return this.recompute("reconcile:first");
|
|
71626
|
+
if (this.now() - this.answer.computedAt > EffectiveSharesService.MAX_AGE_MS) return this.recompute("reconcile:age");
|
|
71627
|
+
let signature;
|
|
71628
|
+
try {
|
|
71629
|
+
signature = await this.deps.topologySignature();
|
|
71630
|
+
} catch (err) {
|
|
71631
|
+
this.deps.logger.debug("load balancing: topology signature unreadable — reconcile skipped", { meta: { error: errMsg$2(err) } });
|
|
71632
|
+
return;
|
|
71633
|
+
}
|
|
71634
|
+
if (signature !== this.answer.signature) return this.recompute("reconcile:topology");
|
|
71635
|
+
}
|
|
71636
|
+
start() {
|
|
71637
|
+
if (this.timer !== null) return;
|
|
71638
|
+
this.timer = setInterval(() => {
|
|
71639
|
+
this.reconcile();
|
|
71640
|
+
}, EffectiveSharesService.RECONCILE_MS);
|
|
71641
|
+
this.timer.unref?.();
|
|
71642
|
+
}
|
|
71643
|
+
stop() {
|
|
71644
|
+
if (this.timer !== null) clearInterval(this.timer);
|
|
71645
|
+
this.timer = null;
|
|
71646
|
+
}
|
|
71647
|
+
};
|
|
71648
|
+
//#endregion
|
|
71649
|
+
//#region src/load-balancing/share-inputs.ts
|
|
71650
|
+
/** The operator's sliders: node `detectWeight` and device `weight` (positive only). */
|
|
71651
|
+
function storedShareWeights(blob) {
|
|
71652
|
+
const nodes = {};
|
|
71653
|
+
const devices = {};
|
|
71654
|
+
for (const [nodeId, settings] of Object.entries(blob)) {
|
|
71655
|
+
if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) nodes[nodeId] = settings.detectWeight;
|
|
71656
|
+
const perDevice = {};
|
|
71657
|
+
for (const [key, entry] of Object.entries(settings.inferenceDevices ?? {})) if (typeof entry.weight === "number" && entry.weight > 0) perDevice[key] = entry.weight;
|
|
71658
|
+
if (Object.keys(perDevice).length > 0) devices[nodeId] = perDevice;
|
|
71659
|
+
}
|
|
71660
|
+
return {
|
|
71661
|
+
nodes,
|
|
71662
|
+
devices
|
|
71663
|
+
};
|
|
71664
|
+
}
|
|
71665
|
+
/**
|
|
71666
|
+
* A string that changes whenever an input of the automatic shares does: the
|
|
71667
|
+
* nodes present, each node's detect capability, which devices are switched on
|
|
71668
|
+
* and which detector model each runs. Sliders and caps are left out — they do
|
|
71669
|
+
* not change capacity.
|
|
71670
|
+
*/
|
|
71671
|
+
function shareTopologySignature(nodeIds, blob) {
|
|
71672
|
+
return [...nodeIds].toSorted().map((nodeId) => {
|
|
71673
|
+
const settings = blob[nodeId];
|
|
71674
|
+
const devices = Object.entries(settings?.inferenceDevices ?? {}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => {
|
|
71675
|
+
const model = entry.steps?.["object-detection"]?.modelId ?? "";
|
|
71676
|
+
return `${key}=${entry.enabled ? "on" : "off"}:${model}`;
|
|
71677
|
+
});
|
|
71678
|
+
return `${nodeId}|detect=${String(settings?.detect ?? "default")}|${devices.join(",")}`;
|
|
71679
|
+
}).join(";");
|
|
71680
|
+
}
|
|
71681
|
+
//#endregion
|
|
71682
|
+
//#region src/stale-device-pins.ts
|
|
71683
|
+
function findStaleDevicePins(pins, nodes) {
|
|
71684
|
+
const usable = /* @__PURE__ */ new Map();
|
|
71685
|
+
for (const node of nodes) for (const d of node.devices) usable.set(d.key, (usable.get(d.key) ?? false) || d.enabled && d.available);
|
|
71686
|
+
return pins.flatMap((pin) => {
|
|
71687
|
+
const state = usable.get(pin.deviceKey);
|
|
71688
|
+
if (state === true) return [];
|
|
71689
|
+
return [{
|
|
71690
|
+
deviceId: pin.deviceId,
|
|
71691
|
+
deviceKey: pin.deviceKey,
|
|
71692
|
+
reason: state === false ? "disabled" : "absent"
|
|
71693
|
+
}];
|
|
71694
|
+
}).toSorted((a, b) => a.deviceId - b.deviceId);
|
|
71695
|
+
}
|
|
71696
|
+
//#endregion
|
|
70611
71697
|
//#region src/index.ts
|
|
70612
71698
|
/**
|
|
70613
71699
|
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
@@ -70677,7 +71763,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70677
71763
|
clusterModelChoiceReady = false;
|
|
70678
71764
|
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
70679
71765
|
devicePeaks = new DevicePeakTracker();
|
|
70680
|
-
|
|
71766
|
+
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71767
|
+
placementStats = new SessionPlacementStats();
|
|
71768
|
+
/** The stored session placement policy (D748, default `share` since D750). */
|
|
71769
|
+
readSessionPlacement = () => DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
71770
|
+
/** Where the shares come from (D750); `auto` before init. */
|
|
71771
|
+
readShareSource = () => DEFAULT_SHARE_SOURCE;
|
|
71772
|
+
autoTuneService = null;
|
|
71773
|
+
effectiveSharesService = null;
|
|
70681
71774
|
/**
|
|
70682
71775
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
70683
71776
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -70926,6 +72019,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70926
72019
|
getEligibleInferenceDevices: (nodeId, read) => this.resolveEligibleInferenceDevices(nodeId, read),
|
|
70927
72020
|
getEligibleInferenceDevicesIfCached: (nodeId) => this.resolveEligibleInferenceDevicesIfCached(nodeId),
|
|
70928
72021
|
forgetInferenceDeviceProbe: (nodeId) => this.forgetInferenceDeviceProbe(nodeId),
|
|
72022
|
+
noteShareTopologyChange: (trigger) => this.noteShareChange(trigger),
|
|
70929
72023
|
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
70930
72024
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
70931
72025
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
@@ -70955,6 +72049,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70955
72049
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
70956
72050
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
70957
72051
|
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
72052
|
+
placementStats: this.placementStats,
|
|
70958
72053
|
setTopology: (topology) => {
|
|
70959
72054
|
this.topology = topology;
|
|
70960
72055
|
},
|
|
@@ -70981,6 +72076,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70981
72076
|
this.nodeStress = controllers.nodeStress;
|
|
70982
72077
|
this.placement = controllers.placement;
|
|
70983
72078
|
this.session = controllers.session;
|
|
72079
|
+
this.readSessionPlacement = controllers.sessionPlacement;
|
|
72080
|
+
this.readShareSource = controllers.shareSource;
|
|
72081
|
+
this.settingsStore.setEffectiveNodeWeights(() => this.effectiveShares().nodeWeights());
|
|
72082
|
+
this.effectiveShares().start();
|
|
72083
|
+
this.noteShareChange("startup");
|
|
70984
72084
|
this.deviceConfig = controllers.deviceConfig;
|
|
70985
72085
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
70986
72086
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71106,6 +72206,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71106
72206
|
this.detectionWiring = null;
|
|
71107
72207
|
this.unsubAgentOffline?.();
|
|
71108
72208
|
this.unsubAgentOffline = null;
|
|
72209
|
+
this.effectiveSharesService?.stop();
|
|
71109
72210
|
this.unsubAgentOnline?.();
|
|
71110
72211
|
this.unsubAgentOnline = null;
|
|
71111
72212
|
this.nodeLifecycle?.dispose();
|
|
@@ -71473,26 +72574,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71473
72574
|
this.reconcile.schedulePendingRetry();
|
|
71474
72575
|
return { success: true };
|
|
71475
72576
|
}
|
|
71476
|
-
async setAgentDetectWeight(input) {
|
|
71477
|
-
const existing = (await this.settingsStore.readAgentSettingsMap())[input.agentNodeId];
|
|
71478
|
-
const weight = input.detectWeight ?? 1;
|
|
71479
|
-
const next = existing ? {
|
|
71480
|
-
...existing,
|
|
71481
|
-
detectWeight: weight
|
|
71482
|
-
} : {
|
|
71483
|
-
maxCameras: null,
|
|
71484
|
-
detectWeight: weight
|
|
71485
|
-
};
|
|
71486
|
-
await this.settingsStore.writeAgentSettings(input.agentNodeId, next);
|
|
71487
|
-
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
71488
|
-
tags: { nodeId: input.agentNodeId },
|
|
71489
|
-
meta: { detectWeight: weight ?? null }
|
|
71490
|
-
});
|
|
71491
|
-
this.rebalance().catch((err) => {
|
|
71492
|
-
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
71493
|
-
});
|
|
71494
|
-
return { success: true };
|
|
71495
|
-
}
|
|
71496
72577
|
/**
|
|
71497
72578
|
* Set one node's placement capabilities — the per-node record that replaced
|
|
71498
72579
|
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
@@ -71528,6 +72609,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71528
72609
|
await this.topology.refreshNodeCapabilities();
|
|
71529
72610
|
this.topology.addKnownRunnerNode(input.agentNodeId);
|
|
71530
72611
|
this.reconcile.schedulePendingRetry();
|
|
72612
|
+
if (input.detect !== void 0) this.noteShareChange(`node-detect:${input.agentNodeId}`);
|
|
71531
72613
|
return { success: true };
|
|
71532
72614
|
}
|
|
71533
72615
|
/**
|
|
@@ -71617,6 +72699,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71617
72699
|
});
|
|
71618
72700
|
await this.resetRemovedModelPins("node-device-write");
|
|
71619
72701
|
this.reconcile.schedulePendingRetry();
|
|
72702
|
+
this.noteShareChange(`node-devices:${input.agentNodeId}`);
|
|
71620
72703
|
return { success: true };
|
|
71621
72704
|
}
|
|
71622
72705
|
/**
|
|
@@ -71697,7 +72780,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71697
72780
|
});
|
|
71698
72781
|
throw new Error(`inference device eligibility: node "${nodeId}" catalog not answered`);
|
|
71699
72782
|
}
|
|
71700
|
-
return this.electableDevices(nodeId, probed, catalog);
|
|
72783
|
+
return this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, probed, catalog));
|
|
71701
72784
|
}
|
|
71702
72785
|
/**
|
|
71703
72786
|
* Forget a node's last device probe (pre-ship review, M4). Called when the
|
|
@@ -71741,7 +72824,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71741
72824
|
this.refreshInferenceDeviceProbe(nodeId);
|
|
71742
72825
|
return {
|
|
71743
72826
|
kind: "eligible",
|
|
71744
|
-
eligible: await this.electableDevices(nodeId, known, catalog)
|
|
72827
|
+
eligible: this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, known, catalog))
|
|
71745
72828
|
};
|
|
71746
72829
|
}
|
|
71747
72830
|
/**
|
|
@@ -72071,10 +73154,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72071
73154
|
clearedCameraOverrides
|
|
72072
73155
|
};
|
|
72073
73156
|
}
|
|
72074
|
-
|
|
72075
|
-
if (this.
|
|
73157
|
+
autoTune() {
|
|
73158
|
+
if (this.autoTuneService !== null) return this.autoTuneService;
|
|
72076
73159
|
const store = this.settingsStore;
|
|
72077
|
-
this.
|
|
73160
|
+
this.autoTuneService = new AutoTuneService({
|
|
72078
73161
|
logger: this.ctx.logger,
|
|
72079
73162
|
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72080
73163
|
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
@@ -72088,8 +73171,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72088
73171
|
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72089
73172
|
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72090
73173
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72091
|
-
cameraLoads: () => this.
|
|
73174
|
+
cameraLoads: () => this.autoTuneCameraLoads(),
|
|
72092
73175
|
peaks: () => this.devicePeaks.peaks(),
|
|
73176
|
+
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
73177
|
+
sessionPlacement: () => this.readSessionPlacement(),
|
|
73178
|
+
readEffectiveWeights: () => this.effectiveShares().effectiveWeights(),
|
|
72093
73179
|
detectorStore: async (input) => {
|
|
72094
73180
|
const api = this.ctx.api;
|
|
72095
73181
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72112,35 +73198,157 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72112
73198
|
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72113
73199
|
now: () => Date.now()
|
|
72114
73200
|
});
|
|
72115
|
-
return this.
|
|
73201
|
+
return this.autoTuneService;
|
|
72116
73202
|
}
|
|
72117
73203
|
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72118
|
-
async
|
|
73204
|
+
async autoTuneCameraLoads() {
|
|
72119
73205
|
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72120
|
-
const cameraMap = await this.settingsStore.readCameraSettingsMap();
|
|
72121
73206
|
const loads = [];
|
|
72122
73207
|
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
72123
|
-
const
|
|
73208
|
+
const nodePin = await this.readPipelinePin(deviceId);
|
|
72124
73209
|
const pinnedKey = await this.readPipelineDevicePin(deviceId);
|
|
73210
|
+
const standingNode = config.detectionMode === "always-on" ? standing.get(deviceId) ?? null : null;
|
|
72125
73211
|
loads.push({
|
|
72126
73212
|
deviceId,
|
|
72127
73213
|
mode: config.detectionMode,
|
|
72128
73214
|
fps: config.detectionFps,
|
|
72129
73215
|
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72130
|
-
standingNodeId:
|
|
72131
|
-
|
|
72132
|
-
|
|
73216
|
+
standingNodeId: standingNode,
|
|
73217
|
+
nodePinId: nodePin,
|
|
73218
|
+
devicePin: pinnedKey !== null ? {
|
|
73219
|
+
nodeId: nodePin ?? standingNode,
|
|
72133
73220
|
deviceKey: pinnedKey
|
|
72134
73221
|
} : null
|
|
72135
73222
|
});
|
|
72136
73223
|
}
|
|
72137
73224
|
return loads;
|
|
72138
73225
|
}
|
|
72139
|
-
async
|
|
72140
|
-
return this.
|
|
73226
|
+
async planAutoTune(input) {
|
|
73227
|
+
return this.autoTune().plan(input);
|
|
73228
|
+
}
|
|
73229
|
+
/**
|
|
73230
|
+
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
73231
|
+
* device with no stored entry is made explicit with its effective
|
|
73232
|
+
* enablement, read from the probe-aware device list. Never touches the
|
|
73233
|
+
* `sessionPlacement` policy.
|
|
73234
|
+
*/
|
|
73235
|
+
async setLoadBalancingShares(input) {
|
|
73236
|
+
const effectiveEnabled = {};
|
|
73237
|
+
for (const nodeId of Object.keys(input.devices)) {
|
|
73238
|
+
const read = await this.getNodeInferenceDevices({ nodeId });
|
|
73239
|
+
effectiveEnabled[nodeId] = Object.fromEntries(read.devices.map((d) => [d.key, d.enabled]));
|
|
73240
|
+
}
|
|
73241
|
+
await this.settingsStore.rewriteAgentSettingsMap((prev) => applyLoadBalancingShares(prev, {
|
|
73242
|
+
nodes: input.nodes,
|
|
73243
|
+
devices: input.devices,
|
|
73244
|
+
effectiveEnabled
|
|
73245
|
+
}));
|
|
73246
|
+
this.ctx.logger.info("load balancing shares updated", { meta: {
|
|
73247
|
+
nodes: JSON.stringify(input.nodes),
|
|
73248
|
+
devices: JSON.stringify(input.devices),
|
|
73249
|
+
policy: this.readSessionPlacement()
|
|
73250
|
+
} });
|
|
73251
|
+
if (Object.keys(input.nodes).length > 0) this.rebalance().catch((err) => {
|
|
73252
|
+
this.ctx.logger.warn("setLoadBalancingShares: rebalance failed", { meta: { error: errMsg$2(err) } });
|
|
73253
|
+
});
|
|
73254
|
+
return { success: true };
|
|
73255
|
+
}
|
|
73256
|
+
async getPlacementStats(input) {
|
|
73257
|
+
const snap = this.placementStats.snapshot(input.windowMs);
|
|
73258
|
+
return {
|
|
73259
|
+
policy: this.readSessionPlacement(),
|
|
73260
|
+
windowMs: input.windowMs,
|
|
73261
|
+
observedForMs: snap.observedForMs,
|
|
73262
|
+
sessionsNow: snap.sessionsNow,
|
|
73263
|
+
clusterPeakSessions: snap.clusterPeakSessions,
|
|
73264
|
+
nodes: snap.nodes,
|
|
73265
|
+
stalePins: await this.listStaleDevicePins()
|
|
73266
|
+
};
|
|
72141
73267
|
}
|
|
72142
|
-
|
|
72143
|
-
|
|
73268
|
+
/** Device pins the session path cannot honour — see `stale-device-pins.ts`. */
|
|
73269
|
+
async listStaleDevicePins() {
|
|
73270
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])];
|
|
73271
|
+
const offered = await Promise.all(nodeIds.map(async (nodeId) => ({ devices: (await this.getNodeInferenceDevices({ nodeId }).catch(() => null))?.devices ?? [] })));
|
|
73272
|
+
const pins = [];
|
|
73273
|
+
for (const deviceId of this.detectionWiring.activeDetections.keys()) {
|
|
73274
|
+
const deviceKey = await this.readPipelineDevicePin(deviceId);
|
|
73275
|
+
if (deviceKey !== null) pins.push({
|
|
73276
|
+
deviceId,
|
|
73277
|
+
deviceKey
|
|
73278
|
+
});
|
|
73279
|
+
}
|
|
73280
|
+
return findStaleDevicePins(pins, offered);
|
|
73281
|
+
}
|
|
73282
|
+
async suggestShares(input) {
|
|
73283
|
+
return this.autoTune().suggestShares(input);
|
|
73284
|
+
}
|
|
73285
|
+
async applyAutoTune(input) {
|
|
73286
|
+
const result = await this.autoTune().apply(input);
|
|
73287
|
+
if (result.kind === "applied") this.noteShareChange("auto-tune-applied");
|
|
73288
|
+
return result;
|
|
73289
|
+
}
|
|
73290
|
+
effectiveShares() {
|
|
73291
|
+
if (this.effectiveSharesService !== null) return this.effectiveSharesService;
|
|
73292
|
+
const store = this.settingsStore;
|
|
73293
|
+
this.effectiveSharesService = new EffectiveSharesService({
|
|
73294
|
+
readSource: () => this.readShareSource(),
|
|
73295
|
+
readStored: async () => storedShareWeights(await store.readAgentSettingsMap()),
|
|
73296
|
+
suggest: () => this.autoTune().suggestShares({ headroom: AUTO_SHARE_HEADROOM }),
|
|
73297
|
+
topologySignature: async () => shareTopologySignature([...new Set(["hub", ...this.topology.collectRunnerNodeIds()])], await store.readAgentSettingsMap()),
|
|
73298
|
+
logger: this.ctx.logger
|
|
73299
|
+
});
|
|
73300
|
+
return this.effectiveSharesService;
|
|
73301
|
+
}
|
|
73302
|
+
/** Something the automatic shares depend on changed: recompute now (best effort). */
|
|
73303
|
+
noteShareChange(trigger) {
|
|
73304
|
+
if (this.settingsStore === null) return;
|
|
73305
|
+
this.effectiveShares().recompute(trigger).catch((err) => {
|
|
73306
|
+
this.ctx.logger.warn("load balancing: share recompute failed", { meta: {
|
|
73307
|
+
trigger,
|
|
73308
|
+
error: errMsg$2(err)
|
|
73309
|
+
} });
|
|
73310
|
+
});
|
|
73311
|
+
}
|
|
73312
|
+
async getEffectiveShares() {
|
|
73313
|
+
const service = this.effectiveShares();
|
|
73314
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])].toSorted();
|
|
73315
|
+
const rows = await Promise.all(nodeIds.map(async (nodeId) => ({
|
|
73316
|
+
nodeId,
|
|
73317
|
+
read: await this.getNodeInferenceDevices({ nodeId }).catch(() => null)
|
|
73318
|
+
})));
|
|
73319
|
+
const detectable = new Set(this.topology.detectionEligibleNodes());
|
|
73320
|
+
const eligibleNodes = rows.filter((r) => r.read !== null && r.read.reachable && detectable.has(r.nodeId) && this.isNodeInferenceUsable(r.nodeId)).map((r) => r.nodeId);
|
|
73321
|
+
const eligibleDevices = {};
|
|
73322
|
+
for (const r of rows) {
|
|
73323
|
+
if (r.read === null) continue;
|
|
73324
|
+
eligibleDevices[r.nodeId] = r.read.devices.filter((d) => d.enabled && d.available && d.exclusion === null).map((d) => d.key);
|
|
73325
|
+
}
|
|
73326
|
+
const auto = service.current();
|
|
73327
|
+
return resolveEffectiveShares({
|
|
73328
|
+
source: this.readShareSource(),
|
|
73329
|
+
policy: this.readSessionPlacement(),
|
|
73330
|
+
nodeIds,
|
|
73331
|
+
stored: storedShareWeights(await this.settingsStore.readAgentSettingsMap()),
|
|
73332
|
+
auto: auto?.weights ?? null,
|
|
73333
|
+
eligibleNodes,
|
|
73334
|
+
eligibleDevices,
|
|
73335
|
+
computedAt: auto?.computedAt ?? null,
|
|
73336
|
+
trigger: auto?.trigger ?? null
|
|
73337
|
+
});
|
|
73338
|
+
}
|
|
73339
|
+
async setShareSource(input) {
|
|
73340
|
+
const previous = this.readShareSource();
|
|
73341
|
+
if (input.source === "manual" && previous === "auto") {
|
|
73342
|
+
const auto = this.effectiveShares().current();
|
|
73343
|
+
if (auto !== null) await this.setLoadBalancingShares(auto.weights);
|
|
73344
|
+
}
|
|
73345
|
+
await this.updateGlobalSettings({ [SHARE_SOURCE_SETTING_KEY]: input.source });
|
|
73346
|
+
this.ctx.logger.info("load balancing: share source changed", { meta: {
|
|
73347
|
+
from: previous,
|
|
73348
|
+
to: input.source
|
|
73349
|
+
} });
|
|
73350
|
+
if (input.source === "auto") this.noteShareChange("source:auto");
|
|
73351
|
+
return { success: true };
|
|
72144
73352
|
}
|
|
72145
73353
|
async getCameraSettings(input) {
|
|
72146
73354
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|