@camstack/addon-pipeline-orchestrator 1.2.290 → 1.2.292
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DHY2k_nC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets--PHyrJOr.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-tJspAxws.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CdkZH5kw.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CcezrkWV.mjs} +1 -1
- package/dist/{hostInit-CRtAczui.mjs → hostInit-D7n994Lm.mjs} +2 -2
- package/dist/index.js +1337 -129
- package/dist/index.mjs +1337 -129
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DWLgJB4I.mjs +0 -26
package/dist/index.mjs
CHANGED
|
@@ -24633,7 +24633,7 @@ var pipelineExecutorCapability = {
|
|
|
24633
24633
|
auth: "admin"
|
|
24634
24634
|
}),
|
|
24635
24635
|
/**
|
|
24636
|
-
*
|
|
24636
|
+
* Auto tune (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24637
24637
|
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24638
24638
|
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24639
24639
|
* running model's own defaults apply. Every other key of a camera's store
|
|
@@ -25005,7 +25005,165 @@ var pipelineExecutorCapability = {
|
|
|
25005
25005
|
}
|
|
25006
25006
|
};
|
|
25007
25007
|
/**
|
|
25008
|
-
*
|
|
25008
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25009
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25010
|
+
* hardware share suggestion and the stale device pins.
|
|
25011
|
+
*/
|
|
25012
|
+
/**
|
|
25013
|
+
* How a new detection session picks its node and device.
|
|
25014
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25015
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25016
|
+
* weight only bites once a node already holds a session.
|
|
25017
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25018
|
+
* full or drowning node yields to a free one (D748).
|
|
25019
|
+
*/
|
|
25020
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25021
|
+
/**
|
|
25022
|
+
* `share` since D750: the percentages are held by default. `least-loaded` stays
|
|
25023
|
+
* selectable.
|
|
25024
|
+
*/
|
|
25025
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "share";
|
|
25026
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25027
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25028
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25029
|
+
function parseSessionPlacementPolicy(value) {
|
|
25030
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25031
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25032
|
+
}
|
|
25033
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25034
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25035
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25036
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25037
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25038
|
+
});
|
|
25039
|
+
var PlacementStatsInputSchema = object({
|
|
25040
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25041
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25042
|
+
var PlacementDeviceStatsSchema = object({
|
|
25043
|
+
deviceKey: string(),
|
|
25044
|
+
/** Sessions running on this device now. */
|
|
25045
|
+
sessionsNow: number(),
|
|
25046
|
+
/** Sessions started on this device inside the window. */
|
|
25047
|
+
sessionsStarted: number(),
|
|
25048
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25049
|
+
observedShare: number().nullable()
|
|
25050
|
+
});
|
|
25051
|
+
var PlacementNodeStatsSchema = object({
|
|
25052
|
+
nodeId: string(),
|
|
25053
|
+
sessionsNow: number(),
|
|
25054
|
+
sessionsStarted: number(),
|
|
25055
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25056
|
+
observedShare: number().nullable(),
|
|
25057
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25058
|
+
peakSessions: number(),
|
|
25059
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25060
|
+
yieldsAway: number(),
|
|
25061
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25062
|
+
});
|
|
25063
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25064
|
+
var StaleDevicePinSchema = object({
|
|
25065
|
+
deviceId: number(),
|
|
25066
|
+
deviceKey: string(),
|
|
25067
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25068
|
+
reason: _enum(["disabled", "absent"])
|
|
25069
|
+
});
|
|
25070
|
+
var PlacementStatsSchema = object({
|
|
25071
|
+
policy: SessionPlacementPolicySchema,
|
|
25072
|
+
windowMs: number(),
|
|
25073
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25074
|
+
observedForMs: number(),
|
|
25075
|
+
sessionsNow: number(),
|
|
25076
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25077
|
+
clusterPeakSessions: number(),
|
|
25078
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25079
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25080
|
+
});
|
|
25081
|
+
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25082
|
+
var SuggestedDeviceShareSchema = object({
|
|
25083
|
+
deviceKey: string(),
|
|
25084
|
+
/** Share of the node (0–100, rounded). */
|
|
25085
|
+
share: number(),
|
|
25086
|
+
modelId: string().nullable(),
|
|
25087
|
+
/** Inferences per second at that model, from the measured table. */
|
|
25088
|
+
fps: number().nullable(),
|
|
25089
|
+
measured: boolean(),
|
|
25090
|
+
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25091
|
+
suggestedMaxSessions: number().nullable()
|
|
25092
|
+
});
|
|
25093
|
+
var SuggestedNodeShareSchema = object({
|
|
25094
|
+
nodeId: string(),
|
|
25095
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25096
|
+
share: number().nullable(),
|
|
25097
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25098
|
+
capacitySessions: number(),
|
|
25099
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25100
|
+
skippedReason: string().nullable()
|
|
25101
|
+
});
|
|
25102
|
+
var SuggestSharesResultSchema = object({
|
|
25103
|
+
headroom: number(),
|
|
25104
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25105
|
+
cameraFps: number(),
|
|
25106
|
+
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25107
|
+
iterations: number(),
|
|
25108
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25109
|
+
});
|
|
25110
|
+
/**
|
|
25111
|
+
* Where the node and device shares come from (D750).
|
|
25112
|
+
* - `auto` — computed by the system from the cluster's measured capacity at
|
|
25113
|
+
* the Auto tune models, and recomputed whenever a node or device joins,
|
|
25114
|
+
* leaves, is switched on or off, or changes model.
|
|
25115
|
+
* - `manual` — the operator's stored sliders. A node or device that cannot
|
|
25116
|
+
* take sessions gives its share to the others in proportion; the stored
|
|
25117
|
+
* values are never rewritten for that.
|
|
25118
|
+
*/
|
|
25119
|
+
var ShareSourceSchema = _enum(["auto", "manual"]);
|
|
25120
|
+
var DEFAULT_SHARE_SOURCE = "auto";
|
|
25121
|
+
/** Headroom the automatic shares are computed at (Auto tune's default). */
|
|
25122
|
+
var AUTO_SHARE_HEADROOM = .4;
|
|
25123
|
+
/** The global-settings key the share source is stored under (orchestrator `load` section). */
|
|
25124
|
+
var SHARE_SOURCE_SETTING_KEY = "shareSource";
|
|
25125
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25126
|
+
function parseShareSource(value) {
|
|
25127
|
+
const parsed = ShareSourceSchema.safeParse(value);
|
|
25128
|
+
return parsed.success ? parsed.data : DEFAULT_SHARE_SOURCE;
|
|
25129
|
+
}
|
|
25130
|
+
var SetShareSourceInputSchema = object({ source: ShareSourceSchema });
|
|
25131
|
+
object({
|
|
25132
|
+
nodes: record(string(), number()),
|
|
25133
|
+
devices: record(string(), record(string(), number()))
|
|
25134
|
+
});
|
|
25135
|
+
var EffectiveDeviceShareSchema = object({
|
|
25136
|
+
deviceKey: string(),
|
|
25137
|
+
/** Can take sessions now (enabled, available, can run the root). */
|
|
25138
|
+
eligible: boolean(),
|
|
25139
|
+
/** The share the source assigns, as a percentage of the node's devices. */
|
|
25140
|
+
configuredPct: number(),
|
|
25141
|
+
/** The share actually used, after an ineligible device's share went to the others. */
|
|
25142
|
+
effectivePct: number()
|
|
25143
|
+
});
|
|
25144
|
+
var EffectiveNodeShareSchema = object({
|
|
25145
|
+
nodeId: string(),
|
|
25146
|
+
eligible: boolean(),
|
|
25147
|
+
configuredPct: number(),
|
|
25148
|
+
effectivePct: number(),
|
|
25149
|
+
devices: array(EffectiveDeviceShareSchema).readonly()
|
|
25150
|
+
});
|
|
25151
|
+
var EffectiveSharesSchema = object({
|
|
25152
|
+
source: ShareSourceSchema,
|
|
25153
|
+
policy: SessionPlacementPolicySchema,
|
|
25154
|
+
/** `auto` with no computed answer yet: the stored sliders are used meanwhile. */
|
|
25155
|
+
autoPending: boolean(),
|
|
25156
|
+
/** When the automatic shares were last computed (ms epoch); null = never. */
|
|
25157
|
+
computedAt: number().nullable(),
|
|
25158
|
+
/** What made the system recompute them last. */
|
|
25159
|
+
trigger: string().nullable(),
|
|
25160
|
+
nodes: array(EffectiveNodeShareSchema).readonly(),
|
|
25161
|
+
/** The relative weights the election reads (before eligibility). */
|
|
25162
|
+
nodeWeights: record(string(), number()),
|
|
25163
|
+
deviceWeights: record(string(), record(string(), number()))
|
|
25164
|
+
});
|
|
25165
|
+
/**
|
|
25166
|
+
* The wire shapes of the Auto tune plan and apply (D745), used by the
|
|
25009
25167
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25010
25168
|
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
25011
25169
|
* schemas are what crosses the cap boundary.
|
|
@@ -25081,7 +25239,9 @@ var DeviceDemandSchema = object({
|
|
|
25081
25239
|
"estimate",
|
|
25082
25240
|
"none"
|
|
25083
25241
|
]),
|
|
25084
|
-
observedForMs: number().nullable()
|
|
25242
|
+
observedForMs: number().nullable(),
|
|
25243
|
+
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25244
|
+
placement: SessionPlacementPolicySchema
|
|
25085
25245
|
});
|
|
25086
25246
|
var DeviceModelChoiceSchema = object({
|
|
25087
25247
|
modelId: string(),
|
|
@@ -25138,7 +25298,7 @@ var DetectorStoreClearSchema = object({
|
|
|
25138
25298
|
deviceId: number(),
|
|
25139
25299
|
keys: array(string()).readonly()
|
|
25140
25300
|
});
|
|
25141
|
-
var
|
|
25301
|
+
var AutoTuneWarningSchema = discriminatedUnion("kind", [
|
|
25142
25302
|
object({
|
|
25143
25303
|
kind: literal("detector-store-cluster-wide"),
|
|
25144
25304
|
skippedNodes: array(string()).readonly()
|
|
@@ -25153,26 +25313,36 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25153
25313
|
deviceKey: string()
|
|
25154
25314
|
})
|
|
25155
25315
|
]);
|
|
25156
|
-
var
|
|
25157
|
-
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25158
|
-
headroom: number().min(.3).max(.5).default(.4)
|
|
25159
|
-
|
|
25316
|
+
var AutoTuneOptionsSchema = object({
|
|
25317
|
+
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25318
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25319
|
+
/**
|
|
25320
|
+
* What-if shares (D748): plan as if these node/device shares were stored
|
|
25321
|
+
* and the policy were `share`. Absent = the stored shares under the stored
|
|
25322
|
+
* policy. A what-if plan is for display; apply always re-plans from what
|
|
25323
|
+
* is stored.
|
|
25324
|
+
*/
|
|
25325
|
+
shares: LoadBalancingSharesInputSchema.optional()
|
|
25326
|
+
});
|
|
25327
|
+
var AutoTunePlanSchema = object({
|
|
25160
25328
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25161
25329
|
planHash: string(),
|
|
25162
25330
|
family: literal("yolov9"),
|
|
25163
25331
|
headroom: number(),
|
|
25332
|
+
/** Planned under what-if shares (D748): for display — apply refuses it as stale. */
|
|
25333
|
+
whatIf: boolean(),
|
|
25164
25334
|
generatedAt: number(),
|
|
25165
25335
|
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25166
25336
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25167
25337
|
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25168
25338
|
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25169
|
-
warnings: array(
|
|
25339
|
+
warnings: array(AutoTuneWarningSchema).readonly()
|
|
25170
25340
|
});
|
|
25171
|
-
var
|
|
25341
|
+
var AutoTuneApplyInputSchema = object({
|
|
25172
25342
|
planHash: string(),
|
|
25173
25343
|
headroom: number().min(.3).max(.5).default(.4)
|
|
25174
25344
|
});
|
|
25175
|
-
var
|
|
25345
|
+
var AutoTunePhaseResultSchema = object({
|
|
25176
25346
|
phase: _enum([
|
|
25177
25347
|
"download",
|
|
25178
25348
|
"camera-overrides",
|
|
@@ -25183,19 +25353,19 @@ var BestSettingsPhaseResultSchema = object({
|
|
|
25183
25353
|
ok: boolean(),
|
|
25184
25354
|
detail: string()
|
|
25185
25355
|
});
|
|
25186
|
-
var
|
|
25356
|
+
var AutoTuneApplyResultSchema = discriminatedUnion("kind", [
|
|
25187
25357
|
object({
|
|
25188
25358
|
kind: literal("applied"),
|
|
25189
25359
|
planHash: string(),
|
|
25190
|
-
phases: array(
|
|
25360
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25191
25361
|
}),
|
|
25192
25362
|
object({
|
|
25193
25363
|
kind: literal("aborted"),
|
|
25194
|
-
phases: array(
|
|
25364
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25195
25365
|
}),
|
|
25196
25366
|
object({
|
|
25197
25367
|
kind: literal("stale"),
|
|
25198
|
-
plan:
|
|
25368
|
+
plan: AutoTunePlanSchema
|
|
25199
25369
|
}),
|
|
25200
25370
|
object({
|
|
25201
25371
|
kind: literal("refused"),
|
|
@@ -26610,7 +26780,6 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26610
26780
|
settings: record(string(), unknown()).readonly().optional()
|
|
26611
26781
|
});
|
|
26612
26782
|
var CameraPipelineSettingsSchema = object({
|
|
26613
|
-
pinnedAgentNodeId: string().optional(),
|
|
26614
26783
|
stepToggles: record(string(), boolean()).optional(),
|
|
26615
26784
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26616
26785
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
@@ -27258,18 +27427,6 @@ var pipelineOrchestratorCapability = {
|
|
|
27258
27427
|
auth: "admin"
|
|
27259
27428
|
}),
|
|
27260
27429
|
/**
|
|
27261
|
-
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
27262
|
-
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
27263
|
-
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
27264
|
-
*/
|
|
27265
|
-
setAgentDetectWeight: method(object({
|
|
27266
|
-
agentNodeId: string(),
|
|
27267
|
-
detectWeight: number().positive().nullable()
|
|
27268
|
-
}), object({ success: literal(true) }), {
|
|
27269
|
-
kind: "mutation",
|
|
27270
|
-
auth: "admin"
|
|
27271
|
-
}),
|
|
27272
|
-
/**
|
|
27273
27430
|
* Set one node's placement CAPABILITIES — the per-node record that
|
|
27274
27431
|
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
27275
27432
|
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
@@ -27389,7 +27546,7 @@ var pipelineOrchestratorCapability = {
|
|
|
27389
27546
|
auth: "admin"
|
|
27390
27547
|
}),
|
|
27391
27548
|
/**
|
|
27392
|
-
*
|
|
27549
|
+
* Auto tune (D745) — DRY RUN. Recommends ONE detector family (YOLOv9)
|
|
27393
27550
|
* for the whole cluster from each node's static hardware class, the
|
|
27394
27551
|
* measured latency table and the detection demand, and lists every write
|
|
27395
27552
|
* the apply would make: the node model pins set, the enrichment pins and
|
|
@@ -27398,14 +27555,57 @@ var pipelineOrchestratorCapability = {
|
|
|
27398
27555
|
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27399
27556
|
* rules are never part of it. `planHash` names exactly this diff.
|
|
27400
27557
|
*/
|
|
27401
|
-
|
|
27558
|
+
planAutoTune: method(AutoTuneOptionsSchema, AutoTunePlanSchema, { auth: "admin" }),
|
|
27402
27559
|
/**
|
|
27403
|
-
*
|
|
27560
|
+
* Auto tune (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27404
27561
|
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27405
27562
|
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27406
27563
|
* nodes are skipped; there is no undo.
|
|
27407
27564
|
*/
|
|
27408
|
-
|
|
27565
|
+
applyAutoTune: method(AutoTuneApplyInputSchema, AutoTuneApplyResultSchema, {
|
|
27566
|
+
kind: "mutation",
|
|
27567
|
+
auth: "admin"
|
|
27568
|
+
}),
|
|
27569
|
+
/**
|
|
27570
|
+
* Load balancing (D748) — write node shares (`detectWeight`) and per-node
|
|
27571
|
+
* device shares (inference-device `weight`) in ONE atomic read-modify-write
|
|
27572
|
+
* of the agent settings — the only writer of node shares (the per-node
|
|
27573
|
+
* `setAgentDetectWeight` it replaced raced its siblings, one RMW each).
|
|
27574
|
+
* A node or device left out keeps its weight. Never changes the
|
|
27575
|
+
* `sessionPlacement` policy.
|
|
27576
|
+
*/
|
|
27577
|
+
setLoadBalancingShares: method(LoadBalancingSharesInputSchema, object({ success: literal(true) }), {
|
|
27578
|
+
kind: "mutation",
|
|
27579
|
+
auth: "admin"
|
|
27580
|
+
}),
|
|
27581
|
+
/**
|
|
27582
|
+
* Load balancing (D748) — what placement actually did: per node and
|
|
27583
|
+
* device, sessions now, sessions started and the time-weighted share of
|
|
27584
|
+
* PARALLEL sessions over `windowMs`; the cluster's parallel peak; yields
|
|
27585
|
+
* a share election made; and every device pin the session path cannot
|
|
27586
|
+
* honour. In memory on the hub — a restart starts a new window.
|
|
27587
|
+
*/
|
|
27588
|
+
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27589
|
+
/**
|
|
27590
|
+
* Load balancing (D748) — shares proportional to each node's measured
|
|
27591
|
+
* capacity at the models Auto tune would pick, iterated (≤ 3 rounds)
|
|
27592
|
+
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27593
|
+
* Writes nothing.
|
|
27594
|
+
*/
|
|
27595
|
+
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27596
|
+
/**
|
|
27597
|
+
* Load balancing (D750) — the shares placement reads now: the source
|
|
27598
|
+
* (automatic capacity shares or the operator's sliders), each node's and
|
|
27599
|
+
* device's configured share, and the effective share after a node or
|
|
27600
|
+
* device that cannot take sessions gave its share to the others.
|
|
27601
|
+
*/
|
|
27602
|
+
getEffectiveShares: method(_void(), EffectiveSharesSchema, { auth: "admin" }),
|
|
27603
|
+
/**
|
|
27604
|
+
* Load balancing (D750) — switch between automatic and manual shares.
|
|
27605
|
+
* Going manual seeds the sliders with the current automatic shares (one
|
|
27606
|
+
* atomic write), so nothing jumps; going automatic recomputes at once.
|
|
27607
|
+
*/
|
|
27608
|
+
setShareSource: method(SetShareSourceInputSchema, object({ success: literal(true) }), {
|
|
27409
27609
|
kind: "mutation",
|
|
27410
27610
|
auth: "admin"
|
|
27411
27611
|
}),
|
|
@@ -43432,7 +43632,7 @@ Object.freeze({
|
|
|
43432
43632
|
addonId: null,
|
|
43433
43633
|
access: "view"
|
|
43434
43634
|
},
|
|
43435
|
-
"pipelineOrchestrator.
|
|
43635
|
+
"pipelineOrchestrator.applyAutoTune": {
|
|
43436
43636
|
capName: "pipeline-orchestrator",
|
|
43437
43637
|
capScope: "system",
|
|
43438
43638
|
addonId: null,
|
|
@@ -43552,6 +43752,12 @@ Object.freeze({
|
|
|
43552
43752
|
addonId: null,
|
|
43553
43753
|
access: "view"
|
|
43554
43754
|
},
|
|
43755
|
+
"pipelineOrchestrator.getEffectiveShares": {
|
|
43756
|
+
capName: "pipeline-orchestrator",
|
|
43757
|
+
capScope: "system",
|
|
43758
|
+
addonId: null,
|
|
43759
|
+
access: "view"
|
|
43760
|
+
},
|
|
43555
43761
|
"pipelineOrchestrator.getGlobalMetrics": {
|
|
43556
43762
|
capName: "pipeline-orchestrator",
|
|
43557
43763
|
capScope: "system",
|
|
@@ -43588,6 +43794,12 @@ Object.freeze({
|
|
|
43588
43794
|
addonId: null,
|
|
43589
43795
|
access: "view"
|
|
43590
43796
|
},
|
|
43797
|
+
"pipelineOrchestrator.getPlacementStats": {
|
|
43798
|
+
capName: "pipeline-orchestrator",
|
|
43799
|
+
capScope: "system",
|
|
43800
|
+
addonId: null,
|
|
43801
|
+
access: "view"
|
|
43802
|
+
},
|
|
43591
43803
|
"pipelineOrchestrator.getReconcileFromDiskStatus": {
|
|
43592
43804
|
capName: "pipeline-orchestrator",
|
|
43593
43805
|
capScope: "system",
|
|
@@ -43612,7 +43824,7 @@ Object.freeze({
|
|
|
43612
43824
|
addonId: null,
|
|
43613
43825
|
access: "create"
|
|
43614
43826
|
},
|
|
43615
|
-
"pipelineOrchestrator.
|
|
43827
|
+
"pipelineOrchestrator.planAutoTune": {
|
|
43616
43828
|
capName: "pipeline-orchestrator",
|
|
43617
43829
|
capScope: "system",
|
|
43618
43830
|
addonId: null,
|
|
@@ -43678,12 +43890,6 @@ Object.freeze({
|
|
|
43678
43890
|
addonId: null,
|
|
43679
43891
|
access: "create"
|
|
43680
43892
|
},
|
|
43681
|
-
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
43682
|
-
capName: "pipeline-orchestrator",
|
|
43683
|
-
capScope: "system",
|
|
43684
|
-
addonId: null,
|
|
43685
|
-
access: "create"
|
|
43686
|
-
},
|
|
43687
43893
|
"pipelineOrchestrator.setAgentInferenceDevices": {
|
|
43688
43894
|
capName: "pipeline-orchestrator",
|
|
43689
43895
|
capScope: "system",
|
|
@@ -43738,12 +43944,30 @@ Object.freeze({
|
|
|
43738
43944
|
addonId: null,
|
|
43739
43945
|
access: "create"
|
|
43740
43946
|
},
|
|
43947
|
+
"pipelineOrchestrator.setLoadBalancingShares": {
|
|
43948
|
+
capName: "pipeline-orchestrator",
|
|
43949
|
+
capScope: "system",
|
|
43950
|
+
addonId: null,
|
|
43951
|
+
access: "create"
|
|
43952
|
+
},
|
|
43741
43953
|
"pipelineOrchestrator.setPipelineDevicePin": {
|
|
43742
43954
|
capName: "pipeline-orchestrator",
|
|
43743
43955
|
capScope: "system",
|
|
43744
43956
|
addonId: null,
|
|
43745
43957
|
access: "create"
|
|
43746
43958
|
},
|
|
43959
|
+
"pipelineOrchestrator.setShareSource": {
|
|
43960
|
+
capName: "pipeline-orchestrator",
|
|
43961
|
+
capScope: "system",
|
|
43962
|
+
addonId: null,
|
|
43963
|
+
access: "create"
|
|
43964
|
+
},
|
|
43965
|
+
"pipelineOrchestrator.suggestShares": {
|
|
43966
|
+
capName: "pipeline-orchestrator",
|
|
43967
|
+
capScope: "system",
|
|
43968
|
+
addonId: null,
|
|
43969
|
+
access: "view"
|
|
43970
|
+
},
|
|
43747
43971
|
"pipelineOrchestrator.unassignAudio": {
|
|
43748
43972
|
capName: "pipeline-orchestrator",
|
|
43749
43973
|
capScope: "system",
|
|
@@ -50976,7 +51200,7 @@ function modelBindingKey(key) {
|
|
|
50976
51200
|
/**
|
|
50977
51201
|
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
50978
51202
|
* flattened (D738). One definition for the runner (which applies the binding)
|
|
50979
|
-
* and the orchestrator's
|
|
51203
|
+
* and the orchestrator's Auto tune (which clears those overrides, D745).
|
|
50980
51204
|
*/
|
|
50981
51205
|
function collectModelBoundKeys(fields) {
|
|
50982
51206
|
return fields.flatMap((field) => {
|
|
@@ -51493,6 +51717,7 @@ function estimatedConcurrentSessions(cameraCount, options) {
|
|
|
51493
51717
|
*/
|
|
51494
51718
|
function estimateDeviceDemand(input) {
|
|
51495
51719
|
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51720
|
+
const placement = input.shares ? "share" : "least-loaded";
|
|
51496
51721
|
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51497
51722
|
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51498
51723
|
const weightByNode = /* @__PURE__ */ new Map();
|
|
@@ -51512,19 +51737,28 @@ function estimateDeviceDemand(input) {
|
|
|
51512
51737
|
const noteMax = (key, fps) => {
|
|
51513
51738
|
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51514
51739
|
};
|
|
51740
|
+
const pinNodeOf = (pin) => {
|
|
51741
|
+
if (pin.nodeId !== null) return pin.nodeId;
|
|
51742
|
+
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51743
|
+
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51744
|
+
};
|
|
51745
|
+
let alwaysOnTotal = 0;
|
|
51515
51746
|
for (const cam of input.cameras) {
|
|
51516
51747
|
if (!charges(cam)) continue;
|
|
51517
|
-
|
|
51518
|
-
|
|
51748
|
+
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51749
|
+
if (cam.devicePin !== null && pinNode !== null) {
|
|
51750
|
+
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51519
51751
|
if (!eligibleById.has(key)) continue;
|
|
51520
51752
|
bump(pinnedFps, key, cam.fps);
|
|
51521
51753
|
noteMax(key, cam.fps);
|
|
51522
51754
|
continue;
|
|
51523
51755
|
}
|
|
51524
|
-
|
|
51525
|
-
|
|
51756
|
+
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51757
|
+
if (standing !== null) {
|
|
51758
|
+
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51759
|
+
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51526
51760
|
for (const d of input.eligible) {
|
|
51527
|
-
if (d.nodeId !==
|
|
51761
|
+
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51528
51762
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51529
51763
|
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51530
51764
|
bump(alwaysOnFps, key, cam.fps * share);
|
|
@@ -51538,13 +51772,23 @@ function estimateDeviceDemand(input) {
|
|
|
51538
51772
|
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51539
51773
|
const nodeCount = weightByNode.size;
|
|
51540
51774
|
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51775
|
+
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51776
|
+
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51777
|
+
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51778
|
+
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51541
51779
|
return input.eligible.map((d) => {
|
|
51542
51780
|
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51543
51781
|
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51544
51782
|
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51545
51783
|
let onMotionSessions = 0;
|
|
51546
51784
|
let onMotionSource = "none";
|
|
51547
|
-
if (onMotionRates.length > 0
|
|
51785
|
+
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51786
|
+
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51787
|
+
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51788
|
+
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51789
|
+
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51790
|
+
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51791
|
+
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51548
51792
|
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51549
51793
|
onMotionSource = "observed-24h";
|
|
51550
51794
|
} else {
|
|
@@ -51562,12 +51806,13 @@ function estimateDeviceDemand(input) {
|
|
|
51562
51806
|
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51563
51807
|
onMotionSessions,
|
|
51564
51808
|
onMotionSource,
|
|
51565
|
-
observedForMs: peak?.observedForMs ?? null
|
|
51809
|
+
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51810
|
+
placement
|
|
51566
51811
|
};
|
|
51567
51812
|
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51568
51813
|
}
|
|
51569
51814
|
/** The family the cluster runs. */
|
|
51570
|
-
var
|
|
51815
|
+
var AUTO_TUNE_FAMILY = "yolov9";
|
|
51571
51816
|
function round(value, digits = 3) {
|
|
51572
51817
|
const f = 10 ** digits;
|
|
51573
51818
|
return Math.round(value * f) / f;
|
|
@@ -51580,7 +51825,7 @@ function fitsOn(cell, demand, headroom) {
|
|
|
51580
51825
|
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51581
51826
|
}
|
|
51582
51827
|
function candidatesFor(contexts, formatsByModel) {
|
|
51583
|
-
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(
|
|
51828
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(AUTO_TUNE_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51584
51829
|
modelId: id,
|
|
51585
51830
|
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51586
51831
|
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
@@ -51791,6 +52036,97 @@ function recommendNode(input) {
|
|
|
51791
52036
|
reasons: nodeReasons
|
|
51792
52037
|
};
|
|
51793
52038
|
}
|
|
52039
|
+
/** Deficits equal up to float noise (0.7 × 2 − 1 is 0.3999…, not 0.4). */
|
|
52040
|
+
var DEFICIT_EPSILON = 1e-9;
|
|
52041
|
+
function byDeficit(a, b) {
|
|
52042
|
+
const diff = b.deficit - a.deficit;
|
|
52043
|
+
return (Math.abs(diff) < DEFICIT_EPSILON ? 0 : diff) || b.candidate.share - a.candidate.share || a.candidate.key.localeCompare(b.candidate.key);
|
|
52044
|
+
}
|
|
52045
|
+
function spareOf(c) {
|
|
52046
|
+
return c.spare ?? Number.POSITIVE_INFINITY;
|
|
52047
|
+
}
|
|
52048
|
+
function electByShare(candidates) {
|
|
52049
|
+
if (candidates.length === 0) return { kind: "none" };
|
|
52050
|
+
const shareSum = candidates.reduce((s, c) => s + Math.max(c.share, 0), 0);
|
|
52051
|
+
const running = candidates.reduce((s, c) => s + Math.max(c.running, 0), 0);
|
|
52052
|
+
const scored = candidates.map((candidate) => ({
|
|
52053
|
+
candidate,
|
|
52054
|
+
deficit: (shareSum > 0 ? Math.max(candidate.share, 0) / shareSum : 1 / candidates.length) * (running + 1) - candidate.running
|
|
52055
|
+
})).toSorted(byDeficit);
|
|
52056
|
+
const turn = scored[0].candidate;
|
|
52057
|
+
if (!turn.capped && !turn.saturated) return {
|
|
52058
|
+
kind: "elected",
|
|
52059
|
+
key: turn.key,
|
|
52060
|
+
reason: "share"
|
|
52061
|
+
};
|
|
52062
|
+
const free = scored.filter((s) => !s.candidate.capped && !s.candidate.saturated).toSorted((a, b) => spareOf(b.candidate) - spareOf(a.candidate) || byDeficit(a, b));
|
|
52063
|
+
const yieldReason = turn.capped ? "at-cap" : "saturated";
|
|
52064
|
+
if (free.length > 0) return {
|
|
52065
|
+
kind: "elected",
|
|
52066
|
+
key: free[0].candidate.key,
|
|
52067
|
+
reason: "yield",
|
|
52068
|
+
turnKey: turn.key,
|
|
52069
|
+
yieldReason
|
|
52070
|
+
};
|
|
52071
|
+
const uncapped = scored.find((s) => !s.candidate.capped);
|
|
52072
|
+
if (uncapped === void 0) return { kind: "none" };
|
|
52073
|
+
return {
|
|
52074
|
+
kind: "elected",
|
|
52075
|
+
key: uncapped.candidate.key,
|
|
52076
|
+
reason: "share"
|
|
52077
|
+
};
|
|
52078
|
+
}
|
|
52079
|
+
/** Weight a node or device has when its source names none (the balancer's own default). */
|
|
52080
|
+
var DEFAULT_WEIGHT = 1;
|
|
52081
|
+
function positive(value) {
|
|
52082
|
+
return value !== void 0 && value > 0 ? value : DEFAULT_WEIGHT;
|
|
52083
|
+
}
|
|
52084
|
+
function percentages(keys, weightOf, counts) {
|
|
52085
|
+
const total = keys.reduce((sum, k) => sum + (counts(k) ? weightOf(k) : 0), 0);
|
|
52086
|
+
return new Map(keys.map((k) => [k, total > 0 && counts(k) ? Math.round(weightOf(k) / total * 1e3) / 10 : 0]));
|
|
52087
|
+
}
|
|
52088
|
+
function resolveEffectiveShares(input) {
|
|
52089
|
+
const weights = input.source === "auto" && input.auto !== null && input.auto !== null ? input.auto : input.stored;
|
|
52090
|
+
const nodeWeights = {};
|
|
52091
|
+
for (const nodeId of input.nodeIds) nodeWeights[nodeId] = positive(weights.nodes[nodeId]);
|
|
52092
|
+
const eligibleNodes = new Set(input.eligibleNodes);
|
|
52093
|
+
const configured = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52094
|
+
const effective = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(k));
|
|
52095
|
+
const deviceWeights = {};
|
|
52096
|
+
const nodes = input.nodeIds.map((nodeId) => {
|
|
52097
|
+
const eligibleKeys = new Set(input.eligibleDevices[nodeId] ?? []);
|
|
52098
|
+
const named = weights.devices[nodeId] ?? {};
|
|
52099
|
+
const keys = [...new Set([...Object.keys(named), ...eligibleKeys])].toSorted();
|
|
52100
|
+
const perDevice = {};
|
|
52101
|
+
for (const key of keys) perDevice[key] = positive(named[key]);
|
|
52102
|
+
if (keys.length > 0) deviceWeights[nodeId] = perDevice;
|
|
52103
|
+
const devConfigured = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52104
|
+
const devEffective = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(nodeId) && eligibleKeys.has(k));
|
|
52105
|
+
const devices = keys.map((deviceKey) => ({
|
|
52106
|
+
deviceKey,
|
|
52107
|
+
eligible: eligibleKeys.has(deviceKey),
|
|
52108
|
+
configuredPct: devConfigured.get(deviceKey) ?? 0,
|
|
52109
|
+
effectivePct: devEffective.get(deviceKey) ?? 0
|
|
52110
|
+
}));
|
|
52111
|
+
return {
|
|
52112
|
+
nodeId,
|
|
52113
|
+
eligible: eligibleNodes.has(nodeId),
|
|
52114
|
+
configuredPct: configured.get(nodeId) ?? 0,
|
|
52115
|
+
effectivePct: effective.get(nodeId) ?? 0,
|
|
52116
|
+
devices
|
|
52117
|
+
};
|
|
52118
|
+
});
|
|
52119
|
+
return {
|
|
52120
|
+
source: input.source,
|
|
52121
|
+
policy: input.policy ?? "share",
|
|
52122
|
+
autoPending: input.source === "auto" && input.auto === null,
|
|
52123
|
+
computedAt: input.computedAt ?? null,
|
|
52124
|
+
trigger: input.trigger ?? null,
|
|
52125
|
+
nodes,
|
|
52126
|
+
nodeWeights,
|
|
52127
|
+
deviceWeights
|
|
52128
|
+
};
|
|
52129
|
+
}
|
|
51794
52130
|
/**
|
|
51795
52131
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
51796
52132
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -59853,6 +60189,35 @@ function pickInferenceDevice(input) {
|
|
|
59853
60189
|
const loadByKey = new Map((input.loads ?? []).map((l) => [l.deviceKey, l]));
|
|
59854
60190
|
const healthyKeys = eligibleKeys.filter((k) => !isDeviceSaturated(loadByKey.get(k)));
|
|
59855
60191
|
const consideredKeys = pin !== null || healthyKeys.length === 0 || healthyKeys.length === eligibleKeys.length ? eligibleKeys : healthyKeys;
|
|
60192
|
+
if (input.policy === "share" && pin === null) {
|
|
60193
|
+
const election = electByShare(eligibleKeys.map((key) => {
|
|
60194
|
+
const load = loadByKey.get(key);
|
|
60195
|
+
const attached = load?.attachedCameras ?? 0;
|
|
60196
|
+
const cap = input.caps?.[key];
|
|
60197
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
60198
|
+
return {
|
|
60199
|
+
key,
|
|
60200
|
+
share: input.eligible[key] ?? 1,
|
|
60201
|
+
running: attached,
|
|
60202
|
+
capped: hasCap && attached >= cap,
|
|
60203
|
+
saturated: isDeviceSaturated(load),
|
|
60204
|
+
...hasCap ? { spare: Math.max(0, cap - attached) } : {}
|
|
60205
|
+
};
|
|
60206
|
+
}));
|
|
60207
|
+
if (election.kind === "none") return {
|
|
60208
|
+
kind: "default-pool",
|
|
60209
|
+
reason: "all-devices-at-cap",
|
|
60210
|
+
eligibleKeys
|
|
60211
|
+
};
|
|
60212
|
+
return {
|
|
60213
|
+
kind: "selected",
|
|
60214
|
+
deviceKey: election.key,
|
|
60215
|
+
reason: "share",
|
|
60216
|
+
tieBreak: null,
|
|
60217
|
+
eligibleKeys,
|
|
60218
|
+
tiedKeys: [election.key]
|
|
60219
|
+
};
|
|
60220
|
+
}
|
|
59856
60221
|
const balancerInput = {
|
|
59857
60222
|
nodes: consideredKeys.map((k) => deviceAsLoad(k, loadByKey.get(k))),
|
|
59858
60223
|
preferredAgent: pin,
|
|
@@ -62156,6 +62521,7 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62156
62521
|
};
|
|
62157
62522
|
const okByNode = new Map(input.verdicts.filter((v) => v.state === "ok").map((v) => [v.nodeId, v]));
|
|
62158
62523
|
const sourceAttached = input.attachedByNode[saturated.nodeId] ?? 0;
|
|
62524
|
+
if (input.shares !== void 0) return planShareTarget(input, input.shares, saturated, movable, okByNode, now, guards);
|
|
62159
62525
|
for (const camera of movable) {
|
|
62160
62526
|
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62161
62527
|
const target = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => {
|
|
@@ -62180,6 +62546,52 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62180
62546
|
nodeId: saturated.nodeId
|
|
62181
62547
|
};
|
|
62182
62548
|
}
|
|
62549
|
+
/** Deficits equal up to float noise. */
|
|
62550
|
+
var SHARE_EPSILON = 1e-9;
|
|
62551
|
+
function underCap(input, nodeId) {
|
|
62552
|
+
const cap = input.nodeCaps?.[nodeId];
|
|
62553
|
+
const attached = input.attachedByNode[nodeId] ?? 0;
|
|
62554
|
+
return cap === null || cap === void 0 || cap <= 0 || attached < cap;
|
|
62555
|
+
}
|
|
62556
|
+
/**
|
|
62557
|
+
* Share placement's failover target (D750): among the camera's eligible,
|
|
62558
|
+
* `ok`, under-cap nodes, the one furthest below its share of the cameras
|
|
62559
|
+
* attached across the source and those nodes — the same deficit rule a new
|
|
62560
|
+
* session is placed by. A node at or over its share is no target, so a move
|
|
62561
|
+
* never undoes the balance. Nothing below its share ⇒ hold, naming the camera.
|
|
62562
|
+
*/
|
|
62563
|
+
function planShareTarget(input, shares, saturated, movable, okByNode, now, guards) {
|
|
62564
|
+
const shareOf = (nodeId) => {
|
|
62565
|
+
const share = shares[nodeId];
|
|
62566
|
+
return share !== void 0 && share > 0 ? share : 0;
|
|
62567
|
+
};
|
|
62568
|
+
for (const camera of movable) {
|
|
62569
|
+
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62570
|
+
const healthy = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => underCap(input, v.nodeId));
|
|
62571
|
+
const set = [saturated.nodeId, ...healthy.map((v) => v.nodeId)];
|
|
62572
|
+
const shareSum = set.reduce((sum, nodeId) => sum + shareOf(nodeId), 0);
|
|
62573
|
+
const attachedSum = set.reduce((sum, nodeId) => sum + (input.attachedByNode[nodeId] ?? 0), 0);
|
|
62574
|
+
if (shareSum <= 0) continue;
|
|
62575
|
+
const deficit = (nodeId) => shareOf(nodeId) / shareSum * attachedSum - (input.attachedByNode[nodeId] ?? 0);
|
|
62576
|
+
const target = healthy.filter((v) => deficit(v.nodeId) > SHARE_EPSILON).toSorted((a, b) => deficit(b.nodeId) - deficit(a.nodeId) || a.signals.score - b.signals.score || a.nodeId.localeCompare(b.nodeId))[0];
|
|
62577
|
+
if (!target) continue;
|
|
62578
|
+
return {
|
|
62579
|
+
kind: "move",
|
|
62580
|
+
deviceId: camera.deviceId,
|
|
62581
|
+
fromNodeId: saturated.nodeId,
|
|
62582
|
+
toNodeId: target.nodeId,
|
|
62583
|
+
fromScore: saturated.signals.score,
|
|
62584
|
+
toScore: target.signals.score
|
|
62585
|
+
};
|
|
62586
|
+
}
|
|
62587
|
+
const first = movable[0];
|
|
62588
|
+
return {
|
|
62589
|
+
kind: "hold",
|
|
62590
|
+
reason: "no-target",
|
|
62591
|
+
nodeId: saturated.nodeId,
|
|
62592
|
+
...first !== void 0 ? { deviceId: first.deviceId } : {}
|
|
62593
|
+
};
|
|
62594
|
+
}
|
|
62183
62595
|
//#endregion
|
|
62184
62596
|
//#region src/node-stress-controller.ts
|
|
62185
62597
|
/** Narrow an unknown settings value to a {@link NodeStressMode}. */
|
|
@@ -62388,17 +62800,27 @@ var NodeStressController = class NodeStressController {
|
|
|
62388
62800
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
62389
62801
|
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
62390
62802
|
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
62803
|
+
const candidates = await this.collectCandidates(now);
|
|
62804
|
+
const shares = await this.deps.readFailoverShares?.() ?? null;
|
|
62391
62805
|
const plan = planStressFailover({
|
|
62392
62806
|
verdicts,
|
|
62393
|
-
candidates
|
|
62807
|
+
candidates,
|
|
62394
62808
|
history,
|
|
62395
62809
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
62396
|
-
attachedByNode: this.attachedByNode()
|
|
62810
|
+
attachedByNode: this.attachedByNode(),
|
|
62811
|
+
...shares !== null ? { shares } : {}
|
|
62397
62812
|
}, now, this.guards());
|
|
62398
62813
|
if (plan.kind === "hold") {
|
|
62399
62814
|
this.deps.logger.warn("node saturated — no failover applied", {
|
|
62400
|
-
tags: {
|
|
62401
|
-
|
|
62815
|
+
tags: {
|
|
62816
|
+
nodeId: plan.nodeId ?? "",
|
|
62817
|
+
...plan.deviceId !== void 0 ? { deviceId: plan.deviceId } : {}
|
|
62818
|
+
},
|
|
62819
|
+
meta: {
|
|
62820
|
+
reason: plan.reason,
|
|
62821
|
+
rule: shares !== null ? "share-deficit" : "fewest-cameras",
|
|
62822
|
+
...plan.reason === "no-target" && shares !== null ? { why: "no healthy node below its share can take it" } : {}
|
|
62823
|
+
}
|
|
62402
62824
|
});
|
|
62403
62825
|
return;
|
|
62404
62826
|
}
|
|
@@ -64176,7 +64598,6 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
64176
64598
|
}).nullable()
|
|
64177
64599
|
});
|
|
64178
64600
|
var StoredCameraPipelineSettingsSchema = object({
|
|
64179
|
-
pinnedAgentNodeId: string().optional(),
|
|
64180
64601
|
stepToggles: record(string(), boolean()).optional(),
|
|
64181
64602
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
64182
64603
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
@@ -65011,12 +65432,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65011
65432
|
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
65012
65433
|
*/
|
|
65013
65434
|
async buildNodeWeights() {
|
|
65435
|
+
if (this.effectiveNodeWeights !== null) return this.effectiveNodeWeights();
|
|
65436
|
+
return this.buildStoredNodeWeights();
|
|
65437
|
+
}
|
|
65438
|
+
/**
|
|
65439
|
+
* The operator's stored node sliders, whatever the share source — what
|
|
65440
|
+
* `manual` applies and what `auto` falls back to before its first answer.
|
|
65441
|
+
*/
|
|
65442
|
+
async buildStoredNodeWeights() {
|
|
65014
65443
|
const blob = await this.agentSettingsState.get();
|
|
65015
65444
|
const weights = {};
|
|
65016
65445
|
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
65017
65446
|
return weights;
|
|
65018
65447
|
}
|
|
65019
65448
|
/**
|
|
65449
|
+
* Where {@link buildNodeWeights} reads from once the share source is wired
|
|
65450
|
+
* (D750): the effective shares (automatic or manual). Every balancer, the
|
|
65451
|
+
* share election and the stress failover read through it, so they can never
|
|
65452
|
+
* disagree on what a node's share is. Unset (tests, early boot) ⇒ stored.
|
|
65453
|
+
*/
|
|
65454
|
+
effectiveNodeWeights = null;
|
|
65455
|
+
setEffectiveNodeWeights(read) {
|
|
65456
|
+
this.effectiveNodeWeights = read;
|
|
65457
|
+
}
|
|
65458
|
+
/**
|
|
65020
65459
|
* The per-(node,device) BASE provisioning (C7.2/C7.4) for one `(node,
|
|
65021
65460
|
* deviceKey)` — the effective `stepId → {modelId?, settings?}` map the
|
|
65022
65461
|
* dispatch provisioning stage attaches onto the resolved step tree before
|
|
@@ -66680,7 +67119,15 @@ var TtlMemo = class {
|
|
|
66680
67119
|
//#endregion
|
|
66681
67120
|
//#region src/detection-session-registry.ts
|
|
66682
67121
|
var DetectionSessionRegistry = class {
|
|
67122
|
+
onChange;
|
|
66683
67123
|
sessions = /* @__PURE__ */ new Map();
|
|
67124
|
+
/**
|
|
67125
|
+
* @param onChange called after every register / deregister / clear with the
|
|
67126
|
+
* running set — feeds the placement statistics (D748). Never on a no-op.
|
|
67127
|
+
*/
|
|
67128
|
+
constructor(onChange = () => {}) {
|
|
67129
|
+
this.onChange = onChange;
|
|
67130
|
+
}
|
|
66684
67131
|
register(deviceId, nodeId, startedAt, triggeredBy, deviceKey) {
|
|
66685
67132
|
const existing = this.sessions.get(deviceId);
|
|
66686
67133
|
if (existing?.teardownTimer) clearTimeout(existing.teardownTimer);
|
|
@@ -66693,6 +67140,7 @@ var DetectionSessionRegistry = class {
|
|
|
66693
67140
|
...deviceKey ? { deviceKey } : {}
|
|
66694
67141
|
};
|
|
66695
67142
|
this.sessions.set(deviceId, session);
|
|
67143
|
+
this.onChange(this.list());
|
|
66696
67144
|
return session;
|
|
66697
67145
|
}
|
|
66698
67146
|
get(deviceId) {
|
|
@@ -66706,6 +67154,7 @@ var DetectionSessionRegistry = class {
|
|
|
66706
67154
|
if (!session) return void 0;
|
|
66707
67155
|
if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66708
67156
|
this.sessions.delete(deviceId);
|
|
67157
|
+
this.onChange(this.list());
|
|
66709
67158
|
return session;
|
|
66710
67159
|
}
|
|
66711
67160
|
setTeardownTimer(deviceId, timer) {
|
|
@@ -66728,9 +67177,39 @@ var DetectionSessionRegistry = class {
|
|
|
66728
67177
|
}
|
|
66729
67178
|
clear() {
|
|
66730
67179
|
for (const session of this.sessions.values()) if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
67180
|
+
const hadSessions = this.sessions.size > 0;
|
|
66731
67181
|
this.sessions.clear();
|
|
67182
|
+
if (hadSessions) this.onChange([]);
|
|
66732
67183
|
}
|
|
66733
67184
|
};
|
|
67185
|
+
//#endregion
|
|
67186
|
+
//#region src/session-share-placement.ts
|
|
67187
|
+
function nodeSaturated(load) {
|
|
67188
|
+
return load.devices.length > 0 && load.devices.every((d) => isDeviceSaturated(d));
|
|
67189
|
+
}
|
|
67190
|
+
function electNodeByShare(input) {
|
|
67191
|
+
return electByShare(input.loads.filter((l) => l.nodeId.length > 0 && input.eligible.includes(l.nodeId)).map((load) => {
|
|
67192
|
+
const cap = input.caps[load.nodeId];
|
|
67193
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
67194
|
+
const weight = input.weights[load.nodeId];
|
|
67195
|
+
return {
|
|
67196
|
+
key: load.nodeId,
|
|
67197
|
+
share: weight !== void 0 && weight > 0 ? weight : 1,
|
|
67198
|
+
running: input.runningByNode.get(load.nodeId) ?? 0,
|
|
67199
|
+
capped: hasCap && load.attachedCameras >= cap,
|
|
67200
|
+
saturated: nodeSaturated(load),
|
|
67201
|
+
...hasCap ? { spare: Math.max(0, cap - load.attachedCameras) } : {}
|
|
67202
|
+
};
|
|
67203
|
+
}));
|
|
67204
|
+
}
|
|
67205
|
+
/** Parallel sessions per node: sessions and standing cameras, each camera once. */
|
|
67206
|
+
function countRunningByNode(placements) {
|
|
67207
|
+
const seen = /* @__PURE__ */ new Map();
|
|
67208
|
+
for (const p of placements) if (!seen.has(p.deviceId)) seen.set(p.deviceId, p.nodeId);
|
|
67209
|
+
const out = /* @__PURE__ */ new Map();
|
|
67210
|
+
for (const nodeId of seen.values()) out.set(nodeId, (out.get(nodeId) ?? 0) + 1);
|
|
67211
|
+
return out;
|
|
67212
|
+
}
|
|
66734
67213
|
/**
|
|
66735
67214
|
* Minimum time a detection session stays open, whatever closes it.
|
|
66736
67215
|
*
|
|
@@ -66843,7 +67322,15 @@ var SessionDispatchController = class {
|
|
|
66843
67322
|
* (not `private`) — see the class doc for the two production/test reasons
|
|
66844
67323
|
* `index.ts` reaches through it directly.
|
|
66845
67324
|
*/
|
|
66846
|
-
sessionRegistry = new DetectionSessionRegistry()
|
|
67325
|
+
sessionRegistry = new DetectionSessionRegistry((sessions) => this.deps.placementStats?.noteSessions(sessions.map((s) => ({
|
|
67326
|
+
nodeId: s.nodeId,
|
|
67327
|
+
deviceKey: s.deviceKey ?? ""
|
|
67328
|
+
}))));
|
|
67329
|
+
/**
|
|
67330
|
+
* `deviceId|deviceKey` of device pins the session path has already said it
|
|
67331
|
+
* ignored — one line per stale pin, not one per session (D748).
|
|
67332
|
+
*/
|
|
67333
|
+
reportedStalePins = /* @__PURE__ */ new Set();
|
|
66847
67334
|
/**
|
|
66848
67335
|
* Per-device async mutex for session dispatch/teardown critical sections —
|
|
66849
67336
|
* backed by the generic `KeyedAsyncLock` (mirrors the audio controller's
|
|
@@ -67304,6 +67791,77 @@ var SessionDispatchController = class {
|
|
|
67304
67791
|
});
|
|
67305
67792
|
}
|
|
67306
67793
|
/**
|
|
67794
|
+
* The session's node. `least-loaded` (and any node pin, under either
|
|
67795
|
+
* policy) is `balance()` exactly as before. `share` holds the node shares
|
|
67796
|
+
* of PARALLEL sessions (D748): sessions now running and standing cameras
|
|
67797
|
+
* count toward their node, and a node at its cap or drowning yields —
|
|
67798
|
+
* logged with the reason, and counted against the node in the stats.
|
|
67799
|
+
*/
|
|
67800
|
+
electSessionNode(input) {
|
|
67801
|
+
const balanced = () => balance({
|
|
67802
|
+
nodes: input.loads,
|
|
67803
|
+
preferredAgent: input.preferredAgent,
|
|
67804
|
+
nodeCaps: input.nodeCaps,
|
|
67805
|
+
weights: input.weights,
|
|
67806
|
+
eligibleNodes: input.eligible
|
|
67807
|
+
});
|
|
67808
|
+
if (input.policy !== "share" || input.preferredAgent) return balanced();
|
|
67809
|
+
const running = countRunningByNode([...this.sessionRegistry.list().map((s) => ({
|
|
67810
|
+
deviceId: s.deviceId,
|
|
67811
|
+
nodeId: s.nodeId
|
|
67812
|
+
})), ...this.deps.ledger.listAssignments().map((a) => ({
|
|
67813
|
+
deviceId: a.deviceId,
|
|
67814
|
+
nodeId: a.agentNodeId
|
|
67815
|
+
}))]);
|
|
67816
|
+
const present = new Set(this.deps.topology.knownRunnerNodeIds());
|
|
67817
|
+
const election = electNodeByShare({
|
|
67818
|
+
loads: input.loads,
|
|
67819
|
+
eligible: input.eligible.filter((nodeId) => present.has(nodeId)),
|
|
67820
|
+
weights: input.weights,
|
|
67821
|
+
caps: input.nodeCaps,
|
|
67822
|
+
runningByNode: running
|
|
67823
|
+
});
|
|
67824
|
+
if (election.kind === "none") return balanced();
|
|
67825
|
+
if (election.reason === "yield") {
|
|
67826
|
+
this.deps.placementStats?.noteYield(election.turnKey);
|
|
67827
|
+
this.deps.logger.info("detection session: share turn yielded — the node had no room", {
|
|
67828
|
+
tags: {
|
|
67829
|
+
deviceId: input.deviceId,
|
|
67830
|
+
nodeId: election.key
|
|
67831
|
+
},
|
|
67832
|
+
meta: {
|
|
67833
|
+
turnNodeId: election.turnKey,
|
|
67834
|
+
reason: election.yieldReason
|
|
67835
|
+
}
|
|
67836
|
+
});
|
|
67837
|
+
}
|
|
67838
|
+
const load = input.loads.find((l) => l.nodeId === election.key);
|
|
67839
|
+
return {
|
|
67840
|
+
kind: "assigned",
|
|
67841
|
+
agentNodeId: election.key,
|
|
67842
|
+
reason: "capacity",
|
|
67843
|
+
score: load ? load.attachedCameras + load.queueDepthTotal : 0
|
|
67844
|
+
};
|
|
67845
|
+
}
|
|
67846
|
+
/**
|
|
67847
|
+
* Say ONCE that a camera's device pin names a device this node does not
|
|
67848
|
+
* offer (disabled, absent): the session path ignores it, and the Load
|
|
67849
|
+
* balancing tab offers to clear it. Never cleared here (D748).
|
|
67850
|
+
*/
|
|
67851
|
+
noteStalePin(deviceId, pinnedDeviceKey, enabledDevices) {
|
|
67852
|
+
if (pinnedDeviceKey === null || pinnedDeviceKey in enabledDevices) return;
|
|
67853
|
+
const key = `${deviceId}|${pinnedDeviceKey}`;
|
|
67854
|
+
if (this.reportedStalePins.has(key)) return;
|
|
67855
|
+
this.reportedStalePins.add(key);
|
|
67856
|
+
this.deps.logger.warn("device pin ignored — the pinned device is not enabled on the elected node", {
|
|
67857
|
+
tags: { deviceId },
|
|
67858
|
+
meta: {
|
|
67859
|
+
pin: pinnedDeviceKey,
|
|
67860
|
+
enabled: Object.keys(enabledDevices).join(",")
|
|
67861
|
+
}
|
|
67862
|
+
});
|
|
67863
|
+
}
|
|
67864
|
+
/**
|
|
67307
67865
|
* Dispatch a session-time detection attach for an on-motion camera whose
|
|
67308
67866
|
* `decideSessionAction` returned `'dispatch'`. Builds a session
|
|
67309
67867
|
* `RunnerCameraConfig` the same way `startDetection` does, but forces
|
|
@@ -67354,12 +67912,17 @@ var SessionDispatchController = class {
|
|
|
67354
67912
|
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
67355
67913
|
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
67356
67914
|
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
67357
|
-
const
|
|
67358
|
-
|
|
67915
|
+
const nodeCaps = await this.nodeCaps.get(CLUSTER_KEY);
|
|
67916
|
+
const weights = await this.nodeWeights.get(CLUSTER_KEY);
|
|
67917
|
+
const policy = this.deps.sessionPlacement?.() ?? "least-loaded";
|
|
67918
|
+
const decision = this.electSessionNode({
|
|
67919
|
+
deviceId,
|
|
67920
|
+
policy,
|
|
67921
|
+
loads,
|
|
67359
67922
|
preferredAgent,
|
|
67360
|
-
nodeCaps
|
|
67361
|
-
weights
|
|
67362
|
-
|
|
67923
|
+
nodeCaps,
|
|
67924
|
+
weights,
|
|
67925
|
+
eligible
|
|
67363
67926
|
});
|
|
67364
67927
|
if (!decision || decision.kind !== "assigned") {
|
|
67365
67928
|
log.warn("detection session: no eligible node — will retry on next motion", { meta: { reason: decision?.kind === "pending" ? decision.reason : "no-runners-online" } });
|
|
@@ -67372,12 +67935,14 @@ var SessionDispatchController = class {
|
|
|
67372
67935
|
const targetLoad = loads.find((n) => n.nodeId === targetNodeId);
|
|
67373
67936
|
const deviceCaps = await this.inferenceDeviceCaps.get(targetNodeId);
|
|
67374
67937
|
const pinnedDeviceKey = await this.deps.readPipelineDevicePin(deviceId);
|
|
67938
|
+
this.noteStalePin(deviceId, pinnedDeviceKey, enabledDevices);
|
|
67375
67939
|
const selection = pickInferenceDevice({
|
|
67376
67940
|
eligible: enabledDevices,
|
|
67377
67941
|
loads: targetLoad?.devices ?? [],
|
|
67378
67942
|
caps: deviceCaps,
|
|
67379
|
-
pinnedDeviceKey,
|
|
67380
|
-
rotation: this.deps.inferenceRotation
|
|
67943
|
+
pinnedDeviceKey: pinnedDeviceKey !== null && pinnedDeviceKey in enabledDevices ? pinnedDeviceKey : null,
|
|
67944
|
+
rotation: this.deps.inferenceRotation,
|
|
67945
|
+
policy
|
|
67381
67946
|
});
|
|
67382
67947
|
const deviceKey = selectedDeviceKey(selection);
|
|
67383
67948
|
log.info("inference device selection", {
|
|
@@ -67465,6 +68030,7 @@ var SessionDispatchController = class {
|
|
|
67465
68030
|
attachedAt = Date.now();
|
|
67466
68031
|
this.deps.recordAssignment(deviceId, targetNodeId, decision.reason, decision.reason === "manual");
|
|
67467
68032
|
this.sessionRegistry.register(deviceId, targetNodeId, Date.now(), triggeredBy, deviceKey);
|
|
68033
|
+
this.deps.placementStats?.noteStart(targetNodeId, deviceKey ?? "");
|
|
67468
68034
|
if (this.sessionRefusals.delete(deviceId)) log.info("detection session refusal cleared — a session opened", {
|
|
67469
68035
|
tags: {
|
|
67470
68036
|
deviceId,
|
|
@@ -68719,7 +69285,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68719
69285
|
attach: (nodeId, config) => placement.attachOn(nodeId, config),
|
|
68720
69286
|
detach: (nodeId, deviceId) => placement.detachOn(nodeId, deviceId),
|
|
68721
69287
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
68722
|
-
readGlobalSettings: () => globalSettings
|
|
69288
|
+
readGlobalSettings: () => globalSettings,
|
|
69289
|
+
readFailoverShares: async () => readSessionPlacement() === "share" ? settingsStore.buildNodeWeights() : null
|
|
68723
69290
|
});
|
|
68724
69291
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
68725
69292
|
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
@@ -68734,6 +69301,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68734
69301
|
}, 6e4);
|
|
68735
69302
|
nodeStress.start();
|
|
68736
69303
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69304
|
+
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
69305
|
+
const readShareSource = () => parseShareSource(globalSettings?.[SHARE_SOURCE_SETTING_KEY]);
|
|
68737
69306
|
/**
|
|
68738
69307
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
68739
69308
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -68817,6 +69386,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68817
69386
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
68818
69387
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
68819
69388
|
inferenceRotation,
|
|
69389
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69390
|
+
placementStats: deps.placementStats,
|
|
68820
69391
|
logger: deps.ctx().logger
|
|
68821
69392
|
});
|
|
68822
69393
|
const deviceConfig = new DeviceConfigContributions({
|
|
@@ -69068,6 +69639,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69068
69639
|
nodeLifecycle.unsubscribeDetectionPipelineReadiness(nodeId);
|
|
69069
69640
|
audio.markNodeOffline(nodeId);
|
|
69070
69641
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
69642
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-offline:${nodeId}`);
|
|
69071
69643
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
69072
69644
|
const msg = errMsg$2(err);
|
|
69073
69645
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
@@ -69111,6 +69683,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69111
69683
|
});
|
|
69112
69684
|
});
|
|
69113
69685
|
reconcile.scheduleReconcile();
|
|
69686
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-online:${nodeId}`);
|
|
69114
69687
|
});
|
|
69115
69688
|
const unsubAgentOnline = () => {
|
|
69116
69689
|
unsubAgentOnlineEvent();
|
|
@@ -69350,7 +69923,9 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69350
69923
|
unsubAgentOffline,
|
|
69351
69924
|
unsubOrchestratorSubscriptions,
|
|
69352
69925
|
pendingRetryTimer,
|
|
69353
|
-
autoRebalanceTimer
|
|
69926
|
+
autoRebalanceTimer,
|
|
69927
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69928
|
+
shareSource: () => readShareSource()
|
|
69354
69929
|
};
|
|
69355
69930
|
}
|
|
69356
69931
|
//#endregion
|
|
@@ -69395,6 +69970,20 @@ function buildGlobalSettingsSections(options) {
|
|
|
69395
69970
|
title: "Load Balancer",
|
|
69396
69971
|
tab: "pipeline",
|
|
69397
69972
|
fields: [
|
|
69973
|
+
{
|
|
69974
|
+
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69975
|
+
type: "select",
|
|
69976
|
+
label: "Session placement",
|
|
69977
|
+
description: "How a new detection session picks its node and device. Honour shares (default) = the share of PARALLEL sessions is held to the node and device shares, and a node at its cap yields to a free one. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy.",
|
|
69978
|
+
options: [{
|
|
69979
|
+
value: "share",
|
|
69980
|
+
label: "Honour shares"
|
|
69981
|
+
}, {
|
|
69982
|
+
value: "least-loaded",
|
|
69983
|
+
label: "Least-loaded"
|
|
69984
|
+
}],
|
|
69985
|
+
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69986
|
+
},
|
|
69398
69987
|
{
|
|
69399
69988
|
key: "fpsMinThreshold",
|
|
69400
69989
|
type: "slider",
|
|
@@ -69953,9 +70542,9 @@ var pipelineOrchestratorWidgets = [{
|
|
|
69953
70542
|
defaultRows: 4
|
|
69954
70543
|
}];
|
|
69955
70544
|
//#endregion
|
|
69956
|
-
//#region src/
|
|
70545
|
+
//#region src/auto-tune/plan-diff.ts
|
|
69957
70546
|
/**
|
|
69958
|
-
* The
|
|
70547
|
+
* The Auto tune writes, as pure transforms of the orchestrator's two
|
|
69959
70548
|
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
69960
70549
|
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
69961
70550
|
*
|
|
@@ -69974,12 +70563,12 @@ var pipelineOrchestratorWidgets = [{
|
|
|
69974
70563
|
* settings: the attach provisions the node pin over the snapshot
|
|
69975
70564
|
* (`applyDeviceProvisioning`: override ?? base), so once the snapshot pin
|
|
69976
70565
|
* is gone the node's recommended model runs.
|
|
69977
|
-
* - never: `stepToggles`,
|
|
70566
|
+
* - never: `stepToggles`, devices' enablement, pins of
|
|
69978
70567
|
* embedding steps (cluster-scoped, changing one forces a re-index).
|
|
69979
70568
|
*/
|
|
69980
70569
|
/** The root detector step. */
|
|
69981
70570
|
var DETECTOR_STEP_ID = "object-detection";
|
|
69982
|
-
/** Steps whose `modelId` pin
|
|
70571
|
+
/** Steps whose `modelId` pin Auto tune never touches. */
|
|
69983
70572
|
function isModelPinKept(stepId) {
|
|
69984
70573
|
return stepId.endsWith("-embedding");
|
|
69985
70574
|
}
|
|
@@ -70031,7 +70620,7 @@ function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
|
70031
70620
|
};
|
|
70032
70621
|
}
|
|
70033
70622
|
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70034
|
-
function
|
|
70623
|
+
function applyNodeAutoTune(blob, input) {
|
|
70035
70624
|
const next = { ...blob };
|
|
70036
70625
|
const changes = [];
|
|
70037
70626
|
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
@@ -70101,7 +70690,7 @@ function scrubTree(steps, input, onClear) {
|
|
|
70101
70690
|
});
|
|
70102
70691
|
}
|
|
70103
70692
|
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70104
|
-
function
|
|
70693
|
+
function applyCameraAutoTune(blob, input) {
|
|
70105
70694
|
const next = {};
|
|
70106
70695
|
const clears = [];
|
|
70107
70696
|
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
@@ -70185,12 +70774,12 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70185
70774
|
};
|
|
70186
70775
|
}
|
|
70187
70776
|
//#endregion
|
|
70188
|
-
//#region src/
|
|
70777
|
+
//#region src/auto-tune/auto-tune-service.ts
|
|
70189
70778
|
/**
|
|
70190
|
-
*
|
|
70779
|
+
* Auto tune (D745): gather the inputs, compute the plan, and apply exactly
|
|
70191
70780
|
* the plan the operator confirmed.
|
|
70192
70781
|
*
|
|
70193
|
-
* The plan is the recommender (`@camstack/types`
|
|
70782
|
+
* The plan is the recommender (`@camstack/types` auto-tune) over every
|
|
70194
70783
|
* runner node, plus the diff of the writes (`plan-diff.ts`) and the
|
|
70195
70784
|
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70196
70785
|
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
@@ -70236,16 +70825,117 @@ function boundKeysOf(catalogs) {
|
|
|
70236
70825
|
}
|
|
70237
70826
|
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70238
70827
|
}
|
|
70239
|
-
|
|
70828
|
+
/** Bounded fixed point of the hardware share suggestion (D748). */
|
|
70829
|
+
var SUGGEST_MAX_ROUNDS = 3;
|
|
70830
|
+
/** Integer shares summing to 100 (largest remainder). */
|
|
70831
|
+
function toPercent(weights) {
|
|
70832
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70833
|
+
const out = /* @__PURE__ */ new Map();
|
|
70834
|
+
if (total <= 0) return out;
|
|
70835
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
70836
|
+
k,
|
|
70837
|
+
exact: w / total * 100
|
|
70838
|
+
}));
|
|
70839
|
+
let assigned = 0;
|
|
70840
|
+
for (const r of raw) {
|
|
70841
|
+
out.set(r.k, Math.floor(r.exact));
|
|
70842
|
+
assigned += Math.floor(r.exact);
|
|
70843
|
+
}
|
|
70844
|
+
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70845
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
70846
|
+
const r = byRemainder[i % byRemainder.length];
|
|
70847
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70848
|
+
}
|
|
70849
|
+
return out;
|
|
70850
|
+
}
|
|
70851
|
+
var AutoTuneService = class {
|
|
70240
70852
|
deps;
|
|
70241
70853
|
applying = false;
|
|
70242
70854
|
constructor(deps) {
|
|
70243
70855
|
this.deps = deps;
|
|
70244
70856
|
}
|
|
70245
70857
|
async plan(input) {
|
|
70246
|
-
return (await this.computePlan(input.headroom)).plan;
|
|
70858
|
+
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70247
70859
|
}
|
|
70248
|
-
|
|
70860
|
+
/**
|
|
70861
|
+
* Shares proportional to each node's capacity at the models Auto tune
|
|
70862
|
+
* would pick under those very shares (D748). Model and load depend on each
|
|
70863
|
+
* other, so it iterates from equal shares until the rounded shares stop
|
|
70864
|
+
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
70865
|
+
* ADVISORY session cap — never written, never enforced.
|
|
70866
|
+
*/
|
|
70867
|
+
async suggestShares(input) {
|
|
70868
|
+
let shares = {
|
|
70869
|
+
nodes: {},
|
|
70870
|
+
devices: {}
|
|
70871
|
+
};
|
|
70872
|
+
let result = [];
|
|
70873
|
+
let cameraFps = 10;
|
|
70874
|
+
let rounds = 0;
|
|
70875
|
+
for (let round = 1; round <= SUGGEST_MAX_ROUNDS; round++) {
|
|
70876
|
+
rounds = round;
|
|
70877
|
+
const rec = await this.recommend(input.headroom, shares);
|
|
70878
|
+
cameraFps = rec.cameraFps;
|
|
70879
|
+
result = this.sharesFrom(rec, input.headroom);
|
|
70880
|
+
const next = {
|
|
70881
|
+
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70882
|
+
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70883
|
+
};
|
|
70884
|
+
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70885
|
+
shares = next;
|
|
70886
|
+
if (settled) break;
|
|
70887
|
+
}
|
|
70888
|
+
return {
|
|
70889
|
+
headroom: input.headroom,
|
|
70890
|
+
cameraFps,
|
|
70891
|
+
iterations: rounds,
|
|
70892
|
+
nodes: [...result]
|
|
70893
|
+
};
|
|
70894
|
+
}
|
|
70895
|
+
sharesFrom(rec, headroom) {
|
|
70896
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70897
|
+
const perNode = rec.recommendations.map((r) => {
|
|
70898
|
+
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70899
|
+
const fps = d.chosen?.fps ?? 0;
|
|
70900
|
+
return {
|
|
70901
|
+
d,
|
|
70902
|
+
fps,
|
|
70903
|
+
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70904
|
+
};
|
|
70905
|
+
});
|
|
70906
|
+
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70907
|
+
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70908
|
+
return {
|
|
70909
|
+
r,
|
|
70910
|
+
devices,
|
|
70911
|
+
capacity
|
|
70912
|
+
};
|
|
70913
|
+
});
|
|
70914
|
+
const nodePct = toPercent(capacityByNode);
|
|
70915
|
+
return perNode.map(({ r, devices, capacity }) => {
|
|
70916
|
+
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70917
|
+
return {
|
|
70918
|
+
nodeId: r.nodeId,
|
|
70919
|
+
share: nodePct.get(r.nodeId) ?? null,
|
|
70920
|
+
capacitySessions: capacity,
|
|
70921
|
+
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70922
|
+
devices: devices.map((x) => ({
|
|
70923
|
+
deviceKey: x.d.deviceKey,
|
|
70924
|
+
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70925
|
+
modelId: x.d.chosen?.modelId ?? null,
|
|
70926
|
+
fps: x.d.chosen?.fps ?? null,
|
|
70927
|
+
measured: x.d.chosen?.measured ?? false,
|
|
70928
|
+
suggestedMaxSessions: x.sessions
|
|
70929
|
+
}))
|
|
70930
|
+
};
|
|
70931
|
+
});
|
|
70932
|
+
}
|
|
70933
|
+
/**
|
|
70934
|
+
* Gather every node and recommend. `shares` (a what-if or the suggestion's
|
|
70935
|
+
* round) forces share placement with those shares overlaid on the stored
|
|
70936
|
+
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70937
|
+
*/
|
|
70938
|
+
async recommend(headroom, shares) {
|
|
70249
70939
|
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70250
70940
|
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70251
70941
|
const [devices, hardware, catalog] = await Promise.all([
|
|
@@ -70260,6 +70950,8 @@ var BestSettingsService = class {
|
|
|
70260
70950
|
catalog
|
|
70261
70951
|
};
|
|
70262
70952
|
}));
|
|
70953
|
+
const effective = await this.deps.readEffectiveWeights();
|
|
70954
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? effective.devices[nodeId]?.[key] ?? stored;
|
|
70263
70955
|
const nodes = gathered.map((g) => ({
|
|
70264
70956
|
nodeId: g.nodeId,
|
|
70265
70957
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70275,18 +70967,37 @@ var BestSettingsService = class {
|
|
|
70275
70967
|
const eligible = gathered.flatMap((g) => g.devices.reachable ? g.devices.devices.filter((d) => d.enabled && d.available).map((d) => ({
|
|
70276
70968
|
nodeId: g.nodeId,
|
|
70277
70969
|
deviceKey: d.key,
|
|
70278
|
-
weight: d.weight
|
|
70970
|
+
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70279
70971
|
})) : []);
|
|
70972
|
+
const cameras = await this.deps.cameraLoads();
|
|
70973
|
+
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70280
70974
|
const demand = estimateDeviceDemand({
|
|
70281
|
-
cameras
|
|
70975
|
+
cameras,
|
|
70282
70976
|
eligible,
|
|
70283
|
-
peaks: this.deps.peaks()
|
|
70977
|
+
peaks: this.deps.peaks(),
|
|
70978
|
+
...policy === "share" ? { shares: {
|
|
70979
|
+
nodes: {
|
|
70980
|
+
...effective.nodes,
|
|
70981
|
+
...shares?.nodes
|
|
70982
|
+
},
|
|
70983
|
+
clusterPeak: this.deps.clusterPeak()
|
|
70984
|
+
} } : {}
|
|
70284
70985
|
});
|
|
70285
70986
|
const recommendations = nodes.map((node) => recommendNode({
|
|
70286
70987
|
node,
|
|
70287
70988
|
demand,
|
|
70288
70989
|
headroom
|
|
70289
70990
|
}));
|
|
70991
|
+
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
70992
|
+
return {
|
|
70993
|
+
gathered,
|
|
70994
|
+
nodes,
|
|
70995
|
+
recommendations,
|
|
70996
|
+
cameraFps: cameraFps > 0 ? cameraFps : 10
|
|
70997
|
+
};
|
|
70998
|
+
}
|
|
70999
|
+
async computePlan(headroom, shares) {
|
|
71000
|
+
const { gathered, nodes, recommendations } = await this.recommend(headroom, shares);
|
|
70290
71001
|
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70291
71002
|
const targets = /* @__PURE__ */ new Map();
|
|
70292
71003
|
const formats = /* @__PURE__ */ new Map();
|
|
@@ -70313,8 +71024,8 @@ var BestSettingsService = class {
|
|
|
70313
71024
|
targets,
|
|
70314
71025
|
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70315
71026
|
};
|
|
70316
|
-
const nodeChanges =
|
|
70317
|
-
const cameraClears =
|
|
71027
|
+
const nodeChanges = applyNodeAutoTune(await this.deps.readAgentSettings(), diffInput).changes;
|
|
71028
|
+
const cameraClears = applyCameraAutoTune(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70318
71029
|
const warnings = [];
|
|
70319
71030
|
let detectorStoreClears = [];
|
|
70320
71031
|
if (touchedNodes.size > 0) try {
|
|
@@ -70348,6 +71059,7 @@ var BestSettingsService = class {
|
|
|
70348
71059
|
planHash: createHash("sha256").update(canonical({
|
|
70349
71060
|
version: 1,
|
|
70350
71061
|
headroom,
|
|
71062
|
+
whatIf: shares ?? null,
|
|
70351
71063
|
picks: recommendations.map((r) => ({
|
|
70352
71064
|
nodeId: r.nodeId,
|
|
70353
71065
|
skipped: r.skipped,
|
|
@@ -70359,6 +71071,7 @@ var BestSettingsService = class {
|
|
|
70359
71071
|
})).digest("hex"),
|
|
70360
71072
|
family: "yolov9",
|
|
70361
71073
|
headroom,
|
|
71074
|
+
whatIf: shares !== void 0,
|
|
70362
71075
|
generatedAt: this.deps.now(),
|
|
70363
71076
|
nodes: recommendations,
|
|
70364
71077
|
nodeChanges: [...nodeChanges],
|
|
@@ -70385,7 +71098,7 @@ var BestSettingsService = class {
|
|
|
70385
71098
|
async applyLocked(input) {
|
|
70386
71099
|
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
70387
71100
|
if (plan.planHash !== input.planHash) {
|
|
70388
|
-
this.deps.logger.warn("
|
|
71101
|
+
this.deps.logger.warn("auto tune: stale plan refused", { meta: {
|
|
70389
71102
|
confirmed: input.planHash,
|
|
70390
71103
|
current: plan.planHash
|
|
70391
71104
|
} });
|
|
@@ -70417,15 +71130,15 @@ var BestSettingsService = class {
|
|
|
70417
71130
|
ok: false,
|
|
70418
71131
|
detail: errMsg(err)
|
|
70419
71132
|
});
|
|
70420
|
-
this.deps.logger.warn("
|
|
71133
|
+
this.deps.logger.warn("auto tune: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70421
71134
|
return {
|
|
70422
71135
|
kind: "aborted",
|
|
70423
71136
|
phases
|
|
70424
71137
|
};
|
|
70425
71138
|
}
|
|
70426
71139
|
try {
|
|
70427
|
-
await this.deps.rewriteCameraSettings((prev) =>
|
|
70428
|
-
for (const c of plan.cameraClears) this.deps.logger.info("
|
|
71140
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraAutoTune(prev, diffInput).next);
|
|
71141
|
+
for (const c of plan.cameraClears) this.deps.logger.info("auto tune: camera override cleared", {
|
|
70429
71142
|
tags: {
|
|
70430
71143
|
deviceId: c.deviceId,
|
|
70431
71144
|
nodeId: c.nodeId
|
|
@@ -70469,8 +71182,8 @@ var BestSettingsService = class {
|
|
|
70469
71182
|
});
|
|
70470
71183
|
}
|
|
70471
71184
|
try {
|
|
70472
|
-
await this.deps.rewriteAgentSettings((prev) =>
|
|
70473
|
-
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("
|
|
71185
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeAutoTune(prev, diffInput).next);
|
|
71186
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("auto tune: node model set", {
|
|
70474
71187
|
tags: { nodeId: c.nodeId },
|
|
70475
71188
|
meta: {
|
|
70476
71189
|
nodeId: c.nodeId,
|
|
@@ -70504,14 +71217,14 @@ var BestSettingsService = class {
|
|
|
70504
71217
|
ok: failed.length === 0,
|
|
70505
71218
|
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
70506
71219
|
});
|
|
70507
|
-
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("
|
|
71220
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("auto tune: node skipped", {
|
|
70508
71221
|
tags: { nodeId: rec.nodeId },
|
|
70509
71222
|
meta: {
|
|
70510
71223
|
nodeId: rec.nodeId,
|
|
70511
71224
|
reasons: rec.reasons
|
|
70512
71225
|
}
|
|
70513
71226
|
});
|
|
70514
|
-
this.deps.logger.info("
|
|
71227
|
+
this.deps.logger.info("auto tune: applied", { meta: {
|
|
70515
71228
|
planHash: plan.planHash,
|
|
70516
71229
|
nodeChanges: plan.nodeChanges.length,
|
|
70517
71230
|
cameraClears: plan.cameraClears.length,
|
|
@@ -70526,11 +71239,12 @@ var BestSettingsService = class {
|
|
|
70526
71239
|
}
|
|
70527
71240
|
};
|
|
70528
71241
|
//#endregion
|
|
70529
|
-
//#region src/
|
|
71242
|
+
//#region src/auto-tune/device-peak-tracker.ts
|
|
70530
71243
|
var BUCKET_MS = 3600 * 1e3;
|
|
70531
71244
|
var DevicePeakTracker = class DevicePeakTracker {
|
|
70532
71245
|
now;
|
|
70533
71246
|
series = /* @__PURE__ */ new Map();
|
|
71247
|
+
cluster = null;
|
|
70534
71248
|
constructor(now = () => Date.now()) {
|
|
70535
71249
|
this.now = now;
|
|
70536
71250
|
}
|
|
@@ -70541,12 +71255,14 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70541
71255
|
noteCensus(loads) {
|
|
70542
71256
|
const now = this.now();
|
|
70543
71257
|
const bucket = now - now % BUCKET_MS;
|
|
71258
|
+
let clusterTotal = 0;
|
|
70544
71259
|
for (const load of loads) {
|
|
70545
71260
|
const rows = load.devices.length > 0 ? load.devices : [{
|
|
70546
71261
|
deviceKey: "",
|
|
70547
71262
|
attachedCameras: load.attachedCameras
|
|
70548
71263
|
}];
|
|
70549
71264
|
for (const device of rows) {
|
|
71265
|
+
clusterTotal += device.attachedCameras;
|
|
70550
71266
|
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
70551
71267
|
let s = this.series.get(key);
|
|
70552
71268
|
if (s === void 0) {
|
|
@@ -70560,6 +71276,26 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70560
71276
|
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
70561
71277
|
}
|
|
70562
71278
|
}
|
|
71279
|
+
if (loads.length === 0) return;
|
|
71280
|
+
const c = this.cluster ?? {
|
|
71281
|
+
firstSeenAt: now,
|
|
71282
|
+
buckets: /* @__PURE__ */ new Map()
|
|
71283
|
+
};
|
|
71284
|
+
this.cluster = c;
|
|
71285
|
+
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71286
|
+
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71287
|
+
}
|
|
71288
|
+
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71289
|
+
clusterPeak() {
|
|
71290
|
+
const c = this.cluster;
|
|
71291
|
+
if (c === null) return null;
|
|
71292
|
+
const now = this.now();
|
|
71293
|
+
let peak = 0;
|
|
71294
|
+
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71295
|
+
return {
|
|
71296
|
+
peakSessions: peak,
|
|
71297
|
+
observedForMs: now - c.firstSeenAt
|
|
71298
|
+
};
|
|
70563
71299
|
}
|
|
70564
71300
|
/** Every observed device's peak over the window and how long it has been watched. */
|
|
70565
71301
|
peaks() {
|
|
@@ -70580,6 +71316,356 @@ var DevicePeakTracker = class DevicePeakTracker {
|
|
|
70580
71316
|
}
|
|
70581
71317
|
};
|
|
70582
71318
|
//#endregion
|
|
71319
|
+
//#region src/session-placement-stats.ts
|
|
71320
|
+
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71321
|
+
function keyOf(nodeId, deviceKey) {
|
|
71322
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
71323
|
+
}
|
|
71324
|
+
function split(key) {
|
|
71325
|
+
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71326
|
+
return {
|
|
71327
|
+
nodeId,
|
|
71328
|
+
deviceKey
|
|
71329
|
+
};
|
|
71330
|
+
}
|
|
71331
|
+
var SessionPlacementStats = class {
|
|
71332
|
+
now;
|
|
71333
|
+
segments = [];
|
|
71334
|
+
starts = [];
|
|
71335
|
+
yields = [];
|
|
71336
|
+
startedAt;
|
|
71337
|
+
constructor(now = () => Date.now()) {
|
|
71338
|
+
this.now = now;
|
|
71339
|
+
this.startedAt = now();
|
|
71340
|
+
}
|
|
71341
|
+
/** The running-session set changed: open a new segment with these counts. */
|
|
71342
|
+
noteSessions(sessions) {
|
|
71343
|
+
const counts = /* @__PURE__ */ new Map();
|
|
71344
|
+
for (const s of sessions) {
|
|
71345
|
+
const k = keyOf(s.nodeId, s.deviceKey);
|
|
71346
|
+
counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
71347
|
+
}
|
|
71348
|
+
this.segments.push({
|
|
71349
|
+
at: this.now(),
|
|
71350
|
+
counts
|
|
71351
|
+
});
|
|
71352
|
+
this.prune();
|
|
71353
|
+
}
|
|
71354
|
+
noteStart(nodeId, deviceKey) {
|
|
71355
|
+
this.starts.push({
|
|
71356
|
+
at: this.now(),
|
|
71357
|
+
nodeId,
|
|
71358
|
+
deviceKey
|
|
71359
|
+
});
|
|
71360
|
+
this.prune();
|
|
71361
|
+
}
|
|
71362
|
+
/** A share election handed `nodeId`'s turn to another node. */
|
|
71363
|
+
noteYield(nodeId) {
|
|
71364
|
+
this.yields.push({
|
|
71365
|
+
at: this.now(),
|
|
71366
|
+
nodeId
|
|
71367
|
+
});
|
|
71368
|
+
this.prune();
|
|
71369
|
+
}
|
|
71370
|
+
/** For tests: how many segments are retained. */
|
|
71371
|
+
segmentCount() {
|
|
71372
|
+
return this.segments.length;
|
|
71373
|
+
}
|
|
71374
|
+
prune() {
|
|
71375
|
+
const floor = this.now() - RETAIN_MS;
|
|
71376
|
+
while (this.segments.length > 1 && this.segments[1].at <= floor) this.segments.shift();
|
|
71377
|
+
while (this.starts.length > 0 && this.starts[0].at < floor) this.starts.shift();
|
|
71378
|
+
while (this.yields.length > 0 && this.yields[0].at < floor) this.yields.shift();
|
|
71379
|
+
}
|
|
71380
|
+
snapshot(windowMs) {
|
|
71381
|
+
const now = this.now();
|
|
71382
|
+
const from = now - windowMs;
|
|
71383
|
+
const integral = /* @__PURE__ */ new Map();
|
|
71384
|
+
let clusterPeak = 0;
|
|
71385
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71386
|
+
const seg = this.segments[i];
|
|
71387
|
+
const end = this.segments[i + 1]?.at ?? now;
|
|
71388
|
+
const start = Math.max(seg.at, from);
|
|
71389
|
+
if (end <= start) continue;
|
|
71390
|
+
let total = 0;
|
|
71391
|
+
for (const [k, n] of seg.counts) {
|
|
71392
|
+
integral.set(k, (integral.get(k) ?? 0) + n * (end - start));
|
|
71393
|
+
total += n;
|
|
71394
|
+
}
|
|
71395
|
+
clusterPeak = Math.max(clusterPeak, total);
|
|
71396
|
+
}
|
|
71397
|
+
const current = this.segments.at(-1)?.counts ?? /* @__PURE__ */ new Map();
|
|
71398
|
+
const nodeIds = /* @__PURE__ */ new Set();
|
|
71399
|
+
for (const k of [...integral.keys(), ...current.keys()]) nodeIds.add(split(k).nodeId);
|
|
71400
|
+
for (const s of this.starts) if (s.at >= from) nodeIds.add(s.nodeId);
|
|
71401
|
+
const clusterIntegral = [...integral.values()].reduce((a, b) => a + b, 0);
|
|
71402
|
+
const nodes = [...nodeIds].toSorted().map((nodeId) => {
|
|
71403
|
+
const deviceKeys = /* @__PURE__ */ new Set();
|
|
71404
|
+
for (const k of [...integral.keys(), ...current.keys()]) {
|
|
71405
|
+
const s = split(k);
|
|
71406
|
+
if (s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71407
|
+
}
|
|
71408
|
+
for (const s of this.starts) if (s.at >= from && s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71409
|
+
const nodeIntegral = [...deviceKeys].reduce((sum, d) => sum + (integral.get(keyOf(nodeId, d)) ?? 0), 0);
|
|
71410
|
+
const devices = [...deviceKeys].toSorted().map((deviceKey) => ({
|
|
71411
|
+
deviceKey,
|
|
71412
|
+
sessionsNow: current.get(keyOf(nodeId, deviceKey)) ?? 0,
|
|
71413
|
+
sessionsStarted: this.starts.filter((s) => s.at >= from && s.nodeId === nodeId && s.deviceKey === deviceKey).length,
|
|
71414
|
+
observedShare: nodeIntegral > 0 ? (integral.get(keyOf(nodeId, deviceKey)) ?? 0) / nodeIntegral : null
|
|
71415
|
+
}));
|
|
71416
|
+
let peak = 0;
|
|
71417
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71418
|
+
const seg = this.segments[i];
|
|
71419
|
+
if ((this.segments[i + 1]?.at ?? now) <= Math.max(seg.at, from)) continue;
|
|
71420
|
+
let n = 0;
|
|
71421
|
+
for (const [k, c] of seg.counts) if (split(k).nodeId === nodeId) n += c;
|
|
71422
|
+
peak = Math.max(peak, n);
|
|
71423
|
+
}
|
|
71424
|
+
return {
|
|
71425
|
+
nodeId,
|
|
71426
|
+
sessionsNow: devices.reduce((s, d) => s + d.sessionsNow, 0),
|
|
71427
|
+
sessionsStarted: devices.reduce((s, d) => s + d.sessionsStarted, 0),
|
|
71428
|
+
observedShare: clusterIntegral > 0 && nodeIntegral > 0 ? nodeIntegral / clusterIntegral : null,
|
|
71429
|
+
peakSessions: peak,
|
|
71430
|
+
yieldsAway: this.yields.filter((y) => y.at >= from && y.nodeId === nodeId).length,
|
|
71431
|
+
devices
|
|
71432
|
+
};
|
|
71433
|
+
});
|
|
71434
|
+
return {
|
|
71435
|
+
observedForMs: Math.min(windowMs, now - this.startedAt),
|
|
71436
|
+
sessionsNow: [...current.values()].reduce((a, b) => a + b, 0),
|
|
71437
|
+
clusterPeakSessions: clusterPeak,
|
|
71438
|
+
nodes
|
|
71439
|
+
};
|
|
71440
|
+
}
|
|
71441
|
+
};
|
|
71442
|
+
//#endregion
|
|
71443
|
+
//#region src/load-balancing-shares.ts
|
|
71444
|
+
function nextDevices(stored, shares, effective) {
|
|
71445
|
+
let changed = false;
|
|
71446
|
+
const out = { ...stored };
|
|
71447
|
+
for (const [key, weight] of Object.entries(shares)) {
|
|
71448
|
+
const existing = stored?.[key];
|
|
71449
|
+
if (existing !== void 0) {
|
|
71450
|
+
out[key] = {
|
|
71451
|
+
...existing,
|
|
71452
|
+
weight
|
|
71453
|
+
};
|
|
71454
|
+
changed = true;
|
|
71455
|
+
continue;
|
|
71456
|
+
}
|
|
71457
|
+
const enabled = effective[key];
|
|
71458
|
+
if (enabled === void 0) continue;
|
|
71459
|
+
out[key] = {
|
|
71460
|
+
enabled,
|
|
71461
|
+
weight
|
|
71462
|
+
};
|
|
71463
|
+
changed = true;
|
|
71464
|
+
}
|
|
71465
|
+
return changed ? out : stored;
|
|
71466
|
+
}
|
|
71467
|
+
function applyLoadBalancingShares(prev, write) {
|
|
71468
|
+
const next = { ...prev };
|
|
71469
|
+
const nodeIds = new Set([...Object.keys(write.nodes), ...Object.keys(write.devices)]);
|
|
71470
|
+
for (const nodeId of nodeIds) {
|
|
71471
|
+
const existing = prev[nodeId];
|
|
71472
|
+
const nodeShare = write.nodes[nodeId];
|
|
71473
|
+
const devices = nextDevices(existing?.inferenceDevices, write.devices[nodeId] ?? {}, write.effectiveEnabled[nodeId] ?? {});
|
|
71474
|
+
if (nodeShare === void 0 && devices === existing?.inferenceDevices) continue;
|
|
71475
|
+
next[nodeId] = {
|
|
71476
|
+
...existing ?? { maxCameras: null },
|
|
71477
|
+
...nodeShare !== void 0 ? { detectWeight: nodeShare } : {},
|
|
71478
|
+
...devices !== void 0 ? { inferenceDevices: devices } : {}
|
|
71479
|
+
};
|
|
71480
|
+
}
|
|
71481
|
+
return next;
|
|
71482
|
+
}
|
|
71483
|
+
//#endregion
|
|
71484
|
+
//#region src/load-balancing/effective-shares-service.ts
|
|
71485
|
+
function weightsFrom(result) {
|
|
71486
|
+
const nodes = {};
|
|
71487
|
+
const devices = {};
|
|
71488
|
+
for (const node of result.nodes) {
|
|
71489
|
+
if (node.share !== null && node.share > 0) nodes[node.nodeId] = node.share;
|
|
71490
|
+
const perDevice = {};
|
|
71491
|
+
for (const d of node.devices) if (d.share > 0) perDevice[d.deviceKey] = d.share;
|
|
71492
|
+
if (Object.keys(perDevice).length > 0) devices[node.nodeId] = perDevice;
|
|
71493
|
+
}
|
|
71494
|
+
return {
|
|
71495
|
+
nodes,
|
|
71496
|
+
devices
|
|
71497
|
+
};
|
|
71498
|
+
}
|
|
71499
|
+
var EffectiveSharesService = class EffectiveSharesService {
|
|
71500
|
+
deps;
|
|
71501
|
+
/** An automatic answer older than this is recomputed by the reconcile. */
|
|
71502
|
+
static MAX_AGE_MS = 15 * 6e4;
|
|
71503
|
+
/** Reconcile cadence. */
|
|
71504
|
+
static RECONCILE_MS = 6e4;
|
|
71505
|
+
answer = null;
|
|
71506
|
+
inFlight = null;
|
|
71507
|
+
pendingTrigger = null;
|
|
71508
|
+
timer = null;
|
|
71509
|
+
constructor(deps) {
|
|
71510
|
+
this.deps = deps;
|
|
71511
|
+
}
|
|
71512
|
+
now() {
|
|
71513
|
+
return this.deps.now?.() ?? Date.now();
|
|
71514
|
+
}
|
|
71515
|
+
/** The last automatic answer; null = never computed. */
|
|
71516
|
+
current() {
|
|
71517
|
+
return this.answer;
|
|
71518
|
+
}
|
|
71519
|
+
/** Node weights the election reads. */
|
|
71520
|
+
async nodeWeights() {
|
|
71521
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return { ...this.answer.weights.nodes };
|
|
71522
|
+
return { ...(await this.deps.readStored()).nodes };
|
|
71523
|
+
}
|
|
71524
|
+
/** The weights of a node's ELIGIBLE devices (`eligible` carries their stored weights). */
|
|
71525
|
+
deviceWeights(nodeId, eligible) {
|
|
71526
|
+
if (this.deps.readSource() !== "auto" || this.answer === null) return { ...eligible };
|
|
71527
|
+
const auto = this.answer.weights.devices[nodeId] ?? {};
|
|
71528
|
+
const out = {};
|
|
71529
|
+
for (const [key, stored] of Object.entries(eligible)) {
|
|
71530
|
+
const share = auto[key];
|
|
71531
|
+
out[key] = share !== void 0 && share > 0 ? share : stored;
|
|
71532
|
+
}
|
|
71533
|
+
return out;
|
|
71534
|
+
}
|
|
71535
|
+
/** Both maps, as the Auto tune recommender charges load by them. */
|
|
71536
|
+
async effectiveWeights() {
|
|
71537
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return this.answer.weights;
|
|
71538
|
+
return this.deps.readStored();
|
|
71539
|
+
}
|
|
71540
|
+
/**
|
|
71541
|
+
* Recompute now (auto only). Concurrent calls coalesce: one runs, and one
|
|
71542
|
+
* trailing run picks up whatever was reported meanwhile.
|
|
71543
|
+
*/
|
|
71544
|
+
async recompute(trigger) {
|
|
71545
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71546
|
+
if (this.inFlight !== null) {
|
|
71547
|
+
this.pendingTrigger = trigger;
|
|
71548
|
+
return this.inFlight;
|
|
71549
|
+
}
|
|
71550
|
+
this.inFlight = this.run(trigger).finally(() => {
|
|
71551
|
+
this.inFlight = null;
|
|
71552
|
+
});
|
|
71553
|
+
await this.inFlight;
|
|
71554
|
+
const trailing = this.pendingTrigger;
|
|
71555
|
+
if (trailing !== null) {
|
|
71556
|
+
this.pendingTrigger = null;
|
|
71557
|
+
await this.recompute(trailing);
|
|
71558
|
+
}
|
|
71559
|
+
}
|
|
71560
|
+
async run(trigger) {
|
|
71561
|
+
let signature;
|
|
71562
|
+
let result;
|
|
71563
|
+
try {
|
|
71564
|
+
signature = await this.deps.topologySignature();
|
|
71565
|
+
result = await this.deps.suggest();
|
|
71566
|
+
} catch (err) {
|
|
71567
|
+
this.deps.logger.warn("load balancing: automatic shares not recomputed — last answer kept", { meta: {
|
|
71568
|
+
trigger,
|
|
71569
|
+
error: errMsg$2(err),
|
|
71570
|
+
hasAnswer: this.answer !== null
|
|
71571
|
+
} });
|
|
71572
|
+
return;
|
|
71573
|
+
}
|
|
71574
|
+
const before = this.answer?.weights ?? null;
|
|
71575
|
+
const after = weightsFrom(result);
|
|
71576
|
+
this.answer = {
|
|
71577
|
+
weights: after,
|
|
71578
|
+
computedAt: this.now(),
|
|
71579
|
+
trigger,
|
|
71580
|
+
signature
|
|
71581
|
+
};
|
|
71582
|
+
const nodeIds = [...new Set([...Object.keys(before?.nodes ?? {}), ...result.nodes.map((n) => n.nodeId)])];
|
|
71583
|
+
for (const nodeId of nodeIds.toSorted()) this.deps.logger.info("load balancing: automatic shares recomputed", {
|
|
71584
|
+
tags: { nodeId },
|
|
71585
|
+
meta: {
|
|
71586
|
+
trigger,
|
|
71587
|
+
before: before?.nodes[nodeId] ?? null,
|
|
71588
|
+
after: after.nodes[nodeId] ?? null,
|
|
71589
|
+
devicesBefore: JSON.stringify(before?.devices[nodeId] ?? {}),
|
|
71590
|
+
devicesAfter: JSON.stringify(after.devices[nodeId] ?? {})
|
|
71591
|
+
}
|
|
71592
|
+
});
|
|
71593
|
+
}
|
|
71594
|
+
/** Recompute if the topology changed or the answer has grown old (auto only). */
|
|
71595
|
+
async reconcile() {
|
|
71596
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71597
|
+
if (this.answer === null) return this.recompute("reconcile:first");
|
|
71598
|
+
if (this.now() - this.answer.computedAt > EffectiveSharesService.MAX_AGE_MS) return this.recompute("reconcile:age");
|
|
71599
|
+
let signature;
|
|
71600
|
+
try {
|
|
71601
|
+
signature = await this.deps.topologySignature();
|
|
71602
|
+
} catch (err) {
|
|
71603
|
+
this.deps.logger.debug("load balancing: topology signature unreadable — reconcile skipped", { meta: { error: errMsg$2(err) } });
|
|
71604
|
+
return;
|
|
71605
|
+
}
|
|
71606
|
+
if (signature !== this.answer.signature) return this.recompute("reconcile:topology");
|
|
71607
|
+
}
|
|
71608
|
+
start() {
|
|
71609
|
+
if (this.timer !== null) return;
|
|
71610
|
+
this.timer = setInterval(() => {
|
|
71611
|
+
this.reconcile();
|
|
71612
|
+
}, EffectiveSharesService.RECONCILE_MS);
|
|
71613
|
+
this.timer.unref?.();
|
|
71614
|
+
}
|
|
71615
|
+
stop() {
|
|
71616
|
+
if (this.timer !== null) clearInterval(this.timer);
|
|
71617
|
+
this.timer = null;
|
|
71618
|
+
}
|
|
71619
|
+
};
|
|
71620
|
+
//#endregion
|
|
71621
|
+
//#region src/load-balancing/share-inputs.ts
|
|
71622
|
+
/** The operator's sliders: node `detectWeight` and device `weight` (positive only). */
|
|
71623
|
+
function storedShareWeights(blob) {
|
|
71624
|
+
const nodes = {};
|
|
71625
|
+
const devices = {};
|
|
71626
|
+
for (const [nodeId, settings] of Object.entries(blob)) {
|
|
71627
|
+
if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) nodes[nodeId] = settings.detectWeight;
|
|
71628
|
+
const perDevice = {};
|
|
71629
|
+
for (const [key, entry] of Object.entries(settings.inferenceDevices ?? {})) if (typeof entry.weight === "number" && entry.weight > 0) perDevice[key] = entry.weight;
|
|
71630
|
+
if (Object.keys(perDevice).length > 0) devices[nodeId] = perDevice;
|
|
71631
|
+
}
|
|
71632
|
+
return {
|
|
71633
|
+
nodes,
|
|
71634
|
+
devices
|
|
71635
|
+
};
|
|
71636
|
+
}
|
|
71637
|
+
/**
|
|
71638
|
+
* A string that changes whenever an input of the automatic shares does: the
|
|
71639
|
+
* nodes present, each node's detect capability, which devices are switched on
|
|
71640
|
+
* and which detector model each runs. Sliders and caps are left out — they do
|
|
71641
|
+
* not change capacity.
|
|
71642
|
+
*/
|
|
71643
|
+
function shareTopologySignature(nodeIds, blob) {
|
|
71644
|
+
return [...nodeIds].toSorted().map((nodeId) => {
|
|
71645
|
+
const settings = blob[nodeId];
|
|
71646
|
+
const devices = Object.entries(settings?.inferenceDevices ?? {}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => {
|
|
71647
|
+
const model = entry.steps?.["object-detection"]?.modelId ?? "";
|
|
71648
|
+
return `${key}=${entry.enabled ? "on" : "off"}:${model}`;
|
|
71649
|
+
});
|
|
71650
|
+
return `${nodeId}|detect=${String(settings?.detect ?? "default")}|${devices.join(",")}`;
|
|
71651
|
+
}).join(";");
|
|
71652
|
+
}
|
|
71653
|
+
//#endregion
|
|
71654
|
+
//#region src/stale-device-pins.ts
|
|
71655
|
+
function findStaleDevicePins(pins, nodes) {
|
|
71656
|
+
const usable = /* @__PURE__ */ new Map();
|
|
71657
|
+
for (const node of nodes) for (const d of node.devices) usable.set(d.key, (usable.get(d.key) ?? false) || d.enabled && d.available);
|
|
71658
|
+
return pins.flatMap((pin) => {
|
|
71659
|
+
const state = usable.get(pin.deviceKey);
|
|
71660
|
+
if (state === true) return [];
|
|
71661
|
+
return [{
|
|
71662
|
+
deviceId: pin.deviceId,
|
|
71663
|
+
deviceKey: pin.deviceKey,
|
|
71664
|
+
reason: state === false ? "disabled" : "absent"
|
|
71665
|
+
}];
|
|
71666
|
+
}).toSorted((a, b) => a.deviceId - b.deviceId);
|
|
71667
|
+
}
|
|
71668
|
+
//#endregion
|
|
70583
71669
|
//#region src/index.ts
|
|
70584
71670
|
/**
|
|
70585
71671
|
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
@@ -70649,7 +71735,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70649
71735
|
clusterModelChoiceReady = false;
|
|
70650
71736
|
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
70651
71737
|
devicePeaks = new DevicePeakTracker();
|
|
70652
|
-
|
|
71738
|
+
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71739
|
+
placementStats = new SessionPlacementStats();
|
|
71740
|
+
/** The stored session placement policy (D748, default `share` since D750). */
|
|
71741
|
+
readSessionPlacement = () => DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
71742
|
+
/** Where the shares come from (D750); `auto` before init. */
|
|
71743
|
+
readShareSource = () => DEFAULT_SHARE_SOURCE;
|
|
71744
|
+
autoTuneService = null;
|
|
71745
|
+
effectiveSharesService = null;
|
|
70653
71746
|
/**
|
|
70654
71747
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
70655
71748
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -70898,6 +71991,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70898
71991
|
getEligibleInferenceDevices: (nodeId, read) => this.resolveEligibleInferenceDevices(nodeId, read),
|
|
70899
71992
|
getEligibleInferenceDevicesIfCached: (nodeId) => this.resolveEligibleInferenceDevicesIfCached(nodeId),
|
|
70900
71993
|
forgetInferenceDeviceProbe: (nodeId) => this.forgetInferenceDeviceProbe(nodeId),
|
|
71994
|
+
noteShareTopologyChange: (trigger) => this.noteShareChange(trigger),
|
|
70901
71995
|
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
70902
71996
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
70903
71997
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
@@ -70927,6 +72021,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70927
72021
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
70928
72022
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
70929
72023
|
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
72024
|
+
placementStats: this.placementStats,
|
|
70930
72025
|
setTopology: (topology) => {
|
|
70931
72026
|
this.topology = topology;
|
|
70932
72027
|
},
|
|
@@ -70953,6 +72048,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70953
72048
|
this.nodeStress = controllers.nodeStress;
|
|
70954
72049
|
this.placement = controllers.placement;
|
|
70955
72050
|
this.session = controllers.session;
|
|
72051
|
+
this.readSessionPlacement = controllers.sessionPlacement;
|
|
72052
|
+
this.readShareSource = controllers.shareSource;
|
|
72053
|
+
this.settingsStore.setEffectiveNodeWeights(() => this.effectiveShares().nodeWeights());
|
|
72054
|
+
this.effectiveShares().start();
|
|
72055
|
+
this.noteShareChange("startup");
|
|
70956
72056
|
this.deviceConfig = controllers.deviceConfig;
|
|
70957
72057
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
70958
72058
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71078,6 +72178,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71078
72178
|
this.detectionWiring = null;
|
|
71079
72179
|
this.unsubAgentOffline?.();
|
|
71080
72180
|
this.unsubAgentOffline = null;
|
|
72181
|
+
this.effectiveSharesService?.stop();
|
|
71081
72182
|
this.unsubAgentOnline?.();
|
|
71082
72183
|
this.unsubAgentOnline = null;
|
|
71083
72184
|
this.nodeLifecycle?.dispose();
|
|
@@ -71445,26 +72546,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71445
72546
|
this.reconcile.schedulePendingRetry();
|
|
71446
72547
|
return { success: true };
|
|
71447
72548
|
}
|
|
71448
|
-
async setAgentDetectWeight(input) {
|
|
71449
|
-
const existing = (await this.settingsStore.readAgentSettingsMap())[input.agentNodeId];
|
|
71450
|
-
const weight = input.detectWeight ?? 1;
|
|
71451
|
-
const next = existing ? {
|
|
71452
|
-
...existing,
|
|
71453
|
-
detectWeight: weight
|
|
71454
|
-
} : {
|
|
71455
|
-
maxCameras: null,
|
|
71456
|
-
detectWeight: weight
|
|
71457
|
-
};
|
|
71458
|
-
await this.settingsStore.writeAgentSettings(input.agentNodeId, next);
|
|
71459
|
-
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
71460
|
-
tags: { nodeId: input.agentNodeId },
|
|
71461
|
-
meta: { detectWeight: weight ?? null }
|
|
71462
|
-
});
|
|
71463
|
-
this.rebalance().catch((err) => {
|
|
71464
|
-
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
71465
|
-
});
|
|
71466
|
-
return { success: true };
|
|
71467
|
-
}
|
|
71468
72549
|
/**
|
|
71469
72550
|
* Set one node's placement capabilities — the per-node record that replaced
|
|
71470
72551
|
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
@@ -71500,6 +72581,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71500
72581
|
await this.topology.refreshNodeCapabilities();
|
|
71501
72582
|
this.topology.addKnownRunnerNode(input.agentNodeId);
|
|
71502
72583
|
this.reconcile.schedulePendingRetry();
|
|
72584
|
+
if (input.detect !== void 0) this.noteShareChange(`node-detect:${input.agentNodeId}`);
|
|
71503
72585
|
return { success: true };
|
|
71504
72586
|
}
|
|
71505
72587
|
/**
|
|
@@ -71589,6 +72671,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71589
72671
|
});
|
|
71590
72672
|
await this.resetRemovedModelPins("node-device-write");
|
|
71591
72673
|
this.reconcile.schedulePendingRetry();
|
|
72674
|
+
this.noteShareChange(`node-devices:${input.agentNodeId}`);
|
|
71592
72675
|
return { success: true };
|
|
71593
72676
|
}
|
|
71594
72677
|
/**
|
|
@@ -71669,7 +72752,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71669
72752
|
});
|
|
71670
72753
|
throw new Error(`inference device eligibility: node "${nodeId}" catalog not answered`);
|
|
71671
72754
|
}
|
|
71672
|
-
return this.electableDevices(nodeId, probed, catalog);
|
|
72755
|
+
return this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, probed, catalog));
|
|
71673
72756
|
}
|
|
71674
72757
|
/**
|
|
71675
72758
|
* Forget a node's last device probe (pre-ship review, M4). Called when the
|
|
@@ -71713,7 +72796,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71713
72796
|
this.refreshInferenceDeviceProbe(nodeId);
|
|
71714
72797
|
return {
|
|
71715
72798
|
kind: "eligible",
|
|
71716
|
-
eligible: await this.electableDevices(nodeId, known, catalog)
|
|
72799
|
+
eligible: this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, known, catalog))
|
|
71717
72800
|
};
|
|
71718
72801
|
}
|
|
71719
72802
|
/**
|
|
@@ -72043,10 +73126,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72043
73126
|
clearedCameraOverrides
|
|
72044
73127
|
};
|
|
72045
73128
|
}
|
|
72046
|
-
|
|
72047
|
-
if (this.
|
|
73129
|
+
autoTune() {
|
|
73130
|
+
if (this.autoTuneService !== null) return this.autoTuneService;
|
|
72048
73131
|
const store = this.settingsStore;
|
|
72049
|
-
this.
|
|
73132
|
+
this.autoTuneService = new AutoTuneService({
|
|
72050
73133
|
logger: this.ctx.logger,
|
|
72051
73134
|
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72052
73135
|
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
@@ -72060,8 +73143,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72060
73143
|
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72061
73144
|
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72062
73145
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72063
|
-
cameraLoads: () => this.
|
|
73146
|
+
cameraLoads: () => this.autoTuneCameraLoads(),
|
|
72064
73147
|
peaks: () => this.devicePeaks.peaks(),
|
|
73148
|
+
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
73149
|
+
sessionPlacement: () => this.readSessionPlacement(),
|
|
73150
|
+
readEffectiveWeights: () => this.effectiveShares().effectiveWeights(),
|
|
72065
73151
|
detectorStore: async (input) => {
|
|
72066
73152
|
const api = this.ctx.api;
|
|
72067
73153
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72084,35 +73170,157 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72084
73170
|
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72085
73171
|
now: () => Date.now()
|
|
72086
73172
|
});
|
|
72087
|
-
return this.
|
|
73173
|
+
return this.autoTuneService;
|
|
72088
73174
|
}
|
|
72089
73175
|
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72090
|
-
async
|
|
73176
|
+
async autoTuneCameraLoads() {
|
|
72091
73177
|
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72092
|
-
const cameraMap = await this.settingsStore.readCameraSettingsMap();
|
|
72093
73178
|
const loads = [];
|
|
72094
73179
|
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
72095
|
-
const
|
|
73180
|
+
const nodePin = await this.readPipelinePin(deviceId);
|
|
72096
73181
|
const pinnedKey = await this.readPipelineDevicePin(deviceId);
|
|
73182
|
+
const standingNode = config.detectionMode === "always-on" ? standing.get(deviceId) ?? null : null;
|
|
72097
73183
|
loads.push({
|
|
72098
73184
|
deviceId,
|
|
72099
73185
|
mode: config.detectionMode,
|
|
72100
73186
|
fps: config.detectionFps,
|
|
72101
73187
|
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72102
|
-
standingNodeId:
|
|
72103
|
-
|
|
72104
|
-
|
|
73188
|
+
standingNodeId: standingNode,
|
|
73189
|
+
nodePinId: nodePin,
|
|
73190
|
+
devicePin: pinnedKey !== null ? {
|
|
73191
|
+
nodeId: nodePin ?? standingNode,
|
|
72105
73192
|
deviceKey: pinnedKey
|
|
72106
73193
|
} : null
|
|
72107
73194
|
});
|
|
72108
73195
|
}
|
|
72109
73196
|
return loads;
|
|
72110
73197
|
}
|
|
72111
|
-
async
|
|
72112
|
-
return this.
|
|
73198
|
+
async planAutoTune(input) {
|
|
73199
|
+
return this.autoTune().plan(input);
|
|
73200
|
+
}
|
|
73201
|
+
/**
|
|
73202
|
+
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
73203
|
+
* device with no stored entry is made explicit with its effective
|
|
73204
|
+
* enablement, read from the probe-aware device list. Never touches the
|
|
73205
|
+
* `sessionPlacement` policy.
|
|
73206
|
+
*/
|
|
73207
|
+
async setLoadBalancingShares(input) {
|
|
73208
|
+
const effectiveEnabled = {};
|
|
73209
|
+
for (const nodeId of Object.keys(input.devices)) {
|
|
73210
|
+
const read = await this.getNodeInferenceDevices({ nodeId });
|
|
73211
|
+
effectiveEnabled[nodeId] = Object.fromEntries(read.devices.map((d) => [d.key, d.enabled]));
|
|
73212
|
+
}
|
|
73213
|
+
await this.settingsStore.rewriteAgentSettingsMap((prev) => applyLoadBalancingShares(prev, {
|
|
73214
|
+
nodes: input.nodes,
|
|
73215
|
+
devices: input.devices,
|
|
73216
|
+
effectiveEnabled
|
|
73217
|
+
}));
|
|
73218
|
+
this.ctx.logger.info("load balancing shares updated", { meta: {
|
|
73219
|
+
nodes: JSON.stringify(input.nodes),
|
|
73220
|
+
devices: JSON.stringify(input.devices),
|
|
73221
|
+
policy: this.readSessionPlacement()
|
|
73222
|
+
} });
|
|
73223
|
+
if (Object.keys(input.nodes).length > 0) this.rebalance().catch((err) => {
|
|
73224
|
+
this.ctx.logger.warn("setLoadBalancingShares: rebalance failed", { meta: { error: errMsg$2(err) } });
|
|
73225
|
+
});
|
|
73226
|
+
return { success: true };
|
|
73227
|
+
}
|
|
73228
|
+
async getPlacementStats(input) {
|
|
73229
|
+
const snap = this.placementStats.snapshot(input.windowMs);
|
|
73230
|
+
return {
|
|
73231
|
+
policy: this.readSessionPlacement(),
|
|
73232
|
+
windowMs: input.windowMs,
|
|
73233
|
+
observedForMs: snap.observedForMs,
|
|
73234
|
+
sessionsNow: snap.sessionsNow,
|
|
73235
|
+
clusterPeakSessions: snap.clusterPeakSessions,
|
|
73236
|
+
nodes: snap.nodes,
|
|
73237
|
+
stalePins: await this.listStaleDevicePins()
|
|
73238
|
+
};
|
|
72113
73239
|
}
|
|
72114
|
-
|
|
72115
|
-
|
|
73240
|
+
/** Device pins the session path cannot honour — see `stale-device-pins.ts`. */
|
|
73241
|
+
async listStaleDevicePins() {
|
|
73242
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])];
|
|
73243
|
+
const offered = await Promise.all(nodeIds.map(async (nodeId) => ({ devices: (await this.getNodeInferenceDevices({ nodeId }).catch(() => null))?.devices ?? [] })));
|
|
73244
|
+
const pins = [];
|
|
73245
|
+
for (const deviceId of this.detectionWiring.activeDetections.keys()) {
|
|
73246
|
+
const deviceKey = await this.readPipelineDevicePin(deviceId);
|
|
73247
|
+
if (deviceKey !== null) pins.push({
|
|
73248
|
+
deviceId,
|
|
73249
|
+
deviceKey
|
|
73250
|
+
});
|
|
73251
|
+
}
|
|
73252
|
+
return findStaleDevicePins(pins, offered);
|
|
73253
|
+
}
|
|
73254
|
+
async suggestShares(input) {
|
|
73255
|
+
return this.autoTune().suggestShares(input);
|
|
73256
|
+
}
|
|
73257
|
+
async applyAutoTune(input) {
|
|
73258
|
+
const result = await this.autoTune().apply(input);
|
|
73259
|
+
if (result.kind === "applied") this.noteShareChange("auto-tune-applied");
|
|
73260
|
+
return result;
|
|
73261
|
+
}
|
|
73262
|
+
effectiveShares() {
|
|
73263
|
+
if (this.effectiveSharesService !== null) return this.effectiveSharesService;
|
|
73264
|
+
const store = this.settingsStore;
|
|
73265
|
+
this.effectiveSharesService = new EffectiveSharesService({
|
|
73266
|
+
readSource: () => this.readShareSource(),
|
|
73267
|
+
readStored: async () => storedShareWeights(await store.readAgentSettingsMap()),
|
|
73268
|
+
suggest: () => this.autoTune().suggestShares({ headroom: AUTO_SHARE_HEADROOM }),
|
|
73269
|
+
topologySignature: async () => shareTopologySignature([...new Set(["hub", ...this.topology.collectRunnerNodeIds()])], await store.readAgentSettingsMap()),
|
|
73270
|
+
logger: this.ctx.logger
|
|
73271
|
+
});
|
|
73272
|
+
return this.effectiveSharesService;
|
|
73273
|
+
}
|
|
73274
|
+
/** Something the automatic shares depend on changed: recompute now (best effort). */
|
|
73275
|
+
noteShareChange(trigger) {
|
|
73276
|
+
if (this.settingsStore === null) return;
|
|
73277
|
+
this.effectiveShares().recompute(trigger).catch((err) => {
|
|
73278
|
+
this.ctx.logger.warn("load balancing: share recompute failed", { meta: {
|
|
73279
|
+
trigger,
|
|
73280
|
+
error: errMsg$2(err)
|
|
73281
|
+
} });
|
|
73282
|
+
});
|
|
73283
|
+
}
|
|
73284
|
+
async getEffectiveShares() {
|
|
73285
|
+
const service = this.effectiveShares();
|
|
73286
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])].toSorted();
|
|
73287
|
+
const rows = await Promise.all(nodeIds.map(async (nodeId) => ({
|
|
73288
|
+
nodeId,
|
|
73289
|
+
read: await this.getNodeInferenceDevices({ nodeId }).catch(() => null)
|
|
73290
|
+
})));
|
|
73291
|
+
const detectable = new Set(this.topology.detectionEligibleNodes());
|
|
73292
|
+
const eligibleNodes = rows.filter((r) => r.read !== null && r.read.reachable && detectable.has(r.nodeId) && this.isNodeInferenceUsable(r.nodeId)).map((r) => r.nodeId);
|
|
73293
|
+
const eligibleDevices = {};
|
|
73294
|
+
for (const r of rows) {
|
|
73295
|
+
if (r.read === null) continue;
|
|
73296
|
+
eligibleDevices[r.nodeId] = r.read.devices.filter((d) => d.enabled && d.available && d.exclusion === null).map((d) => d.key);
|
|
73297
|
+
}
|
|
73298
|
+
const auto = service.current();
|
|
73299
|
+
return resolveEffectiveShares({
|
|
73300
|
+
source: this.readShareSource(),
|
|
73301
|
+
policy: this.readSessionPlacement(),
|
|
73302
|
+
nodeIds,
|
|
73303
|
+
stored: storedShareWeights(await this.settingsStore.readAgentSettingsMap()),
|
|
73304
|
+
auto: auto?.weights ?? null,
|
|
73305
|
+
eligibleNodes,
|
|
73306
|
+
eligibleDevices,
|
|
73307
|
+
computedAt: auto?.computedAt ?? null,
|
|
73308
|
+
trigger: auto?.trigger ?? null
|
|
73309
|
+
});
|
|
73310
|
+
}
|
|
73311
|
+
async setShareSource(input) {
|
|
73312
|
+
const previous = this.readShareSource();
|
|
73313
|
+
if (input.source === "manual" && previous === "auto") {
|
|
73314
|
+
const auto = this.effectiveShares().current();
|
|
73315
|
+
if (auto !== null) await this.setLoadBalancingShares(auto.weights);
|
|
73316
|
+
}
|
|
73317
|
+
await this.updateGlobalSettings({ [SHARE_SOURCE_SETTING_KEY]: input.source });
|
|
73318
|
+
this.ctx.logger.info("load balancing: share source changed", { meta: {
|
|
73319
|
+
from: previous,
|
|
73320
|
+
to: input.source
|
|
73321
|
+
} });
|
|
73322
|
+
if (input.source === "auto") this.noteShareChange("source:auto");
|
|
73323
|
+
return { success: true };
|
|
72116
73324
|
}
|
|
72117
73325
|
async getCameraSettings(input) {
|
|
72118
73326
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|