@camstack/addon-pipeline-orchestrator 1.2.291 → 1.2.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +21 -21
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CNnJE0gg.mjs} +3 -3
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CXr4GmWP.mjs +26 -0
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-Cu7Q-ifj.mjs +26 -0
- package/dist/{hostInit-BpNmAa0Q.mjs → hostInit-S3TibCek.mjs} +3 -3
- package/dist/index.js +1083 -608
- package/dist/index.mjs +1083 -608
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +0 -26
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs +0 -26
package/dist/index.mjs
CHANGED
|
@@ -5382,7 +5382,7 @@ var ZodIssueCode = {
|
|
|
5382
5382
|
var ZodFirstPartyTypeKind;
|
|
5383
5383
|
ZodFirstPartyTypeKind || (ZodFirstPartyTypeKind = {});
|
|
5384
5384
|
//#endregion
|
|
5385
|
-
//#region ../types/dist/sleep-
|
|
5385
|
+
//#region ../types/dist/sleep-DAJ9Er57.mjs
|
|
5386
5386
|
/**
|
|
5387
5387
|
* The audio chunk plane's byte format, and the ONE expansion from a coded
|
|
5388
5388
|
* window to float samples (D455).
|
|
@@ -24633,7 +24633,7 @@ var pipelineExecutorCapability = {
|
|
|
24633
24633
|
auth: "admin"
|
|
24634
24634
|
}),
|
|
24635
24635
|
/**
|
|
24636
|
-
*
|
|
24636
|
+
* Auto tune (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24637
24637
|
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24638
24638
|
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24639
24639
|
* running model's own defaults apply. Every other key of a camera's store
|
|
@@ -25005,111 +25005,17 @@ var pipelineExecutorCapability = {
|
|
|
25005
25005
|
}
|
|
25006
25006
|
};
|
|
25007
25007
|
/**
|
|
25008
|
-
*
|
|
25009
|
-
* policy, the atomic share write, the observed placement statistics, the
|
|
25010
|
-
* hardware share suggestion and the stale device pins.
|
|
25011
|
-
*/
|
|
25012
|
-
/**
|
|
25013
|
-
* How a new detection session picks its node and device.
|
|
25014
|
-
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25015
|
-
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25016
|
-
* weight only bites once a node already holds a session.
|
|
25017
|
-
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25018
|
-
* full or drowning node yields to a free one (D748).
|
|
25019
|
-
*/
|
|
25020
|
-
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25021
|
-
var DEFAULT_SESSION_PLACEMENT_POLICY = "least-loaded";
|
|
25022
|
-
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25023
|
-
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25024
|
-
/** Parse a stored value; anything unknown is the default (nothing changes on deploy). */
|
|
25025
|
-
function parseSessionPlacementPolicy(value) {
|
|
25026
|
-
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25027
|
-
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25028
|
-
}
|
|
25029
|
-
var LoadBalancingSharesInputSchema = object({
|
|
25030
|
-
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25031
|
-
nodes: record(string(), number().positive()).default({}),
|
|
25032
|
-
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25033
|
-
devices: record(string(), record(string(), number().positive())).default({})
|
|
25034
|
-
});
|
|
25035
|
-
var PlacementStatsInputSchema = object({
|
|
25036
|
-
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25037
|
-
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25038
|
-
var PlacementDeviceStatsSchema = object({
|
|
25039
|
-
deviceKey: string(),
|
|
25040
|
-
/** Sessions running on this device now. */
|
|
25041
|
-
sessionsNow: number(),
|
|
25042
|
-
/** Sessions started on this device inside the window. */
|
|
25043
|
-
sessionsStarted: number(),
|
|
25044
|
-
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25045
|
-
observedShare: number().nullable()
|
|
25046
|
-
});
|
|
25047
|
-
var PlacementNodeStatsSchema = object({
|
|
25048
|
-
nodeId: string(),
|
|
25049
|
-
sessionsNow: number(),
|
|
25050
|
-
sessionsStarted: number(),
|
|
25051
|
-
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25052
|
-
observedShare: number().nullable(),
|
|
25053
|
-
/** Most sessions this node ran at once inside the window. */
|
|
25054
|
-
peakSessions: number(),
|
|
25055
|
-
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25056
|
-
yieldsAway: number(),
|
|
25057
|
-
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25058
|
-
});
|
|
25059
|
-
/** A camera's device pin that the session path cannot honour. */
|
|
25060
|
-
var StaleDevicePinSchema = object({
|
|
25061
|
-
deviceId: number(),
|
|
25062
|
-
deviceKey: string(),
|
|
25063
|
-
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25064
|
-
reason: _enum(["disabled", "absent"])
|
|
25065
|
-
});
|
|
25066
|
-
var PlacementStatsSchema = object({
|
|
25067
|
-
policy: SessionPlacementPolicySchema,
|
|
25068
|
-
windowMs: number(),
|
|
25069
|
-
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25070
|
-
observedForMs: number(),
|
|
25071
|
-
sessionsNow: number(),
|
|
25072
|
-
/** Most parallel sessions across the cluster inside the window. */
|
|
25073
|
-
clusterPeakSessions: number(),
|
|
25074
|
-
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25075
|
-
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25076
|
-
});
|
|
25077
|
-
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25078
|
-
var SuggestedDeviceShareSchema = object({
|
|
25079
|
-
deviceKey: string(),
|
|
25080
|
-
/** Share of the node (0–100, rounded). */
|
|
25081
|
-
share: number(),
|
|
25082
|
-
modelId: string().nullable(),
|
|
25083
|
-
/** Inferences per second at that model, from the measured table. */
|
|
25084
|
-
fps: number().nullable(),
|
|
25085
|
-
measured: boolean(),
|
|
25086
|
-
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25087
|
-
suggestedMaxSessions: number().nullable()
|
|
25088
|
-
});
|
|
25089
|
-
var SuggestedNodeShareSchema = object({
|
|
25090
|
-
nodeId: string(),
|
|
25091
|
-
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25092
|
-
share: number().nullable(),
|
|
25093
|
-
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25094
|
-
capacitySessions: number(),
|
|
25095
|
-
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25096
|
-
skippedReason: string().nullable()
|
|
25097
|
-
});
|
|
25098
|
-
var SuggestSharesResultSchema = object({
|
|
25099
|
-
headroom: number(),
|
|
25100
|
-
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25101
|
-
cameraFps: number(),
|
|
25102
|
-
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25103
|
-
iterations: number(),
|
|
25104
|
-
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25105
|
-
});
|
|
25106
|
-
/**
|
|
25107
|
-
* The wire shapes of the Best settings plan and apply (D745), used by the
|
|
25008
|
+
* The wire shapes of the Auto tune plan and apply (D745), used by the
|
|
25108
25009
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25109
|
-
* (`recommend.ts
|
|
25010
|
+
* (`recommend.ts`) are the TS face of the same data; these
|
|
25110
25011
|
* schemas are what crosses the cap boundary.
|
|
25111
25012
|
*/
|
|
25112
25013
|
var ModelFormatSchema$1 = _enum(MODEL_FORMATS);
|
|
25014
|
+
/** The operator's range for it. */
|
|
25015
|
+
var MIN_SESSIONS_PER_DEVICE_RANGE = {
|
|
25016
|
+
min: 1,
|
|
25017
|
+
max: 8
|
|
25018
|
+
};
|
|
25113
25019
|
var HardwareClassIdSchema = _enum([
|
|
25114
25020
|
"intel-core-ultra",
|
|
25115
25021
|
"intel-n-series",
|
|
@@ -25127,8 +25033,9 @@ var AcceleratorKindSchema = _enum([
|
|
|
25127
25033
|
]);
|
|
25128
25034
|
var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
25129
25035
|
object({
|
|
25130
|
-
kind: literal("
|
|
25131
|
-
|
|
25036
|
+
kind: literal("capacity"),
|
|
25037
|
+
sessions: number(),
|
|
25038
|
+
minSessions: number()
|
|
25132
25039
|
}),
|
|
25133
25040
|
object({
|
|
25134
25041
|
kind: literal("tie-faster"),
|
|
@@ -25143,16 +25050,13 @@ var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
|
25143
25050
|
}),
|
|
25144
25051
|
object({
|
|
25145
25052
|
kind: literal("no-fit-lightest"),
|
|
25146
|
-
|
|
25053
|
+
sessions: number(),
|
|
25054
|
+
minSessions: number()
|
|
25147
25055
|
}),
|
|
25148
25056
|
object({
|
|
25149
25057
|
kind: literal("derived-latency"),
|
|
25150
25058
|
modelId: string()
|
|
25151
25059
|
}),
|
|
25152
|
-
object({
|
|
25153
|
-
kind: literal("estimated-load"),
|
|
25154
|
-
observedForMs: number().nullable()
|
|
25155
|
-
}),
|
|
25156
25060
|
object({ kind: literal("cpu-device") }),
|
|
25157
25061
|
object({ kind: literal("coral-untouched") }),
|
|
25158
25062
|
object({ kind: literal("device-disabled") }),
|
|
@@ -25168,28 +25072,13 @@ var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
|
25168
25072
|
format: ModelFormatSchema$1
|
|
25169
25073
|
})
|
|
25170
25074
|
]);
|
|
25171
|
-
var DeviceDemandSchema = object({
|
|
25172
|
-
nodeId: string(),
|
|
25173
|
-
deviceKey: string(),
|
|
25174
|
-
demandFps: number(),
|
|
25175
|
-
maxCameraFps: number(),
|
|
25176
|
-
alwaysOnCameras: number(),
|
|
25177
|
-
onMotionSessions: number(),
|
|
25178
|
-
onMotionSource: _enum([
|
|
25179
|
-
"observed-24h",
|
|
25180
|
-
"estimate",
|
|
25181
|
-
"none"
|
|
25182
|
-
]),
|
|
25183
|
-
observedForMs: number().nullable(),
|
|
25184
|
-
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25185
|
-
placement: SessionPlacementPolicySchema
|
|
25186
|
-
});
|
|
25187
25075
|
var DeviceModelChoiceSchema = object({
|
|
25188
25076
|
modelId: string(),
|
|
25189
25077
|
p50Ms: number(),
|
|
25190
25078
|
fps: number(),
|
|
25191
25079
|
measured: boolean(),
|
|
25192
|
-
|
|
25080
|
+
/** Parallel sessions at the camera rate with the headroom kept free: ⌊fps × (1 − headroom) / camera fps⌋. */
|
|
25081
|
+
capacitySessions: number(),
|
|
25193
25082
|
personRecall: number()
|
|
25194
25083
|
});
|
|
25195
25084
|
var DeviceRecommendationSchema = object({
|
|
@@ -25197,7 +25086,6 @@ var DeviceRecommendationSchema = object({
|
|
|
25197
25086
|
accelerator: AcceleratorKindSchema.nullable(),
|
|
25198
25087
|
enabled: boolean(),
|
|
25199
25088
|
available: boolean(),
|
|
25200
|
-
demand: DeviceDemandSchema.nullable(),
|
|
25201
25089
|
chosen: DeviceModelChoiceSchema.nullable(),
|
|
25202
25090
|
reasons: array(RecommendationReasonSchema).readonly()
|
|
25203
25091
|
});
|
|
@@ -25239,7 +25127,7 @@ var DetectorStoreClearSchema = object({
|
|
|
25239
25127
|
deviceId: number(),
|
|
25240
25128
|
keys: array(string()).readonly()
|
|
25241
25129
|
});
|
|
25242
|
-
var
|
|
25130
|
+
var AutoTuneWarningSchema = discriminatedUnion("kind", [
|
|
25243
25131
|
object({
|
|
25244
25132
|
kind: literal("detector-store-cluster-wide"),
|
|
25245
25133
|
skippedNodes: array(string()).readonly()
|
|
@@ -25254,36 +25142,37 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25254
25142
|
deviceKey: string()
|
|
25255
25143
|
})
|
|
25256
25144
|
]);
|
|
25257
|
-
var
|
|
25145
|
+
var AutoTuneOptionsSchema = object({
|
|
25258
25146
|
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25259
25147
|
headroom: number().min(.3).max(.5).default(.4),
|
|
25260
25148
|
/**
|
|
25261
|
-
*
|
|
25262
|
-
*
|
|
25263
|
-
*
|
|
25264
|
-
* is stored.
|
|
25149
|
+
* The least number of parallel sessions every device must sustain at the
|
|
25150
|
+
* camera rate with the headroom kept free (D755). The most accurate model
|
|
25151
|
+
* that reaches it wins.
|
|
25265
25152
|
*/
|
|
25266
|
-
|
|
25153
|
+
minSessionsPerDevice: number().int().min(MIN_SESSIONS_PER_DEVICE_RANGE.min).max(MIN_SESSIONS_PER_DEVICE_RANGE.max).default(2)
|
|
25267
25154
|
});
|
|
25268
|
-
var
|
|
25155
|
+
var AutoTunePlanSchema = object({
|
|
25269
25156
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25270
25157
|
planHash: string(),
|
|
25271
25158
|
family: literal("yolov9"),
|
|
25272
25159
|
headroom: number(),
|
|
25273
|
-
|
|
25274
|
-
|
|
25160
|
+
minSessionsPerDevice: number(),
|
|
25161
|
+
/** The detection rate a session is counted at (the fastest camera running the pipeline root). */
|
|
25162
|
+
cameraFps: number(),
|
|
25275
25163
|
generatedAt: number(),
|
|
25276
25164
|
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25277
25165
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25278
25166
|
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25279
25167
|
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25280
|
-
warnings: array(
|
|
25168
|
+
warnings: array(AutoTuneWarningSchema).readonly()
|
|
25281
25169
|
});
|
|
25282
|
-
var
|
|
25170
|
+
var AutoTuneApplyInputSchema = object({
|
|
25283
25171
|
planHash: string(),
|
|
25284
|
-
headroom: number().min(.3).max(.5).default(.4)
|
|
25172
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25173
|
+
minSessionsPerDevice: number().int().min(MIN_SESSIONS_PER_DEVICE_RANGE.min).max(MIN_SESSIONS_PER_DEVICE_RANGE.max).default(2)
|
|
25285
25174
|
});
|
|
25286
|
-
var
|
|
25175
|
+
var AutoTunePhaseResultSchema = object({
|
|
25287
25176
|
phase: _enum([
|
|
25288
25177
|
"download",
|
|
25289
25178
|
"camera-overrides",
|
|
@@ -25294,25 +25183,225 @@ var BestSettingsPhaseResultSchema = object({
|
|
|
25294
25183
|
ok: boolean(),
|
|
25295
25184
|
detail: string()
|
|
25296
25185
|
});
|
|
25297
|
-
var
|
|
25186
|
+
var AutoTuneApplyResultSchema = discriminatedUnion("kind", [
|
|
25298
25187
|
object({
|
|
25299
25188
|
kind: literal("applied"),
|
|
25300
25189
|
planHash: string(),
|
|
25301
|
-
phases: array(
|
|
25190
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25302
25191
|
}),
|
|
25303
25192
|
object({
|
|
25304
25193
|
kind: literal("aborted"),
|
|
25305
|
-
phases: array(
|
|
25194
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25306
25195
|
}),
|
|
25307
25196
|
object({
|
|
25308
25197
|
kind: literal("stale"),
|
|
25309
|
-
plan:
|
|
25198
|
+
plan: AutoTunePlanSchema
|
|
25310
25199
|
}),
|
|
25311
25200
|
object({
|
|
25312
25201
|
kind: literal("refused"),
|
|
25313
25202
|
reason: _enum(["apply-in-progress"])
|
|
25314
25203
|
})
|
|
25315
25204
|
]);
|
|
25205
|
+
/**
|
|
25206
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25207
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25208
|
+
* hardware share suggestion and the stale device pins.
|
|
25209
|
+
*/
|
|
25210
|
+
/**
|
|
25211
|
+
* How a new detection session picks its node and device.
|
|
25212
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25213
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25214
|
+
* weight only bites once a node already holds a session.
|
|
25215
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25216
|
+
* full or drowning node yields to a free one (D748).
|
|
25217
|
+
*/
|
|
25218
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25219
|
+
/**
|
|
25220
|
+
* `share` since D750: the percentages are held by default. `least-loaded` stays
|
|
25221
|
+
* selectable.
|
|
25222
|
+
*/
|
|
25223
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "share";
|
|
25224
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25225
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25226
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25227
|
+
function parseSessionPlacementPolicy(value) {
|
|
25228
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25229
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25230
|
+
}
|
|
25231
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25232
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25233
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25234
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25235
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25236
|
+
});
|
|
25237
|
+
var PlacementStatsInputSchema = object({
|
|
25238
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25239
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25240
|
+
var PlacementDeviceStatsSchema = object({
|
|
25241
|
+
deviceKey: string(),
|
|
25242
|
+
/** Sessions running on this device now. */
|
|
25243
|
+
sessionsNow: number(),
|
|
25244
|
+
/** Sessions started on this device inside the window. */
|
|
25245
|
+
sessionsStarted: number(),
|
|
25246
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25247
|
+
observedShare: number().nullable()
|
|
25248
|
+
});
|
|
25249
|
+
var PlacementNodeStatsSchema = object({
|
|
25250
|
+
nodeId: string(),
|
|
25251
|
+
sessionsNow: number(),
|
|
25252
|
+
sessionsStarted: number(),
|
|
25253
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25254
|
+
observedShare: number().nullable(),
|
|
25255
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25256
|
+
peakSessions: number(),
|
|
25257
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25258
|
+
yieldsAway: number(),
|
|
25259
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25260
|
+
});
|
|
25261
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25262
|
+
var StaleDevicePinSchema = object({
|
|
25263
|
+
deviceId: number(),
|
|
25264
|
+
deviceKey: string(),
|
|
25265
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25266
|
+
reason: _enum(["disabled", "absent"])
|
|
25267
|
+
});
|
|
25268
|
+
var PlacementStatsSchema = object({
|
|
25269
|
+
policy: SessionPlacementPolicySchema,
|
|
25270
|
+
windowMs: number(),
|
|
25271
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25272
|
+
observedForMs: number(),
|
|
25273
|
+
sessionsNow: number(),
|
|
25274
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25275
|
+
clusterPeakSessions: number(),
|
|
25276
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25277
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25278
|
+
});
|
|
25279
|
+
/**
|
|
25280
|
+
* Where an automatic share's capacity comes from (D755).
|
|
25281
|
+
* - `hardware` — the static capacity of each device at its Auto tune model:
|
|
25282
|
+
* the measured latency table for the node's hardware class.
|
|
25283
|
+
* - `benchmark` — the node's own measured capacity (node benchmark results).
|
|
25284
|
+
* A node with no results falls back to `hardware`, and says so.
|
|
25285
|
+
*/
|
|
25286
|
+
var CapacitySourceSchema = _enum(["hardware", "benchmark"]);
|
|
25287
|
+
/** Why a node under the `benchmark` source was sized from hardware instead. */
|
|
25288
|
+
var CapacityFallbackSchema = _enum(["no-benchmark-results", "partial-benchmark-results"]);
|
|
25289
|
+
var SuggestSharesInputSchema = object({
|
|
25290
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25291
|
+
/** Default `hardware`. */
|
|
25292
|
+
source: CapacitySourceSchema.default("hardware")
|
|
25293
|
+
});
|
|
25294
|
+
var SuggestedDeviceShareSchema = object({
|
|
25295
|
+
deviceKey: string(),
|
|
25296
|
+
/** Share of the node (0–100, rounded). */
|
|
25297
|
+
share: number(),
|
|
25298
|
+
modelId: string().nullable(),
|
|
25299
|
+
/** Inferences per second at that model, from the measured table; null when sized by a benchmark. */
|
|
25300
|
+
fps: number().nullable(),
|
|
25301
|
+
/** False for a table cell derived from another chip; true for a measured cell or a benchmark. */
|
|
25302
|
+
measured: boolean(),
|
|
25303
|
+
/**
|
|
25304
|
+
* Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋ from the table,
|
|
25305
|
+
* or the benchmark's measured max sessions. Never enforced.
|
|
25306
|
+
*/
|
|
25307
|
+
suggestedMaxSessions: number().nullable()
|
|
25308
|
+
});
|
|
25309
|
+
var SuggestedNodeShareSchema = object({
|
|
25310
|
+
nodeId: string(),
|
|
25311
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25312
|
+
share: number().nullable(),
|
|
25313
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25314
|
+
capacitySessions: number(),
|
|
25315
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25316
|
+
skippedReason: string().nullable(),
|
|
25317
|
+
/** What this node's capacity was read from; null when the node is skipped. */
|
|
25318
|
+
capacitySource: CapacitySourceSchema.nullable(),
|
|
25319
|
+
/** Set when the `benchmark` source was asked for and this node used `hardware`. */
|
|
25320
|
+
capacityFallback: CapacityFallbackSchema.nullable()
|
|
25321
|
+
});
|
|
25322
|
+
var SuggestSharesResultSchema = object({
|
|
25323
|
+
headroom: number(),
|
|
25324
|
+
/** The source asked for; each node says what it actually used. */
|
|
25325
|
+
source: CapacitySourceSchema,
|
|
25326
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25327
|
+
cameraFps: number(),
|
|
25328
|
+
/** The Auto tune capacity floor the models were picked at. */
|
|
25329
|
+
minSessionsPerDevice: number(),
|
|
25330
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25331
|
+
});
|
|
25332
|
+
/**
|
|
25333
|
+
* Where the node and device shares come from (D750, D755).
|
|
25334
|
+
* - `hardware` (default) — computed by the system from each node's static
|
|
25335
|
+
* capacity (its accelerators at their Auto tune models, from the measured
|
|
25336
|
+
* latency table), recomputed whenever a node or device joins, leaves, is
|
|
25337
|
+
* switched on or off, or changes model. No load input.
|
|
25338
|
+
* - `benchmark` — the same, from each node's measured capacity (node
|
|
25339
|
+
* benchmark results); a node without results falls back to `hardware`.
|
|
25340
|
+
* - `manual` — the operator's stored sliders.
|
|
25341
|
+
* Either way a node or device that cannot take sessions gives its share to
|
|
25342
|
+
* the others in proportion; the stored values are never rewritten for that.
|
|
25343
|
+
*/
|
|
25344
|
+
var ShareSourceSchema = _enum([
|
|
25345
|
+
"hardware",
|
|
25346
|
+
"benchmark",
|
|
25347
|
+
"manual"
|
|
25348
|
+
]);
|
|
25349
|
+
var DEFAULT_SHARE_SOURCE = "hardware";
|
|
25350
|
+
/** `hardware` or `benchmark`: the system computes the shares. */
|
|
25351
|
+
function isAutomaticShareSource(source) {
|
|
25352
|
+
return source !== "manual";
|
|
25353
|
+
}
|
|
25354
|
+
/** Headroom the automatic shares are computed at (Auto tune's default). */
|
|
25355
|
+
var AUTO_SHARE_HEADROOM = .4;
|
|
25356
|
+
/** The global-settings key the share source is stored under (orchestrator `load` section). */
|
|
25357
|
+
var SHARE_SOURCE_SETTING_KEY = "shareSource";
|
|
25358
|
+
/**
|
|
25359
|
+
* Parse a stored value; anything unknown is the default. `auto` (D750, before
|
|
25360
|
+
* D755 split it) reads as `hardware` — that is what it computed.
|
|
25361
|
+
*/
|
|
25362
|
+
function parseShareSource(value) {
|
|
25363
|
+
const parsed = ShareSourceSchema.safeParse(value);
|
|
25364
|
+
return parsed.success ? parsed.data : DEFAULT_SHARE_SOURCE;
|
|
25365
|
+
}
|
|
25366
|
+
var SetShareSourceInputSchema = object({ source: ShareSourceSchema });
|
|
25367
|
+
object({
|
|
25368
|
+
nodes: record(string(), number()),
|
|
25369
|
+
devices: record(string(), record(string(), number()))
|
|
25370
|
+
});
|
|
25371
|
+
var EffectiveDeviceShareSchema = object({
|
|
25372
|
+
deviceKey: string(),
|
|
25373
|
+
/** Can take sessions now (enabled, available, can run the root). */
|
|
25374
|
+
eligible: boolean(),
|
|
25375
|
+
/** The share the source assigns, as a percentage of the node's devices. */
|
|
25376
|
+
configuredPct: number(),
|
|
25377
|
+
/** The share actually used, after an ineligible device's share went to the others. */
|
|
25378
|
+
effectivePct: number()
|
|
25379
|
+
});
|
|
25380
|
+
var EffectiveNodeShareSchema = object({
|
|
25381
|
+
nodeId: string(),
|
|
25382
|
+
eligible: boolean(),
|
|
25383
|
+
configuredPct: number(),
|
|
25384
|
+
effectivePct: number(),
|
|
25385
|
+
/** Automatic sources: what this node's share was computed from; null under `manual` or before the first answer. */
|
|
25386
|
+
capacitySource: CapacitySourceSchema.nullable(),
|
|
25387
|
+
/** Automatic `benchmark`: why this node used `hardware` instead. */
|
|
25388
|
+
capacityFallback: CapacityFallbackSchema.nullable(),
|
|
25389
|
+
devices: array(EffectiveDeviceShareSchema).readonly()
|
|
25390
|
+
});
|
|
25391
|
+
var EffectiveSharesSchema = object({
|
|
25392
|
+
source: ShareSourceSchema,
|
|
25393
|
+
policy: SessionPlacementPolicySchema,
|
|
25394
|
+
/** An automatic source with no computed answer yet: the stored sliders are used meanwhile. */
|
|
25395
|
+
autoPending: boolean(),
|
|
25396
|
+
/** When the automatic shares were last computed (ms epoch); null = never. */
|
|
25397
|
+
computedAt: number().nullable(),
|
|
25398
|
+
/** What made the system recompute them last. */
|
|
25399
|
+
trigger: string().nullable(),
|
|
25400
|
+
nodes: array(EffectiveNodeShareSchema).readonly(),
|
|
25401
|
+
/** The relative weights the election reads (before eligibility). */
|
|
25402
|
+
nodeWeights: record(string(), number()),
|
|
25403
|
+
deviceWeights: record(string(), record(string(), number()))
|
|
25404
|
+
});
|
|
25316
25405
|
object({
|
|
25317
25406
|
activeCameras: number(),
|
|
25318
25407
|
throttledCameras: number(),
|
|
@@ -26720,8 +26809,10 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26720
26809
|
modelId: string().optional(),
|
|
26721
26810
|
settings: record(string(), unknown()).readonly().optional()
|
|
26722
26811
|
});
|
|
26812
|
+
var CameraStepSettingsSchema = record(string(), record(string(), unknown()).readonly());
|
|
26723
26813
|
var CameraPipelineSettingsSchema = object({
|
|
26724
26814
|
stepToggles: record(string(), boolean()).optional(),
|
|
26815
|
+
stepSettings: CameraStepSettingsSchema.optional(),
|
|
26725
26816
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26726
26817
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
26727
26818
|
});
|
|
@@ -27487,23 +27578,25 @@ var pipelineOrchestratorCapability = {
|
|
|
27487
27578
|
auth: "admin"
|
|
27488
27579
|
}),
|
|
27489
27580
|
/**
|
|
27490
|
-
*
|
|
27491
|
-
* for the whole cluster from each node's static hardware class
|
|
27492
|
-
* measured latency table
|
|
27581
|
+
* Auto tune (D745, D755) — DRY RUN. Recommends ONE detector family
|
|
27582
|
+
* (YOLOv9) for the whole cluster from each node's static hardware class
|
|
27583
|
+
* and the measured latency table: per device, the most accurate model
|
|
27584
|
+
* that sustains at least `minSessionsPerDevice` parallel sessions at the
|
|
27585
|
+
* camera rate with the headroom kept free (no load input), and lists every write
|
|
27493
27586
|
* the apply would make: the node model pins set, the enrichment pins and
|
|
27494
27587
|
* model-bound thresholds removed (node level and per camera), and the
|
|
27495
27588
|
* per-camera detector thresholds cleared. Writes nothing. Step
|
|
27496
27589
|
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27497
27590
|
* rules are never part of it. `planHash` names exactly this diff.
|
|
27498
27591
|
*/
|
|
27499
|
-
|
|
27592
|
+
planAutoTune: method(AutoTuneOptionsSchema, AutoTunePlanSchema, { auth: "admin" }),
|
|
27500
27593
|
/**
|
|
27501
|
-
*
|
|
27594
|
+
* Auto tune (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27502
27595
|
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27503
27596
|
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27504
27597
|
* nodes are skipped; there is no undo.
|
|
27505
27598
|
*/
|
|
27506
|
-
|
|
27599
|
+
applyAutoTune: method(AutoTuneApplyInputSchema, AutoTuneApplyResultSchema, {
|
|
27507
27600
|
kind: "mutation",
|
|
27508
27601
|
auth: "admin"
|
|
27509
27602
|
}),
|
|
@@ -27528,12 +27621,32 @@ var pipelineOrchestratorCapability = {
|
|
|
27528
27621
|
*/
|
|
27529
27622
|
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27530
27623
|
/**
|
|
27531
|
-
* Load balancing (D748) — shares proportional to each node's
|
|
27532
|
-
* capacity
|
|
27533
|
-
*
|
|
27534
|
-
*
|
|
27624
|
+
* Load balancing (D748, D755) — shares proportional to each node's
|
|
27625
|
+
* capacity in parallel sessions: from the latency table at the models
|
|
27626
|
+
* Auto tune picks (`hardware`), or from node benchmark results
|
|
27627
|
+
* (`benchmark`, each node without results falling back to `hardware` and
|
|
27628
|
+
* saying so). The automatic shares are exactly this answer. Per device,
|
|
27629
|
+
* an ADVISORY session cap. Writes nothing.
|
|
27535
27630
|
*/
|
|
27536
27631
|
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27632
|
+
/**
|
|
27633
|
+
* Load balancing (D750) — the shares placement reads now: the source
|
|
27634
|
+
* (automatic capacity shares or the operator's sliders), each node's and
|
|
27635
|
+
* device's configured share, and the effective share after a node or
|
|
27636
|
+
* device that cannot take sessions gave its share to the others.
|
|
27637
|
+
*/
|
|
27638
|
+
getEffectiveShares: method(_void(), EffectiveSharesSchema, { auth: "admin" }),
|
|
27639
|
+
/**
|
|
27640
|
+
* Load balancing (D750, D755) — switch the share source: `hardware`
|
|
27641
|
+
* (automatic, from static capacity), `benchmark` (automatic, from node
|
|
27642
|
+
* benchmark results) or `manual`. Going manual seeds the sliders with the
|
|
27643
|
+
* current automatic shares (one atomic write), so nothing jumps; going
|
|
27644
|
+
* to an automatic source recomputes at once.
|
|
27645
|
+
*/
|
|
27646
|
+
setShareSource: method(SetShareSourceInputSchema, object({ success: literal(true) }), {
|
|
27647
|
+
kind: "mutation",
|
|
27648
|
+
auth: "admin"
|
|
27649
|
+
}),
|
|
27537
27650
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27538
27651
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27539
27652
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -27563,6 +27676,22 @@ var pipelineOrchestratorCapability = {
|
|
|
27563
27676
|
kind: "mutation",
|
|
27564
27677
|
auth: "admin"
|
|
27565
27678
|
}),
|
|
27679
|
+
/**
|
|
27680
|
+
* Write or clear one step's CAMERA-WIDE settings (D754) — the fields this
|
|
27681
|
+
* camera overrides on every accelerator it runs on. `settings` is the
|
|
27682
|
+
* whole field-level bag for that step (only the overridden keys; a
|
|
27683
|
+
* model-bound field travels with its binding key); `null` or `{}` clears
|
|
27684
|
+
* it. Merged at dispatch between the node base and the per-(node,device)
|
|
27685
|
+
* override, key by key.
|
|
27686
|
+
*/
|
|
27687
|
+
setCameraStepSettings: method(object({
|
|
27688
|
+
deviceId: number(),
|
|
27689
|
+
addonId: string(),
|
|
27690
|
+
settings: record(string(), unknown()).nullable()
|
|
27691
|
+
}), object({ success: literal(true) }), {
|
|
27692
|
+
kind: "mutation",
|
|
27693
|
+
auth: "admin"
|
|
27694
|
+
}),
|
|
27566
27695
|
/** Write or clear the wholesale per-agent pipeline override. Pass `pipeline: null` to drop the override. */
|
|
27567
27696
|
setCameraPipelineForAgent: method(object({
|
|
27568
27697
|
deviceId: number(),
|
|
@@ -43557,7 +43686,7 @@ Object.freeze({
|
|
|
43557
43686
|
addonId: null,
|
|
43558
43687
|
access: "view"
|
|
43559
43688
|
},
|
|
43560
|
-
"pipelineOrchestrator.
|
|
43689
|
+
"pipelineOrchestrator.applyAutoTune": {
|
|
43561
43690
|
capName: "pipeline-orchestrator",
|
|
43562
43691
|
capScope: "system",
|
|
43563
43692
|
addonId: null,
|
|
@@ -43677,6 +43806,12 @@ Object.freeze({
|
|
|
43677
43806
|
addonId: null,
|
|
43678
43807
|
access: "view"
|
|
43679
43808
|
},
|
|
43809
|
+
"pipelineOrchestrator.getEffectiveShares": {
|
|
43810
|
+
capName: "pipeline-orchestrator",
|
|
43811
|
+
capScope: "system",
|
|
43812
|
+
addonId: null,
|
|
43813
|
+
access: "view"
|
|
43814
|
+
},
|
|
43680
43815
|
"pipelineOrchestrator.getGlobalMetrics": {
|
|
43681
43816
|
capName: "pipeline-orchestrator",
|
|
43682
43817
|
capScope: "system",
|
|
@@ -43743,7 +43878,7 @@ Object.freeze({
|
|
|
43743
43878
|
addonId: null,
|
|
43744
43879
|
access: "create"
|
|
43745
43880
|
},
|
|
43746
|
-
"pipelineOrchestrator.
|
|
43881
|
+
"pipelineOrchestrator.planAutoTune": {
|
|
43747
43882
|
capName: "pipeline-orchestrator",
|
|
43748
43883
|
capScope: "system",
|
|
43749
43884
|
addonId: null,
|
|
@@ -43845,6 +43980,12 @@ Object.freeze({
|
|
|
43845
43980
|
addonId: null,
|
|
43846
43981
|
access: "create"
|
|
43847
43982
|
},
|
|
43983
|
+
"pipelineOrchestrator.setCameraStepSettings": {
|
|
43984
|
+
capName: "pipeline-orchestrator",
|
|
43985
|
+
capScope: "system",
|
|
43986
|
+
addonId: null,
|
|
43987
|
+
access: "create"
|
|
43988
|
+
},
|
|
43848
43989
|
"pipelineOrchestrator.setCameraStepToggle": {
|
|
43849
43990
|
capName: "pipeline-orchestrator",
|
|
43850
43991
|
capScope: "system",
|
|
@@ -43875,6 +44016,12 @@ Object.freeze({
|
|
|
43875
44016
|
addonId: null,
|
|
43876
44017
|
access: "create"
|
|
43877
44018
|
},
|
|
44019
|
+
"pipelineOrchestrator.setShareSource": {
|
|
44020
|
+
capName: "pipeline-orchestrator",
|
|
44021
|
+
capScope: "system",
|
|
44022
|
+
addonId: null,
|
|
44023
|
+
access: "create"
|
|
44024
|
+
},
|
|
43878
44025
|
"pipelineOrchestrator.suggestShares": {
|
|
43879
44026
|
capName: "pipeline-orchestrator",
|
|
43880
44027
|
capScope: "system",
|
|
@@ -49315,6 +49462,11 @@ Object.freeze({
|
|
|
49315
49462
|
form: "single",
|
|
49316
49463
|
optional: false
|
|
49317
49464
|
}],
|
|
49465
|
+
"pipelineOrchestrator.setCameraStepSettings": [{
|
|
49466
|
+
name: "deviceId",
|
|
49467
|
+
form: "single",
|
|
49468
|
+
optional: false
|
|
49469
|
+
}],
|
|
49318
49470
|
"pipelineOrchestrator.setCameraStepToggle": [{
|
|
49319
49471
|
name: "deviceId",
|
|
49320
49472
|
form: "single",
|
|
@@ -51113,7 +51265,7 @@ function modelBindingKey(key) {
|
|
|
51113
51265
|
/**
|
|
51114
51266
|
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
51115
51267
|
* flattened (D738). One definition for the runner (which applies the binding)
|
|
51116
|
-
* and the orchestrator's
|
|
51268
|
+
* and the orchestrator's Auto tune (which clears those overrides, D745).
|
|
51117
51269
|
*/
|
|
51118
51270
|
function collectModelBoundKeys(fields) {
|
|
51119
51271
|
return fields.flatMap((field) => {
|
|
@@ -51605,140 +51757,25 @@ var DETECTOR_ACCURACY = {
|
|
|
51605
51757
|
personRecall: .646
|
|
51606
51758
|
}
|
|
51607
51759
|
};
|
|
51608
|
-
var DEFAULT_DEMAND_OPTIONS = {
|
|
51609
|
-
onMotionConcurrency: .25,
|
|
51610
|
-
onMotionMinConcurrent: 2
|
|
51611
|
-
};
|
|
51612
|
-
function deviceId(nodeId, deviceKey) {
|
|
51613
|
-
return `${nodeId}\u0000${deviceKey}`;
|
|
51614
|
-
}
|
|
51615
|
-
function charges(load) {
|
|
51616
|
-
return load.mode !== "disabled" && load.runsPipelineRoot && load.fps > 0;
|
|
51617
|
-
}
|
|
51618
|
-
/** The highest `count` rates, summed. */
|
|
51619
|
-
function topRatesSum(rates, count) {
|
|
51620
|
-
return rates.toSorted((a, b) => b - a).slice(0, Math.max(0, count)).reduce((sum, r) => sum + r, 0);
|
|
51621
|
-
}
|
|
51622
|
-
/** Concurrent on-motion sessions assumed cluster-wide without an observation. */
|
|
51623
|
-
function estimatedConcurrentSessions(cameraCount, options) {
|
|
51624
|
-
if (cameraCount <= 0) return 0;
|
|
51625
|
-
const share = Math.ceil(options.onMotionConcurrency * cameraCount);
|
|
51626
|
-
return Math.min(cameraCount, Math.max(share, Math.min(cameraCount, options.onMotionMinConcurrent)));
|
|
51627
|
-
}
|
|
51628
|
-
/**
|
|
51629
|
-
* Demand on every eligible device of the cluster.
|
|
51630
|
-
*/
|
|
51631
|
-
function estimateDeviceDemand(input) {
|
|
51632
|
-
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51633
|
-
const placement = input.shares ? "share" : "least-loaded";
|
|
51634
|
-
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51635
|
-
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51636
|
-
const weightByNode = /* @__PURE__ */ new Map();
|
|
51637
|
-
const eligibleCountByNode = /* @__PURE__ */ new Map();
|
|
51638
|
-
for (const d of input.eligible) {
|
|
51639
|
-
weightByNode.set(d.nodeId, (weightByNode.get(d.nodeId) ?? 0) + Math.max(d.weight, 0));
|
|
51640
|
-
eligibleCountByNode.set(d.nodeId, (eligibleCountByNode.get(d.nodeId) ?? 0) + 1);
|
|
51641
|
-
}
|
|
51642
|
-
const pinnedFps = /* @__PURE__ */ new Map();
|
|
51643
|
-
const alwaysOnFps = /* @__PURE__ */ new Map();
|
|
51644
|
-
const alwaysOnCount = /* @__PURE__ */ new Map();
|
|
51645
|
-
const maxFps = /* @__PURE__ */ new Map();
|
|
51646
|
-
const onMotionRates = [];
|
|
51647
|
-
const bump = (map, key, by) => {
|
|
51648
|
-
map.set(key, (map.get(key) ?? 0) + by);
|
|
51649
|
-
};
|
|
51650
|
-
const noteMax = (key, fps) => {
|
|
51651
|
-
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51652
|
-
};
|
|
51653
|
-
const pinNodeOf = (pin) => {
|
|
51654
|
-
if (pin.nodeId !== null) return pin.nodeId;
|
|
51655
|
-
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51656
|
-
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51657
|
-
};
|
|
51658
|
-
let alwaysOnTotal = 0;
|
|
51659
|
-
for (const cam of input.cameras) {
|
|
51660
|
-
if (!charges(cam)) continue;
|
|
51661
|
-
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51662
|
-
if (cam.devicePin !== null && pinNode !== null) {
|
|
51663
|
-
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51664
|
-
if (!eligibleById.has(key)) continue;
|
|
51665
|
-
bump(pinnedFps, key, cam.fps);
|
|
51666
|
-
noteMax(key, cam.fps);
|
|
51667
|
-
continue;
|
|
51668
|
-
}
|
|
51669
|
-
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51670
|
-
if (standing !== null) {
|
|
51671
|
-
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51672
|
-
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51673
|
-
for (const d of input.eligible) {
|
|
51674
|
-
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51675
|
-
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51676
|
-
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51677
|
-
bump(alwaysOnFps, key, cam.fps * share);
|
|
51678
|
-
bump(alwaysOnCount, key, share);
|
|
51679
|
-
noteMax(key, cam.fps);
|
|
51680
|
-
}
|
|
51681
|
-
continue;
|
|
51682
|
-
}
|
|
51683
|
-
onMotionRates.push(cam.fps);
|
|
51684
|
-
}
|
|
51685
|
-
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51686
|
-
const nodeCount = weightByNode.size;
|
|
51687
|
-
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51688
|
-
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51689
|
-
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51690
|
-
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51691
|
-
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51692
|
-
return input.eligible.map((d) => {
|
|
51693
|
-
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51694
|
-
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51695
|
-
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51696
|
-
let onMotionSessions = 0;
|
|
51697
|
-
let onMotionSource = "none";
|
|
51698
|
-
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51699
|
-
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51700
|
-
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51701
|
-
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51702
|
-
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51703
|
-
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51704
|
-
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51705
|
-
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51706
|
-
onMotionSource = "observed-24h";
|
|
51707
|
-
} else {
|
|
51708
|
-
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51709
|
-
const share = nodeCount > 0 && nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight / nodeCount : 0;
|
|
51710
|
-
onMotionSessions = Math.ceil(estimatedTotal * share);
|
|
51711
|
-
onMotionSource = "estimate";
|
|
51712
|
-
}
|
|
51713
|
-
const demandFps = (pinnedFps.get(key) ?? 0) + (alwaysOnFps.get(key) ?? 0) + topRatesSum(onMotionRates, onMotionSessions);
|
|
51714
|
-
return {
|
|
51715
|
-
nodeId: d.nodeId,
|
|
51716
|
-
deviceKey: d.deviceKey,
|
|
51717
|
-
demandFps: Math.round(demandFps * 100) / 100,
|
|
51718
|
-
maxCameraFps: Math.max(maxFps.get(key) ?? 0, fastestOnMotion),
|
|
51719
|
-
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51720
|
-
onMotionSessions,
|
|
51721
|
-
onMotionSource,
|
|
51722
|
-
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51723
|
-
placement
|
|
51724
|
-
};
|
|
51725
|
-
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51726
|
-
}
|
|
51727
51760
|
/** The family the cluster runs. */
|
|
51728
|
-
var
|
|
51761
|
+
var AUTO_TUNE_FAMILY = "yolov9";
|
|
51762
|
+
/** Parallel sessions one device sustains at `fps` with `headroom` kept free. */
|
|
51763
|
+
function capacitySessionsOf(fps, headroom, cameraFps) {
|
|
51764
|
+
if (!(fps > 0) || !(cameraFps > 0)) return 0;
|
|
51765
|
+
return Math.max(0, Math.floor(fps * (1 - headroom) / cameraFps + 1e-9));
|
|
51766
|
+
}
|
|
51729
51767
|
function round(value, digits = 3) {
|
|
51730
51768
|
const f = 10 ** digits;
|
|
51731
51769
|
return Math.round(value * f) / f;
|
|
51732
51770
|
}
|
|
51733
|
-
function
|
|
51734
|
-
return cell
|
|
51771
|
+
function sessionsOn(cell, sizing) {
|
|
51772
|
+
return cell === void 0 ? 0 : capacitySessionsOf(cell.fps, sizing.headroom, sizing.cameraFps);
|
|
51735
51773
|
}
|
|
51736
|
-
function fitsOn(cell,
|
|
51737
|
-
|
|
51738
|
-
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51774
|
+
function fitsOn(cell, sizing) {
|
|
51775
|
+
return sessionsOn(cell, sizing) >= sizing.minSessions;
|
|
51739
51776
|
}
|
|
51740
51777
|
function candidatesFor(contexts, formatsByModel) {
|
|
51741
|
-
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(
|
|
51778
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(AUTO_TUNE_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51742
51779
|
modelId: id,
|
|
51743
51780
|
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51744
51781
|
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
@@ -51746,28 +51783,24 @@ function candidatesFor(contexts, formatsByModel) {
|
|
|
51746
51783
|
function totalLatency(contexts, modelId) {
|
|
51747
51784
|
return contexts.reduce((sum, c) => sum + (c.column[modelId]?.p50Ms ?? Number.POSITIVE_INFINITY), 0);
|
|
51748
51785
|
}
|
|
51749
|
-
|
|
51750
|
-
|
|
51751
|
-
|
|
51752
|
-
return Math.max(worst, cell ? utilisationOf(cell, c.demand) : Number.POSITIVE_INFINITY);
|
|
51753
|
-
}, 0);
|
|
51786
|
+
/** The fewest sessions any device of the group sustains with this model. */
|
|
51787
|
+
function worstSessions(contexts, modelId, sizing) {
|
|
51788
|
+
return contexts.reduce((worst, c) => Math.min(worst, sessionsOn(c.column[modelId], sizing)), Number.POSITIVE_INFINITY);
|
|
51754
51789
|
}
|
|
51755
51790
|
/** One model for a group of devices that must agree (step 2–4 of the header). */
|
|
51756
|
-
function pickForGroup(contexts, formatsByModel,
|
|
51791
|
+
function pickForGroup(contexts, formatsByModel, sizing) {
|
|
51757
51792
|
const candidates = candidatesFor(contexts, formatsByModel);
|
|
51758
51793
|
if (candidates.length === 0) return null;
|
|
51759
|
-
const fitting = candidates.filter((c) => contexts.every((ctx) =>
|
|
51760
|
-
const cell = ctx.column[c.modelId];
|
|
51761
|
-
return cell !== void 0 && fitsOn(cell, ctx.demand, headroom);
|
|
51762
|
-
}));
|
|
51794
|
+
const fitting = candidates.filter((c) => contexts.every((ctx) => fitsOn(ctx.column[c.modelId], sizing)));
|
|
51763
51795
|
if (fitting.length === 0) {
|
|
51764
|
-
const lightest = candidates.toSorted((a, b) =>
|
|
51796
|
+
const lightest = candidates.toSorted((a, b) => worstSessions(contexts, b.modelId, sizing) - worstSessions(contexts, a.modelId, sizing) || totalLatency(contexts, a.modelId) - totalLatency(contexts, b.modelId))[0];
|
|
51765
51797
|
if (lightest === void 0) return null;
|
|
51766
51798
|
return {
|
|
51767
51799
|
modelId: lightest.modelId,
|
|
51768
51800
|
reasons: [{
|
|
51769
51801
|
kind: "no-fit-lightest",
|
|
51770
|
-
|
|
51802
|
+
sessions: worstSessions(contexts, lightest.modelId, sizing),
|
|
51803
|
+
minSessions: sizing.minSessions
|
|
51771
51804
|
}]
|
|
51772
51805
|
};
|
|
51773
51806
|
}
|
|
@@ -51777,10 +51810,7 @@ function pickForGroup(contexts, formatsByModel, headroom) {
|
|
|
51777
51810
|
const reasons = [];
|
|
51778
51811
|
const mostAccurate = candidates[0];
|
|
51779
51812
|
if (mostAccurate !== void 0 && mostAccurate.modelId !== best.modelId) {
|
|
51780
|
-
const limitedBy = contexts.find((ctx) =>
|
|
51781
|
-
const cell = ctx.column[mostAccurate.modelId];
|
|
51782
|
-
return cell === void 0 || !fitsOn(cell, ctx.demand, headroom);
|
|
51783
|
-
})?.device.key ?? "";
|
|
51813
|
+
const limitedBy = contexts.find((ctx) => !fitsOn(ctx.column[mostAccurate.modelId], sizing))?.device.key ?? "";
|
|
51784
51814
|
reasons.push({
|
|
51785
51815
|
kind: "stepped-down",
|
|
51786
51816
|
from: mostAccurate.modelId,
|
|
@@ -51798,7 +51828,7 @@ function pickForGroup(contexts, formatsByModel, headroom) {
|
|
|
51798
51828
|
reasons
|
|
51799
51829
|
};
|
|
51800
51830
|
}
|
|
51801
|
-
function choiceOn(ctx, modelId) {
|
|
51831
|
+
function choiceOn(ctx, modelId, sizing) {
|
|
51802
51832
|
const cell = ctx.column[modelId];
|
|
51803
51833
|
if (cell === void 0) return null;
|
|
51804
51834
|
return {
|
|
@@ -51806,23 +51836,20 @@ function choiceOn(ctx, modelId) {
|
|
|
51806
51836
|
p50Ms: cell.p50Ms,
|
|
51807
51837
|
fps: cell.fps,
|
|
51808
51838
|
measured: cell.measured,
|
|
51809
|
-
|
|
51839
|
+
capacitySessions: sessionsOn(cell, sizing),
|
|
51810
51840
|
personRecall: DETECTOR_ACCURACY[modelId]?.personRecall ?? 0
|
|
51811
51841
|
};
|
|
51812
51842
|
}
|
|
51813
|
-
function deviceReasonsFor(
|
|
51843
|
+
function deviceReasonsFor(choice, sizing) {
|
|
51814
51844
|
const out = [{
|
|
51815
|
-
kind: "
|
|
51816
|
-
|
|
51845
|
+
kind: "capacity",
|
|
51846
|
+
sessions: choice.capacitySessions,
|
|
51847
|
+
minSessions: sizing.minSessions
|
|
51817
51848
|
}];
|
|
51818
51849
|
if (!choice.measured) out.push({
|
|
51819
51850
|
kind: "derived-latency",
|
|
51820
51851
|
modelId: choice.modelId
|
|
51821
51852
|
});
|
|
51822
|
-
if (ctx.demand?.onMotionSource === "estimate") out.push({
|
|
51823
|
-
kind: "estimated-load",
|
|
51824
|
-
observedForMs: ctx.demand.observedForMs
|
|
51825
|
-
});
|
|
51826
51853
|
return out;
|
|
51827
51854
|
}
|
|
51828
51855
|
function skippedNode(node, reason, hardwareClass) {
|
|
@@ -51838,7 +51865,6 @@ function skippedNode(node, reason, hardwareClass) {
|
|
|
51838
51865
|
accelerator: node.hardware ? acceleratorKindOf(d.key, node.hardware) : null,
|
|
51839
51866
|
enabled: d.enabled,
|
|
51840
51867
|
available: d.available,
|
|
51841
|
-
demand: null,
|
|
51842
51868
|
chosen: null,
|
|
51843
51869
|
reasons: []
|
|
51844
51870
|
})),
|
|
@@ -51848,7 +51874,11 @@ function skippedNode(node, reason, hardwareClass) {
|
|
|
51848
51874
|
/** Recommend one node. See the module header. */
|
|
51849
51875
|
function recommendNode(input) {
|
|
51850
51876
|
const { node } = input;
|
|
51851
|
-
const
|
|
51877
|
+
const sizing = {
|
|
51878
|
+
headroom: input.headroom ?? .4,
|
|
51879
|
+
cameraFps: input.cameraFps ?? 10,
|
|
51880
|
+
minSessions: input.minSessionsPerDevice ?? 2
|
|
51881
|
+
};
|
|
51852
51882
|
const perf = input.perf ?? DETECTOR_PERF;
|
|
51853
51883
|
if (!node.reachable) return skippedNode(node, { kind: "node-offline" }, null);
|
|
51854
51884
|
if (node.hardware === null) return skippedNode(node, { kind: "no-hardware-facts" }, null);
|
|
@@ -51859,7 +51889,6 @@ function recommendNode(input) {
|
|
|
51859
51889
|
kind: "no-measured-data",
|
|
51860
51890
|
hardwareClass
|
|
51861
51891
|
}, hardwareClass);
|
|
51862
|
-
const demandByKey = new Map(input.demand.filter((d) => d.nodeId === node.nodeId).map((d) => [d.deviceKey, d]));
|
|
51863
51892
|
const devices = node.devices.toSorted((a, b) => a.key.localeCompare(b.key));
|
|
51864
51893
|
const contexts = /* @__PURE__ */ new Map();
|
|
51865
51894
|
const early = /* @__PURE__ */ new Map();
|
|
@@ -51888,8 +51917,7 @@ function recommendNode(input) {
|
|
|
51888
51917
|
contexts.set(device.key, {
|
|
51889
51918
|
device,
|
|
51890
51919
|
accelerator,
|
|
51891
|
-
column
|
|
51892
|
-
demand: demandByKey.get(device.key) ?? null
|
|
51920
|
+
column
|
|
51893
51921
|
});
|
|
51894
51922
|
}
|
|
51895
51923
|
const isCpu = (a) => a === "x86-cpu" || a === "arm-cpu";
|
|
@@ -51899,23 +51927,23 @@ function recommendNode(input) {
|
|
|
51899
51927
|
const reasonsByKey = /* @__PURE__ */ new Map();
|
|
51900
51928
|
const nodeReasons = [];
|
|
51901
51929
|
let acceleratorModelId = null;
|
|
51902
|
-
const groupPick = accelerators.length > 0 ? pickForGroup(accelerators, node.formatsByModel,
|
|
51930
|
+
const groupPick = accelerators.length > 0 ? pickForGroup(accelerators, node.formatsByModel, sizing) : null;
|
|
51903
51931
|
if (groupPick !== null) {
|
|
51904
51932
|
acceleratorModelId = groupPick.modelId;
|
|
51905
51933
|
nodeReasons.push(...groupPick.reasons);
|
|
51906
51934
|
for (const ctx of accelerators) {
|
|
51907
|
-
const choice = choiceOn(ctx, groupPick.modelId);
|
|
51935
|
+
const choice = choiceOn(ctx, groupPick.modelId, sizing);
|
|
51908
51936
|
if (choice === null) continue;
|
|
51909
51937
|
chosenByKey.set(ctx.device.key, choice);
|
|
51910
|
-
reasonsByKey.set(ctx.device.key, deviceReasonsFor(
|
|
51938
|
+
reasonsByKey.set(ctx.device.key, deviceReasonsFor(choice, sizing));
|
|
51911
51939
|
}
|
|
51912
51940
|
} else for (const ctx of accelerators) reasonsByKey.set(ctx.device.key, [{
|
|
51913
51941
|
kind: "no-family-build",
|
|
51914
51942
|
format: ctx.device.format
|
|
51915
51943
|
}]);
|
|
51916
51944
|
for (const ctx of cpus) {
|
|
51917
|
-
const pick = pickForGroup([ctx], node.formatsByModel,
|
|
51918
|
-
const choice = pick === null ? null : choiceOn(ctx, pick.modelId);
|
|
51945
|
+
const pick = pickForGroup([ctx], node.formatsByModel, sizing);
|
|
51946
|
+
const choice = pick === null ? null : choiceOn(ctx, pick.modelId, sizing);
|
|
51919
51947
|
if (pick === null || choice === null) {
|
|
51920
51948
|
reasonsByKey.set(ctx.device.key, [{
|
|
51921
51949
|
kind: "no-family-build",
|
|
@@ -51927,7 +51955,7 @@ function recommendNode(input) {
|
|
|
51927
51955
|
reasonsByKey.set(ctx.device.key, [
|
|
51928
51956
|
{ kind: "cpu-device" },
|
|
51929
51957
|
...pick.reasons,
|
|
51930
|
-
...deviceReasonsFor(
|
|
51958
|
+
...deviceReasonsFor(choice, sizing)
|
|
51931
51959
|
]);
|
|
51932
51960
|
}
|
|
51933
51961
|
return {
|
|
@@ -51942,7 +51970,6 @@ function recommendNode(input) {
|
|
|
51942
51970
|
accelerator: acceleratorKindOf(device.key, hardware),
|
|
51943
51971
|
enabled: device.enabled,
|
|
51944
51972
|
available: device.available,
|
|
51945
|
-
demand: demandByKey.get(device.key) ?? null,
|
|
51946
51973
|
chosen: chosenByKey.get(device.key) ?? null,
|
|
51947
51974
|
reasons: early.get(device.key) ?? reasonsByKey.get(device.key) ?? []
|
|
51948
51975
|
})),
|
|
@@ -51990,6 +52017,204 @@ function electByShare(candidates) {
|
|
|
51990
52017
|
};
|
|
51991
52018
|
}
|
|
51992
52019
|
/**
|
|
52020
|
+
* The shares a session election actually reads, and the view the Load
|
|
52021
|
+
* balancing tab shows (D750). Pure.
|
|
52022
|
+
*
|
|
52023
|
+
* The SOURCE decides the configured weights: the automatic capacity shares
|
|
52024
|
+
* (`hardware` or `benchmark`, D755), or the operator's stored sliders. Eligibility decides the effective ones: a
|
|
52025
|
+
* node or device that cannot take a session now (offline, switched off, no
|
|
52026
|
+
* detect capability, unable to run the root) contributes nothing, and the
|
|
52027
|
+
* others split its share in proportion to their own. That redistribution is
|
|
52028
|
+
* exactly what `electByShare` does over its candidate set, so the view and
|
|
52029
|
+
* the election cannot disagree; the stored values are never rewritten for it.
|
|
52030
|
+
*/
|
|
52031
|
+
/** Weight a node or device has when its source names none (the balancer's own default). */
|
|
52032
|
+
var DEFAULT_WEIGHT = 1;
|
|
52033
|
+
function positive(value) {
|
|
52034
|
+
return value !== void 0 && value > 0 ? value : DEFAULT_WEIGHT;
|
|
52035
|
+
}
|
|
52036
|
+
function percentages(keys, weightOf, counts) {
|
|
52037
|
+
const total = keys.reduce((sum, k) => sum + (counts(k) ? weightOf(k) : 0), 0);
|
|
52038
|
+
return new Map(keys.map((k) => [k, total > 0 && counts(k) ? Math.round(weightOf(k) / total * 1e3) / 10 : 0]));
|
|
52039
|
+
}
|
|
52040
|
+
function resolveEffectiveShares(input) {
|
|
52041
|
+
const useAuto = isAutomaticShareSource(input.source) && input.auto !== null;
|
|
52042
|
+
const weights = useAuto && input.auto !== null ? input.auto : input.stored;
|
|
52043
|
+
const nodeWeights = {};
|
|
52044
|
+
for (const nodeId of input.nodeIds) nodeWeights[nodeId] = positive(weights.nodes[nodeId]);
|
|
52045
|
+
const eligibleNodes = new Set(input.eligibleNodes);
|
|
52046
|
+
const configured = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52047
|
+
const effective = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(k));
|
|
52048
|
+
const deviceWeights = {};
|
|
52049
|
+
const nodes = input.nodeIds.map((nodeId) => {
|
|
52050
|
+
const eligibleKeys = new Set(input.eligibleDevices[nodeId] ?? []);
|
|
52051
|
+
const named = weights.devices[nodeId] ?? {};
|
|
52052
|
+
const keys = [...new Set([...Object.keys(named), ...eligibleKeys])].toSorted();
|
|
52053
|
+
const perDevice = {};
|
|
52054
|
+
for (const key of keys) perDevice[key] = positive(named[key]);
|
|
52055
|
+
if (keys.length > 0) deviceWeights[nodeId] = perDevice;
|
|
52056
|
+
const devConfigured = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52057
|
+
const devEffective = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(nodeId) && eligibleKeys.has(k));
|
|
52058
|
+
const devices = keys.map((deviceKey) => ({
|
|
52059
|
+
deviceKey,
|
|
52060
|
+
eligible: eligibleKeys.has(deviceKey),
|
|
52061
|
+
configuredPct: devConfigured.get(deviceKey) ?? 0,
|
|
52062
|
+
effectivePct: devEffective.get(deviceKey) ?? 0
|
|
52063
|
+
}));
|
|
52064
|
+
const capacity = useAuto ? input.autoCapacity?.[nodeId] : void 0;
|
|
52065
|
+
return {
|
|
52066
|
+
nodeId,
|
|
52067
|
+
eligible: eligibleNodes.has(nodeId),
|
|
52068
|
+
configuredPct: configured.get(nodeId) ?? 0,
|
|
52069
|
+
effectivePct: effective.get(nodeId) ?? 0,
|
|
52070
|
+
capacitySource: capacity?.source ?? null,
|
|
52071
|
+
capacityFallback: capacity?.fallback ?? null,
|
|
52072
|
+
devices
|
|
52073
|
+
};
|
|
52074
|
+
});
|
|
52075
|
+
return {
|
|
52076
|
+
source: input.source,
|
|
52077
|
+
policy: input.policy ?? "share",
|
|
52078
|
+
autoPending: isAutomaticShareSource(input.source) && input.auto === null,
|
|
52079
|
+
computedAt: input.computedAt ?? null,
|
|
52080
|
+
trigger: input.trigger ?? null,
|
|
52081
|
+
nodes,
|
|
52082
|
+
nodeWeights,
|
|
52083
|
+
deviceWeights
|
|
52084
|
+
};
|
|
52085
|
+
}
|
|
52086
|
+
/**
|
|
52087
|
+
* Automatic shares from CAPACITY (D755). Pure.
|
|
52088
|
+
*
|
|
52089
|
+
* A node's share is its capacity in parallel sessions over the cluster's; a
|
|
52090
|
+
* device's share inside its node follows its throughput. There is no load
|
|
52091
|
+
* input: nothing here depends on how many sessions ran, only on what each
|
|
52092
|
+
* node can carry.
|
|
52093
|
+
*
|
|
52094
|
+
* Two places a capacity can come from:
|
|
52095
|
+
* - `hardware` — each enabled accelerator at its Auto tune model, from the
|
|
52096
|
+
* measured latency table: ⌊fps × (1 − headroom) / camera fps⌋ sessions;
|
|
52097
|
+
* - `benchmark` — the node's own measured max sessions per device, read
|
|
52098
|
+
* through a {@link NodeCapacitySource}. A node whose results do not cover
|
|
52099
|
+
* every enabled device it would be sized on is sized from `hardware`
|
|
52100
|
+
* instead, and the answer says so per node — never a silent mix.
|
|
52101
|
+
*/
|
|
52102
|
+
/** A source that has no results — the benchmark store before it exists. */
|
|
52103
|
+
var NO_MEASURED_CAPACITY = { readMeasuredCapacity: async () => /* @__PURE__ */ new Map() };
|
|
52104
|
+
/** Integer shares summing to 100 (largest remainder); ties broken by key. */
|
|
52105
|
+
function toPercentShares(weights) {
|
|
52106
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
52107
|
+
const out = /* @__PURE__ */ new Map();
|
|
52108
|
+
if (total <= 0) return out;
|
|
52109
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
52110
|
+
k,
|
|
52111
|
+
exact: w / total * 100
|
|
52112
|
+
}));
|
|
52113
|
+
let assigned = 0;
|
|
52114
|
+
for (const r of raw) {
|
|
52115
|
+
out.set(r.k, Math.floor(r.exact));
|
|
52116
|
+
assigned += Math.floor(r.exact);
|
|
52117
|
+
}
|
|
52118
|
+
const remainder = (exact) => Math.round((exact - Math.floor(exact)) * 1e6);
|
|
52119
|
+
const byRemainder = raw.toSorted((a, b) => remainder(b.exact) - remainder(a.exact) || a.k.localeCompare(b.k));
|
|
52120
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
52121
|
+
const r = byRemainder[i % byRemainder.length];
|
|
52122
|
+
if (r === void 0) break;
|
|
52123
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
52124
|
+
}
|
|
52125
|
+
return out;
|
|
52126
|
+
}
|
|
52127
|
+
function hardwareDevices(rec, headroom, cameraFps) {
|
|
52128
|
+
return rec.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
52129
|
+
const fps = d.chosen?.fps ?? 0;
|
|
52130
|
+
return {
|
|
52131
|
+
deviceKey: d.deviceKey,
|
|
52132
|
+
sessions: capacitySessionsOf(fps, headroom, cameraFps),
|
|
52133
|
+
weight: Math.max(fps, 0),
|
|
52134
|
+
modelId: d.chosen?.modelId ?? null,
|
|
52135
|
+
fps: d.chosen?.fps ?? null,
|
|
52136
|
+
measured: d.chosen?.measured ?? false
|
|
52137
|
+
};
|
|
52138
|
+
});
|
|
52139
|
+
}
|
|
52140
|
+
function sizeNode(rec, input) {
|
|
52141
|
+
const fromHardware = (fallback) => {
|
|
52142
|
+
const devices = rec.skipped ? [] : hardwareDevices(rec, input.headroom, input.cameraFps);
|
|
52143
|
+
const capacity = devices.reduce((sum, d) => sum + d.sessions, 0);
|
|
52144
|
+
return {
|
|
52145
|
+
rec,
|
|
52146
|
+
devices,
|
|
52147
|
+
capacitySource: rec.skipped ? null : "hardware",
|
|
52148
|
+
capacityFallback: rec.skipped ? null : fallback,
|
|
52149
|
+
skippedReason: rec.skipped ? rec.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity"
|
|
52150
|
+
};
|
|
52151
|
+
};
|
|
52152
|
+
if (input.source === "hardware") return fromHardware(null);
|
|
52153
|
+
if (!rec.reachable) return fromHardware(null);
|
|
52154
|
+
const results = input.measured.get(rec.nodeId);
|
|
52155
|
+
const byKey = new Map((results?.devices ?? []).map((d) => [d.deviceKey, d]));
|
|
52156
|
+
const usable = rec.devices.filter((d) => d.enabled && d.available);
|
|
52157
|
+
const mustCover = rec.devices.filter((d) => d.enabled && d.available && d.chosen !== null);
|
|
52158
|
+
const covered = usable.filter((d) => byKey.has(d.deviceKey));
|
|
52159
|
+
if (covered.length === 0) return fromHardware("no-benchmark-results");
|
|
52160
|
+
if (mustCover.some((d) => !byKey.has(d.deviceKey))) return fromHardware("partial-benchmark-results");
|
|
52161
|
+
const devices = covered.map((d) => {
|
|
52162
|
+
const row = byKey.get(d.deviceKey);
|
|
52163
|
+
const sessions = Math.max(0, Math.floor(row?.maxSessions ?? 0));
|
|
52164
|
+
return {
|
|
52165
|
+
deviceKey: d.deviceKey,
|
|
52166
|
+
sessions,
|
|
52167
|
+
weight: sessions,
|
|
52168
|
+
modelId: row?.modelId ?? null,
|
|
52169
|
+
fps: null,
|
|
52170
|
+
measured: true
|
|
52171
|
+
};
|
|
52172
|
+
});
|
|
52173
|
+
return {
|
|
52174
|
+
rec,
|
|
52175
|
+
devices,
|
|
52176
|
+
capacitySource: "benchmark",
|
|
52177
|
+
capacityFallback: null,
|
|
52178
|
+
skippedReason: devices.reduce((sum, d) => sum + d.sessions, 0) > 0 ? null : "no-capacity"
|
|
52179
|
+
};
|
|
52180
|
+
}
|
|
52181
|
+
/** Shares proportional to capacity — the automatic shares and the suggestion are this. */
|
|
52182
|
+
function computeCapacityShares(input) {
|
|
52183
|
+
const sized = input.recommendations.map((rec) => sizeNode(rec, input));
|
|
52184
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
52185
|
+
for (const node of sized) {
|
|
52186
|
+
const capacity = node.devices.reduce((sum, d) => sum + d.sessions, 0);
|
|
52187
|
+
if (node.skippedReason === null && capacity > 0) capacityByNode.set(node.rec.nodeId, capacity);
|
|
52188
|
+
}
|
|
52189
|
+
const nodePct = toPercentShares(capacityByNode);
|
|
52190
|
+
const nodes = sized.map((node) => {
|
|
52191
|
+
const devicePct = toPercentShares(new Map(node.devices.map((d) => [d.deviceKey, d.weight])));
|
|
52192
|
+
return {
|
|
52193
|
+
nodeId: node.rec.nodeId,
|
|
52194
|
+
share: nodePct.get(node.rec.nodeId) ?? null,
|
|
52195
|
+
capacitySessions: node.devices.reduce((sum, d) => sum + d.sessions, 0),
|
|
52196
|
+
skippedReason: node.skippedReason,
|
|
52197
|
+
capacitySource: node.capacitySource,
|
|
52198
|
+
capacityFallback: node.capacityFallback,
|
|
52199
|
+
devices: node.devices.map((d) => ({
|
|
52200
|
+
deviceKey: d.deviceKey,
|
|
52201
|
+
share: devicePct.get(d.deviceKey) ?? 0,
|
|
52202
|
+
modelId: d.modelId,
|
|
52203
|
+
fps: d.fps,
|
|
52204
|
+
measured: d.measured,
|
|
52205
|
+
suggestedMaxSessions: d.sessions
|
|
52206
|
+
}))
|
|
52207
|
+
};
|
|
52208
|
+
});
|
|
52209
|
+
return {
|
|
52210
|
+
headroom: input.headroom,
|
|
52211
|
+
source: input.source,
|
|
52212
|
+
cameraFps: input.cameraFps,
|
|
52213
|
+
minSessionsPerDevice: input.minSessionsPerDevice,
|
|
52214
|
+
nodes
|
|
52215
|
+
};
|
|
52216
|
+
}
|
|
52217
|
+
/**
|
|
51993
52218
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
51994
52219
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
51995
52220
|
* is fed.
|
|
@@ -62383,6 +62608,7 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62383
62608
|
};
|
|
62384
62609
|
const okByNode = new Map(input.verdicts.filter((v) => v.state === "ok").map((v) => [v.nodeId, v]));
|
|
62385
62610
|
const sourceAttached = input.attachedByNode[saturated.nodeId] ?? 0;
|
|
62611
|
+
if (input.shares !== void 0) return planShareTarget(input, input.shares, saturated, movable, okByNode, now, guards);
|
|
62386
62612
|
for (const camera of movable) {
|
|
62387
62613
|
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62388
62614
|
const target = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => {
|
|
@@ -62407,6 +62633,52 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62407
62633
|
nodeId: saturated.nodeId
|
|
62408
62634
|
};
|
|
62409
62635
|
}
|
|
62636
|
+
/** Deficits equal up to float noise. */
|
|
62637
|
+
var SHARE_EPSILON = 1e-9;
|
|
62638
|
+
function underCap(input, nodeId) {
|
|
62639
|
+
const cap = input.nodeCaps?.[nodeId];
|
|
62640
|
+
const attached = input.attachedByNode[nodeId] ?? 0;
|
|
62641
|
+
return cap === null || cap === void 0 || cap <= 0 || attached < cap;
|
|
62642
|
+
}
|
|
62643
|
+
/**
|
|
62644
|
+
* Share placement's failover target (D750): among the camera's eligible,
|
|
62645
|
+
* `ok`, under-cap nodes, the one furthest below its share of the cameras
|
|
62646
|
+
* attached across the source and those nodes — the same deficit rule a new
|
|
62647
|
+
* session is placed by. A node at or over its share is no target, so a move
|
|
62648
|
+
* never undoes the balance. Nothing below its share ⇒ hold, naming the camera.
|
|
62649
|
+
*/
|
|
62650
|
+
function planShareTarget(input, shares, saturated, movable, okByNode, now, guards) {
|
|
62651
|
+
const shareOf = (nodeId) => {
|
|
62652
|
+
const share = shares[nodeId];
|
|
62653
|
+
return share !== void 0 && share > 0 ? share : 0;
|
|
62654
|
+
};
|
|
62655
|
+
for (const camera of movable) {
|
|
62656
|
+
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62657
|
+
const healthy = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => underCap(input, v.nodeId));
|
|
62658
|
+
const set = [saturated.nodeId, ...healthy.map((v) => v.nodeId)];
|
|
62659
|
+
const shareSum = set.reduce((sum, nodeId) => sum + shareOf(nodeId), 0);
|
|
62660
|
+
const attachedSum = set.reduce((sum, nodeId) => sum + (input.attachedByNode[nodeId] ?? 0), 0);
|
|
62661
|
+
if (shareSum <= 0) continue;
|
|
62662
|
+
const deficit = (nodeId) => shareOf(nodeId) / shareSum * attachedSum - (input.attachedByNode[nodeId] ?? 0);
|
|
62663
|
+
const target = healthy.filter((v) => deficit(v.nodeId) > SHARE_EPSILON).toSorted((a, b) => deficit(b.nodeId) - deficit(a.nodeId) || a.signals.score - b.signals.score || a.nodeId.localeCompare(b.nodeId))[0];
|
|
62664
|
+
if (!target) continue;
|
|
62665
|
+
return {
|
|
62666
|
+
kind: "move",
|
|
62667
|
+
deviceId: camera.deviceId,
|
|
62668
|
+
fromNodeId: saturated.nodeId,
|
|
62669
|
+
toNodeId: target.nodeId,
|
|
62670
|
+
fromScore: saturated.signals.score,
|
|
62671
|
+
toScore: target.signals.score
|
|
62672
|
+
};
|
|
62673
|
+
}
|
|
62674
|
+
const first = movable[0];
|
|
62675
|
+
return {
|
|
62676
|
+
kind: "hold",
|
|
62677
|
+
reason: "no-target",
|
|
62678
|
+
nodeId: saturated.nodeId,
|
|
62679
|
+
...first !== void 0 ? { deviceId: first.deviceId } : {}
|
|
62680
|
+
};
|
|
62681
|
+
}
|
|
62410
62682
|
//#endregion
|
|
62411
62683
|
//#region src/node-stress-controller.ts
|
|
62412
62684
|
/** Narrow an unknown settings value to a {@link NodeStressMode}. */
|
|
@@ -62615,17 +62887,27 @@ var NodeStressController = class NodeStressController {
|
|
|
62615
62887
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
62616
62888
|
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
62617
62889
|
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
62890
|
+
const candidates = await this.collectCandidates(now);
|
|
62891
|
+
const shares = await this.deps.readFailoverShares?.() ?? null;
|
|
62618
62892
|
const plan = planStressFailover({
|
|
62619
62893
|
verdicts,
|
|
62620
|
-
candidates
|
|
62894
|
+
candidates,
|
|
62621
62895
|
history,
|
|
62622
62896
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
62623
|
-
attachedByNode: this.attachedByNode()
|
|
62897
|
+
attachedByNode: this.attachedByNode(),
|
|
62898
|
+
...shares !== null ? { shares } : {}
|
|
62624
62899
|
}, now, this.guards());
|
|
62625
62900
|
if (plan.kind === "hold") {
|
|
62626
62901
|
this.deps.logger.warn("node saturated — no failover applied", {
|
|
62627
|
-
tags: {
|
|
62628
|
-
|
|
62902
|
+
tags: {
|
|
62903
|
+
nodeId: plan.nodeId ?? "",
|
|
62904
|
+
...plan.deviceId !== void 0 ? { deviceId: plan.deviceId } : {}
|
|
62905
|
+
},
|
|
62906
|
+
meta: {
|
|
62907
|
+
reason: plan.reason,
|
|
62908
|
+
rule: shares !== null ? "share-deficit" : "fewest-cameras",
|
|
62909
|
+
...plan.reason === "no-target" && shares !== null ? { why: "no healthy node below its share can take it" } : {}
|
|
62910
|
+
}
|
|
62629
62911
|
});
|
|
62630
62912
|
return;
|
|
62631
62913
|
}
|
|
@@ -64404,6 +64686,7 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
64404
64686
|
});
|
|
64405
64687
|
var StoredCameraPipelineSettingsSchema = object({
|
|
64406
64688
|
stepToggles: record(string(), boolean()).optional(),
|
|
64689
|
+
stepSettings: record(string(), record(string(), unknown()).readonly()).optional(),
|
|
64407
64690
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
64408
64691
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
64409
64692
|
pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
|
|
@@ -64723,6 +65006,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
64723
65006
|
};
|
|
64724
65007
|
});
|
|
64725
65008
|
}
|
|
65009
|
+
/**
|
|
65010
|
+
* Write or clear one camera's CAMERA-WIDE settings for `addonId` (D754): the
|
|
65011
|
+
* field-level bag merged between the node base and the per-(node,device)
|
|
65012
|
+
* override on every accelerator. `null` or an empty bag deletes the step's
|
|
65013
|
+
* entry; an empty layer is pruned. Read-patch-write, immutable.
|
|
65014
|
+
*/
|
|
65015
|
+
async writeCameraStepSettings(deviceId, addonId, settings) {
|
|
65016
|
+
await this.cameraSettingsState.update((prev) => {
|
|
65017
|
+
const key = String(deviceId);
|
|
65018
|
+
const camera = prev[key] ?? {};
|
|
65019
|
+
const nextLayer = { ...camera.stepSettings };
|
|
65020
|
+
if (settings === null || Object.keys(settings).length === 0) delete nextLayer[addonId];
|
|
65021
|
+
else nextLayer[addonId] = { ...settings };
|
|
65022
|
+
const { stepSettings: _drop, ...rest } = camera;
|
|
65023
|
+
const nextCamera = {
|
|
65024
|
+
...rest,
|
|
65025
|
+
...Object.keys(nextLayer).length > 0 ? { stepSettings: nextLayer } : {}
|
|
65026
|
+
};
|
|
65027
|
+
return {
|
|
65028
|
+
...prev,
|
|
65029
|
+
[key]: nextCamera
|
|
65030
|
+
};
|
|
65031
|
+
});
|
|
65032
|
+
}
|
|
64726
65033
|
/** Purge one device's entry through the durable handle (whole-blob round-trip). */
|
|
64727
65034
|
async deleteCameraSettings(deviceId) {
|
|
64728
65035
|
await this.cameraSettingsState.update((prev) => {
|
|
@@ -65237,12 +65544,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65237
65544
|
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
65238
65545
|
*/
|
|
65239
65546
|
async buildNodeWeights() {
|
|
65547
|
+
if (this.effectiveNodeWeights !== null) return this.effectiveNodeWeights();
|
|
65548
|
+
return this.buildStoredNodeWeights();
|
|
65549
|
+
}
|
|
65550
|
+
/**
|
|
65551
|
+
* The operator's stored node sliders, whatever the share source — what
|
|
65552
|
+
* `manual` applies and what `auto` falls back to before its first answer.
|
|
65553
|
+
*/
|
|
65554
|
+
async buildStoredNodeWeights() {
|
|
65240
65555
|
const blob = await this.agentSettingsState.get();
|
|
65241
65556
|
const weights = {};
|
|
65242
65557
|
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
65243
65558
|
return weights;
|
|
65244
65559
|
}
|
|
65245
65560
|
/**
|
|
65561
|
+
* Where {@link buildNodeWeights} reads from once the share source is wired
|
|
65562
|
+
* (D750): the effective shares (automatic or manual). Every balancer, the
|
|
65563
|
+
* share election and the stress failover read through it, so they can never
|
|
65564
|
+
* disagree on what a node's share is. Unset (tests, early boot) ⇒ stored.
|
|
65565
|
+
*/
|
|
65566
|
+
effectiveNodeWeights = null;
|
|
65567
|
+
setEffectiveNodeWeights(read) {
|
|
65568
|
+
this.effectiveNodeWeights = read;
|
|
65569
|
+
}
|
|
65570
|
+
/**
|
|
65246
65571
|
* The per-(node,device) BASE provisioning (C7.2/C7.4) for one `(node,
|
|
65247
65572
|
* deviceKey)` — the effective `stepId → {modelId?, settings?}` map the
|
|
65248
65573
|
* dispatch provisioning stage attaches onto the resolved step tree before
|
|
@@ -65271,13 +65596,14 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65271
65596
|
async readDeviceProvisioning(deviceId, nodeId, deviceKeys) {
|
|
65272
65597
|
const [agentBlob, cameraMap] = await Promise.all([this.agentSettingsState.get(), this.readCameraSettingsMap()]);
|
|
65273
65598
|
const devices = agentBlob[nodeId]?.inferenceDevices;
|
|
65274
|
-
const
|
|
65599
|
+
const camera = cameraMap[String(deviceId)];
|
|
65600
|
+
const byDevice = camera?.stepOverridesByDevice?.[nodeId];
|
|
65275
65601
|
const out = /* @__PURE__ */ new Map();
|
|
65276
65602
|
for (const deviceKey of deviceKeys) {
|
|
65277
|
-
const
|
|
65603
|
+
const perDevice = deviceKey === "" ? void 0 : byDevice?.[deviceKey];
|
|
65278
65604
|
out.set(deviceKey, {
|
|
65279
65605
|
base: foldNodeStepSettings(devices, deviceKey),
|
|
65280
|
-
override:
|
|
65606
|
+
override: layerCameraStepSettings(camera?.stepSettings, perDevice)
|
|
65281
65607
|
});
|
|
65282
65608
|
}
|
|
65283
65609
|
return out;
|
|
@@ -65341,6 +65667,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65341
65667
|
return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
|
|
65342
65668
|
}
|
|
65343
65669
|
};
|
|
65670
|
+
/**
|
|
65671
|
+
* The camera's override for one accelerator, with its CAMERA-WIDE settings
|
|
65672
|
+
* (D754) folded underneath: per step, `settings = { ...camera, ...perDevice }`
|
|
65673
|
+
* (a per-accelerator key wins), and the per-accelerator `modelId` kept as is —
|
|
65674
|
+
* the camera layer carries no model. The result is what
|
|
65675
|
+
* `applyDeviceProvisioning` merges over the node base, so the full order is
|
|
65676
|
+
* node ⊕ camera ⊕ camera-per-accelerator. Fresh objects; a camera with neither
|
|
65677
|
+
* layer yields `{}`.
|
|
65678
|
+
*/
|
|
65679
|
+
function layerCameraStepSettings(cameraWide, perDevice) {
|
|
65680
|
+
const out = { ...perDevice };
|
|
65681
|
+
for (const [stepId, settings] of Object.entries(cameraWide ?? {})) {
|
|
65682
|
+
if (Object.keys(settings).length === 0) continue;
|
|
65683
|
+
const own = out[stepId];
|
|
65684
|
+
out[stepId] = {
|
|
65685
|
+
...own,
|
|
65686
|
+
settings: {
|
|
65687
|
+
...settings,
|
|
65688
|
+
...own?.settings
|
|
65689
|
+
}
|
|
65690
|
+
};
|
|
65691
|
+
}
|
|
65692
|
+
return out;
|
|
65693
|
+
}
|
|
65344
65694
|
//#endregion
|
|
65345
65695
|
//#region src/pipeline-watchdog.ts
|
|
65346
65696
|
var DEFAULT_WATCHDOG_THRESHOLDS = {
|
|
@@ -67600,9 +67950,10 @@ var SessionDispatchController = class {
|
|
|
67600
67950
|
deviceId: a.deviceId,
|
|
67601
67951
|
nodeId: a.agentNodeId
|
|
67602
67952
|
}))]);
|
|
67953
|
+
const present = new Set(this.deps.topology.knownRunnerNodeIds());
|
|
67603
67954
|
const election = electNodeByShare({
|
|
67604
67955
|
loads: input.loads,
|
|
67605
|
-
eligible: input.eligible,
|
|
67956
|
+
eligible: input.eligible.filter((nodeId) => present.has(nodeId)),
|
|
67606
67957
|
weights: input.weights,
|
|
67607
67958
|
caps: input.nodeCaps,
|
|
67608
67959
|
runningByNode: running
|
|
@@ -68962,7 +69313,6 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68962
69313
|
readGlobalSettings: () => globalSettings,
|
|
68963
69314
|
onCensus: (entries) => {
|
|
68964
69315
|
motionSliceGate?.noteCensus(entries);
|
|
68965
|
-
deps.onRunnerCensus(entries.flatMap((e) => e.load === null ? [] : [e.load]));
|
|
68966
69316
|
providerSliceGate?.refreshIfStale("runner-census");
|
|
68967
69317
|
}
|
|
68968
69318
|
});
|
|
@@ -69071,7 +69421,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69071
69421
|
attach: (nodeId, config) => placement.attachOn(nodeId, config),
|
|
69072
69422
|
detach: (nodeId, deviceId) => placement.detachOn(nodeId, deviceId),
|
|
69073
69423
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
69074
|
-
readGlobalSettings: () => globalSettings
|
|
69424
|
+
readGlobalSettings: () => globalSettings,
|
|
69425
|
+
readFailoverShares: async () => readSessionPlacement() === "share" ? settingsStore.buildNodeWeights() : null
|
|
69075
69426
|
});
|
|
69076
69427
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
69077
69428
|
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
@@ -69087,6 +69438,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69087
69438
|
nodeStress.start();
|
|
69088
69439
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69089
69440
|
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
69441
|
+
const readShareSource = () => parseShareSource(globalSettings?.[SHARE_SOURCE_SETTING_KEY]);
|
|
69090
69442
|
/**
|
|
69091
69443
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
69092
69444
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -69423,6 +69775,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69423
69775
|
nodeLifecycle.unsubscribeDetectionPipelineReadiness(nodeId);
|
|
69424
69776
|
audio.markNodeOffline(nodeId);
|
|
69425
69777
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
69778
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-offline:${nodeId}`);
|
|
69426
69779
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
69427
69780
|
const msg = errMsg$2(err);
|
|
69428
69781
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
@@ -69466,6 +69819,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69466
69819
|
});
|
|
69467
69820
|
});
|
|
69468
69821
|
reconcile.scheduleReconcile();
|
|
69822
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-online:${nodeId}`);
|
|
69469
69823
|
});
|
|
69470
69824
|
const unsubAgentOnline = () => {
|
|
69471
69825
|
unsubAgentOnlineEvent();
|
|
@@ -69706,7 +70060,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69706
70060
|
unsubOrchestratorSubscriptions,
|
|
69707
70061
|
pendingRetryTimer,
|
|
69708
70062
|
autoRebalanceTimer,
|
|
69709
|
-
sessionPlacement: () => readSessionPlacement()
|
|
70063
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
70064
|
+
shareSource: () => readShareSource()
|
|
69710
70065
|
};
|
|
69711
70066
|
}
|
|
69712
70067
|
//#endregion
|
|
@@ -69755,13 +70110,13 @@ function buildGlobalSettingsSections(options) {
|
|
|
69755
70110
|
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69756
70111
|
type: "select",
|
|
69757
70112
|
label: "Session placement",
|
|
69758
|
-
description: "How a new detection session picks its node and device.
|
|
70113
|
+
description: "How a new detection session picks its node and device. Honour shares (default) = the share of PARALLEL sessions is held to the node and device shares, and a node at its cap yields to a free one. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy.",
|
|
69759
70114
|
options: [{
|
|
69760
|
-
value: "least-loaded",
|
|
69761
|
-
label: "Least-loaded"
|
|
69762
|
-
}, {
|
|
69763
70115
|
value: "share",
|
|
69764
70116
|
label: "Honour shares"
|
|
70117
|
+
}, {
|
|
70118
|
+
value: "least-loaded",
|
|
70119
|
+
label: "Least-loaded"
|
|
69765
70120
|
}],
|
|
69766
70121
|
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69767
70122
|
},
|
|
@@ -70323,9 +70678,9 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70323
70678
|
defaultRows: 4
|
|
70324
70679
|
}];
|
|
70325
70680
|
//#endregion
|
|
70326
|
-
//#region src/
|
|
70681
|
+
//#region src/auto-tune/plan-diff.ts
|
|
70327
70682
|
/**
|
|
70328
|
-
* The
|
|
70683
|
+
* The Auto tune writes, as pure transforms of the orchestrator's two
|
|
70329
70684
|
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
70330
70685
|
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
70331
70686
|
*
|
|
@@ -70349,7 +70704,7 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70349
70704
|
*/
|
|
70350
70705
|
/** The root detector step. */
|
|
70351
70706
|
var DETECTOR_STEP_ID = "object-detection";
|
|
70352
|
-
/** Steps whose `modelId` pin
|
|
70707
|
+
/** Steps whose `modelId` pin Auto tune never touches. */
|
|
70353
70708
|
function isModelPinKept(stepId) {
|
|
70354
70709
|
return stepId.endsWith("-embedding");
|
|
70355
70710
|
}
|
|
@@ -70401,7 +70756,7 @@ function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
|
70401
70756
|
};
|
|
70402
70757
|
}
|
|
70403
70758
|
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70404
|
-
function
|
|
70759
|
+
function applyNodeAutoTune(blob, input) {
|
|
70405
70760
|
const next = { ...blob };
|
|
70406
70761
|
const changes = [];
|
|
70407
70762
|
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
@@ -70471,7 +70826,7 @@ function scrubTree(steps, input, onClear) {
|
|
|
70471
70826
|
});
|
|
70472
70827
|
}
|
|
70473
70828
|
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70474
|
-
function
|
|
70829
|
+
function applyCameraAutoTune(blob, input) {
|
|
70475
70830
|
const next = {};
|
|
70476
70831
|
const clears = [];
|
|
70477
70832
|
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
@@ -70555,13 +70910,15 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70555
70910
|
};
|
|
70556
70911
|
}
|
|
70557
70912
|
//#endregion
|
|
70558
|
-
//#region src/
|
|
70913
|
+
//#region src/auto-tune/auto-tune-service.ts
|
|
70559
70914
|
/**
|
|
70560
|
-
*
|
|
70561
|
-
* the plan the operator confirmed.
|
|
70915
|
+
* Auto tune (D745, D755): gather the inputs, compute the plan, and apply
|
|
70916
|
+
* exactly the plan the operator confirmed.
|
|
70562
70917
|
*
|
|
70563
|
-
* The plan is the recommender (`@camstack/types`
|
|
70564
|
-
* runner node
|
|
70918
|
+
* The plan is the recommender (`@camstack/types` auto-tune) over every
|
|
70919
|
+
* runner node — sized by a capacity floor (`minSessionsPerDevice` parallel
|
|
70920
|
+
* sessions per device at the camera rate), never by observed or estimated
|
|
70921
|
+
* load — plus the diff of the writes (`plan-diff.ts`) and the
|
|
70565
70922
|
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70566
70923
|
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
70567
70924
|
* and refuses a different hash, so a configuration change between the two is
|
|
@@ -70578,6 +70935,9 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70578
70935
|
* A failure after phase 2 leaves "overrides cleared, previous node model",
|
|
70579
70936
|
* which is a valid configuration (the defaults run); re-running converges.
|
|
70580
70937
|
*/
|
|
70938
|
+
function minSessionsOf(input) {
|
|
70939
|
+
return input.minSessionsPerDevice ?? 2;
|
|
70940
|
+
}
|
|
70581
70941
|
function errMsg(err) {
|
|
70582
70942
|
return err instanceof Error ? err.message : String(err);
|
|
70583
70943
|
}
|
|
@@ -70606,117 +70966,45 @@ function boundKeysOf(catalogs) {
|
|
|
70606
70966
|
}
|
|
70607
70967
|
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70608
70968
|
}
|
|
70609
|
-
|
|
70610
|
-
var SUGGEST_MAX_ROUNDS = 3;
|
|
70611
|
-
/** Integer shares summing to 100 (largest remainder). */
|
|
70612
|
-
function toPercent(weights) {
|
|
70613
|
-
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70614
|
-
const out = /* @__PURE__ */ new Map();
|
|
70615
|
-
if (total <= 0) return out;
|
|
70616
|
-
const raw = [...weights].map(([k, w]) => ({
|
|
70617
|
-
k,
|
|
70618
|
-
exact: w / total * 100
|
|
70619
|
-
}));
|
|
70620
|
-
let assigned = 0;
|
|
70621
|
-
for (const r of raw) {
|
|
70622
|
-
out.set(r.k, Math.floor(r.exact));
|
|
70623
|
-
assigned += Math.floor(r.exact);
|
|
70624
|
-
}
|
|
70625
|
-
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70626
|
-
for (let i = 0; i < 100 - assigned; i++) {
|
|
70627
|
-
const r = byRemainder[i % byRemainder.length];
|
|
70628
|
-
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70629
|
-
}
|
|
70630
|
-
return out;
|
|
70631
|
-
}
|
|
70632
|
-
var BestSettingsService = class {
|
|
70969
|
+
var AutoTuneService = class {
|
|
70633
70970
|
deps;
|
|
70634
70971
|
applying = false;
|
|
70635
70972
|
constructor(deps) {
|
|
70636
70973
|
this.deps = deps;
|
|
70637
70974
|
}
|
|
70638
70975
|
async plan(input) {
|
|
70639
|
-
return (await this.computePlan(input.headroom, input
|
|
70976
|
+
return (await this.computePlan(input.headroom, minSessionsOf(input))).plan;
|
|
70640
70977
|
}
|
|
70641
70978
|
/**
|
|
70642
|
-
* Shares proportional to each node's capacity
|
|
70643
|
-
*
|
|
70644
|
-
*
|
|
70645
|
-
*
|
|
70646
|
-
*
|
|
70979
|
+
* Shares proportional to each node's capacity in parallel sessions (D755):
|
|
70980
|
+
* from the latency table at the models Auto tune picks at its defaults
|
|
70981
|
+
* (`hardware`), or from node benchmark results (`benchmark`), a node
|
|
70982
|
+
* without results falling back to `hardware` and saying so. ONE pass, and
|
|
70983
|
+
* nothing but capacity goes in — the automatic shares are exactly this.
|
|
70984
|
+
* Per device it reports an ADVISORY session cap — never written, never
|
|
70985
|
+
* enforced.
|
|
70647
70986
|
*/
|
|
70648
70987
|
async suggestShares(input) {
|
|
70649
|
-
|
|
70650
|
-
|
|
70651
|
-
|
|
70652
|
-
|
|
70653
|
-
|
|
70654
|
-
|
|
70655
|
-
|
|
70656
|
-
|
|
70657
|
-
rounds = round;
|
|
70658
|
-
const rec = await this.recommend(input.headroom, shares);
|
|
70659
|
-
cameraFps = rec.cameraFps;
|
|
70660
|
-
result = this.sharesFrom(rec, input.headroom);
|
|
70661
|
-
const next = {
|
|
70662
|
-
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70663
|
-
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70664
|
-
};
|
|
70665
|
-
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70666
|
-
shares = next;
|
|
70667
|
-
if (settled) break;
|
|
70988
|
+
const source = input.source ?? "hardware";
|
|
70989
|
+
const minSessionsPerDevice = 2;
|
|
70990
|
+
const rec = await this.recommend(input.headroom, minSessionsPerDevice);
|
|
70991
|
+
let measured = /* @__PURE__ */ new Map();
|
|
70992
|
+
if (source === "benchmark") try {
|
|
70993
|
+
measured = await this.deps.measuredCapacity.readMeasuredCapacity(rec.recommendations.map((r) => r.nodeId));
|
|
70994
|
+
} catch (err) {
|
|
70995
|
+
this.deps.logger.warn("load balancing: benchmark results unreadable — sized from hardware", { meta: { error: errMsg(err) } });
|
|
70668
70996
|
}
|
|
70669
|
-
return {
|
|
70997
|
+
return computeCapacityShares({
|
|
70998
|
+
recommendations: rec.recommendations,
|
|
70999
|
+
source,
|
|
71000
|
+
measured,
|
|
70670
71001
|
headroom: input.headroom,
|
|
70671
|
-
cameraFps,
|
|
70672
|
-
|
|
70673
|
-
nodes: [...result]
|
|
70674
|
-
};
|
|
70675
|
-
}
|
|
70676
|
-
sharesFrom(rec, headroom) {
|
|
70677
|
-
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70678
|
-
const perNode = rec.recommendations.map((r) => {
|
|
70679
|
-
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70680
|
-
const fps = d.chosen?.fps ?? 0;
|
|
70681
|
-
return {
|
|
70682
|
-
d,
|
|
70683
|
-
fps,
|
|
70684
|
-
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70685
|
-
};
|
|
70686
|
-
});
|
|
70687
|
-
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70688
|
-
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70689
|
-
return {
|
|
70690
|
-
r,
|
|
70691
|
-
devices,
|
|
70692
|
-
capacity
|
|
70693
|
-
};
|
|
70694
|
-
});
|
|
70695
|
-
const nodePct = toPercent(capacityByNode);
|
|
70696
|
-
return perNode.map(({ r, devices, capacity }) => {
|
|
70697
|
-
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70698
|
-
return {
|
|
70699
|
-
nodeId: r.nodeId,
|
|
70700
|
-
share: nodePct.get(r.nodeId) ?? null,
|
|
70701
|
-
capacitySessions: capacity,
|
|
70702
|
-
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70703
|
-
devices: devices.map((x) => ({
|
|
70704
|
-
deviceKey: x.d.deviceKey,
|
|
70705
|
-
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70706
|
-
modelId: x.d.chosen?.modelId ?? null,
|
|
70707
|
-
fps: x.d.chosen?.fps ?? null,
|
|
70708
|
-
measured: x.d.chosen?.measured ?? false,
|
|
70709
|
-
suggestedMaxSessions: x.sessions
|
|
70710
|
-
}))
|
|
70711
|
-
};
|
|
71002
|
+
cameraFps: rec.cameraFps,
|
|
71003
|
+
minSessionsPerDevice
|
|
70712
71004
|
});
|
|
70713
71005
|
}
|
|
70714
|
-
/**
|
|
70715
|
-
|
|
70716
|
-
* round) forces share placement with those shares overlaid on the stored
|
|
70717
|
-
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70718
|
-
*/
|
|
70719
|
-
async recommend(headroom, shares) {
|
|
71006
|
+
/** Gather every node and recommend. Nothing but hardware, catalog and the sizing goes in. */
|
|
71007
|
+
async recommend(headroom, minSessionsPerDevice) {
|
|
70720
71008
|
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70721
71009
|
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70722
71010
|
const [devices, hardware, catalog] = await Promise.all([
|
|
@@ -70731,7 +71019,6 @@ var BestSettingsService = class {
|
|
|
70731
71019
|
catalog
|
|
70732
71020
|
};
|
|
70733
71021
|
}));
|
|
70734
|
-
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? stored;
|
|
70735
71022
|
const nodes = gathered.map((g) => ({
|
|
70736
71023
|
nodeId: g.nodeId,
|
|
70737
71024
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70744,40 +71031,22 @@ var BestSettingsService = class {
|
|
|
70744
71031
|
})),
|
|
70745
71032
|
formatsByModel: formatsByModelOf(g.catalog)
|
|
70746
71033
|
}));
|
|
70747
|
-
const
|
|
70748
|
-
|
|
70749
|
-
deviceKey: d.key,
|
|
70750
|
-
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70751
|
-
})) : []);
|
|
70752
|
-
const cameras = await this.deps.cameraLoads();
|
|
70753
|
-
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70754
|
-
const demand = estimateDeviceDemand({
|
|
70755
|
-
cameras,
|
|
70756
|
-
eligible,
|
|
70757
|
-
peaks: this.deps.peaks(),
|
|
70758
|
-
...policy === "share" ? { shares: {
|
|
70759
|
-
nodes: {
|
|
70760
|
-
...await this.deps.readNodeShares(),
|
|
70761
|
-
...shares?.nodes
|
|
70762
|
-
},
|
|
70763
|
-
clusterPeak: this.deps.clusterPeak()
|
|
70764
|
-
} } : {}
|
|
70765
|
-
});
|
|
70766
|
-
const recommendations = nodes.map((node) => recommendNode({
|
|
70767
|
-
node,
|
|
70768
|
-
demand,
|
|
70769
|
-
headroom
|
|
70770
|
-
}));
|
|
70771
|
-
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
71034
|
+
const observedFps = await this.deps.cameraFps();
|
|
71035
|
+
const cameraFps = observedFps !== null && observedFps > 0 ? observedFps : 10;
|
|
70772
71036
|
return {
|
|
70773
71037
|
gathered,
|
|
70774
71038
|
nodes,
|
|
70775
|
-
recommendations
|
|
70776
|
-
|
|
71039
|
+
recommendations: nodes.map((node) => recommendNode({
|
|
71040
|
+
node,
|
|
71041
|
+
headroom,
|
|
71042
|
+
minSessionsPerDevice,
|
|
71043
|
+
cameraFps
|
|
71044
|
+
})),
|
|
71045
|
+
cameraFps
|
|
70777
71046
|
};
|
|
70778
71047
|
}
|
|
70779
|
-
async computePlan(headroom,
|
|
70780
|
-
const { gathered, nodes, recommendations } = await this.recommend(headroom,
|
|
71048
|
+
async computePlan(headroom, minSessionsPerDevice) {
|
|
71049
|
+
const { gathered, nodes, recommendations, cameraFps } = await this.recommend(headroom, minSessionsPerDevice);
|
|
70781
71050
|
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70782
71051
|
const targets = /* @__PURE__ */ new Map();
|
|
70783
71052
|
const formats = /* @__PURE__ */ new Map();
|
|
@@ -70804,8 +71073,8 @@ var BestSettingsService = class {
|
|
|
70804
71073
|
targets,
|
|
70805
71074
|
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70806
71075
|
};
|
|
70807
|
-
const nodeChanges =
|
|
70808
|
-
const cameraClears =
|
|
71076
|
+
const nodeChanges = applyNodeAutoTune(await this.deps.readAgentSettings(), diffInput).changes;
|
|
71077
|
+
const cameraClears = applyCameraAutoTune(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70809
71078
|
const warnings = [];
|
|
70810
71079
|
let detectorStoreClears = [];
|
|
70811
71080
|
if (touchedNodes.size > 0) try {
|
|
@@ -70837,9 +71106,10 @@ var BestSettingsService = class {
|
|
|
70837
71106
|
return {
|
|
70838
71107
|
plan: {
|
|
70839
71108
|
planHash: createHash("sha256").update(canonical({
|
|
70840
|
-
version:
|
|
71109
|
+
version: 2,
|
|
70841
71110
|
headroom,
|
|
70842
|
-
|
|
71111
|
+
minSessionsPerDevice,
|
|
71112
|
+
cameraFps,
|
|
70843
71113
|
picks: recommendations.map((r) => ({
|
|
70844
71114
|
nodeId: r.nodeId,
|
|
70845
71115
|
skipped: r.skipped,
|
|
@@ -70851,7 +71121,8 @@ var BestSettingsService = class {
|
|
|
70851
71121
|
})).digest("hex"),
|
|
70852
71122
|
family: "yolov9",
|
|
70853
71123
|
headroom,
|
|
70854
|
-
|
|
71124
|
+
minSessionsPerDevice,
|
|
71125
|
+
cameraFps,
|
|
70855
71126
|
generatedAt: this.deps.now(),
|
|
70856
71127
|
nodes: recommendations,
|
|
70857
71128
|
nodeChanges: [...nodeChanges],
|
|
@@ -70876,9 +71147,9 @@ var BestSettingsService = class {
|
|
|
70876
71147
|
}
|
|
70877
71148
|
}
|
|
70878
71149
|
async applyLocked(input) {
|
|
70879
|
-
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
71150
|
+
const { plan, diffInput, formats } = await this.computePlan(input.headroom, minSessionsOf(input));
|
|
70880
71151
|
if (plan.planHash !== input.planHash) {
|
|
70881
|
-
this.deps.logger.warn("
|
|
71152
|
+
this.deps.logger.warn("auto tune: stale plan refused", { meta: {
|
|
70882
71153
|
confirmed: input.planHash,
|
|
70883
71154
|
current: plan.planHash
|
|
70884
71155
|
} });
|
|
@@ -70910,15 +71181,15 @@ var BestSettingsService = class {
|
|
|
70910
71181
|
ok: false,
|
|
70911
71182
|
detail: errMsg(err)
|
|
70912
71183
|
});
|
|
70913
|
-
this.deps.logger.warn("
|
|
71184
|
+
this.deps.logger.warn("auto tune: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70914
71185
|
return {
|
|
70915
71186
|
kind: "aborted",
|
|
70916
71187
|
phases
|
|
70917
71188
|
};
|
|
70918
71189
|
}
|
|
70919
71190
|
try {
|
|
70920
|
-
await this.deps.rewriteCameraSettings((prev) =>
|
|
70921
|
-
for (const c of plan.cameraClears) this.deps.logger.info("
|
|
71191
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraAutoTune(prev, diffInput).next);
|
|
71192
|
+
for (const c of plan.cameraClears) this.deps.logger.info("auto tune: camera override cleared", {
|
|
70922
71193
|
tags: {
|
|
70923
71194
|
deviceId: c.deviceId,
|
|
70924
71195
|
nodeId: c.nodeId
|
|
@@ -70962,8 +71233,8 @@ var BestSettingsService = class {
|
|
|
70962
71233
|
});
|
|
70963
71234
|
}
|
|
70964
71235
|
try {
|
|
70965
|
-
await this.deps.rewriteAgentSettings((prev) =>
|
|
70966
|
-
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("
|
|
71236
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeAutoTune(prev, diffInput).next);
|
|
71237
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("auto tune: node model set", {
|
|
70967
71238
|
tags: { nodeId: c.nodeId },
|
|
70968
71239
|
meta: {
|
|
70969
71240
|
nodeId: c.nodeId,
|
|
@@ -70997,14 +71268,14 @@ var BestSettingsService = class {
|
|
|
70997
71268
|
ok: failed.length === 0,
|
|
70998
71269
|
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
70999
71270
|
});
|
|
71000
|
-
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("
|
|
71271
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("auto tune: node skipped", {
|
|
71001
71272
|
tags: { nodeId: rec.nodeId },
|
|
71002
71273
|
meta: {
|
|
71003
71274
|
nodeId: rec.nodeId,
|
|
71004
71275
|
reasons: rec.reasons
|
|
71005
71276
|
}
|
|
71006
71277
|
});
|
|
71007
|
-
this.deps.logger.info("
|
|
71278
|
+
this.deps.logger.info("auto tune: applied", { meta: {
|
|
71008
71279
|
planHash: plan.planHash,
|
|
71009
71280
|
nodeChanges: plan.nodeChanges.length,
|
|
71010
71281
|
cameraClears: plan.cameraClears.length,
|
|
@@ -71019,83 +71290,6 @@ var BestSettingsService = class {
|
|
|
71019
71290
|
}
|
|
71020
71291
|
};
|
|
71021
71292
|
//#endregion
|
|
71022
|
-
//#region src/best-settings/device-peak-tracker.ts
|
|
71023
|
-
var BUCKET_MS = 3600 * 1e3;
|
|
71024
|
-
var DevicePeakTracker = class DevicePeakTracker {
|
|
71025
|
-
now;
|
|
71026
|
-
series = /* @__PURE__ */ new Map();
|
|
71027
|
-
cluster = null;
|
|
71028
|
-
constructor(now = () => Date.now()) {
|
|
71029
|
-
this.now = now;
|
|
71030
|
-
}
|
|
71031
|
-
static key(nodeId, deviceKey) {
|
|
71032
|
-
return `${nodeId}\u0000${deviceKey}`;
|
|
71033
|
-
}
|
|
71034
|
-
/** Fold one runner census in. A runner that did not answer is simply absent. */
|
|
71035
|
-
noteCensus(loads) {
|
|
71036
|
-
const now = this.now();
|
|
71037
|
-
const bucket = now - now % BUCKET_MS;
|
|
71038
|
-
let clusterTotal = 0;
|
|
71039
|
-
for (const load of loads) {
|
|
71040
|
-
const rows = load.devices.length > 0 ? load.devices : [{
|
|
71041
|
-
deviceKey: "",
|
|
71042
|
-
attachedCameras: load.attachedCameras
|
|
71043
|
-
}];
|
|
71044
|
-
for (const device of rows) {
|
|
71045
|
-
clusterTotal += device.attachedCameras;
|
|
71046
|
-
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
71047
|
-
let s = this.series.get(key);
|
|
71048
|
-
if (s === void 0) {
|
|
71049
|
-
s = {
|
|
71050
|
-
firstSeenAt: now,
|
|
71051
|
-
buckets: /* @__PURE__ */ new Map()
|
|
71052
|
-
};
|
|
71053
|
-
this.series.set(key, s);
|
|
71054
|
-
}
|
|
71055
|
-
s.buckets.set(bucket, Math.max(s.buckets.get(bucket) ?? 0, device.attachedCameras));
|
|
71056
|
-
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
71057
|
-
}
|
|
71058
|
-
}
|
|
71059
|
-
if (loads.length === 0) return;
|
|
71060
|
-
const c = this.cluster ?? {
|
|
71061
|
-
firstSeenAt: now,
|
|
71062
|
-
buckets: /* @__PURE__ */ new Map()
|
|
71063
|
-
};
|
|
71064
|
-
this.cluster = c;
|
|
71065
|
-
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71066
|
-
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71067
|
-
}
|
|
71068
|
-
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71069
|
-
clusterPeak() {
|
|
71070
|
-
const c = this.cluster;
|
|
71071
|
-
if (c === null) return null;
|
|
71072
|
-
const now = this.now();
|
|
71073
|
-
let peak = 0;
|
|
71074
|
-
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71075
|
-
return {
|
|
71076
|
-
peakSessions: peak,
|
|
71077
|
-
observedForMs: now - c.firstSeenAt
|
|
71078
|
-
};
|
|
71079
|
-
}
|
|
71080
|
-
/** Every observed device's peak over the window and how long it has been watched. */
|
|
71081
|
-
peaks() {
|
|
71082
|
-
const now = this.now();
|
|
71083
|
-
const out = [];
|
|
71084
|
-
for (const [key, s] of this.series) {
|
|
71085
|
-
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71086
|
-
let peak = 0;
|
|
71087
|
-
for (const [start, value] of s.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71088
|
-
out.push({
|
|
71089
|
-
nodeId,
|
|
71090
|
-
deviceKey,
|
|
71091
|
-
peakSessions: peak,
|
|
71092
|
-
observedForMs: now - s.firstSeenAt
|
|
71093
|
-
});
|
|
71094
|
-
}
|
|
71095
|
-
return out.toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
71096
|
-
}
|
|
71097
|
-
};
|
|
71098
|
-
//#endregion
|
|
71099
71293
|
//#region src/session-placement-stats.ts
|
|
71100
71294
|
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71101
71295
|
function keyOf(nodeId, deviceKey) {
|
|
@@ -71261,6 +71455,212 @@ function applyLoadBalancingShares(prev, write) {
|
|
|
71261
71455
|
return next;
|
|
71262
71456
|
}
|
|
71263
71457
|
//#endregion
|
|
71458
|
+
//#region src/load-balancing/effective-shares-service.ts
|
|
71459
|
+
/** Skips that mean "not readable yet", never "cannot take sessions". */
|
|
71460
|
+
var NOT_READY_REASONS = new Set(["node-offline", "no-hardware-facts"]);
|
|
71461
|
+
function capacityFrom(result) {
|
|
71462
|
+
const out = {};
|
|
71463
|
+
for (const node of result.nodes) if (node.capacitySource !== null) out[node.nodeId] = {
|
|
71464
|
+
source: node.capacitySource,
|
|
71465
|
+
fallback: node.capacityFallback
|
|
71466
|
+
};
|
|
71467
|
+
return out;
|
|
71468
|
+
}
|
|
71469
|
+
function weightsFrom(result) {
|
|
71470
|
+
const nodes = {};
|
|
71471
|
+
const devices = {};
|
|
71472
|
+
for (const node of result.nodes) {
|
|
71473
|
+
if (node.share !== null && node.share > 0) nodes[node.nodeId] = node.share;
|
|
71474
|
+
const perDevice = {};
|
|
71475
|
+
for (const d of node.devices) if (d.share > 0) perDevice[d.deviceKey] = d.share;
|
|
71476
|
+
if (Object.keys(perDevice).length > 0) devices[node.nodeId] = perDevice;
|
|
71477
|
+
}
|
|
71478
|
+
return {
|
|
71479
|
+
nodes,
|
|
71480
|
+
devices
|
|
71481
|
+
};
|
|
71482
|
+
}
|
|
71483
|
+
var EffectiveSharesService = class EffectiveSharesService {
|
|
71484
|
+
deps;
|
|
71485
|
+
/** An automatic answer older than this is recomputed by the reconcile. */
|
|
71486
|
+
static MAX_AGE_MS = 15 * 6e4;
|
|
71487
|
+
/** Reconcile cadence. */
|
|
71488
|
+
static RECONCILE_MS = 6e4;
|
|
71489
|
+
answer = null;
|
|
71490
|
+
inFlight = null;
|
|
71491
|
+
pendingTrigger = null;
|
|
71492
|
+
timer = null;
|
|
71493
|
+
constructor(deps) {
|
|
71494
|
+
this.deps = deps;
|
|
71495
|
+
}
|
|
71496
|
+
now() {
|
|
71497
|
+
return this.deps.now?.() ?? Date.now();
|
|
71498
|
+
}
|
|
71499
|
+
/** The last automatic answer for the CURRENT source; null = none yet. */
|
|
71500
|
+
current() {
|
|
71501
|
+
return this.usable();
|
|
71502
|
+
}
|
|
71503
|
+
/** The answer, when the source is automatic and the answer was computed for it. */
|
|
71504
|
+
usable() {
|
|
71505
|
+
const source = this.deps.readSource();
|
|
71506
|
+
if (!isAutomaticShareSource(source)) return null;
|
|
71507
|
+
return this.answer !== null && this.answer.source === source ? this.answer : null;
|
|
71508
|
+
}
|
|
71509
|
+
/** Node weights the election reads. */
|
|
71510
|
+
async nodeWeights() {
|
|
71511
|
+
const answer = this.usable();
|
|
71512
|
+
if (answer !== null) return { ...answer.weights.nodes };
|
|
71513
|
+
return { ...(await this.deps.readStored()).nodes };
|
|
71514
|
+
}
|
|
71515
|
+
/** The weights of a node's ELIGIBLE devices (`eligible` carries their stored weights). */
|
|
71516
|
+
deviceWeights(nodeId, eligible) {
|
|
71517
|
+
const answer = this.usable();
|
|
71518
|
+
if (answer === null) return { ...eligible };
|
|
71519
|
+
const auto = answer.weights.devices[nodeId] ?? {};
|
|
71520
|
+
const out = {};
|
|
71521
|
+
for (const [key, stored] of Object.entries(eligible)) {
|
|
71522
|
+
const share = auto[key];
|
|
71523
|
+
out[key] = share !== void 0 && share > 0 ? share : stored;
|
|
71524
|
+
}
|
|
71525
|
+
return out;
|
|
71526
|
+
}
|
|
71527
|
+
/** Both maps, as the Auto tune recommender charges load by them. */
|
|
71528
|
+
async effectiveWeights() {
|
|
71529
|
+
return this.usable()?.weights ?? this.deps.readStored();
|
|
71530
|
+
}
|
|
71531
|
+
/**
|
|
71532
|
+
* Recompute now (automatic sources only). Concurrent calls coalesce: one
|
|
71533
|
+
* runs, and one trailing run picks up whatever was reported meanwhile.
|
|
71534
|
+
*/
|
|
71535
|
+
async recompute(trigger) {
|
|
71536
|
+
if (!isAutomaticShareSource(this.deps.readSource())) return;
|
|
71537
|
+
if (this.inFlight !== null) {
|
|
71538
|
+
this.pendingTrigger = trigger;
|
|
71539
|
+
return this.inFlight;
|
|
71540
|
+
}
|
|
71541
|
+
this.inFlight = this.run(trigger).finally(() => {
|
|
71542
|
+
this.inFlight = null;
|
|
71543
|
+
});
|
|
71544
|
+
await this.inFlight;
|
|
71545
|
+
const trailing = this.pendingTrigger;
|
|
71546
|
+
if (trailing !== null) {
|
|
71547
|
+
this.pendingTrigger = null;
|
|
71548
|
+
await this.recompute(trailing);
|
|
71549
|
+
}
|
|
71550
|
+
}
|
|
71551
|
+
async run(trigger) {
|
|
71552
|
+
const source = this.deps.readSource();
|
|
71553
|
+
if (!isAutomaticShareSource(source)) return;
|
|
71554
|
+
let signature;
|
|
71555
|
+
let result;
|
|
71556
|
+
try {
|
|
71557
|
+
signature = await this.deps.topologySignature();
|
|
71558
|
+
result = await this.deps.suggest(source);
|
|
71559
|
+
} catch (err) {
|
|
71560
|
+
this.deps.logger.warn("load balancing: automatic shares not recomputed — last answer kept", { meta: {
|
|
71561
|
+
trigger,
|
|
71562
|
+
error: errMsg$2(err),
|
|
71563
|
+
hasAnswer: this.answer !== null
|
|
71564
|
+
} });
|
|
71565
|
+
return;
|
|
71566
|
+
}
|
|
71567
|
+
const before = this.answer?.weights ?? null;
|
|
71568
|
+
const after = weightsFrom(result);
|
|
71569
|
+
const notReady = result.nodes.filter((n) => n.share === null && NOT_READY_REASONS.has(n.skippedReason ?? "")).map((n) => n.nodeId).toSorted();
|
|
71570
|
+
const capacity = capacityFrom(result);
|
|
71571
|
+
this.answer = {
|
|
71572
|
+
weights: after,
|
|
71573
|
+
capacity,
|
|
71574
|
+
source,
|
|
71575
|
+
computedAt: this.now(),
|
|
71576
|
+
trigger,
|
|
71577
|
+
signature,
|
|
71578
|
+
notReady
|
|
71579
|
+
};
|
|
71580
|
+
const nodeIds = [...new Set([...Object.keys(before?.nodes ?? {}), ...result.nodes.map((n) => n.nodeId)])];
|
|
71581
|
+
const log = before !== null && JSON.stringify(before) === JSON.stringify(after) && trigger.startsWith("reconcile:not-ready") ? this.deps.logger.debug.bind(this.deps.logger) : this.deps.logger.info.bind(this.deps.logger);
|
|
71582
|
+
for (const nodeId of nodeIds.toSorted()) log("load balancing: automatic shares recomputed", {
|
|
71583
|
+
tags: { nodeId },
|
|
71584
|
+
meta: {
|
|
71585
|
+
trigger,
|
|
71586
|
+
source,
|
|
71587
|
+
capacitySource: capacity[nodeId]?.source ?? null,
|
|
71588
|
+
capacityFallback: capacity[nodeId]?.fallback ?? null,
|
|
71589
|
+
notReady: notReady.includes(nodeId),
|
|
71590
|
+
before: before?.nodes[nodeId] ?? null,
|
|
71591
|
+
after: after.nodes[nodeId] ?? null,
|
|
71592
|
+
devicesBefore: JSON.stringify(before?.devices[nodeId] ?? {}),
|
|
71593
|
+
devicesAfter: JSON.stringify(after.devices[nodeId] ?? {})
|
|
71594
|
+
}
|
|
71595
|
+
});
|
|
71596
|
+
}
|
|
71597
|
+
/**
|
|
71598
|
+
* Recompute if there is no answer for the current source, a node was not
|
|
71599
|
+
* readable yet, the topology changed or the answer has grown old
|
|
71600
|
+
* (automatic sources only).
|
|
71601
|
+
*/
|
|
71602
|
+
async reconcile() {
|
|
71603
|
+
const source = this.deps.readSource();
|
|
71604
|
+
if (!isAutomaticShareSource(source)) return;
|
|
71605
|
+
if (this.answer === null) return this.recompute("reconcile:first");
|
|
71606
|
+
if (this.answer.source !== source) return this.recompute(`reconcile:source:${source}`);
|
|
71607
|
+
if (this.answer.notReady.length > 0) return this.recompute(`reconcile:not-ready:${this.answer.notReady.join(",")}`);
|
|
71608
|
+
if (this.now() - this.answer.computedAt > EffectiveSharesService.MAX_AGE_MS) return this.recompute("reconcile:age");
|
|
71609
|
+
let signature;
|
|
71610
|
+
try {
|
|
71611
|
+
signature = await this.deps.topologySignature();
|
|
71612
|
+
} catch (err) {
|
|
71613
|
+
this.deps.logger.debug("load balancing: topology signature unreadable — reconcile skipped", { meta: { error: errMsg$2(err) } });
|
|
71614
|
+
return;
|
|
71615
|
+
}
|
|
71616
|
+
if (signature !== this.answer.signature) return this.recompute("reconcile:topology");
|
|
71617
|
+
}
|
|
71618
|
+
start() {
|
|
71619
|
+
if (this.timer !== null) return;
|
|
71620
|
+
this.timer = setInterval(() => {
|
|
71621
|
+
this.reconcile();
|
|
71622
|
+
}, EffectiveSharesService.RECONCILE_MS);
|
|
71623
|
+
this.timer.unref?.();
|
|
71624
|
+
}
|
|
71625
|
+
stop() {
|
|
71626
|
+
if (this.timer !== null) clearInterval(this.timer);
|
|
71627
|
+
this.timer = null;
|
|
71628
|
+
}
|
|
71629
|
+
};
|
|
71630
|
+
//#endregion
|
|
71631
|
+
//#region src/load-balancing/share-inputs.ts
|
|
71632
|
+
/** The operator's sliders: node `detectWeight` and device `weight` (positive only). */
|
|
71633
|
+
function storedShareWeights(blob) {
|
|
71634
|
+
const nodes = {};
|
|
71635
|
+
const devices = {};
|
|
71636
|
+
for (const [nodeId, settings] of Object.entries(blob)) {
|
|
71637
|
+
if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) nodes[nodeId] = settings.detectWeight;
|
|
71638
|
+
const perDevice = {};
|
|
71639
|
+
for (const [key, entry] of Object.entries(settings.inferenceDevices ?? {})) if (typeof entry.weight === "number" && entry.weight > 0) perDevice[key] = entry.weight;
|
|
71640
|
+
if (Object.keys(perDevice).length > 0) devices[nodeId] = perDevice;
|
|
71641
|
+
}
|
|
71642
|
+
return {
|
|
71643
|
+
nodes,
|
|
71644
|
+
devices
|
|
71645
|
+
};
|
|
71646
|
+
}
|
|
71647
|
+
/**
|
|
71648
|
+
* A string that changes whenever an input of the automatic shares does: the
|
|
71649
|
+
* nodes present, each node's detect capability, which devices are switched on
|
|
71650
|
+
* and which detector model each runs. Sliders and caps are left out — they do
|
|
71651
|
+
* not change capacity.
|
|
71652
|
+
*/
|
|
71653
|
+
function shareTopologySignature(nodeIds, blob) {
|
|
71654
|
+
return [...nodeIds].toSorted().map((nodeId) => {
|
|
71655
|
+
const settings = blob[nodeId];
|
|
71656
|
+
const devices = Object.entries(settings?.inferenceDevices ?? {}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => {
|
|
71657
|
+
const model = entry.steps?.["object-detection"]?.modelId ?? "";
|
|
71658
|
+
return `${key}=${entry.enabled ? "on" : "off"}:${model}`;
|
|
71659
|
+
});
|
|
71660
|
+
return `${nodeId}|detect=${String(settings?.detect ?? "default")}|${devices.join(",")}`;
|
|
71661
|
+
}).join(";");
|
|
71662
|
+
}
|
|
71663
|
+
//#endregion
|
|
71264
71664
|
//#region src/stale-device-pins.ts
|
|
71265
71665
|
function findStaleDevicePins(pins, nodes) {
|
|
71266
71666
|
const usable = /* @__PURE__ */ new Map();
|
|
@@ -71344,12 +71744,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71344
71744
|
*/
|
|
71345
71745
|
clusterModelChoiceReady = false;
|
|
71346
71746
|
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
71347
|
-
devicePeaks = new DevicePeakTracker();
|
|
71348
71747
|
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71349
71748
|
placementStats = new SessionPlacementStats();
|
|
71350
|
-
/** The stored session placement policy (D748
|
|
71351
|
-
readSessionPlacement = () =>
|
|
71352
|
-
|
|
71749
|
+
/** The stored session placement policy (D748, default `share` since D750). */
|
|
71750
|
+
readSessionPlacement = () => DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
71751
|
+
/** Where the shares come from (D750); `auto` before init. */
|
|
71752
|
+
readShareSource = () => DEFAULT_SHARE_SOURCE;
|
|
71753
|
+
autoTuneService = null;
|
|
71754
|
+
effectiveSharesService = null;
|
|
71353
71755
|
/**
|
|
71354
71756
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
71355
71757
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -71598,6 +72000,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71598
72000
|
getEligibleInferenceDevices: (nodeId, read) => this.resolveEligibleInferenceDevices(nodeId, read),
|
|
71599
72001
|
getEligibleInferenceDevicesIfCached: (nodeId) => this.resolveEligibleInferenceDevicesIfCached(nodeId),
|
|
71600
72002
|
forgetInferenceDeviceProbe: (nodeId) => this.forgetInferenceDeviceProbe(nodeId),
|
|
72003
|
+
noteShareTopologyChange: (trigger) => this.noteShareChange(trigger),
|
|
71601
72004
|
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
71602
72005
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
71603
72006
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
@@ -71626,7 +72029,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71626
72029
|
rebalance: (opts) => this.rebalance(opts),
|
|
71627
72030
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
71628
72031
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
71629
|
-
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
71630
72032
|
placementStats: this.placementStats,
|
|
71631
72033
|
setTopology: (topology) => {
|
|
71632
72034
|
this.topology = topology;
|
|
@@ -71655,6 +72057,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71655
72057
|
this.placement = controllers.placement;
|
|
71656
72058
|
this.session = controllers.session;
|
|
71657
72059
|
this.readSessionPlacement = controllers.sessionPlacement;
|
|
72060
|
+
this.readShareSource = controllers.shareSource;
|
|
72061
|
+
this.settingsStore.setEffectiveNodeWeights(() => this.effectiveShares().nodeWeights());
|
|
72062
|
+
this.effectiveShares().start();
|
|
72063
|
+
this.noteShareChange("startup");
|
|
71658
72064
|
this.deviceConfig = controllers.deviceConfig;
|
|
71659
72065
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
71660
72066
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71780,6 +72186,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71780
72186
|
this.detectionWiring = null;
|
|
71781
72187
|
this.unsubAgentOffline?.();
|
|
71782
72188
|
this.unsubAgentOffline = null;
|
|
72189
|
+
this.effectiveSharesService?.stop();
|
|
71783
72190
|
this.unsubAgentOnline?.();
|
|
71784
72191
|
this.unsubAgentOnline = null;
|
|
71785
72192
|
this.nodeLifecycle?.dispose();
|
|
@@ -72182,6 +72589,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72182
72589
|
await this.topology.refreshNodeCapabilities();
|
|
72183
72590
|
this.topology.addKnownRunnerNode(input.agentNodeId);
|
|
72184
72591
|
this.reconcile.schedulePendingRetry();
|
|
72592
|
+
if (input.detect !== void 0) this.noteShareChange(`node-detect:${input.agentNodeId}`);
|
|
72185
72593
|
return { success: true };
|
|
72186
72594
|
}
|
|
72187
72595
|
/**
|
|
@@ -72271,6 +72679,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72271
72679
|
});
|
|
72272
72680
|
await this.resetRemovedModelPins("node-device-write");
|
|
72273
72681
|
this.reconcile.schedulePendingRetry();
|
|
72682
|
+
this.noteShareChange(`node-devices:${input.agentNodeId}`);
|
|
72274
72683
|
return { success: true };
|
|
72275
72684
|
}
|
|
72276
72685
|
/**
|
|
@@ -72351,7 +72760,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72351
72760
|
});
|
|
72352
72761
|
throw new Error(`inference device eligibility: node "${nodeId}" catalog not answered`);
|
|
72353
72762
|
}
|
|
72354
|
-
return this.electableDevices(nodeId, probed, catalog);
|
|
72763
|
+
return this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, probed, catalog));
|
|
72355
72764
|
}
|
|
72356
72765
|
/**
|
|
72357
72766
|
* Forget a node's last device probe (pre-ship review, M4). Called when the
|
|
@@ -72395,7 +72804,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72395
72804
|
this.refreshInferenceDeviceProbe(nodeId);
|
|
72396
72805
|
return {
|
|
72397
72806
|
kind: "eligible",
|
|
72398
|
-
eligible: await this.electableDevices(nodeId, known, catalog)
|
|
72807
|
+
eligible: this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, known, catalog))
|
|
72399
72808
|
};
|
|
72400
72809
|
}
|
|
72401
72810
|
/**
|
|
@@ -72725,10 +73134,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72725
73134
|
clearedCameraOverrides
|
|
72726
73135
|
};
|
|
72727
73136
|
}
|
|
72728
|
-
|
|
72729
|
-
if (this.
|
|
73137
|
+
autoTune() {
|
|
73138
|
+
if (this.autoTuneService !== null) return this.autoTuneService;
|
|
72730
73139
|
const store = this.settingsStore;
|
|
72731
|
-
this.
|
|
73140
|
+
this.autoTuneService = new AutoTuneService({
|
|
72732
73141
|
logger: this.ctx.logger,
|
|
72733
73142
|
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72734
73143
|
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
@@ -72742,11 +73151,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72742
73151
|
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72743
73152
|
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72744
73153
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72745
|
-
|
|
72746
|
-
|
|
72747
|
-
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72748
|
-
sessionPlacement: () => this.readSessionPlacement(),
|
|
72749
|
-
readNodeShares: () => store.buildNodeWeights(),
|
|
73154
|
+
cameraFps: async () => this.autoTuneCameraFps(),
|
|
73155
|
+
measuredCapacity: NO_MEASURED_CAPACITY,
|
|
72750
73156
|
detectorStore: async (input) => {
|
|
72751
73157
|
const api = this.ctx.api;
|
|
72752
73158
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72769,33 +73175,23 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72769
73175
|
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72770
73176
|
now: () => Date.now()
|
|
72771
73177
|
});
|
|
72772
|
-
return this.
|
|
72773
|
-
}
|
|
72774
|
-
/**
|
|
72775
|
-
|
|
72776
|
-
|
|
72777
|
-
|
|
72778
|
-
|
|
72779
|
-
|
|
72780
|
-
|
|
72781
|
-
|
|
72782
|
-
|
|
72783
|
-
|
|
72784
|
-
mode: config.detectionMode,
|
|
72785
|
-
fps: config.detectionFps,
|
|
72786
|
-
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72787
|
-
standingNodeId: standingNode,
|
|
72788
|
-
nodePinId: nodePin,
|
|
72789
|
-
devicePin: pinnedKey !== null ? {
|
|
72790
|
-
nodeId: nodePin ?? standingNode,
|
|
72791
|
-
deviceKey: pinnedKey
|
|
72792
|
-
} : null
|
|
72793
|
-
});
|
|
73178
|
+
return this.autoTuneService;
|
|
73179
|
+
}
|
|
73180
|
+
/**
|
|
73181
|
+
* The rate one detection session is counted at (D755): the fastest camera
|
|
73182
|
+
* with detection started that runs the pipeline root; null when none does.
|
|
73183
|
+
*/
|
|
73184
|
+
autoTuneCameraFps() {
|
|
73185
|
+
let fps = 0;
|
|
73186
|
+
for (const config of this.detectionWiring.activeDetections.values()) {
|
|
73187
|
+
if (config.detectionMode === "disabled") continue;
|
|
73188
|
+
if (!(config.detectionSources ?? ["pipeline"]).includes("pipeline")) continue;
|
|
73189
|
+
fps = Math.max(fps, config.detectionFps);
|
|
72794
73190
|
}
|
|
72795
|
-
return
|
|
73191
|
+
return fps > 0 ? fps : null;
|
|
72796
73192
|
}
|
|
72797
|
-
async
|
|
72798
|
-
return this.
|
|
73193
|
+
async planAutoTune(input) {
|
|
73194
|
+
return this.autoTune().plan(input);
|
|
72799
73195
|
}
|
|
72800
73196
|
/**
|
|
72801
73197
|
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
@@ -72851,10 +73247,79 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72851
73247
|
return findStaleDevicePins(pins, offered);
|
|
72852
73248
|
}
|
|
72853
73249
|
async suggestShares(input) {
|
|
72854
|
-
return this.
|
|
73250
|
+
return this.autoTune().suggestShares(input);
|
|
72855
73251
|
}
|
|
72856
|
-
async
|
|
72857
|
-
|
|
73252
|
+
async applyAutoTune(input) {
|
|
73253
|
+
const result = await this.autoTune().apply(input);
|
|
73254
|
+
if (result.kind === "applied") this.noteShareChange("auto-tune-applied");
|
|
73255
|
+
return result;
|
|
73256
|
+
}
|
|
73257
|
+
effectiveShares() {
|
|
73258
|
+
if (this.effectiveSharesService !== null) return this.effectiveSharesService;
|
|
73259
|
+
const store = this.settingsStore;
|
|
73260
|
+
this.effectiveSharesService = new EffectiveSharesService({
|
|
73261
|
+
readSource: () => this.readShareSource(),
|
|
73262
|
+
readStored: async () => storedShareWeights(await store.readAgentSettingsMap()),
|
|
73263
|
+
suggest: (source) => this.autoTune().suggestShares({
|
|
73264
|
+
headroom: AUTO_SHARE_HEADROOM,
|
|
73265
|
+
source
|
|
73266
|
+
}),
|
|
73267
|
+
topologySignature: async () => shareTopologySignature([...new Set(["hub", ...this.topology.collectRunnerNodeIds()])], await store.readAgentSettingsMap()),
|
|
73268
|
+
logger: this.ctx.logger
|
|
73269
|
+
});
|
|
73270
|
+
return this.effectiveSharesService;
|
|
73271
|
+
}
|
|
73272
|
+
/** Something the automatic shares depend on changed: recompute now (best effort). */
|
|
73273
|
+
noteShareChange(trigger) {
|
|
73274
|
+
if (this.settingsStore === null) return;
|
|
73275
|
+
this.effectiveShares().recompute(trigger).catch((err) => {
|
|
73276
|
+
this.ctx.logger.warn("load balancing: share recompute failed", { meta: {
|
|
73277
|
+
trigger,
|
|
73278
|
+
error: errMsg$2(err)
|
|
73279
|
+
} });
|
|
73280
|
+
});
|
|
73281
|
+
}
|
|
73282
|
+
async getEffectiveShares() {
|
|
73283
|
+
const service = this.effectiveShares();
|
|
73284
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])].toSorted();
|
|
73285
|
+
const rows = await Promise.all(nodeIds.map(async (nodeId) => ({
|
|
73286
|
+
nodeId,
|
|
73287
|
+
read: await this.getNodeInferenceDevices({ nodeId }).catch(() => null)
|
|
73288
|
+
})));
|
|
73289
|
+
const detectable = new Set(this.topology.detectionEligibleNodes());
|
|
73290
|
+
const eligibleNodes = rows.filter((r) => r.read !== null && r.read.reachable && detectable.has(r.nodeId) && this.isNodeInferenceUsable(r.nodeId)).map((r) => r.nodeId);
|
|
73291
|
+
const eligibleDevices = {};
|
|
73292
|
+
for (const r of rows) {
|
|
73293
|
+
if (r.read === null) continue;
|
|
73294
|
+
eligibleDevices[r.nodeId] = r.read.devices.filter((d) => d.enabled && d.available && d.exclusion === null).map((d) => d.key);
|
|
73295
|
+
}
|
|
73296
|
+
const auto = service.current();
|
|
73297
|
+
return resolveEffectiveShares({
|
|
73298
|
+
source: this.readShareSource(),
|
|
73299
|
+
policy: this.readSessionPlacement(),
|
|
73300
|
+
nodeIds,
|
|
73301
|
+
stored: storedShareWeights(await this.settingsStore.readAgentSettingsMap()),
|
|
73302
|
+
auto: auto?.weights ?? null,
|
|
73303
|
+
...auto !== null ? { autoCapacity: auto.capacity } : {},
|
|
73304
|
+
eligibleNodes,
|
|
73305
|
+
eligibleDevices,
|
|
73306
|
+
computedAt: auto?.computedAt ?? null,
|
|
73307
|
+
trigger: auto?.trigger ?? null
|
|
73308
|
+
});
|
|
73309
|
+
}
|
|
73310
|
+
async setShareSource(input) {
|
|
73311
|
+
const previous = this.readShareSource();
|
|
73312
|
+
if (input.source === "manual" && isAutomaticShareSource(previous)) {
|
|
73313
|
+
const auto = this.effectiveShares().current();
|
|
73314
|
+
if (auto !== null) await this.setLoadBalancingShares(auto.weights);
|
|
73315
|
+
}
|
|
73316
|
+
await this.updateGlobalSettings({ [SHARE_SOURCE_SETTING_KEY]: input.source });
|
|
73317
|
+
this.ctx.logger.info("load balancing: share source changed", { meta: {
|
|
73318
|
+
from: previous,
|
|
73319
|
+
to: input.source
|
|
73320
|
+
} });
|
|
73321
|
+
if (isAutomaticShareSource(input.source)) this.noteShareChange(`source:${input.source}`);
|
|
73322
|
+
return { success: true };
|
|
72858
73323
|
}
|
|
72859
73324
|
async getCameraSettings(input) {
|
|
72860
73325
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|
|
@@ -72885,6 +73350,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72885
73350
|
await this.emitResolvedCameraUpdated(input.deviceId);
|
|
72886
73351
|
return { success: true };
|
|
72887
73352
|
}
|
|
73353
|
+
/**
|
|
73354
|
+
* D754 — write or clear one step's CAMERA-WIDE settings (field-level bag,
|
|
73355
|
+
* merged under the per-(node,device) override on every accelerator). No
|
|
73356
|
+
* deviceKey to validate: the layer is node-independent by construction.
|
|
73357
|
+
*/
|
|
73358
|
+
async setCameraStepSettings(input) {
|
|
73359
|
+
await this.settingsStore.writeCameraStepSettings(input.deviceId, input.addonId, input.settings);
|
|
73360
|
+
await this.emitResolvedCameraUpdated(input.deviceId);
|
|
73361
|
+
return { success: true };
|
|
73362
|
+
}
|
|
72888
73363
|
async setCameraPipelineForAgent(input) {
|
|
72889
73364
|
const existing = (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? {};
|
|
72890
73365
|
const nextByAgent = { ...existing.pipelineByAgent };
|