@camstack/addon-pipeline-orchestrator 1.2.291 → 1.2.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +21 -21
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CNnJE0gg.mjs} +3 -3
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CXr4GmWP.mjs +26 -0
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-Cu7Q-ifj.mjs +26 -0
- package/dist/{hostInit-BpNmAa0Q.mjs → hostInit-S3TibCek.mjs} +3 -3
- package/dist/index.js +1083 -608
- package/dist/index.mjs +1083 -608
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +0 -26
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs +0 -26
package/dist/index.js
CHANGED
|
@@ -5410,7 +5410,7 @@ var ZodIssueCode = {
|
|
|
5410
5410
|
var ZodFirstPartyTypeKind;
|
|
5411
5411
|
ZodFirstPartyTypeKind || (ZodFirstPartyTypeKind = {});
|
|
5412
5412
|
//#endregion
|
|
5413
|
-
//#region ../types/dist/sleep-
|
|
5413
|
+
//#region ../types/dist/sleep-DAJ9Er57.mjs
|
|
5414
5414
|
/**
|
|
5415
5415
|
* The audio chunk plane's byte format, and the ONE expansion from a coded
|
|
5416
5416
|
* window to float samples (D455).
|
|
@@ -24661,7 +24661,7 @@ var pipelineExecutorCapability = {
|
|
|
24661
24661
|
auth: "admin"
|
|
24662
24662
|
}),
|
|
24663
24663
|
/**
|
|
24664
|
-
*
|
|
24664
|
+
* Auto tune (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24665
24665
|
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24666
24666
|
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24667
24667
|
* running model's own defaults apply. Every other key of a camera's store
|
|
@@ -25033,111 +25033,17 @@ var pipelineExecutorCapability = {
|
|
|
25033
25033
|
}
|
|
25034
25034
|
};
|
|
25035
25035
|
/**
|
|
25036
|
-
*
|
|
25037
|
-
* policy, the atomic share write, the observed placement statistics, the
|
|
25038
|
-
* hardware share suggestion and the stale device pins.
|
|
25039
|
-
*/
|
|
25040
|
-
/**
|
|
25041
|
-
* How a new detection session picks its node and device.
|
|
25042
|
-
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25043
|
-
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25044
|
-
* weight only bites once a node already holds a session.
|
|
25045
|
-
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25046
|
-
* full or drowning node yields to a free one (D748).
|
|
25047
|
-
*/
|
|
25048
|
-
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25049
|
-
var DEFAULT_SESSION_PLACEMENT_POLICY = "least-loaded";
|
|
25050
|
-
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25051
|
-
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25052
|
-
/** Parse a stored value; anything unknown is the default (nothing changes on deploy). */
|
|
25053
|
-
function parseSessionPlacementPolicy(value) {
|
|
25054
|
-
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25055
|
-
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25056
|
-
}
|
|
25057
|
-
var LoadBalancingSharesInputSchema = object({
|
|
25058
|
-
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25059
|
-
nodes: record(string(), number().positive()).default({}),
|
|
25060
|
-
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25061
|
-
devices: record(string(), record(string(), number().positive())).default({})
|
|
25062
|
-
});
|
|
25063
|
-
var PlacementStatsInputSchema = object({
|
|
25064
|
-
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25065
|
-
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25066
|
-
var PlacementDeviceStatsSchema = object({
|
|
25067
|
-
deviceKey: string(),
|
|
25068
|
-
/** Sessions running on this device now. */
|
|
25069
|
-
sessionsNow: number(),
|
|
25070
|
-
/** Sessions started on this device inside the window. */
|
|
25071
|
-
sessionsStarted: number(),
|
|
25072
|
-
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25073
|
-
observedShare: number().nullable()
|
|
25074
|
-
});
|
|
25075
|
-
var PlacementNodeStatsSchema = object({
|
|
25076
|
-
nodeId: string(),
|
|
25077
|
-
sessionsNow: number(),
|
|
25078
|
-
sessionsStarted: number(),
|
|
25079
|
-
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25080
|
-
observedShare: number().nullable(),
|
|
25081
|
-
/** Most sessions this node ran at once inside the window. */
|
|
25082
|
-
peakSessions: number(),
|
|
25083
|
-
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25084
|
-
yieldsAway: number(),
|
|
25085
|
-
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25086
|
-
});
|
|
25087
|
-
/** A camera's device pin that the session path cannot honour. */
|
|
25088
|
-
var StaleDevicePinSchema = object({
|
|
25089
|
-
deviceId: number(),
|
|
25090
|
-
deviceKey: string(),
|
|
25091
|
-
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25092
|
-
reason: _enum(["disabled", "absent"])
|
|
25093
|
-
});
|
|
25094
|
-
var PlacementStatsSchema = object({
|
|
25095
|
-
policy: SessionPlacementPolicySchema,
|
|
25096
|
-
windowMs: number(),
|
|
25097
|
-
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25098
|
-
observedForMs: number(),
|
|
25099
|
-
sessionsNow: number(),
|
|
25100
|
-
/** Most parallel sessions across the cluster inside the window. */
|
|
25101
|
-
clusterPeakSessions: number(),
|
|
25102
|
-
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25103
|
-
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25104
|
-
});
|
|
25105
|
-
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25106
|
-
var SuggestedDeviceShareSchema = object({
|
|
25107
|
-
deviceKey: string(),
|
|
25108
|
-
/** Share of the node (0–100, rounded). */
|
|
25109
|
-
share: number(),
|
|
25110
|
-
modelId: string().nullable(),
|
|
25111
|
-
/** Inferences per second at that model, from the measured table. */
|
|
25112
|
-
fps: number().nullable(),
|
|
25113
|
-
measured: boolean(),
|
|
25114
|
-
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25115
|
-
suggestedMaxSessions: number().nullable()
|
|
25116
|
-
});
|
|
25117
|
-
var SuggestedNodeShareSchema = object({
|
|
25118
|
-
nodeId: string(),
|
|
25119
|
-
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25120
|
-
share: number().nullable(),
|
|
25121
|
-
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25122
|
-
capacitySessions: number(),
|
|
25123
|
-
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25124
|
-
skippedReason: string().nullable()
|
|
25125
|
-
});
|
|
25126
|
-
var SuggestSharesResultSchema = object({
|
|
25127
|
-
headroom: number(),
|
|
25128
|
-
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25129
|
-
cameraFps: number(),
|
|
25130
|
-
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25131
|
-
iterations: number(),
|
|
25132
|
-
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25133
|
-
});
|
|
25134
|
-
/**
|
|
25135
|
-
* The wire shapes of the Best settings plan and apply (D745), used by the
|
|
25036
|
+
* The wire shapes of the Auto tune plan and apply (D745), used by the
|
|
25136
25037
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25137
|
-
* (`recommend.ts
|
|
25038
|
+
* (`recommend.ts`) are the TS face of the same data; these
|
|
25138
25039
|
* schemas are what crosses the cap boundary.
|
|
25139
25040
|
*/
|
|
25140
25041
|
var ModelFormatSchema$1 = _enum(MODEL_FORMATS);
|
|
25042
|
+
/** The operator's range for it. */
|
|
25043
|
+
var MIN_SESSIONS_PER_DEVICE_RANGE = {
|
|
25044
|
+
min: 1,
|
|
25045
|
+
max: 8
|
|
25046
|
+
};
|
|
25141
25047
|
var HardwareClassIdSchema = _enum([
|
|
25142
25048
|
"intel-core-ultra",
|
|
25143
25049
|
"intel-n-series",
|
|
@@ -25155,8 +25061,9 @@ var AcceleratorKindSchema = _enum([
|
|
|
25155
25061
|
]);
|
|
25156
25062
|
var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
25157
25063
|
object({
|
|
25158
|
-
kind: literal("
|
|
25159
|
-
|
|
25064
|
+
kind: literal("capacity"),
|
|
25065
|
+
sessions: number(),
|
|
25066
|
+
minSessions: number()
|
|
25160
25067
|
}),
|
|
25161
25068
|
object({
|
|
25162
25069
|
kind: literal("tie-faster"),
|
|
@@ -25171,16 +25078,13 @@ var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
|
25171
25078
|
}),
|
|
25172
25079
|
object({
|
|
25173
25080
|
kind: literal("no-fit-lightest"),
|
|
25174
|
-
|
|
25081
|
+
sessions: number(),
|
|
25082
|
+
minSessions: number()
|
|
25175
25083
|
}),
|
|
25176
25084
|
object({
|
|
25177
25085
|
kind: literal("derived-latency"),
|
|
25178
25086
|
modelId: string()
|
|
25179
25087
|
}),
|
|
25180
|
-
object({
|
|
25181
|
-
kind: literal("estimated-load"),
|
|
25182
|
-
observedForMs: number().nullable()
|
|
25183
|
-
}),
|
|
25184
25088
|
object({ kind: literal("cpu-device") }),
|
|
25185
25089
|
object({ kind: literal("coral-untouched") }),
|
|
25186
25090
|
object({ kind: literal("device-disabled") }),
|
|
@@ -25196,28 +25100,13 @@ var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
|
25196
25100
|
format: ModelFormatSchema$1
|
|
25197
25101
|
})
|
|
25198
25102
|
]);
|
|
25199
|
-
var DeviceDemandSchema = object({
|
|
25200
|
-
nodeId: string(),
|
|
25201
|
-
deviceKey: string(),
|
|
25202
|
-
demandFps: number(),
|
|
25203
|
-
maxCameraFps: number(),
|
|
25204
|
-
alwaysOnCameras: number(),
|
|
25205
|
-
onMotionSessions: number(),
|
|
25206
|
-
onMotionSource: _enum([
|
|
25207
|
-
"observed-24h",
|
|
25208
|
-
"estimate",
|
|
25209
|
-
"none"
|
|
25210
|
-
]),
|
|
25211
|
-
observedForMs: number().nullable(),
|
|
25212
|
-
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25213
|
-
placement: SessionPlacementPolicySchema
|
|
25214
|
-
});
|
|
25215
25103
|
var DeviceModelChoiceSchema = object({
|
|
25216
25104
|
modelId: string(),
|
|
25217
25105
|
p50Ms: number(),
|
|
25218
25106
|
fps: number(),
|
|
25219
25107
|
measured: boolean(),
|
|
25220
|
-
|
|
25108
|
+
/** Parallel sessions at the camera rate with the headroom kept free: ⌊fps × (1 − headroom) / camera fps⌋. */
|
|
25109
|
+
capacitySessions: number(),
|
|
25221
25110
|
personRecall: number()
|
|
25222
25111
|
});
|
|
25223
25112
|
var DeviceRecommendationSchema = object({
|
|
@@ -25225,7 +25114,6 @@ var DeviceRecommendationSchema = object({
|
|
|
25225
25114
|
accelerator: AcceleratorKindSchema.nullable(),
|
|
25226
25115
|
enabled: boolean(),
|
|
25227
25116
|
available: boolean(),
|
|
25228
|
-
demand: DeviceDemandSchema.nullable(),
|
|
25229
25117
|
chosen: DeviceModelChoiceSchema.nullable(),
|
|
25230
25118
|
reasons: array(RecommendationReasonSchema).readonly()
|
|
25231
25119
|
});
|
|
@@ -25267,7 +25155,7 @@ var DetectorStoreClearSchema = object({
|
|
|
25267
25155
|
deviceId: number(),
|
|
25268
25156
|
keys: array(string()).readonly()
|
|
25269
25157
|
});
|
|
25270
|
-
var
|
|
25158
|
+
var AutoTuneWarningSchema = discriminatedUnion("kind", [
|
|
25271
25159
|
object({
|
|
25272
25160
|
kind: literal("detector-store-cluster-wide"),
|
|
25273
25161
|
skippedNodes: array(string()).readonly()
|
|
@@ -25282,36 +25170,37 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25282
25170
|
deviceKey: string()
|
|
25283
25171
|
})
|
|
25284
25172
|
]);
|
|
25285
|
-
var
|
|
25173
|
+
var AutoTuneOptionsSchema = object({
|
|
25286
25174
|
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25287
25175
|
headroom: number().min(.3).max(.5).default(.4),
|
|
25288
25176
|
/**
|
|
25289
|
-
*
|
|
25290
|
-
*
|
|
25291
|
-
*
|
|
25292
|
-
* is stored.
|
|
25177
|
+
* The least number of parallel sessions every device must sustain at the
|
|
25178
|
+
* camera rate with the headroom kept free (D755). The most accurate model
|
|
25179
|
+
* that reaches it wins.
|
|
25293
25180
|
*/
|
|
25294
|
-
|
|
25181
|
+
minSessionsPerDevice: number().int().min(MIN_SESSIONS_PER_DEVICE_RANGE.min).max(MIN_SESSIONS_PER_DEVICE_RANGE.max).default(2)
|
|
25295
25182
|
});
|
|
25296
|
-
var
|
|
25183
|
+
var AutoTunePlanSchema = object({
|
|
25297
25184
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25298
25185
|
planHash: string(),
|
|
25299
25186
|
family: literal("yolov9"),
|
|
25300
25187
|
headroom: number(),
|
|
25301
|
-
|
|
25302
|
-
|
|
25188
|
+
minSessionsPerDevice: number(),
|
|
25189
|
+
/** The detection rate a session is counted at (the fastest camera running the pipeline root). */
|
|
25190
|
+
cameraFps: number(),
|
|
25303
25191
|
generatedAt: number(),
|
|
25304
25192
|
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25305
25193
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25306
25194
|
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25307
25195
|
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25308
|
-
warnings: array(
|
|
25196
|
+
warnings: array(AutoTuneWarningSchema).readonly()
|
|
25309
25197
|
});
|
|
25310
|
-
var
|
|
25198
|
+
var AutoTuneApplyInputSchema = object({
|
|
25311
25199
|
planHash: string(),
|
|
25312
|
-
headroom: number().min(.3).max(.5).default(.4)
|
|
25200
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25201
|
+
minSessionsPerDevice: number().int().min(MIN_SESSIONS_PER_DEVICE_RANGE.min).max(MIN_SESSIONS_PER_DEVICE_RANGE.max).default(2)
|
|
25313
25202
|
});
|
|
25314
|
-
var
|
|
25203
|
+
var AutoTunePhaseResultSchema = object({
|
|
25315
25204
|
phase: _enum([
|
|
25316
25205
|
"download",
|
|
25317
25206
|
"camera-overrides",
|
|
@@ -25322,25 +25211,225 @@ var BestSettingsPhaseResultSchema = object({
|
|
|
25322
25211
|
ok: boolean(),
|
|
25323
25212
|
detail: string()
|
|
25324
25213
|
});
|
|
25325
|
-
var
|
|
25214
|
+
var AutoTuneApplyResultSchema = discriminatedUnion("kind", [
|
|
25326
25215
|
object({
|
|
25327
25216
|
kind: literal("applied"),
|
|
25328
25217
|
planHash: string(),
|
|
25329
|
-
phases: array(
|
|
25218
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25330
25219
|
}),
|
|
25331
25220
|
object({
|
|
25332
25221
|
kind: literal("aborted"),
|
|
25333
|
-
phases: array(
|
|
25222
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25334
25223
|
}),
|
|
25335
25224
|
object({
|
|
25336
25225
|
kind: literal("stale"),
|
|
25337
|
-
plan:
|
|
25226
|
+
plan: AutoTunePlanSchema
|
|
25338
25227
|
}),
|
|
25339
25228
|
object({
|
|
25340
25229
|
kind: literal("refused"),
|
|
25341
25230
|
reason: _enum(["apply-in-progress"])
|
|
25342
25231
|
})
|
|
25343
25232
|
]);
|
|
25233
|
+
/**
|
|
25234
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25235
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25236
|
+
* hardware share suggestion and the stale device pins.
|
|
25237
|
+
*/
|
|
25238
|
+
/**
|
|
25239
|
+
* How a new detection session picks its node and device.
|
|
25240
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25241
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25242
|
+
* weight only bites once a node already holds a session.
|
|
25243
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25244
|
+
* full or drowning node yields to a free one (D748).
|
|
25245
|
+
*/
|
|
25246
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25247
|
+
/**
|
|
25248
|
+
* `share` since D750: the percentages are held by default. `least-loaded` stays
|
|
25249
|
+
* selectable.
|
|
25250
|
+
*/
|
|
25251
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "share";
|
|
25252
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25253
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25254
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25255
|
+
function parseSessionPlacementPolicy(value) {
|
|
25256
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25257
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25258
|
+
}
|
|
25259
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25260
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25261
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25262
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25263
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25264
|
+
});
|
|
25265
|
+
var PlacementStatsInputSchema = object({
|
|
25266
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25267
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25268
|
+
var PlacementDeviceStatsSchema = object({
|
|
25269
|
+
deviceKey: string(),
|
|
25270
|
+
/** Sessions running on this device now. */
|
|
25271
|
+
sessionsNow: number(),
|
|
25272
|
+
/** Sessions started on this device inside the window. */
|
|
25273
|
+
sessionsStarted: number(),
|
|
25274
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25275
|
+
observedShare: number().nullable()
|
|
25276
|
+
});
|
|
25277
|
+
var PlacementNodeStatsSchema = object({
|
|
25278
|
+
nodeId: string(),
|
|
25279
|
+
sessionsNow: number(),
|
|
25280
|
+
sessionsStarted: number(),
|
|
25281
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25282
|
+
observedShare: number().nullable(),
|
|
25283
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25284
|
+
peakSessions: number(),
|
|
25285
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25286
|
+
yieldsAway: number(),
|
|
25287
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25288
|
+
});
|
|
25289
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25290
|
+
var StaleDevicePinSchema = object({
|
|
25291
|
+
deviceId: number(),
|
|
25292
|
+
deviceKey: string(),
|
|
25293
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25294
|
+
reason: _enum(["disabled", "absent"])
|
|
25295
|
+
});
|
|
25296
|
+
var PlacementStatsSchema = object({
|
|
25297
|
+
policy: SessionPlacementPolicySchema,
|
|
25298
|
+
windowMs: number(),
|
|
25299
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25300
|
+
observedForMs: number(),
|
|
25301
|
+
sessionsNow: number(),
|
|
25302
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25303
|
+
clusterPeakSessions: number(),
|
|
25304
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25305
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25306
|
+
});
|
|
25307
|
+
/**
|
|
25308
|
+
* Where an automatic share's capacity comes from (D755).
|
|
25309
|
+
* - `hardware` — the static capacity of each device at its Auto tune model:
|
|
25310
|
+
* the measured latency table for the node's hardware class.
|
|
25311
|
+
* - `benchmark` — the node's own measured capacity (node benchmark results).
|
|
25312
|
+
* A node with no results falls back to `hardware`, and says so.
|
|
25313
|
+
*/
|
|
25314
|
+
var CapacitySourceSchema = _enum(["hardware", "benchmark"]);
|
|
25315
|
+
/** Why a node under the `benchmark` source was sized from hardware instead. */
|
|
25316
|
+
var CapacityFallbackSchema = _enum(["no-benchmark-results", "partial-benchmark-results"]);
|
|
25317
|
+
var SuggestSharesInputSchema = object({
|
|
25318
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25319
|
+
/** Default `hardware`. */
|
|
25320
|
+
source: CapacitySourceSchema.default("hardware")
|
|
25321
|
+
});
|
|
25322
|
+
var SuggestedDeviceShareSchema = object({
|
|
25323
|
+
deviceKey: string(),
|
|
25324
|
+
/** Share of the node (0–100, rounded). */
|
|
25325
|
+
share: number(),
|
|
25326
|
+
modelId: string().nullable(),
|
|
25327
|
+
/** Inferences per second at that model, from the measured table; null when sized by a benchmark. */
|
|
25328
|
+
fps: number().nullable(),
|
|
25329
|
+
/** False for a table cell derived from another chip; true for a measured cell or a benchmark. */
|
|
25330
|
+
measured: boolean(),
|
|
25331
|
+
/**
|
|
25332
|
+
* Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋ from the table,
|
|
25333
|
+
* or the benchmark's measured max sessions. Never enforced.
|
|
25334
|
+
*/
|
|
25335
|
+
suggestedMaxSessions: number().nullable()
|
|
25336
|
+
});
|
|
25337
|
+
var SuggestedNodeShareSchema = object({
|
|
25338
|
+
nodeId: string(),
|
|
25339
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25340
|
+
share: number().nullable(),
|
|
25341
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25342
|
+
capacitySessions: number(),
|
|
25343
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25344
|
+
skippedReason: string().nullable(),
|
|
25345
|
+
/** What this node's capacity was read from; null when the node is skipped. */
|
|
25346
|
+
capacitySource: CapacitySourceSchema.nullable(),
|
|
25347
|
+
/** Set when the `benchmark` source was asked for and this node used `hardware`. */
|
|
25348
|
+
capacityFallback: CapacityFallbackSchema.nullable()
|
|
25349
|
+
});
|
|
25350
|
+
var SuggestSharesResultSchema = object({
|
|
25351
|
+
headroom: number(),
|
|
25352
|
+
/** The source asked for; each node says what it actually used. */
|
|
25353
|
+
source: CapacitySourceSchema,
|
|
25354
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25355
|
+
cameraFps: number(),
|
|
25356
|
+
/** The Auto tune capacity floor the models were picked at. */
|
|
25357
|
+
minSessionsPerDevice: number(),
|
|
25358
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25359
|
+
});
|
|
25360
|
+
/**
|
|
25361
|
+
* Where the node and device shares come from (D750, D755).
|
|
25362
|
+
* - `hardware` (default) — computed by the system from each node's static
|
|
25363
|
+
* capacity (its accelerators at their Auto tune models, from the measured
|
|
25364
|
+
* latency table), recomputed whenever a node or device joins, leaves, is
|
|
25365
|
+
* switched on or off, or changes model. No load input.
|
|
25366
|
+
* - `benchmark` — the same, from each node's measured capacity (node
|
|
25367
|
+
* benchmark results); a node without results falls back to `hardware`.
|
|
25368
|
+
* - `manual` — the operator's stored sliders.
|
|
25369
|
+
* Either way a node or device that cannot take sessions gives its share to
|
|
25370
|
+
* the others in proportion; the stored values are never rewritten for that.
|
|
25371
|
+
*/
|
|
25372
|
+
var ShareSourceSchema = _enum([
|
|
25373
|
+
"hardware",
|
|
25374
|
+
"benchmark",
|
|
25375
|
+
"manual"
|
|
25376
|
+
]);
|
|
25377
|
+
var DEFAULT_SHARE_SOURCE = "hardware";
|
|
25378
|
+
/** `hardware` or `benchmark`: the system computes the shares. */
|
|
25379
|
+
function isAutomaticShareSource(source) {
|
|
25380
|
+
return source !== "manual";
|
|
25381
|
+
}
|
|
25382
|
+
/** Headroom the automatic shares are computed at (Auto tune's default). */
|
|
25383
|
+
var AUTO_SHARE_HEADROOM = .4;
|
|
25384
|
+
/** The global-settings key the share source is stored under (orchestrator `load` section). */
|
|
25385
|
+
var SHARE_SOURCE_SETTING_KEY = "shareSource";
|
|
25386
|
+
/**
|
|
25387
|
+
* Parse a stored value; anything unknown is the default. `auto` (D750, before
|
|
25388
|
+
* D755 split it) reads as `hardware` — that is what it computed.
|
|
25389
|
+
*/
|
|
25390
|
+
function parseShareSource(value) {
|
|
25391
|
+
const parsed = ShareSourceSchema.safeParse(value);
|
|
25392
|
+
return parsed.success ? parsed.data : DEFAULT_SHARE_SOURCE;
|
|
25393
|
+
}
|
|
25394
|
+
var SetShareSourceInputSchema = object({ source: ShareSourceSchema });
|
|
25395
|
+
object({
|
|
25396
|
+
nodes: record(string(), number()),
|
|
25397
|
+
devices: record(string(), record(string(), number()))
|
|
25398
|
+
});
|
|
25399
|
+
var EffectiveDeviceShareSchema = object({
|
|
25400
|
+
deviceKey: string(),
|
|
25401
|
+
/** Can take sessions now (enabled, available, can run the root). */
|
|
25402
|
+
eligible: boolean(),
|
|
25403
|
+
/** The share the source assigns, as a percentage of the node's devices. */
|
|
25404
|
+
configuredPct: number(),
|
|
25405
|
+
/** The share actually used, after an ineligible device's share went to the others. */
|
|
25406
|
+
effectivePct: number()
|
|
25407
|
+
});
|
|
25408
|
+
var EffectiveNodeShareSchema = object({
|
|
25409
|
+
nodeId: string(),
|
|
25410
|
+
eligible: boolean(),
|
|
25411
|
+
configuredPct: number(),
|
|
25412
|
+
effectivePct: number(),
|
|
25413
|
+
/** Automatic sources: what this node's share was computed from; null under `manual` or before the first answer. */
|
|
25414
|
+
capacitySource: CapacitySourceSchema.nullable(),
|
|
25415
|
+
/** Automatic `benchmark`: why this node used `hardware` instead. */
|
|
25416
|
+
capacityFallback: CapacityFallbackSchema.nullable(),
|
|
25417
|
+
devices: array(EffectiveDeviceShareSchema).readonly()
|
|
25418
|
+
});
|
|
25419
|
+
var EffectiveSharesSchema = object({
|
|
25420
|
+
source: ShareSourceSchema,
|
|
25421
|
+
policy: SessionPlacementPolicySchema,
|
|
25422
|
+
/** An automatic source with no computed answer yet: the stored sliders are used meanwhile. */
|
|
25423
|
+
autoPending: boolean(),
|
|
25424
|
+
/** When the automatic shares were last computed (ms epoch); null = never. */
|
|
25425
|
+
computedAt: number().nullable(),
|
|
25426
|
+
/** What made the system recompute them last. */
|
|
25427
|
+
trigger: string().nullable(),
|
|
25428
|
+
nodes: array(EffectiveNodeShareSchema).readonly(),
|
|
25429
|
+
/** The relative weights the election reads (before eligibility). */
|
|
25430
|
+
nodeWeights: record(string(), number()),
|
|
25431
|
+
deviceWeights: record(string(), record(string(), number()))
|
|
25432
|
+
});
|
|
25344
25433
|
object({
|
|
25345
25434
|
activeCameras: number(),
|
|
25346
25435
|
throttledCameras: number(),
|
|
@@ -26748,8 +26837,10 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26748
26837
|
modelId: string().optional(),
|
|
26749
26838
|
settings: record(string(), unknown()).readonly().optional()
|
|
26750
26839
|
});
|
|
26840
|
+
var CameraStepSettingsSchema = record(string(), record(string(), unknown()).readonly());
|
|
26751
26841
|
var CameraPipelineSettingsSchema = object({
|
|
26752
26842
|
stepToggles: record(string(), boolean()).optional(),
|
|
26843
|
+
stepSettings: CameraStepSettingsSchema.optional(),
|
|
26753
26844
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26754
26845
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
26755
26846
|
});
|
|
@@ -27515,23 +27606,25 @@ var pipelineOrchestratorCapability = {
|
|
|
27515
27606
|
auth: "admin"
|
|
27516
27607
|
}),
|
|
27517
27608
|
/**
|
|
27518
|
-
*
|
|
27519
|
-
* for the whole cluster from each node's static hardware class
|
|
27520
|
-
* measured latency table
|
|
27609
|
+
* Auto tune (D745, D755) — DRY RUN. Recommends ONE detector family
|
|
27610
|
+
* (YOLOv9) for the whole cluster from each node's static hardware class
|
|
27611
|
+
* and the measured latency table: per device, the most accurate model
|
|
27612
|
+
* that sustains at least `minSessionsPerDevice` parallel sessions at the
|
|
27613
|
+
* camera rate with the headroom kept free (no load input), and lists every write
|
|
27521
27614
|
* the apply would make: the node model pins set, the enrichment pins and
|
|
27522
27615
|
* model-bound thresholds removed (node level and per camera), and the
|
|
27523
27616
|
* per-camera detector thresholds cleared. Writes nothing. Step
|
|
27524
27617
|
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27525
27618
|
* rules are never part of it. `planHash` names exactly this diff.
|
|
27526
27619
|
*/
|
|
27527
|
-
|
|
27620
|
+
planAutoTune: method(AutoTuneOptionsSchema, AutoTunePlanSchema, { auth: "admin" }),
|
|
27528
27621
|
/**
|
|
27529
|
-
*
|
|
27622
|
+
* Auto tune (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27530
27623
|
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27531
27624
|
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27532
27625
|
* nodes are skipped; there is no undo.
|
|
27533
27626
|
*/
|
|
27534
|
-
|
|
27627
|
+
applyAutoTune: method(AutoTuneApplyInputSchema, AutoTuneApplyResultSchema, {
|
|
27535
27628
|
kind: "mutation",
|
|
27536
27629
|
auth: "admin"
|
|
27537
27630
|
}),
|
|
@@ -27556,12 +27649,32 @@ var pipelineOrchestratorCapability = {
|
|
|
27556
27649
|
*/
|
|
27557
27650
|
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27558
27651
|
/**
|
|
27559
|
-
* Load balancing (D748) — shares proportional to each node's
|
|
27560
|
-
* capacity
|
|
27561
|
-
*
|
|
27562
|
-
*
|
|
27652
|
+
* Load balancing (D748, D755) — shares proportional to each node's
|
|
27653
|
+
* capacity in parallel sessions: from the latency table at the models
|
|
27654
|
+
* Auto tune picks (`hardware`), or from node benchmark results
|
|
27655
|
+
* (`benchmark`, each node without results falling back to `hardware` and
|
|
27656
|
+
* saying so). The automatic shares are exactly this answer. Per device,
|
|
27657
|
+
* an ADVISORY session cap. Writes nothing.
|
|
27563
27658
|
*/
|
|
27564
27659
|
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27660
|
+
/**
|
|
27661
|
+
* Load balancing (D750) — the shares placement reads now: the source
|
|
27662
|
+
* (automatic capacity shares or the operator's sliders), each node's and
|
|
27663
|
+
* device's configured share, and the effective share after a node or
|
|
27664
|
+
* device that cannot take sessions gave its share to the others.
|
|
27665
|
+
*/
|
|
27666
|
+
getEffectiveShares: method(_void(), EffectiveSharesSchema, { auth: "admin" }),
|
|
27667
|
+
/**
|
|
27668
|
+
* Load balancing (D750, D755) — switch the share source: `hardware`
|
|
27669
|
+
* (automatic, from static capacity), `benchmark` (automatic, from node
|
|
27670
|
+
* benchmark results) or `manual`. Going manual seeds the sliders with the
|
|
27671
|
+
* current automatic shares (one atomic write), so nothing jumps; going
|
|
27672
|
+
* to an automatic source recomputes at once.
|
|
27673
|
+
*/
|
|
27674
|
+
setShareSource: method(SetShareSourceInputSchema, object({ success: literal(true) }), {
|
|
27675
|
+
kind: "mutation",
|
|
27676
|
+
auth: "admin"
|
|
27677
|
+
}),
|
|
27565
27678
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27566
27679
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27567
27680
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -27591,6 +27704,22 @@ var pipelineOrchestratorCapability = {
|
|
|
27591
27704
|
kind: "mutation",
|
|
27592
27705
|
auth: "admin"
|
|
27593
27706
|
}),
|
|
27707
|
+
/**
|
|
27708
|
+
* Write or clear one step's CAMERA-WIDE settings (D754) — the fields this
|
|
27709
|
+
* camera overrides on every accelerator it runs on. `settings` is the
|
|
27710
|
+
* whole field-level bag for that step (only the overridden keys; a
|
|
27711
|
+
* model-bound field travels with its binding key); `null` or `{}` clears
|
|
27712
|
+
* it. Merged at dispatch between the node base and the per-(node,device)
|
|
27713
|
+
* override, key by key.
|
|
27714
|
+
*/
|
|
27715
|
+
setCameraStepSettings: method(object({
|
|
27716
|
+
deviceId: number(),
|
|
27717
|
+
addonId: string(),
|
|
27718
|
+
settings: record(string(), unknown()).nullable()
|
|
27719
|
+
}), object({ success: literal(true) }), {
|
|
27720
|
+
kind: "mutation",
|
|
27721
|
+
auth: "admin"
|
|
27722
|
+
}),
|
|
27594
27723
|
/** Write or clear the wholesale per-agent pipeline override. Pass `pipeline: null` to drop the override. */
|
|
27595
27724
|
setCameraPipelineForAgent: method(object({
|
|
27596
27725
|
deviceId: number(),
|
|
@@ -43585,7 +43714,7 @@ Object.freeze({
|
|
|
43585
43714
|
addonId: null,
|
|
43586
43715
|
access: "view"
|
|
43587
43716
|
},
|
|
43588
|
-
"pipelineOrchestrator.
|
|
43717
|
+
"pipelineOrchestrator.applyAutoTune": {
|
|
43589
43718
|
capName: "pipeline-orchestrator",
|
|
43590
43719
|
capScope: "system",
|
|
43591
43720
|
addonId: null,
|
|
@@ -43705,6 +43834,12 @@ Object.freeze({
|
|
|
43705
43834
|
addonId: null,
|
|
43706
43835
|
access: "view"
|
|
43707
43836
|
},
|
|
43837
|
+
"pipelineOrchestrator.getEffectiveShares": {
|
|
43838
|
+
capName: "pipeline-orchestrator",
|
|
43839
|
+
capScope: "system",
|
|
43840
|
+
addonId: null,
|
|
43841
|
+
access: "view"
|
|
43842
|
+
},
|
|
43708
43843
|
"pipelineOrchestrator.getGlobalMetrics": {
|
|
43709
43844
|
capName: "pipeline-orchestrator",
|
|
43710
43845
|
capScope: "system",
|
|
@@ -43771,7 +43906,7 @@ Object.freeze({
|
|
|
43771
43906
|
addonId: null,
|
|
43772
43907
|
access: "create"
|
|
43773
43908
|
},
|
|
43774
|
-
"pipelineOrchestrator.
|
|
43909
|
+
"pipelineOrchestrator.planAutoTune": {
|
|
43775
43910
|
capName: "pipeline-orchestrator",
|
|
43776
43911
|
capScope: "system",
|
|
43777
43912
|
addonId: null,
|
|
@@ -43873,6 +44008,12 @@ Object.freeze({
|
|
|
43873
44008
|
addonId: null,
|
|
43874
44009
|
access: "create"
|
|
43875
44010
|
},
|
|
44011
|
+
"pipelineOrchestrator.setCameraStepSettings": {
|
|
44012
|
+
capName: "pipeline-orchestrator",
|
|
44013
|
+
capScope: "system",
|
|
44014
|
+
addonId: null,
|
|
44015
|
+
access: "create"
|
|
44016
|
+
},
|
|
43876
44017
|
"pipelineOrchestrator.setCameraStepToggle": {
|
|
43877
44018
|
capName: "pipeline-orchestrator",
|
|
43878
44019
|
capScope: "system",
|
|
@@ -43903,6 +44044,12 @@ Object.freeze({
|
|
|
43903
44044
|
addonId: null,
|
|
43904
44045
|
access: "create"
|
|
43905
44046
|
},
|
|
44047
|
+
"pipelineOrchestrator.setShareSource": {
|
|
44048
|
+
capName: "pipeline-orchestrator",
|
|
44049
|
+
capScope: "system",
|
|
44050
|
+
addonId: null,
|
|
44051
|
+
access: "create"
|
|
44052
|
+
},
|
|
43906
44053
|
"pipelineOrchestrator.suggestShares": {
|
|
43907
44054
|
capName: "pipeline-orchestrator",
|
|
43908
44055
|
capScope: "system",
|
|
@@ -49343,6 +49490,11 @@ Object.freeze({
|
|
|
49343
49490
|
form: "single",
|
|
49344
49491
|
optional: false
|
|
49345
49492
|
}],
|
|
49493
|
+
"pipelineOrchestrator.setCameraStepSettings": [{
|
|
49494
|
+
name: "deviceId",
|
|
49495
|
+
form: "single",
|
|
49496
|
+
optional: false
|
|
49497
|
+
}],
|
|
49346
49498
|
"pipelineOrchestrator.setCameraStepToggle": [{
|
|
49347
49499
|
name: "deviceId",
|
|
49348
49500
|
form: "single",
|
|
@@ -51141,7 +51293,7 @@ function modelBindingKey(key) {
|
|
|
51141
51293
|
/**
|
|
51142
51294
|
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
51143
51295
|
* flattened (D738). One definition for the runner (which applies the binding)
|
|
51144
|
-
* and the orchestrator's
|
|
51296
|
+
* and the orchestrator's Auto tune (which clears those overrides, D745).
|
|
51145
51297
|
*/
|
|
51146
51298
|
function collectModelBoundKeys(fields) {
|
|
51147
51299
|
return fields.flatMap((field) => {
|
|
@@ -51633,140 +51785,25 @@ var DETECTOR_ACCURACY = {
|
|
|
51633
51785
|
personRecall: .646
|
|
51634
51786
|
}
|
|
51635
51787
|
};
|
|
51636
|
-
var DEFAULT_DEMAND_OPTIONS = {
|
|
51637
|
-
onMotionConcurrency: .25,
|
|
51638
|
-
onMotionMinConcurrent: 2
|
|
51639
|
-
};
|
|
51640
|
-
function deviceId(nodeId, deviceKey) {
|
|
51641
|
-
return `${nodeId}\u0000${deviceKey}`;
|
|
51642
|
-
}
|
|
51643
|
-
function charges(load) {
|
|
51644
|
-
return load.mode !== "disabled" && load.runsPipelineRoot && load.fps > 0;
|
|
51645
|
-
}
|
|
51646
|
-
/** The highest `count` rates, summed. */
|
|
51647
|
-
function topRatesSum(rates, count) {
|
|
51648
|
-
return rates.toSorted((a, b) => b - a).slice(0, Math.max(0, count)).reduce((sum, r) => sum + r, 0);
|
|
51649
|
-
}
|
|
51650
|
-
/** Concurrent on-motion sessions assumed cluster-wide without an observation. */
|
|
51651
|
-
function estimatedConcurrentSessions(cameraCount, options) {
|
|
51652
|
-
if (cameraCount <= 0) return 0;
|
|
51653
|
-
const share = Math.ceil(options.onMotionConcurrency * cameraCount);
|
|
51654
|
-
return Math.min(cameraCount, Math.max(share, Math.min(cameraCount, options.onMotionMinConcurrent)));
|
|
51655
|
-
}
|
|
51656
|
-
/**
|
|
51657
|
-
* Demand on every eligible device of the cluster.
|
|
51658
|
-
*/
|
|
51659
|
-
function estimateDeviceDemand(input) {
|
|
51660
|
-
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51661
|
-
const placement = input.shares ? "share" : "least-loaded";
|
|
51662
|
-
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51663
|
-
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51664
|
-
const weightByNode = /* @__PURE__ */ new Map();
|
|
51665
|
-
const eligibleCountByNode = /* @__PURE__ */ new Map();
|
|
51666
|
-
for (const d of input.eligible) {
|
|
51667
|
-
weightByNode.set(d.nodeId, (weightByNode.get(d.nodeId) ?? 0) + Math.max(d.weight, 0));
|
|
51668
|
-
eligibleCountByNode.set(d.nodeId, (eligibleCountByNode.get(d.nodeId) ?? 0) + 1);
|
|
51669
|
-
}
|
|
51670
|
-
const pinnedFps = /* @__PURE__ */ new Map();
|
|
51671
|
-
const alwaysOnFps = /* @__PURE__ */ new Map();
|
|
51672
|
-
const alwaysOnCount = /* @__PURE__ */ new Map();
|
|
51673
|
-
const maxFps = /* @__PURE__ */ new Map();
|
|
51674
|
-
const onMotionRates = [];
|
|
51675
|
-
const bump = (map, key, by) => {
|
|
51676
|
-
map.set(key, (map.get(key) ?? 0) + by);
|
|
51677
|
-
};
|
|
51678
|
-
const noteMax = (key, fps) => {
|
|
51679
|
-
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51680
|
-
};
|
|
51681
|
-
const pinNodeOf = (pin) => {
|
|
51682
|
-
if (pin.nodeId !== null) return pin.nodeId;
|
|
51683
|
-
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51684
|
-
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51685
|
-
};
|
|
51686
|
-
let alwaysOnTotal = 0;
|
|
51687
|
-
for (const cam of input.cameras) {
|
|
51688
|
-
if (!charges(cam)) continue;
|
|
51689
|
-
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51690
|
-
if (cam.devicePin !== null && pinNode !== null) {
|
|
51691
|
-
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51692
|
-
if (!eligibleById.has(key)) continue;
|
|
51693
|
-
bump(pinnedFps, key, cam.fps);
|
|
51694
|
-
noteMax(key, cam.fps);
|
|
51695
|
-
continue;
|
|
51696
|
-
}
|
|
51697
|
-
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51698
|
-
if (standing !== null) {
|
|
51699
|
-
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51700
|
-
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51701
|
-
for (const d of input.eligible) {
|
|
51702
|
-
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51703
|
-
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51704
|
-
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51705
|
-
bump(alwaysOnFps, key, cam.fps * share);
|
|
51706
|
-
bump(alwaysOnCount, key, share);
|
|
51707
|
-
noteMax(key, cam.fps);
|
|
51708
|
-
}
|
|
51709
|
-
continue;
|
|
51710
|
-
}
|
|
51711
|
-
onMotionRates.push(cam.fps);
|
|
51712
|
-
}
|
|
51713
|
-
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51714
|
-
const nodeCount = weightByNode.size;
|
|
51715
|
-
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51716
|
-
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51717
|
-
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51718
|
-
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51719
|
-
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51720
|
-
return input.eligible.map((d) => {
|
|
51721
|
-
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51722
|
-
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51723
|
-
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51724
|
-
let onMotionSessions = 0;
|
|
51725
|
-
let onMotionSource = "none";
|
|
51726
|
-
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51727
|
-
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51728
|
-
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51729
|
-
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51730
|
-
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51731
|
-
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51732
|
-
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51733
|
-
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51734
|
-
onMotionSource = "observed-24h";
|
|
51735
|
-
} else {
|
|
51736
|
-
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51737
|
-
const share = nodeCount > 0 && nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight / nodeCount : 0;
|
|
51738
|
-
onMotionSessions = Math.ceil(estimatedTotal * share);
|
|
51739
|
-
onMotionSource = "estimate";
|
|
51740
|
-
}
|
|
51741
|
-
const demandFps = (pinnedFps.get(key) ?? 0) + (alwaysOnFps.get(key) ?? 0) + topRatesSum(onMotionRates, onMotionSessions);
|
|
51742
|
-
return {
|
|
51743
|
-
nodeId: d.nodeId,
|
|
51744
|
-
deviceKey: d.deviceKey,
|
|
51745
|
-
demandFps: Math.round(demandFps * 100) / 100,
|
|
51746
|
-
maxCameraFps: Math.max(maxFps.get(key) ?? 0, fastestOnMotion),
|
|
51747
|
-
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51748
|
-
onMotionSessions,
|
|
51749
|
-
onMotionSource,
|
|
51750
|
-
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51751
|
-
placement
|
|
51752
|
-
};
|
|
51753
|
-
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51754
|
-
}
|
|
51755
51788
|
/** The family the cluster runs. */
|
|
51756
|
-
var
|
|
51789
|
+
var AUTO_TUNE_FAMILY = "yolov9";
|
|
51790
|
+
/** Parallel sessions one device sustains at `fps` with `headroom` kept free. */
|
|
51791
|
+
function capacitySessionsOf(fps, headroom, cameraFps) {
|
|
51792
|
+
if (!(fps > 0) || !(cameraFps > 0)) return 0;
|
|
51793
|
+
return Math.max(0, Math.floor(fps * (1 - headroom) / cameraFps + 1e-9));
|
|
51794
|
+
}
|
|
51757
51795
|
function round(value, digits = 3) {
|
|
51758
51796
|
const f = 10 ** digits;
|
|
51759
51797
|
return Math.round(value * f) / f;
|
|
51760
51798
|
}
|
|
51761
|
-
function
|
|
51762
|
-
return cell
|
|
51799
|
+
function sessionsOn(cell, sizing) {
|
|
51800
|
+
return cell === void 0 ? 0 : capacitySessionsOf(cell.fps, sizing.headroom, sizing.cameraFps);
|
|
51763
51801
|
}
|
|
51764
|
-
function fitsOn(cell,
|
|
51765
|
-
|
|
51766
|
-
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51802
|
+
function fitsOn(cell, sizing) {
|
|
51803
|
+
return sessionsOn(cell, sizing) >= sizing.minSessions;
|
|
51767
51804
|
}
|
|
51768
51805
|
function candidatesFor(contexts, formatsByModel) {
|
|
51769
|
-
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(
|
|
51806
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(AUTO_TUNE_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51770
51807
|
modelId: id,
|
|
51771
51808
|
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51772
51809
|
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
@@ -51774,28 +51811,24 @@ function candidatesFor(contexts, formatsByModel) {
|
|
|
51774
51811
|
function totalLatency(contexts, modelId) {
|
|
51775
51812
|
return contexts.reduce((sum, c) => sum + (c.column[modelId]?.p50Ms ?? Number.POSITIVE_INFINITY), 0);
|
|
51776
51813
|
}
|
|
51777
|
-
|
|
51778
|
-
|
|
51779
|
-
|
|
51780
|
-
return Math.max(worst, cell ? utilisationOf(cell, c.demand) : Number.POSITIVE_INFINITY);
|
|
51781
|
-
}, 0);
|
|
51814
|
+
/** The fewest sessions any device of the group sustains with this model. */
|
|
51815
|
+
function worstSessions(contexts, modelId, sizing) {
|
|
51816
|
+
return contexts.reduce((worst, c) => Math.min(worst, sessionsOn(c.column[modelId], sizing)), Number.POSITIVE_INFINITY);
|
|
51782
51817
|
}
|
|
51783
51818
|
/** One model for a group of devices that must agree (step 2–4 of the header). */
|
|
51784
|
-
function pickForGroup(contexts, formatsByModel,
|
|
51819
|
+
function pickForGroup(contexts, formatsByModel, sizing) {
|
|
51785
51820
|
const candidates = candidatesFor(contexts, formatsByModel);
|
|
51786
51821
|
if (candidates.length === 0) return null;
|
|
51787
|
-
const fitting = candidates.filter((c) => contexts.every((ctx) =>
|
|
51788
|
-
const cell = ctx.column[c.modelId];
|
|
51789
|
-
return cell !== void 0 && fitsOn(cell, ctx.demand, headroom);
|
|
51790
|
-
}));
|
|
51822
|
+
const fitting = candidates.filter((c) => contexts.every((ctx) => fitsOn(ctx.column[c.modelId], sizing)));
|
|
51791
51823
|
if (fitting.length === 0) {
|
|
51792
|
-
const lightest = candidates.toSorted((a, b) =>
|
|
51824
|
+
const lightest = candidates.toSorted((a, b) => worstSessions(contexts, b.modelId, sizing) - worstSessions(contexts, a.modelId, sizing) || totalLatency(contexts, a.modelId) - totalLatency(contexts, b.modelId))[0];
|
|
51793
51825
|
if (lightest === void 0) return null;
|
|
51794
51826
|
return {
|
|
51795
51827
|
modelId: lightest.modelId,
|
|
51796
51828
|
reasons: [{
|
|
51797
51829
|
kind: "no-fit-lightest",
|
|
51798
|
-
|
|
51830
|
+
sessions: worstSessions(contexts, lightest.modelId, sizing),
|
|
51831
|
+
minSessions: sizing.minSessions
|
|
51799
51832
|
}]
|
|
51800
51833
|
};
|
|
51801
51834
|
}
|
|
@@ -51805,10 +51838,7 @@ function pickForGroup(contexts, formatsByModel, headroom) {
|
|
|
51805
51838
|
const reasons = [];
|
|
51806
51839
|
const mostAccurate = candidates[0];
|
|
51807
51840
|
if (mostAccurate !== void 0 && mostAccurate.modelId !== best.modelId) {
|
|
51808
|
-
const limitedBy = contexts.find((ctx) =>
|
|
51809
|
-
const cell = ctx.column[mostAccurate.modelId];
|
|
51810
|
-
return cell === void 0 || !fitsOn(cell, ctx.demand, headroom);
|
|
51811
|
-
})?.device.key ?? "";
|
|
51841
|
+
const limitedBy = contexts.find((ctx) => !fitsOn(ctx.column[mostAccurate.modelId], sizing))?.device.key ?? "";
|
|
51812
51842
|
reasons.push({
|
|
51813
51843
|
kind: "stepped-down",
|
|
51814
51844
|
from: mostAccurate.modelId,
|
|
@@ -51826,7 +51856,7 @@ function pickForGroup(contexts, formatsByModel, headroom) {
|
|
|
51826
51856
|
reasons
|
|
51827
51857
|
};
|
|
51828
51858
|
}
|
|
51829
|
-
function choiceOn(ctx, modelId) {
|
|
51859
|
+
function choiceOn(ctx, modelId, sizing) {
|
|
51830
51860
|
const cell = ctx.column[modelId];
|
|
51831
51861
|
if (cell === void 0) return null;
|
|
51832
51862
|
return {
|
|
@@ -51834,23 +51864,20 @@ function choiceOn(ctx, modelId) {
|
|
|
51834
51864
|
p50Ms: cell.p50Ms,
|
|
51835
51865
|
fps: cell.fps,
|
|
51836
51866
|
measured: cell.measured,
|
|
51837
|
-
|
|
51867
|
+
capacitySessions: sessionsOn(cell, sizing),
|
|
51838
51868
|
personRecall: DETECTOR_ACCURACY[modelId]?.personRecall ?? 0
|
|
51839
51869
|
};
|
|
51840
51870
|
}
|
|
51841
|
-
function deviceReasonsFor(
|
|
51871
|
+
function deviceReasonsFor(choice, sizing) {
|
|
51842
51872
|
const out = [{
|
|
51843
|
-
kind: "
|
|
51844
|
-
|
|
51873
|
+
kind: "capacity",
|
|
51874
|
+
sessions: choice.capacitySessions,
|
|
51875
|
+
minSessions: sizing.minSessions
|
|
51845
51876
|
}];
|
|
51846
51877
|
if (!choice.measured) out.push({
|
|
51847
51878
|
kind: "derived-latency",
|
|
51848
51879
|
modelId: choice.modelId
|
|
51849
51880
|
});
|
|
51850
|
-
if (ctx.demand?.onMotionSource === "estimate") out.push({
|
|
51851
|
-
kind: "estimated-load",
|
|
51852
|
-
observedForMs: ctx.demand.observedForMs
|
|
51853
|
-
});
|
|
51854
51881
|
return out;
|
|
51855
51882
|
}
|
|
51856
51883
|
function skippedNode(node, reason, hardwareClass) {
|
|
@@ -51866,7 +51893,6 @@ function skippedNode(node, reason, hardwareClass) {
|
|
|
51866
51893
|
accelerator: node.hardware ? acceleratorKindOf(d.key, node.hardware) : null,
|
|
51867
51894
|
enabled: d.enabled,
|
|
51868
51895
|
available: d.available,
|
|
51869
|
-
demand: null,
|
|
51870
51896
|
chosen: null,
|
|
51871
51897
|
reasons: []
|
|
51872
51898
|
})),
|
|
@@ -51876,7 +51902,11 @@ function skippedNode(node, reason, hardwareClass) {
|
|
|
51876
51902
|
/** Recommend one node. See the module header. */
|
|
51877
51903
|
function recommendNode(input) {
|
|
51878
51904
|
const { node } = input;
|
|
51879
|
-
const
|
|
51905
|
+
const sizing = {
|
|
51906
|
+
headroom: input.headroom ?? .4,
|
|
51907
|
+
cameraFps: input.cameraFps ?? 10,
|
|
51908
|
+
minSessions: input.minSessionsPerDevice ?? 2
|
|
51909
|
+
};
|
|
51880
51910
|
const perf = input.perf ?? DETECTOR_PERF;
|
|
51881
51911
|
if (!node.reachable) return skippedNode(node, { kind: "node-offline" }, null);
|
|
51882
51912
|
if (node.hardware === null) return skippedNode(node, { kind: "no-hardware-facts" }, null);
|
|
@@ -51887,7 +51917,6 @@ function recommendNode(input) {
|
|
|
51887
51917
|
kind: "no-measured-data",
|
|
51888
51918
|
hardwareClass
|
|
51889
51919
|
}, hardwareClass);
|
|
51890
|
-
const demandByKey = new Map(input.demand.filter((d) => d.nodeId === node.nodeId).map((d) => [d.deviceKey, d]));
|
|
51891
51920
|
const devices = node.devices.toSorted((a, b) => a.key.localeCompare(b.key));
|
|
51892
51921
|
const contexts = /* @__PURE__ */ new Map();
|
|
51893
51922
|
const early = /* @__PURE__ */ new Map();
|
|
@@ -51916,8 +51945,7 @@ function recommendNode(input) {
|
|
|
51916
51945
|
contexts.set(device.key, {
|
|
51917
51946
|
device,
|
|
51918
51947
|
accelerator,
|
|
51919
|
-
column
|
|
51920
|
-
demand: demandByKey.get(device.key) ?? null
|
|
51948
|
+
column
|
|
51921
51949
|
});
|
|
51922
51950
|
}
|
|
51923
51951
|
const isCpu = (a) => a === "x86-cpu" || a === "arm-cpu";
|
|
@@ -51927,23 +51955,23 @@ function recommendNode(input) {
|
|
|
51927
51955
|
const reasonsByKey = /* @__PURE__ */ new Map();
|
|
51928
51956
|
const nodeReasons = [];
|
|
51929
51957
|
let acceleratorModelId = null;
|
|
51930
|
-
const groupPick = accelerators.length > 0 ? pickForGroup(accelerators, node.formatsByModel,
|
|
51958
|
+
const groupPick = accelerators.length > 0 ? pickForGroup(accelerators, node.formatsByModel, sizing) : null;
|
|
51931
51959
|
if (groupPick !== null) {
|
|
51932
51960
|
acceleratorModelId = groupPick.modelId;
|
|
51933
51961
|
nodeReasons.push(...groupPick.reasons);
|
|
51934
51962
|
for (const ctx of accelerators) {
|
|
51935
|
-
const choice = choiceOn(ctx, groupPick.modelId);
|
|
51963
|
+
const choice = choiceOn(ctx, groupPick.modelId, sizing);
|
|
51936
51964
|
if (choice === null) continue;
|
|
51937
51965
|
chosenByKey.set(ctx.device.key, choice);
|
|
51938
|
-
reasonsByKey.set(ctx.device.key, deviceReasonsFor(
|
|
51966
|
+
reasonsByKey.set(ctx.device.key, deviceReasonsFor(choice, sizing));
|
|
51939
51967
|
}
|
|
51940
51968
|
} else for (const ctx of accelerators) reasonsByKey.set(ctx.device.key, [{
|
|
51941
51969
|
kind: "no-family-build",
|
|
51942
51970
|
format: ctx.device.format
|
|
51943
51971
|
}]);
|
|
51944
51972
|
for (const ctx of cpus) {
|
|
51945
|
-
const pick = pickForGroup([ctx], node.formatsByModel,
|
|
51946
|
-
const choice = pick === null ? null : choiceOn(ctx, pick.modelId);
|
|
51973
|
+
const pick = pickForGroup([ctx], node.formatsByModel, sizing);
|
|
51974
|
+
const choice = pick === null ? null : choiceOn(ctx, pick.modelId, sizing);
|
|
51947
51975
|
if (pick === null || choice === null) {
|
|
51948
51976
|
reasonsByKey.set(ctx.device.key, [{
|
|
51949
51977
|
kind: "no-family-build",
|
|
@@ -51955,7 +51983,7 @@ function recommendNode(input) {
|
|
|
51955
51983
|
reasonsByKey.set(ctx.device.key, [
|
|
51956
51984
|
{ kind: "cpu-device" },
|
|
51957
51985
|
...pick.reasons,
|
|
51958
|
-
...deviceReasonsFor(
|
|
51986
|
+
...deviceReasonsFor(choice, sizing)
|
|
51959
51987
|
]);
|
|
51960
51988
|
}
|
|
51961
51989
|
return {
|
|
@@ -51970,7 +51998,6 @@ function recommendNode(input) {
|
|
|
51970
51998
|
accelerator: acceleratorKindOf(device.key, hardware),
|
|
51971
51999
|
enabled: device.enabled,
|
|
51972
52000
|
available: device.available,
|
|
51973
|
-
demand: demandByKey.get(device.key) ?? null,
|
|
51974
52001
|
chosen: chosenByKey.get(device.key) ?? null,
|
|
51975
52002
|
reasons: early.get(device.key) ?? reasonsByKey.get(device.key) ?? []
|
|
51976
52003
|
})),
|
|
@@ -52018,6 +52045,204 @@ function electByShare(candidates) {
|
|
|
52018
52045
|
};
|
|
52019
52046
|
}
|
|
52020
52047
|
/**
|
|
52048
|
+
* The shares a session election actually reads, and the view the Load
|
|
52049
|
+
* balancing tab shows (D750). Pure.
|
|
52050
|
+
*
|
|
52051
|
+
* The SOURCE decides the configured weights: the automatic capacity shares
|
|
52052
|
+
* (`hardware` or `benchmark`, D755), or the operator's stored sliders. Eligibility decides the effective ones: a
|
|
52053
|
+
* node or device that cannot take a session now (offline, switched off, no
|
|
52054
|
+
* detect capability, unable to run the root) contributes nothing, and the
|
|
52055
|
+
* others split its share in proportion to their own. That redistribution is
|
|
52056
|
+
* exactly what `electByShare` does over its candidate set, so the view and
|
|
52057
|
+
* the election cannot disagree; the stored values are never rewritten for it.
|
|
52058
|
+
*/
|
|
52059
|
+
/** Weight a node or device has when its source names none (the balancer's own default). */
|
|
52060
|
+
var DEFAULT_WEIGHT = 1;
|
|
52061
|
+
function positive(value) {
|
|
52062
|
+
return value !== void 0 && value > 0 ? value : DEFAULT_WEIGHT;
|
|
52063
|
+
}
|
|
52064
|
+
function percentages(keys, weightOf, counts) {
|
|
52065
|
+
const total = keys.reduce((sum, k) => sum + (counts(k) ? weightOf(k) : 0), 0);
|
|
52066
|
+
return new Map(keys.map((k) => [k, total > 0 && counts(k) ? Math.round(weightOf(k) / total * 1e3) / 10 : 0]));
|
|
52067
|
+
}
|
|
52068
|
+
function resolveEffectiveShares(input) {
|
|
52069
|
+
const useAuto = isAutomaticShareSource(input.source) && input.auto !== null;
|
|
52070
|
+
const weights = useAuto && input.auto !== null ? input.auto : input.stored;
|
|
52071
|
+
const nodeWeights = {};
|
|
52072
|
+
for (const nodeId of input.nodeIds) nodeWeights[nodeId] = positive(weights.nodes[nodeId]);
|
|
52073
|
+
const eligibleNodes = new Set(input.eligibleNodes);
|
|
52074
|
+
const configured = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52075
|
+
const effective = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(k));
|
|
52076
|
+
const deviceWeights = {};
|
|
52077
|
+
const nodes = input.nodeIds.map((nodeId) => {
|
|
52078
|
+
const eligibleKeys = new Set(input.eligibleDevices[nodeId] ?? []);
|
|
52079
|
+
const named = weights.devices[nodeId] ?? {};
|
|
52080
|
+
const keys = [...new Set([...Object.keys(named), ...eligibleKeys])].toSorted();
|
|
52081
|
+
const perDevice = {};
|
|
52082
|
+
for (const key of keys) perDevice[key] = positive(named[key]);
|
|
52083
|
+
if (keys.length > 0) deviceWeights[nodeId] = perDevice;
|
|
52084
|
+
const devConfigured = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52085
|
+
const devEffective = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(nodeId) && eligibleKeys.has(k));
|
|
52086
|
+
const devices = keys.map((deviceKey) => ({
|
|
52087
|
+
deviceKey,
|
|
52088
|
+
eligible: eligibleKeys.has(deviceKey),
|
|
52089
|
+
configuredPct: devConfigured.get(deviceKey) ?? 0,
|
|
52090
|
+
effectivePct: devEffective.get(deviceKey) ?? 0
|
|
52091
|
+
}));
|
|
52092
|
+
const capacity = useAuto ? input.autoCapacity?.[nodeId] : void 0;
|
|
52093
|
+
return {
|
|
52094
|
+
nodeId,
|
|
52095
|
+
eligible: eligibleNodes.has(nodeId),
|
|
52096
|
+
configuredPct: configured.get(nodeId) ?? 0,
|
|
52097
|
+
effectivePct: effective.get(nodeId) ?? 0,
|
|
52098
|
+
capacitySource: capacity?.source ?? null,
|
|
52099
|
+
capacityFallback: capacity?.fallback ?? null,
|
|
52100
|
+
devices
|
|
52101
|
+
};
|
|
52102
|
+
});
|
|
52103
|
+
return {
|
|
52104
|
+
source: input.source,
|
|
52105
|
+
policy: input.policy ?? "share",
|
|
52106
|
+
autoPending: isAutomaticShareSource(input.source) && input.auto === null,
|
|
52107
|
+
computedAt: input.computedAt ?? null,
|
|
52108
|
+
trigger: input.trigger ?? null,
|
|
52109
|
+
nodes,
|
|
52110
|
+
nodeWeights,
|
|
52111
|
+
deviceWeights
|
|
52112
|
+
};
|
|
52113
|
+
}
|
|
52114
|
+
/**
|
|
52115
|
+
* Automatic shares from CAPACITY (D755). Pure.
|
|
52116
|
+
*
|
|
52117
|
+
* A node's share is its capacity in parallel sessions over the cluster's; a
|
|
52118
|
+
* device's share inside its node follows its throughput. There is no load
|
|
52119
|
+
* input: nothing here depends on how many sessions ran, only on what each
|
|
52120
|
+
* node can carry.
|
|
52121
|
+
*
|
|
52122
|
+
* Two places a capacity can come from:
|
|
52123
|
+
* - `hardware` — each enabled accelerator at its Auto tune model, from the
|
|
52124
|
+
* measured latency table: ⌊fps × (1 − headroom) / camera fps⌋ sessions;
|
|
52125
|
+
* - `benchmark` — the node's own measured max sessions per device, read
|
|
52126
|
+
* through a {@link NodeCapacitySource}. A node whose results do not cover
|
|
52127
|
+
* every enabled device it would be sized on is sized from `hardware`
|
|
52128
|
+
* instead, and the answer says so per node — never a silent mix.
|
|
52129
|
+
*/
|
|
52130
|
+
/** A source that has no results — the benchmark store before it exists. */
|
|
52131
|
+
var NO_MEASURED_CAPACITY = { readMeasuredCapacity: async () => /* @__PURE__ */ new Map() };
|
|
52132
|
+
/** Integer shares summing to 100 (largest remainder); ties broken by key. */
|
|
52133
|
+
function toPercentShares(weights) {
|
|
52134
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
52135
|
+
const out = /* @__PURE__ */ new Map();
|
|
52136
|
+
if (total <= 0) return out;
|
|
52137
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
52138
|
+
k,
|
|
52139
|
+
exact: w / total * 100
|
|
52140
|
+
}));
|
|
52141
|
+
let assigned = 0;
|
|
52142
|
+
for (const r of raw) {
|
|
52143
|
+
out.set(r.k, Math.floor(r.exact));
|
|
52144
|
+
assigned += Math.floor(r.exact);
|
|
52145
|
+
}
|
|
52146
|
+
const remainder = (exact) => Math.round((exact - Math.floor(exact)) * 1e6);
|
|
52147
|
+
const byRemainder = raw.toSorted((a, b) => remainder(b.exact) - remainder(a.exact) || a.k.localeCompare(b.k));
|
|
52148
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
52149
|
+
const r = byRemainder[i % byRemainder.length];
|
|
52150
|
+
if (r === void 0) break;
|
|
52151
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
52152
|
+
}
|
|
52153
|
+
return out;
|
|
52154
|
+
}
|
|
52155
|
+
function hardwareDevices(rec, headroom, cameraFps) {
|
|
52156
|
+
return rec.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
52157
|
+
const fps = d.chosen?.fps ?? 0;
|
|
52158
|
+
return {
|
|
52159
|
+
deviceKey: d.deviceKey,
|
|
52160
|
+
sessions: capacitySessionsOf(fps, headroom, cameraFps),
|
|
52161
|
+
weight: Math.max(fps, 0),
|
|
52162
|
+
modelId: d.chosen?.modelId ?? null,
|
|
52163
|
+
fps: d.chosen?.fps ?? null,
|
|
52164
|
+
measured: d.chosen?.measured ?? false
|
|
52165
|
+
};
|
|
52166
|
+
});
|
|
52167
|
+
}
|
|
52168
|
+
function sizeNode(rec, input) {
|
|
52169
|
+
const fromHardware = (fallback) => {
|
|
52170
|
+
const devices = rec.skipped ? [] : hardwareDevices(rec, input.headroom, input.cameraFps);
|
|
52171
|
+
const capacity = devices.reduce((sum, d) => sum + d.sessions, 0);
|
|
52172
|
+
return {
|
|
52173
|
+
rec,
|
|
52174
|
+
devices,
|
|
52175
|
+
capacitySource: rec.skipped ? null : "hardware",
|
|
52176
|
+
capacityFallback: rec.skipped ? null : fallback,
|
|
52177
|
+
skippedReason: rec.skipped ? rec.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity"
|
|
52178
|
+
};
|
|
52179
|
+
};
|
|
52180
|
+
if (input.source === "hardware") return fromHardware(null);
|
|
52181
|
+
if (!rec.reachable) return fromHardware(null);
|
|
52182
|
+
const results = input.measured.get(rec.nodeId);
|
|
52183
|
+
const byKey = new Map((results?.devices ?? []).map((d) => [d.deviceKey, d]));
|
|
52184
|
+
const usable = rec.devices.filter((d) => d.enabled && d.available);
|
|
52185
|
+
const mustCover = rec.devices.filter((d) => d.enabled && d.available && d.chosen !== null);
|
|
52186
|
+
const covered = usable.filter((d) => byKey.has(d.deviceKey));
|
|
52187
|
+
if (covered.length === 0) return fromHardware("no-benchmark-results");
|
|
52188
|
+
if (mustCover.some((d) => !byKey.has(d.deviceKey))) return fromHardware("partial-benchmark-results");
|
|
52189
|
+
const devices = covered.map((d) => {
|
|
52190
|
+
const row = byKey.get(d.deviceKey);
|
|
52191
|
+
const sessions = Math.max(0, Math.floor(row?.maxSessions ?? 0));
|
|
52192
|
+
return {
|
|
52193
|
+
deviceKey: d.deviceKey,
|
|
52194
|
+
sessions,
|
|
52195
|
+
weight: sessions,
|
|
52196
|
+
modelId: row?.modelId ?? null,
|
|
52197
|
+
fps: null,
|
|
52198
|
+
measured: true
|
|
52199
|
+
};
|
|
52200
|
+
});
|
|
52201
|
+
return {
|
|
52202
|
+
rec,
|
|
52203
|
+
devices,
|
|
52204
|
+
capacitySource: "benchmark",
|
|
52205
|
+
capacityFallback: null,
|
|
52206
|
+
skippedReason: devices.reduce((sum, d) => sum + d.sessions, 0) > 0 ? null : "no-capacity"
|
|
52207
|
+
};
|
|
52208
|
+
}
|
|
52209
|
+
/** Shares proportional to capacity — the automatic shares and the suggestion are this. */
|
|
52210
|
+
function computeCapacityShares(input) {
|
|
52211
|
+
const sized = input.recommendations.map((rec) => sizeNode(rec, input));
|
|
52212
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
52213
|
+
for (const node of sized) {
|
|
52214
|
+
const capacity = node.devices.reduce((sum, d) => sum + d.sessions, 0);
|
|
52215
|
+
if (node.skippedReason === null && capacity > 0) capacityByNode.set(node.rec.nodeId, capacity);
|
|
52216
|
+
}
|
|
52217
|
+
const nodePct = toPercentShares(capacityByNode);
|
|
52218
|
+
const nodes = sized.map((node) => {
|
|
52219
|
+
const devicePct = toPercentShares(new Map(node.devices.map((d) => [d.deviceKey, d.weight])));
|
|
52220
|
+
return {
|
|
52221
|
+
nodeId: node.rec.nodeId,
|
|
52222
|
+
share: nodePct.get(node.rec.nodeId) ?? null,
|
|
52223
|
+
capacitySessions: node.devices.reduce((sum, d) => sum + d.sessions, 0),
|
|
52224
|
+
skippedReason: node.skippedReason,
|
|
52225
|
+
capacitySource: node.capacitySource,
|
|
52226
|
+
capacityFallback: node.capacityFallback,
|
|
52227
|
+
devices: node.devices.map((d) => ({
|
|
52228
|
+
deviceKey: d.deviceKey,
|
|
52229
|
+
share: devicePct.get(d.deviceKey) ?? 0,
|
|
52230
|
+
modelId: d.modelId,
|
|
52231
|
+
fps: d.fps,
|
|
52232
|
+
measured: d.measured,
|
|
52233
|
+
suggestedMaxSessions: d.sessions
|
|
52234
|
+
}))
|
|
52235
|
+
};
|
|
52236
|
+
});
|
|
52237
|
+
return {
|
|
52238
|
+
headroom: input.headroom,
|
|
52239
|
+
source: input.source,
|
|
52240
|
+
cameraFps: input.cameraFps,
|
|
52241
|
+
minSessionsPerDevice: input.minSessionsPerDevice,
|
|
52242
|
+
nodes
|
|
52243
|
+
};
|
|
52244
|
+
}
|
|
52245
|
+
/**
|
|
52021
52246
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
52022
52247
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
52023
52248
|
* is fed.
|
|
@@ -62411,6 +62636,7 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62411
62636
|
};
|
|
62412
62637
|
const okByNode = new Map(input.verdicts.filter((v) => v.state === "ok").map((v) => [v.nodeId, v]));
|
|
62413
62638
|
const sourceAttached = input.attachedByNode[saturated.nodeId] ?? 0;
|
|
62639
|
+
if (input.shares !== void 0) return planShareTarget(input, input.shares, saturated, movable, okByNode, now, guards);
|
|
62414
62640
|
for (const camera of movable) {
|
|
62415
62641
|
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62416
62642
|
const target = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => {
|
|
@@ -62435,6 +62661,52 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62435
62661
|
nodeId: saturated.nodeId
|
|
62436
62662
|
};
|
|
62437
62663
|
}
|
|
62664
|
+
/** Deficits equal up to float noise. */
|
|
62665
|
+
var SHARE_EPSILON = 1e-9;
|
|
62666
|
+
function underCap(input, nodeId) {
|
|
62667
|
+
const cap = input.nodeCaps?.[nodeId];
|
|
62668
|
+
const attached = input.attachedByNode[nodeId] ?? 0;
|
|
62669
|
+
return cap === null || cap === void 0 || cap <= 0 || attached < cap;
|
|
62670
|
+
}
|
|
62671
|
+
/**
|
|
62672
|
+
* Share placement's failover target (D750): among the camera's eligible,
|
|
62673
|
+
* `ok`, under-cap nodes, the one furthest below its share of the cameras
|
|
62674
|
+
* attached across the source and those nodes — the same deficit rule a new
|
|
62675
|
+
* session is placed by. A node at or over its share is no target, so a move
|
|
62676
|
+
* never undoes the balance. Nothing below its share ⇒ hold, naming the camera.
|
|
62677
|
+
*/
|
|
62678
|
+
function planShareTarget(input, shares, saturated, movable, okByNode, now, guards) {
|
|
62679
|
+
const shareOf = (nodeId) => {
|
|
62680
|
+
const share = shares[nodeId];
|
|
62681
|
+
return share !== void 0 && share > 0 ? share : 0;
|
|
62682
|
+
};
|
|
62683
|
+
for (const camera of movable) {
|
|
62684
|
+
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62685
|
+
const healthy = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => underCap(input, v.nodeId));
|
|
62686
|
+
const set = [saturated.nodeId, ...healthy.map((v) => v.nodeId)];
|
|
62687
|
+
const shareSum = set.reduce((sum, nodeId) => sum + shareOf(nodeId), 0);
|
|
62688
|
+
const attachedSum = set.reduce((sum, nodeId) => sum + (input.attachedByNode[nodeId] ?? 0), 0);
|
|
62689
|
+
if (shareSum <= 0) continue;
|
|
62690
|
+
const deficit = (nodeId) => shareOf(nodeId) / shareSum * attachedSum - (input.attachedByNode[nodeId] ?? 0);
|
|
62691
|
+
const target = healthy.filter((v) => deficit(v.nodeId) > SHARE_EPSILON).toSorted((a, b) => deficit(b.nodeId) - deficit(a.nodeId) || a.signals.score - b.signals.score || a.nodeId.localeCompare(b.nodeId))[0];
|
|
62692
|
+
if (!target) continue;
|
|
62693
|
+
return {
|
|
62694
|
+
kind: "move",
|
|
62695
|
+
deviceId: camera.deviceId,
|
|
62696
|
+
fromNodeId: saturated.nodeId,
|
|
62697
|
+
toNodeId: target.nodeId,
|
|
62698
|
+
fromScore: saturated.signals.score,
|
|
62699
|
+
toScore: target.signals.score
|
|
62700
|
+
};
|
|
62701
|
+
}
|
|
62702
|
+
const first = movable[0];
|
|
62703
|
+
return {
|
|
62704
|
+
kind: "hold",
|
|
62705
|
+
reason: "no-target",
|
|
62706
|
+
nodeId: saturated.nodeId,
|
|
62707
|
+
...first !== void 0 ? { deviceId: first.deviceId } : {}
|
|
62708
|
+
};
|
|
62709
|
+
}
|
|
62438
62710
|
//#endregion
|
|
62439
62711
|
//#region src/node-stress-controller.ts
|
|
62440
62712
|
/** Narrow an unknown settings value to a {@link NodeStressMode}. */
|
|
@@ -62643,17 +62915,27 @@ var NodeStressController = class NodeStressController {
|
|
|
62643
62915
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
62644
62916
|
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
62645
62917
|
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
62918
|
+
const candidates = await this.collectCandidates(now);
|
|
62919
|
+
const shares = await this.deps.readFailoverShares?.() ?? null;
|
|
62646
62920
|
const plan = planStressFailover({
|
|
62647
62921
|
verdicts,
|
|
62648
|
-
candidates
|
|
62922
|
+
candidates,
|
|
62649
62923
|
history,
|
|
62650
62924
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
62651
|
-
attachedByNode: this.attachedByNode()
|
|
62925
|
+
attachedByNode: this.attachedByNode(),
|
|
62926
|
+
...shares !== null ? { shares } : {}
|
|
62652
62927
|
}, now, this.guards());
|
|
62653
62928
|
if (plan.kind === "hold") {
|
|
62654
62929
|
this.deps.logger.warn("node saturated — no failover applied", {
|
|
62655
|
-
tags: {
|
|
62656
|
-
|
|
62930
|
+
tags: {
|
|
62931
|
+
nodeId: plan.nodeId ?? "",
|
|
62932
|
+
...plan.deviceId !== void 0 ? { deviceId: plan.deviceId } : {}
|
|
62933
|
+
},
|
|
62934
|
+
meta: {
|
|
62935
|
+
reason: plan.reason,
|
|
62936
|
+
rule: shares !== null ? "share-deficit" : "fewest-cameras",
|
|
62937
|
+
...plan.reason === "no-target" && shares !== null ? { why: "no healthy node below its share can take it" } : {}
|
|
62938
|
+
}
|
|
62657
62939
|
});
|
|
62658
62940
|
return;
|
|
62659
62941
|
}
|
|
@@ -64432,6 +64714,7 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
64432
64714
|
});
|
|
64433
64715
|
var StoredCameraPipelineSettingsSchema = object({
|
|
64434
64716
|
stepToggles: record(string(), boolean()).optional(),
|
|
64717
|
+
stepSettings: record(string(), record(string(), unknown()).readonly()).optional(),
|
|
64435
64718
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
64436
64719
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
64437
64720
|
pipelineByAgent: record(string(), StoredCameraPipelineForAgentSchema).optional(),
|
|
@@ -64751,6 +65034,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
64751
65034
|
};
|
|
64752
65035
|
});
|
|
64753
65036
|
}
|
|
65037
|
+
/**
|
|
65038
|
+
* Write or clear one camera's CAMERA-WIDE settings for `addonId` (D754): the
|
|
65039
|
+
* field-level bag merged between the node base and the per-(node,device)
|
|
65040
|
+
* override on every accelerator. `null` or an empty bag deletes the step's
|
|
65041
|
+
* entry; an empty layer is pruned. Read-patch-write, immutable.
|
|
65042
|
+
*/
|
|
65043
|
+
async writeCameraStepSettings(deviceId, addonId, settings) {
|
|
65044
|
+
await this.cameraSettingsState.update((prev) => {
|
|
65045
|
+
const key = String(deviceId);
|
|
65046
|
+
const camera = prev[key] ?? {};
|
|
65047
|
+
const nextLayer = { ...camera.stepSettings };
|
|
65048
|
+
if (settings === null || Object.keys(settings).length === 0) delete nextLayer[addonId];
|
|
65049
|
+
else nextLayer[addonId] = { ...settings };
|
|
65050
|
+
const { stepSettings: _drop, ...rest } = camera;
|
|
65051
|
+
const nextCamera = {
|
|
65052
|
+
...rest,
|
|
65053
|
+
...Object.keys(nextLayer).length > 0 ? { stepSettings: nextLayer } : {}
|
|
65054
|
+
};
|
|
65055
|
+
return {
|
|
65056
|
+
...prev,
|
|
65057
|
+
[key]: nextCamera
|
|
65058
|
+
};
|
|
65059
|
+
});
|
|
65060
|
+
}
|
|
64754
65061
|
/** Purge one device's entry through the durable handle (whole-blob round-trip). */
|
|
64755
65062
|
async deleteCameraSettings(deviceId) {
|
|
64756
65063
|
await this.cameraSettingsState.update((prev) => {
|
|
@@ -65265,12 +65572,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65265
65572
|
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
65266
65573
|
*/
|
|
65267
65574
|
async buildNodeWeights() {
|
|
65575
|
+
if (this.effectiveNodeWeights !== null) return this.effectiveNodeWeights();
|
|
65576
|
+
return this.buildStoredNodeWeights();
|
|
65577
|
+
}
|
|
65578
|
+
/**
|
|
65579
|
+
* The operator's stored node sliders, whatever the share source — what
|
|
65580
|
+
* `manual` applies and what `auto` falls back to before its first answer.
|
|
65581
|
+
*/
|
|
65582
|
+
async buildStoredNodeWeights() {
|
|
65268
65583
|
const blob = await this.agentSettingsState.get();
|
|
65269
65584
|
const weights = {};
|
|
65270
65585
|
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
65271
65586
|
return weights;
|
|
65272
65587
|
}
|
|
65273
65588
|
/**
|
|
65589
|
+
* Where {@link buildNodeWeights} reads from once the share source is wired
|
|
65590
|
+
* (D750): the effective shares (automatic or manual). Every balancer, the
|
|
65591
|
+
* share election and the stress failover read through it, so they can never
|
|
65592
|
+
* disagree on what a node's share is. Unset (tests, early boot) ⇒ stored.
|
|
65593
|
+
*/
|
|
65594
|
+
effectiveNodeWeights = null;
|
|
65595
|
+
setEffectiveNodeWeights(read) {
|
|
65596
|
+
this.effectiveNodeWeights = read;
|
|
65597
|
+
}
|
|
65598
|
+
/**
|
|
65274
65599
|
* The per-(node,device) BASE provisioning (C7.2/C7.4) for one `(node,
|
|
65275
65600
|
* deviceKey)` — the effective `stepId → {modelId?, settings?}` map the
|
|
65276
65601
|
* dispatch provisioning stage attaches onto the resolved step tree before
|
|
@@ -65299,13 +65624,14 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65299
65624
|
async readDeviceProvisioning(deviceId, nodeId, deviceKeys) {
|
|
65300
65625
|
const [agentBlob, cameraMap] = await Promise.all([this.agentSettingsState.get(), this.readCameraSettingsMap()]);
|
|
65301
65626
|
const devices = agentBlob[nodeId]?.inferenceDevices;
|
|
65302
|
-
const
|
|
65627
|
+
const camera = cameraMap[String(deviceId)];
|
|
65628
|
+
const byDevice = camera?.stepOverridesByDevice?.[nodeId];
|
|
65303
65629
|
const out = /* @__PURE__ */ new Map();
|
|
65304
65630
|
for (const deviceKey of deviceKeys) {
|
|
65305
|
-
const
|
|
65631
|
+
const perDevice = deviceKey === "" ? void 0 : byDevice?.[deviceKey];
|
|
65306
65632
|
out.set(deviceKey, {
|
|
65307
65633
|
base: foldNodeStepSettings(devices, deviceKey),
|
|
65308
|
-
override:
|
|
65634
|
+
override: layerCameraStepSettings(camera?.stepSettings, perDevice)
|
|
65309
65635
|
});
|
|
65310
65636
|
}
|
|
65311
65637
|
return out;
|
|
@@ -65369,6 +65695,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65369
65695
|
return `tpl_${Date.now().toString(36).slice(-6)}${rand}`;
|
|
65370
65696
|
}
|
|
65371
65697
|
};
|
|
65698
|
+
/**
|
|
65699
|
+
* The camera's override for one accelerator, with its CAMERA-WIDE settings
|
|
65700
|
+
* (D754) folded underneath: per step, `settings = { ...camera, ...perDevice }`
|
|
65701
|
+
* (a per-accelerator key wins), and the per-accelerator `modelId` kept as is —
|
|
65702
|
+
* the camera layer carries no model. The result is what
|
|
65703
|
+
* `applyDeviceProvisioning` merges over the node base, so the full order is
|
|
65704
|
+
* node ⊕ camera ⊕ camera-per-accelerator. Fresh objects; a camera with neither
|
|
65705
|
+
* layer yields `{}`.
|
|
65706
|
+
*/
|
|
65707
|
+
function layerCameraStepSettings(cameraWide, perDevice) {
|
|
65708
|
+
const out = { ...perDevice };
|
|
65709
|
+
for (const [stepId, settings] of Object.entries(cameraWide ?? {})) {
|
|
65710
|
+
if (Object.keys(settings).length === 0) continue;
|
|
65711
|
+
const own = out[stepId];
|
|
65712
|
+
out[stepId] = {
|
|
65713
|
+
...own,
|
|
65714
|
+
settings: {
|
|
65715
|
+
...settings,
|
|
65716
|
+
...own?.settings
|
|
65717
|
+
}
|
|
65718
|
+
};
|
|
65719
|
+
}
|
|
65720
|
+
return out;
|
|
65721
|
+
}
|
|
65372
65722
|
//#endregion
|
|
65373
65723
|
//#region src/pipeline-watchdog.ts
|
|
65374
65724
|
var DEFAULT_WATCHDOG_THRESHOLDS = {
|
|
@@ -67628,9 +67978,10 @@ var SessionDispatchController = class {
|
|
|
67628
67978
|
deviceId: a.deviceId,
|
|
67629
67979
|
nodeId: a.agentNodeId
|
|
67630
67980
|
}))]);
|
|
67981
|
+
const present = new Set(this.deps.topology.knownRunnerNodeIds());
|
|
67631
67982
|
const election = electNodeByShare({
|
|
67632
67983
|
loads: input.loads,
|
|
67633
|
-
eligible: input.eligible,
|
|
67984
|
+
eligible: input.eligible.filter((nodeId) => present.has(nodeId)),
|
|
67634
67985
|
weights: input.weights,
|
|
67635
67986
|
caps: input.nodeCaps,
|
|
67636
67987
|
runningByNode: running
|
|
@@ -68990,7 +69341,6 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68990
69341
|
readGlobalSettings: () => globalSettings,
|
|
68991
69342
|
onCensus: (entries) => {
|
|
68992
69343
|
motionSliceGate?.noteCensus(entries);
|
|
68993
|
-
deps.onRunnerCensus(entries.flatMap((e) => e.load === null ? [] : [e.load]));
|
|
68994
69344
|
providerSliceGate?.refreshIfStale("runner-census");
|
|
68995
69345
|
}
|
|
68996
69346
|
});
|
|
@@ -69099,7 +69449,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69099
69449
|
attach: (nodeId, config) => placement.attachOn(nodeId, config),
|
|
69100
69450
|
detach: (nodeId, deviceId) => placement.detachOn(nodeId, deviceId),
|
|
69101
69451
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
69102
|
-
readGlobalSettings: () => globalSettings
|
|
69452
|
+
readGlobalSettings: () => globalSettings,
|
|
69453
|
+
readFailoverShares: async () => readSessionPlacement() === "share" ? settingsStore.buildNodeWeights() : null
|
|
69103
69454
|
});
|
|
69104
69455
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
69105
69456
|
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
@@ -69115,6 +69466,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69115
69466
|
nodeStress.start();
|
|
69116
69467
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69117
69468
|
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
69469
|
+
const readShareSource = () => parseShareSource(globalSettings?.[SHARE_SOURCE_SETTING_KEY]);
|
|
69118
69470
|
/**
|
|
69119
69471
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
69120
69472
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -69451,6 +69803,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69451
69803
|
nodeLifecycle.unsubscribeDetectionPipelineReadiness(nodeId);
|
|
69452
69804
|
audio.markNodeOffline(nodeId);
|
|
69453
69805
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
69806
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-offline:${nodeId}`);
|
|
69454
69807
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
69455
69808
|
const msg = errMsg$2(err);
|
|
69456
69809
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
@@ -69494,6 +69847,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69494
69847
|
});
|
|
69495
69848
|
});
|
|
69496
69849
|
reconcile.scheduleReconcile();
|
|
69850
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-online:${nodeId}`);
|
|
69497
69851
|
});
|
|
69498
69852
|
const unsubAgentOnline = () => {
|
|
69499
69853
|
unsubAgentOnlineEvent();
|
|
@@ -69734,7 +70088,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69734
70088
|
unsubOrchestratorSubscriptions,
|
|
69735
70089
|
pendingRetryTimer,
|
|
69736
70090
|
autoRebalanceTimer,
|
|
69737
|
-
sessionPlacement: () => readSessionPlacement()
|
|
70091
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
70092
|
+
shareSource: () => readShareSource()
|
|
69738
70093
|
};
|
|
69739
70094
|
}
|
|
69740
70095
|
//#endregion
|
|
@@ -69783,13 +70138,13 @@ function buildGlobalSettingsSections(options) {
|
|
|
69783
70138
|
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69784
70139
|
type: "select",
|
|
69785
70140
|
label: "Session placement",
|
|
69786
|
-
description: "How a new detection session picks its node and device.
|
|
70141
|
+
description: "How a new detection session picks its node and device. Honour shares (default) = the share of PARALLEL sessions is held to the node and device shares, and a node at its cap yields to a free one. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy.",
|
|
69787
70142
|
options: [{
|
|
69788
|
-
value: "least-loaded",
|
|
69789
|
-
label: "Least-loaded"
|
|
69790
|
-
}, {
|
|
69791
70143
|
value: "share",
|
|
69792
70144
|
label: "Honour shares"
|
|
70145
|
+
}, {
|
|
70146
|
+
value: "least-loaded",
|
|
70147
|
+
label: "Least-loaded"
|
|
69793
70148
|
}],
|
|
69794
70149
|
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69795
70150
|
},
|
|
@@ -70351,9 +70706,9 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70351
70706
|
defaultRows: 4
|
|
70352
70707
|
}];
|
|
70353
70708
|
//#endregion
|
|
70354
|
-
//#region src/
|
|
70709
|
+
//#region src/auto-tune/plan-diff.ts
|
|
70355
70710
|
/**
|
|
70356
|
-
* The
|
|
70711
|
+
* The Auto tune writes, as pure transforms of the orchestrator's two
|
|
70357
70712
|
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
70358
70713
|
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
70359
70714
|
*
|
|
@@ -70377,7 +70732,7 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70377
70732
|
*/
|
|
70378
70733
|
/** The root detector step. */
|
|
70379
70734
|
var DETECTOR_STEP_ID = "object-detection";
|
|
70380
|
-
/** Steps whose `modelId` pin
|
|
70735
|
+
/** Steps whose `modelId` pin Auto tune never touches. */
|
|
70381
70736
|
function isModelPinKept(stepId) {
|
|
70382
70737
|
return stepId.endsWith("-embedding");
|
|
70383
70738
|
}
|
|
@@ -70429,7 +70784,7 @@ function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
|
70429
70784
|
};
|
|
70430
70785
|
}
|
|
70431
70786
|
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70432
|
-
function
|
|
70787
|
+
function applyNodeAutoTune(blob, input) {
|
|
70433
70788
|
const next = { ...blob };
|
|
70434
70789
|
const changes = [];
|
|
70435
70790
|
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
@@ -70499,7 +70854,7 @@ function scrubTree(steps, input, onClear) {
|
|
|
70499
70854
|
});
|
|
70500
70855
|
}
|
|
70501
70856
|
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70502
|
-
function
|
|
70857
|
+
function applyCameraAutoTune(blob, input) {
|
|
70503
70858
|
const next = {};
|
|
70504
70859
|
const clears = [];
|
|
70505
70860
|
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
@@ -70583,13 +70938,15 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70583
70938
|
};
|
|
70584
70939
|
}
|
|
70585
70940
|
//#endregion
|
|
70586
|
-
//#region src/
|
|
70941
|
+
//#region src/auto-tune/auto-tune-service.ts
|
|
70587
70942
|
/**
|
|
70588
|
-
*
|
|
70589
|
-
* the plan the operator confirmed.
|
|
70943
|
+
* Auto tune (D745, D755): gather the inputs, compute the plan, and apply
|
|
70944
|
+
* exactly the plan the operator confirmed.
|
|
70590
70945
|
*
|
|
70591
|
-
* The plan is the recommender (`@camstack/types`
|
|
70592
|
-
* runner node
|
|
70946
|
+
* The plan is the recommender (`@camstack/types` auto-tune) over every
|
|
70947
|
+
* runner node — sized by a capacity floor (`minSessionsPerDevice` parallel
|
|
70948
|
+
* sessions per device at the camera rate), never by observed or estimated
|
|
70949
|
+
* load — plus the diff of the writes (`plan-diff.ts`) and the
|
|
70593
70950
|
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70594
70951
|
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
70595
70952
|
* and refuses a different hash, so a configuration change between the two is
|
|
@@ -70606,6 +70963,9 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70606
70963
|
* A failure after phase 2 leaves "overrides cleared, previous node model",
|
|
70607
70964
|
* which is a valid configuration (the defaults run); re-running converges.
|
|
70608
70965
|
*/
|
|
70966
|
+
function minSessionsOf(input) {
|
|
70967
|
+
return input.minSessionsPerDevice ?? 2;
|
|
70968
|
+
}
|
|
70609
70969
|
function errMsg(err) {
|
|
70610
70970
|
return err instanceof Error ? err.message : String(err);
|
|
70611
70971
|
}
|
|
@@ -70634,117 +70994,45 @@ function boundKeysOf(catalogs) {
|
|
|
70634
70994
|
}
|
|
70635
70995
|
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70636
70996
|
}
|
|
70637
|
-
|
|
70638
|
-
var SUGGEST_MAX_ROUNDS = 3;
|
|
70639
|
-
/** Integer shares summing to 100 (largest remainder). */
|
|
70640
|
-
function toPercent(weights) {
|
|
70641
|
-
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70642
|
-
const out = /* @__PURE__ */ new Map();
|
|
70643
|
-
if (total <= 0) return out;
|
|
70644
|
-
const raw = [...weights].map(([k, w]) => ({
|
|
70645
|
-
k,
|
|
70646
|
-
exact: w / total * 100
|
|
70647
|
-
}));
|
|
70648
|
-
let assigned = 0;
|
|
70649
|
-
for (const r of raw) {
|
|
70650
|
-
out.set(r.k, Math.floor(r.exact));
|
|
70651
|
-
assigned += Math.floor(r.exact);
|
|
70652
|
-
}
|
|
70653
|
-
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70654
|
-
for (let i = 0; i < 100 - assigned; i++) {
|
|
70655
|
-
const r = byRemainder[i % byRemainder.length];
|
|
70656
|
-
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70657
|
-
}
|
|
70658
|
-
return out;
|
|
70659
|
-
}
|
|
70660
|
-
var BestSettingsService = class {
|
|
70997
|
+
var AutoTuneService = class {
|
|
70661
70998
|
deps;
|
|
70662
70999
|
applying = false;
|
|
70663
71000
|
constructor(deps) {
|
|
70664
71001
|
this.deps = deps;
|
|
70665
71002
|
}
|
|
70666
71003
|
async plan(input) {
|
|
70667
|
-
return (await this.computePlan(input.headroom, input
|
|
71004
|
+
return (await this.computePlan(input.headroom, minSessionsOf(input))).plan;
|
|
70668
71005
|
}
|
|
70669
71006
|
/**
|
|
70670
|
-
* Shares proportional to each node's capacity
|
|
70671
|
-
*
|
|
70672
|
-
*
|
|
70673
|
-
*
|
|
70674
|
-
*
|
|
71007
|
+
* Shares proportional to each node's capacity in parallel sessions (D755):
|
|
71008
|
+
* from the latency table at the models Auto tune picks at its defaults
|
|
71009
|
+
* (`hardware`), or from node benchmark results (`benchmark`), a node
|
|
71010
|
+
* without results falling back to `hardware` and saying so. ONE pass, and
|
|
71011
|
+
* nothing but capacity goes in — the automatic shares are exactly this.
|
|
71012
|
+
* Per device it reports an ADVISORY session cap — never written, never
|
|
71013
|
+
* enforced.
|
|
70675
71014
|
*/
|
|
70676
71015
|
async suggestShares(input) {
|
|
70677
|
-
|
|
70678
|
-
|
|
70679
|
-
|
|
70680
|
-
|
|
70681
|
-
|
|
70682
|
-
|
|
70683
|
-
|
|
70684
|
-
|
|
70685
|
-
rounds = round;
|
|
70686
|
-
const rec = await this.recommend(input.headroom, shares);
|
|
70687
|
-
cameraFps = rec.cameraFps;
|
|
70688
|
-
result = this.sharesFrom(rec, input.headroom);
|
|
70689
|
-
const next = {
|
|
70690
|
-
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70691
|
-
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70692
|
-
};
|
|
70693
|
-
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70694
|
-
shares = next;
|
|
70695
|
-
if (settled) break;
|
|
71016
|
+
const source = input.source ?? "hardware";
|
|
71017
|
+
const minSessionsPerDevice = 2;
|
|
71018
|
+
const rec = await this.recommend(input.headroom, minSessionsPerDevice);
|
|
71019
|
+
let measured = /* @__PURE__ */ new Map();
|
|
71020
|
+
if (source === "benchmark") try {
|
|
71021
|
+
measured = await this.deps.measuredCapacity.readMeasuredCapacity(rec.recommendations.map((r) => r.nodeId));
|
|
71022
|
+
} catch (err) {
|
|
71023
|
+
this.deps.logger.warn("load balancing: benchmark results unreadable — sized from hardware", { meta: { error: errMsg(err) } });
|
|
70696
71024
|
}
|
|
70697
|
-
return {
|
|
71025
|
+
return computeCapacityShares({
|
|
71026
|
+
recommendations: rec.recommendations,
|
|
71027
|
+
source,
|
|
71028
|
+
measured,
|
|
70698
71029
|
headroom: input.headroom,
|
|
70699
|
-
cameraFps,
|
|
70700
|
-
|
|
70701
|
-
nodes: [...result]
|
|
70702
|
-
};
|
|
70703
|
-
}
|
|
70704
|
-
sharesFrom(rec, headroom) {
|
|
70705
|
-
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70706
|
-
const perNode = rec.recommendations.map((r) => {
|
|
70707
|
-
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70708
|
-
const fps = d.chosen?.fps ?? 0;
|
|
70709
|
-
return {
|
|
70710
|
-
d,
|
|
70711
|
-
fps,
|
|
70712
|
-
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70713
|
-
};
|
|
70714
|
-
});
|
|
70715
|
-
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70716
|
-
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70717
|
-
return {
|
|
70718
|
-
r,
|
|
70719
|
-
devices,
|
|
70720
|
-
capacity
|
|
70721
|
-
};
|
|
70722
|
-
});
|
|
70723
|
-
const nodePct = toPercent(capacityByNode);
|
|
70724
|
-
return perNode.map(({ r, devices, capacity }) => {
|
|
70725
|
-
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70726
|
-
return {
|
|
70727
|
-
nodeId: r.nodeId,
|
|
70728
|
-
share: nodePct.get(r.nodeId) ?? null,
|
|
70729
|
-
capacitySessions: capacity,
|
|
70730
|
-
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70731
|
-
devices: devices.map((x) => ({
|
|
70732
|
-
deviceKey: x.d.deviceKey,
|
|
70733
|
-
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70734
|
-
modelId: x.d.chosen?.modelId ?? null,
|
|
70735
|
-
fps: x.d.chosen?.fps ?? null,
|
|
70736
|
-
measured: x.d.chosen?.measured ?? false,
|
|
70737
|
-
suggestedMaxSessions: x.sessions
|
|
70738
|
-
}))
|
|
70739
|
-
};
|
|
71030
|
+
cameraFps: rec.cameraFps,
|
|
71031
|
+
minSessionsPerDevice
|
|
70740
71032
|
});
|
|
70741
71033
|
}
|
|
70742
|
-
/**
|
|
70743
|
-
|
|
70744
|
-
* round) forces share placement with those shares overlaid on the stored
|
|
70745
|
-
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70746
|
-
*/
|
|
70747
|
-
async recommend(headroom, shares) {
|
|
71034
|
+
/** Gather every node and recommend. Nothing but hardware, catalog and the sizing goes in. */
|
|
71035
|
+
async recommend(headroom, minSessionsPerDevice) {
|
|
70748
71036
|
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70749
71037
|
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70750
71038
|
const [devices, hardware, catalog] = await Promise.all([
|
|
@@ -70759,7 +71047,6 @@ var BestSettingsService = class {
|
|
|
70759
71047
|
catalog
|
|
70760
71048
|
};
|
|
70761
71049
|
}));
|
|
70762
|
-
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? stored;
|
|
70763
71050
|
const nodes = gathered.map((g) => ({
|
|
70764
71051
|
nodeId: g.nodeId,
|
|
70765
71052
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70772,40 +71059,22 @@ var BestSettingsService = class {
|
|
|
70772
71059
|
})),
|
|
70773
71060
|
formatsByModel: formatsByModelOf(g.catalog)
|
|
70774
71061
|
}));
|
|
70775
|
-
const
|
|
70776
|
-
|
|
70777
|
-
deviceKey: d.key,
|
|
70778
|
-
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70779
|
-
})) : []);
|
|
70780
|
-
const cameras = await this.deps.cameraLoads();
|
|
70781
|
-
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70782
|
-
const demand = estimateDeviceDemand({
|
|
70783
|
-
cameras,
|
|
70784
|
-
eligible,
|
|
70785
|
-
peaks: this.deps.peaks(),
|
|
70786
|
-
...policy === "share" ? { shares: {
|
|
70787
|
-
nodes: {
|
|
70788
|
-
...await this.deps.readNodeShares(),
|
|
70789
|
-
...shares?.nodes
|
|
70790
|
-
},
|
|
70791
|
-
clusterPeak: this.deps.clusterPeak()
|
|
70792
|
-
} } : {}
|
|
70793
|
-
});
|
|
70794
|
-
const recommendations = nodes.map((node) => recommendNode({
|
|
70795
|
-
node,
|
|
70796
|
-
demand,
|
|
70797
|
-
headroom
|
|
70798
|
-
}));
|
|
70799
|
-
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
71062
|
+
const observedFps = await this.deps.cameraFps();
|
|
71063
|
+
const cameraFps = observedFps !== null && observedFps > 0 ? observedFps : 10;
|
|
70800
71064
|
return {
|
|
70801
71065
|
gathered,
|
|
70802
71066
|
nodes,
|
|
70803
|
-
recommendations
|
|
70804
|
-
|
|
71067
|
+
recommendations: nodes.map((node) => recommendNode({
|
|
71068
|
+
node,
|
|
71069
|
+
headroom,
|
|
71070
|
+
minSessionsPerDevice,
|
|
71071
|
+
cameraFps
|
|
71072
|
+
})),
|
|
71073
|
+
cameraFps
|
|
70805
71074
|
};
|
|
70806
71075
|
}
|
|
70807
|
-
async computePlan(headroom,
|
|
70808
|
-
const { gathered, nodes, recommendations } = await this.recommend(headroom,
|
|
71076
|
+
async computePlan(headroom, minSessionsPerDevice) {
|
|
71077
|
+
const { gathered, nodes, recommendations, cameraFps } = await this.recommend(headroom, minSessionsPerDevice);
|
|
70809
71078
|
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70810
71079
|
const targets = /* @__PURE__ */ new Map();
|
|
70811
71080
|
const formats = /* @__PURE__ */ new Map();
|
|
@@ -70832,8 +71101,8 @@ var BestSettingsService = class {
|
|
|
70832
71101
|
targets,
|
|
70833
71102
|
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70834
71103
|
};
|
|
70835
|
-
const nodeChanges =
|
|
70836
|
-
const cameraClears =
|
|
71104
|
+
const nodeChanges = applyNodeAutoTune(await this.deps.readAgentSettings(), diffInput).changes;
|
|
71105
|
+
const cameraClears = applyCameraAutoTune(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70837
71106
|
const warnings = [];
|
|
70838
71107
|
let detectorStoreClears = [];
|
|
70839
71108
|
if (touchedNodes.size > 0) try {
|
|
@@ -70865,9 +71134,10 @@ var BestSettingsService = class {
|
|
|
70865
71134
|
return {
|
|
70866
71135
|
plan: {
|
|
70867
71136
|
planHash: (0, node_crypto.createHash)("sha256").update(canonical({
|
|
70868
|
-
version:
|
|
71137
|
+
version: 2,
|
|
70869
71138
|
headroom,
|
|
70870
|
-
|
|
71139
|
+
minSessionsPerDevice,
|
|
71140
|
+
cameraFps,
|
|
70871
71141
|
picks: recommendations.map((r) => ({
|
|
70872
71142
|
nodeId: r.nodeId,
|
|
70873
71143
|
skipped: r.skipped,
|
|
@@ -70879,7 +71149,8 @@ var BestSettingsService = class {
|
|
|
70879
71149
|
})).digest("hex"),
|
|
70880
71150
|
family: "yolov9",
|
|
70881
71151
|
headroom,
|
|
70882
|
-
|
|
71152
|
+
minSessionsPerDevice,
|
|
71153
|
+
cameraFps,
|
|
70883
71154
|
generatedAt: this.deps.now(),
|
|
70884
71155
|
nodes: recommendations,
|
|
70885
71156
|
nodeChanges: [...nodeChanges],
|
|
@@ -70904,9 +71175,9 @@ var BestSettingsService = class {
|
|
|
70904
71175
|
}
|
|
70905
71176
|
}
|
|
70906
71177
|
async applyLocked(input) {
|
|
70907
|
-
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
71178
|
+
const { plan, diffInput, formats } = await this.computePlan(input.headroom, minSessionsOf(input));
|
|
70908
71179
|
if (plan.planHash !== input.planHash) {
|
|
70909
|
-
this.deps.logger.warn("
|
|
71180
|
+
this.deps.logger.warn("auto tune: stale plan refused", { meta: {
|
|
70910
71181
|
confirmed: input.planHash,
|
|
70911
71182
|
current: plan.planHash
|
|
70912
71183
|
} });
|
|
@@ -70938,15 +71209,15 @@ var BestSettingsService = class {
|
|
|
70938
71209
|
ok: false,
|
|
70939
71210
|
detail: errMsg(err)
|
|
70940
71211
|
});
|
|
70941
|
-
this.deps.logger.warn("
|
|
71212
|
+
this.deps.logger.warn("auto tune: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70942
71213
|
return {
|
|
70943
71214
|
kind: "aborted",
|
|
70944
71215
|
phases
|
|
70945
71216
|
};
|
|
70946
71217
|
}
|
|
70947
71218
|
try {
|
|
70948
|
-
await this.deps.rewriteCameraSettings((prev) =>
|
|
70949
|
-
for (const c of plan.cameraClears) this.deps.logger.info("
|
|
71219
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraAutoTune(prev, diffInput).next);
|
|
71220
|
+
for (const c of plan.cameraClears) this.deps.logger.info("auto tune: camera override cleared", {
|
|
70950
71221
|
tags: {
|
|
70951
71222
|
deviceId: c.deviceId,
|
|
70952
71223
|
nodeId: c.nodeId
|
|
@@ -70990,8 +71261,8 @@ var BestSettingsService = class {
|
|
|
70990
71261
|
});
|
|
70991
71262
|
}
|
|
70992
71263
|
try {
|
|
70993
|
-
await this.deps.rewriteAgentSettings((prev) =>
|
|
70994
|
-
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("
|
|
71264
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeAutoTune(prev, diffInput).next);
|
|
71265
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("auto tune: node model set", {
|
|
70995
71266
|
tags: { nodeId: c.nodeId },
|
|
70996
71267
|
meta: {
|
|
70997
71268
|
nodeId: c.nodeId,
|
|
@@ -71025,14 +71296,14 @@ var BestSettingsService = class {
|
|
|
71025
71296
|
ok: failed.length === 0,
|
|
71026
71297
|
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
71027
71298
|
});
|
|
71028
|
-
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("
|
|
71299
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("auto tune: node skipped", {
|
|
71029
71300
|
tags: { nodeId: rec.nodeId },
|
|
71030
71301
|
meta: {
|
|
71031
71302
|
nodeId: rec.nodeId,
|
|
71032
71303
|
reasons: rec.reasons
|
|
71033
71304
|
}
|
|
71034
71305
|
});
|
|
71035
|
-
this.deps.logger.info("
|
|
71306
|
+
this.deps.logger.info("auto tune: applied", { meta: {
|
|
71036
71307
|
planHash: plan.planHash,
|
|
71037
71308
|
nodeChanges: plan.nodeChanges.length,
|
|
71038
71309
|
cameraClears: plan.cameraClears.length,
|
|
@@ -71047,83 +71318,6 @@ var BestSettingsService = class {
|
|
|
71047
71318
|
}
|
|
71048
71319
|
};
|
|
71049
71320
|
//#endregion
|
|
71050
|
-
//#region src/best-settings/device-peak-tracker.ts
|
|
71051
|
-
var BUCKET_MS = 3600 * 1e3;
|
|
71052
|
-
var DevicePeakTracker = class DevicePeakTracker {
|
|
71053
|
-
now;
|
|
71054
|
-
series = /* @__PURE__ */ new Map();
|
|
71055
|
-
cluster = null;
|
|
71056
|
-
constructor(now = () => Date.now()) {
|
|
71057
|
-
this.now = now;
|
|
71058
|
-
}
|
|
71059
|
-
static key(nodeId, deviceKey) {
|
|
71060
|
-
return `${nodeId}\u0000${deviceKey}`;
|
|
71061
|
-
}
|
|
71062
|
-
/** Fold one runner census in. A runner that did not answer is simply absent. */
|
|
71063
|
-
noteCensus(loads) {
|
|
71064
|
-
const now = this.now();
|
|
71065
|
-
const bucket = now - now % BUCKET_MS;
|
|
71066
|
-
let clusterTotal = 0;
|
|
71067
|
-
for (const load of loads) {
|
|
71068
|
-
const rows = load.devices.length > 0 ? load.devices : [{
|
|
71069
|
-
deviceKey: "",
|
|
71070
|
-
attachedCameras: load.attachedCameras
|
|
71071
|
-
}];
|
|
71072
|
-
for (const device of rows) {
|
|
71073
|
-
clusterTotal += device.attachedCameras;
|
|
71074
|
-
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
71075
|
-
let s = this.series.get(key);
|
|
71076
|
-
if (s === void 0) {
|
|
71077
|
-
s = {
|
|
71078
|
-
firstSeenAt: now,
|
|
71079
|
-
buckets: /* @__PURE__ */ new Map()
|
|
71080
|
-
};
|
|
71081
|
-
this.series.set(key, s);
|
|
71082
|
-
}
|
|
71083
|
-
s.buckets.set(bucket, Math.max(s.buckets.get(bucket) ?? 0, device.attachedCameras));
|
|
71084
|
-
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
71085
|
-
}
|
|
71086
|
-
}
|
|
71087
|
-
if (loads.length === 0) return;
|
|
71088
|
-
const c = this.cluster ?? {
|
|
71089
|
-
firstSeenAt: now,
|
|
71090
|
-
buckets: /* @__PURE__ */ new Map()
|
|
71091
|
-
};
|
|
71092
|
-
this.cluster = c;
|
|
71093
|
-
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71094
|
-
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71095
|
-
}
|
|
71096
|
-
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71097
|
-
clusterPeak() {
|
|
71098
|
-
const c = this.cluster;
|
|
71099
|
-
if (c === null) return null;
|
|
71100
|
-
const now = this.now();
|
|
71101
|
-
let peak = 0;
|
|
71102
|
-
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71103
|
-
return {
|
|
71104
|
-
peakSessions: peak,
|
|
71105
|
-
observedForMs: now - c.firstSeenAt
|
|
71106
|
-
};
|
|
71107
|
-
}
|
|
71108
|
-
/** Every observed device's peak over the window and how long it has been watched. */
|
|
71109
|
-
peaks() {
|
|
71110
|
-
const now = this.now();
|
|
71111
|
-
const out = [];
|
|
71112
|
-
for (const [key, s] of this.series) {
|
|
71113
|
-
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71114
|
-
let peak = 0;
|
|
71115
|
-
for (const [start, value] of s.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71116
|
-
out.push({
|
|
71117
|
-
nodeId,
|
|
71118
|
-
deviceKey,
|
|
71119
|
-
peakSessions: peak,
|
|
71120
|
-
observedForMs: now - s.firstSeenAt
|
|
71121
|
-
});
|
|
71122
|
-
}
|
|
71123
|
-
return out.toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
71124
|
-
}
|
|
71125
|
-
};
|
|
71126
|
-
//#endregion
|
|
71127
71321
|
//#region src/session-placement-stats.ts
|
|
71128
71322
|
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71129
71323
|
function keyOf(nodeId, deviceKey) {
|
|
@@ -71289,6 +71483,212 @@ function applyLoadBalancingShares(prev, write) {
|
|
|
71289
71483
|
return next;
|
|
71290
71484
|
}
|
|
71291
71485
|
//#endregion
|
|
71486
|
+
//#region src/load-balancing/effective-shares-service.ts
|
|
71487
|
+
/** Skips that mean "not readable yet", never "cannot take sessions". */
|
|
71488
|
+
var NOT_READY_REASONS = new Set(["node-offline", "no-hardware-facts"]);
|
|
71489
|
+
function capacityFrom(result) {
|
|
71490
|
+
const out = {};
|
|
71491
|
+
for (const node of result.nodes) if (node.capacitySource !== null) out[node.nodeId] = {
|
|
71492
|
+
source: node.capacitySource,
|
|
71493
|
+
fallback: node.capacityFallback
|
|
71494
|
+
};
|
|
71495
|
+
return out;
|
|
71496
|
+
}
|
|
71497
|
+
function weightsFrom(result) {
|
|
71498
|
+
const nodes = {};
|
|
71499
|
+
const devices = {};
|
|
71500
|
+
for (const node of result.nodes) {
|
|
71501
|
+
if (node.share !== null && node.share > 0) nodes[node.nodeId] = node.share;
|
|
71502
|
+
const perDevice = {};
|
|
71503
|
+
for (const d of node.devices) if (d.share > 0) perDevice[d.deviceKey] = d.share;
|
|
71504
|
+
if (Object.keys(perDevice).length > 0) devices[node.nodeId] = perDevice;
|
|
71505
|
+
}
|
|
71506
|
+
return {
|
|
71507
|
+
nodes,
|
|
71508
|
+
devices
|
|
71509
|
+
};
|
|
71510
|
+
}
|
|
71511
|
+
var EffectiveSharesService = class EffectiveSharesService {
|
|
71512
|
+
deps;
|
|
71513
|
+
/** An automatic answer older than this is recomputed by the reconcile. */
|
|
71514
|
+
static MAX_AGE_MS = 15 * 6e4;
|
|
71515
|
+
/** Reconcile cadence. */
|
|
71516
|
+
static RECONCILE_MS = 6e4;
|
|
71517
|
+
answer = null;
|
|
71518
|
+
inFlight = null;
|
|
71519
|
+
pendingTrigger = null;
|
|
71520
|
+
timer = null;
|
|
71521
|
+
constructor(deps) {
|
|
71522
|
+
this.deps = deps;
|
|
71523
|
+
}
|
|
71524
|
+
now() {
|
|
71525
|
+
return this.deps.now?.() ?? Date.now();
|
|
71526
|
+
}
|
|
71527
|
+
/** The last automatic answer for the CURRENT source; null = none yet. */
|
|
71528
|
+
current() {
|
|
71529
|
+
return this.usable();
|
|
71530
|
+
}
|
|
71531
|
+
/** The answer, when the source is automatic and the answer was computed for it. */
|
|
71532
|
+
usable() {
|
|
71533
|
+
const source = this.deps.readSource();
|
|
71534
|
+
if (!isAutomaticShareSource(source)) return null;
|
|
71535
|
+
return this.answer !== null && this.answer.source === source ? this.answer : null;
|
|
71536
|
+
}
|
|
71537
|
+
/** Node weights the election reads. */
|
|
71538
|
+
async nodeWeights() {
|
|
71539
|
+
const answer = this.usable();
|
|
71540
|
+
if (answer !== null) return { ...answer.weights.nodes };
|
|
71541
|
+
return { ...(await this.deps.readStored()).nodes };
|
|
71542
|
+
}
|
|
71543
|
+
/** The weights of a node's ELIGIBLE devices (`eligible` carries their stored weights). */
|
|
71544
|
+
deviceWeights(nodeId, eligible) {
|
|
71545
|
+
const answer = this.usable();
|
|
71546
|
+
if (answer === null) return { ...eligible };
|
|
71547
|
+
const auto = answer.weights.devices[nodeId] ?? {};
|
|
71548
|
+
const out = {};
|
|
71549
|
+
for (const [key, stored] of Object.entries(eligible)) {
|
|
71550
|
+
const share = auto[key];
|
|
71551
|
+
out[key] = share !== void 0 && share > 0 ? share : stored;
|
|
71552
|
+
}
|
|
71553
|
+
return out;
|
|
71554
|
+
}
|
|
71555
|
+
/** Both maps, as the Auto tune recommender charges load by them. */
|
|
71556
|
+
async effectiveWeights() {
|
|
71557
|
+
return this.usable()?.weights ?? this.deps.readStored();
|
|
71558
|
+
}
|
|
71559
|
+
/**
|
|
71560
|
+
* Recompute now (automatic sources only). Concurrent calls coalesce: one
|
|
71561
|
+
* runs, and one trailing run picks up whatever was reported meanwhile.
|
|
71562
|
+
*/
|
|
71563
|
+
async recompute(trigger) {
|
|
71564
|
+
if (!isAutomaticShareSource(this.deps.readSource())) return;
|
|
71565
|
+
if (this.inFlight !== null) {
|
|
71566
|
+
this.pendingTrigger = trigger;
|
|
71567
|
+
return this.inFlight;
|
|
71568
|
+
}
|
|
71569
|
+
this.inFlight = this.run(trigger).finally(() => {
|
|
71570
|
+
this.inFlight = null;
|
|
71571
|
+
});
|
|
71572
|
+
await this.inFlight;
|
|
71573
|
+
const trailing = this.pendingTrigger;
|
|
71574
|
+
if (trailing !== null) {
|
|
71575
|
+
this.pendingTrigger = null;
|
|
71576
|
+
await this.recompute(trailing);
|
|
71577
|
+
}
|
|
71578
|
+
}
|
|
71579
|
+
async run(trigger) {
|
|
71580
|
+
const source = this.deps.readSource();
|
|
71581
|
+
if (!isAutomaticShareSource(source)) return;
|
|
71582
|
+
let signature;
|
|
71583
|
+
let result;
|
|
71584
|
+
try {
|
|
71585
|
+
signature = await this.deps.topologySignature();
|
|
71586
|
+
result = await this.deps.suggest(source);
|
|
71587
|
+
} catch (err) {
|
|
71588
|
+
this.deps.logger.warn("load balancing: automatic shares not recomputed — last answer kept", { meta: {
|
|
71589
|
+
trigger,
|
|
71590
|
+
error: errMsg$2(err),
|
|
71591
|
+
hasAnswer: this.answer !== null
|
|
71592
|
+
} });
|
|
71593
|
+
return;
|
|
71594
|
+
}
|
|
71595
|
+
const before = this.answer?.weights ?? null;
|
|
71596
|
+
const after = weightsFrom(result);
|
|
71597
|
+
const notReady = result.nodes.filter((n) => n.share === null && NOT_READY_REASONS.has(n.skippedReason ?? "")).map((n) => n.nodeId).toSorted();
|
|
71598
|
+
const capacity = capacityFrom(result);
|
|
71599
|
+
this.answer = {
|
|
71600
|
+
weights: after,
|
|
71601
|
+
capacity,
|
|
71602
|
+
source,
|
|
71603
|
+
computedAt: this.now(),
|
|
71604
|
+
trigger,
|
|
71605
|
+
signature,
|
|
71606
|
+
notReady
|
|
71607
|
+
};
|
|
71608
|
+
const nodeIds = [...new Set([...Object.keys(before?.nodes ?? {}), ...result.nodes.map((n) => n.nodeId)])];
|
|
71609
|
+
const log = before !== null && JSON.stringify(before) === JSON.stringify(after) && trigger.startsWith("reconcile:not-ready") ? this.deps.logger.debug.bind(this.deps.logger) : this.deps.logger.info.bind(this.deps.logger);
|
|
71610
|
+
for (const nodeId of nodeIds.toSorted()) log("load balancing: automatic shares recomputed", {
|
|
71611
|
+
tags: { nodeId },
|
|
71612
|
+
meta: {
|
|
71613
|
+
trigger,
|
|
71614
|
+
source,
|
|
71615
|
+
capacitySource: capacity[nodeId]?.source ?? null,
|
|
71616
|
+
capacityFallback: capacity[nodeId]?.fallback ?? null,
|
|
71617
|
+
notReady: notReady.includes(nodeId),
|
|
71618
|
+
before: before?.nodes[nodeId] ?? null,
|
|
71619
|
+
after: after.nodes[nodeId] ?? null,
|
|
71620
|
+
devicesBefore: JSON.stringify(before?.devices[nodeId] ?? {}),
|
|
71621
|
+
devicesAfter: JSON.stringify(after.devices[nodeId] ?? {})
|
|
71622
|
+
}
|
|
71623
|
+
});
|
|
71624
|
+
}
|
|
71625
|
+
/**
|
|
71626
|
+
* Recompute if there is no answer for the current source, a node was not
|
|
71627
|
+
* readable yet, the topology changed or the answer has grown old
|
|
71628
|
+
* (automatic sources only).
|
|
71629
|
+
*/
|
|
71630
|
+
async reconcile() {
|
|
71631
|
+
const source = this.deps.readSource();
|
|
71632
|
+
if (!isAutomaticShareSource(source)) return;
|
|
71633
|
+
if (this.answer === null) return this.recompute("reconcile:first");
|
|
71634
|
+
if (this.answer.source !== source) return this.recompute(`reconcile:source:${source}`);
|
|
71635
|
+
if (this.answer.notReady.length > 0) return this.recompute(`reconcile:not-ready:${this.answer.notReady.join(",")}`);
|
|
71636
|
+
if (this.now() - this.answer.computedAt > EffectiveSharesService.MAX_AGE_MS) return this.recompute("reconcile:age");
|
|
71637
|
+
let signature;
|
|
71638
|
+
try {
|
|
71639
|
+
signature = await this.deps.topologySignature();
|
|
71640
|
+
} catch (err) {
|
|
71641
|
+
this.deps.logger.debug("load balancing: topology signature unreadable — reconcile skipped", { meta: { error: errMsg$2(err) } });
|
|
71642
|
+
return;
|
|
71643
|
+
}
|
|
71644
|
+
if (signature !== this.answer.signature) return this.recompute("reconcile:topology");
|
|
71645
|
+
}
|
|
71646
|
+
start() {
|
|
71647
|
+
if (this.timer !== null) return;
|
|
71648
|
+
this.timer = setInterval(() => {
|
|
71649
|
+
this.reconcile();
|
|
71650
|
+
}, EffectiveSharesService.RECONCILE_MS);
|
|
71651
|
+
this.timer.unref?.();
|
|
71652
|
+
}
|
|
71653
|
+
stop() {
|
|
71654
|
+
if (this.timer !== null) clearInterval(this.timer);
|
|
71655
|
+
this.timer = null;
|
|
71656
|
+
}
|
|
71657
|
+
};
|
|
71658
|
+
//#endregion
|
|
71659
|
+
//#region src/load-balancing/share-inputs.ts
|
|
71660
|
+
/** The operator's sliders: node `detectWeight` and device `weight` (positive only). */
|
|
71661
|
+
function storedShareWeights(blob) {
|
|
71662
|
+
const nodes = {};
|
|
71663
|
+
const devices = {};
|
|
71664
|
+
for (const [nodeId, settings] of Object.entries(blob)) {
|
|
71665
|
+
if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) nodes[nodeId] = settings.detectWeight;
|
|
71666
|
+
const perDevice = {};
|
|
71667
|
+
for (const [key, entry] of Object.entries(settings.inferenceDevices ?? {})) if (typeof entry.weight === "number" && entry.weight > 0) perDevice[key] = entry.weight;
|
|
71668
|
+
if (Object.keys(perDevice).length > 0) devices[nodeId] = perDevice;
|
|
71669
|
+
}
|
|
71670
|
+
return {
|
|
71671
|
+
nodes,
|
|
71672
|
+
devices
|
|
71673
|
+
};
|
|
71674
|
+
}
|
|
71675
|
+
/**
|
|
71676
|
+
* A string that changes whenever an input of the automatic shares does: the
|
|
71677
|
+
* nodes present, each node's detect capability, which devices are switched on
|
|
71678
|
+
* and which detector model each runs. Sliders and caps are left out — they do
|
|
71679
|
+
* not change capacity.
|
|
71680
|
+
*/
|
|
71681
|
+
function shareTopologySignature(nodeIds, blob) {
|
|
71682
|
+
return [...nodeIds].toSorted().map((nodeId) => {
|
|
71683
|
+
const settings = blob[nodeId];
|
|
71684
|
+
const devices = Object.entries(settings?.inferenceDevices ?? {}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => {
|
|
71685
|
+
const model = entry.steps?.["object-detection"]?.modelId ?? "";
|
|
71686
|
+
return `${key}=${entry.enabled ? "on" : "off"}:${model}`;
|
|
71687
|
+
});
|
|
71688
|
+
return `${nodeId}|detect=${String(settings?.detect ?? "default")}|${devices.join(",")}`;
|
|
71689
|
+
}).join(";");
|
|
71690
|
+
}
|
|
71691
|
+
//#endregion
|
|
71292
71692
|
//#region src/stale-device-pins.ts
|
|
71293
71693
|
function findStaleDevicePins(pins, nodes) {
|
|
71294
71694
|
const usable = /* @__PURE__ */ new Map();
|
|
@@ -71372,12 +71772,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71372
71772
|
*/
|
|
71373
71773
|
clusterModelChoiceReady = false;
|
|
71374
71774
|
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
71375
|
-
devicePeaks = new DevicePeakTracker();
|
|
71376
71775
|
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71377
71776
|
placementStats = new SessionPlacementStats();
|
|
71378
|
-
/** The stored session placement policy (D748
|
|
71379
|
-
readSessionPlacement = () =>
|
|
71380
|
-
|
|
71777
|
+
/** The stored session placement policy (D748, default `share` since D750). */
|
|
71778
|
+
readSessionPlacement = () => DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
71779
|
+
/** Where the shares come from (D750); `auto` before init. */
|
|
71780
|
+
readShareSource = () => DEFAULT_SHARE_SOURCE;
|
|
71781
|
+
autoTuneService = null;
|
|
71782
|
+
effectiveSharesService = null;
|
|
71381
71783
|
/**
|
|
71382
71784
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
71383
71785
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -71626,6 +72028,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71626
72028
|
getEligibleInferenceDevices: (nodeId, read) => this.resolveEligibleInferenceDevices(nodeId, read),
|
|
71627
72029
|
getEligibleInferenceDevicesIfCached: (nodeId) => this.resolveEligibleInferenceDevicesIfCached(nodeId),
|
|
71628
72030
|
forgetInferenceDeviceProbe: (nodeId) => this.forgetInferenceDeviceProbe(nodeId),
|
|
72031
|
+
noteShareTopologyChange: (trigger) => this.noteShareChange(trigger),
|
|
71629
72032
|
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
71630
72033
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
71631
72034
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
@@ -71654,7 +72057,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71654
72057
|
rebalance: (opts) => this.rebalance(opts),
|
|
71655
72058
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
71656
72059
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
71657
|
-
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
71658
72060
|
placementStats: this.placementStats,
|
|
71659
72061
|
setTopology: (topology) => {
|
|
71660
72062
|
this.topology = topology;
|
|
@@ -71683,6 +72085,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71683
72085
|
this.placement = controllers.placement;
|
|
71684
72086
|
this.session = controllers.session;
|
|
71685
72087
|
this.readSessionPlacement = controllers.sessionPlacement;
|
|
72088
|
+
this.readShareSource = controllers.shareSource;
|
|
72089
|
+
this.settingsStore.setEffectiveNodeWeights(() => this.effectiveShares().nodeWeights());
|
|
72090
|
+
this.effectiveShares().start();
|
|
72091
|
+
this.noteShareChange("startup");
|
|
71686
72092
|
this.deviceConfig = controllers.deviceConfig;
|
|
71687
72093
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
71688
72094
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71808,6 +72214,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71808
72214
|
this.detectionWiring = null;
|
|
71809
72215
|
this.unsubAgentOffline?.();
|
|
71810
72216
|
this.unsubAgentOffline = null;
|
|
72217
|
+
this.effectiveSharesService?.stop();
|
|
71811
72218
|
this.unsubAgentOnline?.();
|
|
71812
72219
|
this.unsubAgentOnline = null;
|
|
71813
72220
|
this.nodeLifecycle?.dispose();
|
|
@@ -72210,6 +72617,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72210
72617
|
await this.topology.refreshNodeCapabilities();
|
|
72211
72618
|
this.topology.addKnownRunnerNode(input.agentNodeId);
|
|
72212
72619
|
this.reconcile.schedulePendingRetry();
|
|
72620
|
+
if (input.detect !== void 0) this.noteShareChange(`node-detect:${input.agentNodeId}`);
|
|
72213
72621
|
return { success: true };
|
|
72214
72622
|
}
|
|
72215
72623
|
/**
|
|
@@ -72299,6 +72707,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72299
72707
|
});
|
|
72300
72708
|
await this.resetRemovedModelPins("node-device-write");
|
|
72301
72709
|
this.reconcile.schedulePendingRetry();
|
|
72710
|
+
this.noteShareChange(`node-devices:${input.agentNodeId}`);
|
|
72302
72711
|
return { success: true };
|
|
72303
72712
|
}
|
|
72304
72713
|
/**
|
|
@@ -72379,7 +72788,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72379
72788
|
});
|
|
72380
72789
|
throw new Error(`inference device eligibility: node "${nodeId}" catalog not answered`);
|
|
72381
72790
|
}
|
|
72382
|
-
return this.electableDevices(nodeId, probed, catalog);
|
|
72791
|
+
return this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, probed, catalog));
|
|
72383
72792
|
}
|
|
72384
72793
|
/**
|
|
72385
72794
|
* Forget a node's last device probe (pre-ship review, M4). Called when the
|
|
@@ -72423,7 +72832,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72423
72832
|
this.refreshInferenceDeviceProbe(nodeId);
|
|
72424
72833
|
return {
|
|
72425
72834
|
kind: "eligible",
|
|
72426
|
-
eligible: await this.electableDevices(nodeId, known, catalog)
|
|
72835
|
+
eligible: this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, known, catalog))
|
|
72427
72836
|
};
|
|
72428
72837
|
}
|
|
72429
72838
|
/**
|
|
@@ -72753,10 +73162,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72753
73162
|
clearedCameraOverrides
|
|
72754
73163
|
};
|
|
72755
73164
|
}
|
|
72756
|
-
|
|
72757
|
-
if (this.
|
|
73165
|
+
autoTune() {
|
|
73166
|
+
if (this.autoTuneService !== null) return this.autoTuneService;
|
|
72758
73167
|
const store = this.settingsStore;
|
|
72759
|
-
this.
|
|
73168
|
+
this.autoTuneService = new AutoTuneService({
|
|
72760
73169
|
logger: this.ctx.logger,
|
|
72761
73170
|
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72762
73171
|
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
@@ -72770,11 +73179,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72770
73179
|
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72771
73180
|
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72772
73181
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72773
|
-
|
|
72774
|
-
|
|
72775
|
-
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72776
|
-
sessionPlacement: () => this.readSessionPlacement(),
|
|
72777
|
-
readNodeShares: () => store.buildNodeWeights(),
|
|
73182
|
+
cameraFps: async () => this.autoTuneCameraFps(),
|
|
73183
|
+
measuredCapacity: NO_MEASURED_CAPACITY,
|
|
72778
73184
|
detectorStore: async (input) => {
|
|
72779
73185
|
const api = this.ctx.api;
|
|
72780
73186
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72797,33 +73203,23 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72797
73203
|
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72798
73204
|
now: () => Date.now()
|
|
72799
73205
|
});
|
|
72800
|
-
return this.
|
|
72801
|
-
}
|
|
72802
|
-
/**
|
|
72803
|
-
|
|
72804
|
-
|
|
72805
|
-
|
|
72806
|
-
|
|
72807
|
-
|
|
72808
|
-
|
|
72809
|
-
|
|
72810
|
-
|
|
72811
|
-
|
|
72812
|
-
mode: config.detectionMode,
|
|
72813
|
-
fps: config.detectionFps,
|
|
72814
|
-
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72815
|
-
standingNodeId: standingNode,
|
|
72816
|
-
nodePinId: nodePin,
|
|
72817
|
-
devicePin: pinnedKey !== null ? {
|
|
72818
|
-
nodeId: nodePin ?? standingNode,
|
|
72819
|
-
deviceKey: pinnedKey
|
|
72820
|
-
} : null
|
|
72821
|
-
});
|
|
73206
|
+
return this.autoTuneService;
|
|
73207
|
+
}
|
|
73208
|
+
/**
|
|
73209
|
+
* The rate one detection session is counted at (D755): the fastest camera
|
|
73210
|
+
* with detection started that runs the pipeline root; null when none does.
|
|
73211
|
+
*/
|
|
73212
|
+
autoTuneCameraFps() {
|
|
73213
|
+
let fps = 0;
|
|
73214
|
+
for (const config of this.detectionWiring.activeDetections.values()) {
|
|
73215
|
+
if (config.detectionMode === "disabled") continue;
|
|
73216
|
+
if (!(config.detectionSources ?? ["pipeline"]).includes("pipeline")) continue;
|
|
73217
|
+
fps = Math.max(fps, config.detectionFps);
|
|
72822
73218
|
}
|
|
72823
|
-
return
|
|
73219
|
+
return fps > 0 ? fps : null;
|
|
72824
73220
|
}
|
|
72825
|
-
async
|
|
72826
|
-
return this.
|
|
73221
|
+
async planAutoTune(input) {
|
|
73222
|
+
return this.autoTune().plan(input);
|
|
72827
73223
|
}
|
|
72828
73224
|
/**
|
|
72829
73225
|
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
@@ -72879,10 +73275,79 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72879
73275
|
return findStaleDevicePins(pins, offered);
|
|
72880
73276
|
}
|
|
72881
73277
|
async suggestShares(input) {
|
|
72882
|
-
return this.
|
|
73278
|
+
return this.autoTune().suggestShares(input);
|
|
72883
73279
|
}
|
|
72884
|
-
async
|
|
72885
|
-
|
|
73280
|
+
async applyAutoTune(input) {
|
|
73281
|
+
const result = await this.autoTune().apply(input);
|
|
73282
|
+
if (result.kind === "applied") this.noteShareChange("auto-tune-applied");
|
|
73283
|
+
return result;
|
|
73284
|
+
}
|
|
73285
|
+
effectiveShares() {
|
|
73286
|
+
if (this.effectiveSharesService !== null) return this.effectiveSharesService;
|
|
73287
|
+
const store = this.settingsStore;
|
|
73288
|
+
this.effectiveSharesService = new EffectiveSharesService({
|
|
73289
|
+
readSource: () => this.readShareSource(),
|
|
73290
|
+
readStored: async () => storedShareWeights(await store.readAgentSettingsMap()),
|
|
73291
|
+
suggest: (source) => this.autoTune().suggestShares({
|
|
73292
|
+
headroom: AUTO_SHARE_HEADROOM,
|
|
73293
|
+
source
|
|
73294
|
+
}),
|
|
73295
|
+
topologySignature: async () => shareTopologySignature([...new Set(["hub", ...this.topology.collectRunnerNodeIds()])], await store.readAgentSettingsMap()),
|
|
73296
|
+
logger: this.ctx.logger
|
|
73297
|
+
});
|
|
73298
|
+
return this.effectiveSharesService;
|
|
73299
|
+
}
|
|
73300
|
+
/** Something the automatic shares depend on changed: recompute now (best effort). */
|
|
73301
|
+
noteShareChange(trigger) {
|
|
73302
|
+
if (this.settingsStore === null) return;
|
|
73303
|
+
this.effectiveShares().recompute(trigger).catch((err) => {
|
|
73304
|
+
this.ctx.logger.warn("load balancing: share recompute failed", { meta: {
|
|
73305
|
+
trigger,
|
|
73306
|
+
error: errMsg$2(err)
|
|
73307
|
+
} });
|
|
73308
|
+
});
|
|
73309
|
+
}
|
|
73310
|
+
async getEffectiveShares() {
|
|
73311
|
+
const service = this.effectiveShares();
|
|
73312
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])].toSorted();
|
|
73313
|
+
const rows = await Promise.all(nodeIds.map(async (nodeId) => ({
|
|
73314
|
+
nodeId,
|
|
73315
|
+
read: await this.getNodeInferenceDevices({ nodeId }).catch(() => null)
|
|
73316
|
+
})));
|
|
73317
|
+
const detectable = new Set(this.topology.detectionEligibleNodes());
|
|
73318
|
+
const eligibleNodes = rows.filter((r) => r.read !== null && r.read.reachable && detectable.has(r.nodeId) && this.isNodeInferenceUsable(r.nodeId)).map((r) => r.nodeId);
|
|
73319
|
+
const eligibleDevices = {};
|
|
73320
|
+
for (const r of rows) {
|
|
73321
|
+
if (r.read === null) continue;
|
|
73322
|
+
eligibleDevices[r.nodeId] = r.read.devices.filter((d) => d.enabled && d.available && d.exclusion === null).map((d) => d.key);
|
|
73323
|
+
}
|
|
73324
|
+
const auto = service.current();
|
|
73325
|
+
return resolveEffectiveShares({
|
|
73326
|
+
source: this.readShareSource(),
|
|
73327
|
+
policy: this.readSessionPlacement(),
|
|
73328
|
+
nodeIds,
|
|
73329
|
+
stored: storedShareWeights(await this.settingsStore.readAgentSettingsMap()),
|
|
73330
|
+
auto: auto?.weights ?? null,
|
|
73331
|
+
...auto !== null ? { autoCapacity: auto.capacity } : {},
|
|
73332
|
+
eligibleNodes,
|
|
73333
|
+
eligibleDevices,
|
|
73334
|
+
computedAt: auto?.computedAt ?? null,
|
|
73335
|
+
trigger: auto?.trigger ?? null
|
|
73336
|
+
});
|
|
73337
|
+
}
|
|
73338
|
+
async setShareSource(input) {
|
|
73339
|
+
const previous = this.readShareSource();
|
|
73340
|
+
if (input.source === "manual" && isAutomaticShareSource(previous)) {
|
|
73341
|
+
const auto = this.effectiveShares().current();
|
|
73342
|
+
if (auto !== null) await this.setLoadBalancingShares(auto.weights);
|
|
73343
|
+
}
|
|
73344
|
+
await this.updateGlobalSettings({ [SHARE_SOURCE_SETTING_KEY]: input.source });
|
|
73345
|
+
this.ctx.logger.info("load balancing: share source changed", { meta: {
|
|
73346
|
+
from: previous,
|
|
73347
|
+
to: input.source
|
|
73348
|
+
} });
|
|
73349
|
+
if (isAutomaticShareSource(input.source)) this.noteShareChange(`source:${input.source}`);
|
|
73350
|
+
return { success: true };
|
|
72886
73351
|
}
|
|
72887
73352
|
async getCameraSettings(input) {
|
|
72888
73353
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|
|
@@ -72913,6 +73378,16 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72913
73378
|
await this.emitResolvedCameraUpdated(input.deviceId);
|
|
72914
73379
|
return { success: true };
|
|
72915
73380
|
}
|
|
73381
|
+
/**
|
|
73382
|
+
* D754 — write or clear one step's CAMERA-WIDE settings (field-level bag,
|
|
73383
|
+
* merged under the per-(node,device) override on every accelerator). No
|
|
73384
|
+
* deviceKey to validate: the layer is node-independent by construction.
|
|
73385
|
+
*/
|
|
73386
|
+
async setCameraStepSettings(input) {
|
|
73387
|
+
await this.settingsStore.writeCameraStepSettings(input.deviceId, input.addonId, input.settings);
|
|
73388
|
+
await this.emitResolvedCameraUpdated(input.deviceId);
|
|
73389
|
+
return { success: true };
|
|
73390
|
+
}
|
|
72916
73391
|
async setCameraPipelineForAgent(input) {
|
|
72917
73392
|
const existing = (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? {};
|
|
72918
73393
|
const nextByAgent = { ...existing.pipelineByAgent };
|