@camstack/addon-pipeline-orchestrator 1.2.291 → 1.2.292
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets--PHyrJOr.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-tJspAxws.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CcezrkWV.mjs} +1 -1
- package/dist/{hostInit-BpNmAa0Q.mjs → hostInit-D7n994Lm.mjs} +2 -2
- package/dist/index.js +541 -75
- package/dist/index.mjs +541 -75
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +0 -26
package/dist/index.js
CHANGED
|
@@ -24661,7 +24661,7 @@ var pipelineExecutorCapability = {
|
|
|
24661
24661
|
auth: "admin"
|
|
24662
24662
|
}),
|
|
24663
24663
|
/**
|
|
24664
|
-
*
|
|
24664
|
+
* Auto tune (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24665
24665
|
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24666
24666
|
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24667
24667
|
* running model's own defaults apply. Every other key of a camera's store
|
|
@@ -25046,10 +25046,14 @@ var pipelineExecutorCapability = {
|
|
|
25046
25046
|
* full or drowning node yields to a free one (D748).
|
|
25047
25047
|
*/
|
|
25048
25048
|
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25049
|
-
|
|
25049
|
+
/**
|
|
25050
|
+
* `share` since D750: the percentages are held by default. `least-loaded` stays
|
|
25051
|
+
* selectable.
|
|
25052
|
+
*/
|
|
25053
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "share";
|
|
25050
25054
|
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25051
25055
|
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25052
|
-
/** Parse a stored value; anything unknown is the default
|
|
25056
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25053
25057
|
function parseSessionPlacementPolicy(value) {
|
|
25054
25058
|
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25055
25059
|
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
@@ -25132,7 +25136,62 @@ var SuggestSharesResultSchema = object({
|
|
|
25132
25136
|
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25133
25137
|
});
|
|
25134
25138
|
/**
|
|
25135
|
-
*
|
|
25139
|
+
* Where the node and device shares come from (D750).
|
|
25140
|
+
* - `auto` — computed by the system from the cluster's measured capacity at
|
|
25141
|
+
* the Auto tune models, and recomputed whenever a node or device joins,
|
|
25142
|
+
* leaves, is switched on or off, or changes model.
|
|
25143
|
+
* - `manual` — the operator's stored sliders. A node or device that cannot
|
|
25144
|
+
* take sessions gives its share to the others in proportion; the stored
|
|
25145
|
+
* values are never rewritten for that.
|
|
25146
|
+
*/
|
|
25147
|
+
var ShareSourceSchema = _enum(["auto", "manual"]);
|
|
25148
|
+
var DEFAULT_SHARE_SOURCE = "auto";
|
|
25149
|
+
/** Headroom the automatic shares are computed at (Auto tune's default). */
|
|
25150
|
+
var AUTO_SHARE_HEADROOM = .4;
|
|
25151
|
+
/** The global-settings key the share source is stored under (orchestrator `load` section). */
|
|
25152
|
+
var SHARE_SOURCE_SETTING_KEY = "shareSource";
|
|
25153
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25154
|
+
function parseShareSource(value) {
|
|
25155
|
+
const parsed = ShareSourceSchema.safeParse(value);
|
|
25156
|
+
return parsed.success ? parsed.data : DEFAULT_SHARE_SOURCE;
|
|
25157
|
+
}
|
|
25158
|
+
var SetShareSourceInputSchema = object({ source: ShareSourceSchema });
|
|
25159
|
+
object({
|
|
25160
|
+
nodes: record(string(), number()),
|
|
25161
|
+
devices: record(string(), record(string(), number()))
|
|
25162
|
+
});
|
|
25163
|
+
var EffectiveDeviceShareSchema = object({
|
|
25164
|
+
deviceKey: string(),
|
|
25165
|
+
/** Can take sessions now (enabled, available, can run the root). */
|
|
25166
|
+
eligible: boolean(),
|
|
25167
|
+
/** The share the source assigns, as a percentage of the node's devices. */
|
|
25168
|
+
configuredPct: number(),
|
|
25169
|
+
/** The share actually used, after an ineligible device's share went to the others. */
|
|
25170
|
+
effectivePct: number()
|
|
25171
|
+
});
|
|
25172
|
+
var EffectiveNodeShareSchema = object({
|
|
25173
|
+
nodeId: string(),
|
|
25174
|
+
eligible: boolean(),
|
|
25175
|
+
configuredPct: number(),
|
|
25176
|
+
effectivePct: number(),
|
|
25177
|
+
devices: array(EffectiveDeviceShareSchema).readonly()
|
|
25178
|
+
});
|
|
25179
|
+
var EffectiveSharesSchema = object({
|
|
25180
|
+
source: ShareSourceSchema,
|
|
25181
|
+
policy: SessionPlacementPolicySchema,
|
|
25182
|
+
/** `auto` with no computed answer yet: the stored sliders are used meanwhile. */
|
|
25183
|
+
autoPending: boolean(),
|
|
25184
|
+
/** When the automatic shares were last computed (ms epoch); null = never. */
|
|
25185
|
+
computedAt: number().nullable(),
|
|
25186
|
+
/** What made the system recompute them last. */
|
|
25187
|
+
trigger: string().nullable(),
|
|
25188
|
+
nodes: array(EffectiveNodeShareSchema).readonly(),
|
|
25189
|
+
/** The relative weights the election reads (before eligibility). */
|
|
25190
|
+
nodeWeights: record(string(), number()),
|
|
25191
|
+
deviceWeights: record(string(), record(string(), number()))
|
|
25192
|
+
});
|
|
25193
|
+
/**
|
|
25194
|
+
* The wire shapes of the Auto tune plan and apply (D745), used by the
|
|
25136
25195
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25137
25196
|
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
25138
25197
|
* schemas are what crosses the cap boundary.
|
|
@@ -25267,7 +25326,7 @@ var DetectorStoreClearSchema = object({
|
|
|
25267
25326
|
deviceId: number(),
|
|
25268
25327
|
keys: array(string()).readonly()
|
|
25269
25328
|
});
|
|
25270
|
-
var
|
|
25329
|
+
var AutoTuneWarningSchema = discriminatedUnion("kind", [
|
|
25271
25330
|
object({
|
|
25272
25331
|
kind: literal("detector-store-cluster-wide"),
|
|
25273
25332
|
skippedNodes: array(string()).readonly()
|
|
@@ -25282,7 +25341,7 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25282
25341
|
deviceKey: string()
|
|
25283
25342
|
})
|
|
25284
25343
|
]);
|
|
25285
|
-
var
|
|
25344
|
+
var AutoTuneOptionsSchema = object({
|
|
25286
25345
|
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25287
25346
|
headroom: number().min(.3).max(.5).default(.4),
|
|
25288
25347
|
/**
|
|
@@ -25293,7 +25352,7 @@ var BestSettingsOptionsSchema = object({
|
|
|
25293
25352
|
*/
|
|
25294
25353
|
shares: LoadBalancingSharesInputSchema.optional()
|
|
25295
25354
|
});
|
|
25296
|
-
var
|
|
25355
|
+
var AutoTunePlanSchema = object({
|
|
25297
25356
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25298
25357
|
planHash: string(),
|
|
25299
25358
|
family: literal("yolov9"),
|
|
@@ -25305,13 +25364,13 @@ var BestSettingsPlanSchema = object({
|
|
|
25305
25364
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25306
25365
|
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25307
25366
|
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25308
|
-
warnings: array(
|
|
25367
|
+
warnings: array(AutoTuneWarningSchema).readonly()
|
|
25309
25368
|
});
|
|
25310
|
-
var
|
|
25369
|
+
var AutoTuneApplyInputSchema = object({
|
|
25311
25370
|
planHash: string(),
|
|
25312
25371
|
headroom: number().min(.3).max(.5).default(.4)
|
|
25313
25372
|
});
|
|
25314
|
-
var
|
|
25373
|
+
var AutoTunePhaseResultSchema = object({
|
|
25315
25374
|
phase: _enum([
|
|
25316
25375
|
"download",
|
|
25317
25376
|
"camera-overrides",
|
|
@@ -25322,19 +25381,19 @@ var BestSettingsPhaseResultSchema = object({
|
|
|
25322
25381
|
ok: boolean(),
|
|
25323
25382
|
detail: string()
|
|
25324
25383
|
});
|
|
25325
|
-
var
|
|
25384
|
+
var AutoTuneApplyResultSchema = discriminatedUnion("kind", [
|
|
25326
25385
|
object({
|
|
25327
25386
|
kind: literal("applied"),
|
|
25328
25387
|
planHash: string(),
|
|
25329
|
-
phases: array(
|
|
25388
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25330
25389
|
}),
|
|
25331
25390
|
object({
|
|
25332
25391
|
kind: literal("aborted"),
|
|
25333
|
-
phases: array(
|
|
25392
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25334
25393
|
}),
|
|
25335
25394
|
object({
|
|
25336
25395
|
kind: literal("stale"),
|
|
25337
|
-
plan:
|
|
25396
|
+
plan: AutoTunePlanSchema
|
|
25338
25397
|
}),
|
|
25339
25398
|
object({
|
|
25340
25399
|
kind: literal("refused"),
|
|
@@ -27515,7 +27574,7 @@ var pipelineOrchestratorCapability = {
|
|
|
27515
27574
|
auth: "admin"
|
|
27516
27575
|
}),
|
|
27517
27576
|
/**
|
|
27518
|
-
*
|
|
27577
|
+
* Auto tune (D745) — DRY RUN. Recommends ONE detector family (YOLOv9)
|
|
27519
27578
|
* for the whole cluster from each node's static hardware class, the
|
|
27520
27579
|
* measured latency table and the detection demand, and lists every write
|
|
27521
27580
|
* the apply would make: the node model pins set, the enrichment pins and
|
|
@@ -27524,14 +27583,14 @@ var pipelineOrchestratorCapability = {
|
|
|
27524
27583
|
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27525
27584
|
* rules are never part of it. `planHash` names exactly this diff.
|
|
27526
27585
|
*/
|
|
27527
|
-
|
|
27586
|
+
planAutoTune: method(AutoTuneOptionsSchema, AutoTunePlanSchema, { auth: "admin" }),
|
|
27528
27587
|
/**
|
|
27529
|
-
*
|
|
27588
|
+
* Auto tune (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27530
27589
|
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27531
27590
|
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27532
27591
|
* nodes are skipped; there is no undo.
|
|
27533
27592
|
*/
|
|
27534
|
-
|
|
27593
|
+
applyAutoTune: method(AutoTuneApplyInputSchema, AutoTuneApplyResultSchema, {
|
|
27535
27594
|
kind: "mutation",
|
|
27536
27595
|
auth: "admin"
|
|
27537
27596
|
}),
|
|
@@ -27557,11 +27616,27 @@ var pipelineOrchestratorCapability = {
|
|
|
27557
27616
|
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27558
27617
|
/**
|
|
27559
27618
|
* Load balancing (D748) — shares proportional to each node's measured
|
|
27560
|
-
* capacity at the models
|
|
27619
|
+
* capacity at the models Auto tune would pick, iterated (≤ 3 rounds)
|
|
27561
27620
|
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27562
27621
|
* Writes nothing.
|
|
27563
27622
|
*/
|
|
27564
27623
|
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27624
|
+
/**
|
|
27625
|
+
* Load balancing (D750) — the shares placement reads now: the source
|
|
27626
|
+
* (automatic capacity shares or the operator's sliders), each node's and
|
|
27627
|
+
* device's configured share, and the effective share after a node or
|
|
27628
|
+
* device that cannot take sessions gave its share to the others.
|
|
27629
|
+
*/
|
|
27630
|
+
getEffectiveShares: method(_void(), EffectiveSharesSchema, { auth: "admin" }),
|
|
27631
|
+
/**
|
|
27632
|
+
* Load balancing (D750) — switch between automatic and manual shares.
|
|
27633
|
+
* Going manual seeds the sliders with the current automatic shares (one
|
|
27634
|
+
* atomic write), so nothing jumps; going automatic recomputes at once.
|
|
27635
|
+
*/
|
|
27636
|
+
setShareSource: method(SetShareSourceInputSchema, object({ success: literal(true) }), {
|
|
27637
|
+
kind: "mutation",
|
|
27638
|
+
auth: "admin"
|
|
27639
|
+
}),
|
|
27565
27640
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27566
27641
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27567
27642
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -43585,7 +43660,7 @@ Object.freeze({
|
|
|
43585
43660
|
addonId: null,
|
|
43586
43661
|
access: "view"
|
|
43587
43662
|
},
|
|
43588
|
-
"pipelineOrchestrator.
|
|
43663
|
+
"pipelineOrchestrator.applyAutoTune": {
|
|
43589
43664
|
capName: "pipeline-orchestrator",
|
|
43590
43665
|
capScope: "system",
|
|
43591
43666
|
addonId: null,
|
|
@@ -43705,6 +43780,12 @@ Object.freeze({
|
|
|
43705
43780
|
addonId: null,
|
|
43706
43781
|
access: "view"
|
|
43707
43782
|
},
|
|
43783
|
+
"pipelineOrchestrator.getEffectiveShares": {
|
|
43784
|
+
capName: "pipeline-orchestrator",
|
|
43785
|
+
capScope: "system",
|
|
43786
|
+
addonId: null,
|
|
43787
|
+
access: "view"
|
|
43788
|
+
},
|
|
43708
43789
|
"pipelineOrchestrator.getGlobalMetrics": {
|
|
43709
43790
|
capName: "pipeline-orchestrator",
|
|
43710
43791
|
capScope: "system",
|
|
@@ -43771,7 +43852,7 @@ Object.freeze({
|
|
|
43771
43852
|
addonId: null,
|
|
43772
43853
|
access: "create"
|
|
43773
43854
|
},
|
|
43774
|
-
"pipelineOrchestrator.
|
|
43855
|
+
"pipelineOrchestrator.planAutoTune": {
|
|
43775
43856
|
capName: "pipeline-orchestrator",
|
|
43776
43857
|
capScope: "system",
|
|
43777
43858
|
addonId: null,
|
|
@@ -43903,6 +43984,12 @@ Object.freeze({
|
|
|
43903
43984
|
addonId: null,
|
|
43904
43985
|
access: "create"
|
|
43905
43986
|
},
|
|
43987
|
+
"pipelineOrchestrator.setShareSource": {
|
|
43988
|
+
capName: "pipeline-orchestrator",
|
|
43989
|
+
capScope: "system",
|
|
43990
|
+
addonId: null,
|
|
43991
|
+
access: "create"
|
|
43992
|
+
},
|
|
43906
43993
|
"pipelineOrchestrator.suggestShares": {
|
|
43907
43994
|
capName: "pipeline-orchestrator",
|
|
43908
43995
|
capScope: "system",
|
|
@@ -51141,7 +51228,7 @@ function modelBindingKey(key) {
|
|
|
51141
51228
|
/**
|
|
51142
51229
|
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
51143
51230
|
* flattened (D738). One definition for the runner (which applies the binding)
|
|
51144
|
-
* and the orchestrator's
|
|
51231
|
+
* and the orchestrator's Auto tune (which clears those overrides, D745).
|
|
51145
51232
|
*/
|
|
51146
51233
|
function collectModelBoundKeys(fields) {
|
|
51147
51234
|
return fields.flatMap((field) => {
|
|
@@ -51753,7 +51840,7 @@ function estimateDeviceDemand(input) {
|
|
|
51753
51840
|
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51754
51841
|
}
|
|
51755
51842
|
/** The family the cluster runs. */
|
|
51756
|
-
var
|
|
51843
|
+
var AUTO_TUNE_FAMILY = "yolov9";
|
|
51757
51844
|
function round(value, digits = 3) {
|
|
51758
51845
|
const f = 10 ** digits;
|
|
51759
51846
|
return Math.round(value * f) / f;
|
|
@@ -51766,7 +51853,7 @@ function fitsOn(cell, demand, headroom) {
|
|
|
51766
51853
|
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51767
51854
|
}
|
|
51768
51855
|
function candidatesFor(contexts, formatsByModel) {
|
|
51769
|
-
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(
|
|
51856
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(AUTO_TUNE_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51770
51857
|
modelId: id,
|
|
51771
51858
|
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51772
51859
|
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
@@ -52017,6 +52104,57 @@ function electByShare(candidates) {
|
|
|
52017
52104
|
reason: "share"
|
|
52018
52105
|
};
|
|
52019
52106
|
}
|
|
52107
|
+
/** Weight a node or device has when its source names none (the balancer's own default). */
|
|
52108
|
+
var DEFAULT_WEIGHT = 1;
|
|
52109
|
+
function positive(value) {
|
|
52110
|
+
return value !== void 0 && value > 0 ? value : DEFAULT_WEIGHT;
|
|
52111
|
+
}
|
|
52112
|
+
function percentages(keys, weightOf, counts) {
|
|
52113
|
+
const total = keys.reduce((sum, k) => sum + (counts(k) ? weightOf(k) : 0), 0);
|
|
52114
|
+
return new Map(keys.map((k) => [k, total > 0 && counts(k) ? Math.round(weightOf(k) / total * 1e3) / 10 : 0]));
|
|
52115
|
+
}
|
|
52116
|
+
function resolveEffectiveShares(input) {
|
|
52117
|
+
const weights = input.source === "auto" && input.auto !== null && input.auto !== null ? input.auto : input.stored;
|
|
52118
|
+
const nodeWeights = {};
|
|
52119
|
+
for (const nodeId of input.nodeIds) nodeWeights[nodeId] = positive(weights.nodes[nodeId]);
|
|
52120
|
+
const eligibleNodes = new Set(input.eligibleNodes);
|
|
52121
|
+
const configured = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52122
|
+
const effective = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(k));
|
|
52123
|
+
const deviceWeights = {};
|
|
52124
|
+
const nodes = input.nodeIds.map((nodeId) => {
|
|
52125
|
+
const eligibleKeys = new Set(input.eligibleDevices[nodeId] ?? []);
|
|
52126
|
+
const named = weights.devices[nodeId] ?? {};
|
|
52127
|
+
const keys = [...new Set([...Object.keys(named), ...eligibleKeys])].toSorted();
|
|
52128
|
+
const perDevice = {};
|
|
52129
|
+
for (const key of keys) perDevice[key] = positive(named[key]);
|
|
52130
|
+
if (keys.length > 0) deviceWeights[nodeId] = perDevice;
|
|
52131
|
+
const devConfigured = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52132
|
+
const devEffective = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(nodeId) && eligibleKeys.has(k));
|
|
52133
|
+
const devices = keys.map((deviceKey) => ({
|
|
52134
|
+
deviceKey,
|
|
52135
|
+
eligible: eligibleKeys.has(deviceKey),
|
|
52136
|
+
configuredPct: devConfigured.get(deviceKey) ?? 0,
|
|
52137
|
+
effectivePct: devEffective.get(deviceKey) ?? 0
|
|
52138
|
+
}));
|
|
52139
|
+
return {
|
|
52140
|
+
nodeId,
|
|
52141
|
+
eligible: eligibleNodes.has(nodeId),
|
|
52142
|
+
configuredPct: configured.get(nodeId) ?? 0,
|
|
52143
|
+
effectivePct: effective.get(nodeId) ?? 0,
|
|
52144
|
+
devices
|
|
52145
|
+
};
|
|
52146
|
+
});
|
|
52147
|
+
return {
|
|
52148
|
+
source: input.source,
|
|
52149
|
+
policy: input.policy ?? "share",
|
|
52150
|
+
autoPending: input.source === "auto" && input.auto === null,
|
|
52151
|
+
computedAt: input.computedAt ?? null,
|
|
52152
|
+
trigger: input.trigger ?? null,
|
|
52153
|
+
nodes,
|
|
52154
|
+
nodeWeights,
|
|
52155
|
+
deviceWeights
|
|
52156
|
+
};
|
|
52157
|
+
}
|
|
52020
52158
|
/**
|
|
52021
52159
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
52022
52160
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -62411,6 +62549,7 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62411
62549
|
};
|
|
62412
62550
|
const okByNode = new Map(input.verdicts.filter((v) => v.state === "ok").map((v) => [v.nodeId, v]));
|
|
62413
62551
|
const sourceAttached = input.attachedByNode[saturated.nodeId] ?? 0;
|
|
62552
|
+
if (input.shares !== void 0) return planShareTarget(input, input.shares, saturated, movable, okByNode, now, guards);
|
|
62414
62553
|
for (const camera of movable) {
|
|
62415
62554
|
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62416
62555
|
const target = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => {
|
|
@@ -62435,6 +62574,52 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62435
62574
|
nodeId: saturated.nodeId
|
|
62436
62575
|
};
|
|
62437
62576
|
}
|
|
62577
|
+
/** Deficits equal up to float noise. */
|
|
62578
|
+
var SHARE_EPSILON = 1e-9;
|
|
62579
|
+
function underCap(input, nodeId) {
|
|
62580
|
+
const cap = input.nodeCaps?.[nodeId];
|
|
62581
|
+
const attached = input.attachedByNode[nodeId] ?? 0;
|
|
62582
|
+
return cap === null || cap === void 0 || cap <= 0 || attached < cap;
|
|
62583
|
+
}
|
|
62584
|
+
/**
|
|
62585
|
+
* Share placement's failover target (D750): among the camera's eligible,
|
|
62586
|
+
* `ok`, under-cap nodes, the one furthest below its share of the cameras
|
|
62587
|
+
* attached across the source and those nodes — the same deficit rule a new
|
|
62588
|
+
* session is placed by. A node at or over its share is no target, so a move
|
|
62589
|
+
* never undoes the balance. Nothing below its share ⇒ hold, naming the camera.
|
|
62590
|
+
*/
|
|
62591
|
+
function planShareTarget(input, shares, saturated, movable, okByNode, now, guards) {
|
|
62592
|
+
const shareOf = (nodeId) => {
|
|
62593
|
+
const share = shares[nodeId];
|
|
62594
|
+
return share !== void 0 && share > 0 ? share : 0;
|
|
62595
|
+
};
|
|
62596
|
+
for (const camera of movable) {
|
|
62597
|
+
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62598
|
+
const healthy = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => underCap(input, v.nodeId));
|
|
62599
|
+
const set = [saturated.nodeId, ...healthy.map((v) => v.nodeId)];
|
|
62600
|
+
const shareSum = set.reduce((sum, nodeId) => sum + shareOf(nodeId), 0);
|
|
62601
|
+
const attachedSum = set.reduce((sum, nodeId) => sum + (input.attachedByNode[nodeId] ?? 0), 0);
|
|
62602
|
+
if (shareSum <= 0) continue;
|
|
62603
|
+
const deficit = (nodeId) => shareOf(nodeId) / shareSum * attachedSum - (input.attachedByNode[nodeId] ?? 0);
|
|
62604
|
+
const target = healthy.filter((v) => deficit(v.nodeId) > SHARE_EPSILON).toSorted((a, b) => deficit(b.nodeId) - deficit(a.nodeId) || a.signals.score - b.signals.score || a.nodeId.localeCompare(b.nodeId))[0];
|
|
62605
|
+
if (!target) continue;
|
|
62606
|
+
return {
|
|
62607
|
+
kind: "move",
|
|
62608
|
+
deviceId: camera.deviceId,
|
|
62609
|
+
fromNodeId: saturated.nodeId,
|
|
62610
|
+
toNodeId: target.nodeId,
|
|
62611
|
+
fromScore: saturated.signals.score,
|
|
62612
|
+
toScore: target.signals.score
|
|
62613
|
+
};
|
|
62614
|
+
}
|
|
62615
|
+
const first = movable[0];
|
|
62616
|
+
return {
|
|
62617
|
+
kind: "hold",
|
|
62618
|
+
reason: "no-target",
|
|
62619
|
+
nodeId: saturated.nodeId,
|
|
62620
|
+
...first !== void 0 ? { deviceId: first.deviceId } : {}
|
|
62621
|
+
};
|
|
62622
|
+
}
|
|
62438
62623
|
//#endregion
|
|
62439
62624
|
//#region src/node-stress-controller.ts
|
|
62440
62625
|
/** Narrow an unknown settings value to a {@link NodeStressMode}. */
|
|
@@ -62643,17 +62828,27 @@ var NodeStressController = class NodeStressController {
|
|
|
62643
62828
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
62644
62829
|
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
62645
62830
|
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
62831
|
+
const candidates = await this.collectCandidates(now);
|
|
62832
|
+
const shares = await this.deps.readFailoverShares?.() ?? null;
|
|
62646
62833
|
const plan = planStressFailover({
|
|
62647
62834
|
verdicts,
|
|
62648
|
-
candidates
|
|
62835
|
+
candidates,
|
|
62649
62836
|
history,
|
|
62650
62837
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
62651
|
-
attachedByNode: this.attachedByNode()
|
|
62838
|
+
attachedByNode: this.attachedByNode(),
|
|
62839
|
+
...shares !== null ? { shares } : {}
|
|
62652
62840
|
}, now, this.guards());
|
|
62653
62841
|
if (plan.kind === "hold") {
|
|
62654
62842
|
this.deps.logger.warn("node saturated — no failover applied", {
|
|
62655
|
-
tags: {
|
|
62656
|
-
|
|
62843
|
+
tags: {
|
|
62844
|
+
nodeId: plan.nodeId ?? "",
|
|
62845
|
+
...plan.deviceId !== void 0 ? { deviceId: plan.deviceId } : {}
|
|
62846
|
+
},
|
|
62847
|
+
meta: {
|
|
62848
|
+
reason: plan.reason,
|
|
62849
|
+
rule: shares !== null ? "share-deficit" : "fewest-cameras",
|
|
62850
|
+
...plan.reason === "no-target" && shares !== null ? { why: "no healthy node below its share can take it" } : {}
|
|
62851
|
+
}
|
|
62657
62852
|
});
|
|
62658
62853
|
return;
|
|
62659
62854
|
}
|
|
@@ -65265,12 +65460,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65265
65460
|
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
65266
65461
|
*/
|
|
65267
65462
|
async buildNodeWeights() {
|
|
65463
|
+
if (this.effectiveNodeWeights !== null) return this.effectiveNodeWeights();
|
|
65464
|
+
return this.buildStoredNodeWeights();
|
|
65465
|
+
}
|
|
65466
|
+
/**
|
|
65467
|
+
* The operator's stored node sliders, whatever the share source — what
|
|
65468
|
+
* `manual` applies and what `auto` falls back to before its first answer.
|
|
65469
|
+
*/
|
|
65470
|
+
async buildStoredNodeWeights() {
|
|
65268
65471
|
const blob = await this.agentSettingsState.get();
|
|
65269
65472
|
const weights = {};
|
|
65270
65473
|
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
65271
65474
|
return weights;
|
|
65272
65475
|
}
|
|
65273
65476
|
/**
|
|
65477
|
+
* Where {@link buildNodeWeights} reads from once the share source is wired
|
|
65478
|
+
* (D750): the effective shares (automatic or manual). Every balancer, the
|
|
65479
|
+
* share election and the stress failover read through it, so they can never
|
|
65480
|
+
* disagree on what a node's share is. Unset (tests, early boot) ⇒ stored.
|
|
65481
|
+
*/
|
|
65482
|
+
effectiveNodeWeights = null;
|
|
65483
|
+
setEffectiveNodeWeights(read) {
|
|
65484
|
+
this.effectiveNodeWeights = read;
|
|
65485
|
+
}
|
|
65486
|
+
/**
|
|
65274
65487
|
* The per-(node,device) BASE provisioning (C7.2/C7.4) for one `(node,
|
|
65275
65488
|
* deviceKey)` — the effective `stepId → {modelId?, settings?}` map the
|
|
65276
65489
|
* dispatch provisioning stage attaches onto the resolved step tree before
|
|
@@ -67628,9 +67841,10 @@ var SessionDispatchController = class {
|
|
|
67628
67841
|
deviceId: a.deviceId,
|
|
67629
67842
|
nodeId: a.agentNodeId
|
|
67630
67843
|
}))]);
|
|
67844
|
+
const present = new Set(this.deps.topology.knownRunnerNodeIds());
|
|
67631
67845
|
const election = electNodeByShare({
|
|
67632
67846
|
loads: input.loads,
|
|
67633
|
-
eligible: input.eligible,
|
|
67847
|
+
eligible: input.eligible.filter((nodeId) => present.has(nodeId)),
|
|
67634
67848
|
weights: input.weights,
|
|
67635
67849
|
caps: input.nodeCaps,
|
|
67636
67850
|
runningByNode: running
|
|
@@ -69099,7 +69313,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69099
69313
|
attach: (nodeId, config) => placement.attachOn(nodeId, config),
|
|
69100
69314
|
detach: (nodeId, deviceId) => placement.detachOn(nodeId, deviceId),
|
|
69101
69315
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
69102
|
-
readGlobalSettings: () => globalSettings
|
|
69316
|
+
readGlobalSettings: () => globalSettings,
|
|
69317
|
+
readFailoverShares: async () => readSessionPlacement() === "share" ? settingsStore.buildNodeWeights() : null
|
|
69103
69318
|
});
|
|
69104
69319
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
69105
69320
|
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
@@ -69115,6 +69330,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69115
69330
|
nodeStress.start();
|
|
69116
69331
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69117
69332
|
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
69333
|
+
const readShareSource = () => parseShareSource(globalSettings?.[SHARE_SOURCE_SETTING_KEY]);
|
|
69118
69334
|
/**
|
|
69119
69335
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
69120
69336
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -69451,6 +69667,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69451
69667
|
nodeLifecycle.unsubscribeDetectionPipelineReadiness(nodeId);
|
|
69452
69668
|
audio.markNodeOffline(nodeId);
|
|
69453
69669
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
69670
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-offline:${nodeId}`);
|
|
69454
69671
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
69455
69672
|
const msg = errMsg$2(err);
|
|
69456
69673
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
@@ -69494,6 +69711,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69494
69711
|
});
|
|
69495
69712
|
});
|
|
69496
69713
|
reconcile.scheduleReconcile();
|
|
69714
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-online:${nodeId}`);
|
|
69497
69715
|
});
|
|
69498
69716
|
const unsubAgentOnline = () => {
|
|
69499
69717
|
unsubAgentOnlineEvent();
|
|
@@ -69734,7 +69952,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69734
69952
|
unsubOrchestratorSubscriptions,
|
|
69735
69953
|
pendingRetryTimer,
|
|
69736
69954
|
autoRebalanceTimer,
|
|
69737
|
-
sessionPlacement: () => readSessionPlacement()
|
|
69955
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69956
|
+
shareSource: () => readShareSource()
|
|
69738
69957
|
};
|
|
69739
69958
|
}
|
|
69740
69959
|
//#endregion
|
|
@@ -69783,13 +70002,13 @@ function buildGlobalSettingsSections(options) {
|
|
|
69783
70002
|
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69784
70003
|
type: "select",
|
|
69785
70004
|
label: "Session placement",
|
|
69786
|
-
description: "How a new detection session picks its node and device.
|
|
70005
|
+
description: "How a new detection session picks its node and device. Honour shares (default) = the share of PARALLEL sessions is held to the node and device shares, and a node at its cap yields to a free one. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy.",
|
|
69787
70006
|
options: [{
|
|
69788
|
-
value: "least-loaded",
|
|
69789
|
-
label: "Least-loaded"
|
|
69790
|
-
}, {
|
|
69791
70007
|
value: "share",
|
|
69792
70008
|
label: "Honour shares"
|
|
70009
|
+
}, {
|
|
70010
|
+
value: "least-loaded",
|
|
70011
|
+
label: "Least-loaded"
|
|
69793
70012
|
}],
|
|
69794
70013
|
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69795
70014
|
},
|
|
@@ -70351,9 +70570,9 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70351
70570
|
defaultRows: 4
|
|
70352
70571
|
}];
|
|
70353
70572
|
//#endregion
|
|
70354
|
-
//#region src/
|
|
70573
|
+
//#region src/auto-tune/plan-diff.ts
|
|
70355
70574
|
/**
|
|
70356
|
-
* The
|
|
70575
|
+
* The Auto tune writes, as pure transforms of the orchestrator's two
|
|
70357
70576
|
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
70358
70577
|
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
70359
70578
|
*
|
|
@@ -70377,7 +70596,7 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70377
70596
|
*/
|
|
70378
70597
|
/** The root detector step. */
|
|
70379
70598
|
var DETECTOR_STEP_ID = "object-detection";
|
|
70380
|
-
/** Steps whose `modelId` pin
|
|
70599
|
+
/** Steps whose `modelId` pin Auto tune never touches. */
|
|
70381
70600
|
function isModelPinKept(stepId) {
|
|
70382
70601
|
return stepId.endsWith("-embedding");
|
|
70383
70602
|
}
|
|
@@ -70429,7 +70648,7 @@ function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
|
70429
70648
|
};
|
|
70430
70649
|
}
|
|
70431
70650
|
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70432
|
-
function
|
|
70651
|
+
function applyNodeAutoTune(blob, input) {
|
|
70433
70652
|
const next = { ...blob };
|
|
70434
70653
|
const changes = [];
|
|
70435
70654
|
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
@@ -70499,7 +70718,7 @@ function scrubTree(steps, input, onClear) {
|
|
|
70499
70718
|
});
|
|
70500
70719
|
}
|
|
70501
70720
|
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70502
|
-
function
|
|
70721
|
+
function applyCameraAutoTune(blob, input) {
|
|
70503
70722
|
const next = {};
|
|
70504
70723
|
const clears = [];
|
|
70505
70724
|
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
@@ -70583,12 +70802,12 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70583
70802
|
};
|
|
70584
70803
|
}
|
|
70585
70804
|
//#endregion
|
|
70586
|
-
//#region src/
|
|
70805
|
+
//#region src/auto-tune/auto-tune-service.ts
|
|
70587
70806
|
/**
|
|
70588
|
-
*
|
|
70807
|
+
* Auto tune (D745): gather the inputs, compute the plan, and apply exactly
|
|
70589
70808
|
* the plan the operator confirmed.
|
|
70590
70809
|
*
|
|
70591
|
-
* The plan is the recommender (`@camstack/types`
|
|
70810
|
+
* The plan is the recommender (`@camstack/types` auto-tune) over every
|
|
70592
70811
|
* runner node, plus the diff of the writes (`plan-diff.ts`) and the
|
|
70593
70812
|
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70594
70813
|
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
@@ -70657,7 +70876,7 @@ function toPercent(weights) {
|
|
|
70657
70876
|
}
|
|
70658
70877
|
return out;
|
|
70659
70878
|
}
|
|
70660
|
-
var
|
|
70879
|
+
var AutoTuneService = class {
|
|
70661
70880
|
deps;
|
|
70662
70881
|
applying = false;
|
|
70663
70882
|
constructor(deps) {
|
|
@@ -70667,7 +70886,7 @@ var BestSettingsService = class {
|
|
|
70667
70886
|
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70668
70887
|
}
|
|
70669
70888
|
/**
|
|
70670
|
-
* Shares proportional to each node's capacity at the models
|
|
70889
|
+
* Shares proportional to each node's capacity at the models Auto tune
|
|
70671
70890
|
* would pick under those very shares (D748). Model and load depend on each
|
|
70672
70891
|
* other, so it iterates from equal shares until the rounded shares stop
|
|
70673
70892
|
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
@@ -70759,7 +70978,8 @@ var BestSettingsService = class {
|
|
|
70759
70978
|
catalog
|
|
70760
70979
|
};
|
|
70761
70980
|
}));
|
|
70762
|
-
const
|
|
70981
|
+
const effective = await this.deps.readEffectiveWeights();
|
|
70982
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? effective.devices[nodeId]?.[key] ?? stored;
|
|
70763
70983
|
const nodes = gathered.map((g) => ({
|
|
70764
70984
|
nodeId: g.nodeId,
|
|
70765
70985
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70785,7 +71005,7 @@ var BestSettingsService = class {
|
|
|
70785
71005
|
peaks: this.deps.peaks(),
|
|
70786
71006
|
...policy === "share" ? { shares: {
|
|
70787
71007
|
nodes: {
|
|
70788
|
-
...
|
|
71008
|
+
...effective.nodes,
|
|
70789
71009
|
...shares?.nodes
|
|
70790
71010
|
},
|
|
70791
71011
|
clusterPeak: this.deps.clusterPeak()
|
|
@@ -70832,8 +71052,8 @@ var BestSettingsService = class {
|
|
|
70832
71052
|
targets,
|
|
70833
71053
|
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70834
71054
|
};
|
|
70835
|
-
const nodeChanges =
|
|
70836
|
-
const cameraClears =
|
|
71055
|
+
const nodeChanges = applyNodeAutoTune(await this.deps.readAgentSettings(), diffInput).changes;
|
|
71056
|
+
const cameraClears = applyCameraAutoTune(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70837
71057
|
const warnings = [];
|
|
70838
71058
|
let detectorStoreClears = [];
|
|
70839
71059
|
if (touchedNodes.size > 0) try {
|
|
@@ -70906,7 +71126,7 @@ var BestSettingsService = class {
|
|
|
70906
71126
|
async applyLocked(input) {
|
|
70907
71127
|
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
70908
71128
|
if (plan.planHash !== input.planHash) {
|
|
70909
|
-
this.deps.logger.warn("
|
|
71129
|
+
this.deps.logger.warn("auto tune: stale plan refused", { meta: {
|
|
70910
71130
|
confirmed: input.planHash,
|
|
70911
71131
|
current: plan.planHash
|
|
70912
71132
|
} });
|
|
@@ -70938,15 +71158,15 @@ var BestSettingsService = class {
|
|
|
70938
71158
|
ok: false,
|
|
70939
71159
|
detail: errMsg(err)
|
|
70940
71160
|
});
|
|
70941
|
-
this.deps.logger.warn("
|
|
71161
|
+
this.deps.logger.warn("auto tune: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70942
71162
|
return {
|
|
70943
71163
|
kind: "aborted",
|
|
70944
71164
|
phases
|
|
70945
71165
|
};
|
|
70946
71166
|
}
|
|
70947
71167
|
try {
|
|
70948
|
-
await this.deps.rewriteCameraSettings((prev) =>
|
|
70949
|
-
for (const c of plan.cameraClears) this.deps.logger.info("
|
|
71168
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraAutoTune(prev, diffInput).next);
|
|
71169
|
+
for (const c of plan.cameraClears) this.deps.logger.info("auto tune: camera override cleared", {
|
|
70950
71170
|
tags: {
|
|
70951
71171
|
deviceId: c.deviceId,
|
|
70952
71172
|
nodeId: c.nodeId
|
|
@@ -70990,8 +71210,8 @@ var BestSettingsService = class {
|
|
|
70990
71210
|
});
|
|
70991
71211
|
}
|
|
70992
71212
|
try {
|
|
70993
|
-
await this.deps.rewriteAgentSettings((prev) =>
|
|
70994
|
-
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("
|
|
71213
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeAutoTune(prev, diffInput).next);
|
|
71214
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("auto tune: node model set", {
|
|
70995
71215
|
tags: { nodeId: c.nodeId },
|
|
70996
71216
|
meta: {
|
|
70997
71217
|
nodeId: c.nodeId,
|
|
@@ -71025,14 +71245,14 @@ var BestSettingsService = class {
|
|
|
71025
71245
|
ok: failed.length === 0,
|
|
71026
71246
|
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
71027
71247
|
});
|
|
71028
|
-
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("
|
|
71248
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("auto tune: node skipped", {
|
|
71029
71249
|
tags: { nodeId: rec.nodeId },
|
|
71030
71250
|
meta: {
|
|
71031
71251
|
nodeId: rec.nodeId,
|
|
71032
71252
|
reasons: rec.reasons
|
|
71033
71253
|
}
|
|
71034
71254
|
});
|
|
71035
|
-
this.deps.logger.info("
|
|
71255
|
+
this.deps.logger.info("auto tune: applied", { meta: {
|
|
71036
71256
|
planHash: plan.planHash,
|
|
71037
71257
|
nodeChanges: plan.nodeChanges.length,
|
|
71038
71258
|
cameraClears: plan.cameraClears.length,
|
|
@@ -71047,7 +71267,7 @@ var BestSettingsService = class {
|
|
|
71047
71267
|
}
|
|
71048
71268
|
};
|
|
71049
71269
|
//#endregion
|
|
71050
|
-
//#region src/
|
|
71270
|
+
//#region src/auto-tune/device-peak-tracker.ts
|
|
71051
71271
|
var BUCKET_MS = 3600 * 1e3;
|
|
71052
71272
|
var DevicePeakTracker = class DevicePeakTracker {
|
|
71053
71273
|
now;
|
|
@@ -71289,6 +71509,176 @@ function applyLoadBalancingShares(prev, write) {
|
|
|
71289
71509
|
return next;
|
|
71290
71510
|
}
|
|
71291
71511
|
//#endregion
|
|
71512
|
+
//#region src/load-balancing/effective-shares-service.ts
|
|
71513
|
+
function weightsFrom(result) {
|
|
71514
|
+
const nodes = {};
|
|
71515
|
+
const devices = {};
|
|
71516
|
+
for (const node of result.nodes) {
|
|
71517
|
+
if (node.share !== null && node.share > 0) nodes[node.nodeId] = node.share;
|
|
71518
|
+
const perDevice = {};
|
|
71519
|
+
for (const d of node.devices) if (d.share > 0) perDevice[d.deviceKey] = d.share;
|
|
71520
|
+
if (Object.keys(perDevice).length > 0) devices[node.nodeId] = perDevice;
|
|
71521
|
+
}
|
|
71522
|
+
return {
|
|
71523
|
+
nodes,
|
|
71524
|
+
devices
|
|
71525
|
+
};
|
|
71526
|
+
}
|
|
71527
|
+
var EffectiveSharesService = class EffectiveSharesService {
|
|
71528
|
+
deps;
|
|
71529
|
+
/** An automatic answer older than this is recomputed by the reconcile. */
|
|
71530
|
+
static MAX_AGE_MS = 15 * 6e4;
|
|
71531
|
+
/** Reconcile cadence. */
|
|
71532
|
+
static RECONCILE_MS = 6e4;
|
|
71533
|
+
answer = null;
|
|
71534
|
+
inFlight = null;
|
|
71535
|
+
pendingTrigger = null;
|
|
71536
|
+
timer = null;
|
|
71537
|
+
constructor(deps) {
|
|
71538
|
+
this.deps = deps;
|
|
71539
|
+
}
|
|
71540
|
+
now() {
|
|
71541
|
+
return this.deps.now?.() ?? Date.now();
|
|
71542
|
+
}
|
|
71543
|
+
/** The last automatic answer; null = never computed. */
|
|
71544
|
+
current() {
|
|
71545
|
+
return this.answer;
|
|
71546
|
+
}
|
|
71547
|
+
/** Node weights the election reads. */
|
|
71548
|
+
async nodeWeights() {
|
|
71549
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return { ...this.answer.weights.nodes };
|
|
71550
|
+
return { ...(await this.deps.readStored()).nodes };
|
|
71551
|
+
}
|
|
71552
|
+
/** The weights of a node's ELIGIBLE devices (`eligible` carries their stored weights). */
|
|
71553
|
+
deviceWeights(nodeId, eligible) {
|
|
71554
|
+
if (this.deps.readSource() !== "auto" || this.answer === null) return { ...eligible };
|
|
71555
|
+
const auto = this.answer.weights.devices[nodeId] ?? {};
|
|
71556
|
+
const out = {};
|
|
71557
|
+
for (const [key, stored] of Object.entries(eligible)) {
|
|
71558
|
+
const share = auto[key];
|
|
71559
|
+
out[key] = share !== void 0 && share > 0 ? share : stored;
|
|
71560
|
+
}
|
|
71561
|
+
return out;
|
|
71562
|
+
}
|
|
71563
|
+
/** Both maps, as the Auto tune recommender charges load by them. */
|
|
71564
|
+
async effectiveWeights() {
|
|
71565
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return this.answer.weights;
|
|
71566
|
+
return this.deps.readStored();
|
|
71567
|
+
}
|
|
71568
|
+
/**
|
|
71569
|
+
* Recompute now (auto only). Concurrent calls coalesce: one runs, and one
|
|
71570
|
+
* trailing run picks up whatever was reported meanwhile.
|
|
71571
|
+
*/
|
|
71572
|
+
async recompute(trigger) {
|
|
71573
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71574
|
+
if (this.inFlight !== null) {
|
|
71575
|
+
this.pendingTrigger = trigger;
|
|
71576
|
+
return this.inFlight;
|
|
71577
|
+
}
|
|
71578
|
+
this.inFlight = this.run(trigger).finally(() => {
|
|
71579
|
+
this.inFlight = null;
|
|
71580
|
+
});
|
|
71581
|
+
await this.inFlight;
|
|
71582
|
+
const trailing = this.pendingTrigger;
|
|
71583
|
+
if (trailing !== null) {
|
|
71584
|
+
this.pendingTrigger = null;
|
|
71585
|
+
await this.recompute(trailing);
|
|
71586
|
+
}
|
|
71587
|
+
}
|
|
71588
|
+
async run(trigger) {
|
|
71589
|
+
let signature;
|
|
71590
|
+
let result;
|
|
71591
|
+
try {
|
|
71592
|
+
signature = await this.deps.topologySignature();
|
|
71593
|
+
result = await this.deps.suggest();
|
|
71594
|
+
} catch (err) {
|
|
71595
|
+
this.deps.logger.warn("load balancing: automatic shares not recomputed — last answer kept", { meta: {
|
|
71596
|
+
trigger,
|
|
71597
|
+
error: errMsg$2(err),
|
|
71598
|
+
hasAnswer: this.answer !== null
|
|
71599
|
+
} });
|
|
71600
|
+
return;
|
|
71601
|
+
}
|
|
71602
|
+
const before = this.answer?.weights ?? null;
|
|
71603
|
+
const after = weightsFrom(result);
|
|
71604
|
+
this.answer = {
|
|
71605
|
+
weights: after,
|
|
71606
|
+
computedAt: this.now(),
|
|
71607
|
+
trigger,
|
|
71608
|
+
signature
|
|
71609
|
+
};
|
|
71610
|
+
const nodeIds = [...new Set([...Object.keys(before?.nodes ?? {}), ...result.nodes.map((n) => n.nodeId)])];
|
|
71611
|
+
for (const nodeId of nodeIds.toSorted()) this.deps.logger.info("load balancing: automatic shares recomputed", {
|
|
71612
|
+
tags: { nodeId },
|
|
71613
|
+
meta: {
|
|
71614
|
+
trigger,
|
|
71615
|
+
before: before?.nodes[nodeId] ?? null,
|
|
71616
|
+
after: after.nodes[nodeId] ?? null,
|
|
71617
|
+
devicesBefore: JSON.stringify(before?.devices[nodeId] ?? {}),
|
|
71618
|
+
devicesAfter: JSON.stringify(after.devices[nodeId] ?? {})
|
|
71619
|
+
}
|
|
71620
|
+
});
|
|
71621
|
+
}
|
|
71622
|
+
/** Recompute if the topology changed or the answer has grown old (auto only). */
|
|
71623
|
+
async reconcile() {
|
|
71624
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71625
|
+
if (this.answer === null) return this.recompute("reconcile:first");
|
|
71626
|
+
if (this.now() - this.answer.computedAt > EffectiveSharesService.MAX_AGE_MS) return this.recompute("reconcile:age");
|
|
71627
|
+
let signature;
|
|
71628
|
+
try {
|
|
71629
|
+
signature = await this.deps.topologySignature();
|
|
71630
|
+
} catch (err) {
|
|
71631
|
+
this.deps.logger.debug("load balancing: topology signature unreadable — reconcile skipped", { meta: { error: errMsg$2(err) } });
|
|
71632
|
+
return;
|
|
71633
|
+
}
|
|
71634
|
+
if (signature !== this.answer.signature) return this.recompute("reconcile:topology");
|
|
71635
|
+
}
|
|
71636
|
+
start() {
|
|
71637
|
+
if (this.timer !== null) return;
|
|
71638
|
+
this.timer = setInterval(() => {
|
|
71639
|
+
this.reconcile();
|
|
71640
|
+
}, EffectiveSharesService.RECONCILE_MS);
|
|
71641
|
+
this.timer.unref?.();
|
|
71642
|
+
}
|
|
71643
|
+
stop() {
|
|
71644
|
+
if (this.timer !== null) clearInterval(this.timer);
|
|
71645
|
+
this.timer = null;
|
|
71646
|
+
}
|
|
71647
|
+
};
|
|
71648
|
+
//#endregion
|
|
71649
|
+
//#region src/load-balancing/share-inputs.ts
|
|
71650
|
+
/** The operator's sliders: node `detectWeight` and device `weight` (positive only). */
|
|
71651
|
+
function storedShareWeights(blob) {
|
|
71652
|
+
const nodes = {};
|
|
71653
|
+
const devices = {};
|
|
71654
|
+
for (const [nodeId, settings] of Object.entries(blob)) {
|
|
71655
|
+
if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) nodes[nodeId] = settings.detectWeight;
|
|
71656
|
+
const perDevice = {};
|
|
71657
|
+
for (const [key, entry] of Object.entries(settings.inferenceDevices ?? {})) if (typeof entry.weight === "number" && entry.weight > 0) perDevice[key] = entry.weight;
|
|
71658
|
+
if (Object.keys(perDevice).length > 0) devices[nodeId] = perDevice;
|
|
71659
|
+
}
|
|
71660
|
+
return {
|
|
71661
|
+
nodes,
|
|
71662
|
+
devices
|
|
71663
|
+
};
|
|
71664
|
+
}
|
|
71665
|
+
/**
|
|
71666
|
+
* A string that changes whenever an input of the automatic shares does: the
|
|
71667
|
+
* nodes present, each node's detect capability, which devices are switched on
|
|
71668
|
+
* and which detector model each runs. Sliders and caps are left out — they do
|
|
71669
|
+
* not change capacity.
|
|
71670
|
+
*/
|
|
71671
|
+
function shareTopologySignature(nodeIds, blob) {
|
|
71672
|
+
return [...nodeIds].toSorted().map((nodeId) => {
|
|
71673
|
+
const settings = blob[nodeId];
|
|
71674
|
+
const devices = Object.entries(settings?.inferenceDevices ?? {}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => {
|
|
71675
|
+
const model = entry.steps?.["object-detection"]?.modelId ?? "";
|
|
71676
|
+
return `${key}=${entry.enabled ? "on" : "off"}:${model}`;
|
|
71677
|
+
});
|
|
71678
|
+
return `${nodeId}|detect=${String(settings?.detect ?? "default")}|${devices.join(",")}`;
|
|
71679
|
+
}).join(";");
|
|
71680
|
+
}
|
|
71681
|
+
//#endregion
|
|
71292
71682
|
//#region src/stale-device-pins.ts
|
|
71293
71683
|
function findStaleDevicePins(pins, nodes) {
|
|
71294
71684
|
const usable = /* @__PURE__ */ new Map();
|
|
@@ -71375,9 +71765,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71375
71765
|
devicePeaks = new DevicePeakTracker();
|
|
71376
71766
|
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71377
71767
|
placementStats = new SessionPlacementStats();
|
|
71378
|
-
/** The stored session placement policy (D748
|
|
71379
|
-
readSessionPlacement = () =>
|
|
71380
|
-
|
|
71768
|
+
/** The stored session placement policy (D748, default `share` since D750). */
|
|
71769
|
+
readSessionPlacement = () => DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
71770
|
+
/** Where the shares come from (D750); `auto` before init. */
|
|
71771
|
+
readShareSource = () => DEFAULT_SHARE_SOURCE;
|
|
71772
|
+
autoTuneService = null;
|
|
71773
|
+
effectiveSharesService = null;
|
|
71381
71774
|
/**
|
|
71382
71775
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
71383
71776
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -71626,6 +72019,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71626
72019
|
getEligibleInferenceDevices: (nodeId, read) => this.resolveEligibleInferenceDevices(nodeId, read),
|
|
71627
72020
|
getEligibleInferenceDevicesIfCached: (nodeId) => this.resolveEligibleInferenceDevicesIfCached(nodeId),
|
|
71628
72021
|
forgetInferenceDeviceProbe: (nodeId) => this.forgetInferenceDeviceProbe(nodeId),
|
|
72022
|
+
noteShareTopologyChange: (trigger) => this.noteShareChange(trigger),
|
|
71629
72023
|
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
71630
72024
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
71631
72025
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
@@ -71683,6 +72077,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71683
72077
|
this.placement = controllers.placement;
|
|
71684
72078
|
this.session = controllers.session;
|
|
71685
72079
|
this.readSessionPlacement = controllers.sessionPlacement;
|
|
72080
|
+
this.readShareSource = controllers.shareSource;
|
|
72081
|
+
this.settingsStore.setEffectiveNodeWeights(() => this.effectiveShares().nodeWeights());
|
|
72082
|
+
this.effectiveShares().start();
|
|
72083
|
+
this.noteShareChange("startup");
|
|
71686
72084
|
this.deviceConfig = controllers.deviceConfig;
|
|
71687
72085
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
71688
72086
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71808,6 +72206,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71808
72206
|
this.detectionWiring = null;
|
|
71809
72207
|
this.unsubAgentOffline?.();
|
|
71810
72208
|
this.unsubAgentOffline = null;
|
|
72209
|
+
this.effectiveSharesService?.stop();
|
|
71811
72210
|
this.unsubAgentOnline?.();
|
|
71812
72211
|
this.unsubAgentOnline = null;
|
|
71813
72212
|
this.nodeLifecycle?.dispose();
|
|
@@ -72210,6 +72609,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72210
72609
|
await this.topology.refreshNodeCapabilities();
|
|
72211
72610
|
this.topology.addKnownRunnerNode(input.agentNodeId);
|
|
72212
72611
|
this.reconcile.schedulePendingRetry();
|
|
72612
|
+
if (input.detect !== void 0) this.noteShareChange(`node-detect:${input.agentNodeId}`);
|
|
72213
72613
|
return { success: true };
|
|
72214
72614
|
}
|
|
72215
72615
|
/**
|
|
@@ -72299,6 +72699,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72299
72699
|
});
|
|
72300
72700
|
await this.resetRemovedModelPins("node-device-write");
|
|
72301
72701
|
this.reconcile.schedulePendingRetry();
|
|
72702
|
+
this.noteShareChange(`node-devices:${input.agentNodeId}`);
|
|
72302
72703
|
return { success: true };
|
|
72303
72704
|
}
|
|
72304
72705
|
/**
|
|
@@ -72379,7 +72780,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72379
72780
|
});
|
|
72380
72781
|
throw new Error(`inference device eligibility: node "${nodeId}" catalog not answered`);
|
|
72381
72782
|
}
|
|
72382
|
-
return this.electableDevices(nodeId, probed, catalog);
|
|
72783
|
+
return this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, probed, catalog));
|
|
72383
72784
|
}
|
|
72384
72785
|
/**
|
|
72385
72786
|
* Forget a node's last device probe (pre-ship review, M4). Called when the
|
|
@@ -72423,7 +72824,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72423
72824
|
this.refreshInferenceDeviceProbe(nodeId);
|
|
72424
72825
|
return {
|
|
72425
72826
|
kind: "eligible",
|
|
72426
|
-
eligible: await this.electableDevices(nodeId, known, catalog)
|
|
72827
|
+
eligible: this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, known, catalog))
|
|
72427
72828
|
};
|
|
72428
72829
|
}
|
|
72429
72830
|
/**
|
|
@@ -72753,10 +73154,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72753
73154
|
clearedCameraOverrides
|
|
72754
73155
|
};
|
|
72755
73156
|
}
|
|
72756
|
-
|
|
72757
|
-
if (this.
|
|
73157
|
+
autoTune() {
|
|
73158
|
+
if (this.autoTuneService !== null) return this.autoTuneService;
|
|
72758
73159
|
const store = this.settingsStore;
|
|
72759
|
-
this.
|
|
73160
|
+
this.autoTuneService = new AutoTuneService({
|
|
72760
73161
|
logger: this.ctx.logger,
|
|
72761
73162
|
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72762
73163
|
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
@@ -72770,11 +73171,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72770
73171
|
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72771
73172
|
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72772
73173
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72773
|
-
cameraLoads: () => this.
|
|
73174
|
+
cameraLoads: () => this.autoTuneCameraLoads(),
|
|
72774
73175
|
peaks: () => this.devicePeaks.peaks(),
|
|
72775
73176
|
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72776
73177
|
sessionPlacement: () => this.readSessionPlacement(),
|
|
72777
|
-
|
|
73178
|
+
readEffectiveWeights: () => this.effectiveShares().effectiveWeights(),
|
|
72778
73179
|
detectorStore: async (input) => {
|
|
72779
73180
|
const api = this.ctx.api;
|
|
72780
73181
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72797,10 +73198,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72797
73198
|
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72798
73199
|
now: () => Date.now()
|
|
72799
73200
|
});
|
|
72800
|
-
return this.
|
|
73201
|
+
return this.autoTuneService;
|
|
72801
73202
|
}
|
|
72802
73203
|
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72803
|
-
async
|
|
73204
|
+
async autoTuneCameraLoads() {
|
|
72804
73205
|
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72805
73206
|
const loads = [];
|
|
72806
73207
|
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
@@ -72822,8 +73223,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72822
73223
|
}
|
|
72823
73224
|
return loads;
|
|
72824
73225
|
}
|
|
72825
|
-
async
|
|
72826
|
-
return this.
|
|
73226
|
+
async planAutoTune(input) {
|
|
73227
|
+
return this.autoTune().plan(input);
|
|
72827
73228
|
}
|
|
72828
73229
|
/**
|
|
72829
73230
|
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
@@ -72879,10 +73280,75 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72879
73280
|
return findStaleDevicePins(pins, offered);
|
|
72880
73281
|
}
|
|
72881
73282
|
async suggestShares(input) {
|
|
72882
|
-
return this.
|
|
73283
|
+
return this.autoTune().suggestShares(input);
|
|
72883
73284
|
}
|
|
72884
|
-
async
|
|
72885
|
-
|
|
73285
|
+
async applyAutoTune(input) {
|
|
73286
|
+
const result = await this.autoTune().apply(input);
|
|
73287
|
+
if (result.kind === "applied") this.noteShareChange("auto-tune-applied");
|
|
73288
|
+
return result;
|
|
73289
|
+
}
|
|
73290
|
+
effectiveShares() {
|
|
73291
|
+
if (this.effectiveSharesService !== null) return this.effectiveSharesService;
|
|
73292
|
+
const store = this.settingsStore;
|
|
73293
|
+
this.effectiveSharesService = new EffectiveSharesService({
|
|
73294
|
+
readSource: () => this.readShareSource(),
|
|
73295
|
+
readStored: async () => storedShareWeights(await store.readAgentSettingsMap()),
|
|
73296
|
+
suggest: () => this.autoTune().suggestShares({ headroom: AUTO_SHARE_HEADROOM }),
|
|
73297
|
+
topologySignature: async () => shareTopologySignature([...new Set(["hub", ...this.topology.collectRunnerNodeIds()])], await store.readAgentSettingsMap()),
|
|
73298
|
+
logger: this.ctx.logger
|
|
73299
|
+
});
|
|
73300
|
+
return this.effectiveSharesService;
|
|
73301
|
+
}
|
|
73302
|
+
/** Something the automatic shares depend on changed: recompute now (best effort). */
|
|
73303
|
+
noteShareChange(trigger) {
|
|
73304
|
+
if (this.settingsStore === null) return;
|
|
73305
|
+
this.effectiveShares().recompute(trigger).catch((err) => {
|
|
73306
|
+
this.ctx.logger.warn("load balancing: share recompute failed", { meta: {
|
|
73307
|
+
trigger,
|
|
73308
|
+
error: errMsg$2(err)
|
|
73309
|
+
} });
|
|
73310
|
+
});
|
|
73311
|
+
}
|
|
73312
|
+
async getEffectiveShares() {
|
|
73313
|
+
const service = this.effectiveShares();
|
|
73314
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])].toSorted();
|
|
73315
|
+
const rows = await Promise.all(nodeIds.map(async (nodeId) => ({
|
|
73316
|
+
nodeId,
|
|
73317
|
+
read: await this.getNodeInferenceDevices({ nodeId }).catch(() => null)
|
|
73318
|
+
})));
|
|
73319
|
+
const detectable = new Set(this.topology.detectionEligibleNodes());
|
|
73320
|
+
const eligibleNodes = rows.filter((r) => r.read !== null && r.read.reachable && detectable.has(r.nodeId) && this.isNodeInferenceUsable(r.nodeId)).map((r) => r.nodeId);
|
|
73321
|
+
const eligibleDevices = {};
|
|
73322
|
+
for (const r of rows) {
|
|
73323
|
+
if (r.read === null) continue;
|
|
73324
|
+
eligibleDevices[r.nodeId] = r.read.devices.filter((d) => d.enabled && d.available && d.exclusion === null).map((d) => d.key);
|
|
73325
|
+
}
|
|
73326
|
+
const auto = service.current();
|
|
73327
|
+
return resolveEffectiveShares({
|
|
73328
|
+
source: this.readShareSource(),
|
|
73329
|
+
policy: this.readSessionPlacement(),
|
|
73330
|
+
nodeIds,
|
|
73331
|
+
stored: storedShareWeights(await this.settingsStore.readAgentSettingsMap()),
|
|
73332
|
+
auto: auto?.weights ?? null,
|
|
73333
|
+
eligibleNodes,
|
|
73334
|
+
eligibleDevices,
|
|
73335
|
+
computedAt: auto?.computedAt ?? null,
|
|
73336
|
+
trigger: auto?.trigger ?? null
|
|
73337
|
+
});
|
|
73338
|
+
}
|
|
73339
|
+
async setShareSource(input) {
|
|
73340
|
+
const previous = this.readShareSource();
|
|
73341
|
+
if (input.source === "manual" && previous === "auto") {
|
|
73342
|
+
const auto = this.effectiveShares().current();
|
|
73343
|
+
if (auto !== null) await this.setLoadBalancingShares(auto.weights);
|
|
73344
|
+
}
|
|
73345
|
+
await this.updateGlobalSettings({ [SHARE_SOURCE_SETTING_KEY]: input.source });
|
|
73346
|
+
this.ctx.logger.info("load balancing: share source changed", { meta: {
|
|
73347
|
+
from: previous,
|
|
73348
|
+
to: input.source
|
|
73349
|
+
} });
|
|
73350
|
+
if (input.source === "auto") this.noteShareChange("source:auto");
|
|
73351
|
+
return { success: true };
|
|
72886
73352
|
}
|
|
72887
73353
|
async getCameraSettings(input) {
|
|
72888
73354
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|