@camstack/addon-pipeline-orchestrator 1.2.291 → 1.2.292
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets--PHyrJOr.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-tJspAxws.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CcezrkWV.mjs} +1 -1
- package/dist/{hostInit-BpNmAa0Q.mjs → hostInit-D7n994Lm.mjs} +2 -2
- package/dist/index.js +541 -75
- package/dist/index.mjs +541 -75
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +0 -26
package/dist/index.mjs
CHANGED
|
@@ -24633,7 +24633,7 @@ var pipelineExecutorCapability = {
|
|
|
24633
24633
|
auth: "admin"
|
|
24634
24634
|
}),
|
|
24635
24635
|
/**
|
|
24636
|
-
*
|
|
24636
|
+
* Auto tune (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24637
24637
|
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24638
24638
|
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24639
24639
|
* running model's own defaults apply. Every other key of a camera's store
|
|
@@ -25018,10 +25018,14 @@ var pipelineExecutorCapability = {
|
|
|
25018
25018
|
* full or drowning node yields to a free one (D748).
|
|
25019
25019
|
*/
|
|
25020
25020
|
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25021
|
-
|
|
25021
|
+
/**
|
|
25022
|
+
* `share` since D750: the percentages are held by default. `least-loaded` stays
|
|
25023
|
+
* selectable.
|
|
25024
|
+
*/
|
|
25025
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "share";
|
|
25022
25026
|
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25023
25027
|
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25024
|
-
/** Parse a stored value; anything unknown is the default
|
|
25028
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25025
25029
|
function parseSessionPlacementPolicy(value) {
|
|
25026
25030
|
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25027
25031
|
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
@@ -25104,7 +25108,62 @@ var SuggestSharesResultSchema = object({
|
|
|
25104
25108
|
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25105
25109
|
});
|
|
25106
25110
|
/**
|
|
25107
|
-
*
|
|
25111
|
+
* Where the node and device shares come from (D750).
|
|
25112
|
+
* - `auto` — computed by the system from the cluster's measured capacity at
|
|
25113
|
+
* the Auto tune models, and recomputed whenever a node or device joins,
|
|
25114
|
+
* leaves, is switched on or off, or changes model.
|
|
25115
|
+
* - `manual` — the operator's stored sliders. A node or device that cannot
|
|
25116
|
+
* take sessions gives its share to the others in proportion; the stored
|
|
25117
|
+
* values are never rewritten for that.
|
|
25118
|
+
*/
|
|
25119
|
+
var ShareSourceSchema = _enum(["auto", "manual"]);
|
|
25120
|
+
var DEFAULT_SHARE_SOURCE = "auto";
|
|
25121
|
+
/** Headroom the automatic shares are computed at (Auto tune's default). */
|
|
25122
|
+
var AUTO_SHARE_HEADROOM = .4;
|
|
25123
|
+
/** The global-settings key the share source is stored under (orchestrator `load` section). */
|
|
25124
|
+
var SHARE_SOURCE_SETTING_KEY = "shareSource";
|
|
25125
|
+
/** Parse a stored value; anything unknown is the default. */
|
|
25126
|
+
function parseShareSource(value) {
|
|
25127
|
+
const parsed = ShareSourceSchema.safeParse(value);
|
|
25128
|
+
return parsed.success ? parsed.data : DEFAULT_SHARE_SOURCE;
|
|
25129
|
+
}
|
|
25130
|
+
var SetShareSourceInputSchema = object({ source: ShareSourceSchema });
|
|
25131
|
+
object({
|
|
25132
|
+
nodes: record(string(), number()),
|
|
25133
|
+
devices: record(string(), record(string(), number()))
|
|
25134
|
+
});
|
|
25135
|
+
var EffectiveDeviceShareSchema = object({
|
|
25136
|
+
deviceKey: string(),
|
|
25137
|
+
/** Can take sessions now (enabled, available, can run the root). */
|
|
25138
|
+
eligible: boolean(),
|
|
25139
|
+
/** The share the source assigns, as a percentage of the node's devices. */
|
|
25140
|
+
configuredPct: number(),
|
|
25141
|
+
/** The share actually used, after an ineligible device's share went to the others. */
|
|
25142
|
+
effectivePct: number()
|
|
25143
|
+
});
|
|
25144
|
+
var EffectiveNodeShareSchema = object({
|
|
25145
|
+
nodeId: string(),
|
|
25146
|
+
eligible: boolean(),
|
|
25147
|
+
configuredPct: number(),
|
|
25148
|
+
effectivePct: number(),
|
|
25149
|
+
devices: array(EffectiveDeviceShareSchema).readonly()
|
|
25150
|
+
});
|
|
25151
|
+
var EffectiveSharesSchema = object({
|
|
25152
|
+
source: ShareSourceSchema,
|
|
25153
|
+
policy: SessionPlacementPolicySchema,
|
|
25154
|
+
/** `auto` with no computed answer yet: the stored sliders are used meanwhile. */
|
|
25155
|
+
autoPending: boolean(),
|
|
25156
|
+
/** When the automatic shares were last computed (ms epoch); null = never. */
|
|
25157
|
+
computedAt: number().nullable(),
|
|
25158
|
+
/** What made the system recompute them last. */
|
|
25159
|
+
trigger: string().nullable(),
|
|
25160
|
+
nodes: array(EffectiveNodeShareSchema).readonly(),
|
|
25161
|
+
/** The relative weights the election reads (before eligibility). */
|
|
25162
|
+
nodeWeights: record(string(), number()),
|
|
25163
|
+
deviceWeights: record(string(), record(string(), number()))
|
|
25164
|
+
});
|
|
25165
|
+
/**
|
|
25166
|
+
* The wire shapes of the Auto tune plan and apply (D745), used by the
|
|
25108
25167
|
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25109
25168
|
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
25110
25169
|
* schemas are what crosses the cap boundary.
|
|
@@ -25239,7 +25298,7 @@ var DetectorStoreClearSchema = object({
|
|
|
25239
25298
|
deviceId: number(),
|
|
25240
25299
|
keys: array(string()).readonly()
|
|
25241
25300
|
});
|
|
25242
|
-
var
|
|
25301
|
+
var AutoTuneWarningSchema = discriminatedUnion("kind", [
|
|
25243
25302
|
object({
|
|
25244
25303
|
kind: literal("detector-store-cluster-wide"),
|
|
25245
25304
|
skippedNodes: array(string()).readonly()
|
|
@@ -25254,7 +25313,7 @@ var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
|
25254
25313
|
deviceKey: string()
|
|
25255
25314
|
})
|
|
25256
25315
|
]);
|
|
25257
|
-
var
|
|
25316
|
+
var AutoTuneOptionsSchema = object({
|
|
25258
25317
|
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25259
25318
|
headroom: number().min(.3).max(.5).default(.4),
|
|
25260
25319
|
/**
|
|
@@ -25265,7 +25324,7 @@ var BestSettingsOptionsSchema = object({
|
|
|
25265
25324
|
*/
|
|
25266
25325
|
shares: LoadBalancingSharesInputSchema.optional()
|
|
25267
25326
|
});
|
|
25268
|
-
var
|
|
25327
|
+
var AutoTunePlanSchema = object({
|
|
25269
25328
|
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25270
25329
|
planHash: string(),
|
|
25271
25330
|
family: literal("yolov9"),
|
|
@@ -25277,13 +25336,13 @@ var BestSettingsPlanSchema = object({
|
|
|
25277
25336
|
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25278
25337
|
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25279
25338
|
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25280
|
-
warnings: array(
|
|
25339
|
+
warnings: array(AutoTuneWarningSchema).readonly()
|
|
25281
25340
|
});
|
|
25282
|
-
var
|
|
25341
|
+
var AutoTuneApplyInputSchema = object({
|
|
25283
25342
|
planHash: string(),
|
|
25284
25343
|
headroom: number().min(.3).max(.5).default(.4)
|
|
25285
25344
|
});
|
|
25286
|
-
var
|
|
25345
|
+
var AutoTunePhaseResultSchema = object({
|
|
25287
25346
|
phase: _enum([
|
|
25288
25347
|
"download",
|
|
25289
25348
|
"camera-overrides",
|
|
@@ -25294,19 +25353,19 @@ var BestSettingsPhaseResultSchema = object({
|
|
|
25294
25353
|
ok: boolean(),
|
|
25295
25354
|
detail: string()
|
|
25296
25355
|
});
|
|
25297
|
-
var
|
|
25356
|
+
var AutoTuneApplyResultSchema = discriminatedUnion("kind", [
|
|
25298
25357
|
object({
|
|
25299
25358
|
kind: literal("applied"),
|
|
25300
25359
|
planHash: string(),
|
|
25301
|
-
phases: array(
|
|
25360
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25302
25361
|
}),
|
|
25303
25362
|
object({
|
|
25304
25363
|
kind: literal("aborted"),
|
|
25305
|
-
phases: array(
|
|
25364
|
+
phases: array(AutoTunePhaseResultSchema).readonly()
|
|
25306
25365
|
}),
|
|
25307
25366
|
object({
|
|
25308
25367
|
kind: literal("stale"),
|
|
25309
|
-
plan:
|
|
25368
|
+
plan: AutoTunePlanSchema
|
|
25310
25369
|
}),
|
|
25311
25370
|
object({
|
|
25312
25371
|
kind: literal("refused"),
|
|
@@ -27487,7 +27546,7 @@ var pipelineOrchestratorCapability = {
|
|
|
27487
27546
|
auth: "admin"
|
|
27488
27547
|
}),
|
|
27489
27548
|
/**
|
|
27490
|
-
*
|
|
27549
|
+
* Auto tune (D745) — DRY RUN. Recommends ONE detector family (YOLOv9)
|
|
27491
27550
|
* for the whole cluster from each node's static hardware class, the
|
|
27492
27551
|
* measured latency table and the detection demand, and lists every write
|
|
27493
27552
|
* the apply would make: the node model pins set, the enrichment pins and
|
|
@@ -27496,14 +27555,14 @@ var pipelineOrchestratorCapability = {
|
|
|
27496
27555
|
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27497
27556
|
* rules are never part of it. `planHash` names exactly this diff.
|
|
27498
27557
|
*/
|
|
27499
|
-
|
|
27558
|
+
planAutoTune: method(AutoTuneOptionsSchema, AutoTunePlanSchema, { auth: "admin" }),
|
|
27500
27559
|
/**
|
|
27501
|
-
*
|
|
27560
|
+
* Auto tune (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27502
27561
|
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27503
27562
|
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27504
27563
|
* nodes are skipped; there is no undo.
|
|
27505
27564
|
*/
|
|
27506
|
-
|
|
27565
|
+
applyAutoTune: method(AutoTuneApplyInputSchema, AutoTuneApplyResultSchema, {
|
|
27507
27566
|
kind: "mutation",
|
|
27508
27567
|
auth: "admin"
|
|
27509
27568
|
}),
|
|
@@ -27529,11 +27588,27 @@ var pipelineOrchestratorCapability = {
|
|
|
27529
27588
|
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27530
27589
|
/**
|
|
27531
27590
|
* Load balancing (D748) — shares proportional to each node's measured
|
|
27532
|
-
* capacity at the models
|
|
27591
|
+
* capacity at the models Auto tune would pick, iterated (≤ 3 rounds)
|
|
27533
27592
|
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27534
27593
|
* Writes nothing.
|
|
27535
27594
|
*/
|
|
27536
27595
|
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27596
|
+
/**
|
|
27597
|
+
* Load balancing (D750) — the shares placement reads now: the source
|
|
27598
|
+
* (automatic capacity shares or the operator's sliders), each node's and
|
|
27599
|
+
* device's configured share, and the effective share after a node or
|
|
27600
|
+
* device that cannot take sessions gave its share to the others.
|
|
27601
|
+
*/
|
|
27602
|
+
getEffectiveShares: method(_void(), EffectiveSharesSchema, { auth: "admin" }),
|
|
27603
|
+
/**
|
|
27604
|
+
* Load balancing (D750) — switch between automatic and manual shares.
|
|
27605
|
+
* Going manual seeds the sliders with the current automatic shares (one
|
|
27606
|
+
* atomic write), so nothing jumps; going automatic recomputes at once.
|
|
27607
|
+
*/
|
|
27608
|
+
setShareSource: method(SetShareSourceInputSchema, object({ success: literal(true) }), {
|
|
27609
|
+
kind: "mutation",
|
|
27610
|
+
auth: "admin"
|
|
27611
|
+
}),
|
|
27537
27612
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27538
27613
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27539
27614
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -43557,7 +43632,7 @@ Object.freeze({
|
|
|
43557
43632
|
addonId: null,
|
|
43558
43633
|
access: "view"
|
|
43559
43634
|
},
|
|
43560
|
-
"pipelineOrchestrator.
|
|
43635
|
+
"pipelineOrchestrator.applyAutoTune": {
|
|
43561
43636
|
capName: "pipeline-orchestrator",
|
|
43562
43637
|
capScope: "system",
|
|
43563
43638
|
addonId: null,
|
|
@@ -43677,6 +43752,12 @@ Object.freeze({
|
|
|
43677
43752
|
addonId: null,
|
|
43678
43753
|
access: "view"
|
|
43679
43754
|
},
|
|
43755
|
+
"pipelineOrchestrator.getEffectiveShares": {
|
|
43756
|
+
capName: "pipeline-orchestrator",
|
|
43757
|
+
capScope: "system",
|
|
43758
|
+
addonId: null,
|
|
43759
|
+
access: "view"
|
|
43760
|
+
},
|
|
43680
43761
|
"pipelineOrchestrator.getGlobalMetrics": {
|
|
43681
43762
|
capName: "pipeline-orchestrator",
|
|
43682
43763
|
capScope: "system",
|
|
@@ -43743,7 +43824,7 @@ Object.freeze({
|
|
|
43743
43824
|
addonId: null,
|
|
43744
43825
|
access: "create"
|
|
43745
43826
|
},
|
|
43746
|
-
"pipelineOrchestrator.
|
|
43827
|
+
"pipelineOrchestrator.planAutoTune": {
|
|
43747
43828
|
capName: "pipeline-orchestrator",
|
|
43748
43829
|
capScope: "system",
|
|
43749
43830
|
addonId: null,
|
|
@@ -43875,6 +43956,12 @@ Object.freeze({
|
|
|
43875
43956
|
addonId: null,
|
|
43876
43957
|
access: "create"
|
|
43877
43958
|
},
|
|
43959
|
+
"pipelineOrchestrator.setShareSource": {
|
|
43960
|
+
capName: "pipeline-orchestrator",
|
|
43961
|
+
capScope: "system",
|
|
43962
|
+
addonId: null,
|
|
43963
|
+
access: "create"
|
|
43964
|
+
},
|
|
43878
43965
|
"pipelineOrchestrator.suggestShares": {
|
|
43879
43966
|
capName: "pipeline-orchestrator",
|
|
43880
43967
|
capScope: "system",
|
|
@@ -51113,7 +51200,7 @@ function modelBindingKey(key) {
|
|
|
51113
51200
|
/**
|
|
51114
51201
|
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
51115
51202
|
* flattened (D738). One definition for the runner (which applies the binding)
|
|
51116
|
-
* and the orchestrator's
|
|
51203
|
+
* and the orchestrator's Auto tune (which clears those overrides, D745).
|
|
51117
51204
|
*/
|
|
51118
51205
|
function collectModelBoundKeys(fields) {
|
|
51119
51206
|
return fields.flatMap((field) => {
|
|
@@ -51725,7 +51812,7 @@ function estimateDeviceDemand(input) {
|
|
|
51725
51812
|
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51726
51813
|
}
|
|
51727
51814
|
/** The family the cluster runs. */
|
|
51728
|
-
var
|
|
51815
|
+
var AUTO_TUNE_FAMILY = "yolov9";
|
|
51729
51816
|
function round(value, digits = 3) {
|
|
51730
51817
|
const f = 10 ** digits;
|
|
51731
51818
|
return Math.round(value * f) / f;
|
|
@@ -51738,7 +51825,7 @@ function fitsOn(cell, demand, headroom) {
|
|
|
51738
51825
|
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51739
51826
|
}
|
|
51740
51827
|
function candidatesFor(contexts, formatsByModel) {
|
|
51741
|
-
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(
|
|
51828
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(AUTO_TUNE_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51742
51829
|
modelId: id,
|
|
51743
51830
|
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51744
51831
|
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
@@ -51989,6 +52076,57 @@ function electByShare(candidates) {
|
|
|
51989
52076
|
reason: "share"
|
|
51990
52077
|
};
|
|
51991
52078
|
}
|
|
52079
|
+
/** Weight a node or device has when its source names none (the balancer's own default). */
|
|
52080
|
+
var DEFAULT_WEIGHT = 1;
|
|
52081
|
+
function positive(value) {
|
|
52082
|
+
return value !== void 0 && value > 0 ? value : DEFAULT_WEIGHT;
|
|
52083
|
+
}
|
|
52084
|
+
function percentages(keys, weightOf, counts) {
|
|
52085
|
+
const total = keys.reduce((sum, k) => sum + (counts(k) ? weightOf(k) : 0), 0);
|
|
52086
|
+
return new Map(keys.map((k) => [k, total > 0 && counts(k) ? Math.round(weightOf(k) / total * 1e3) / 10 : 0]));
|
|
52087
|
+
}
|
|
52088
|
+
function resolveEffectiveShares(input) {
|
|
52089
|
+
const weights = input.source === "auto" && input.auto !== null && input.auto !== null ? input.auto : input.stored;
|
|
52090
|
+
const nodeWeights = {};
|
|
52091
|
+
for (const nodeId of input.nodeIds) nodeWeights[nodeId] = positive(weights.nodes[nodeId]);
|
|
52092
|
+
const eligibleNodes = new Set(input.eligibleNodes);
|
|
52093
|
+
const configured = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52094
|
+
const effective = percentages(input.nodeIds, (k) => nodeWeights[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(k));
|
|
52095
|
+
const deviceWeights = {};
|
|
52096
|
+
const nodes = input.nodeIds.map((nodeId) => {
|
|
52097
|
+
const eligibleKeys = new Set(input.eligibleDevices[nodeId] ?? []);
|
|
52098
|
+
const named = weights.devices[nodeId] ?? {};
|
|
52099
|
+
const keys = [...new Set([...Object.keys(named), ...eligibleKeys])].toSorted();
|
|
52100
|
+
const perDevice = {};
|
|
52101
|
+
for (const key of keys) perDevice[key] = positive(named[key]);
|
|
52102
|
+
if (keys.length > 0) deviceWeights[nodeId] = perDevice;
|
|
52103
|
+
const devConfigured = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, () => true);
|
|
52104
|
+
const devEffective = percentages(keys, (k) => perDevice[k] ?? DEFAULT_WEIGHT, (k) => eligibleNodes.has(nodeId) && eligibleKeys.has(k));
|
|
52105
|
+
const devices = keys.map((deviceKey) => ({
|
|
52106
|
+
deviceKey,
|
|
52107
|
+
eligible: eligibleKeys.has(deviceKey),
|
|
52108
|
+
configuredPct: devConfigured.get(deviceKey) ?? 0,
|
|
52109
|
+
effectivePct: devEffective.get(deviceKey) ?? 0
|
|
52110
|
+
}));
|
|
52111
|
+
return {
|
|
52112
|
+
nodeId,
|
|
52113
|
+
eligible: eligibleNodes.has(nodeId),
|
|
52114
|
+
configuredPct: configured.get(nodeId) ?? 0,
|
|
52115
|
+
effectivePct: effective.get(nodeId) ?? 0,
|
|
52116
|
+
devices
|
|
52117
|
+
};
|
|
52118
|
+
});
|
|
52119
|
+
return {
|
|
52120
|
+
source: input.source,
|
|
52121
|
+
policy: input.policy ?? "share",
|
|
52122
|
+
autoPending: input.source === "auto" && input.auto === null,
|
|
52123
|
+
computedAt: input.computedAt ?? null,
|
|
52124
|
+
trigger: input.trigger ?? null,
|
|
52125
|
+
nodes,
|
|
52126
|
+
nodeWeights,
|
|
52127
|
+
deviceWeights
|
|
52128
|
+
};
|
|
52129
|
+
}
|
|
51992
52130
|
/**
|
|
51993
52131
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
51994
52132
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -62383,6 +62521,7 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62383
62521
|
};
|
|
62384
62522
|
const okByNode = new Map(input.verdicts.filter((v) => v.state === "ok").map((v) => [v.nodeId, v]));
|
|
62385
62523
|
const sourceAttached = input.attachedByNode[saturated.nodeId] ?? 0;
|
|
62524
|
+
if (input.shares !== void 0) return planShareTarget(input, input.shares, saturated, movable, okByNode, now, guards);
|
|
62386
62525
|
for (const camera of movable) {
|
|
62387
62526
|
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62388
62527
|
const target = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => {
|
|
@@ -62407,6 +62546,52 @@ function planStressFailover(input, now, guards = DEFAULT_STRESS_FAILOVER_GUARDS)
|
|
|
62407
62546
|
nodeId: saturated.nodeId
|
|
62408
62547
|
};
|
|
62409
62548
|
}
|
|
62549
|
+
/** Deficits equal up to float noise. */
|
|
62550
|
+
var SHARE_EPSILON = 1e-9;
|
|
62551
|
+
function underCap(input, nodeId) {
|
|
62552
|
+
const cap = input.nodeCaps?.[nodeId];
|
|
62553
|
+
const attached = input.attachedByNode[nodeId] ?? 0;
|
|
62554
|
+
return cap === null || cap === void 0 || cap <= 0 || attached < cap;
|
|
62555
|
+
}
|
|
62556
|
+
/**
|
|
62557
|
+
* Share placement's failover target (D750): among the camera's eligible,
|
|
62558
|
+
* `ok`, under-cap nodes, the one furthest below its share of the cameras
|
|
62559
|
+
* attached across the source and those nodes — the same deficit rule a new
|
|
62560
|
+
* session is placed by. A node at or over its share is no target, so a move
|
|
62561
|
+
* never undoes the balance. Nothing below its share ⇒ hold, naming the camera.
|
|
62562
|
+
*/
|
|
62563
|
+
function planShareTarget(input, shares, saturated, movable, okByNode, now, guards) {
|
|
62564
|
+
const shareOf = (nodeId) => {
|
|
62565
|
+
const share = shares[nodeId];
|
|
62566
|
+
return share !== void 0 && share > 0 ? share : 0;
|
|
62567
|
+
};
|
|
62568
|
+
for (const camera of movable) {
|
|
62569
|
+
const banned = new Set(input.history.filter((h) => h.deviceId === camera.deviceId && now - h.at < guards.returnBanMs).map((h) => h.fromNodeId));
|
|
62570
|
+
const healthy = camera.eligibleNodes.filter((nodeId) => nodeId !== saturated.nodeId && !banned.has(nodeId)).map((nodeId) => okByNode.get(nodeId)).filter((v) => v !== void 0).filter((v) => underCap(input, v.nodeId));
|
|
62571
|
+
const set = [saturated.nodeId, ...healthy.map((v) => v.nodeId)];
|
|
62572
|
+
const shareSum = set.reduce((sum, nodeId) => sum + shareOf(nodeId), 0);
|
|
62573
|
+
const attachedSum = set.reduce((sum, nodeId) => sum + (input.attachedByNode[nodeId] ?? 0), 0);
|
|
62574
|
+
if (shareSum <= 0) continue;
|
|
62575
|
+
const deficit = (nodeId) => shareOf(nodeId) / shareSum * attachedSum - (input.attachedByNode[nodeId] ?? 0);
|
|
62576
|
+
const target = healthy.filter((v) => deficit(v.nodeId) > SHARE_EPSILON).toSorted((a, b) => deficit(b.nodeId) - deficit(a.nodeId) || a.signals.score - b.signals.score || a.nodeId.localeCompare(b.nodeId))[0];
|
|
62577
|
+
if (!target) continue;
|
|
62578
|
+
return {
|
|
62579
|
+
kind: "move",
|
|
62580
|
+
deviceId: camera.deviceId,
|
|
62581
|
+
fromNodeId: saturated.nodeId,
|
|
62582
|
+
toNodeId: target.nodeId,
|
|
62583
|
+
fromScore: saturated.signals.score,
|
|
62584
|
+
toScore: target.signals.score
|
|
62585
|
+
};
|
|
62586
|
+
}
|
|
62587
|
+
const first = movable[0];
|
|
62588
|
+
return {
|
|
62589
|
+
kind: "hold",
|
|
62590
|
+
reason: "no-target",
|
|
62591
|
+
nodeId: saturated.nodeId,
|
|
62592
|
+
...first !== void 0 ? { deviceId: first.deviceId } : {}
|
|
62593
|
+
};
|
|
62594
|
+
}
|
|
62410
62595
|
//#endregion
|
|
62411
62596
|
//#region src/node-stress-controller.ts
|
|
62412
62597
|
/** Narrow an unknown settings value to a {@link NodeStressMode}. */
|
|
@@ -62615,17 +62800,27 @@ var NodeStressController = class NodeStressController {
|
|
|
62615
62800
|
if (!verdicts.some((v) => v.state === "saturated")) return;
|
|
62616
62801
|
this.deps.moves.pruneOlderThan(now - NodeStressController.HISTORY_TTL_MS);
|
|
62617
62802
|
const history = this.deps.moves.entries().filter((h) => now - h.at < NodeStressController.HISTORY_TTL_MS);
|
|
62803
|
+
const candidates = await this.collectCandidates(now);
|
|
62804
|
+
const shares = await this.deps.readFailoverShares?.() ?? null;
|
|
62618
62805
|
const plan = planStressFailover({
|
|
62619
62806
|
verdicts,
|
|
62620
|
-
candidates
|
|
62807
|
+
candidates,
|
|
62621
62808
|
history,
|
|
62622
62809
|
nodeCaps: await this.deps.settingsStore.buildNodeCaps(),
|
|
62623
|
-
attachedByNode: this.attachedByNode()
|
|
62810
|
+
attachedByNode: this.attachedByNode(),
|
|
62811
|
+
...shares !== null ? { shares } : {}
|
|
62624
62812
|
}, now, this.guards());
|
|
62625
62813
|
if (plan.kind === "hold") {
|
|
62626
62814
|
this.deps.logger.warn("node saturated — no failover applied", {
|
|
62627
|
-
tags: {
|
|
62628
|
-
|
|
62815
|
+
tags: {
|
|
62816
|
+
nodeId: plan.nodeId ?? "",
|
|
62817
|
+
...plan.deviceId !== void 0 ? { deviceId: plan.deviceId } : {}
|
|
62818
|
+
},
|
|
62819
|
+
meta: {
|
|
62820
|
+
reason: plan.reason,
|
|
62821
|
+
rule: shares !== null ? "share-deficit" : "fewest-cameras",
|
|
62822
|
+
...plan.reason === "no-target" && shares !== null ? { why: "no healthy node below its share can take it" } : {}
|
|
62823
|
+
}
|
|
62629
62824
|
});
|
|
62630
62825
|
return;
|
|
62631
62826
|
}
|
|
@@ -65237,12 +65432,30 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
65237
65432
|
* `balance()` call so unpinned cameras distribute proportionally to weight.
|
|
65238
65433
|
*/
|
|
65239
65434
|
async buildNodeWeights() {
|
|
65435
|
+
if (this.effectiveNodeWeights !== null) return this.effectiveNodeWeights();
|
|
65436
|
+
return this.buildStoredNodeWeights();
|
|
65437
|
+
}
|
|
65438
|
+
/**
|
|
65439
|
+
* The operator's stored node sliders, whatever the share source — what
|
|
65440
|
+
* `manual` applies and what `auto` falls back to before its first answer.
|
|
65441
|
+
*/
|
|
65442
|
+
async buildStoredNodeWeights() {
|
|
65240
65443
|
const blob = await this.agentSettingsState.get();
|
|
65241
65444
|
const weights = {};
|
|
65242
65445
|
for (const [nodeId, settings] of Object.entries(blob)) if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) weights[nodeId] = settings.detectWeight;
|
|
65243
65446
|
return weights;
|
|
65244
65447
|
}
|
|
65245
65448
|
/**
|
|
65449
|
+
* Where {@link buildNodeWeights} reads from once the share source is wired
|
|
65450
|
+
* (D750): the effective shares (automatic or manual). Every balancer, the
|
|
65451
|
+
* share election and the stress failover read through it, so they can never
|
|
65452
|
+
* disagree on what a node's share is. Unset (tests, early boot) ⇒ stored.
|
|
65453
|
+
*/
|
|
65454
|
+
effectiveNodeWeights = null;
|
|
65455
|
+
setEffectiveNodeWeights(read) {
|
|
65456
|
+
this.effectiveNodeWeights = read;
|
|
65457
|
+
}
|
|
65458
|
+
/**
|
|
65246
65459
|
* The per-(node,device) BASE provisioning (C7.2/C7.4) for one `(node,
|
|
65247
65460
|
* deviceKey)` — the effective `stepId → {modelId?, settings?}` map the
|
|
65248
65461
|
* dispatch provisioning stage attaches onto the resolved step tree before
|
|
@@ -67600,9 +67813,10 @@ var SessionDispatchController = class {
|
|
|
67600
67813
|
deviceId: a.deviceId,
|
|
67601
67814
|
nodeId: a.agentNodeId
|
|
67602
67815
|
}))]);
|
|
67816
|
+
const present = new Set(this.deps.topology.knownRunnerNodeIds());
|
|
67603
67817
|
const election = electNodeByShare({
|
|
67604
67818
|
loads: input.loads,
|
|
67605
|
-
eligible: input.eligible,
|
|
67819
|
+
eligible: input.eligible.filter((nodeId) => present.has(nodeId)),
|
|
67606
67820
|
weights: input.weights,
|
|
67607
67821
|
caps: input.nodeCaps,
|
|
67608
67822
|
runningByNode: running
|
|
@@ -69071,7 +69285,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69071
69285
|
attach: (nodeId, config) => placement.attachOn(nodeId, config),
|
|
69072
69286
|
detach: (nodeId, deviceId) => placement.detachOn(nodeId, deviceId),
|
|
69073
69287
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
69074
|
-
readGlobalSettings: () => globalSettings
|
|
69288
|
+
readGlobalSettings: () => globalSettings,
|
|
69289
|
+
readFailoverShares: async () => readSessionPlacement() === "share" ? settingsStore.buildNodeWeights() : null
|
|
69075
69290
|
});
|
|
69076
69291
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
69077
69292
|
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
@@ -69087,6 +69302,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69087
69302
|
nodeStress.start();
|
|
69088
69303
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69089
69304
|
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
69305
|
+
const readShareSource = () => parseShareSource(globalSettings?.[SHARE_SOURCE_SETTING_KEY]);
|
|
69090
69306
|
/**
|
|
69091
69307
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
69092
69308
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -69423,6 +69639,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69423
69639
|
nodeLifecycle.unsubscribeDetectionPipelineReadiness(nodeId);
|
|
69424
69640
|
audio.markNodeOffline(nodeId);
|
|
69425
69641
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
69642
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-offline:${nodeId}`);
|
|
69426
69643
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
69427
69644
|
const msg = errMsg$2(err);
|
|
69428
69645
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
@@ -69466,6 +69683,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69466
69683
|
});
|
|
69467
69684
|
});
|
|
69468
69685
|
reconcile.scheduleReconcile();
|
|
69686
|
+
if (!nodeId.includes("/")) deps.noteShareTopologyChange(`node-online:${nodeId}`);
|
|
69469
69687
|
});
|
|
69470
69688
|
const unsubAgentOnline = () => {
|
|
69471
69689
|
unsubAgentOnlineEvent();
|
|
@@ -69706,7 +69924,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
69706
69924
|
unsubOrchestratorSubscriptions,
|
|
69707
69925
|
pendingRetryTimer,
|
|
69708
69926
|
autoRebalanceTimer,
|
|
69709
|
-
sessionPlacement: () => readSessionPlacement()
|
|
69927
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69928
|
+
shareSource: () => readShareSource()
|
|
69710
69929
|
};
|
|
69711
69930
|
}
|
|
69712
69931
|
//#endregion
|
|
@@ -69755,13 +69974,13 @@ function buildGlobalSettingsSections(options) {
|
|
|
69755
69974
|
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69756
69975
|
type: "select",
|
|
69757
69976
|
label: "Session placement",
|
|
69758
|
-
description: "How a new detection session picks its node and device.
|
|
69977
|
+
description: "How a new detection session picks its node and device. Honour shares (default) = the share of PARALLEL sessions is held to the node and device shares, and a node at its cap yields to a free one. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy.",
|
|
69759
69978
|
options: [{
|
|
69760
|
-
value: "least-loaded",
|
|
69761
|
-
label: "Least-loaded"
|
|
69762
|
-
}, {
|
|
69763
69979
|
value: "share",
|
|
69764
69980
|
label: "Honour shares"
|
|
69981
|
+
}, {
|
|
69982
|
+
value: "least-loaded",
|
|
69983
|
+
label: "Least-loaded"
|
|
69765
69984
|
}],
|
|
69766
69985
|
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69767
69986
|
},
|
|
@@ -70323,9 +70542,9 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70323
70542
|
defaultRows: 4
|
|
70324
70543
|
}];
|
|
70325
70544
|
//#endregion
|
|
70326
|
-
//#region src/
|
|
70545
|
+
//#region src/auto-tune/plan-diff.ts
|
|
70327
70546
|
/**
|
|
70328
|
-
* The
|
|
70547
|
+
* The Auto tune writes, as pure transforms of the orchestrator's two
|
|
70329
70548
|
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
70330
70549
|
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
70331
70550
|
*
|
|
@@ -70349,7 +70568,7 @@ var pipelineOrchestratorWidgets = [{
|
|
|
70349
70568
|
*/
|
|
70350
70569
|
/** The root detector step. */
|
|
70351
70570
|
var DETECTOR_STEP_ID = "object-detection";
|
|
70352
|
-
/** Steps whose `modelId` pin
|
|
70571
|
+
/** Steps whose `modelId` pin Auto tune never touches. */
|
|
70353
70572
|
function isModelPinKept(stepId) {
|
|
70354
70573
|
return stepId.endsWith("-embedding");
|
|
70355
70574
|
}
|
|
@@ -70401,7 +70620,7 @@ function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
|
70401
70620
|
};
|
|
70402
70621
|
}
|
|
70403
70622
|
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70404
|
-
function
|
|
70623
|
+
function applyNodeAutoTune(blob, input) {
|
|
70405
70624
|
const next = { ...blob };
|
|
70406
70625
|
const changes = [];
|
|
70407
70626
|
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
@@ -70471,7 +70690,7 @@ function scrubTree(steps, input, onClear) {
|
|
|
70471
70690
|
});
|
|
70472
70691
|
}
|
|
70473
70692
|
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70474
|
-
function
|
|
70693
|
+
function applyCameraAutoTune(blob, input) {
|
|
70475
70694
|
const next = {};
|
|
70476
70695
|
const clears = [];
|
|
70477
70696
|
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
@@ -70555,12 +70774,12 @@ function applyCameraBestSettings(blob, input) {
|
|
|
70555
70774
|
};
|
|
70556
70775
|
}
|
|
70557
70776
|
//#endregion
|
|
70558
|
-
//#region src/
|
|
70777
|
+
//#region src/auto-tune/auto-tune-service.ts
|
|
70559
70778
|
/**
|
|
70560
|
-
*
|
|
70779
|
+
* Auto tune (D745): gather the inputs, compute the plan, and apply exactly
|
|
70561
70780
|
* the plan the operator confirmed.
|
|
70562
70781
|
*
|
|
70563
|
-
* The plan is the recommender (`@camstack/types`
|
|
70782
|
+
* The plan is the recommender (`@camstack/types` auto-tune) over every
|
|
70564
70783
|
* runner node, plus the diff of the writes (`plan-diff.ts`) and the
|
|
70565
70784
|
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70566
70785
|
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
@@ -70629,7 +70848,7 @@ function toPercent(weights) {
|
|
|
70629
70848
|
}
|
|
70630
70849
|
return out;
|
|
70631
70850
|
}
|
|
70632
|
-
var
|
|
70851
|
+
var AutoTuneService = class {
|
|
70633
70852
|
deps;
|
|
70634
70853
|
applying = false;
|
|
70635
70854
|
constructor(deps) {
|
|
@@ -70639,7 +70858,7 @@ var BestSettingsService = class {
|
|
|
70639
70858
|
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70640
70859
|
}
|
|
70641
70860
|
/**
|
|
70642
|
-
* Shares proportional to each node's capacity at the models
|
|
70861
|
+
* Shares proportional to each node's capacity at the models Auto tune
|
|
70643
70862
|
* would pick under those very shares (D748). Model and load depend on each
|
|
70644
70863
|
* other, so it iterates from equal shares until the rounded shares stop
|
|
70645
70864
|
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
@@ -70731,7 +70950,8 @@ var BestSettingsService = class {
|
|
|
70731
70950
|
catalog
|
|
70732
70951
|
};
|
|
70733
70952
|
}));
|
|
70734
|
-
const
|
|
70953
|
+
const effective = await this.deps.readEffectiveWeights();
|
|
70954
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? effective.devices[nodeId]?.[key] ?? stored;
|
|
70735
70955
|
const nodes = gathered.map((g) => ({
|
|
70736
70956
|
nodeId: g.nodeId,
|
|
70737
70957
|
reachable: g.devices.reachable && g.catalog !== null,
|
|
@@ -70757,7 +70977,7 @@ var BestSettingsService = class {
|
|
|
70757
70977
|
peaks: this.deps.peaks(),
|
|
70758
70978
|
...policy === "share" ? { shares: {
|
|
70759
70979
|
nodes: {
|
|
70760
|
-
...
|
|
70980
|
+
...effective.nodes,
|
|
70761
70981
|
...shares?.nodes
|
|
70762
70982
|
},
|
|
70763
70983
|
clusterPeak: this.deps.clusterPeak()
|
|
@@ -70804,8 +71024,8 @@ var BestSettingsService = class {
|
|
|
70804
71024
|
targets,
|
|
70805
71025
|
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70806
71026
|
};
|
|
70807
|
-
const nodeChanges =
|
|
70808
|
-
const cameraClears =
|
|
71027
|
+
const nodeChanges = applyNodeAutoTune(await this.deps.readAgentSettings(), diffInput).changes;
|
|
71028
|
+
const cameraClears = applyCameraAutoTune(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70809
71029
|
const warnings = [];
|
|
70810
71030
|
let detectorStoreClears = [];
|
|
70811
71031
|
if (touchedNodes.size > 0) try {
|
|
@@ -70878,7 +71098,7 @@ var BestSettingsService = class {
|
|
|
70878
71098
|
async applyLocked(input) {
|
|
70879
71099
|
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
70880
71100
|
if (plan.planHash !== input.planHash) {
|
|
70881
|
-
this.deps.logger.warn("
|
|
71101
|
+
this.deps.logger.warn("auto tune: stale plan refused", { meta: {
|
|
70882
71102
|
confirmed: input.planHash,
|
|
70883
71103
|
current: plan.planHash
|
|
70884
71104
|
} });
|
|
@@ -70910,15 +71130,15 @@ var BestSettingsService = class {
|
|
|
70910
71130
|
ok: false,
|
|
70911
71131
|
detail: errMsg(err)
|
|
70912
71132
|
});
|
|
70913
|
-
this.deps.logger.warn("
|
|
71133
|
+
this.deps.logger.warn("auto tune: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70914
71134
|
return {
|
|
70915
71135
|
kind: "aborted",
|
|
70916
71136
|
phases
|
|
70917
71137
|
};
|
|
70918
71138
|
}
|
|
70919
71139
|
try {
|
|
70920
|
-
await this.deps.rewriteCameraSettings((prev) =>
|
|
70921
|
-
for (const c of plan.cameraClears) this.deps.logger.info("
|
|
71140
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraAutoTune(prev, diffInput).next);
|
|
71141
|
+
for (const c of plan.cameraClears) this.deps.logger.info("auto tune: camera override cleared", {
|
|
70922
71142
|
tags: {
|
|
70923
71143
|
deviceId: c.deviceId,
|
|
70924
71144
|
nodeId: c.nodeId
|
|
@@ -70962,8 +71182,8 @@ var BestSettingsService = class {
|
|
|
70962
71182
|
});
|
|
70963
71183
|
}
|
|
70964
71184
|
try {
|
|
70965
|
-
await this.deps.rewriteAgentSettings((prev) =>
|
|
70966
|
-
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("
|
|
71185
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeAutoTune(prev, diffInput).next);
|
|
71186
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("auto tune: node model set", {
|
|
70967
71187
|
tags: { nodeId: c.nodeId },
|
|
70968
71188
|
meta: {
|
|
70969
71189
|
nodeId: c.nodeId,
|
|
@@ -70997,14 +71217,14 @@ var BestSettingsService = class {
|
|
|
70997
71217
|
ok: failed.length === 0,
|
|
70998
71218
|
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
70999
71219
|
});
|
|
71000
|
-
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("
|
|
71220
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("auto tune: node skipped", {
|
|
71001
71221
|
tags: { nodeId: rec.nodeId },
|
|
71002
71222
|
meta: {
|
|
71003
71223
|
nodeId: rec.nodeId,
|
|
71004
71224
|
reasons: rec.reasons
|
|
71005
71225
|
}
|
|
71006
71226
|
});
|
|
71007
|
-
this.deps.logger.info("
|
|
71227
|
+
this.deps.logger.info("auto tune: applied", { meta: {
|
|
71008
71228
|
planHash: plan.planHash,
|
|
71009
71229
|
nodeChanges: plan.nodeChanges.length,
|
|
71010
71230
|
cameraClears: plan.cameraClears.length,
|
|
@@ -71019,7 +71239,7 @@ var BestSettingsService = class {
|
|
|
71019
71239
|
}
|
|
71020
71240
|
};
|
|
71021
71241
|
//#endregion
|
|
71022
|
-
//#region src/
|
|
71242
|
+
//#region src/auto-tune/device-peak-tracker.ts
|
|
71023
71243
|
var BUCKET_MS = 3600 * 1e3;
|
|
71024
71244
|
var DevicePeakTracker = class DevicePeakTracker {
|
|
71025
71245
|
now;
|
|
@@ -71261,6 +71481,176 @@ function applyLoadBalancingShares(prev, write) {
|
|
|
71261
71481
|
return next;
|
|
71262
71482
|
}
|
|
71263
71483
|
//#endregion
|
|
71484
|
+
//#region src/load-balancing/effective-shares-service.ts
|
|
71485
|
+
function weightsFrom(result) {
|
|
71486
|
+
const nodes = {};
|
|
71487
|
+
const devices = {};
|
|
71488
|
+
for (const node of result.nodes) {
|
|
71489
|
+
if (node.share !== null && node.share > 0) nodes[node.nodeId] = node.share;
|
|
71490
|
+
const perDevice = {};
|
|
71491
|
+
for (const d of node.devices) if (d.share > 0) perDevice[d.deviceKey] = d.share;
|
|
71492
|
+
if (Object.keys(perDevice).length > 0) devices[node.nodeId] = perDevice;
|
|
71493
|
+
}
|
|
71494
|
+
return {
|
|
71495
|
+
nodes,
|
|
71496
|
+
devices
|
|
71497
|
+
};
|
|
71498
|
+
}
|
|
71499
|
+
var EffectiveSharesService = class EffectiveSharesService {
|
|
71500
|
+
deps;
|
|
71501
|
+
/** An automatic answer older than this is recomputed by the reconcile. */
|
|
71502
|
+
static MAX_AGE_MS = 15 * 6e4;
|
|
71503
|
+
/** Reconcile cadence. */
|
|
71504
|
+
static RECONCILE_MS = 6e4;
|
|
71505
|
+
answer = null;
|
|
71506
|
+
inFlight = null;
|
|
71507
|
+
pendingTrigger = null;
|
|
71508
|
+
timer = null;
|
|
71509
|
+
constructor(deps) {
|
|
71510
|
+
this.deps = deps;
|
|
71511
|
+
}
|
|
71512
|
+
now() {
|
|
71513
|
+
return this.deps.now?.() ?? Date.now();
|
|
71514
|
+
}
|
|
71515
|
+
/** The last automatic answer; null = never computed. */
|
|
71516
|
+
current() {
|
|
71517
|
+
return this.answer;
|
|
71518
|
+
}
|
|
71519
|
+
/** Node weights the election reads. */
|
|
71520
|
+
async nodeWeights() {
|
|
71521
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return { ...this.answer.weights.nodes };
|
|
71522
|
+
return { ...(await this.deps.readStored()).nodes };
|
|
71523
|
+
}
|
|
71524
|
+
/** The weights of a node's ELIGIBLE devices (`eligible` carries their stored weights). */
|
|
71525
|
+
deviceWeights(nodeId, eligible) {
|
|
71526
|
+
if (this.deps.readSource() !== "auto" || this.answer === null) return { ...eligible };
|
|
71527
|
+
const auto = this.answer.weights.devices[nodeId] ?? {};
|
|
71528
|
+
const out = {};
|
|
71529
|
+
for (const [key, stored] of Object.entries(eligible)) {
|
|
71530
|
+
const share = auto[key];
|
|
71531
|
+
out[key] = share !== void 0 && share > 0 ? share : stored;
|
|
71532
|
+
}
|
|
71533
|
+
return out;
|
|
71534
|
+
}
|
|
71535
|
+
/** Both maps, as the Auto tune recommender charges load by them. */
|
|
71536
|
+
async effectiveWeights() {
|
|
71537
|
+
if (this.deps.readSource() === "auto" && this.answer !== null) return this.answer.weights;
|
|
71538
|
+
return this.deps.readStored();
|
|
71539
|
+
}
|
|
71540
|
+
/**
|
|
71541
|
+
* Recompute now (auto only). Concurrent calls coalesce: one runs, and one
|
|
71542
|
+
* trailing run picks up whatever was reported meanwhile.
|
|
71543
|
+
*/
|
|
71544
|
+
async recompute(trigger) {
|
|
71545
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71546
|
+
if (this.inFlight !== null) {
|
|
71547
|
+
this.pendingTrigger = trigger;
|
|
71548
|
+
return this.inFlight;
|
|
71549
|
+
}
|
|
71550
|
+
this.inFlight = this.run(trigger).finally(() => {
|
|
71551
|
+
this.inFlight = null;
|
|
71552
|
+
});
|
|
71553
|
+
await this.inFlight;
|
|
71554
|
+
const trailing = this.pendingTrigger;
|
|
71555
|
+
if (trailing !== null) {
|
|
71556
|
+
this.pendingTrigger = null;
|
|
71557
|
+
await this.recompute(trailing);
|
|
71558
|
+
}
|
|
71559
|
+
}
|
|
71560
|
+
async run(trigger) {
|
|
71561
|
+
let signature;
|
|
71562
|
+
let result;
|
|
71563
|
+
try {
|
|
71564
|
+
signature = await this.deps.topologySignature();
|
|
71565
|
+
result = await this.deps.suggest();
|
|
71566
|
+
} catch (err) {
|
|
71567
|
+
this.deps.logger.warn("load balancing: automatic shares not recomputed — last answer kept", { meta: {
|
|
71568
|
+
trigger,
|
|
71569
|
+
error: errMsg$2(err),
|
|
71570
|
+
hasAnswer: this.answer !== null
|
|
71571
|
+
} });
|
|
71572
|
+
return;
|
|
71573
|
+
}
|
|
71574
|
+
const before = this.answer?.weights ?? null;
|
|
71575
|
+
const after = weightsFrom(result);
|
|
71576
|
+
this.answer = {
|
|
71577
|
+
weights: after,
|
|
71578
|
+
computedAt: this.now(),
|
|
71579
|
+
trigger,
|
|
71580
|
+
signature
|
|
71581
|
+
};
|
|
71582
|
+
const nodeIds = [...new Set([...Object.keys(before?.nodes ?? {}), ...result.nodes.map((n) => n.nodeId)])];
|
|
71583
|
+
for (const nodeId of nodeIds.toSorted()) this.deps.logger.info("load balancing: automatic shares recomputed", {
|
|
71584
|
+
tags: { nodeId },
|
|
71585
|
+
meta: {
|
|
71586
|
+
trigger,
|
|
71587
|
+
before: before?.nodes[nodeId] ?? null,
|
|
71588
|
+
after: after.nodes[nodeId] ?? null,
|
|
71589
|
+
devicesBefore: JSON.stringify(before?.devices[nodeId] ?? {}),
|
|
71590
|
+
devicesAfter: JSON.stringify(after.devices[nodeId] ?? {})
|
|
71591
|
+
}
|
|
71592
|
+
});
|
|
71593
|
+
}
|
|
71594
|
+
/** Recompute if the topology changed or the answer has grown old (auto only). */
|
|
71595
|
+
async reconcile() {
|
|
71596
|
+
if (this.deps.readSource() !== "auto") return;
|
|
71597
|
+
if (this.answer === null) return this.recompute("reconcile:first");
|
|
71598
|
+
if (this.now() - this.answer.computedAt > EffectiveSharesService.MAX_AGE_MS) return this.recompute("reconcile:age");
|
|
71599
|
+
let signature;
|
|
71600
|
+
try {
|
|
71601
|
+
signature = await this.deps.topologySignature();
|
|
71602
|
+
} catch (err) {
|
|
71603
|
+
this.deps.logger.debug("load balancing: topology signature unreadable — reconcile skipped", { meta: { error: errMsg$2(err) } });
|
|
71604
|
+
return;
|
|
71605
|
+
}
|
|
71606
|
+
if (signature !== this.answer.signature) return this.recompute("reconcile:topology");
|
|
71607
|
+
}
|
|
71608
|
+
start() {
|
|
71609
|
+
if (this.timer !== null) return;
|
|
71610
|
+
this.timer = setInterval(() => {
|
|
71611
|
+
this.reconcile();
|
|
71612
|
+
}, EffectiveSharesService.RECONCILE_MS);
|
|
71613
|
+
this.timer.unref?.();
|
|
71614
|
+
}
|
|
71615
|
+
stop() {
|
|
71616
|
+
if (this.timer !== null) clearInterval(this.timer);
|
|
71617
|
+
this.timer = null;
|
|
71618
|
+
}
|
|
71619
|
+
};
|
|
71620
|
+
//#endregion
|
|
71621
|
+
//#region src/load-balancing/share-inputs.ts
|
|
71622
|
+
/** The operator's sliders: node `detectWeight` and device `weight` (positive only). */
|
|
71623
|
+
function storedShareWeights(blob) {
|
|
71624
|
+
const nodes = {};
|
|
71625
|
+
const devices = {};
|
|
71626
|
+
for (const [nodeId, settings] of Object.entries(blob)) {
|
|
71627
|
+
if (typeof settings.detectWeight === "number" && settings.detectWeight > 0) nodes[nodeId] = settings.detectWeight;
|
|
71628
|
+
const perDevice = {};
|
|
71629
|
+
for (const [key, entry] of Object.entries(settings.inferenceDevices ?? {})) if (typeof entry.weight === "number" && entry.weight > 0) perDevice[key] = entry.weight;
|
|
71630
|
+
if (Object.keys(perDevice).length > 0) devices[nodeId] = perDevice;
|
|
71631
|
+
}
|
|
71632
|
+
return {
|
|
71633
|
+
nodes,
|
|
71634
|
+
devices
|
|
71635
|
+
};
|
|
71636
|
+
}
|
|
71637
|
+
/**
|
|
71638
|
+
* A string that changes whenever an input of the automatic shares does: the
|
|
71639
|
+
* nodes present, each node's detect capability, which devices are switched on
|
|
71640
|
+
* and which detector model each runs. Sliders and caps are left out — they do
|
|
71641
|
+
* not change capacity.
|
|
71642
|
+
*/
|
|
71643
|
+
function shareTopologySignature(nodeIds, blob) {
|
|
71644
|
+
return [...nodeIds].toSorted().map((nodeId) => {
|
|
71645
|
+
const settings = blob[nodeId];
|
|
71646
|
+
const devices = Object.entries(settings?.inferenceDevices ?? {}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => {
|
|
71647
|
+
const model = entry.steps?.["object-detection"]?.modelId ?? "";
|
|
71648
|
+
return `${key}=${entry.enabled ? "on" : "off"}:${model}`;
|
|
71649
|
+
});
|
|
71650
|
+
return `${nodeId}|detect=${String(settings?.detect ?? "default")}|${devices.join(",")}`;
|
|
71651
|
+
}).join(";");
|
|
71652
|
+
}
|
|
71653
|
+
//#endregion
|
|
71264
71654
|
//#region src/stale-device-pins.ts
|
|
71265
71655
|
function findStaleDevicePins(pins, nodes) {
|
|
71266
71656
|
const usable = /* @__PURE__ */ new Map();
|
|
@@ -71347,9 +71737,12 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71347
71737
|
devicePeaks = new DevicePeakTracker();
|
|
71348
71738
|
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71349
71739
|
placementStats = new SessionPlacementStats();
|
|
71350
|
-
/** The stored session placement policy (D748
|
|
71351
|
-
readSessionPlacement = () =>
|
|
71352
|
-
|
|
71740
|
+
/** The stored session placement policy (D748, default `share` since D750). */
|
|
71741
|
+
readSessionPlacement = () => DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
71742
|
+
/** Where the shares come from (D750); `auto` before init. */
|
|
71743
|
+
readShareSource = () => DEFAULT_SHARE_SOURCE;
|
|
71744
|
+
autoTuneService = null;
|
|
71745
|
+
effectiveSharesService = null;
|
|
71353
71746
|
/**
|
|
71354
71747
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
71355
71748
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -71598,6 +71991,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71598
71991
|
getEligibleInferenceDevices: (nodeId, read) => this.resolveEligibleInferenceDevices(nodeId, read),
|
|
71599
71992
|
getEligibleInferenceDevicesIfCached: (nodeId) => this.resolveEligibleInferenceDevicesIfCached(nodeId),
|
|
71600
71993
|
forgetInferenceDeviceProbe: (nodeId) => this.forgetInferenceDeviceProbe(nodeId),
|
|
71994
|
+
noteShareTopologyChange: (trigger) => this.noteShareChange(trigger),
|
|
71601
71995
|
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
71602
71996
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
71603
71997
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
@@ -71655,6 +72049,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71655
72049
|
this.placement = controllers.placement;
|
|
71656
72050
|
this.session = controllers.session;
|
|
71657
72051
|
this.readSessionPlacement = controllers.sessionPlacement;
|
|
72052
|
+
this.readShareSource = controllers.shareSource;
|
|
72053
|
+
this.settingsStore.setEffectiveNodeWeights(() => this.effectiveShares().nodeWeights());
|
|
72054
|
+
this.effectiveShares().start();
|
|
72055
|
+
this.noteShareChange("startup");
|
|
71658
72056
|
this.deviceConfig = controllers.deviceConfig;
|
|
71659
72057
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
71660
72058
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -71780,6 +72178,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
71780
72178
|
this.detectionWiring = null;
|
|
71781
72179
|
this.unsubAgentOffline?.();
|
|
71782
72180
|
this.unsubAgentOffline = null;
|
|
72181
|
+
this.effectiveSharesService?.stop();
|
|
71783
72182
|
this.unsubAgentOnline?.();
|
|
71784
72183
|
this.unsubAgentOnline = null;
|
|
71785
72184
|
this.nodeLifecycle?.dispose();
|
|
@@ -72182,6 +72581,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72182
72581
|
await this.topology.refreshNodeCapabilities();
|
|
72183
72582
|
this.topology.addKnownRunnerNode(input.agentNodeId);
|
|
72184
72583
|
this.reconcile.schedulePendingRetry();
|
|
72584
|
+
if (input.detect !== void 0) this.noteShareChange(`node-detect:${input.agentNodeId}`);
|
|
72185
72585
|
return { success: true };
|
|
72186
72586
|
}
|
|
72187
72587
|
/**
|
|
@@ -72271,6 +72671,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72271
72671
|
});
|
|
72272
72672
|
await this.resetRemovedModelPins("node-device-write");
|
|
72273
72673
|
this.reconcile.schedulePendingRetry();
|
|
72674
|
+
this.noteShareChange(`node-devices:${input.agentNodeId}`);
|
|
72274
72675
|
return { success: true };
|
|
72275
72676
|
}
|
|
72276
72677
|
/**
|
|
@@ -72351,7 +72752,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72351
72752
|
});
|
|
72352
72753
|
throw new Error(`inference device eligibility: node "${nodeId}" catalog not answered`);
|
|
72353
72754
|
}
|
|
72354
|
-
return this.electableDevices(nodeId, probed, catalog);
|
|
72755
|
+
return this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, probed, catalog));
|
|
72355
72756
|
}
|
|
72356
72757
|
/**
|
|
72357
72758
|
* Forget a node's last device probe (pre-ship review, M4). Called when the
|
|
@@ -72395,7 +72796,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72395
72796
|
this.refreshInferenceDeviceProbe(nodeId);
|
|
72396
72797
|
return {
|
|
72397
72798
|
kind: "eligible",
|
|
72398
|
-
eligible: await this.electableDevices(nodeId, known, catalog)
|
|
72799
|
+
eligible: this.effectiveShares().deviceWeights(nodeId, await this.electableDevices(nodeId, known, catalog))
|
|
72399
72800
|
};
|
|
72400
72801
|
}
|
|
72401
72802
|
/**
|
|
@@ -72725,10 +73126,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72725
73126
|
clearedCameraOverrides
|
|
72726
73127
|
};
|
|
72727
73128
|
}
|
|
72728
|
-
|
|
72729
|
-
if (this.
|
|
73129
|
+
autoTune() {
|
|
73130
|
+
if (this.autoTuneService !== null) return this.autoTuneService;
|
|
72730
73131
|
const store = this.settingsStore;
|
|
72731
|
-
this.
|
|
73132
|
+
this.autoTuneService = new AutoTuneService({
|
|
72732
73133
|
logger: this.ctx.logger,
|
|
72733
73134
|
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72734
73135
|
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
@@ -72742,11 +73143,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72742
73143
|
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72743
73144
|
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72744
73145
|
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72745
|
-
cameraLoads: () => this.
|
|
73146
|
+
cameraLoads: () => this.autoTuneCameraLoads(),
|
|
72746
73147
|
peaks: () => this.devicePeaks.peaks(),
|
|
72747
73148
|
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72748
73149
|
sessionPlacement: () => this.readSessionPlacement(),
|
|
72749
|
-
|
|
73150
|
+
readEffectiveWeights: () => this.effectiveShares().effectiveWeights(),
|
|
72750
73151
|
detectorStore: async (input) => {
|
|
72751
73152
|
const api = this.ctx.api;
|
|
72752
73153
|
if (!api) throw new Error("ctx.api unavailable");
|
|
@@ -72769,10 +73170,10 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72769
73170
|
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72770
73171
|
now: () => Date.now()
|
|
72771
73172
|
});
|
|
72772
|
-
return this.
|
|
73173
|
+
return this.autoTuneService;
|
|
72773
73174
|
}
|
|
72774
73175
|
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72775
|
-
async
|
|
73176
|
+
async autoTuneCameraLoads() {
|
|
72776
73177
|
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72777
73178
|
const loads = [];
|
|
72778
73179
|
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
@@ -72794,8 +73195,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72794
73195
|
}
|
|
72795
73196
|
return loads;
|
|
72796
73197
|
}
|
|
72797
|
-
async
|
|
72798
|
-
return this.
|
|
73198
|
+
async planAutoTune(input) {
|
|
73199
|
+
return this.autoTune().plan(input);
|
|
72799
73200
|
}
|
|
72800
73201
|
/**
|
|
72801
73202
|
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
@@ -72851,10 +73252,75 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
72851
73252
|
return findStaleDevicePins(pins, offered);
|
|
72852
73253
|
}
|
|
72853
73254
|
async suggestShares(input) {
|
|
72854
|
-
return this.
|
|
73255
|
+
return this.autoTune().suggestShares(input);
|
|
72855
73256
|
}
|
|
72856
|
-
async
|
|
72857
|
-
|
|
73257
|
+
async applyAutoTune(input) {
|
|
73258
|
+
const result = await this.autoTune().apply(input);
|
|
73259
|
+
if (result.kind === "applied") this.noteShareChange("auto-tune-applied");
|
|
73260
|
+
return result;
|
|
73261
|
+
}
|
|
73262
|
+
effectiveShares() {
|
|
73263
|
+
if (this.effectiveSharesService !== null) return this.effectiveSharesService;
|
|
73264
|
+
const store = this.settingsStore;
|
|
73265
|
+
this.effectiveSharesService = new EffectiveSharesService({
|
|
73266
|
+
readSource: () => this.readShareSource(),
|
|
73267
|
+
readStored: async () => storedShareWeights(await store.readAgentSettingsMap()),
|
|
73268
|
+
suggest: () => this.autoTune().suggestShares({ headroom: AUTO_SHARE_HEADROOM }),
|
|
73269
|
+
topologySignature: async () => shareTopologySignature([...new Set(["hub", ...this.topology.collectRunnerNodeIds()])], await store.readAgentSettingsMap()),
|
|
73270
|
+
logger: this.ctx.logger
|
|
73271
|
+
});
|
|
73272
|
+
return this.effectiveSharesService;
|
|
73273
|
+
}
|
|
73274
|
+
/** Something the automatic shares depend on changed: recompute now (best effort). */
|
|
73275
|
+
noteShareChange(trigger) {
|
|
73276
|
+
if (this.settingsStore === null) return;
|
|
73277
|
+
this.effectiveShares().recompute(trigger).catch((err) => {
|
|
73278
|
+
this.ctx.logger.warn("load balancing: share recompute failed", { meta: {
|
|
73279
|
+
trigger,
|
|
73280
|
+
error: errMsg$2(err)
|
|
73281
|
+
} });
|
|
73282
|
+
});
|
|
73283
|
+
}
|
|
73284
|
+
async getEffectiveShares() {
|
|
73285
|
+
const service = this.effectiveShares();
|
|
73286
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])].toSorted();
|
|
73287
|
+
const rows = await Promise.all(nodeIds.map(async (nodeId) => ({
|
|
73288
|
+
nodeId,
|
|
73289
|
+
read: await this.getNodeInferenceDevices({ nodeId }).catch(() => null)
|
|
73290
|
+
})));
|
|
73291
|
+
const detectable = new Set(this.topology.detectionEligibleNodes());
|
|
73292
|
+
const eligibleNodes = rows.filter((r) => r.read !== null && r.read.reachable && detectable.has(r.nodeId) && this.isNodeInferenceUsable(r.nodeId)).map((r) => r.nodeId);
|
|
73293
|
+
const eligibleDevices = {};
|
|
73294
|
+
for (const r of rows) {
|
|
73295
|
+
if (r.read === null) continue;
|
|
73296
|
+
eligibleDevices[r.nodeId] = r.read.devices.filter((d) => d.enabled && d.available && d.exclusion === null).map((d) => d.key);
|
|
73297
|
+
}
|
|
73298
|
+
const auto = service.current();
|
|
73299
|
+
return resolveEffectiveShares({
|
|
73300
|
+
source: this.readShareSource(),
|
|
73301
|
+
policy: this.readSessionPlacement(),
|
|
73302
|
+
nodeIds,
|
|
73303
|
+
stored: storedShareWeights(await this.settingsStore.readAgentSettingsMap()),
|
|
73304
|
+
auto: auto?.weights ?? null,
|
|
73305
|
+
eligibleNodes,
|
|
73306
|
+
eligibleDevices,
|
|
73307
|
+
computedAt: auto?.computedAt ?? null,
|
|
73308
|
+
trigger: auto?.trigger ?? null
|
|
73309
|
+
});
|
|
73310
|
+
}
|
|
73311
|
+
async setShareSource(input) {
|
|
73312
|
+
const previous = this.readShareSource();
|
|
73313
|
+
if (input.source === "manual" && previous === "auto") {
|
|
73314
|
+
const auto = this.effectiveShares().current();
|
|
73315
|
+
if (auto !== null) await this.setLoadBalancingShares(auto.weights);
|
|
73316
|
+
}
|
|
73317
|
+
await this.updateGlobalSettings({ [SHARE_SOURCE_SETTING_KEY]: input.source });
|
|
73318
|
+
this.ctx.logger.info("load balancing: share source changed", { meta: {
|
|
73319
|
+
from: previous,
|
|
73320
|
+
to: input.source
|
|
73321
|
+
} });
|
|
73322
|
+
if (input.source === "auto") this.noteShareChange("source:auto");
|
|
73323
|
+
return { success: true };
|
|
72858
73324
|
}
|
|
72859
73325
|
async getCameraSettings(input) {
|
|
72860
73326
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|