@camstack/addon-pipeline-orchestrator 1.2.289 → 1.2.291
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_stub.js +2 -2
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-B17_Um7q.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CJCGMfMC.mjs} +2 -2
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-bgphbyCd.mjs +26 -0
- package/dist/{_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-CvQcqdZe.mjs → _virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-pEikYuiA.mjs} +1 -1
- package/dist/{hostInit-Bd-NWr7j.mjs → hostInit-BpNmAa0Q.mjs} +2 -2
- package/dist/index.js +2791 -177
- package/dist/index.mjs +2792 -178
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/_virtual_mf___mfe_internal__addon_pipeline_orchestrator_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-3MI4PP9B.mjs +0 -26
package/dist/index.js
CHANGED
|
@@ -8076,7 +8076,7 @@ import { errMsg } from '@camstack/types'
|
|
|
8076
8076
|
* Extract a human-readable message from an unknown error value.
|
|
8077
8077
|
* Replaces the ubiquitous `errMsg(err)` pattern.
|
|
8078
8078
|
*/
|
|
8079
|
-
function errMsg$
|
|
8079
|
+
function errMsg$2(err) {
|
|
8080
8080
|
if (err instanceof Error) return err.message;
|
|
8081
8081
|
if (typeof err === "string") return err;
|
|
8082
8082
|
return String(err);
|
|
@@ -11579,7 +11579,53 @@ var ModelFormatEntrySchema = object({
|
|
|
11579
11579
|
/** Multi-file payload (directory members or sibling files). */
|
|
11580
11580
|
files: array(string()).readonly().optional(),
|
|
11581
11581
|
/** Runtime(s) that can use this format. If omitted, inferred from ModelFormat key */
|
|
11582
|
-
runtimes: array(_enum(["python"])).readonly().optional()
|
|
11582
|
+
runtimes: array(_enum(["python"])).readonly().optional(),
|
|
11583
|
+
/**
|
|
11584
|
+
* Content revision of the artefact, for one hosted OFF the CamStack mirror.
|
|
11585
|
+
* A format on the mirror leaves it unset: its revision (the sha256 of its
|
|
11586
|
+
* weight file) comes from the downloader's generated table, kept in step
|
|
11587
|
+
* with the mirror by `scripts/check-hosted-model-revisions.ts` (D744). A
|
|
11588
|
+
* node records the revision it downloaded next to the file
|
|
11589
|
+
* (`<file>.revision`); a copy with no record or a different one is stale
|
|
11590
|
+
* unless its weight file hashes to the revision, and a stale copy is
|
|
11591
|
+
* downloaded again, the old one serving until the new one is complete. No
|
|
11592
|
+
* revision at all ⇒ a file on disk is current, whatever it holds (D743).
|
|
11593
|
+
*/
|
|
11594
|
+
revision: string().min(1).optional()
|
|
11595
|
+
});
|
|
11596
|
+
/**
|
|
11597
|
+
* What a node holds for one (model, format) (D746). `current` is a complete
|
|
11598
|
+
* copy at its catalog revision — exactly what `downloaded: true` means.
|
|
11599
|
+
* `stale` is bytes on disk that are not (yet) proven current under D743/D744:
|
|
11600
|
+
* no or a different revision record, or a missing sibling. `missing` is no
|
|
11601
|
+
* copy. `downloaded` alone folds `stale` into "not downloaded", which hid
|
|
11602
|
+
* reclaimable bytes and a copy about to be replaced.
|
|
11603
|
+
*/
|
|
11604
|
+
var ModelOnDiskStateSchema = _enum([
|
|
11605
|
+
"current",
|
|
11606
|
+
"stale",
|
|
11607
|
+
"missing"
|
|
11608
|
+
]);
|
|
11609
|
+
/**
|
|
11610
|
+
* The numeric precision of one format's BUILD (D746) — reported only when the
|
|
11611
|
+
* build states it (file-name token, an INT8 variant, an EdgeTPU compile, a
|
|
11612
|
+
* CamStack Core ML export). Absent means "not stated", never "fp32".
|
|
11613
|
+
*/
|
|
11614
|
+
var ModelFormatPrecisionSchema = _enum([
|
|
11615
|
+
"fp32",
|
|
11616
|
+
"fp16",
|
|
11617
|
+
"int8"
|
|
11618
|
+
]);
|
|
11619
|
+
/**
|
|
11620
|
+
* One format of one model as a node's `pipelineExecutor.getSchema` reports it.
|
|
11621
|
+
* `onDisk` / `precision` are optional for version skew: a runner that predates
|
|
11622
|
+
* them omits them, and a reader falls back to `downloaded`.
|
|
11623
|
+
*/
|
|
11624
|
+
var PipelineModelFormatStatusSchema = object({
|
|
11625
|
+
downloaded: boolean(),
|
|
11626
|
+
sizeMB: number(),
|
|
11627
|
+
onDisk: ModelOnDiskStateSchema.optional(),
|
|
11628
|
+
precision: ModelFormatPrecisionSchema.optional()
|
|
11583
11629
|
});
|
|
11584
11630
|
/**
|
|
11585
11631
|
* Extra file that must be downloaded alongside the model (e.g., labels JSON, dict.txt).
|
|
@@ -24325,7 +24371,7 @@ var FrameRefSchema = object({
|
|
|
24325
24371
|
timestamp: number(),
|
|
24326
24372
|
capturedAt: number().optional()
|
|
24327
24373
|
});
|
|
24328
|
-
var ModelFormatSchema$
|
|
24374
|
+
var ModelFormatSchema$2 = _enum([
|
|
24329
24375
|
"onnx",
|
|
24330
24376
|
"coreml",
|
|
24331
24377
|
"openvino",
|
|
@@ -24343,7 +24389,7 @@ var PipelineSlotSchema = _enum([
|
|
|
24343
24389
|
var PipelineEngineChoiceSchema = object({
|
|
24344
24390
|
runtime: literal("python"),
|
|
24345
24391
|
backend: string(),
|
|
24346
|
-
format: ModelFormatSchema$
|
|
24392
|
+
format: ModelFormatSchema$2,
|
|
24347
24393
|
device: string().optional()
|
|
24348
24394
|
});
|
|
24349
24395
|
/**
|
|
@@ -24400,10 +24446,7 @@ var PipelineTemplateSchema$1 = object({
|
|
|
24400
24446
|
var PipelineModelOptionSchema = object({
|
|
24401
24447
|
id: string(),
|
|
24402
24448
|
name: string(),
|
|
24403
|
-
formats: record(string(),
|
|
24404
|
-
downloaded: boolean(),
|
|
24405
|
-
sizeMB: number()
|
|
24406
|
-
})),
|
|
24449
|
+
formats: record(string(), PipelineModelFormatStatusSchema),
|
|
24407
24450
|
group: ModelVariantGroupSchema.optional(),
|
|
24408
24451
|
legacy: boolean().optional(),
|
|
24409
24452
|
provider: ModelProviderIdSchema.optional(),
|
|
@@ -24618,6 +24661,32 @@ var pipelineExecutorCapability = {
|
|
|
24618
24661
|
auth: "admin"
|
|
24619
24662
|
}),
|
|
24620
24663
|
/**
|
|
24664
|
+
* Best settings (D745): the MODEL-BOUND thresholds (`bindsToModel` keys of
|
|
24665
|
+
* the object-detection root, and their binding keys) stored in THIS node's
|
|
24666
|
+
* per-camera detector settings — and, unless `dryRun`, remove them so the
|
|
24667
|
+
* running model's own defaults apply. Every other key of a camera's store
|
|
24668
|
+
* is kept. `nodeId` is the ROUTING key. `deviceIds` absent ⇒ every camera
|
|
24669
|
+
* with a store. A camera whose store could not be read is `refused`, never
|
|
24670
|
+
* reported as clean.
|
|
24671
|
+
*/
|
|
24672
|
+
clearModelBoundDeviceOverrides: method(object({
|
|
24673
|
+
nodeId: string(),
|
|
24674
|
+
deviceIds: array(number()).readonly().optional(),
|
|
24675
|
+
dryRun: boolean()
|
|
24676
|
+
}), object({
|
|
24677
|
+
entries: array(object({
|
|
24678
|
+
deviceId: number(),
|
|
24679
|
+
keys: array(string()).readonly()
|
|
24680
|
+
})).readonly(),
|
|
24681
|
+
refused: array(object({
|
|
24682
|
+
deviceId: number(),
|
|
24683
|
+
reason: string()
|
|
24684
|
+
})).readonly()
|
|
24685
|
+
}), {
|
|
24686
|
+
kind: "mutation",
|
|
24687
|
+
auth: "admin"
|
|
24688
|
+
}),
|
|
24689
|
+
/**
|
|
24621
24690
|
* Which of THIS node's inference devices the executor currently refuses,
|
|
24622
24691
|
* and why. `nodeId` is the ROUTING key (stripped by the generated router).
|
|
24623
24692
|
*
|
|
@@ -24712,12 +24781,12 @@ var pipelineExecutorCapability = {
|
|
|
24712
24781
|
downloadModel: method(object({
|
|
24713
24782
|
addonId: string(),
|
|
24714
24783
|
modelId: string(),
|
|
24715
|
-
format: ModelFormatSchema$
|
|
24784
|
+
format: ModelFormatSchema$2
|
|
24716
24785
|
}), DownloadModelResultSchema, { kind: "mutation" }),
|
|
24717
24786
|
deleteModel: method(object({
|
|
24718
24787
|
addonId: string(),
|
|
24719
24788
|
modelId: string(),
|
|
24720
|
-
format: ModelFormatSchema$
|
|
24789
|
+
format: ModelFormatSchema$2
|
|
24721
24790
|
}), object({ success: literal(true) }), { kind: "mutation" }),
|
|
24722
24791
|
/**
|
|
24723
24792
|
* Stateless single-frame execution. Callers (runner, benchmark) pass
|
|
@@ -24963,6 +25032,315 @@ var pipelineExecutorCapability = {
|
|
|
24963
25032
|
getDetectionConfigSchema: method(_void(), ConfigUISchemaNullableBridge)
|
|
24964
25033
|
}
|
|
24965
25034
|
};
|
|
25035
|
+
/**
|
|
25036
|
+
* Wire shapes of the Load balancing surface (D748): the session placement
|
|
25037
|
+
* policy, the atomic share write, the observed placement statistics, the
|
|
25038
|
+
* hardware share suggestion and the stale device pins.
|
|
25039
|
+
*/
|
|
25040
|
+
/**
|
|
25041
|
+
* How a new detection session picks its node and device.
|
|
25042
|
+
* - `least-loaded` — the historical rule: lowest `attached / weight`, a
|
|
25043
|
+
* constant node order at a tie (D37). An idle node always ties at 0, so a
|
|
25044
|
+
* weight only bites once a node already holds a session.
|
|
25045
|
+
* - `share` — the share of PARALLEL sessions is held to the weights, and a
|
|
25046
|
+
* full or drowning node yields to a free one (D748).
|
|
25047
|
+
*/
|
|
25048
|
+
var SessionPlacementPolicySchema = _enum(["least-loaded", "share"]);
|
|
25049
|
+
var DEFAULT_SESSION_PLACEMENT_POLICY = "least-loaded";
|
|
25050
|
+
/** The global-settings key the policy is stored under (orchestrator `load` section). */
|
|
25051
|
+
var SESSION_PLACEMENT_SETTING_KEY = "sessionPlacement";
|
|
25052
|
+
/** Parse a stored value; anything unknown is the default (nothing changes on deploy). */
|
|
25053
|
+
function parseSessionPlacementPolicy(value) {
|
|
25054
|
+
const parsed = SessionPlacementPolicySchema.safeParse(value);
|
|
25055
|
+
return parsed.success ? parsed.data : DEFAULT_SESSION_PLACEMENT_POLICY;
|
|
25056
|
+
}
|
|
25057
|
+
var LoadBalancingSharesInputSchema = object({
|
|
25058
|
+
/** nodeId → share (> 0). A node left out keeps its weight. */
|
|
25059
|
+
nodes: record(string(), number().positive()).default({}),
|
|
25060
|
+
/** nodeId → deviceKey → share (> 0). A device left out keeps its weight. */
|
|
25061
|
+
devices: record(string(), record(string(), number().positive())).default({})
|
|
25062
|
+
});
|
|
25063
|
+
var PlacementStatsInputSchema = object({
|
|
25064
|
+
/** Window the observed shares are averaged over (default 1 h, max 24 h). */
|
|
25065
|
+
windowMs: number().int().min(6e4).max(1440 * 60 * 1e3).default(3600 * 1e3) });
|
|
25066
|
+
var PlacementDeviceStatsSchema = object({
|
|
25067
|
+
deviceKey: string(),
|
|
25068
|
+
/** Sessions running on this device now. */
|
|
25069
|
+
sessionsNow: number(),
|
|
25070
|
+
/** Sessions started on this device inside the window. */
|
|
25071
|
+
sessionsStarted: number(),
|
|
25072
|
+
/** Time-weighted share of the node's parallel sessions over the window (0–1); null = no session ran. */
|
|
25073
|
+
observedShare: number().nullable()
|
|
25074
|
+
});
|
|
25075
|
+
var PlacementNodeStatsSchema = object({
|
|
25076
|
+
nodeId: string(),
|
|
25077
|
+
sessionsNow: number(),
|
|
25078
|
+
sessionsStarted: number(),
|
|
25079
|
+
/** Time-weighted share of the cluster's parallel sessions over the window (0–1); null = none ran. */
|
|
25080
|
+
observedShare: number().nullable(),
|
|
25081
|
+
/** Most sessions this node ran at once inside the window. */
|
|
25082
|
+
peakSessions: number(),
|
|
25083
|
+
/** Sessions a share election handed elsewhere because this node was full or drowning. */
|
|
25084
|
+
yieldsAway: number(),
|
|
25085
|
+
devices: array(PlacementDeviceStatsSchema).readonly()
|
|
25086
|
+
});
|
|
25087
|
+
/** A camera's device pin that the session path cannot honour. */
|
|
25088
|
+
var StaleDevicePinSchema = object({
|
|
25089
|
+
deviceId: number(),
|
|
25090
|
+
deviceKey: string(),
|
|
25091
|
+
/** `disabled` — the key exists on some node but is not enabled anywhere; `absent` — no node has it. */
|
|
25092
|
+
reason: _enum(["disabled", "absent"])
|
|
25093
|
+
});
|
|
25094
|
+
var PlacementStatsSchema = object({
|
|
25095
|
+
policy: SessionPlacementPolicySchema,
|
|
25096
|
+
windowMs: number(),
|
|
25097
|
+
/** How much of the window was actually observed (the hub keeps it in memory). */
|
|
25098
|
+
observedForMs: number(),
|
|
25099
|
+
sessionsNow: number(),
|
|
25100
|
+
/** Most parallel sessions across the cluster inside the window. */
|
|
25101
|
+
clusterPeakSessions: number(),
|
|
25102
|
+
nodes: array(PlacementNodeStatsSchema).readonly(),
|
|
25103
|
+
stalePins: array(StaleDevicePinSchema).readonly()
|
|
25104
|
+
});
|
|
25105
|
+
var SuggestSharesInputSchema = object({ headroom: number().min(.3).max(.5).default(.4) });
|
|
25106
|
+
var SuggestedDeviceShareSchema = object({
|
|
25107
|
+
deviceKey: string(),
|
|
25108
|
+
/** Share of the node (0–100, rounded). */
|
|
25109
|
+
share: number(),
|
|
25110
|
+
modelId: string().nullable(),
|
|
25111
|
+
/** Inferences per second at that model, from the measured table. */
|
|
25112
|
+
fps: number().nullable(),
|
|
25113
|
+
measured: boolean(),
|
|
25114
|
+
/** Advisory session cap: ⌊fps × (1 − headroom) / camera fps⌋. Never enforced. */
|
|
25115
|
+
suggestedMaxSessions: number().nullable()
|
|
25116
|
+
});
|
|
25117
|
+
var SuggestedNodeShareSchema = object({
|
|
25118
|
+
nodeId: string(),
|
|
25119
|
+
/** Share of the cluster (0–100, rounded); null = node skipped (offline, unknown hardware). */
|
|
25120
|
+
share: number().nullable(),
|
|
25121
|
+
/** Parallel sessions the node sustains at the suggested models with the headroom kept free. */
|
|
25122
|
+
capacitySessions: number(),
|
|
25123
|
+
devices: array(SuggestedDeviceShareSchema).readonly(),
|
|
25124
|
+
skippedReason: string().nullable()
|
|
25125
|
+
});
|
|
25126
|
+
var SuggestSharesResultSchema = object({
|
|
25127
|
+
headroom: number(),
|
|
25128
|
+
/** Camera detection rate the capacity is counted in (the fastest camera's fps). */
|
|
25129
|
+
cameraFps: number(),
|
|
25130
|
+
/** How many share → model → capacity rounds ran before the shares stopped moving (≤ 3). */
|
|
25131
|
+
iterations: number(),
|
|
25132
|
+
nodes: array(SuggestedNodeShareSchema).readonly()
|
|
25133
|
+
});
|
|
25134
|
+
/**
|
|
25135
|
+
* The wire shapes of the Best settings plan and apply (D745), used by the
|
|
25136
|
+
* `pipeline-orchestrator` cap. The recommender's own types
|
|
25137
|
+
* (`recommend.ts`, `demand.ts`) are the TS face of the same data; these
|
|
25138
|
+
* schemas are what crosses the cap boundary.
|
|
25139
|
+
*/
|
|
25140
|
+
var ModelFormatSchema$1 = _enum(MODEL_FORMATS);
|
|
25141
|
+
var HardwareClassIdSchema = _enum([
|
|
25142
|
+
"intel-core-ultra",
|
|
25143
|
+
"intel-n-series",
|
|
25144
|
+
"apple-m-base",
|
|
25145
|
+
"apple-m-pro",
|
|
25146
|
+
"unknown"
|
|
25147
|
+
]);
|
|
25148
|
+
var AcceleratorKindSchema = _enum([
|
|
25149
|
+
"intel-npu",
|
|
25150
|
+
"intel-igpu",
|
|
25151
|
+
"x86-cpu",
|
|
25152
|
+
"arm-cpu",
|
|
25153
|
+
"apple-ane",
|
|
25154
|
+
"coral"
|
|
25155
|
+
]);
|
|
25156
|
+
var RecommendationReasonSchema = discriminatedUnion("kind", [
|
|
25157
|
+
object({
|
|
25158
|
+
kind: literal("fits"),
|
|
25159
|
+
utilisation: number()
|
|
25160
|
+
}),
|
|
25161
|
+
object({
|
|
25162
|
+
kind: literal("tie-faster"),
|
|
25163
|
+
over: string(),
|
|
25164
|
+
recallDelta: number()
|
|
25165
|
+
}),
|
|
25166
|
+
object({
|
|
25167
|
+
kind: literal("stepped-down"),
|
|
25168
|
+
from: string(),
|
|
25169
|
+
to: string(),
|
|
25170
|
+
limitedBy: string()
|
|
25171
|
+
}),
|
|
25172
|
+
object({
|
|
25173
|
+
kind: literal("no-fit-lightest"),
|
|
25174
|
+
utilisation: number()
|
|
25175
|
+
}),
|
|
25176
|
+
object({
|
|
25177
|
+
kind: literal("derived-latency"),
|
|
25178
|
+
modelId: string()
|
|
25179
|
+
}),
|
|
25180
|
+
object({
|
|
25181
|
+
kind: literal("estimated-load"),
|
|
25182
|
+
observedForMs: number().nullable()
|
|
25183
|
+
}),
|
|
25184
|
+
object({ kind: literal("cpu-device") }),
|
|
25185
|
+
object({ kind: literal("coral-untouched") }),
|
|
25186
|
+
object({ kind: literal("device-disabled") }),
|
|
25187
|
+
object({ kind: literal("device-unavailable") }),
|
|
25188
|
+
object({ kind: literal("node-offline") }),
|
|
25189
|
+
object({ kind: literal("no-hardware-facts") }),
|
|
25190
|
+
object({
|
|
25191
|
+
kind: literal("no-measured-data"),
|
|
25192
|
+
hardwareClass: HardwareClassIdSchema
|
|
25193
|
+
}),
|
|
25194
|
+
object({
|
|
25195
|
+
kind: literal("no-family-build"),
|
|
25196
|
+
format: ModelFormatSchema$1
|
|
25197
|
+
})
|
|
25198
|
+
]);
|
|
25199
|
+
var DeviceDemandSchema = object({
|
|
25200
|
+
nodeId: string(),
|
|
25201
|
+
deviceKey: string(),
|
|
25202
|
+
demandFps: number(),
|
|
25203
|
+
maxCameraFps: number(),
|
|
25204
|
+
alwaysOnCameras: number(),
|
|
25205
|
+
onMotionSessions: number(),
|
|
25206
|
+
onMotionSource: _enum([
|
|
25207
|
+
"observed-24h",
|
|
25208
|
+
"estimate",
|
|
25209
|
+
"none"
|
|
25210
|
+
]),
|
|
25211
|
+
observedForMs: number().nullable(),
|
|
25212
|
+
/** How on-motion sessions were charged: per-device observation, or split by share (D748). */
|
|
25213
|
+
placement: SessionPlacementPolicySchema
|
|
25214
|
+
});
|
|
25215
|
+
var DeviceModelChoiceSchema = object({
|
|
25216
|
+
modelId: string(),
|
|
25217
|
+
p50Ms: number(),
|
|
25218
|
+
fps: number(),
|
|
25219
|
+
measured: boolean(),
|
|
25220
|
+
utilisation: number(),
|
|
25221
|
+
personRecall: number()
|
|
25222
|
+
});
|
|
25223
|
+
var DeviceRecommendationSchema = object({
|
|
25224
|
+
deviceKey: string(),
|
|
25225
|
+
accelerator: AcceleratorKindSchema.nullable(),
|
|
25226
|
+
enabled: boolean(),
|
|
25227
|
+
available: boolean(),
|
|
25228
|
+
demand: DeviceDemandSchema.nullable(),
|
|
25229
|
+
chosen: DeviceModelChoiceSchema.nullable(),
|
|
25230
|
+
reasons: array(RecommendationReasonSchema).readonly()
|
|
25231
|
+
});
|
|
25232
|
+
var NodeRecommendationSchema = object({
|
|
25233
|
+
nodeId: string(),
|
|
25234
|
+
reachable: boolean(),
|
|
25235
|
+
hardwareClass: HardwareClassIdSchema.nullable(),
|
|
25236
|
+
cpuModel: string().nullable(),
|
|
25237
|
+
skipped: boolean(),
|
|
25238
|
+
acceleratorModelId: string().nullable(),
|
|
25239
|
+
devices: array(DeviceRecommendationSchema).readonly(),
|
|
25240
|
+
reasons: array(RecommendationReasonSchema).readonly()
|
|
25241
|
+
});
|
|
25242
|
+
var ModelPinChangeSchema = object({
|
|
25243
|
+
stepId: string(),
|
|
25244
|
+
before: string().nullable(),
|
|
25245
|
+
after: string().nullable()
|
|
25246
|
+
});
|
|
25247
|
+
var NodeDeviceChangeSchema = object({
|
|
25248
|
+
nodeId: string(),
|
|
25249
|
+
deviceKey: string(),
|
|
25250
|
+
madeExplicit: boolean(),
|
|
25251
|
+
modelPins: array(ModelPinChangeSchema).readonly(),
|
|
25252
|
+
thresholds: array(object({
|
|
25253
|
+
stepId: string(),
|
|
25254
|
+
keys: array(string()).readonly()
|
|
25255
|
+
})).readonly()
|
|
25256
|
+
});
|
|
25257
|
+
var CameraOverrideClearSchema = object({
|
|
25258
|
+
deviceId: number(),
|
|
25259
|
+
source: _enum(["step-override", "wholesale"]),
|
|
25260
|
+
nodeId: string(),
|
|
25261
|
+
deviceKey: string().nullable(),
|
|
25262
|
+
stepId: string(),
|
|
25263
|
+
modelIdBefore: string().nullable(),
|
|
25264
|
+
keys: array(string()).readonly()
|
|
25265
|
+
});
|
|
25266
|
+
var DetectorStoreClearSchema = object({
|
|
25267
|
+
deviceId: number(),
|
|
25268
|
+
keys: array(string()).readonly()
|
|
25269
|
+
});
|
|
25270
|
+
var BestSettingsWarningSchema = discriminatedUnion("kind", [
|
|
25271
|
+
object({
|
|
25272
|
+
kind: literal("detector-store-cluster-wide"),
|
|
25273
|
+
skippedNodes: array(string()).readonly()
|
|
25274
|
+
}),
|
|
25275
|
+
object({
|
|
25276
|
+
kind: literal("detector-store-unreadable"),
|
|
25277
|
+
error: string()
|
|
25278
|
+
}),
|
|
25279
|
+
object({
|
|
25280
|
+
kind: literal("coral-enabled"),
|
|
25281
|
+
nodeId: string(),
|
|
25282
|
+
deviceKey: string()
|
|
25283
|
+
})
|
|
25284
|
+
]);
|
|
25285
|
+
var BestSettingsOptionsSchema = object({
|
|
25286
|
+
/** Share of each accelerator's throughput kept free (0.3–0.5). */
|
|
25287
|
+
headroom: number().min(.3).max(.5).default(.4),
|
|
25288
|
+
/**
|
|
25289
|
+
* What-if shares (D748): plan as if these node/device shares were stored
|
|
25290
|
+
* and the policy were `share`. Absent = the stored shares under the stored
|
|
25291
|
+
* policy. A what-if plan is for display; apply always re-plans from what
|
|
25292
|
+
* is stored.
|
|
25293
|
+
*/
|
|
25294
|
+
shares: LoadBalancingSharesInputSchema.optional()
|
|
25295
|
+
});
|
|
25296
|
+
var BestSettingsPlanSchema = object({
|
|
25297
|
+
/** Hash of everything the apply will write — the apply refuses a different one. */
|
|
25298
|
+
planHash: string(),
|
|
25299
|
+
family: literal("yolov9"),
|
|
25300
|
+
headroom: number(),
|
|
25301
|
+
/** Planned under what-if shares (D748): for display — apply refuses it as stale. */
|
|
25302
|
+
whatIf: boolean(),
|
|
25303
|
+
generatedAt: number(),
|
|
25304
|
+
nodes: array(NodeRecommendationSchema).readonly(),
|
|
25305
|
+
nodeChanges: array(NodeDeviceChangeSchema).readonly(),
|
|
25306
|
+
cameraClears: array(CameraOverrideClearSchema).readonly(),
|
|
25307
|
+
detectorStoreClears: array(DetectorStoreClearSchema).readonly(),
|
|
25308
|
+
warnings: array(BestSettingsWarningSchema).readonly()
|
|
25309
|
+
});
|
|
25310
|
+
var BestSettingsApplyInputSchema = object({
|
|
25311
|
+
planHash: string(),
|
|
25312
|
+
headroom: number().min(.3).max(.5).default(.4)
|
|
25313
|
+
});
|
|
25314
|
+
var BestSettingsPhaseResultSchema = object({
|
|
25315
|
+
phase: _enum([
|
|
25316
|
+
"download",
|
|
25317
|
+
"camera-overrides",
|
|
25318
|
+
"detector-thresholds",
|
|
25319
|
+
"node-models",
|
|
25320
|
+
"redispatch"
|
|
25321
|
+
]),
|
|
25322
|
+
ok: boolean(),
|
|
25323
|
+
detail: string()
|
|
25324
|
+
});
|
|
25325
|
+
var BestSettingsApplyResultSchema = discriminatedUnion("kind", [
|
|
25326
|
+
object({
|
|
25327
|
+
kind: literal("applied"),
|
|
25328
|
+
planHash: string(),
|
|
25329
|
+
phases: array(BestSettingsPhaseResultSchema).readonly()
|
|
25330
|
+
}),
|
|
25331
|
+
object({
|
|
25332
|
+
kind: literal("aborted"),
|
|
25333
|
+
phases: array(BestSettingsPhaseResultSchema).readonly()
|
|
25334
|
+
}),
|
|
25335
|
+
object({
|
|
25336
|
+
kind: literal("stale"),
|
|
25337
|
+
plan: BestSettingsPlanSchema
|
|
25338
|
+
}),
|
|
25339
|
+
object({
|
|
25340
|
+
kind: literal("refused"),
|
|
25341
|
+
reason: _enum(["apply-in-progress"])
|
|
25342
|
+
})
|
|
25343
|
+
]);
|
|
24966
25344
|
object({
|
|
24967
25345
|
activeCameras: number(),
|
|
24968
25346
|
throttledCameras: number(),
|
|
@@ -26371,7 +26749,6 @@ var CameraStepOverridePatchSchema = object({
|
|
|
26371
26749
|
settings: record(string(), unknown()).readonly().optional()
|
|
26372
26750
|
});
|
|
26373
26751
|
var CameraPipelineSettingsSchema = object({
|
|
26374
|
-
pinnedAgentNodeId: string().optional(),
|
|
26375
26752
|
stepToggles: record(string(), boolean()).optional(),
|
|
26376
26753
|
stepOverridesByDevice: record(string(), record(string(), record(string(), CameraStepOverridePatchSchema))).optional(),
|
|
26377
26754
|
pipelineByAgent: record(string(), CameraPipelineForAgentSchema).optional()
|
|
@@ -27019,18 +27396,6 @@ var pipelineOrchestratorCapability = {
|
|
|
27019
27396
|
auth: "admin"
|
|
27020
27397
|
}),
|
|
27021
27398
|
/**
|
|
27022
|
-
* Set a node's detection WEIGHT (relative share for the quota balancer).
|
|
27023
|
-
* `null` clears it (back to the implicit weight 1). Unpinned cameras
|
|
27024
|
-
* distribute proportionally to weight; `maxCameras` still clamps on top.
|
|
27025
|
-
*/
|
|
27026
|
-
setAgentDetectWeight: method(object({
|
|
27027
|
-
agentNodeId: string(),
|
|
27028
|
-
detectWeight: number().positive().nullable()
|
|
27029
|
-
}), object({ success: literal(true) }), {
|
|
27030
|
-
kind: "mutation",
|
|
27031
|
-
auth: "admin"
|
|
27032
|
-
}),
|
|
27033
|
-
/**
|
|
27034
27399
|
* Set one node's placement CAPABILITIES — the per-node record that
|
|
27035
27400
|
* replaced the four flat orchestrator lists (`enabledNodes`,
|
|
27036
27401
|
* `enabledDecoderNodes`, `enabledAudioNodes`, `remoteSourcingNodes`).
|
|
@@ -27149,6 +27514,54 @@ var pipelineOrchestratorCapability = {
|
|
|
27149
27514
|
kind: "mutation",
|
|
27150
27515
|
auth: "admin"
|
|
27151
27516
|
}),
|
|
27517
|
+
/**
|
|
27518
|
+
* Best settings (D745) — DRY RUN. Recommends ONE detector family (YOLOv9)
|
|
27519
|
+
* for the whole cluster from each node's static hardware class, the
|
|
27520
|
+
* measured latency table and the detection demand, and lists every write
|
|
27521
|
+
* the apply would make: the node model pins set, the enrichment pins and
|
|
27522
|
+
* model-bound thresholds removed (node level and per camera), and the
|
|
27523
|
+
* per-camera detector thresholds cleared. Writes nothing. Step
|
|
27524
|
+
* enablement, device enablement, pins, recording, notifications, zones and
|
|
27525
|
+
* rules are never part of it. `planHash` names exactly this diff.
|
|
27526
|
+
*/
|
|
27527
|
+
planBestSettings: method(BestSettingsOptionsSchema, BestSettingsPlanSchema, { auth: "admin" }),
|
|
27528
|
+
/**
|
|
27529
|
+
* Best settings (D745) — APPLY the plan the operator confirmed. Recomputes
|
|
27530
|
+
* the plan and refuses (`stale`, with the fresh plan) when its hash is not
|
|
27531
|
+
* `planHash`, so what is written is exactly the confirmed diff. Offline
|
|
27532
|
+
* nodes are skipped; there is no undo.
|
|
27533
|
+
*/
|
|
27534
|
+
applyBestSettings: method(BestSettingsApplyInputSchema, BestSettingsApplyResultSchema, {
|
|
27535
|
+
kind: "mutation",
|
|
27536
|
+
auth: "admin"
|
|
27537
|
+
}),
|
|
27538
|
+
/**
|
|
27539
|
+
* Load balancing (D748) — write node shares (`detectWeight`) and per-node
|
|
27540
|
+
* device shares (inference-device `weight`) in ONE atomic read-modify-write
|
|
27541
|
+
* of the agent settings — the only writer of node shares (the per-node
|
|
27542
|
+
* `setAgentDetectWeight` it replaced raced its siblings, one RMW each).
|
|
27543
|
+
* A node or device left out keeps its weight. Never changes the
|
|
27544
|
+
* `sessionPlacement` policy.
|
|
27545
|
+
*/
|
|
27546
|
+
setLoadBalancingShares: method(LoadBalancingSharesInputSchema, object({ success: literal(true) }), {
|
|
27547
|
+
kind: "mutation",
|
|
27548
|
+
auth: "admin"
|
|
27549
|
+
}),
|
|
27550
|
+
/**
|
|
27551
|
+
* Load balancing (D748) — what placement actually did: per node and
|
|
27552
|
+
* device, sessions now, sessions started and the time-weighted share of
|
|
27553
|
+
* PARALLEL sessions over `windowMs`; the cluster's parallel peak; yields
|
|
27554
|
+
* a share election made; and every device pin the session path cannot
|
|
27555
|
+
* honour. In memory on the hub — a restart starts a new window.
|
|
27556
|
+
*/
|
|
27557
|
+
getPlacementStats: method(PlacementStatsInputSchema, PlacementStatsSchema, { auth: "admin" }),
|
|
27558
|
+
/**
|
|
27559
|
+
* Load balancing (D748) — shares proportional to each node's measured
|
|
27560
|
+
* capacity at the models Best settings would pick, iterated (≤ 3 rounds)
|
|
27561
|
+
* until the shares stop moving; per device, an ADVISORY session cap.
|
|
27562
|
+
* Writes nothing.
|
|
27563
|
+
*/
|
|
27564
|
+
suggestShares: method(SuggestSharesInputSchema, SuggestSharesResultSchema, { auth: "admin" }),
|
|
27152
27565
|
/** Read one camera's settings. Null when never touched (inherits agent defaults fully). */
|
|
27153
27566
|
getCameraSettings: method(object({ deviceId: number() }), CameraPipelineSettingsSchema.nullable()),
|
|
27154
27567
|
/** Set or clear the 3-state toggle for one (camera, addonId). Pass `enabled: null` to clear and revert to agent default. */
|
|
@@ -42950,6 +43363,12 @@ Object.freeze({
|
|
|
42950
43363
|
addonId: null,
|
|
42951
43364
|
access: "delete"
|
|
42952
43365
|
},
|
|
43366
|
+
"pipelineExecutor.clearModelBoundDeviceOverrides": {
|
|
43367
|
+
capName: "pipeline-executor",
|
|
43368
|
+
capScope: "system",
|
|
43369
|
+
addonId: null,
|
|
43370
|
+
access: "delete"
|
|
43371
|
+
},
|
|
42953
43372
|
"pipelineExecutor.deleteModel": {
|
|
42954
43373
|
capName: "pipeline-executor",
|
|
42955
43374
|
capScope: "system",
|
|
@@ -43166,6 +43585,12 @@ Object.freeze({
|
|
|
43166
43585
|
addonId: null,
|
|
43167
43586
|
access: "view"
|
|
43168
43587
|
},
|
|
43588
|
+
"pipelineOrchestrator.applyBestSettings": {
|
|
43589
|
+
capName: "pipeline-orchestrator",
|
|
43590
|
+
capScope: "system",
|
|
43591
|
+
addonId: null,
|
|
43592
|
+
access: "create"
|
|
43593
|
+
},
|
|
43169
43594
|
"pipelineOrchestrator.applyDeviceSettingsPatch": {
|
|
43170
43595
|
capName: "pipeline-orchestrator",
|
|
43171
43596
|
capScope: "system",
|
|
@@ -43316,6 +43741,12 @@ Object.freeze({
|
|
|
43316
43741
|
addonId: null,
|
|
43317
43742
|
access: "view"
|
|
43318
43743
|
},
|
|
43744
|
+
"pipelineOrchestrator.getPlacementStats": {
|
|
43745
|
+
capName: "pipeline-orchestrator",
|
|
43746
|
+
capScope: "system",
|
|
43747
|
+
addonId: null,
|
|
43748
|
+
access: "view"
|
|
43749
|
+
},
|
|
43319
43750
|
"pipelineOrchestrator.getReconcileFromDiskStatus": {
|
|
43320
43751
|
capName: "pipeline-orchestrator",
|
|
43321
43752
|
capScope: "system",
|
|
@@ -43340,6 +43771,12 @@ Object.freeze({
|
|
|
43340
43771
|
addonId: null,
|
|
43341
43772
|
access: "create"
|
|
43342
43773
|
},
|
|
43774
|
+
"pipelineOrchestrator.planBestSettings": {
|
|
43775
|
+
capName: "pipeline-orchestrator",
|
|
43776
|
+
capScope: "system",
|
|
43777
|
+
addonId: null,
|
|
43778
|
+
access: "view"
|
|
43779
|
+
},
|
|
43343
43780
|
"pipelineOrchestrator.rebalance": {
|
|
43344
43781
|
capName: "pipeline-orchestrator",
|
|
43345
43782
|
capScope: "system",
|
|
@@ -43400,12 +43837,6 @@ Object.freeze({
|
|
|
43400
43837
|
addonId: null,
|
|
43401
43838
|
access: "create"
|
|
43402
43839
|
},
|
|
43403
|
-
"pipelineOrchestrator.setAgentDetectWeight": {
|
|
43404
|
-
capName: "pipeline-orchestrator",
|
|
43405
|
-
capScope: "system",
|
|
43406
|
-
addonId: null,
|
|
43407
|
-
access: "create"
|
|
43408
|
-
},
|
|
43409
43840
|
"pipelineOrchestrator.setAgentInferenceDevices": {
|
|
43410
43841
|
capName: "pipeline-orchestrator",
|
|
43411
43842
|
capScope: "system",
|
|
@@ -43460,12 +43891,24 @@ Object.freeze({
|
|
|
43460
43891
|
addonId: null,
|
|
43461
43892
|
access: "create"
|
|
43462
43893
|
},
|
|
43894
|
+
"pipelineOrchestrator.setLoadBalancingShares": {
|
|
43895
|
+
capName: "pipeline-orchestrator",
|
|
43896
|
+
capScope: "system",
|
|
43897
|
+
addonId: null,
|
|
43898
|
+
access: "create"
|
|
43899
|
+
},
|
|
43463
43900
|
"pipelineOrchestrator.setPipelineDevicePin": {
|
|
43464
43901
|
capName: "pipeline-orchestrator",
|
|
43465
43902
|
capScope: "system",
|
|
43466
43903
|
addonId: null,
|
|
43467
43904
|
access: "create"
|
|
43468
43905
|
},
|
|
43906
|
+
"pipelineOrchestrator.suggestShares": {
|
|
43907
|
+
capName: "pipeline-orchestrator",
|
|
43908
|
+
capScope: "system",
|
|
43909
|
+
addonId: null,
|
|
43910
|
+
access: "view"
|
|
43911
|
+
},
|
|
43469
43912
|
"pipelineOrchestrator.unassignAudio": {
|
|
43470
43913
|
capName: "pipeline-orchestrator",
|
|
43471
43914
|
capScope: "system",
|
|
@@ -48790,6 +49233,11 @@ Object.freeze({
|
|
|
48790
49233
|
form: "single",
|
|
48791
49234
|
optional: false
|
|
48792
49235
|
}],
|
|
49236
|
+
"pipelineExecutor.clearModelBoundDeviceOverrides": [{
|
|
49237
|
+
name: "deviceIds",
|
|
49238
|
+
form: "array",
|
|
49239
|
+
optional: true
|
|
49240
|
+
}],
|
|
48793
49241
|
"pipelineExecutor.runPipeline": [{
|
|
48794
49242
|
name: "deviceId",
|
|
48795
49243
|
form: "single",
|
|
@@ -50684,6 +51132,891 @@ var REMOVED_MODEL_IDS = {
|
|
|
50684
51132
|
function isRemovedModelId(stepId, modelId) {
|
|
50685
51133
|
return REMOVED_MODEL_IDS[stepId]?.includes(modelId) ?? false;
|
|
50686
51134
|
}
|
|
51135
|
+
/** Suffix of the sibling key that names the model a stored value was made for. */
|
|
51136
|
+
var MODEL_BINDING_SUFFIX = "ModelId";
|
|
51137
|
+
/** `minConfidencePerson` → `minConfidencePersonModelId`. */
|
|
51138
|
+
function modelBindingKey(key) {
|
|
51139
|
+
return `${key}${MODEL_BINDING_SUFFIX}`;
|
|
51140
|
+
}
|
|
51141
|
+
/**
|
|
51142
|
+
* The keys of every `bindsToModel` field of a step's config schema, groups
|
|
51143
|
+
* flattened (D738). One definition for the runner (which applies the binding)
|
|
51144
|
+
* and the orchestrator's Best settings (which clears those overrides, D745).
|
|
51145
|
+
*/
|
|
51146
|
+
function collectModelBoundKeys(fields) {
|
|
51147
|
+
return fields.flatMap((field) => {
|
|
51148
|
+
if (field.type === "group") return collectModelBoundKeys(field.fields);
|
|
51149
|
+
return "key" in field && "bindsToModel" in field && field.bindsToModel === true ? [field.key] : [];
|
|
51150
|
+
});
|
|
51151
|
+
}
|
|
51152
|
+
var APPLE_CHIP = /^Apple M(\d+)(?:\s+(Pro|Max|Ultra))?\b/i;
|
|
51153
|
+
var INTEL_CORE_ULTRA = /Core\(TM\)\s+Ultra|Core Ultra/i;
|
|
51154
|
+
var INTEL_N_SERIES = /Intel\(R\)\s+N\d{2,3}\b|\bIntel\s+N\d{2,3}\b/i;
|
|
51155
|
+
function parseAppleChip(cpuModel) {
|
|
51156
|
+
const match = APPLE_CHIP.exec(cpuModel.trim());
|
|
51157
|
+
if (match === null) return null;
|
|
51158
|
+
const generation = Number(match[1]);
|
|
51159
|
+
const variantRaw = (match[2] ?? "base").toLowerCase();
|
|
51160
|
+
return {
|
|
51161
|
+
generation,
|
|
51162
|
+
variant: variantRaw === "pro" || variantRaw === "max" || variantRaw === "ultra" ? variantRaw : "base"
|
|
51163
|
+
};
|
|
51164
|
+
}
|
|
51165
|
+
/**
|
|
51166
|
+
* Classify a node. Apple base chips (M1–M4) map to the measured M2 column, the
|
|
51167
|
+
* Pro/Max/Ultra chips to the measured M3 Pro column — the M2 was measured at
|
|
51168
|
+
* 1.0–1.2× the M3 Pro on the Neural Engine, so the generation inside a variant
|
|
51169
|
+
* matters less than the column's own error.
|
|
51170
|
+
*/
|
|
51171
|
+
function classifyInferenceHardware(hw) {
|
|
51172
|
+
const cpuModel = hw.cpuModel;
|
|
51173
|
+
const apple = parseAppleChip(cpuModel);
|
|
51174
|
+
if (apple !== null) return {
|
|
51175
|
+
hardwareClass: apple.variant === "base" ? "apple-m-base" : "apple-m-pro",
|
|
51176
|
+
cpuModel,
|
|
51177
|
+
apple
|
|
51178
|
+
};
|
|
51179
|
+
if (INTEL_CORE_ULTRA.test(cpuModel)) return {
|
|
51180
|
+
hardwareClass: "intel-core-ultra",
|
|
51181
|
+
cpuModel
|
|
51182
|
+
};
|
|
51183
|
+
if (INTEL_N_SERIES.test(cpuModel)) return {
|
|
51184
|
+
hardwareClass: "intel-n-series",
|
|
51185
|
+
cpuModel
|
|
51186
|
+
};
|
|
51187
|
+
return {
|
|
51188
|
+
hardwareClass: "unknown",
|
|
51189
|
+
cpuModel
|
|
51190
|
+
};
|
|
51191
|
+
}
|
|
51192
|
+
/**
|
|
51193
|
+
* The accelerator kind of one of a node's device keys (`openvino:npu`,
|
|
51194
|
+
* `openvino:gpu`, `coreml:all`, `edgetpu:usb`, `cpu`, `onnx:cpu`, …). `null`
|
|
51195
|
+
* for a key this table has no column for.
|
|
51196
|
+
*/
|
|
51197
|
+
function acceleratorKindOf(deviceKey, hw) {
|
|
51198
|
+
const [backend = "", device = ""] = deviceKey.split(":");
|
|
51199
|
+
if (backend === "edgetpu") return "coral";
|
|
51200
|
+
if (backend === "coreml") return device === "cpu" ? "arm-cpu" : "apple-ane";
|
|
51201
|
+
if (backend === "openvino" && device === "npu") return "intel-npu";
|
|
51202
|
+
if (backend === "openvino" && device === "gpu") return "intel-igpu";
|
|
51203
|
+
if (backend === "cpu" || device === "cpu") return hw.arch === "arm64" ? "arm-cpu" : "x86-cpu";
|
|
51204
|
+
return null;
|
|
51205
|
+
}
|
|
51206
|
+
var DETECTOR_PERF = {
|
|
51207
|
+
"intel-core-ultra": {
|
|
51208
|
+
"intel-npu": {
|
|
51209
|
+
"yolov9t-320": {
|
|
51210
|
+
p50Ms: 2.9,
|
|
51211
|
+
fps: 383,
|
|
51212
|
+
measured: true
|
|
51213
|
+
},
|
|
51214
|
+
"yolov9t-320-int8": {
|
|
51215
|
+
p50Ms: 3.8,
|
|
51216
|
+
fps: 333,
|
|
51217
|
+
measured: true
|
|
51218
|
+
},
|
|
51219
|
+
"yolov9t-640": {
|
|
51220
|
+
p50Ms: 7.4,
|
|
51221
|
+
fps: 139,
|
|
51222
|
+
measured: true
|
|
51223
|
+
},
|
|
51224
|
+
"yolov9t-640-int8": {
|
|
51225
|
+
p50Ms: 7.5,
|
|
51226
|
+
fps: 144,
|
|
51227
|
+
measured: true
|
|
51228
|
+
},
|
|
51229
|
+
"yolov9s-320": {
|
|
51230
|
+
p50Ms: 4.4,
|
|
51231
|
+
fps: 241,
|
|
51232
|
+
measured: true
|
|
51233
|
+
},
|
|
51234
|
+
"yolov9s-320-int8": {
|
|
51235
|
+
p50Ms: 4.1,
|
|
51236
|
+
fps: 264,
|
|
51237
|
+
measured: true
|
|
51238
|
+
},
|
|
51239
|
+
"yolov9s-640": {
|
|
51240
|
+
p50Ms: 12.7,
|
|
51241
|
+
fps: 82,
|
|
51242
|
+
measured: true
|
|
51243
|
+
},
|
|
51244
|
+
"yolov9s-640-int8": {
|
|
51245
|
+
p50Ms: 10.7,
|
|
51246
|
+
fps: 99,
|
|
51247
|
+
measured: true
|
|
51248
|
+
},
|
|
51249
|
+
"yolov9m-320": {
|
|
51250
|
+
p50Ms: 8.4,
|
|
51251
|
+
fps: 123,
|
|
51252
|
+
measured: true
|
|
51253
|
+
},
|
|
51254
|
+
"yolov9m-320-int8": {
|
|
51255
|
+
p50Ms: 6.5,
|
|
51256
|
+
fps: 165,
|
|
51257
|
+
measured: true
|
|
51258
|
+
},
|
|
51259
|
+
"yolov9m-640": {
|
|
51260
|
+
p50Ms: 29.5,
|
|
51261
|
+
fps: 32,
|
|
51262
|
+
measured: true
|
|
51263
|
+
},
|
|
51264
|
+
"yolov9m-640-int8": {
|
|
51265
|
+
p50Ms: 19.5,
|
|
51266
|
+
fps: 52,
|
|
51267
|
+
measured: true
|
|
51268
|
+
}
|
|
51269
|
+
},
|
|
51270
|
+
"intel-igpu": {
|
|
51271
|
+
"yolov9t-320": {
|
|
51272
|
+
p50Ms: 2.6,
|
|
51273
|
+
fps: 405,
|
|
51274
|
+
measured: true
|
|
51275
|
+
},
|
|
51276
|
+
"yolov9t-320-int8": {
|
|
51277
|
+
p50Ms: 2.8,
|
|
51278
|
+
fps: 324,
|
|
51279
|
+
measured: true
|
|
51280
|
+
},
|
|
51281
|
+
"yolov9t-640": {
|
|
51282
|
+
p50Ms: 6.5,
|
|
51283
|
+
fps: 157,
|
|
51284
|
+
measured: true
|
|
51285
|
+
},
|
|
51286
|
+
"yolov9t-640-int8": {
|
|
51287
|
+
p50Ms: 5.6,
|
|
51288
|
+
fps: 185,
|
|
51289
|
+
measured: true
|
|
51290
|
+
},
|
|
51291
|
+
"yolov9s-320": {
|
|
51292
|
+
p50Ms: 4.6,
|
|
51293
|
+
fps: 200,
|
|
51294
|
+
measured: true
|
|
51295
|
+
},
|
|
51296
|
+
"yolov9s-320-int8": {
|
|
51297
|
+
p50Ms: 4.1,
|
|
51298
|
+
fps: 239,
|
|
51299
|
+
measured: true
|
|
51300
|
+
},
|
|
51301
|
+
"yolov9s-640": {
|
|
51302
|
+
p50Ms: 15.2,
|
|
51303
|
+
fps: 69,
|
|
51304
|
+
measured: true
|
|
51305
|
+
},
|
|
51306
|
+
"yolov9s-640-int8": {
|
|
51307
|
+
p50Ms: 10.5,
|
|
51308
|
+
fps: 99,
|
|
51309
|
+
measured: true
|
|
51310
|
+
},
|
|
51311
|
+
"yolov9m-320": {
|
|
51312
|
+
p50Ms: 11.4,
|
|
51313
|
+
fps: 88,
|
|
51314
|
+
measured: true
|
|
51315
|
+
},
|
|
51316
|
+
"yolov9m-320-int8": {
|
|
51317
|
+
p50Ms: 7,
|
|
51318
|
+
fps: 127,
|
|
51319
|
+
measured: true
|
|
51320
|
+
},
|
|
51321
|
+
"yolov9m-640": {
|
|
51322
|
+
p50Ms: 40,
|
|
51323
|
+
fps: 25,
|
|
51324
|
+
measured: true
|
|
51325
|
+
},
|
|
51326
|
+
"yolov9m-640-int8": {
|
|
51327
|
+
p50Ms: 23.5,
|
|
51328
|
+
fps: 44,
|
|
51329
|
+
measured: true
|
|
51330
|
+
}
|
|
51331
|
+
},
|
|
51332
|
+
"x86-cpu": {
|
|
51333
|
+
"yolov9t-320": {
|
|
51334
|
+
p50Ms: 6.9,
|
|
51335
|
+
fps: 145,
|
|
51336
|
+
measured: true
|
|
51337
|
+
},
|
|
51338
|
+
"yolov9t-320-int8": {
|
|
51339
|
+
p50Ms: 2.7,
|
|
51340
|
+
fps: 405,
|
|
51341
|
+
measured: true
|
|
51342
|
+
},
|
|
51343
|
+
"yolov9t-640": {
|
|
51344
|
+
p50Ms: 23,
|
|
51345
|
+
fps: 44,
|
|
51346
|
+
measured: true
|
|
51347
|
+
},
|
|
51348
|
+
"yolov9t-640-int8": {
|
|
51349
|
+
p50Ms: 8.8,
|
|
51350
|
+
fps: 124,
|
|
51351
|
+
measured: true
|
|
51352
|
+
},
|
|
51353
|
+
"yolov9s-320": {
|
|
51354
|
+
p50Ms: 15.8,
|
|
51355
|
+
fps: 63,
|
|
51356
|
+
measured: true
|
|
51357
|
+
},
|
|
51358
|
+
"yolov9s-320-int8": {
|
|
51359
|
+
p50Ms: 6.5,
|
|
51360
|
+
fps: 180,
|
|
51361
|
+
measured: true
|
|
51362
|
+
},
|
|
51363
|
+
"yolov9s-640": {
|
|
51364
|
+
p50Ms: 58.2,
|
|
51365
|
+
fps: 17,
|
|
51366
|
+
measured: true
|
|
51367
|
+
},
|
|
51368
|
+
"yolov9s-640-int8": {
|
|
51369
|
+
p50Ms: 20.1,
|
|
51370
|
+
fps: 52,
|
|
51371
|
+
measured: true
|
|
51372
|
+
},
|
|
51373
|
+
"yolov9m-320": {
|
|
51374
|
+
p50Ms: 40,
|
|
51375
|
+
fps: 25,
|
|
51376
|
+
measured: true
|
|
51377
|
+
},
|
|
51378
|
+
"yolov9m-320-int8": {
|
|
51379
|
+
p50Ms: 14.2,
|
|
51380
|
+
fps: 74,
|
|
51381
|
+
measured: true
|
|
51382
|
+
},
|
|
51383
|
+
"yolov9m-640": {
|
|
51384
|
+
p50Ms: 155.8,
|
|
51385
|
+
fps: 6,
|
|
51386
|
+
measured: true
|
|
51387
|
+
},
|
|
51388
|
+
"yolov9m-640-int8": {
|
|
51389
|
+
p50Ms: 52.4,
|
|
51390
|
+
fps: 19,
|
|
51391
|
+
measured: true
|
|
51392
|
+
}
|
|
51393
|
+
}
|
|
51394
|
+
},
|
|
51395
|
+
"intel-n-series": {
|
|
51396
|
+
"intel-igpu": {
|
|
51397
|
+
"yolov9t-320": {
|
|
51398
|
+
p50Ms: 15,
|
|
51399
|
+
fps: 48,
|
|
51400
|
+
measured: true
|
|
51401
|
+
},
|
|
51402
|
+
"yolov9t-320-int8": {
|
|
51403
|
+
p50Ms: 14.1,
|
|
51404
|
+
fps: 67,
|
|
51405
|
+
measured: true
|
|
51406
|
+
},
|
|
51407
|
+
"yolov9t-640": {
|
|
51408
|
+
p50Ms: 38.5,
|
|
51409
|
+
fps: 26,
|
|
51410
|
+
measured: true
|
|
51411
|
+
},
|
|
51412
|
+
"yolov9t-640-int8": {
|
|
51413
|
+
p50Ms: 33.1,
|
|
51414
|
+
fps: 33,
|
|
51415
|
+
measured: true
|
|
51416
|
+
},
|
|
51417
|
+
"yolov9s-320": {
|
|
51418
|
+
p50Ms: 41.4,
|
|
51419
|
+
fps: 24,
|
|
51420
|
+
measured: true
|
|
51421
|
+
},
|
|
51422
|
+
"yolov9s-320-int8": {
|
|
51423
|
+
p50Ms: 23.3,
|
|
51424
|
+
fps: 34,
|
|
51425
|
+
measured: true
|
|
51426
|
+
},
|
|
51427
|
+
"yolov9s-640": {
|
|
51428
|
+
p50Ms: 127.4,
|
|
51429
|
+
fps: 8,
|
|
51430
|
+
measured: true
|
|
51431
|
+
},
|
|
51432
|
+
"yolov9s-640-int8": {
|
|
51433
|
+
p50Ms: 92.9,
|
|
51434
|
+
fps: 11,
|
|
51435
|
+
measured: true
|
|
51436
|
+
},
|
|
51437
|
+
"yolov9m-320": {
|
|
51438
|
+
p50Ms: 63.5,
|
|
51439
|
+
fps: 16,
|
|
51440
|
+
measured: true
|
|
51441
|
+
},
|
|
51442
|
+
"yolov9m-320-int8": {
|
|
51443
|
+
p50Ms: 39.4,
|
|
51444
|
+
fps: 26,
|
|
51445
|
+
measured: true
|
|
51446
|
+
},
|
|
51447
|
+
"yolov9m-640": {
|
|
51448
|
+
p50Ms: 215.5,
|
|
51449
|
+
fps: 5,
|
|
51450
|
+
measured: true
|
|
51451
|
+
},
|
|
51452
|
+
"yolov9m-640-int8": {
|
|
51453
|
+
p50Ms: 139.1,
|
|
51454
|
+
fps: 7,
|
|
51455
|
+
measured: true
|
|
51456
|
+
}
|
|
51457
|
+
},
|
|
51458
|
+
"x86-cpu": {
|
|
51459
|
+
"yolov9t-320": {
|
|
51460
|
+
p50Ms: 86.2,
|
|
51461
|
+
fps: 12,
|
|
51462
|
+
measured: true
|
|
51463
|
+
},
|
|
51464
|
+
"yolov9t-320-int8": {
|
|
51465
|
+
p50Ms: 52.7,
|
|
51466
|
+
fps: 20,
|
|
51467
|
+
measured: true
|
|
51468
|
+
},
|
|
51469
|
+
"yolov9t-640": {
|
|
51470
|
+
p50Ms: 322.1,
|
|
51471
|
+
fps: 3,
|
|
51472
|
+
measured: true
|
|
51473
|
+
},
|
|
51474
|
+
"yolov9t-640-int8": {
|
|
51475
|
+
p50Ms: 205.6,
|
|
51476
|
+
fps: 5,
|
|
51477
|
+
measured: true
|
|
51478
|
+
},
|
|
51479
|
+
"yolov9s-320": {
|
|
51480
|
+
p50Ms: 164.4,
|
|
51481
|
+
fps: 6,
|
|
51482
|
+
measured: true
|
|
51483
|
+
},
|
|
51484
|
+
"yolov9s-320-int8": {
|
|
51485
|
+
p50Ms: 86.9,
|
|
51486
|
+
fps: 12,
|
|
51487
|
+
measured: true
|
|
51488
|
+
},
|
|
51489
|
+
"yolov9s-640": {
|
|
51490
|
+
p50Ms: 341.3,
|
|
51491
|
+
fps: 3,
|
|
51492
|
+
measured: true
|
|
51493
|
+
},
|
|
51494
|
+
"yolov9s-640-int8": {
|
|
51495
|
+
p50Ms: 185.3,
|
|
51496
|
+
fps: 5,
|
|
51497
|
+
measured: true
|
|
51498
|
+
},
|
|
51499
|
+
"yolov9m-320": {
|
|
51500
|
+
p50Ms: 166.8,
|
|
51501
|
+
fps: 6,
|
|
51502
|
+
measured: true
|
|
51503
|
+
},
|
|
51504
|
+
"yolov9m-320-int8": {
|
|
51505
|
+
p50Ms: 226.3,
|
|
51506
|
+
fps: 4,
|
|
51507
|
+
measured: true
|
|
51508
|
+
},
|
|
51509
|
+
"yolov9m-640": {
|
|
51510
|
+
p50Ms: 663.5,
|
|
51511
|
+
fps: 2,
|
|
51512
|
+
measured: true
|
|
51513
|
+
},
|
|
51514
|
+
"yolov9m-640-int8": {
|
|
51515
|
+
p50Ms: 324.4,
|
|
51516
|
+
fps: 3,
|
|
51517
|
+
measured: true
|
|
51518
|
+
}
|
|
51519
|
+
}
|
|
51520
|
+
},
|
|
51521
|
+
"apple-m-pro": { "apple-ane": {
|
|
51522
|
+
"yolov9t-320": {
|
|
51523
|
+
p50Ms: 1.1,
|
|
51524
|
+
fps: 952,
|
|
51525
|
+
measured: true
|
|
51526
|
+
},
|
|
51527
|
+
"yolov9t-640": {
|
|
51528
|
+
p50Ms: 3,
|
|
51529
|
+
fps: 333,
|
|
51530
|
+
measured: true
|
|
51531
|
+
},
|
|
51532
|
+
"yolov9s-320": {
|
|
51533
|
+
p50Ms: 1.5,
|
|
51534
|
+
fps: 667,
|
|
51535
|
+
measured: true
|
|
51536
|
+
},
|
|
51537
|
+
"yolov9s-640": {
|
|
51538
|
+
p50Ms: 5.1,
|
|
51539
|
+
fps: 196,
|
|
51540
|
+
measured: true
|
|
51541
|
+
},
|
|
51542
|
+
"yolov9m-320": {
|
|
51543
|
+
p50Ms: 3.2,
|
|
51544
|
+
fps: 312,
|
|
51545
|
+
measured: true
|
|
51546
|
+
},
|
|
51547
|
+
"yolov9m-640": {
|
|
51548
|
+
p50Ms: 10.2,
|
|
51549
|
+
fps: 98,
|
|
51550
|
+
measured: true
|
|
51551
|
+
}
|
|
51552
|
+
} },
|
|
51553
|
+
"apple-m-base": { "apple-ane": {
|
|
51554
|
+
"yolov9t-320": {
|
|
51555
|
+
p50Ms: 1.32,
|
|
51556
|
+
fps: 758,
|
|
51557
|
+
measured: false
|
|
51558
|
+
},
|
|
51559
|
+
"yolov9t-640": {
|
|
51560
|
+
p50Ms: 3.45,
|
|
51561
|
+
fps: 290,
|
|
51562
|
+
measured: true
|
|
51563
|
+
},
|
|
51564
|
+
"yolov9s-320": {
|
|
51565
|
+
p50Ms: 1.8,
|
|
51566
|
+
fps: 556,
|
|
51567
|
+
measured: false
|
|
51568
|
+
},
|
|
51569
|
+
"yolov9s-640": {
|
|
51570
|
+
p50Ms: 5.46,
|
|
51571
|
+
fps: 183,
|
|
51572
|
+
measured: true
|
|
51573
|
+
},
|
|
51574
|
+
"yolov9m-320": {
|
|
51575
|
+
p50Ms: 3.79,
|
|
51576
|
+
fps: 264,
|
|
51577
|
+
measured: true
|
|
51578
|
+
},
|
|
51579
|
+
"yolov9m-640": {
|
|
51580
|
+
p50Ms: 10.34,
|
|
51581
|
+
fps: 97,
|
|
51582
|
+
measured: true
|
|
51583
|
+
}
|
|
51584
|
+
} }
|
|
51585
|
+
};
|
|
51586
|
+
var DETECTOR_ACCURACY = {
|
|
51587
|
+
"yolov9m-320": {
|
|
51588
|
+
personPrecision: .979,
|
|
51589
|
+
personRecall: .649
|
|
51590
|
+
},
|
|
51591
|
+
"yolov9m-320-int8": {
|
|
51592
|
+
personPrecision: .974,
|
|
51593
|
+
personRecall: .684
|
|
51594
|
+
},
|
|
51595
|
+
"yolov9m-640": {
|
|
51596
|
+
personPrecision: .976,
|
|
51597
|
+
personRecall: .854
|
|
51598
|
+
},
|
|
51599
|
+
"yolov9m-640-int8": {
|
|
51600
|
+
personPrecision: .976,
|
|
51601
|
+
personRecall: .847
|
|
51602
|
+
},
|
|
51603
|
+
"yolov9s-320": {
|
|
51604
|
+
personPrecision: .958,
|
|
51605
|
+
personRecall: .479
|
|
51606
|
+
},
|
|
51607
|
+
"yolov9s-320-int8": {
|
|
51608
|
+
personPrecision: .969,
|
|
51609
|
+
personRecall: .438
|
|
51610
|
+
},
|
|
51611
|
+
"yolov9s-640": {
|
|
51612
|
+
personPrecision: .986,
|
|
51613
|
+
personRecall: .753
|
|
51614
|
+
},
|
|
51615
|
+
"yolov9s-640-int8": {
|
|
51616
|
+
personPrecision: .963,
|
|
51617
|
+
personRecall: .823
|
|
51618
|
+
},
|
|
51619
|
+
"yolov9t-320": {
|
|
51620
|
+
personPrecision: .946,
|
|
51621
|
+
personRecall: .368
|
|
51622
|
+
},
|
|
51623
|
+
"yolov9t-320-int8": {
|
|
51624
|
+
personPrecision: .943,
|
|
51625
|
+
personRecall: .347
|
|
51626
|
+
},
|
|
51627
|
+
"yolov9t-640": {
|
|
51628
|
+
personPrecision: .979,
|
|
51629
|
+
personRecall: .646
|
|
51630
|
+
},
|
|
51631
|
+
"yolov9t-640-int8": {
|
|
51632
|
+
personPrecision: .979,
|
|
51633
|
+
personRecall: .646
|
|
51634
|
+
}
|
|
51635
|
+
};
|
|
51636
|
+
var DEFAULT_DEMAND_OPTIONS = {
|
|
51637
|
+
onMotionConcurrency: .25,
|
|
51638
|
+
onMotionMinConcurrent: 2
|
|
51639
|
+
};
|
|
51640
|
+
function deviceId(nodeId, deviceKey) {
|
|
51641
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
51642
|
+
}
|
|
51643
|
+
function charges(load) {
|
|
51644
|
+
return load.mode !== "disabled" && load.runsPipelineRoot && load.fps > 0;
|
|
51645
|
+
}
|
|
51646
|
+
/** The highest `count` rates, summed. */
|
|
51647
|
+
function topRatesSum(rates, count) {
|
|
51648
|
+
return rates.toSorted((a, b) => b - a).slice(0, Math.max(0, count)).reduce((sum, r) => sum + r, 0);
|
|
51649
|
+
}
|
|
51650
|
+
/** Concurrent on-motion sessions assumed cluster-wide without an observation. */
|
|
51651
|
+
function estimatedConcurrentSessions(cameraCount, options) {
|
|
51652
|
+
if (cameraCount <= 0) return 0;
|
|
51653
|
+
const share = Math.ceil(options.onMotionConcurrency * cameraCount);
|
|
51654
|
+
return Math.min(cameraCount, Math.max(share, Math.min(cameraCount, options.onMotionMinConcurrent)));
|
|
51655
|
+
}
|
|
51656
|
+
/**
|
|
51657
|
+
* Demand on every eligible device of the cluster.
|
|
51658
|
+
*/
|
|
51659
|
+
function estimateDeviceDemand(input) {
|
|
51660
|
+
const options = input.options ?? DEFAULT_DEMAND_OPTIONS;
|
|
51661
|
+
const placement = input.shares ? "share" : "least-loaded";
|
|
51662
|
+
const eligibleById = new Map(input.eligible.map((d) => [deviceId(d.nodeId, d.deviceKey), d]));
|
|
51663
|
+
const peakById = new Map(input.peaks.map((p) => [deviceId(p.nodeId, p.deviceKey), p]));
|
|
51664
|
+
const weightByNode = /* @__PURE__ */ new Map();
|
|
51665
|
+
const eligibleCountByNode = /* @__PURE__ */ new Map();
|
|
51666
|
+
for (const d of input.eligible) {
|
|
51667
|
+
weightByNode.set(d.nodeId, (weightByNode.get(d.nodeId) ?? 0) + Math.max(d.weight, 0));
|
|
51668
|
+
eligibleCountByNode.set(d.nodeId, (eligibleCountByNode.get(d.nodeId) ?? 0) + 1);
|
|
51669
|
+
}
|
|
51670
|
+
const pinnedFps = /* @__PURE__ */ new Map();
|
|
51671
|
+
const alwaysOnFps = /* @__PURE__ */ new Map();
|
|
51672
|
+
const alwaysOnCount = /* @__PURE__ */ new Map();
|
|
51673
|
+
const maxFps = /* @__PURE__ */ new Map();
|
|
51674
|
+
const onMotionRates = [];
|
|
51675
|
+
const bump = (map, key, by) => {
|
|
51676
|
+
map.set(key, (map.get(key) ?? 0) + by);
|
|
51677
|
+
};
|
|
51678
|
+
const noteMax = (key, fps) => {
|
|
51679
|
+
maxFps.set(key, Math.max(maxFps.get(key) ?? 0, fps));
|
|
51680
|
+
};
|
|
51681
|
+
const pinNodeOf = (pin) => {
|
|
51682
|
+
if (pin.nodeId !== null) return pin.nodeId;
|
|
51683
|
+
const holders = input.eligible.filter((d) => d.deviceKey === pin.deviceKey);
|
|
51684
|
+
return holders.length === 1 ? holders[0].nodeId : null;
|
|
51685
|
+
};
|
|
51686
|
+
let alwaysOnTotal = 0;
|
|
51687
|
+
for (const cam of input.cameras) {
|
|
51688
|
+
if (!charges(cam)) continue;
|
|
51689
|
+
const pinNode = cam.devicePin !== null ? pinNodeOf(cam.devicePin) : null;
|
|
51690
|
+
if (cam.devicePin !== null && pinNode !== null) {
|
|
51691
|
+
const key = deviceId(pinNode, cam.devicePin.deviceKey);
|
|
51692
|
+
if (!eligibleById.has(key)) continue;
|
|
51693
|
+
bump(pinnedFps, key, cam.fps);
|
|
51694
|
+
noteMax(key, cam.fps);
|
|
51695
|
+
continue;
|
|
51696
|
+
}
|
|
51697
|
+
const standing = cam.mode === "always-on" && cam.standingNodeId !== null ? cam.standingNodeId : cam.nodePinId;
|
|
51698
|
+
if (standing !== null) {
|
|
51699
|
+
if (cam.mode === "always-on") alwaysOnTotal += 1;
|
|
51700
|
+
const nodeWeight = weightByNode.get(standing) ?? 0;
|
|
51701
|
+
for (const d of input.eligible) {
|
|
51702
|
+
if (d.nodeId !== standing || nodeWeight <= 0) continue;
|
|
51703
|
+
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51704
|
+
const share = Math.max(d.weight, 0) / nodeWeight;
|
|
51705
|
+
bump(alwaysOnFps, key, cam.fps * share);
|
|
51706
|
+
bump(alwaysOnCount, key, share);
|
|
51707
|
+
noteMax(key, cam.fps);
|
|
51708
|
+
}
|
|
51709
|
+
continue;
|
|
51710
|
+
}
|
|
51711
|
+
onMotionRates.push(cam.fps);
|
|
51712
|
+
}
|
|
51713
|
+
const fastestOnMotion = onMotionRates.reduce((m, r) => Math.max(m, r), 0);
|
|
51714
|
+
const nodeCount = weightByNode.size;
|
|
51715
|
+
const estimatedTotal = estimatedConcurrentSessions(onMotionRates.length, options);
|
|
51716
|
+
const clusterPeak = input.shares?.clusterPeak ?? null;
|
|
51717
|
+
const clusterObserved = clusterPeak !== null && clusterPeak.observedForMs >= 864e5;
|
|
51718
|
+
const clusterSessions = clusterObserved ? Math.max(0, clusterPeak.peakSessions - alwaysOnTotal) : estimatedTotal;
|
|
51719
|
+
const shareSum = [...weightByNode.keys()].reduce((sum, nodeId) => sum + Math.max(input.shares?.nodes[nodeId] ?? 0, 0), 0);
|
|
51720
|
+
return input.eligible.map((d) => {
|
|
51721
|
+
const key = deviceId(d.nodeId, d.deviceKey);
|
|
51722
|
+
const peak = peakById.get(key) ?? (eligibleCountByNode.get(d.nodeId) === 1 ? peakById.get(deviceId(d.nodeId, "")) : void 0);
|
|
51723
|
+
const alwaysOn = alwaysOnCount.get(key) ?? 0;
|
|
51724
|
+
let onMotionSessions = 0;
|
|
51725
|
+
let onMotionSource = "none";
|
|
51726
|
+
if (onMotionRates.length > 0 && input.shares !== void 0) {
|
|
51727
|
+
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51728
|
+
const nodeShare = shareSum > 0 ? Math.max(input.shares.nodes[d.nodeId] ?? 0, 0) / shareSum : 0;
|
|
51729
|
+
const deviceShare = nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight : 0;
|
|
51730
|
+
onMotionSessions = Math.max(0, Math.ceil(clusterSessions * nodeShare * deviceShare - 1e-9));
|
|
51731
|
+
onMotionSource = clusterObserved ? "observed-24h" : "estimate";
|
|
51732
|
+
} else if (onMotionRates.length > 0) if (peak !== void 0 && peak.observedForMs >= 864e5) {
|
|
51733
|
+
onMotionSessions = Math.max(0, Math.round(peak.peakSessions - alwaysOn));
|
|
51734
|
+
onMotionSource = "observed-24h";
|
|
51735
|
+
} else {
|
|
51736
|
+
const nodeWeight = weightByNode.get(d.nodeId) ?? 0;
|
|
51737
|
+
const share = nodeCount > 0 && nodeWeight > 0 ? Math.max(d.weight, 0) / nodeWeight / nodeCount : 0;
|
|
51738
|
+
onMotionSessions = Math.ceil(estimatedTotal * share);
|
|
51739
|
+
onMotionSource = "estimate";
|
|
51740
|
+
}
|
|
51741
|
+
const demandFps = (pinnedFps.get(key) ?? 0) + (alwaysOnFps.get(key) ?? 0) + topRatesSum(onMotionRates, onMotionSessions);
|
|
51742
|
+
return {
|
|
51743
|
+
nodeId: d.nodeId,
|
|
51744
|
+
deviceKey: d.deviceKey,
|
|
51745
|
+
demandFps: Math.round(demandFps * 100) / 100,
|
|
51746
|
+
maxCameraFps: Math.max(maxFps.get(key) ?? 0, fastestOnMotion),
|
|
51747
|
+
alwaysOnCameras: Math.round(alwaysOn * 100) / 100,
|
|
51748
|
+
onMotionSessions,
|
|
51749
|
+
onMotionSource,
|
|
51750
|
+
observedForMs: input.shares !== void 0 ? clusterPeak?.observedForMs ?? null : peak?.observedForMs ?? null,
|
|
51751
|
+
placement
|
|
51752
|
+
};
|
|
51753
|
+
}).toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
51754
|
+
}
|
|
51755
|
+
/** The family the cluster runs. */
|
|
51756
|
+
var BEST_SETTINGS_FAMILY = "yolov9";
|
|
51757
|
+
function round(value, digits = 3) {
|
|
51758
|
+
const f = 10 ** digits;
|
|
51759
|
+
return Math.round(value * f) / f;
|
|
51760
|
+
}
|
|
51761
|
+
function utilisationOf(cell, demand) {
|
|
51762
|
+
return cell.fps > 0 ? round((demand?.demandFps ?? 0) / cell.fps) : Number.POSITIVE_INFINITY;
|
|
51763
|
+
}
|
|
51764
|
+
function fitsOn(cell, demand, headroom) {
|
|
51765
|
+
const budget = cell.fps * (1 - headroom);
|
|
51766
|
+
return budget >= (demand?.demandFps ?? 0) && budget >= (demand?.maxCameraFps ?? 0);
|
|
51767
|
+
}
|
|
51768
|
+
function candidatesFor(contexts, formatsByModel) {
|
|
51769
|
+
return Object.keys(DETECTOR_ACCURACY).filter((id) => id.startsWith(BEST_SETTINGS_FAMILY)).filter((id) => contexts.every((c) => c.column[id] !== void 0 && (formatsByModel[id] ?? []).includes(c.device.format))).map((id) => ({
|
|
51770
|
+
modelId: id,
|
|
51771
|
+
personRecall: DETECTOR_ACCURACY[id]?.personRecall ?? 0
|
|
51772
|
+
})).toSorted((a, b) => b.personRecall - a.personRecall || a.modelId.localeCompare(b.modelId));
|
|
51773
|
+
}
|
|
51774
|
+
function totalLatency(contexts, modelId) {
|
|
51775
|
+
return contexts.reduce((sum, c) => sum + (c.column[modelId]?.p50Ms ?? Number.POSITIVE_INFINITY), 0);
|
|
51776
|
+
}
|
|
51777
|
+
function worstUtilisation(contexts, modelId) {
|
|
51778
|
+
return contexts.reduce((worst, c) => {
|
|
51779
|
+
const cell = c.column[modelId];
|
|
51780
|
+
return Math.max(worst, cell ? utilisationOf(cell, c.demand) : Number.POSITIVE_INFINITY);
|
|
51781
|
+
}, 0);
|
|
51782
|
+
}
|
|
51783
|
+
/** One model for a group of devices that must agree (step 2–4 of the header). */
|
|
51784
|
+
function pickForGroup(contexts, formatsByModel, headroom) {
|
|
51785
|
+
const candidates = candidatesFor(contexts, formatsByModel);
|
|
51786
|
+
if (candidates.length === 0) return null;
|
|
51787
|
+
const fitting = candidates.filter((c) => contexts.every((ctx) => {
|
|
51788
|
+
const cell = ctx.column[c.modelId];
|
|
51789
|
+
return cell !== void 0 && fitsOn(cell, ctx.demand, headroom);
|
|
51790
|
+
}));
|
|
51791
|
+
if (fitting.length === 0) {
|
|
51792
|
+
const lightest = candidates.toSorted((a, b) => worstUtilisation(contexts, a.modelId) - worstUtilisation(contexts, b.modelId))[0];
|
|
51793
|
+
if (lightest === void 0) return null;
|
|
51794
|
+
return {
|
|
51795
|
+
modelId: lightest.modelId,
|
|
51796
|
+
reasons: [{
|
|
51797
|
+
kind: "no-fit-lightest",
|
|
51798
|
+
utilisation: worstUtilisation(contexts, lightest.modelId)
|
|
51799
|
+
}]
|
|
51800
|
+
};
|
|
51801
|
+
}
|
|
51802
|
+
const best = fitting[0];
|
|
51803
|
+
if (best === void 0) return null;
|
|
51804
|
+
const chosen = fitting.filter((c) => best.personRecall - c.personRecall <= .02).toSorted((a, b) => totalLatency(contexts, a.modelId) - totalLatency(contexts, b.modelId) || b.personRecall - a.personRecall)[0] ?? best;
|
|
51805
|
+
const reasons = [];
|
|
51806
|
+
const mostAccurate = candidates[0];
|
|
51807
|
+
if (mostAccurate !== void 0 && mostAccurate.modelId !== best.modelId) {
|
|
51808
|
+
const limitedBy = contexts.find((ctx) => {
|
|
51809
|
+
const cell = ctx.column[mostAccurate.modelId];
|
|
51810
|
+
return cell === void 0 || !fitsOn(cell, ctx.demand, headroom);
|
|
51811
|
+
})?.device.key ?? "";
|
|
51812
|
+
reasons.push({
|
|
51813
|
+
kind: "stepped-down",
|
|
51814
|
+
from: mostAccurate.modelId,
|
|
51815
|
+
to: best.modelId,
|
|
51816
|
+
limitedBy
|
|
51817
|
+
});
|
|
51818
|
+
}
|
|
51819
|
+
if (chosen.modelId !== best.modelId) reasons.push({
|
|
51820
|
+
kind: "tie-faster",
|
|
51821
|
+
over: best.modelId,
|
|
51822
|
+
recallDelta: round(best.personRecall - chosen.personRecall)
|
|
51823
|
+
});
|
|
51824
|
+
return {
|
|
51825
|
+
modelId: chosen.modelId,
|
|
51826
|
+
reasons
|
|
51827
|
+
};
|
|
51828
|
+
}
|
|
51829
|
+
function choiceOn(ctx, modelId) {
|
|
51830
|
+
const cell = ctx.column[modelId];
|
|
51831
|
+
if (cell === void 0) return null;
|
|
51832
|
+
return {
|
|
51833
|
+
modelId,
|
|
51834
|
+
p50Ms: cell.p50Ms,
|
|
51835
|
+
fps: cell.fps,
|
|
51836
|
+
measured: cell.measured,
|
|
51837
|
+
utilisation: utilisationOf(cell, ctx.demand),
|
|
51838
|
+
personRecall: DETECTOR_ACCURACY[modelId]?.personRecall ?? 0
|
|
51839
|
+
};
|
|
51840
|
+
}
|
|
51841
|
+
function deviceReasonsFor(ctx, choice) {
|
|
51842
|
+
const out = [{
|
|
51843
|
+
kind: "fits",
|
|
51844
|
+
utilisation: choice.utilisation
|
|
51845
|
+
}];
|
|
51846
|
+
if (!choice.measured) out.push({
|
|
51847
|
+
kind: "derived-latency",
|
|
51848
|
+
modelId: choice.modelId
|
|
51849
|
+
});
|
|
51850
|
+
if (ctx.demand?.onMotionSource === "estimate") out.push({
|
|
51851
|
+
kind: "estimated-load",
|
|
51852
|
+
observedForMs: ctx.demand.observedForMs
|
|
51853
|
+
});
|
|
51854
|
+
return out;
|
|
51855
|
+
}
|
|
51856
|
+
function skippedNode(node, reason, hardwareClass) {
|
|
51857
|
+
return {
|
|
51858
|
+
nodeId: node.nodeId,
|
|
51859
|
+
reachable: node.reachable,
|
|
51860
|
+
hardwareClass,
|
|
51861
|
+
cpuModel: node.hardware?.cpuModel ?? null,
|
|
51862
|
+
skipped: true,
|
|
51863
|
+
acceleratorModelId: null,
|
|
51864
|
+
devices: node.devices.map((d) => ({
|
|
51865
|
+
deviceKey: d.key,
|
|
51866
|
+
accelerator: node.hardware ? acceleratorKindOf(d.key, node.hardware) : null,
|
|
51867
|
+
enabled: d.enabled,
|
|
51868
|
+
available: d.available,
|
|
51869
|
+
demand: null,
|
|
51870
|
+
chosen: null,
|
|
51871
|
+
reasons: []
|
|
51872
|
+
})),
|
|
51873
|
+
reasons: [reason]
|
|
51874
|
+
};
|
|
51875
|
+
}
|
|
51876
|
+
/** Recommend one node. See the module header. */
|
|
51877
|
+
function recommendNode(input) {
|
|
51878
|
+
const { node } = input;
|
|
51879
|
+
const headroom = input.headroom ?? .4;
|
|
51880
|
+
const perf = input.perf ?? DETECTOR_PERF;
|
|
51881
|
+
if (!node.reachable) return skippedNode(node, { kind: "node-offline" }, null);
|
|
51882
|
+
if (node.hardware === null) return skippedNode(node, { kind: "no-hardware-facts" }, null);
|
|
51883
|
+
const hardware = node.hardware;
|
|
51884
|
+
const { hardwareClass, cpuModel } = classifyInferenceHardware(hardware);
|
|
51885
|
+
const classTable = perf[hardwareClass];
|
|
51886
|
+
if (hardwareClass === "unknown" || classTable === void 0) return skippedNode(node, {
|
|
51887
|
+
kind: "no-measured-data",
|
|
51888
|
+
hardwareClass
|
|
51889
|
+
}, hardwareClass);
|
|
51890
|
+
const demandByKey = new Map(input.demand.filter((d) => d.nodeId === node.nodeId).map((d) => [d.deviceKey, d]));
|
|
51891
|
+
const devices = node.devices.toSorted((a, b) => a.key.localeCompare(b.key));
|
|
51892
|
+
const contexts = /* @__PURE__ */ new Map();
|
|
51893
|
+
const early = /* @__PURE__ */ new Map();
|
|
51894
|
+
for (const device of devices) {
|
|
51895
|
+
const accelerator = acceleratorKindOf(device.key, hardware);
|
|
51896
|
+
if (accelerator === "coral") {
|
|
51897
|
+
early.set(device.key, device.enabled ? [{ kind: "coral-untouched" }] : []);
|
|
51898
|
+
continue;
|
|
51899
|
+
}
|
|
51900
|
+
if (!device.enabled) {
|
|
51901
|
+
early.set(device.key, [{ kind: "device-disabled" }]);
|
|
51902
|
+
continue;
|
|
51903
|
+
}
|
|
51904
|
+
if (!device.available) {
|
|
51905
|
+
early.set(device.key, [{ kind: "device-unavailable" }]);
|
|
51906
|
+
continue;
|
|
51907
|
+
}
|
|
51908
|
+
const column = accelerator === null ? void 0 : classTable[accelerator];
|
|
51909
|
+
if (accelerator === null || column === void 0) {
|
|
51910
|
+
early.set(device.key, [{
|
|
51911
|
+
kind: "no-family-build",
|
|
51912
|
+
format: device.format
|
|
51913
|
+
}]);
|
|
51914
|
+
continue;
|
|
51915
|
+
}
|
|
51916
|
+
contexts.set(device.key, {
|
|
51917
|
+
device,
|
|
51918
|
+
accelerator,
|
|
51919
|
+
column,
|
|
51920
|
+
demand: demandByKey.get(device.key) ?? null
|
|
51921
|
+
});
|
|
51922
|
+
}
|
|
51923
|
+
const isCpu = (a) => a === "x86-cpu" || a === "arm-cpu";
|
|
51924
|
+
const accelerators = [...contexts.values()].filter((c) => !isCpu(c.accelerator));
|
|
51925
|
+
const cpus = [...contexts.values()].filter((c) => isCpu(c.accelerator));
|
|
51926
|
+
const chosenByKey = /* @__PURE__ */ new Map();
|
|
51927
|
+
const reasonsByKey = /* @__PURE__ */ new Map();
|
|
51928
|
+
const nodeReasons = [];
|
|
51929
|
+
let acceleratorModelId = null;
|
|
51930
|
+
const groupPick = accelerators.length > 0 ? pickForGroup(accelerators, node.formatsByModel, headroom) : null;
|
|
51931
|
+
if (groupPick !== null) {
|
|
51932
|
+
acceleratorModelId = groupPick.modelId;
|
|
51933
|
+
nodeReasons.push(...groupPick.reasons);
|
|
51934
|
+
for (const ctx of accelerators) {
|
|
51935
|
+
const choice = choiceOn(ctx, groupPick.modelId);
|
|
51936
|
+
if (choice === null) continue;
|
|
51937
|
+
chosenByKey.set(ctx.device.key, choice);
|
|
51938
|
+
reasonsByKey.set(ctx.device.key, deviceReasonsFor(ctx, choice));
|
|
51939
|
+
}
|
|
51940
|
+
} else for (const ctx of accelerators) reasonsByKey.set(ctx.device.key, [{
|
|
51941
|
+
kind: "no-family-build",
|
|
51942
|
+
format: ctx.device.format
|
|
51943
|
+
}]);
|
|
51944
|
+
for (const ctx of cpus) {
|
|
51945
|
+
const pick = pickForGroup([ctx], node.formatsByModel, headroom);
|
|
51946
|
+
const choice = pick === null ? null : choiceOn(ctx, pick.modelId);
|
|
51947
|
+
if (pick === null || choice === null) {
|
|
51948
|
+
reasonsByKey.set(ctx.device.key, [{
|
|
51949
|
+
kind: "no-family-build",
|
|
51950
|
+
format: ctx.device.format
|
|
51951
|
+
}]);
|
|
51952
|
+
continue;
|
|
51953
|
+
}
|
|
51954
|
+
chosenByKey.set(ctx.device.key, choice);
|
|
51955
|
+
reasonsByKey.set(ctx.device.key, [
|
|
51956
|
+
{ kind: "cpu-device" },
|
|
51957
|
+
...pick.reasons,
|
|
51958
|
+
...deviceReasonsFor(ctx, choice)
|
|
51959
|
+
]);
|
|
51960
|
+
}
|
|
51961
|
+
return {
|
|
51962
|
+
nodeId: node.nodeId,
|
|
51963
|
+
reachable: true,
|
|
51964
|
+
hardwareClass,
|
|
51965
|
+
cpuModel,
|
|
51966
|
+
skipped: chosenByKey.size === 0,
|
|
51967
|
+
acceleratorModelId,
|
|
51968
|
+
devices: devices.map((device) => ({
|
|
51969
|
+
deviceKey: device.key,
|
|
51970
|
+
accelerator: acceleratorKindOf(device.key, hardware),
|
|
51971
|
+
enabled: device.enabled,
|
|
51972
|
+
available: device.available,
|
|
51973
|
+
demand: demandByKey.get(device.key) ?? null,
|
|
51974
|
+
chosen: chosenByKey.get(device.key) ?? null,
|
|
51975
|
+
reasons: early.get(device.key) ?? reasonsByKey.get(device.key) ?? []
|
|
51976
|
+
})),
|
|
51977
|
+
reasons: nodeReasons
|
|
51978
|
+
};
|
|
51979
|
+
}
|
|
51980
|
+
/** Deficits equal up to float noise (0.7 × 2 − 1 is 0.3999…, not 0.4). */
|
|
51981
|
+
var DEFICIT_EPSILON = 1e-9;
|
|
51982
|
+
function byDeficit(a, b) {
|
|
51983
|
+
const diff = b.deficit - a.deficit;
|
|
51984
|
+
return (Math.abs(diff) < DEFICIT_EPSILON ? 0 : diff) || b.candidate.share - a.candidate.share || a.candidate.key.localeCompare(b.candidate.key);
|
|
51985
|
+
}
|
|
51986
|
+
function spareOf(c) {
|
|
51987
|
+
return c.spare ?? Number.POSITIVE_INFINITY;
|
|
51988
|
+
}
|
|
51989
|
+
function electByShare(candidates) {
|
|
51990
|
+
if (candidates.length === 0) return { kind: "none" };
|
|
51991
|
+
const shareSum = candidates.reduce((s, c) => s + Math.max(c.share, 0), 0);
|
|
51992
|
+
const running = candidates.reduce((s, c) => s + Math.max(c.running, 0), 0);
|
|
51993
|
+
const scored = candidates.map((candidate) => ({
|
|
51994
|
+
candidate,
|
|
51995
|
+
deficit: (shareSum > 0 ? Math.max(candidate.share, 0) / shareSum : 1 / candidates.length) * (running + 1) - candidate.running
|
|
51996
|
+
})).toSorted(byDeficit);
|
|
51997
|
+
const turn = scored[0].candidate;
|
|
51998
|
+
if (!turn.capped && !turn.saturated) return {
|
|
51999
|
+
kind: "elected",
|
|
52000
|
+
key: turn.key,
|
|
52001
|
+
reason: "share"
|
|
52002
|
+
};
|
|
52003
|
+
const free = scored.filter((s) => !s.candidate.capped && !s.candidate.saturated).toSorted((a, b) => spareOf(b.candidate) - spareOf(a.candidate) || byDeficit(a, b));
|
|
52004
|
+
const yieldReason = turn.capped ? "at-cap" : "saturated";
|
|
52005
|
+
if (free.length > 0) return {
|
|
52006
|
+
kind: "elected",
|
|
52007
|
+
key: free[0].candidate.key,
|
|
52008
|
+
reason: "yield",
|
|
52009
|
+
turnKey: turn.key,
|
|
52010
|
+
yieldReason
|
|
52011
|
+
};
|
|
52012
|
+
const uncapped = scored.find((s) => !s.candidate.capped);
|
|
52013
|
+
if (uncapped === void 0) return { kind: "none" };
|
|
52014
|
+
return {
|
|
52015
|
+
kind: "elected",
|
|
52016
|
+
key: uncapped.candidate.key,
|
|
52017
|
+
reason: "share"
|
|
52018
|
+
};
|
|
52019
|
+
}
|
|
50687
52020
|
/**
|
|
50688
52021
|
* THE detail-crop convention — the single derivation of the rectangle a
|
|
50689
52022
|
* detail/enrichment step (clip-embedding, face-detection, plate-detection…)
|
|
@@ -51458,7 +52791,7 @@ var AudioSubscriptionController = class {
|
|
|
51458
52791
|
} catch (err) {
|
|
51459
52792
|
this.deps.logger.error("audio re-attach failed", {
|
|
51460
52793
|
tags: { deviceId },
|
|
51461
|
-
meta: { error: errMsg$
|
|
52794
|
+
meta: { error: errMsg$2(err) }
|
|
51462
52795
|
});
|
|
51463
52796
|
}
|
|
51464
52797
|
}
|
|
@@ -51569,7 +52902,7 @@ var AudioSubscriptionController = class {
|
|
|
51569
52902
|
this.closeMotionAudioWindow(deviceId, "motion-cooldown-elapsed").catch((err) => {
|
|
51570
52903
|
this.deps.logger.warn("lazy audio: cooldown teardown failed", {
|
|
51571
52904
|
tags: { deviceId },
|
|
51572
|
-
meta: { error: errMsg$
|
|
52905
|
+
meta: { error: errMsg$2(err) }
|
|
51573
52906
|
});
|
|
51574
52907
|
});
|
|
51575
52908
|
}, cooldownMs);
|
|
@@ -51588,7 +52921,7 @@ var AudioSubscriptionController = class {
|
|
|
51588
52921
|
this.closeMotionAudioWindow(deviceId, "quiet-window-elapsed").catch((err) => {
|
|
51589
52922
|
this.deps.logger.warn("lazy audio: quiet-window teardown failed", {
|
|
51590
52923
|
tags: { deviceId },
|
|
51591
|
-
meta: { error: errMsg$
|
|
52924
|
+
meta: { error: errMsg$2(err) }
|
|
51592
52925
|
});
|
|
51593
52926
|
});
|
|
51594
52927
|
}, windowMs);
|
|
@@ -51790,7 +53123,7 @@ var AudioSubscriptionController = class {
|
|
|
51790
53123
|
meta: {
|
|
51791
53124
|
audioNodeId,
|
|
51792
53125
|
brokerId: audioBrokerId,
|
|
51793
|
-
error: errMsg$
|
|
53126
|
+
error: errMsg$2(err)
|
|
51794
53127
|
}
|
|
51795
53128
|
});
|
|
51796
53129
|
});
|
|
@@ -51865,7 +53198,7 @@ async function readOrRefuse(deps, deviceId) {
|
|
|
51865
53198
|
try {
|
|
51866
53199
|
device = await deps.readDevice(deviceId);
|
|
51867
53200
|
} catch (err) {
|
|
51868
|
-
return refuse(deps, deviceId, `could not read device ${deviceId}: ${errMsg$
|
|
53201
|
+
return refuse(deps, deviceId, `could not read device ${deviceId}: ${errMsg$2(err)}`);
|
|
51869
53202
|
}
|
|
51870
53203
|
if (device === null) return refuse(deps, deviceId, `no device ${deviceId}`);
|
|
51871
53204
|
return device;
|
|
@@ -53195,7 +54528,7 @@ var LoadShedController = class LoadShedController {
|
|
|
53195
54528
|
if (await this.tryClusterRelocate(deviceId).catch((err) => {
|
|
53196
54529
|
this.deps.logger.warn("Load management: cluster relocate failed", {
|
|
53197
54530
|
tags: { deviceId },
|
|
53198
|
-
meta: { error: errMsg$
|
|
54531
|
+
meta: { error: errMsg$2(err) }
|
|
53199
54532
|
});
|
|
53200
54533
|
return false;
|
|
53201
54534
|
})) {
|
|
@@ -53209,7 +54542,7 @@ var LoadShedController = class LoadShedController {
|
|
|
53209
54542
|
this.deps.api()?.pipelineRunner.detachCamera.mutate({ deviceId }).catch((err) => {
|
|
53210
54543
|
this.deps.logger.warn("Load management: detachCamera failed", {
|
|
53211
54544
|
tags: { deviceId },
|
|
53212
|
-
meta: { error: errMsg$
|
|
54545
|
+
meta: { error: errMsg$2(err) }
|
|
53213
54546
|
});
|
|
53214
54547
|
});
|
|
53215
54548
|
this.ensureLoadShedResumeTimer();
|
|
@@ -53241,7 +54574,7 @@ var LoadShedController = class LoadShedController {
|
|
|
53241
54574
|
tags: { deviceId },
|
|
53242
54575
|
meta: {
|
|
53243
54576
|
from: currentNode,
|
|
53244
|
-
error: errMsg$
|
|
54577
|
+
error: errMsg$2(err)
|
|
53245
54578
|
}
|
|
53246
54579
|
});
|
|
53247
54580
|
});
|
|
@@ -53284,7 +54617,7 @@ var LoadShedController = class LoadShedController {
|
|
|
53284
54617
|
this.deps.redispatchSingleCamera(deviceId).catch((err) => {
|
|
53285
54618
|
this.deps.logger.warn("Load management: auto-resume dispatch failed", {
|
|
53286
54619
|
tags: { deviceId },
|
|
53287
|
-
meta: { error: errMsg$
|
|
54620
|
+
meta: { error: errMsg$2(err) }
|
|
53288
54621
|
});
|
|
53289
54622
|
});
|
|
53290
54623
|
}
|
|
@@ -53449,7 +54782,7 @@ var AgentLoadService = class AgentLoadService {
|
|
|
53449
54782
|
ms,
|
|
53450
54783
|
outcome: "failed",
|
|
53451
54784
|
load: null,
|
|
53452
|
-
error: errMsg$
|
|
54785
|
+
error: errMsg$2(outcome.error)
|
|
53453
54786
|
};
|
|
53454
54787
|
return {
|
|
53455
54788
|
nodeId,
|
|
@@ -53916,7 +55249,7 @@ function buildCameraSensorsPorts(api, logger) {
|
|
|
53916
55249
|
} catch (err) {
|
|
53917
55250
|
return {
|
|
53918
55251
|
ok: false,
|
|
53919
|
-
error: errMsg$
|
|
55252
|
+
error: errMsg$2(err)
|
|
53920
55253
|
};
|
|
53921
55254
|
}
|
|
53922
55255
|
};
|
|
@@ -54180,7 +55513,7 @@ var CameraSensorsController = class {
|
|
|
54180
55513
|
async #readFleet() {
|
|
54181
55514
|
const read = await this.#deps.listCameraIds().catch((err) => ({
|
|
54182
55515
|
kind: "unreadable",
|
|
54183
|
-
error: errMsg$
|
|
55516
|
+
error: errMsg$2(err)
|
|
54184
55517
|
}));
|
|
54185
55518
|
if (read.kind === "unreadable") {
|
|
54186
55519
|
this.#deps.logger.warn("camera-sensors: camera list unreadable — mappings stay unloaded", { meta: { error: read.error } });
|
|
@@ -54207,7 +55540,7 @@ var CameraSensorsController = class {
|
|
|
54207
55540
|
} catch (err) {
|
|
54208
55541
|
this.#deps.logger.error("camera-sensors: refresh failed — nothing applied", {
|
|
54209
55542
|
tags: { deviceId: cameraId },
|
|
54210
|
-
meta: { error: errMsg$
|
|
55543
|
+
meta: { error: errMsg$2(err) }
|
|
54211
55544
|
});
|
|
54212
55545
|
}
|
|
54213
55546
|
}
|
|
@@ -54300,7 +55633,7 @@ async function readCameraSensorsSetting(read) {
|
|
|
54300
55633
|
} catch (err) {
|
|
54301
55634
|
return {
|
|
54302
55635
|
state: "unreadable",
|
|
54303
|
-
reason: `settings read failed — ${errMsg$
|
|
55636
|
+
reason: `settings read failed — ${errMsg$2(err)}`
|
|
54304
55637
|
};
|
|
54305
55638
|
}
|
|
54306
55639
|
}
|
|
@@ -54610,7 +55943,7 @@ var CameraMotionSlice = class {
|
|
|
54610
55943
|
st.chain = st.chain.then(job).catch((err) => {
|
|
54611
55944
|
this.#deps.logger.error("camera motion slice: write job failed", {
|
|
54612
55945
|
tags: { deviceId },
|
|
54613
|
-
meta: { error: errMsg$
|
|
55946
|
+
meta: { error: errMsg$2(err) }
|
|
54614
55947
|
});
|
|
54615
55948
|
});
|
|
54616
55949
|
}
|
|
@@ -54674,7 +56007,7 @@ var CameraMotionSlice = class {
|
|
|
54674
56007
|
meta: {
|
|
54675
56008
|
detected: slice.detected,
|
|
54676
56009
|
reason,
|
|
54677
|
-
error: errMsg$
|
|
56010
|
+
error: errMsg$2(err)
|
|
54678
56011
|
}
|
|
54679
56012
|
});
|
|
54680
56013
|
return;
|
|
@@ -54704,7 +56037,7 @@ var CameraMotionSlice = class {
|
|
|
54704
56037
|
async #republishAll() {
|
|
54705
56038
|
const listed = await this.#deps.listCameraIds().catch((err) => ({
|
|
54706
56039
|
kind: "unreadable",
|
|
54707
|
-
error: errMsg$
|
|
56040
|
+
error: errMsg$2(err)
|
|
54708
56041
|
}));
|
|
54709
56042
|
if (!this.#open || this.#stopped) return;
|
|
54710
56043
|
if (listed.kind === "unreadable") this.#deps.logger.warn("camera motion slice: camera list unreadable — only cameras with motion since boot are re-published", { meta: { error: listed.error } });
|
|
@@ -54826,7 +56159,7 @@ function hubMotionSliceStore(api) {
|
|
|
54826
56159
|
} catch (err) {
|
|
54827
56160
|
return {
|
|
54828
56161
|
kind: "unreadable",
|
|
54829
|
-
error: errMsg$
|
|
56162
|
+
error: errMsg$2(err)
|
|
54830
56163
|
};
|
|
54831
56164
|
}
|
|
54832
56165
|
}
|
|
@@ -56698,7 +58031,7 @@ function warnSampled(deps, deviceId, source, message, meta) {
|
|
|
56698
58031
|
function warnUnreachable(deps, deviceId, source, err) {
|
|
56699
58032
|
warnSampled(deps, deviceId, source, UNREACHABLE_MESSAGE, {
|
|
56700
58033
|
source,
|
|
56701
|
-
error: errMsg$
|
|
58034
|
+
error: errMsg$2(err)
|
|
56702
58035
|
});
|
|
56703
58036
|
}
|
|
56704
58037
|
/**
|
|
@@ -57972,7 +59305,7 @@ var ONBOARD_DETECTION_SOURCE = "onboard";
|
|
|
57972
59305
|
* configs, and it reads them the same way the runner does.
|
|
57973
59306
|
*/
|
|
57974
59307
|
var DETECTION_SOURCES_WHEN_ABSENT = ["pipeline"];
|
|
57975
|
-
function errMsg(err) {
|
|
59308
|
+
function errMsg$1(err) {
|
|
57976
59309
|
return err instanceof Error ? err.message : String(err);
|
|
57977
59310
|
}
|
|
57978
59311
|
/**
|
|
@@ -57991,7 +59324,7 @@ async function reconcileOnboardForwarding(input, deps) {
|
|
|
57991
59324
|
log.warn("onboard forwarding not reconciled — could not tell whether the camera has the cap", { meta: {
|
|
57992
59325
|
wantsOnboard,
|
|
57993
59326
|
detectionSources: [...detectionSources],
|
|
57994
|
-
error: errMsg(err)
|
|
59327
|
+
error: errMsg$1(err)
|
|
57995
59328
|
} });
|
|
57996
59329
|
return {
|
|
57997
59330
|
kind: "read-failed",
|
|
@@ -58009,7 +59342,7 @@ async function reconcileOnboardForwarding(input, deps) {
|
|
|
58009
59342
|
log.warn("onboard forwarding not reconciled — the vendor flag could not be read", { meta: {
|
|
58010
59343
|
wantsOnboard,
|
|
58011
59344
|
detectionSources: [...detectionSources],
|
|
58012
|
-
error: errMsg(err)
|
|
59345
|
+
error: errMsg$1(err)
|
|
58013
59346
|
} });
|
|
58014
59347
|
return {
|
|
58015
59348
|
kind: "read-failed",
|
|
@@ -58027,7 +59360,7 @@ async function reconcileOnboardForwarding(input, deps) {
|
|
|
58027
59360
|
was: current,
|
|
58028
59361
|
wanted: wantsOnboard,
|
|
58029
59362
|
detectionSources: [...detectionSources],
|
|
58030
|
-
error: errMsg(err)
|
|
59363
|
+
error: errMsg$1(err)
|
|
58031
59364
|
} });
|
|
58032
59365
|
return {
|
|
58033
59366
|
kind: "write-failed",
|
|
@@ -58223,7 +59556,7 @@ var DetectionWiringController = class {
|
|
|
58223
59556
|
try {
|
|
58224
59557
|
raw = await this.deps.ctx().settings.readDeviceStore(deviceId);
|
|
58225
59558
|
} catch (err) {
|
|
58226
|
-
const msg = errMsg$
|
|
59559
|
+
const msg = errMsg$2(err);
|
|
58227
59560
|
this.deps.logger.error("resolveDeviceDetectionSettings failed", {
|
|
58228
59561
|
tags: { deviceId },
|
|
58229
59562
|
meta: { error: msg }
|
|
@@ -58254,7 +59587,7 @@ var DetectionWiringController = class {
|
|
|
58254
59587
|
motionDetectionEnabled
|
|
58255
59588
|
});
|
|
58256
59589
|
} catch (err) {
|
|
58257
|
-
const msg = errMsg$
|
|
59590
|
+
const msg = errMsg$2(err);
|
|
58258
59591
|
this.deps.logger.error("resolveDeviceDetectionSettings narrowing failed", {
|
|
58259
59592
|
tags: { deviceId },
|
|
58260
59593
|
meta: { error: msg }
|
|
@@ -58289,7 +59622,7 @@ var DetectionWiringController = class {
|
|
|
58289
59622
|
} catch (err) {
|
|
58290
59623
|
this.deps.logger.debug("fetchAssignedProfiles: slot read failed (transient) — treating as unknown", {
|
|
58291
59624
|
tags: { deviceId },
|
|
58292
|
-
meta: { error: errMsg$
|
|
59625
|
+
meta: { error: errMsg$2(err) }
|
|
58293
59626
|
});
|
|
58294
59627
|
return null;
|
|
58295
59628
|
}
|
|
@@ -58365,7 +59698,7 @@ var DetectionWiringController = class {
|
|
|
58365
59698
|
if (config.motionSources.includes("analyzer")) try {
|
|
58366
59699
|
await this.deps.session.ensureMotionWatch(deviceId, config);
|
|
58367
59700
|
} catch (err) {
|
|
58368
|
-
log.error("ensureMotionWatch failed", { meta: { error: errMsg$
|
|
59701
|
+
log.error("ensureMotionWatch failed", { meta: { error: errMsg$2(err) } });
|
|
58369
59702
|
}
|
|
58370
59703
|
log.info("detection armed (session-time) — watching for motion", { meta: { motion: config.motionSources.join("+") } });
|
|
58371
59704
|
} else {
|
|
@@ -58386,14 +59719,14 @@ var DetectionWiringController = class {
|
|
|
58386
59719
|
try {
|
|
58387
59720
|
await this.deps.placement.dispatchCamera(runnerConfig);
|
|
58388
59721
|
} catch (err) {
|
|
58389
|
-
const msg = errMsg$
|
|
59722
|
+
const msg = errMsg$2(err);
|
|
58390
59723
|
log.error("dispatchCamera failed", { meta: { error: msg } });
|
|
58391
59724
|
}
|
|
58392
59725
|
}
|
|
58393
59726
|
try {
|
|
58394
59727
|
await this.deps.audio.resubscribe(deviceId, config);
|
|
58395
59728
|
} catch (err) {
|
|
58396
|
-
const msg = errMsg$
|
|
59729
|
+
const msg = errMsg$2(err);
|
|
58397
59730
|
log.error("subscribeAudioStream failed", { meta: { error: msg } });
|
|
58398
59731
|
}
|
|
58399
59732
|
if (config.detectionMode !== "on-motion") log.info("Detection dispatched", { meta: {
|
|
@@ -58409,7 +59742,7 @@ var DetectionWiringController = class {
|
|
|
58409
59742
|
try {
|
|
58410
59743
|
await this.deps.placement.releaseCamera({ deviceId });
|
|
58411
59744
|
} catch (err) {
|
|
58412
|
-
const msg = errMsg$
|
|
59745
|
+
const msg = errMsg$2(err);
|
|
58413
59746
|
this.deps.logger.error("releaseCamera failed", {
|
|
58414
59747
|
tags: { deviceId },
|
|
58415
59748
|
meta: { error: msg }
|
|
@@ -58435,7 +59768,7 @@ var DetectionWiringController = class {
|
|
|
58435
59768
|
} catch (err) {
|
|
58436
59769
|
this.deps.logger.warn("failed to clear per-device orchestrator settings on removal", {
|
|
58437
59770
|
tags: { deviceId },
|
|
58438
|
-
meta: { error: errMsg$
|
|
59771
|
+
meta: { error: errMsg$2(err) }
|
|
58439
59772
|
});
|
|
58440
59773
|
}
|
|
58441
59774
|
this.deps.logger.info("Purged pipeline config for removed device", { tags: { deviceId } });
|
|
@@ -58459,7 +59792,7 @@ var DetectionWiringController = class {
|
|
|
58459
59792
|
} catch (err) {
|
|
58460
59793
|
this.deps.logger.debug("device existence check failed — proceeding as before", {
|
|
58461
59794
|
tags: { deviceId },
|
|
58462
|
-
meta: { error: errMsg$
|
|
59795
|
+
meta: { error: errMsg$2(err) }
|
|
58463
59796
|
});
|
|
58464
59797
|
return false;
|
|
58465
59798
|
}
|
|
@@ -58576,7 +59909,7 @@ var DetectionWiringController = class {
|
|
|
58576
59909
|
await this.deps.attachOn(assignment.agentNodeId, nextConfig);
|
|
58577
59910
|
log.info("pipeline.camera-updated: hot-reloaded", { tags: { nodeId: assignment.agentNodeId } });
|
|
58578
59911
|
} catch (err) {
|
|
58579
|
-
const msg = errMsg$
|
|
59912
|
+
const msg = errMsg$2(err);
|
|
58580
59913
|
log.error("pipeline.camera-updated: attachOn failed", {
|
|
58581
59914
|
tags: { nodeId: assignment.agentNodeId },
|
|
58582
59915
|
meta: { error: msg }
|
|
@@ -58652,7 +59985,7 @@ var DetectionWiringController = class {
|
|
|
58652
59985
|
this.deps.audio.resubscribe(deviceId, audioCfg).catch((err) => {
|
|
58653
59986
|
this.deps.logger.warn("watchdog audio re-subscribe failed", {
|
|
58654
59987
|
tags: { deviceId },
|
|
58655
|
-
meta: { error: errMsg$
|
|
59988
|
+
meta: { error: errMsg$2(err) }
|
|
58656
59989
|
});
|
|
58657
59990
|
});
|
|
58658
59991
|
}
|
|
@@ -58746,6 +60079,35 @@ function pickInferenceDevice(input) {
|
|
|
58746
60079
|
const loadByKey = new Map((input.loads ?? []).map((l) => [l.deviceKey, l]));
|
|
58747
60080
|
const healthyKeys = eligibleKeys.filter((k) => !isDeviceSaturated(loadByKey.get(k)));
|
|
58748
60081
|
const consideredKeys = pin !== null || healthyKeys.length === 0 || healthyKeys.length === eligibleKeys.length ? eligibleKeys : healthyKeys;
|
|
60082
|
+
if (input.policy === "share" && pin === null) {
|
|
60083
|
+
const election = electByShare(eligibleKeys.map((key) => {
|
|
60084
|
+
const load = loadByKey.get(key);
|
|
60085
|
+
const attached = load?.attachedCameras ?? 0;
|
|
60086
|
+
const cap = input.caps?.[key];
|
|
60087
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
60088
|
+
return {
|
|
60089
|
+
key,
|
|
60090
|
+
share: input.eligible[key] ?? 1,
|
|
60091
|
+
running: attached,
|
|
60092
|
+
capped: hasCap && attached >= cap,
|
|
60093
|
+
saturated: isDeviceSaturated(load),
|
|
60094
|
+
...hasCap ? { spare: Math.max(0, cap - attached) } : {}
|
|
60095
|
+
};
|
|
60096
|
+
}));
|
|
60097
|
+
if (election.kind === "none") return {
|
|
60098
|
+
kind: "default-pool",
|
|
60099
|
+
reason: "all-devices-at-cap",
|
|
60100
|
+
eligibleKeys
|
|
60101
|
+
};
|
|
60102
|
+
return {
|
|
60103
|
+
kind: "selected",
|
|
60104
|
+
deviceKey: election.key,
|
|
60105
|
+
reason: "share",
|
|
60106
|
+
tieBreak: null,
|
|
60107
|
+
eligibleKeys,
|
|
60108
|
+
tiedKeys: [election.key]
|
|
60109
|
+
};
|
|
60110
|
+
}
|
|
58749
60111
|
const balancerInput = {
|
|
58750
60112
|
nodes: consideredKeys.map((k) => deviceAsLoad(k, loadByKey.get(k))),
|
|
58751
60113
|
preferredAgent: pin,
|
|
@@ -58851,7 +60213,7 @@ var DeviceConfigContributions = class {
|
|
|
58851
60213
|
issue: configIssue
|
|
58852
60214
|
} });
|
|
58853
60215
|
this.deps.topology.refreshNodeCapabilities().then(() => this.deps.redispatchAllActiveCameras("ingest-owner-changed")).catch((err) => {
|
|
58854
|
-
this.deps.ctx().logger.warn("ingest-owner-changed re-dispatch failed", { meta: { error: errMsg$
|
|
60216
|
+
this.deps.ctx().logger.warn("ingest-owner-changed re-dispatch failed", { meta: { error: errMsg$2(err) } });
|
|
58855
60217
|
});
|
|
58856
60218
|
}
|
|
58857
60219
|
this.deps.clearOnSettingsChange();
|
|
@@ -59305,10 +60667,25 @@ function rootRefusal(input, droppedRoots) {
|
|
|
59305
60667
|
droppedRoots
|
|
59306
60668
|
};
|
|
59307
60669
|
}
|
|
60670
|
+
/**
|
|
60671
|
+
* Whose provisioning an attach applies. The elected device when there is one.
|
|
60672
|
+
* With NO elected device the runner's default pool serves the camera; when the
|
|
60673
|
+
* node has exactly one eligible device the balancer returns
|
|
60674
|
+
* `single-eligible-device` and that default pool IS that device (a Mac's
|
|
60675
|
+
* `coreml:all`, an N100 with only its iGPU on), so its model pin and the
|
|
60676
|
+
* camera's override stored under its key apply (D745 — before, the empty key
|
|
60677
|
+
* was read, which no store writes, and the pin was silently never run).
|
|
60678
|
+
* Otherwise `''`: the node's folded settings with no hardware fields.
|
|
60679
|
+
*/
|
|
60680
|
+
function provisioningKeyFor(input) {
|
|
60681
|
+
if (input.deviceKey !== void 0) return input.deviceKey;
|
|
60682
|
+
const eligibleKeys = Object.keys(input.eligible);
|
|
60683
|
+
return eligibleKeys.length === 1 ? eligibleKeys[0] ?? "" : "";
|
|
60684
|
+
}
|
|
59308
60685
|
/** Steps 3 of the module header: provision the resolved structure. */
|
|
59309
60686
|
async function provisionAttachPipeline(input, roster, resolved, store) {
|
|
59310
60687
|
const { deviceId, nodeId, deviceKey } = input;
|
|
59311
|
-
const electedKey =
|
|
60688
|
+
const electedKey = provisioningKeyFor(input);
|
|
59312
60689
|
const rosterKeys = (roster ?? []).map((d) => d.deviceKey);
|
|
59313
60690
|
const sources = await store.readDeviceProvisioning(deviceId, nodeId, [...new Set([electedKey, ...rosterKeys])]);
|
|
59314
60691
|
const elected = sources.get(electedKey);
|
|
@@ -59916,7 +61293,7 @@ var PlacementService = class {
|
|
|
59916
61293
|
async releaseCamera(input) {
|
|
59917
61294
|
const current = this.deps.ledger.getAssignment(input.deviceId);
|
|
59918
61295
|
if (current) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
|
|
59919
|
-
const msg = errMsg$
|
|
61296
|
+
const msg = errMsg$2(err);
|
|
59920
61297
|
this.deps.ctx().logger.debug("releaseCamera detach failed", {
|
|
59921
61298
|
tags: { deviceId: input.deviceId },
|
|
59922
61299
|
meta: { error: msg }
|
|
@@ -59944,7 +61321,7 @@ var PlacementService = class {
|
|
|
59944
61321
|
pipelineNodeId: input.agentNodeId,
|
|
59945
61322
|
[PREFERRED_AGENT_SETTING]: null
|
|
59946
61323
|
}).catch((err) => {
|
|
59947
|
-
const msg = errMsg$
|
|
61324
|
+
const msg = errMsg$2(err);
|
|
59948
61325
|
ctx.logger.warn("assignPipeline: failed to persist pin", {
|
|
59949
61326
|
tags: { deviceId: input.deviceId },
|
|
59950
61327
|
meta: { error: msg }
|
|
@@ -59976,7 +61353,7 @@ var PlacementService = class {
|
|
|
59976
61353
|
}
|
|
59977
61354
|
const current = this.deps.ledger.getAssignment(input.deviceId);
|
|
59978
61355
|
if (current && current.agentNodeId !== input.agentNodeId) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
|
|
59979
|
-
const msg = errMsg$
|
|
61356
|
+
const msg = errMsg$2(err);
|
|
59980
61357
|
ctx.logger.debug("assignPipeline detach-old failed", {
|
|
59981
61358
|
tags: { deviceId: input.deviceId },
|
|
59982
61359
|
meta: { error: msg }
|
|
@@ -60001,7 +61378,7 @@ var PlacementService = class {
|
|
|
60001
61378
|
await ctx.settings?.writeDeviceStore(input.deviceId, { pipelineDeviceKey: clearing ? "auto" : input.deviceKey }).catch((err) => {
|
|
60002
61379
|
ctx.logger.warn("setPipelineDevicePin: failed to persist device pin", {
|
|
60003
61380
|
tags: { deviceId: input.deviceId },
|
|
60004
|
-
meta: { error: errMsg$
|
|
61381
|
+
meta: { error: errMsg$2(err) }
|
|
60005
61382
|
});
|
|
60006
61383
|
});
|
|
60007
61384
|
const sessionNode = this.deps.activeSessionNode(input.deviceId);
|
|
@@ -60024,7 +61401,7 @@ var PlacementService = class {
|
|
|
60024
61401
|
[PREFERRED_AGENT_SETTING]: null,
|
|
60025
61402
|
pipelineNodeId: "auto"
|
|
60026
61403
|
}).catch((err) => {
|
|
60027
|
-
const msg = errMsg$
|
|
61404
|
+
const msg = errMsg$2(err);
|
|
60028
61405
|
ctx.logger.warn("unassignPipeline: failed to clear pin", {
|
|
60029
61406
|
tags: { deviceId: input.deviceId },
|
|
60030
61407
|
meta: { error: msg }
|
|
@@ -60045,7 +61422,7 @@ var PlacementService = class {
|
|
|
60045
61422
|
else {
|
|
60046
61423
|
const targetNodeId = decision.agentNodeId;
|
|
60047
61424
|
if (current && current.agentNodeId !== targetNodeId) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
|
|
60048
|
-
const msg = errMsg$
|
|
61425
|
+
const msg = errMsg$2(err);
|
|
60049
61426
|
ctx.logger.debug("unassignPipeline detach-old failed", {
|
|
60050
61427
|
tags: { deviceId: input.deviceId },
|
|
60051
61428
|
meta: { error: msg }
|
|
@@ -60065,7 +61442,7 @@ var PlacementService = class {
|
|
|
60065
61442
|
}
|
|
60066
61443
|
}
|
|
60067
61444
|
if (current) await this.detachOn(current.agentNodeId, input.deviceId).catch((err) => {
|
|
60068
|
-
const msg = errMsg$
|
|
61445
|
+
const msg = errMsg$2(err);
|
|
60069
61446
|
ctx.logger.debug("unassignPipeline detach failed", {
|
|
60070
61447
|
tags: { deviceId: input.deviceId },
|
|
60071
61448
|
meta: { error: msg }
|
|
@@ -60084,7 +61461,7 @@ var PlacementService = class {
|
|
|
60084
61461
|
async runAutoRebalance() {
|
|
60085
61462
|
if (!this.deps.ctxIfReady()) return;
|
|
60086
61463
|
await this.deps.reconcilePlacementFromRunners().catch((err) => {
|
|
60087
|
-
this.deps.ctx().logger.debug("auto-rebalance: attachment reconcile failed", { meta: { error: errMsg$
|
|
61464
|
+
this.deps.ctx().logger.debug("auto-rebalance: attachment reconcile failed", { meta: { error: errMsg$2(err) } });
|
|
60088
61465
|
});
|
|
60089
61466
|
if (this.autoRebalanceInFlight) return;
|
|
60090
61467
|
if (this.deps.topology.enabledNodes.length < 2) return;
|
|
@@ -60093,7 +61470,7 @@ var PlacementService = class {
|
|
|
60093
61470
|
const { migrated } = await this.rebalance({ minImprovement: AUTO_REBALANCE_MIN_IMPROVEMENT });
|
|
60094
61471
|
if (migrated > 0) this.deps.ctx().logger.info("auto-rebalance: corrected placement drift", { meta: { migrated } });
|
|
60095
61472
|
} catch (err) {
|
|
60096
|
-
this.deps.ctx().logger.debug("auto-rebalance pass failed", { meta: { error: errMsg$
|
|
61473
|
+
this.deps.ctx().logger.debug("auto-rebalance pass failed", { meta: { error: errMsg$2(err) } });
|
|
60097
61474
|
} finally {
|
|
60098
61475
|
this.autoRebalanceInFlight = false;
|
|
60099
61476
|
}
|
|
@@ -60165,7 +61542,7 @@ var PlacementService = class {
|
|
|
60165
61542
|
const targetConfig = withAttachPipeline(config, placement.pipeline);
|
|
60166
61543
|
if (current) {
|
|
60167
61544
|
await this.detachOn(current.agentNodeId, deviceId).catch((err) => {
|
|
60168
|
-
const msg = errMsg$
|
|
61545
|
+
const msg = errMsg$2(err);
|
|
60169
61546
|
ctx.logger.debug("rebalance detach-old failed", {
|
|
60170
61547
|
tags: { deviceId },
|
|
60171
61548
|
meta: { error: msg }
|
|
@@ -60570,7 +61947,7 @@ var NodeLifecycleHandler = class {
|
|
|
60570
61947
|
nodeId: decision.agentNodeId
|
|
60571
61948
|
} });
|
|
60572
61949
|
} catch (err) {
|
|
60573
|
-
const msg = errMsg$
|
|
61950
|
+
const msg = errMsg$2(err);
|
|
60574
61951
|
this.deps.logger.error("Failover: attach failed", {
|
|
60575
61952
|
tags: {
|
|
60576
61953
|
deviceId,
|
|
@@ -60600,7 +61977,7 @@ var NodeLifecycleHandler = class {
|
|
|
60600
61977
|
this.deps.schedulePendingRetry();
|
|
60601
61978
|
if (failoverPolicy.onReconnect === "rebalance") {
|
|
60602
61979
|
await this.deps.rebalance().catch((err) => {
|
|
60603
|
-
const msg = errMsg$
|
|
61980
|
+
const msg = errMsg$2(err);
|
|
60604
61981
|
this.deps.logger.error("rebalance after reconnect failed", { meta: { error: msg } });
|
|
60605
61982
|
});
|
|
60606
61983
|
return;
|
|
@@ -60626,7 +62003,7 @@ var NodeLifecycleHandler = class {
|
|
|
60626
62003
|
this.deps.ledger.recordAssignment(deviceId, nodeId, "failover", false);
|
|
60627
62004
|
restored++;
|
|
60628
62005
|
} catch (err) {
|
|
60629
|
-
const msg = errMsg$
|
|
62006
|
+
const msg = errMsg$2(err);
|
|
60630
62007
|
this.deps.logger.error("restore attach failed", {
|
|
60631
62008
|
tags: {
|
|
60632
62009
|
deviceId,
|
|
@@ -60720,7 +62097,7 @@ var NodeLifecycleHandler = class {
|
|
|
60720
62097
|
deviceId,
|
|
60721
62098
|
nodeId
|
|
60722
62099
|
},
|
|
60723
|
-
meta: { error: errMsg$
|
|
62100
|
+
meta: { error: errMsg$2(err) }
|
|
60724
62101
|
});
|
|
60725
62102
|
}
|
|
60726
62103
|
this.deps.logger.info("readiness: detection-pipeline reseeded agent defaults + redispatched cameras", {
|
|
@@ -60734,7 +62111,7 @@ var NodeLifecycleHandler = class {
|
|
|
60734
62111
|
this.reattachStaleEmpties(nodeId).catch((err) => {
|
|
60735
62112
|
this.deps.logger.debug("readiness deferred sweep failed", {
|
|
60736
62113
|
tags: { nodeId },
|
|
60737
|
-
meta: { error: errMsg$
|
|
62114
|
+
meta: { error: errMsg$2(err) }
|
|
60738
62115
|
});
|
|
60739
62116
|
});
|
|
60740
62117
|
}, 3e3);
|
|
@@ -60745,7 +62122,7 @@ var NodeLifecycleHandler = class {
|
|
|
60745
62122
|
tags: { nodeId },
|
|
60746
62123
|
meta: {
|
|
60747
62124
|
epoch: t.epoch,
|
|
60748
|
-
error: errMsg$
|
|
62125
|
+
error: errMsg$2(err)
|
|
60749
62126
|
}
|
|
60750
62127
|
});
|
|
60751
62128
|
}
|
|
@@ -60824,7 +62201,7 @@ var NodeLifecycleHandler = class {
|
|
|
60824
62201
|
deviceId,
|
|
60825
62202
|
nodeId
|
|
60826
62203
|
},
|
|
60827
|
-
meta: { error: errMsg$
|
|
62204
|
+
meta: { error: errMsg$2(err) }
|
|
60828
62205
|
});
|
|
60829
62206
|
}
|
|
60830
62207
|
}
|
|
@@ -61146,7 +62523,7 @@ var NodeStressController = class NodeStressController {
|
|
|
61146
62523
|
if (this.timer) return;
|
|
61147
62524
|
this.timer = setInterval(() => {
|
|
61148
62525
|
this.sweep().catch((err) => {
|
|
61149
|
-
this.deps.logger.warn("node-stress sweep failed", { meta: { error: errMsg$
|
|
62526
|
+
this.deps.logger.warn("node-stress sweep failed", { meta: { error: errMsg$2(err) } });
|
|
61150
62527
|
});
|
|
61151
62528
|
}, NodeStressController.SWEEP_INTERVAL_MS);
|
|
61152
62529
|
}
|
|
@@ -61319,7 +62696,7 @@ var NodeStressController = class NodeStressController {
|
|
|
61319
62696
|
meta: {
|
|
61320
62697
|
from: plan.fromNodeId,
|
|
61321
62698
|
to: plan.toNodeId,
|
|
61322
|
-
error: errMsg$
|
|
62699
|
+
error: errMsg$2(err)
|
|
61323
62700
|
}
|
|
61324
62701
|
});
|
|
61325
62702
|
} finally {
|
|
@@ -62016,7 +63393,7 @@ var NodeTopologyService = class {
|
|
|
62016
63393
|
try {
|
|
62017
63394
|
blob = await this.deps.readAgentSettingsMap();
|
|
62018
63395
|
} catch (err) {
|
|
62019
|
-
this.deps.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg$
|
|
63396
|
+
this.deps.logger.warn("refreshNodeCapabilities: agent settings read failed", { meta: { error: errMsg$2(err) } });
|
|
62020
63397
|
}
|
|
62021
63398
|
const nodeIds = new Set(["hub"]);
|
|
62022
63399
|
for (const nodeId of Object.keys(blob)) if (typeof nodeId === "string" && nodeId.length > 0 && !nodeId.includes("/")) nodeIds.add(nodeId);
|
|
@@ -62190,7 +63567,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62190
63567
|
const unsubInferenceResult = deps.eventBus.subscribe({ category: EventCategory.PipelineInferenceResult }, (event) => {
|
|
62191
63568
|
if (!isEvent(event, EventCategory.PipelineInferenceResult)) return;
|
|
62192
63569
|
deps.handleInferenceResult(event.data).catch((err) => {
|
|
62193
|
-
const msg = errMsg$
|
|
63570
|
+
const msg = errMsg$2(err);
|
|
62194
63571
|
deps.logger.error("PipelineInferenceResult post-processing failed", {
|
|
62195
63572
|
tags: { deviceId: event.data.deviceId },
|
|
62196
63573
|
meta: { error: msg }
|
|
@@ -62209,7 +63586,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62209
63586
|
deps.logger.info("received onProfileSlotsChanged", { tags: { deviceId } });
|
|
62210
63587
|
deps.handleDeviceRegistered(deviceId).catch((err) => {
|
|
62211
63588
|
if (isTornDown) return;
|
|
62212
|
-
const msg = errMsg$
|
|
63589
|
+
const msg = errMsg$2(err);
|
|
62213
63590
|
deps.logger.error("handleDeviceRegistered failed", {
|
|
62214
63591
|
tags: { deviceId },
|
|
62215
63592
|
meta: { error: msg }
|
|
@@ -62221,7 +63598,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62221
63598
|
const unsubDeviceUnregistered = deps.eventBus.subscribe({ category: EventCategory.DeviceUnregistered }, (event) => {
|
|
62222
63599
|
const { deviceId } = event.data;
|
|
62223
63600
|
deps.handleDeviceUnregistered(deviceId).catch((err) => {
|
|
62224
|
-
const msg = errMsg$
|
|
63601
|
+
const msg = errMsg$2(err);
|
|
62225
63602
|
deps.logger.error("handleDeviceUnregistered failed", {
|
|
62226
63603
|
tags: { deviceId },
|
|
62227
63604
|
meta: { error: msg }
|
|
@@ -62234,7 +63611,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62234
63611
|
deps.cameraSensors.requestCamera(deviceId, "settings-updated");
|
|
62235
63612
|
deps.handleSettingsChanged(deviceId).catch((err) => {
|
|
62236
63613
|
if (isTornDown) return;
|
|
62237
|
-
const msg = errMsg$
|
|
63614
|
+
const msg = errMsg$2(err);
|
|
62238
63615
|
deps.logger.error("handleSettingsChanged failed", {
|
|
62239
63616
|
tags: { deviceId },
|
|
62240
63617
|
meta: { error: msg }
|
|
@@ -62244,7 +63621,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62244
63621
|
const unsubCameraUpdated = deps.eventBus.subscribe({ category: EventCategory.PipelineCameraUpdated }, (event) => {
|
|
62245
63622
|
if (!isEvent(event, EventCategory.PipelineCameraUpdated)) return;
|
|
62246
63623
|
deps.handlePipelineCameraUpdated(event.data.deviceId, event.data.config).catch((err) => {
|
|
62247
|
-
const msg = errMsg$
|
|
63624
|
+
const msg = errMsg$2(err);
|
|
62248
63625
|
deps.logger.error("handlePipelineCameraUpdated failed", {
|
|
62249
63626
|
tags: { deviceId: event.data.deviceId },
|
|
62250
63627
|
meta: { error: msg }
|
|
@@ -62257,7 +63634,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62257
63634
|
if (!(reason === "wrapper-activated" || reason === "wrapper-deactivated") || !(capName === "detection-pipeline" || capName === "audio-analysis" || capName === "motion-detection")) return;
|
|
62258
63635
|
deps.handleSettingsChanged(deviceId).catch((err) => {
|
|
62259
63636
|
if (isTornDown) return;
|
|
62260
|
-
const msg = errMsg$
|
|
63637
|
+
const msg = errMsg$2(err);
|
|
62261
63638
|
deps.logger.error("binding-changed re-dispatch failed", {
|
|
62262
63639
|
tags: { deviceId },
|
|
62263
63640
|
meta: {
|
|
@@ -62285,7 +63662,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62285
63662
|
tags: { deviceId },
|
|
62286
63663
|
meta: {
|
|
62287
63664
|
detected,
|
|
62288
|
-
error: errMsg$
|
|
63665
|
+
error: errMsg$2(err)
|
|
62289
63666
|
}
|
|
62290
63667
|
});
|
|
62291
63668
|
});
|
|
@@ -62332,7 +63709,7 @@ function wireOrchestratorSubscriptions(deps) {
|
|
|
62332
63709
|
tags: { deviceId },
|
|
62333
63710
|
meta: {
|
|
62334
63711
|
detected,
|
|
62335
|
-
error: errMsg$
|
|
63712
|
+
error: errMsg$2(err)
|
|
62336
63713
|
}
|
|
62337
63714
|
});
|
|
62338
63715
|
});
|
|
@@ -62598,7 +63975,7 @@ async function reattachAfterPackageZoneChange(deps, deviceId) {
|
|
|
62598
63975
|
deviceId,
|
|
62599
63976
|
nodeId
|
|
62600
63977
|
},
|
|
62601
|
-
meta: { error: errMsg$
|
|
63978
|
+
meta: { error: errMsg$2(err) }
|
|
62602
63979
|
});
|
|
62603
63980
|
}
|
|
62604
63981
|
}
|
|
@@ -62879,7 +64256,7 @@ async function resetRemovedModelPins(deps, trigger) {
|
|
|
62879
64256
|
} catch (err) {
|
|
62880
64257
|
deps.logger.warn("removed-model pin reset skipped — custom model registry unreadable", { meta: {
|
|
62881
64258
|
trigger,
|
|
62882
|
-
error: errMsg$
|
|
64259
|
+
error: errMsg$2(err)
|
|
62883
64260
|
} });
|
|
62884
64261
|
return [];
|
|
62885
64262
|
}
|
|
@@ -62898,7 +64275,7 @@ async function resetRemovedModelPins(deps, trigger) {
|
|
|
62898
64275
|
} catch (err) {
|
|
62899
64276
|
deps.logger.warn("removed-model pin reset failed", { meta: {
|
|
62900
64277
|
trigger,
|
|
62901
|
-
error: errMsg$
|
|
64278
|
+
error: errMsg$2(err)
|
|
62902
64279
|
} });
|
|
62903
64280
|
return [];
|
|
62904
64281
|
}
|
|
@@ -63054,7 +64431,6 @@ var StoredCameraPipelineForAgentSchema = object({
|
|
|
63054
64431
|
}).nullable()
|
|
63055
64432
|
});
|
|
63056
64433
|
var StoredCameraPipelineSettingsSchema = object({
|
|
63057
|
-
pinnedAgentNodeId: string().optional(),
|
|
63058
64434
|
stepToggles: record(string(), boolean()).optional(),
|
|
63059
64435
|
stepOverridesByDevice: record(string(), record(string(), record(string(), StoredCameraStepOverridePatchSchema))).optional(),
|
|
63060
64436
|
stepOverridesByAgent: record(string(), record(string(), LegacyCameraStepOverridePatchSchema)).optional(),
|
|
@@ -63257,6 +64633,21 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
63257
64633
|
});
|
|
63258
64634
|
}
|
|
63259
64635
|
/**
|
|
64636
|
+
* Rewrite the whole agent-settings map in ONE read-modify-write under
|
|
64637
|
+
* {@link blobWriteLock} — atomic across every node of the blob (Best
|
|
64638
|
+
* settings, D745, writes several nodes' devices at once). `transform` must
|
|
64639
|
+
* be pure: it may run on a fresh read.
|
|
64640
|
+
*/
|
|
64641
|
+
async rewriteAgentSettingsMap(transform) {
|
|
64642
|
+
await this.blobWriteLock.run(PipelineSettingsStore.AGENT_SETTINGS_LOCK_KEY, async () => {
|
|
64643
|
+
await this.agentSettingsState.set(transform(await this.readAgentSettingsMap()));
|
|
64644
|
+
});
|
|
64645
|
+
}
|
|
64646
|
+
/** Rewrite the whole camera-settings map in one durable `update()` (D745). */
|
|
64647
|
+
async rewriteCameraSettingsMap(transform) {
|
|
64648
|
+
await this.cameraSettingsState.update(transform);
|
|
64649
|
+
}
|
|
64650
|
+
/**
|
|
63260
64651
|
* Drop one agent's persisted settings entry. Returns whether an entry
|
|
63261
64652
|
* existed (`false` also when the store is already disposed — the locked
|
|
63262
64653
|
* section becomes a no-op then). Serialized via {@link blobWriteLock} for
|
|
@@ -63513,6 +64904,10 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
63513
64904
|
* epochs and operator resets — a bound, not a budget.
|
|
63514
64905
|
*/
|
|
63515
64906
|
static CATALOG_FENCE_REREADS = 3;
|
|
64907
|
+
/** The node's catalog, read through the cache (awaits the node when cold). */
|
|
64908
|
+
async readCatalogForNode(nodeId) {
|
|
64909
|
+
return this.getCatalogForAgent(nodeId);
|
|
64910
|
+
}
|
|
63516
64911
|
async getCatalogForAgent(nodeId) {
|
|
63517
64912
|
const api = this.deps.api();
|
|
63518
64913
|
if (!api) return null;
|
|
@@ -63530,7 +64925,7 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
63530
64925
|
this.catalogReadFailing.add(nodeId);
|
|
63531
64926
|
this.deps.logger.warn("node catalog read failed — placements on it wait or stay pending", {
|
|
63532
64927
|
tags: { nodeId },
|
|
63533
|
-
meta: { error: errMsg$
|
|
64928
|
+
meta: { error: errMsg$2(err) }
|
|
63534
64929
|
});
|
|
63535
64930
|
}
|
|
63536
64931
|
return null;
|
|
@@ -63617,7 +65012,7 @@ var PipelineSettingsStore = class PipelineSettingsStore {
|
|
|
63617
65012
|
tags: { deviceId },
|
|
63618
65013
|
meta: {
|
|
63619
65014
|
capName: "detection-pipeline",
|
|
63620
|
-
error: errMsg$
|
|
65015
|
+
error: errMsg$2(err)
|
|
63621
65016
|
}
|
|
63622
65017
|
});
|
|
63623
65018
|
return true;
|
|
@@ -64450,7 +65845,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64450
65845
|
try {
|
|
64451
65846
|
slots = await api.streamBroker.listAllProfileSlots.query();
|
|
64452
65847
|
} catch (err) {
|
|
64453
|
-
this.deps.logger.debug("dispatch reconcile skipped — stream-broker not ready", { meta: { error: errMsg$
|
|
65848
|
+
this.deps.logger.debug("dispatch reconcile skipped — stream-broker not ready", { meta: { error: errMsg$2(err) } });
|
|
64454
65849
|
return;
|
|
64455
65850
|
}
|
|
64456
65851
|
const desired = desiredDeviceIdsFromSlots(slots);
|
|
@@ -64477,7 +65872,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64477
65872
|
} catch (err) {
|
|
64478
65873
|
this.deps.logger.warn("dispatch reconcile: handleDeviceRegistered failed", {
|
|
64479
65874
|
tags: { deviceId },
|
|
64480
|
-
meta: { error: errMsg$
|
|
65875
|
+
meta: { error: errMsg$2(err) }
|
|
64481
65876
|
});
|
|
64482
65877
|
}
|
|
64483
65878
|
await this.reconcilePlacementFromRunners();
|
|
@@ -64505,7 +65900,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64505
65900
|
const devices = await api.deviceManager.listAll.query({ projection: "slim" });
|
|
64506
65901
|
return new Set(devices.map((device) => device.id));
|
|
64507
65902
|
} catch (err) {
|
|
64508
|
-
this.deps.logger.debug("dispatch reconcile: deviceManager.listAll failed", { meta: { error: errMsg$
|
|
65903
|
+
this.deps.logger.debug("dispatch reconcile: deviceManager.listAll failed", { meta: { error: errMsg$2(err) } });
|
|
64509
65904
|
return null;
|
|
64510
65905
|
}
|
|
64511
65906
|
}
|
|
@@ -64531,7 +65926,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64531
65926
|
} catch (err) {
|
|
64532
65927
|
this.deps.logger.warn("dispatch reconcile: teardown of vanished device failed", {
|
|
64533
65928
|
tags: { deviceId },
|
|
64534
|
-
meta: { error: errMsg$
|
|
65929
|
+
meta: { error: errMsg$2(err) }
|
|
64535
65930
|
});
|
|
64536
65931
|
}
|
|
64537
65932
|
}
|
|
@@ -64588,7 +65983,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64588
65983
|
this.deps.logger.debug("placement reconcile skipped node — getLocalCameras failed", {
|
|
64589
65984
|
tags: { nodeId },
|
|
64590
65985
|
meta: {
|
|
64591
|
-
error: outcome.kind === "rejected" ? errMsg$
|
|
65986
|
+
error: outcome.kind === "rejected" ? errMsg$2(outcome.error) : "budget exceeded",
|
|
64592
65987
|
ms
|
|
64593
65988
|
}
|
|
64594
65989
|
});
|
|
@@ -64618,7 +66013,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64618
66013
|
nodeId,
|
|
64619
66014
|
deviceId
|
|
64620
66015
|
},
|
|
64621
|
-
meta: { error: errMsg$
|
|
66016
|
+
meta: { error: errMsg$2(err) }
|
|
64622
66017
|
});
|
|
64623
66018
|
})));
|
|
64624
66019
|
}
|
|
@@ -64660,7 +66055,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64660
66055
|
this.deps.handleDeviceRegistered(deviceId).catch((err) => {
|
|
64661
66056
|
this.deps.logger.debug("slot-read retry: handleDeviceRegistered failed", {
|
|
64662
66057
|
tags: { deviceId },
|
|
64663
|
-
meta: { error: errMsg$
|
|
66058
|
+
meta: { error: errMsg$2(err) }
|
|
64664
66059
|
});
|
|
64665
66060
|
});
|
|
64666
66061
|
}, delay);
|
|
@@ -64738,7 +66133,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64738
66133
|
} catch (err) {
|
|
64739
66134
|
this.deps.logger.warn("pending retry: dispatchCamera failed", {
|
|
64740
66135
|
tags: { deviceId },
|
|
64741
|
-
meta: { error: errMsg$
|
|
66136
|
+
meta: { error: errMsg$2(err) }
|
|
64742
66137
|
});
|
|
64743
66138
|
stillPending++;
|
|
64744
66139
|
}
|
|
@@ -64816,7 +66211,7 @@ var ReconcileController = class ReconcileController {
|
|
|
64816
66211
|
await this.deps.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
|
|
64817
66212
|
this.deps.logger.debug("remote-health exhausted-detach failed", {
|
|
64818
66213
|
tags: { deviceId },
|
|
64819
|
-
meta: { error: errMsg$
|
|
66214
|
+
meta: { error: errMsg$2(err) }
|
|
64820
66215
|
});
|
|
64821
66216
|
});
|
|
64822
66217
|
return;
|
|
@@ -64839,14 +66234,14 @@ var ReconcileController = class ReconcileController {
|
|
|
64839
66234
|
await this.deps.detachOn(assignment.agentNodeId, deviceId).catch((err) => {
|
|
64840
66235
|
this.deps.logger.warn("remote-health detach failed", {
|
|
64841
66236
|
tags: { deviceId },
|
|
64842
|
-
meta: { error: errMsg$
|
|
66237
|
+
meta: { error: errMsg$2(err) }
|
|
64843
66238
|
});
|
|
64844
66239
|
});
|
|
64845
66240
|
this.deps.ledger.dropAssignment(deviceId, "silent");
|
|
64846
66241
|
await this.deps.redispatchCamera(cfg).catch((err) => {
|
|
64847
66242
|
this.deps.logger.warn("remote-health re-dispatch failed", {
|
|
64848
66243
|
tags: { deviceId },
|
|
64849
|
-
meta: { error: errMsg$
|
|
66244
|
+
meta: { error: errMsg$2(err) }
|
|
64850
66245
|
});
|
|
64851
66246
|
});
|
|
64852
66247
|
}
|
|
@@ -65021,7 +66416,7 @@ var OccupancyRecheckController = class {
|
|
|
65021
66416
|
tags: { deviceId },
|
|
65022
66417
|
meta: {
|
|
65023
66418
|
deviceId,
|
|
65024
|
-
error: errMsg$
|
|
66419
|
+
error: errMsg$2(err)
|
|
65025
66420
|
}
|
|
65026
66421
|
});
|
|
65027
66422
|
});
|
|
@@ -65155,7 +66550,7 @@ var OccupancyRecheckController = class {
|
|
|
65155
66550
|
tags: { deviceId },
|
|
65156
66551
|
meta: {
|
|
65157
66552
|
deviceId,
|
|
65158
|
-
error: errMsg$
|
|
66553
|
+
error: errMsg$2(err)
|
|
65159
66554
|
}
|
|
65160
66555
|
});
|
|
65161
66556
|
});
|
|
@@ -65210,7 +66605,7 @@ var OccupancyRecheckController = class {
|
|
|
65210
66605
|
meta: {
|
|
65211
66606
|
deviceId,
|
|
65212
66607
|
nodeId,
|
|
65213
|
-
error: errMsg$
|
|
66608
|
+
error: errMsg$2(err),
|
|
65214
66609
|
kind
|
|
65215
66610
|
}
|
|
65216
66611
|
});
|
|
@@ -65219,7 +66614,7 @@ var OccupancyRecheckController = class {
|
|
|
65219
66614
|
}
|
|
65220
66615
|
const answer = await this.deps.requestBurstOn(nodeId, deviceId, frames, reasonTag).catch((err) => ({
|
|
65221
66616
|
started: false,
|
|
65222
|
-
error: errMsg$
|
|
66617
|
+
error: errMsg$2(err)
|
|
65223
66618
|
}));
|
|
65224
66619
|
if (!answer.started) {
|
|
65225
66620
|
const refusedBecause = "refusedBecause" in answer ? answer.refusedBecause : void 0;
|
|
@@ -65267,7 +66662,7 @@ var OccupancyRecheckController = class {
|
|
|
65267
66662
|
meta: {
|
|
65268
66663
|
deviceId,
|
|
65269
66664
|
nodeId,
|
|
65270
|
-
error: errMsg$
|
|
66665
|
+
error: errMsg$2(err)
|
|
65271
66666
|
}
|
|
65272
66667
|
});
|
|
65273
66668
|
});
|
|
@@ -65324,7 +66719,7 @@ var OccupancyRecheckController = class {
|
|
|
65324
66719
|
meta: {
|
|
65325
66720
|
deviceId,
|
|
65326
66721
|
nodeId: burst.nodeId,
|
|
65327
|
-
error: errMsg$
|
|
66722
|
+
error: errMsg$2(err)
|
|
65328
66723
|
}
|
|
65329
66724
|
});
|
|
65330
66725
|
});
|
|
@@ -65539,7 +66934,15 @@ var TtlMemo = class {
|
|
|
65539
66934
|
//#endregion
|
|
65540
66935
|
//#region src/detection-session-registry.ts
|
|
65541
66936
|
var DetectionSessionRegistry = class {
|
|
66937
|
+
onChange;
|
|
65542
66938
|
sessions = /* @__PURE__ */ new Map();
|
|
66939
|
+
/**
|
|
66940
|
+
* @param onChange called after every register / deregister / clear with the
|
|
66941
|
+
* running set — feeds the placement statistics (D748). Never on a no-op.
|
|
66942
|
+
*/
|
|
66943
|
+
constructor(onChange = () => {}) {
|
|
66944
|
+
this.onChange = onChange;
|
|
66945
|
+
}
|
|
65543
66946
|
register(deviceId, nodeId, startedAt, triggeredBy, deviceKey) {
|
|
65544
66947
|
const existing = this.sessions.get(deviceId);
|
|
65545
66948
|
if (existing?.teardownTimer) clearTimeout(existing.teardownTimer);
|
|
@@ -65552,6 +66955,7 @@ var DetectionSessionRegistry = class {
|
|
|
65552
66955
|
...deviceKey ? { deviceKey } : {}
|
|
65553
66956
|
};
|
|
65554
66957
|
this.sessions.set(deviceId, session);
|
|
66958
|
+
this.onChange(this.list());
|
|
65555
66959
|
return session;
|
|
65556
66960
|
}
|
|
65557
66961
|
get(deviceId) {
|
|
@@ -65565,6 +66969,7 @@ var DetectionSessionRegistry = class {
|
|
|
65565
66969
|
if (!session) return void 0;
|
|
65566
66970
|
if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
65567
66971
|
this.sessions.delete(deviceId);
|
|
66972
|
+
this.onChange(this.list());
|
|
65568
66973
|
return session;
|
|
65569
66974
|
}
|
|
65570
66975
|
setTeardownTimer(deviceId, timer) {
|
|
@@ -65587,9 +66992,39 @@ var DetectionSessionRegistry = class {
|
|
|
65587
66992
|
}
|
|
65588
66993
|
clear() {
|
|
65589
66994
|
for (const session of this.sessions.values()) if (session.teardownTimer) clearTimeout(session.teardownTimer);
|
|
66995
|
+
const hadSessions = this.sessions.size > 0;
|
|
65590
66996
|
this.sessions.clear();
|
|
66997
|
+
if (hadSessions) this.onChange([]);
|
|
65591
66998
|
}
|
|
65592
66999
|
};
|
|
67000
|
+
//#endregion
|
|
67001
|
+
//#region src/session-share-placement.ts
|
|
67002
|
+
function nodeSaturated(load) {
|
|
67003
|
+
return load.devices.length > 0 && load.devices.every((d) => isDeviceSaturated(d));
|
|
67004
|
+
}
|
|
67005
|
+
function electNodeByShare(input) {
|
|
67006
|
+
return electByShare(input.loads.filter((l) => l.nodeId.length > 0 && input.eligible.includes(l.nodeId)).map((load) => {
|
|
67007
|
+
const cap = input.caps[load.nodeId];
|
|
67008
|
+
const hasCap = cap !== null && cap !== void 0 && cap > 0;
|
|
67009
|
+
const weight = input.weights[load.nodeId];
|
|
67010
|
+
return {
|
|
67011
|
+
key: load.nodeId,
|
|
67012
|
+
share: weight !== void 0 && weight > 0 ? weight : 1,
|
|
67013
|
+
running: input.runningByNode.get(load.nodeId) ?? 0,
|
|
67014
|
+
capped: hasCap && load.attachedCameras >= cap,
|
|
67015
|
+
saturated: nodeSaturated(load),
|
|
67016
|
+
...hasCap ? { spare: Math.max(0, cap - load.attachedCameras) } : {}
|
|
67017
|
+
};
|
|
67018
|
+
}));
|
|
67019
|
+
}
|
|
67020
|
+
/** Parallel sessions per node: sessions and standing cameras, each camera once. */
|
|
67021
|
+
function countRunningByNode(placements) {
|
|
67022
|
+
const seen = /* @__PURE__ */ new Map();
|
|
67023
|
+
for (const p of placements) if (!seen.has(p.deviceId)) seen.set(p.deviceId, p.nodeId);
|
|
67024
|
+
const out = /* @__PURE__ */ new Map();
|
|
67025
|
+
for (const nodeId of seen.values()) out.set(nodeId, (out.get(nodeId) ?? 0) + 1);
|
|
67026
|
+
return out;
|
|
67027
|
+
}
|
|
65593
67028
|
/**
|
|
65594
67029
|
* Minimum time a detection session stays open, whatever closes it.
|
|
65595
67030
|
*
|
|
@@ -65702,7 +67137,15 @@ var SessionDispatchController = class {
|
|
|
65702
67137
|
* (not `private`) — see the class doc for the two production/test reasons
|
|
65703
67138
|
* `index.ts` reaches through it directly.
|
|
65704
67139
|
*/
|
|
65705
|
-
sessionRegistry = new DetectionSessionRegistry()
|
|
67140
|
+
sessionRegistry = new DetectionSessionRegistry((sessions) => this.deps.placementStats?.noteSessions(sessions.map((s) => ({
|
|
67141
|
+
nodeId: s.nodeId,
|
|
67142
|
+
deviceKey: s.deviceKey ?? ""
|
|
67143
|
+
}))));
|
|
67144
|
+
/**
|
|
67145
|
+
* `deviceId|deviceKey` of device pins the session path has already said it
|
|
67146
|
+
* ignored — one line per stale pin, not one per session (D748).
|
|
67147
|
+
*/
|
|
67148
|
+
reportedStalePins = /* @__PURE__ */ new Set();
|
|
65706
67149
|
/**
|
|
65707
67150
|
* Per-device async mutex for session dispatch/teardown critical sections —
|
|
65708
67151
|
* backed by the generic `KeyedAsyncLock` (mirrors the audio controller's
|
|
@@ -65981,7 +67424,7 @@ var SessionDispatchController = class {
|
|
|
65981
67424
|
deviceId,
|
|
65982
67425
|
nodeId: session.nodeId
|
|
65983
67426
|
},
|
|
65984
|
-
meta: { error: errMsg$
|
|
67427
|
+
meta: { error: errMsg$2(err) }
|
|
65985
67428
|
});
|
|
65986
67429
|
});
|
|
65987
67430
|
this.deps.dropAssignment(deviceId, "silent");
|
|
@@ -66163,6 +67606,76 @@ var SessionDispatchController = class {
|
|
|
66163
67606
|
});
|
|
66164
67607
|
}
|
|
66165
67608
|
/**
|
|
67609
|
+
* The session's node. `least-loaded` (and any node pin, under either
|
|
67610
|
+
* policy) is `balance()` exactly as before. `share` holds the node shares
|
|
67611
|
+
* of PARALLEL sessions (D748): sessions now running and standing cameras
|
|
67612
|
+
* count toward their node, and a node at its cap or drowning yields —
|
|
67613
|
+
* logged with the reason, and counted against the node in the stats.
|
|
67614
|
+
*/
|
|
67615
|
+
electSessionNode(input) {
|
|
67616
|
+
const balanced = () => balance({
|
|
67617
|
+
nodes: input.loads,
|
|
67618
|
+
preferredAgent: input.preferredAgent,
|
|
67619
|
+
nodeCaps: input.nodeCaps,
|
|
67620
|
+
weights: input.weights,
|
|
67621
|
+
eligibleNodes: input.eligible
|
|
67622
|
+
});
|
|
67623
|
+
if (input.policy !== "share" || input.preferredAgent) return balanced();
|
|
67624
|
+
const running = countRunningByNode([...this.sessionRegistry.list().map((s) => ({
|
|
67625
|
+
deviceId: s.deviceId,
|
|
67626
|
+
nodeId: s.nodeId
|
|
67627
|
+
})), ...this.deps.ledger.listAssignments().map((a) => ({
|
|
67628
|
+
deviceId: a.deviceId,
|
|
67629
|
+
nodeId: a.agentNodeId
|
|
67630
|
+
}))]);
|
|
67631
|
+
const election = electNodeByShare({
|
|
67632
|
+
loads: input.loads,
|
|
67633
|
+
eligible: input.eligible,
|
|
67634
|
+
weights: input.weights,
|
|
67635
|
+
caps: input.nodeCaps,
|
|
67636
|
+
runningByNode: running
|
|
67637
|
+
});
|
|
67638
|
+
if (election.kind === "none") return balanced();
|
|
67639
|
+
if (election.reason === "yield") {
|
|
67640
|
+
this.deps.placementStats?.noteYield(election.turnKey);
|
|
67641
|
+
this.deps.logger.info("detection session: share turn yielded — the node had no room", {
|
|
67642
|
+
tags: {
|
|
67643
|
+
deviceId: input.deviceId,
|
|
67644
|
+
nodeId: election.key
|
|
67645
|
+
},
|
|
67646
|
+
meta: {
|
|
67647
|
+
turnNodeId: election.turnKey,
|
|
67648
|
+
reason: election.yieldReason
|
|
67649
|
+
}
|
|
67650
|
+
});
|
|
67651
|
+
}
|
|
67652
|
+
const load = input.loads.find((l) => l.nodeId === election.key);
|
|
67653
|
+
return {
|
|
67654
|
+
kind: "assigned",
|
|
67655
|
+
agentNodeId: election.key,
|
|
67656
|
+
reason: "capacity",
|
|
67657
|
+
score: load ? load.attachedCameras + load.queueDepthTotal : 0
|
|
67658
|
+
};
|
|
67659
|
+
}
|
|
67660
|
+
/**
|
|
67661
|
+
* Say ONCE that a camera's device pin names a device this node does not
|
|
67662
|
+
* offer (disabled, absent): the session path ignores it, and the Load
|
|
67663
|
+
* balancing tab offers to clear it. Never cleared here (D748).
|
|
67664
|
+
*/
|
|
67665
|
+
noteStalePin(deviceId, pinnedDeviceKey, enabledDevices) {
|
|
67666
|
+
if (pinnedDeviceKey === null || pinnedDeviceKey in enabledDevices) return;
|
|
67667
|
+
const key = `${deviceId}|${pinnedDeviceKey}`;
|
|
67668
|
+
if (this.reportedStalePins.has(key)) return;
|
|
67669
|
+
this.reportedStalePins.add(key);
|
|
67670
|
+
this.deps.logger.warn("device pin ignored — the pinned device is not enabled on the elected node", {
|
|
67671
|
+
tags: { deviceId },
|
|
67672
|
+
meta: {
|
|
67673
|
+
pin: pinnedDeviceKey,
|
|
67674
|
+
enabled: Object.keys(enabledDevices).join(",")
|
|
67675
|
+
}
|
|
67676
|
+
});
|
|
67677
|
+
}
|
|
67678
|
+
/**
|
|
66166
67679
|
* Dispatch a session-time detection attach for an on-motion camera whose
|
|
66167
67680
|
* `decideSessionAction` returned `'dispatch'`. Builds a session
|
|
66168
67681
|
* `RunnerCameraConfig` the same way `startDetection` does, but forces
|
|
@@ -66198,7 +67711,7 @@ var SessionDispatchController = class {
|
|
|
66198
67711
|
};
|
|
66199
67712
|
if (reconcileBudgetExceeded) log.warn("detection session: placement census overran its budget — placing without it", { meta: { budgetMs: 500 } });
|
|
66200
67713
|
else if (censusOutcome.kind === "rejected") log.warn("detection session: placement census failed — placing on the ledger as it is", { meta: {
|
|
66201
|
-
error: errMsg$
|
|
67714
|
+
error: errMsg$2(censusOutcome.error),
|
|
66202
67715
|
reconcileMs
|
|
66203
67716
|
} });
|
|
66204
67717
|
const balanceStartedAt = Date.now();
|
|
@@ -66213,12 +67726,17 @@ var SessionDispatchController = class {
|
|
|
66213
67726
|
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
66214
67727
|
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
66215
67728
|
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
66216
|
-
const
|
|
66217
|
-
|
|
67729
|
+
const nodeCaps = await this.nodeCaps.get(CLUSTER_KEY);
|
|
67730
|
+
const weights = await this.nodeWeights.get(CLUSTER_KEY);
|
|
67731
|
+
const policy = this.deps.sessionPlacement?.() ?? "least-loaded";
|
|
67732
|
+
const decision = this.electSessionNode({
|
|
67733
|
+
deviceId,
|
|
67734
|
+
policy,
|
|
67735
|
+
loads,
|
|
66218
67736
|
preferredAgent,
|
|
66219
|
-
nodeCaps
|
|
66220
|
-
weights
|
|
66221
|
-
|
|
67737
|
+
nodeCaps,
|
|
67738
|
+
weights,
|
|
67739
|
+
eligible
|
|
66222
67740
|
});
|
|
66223
67741
|
if (!decision || decision.kind !== "assigned") {
|
|
66224
67742
|
log.warn("detection session: no eligible node — will retry on next motion", { meta: { reason: decision?.kind === "pending" ? decision.reason : "no-runners-online" } });
|
|
@@ -66231,12 +67749,14 @@ var SessionDispatchController = class {
|
|
|
66231
67749
|
const targetLoad = loads.find((n) => n.nodeId === targetNodeId);
|
|
66232
67750
|
const deviceCaps = await this.inferenceDeviceCaps.get(targetNodeId);
|
|
66233
67751
|
const pinnedDeviceKey = await this.deps.readPipelineDevicePin(deviceId);
|
|
67752
|
+
this.noteStalePin(deviceId, pinnedDeviceKey, enabledDevices);
|
|
66234
67753
|
const selection = pickInferenceDevice({
|
|
66235
67754
|
eligible: enabledDevices,
|
|
66236
67755
|
loads: targetLoad?.devices ?? [],
|
|
66237
67756
|
caps: deviceCaps,
|
|
66238
|
-
pinnedDeviceKey,
|
|
66239
|
-
rotation: this.deps.inferenceRotation
|
|
67757
|
+
pinnedDeviceKey: pinnedDeviceKey !== null && pinnedDeviceKey in enabledDevices ? pinnedDeviceKey : null,
|
|
67758
|
+
rotation: this.deps.inferenceRotation,
|
|
67759
|
+
policy
|
|
66240
67760
|
});
|
|
66241
67761
|
const deviceKey = selectedDeviceKey(selection);
|
|
66242
67762
|
log.info("inference device selection", {
|
|
@@ -66324,6 +67844,7 @@ var SessionDispatchController = class {
|
|
|
66324
67844
|
attachedAt = Date.now();
|
|
66325
67845
|
this.deps.recordAssignment(deviceId, targetNodeId, decision.reason, decision.reason === "manual");
|
|
66326
67846
|
this.sessionRegistry.register(deviceId, targetNodeId, Date.now(), triggeredBy, deviceKey);
|
|
67847
|
+
this.deps.placementStats?.noteStart(targetNodeId, deviceKey ?? "");
|
|
66327
67848
|
if (this.sessionRefusals.delete(deviceId)) log.info("detection session refusal cleared — a session opened", {
|
|
66328
67849
|
tags: {
|
|
66329
67850
|
deviceId,
|
|
@@ -66392,7 +67913,7 @@ var SessionDispatchController = class {
|
|
|
66392
67913
|
await this.resolveSessionLifecycleLocked(deviceId, reason);
|
|
66393
67914
|
}).catch((err) => {
|
|
66394
67915
|
log.warn("detection session: lifecycle re-check failed", { meta: {
|
|
66395
|
-
error: errMsg$
|
|
67916
|
+
error: errMsg$2(err),
|
|
66396
67917
|
reason
|
|
66397
67918
|
} });
|
|
66398
67919
|
});
|
|
@@ -66497,7 +68018,7 @@ var SessionDispatchController = class {
|
|
|
66497
68018
|
log.warn("detection session: teardown detach failed", {
|
|
66498
68019
|
tags: { nodeId: session.nodeId },
|
|
66499
68020
|
meta: {
|
|
66500
|
-
error: errMsg$
|
|
68021
|
+
error: errMsg$2(err),
|
|
66501
68022
|
reason
|
|
66502
68023
|
}
|
|
66503
68024
|
});
|
|
@@ -66604,7 +68125,7 @@ var SessionDispatchController = class {
|
|
|
66604
68125
|
deviceId,
|
|
66605
68126
|
nodeId: session.nodeId
|
|
66606
68127
|
},
|
|
66607
|
-
meta: { error: errMsg$
|
|
68128
|
+
meta: { error: errMsg$2(err) }
|
|
66608
68129
|
});
|
|
66609
68130
|
});
|
|
66610
68131
|
this.sessionRegistry.deregister(deviceId);
|
|
@@ -66625,7 +68146,7 @@ var SessionDispatchController = class {
|
|
|
66625
68146
|
deviceId,
|
|
66626
68147
|
nodeId: watchNodeId
|
|
66627
68148
|
},
|
|
66628
|
-
meta: { error: errMsg$
|
|
68149
|
+
meta: { error: errMsg$2(err) }
|
|
66629
68150
|
});
|
|
66630
68151
|
});
|
|
66631
68152
|
this.motionWatchByDevice.delete(deviceId);
|
|
@@ -66808,7 +68329,7 @@ async function sweepMirror(deps, hydrator, cameraIds) {
|
|
|
66808
68329
|
tags: { deviceId },
|
|
66809
68330
|
meta: {
|
|
66810
68331
|
mirror: hydrator.mirror,
|
|
66811
|
-
error: errMsg$
|
|
68332
|
+
error: errMsg$2(err)
|
|
66812
68333
|
}
|
|
66813
68334
|
});
|
|
66814
68335
|
return "threw";
|
|
@@ -66848,7 +68369,7 @@ async function readCameraIds(deps) {
|
|
|
66848
68369
|
lastError = err;
|
|
66849
68370
|
}
|
|
66850
68371
|
if (Date.now() >= deadline) {
|
|
66851
|
-
deps.logger.warn("zones mirror boot hydration SKIPPED — camera list never became readable; mirror-only consumers keep whatever the last write left", { meta: { error: lastError === null ? "ctx.api unavailable" : errMsg$
|
|
68372
|
+
deps.logger.warn("zones mirror boot hydration SKIPPED — camera list never became readable; mirror-only consumers keep whatever the last write left", { meta: { error: lastError === null ? "ctx.api unavailable" : errMsg$2(lastError) } });
|
|
66852
68373
|
return null;
|
|
66853
68374
|
}
|
|
66854
68375
|
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
@@ -67469,6 +68990,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67469
68990
|
readGlobalSettings: () => globalSettings,
|
|
67470
68991
|
onCensus: (entries) => {
|
|
67471
68992
|
motionSliceGate?.noteCensus(entries);
|
|
68993
|
+
deps.onRunnerCensus(entries.flatMap((e) => e.load === null ? [] : [e.load]));
|
|
67472
68994
|
providerSliceGate?.refreshIfStale("runner-census");
|
|
67473
68995
|
}
|
|
67474
68996
|
});
|
|
@@ -67495,7 +69017,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67495
69017
|
} catch (err) {
|
|
67496
69018
|
return {
|
|
67497
69019
|
kind: "unknown",
|
|
67498
|
-
reason: errMsg$
|
|
69020
|
+
reason: errMsg$2(err)
|
|
67499
69021
|
};
|
|
67500
69022
|
}
|
|
67501
69023
|
};
|
|
@@ -67580,18 +69102,19 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67580
69102
|
readGlobalSettings: () => globalSettings
|
|
67581
69103
|
});
|
|
67582
69104
|
await NodeStressMoveLedger.declare(deps.ctx().api.settingsStore).then(() => nodeStress.hydrate()).catch((err) => {
|
|
67583
|
-
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$
|
|
69105
|
+
deps.ctx().logger.warn("node-stress move history unavailable — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
67584
69106
|
});
|
|
67585
69107
|
await NodeStressLts.declare(deps.ctx().api.settingsStore).catch((err) => {
|
|
67586
|
-
deps.ctx().logger.warn("node-stress statistics unavailable — no baseline this boot", { meta: { error: errMsg$
|
|
69108
|
+
deps.ctx().logger.warn("node-stress statistics unavailable — no baseline this boot", { meta: { error: errMsg$2(err) } });
|
|
67587
69109
|
});
|
|
67588
69110
|
const nodeStressLtsTimer = setInterval(() => {
|
|
67589
69111
|
nodeStressLts.flushDue().catch((err) => {
|
|
67590
|
-
deps.ctx().logger.warn("node-stress statistics flush failed", { meta: { error: errMsg$
|
|
69112
|
+
deps.ctx().logger.warn("node-stress statistics flush failed", { meta: { error: errMsg$2(err) } });
|
|
67591
69113
|
});
|
|
67592
69114
|
}, 6e4);
|
|
67593
69115
|
nodeStress.start();
|
|
67594
69116
|
const inferenceRotation = new RoundRobinInferenceDeviceRotation();
|
|
69117
|
+
const readSessionPlacement = () => parseSessionPlacementPolicy(globalSettings?.[SESSION_PLACEMENT_SETTING_KEY]);
|
|
67595
69118
|
/**
|
|
67596
69119
|
* Can the watchdog's audio stage ever receive a signal for this camera?
|
|
67597
69120
|
* Only `absent` — POSITIVE evidence of no audio track — disarms it.
|
|
@@ -67675,6 +69198,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67675
69198
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
67676
69199
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
67677
69200
|
inferenceRotation,
|
|
69201
|
+
sessionPlacement: () => readSessionPlacement(),
|
|
69202
|
+
placementStats: deps.placementStats,
|
|
67678
69203
|
logger: deps.ctx().logger
|
|
67679
69204
|
});
|
|
67680
69205
|
const deviceConfig = new DeviceConfigContributions({
|
|
@@ -67737,7 +69262,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67737
69262
|
} catch (err) {
|
|
67738
69263
|
return {
|
|
67739
69264
|
kind: "unreadable",
|
|
67740
|
-
error: errMsg$
|
|
69265
|
+
error: errMsg$2(err)
|
|
67741
69266
|
};
|
|
67742
69267
|
}
|
|
67743
69268
|
};
|
|
@@ -67771,7 +69296,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67771
69296
|
} catch (err) {
|
|
67772
69297
|
return {
|
|
67773
69298
|
kind: "unreadable",
|
|
67774
|
-
error: errMsg$
|
|
69299
|
+
error: errMsg$2(err)
|
|
67775
69300
|
};
|
|
67776
69301
|
}
|
|
67777
69302
|
},
|
|
@@ -67802,7 +69327,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67802
69327
|
} catch (err) {
|
|
67803
69328
|
return {
|
|
67804
69329
|
kind: "unreadable",
|
|
67805
|
-
error: errMsg$
|
|
69330
|
+
error: errMsg$2(err)
|
|
67806
69331
|
};
|
|
67807
69332
|
}
|
|
67808
69333
|
},
|
|
@@ -67815,7 +69340,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67815
69340
|
} catch (err) {
|
|
67816
69341
|
return {
|
|
67817
69342
|
kind: "unreadable",
|
|
67818
|
-
error: errMsg$
|
|
69343
|
+
error: errMsg$2(err)
|
|
67819
69344
|
};
|
|
67820
69345
|
}
|
|
67821
69346
|
},
|
|
@@ -67886,7 +69411,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67886
69411
|
deps.applyRuntimeSettings(globalSettings);
|
|
67887
69412
|
await topology.refreshNodeCapabilities();
|
|
67888
69413
|
} catch (err) {
|
|
67889
|
-
const msg = errMsg$
|
|
69414
|
+
const msg = errMsg$2(err);
|
|
67890
69415
|
deps.ctx().logger.warn("orchestrator settings load failed — using defaults", { meta: { error: msg } });
|
|
67891
69416
|
}
|
|
67892
69417
|
const rawNodeId = deps.ctx().kernel.localNodeId ?? "hub";
|
|
@@ -67927,7 +69452,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67927
69452
|
audio.markNodeOffline(nodeId);
|
|
67928
69453
|
deps.forgetInferenceDeviceProbe(nodeId);
|
|
67929
69454
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
67930
|
-
const msg = errMsg$
|
|
69455
|
+
const msg = errMsg$2(err);
|
|
67931
69456
|
deps.ctx().logger.error("failover after disconnect failed", {
|
|
67932
69457
|
tags: { nodeId },
|
|
67933
69458
|
meta: { error: msg }
|
|
@@ -67962,7 +69487,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67962
69487
|
deps.subscribeToAudioAnalyzerReadiness(nodeId);
|
|
67963
69488
|
}
|
|
67964
69489
|
nodeLifecycle.handleNodeConnect(nodeId).catch((err) => {
|
|
67965
|
-
const msg = errMsg$
|
|
69490
|
+
const msg = errMsg$2(err);
|
|
67966
69491
|
deps.ctx().logger.error("reconnect handling failed", {
|
|
67967
69492
|
tags: { nodeId },
|
|
67968
69493
|
meta: { error: msg }
|
|
@@ -67981,7 +69506,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
67981
69506
|
meta: { capName }
|
|
67982
69507
|
});
|
|
67983
69508
|
nodeLifecycle.handleNodeDisconnect(nodeId).catch((err) => {
|
|
67984
|
-
const msg = errMsg$
|
|
69509
|
+
const msg = errMsg$2(err);
|
|
67985
69510
|
deps.ctx().logger.error("failover after cap-down failed", {
|
|
67986
69511
|
tags: { nodeId },
|
|
67987
69512
|
meta: {
|
|
@@ -68044,7 +69569,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68044
69569
|
doneFlag: settingsStore.bindingsMigrationDoneState,
|
|
68045
69570
|
logger: deps.ctx().logger
|
|
68046
69571
|
}).catch((err) => {
|
|
68047
|
-
deps.ctxIfReady()?.logger.warn("bindings migration failed — will retry on next boot", { meta: { error: errMsg$
|
|
69572
|
+
deps.ctxIfReady()?.logger.warn("bindings migration failed — will retry on next boot", { meta: { error: errMsg$2(err) } });
|
|
68048
69573
|
});
|
|
68049
69574
|
migrateStepGating({
|
|
68050
69575
|
readCameraSettings: () => settingsStore.readCameraSettingsMap(),
|
|
@@ -68057,7 +69582,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68057
69582
|
for (let i = 0; deps.ctxIfReady()?.api == null && i < 30; i++) await new Promise((r) => setTimeout(r, 200));
|
|
68058
69583
|
if (deps.ctxIfReady() !== null) await deps.resetRemovedModelPins("boot");
|
|
68059
69584
|
}).catch((err) => {
|
|
68060
|
-
deps.ctxIfReady()?.logger.warn("step-gating / per-node-device migration failed — retry next boot", { meta: { error: errMsg$
|
|
69585
|
+
deps.ctxIfReady()?.logger.warn("step-gating / per-node-device migration failed — retry next boot", { meta: { error: errMsg$2(err) } });
|
|
68061
69586
|
});
|
|
68062
69587
|
const zoneRulesProvider = new ZoneRulesProvider({
|
|
68063
69588
|
logger: deps.ctx().logger.child("zone-rules"),
|
|
@@ -68099,7 +69624,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68099
69624
|
deviceId,
|
|
68100
69625
|
nodeId: assignment.agentNodeId
|
|
68101
69626
|
},
|
|
68102
|
-
meta: { error: errMsg$
|
|
69627
|
+
meta: { error: errMsg$2(err) }
|
|
68103
69628
|
});
|
|
68104
69629
|
});
|
|
68105
69630
|
}
|
|
@@ -68115,7 +69640,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68115
69640
|
}],
|
|
68116
69641
|
logger: deps.ctx().logger.child("zones")
|
|
68117
69642
|
}).catch((err) => {
|
|
68118
|
-
deps.ctxIfReady()?.logger.warn("zones mirror boot hydration failed", { meta: { error: errMsg$
|
|
69643
|
+
deps.ctxIfReady()?.logger.warn("zones mirror boot hydration failed", { meta: { error: errMsg$2(err) } });
|
|
68119
69644
|
});
|
|
68120
69645
|
const unsubOrchestratorSubscriptions = wireOrchestratorSubscriptions({
|
|
68121
69646
|
eventBus: deps.ctx().eventBus,
|
|
@@ -68176,10 +69701,10 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68176
69701
|
deviceSettingsSchema: () => deps.deviceSettingsSchema()
|
|
68177
69702
|
});
|
|
68178
69703
|
await detectionWiring.hydrateFeaturesMirror().catch((err) => {
|
|
68179
|
-
deps.ctx().logger.warn("device-features mirror not restored — this boot starts cold", { meta: { error: errMsg$
|
|
69704
|
+
deps.ctx().logger.warn("device-features mirror not restored — this boot starts cold", { meta: { error: errMsg$2(err) } });
|
|
68180
69705
|
});
|
|
68181
69706
|
cameraSensors.start().catch((err) => {
|
|
68182
|
-
deps.ctx().logger.warn("camera sensors: boot refresh failed — waiting for the next lifecycle event", { meta: { error: errMsg$
|
|
69707
|
+
deps.ctx().logger.warn("camera sensors: boot refresh failed — waiting for the next lifecycle event", { meta: { error: errMsg$2(err) } });
|
|
68183
69708
|
});
|
|
68184
69709
|
return {
|
|
68185
69710
|
ledger,
|
|
@@ -68208,7 +69733,8 @@ async function buildOrchestratorControllers(deps) {
|
|
|
68208
69733
|
unsubAgentOffline,
|
|
68209
69734
|
unsubOrchestratorSubscriptions,
|
|
68210
69735
|
pendingRetryTimer,
|
|
68211
|
-
autoRebalanceTimer
|
|
69736
|
+
autoRebalanceTimer,
|
|
69737
|
+
sessionPlacement: () => readSessionPlacement()
|
|
68212
69738
|
};
|
|
68213
69739
|
}
|
|
68214
69740
|
//#endregion
|
|
@@ -68253,6 +69779,20 @@ function buildGlobalSettingsSections(options) {
|
|
|
68253
69779
|
title: "Load Balancer",
|
|
68254
69780
|
tab: "pipeline",
|
|
68255
69781
|
fields: [
|
|
69782
|
+
{
|
|
69783
|
+
key: SESSION_PLACEMENT_SETTING_KEY,
|
|
69784
|
+
type: "select",
|
|
69785
|
+
label: "Session placement",
|
|
69786
|
+
description: "How a new detection session picks its node and device. Least-loaded = the node with the fewest sessions per unit of weight; an idle node always wins the tie, so weights only matter once nodes are busy. Honour shares = the share of PARALLEL sessions is held to the node and device weights, and a node at its cap yields to a free one. Moving a weight slider never changes this policy.",
|
|
69787
|
+
options: [{
|
|
69788
|
+
value: "least-loaded",
|
|
69789
|
+
label: "Least-loaded"
|
|
69790
|
+
}, {
|
|
69791
|
+
value: "share",
|
|
69792
|
+
label: "Honour shares"
|
|
69793
|
+
}],
|
|
69794
|
+
default: DEFAULT_SESSION_PLACEMENT_POLICY
|
|
69795
|
+
},
|
|
68256
69796
|
{
|
|
68257
69797
|
key: "fpsMinThreshold",
|
|
68258
69798
|
type: "slider",
|
|
@@ -68811,6 +70351,959 @@ var pipelineOrchestratorWidgets = [{
|
|
|
68811
70351
|
defaultRows: 4
|
|
68812
70352
|
}];
|
|
68813
70353
|
//#endregion
|
|
70354
|
+
//#region src/best-settings/plan-diff.ts
|
|
70355
|
+
/**
|
|
70356
|
+
* The Best settings writes, as pure transforms of the orchestrator's two
|
|
70357
|
+
* settings blobs (D745). The plan runs them on the current blobs to SHOW the
|
|
70358
|
+
* diff; the apply runs the same functions inside each blob's read-modify-write.
|
|
70359
|
+
*
|
|
70360
|
+
* What changes — and nothing else:
|
|
70361
|
+
* - node level (`agentSettings[node].inferenceDevices[deviceKey].steps`): the
|
|
70362
|
+
* object-detection `modelId` of every recommended device is set; the
|
|
70363
|
+
* `modelId` pin of every other in-scope step is removed (each runs its
|
|
70364
|
+
* step default again); every model-bound threshold (and its binding key) is
|
|
70365
|
+
* removed from every device's step `settings`, so the running model's own
|
|
70366
|
+
* defaults apply (D738–D741). `enabled`, `weight`, `maxSessions`,
|
|
70367
|
+
* `jumpDeviceKey` and every other setting are kept.
|
|
70368
|
+
* - camera level, scoped to the touched nodes: the same `modelId` pins and
|
|
70369
|
+
* model-bound thresholds are removed from `stepOverridesByDevice[node]` and
|
|
70370
|
+
* from the wholesale `pipelineByAgent[node]` snapshot (recursively). A
|
|
70371
|
+
* wholesale step keeps `enabled`, `children`, `jumpDeviceKey` and its other
|
|
70372
|
+
* settings: the attach provisions the node pin over the snapshot
|
|
70373
|
+
* (`applyDeviceProvisioning`: override ?? base), so once the snapshot pin
|
|
70374
|
+
* is gone the node's recommended model runs.
|
|
70375
|
+
* - never: `stepToggles`, devices' enablement, pins of
|
|
70376
|
+
* embedding steps (cluster-scoped, changing one forces a re-index).
|
|
70377
|
+
*/
|
|
70378
|
+
/** The root detector step. */
|
|
70379
|
+
var DETECTOR_STEP_ID = "object-detection";
|
|
70380
|
+
/** Steps whose `modelId` pin Best settings never touches. */
|
|
70381
|
+
function isModelPinKept(stepId) {
|
|
70382
|
+
return stepId.endsWith("-embedding");
|
|
70383
|
+
}
|
|
70384
|
+
function boundKeysFor(input, stepId) {
|
|
70385
|
+
return input.boundKeysByStep[stepId] ?? [];
|
|
70386
|
+
}
|
|
70387
|
+
/** The model-bound keys present in `settings` (values or bindings), and the rest. */
|
|
70388
|
+
function stripBound(settings, boundKeys) {
|
|
70389
|
+
if (settings === void 0) return {
|
|
70390
|
+
kept: void 0,
|
|
70391
|
+
removed: []
|
|
70392
|
+
};
|
|
70393
|
+
const kept = { ...settings };
|
|
70394
|
+
const removed = [];
|
|
70395
|
+
for (const key of boundKeys) for (const k of [key, modelBindingKey(key)]) if (k in kept) {
|
|
70396
|
+
delete kept[k];
|
|
70397
|
+
removed.push(k);
|
|
70398
|
+
}
|
|
70399
|
+
return {
|
|
70400
|
+
kept,
|
|
70401
|
+
removed: removed.toSorted()
|
|
70402
|
+
};
|
|
70403
|
+
}
|
|
70404
|
+
function pinInScope(stepId) {
|
|
70405
|
+
return !isModelPinKept(stepId);
|
|
70406
|
+
}
|
|
70407
|
+
/** Rebuild one stored step without the in-scope pin and bound keys. */
|
|
70408
|
+
function scrubStep(stepId, step, boundKeys, targetModel) {
|
|
70409
|
+
const before = step.modelId ?? null;
|
|
70410
|
+
let modelId = step.modelId;
|
|
70411
|
+
if (stepId === "object-detection") {
|
|
70412
|
+
if (targetModel !== void 0) modelId = targetModel;
|
|
70413
|
+
} else if (pinInScope(stepId)) modelId = void 0;
|
|
70414
|
+
const { kept, removed } = stripBound(step.settings, boundKeys);
|
|
70415
|
+
const after = modelId ?? null;
|
|
70416
|
+
const next = {
|
|
70417
|
+
...modelId !== void 0 ? { modelId } : {},
|
|
70418
|
+
...kept !== void 0 && Object.keys(kept).length > 0 ? { settings: kept } : {},
|
|
70419
|
+
...step.jumpDeviceKey !== void 0 ? { jumpDeviceKey: step.jumpDeviceKey } : {}
|
|
70420
|
+
};
|
|
70421
|
+
return {
|
|
70422
|
+
next: Object.keys(next).length > 0 ? next : null,
|
|
70423
|
+
pin: before !== after ? {
|
|
70424
|
+
stepId,
|
|
70425
|
+
before,
|
|
70426
|
+
after
|
|
70427
|
+
} : null,
|
|
70428
|
+
cleared: removed
|
|
70429
|
+
};
|
|
70430
|
+
}
|
|
70431
|
+
/** The node-level transform. Returns the next blob and every change it made. */
|
|
70432
|
+
function applyNodeBestSettings(blob, input) {
|
|
70433
|
+
const next = { ...blob };
|
|
70434
|
+
const changes = [];
|
|
70435
|
+
for (const nodeId of [...input.touchedNodes].toSorted()) {
|
|
70436
|
+
const entry = blob[nodeId];
|
|
70437
|
+
const targets = new Map((input.targets.get(nodeId) ?? []).map((t) => [t.deviceKey, t]));
|
|
70438
|
+
const stored = entry?.inferenceDevices ?? {};
|
|
70439
|
+
const deviceKeys = [...new Set([...Object.keys(stored), ...targets.keys()])].toSorted();
|
|
70440
|
+
const nextDevices = {};
|
|
70441
|
+
for (const deviceKey of deviceKeys) {
|
|
70442
|
+
const device = stored[deviceKey];
|
|
70443
|
+
const target = targets.get(deviceKey);
|
|
70444
|
+
const steps = { ...device?.steps };
|
|
70445
|
+
if (target !== void 0 && steps["object-detection"] === void 0) steps[DETECTOR_STEP_ID] = {};
|
|
70446
|
+
const nextSteps = {};
|
|
70447
|
+
const pins = [];
|
|
70448
|
+
const thresholds = [];
|
|
70449
|
+
for (const stepId of Object.keys(steps).toSorted()) {
|
|
70450
|
+
const step = steps[stepId];
|
|
70451
|
+
if (step === void 0) continue;
|
|
70452
|
+
const scrubbed = scrubStep(stepId, step, boundKeysFor(input, stepId), target?.modelId);
|
|
70453
|
+
if (scrubbed.next !== null) nextSteps[stepId] = scrubbed.next;
|
|
70454
|
+
if (scrubbed.pin !== null) pins.push(scrubbed.pin);
|
|
70455
|
+
if (scrubbed.cleared.length > 0) thresholds.push({
|
|
70456
|
+
stepId,
|
|
70457
|
+
keys: scrubbed.cleared
|
|
70458
|
+
});
|
|
70459
|
+
}
|
|
70460
|
+
const madeExplicit = device === void 0;
|
|
70461
|
+
nextDevices[deviceKey] = {
|
|
70462
|
+
...device !== void 0 ? (({ steps: _steps, ...rest }) => rest)(device) : { enabled: target?.effectiveEnabled ?? false },
|
|
70463
|
+
...Object.keys(nextSteps).length > 0 ? { steps: nextSteps } : {}
|
|
70464
|
+
};
|
|
70465
|
+
if (pins.length > 0 || thresholds.length > 0) changes.push({
|
|
70466
|
+
nodeId,
|
|
70467
|
+
deviceKey,
|
|
70468
|
+
madeExplicit,
|
|
70469
|
+
modelPins: pins,
|
|
70470
|
+
thresholds
|
|
70471
|
+
});
|
|
70472
|
+
}
|
|
70473
|
+
if (entry === void 0 && deviceKeys.length === 0) continue;
|
|
70474
|
+
next[nodeId] = {
|
|
70475
|
+
...entry ?? { maxCameras: null },
|
|
70476
|
+
...deviceKeys.length > 0 ? { inferenceDevices: nextDevices } : {}
|
|
70477
|
+
};
|
|
70478
|
+
}
|
|
70479
|
+
return {
|
|
70480
|
+
next,
|
|
70481
|
+
changes
|
|
70482
|
+
};
|
|
70483
|
+
}
|
|
70484
|
+
/** Scrub a wholesale snapshot tree; collects one clear per changed step. */
|
|
70485
|
+
function scrubTree(steps, input, onClear) {
|
|
70486
|
+
return steps.map((step) => {
|
|
70487
|
+
const children = step.children ? scrubTree(step.children, input, onClear) : void 0;
|
|
70488
|
+
const inScope = step.addonId === "object-detection" || pinInScope(step.addonId);
|
|
70489
|
+
const { kept, removed } = stripBound(step.settings, boundKeysFor(input, step.addonId));
|
|
70490
|
+
const dropPin = inScope && step.modelId !== void 0;
|
|
70491
|
+
if (dropPin || removed.length > 0) onClear(step.addonId, dropPin ? step.modelId ?? null : null, removed);
|
|
70492
|
+
const { modelId: _modelId, settings: _settings, children: _children, ...rest } = step;
|
|
70493
|
+
return {
|
|
70494
|
+
...rest,
|
|
70495
|
+
...!dropPin && step.modelId !== void 0 ? { modelId: step.modelId } : {},
|
|
70496
|
+
...kept !== void 0 && Object.keys(kept).length > 0 ? { settings: kept } : {},
|
|
70497
|
+
...children !== void 0 ? { children } : {}
|
|
70498
|
+
};
|
|
70499
|
+
});
|
|
70500
|
+
}
|
|
70501
|
+
/** The camera-level transform. Returns the next blob and every override it removed. */
|
|
70502
|
+
function applyCameraBestSettings(blob, input) {
|
|
70503
|
+
const next = {};
|
|
70504
|
+
const clears = [];
|
|
70505
|
+
for (const key of Object.keys(blob).toSorted((a, b) => Number(a) - Number(b))) {
|
|
70506
|
+
const camera = blob[key];
|
|
70507
|
+
if (camera === void 0) continue;
|
|
70508
|
+
const deviceId = Number(key);
|
|
70509
|
+
let nextCamera = camera;
|
|
70510
|
+
const byNode = camera.stepOverridesByDevice;
|
|
70511
|
+
if (byNode !== void 0) {
|
|
70512
|
+
const nextByNode = {};
|
|
70513
|
+
for (const [nodeId, byDevice] of Object.entries(byNode)) {
|
|
70514
|
+
if (!input.touchedNodes.has(nodeId)) {
|
|
70515
|
+
nextByNode[nodeId] = byDevice;
|
|
70516
|
+
continue;
|
|
70517
|
+
}
|
|
70518
|
+
const nextByDevice = {};
|
|
70519
|
+
for (const [deviceKey, byStep] of Object.entries(byDevice)) {
|
|
70520
|
+
const nextByStep = {};
|
|
70521
|
+
for (const [stepId, patch] of Object.entries(byStep)) {
|
|
70522
|
+
const inScope = stepId === "object-detection" || pinInScope(stepId);
|
|
70523
|
+
const { kept, removed } = stripBound(patch.settings, boundKeysFor(input, stepId));
|
|
70524
|
+
const dropPin = inScope && patch.modelId !== void 0;
|
|
70525
|
+
if (dropPin || removed.length > 0) clears.push({
|
|
70526
|
+
deviceId,
|
|
70527
|
+
source: "step-override",
|
|
70528
|
+
nodeId,
|
|
70529
|
+
deviceKey,
|
|
70530
|
+
stepId,
|
|
70531
|
+
modelIdBefore: dropPin ? patch.modelId ?? null : null,
|
|
70532
|
+
keys: removed
|
|
70533
|
+
});
|
|
70534
|
+
const rebuilt = {
|
|
70535
|
+
...!dropPin && patch.modelId !== void 0 ? { modelId: patch.modelId } : {},
|
|
70536
|
+
...kept !== void 0 && Object.keys(kept).length > 0 ? { settings: kept } : {}
|
|
70537
|
+
};
|
|
70538
|
+
if (Object.keys(rebuilt).length > 0) nextByStep[stepId] = rebuilt;
|
|
70539
|
+
}
|
|
70540
|
+
if (Object.keys(nextByStep).length > 0) nextByDevice[deviceKey] = nextByStep;
|
|
70541
|
+
}
|
|
70542
|
+
if (Object.keys(nextByDevice).length > 0) nextByNode[nodeId] = nextByDevice;
|
|
70543
|
+
}
|
|
70544
|
+
const { stepOverridesByDevice: _drop, ...rest } = nextCamera;
|
|
70545
|
+
nextCamera = {
|
|
70546
|
+
...rest,
|
|
70547
|
+
...Object.keys(nextByNode).length > 0 ? { stepOverridesByDevice: nextByNode } : {}
|
|
70548
|
+
};
|
|
70549
|
+
}
|
|
70550
|
+
const wholesale = camera.pipelineByAgent;
|
|
70551
|
+
if (wholesale !== void 0) {
|
|
70552
|
+
const nextWholesale = {};
|
|
70553
|
+
for (const [nodeId, snapshot] of Object.entries(wholesale)) {
|
|
70554
|
+
if (!input.touchedNodes.has(nodeId)) {
|
|
70555
|
+
nextWholesale[nodeId] = snapshot;
|
|
70556
|
+
continue;
|
|
70557
|
+
}
|
|
70558
|
+
nextWholesale[nodeId] = {
|
|
70559
|
+
...snapshot,
|
|
70560
|
+
steps: scrubTree(snapshot.steps, input, (stepId, modelIdBefore, keys) => {
|
|
70561
|
+
clears.push({
|
|
70562
|
+
deviceId,
|
|
70563
|
+
source: "wholesale",
|
|
70564
|
+
nodeId,
|
|
70565
|
+
deviceKey: null,
|
|
70566
|
+
stepId,
|
|
70567
|
+
modelIdBefore,
|
|
70568
|
+
keys
|
|
70569
|
+
});
|
|
70570
|
+
})
|
|
70571
|
+
};
|
|
70572
|
+
}
|
|
70573
|
+
nextCamera = {
|
|
70574
|
+
...nextCamera,
|
|
70575
|
+
pipelineByAgent: nextWholesale
|
|
70576
|
+
};
|
|
70577
|
+
}
|
|
70578
|
+
next[key] = nextCamera;
|
|
70579
|
+
}
|
|
70580
|
+
return {
|
|
70581
|
+
next,
|
|
70582
|
+
clears
|
|
70583
|
+
};
|
|
70584
|
+
}
|
|
70585
|
+
//#endregion
|
|
70586
|
+
//#region src/best-settings/best-settings-service.ts
|
|
70587
|
+
/**
|
|
70588
|
+
* Best settings (D745): gather the inputs, compute the plan, and apply exactly
|
|
70589
|
+
* the plan the operator confirmed.
|
|
70590
|
+
*
|
|
70591
|
+
* The plan is the recommender (`@camstack/types` best-settings) over every
|
|
70592
|
+
* runner node, plus the diff of the writes (`plan-diff.ts`) and the
|
|
70593
|
+
* per-camera detector thresholds the hub's executor would clear. Its
|
|
70594
|
+
* `planHash` names everything the apply writes; the apply recomputes the plan
|
|
70595
|
+
* and refuses a different hash, so a configuration change between the two is
|
|
70596
|
+
* shown to the operator again instead of being written over.
|
|
70597
|
+
*
|
|
70598
|
+
* Apply order, each phase reported:
|
|
70599
|
+
* 1. download every newly picked detector on its node FIRST — a node never
|
|
70600
|
+
* switches to a model it does not have (an N100 compile takes 5–25 s, a
|
|
70601
|
+
* Core ML package 40 MB); any failure stops the apply before a write;
|
|
70602
|
+
* 2. camera overrides (one camera-blob read-modify-write);
|
|
70603
|
+
* 3. per-camera detector thresholds (the hub executor's store);
|
|
70604
|
+
* 4. node model pins (one agent-blob read-modify-write);
|
|
70605
|
+
* 5. re-dispatch the cameras of every touched node.
|
|
70606
|
+
* A failure after phase 2 leaves "overrides cleared, previous node model",
|
|
70607
|
+
* which is a valid configuration (the defaults run); re-running converges.
|
|
70608
|
+
*/
|
|
70609
|
+
function errMsg(err) {
|
|
70610
|
+
return err instanceof Error ? err.message : String(err);
|
|
70611
|
+
}
|
|
70612
|
+
/** Canonical JSON (sorted keys) — the plan hash must not depend on key order. */
|
|
70613
|
+
function canonical(value) {
|
|
70614
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
|
70615
|
+
if (typeof value === "object" && value !== null) return `{${Object.entries(value).filter(([, v]) => v !== void 0).toSorted(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`;
|
|
70616
|
+
return JSON.stringify(value);
|
|
70617
|
+
}
|
|
70618
|
+
function formatsByModelOf(catalog) {
|
|
70619
|
+
const out = {};
|
|
70620
|
+
for (const slot of catalog?.slots ?? []) for (const addon of slot.addons) {
|
|
70621
|
+
if (addon.id !== "object-detection") continue;
|
|
70622
|
+
for (const model of addon.models) out[model.id] = MODEL_FORMATS.filter((f) => f in model.formats);
|
|
70623
|
+
}
|
|
70624
|
+
return out;
|
|
70625
|
+
}
|
|
70626
|
+
function boundKeysOf(catalogs) {
|
|
70627
|
+
const byStep = /* @__PURE__ */ new Map();
|
|
70628
|
+
for (const catalog of catalogs) for (const slot of catalog?.slots ?? []) for (const addon of slot.addons) {
|
|
70629
|
+
const keys = collectModelBoundKeys(addon.configSchema ?? []);
|
|
70630
|
+
if (keys.length === 0) continue;
|
|
70631
|
+
const set = byStep.get(addon.id) ?? /* @__PURE__ */ new Set();
|
|
70632
|
+
for (const k of keys) set.add(k);
|
|
70633
|
+
byStep.set(addon.id, set);
|
|
70634
|
+
}
|
|
70635
|
+
return Object.fromEntries([...byStep].map(([step, set]) => [step, [...set].toSorted()]));
|
|
70636
|
+
}
|
|
70637
|
+
/** Bounded fixed point of the hardware share suggestion (D748). */
|
|
70638
|
+
var SUGGEST_MAX_ROUNDS = 3;
|
|
70639
|
+
/** Integer shares summing to 100 (largest remainder). */
|
|
70640
|
+
function toPercent(weights) {
|
|
70641
|
+
const total = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
70642
|
+
const out = /* @__PURE__ */ new Map();
|
|
70643
|
+
if (total <= 0) return out;
|
|
70644
|
+
const raw = [...weights].map(([k, w]) => ({
|
|
70645
|
+
k,
|
|
70646
|
+
exact: w / total * 100
|
|
70647
|
+
}));
|
|
70648
|
+
let assigned = 0;
|
|
70649
|
+
for (const r of raw) {
|
|
70650
|
+
out.set(r.k, Math.floor(r.exact));
|
|
70651
|
+
assigned += Math.floor(r.exact);
|
|
70652
|
+
}
|
|
70653
|
+
const byRemainder = raw.toSorted((a, b) => b.exact - Math.floor(b.exact) - (a.exact - Math.floor(a.exact)) || a.k.localeCompare(b.k));
|
|
70654
|
+
for (let i = 0; i < 100 - assigned; i++) {
|
|
70655
|
+
const r = byRemainder[i % byRemainder.length];
|
|
70656
|
+
out.set(r.k, (out.get(r.k) ?? 0) + 1);
|
|
70657
|
+
}
|
|
70658
|
+
return out;
|
|
70659
|
+
}
|
|
70660
|
+
var BestSettingsService = class {
|
|
70661
|
+
deps;
|
|
70662
|
+
applying = false;
|
|
70663
|
+
constructor(deps) {
|
|
70664
|
+
this.deps = deps;
|
|
70665
|
+
}
|
|
70666
|
+
async plan(input) {
|
|
70667
|
+
return (await this.computePlan(input.headroom, input.shares)).plan;
|
|
70668
|
+
}
|
|
70669
|
+
/**
|
|
70670
|
+
* Shares proportional to each node's capacity at the models Best settings
|
|
70671
|
+
* would pick under those very shares (D748). Model and load depend on each
|
|
70672
|
+
* other, so it iterates from equal shares until the rounded shares stop
|
|
70673
|
+
* moving, at most {@link SUGGEST_MAX_ROUNDS} rounds. Per device it reports an
|
|
70674
|
+
* ADVISORY session cap — never written, never enforced.
|
|
70675
|
+
*/
|
|
70676
|
+
async suggestShares(input) {
|
|
70677
|
+
let shares = {
|
|
70678
|
+
nodes: {},
|
|
70679
|
+
devices: {}
|
|
70680
|
+
};
|
|
70681
|
+
let result = [];
|
|
70682
|
+
let cameraFps = 10;
|
|
70683
|
+
let rounds = 0;
|
|
70684
|
+
for (let round = 1; round <= SUGGEST_MAX_ROUNDS; round++) {
|
|
70685
|
+
rounds = round;
|
|
70686
|
+
const rec = await this.recommend(input.headroom, shares);
|
|
70687
|
+
cameraFps = rec.cameraFps;
|
|
70688
|
+
result = this.sharesFrom(rec, input.headroom);
|
|
70689
|
+
const next = {
|
|
70690
|
+
nodes: Object.fromEntries(result.flatMap((n) => n.share !== null && n.share > 0 ? [[n.nodeId, n.share]] : [])),
|
|
70691
|
+
devices: Object.fromEntries(result.map((n) => [n.nodeId, Object.fromEntries(n.devices.flatMap((d) => d.share > 0 ? [[d.deviceKey, d.share]] : []))]))
|
|
70692
|
+
};
|
|
70693
|
+
const settled = JSON.stringify(next) === JSON.stringify(shares);
|
|
70694
|
+
shares = next;
|
|
70695
|
+
if (settled) break;
|
|
70696
|
+
}
|
|
70697
|
+
return {
|
|
70698
|
+
headroom: input.headroom,
|
|
70699
|
+
cameraFps,
|
|
70700
|
+
iterations: rounds,
|
|
70701
|
+
nodes: [...result]
|
|
70702
|
+
};
|
|
70703
|
+
}
|
|
70704
|
+
sharesFrom(rec, headroom) {
|
|
70705
|
+
const capacityByNode = /* @__PURE__ */ new Map();
|
|
70706
|
+
const perNode = rec.recommendations.map((r) => {
|
|
70707
|
+
const devices = r.devices.filter((d) => d.enabled && d.available && d.chosen !== null).map((d) => {
|
|
70708
|
+
const fps = d.chosen?.fps ?? 0;
|
|
70709
|
+
return {
|
|
70710
|
+
d,
|
|
70711
|
+
fps,
|
|
70712
|
+
sessions: rec.cameraFps > 0 ? Math.floor(fps * (1 - headroom) / rec.cameraFps) : 0
|
|
70713
|
+
};
|
|
70714
|
+
});
|
|
70715
|
+
const capacity = devices.reduce((sum, x) => sum + x.sessions, 0);
|
|
70716
|
+
if (!r.skipped && capacity > 0) capacityByNode.set(r.nodeId, capacity);
|
|
70717
|
+
return {
|
|
70718
|
+
r,
|
|
70719
|
+
devices,
|
|
70720
|
+
capacity
|
|
70721
|
+
};
|
|
70722
|
+
});
|
|
70723
|
+
const nodePct = toPercent(capacityByNode);
|
|
70724
|
+
return perNode.map(({ r, devices, capacity }) => {
|
|
70725
|
+
const devicePct = toPercent(new Map(devices.map((x) => [x.d.deviceKey, Math.max(x.fps, 0)])));
|
|
70726
|
+
return {
|
|
70727
|
+
nodeId: r.nodeId,
|
|
70728
|
+
share: nodePct.get(r.nodeId) ?? null,
|
|
70729
|
+
capacitySessions: capacity,
|
|
70730
|
+
skippedReason: r.skipped ? r.reasons[0]?.kind ?? "skipped" : capacity > 0 ? null : "no-capacity",
|
|
70731
|
+
devices: devices.map((x) => ({
|
|
70732
|
+
deviceKey: x.d.deviceKey,
|
|
70733
|
+
share: devicePct.get(x.d.deviceKey) ?? 0,
|
|
70734
|
+
modelId: x.d.chosen?.modelId ?? null,
|
|
70735
|
+
fps: x.d.chosen?.fps ?? null,
|
|
70736
|
+
measured: x.d.chosen?.measured ?? false,
|
|
70737
|
+
suggestedMaxSessions: x.sessions
|
|
70738
|
+
}))
|
|
70739
|
+
};
|
|
70740
|
+
});
|
|
70741
|
+
}
|
|
70742
|
+
/**
|
|
70743
|
+
* Gather every node and recommend. `shares` (a what-if or the suggestion's
|
|
70744
|
+
* round) forces share placement with those shares overlaid on the stored
|
|
70745
|
+
* ones; absent, the STORED policy decides how on-motion load is charged.
|
|
70746
|
+
*/
|
|
70747
|
+
async recommend(headroom, shares) {
|
|
70748
|
+
const nodeIds = [...new Set(["hub", ...this.deps.runnerNodeIds()])].toSorted();
|
|
70749
|
+
const gathered = await Promise.all(nodeIds.map(async (nodeId) => {
|
|
70750
|
+
const [devices, hardware, catalog] = await Promise.all([
|
|
70751
|
+
this.deps.readNodeDevices(nodeId),
|
|
70752
|
+
this.deps.readHardware(nodeId).catch(() => null),
|
|
70753
|
+
this.deps.readCatalog(nodeId).catch(() => null)
|
|
70754
|
+
]);
|
|
70755
|
+
return {
|
|
70756
|
+
nodeId,
|
|
70757
|
+
devices,
|
|
70758
|
+
hardware,
|
|
70759
|
+
catalog
|
|
70760
|
+
};
|
|
70761
|
+
}));
|
|
70762
|
+
const deviceShare = (nodeId, key, stored) => shares?.devices[nodeId]?.[key] ?? stored;
|
|
70763
|
+
const nodes = gathered.map((g) => ({
|
|
70764
|
+
nodeId: g.nodeId,
|
|
70765
|
+
reachable: g.devices.reachable && g.catalog !== null,
|
|
70766
|
+
hardware: g.hardware,
|
|
70767
|
+
devices: g.devices.devices.map((d) => ({
|
|
70768
|
+
key: d.key,
|
|
70769
|
+
format: d.format,
|
|
70770
|
+
enabled: d.enabled,
|
|
70771
|
+
available: d.available
|
|
70772
|
+
})),
|
|
70773
|
+
formatsByModel: formatsByModelOf(g.catalog)
|
|
70774
|
+
}));
|
|
70775
|
+
const eligible = gathered.flatMap((g) => g.devices.reachable ? g.devices.devices.filter((d) => d.enabled && d.available).map((d) => ({
|
|
70776
|
+
nodeId: g.nodeId,
|
|
70777
|
+
deviceKey: d.key,
|
|
70778
|
+
weight: deviceShare(g.nodeId, d.key, d.weight)
|
|
70779
|
+
})) : []);
|
|
70780
|
+
const cameras = await this.deps.cameraLoads();
|
|
70781
|
+
const policy = shares ? "share" : this.deps.sessionPlacement();
|
|
70782
|
+
const demand = estimateDeviceDemand({
|
|
70783
|
+
cameras,
|
|
70784
|
+
eligible,
|
|
70785
|
+
peaks: this.deps.peaks(),
|
|
70786
|
+
...policy === "share" ? { shares: {
|
|
70787
|
+
nodes: {
|
|
70788
|
+
...await this.deps.readNodeShares(),
|
|
70789
|
+
...shares?.nodes
|
|
70790
|
+
},
|
|
70791
|
+
clusterPeak: this.deps.clusterPeak()
|
|
70792
|
+
} } : {}
|
|
70793
|
+
});
|
|
70794
|
+
const recommendations = nodes.map((node) => recommendNode({
|
|
70795
|
+
node,
|
|
70796
|
+
demand,
|
|
70797
|
+
headroom
|
|
70798
|
+
}));
|
|
70799
|
+
const cameraFps = cameras.reduce((m, c) => c.mode !== "disabled" && c.runsPipelineRoot ? Math.max(m, c.fps) : m, 0);
|
|
70800
|
+
return {
|
|
70801
|
+
gathered,
|
|
70802
|
+
nodes,
|
|
70803
|
+
recommendations,
|
|
70804
|
+
cameraFps: cameraFps > 0 ? cameraFps : 10
|
|
70805
|
+
};
|
|
70806
|
+
}
|
|
70807
|
+
async computePlan(headroom, shares) {
|
|
70808
|
+
const { gathered, nodes, recommendations } = await this.recommend(headroom, shares);
|
|
70809
|
+
const touchedNodes = new Set(recommendations.filter((r) => !r.skipped).map((r) => r.nodeId));
|
|
70810
|
+
const targets = /* @__PURE__ */ new Map();
|
|
70811
|
+
const formats = /* @__PURE__ */ new Map();
|
|
70812
|
+
for (const rec of recommendations) {
|
|
70813
|
+
if (rec.skipped) continue;
|
|
70814
|
+
const node = nodes.find((n) => n.nodeId === rec.nodeId);
|
|
70815
|
+
const list = [];
|
|
70816
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
70817
|
+
for (const d of rec.devices) {
|
|
70818
|
+
if (d.chosen === null) continue;
|
|
70819
|
+
list.push({
|
|
70820
|
+
deviceKey: d.deviceKey,
|
|
70821
|
+
modelId: d.chosen.modelId,
|
|
70822
|
+
effectiveEnabled: d.enabled
|
|
70823
|
+
});
|
|
70824
|
+
const format = node?.devices.find((x) => x.key === d.deviceKey)?.format;
|
|
70825
|
+
if (format !== void 0) byKey.set(d.deviceKey, format);
|
|
70826
|
+
}
|
|
70827
|
+
targets.set(rec.nodeId, list);
|
|
70828
|
+
formats.set(rec.nodeId, byKey);
|
|
70829
|
+
}
|
|
70830
|
+
const diffInput = {
|
|
70831
|
+
touchedNodes,
|
|
70832
|
+
targets,
|
|
70833
|
+
boundKeysByStep: boundKeysOf(gathered.map((g) => g.catalog))
|
|
70834
|
+
};
|
|
70835
|
+
const nodeChanges = applyNodeBestSettings(await this.deps.readAgentSettings(), diffInput).changes;
|
|
70836
|
+
const cameraClears = applyCameraBestSettings(await this.deps.readCameraSettings(), diffInput).clears;
|
|
70837
|
+
const warnings = [];
|
|
70838
|
+
let detectorStoreClears = [];
|
|
70839
|
+
if (touchedNodes.size > 0) try {
|
|
70840
|
+
const answer = await this.deps.detectorStore({ dryRun: true });
|
|
70841
|
+
detectorStoreClears = answer.entries.map((e) => ({
|
|
70842
|
+
deviceId: e.deviceId,
|
|
70843
|
+
keys: [...e.keys]
|
|
70844
|
+
}));
|
|
70845
|
+
if (answer.refused.length > 0) warnings.push({
|
|
70846
|
+
kind: "detector-store-unreadable",
|
|
70847
|
+
error: `${answer.refused.length} camera store(s) unreadable`
|
|
70848
|
+
});
|
|
70849
|
+
} catch (err) {
|
|
70850
|
+
warnings.push({
|
|
70851
|
+
kind: "detector-store-unreadable",
|
|
70852
|
+
error: errMsg(err)
|
|
70853
|
+
});
|
|
70854
|
+
}
|
|
70855
|
+
const skippedNodes = recommendations.filter((r) => r.skipped).map((r) => r.nodeId);
|
|
70856
|
+
if (detectorStoreClears.length > 0 && skippedNodes.length > 0) warnings.push({
|
|
70857
|
+
kind: "detector-store-cluster-wide",
|
|
70858
|
+
skippedNodes
|
|
70859
|
+
});
|
|
70860
|
+
for (const rec of recommendations) for (const d of rec.devices) if (d.accelerator === "coral" && d.enabled) warnings.push({
|
|
70861
|
+
kind: "coral-enabled",
|
|
70862
|
+
nodeId: rec.nodeId,
|
|
70863
|
+
deviceKey: d.deviceKey
|
|
70864
|
+
});
|
|
70865
|
+
return {
|
|
70866
|
+
plan: {
|
|
70867
|
+
planHash: (0, node_crypto.createHash)("sha256").update(canonical({
|
|
70868
|
+
version: 1,
|
|
70869
|
+
headroom,
|
|
70870
|
+
whatIf: shares ?? null,
|
|
70871
|
+
picks: recommendations.map((r) => ({
|
|
70872
|
+
nodeId: r.nodeId,
|
|
70873
|
+
skipped: r.skipped,
|
|
70874
|
+
devices: r.devices.map((d) => [d.deviceKey, d.chosen?.modelId ?? null])
|
|
70875
|
+
})),
|
|
70876
|
+
nodeChanges,
|
|
70877
|
+
cameraClears,
|
|
70878
|
+
detectorStoreClears
|
|
70879
|
+
})).digest("hex"),
|
|
70880
|
+
family: "yolov9",
|
|
70881
|
+
headroom,
|
|
70882
|
+
whatIf: shares !== void 0,
|
|
70883
|
+
generatedAt: this.deps.now(),
|
|
70884
|
+
nodes: recommendations,
|
|
70885
|
+
nodeChanges: [...nodeChanges],
|
|
70886
|
+
cameraClears: [...cameraClears],
|
|
70887
|
+
detectorStoreClears,
|
|
70888
|
+
warnings
|
|
70889
|
+
},
|
|
70890
|
+
diffInput,
|
|
70891
|
+
formats
|
|
70892
|
+
};
|
|
70893
|
+
}
|
|
70894
|
+
async apply(input) {
|
|
70895
|
+
if (this.applying) return {
|
|
70896
|
+
kind: "refused",
|
|
70897
|
+
reason: "apply-in-progress"
|
|
70898
|
+
};
|
|
70899
|
+
this.applying = true;
|
|
70900
|
+
try {
|
|
70901
|
+
return await this.applyLocked(input);
|
|
70902
|
+
} finally {
|
|
70903
|
+
this.applying = false;
|
|
70904
|
+
}
|
|
70905
|
+
}
|
|
70906
|
+
async applyLocked(input) {
|
|
70907
|
+
const { plan, diffInput, formats } = await this.computePlan(input.headroom);
|
|
70908
|
+
if (plan.planHash !== input.planHash) {
|
|
70909
|
+
this.deps.logger.warn("best settings: stale plan refused", { meta: {
|
|
70910
|
+
confirmed: input.planHash,
|
|
70911
|
+
current: plan.planHash
|
|
70912
|
+
} });
|
|
70913
|
+
return {
|
|
70914
|
+
kind: "stale",
|
|
70915
|
+
plan
|
|
70916
|
+
};
|
|
70917
|
+
}
|
|
70918
|
+
const phases = [];
|
|
70919
|
+
const downloads = plan.nodeChanges.flatMap((c) => c.modelPins.filter((p) => p.stepId === "object-detection" && p.after !== null && p.after !== p.before).map((p) => ({
|
|
70920
|
+
nodeId: c.nodeId,
|
|
70921
|
+
modelId: p.after ?? "",
|
|
70922
|
+
format: formats.get(c.nodeId)?.get(c.deviceKey)
|
|
70923
|
+
})));
|
|
70924
|
+
const unique = new Map(downloads.map((d) => [`${d.nodeId}|${d.modelId}|${d.format}`, d]));
|
|
70925
|
+
try {
|
|
70926
|
+
for (const d of unique.values()) {
|
|
70927
|
+
if (d.format === void 0) continue;
|
|
70928
|
+
await this.deps.downloadModel(d.nodeId, d.modelId, d.format);
|
|
70929
|
+
}
|
|
70930
|
+
phases.push({
|
|
70931
|
+
phase: "download",
|
|
70932
|
+
ok: true,
|
|
70933
|
+
detail: `${unique.size} model(s) present on their nodes`
|
|
70934
|
+
});
|
|
70935
|
+
} catch (err) {
|
|
70936
|
+
phases.push({
|
|
70937
|
+
phase: "download",
|
|
70938
|
+
ok: false,
|
|
70939
|
+
detail: errMsg(err)
|
|
70940
|
+
});
|
|
70941
|
+
this.deps.logger.warn("best settings: a model download failed — nothing was written", { meta: { error: errMsg(err) } });
|
|
70942
|
+
return {
|
|
70943
|
+
kind: "aborted",
|
|
70944
|
+
phases
|
|
70945
|
+
};
|
|
70946
|
+
}
|
|
70947
|
+
try {
|
|
70948
|
+
await this.deps.rewriteCameraSettings((prev) => applyCameraBestSettings(prev, diffInput).next);
|
|
70949
|
+
for (const c of plan.cameraClears) this.deps.logger.info("best settings: camera override cleared", {
|
|
70950
|
+
tags: {
|
|
70951
|
+
deviceId: c.deviceId,
|
|
70952
|
+
nodeId: c.nodeId
|
|
70953
|
+
},
|
|
70954
|
+
meta: {
|
|
70955
|
+
deviceId: c.deviceId,
|
|
70956
|
+
source: c.source,
|
|
70957
|
+
deviceKey: c.deviceKey,
|
|
70958
|
+
stepId: c.stepId,
|
|
70959
|
+
modelIdBefore: c.modelIdBefore,
|
|
70960
|
+
keys: c.keys
|
|
70961
|
+
}
|
|
70962
|
+
});
|
|
70963
|
+
phases.push({
|
|
70964
|
+
phase: "camera-overrides",
|
|
70965
|
+
ok: true,
|
|
70966
|
+
detail: `${plan.cameraClears.length} cleared`
|
|
70967
|
+
});
|
|
70968
|
+
} catch (err) {
|
|
70969
|
+
phases.push({
|
|
70970
|
+
phase: "camera-overrides",
|
|
70971
|
+
ok: false,
|
|
70972
|
+
detail: errMsg(err)
|
|
70973
|
+
});
|
|
70974
|
+
}
|
|
70975
|
+
if (plan.detectorStoreClears.length > 0) try {
|
|
70976
|
+
const answer = await this.deps.detectorStore({
|
|
70977
|
+
dryRun: false,
|
|
70978
|
+
deviceIds: plan.detectorStoreClears.map((c) => c.deviceId)
|
|
70979
|
+
});
|
|
70980
|
+
phases.push({
|
|
70981
|
+
phase: "detector-thresholds",
|
|
70982
|
+
ok: answer.refused.length === 0,
|
|
70983
|
+
detail: `${answer.entries.length} cleared, ${answer.refused.length} refused`
|
|
70984
|
+
});
|
|
70985
|
+
} catch (err) {
|
|
70986
|
+
phases.push({
|
|
70987
|
+
phase: "detector-thresholds",
|
|
70988
|
+
ok: false,
|
|
70989
|
+
detail: errMsg(err)
|
|
70990
|
+
});
|
|
70991
|
+
}
|
|
70992
|
+
try {
|
|
70993
|
+
await this.deps.rewriteAgentSettings((prev) => applyNodeBestSettings(prev, diffInput).next);
|
|
70994
|
+
for (const c of plan.nodeChanges) for (const pin of c.modelPins) this.deps.logger.info("best settings: node model set", {
|
|
70995
|
+
tags: { nodeId: c.nodeId },
|
|
70996
|
+
meta: {
|
|
70997
|
+
nodeId: c.nodeId,
|
|
70998
|
+
deviceKey: c.deviceKey,
|
|
70999
|
+
stepId: pin.stepId,
|
|
71000
|
+
from: pin.before,
|
|
71001
|
+
to: pin.after
|
|
71002
|
+
}
|
|
71003
|
+
});
|
|
71004
|
+
phases.push({
|
|
71005
|
+
phase: "node-models",
|
|
71006
|
+
ok: true,
|
|
71007
|
+
detail: `${plan.nodeChanges.length} device(s) updated`
|
|
71008
|
+
});
|
|
71009
|
+
} catch (err) {
|
|
71010
|
+
phases.push({
|
|
71011
|
+
phase: "node-models",
|
|
71012
|
+
ok: false,
|
|
71013
|
+
detail: errMsg(err)
|
|
71014
|
+
});
|
|
71015
|
+
}
|
|
71016
|
+
this.deps.invalidateCatalog();
|
|
71017
|
+
const failed = [];
|
|
71018
|
+
for (const nodeId of [...diffInput.touchedNodes].toSorted()) try {
|
|
71019
|
+
await this.deps.redispatch(nodeId);
|
|
71020
|
+
} catch (err) {
|
|
71021
|
+
failed.push(`${nodeId}: ${errMsg(err)}`);
|
|
71022
|
+
}
|
|
71023
|
+
phases.push({
|
|
71024
|
+
phase: "redispatch",
|
|
71025
|
+
ok: failed.length === 0,
|
|
71026
|
+
detail: failed.length === 0 ? `${diffInput.touchedNodes.size} node(s)` : failed.join("; ")
|
|
71027
|
+
});
|
|
71028
|
+
for (const rec of plan.nodes.filter((r) => r.skipped)) this.deps.logger.info("best settings: node skipped", {
|
|
71029
|
+
tags: { nodeId: rec.nodeId },
|
|
71030
|
+
meta: {
|
|
71031
|
+
nodeId: rec.nodeId,
|
|
71032
|
+
reasons: rec.reasons
|
|
71033
|
+
}
|
|
71034
|
+
});
|
|
71035
|
+
this.deps.logger.info("best settings: applied", { meta: {
|
|
71036
|
+
planHash: plan.planHash,
|
|
71037
|
+
nodeChanges: plan.nodeChanges.length,
|
|
71038
|
+
cameraClears: plan.cameraClears.length,
|
|
71039
|
+
detectorStoreClears: plan.detectorStoreClears.length,
|
|
71040
|
+
phases
|
|
71041
|
+
} });
|
|
71042
|
+
return {
|
|
71043
|
+
kind: "applied",
|
|
71044
|
+
planHash: plan.planHash,
|
|
71045
|
+
phases
|
|
71046
|
+
};
|
|
71047
|
+
}
|
|
71048
|
+
};
|
|
71049
|
+
//#endregion
|
|
71050
|
+
//#region src/best-settings/device-peak-tracker.ts
|
|
71051
|
+
var BUCKET_MS = 3600 * 1e3;
|
|
71052
|
+
var DevicePeakTracker = class DevicePeakTracker {
|
|
71053
|
+
now;
|
|
71054
|
+
series = /* @__PURE__ */ new Map();
|
|
71055
|
+
cluster = null;
|
|
71056
|
+
constructor(now = () => Date.now()) {
|
|
71057
|
+
this.now = now;
|
|
71058
|
+
}
|
|
71059
|
+
static key(nodeId, deviceKey) {
|
|
71060
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
71061
|
+
}
|
|
71062
|
+
/** Fold one runner census in. A runner that did not answer is simply absent. */
|
|
71063
|
+
noteCensus(loads) {
|
|
71064
|
+
const now = this.now();
|
|
71065
|
+
const bucket = now - now % BUCKET_MS;
|
|
71066
|
+
let clusterTotal = 0;
|
|
71067
|
+
for (const load of loads) {
|
|
71068
|
+
const rows = load.devices.length > 0 ? load.devices : [{
|
|
71069
|
+
deviceKey: "",
|
|
71070
|
+
attachedCameras: load.attachedCameras
|
|
71071
|
+
}];
|
|
71072
|
+
for (const device of rows) {
|
|
71073
|
+
clusterTotal += device.attachedCameras;
|
|
71074
|
+
const key = DevicePeakTracker.key(load.nodeId, device.deviceKey);
|
|
71075
|
+
let s = this.series.get(key);
|
|
71076
|
+
if (s === void 0) {
|
|
71077
|
+
s = {
|
|
71078
|
+
firstSeenAt: now,
|
|
71079
|
+
buckets: /* @__PURE__ */ new Map()
|
|
71080
|
+
};
|
|
71081
|
+
this.series.set(key, s);
|
|
71082
|
+
}
|
|
71083
|
+
s.buckets.set(bucket, Math.max(s.buckets.get(bucket) ?? 0, device.attachedCameras));
|
|
71084
|
+
for (const start of s.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) s.buckets.delete(start);
|
|
71085
|
+
}
|
|
71086
|
+
}
|
|
71087
|
+
if (loads.length === 0) return;
|
|
71088
|
+
const c = this.cluster ?? {
|
|
71089
|
+
firstSeenAt: now,
|
|
71090
|
+
buckets: /* @__PURE__ */ new Map()
|
|
71091
|
+
};
|
|
71092
|
+
this.cluster = c;
|
|
71093
|
+
c.buckets.set(bucket, Math.max(c.buckets.get(bucket) ?? 0, clusterTotal));
|
|
71094
|
+
for (const start of c.buckets.keys()) if (start + BUCKET_MS <= now - 864e5) c.buckets.delete(start);
|
|
71095
|
+
}
|
|
71096
|
+
/** The cluster-wide parallel peak over the window, or null before any census. */
|
|
71097
|
+
clusterPeak() {
|
|
71098
|
+
const c = this.cluster;
|
|
71099
|
+
if (c === null) return null;
|
|
71100
|
+
const now = this.now();
|
|
71101
|
+
let peak = 0;
|
|
71102
|
+
for (const [start, value] of c.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71103
|
+
return {
|
|
71104
|
+
peakSessions: peak,
|
|
71105
|
+
observedForMs: now - c.firstSeenAt
|
|
71106
|
+
};
|
|
71107
|
+
}
|
|
71108
|
+
/** Every observed device's peak over the window and how long it has been watched. */
|
|
71109
|
+
peaks() {
|
|
71110
|
+
const now = this.now();
|
|
71111
|
+
const out = [];
|
|
71112
|
+
for (const [key, s] of this.series) {
|
|
71113
|
+
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71114
|
+
let peak = 0;
|
|
71115
|
+
for (const [start, value] of s.buckets) if (start + BUCKET_MS > now - 864e5) peak = Math.max(peak, value);
|
|
71116
|
+
out.push({
|
|
71117
|
+
nodeId,
|
|
71118
|
+
deviceKey,
|
|
71119
|
+
peakSessions: peak,
|
|
71120
|
+
observedForMs: now - s.firstSeenAt
|
|
71121
|
+
});
|
|
71122
|
+
}
|
|
71123
|
+
return out.toSorted((a, b) => a.nodeId === b.nodeId ? a.deviceKey.localeCompare(b.deviceKey) : a.nodeId.localeCompare(b.nodeId));
|
|
71124
|
+
}
|
|
71125
|
+
};
|
|
71126
|
+
//#endregion
|
|
71127
|
+
//#region src/session-placement-stats.ts
|
|
71128
|
+
var RETAIN_MS = 1440 * 60 * 1e3;
|
|
71129
|
+
function keyOf(nodeId, deviceKey) {
|
|
71130
|
+
return `${nodeId}\u0000${deviceKey}`;
|
|
71131
|
+
}
|
|
71132
|
+
function split(key) {
|
|
71133
|
+
const [nodeId = "", deviceKey = ""] = key.split("\0");
|
|
71134
|
+
return {
|
|
71135
|
+
nodeId,
|
|
71136
|
+
deviceKey
|
|
71137
|
+
};
|
|
71138
|
+
}
|
|
71139
|
+
var SessionPlacementStats = class {
|
|
71140
|
+
now;
|
|
71141
|
+
segments = [];
|
|
71142
|
+
starts = [];
|
|
71143
|
+
yields = [];
|
|
71144
|
+
startedAt;
|
|
71145
|
+
constructor(now = () => Date.now()) {
|
|
71146
|
+
this.now = now;
|
|
71147
|
+
this.startedAt = now();
|
|
71148
|
+
}
|
|
71149
|
+
/** The running-session set changed: open a new segment with these counts. */
|
|
71150
|
+
noteSessions(sessions) {
|
|
71151
|
+
const counts = /* @__PURE__ */ new Map();
|
|
71152
|
+
for (const s of sessions) {
|
|
71153
|
+
const k = keyOf(s.nodeId, s.deviceKey);
|
|
71154
|
+
counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
71155
|
+
}
|
|
71156
|
+
this.segments.push({
|
|
71157
|
+
at: this.now(),
|
|
71158
|
+
counts
|
|
71159
|
+
});
|
|
71160
|
+
this.prune();
|
|
71161
|
+
}
|
|
71162
|
+
noteStart(nodeId, deviceKey) {
|
|
71163
|
+
this.starts.push({
|
|
71164
|
+
at: this.now(),
|
|
71165
|
+
nodeId,
|
|
71166
|
+
deviceKey
|
|
71167
|
+
});
|
|
71168
|
+
this.prune();
|
|
71169
|
+
}
|
|
71170
|
+
/** A share election handed `nodeId`'s turn to another node. */
|
|
71171
|
+
noteYield(nodeId) {
|
|
71172
|
+
this.yields.push({
|
|
71173
|
+
at: this.now(),
|
|
71174
|
+
nodeId
|
|
71175
|
+
});
|
|
71176
|
+
this.prune();
|
|
71177
|
+
}
|
|
71178
|
+
/** For tests: how many segments are retained. */
|
|
71179
|
+
segmentCount() {
|
|
71180
|
+
return this.segments.length;
|
|
71181
|
+
}
|
|
71182
|
+
prune() {
|
|
71183
|
+
const floor = this.now() - RETAIN_MS;
|
|
71184
|
+
while (this.segments.length > 1 && this.segments[1].at <= floor) this.segments.shift();
|
|
71185
|
+
while (this.starts.length > 0 && this.starts[0].at < floor) this.starts.shift();
|
|
71186
|
+
while (this.yields.length > 0 && this.yields[0].at < floor) this.yields.shift();
|
|
71187
|
+
}
|
|
71188
|
+
snapshot(windowMs) {
|
|
71189
|
+
const now = this.now();
|
|
71190
|
+
const from = now - windowMs;
|
|
71191
|
+
const integral = /* @__PURE__ */ new Map();
|
|
71192
|
+
let clusterPeak = 0;
|
|
71193
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71194
|
+
const seg = this.segments[i];
|
|
71195
|
+
const end = this.segments[i + 1]?.at ?? now;
|
|
71196
|
+
const start = Math.max(seg.at, from);
|
|
71197
|
+
if (end <= start) continue;
|
|
71198
|
+
let total = 0;
|
|
71199
|
+
for (const [k, n] of seg.counts) {
|
|
71200
|
+
integral.set(k, (integral.get(k) ?? 0) + n * (end - start));
|
|
71201
|
+
total += n;
|
|
71202
|
+
}
|
|
71203
|
+
clusterPeak = Math.max(clusterPeak, total);
|
|
71204
|
+
}
|
|
71205
|
+
const current = this.segments.at(-1)?.counts ?? /* @__PURE__ */ new Map();
|
|
71206
|
+
const nodeIds = /* @__PURE__ */ new Set();
|
|
71207
|
+
for (const k of [...integral.keys(), ...current.keys()]) nodeIds.add(split(k).nodeId);
|
|
71208
|
+
for (const s of this.starts) if (s.at >= from) nodeIds.add(s.nodeId);
|
|
71209
|
+
const clusterIntegral = [...integral.values()].reduce((a, b) => a + b, 0);
|
|
71210
|
+
const nodes = [...nodeIds].toSorted().map((nodeId) => {
|
|
71211
|
+
const deviceKeys = /* @__PURE__ */ new Set();
|
|
71212
|
+
for (const k of [...integral.keys(), ...current.keys()]) {
|
|
71213
|
+
const s = split(k);
|
|
71214
|
+
if (s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71215
|
+
}
|
|
71216
|
+
for (const s of this.starts) if (s.at >= from && s.nodeId === nodeId) deviceKeys.add(s.deviceKey);
|
|
71217
|
+
const nodeIntegral = [...deviceKeys].reduce((sum, d) => sum + (integral.get(keyOf(nodeId, d)) ?? 0), 0);
|
|
71218
|
+
const devices = [...deviceKeys].toSorted().map((deviceKey) => ({
|
|
71219
|
+
deviceKey,
|
|
71220
|
+
sessionsNow: current.get(keyOf(nodeId, deviceKey)) ?? 0,
|
|
71221
|
+
sessionsStarted: this.starts.filter((s) => s.at >= from && s.nodeId === nodeId && s.deviceKey === deviceKey).length,
|
|
71222
|
+
observedShare: nodeIntegral > 0 ? (integral.get(keyOf(nodeId, deviceKey)) ?? 0) / nodeIntegral : null
|
|
71223
|
+
}));
|
|
71224
|
+
let peak = 0;
|
|
71225
|
+
for (let i = 0; i < this.segments.length; i++) {
|
|
71226
|
+
const seg = this.segments[i];
|
|
71227
|
+
if ((this.segments[i + 1]?.at ?? now) <= Math.max(seg.at, from)) continue;
|
|
71228
|
+
let n = 0;
|
|
71229
|
+
for (const [k, c] of seg.counts) if (split(k).nodeId === nodeId) n += c;
|
|
71230
|
+
peak = Math.max(peak, n);
|
|
71231
|
+
}
|
|
71232
|
+
return {
|
|
71233
|
+
nodeId,
|
|
71234
|
+
sessionsNow: devices.reduce((s, d) => s + d.sessionsNow, 0),
|
|
71235
|
+
sessionsStarted: devices.reduce((s, d) => s + d.sessionsStarted, 0),
|
|
71236
|
+
observedShare: clusterIntegral > 0 && nodeIntegral > 0 ? nodeIntegral / clusterIntegral : null,
|
|
71237
|
+
peakSessions: peak,
|
|
71238
|
+
yieldsAway: this.yields.filter((y) => y.at >= from && y.nodeId === nodeId).length,
|
|
71239
|
+
devices
|
|
71240
|
+
};
|
|
71241
|
+
});
|
|
71242
|
+
return {
|
|
71243
|
+
observedForMs: Math.min(windowMs, now - this.startedAt),
|
|
71244
|
+
sessionsNow: [...current.values()].reduce((a, b) => a + b, 0),
|
|
71245
|
+
clusterPeakSessions: clusterPeak,
|
|
71246
|
+
nodes
|
|
71247
|
+
};
|
|
71248
|
+
}
|
|
71249
|
+
};
|
|
71250
|
+
//#endregion
|
|
71251
|
+
//#region src/load-balancing-shares.ts
|
|
71252
|
+
function nextDevices(stored, shares, effective) {
|
|
71253
|
+
let changed = false;
|
|
71254
|
+
const out = { ...stored };
|
|
71255
|
+
for (const [key, weight] of Object.entries(shares)) {
|
|
71256
|
+
const existing = stored?.[key];
|
|
71257
|
+
if (existing !== void 0) {
|
|
71258
|
+
out[key] = {
|
|
71259
|
+
...existing,
|
|
71260
|
+
weight
|
|
71261
|
+
};
|
|
71262
|
+
changed = true;
|
|
71263
|
+
continue;
|
|
71264
|
+
}
|
|
71265
|
+
const enabled = effective[key];
|
|
71266
|
+
if (enabled === void 0) continue;
|
|
71267
|
+
out[key] = {
|
|
71268
|
+
enabled,
|
|
71269
|
+
weight
|
|
71270
|
+
};
|
|
71271
|
+
changed = true;
|
|
71272
|
+
}
|
|
71273
|
+
return changed ? out : stored;
|
|
71274
|
+
}
|
|
71275
|
+
function applyLoadBalancingShares(prev, write) {
|
|
71276
|
+
const next = { ...prev };
|
|
71277
|
+
const nodeIds = new Set([...Object.keys(write.nodes), ...Object.keys(write.devices)]);
|
|
71278
|
+
for (const nodeId of nodeIds) {
|
|
71279
|
+
const existing = prev[nodeId];
|
|
71280
|
+
const nodeShare = write.nodes[nodeId];
|
|
71281
|
+
const devices = nextDevices(existing?.inferenceDevices, write.devices[nodeId] ?? {}, write.effectiveEnabled[nodeId] ?? {});
|
|
71282
|
+
if (nodeShare === void 0 && devices === existing?.inferenceDevices) continue;
|
|
71283
|
+
next[nodeId] = {
|
|
71284
|
+
...existing ?? { maxCameras: null },
|
|
71285
|
+
...nodeShare !== void 0 ? { detectWeight: nodeShare } : {},
|
|
71286
|
+
...devices !== void 0 ? { inferenceDevices: devices } : {}
|
|
71287
|
+
};
|
|
71288
|
+
}
|
|
71289
|
+
return next;
|
|
71290
|
+
}
|
|
71291
|
+
//#endregion
|
|
71292
|
+
//#region src/stale-device-pins.ts
|
|
71293
|
+
function findStaleDevicePins(pins, nodes) {
|
|
71294
|
+
const usable = /* @__PURE__ */ new Map();
|
|
71295
|
+
for (const node of nodes) for (const d of node.devices) usable.set(d.key, (usable.get(d.key) ?? false) || d.enabled && d.available);
|
|
71296
|
+
return pins.flatMap((pin) => {
|
|
71297
|
+
const state = usable.get(pin.deviceKey);
|
|
71298
|
+
if (state === true) return [];
|
|
71299
|
+
return [{
|
|
71300
|
+
deviceId: pin.deviceId,
|
|
71301
|
+
deviceKey: pin.deviceKey,
|
|
71302
|
+
reason: state === false ? "disabled" : "absent"
|
|
71303
|
+
}];
|
|
71304
|
+
}).toSorted((a, b) => a.deviceId - b.deviceId);
|
|
71305
|
+
}
|
|
71306
|
+
//#endregion
|
|
68814
71307
|
//#region src/index.ts
|
|
68815
71308
|
/**
|
|
68816
71309
|
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
@@ -68878,6 +71371,13 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
68878
71371
|
* not a claim this flag makes.
|
|
68879
71372
|
*/
|
|
68880
71373
|
clusterModelChoiceReady = false;
|
|
71374
|
+
/** The per-device 24 h peak of concurrent sessions, fed by every runner census (D745). */
|
|
71375
|
+
devicePeaks = new DevicePeakTracker();
|
|
71376
|
+
/** What session placement did (D748): the Load balancing tab's "actual" column. */
|
|
71377
|
+
placementStats = new SessionPlacementStats();
|
|
71378
|
+
/** The stored session placement policy (D748); `least-loaded` before init. */
|
|
71379
|
+
readSessionPlacement = () => "least-loaded";
|
|
71380
|
+
bestSettingsService = null;
|
|
68881
71381
|
/**
|
|
68882
71382
|
* Single owner of the shared placement state (`assignments`/`cameraConfigs`/
|
|
68883
71383
|
* `pendingReasons`/`dispatchingDevices`) — see `placement-ledger.ts` (S6
|
|
@@ -69154,6 +71654,8 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
69154
71654
|
rebalance: (opts) => this.rebalance(opts),
|
|
69155
71655
|
setCameraPipelineForAgent: (input) => this.setCameraPipelineForAgent(input),
|
|
69156
71656
|
maintenancePaused: () => this.storageMigrationLeaseId !== null,
|
|
71657
|
+
onRunnerCensus: (loads) => this.devicePeaks.noteCensus(loads),
|
|
71658
|
+
placementStats: this.placementStats,
|
|
69157
71659
|
setTopology: (topology) => {
|
|
69158
71660
|
this.topology = topology;
|
|
69159
71661
|
},
|
|
@@ -69180,6 +71682,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
69180
71682
|
this.nodeStress = controllers.nodeStress;
|
|
69181
71683
|
this.placement = controllers.placement;
|
|
69182
71684
|
this.session = controllers.session;
|
|
71685
|
+
this.readSessionPlacement = controllers.sessionPlacement;
|
|
69183
71686
|
this.deviceConfig = controllers.deviceConfig;
|
|
69184
71687
|
this.cameraStatusService = controllers.cameraStatusService;
|
|
69185
71688
|
this.cameraSwitchService = controllers.cameraSwitchService;
|
|
@@ -69295,7 +71798,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
69295
71798
|
for (const deviceId of this.detectionWiring.activeDetectionDeviceIds()) try {
|
|
69296
71799
|
await this.releaseCamera({ deviceId });
|
|
69297
71800
|
} catch (err) {
|
|
69298
|
-
const msg = errMsg$
|
|
71801
|
+
const msg = errMsg$2(err);
|
|
69299
71802
|
this.ctx.logger.debug("releaseCamera failed during shutdown", {
|
|
69300
71803
|
tags: { deviceId },
|
|
69301
71804
|
meta: { error: msg }
|
|
@@ -69672,26 +72175,6 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
69672
72175
|
this.reconcile.schedulePendingRetry();
|
|
69673
72176
|
return { success: true };
|
|
69674
72177
|
}
|
|
69675
|
-
async setAgentDetectWeight(input) {
|
|
69676
|
-
const existing = (await this.settingsStore.readAgentSettingsMap())[input.agentNodeId];
|
|
69677
|
-
const weight = input.detectWeight ?? 1;
|
|
69678
|
-
const next = existing ? {
|
|
69679
|
-
...existing,
|
|
69680
|
-
detectWeight: weight
|
|
69681
|
-
} : {
|
|
69682
|
-
maxCameras: null,
|
|
69683
|
-
detectWeight: weight
|
|
69684
|
-
};
|
|
69685
|
-
await this.settingsStore.writeAgentSettings(input.agentNodeId, next);
|
|
69686
|
-
this.ctx.logger.info("agentSettings.detectWeight updated", {
|
|
69687
|
-
tags: { nodeId: input.agentNodeId },
|
|
69688
|
-
meta: { detectWeight: weight ?? null }
|
|
69689
|
-
});
|
|
69690
|
-
this.rebalance().catch((err) => {
|
|
69691
|
-
this.ctx.logger.warn("setAgentDetectWeight: rebalance failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
69692
|
-
});
|
|
69693
|
-
return { success: true };
|
|
69694
|
-
}
|
|
69695
72178
|
/**
|
|
69696
72179
|
* Set one node's placement capabilities — the per-node record that replaced
|
|
69697
72180
|
* the four flat orchestrator lists. Each flag is a tri-state patch: omit to
|
|
@@ -70090,7 +72573,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70090
72573
|
tags: { nodeId },
|
|
70091
72574
|
meta: {
|
|
70092
72575
|
nodeId,
|
|
70093
|
-
error: errMsg$
|
|
72576
|
+
error: errMsg$2(err)
|
|
70094
72577
|
}
|
|
70095
72578
|
});
|
|
70096
72579
|
return null;
|
|
@@ -70196,7 +72679,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70196
72679
|
tags: { nodeId },
|
|
70197
72680
|
meta: {
|
|
70198
72681
|
nodeId,
|
|
70199
|
-
error: errMsg$
|
|
72682
|
+
error: errMsg$2(err)
|
|
70200
72683
|
}
|
|
70201
72684
|
});
|
|
70202
72685
|
return {
|
|
@@ -70243,7 +72726,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70243
72726
|
} catch (err) {
|
|
70244
72727
|
this.ctx.logger.warn("resetNodePipelineDefaults: executor clearDeviceOverrides failed (node offline?)", {
|
|
70245
72728
|
tags: { nodeId: node },
|
|
70246
|
-
meta: { error: errMsg$
|
|
72729
|
+
meta: { error: errMsg$2(err) }
|
|
70247
72730
|
});
|
|
70248
72731
|
}
|
|
70249
72732
|
this.settingsStore.invalidateCatalog();
|
|
@@ -70254,7 +72737,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70254
72737
|
} catch (err) {
|
|
70255
72738
|
this.ctx.logger.warn("resetNodePipelineDefaults: effective-model probe failed (node offline?)", {
|
|
70256
72739
|
tags: { nodeId: node },
|
|
70257
|
-
meta: { error: errMsg$
|
|
72740
|
+
meta: { error: errMsg$2(err) }
|
|
70258
72741
|
});
|
|
70259
72742
|
}
|
|
70260
72743
|
this.ctx.logger.info("node pipeline reset to hardware-aware defaults", {
|
|
@@ -70270,6 +72753,137 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70270
72753
|
clearedCameraOverrides
|
|
70271
72754
|
};
|
|
70272
72755
|
}
|
|
72756
|
+
bestSettings() {
|
|
72757
|
+
if (this.bestSettingsService !== null) return this.bestSettingsService;
|
|
72758
|
+
const store = this.settingsStore;
|
|
72759
|
+
this.bestSettingsService = new BestSettingsService({
|
|
72760
|
+
logger: this.ctx.logger,
|
|
72761
|
+
runnerNodeIds: () => this.topology.collectRunnerNodeIds(),
|
|
72762
|
+
readNodeDevices: (nodeId) => this.getNodeInferenceDevices({ nodeId }),
|
|
72763
|
+
readHardware: async (nodeId) => {
|
|
72764
|
+
const api = this.ctx.api;
|
|
72765
|
+
if (!api) return null;
|
|
72766
|
+
return nodeId === this.localNodeId ? api.platformProbe.getHardware.query() : api.platformProbe.getHardware.query({ nodeId });
|
|
72767
|
+
},
|
|
72768
|
+
readCatalog: (nodeId) => store.readCatalogForNode(nodeId),
|
|
72769
|
+
readAgentSettings: () => store.readAgentSettingsMap(),
|
|
72770
|
+
readCameraSettings: () => store.readCameraSettingsMap(),
|
|
72771
|
+
rewriteAgentSettings: (transform) => store.rewriteAgentSettingsMap(transform),
|
|
72772
|
+
rewriteCameraSettings: (transform) => store.rewriteCameraSettingsMap(transform),
|
|
72773
|
+
cameraLoads: () => this.bestSettingsCameraLoads(),
|
|
72774
|
+
peaks: () => this.devicePeaks.peaks(),
|
|
72775
|
+
clusterPeak: () => this.devicePeaks.clusterPeak(),
|
|
72776
|
+
sessionPlacement: () => this.readSessionPlacement(),
|
|
72777
|
+
readNodeShares: () => store.buildNodeWeights(),
|
|
72778
|
+
detectorStore: async (input) => {
|
|
72779
|
+
const api = this.ctx.api;
|
|
72780
|
+
if (!api) throw new Error("ctx.api unavailable");
|
|
72781
|
+
return api.pipelineExecutor.clearModelBoundDeviceOverrides.mutate({
|
|
72782
|
+
nodeId: this.localNodeId,
|
|
72783
|
+
dryRun: input.dryRun,
|
|
72784
|
+
...input.deviceIds !== void 0 ? { deviceIds: input.deviceIds } : {}
|
|
72785
|
+
});
|
|
72786
|
+
},
|
|
72787
|
+
downloadModel: async (nodeId, modelId, format) => {
|
|
72788
|
+
const api = this.ctx.api;
|
|
72789
|
+
if (!api) throw new Error("ctx.api unavailable");
|
|
72790
|
+
await api.pipelineExecutor.downloadModel.mutate({
|
|
72791
|
+
addonId: "object-detection",
|
|
72792
|
+
modelId,
|
|
72793
|
+
format
|
|
72794
|
+
}, nodePin(nodeId));
|
|
72795
|
+
},
|
|
72796
|
+
invalidateCatalog: () => store.invalidateCatalog(),
|
|
72797
|
+
redispatch: (nodeId) => this.placement.redispatchCamerasOnAgent(nodeId),
|
|
72798
|
+
now: () => Date.now()
|
|
72799
|
+
});
|
|
72800
|
+
return this.bestSettingsService;
|
|
72801
|
+
}
|
|
72802
|
+
/** Every camera with detection started, as the recommender charges it (D745). */
|
|
72803
|
+
async bestSettingsCameraLoads() {
|
|
72804
|
+
const standing = new Map(this.getPipelineAssignments().map((a) => [a.deviceId, a.agentNodeId]));
|
|
72805
|
+
const loads = [];
|
|
72806
|
+
for (const [deviceId, config] of this.detectionWiring.activeDetections) {
|
|
72807
|
+
const nodePin = await this.readPipelinePin(deviceId);
|
|
72808
|
+
const pinnedKey = await this.readPipelineDevicePin(deviceId);
|
|
72809
|
+
const standingNode = config.detectionMode === "always-on" ? standing.get(deviceId) ?? null : null;
|
|
72810
|
+
loads.push({
|
|
72811
|
+
deviceId,
|
|
72812
|
+
mode: config.detectionMode,
|
|
72813
|
+
fps: config.detectionFps,
|
|
72814
|
+
runsPipelineRoot: (config.detectionSources ?? ["pipeline"]).includes("pipeline"),
|
|
72815
|
+
standingNodeId: standingNode,
|
|
72816
|
+
nodePinId: nodePin,
|
|
72817
|
+
devicePin: pinnedKey !== null ? {
|
|
72818
|
+
nodeId: nodePin ?? standingNode,
|
|
72819
|
+
deviceKey: pinnedKey
|
|
72820
|
+
} : null
|
|
72821
|
+
});
|
|
72822
|
+
}
|
|
72823
|
+
return loads;
|
|
72824
|
+
}
|
|
72825
|
+
async planBestSettings(input) {
|
|
72826
|
+
return this.bestSettings().plan(input);
|
|
72827
|
+
}
|
|
72828
|
+
/**
|
|
72829
|
+
* Node + device shares in ONE read-modify-write of the agent settings. A
|
|
72830
|
+
* device with no stored entry is made explicit with its effective
|
|
72831
|
+
* enablement, read from the probe-aware device list. Never touches the
|
|
72832
|
+
* `sessionPlacement` policy.
|
|
72833
|
+
*/
|
|
72834
|
+
async setLoadBalancingShares(input) {
|
|
72835
|
+
const effectiveEnabled = {};
|
|
72836
|
+
for (const nodeId of Object.keys(input.devices)) {
|
|
72837
|
+
const read = await this.getNodeInferenceDevices({ nodeId });
|
|
72838
|
+
effectiveEnabled[nodeId] = Object.fromEntries(read.devices.map((d) => [d.key, d.enabled]));
|
|
72839
|
+
}
|
|
72840
|
+
await this.settingsStore.rewriteAgentSettingsMap((prev) => applyLoadBalancingShares(prev, {
|
|
72841
|
+
nodes: input.nodes,
|
|
72842
|
+
devices: input.devices,
|
|
72843
|
+
effectiveEnabled
|
|
72844
|
+
}));
|
|
72845
|
+
this.ctx.logger.info("load balancing shares updated", { meta: {
|
|
72846
|
+
nodes: JSON.stringify(input.nodes),
|
|
72847
|
+
devices: JSON.stringify(input.devices),
|
|
72848
|
+
policy: this.readSessionPlacement()
|
|
72849
|
+
} });
|
|
72850
|
+
if (Object.keys(input.nodes).length > 0) this.rebalance().catch((err) => {
|
|
72851
|
+
this.ctx.logger.warn("setLoadBalancingShares: rebalance failed", { meta: { error: errMsg$2(err) } });
|
|
72852
|
+
});
|
|
72853
|
+
return { success: true };
|
|
72854
|
+
}
|
|
72855
|
+
async getPlacementStats(input) {
|
|
72856
|
+
const snap = this.placementStats.snapshot(input.windowMs);
|
|
72857
|
+
return {
|
|
72858
|
+
policy: this.readSessionPlacement(),
|
|
72859
|
+
windowMs: input.windowMs,
|
|
72860
|
+
observedForMs: snap.observedForMs,
|
|
72861
|
+
sessionsNow: snap.sessionsNow,
|
|
72862
|
+
clusterPeakSessions: snap.clusterPeakSessions,
|
|
72863
|
+
nodes: snap.nodes,
|
|
72864
|
+
stalePins: await this.listStaleDevicePins()
|
|
72865
|
+
};
|
|
72866
|
+
}
|
|
72867
|
+
/** Device pins the session path cannot honour — see `stale-device-pins.ts`. */
|
|
72868
|
+
async listStaleDevicePins() {
|
|
72869
|
+
const nodeIds = [...new Set(["hub", ...this.topology.collectRunnerNodeIds()])];
|
|
72870
|
+
const offered = await Promise.all(nodeIds.map(async (nodeId) => ({ devices: (await this.getNodeInferenceDevices({ nodeId }).catch(() => null))?.devices ?? [] })));
|
|
72871
|
+
const pins = [];
|
|
72872
|
+
for (const deviceId of this.detectionWiring.activeDetections.keys()) {
|
|
72873
|
+
const deviceKey = await this.readPipelineDevicePin(deviceId);
|
|
72874
|
+
if (deviceKey !== null) pins.push({
|
|
72875
|
+
deviceId,
|
|
72876
|
+
deviceKey
|
|
72877
|
+
});
|
|
72878
|
+
}
|
|
72879
|
+
return findStaleDevicePins(pins, offered);
|
|
72880
|
+
}
|
|
72881
|
+
async suggestShares(input) {
|
|
72882
|
+
return this.bestSettings().suggestShares(input);
|
|
72883
|
+
}
|
|
72884
|
+
async applyBestSettings(input) {
|
|
72885
|
+
return this.bestSettings().apply(input);
|
|
72886
|
+
}
|
|
70273
72887
|
async getCameraSettings(input) {
|
|
70274
72888
|
return (await this.settingsStore.readCameraSettingsMap())[String(input.deviceId)] ?? null;
|
|
70275
72889
|
}
|
|
@@ -70526,7 +73140,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
70526
73140
|
tags: { deviceId },
|
|
70527
73141
|
meta: {
|
|
70528
73142
|
capName,
|
|
70529
|
-
error: errMsg$
|
|
73143
|
+
error: errMsg$2(err)
|
|
70530
73144
|
}
|
|
70531
73145
|
});
|
|
70532
73146
|
return true;
|