@camstack/addon-pipeline-orchestrator 1.2.55 → 1.2.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-DQ-9VJzz.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CIgtalSC.mjs} +1 -1
- package/dist/{hostInit-DEefQ-oR.mjs → hostInit-BfvdbNXJ.mjs} +1 -1
- package/dist/index.js +475 -7
- package/dist/index.mjs +475 -7
- package/dist/remoteEntry.js +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -24,12 +24,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
enumerable: true
|
|
25
25
|
}) : target, mod));
|
|
26
26
|
//#endregion
|
|
27
|
+
let node_crypto = require("node:crypto");
|
|
27
28
|
let node_fs = require("node:fs");
|
|
28
29
|
node_fs = __toESM(node_fs);
|
|
29
30
|
let node_path = require("node:path");
|
|
30
31
|
node_path = __toESM(node_path);
|
|
31
32
|
let node_url = require("node:url");
|
|
32
|
-
//#region ../types/dist/event-category-
|
|
33
|
+
//#region ../types/dist/event-category-Bxo5yJjt.mjs
|
|
33
34
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
34
35
|
EventCategory["SystemBoot"] = "system.boot";
|
|
35
36
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -236,6 +237,33 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
|
236
237
|
EventCategory["PipelineCameraAssigned"] = "pipeline.camera-assigned";
|
|
237
238
|
EventCategory["PipelineCameraUnassigned"] = "pipeline.camera-unassigned";
|
|
238
239
|
/**
|
|
240
|
+
* A node the orchestrator would otherwise place cameras on has NO usable
|
|
241
|
+
* inference device: the operator enabled one or more accelerators there and
|
|
242
|
+
* the live probe reports every one of them unavailable. Emitted once per
|
|
243
|
+
* TRANSITION into that state (never per dispatch), and the node is dropped
|
|
244
|
+
* from the placement candidate set for as long as it holds.
|
|
245
|
+
*
|
|
246
|
+
* This exists because the state was previously invisible: little-unraid
|
|
247
|
+
* absorbed 283k inference errors in a day while still being handed cameras,
|
|
248
|
+
* and nothing in the system said so.
|
|
249
|
+
*
|
|
250
|
+
* A node with no accelerators configured at all is NOT this — its devices
|
|
251
|
+
* are `disabled`, not `unavailable`, and the runner's default CPU pool
|
|
252
|
+
* serves it exactly as before.
|
|
253
|
+
*/
|
|
254
|
+
EventCategory["PipelineNodeInferenceUnavailable"] = "pipeline.node-inference-unavailable";
|
|
255
|
+
/**
|
|
256
|
+
* A camera has an OPEN detection session and has produced no detection at
|
|
257
|
+
* all for longer than the blind threshold — the camera is being decoded and
|
|
258
|
+
* inferred and is returning nothing. Emitted once per transition into blind,
|
|
259
|
+
* per camera.
|
|
260
|
+
*
|
|
261
|
+
* The failure it reports: a 1h43 detection blackout on the entrance camera
|
|
262
|
+
* that nobody noticed, because "a camera that detects nothing" and "a quiet
|
|
263
|
+
* camera" produce byte-identical silence.
|
|
264
|
+
*/
|
|
265
|
+
EventCategory["PipelineDetectionBlind"] = "pipeline.detection-blind";
|
|
266
|
+
/**
|
|
239
267
|
* Per-camera pipeline config was mutated by the orchestrator
|
|
240
268
|
* (3-level settings change via `setAgentAddonDefaults` /
|
|
241
269
|
* `setCameraStepToggle` / `setCameraPipelineForAgent` or a
|
|
@@ -14927,6 +14955,8 @@ var NcSystemEventKindSchema = _enum([
|
|
|
14927
14955
|
"stream-offline",
|
|
14928
14956
|
"node-online",
|
|
14929
14957
|
"node-offline",
|
|
14958
|
+
"node-inference-unavailable",
|
|
14959
|
+
"detection-blind",
|
|
14930
14960
|
"addon-update-available",
|
|
14931
14961
|
"server-update-available",
|
|
14932
14962
|
"alarm-triggered",
|
|
@@ -15698,7 +15728,50 @@ var NcRuleInputSchema = object({
|
|
|
15698
15728
|
* tests every zone the track visited and a `crossing` condition can no longer
|
|
15699
15729
|
* be satisfied, because a closed track carries no crossing.
|
|
15700
15730
|
*/
|
|
15701
|
-
waitForEnhancement: boolean().optional()
|
|
15731
|
+
waitForEnhancement: boolean().optional(),
|
|
15732
|
+
/**
|
|
15733
|
+
* GROUP a burst of subjects into ONE notification that grows.
|
|
15734
|
+
*
|
|
15735
|
+
* Seconds of quiet after the last matching subject before the burst is
|
|
15736
|
+
* considered over. While it is open, the first subject enqueues immediately —
|
|
15737
|
+
* **exactly as today, with no added latency** — and every real growth (a new
|
|
15738
|
+
* subject, or a name confirmed on one already in it) REPLACES that
|
|
15739
|
+
* notification with an updated one naming everybody. The push carries the
|
|
15740
|
+
* group's own coalescing tag, so the phone replaces rather than stacks.
|
|
15741
|
+
*
|
|
15742
|
+
* `0` / absent = off, and off is today's behaviour byte for byte.
|
|
15743
|
+
*
|
|
15744
|
+
* ### Why an idle cutoff and not a window
|
|
15745
|
+
*
|
|
15746
|
+
* The measured seven-person arrival on device 590 spans 110 s with every
|
|
15747
|
+
* internal gap under 30 s. A 12 s fixed window cuts it into three groups; an
|
|
15748
|
+
* idle cutoff holds it as one and ends it when the arrival actually ends.
|
|
15749
|
+
* 30 is Frigate's shipped value for the same decision.
|
|
15750
|
+
*
|
|
15751
|
+
* ### What it replaces
|
|
15752
|
+
*
|
|
15753
|
+
* The blind cooldown, which collapses a burst by DISCARDING it. Measured on
|
|
15754
|
+
* device 615 / *Persona su Uscio* over six days: 116 qualifying tracks → 74
|
|
15755
|
+
* notifications, **44 (37.9%) suppressed outright**, 23 of them overlapping a
|
|
15756
|
+
* track that did fire and 7 carrying a confirmed identity nobody heard about.
|
|
15757
|
+
* A group collapses the same volume by MERGING, so the cooldown becomes a
|
|
15758
|
+
* budget over GROUPS — which is what it always meant — and a growth is never
|
|
15759
|
+
* throttled by the window its own first member spent.
|
|
15760
|
+
*
|
|
15761
|
+
* ### Interaction with {@link waitForEnhancement}
|
|
15762
|
+
*
|
|
15763
|
+
* They compose, and the order matters. `waitForEnhancement` defers the rule to
|
|
15764
|
+
* TRACK CLOSE, so with both set the group is opened by the first member to
|
|
15765
|
+
* CLOSE — already carrying its name — and grows as later members close. That
|
|
15766
|
+
* is later, and complete. With grouping alone the group opens on the first
|
|
15767
|
+
* object event and picks up names as they are confirmed, through the growth
|
|
15768
|
+
* path. Neither combination fires twice for one subject.
|
|
15769
|
+
*
|
|
15770
|
+
* `.optional()` and deliberately NOT `.default()`: a Zod default does not run
|
|
15771
|
+
* on the addon cap path, so absent must keep meaning what it meant before this
|
|
15772
|
+
* field existed.
|
|
15773
|
+
*/
|
|
15774
|
+
groupIdleSec: number().int().min(0).max(600).optional()
|
|
15702
15775
|
});
|
|
15703
15776
|
/**
|
|
15704
15777
|
* Partial patch for `updateRule` — any subset of the input fields, plus the
|
|
@@ -16624,7 +16697,7 @@ var TrackEnvelopeSchema = object({
|
|
|
16624
16697
|
* `snapshots[]` references — megabytes across a page of tracks. `slim`
|
|
16625
16698
|
* keeps every scalar the list surfaces actually render (ids, class(es),
|
|
16626
16699
|
* label / audioLabels / importance enrichment, firstSeen/lastSeen, state,
|
|
16627
|
-
* zonesVisited, bestEventId, envelope, hasFace) and returns `positions` /
|
|
16700
|
+
* zonesVisited, bestEventId, envelope, hasFace, hasRider) and returns `positions` /
|
|
16628
16701
|
* `snapshots` as EMPTY arrays — detail views re-fetch the full row via
|
|
16629
16702
|
* `getTrack`. Mirrors the event-store `projection` convention
|
|
16630
16703
|
* (`getObjectEvents` et al.).
|
|
@@ -16911,6 +16984,28 @@ var TrackSchema = object({
|
|
|
16911
16984
|
* `=== true` and render nothing otherwise, never infer "no face".
|
|
16912
16985
|
*/
|
|
16913
16986
|
hasFace: boolean().optional(),
|
|
16987
|
+
/**
|
|
16988
|
+
* This subject CONTAINS a folded rider — a person the rider-pairing step
|
|
16989
|
+
* ([D34](../decisions/adr-0034.md)) removed from the frame BEFORE the tracker,
|
|
16990
|
+
* so the passage is tracked once and as a VEHICLE.
|
|
16991
|
+
*
|
|
16992
|
+
* It exists because the fold's record was dishonest. D34 and the code both
|
|
16993
|
+
* said "the person is not lost — it is reported so both entities stay on the
|
|
16994
|
+
* record"; in fact the pair went into a per-processor RAM field behind an
|
|
16995
|
+
* accessor nobody called, and every durable surface said `vehicle`, full
|
|
16996
|
+
* stop. This is the composition note that makes the row true.
|
|
16997
|
+
*
|
|
16998
|
+
* A COMPOSITION, never a class and never a label. "This vehicle contains a
|
|
16999
|
+
* person" is not an answer to "what is this" — both label tiers would refuse
|
|
17000
|
+
* a macro token anyway (D89), and correctly. Nothing here changes what the
|
|
17001
|
+
* subject IS: a cyclist stays one vehicle track, occupancy still counts one,
|
|
17002
|
+
* and a `person` rule still does not fire for someone cycling past.
|
|
17003
|
+
*
|
|
17004
|
+
* **Absent ≠ false**, exactly like {@link hasFace}: every row written before
|
|
17005
|
+
* the column, and every hub that predates the field, omits it. Test
|
|
17006
|
+
* `=== true` and render nothing otherwise — never infer "no rider".
|
|
17007
|
+
*/
|
|
17008
|
+
hasRider: boolean().optional(),
|
|
16914
17009
|
...TrackFlagFields,
|
|
16915
17010
|
...TrackRetrainFields
|
|
16916
17011
|
});
|
|
@@ -34805,6 +34900,15 @@ function resolveInferenceDeviceEligibility(probed, stored, canRunRoot) {
|
|
|
34805
34900
|
excluded
|
|
34806
34901
|
};
|
|
34807
34902
|
}
|
|
34903
|
+
function resolveNodeInferenceUsability(eligibility) {
|
|
34904
|
+
const eligibleKeys = Object.keys(eligibility.eligible).toSorted();
|
|
34905
|
+
const unavailableKeys = eligibility.excluded.filter((e) => e.reason === "unavailable").map((e) => e.key).toSorted();
|
|
34906
|
+
return {
|
|
34907
|
+
usable: eligibleKeys.length > 0 || unavailableKeys.length === 0,
|
|
34908
|
+
unavailableKeys,
|
|
34909
|
+
eligibleKeys
|
|
34910
|
+
};
|
|
34911
|
+
}
|
|
34808
34912
|
/**
|
|
34809
34913
|
* Step-tree device jump (phase 1): the attach-payload roster of a node's
|
|
34810
34914
|
* enabled∧available inference devices with the balancer knobs (`weight`,
|
|
@@ -34823,6 +34927,67 @@ function buildInferenceDeviceRoster(eligible, caps) {
|
|
|
34823
34927
|
})).toSorted((a, b) => a.deviceKey < b.deviceKey ? -1 : a.deviceKey > b.deviceKey ? 1 : 0);
|
|
34824
34928
|
}
|
|
34825
34929
|
//#endregion
|
|
34930
|
+
//#region src/node-inference-usability-mirror.ts
|
|
34931
|
+
var NodeInferenceUsabilityMirror = class {
|
|
34932
|
+
state = /* @__PURE__ */ new Map();
|
|
34933
|
+
/**
|
|
34934
|
+
* Fold one observation in and report whether the caller should act.
|
|
34935
|
+
* Never throws.
|
|
34936
|
+
*/
|
|
34937
|
+
observe(nodeId, usable) {
|
|
34938
|
+
const prev = this.state.get(nodeId);
|
|
34939
|
+
if (usable) {
|
|
34940
|
+
this.state.set(nodeId, {
|
|
34941
|
+
usable: true,
|
|
34942
|
+
armed: false
|
|
34943
|
+
});
|
|
34944
|
+
return prev !== void 0 && !prev.usable ? "recovered" : null;
|
|
34945
|
+
}
|
|
34946
|
+
if (prev === void 0) {
|
|
34947
|
+
this.state.set(nodeId, {
|
|
34948
|
+
usable: true,
|
|
34949
|
+
armed: true
|
|
34950
|
+
});
|
|
34951
|
+
return null;
|
|
34952
|
+
}
|
|
34953
|
+
if (!prev.usable) {
|
|
34954
|
+
this.state.set(nodeId, {
|
|
34955
|
+
usable: false,
|
|
34956
|
+
armed: true
|
|
34957
|
+
});
|
|
34958
|
+
return null;
|
|
34959
|
+
}
|
|
34960
|
+
if (!prev.armed) {
|
|
34961
|
+
this.state.set(nodeId, {
|
|
34962
|
+
usable: true,
|
|
34963
|
+
armed: true
|
|
34964
|
+
});
|
|
34965
|
+
return null;
|
|
34966
|
+
}
|
|
34967
|
+
this.state.set(nodeId, {
|
|
34968
|
+
usable: false,
|
|
34969
|
+
armed: true
|
|
34970
|
+
});
|
|
34971
|
+
return "became-unusable";
|
|
34972
|
+
}
|
|
34973
|
+
/** Can this node be given cameras? Unknown nodes answer YES. */
|
|
34974
|
+
isUsable(nodeId) {
|
|
34975
|
+
return this.state.get(nodeId)?.usable ?? true;
|
|
34976
|
+
}
|
|
34977
|
+
/** Nodes currently excluded — for the placement log and diagnostics. */
|
|
34978
|
+
unusableNodeIds() {
|
|
34979
|
+
const out = [];
|
|
34980
|
+
for (const [nodeId, s] of this.state) if (!s.usable) out.push(nodeId);
|
|
34981
|
+
return out.toSorted();
|
|
34982
|
+
}
|
|
34983
|
+
forget(nodeId) {
|
|
34984
|
+
this.state.delete(nodeId);
|
|
34985
|
+
}
|
|
34986
|
+
reset() {
|
|
34987
|
+
this.state.clear();
|
|
34988
|
+
}
|
|
34989
|
+
};
|
|
34990
|
+
//#endregion
|
|
34826
34991
|
//#region src/orchestrator-types.ts
|
|
34827
34992
|
var PHASE_MODE_VALUES = new Set([
|
|
34828
34993
|
"disabled",
|
|
@@ -36333,6 +36498,69 @@ var LoadShedController = class LoadShedController {
|
|
|
36333
36498
|
}
|
|
36334
36499
|
}
|
|
36335
36500
|
};
|
|
36501
|
+
var DEFAULT_DETECTION_BLIND_CONFIG = {
|
|
36502
|
+
thresholdMs: 60 * 6e4,
|
|
36503
|
+
frameFreshnessMs: 5 * 6e4
|
|
36504
|
+
};
|
|
36505
|
+
var DetectionBlindMonitor = class {
|
|
36506
|
+
config;
|
|
36507
|
+
state = /* @__PURE__ */ new Map();
|
|
36508
|
+
constructor(config = DEFAULT_DETECTION_BLIND_CONFIG) {
|
|
36509
|
+
this.config = config;
|
|
36510
|
+
}
|
|
36511
|
+
/**
|
|
36512
|
+
* One inference frame arrived for a device. `hadDetection` is false for a
|
|
36513
|
+
* frame that returned nothing — which is the whole point: an empty frame
|
|
36514
|
+
* still proves the pipeline is alive, and it is the ONLY evidence that
|
|
36515
|
+
* distinguishes blind from detached.
|
|
36516
|
+
*/
|
|
36517
|
+
noteFrame(deviceId, hadDetection, now) {
|
|
36518
|
+
const prev = this.state.get(deviceId);
|
|
36519
|
+
if (hadDetection) {
|
|
36520
|
+
this.state.set(deviceId, {
|
|
36521
|
+
lastFrameAt: now,
|
|
36522
|
+
lastDetectionAt: now,
|
|
36523
|
+
reported: false
|
|
36524
|
+
});
|
|
36525
|
+
return;
|
|
36526
|
+
}
|
|
36527
|
+
this.state.set(deviceId, {
|
|
36528
|
+
lastFrameAt: now,
|
|
36529
|
+
lastDetectionAt: prev?.lastDetectionAt ?? now,
|
|
36530
|
+
reported: prev?.reported ?? false
|
|
36531
|
+
});
|
|
36532
|
+
}
|
|
36533
|
+
/**
|
|
36534
|
+
* Devices that have JUST crossed into blind. Call from an existing periodic
|
|
36535
|
+
* sweep; it allocates nothing on the healthy path.
|
|
36536
|
+
*/
|
|
36537
|
+
sweep(now) {
|
|
36538
|
+
const out = [];
|
|
36539
|
+
for (const [deviceId, s] of this.state) {
|
|
36540
|
+
if (s.reported) continue;
|
|
36541
|
+
if (now - s.lastFrameAt > this.config.frameFreshnessMs) continue;
|
|
36542
|
+
const blindForMs = now - s.lastDetectionAt;
|
|
36543
|
+
if (blindForMs < this.config.thresholdMs) continue;
|
|
36544
|
+
this.state.set(deviceId, {
|
|
36545
|
+
...s,
|
|
36546
|
+
reported: true
|
|
36547
|
+
});
|
|
36548
|
+
out.push({
|
|
36549
|
+
deviceId,
|
|
36550
|
+
blindForMs,
|
|
36551
|
+
thresholdMs: this.config.thresholdMs
|
|
36552
|
+
});
|
|
36553
|
+
}
|
|
36554
|
+
return out;
|
|
36555
|
+
}
|
|
36556
|
+
/** Drop a device (detach / unbind), so a re-attach starts a fresh clock. */
|
|
36557
|
+
forget(deviceId) {
|
|
36558
|
+
this.state.delete(deviceId);
|
|
36559
|
+
}
|
|
36560
|
+
reset() {
|
|
36561
|
+
this.state.clear();
|
|
36562
|
+
}
|
|
36563
|
+
};
|
|
36336
36564
|
//#endregion
|
|
36337
36565
|
//#region src/agent-load-service.ts
|
|
36338
36566
|
var AgentLoadService = class AgentLoadService {
|
|
@@ -38585,6 +38813,7 @@ var DetectionWiringController = class {
|
|
|
38585
38813
|
if (!this.deps.ctx()) return;
|
|
38586
38814
|
const { deviceId, frame, frameHandle, capturedAt } = payload;
|
|
38587
38815
|
this.deps.watchdog()?.noteSignal(deviceId, "detection");
|
|
38816
|
+
this.deps.blindMonitor()?.noteFrame(deviceId, frame.detections.length > 0, Date.now());
|
|
38588
38817
|
if (frame.detections.length === 0 && (frame.discarded?.length ?? 0) === 0) return;
|
|
38589
38818
|
this.deps.eventBus.emit({
|
|
38590
38819
|
id: `detection-${deviceId}-${Date.now()}`,
|
|
@@ -40372,6 +40601,34 @@ var PlacementService = class {
|
|
|
40372
40601
|
constructor(deps) {
|
|
40373
40602
|
this.deps = deps;
|
|
40374
40603
|
}
|
|
40604
|
+
/**
|
|
40605
|
+
* Drop nodes that cannot infer from a candidate set.
|
|
40606
|
+
*
|
|
40607
|
+
* Fails OPEN in one specific way that matters: if the filter would leave NO
|
|
40608
|
+
* candidate at all, the unfiltered set is used. Placing a camera on a node
|
|
40609
|
+
* that might fail its inference is bad; refusing to place it anywhere
|
|
40610
|
+
* because every node is currently mid-probe is worse, and that is the exact
|
|
40611
|
+
* shape a cold cluster start has.
|
|
40612
|
+
*/
|
|
40613
|
+
filterInferenceUsable(nodes, deviceId) {
|
|
40614
|
+
const usable = nodes.filter((nodeId) => this.deps.isNodeInferenceUsable(nodeId));
|
|
40615
|
+
if (usable.length === nodes.length) return nodes;
|
|
40616
|
+
if (usable.length === 0) {
|
|
40617
|
+
this.deps.ctx().logger.warn("every candidate node reports no usable inference device — placing anyway", {
|
|
40618
|
+
tags: { deviceId },
|
|
40619
|
+
meta: { nodes: [...nodes] }
|
|
40620
|
+
});
|
|
40621
|
+
return nodes;
|
|
40622
|
+
}
|
|
40623
|
+
this.deps.ctx().logger.info("nodes dropped from placement — no usable inference device", {
|
|
40624
|
+
tags: { deviceId },
|
|
40625
|
+
meta: {
|
|
40626
|
+
dropped: nodes.filter((n) => !usable.includes(n)),
|
|
40627
|
+
kept: usable
|
|
40628
|
+
}
|
|
40629
|
+
});
|
|
40630
|
+
return usable;
|
|
40631
|
+
}
|
|
40375
40632
|
async dispatchCamera(runnerConfig) {
|
|
40376
40633
|
if (!this.deps.ctx()) throw new Error("PipelineOrchestrator: dispatchCamera called before initialize");
|
|
40377
40634
|
if (this.deps.maintenancePaused?.() === true) return {
|
|
@@ -40386,7 +40643,7 @@ var PlacementService = class {
|
|
|
40386
40643
|
};
|
|
40387
40644
|
const preferredAgent = await this.deps.readPipelinePin(runnerConfig.deviceId);
|
|
40388
40645
|
const loads = await this.deps.loadService.collectAgentLoad({ onlyEnabled: true });
|
|
40389
|
-
const eligible = this.deps.topology.detectionEligibleNodes(runnerConfig.deviceId);
|
|
40646
|
+
const eligible = this.filterInferenceUsable(this.deps.topology.detectionEligibleNodes(runnerConfig.deviceId), runnerConfig.deviceId);
|
|
40390
40647
|
const nodeCaps = await this.deps.settingsStore.buildNodeCaps();
|
|
40391
40648
|
const motionRoleCandidate = pickMotionRolePlacement({
|
|
40392
40649
|
isAnalyzer: runnerConfig.motionSources.includes("analyzer"),
|
|
@@ -44259,7 +44516,10 @@ var SessionDispatchController = class {
|
|
|
44259
44516
|
await this.deps.reconcilePlacementFromRunners();
|
|
44260
44517
|
const preferredAgent = await this.deps.readPipelinePin(deviceId);
|
|
44261
44518
|
const loads = await this.agentLoads.get(CLUSTER_KEY);
|
|
44262
|
-
const
|
|
44519
|
+
const allEligible = this.deps.topology.detectionEligibleNodes(deviceId);
|
|
44520
|
+
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
44521
|
+
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
44522
|
+
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
44263
44523
|
const decision = balance({
|
|
44264
44524
|
nodes: loads,
|
|
44265
44525
|
preferredAgent,
|
|
@@ -45181,6 +45441,78 @@ var ZonesProvider = class {
|
|
|
45181
45441
|
//#endregion
|
|
45182
45442
|
//#region src/orchestrator-bootstrap.ts
|
|
45183
45443
|
/**
|
|
45444
|
+
* `orchestrator-bootstrap.ts` — the controller-construction body of
|
|
45445
|
+
* `onInitialize` (S11 Task 4, last extraction of the S11 series).
|
|
45446
|
+
*
|
|
45447
|
+
* `buildOrchestratorControllers(deps)` constructs all ~16 controllers the
|
|
45448
|
+
* addon owns, in the EXACT order `onInitialize` built them in before this
|
|
45449
|
+
* extraction (ledger → topology → loadService → audio → settingsStore →
|
|
45450
|
+
* loadShed → placement → session → deviceConfig → cameraStatusService →
|
|
45451
|
+
* reconcile → [settings load + localNodeId resolve] → nodeLifecycle →
|
|
45452
|
+
* [event subs + readiness seed] → pipelineWatchdog → [timers + legacy
|
|
45453
|
+
* migration] → zoneRulesProvider → zonesProvider →
|
|
45454
|
+
* [wireOrchestratorSubscriptions + eager reconcile] → detectionWiring).
|
|
45455
|
+
* Every dep closure is carried over byte-for-byte — only `this.X` became
|
|
45456
|
+
* either a local variable (for a controller already constructed earlier in
|
|
45457
|
+
* THIS function) or a `deps.X` accessor (for addon-owned state/methods that
|
|
45458
|
+
* stay on `PipelineOrchestratorAddon`, or for a controller referenced
|
|
45459
|
+
* before its own construction point — a genuine forward ref, always via a
|
|
45460
|
+
* LAZY closure, exactly as it was in `onInitialize`).
|
|
45461
|
+
*
|
|
45462
|
+
* Three things could not simply become local variables, because an EXISTING
|
|
45463
|
+
* (unmoved) addon method reads the REAL class field synchronously, mid-
|
|
45464
|
+
* construction, not through a lazy closure:
|
|
45465
|
+
* - `applyRuntimeSettings` (stays on the addon) reads `this.topology!`
|
|
45466
|
+
* and `this.reconcile!` directly — called synchronously from the
|
|
45467
|
+
* settings-load block below. `deps.setTopology`/`deps.setReconcile`
|
|
45468
|
+
* progressively sync the REAL addon fields the instant each is built,
|
|
45469
|
+
* so `applyRuntimeSettings` sees them.
|
|
45470
|
+
* - `subscribeToAudioAnalyzerReadiness` (stays on the addon) reads
|
|
45471
|
+
* `this.nodeLifecycle!.readinessRegistry` AND `this.audio!` directly —
|
|
45472
|
+
* called synchronously right after `nodeLifecycle` is built (readiness
|
|
45473
|
+
* seed + the `AgentOnline` handler). `deps.setNodeLifecycle` /
|
|
45474
|
+
* `deps.setAudio` cover it the same way. This one bit LIVE: `audio` is
|
|
45475
|
+
* constructed early (fourth) but `subscribeToAudioAnalyzerReadiness`
|
|
45476
|
+
* wasn't called until much later (after `nodeLifecycle` exists), so the
|
|
45477
|
+
* old code never noticed `this.audio` was still unsynced — until
|
|
45478
|
+
* `ReadinessRegistry.onReadyState`'s `queueMicrotask` hydration replay
|
|
45479
|
+
* (an already-`ready` cap fires the handler once, asynchronously, on
|
|
45480
|
+
* subscribe) won the race against the `await buildOrchestratorControllers(...)`
|
|
45481
|
+
* continuation in `onInitialize` that assigns `this.audio` — both are
|
|
45482
|
+
* microtasks, and the replay's `queueMicrotask` was enqueued (during
|
|
45483
|
+
* this still-synchronous function body) strictly before this function's
|
|
45484
|
+
* `return` resolves the outer promise, so it always runs first. Fixed
|
|
45485
|
+
* by syncing `this.audio` the instant it's constructed, exactly like
|
|
45486
|
+
* `topology`/`reconcile`/`nodeLifecycle` — no lazy closure ever reads a
|
|
45487
|
+
* controller before this function has assigned its real field.
|
|
45488
|
+
* Every other controller-to-controller reference in this function — even
|
|
45489
|
+
* ones that look like a forward ref (e.g. `loadShed`'s `attach` closure
|
|
45490
|
+
* onto `placement`, built one step later) — is a LAZY closure that is
|
|
45491
|
+
* never invoked synchronously during construction (only later, off
|
|
45492
|
+
* event-bus subscriptions, timers, or RPC handlers), so a plain local
|
|
45493
|
+
* `let` variable captured by reference is sufficient and exactly
|
|
45494
|
+
* preserves the original semantics: see each field's inline comment in
|
|
45495
|
+
* `index.ts` for the original construction-order reasoning, unchanged
|
|
45496
|
+
* here.
|
|
45497
|
+
*
|
|
45498
|
+
* `globalSettings` is a purely local variable — read/written ONLY by
|
|
45499
|
+
* closures built inside this same function (the one exception,
|
|
45500
|
+
* `deviceConfig`'s `setGlobalSettings`, is ALSO built here, so the local
|
|
45501
|
+
* variable stays the single source of truth for every reader). No code
|
|
45502
|
+
* outside this closure network ever read the former `this.globalSettings`
|
|
45503
|
+
* class field, so it isn't part of the returned `OrchestratorControllers`
|
|
45504
|
+
* — the field was removed from the class entirely (S11 Task 4 cleanup).
|
|
45505
|
+
* `localNodeId` IS returned — `index.ts` still reads `this.localNodeId`
|
|
45506
|
+
* from a couple of unrelated cap methods (`dumpDiagnostics`,
|
|
45507
|
+
* `getDecoderAssignments`), so the class field is populated once, from the
|
|
45508
|
+
* returned `OrchestratorControllers`, after this function returns.
|
|
45509
|
+
* `failoverPolicy` is the opposite case from `globalSettings` —
|
|
45510
|
+
* `applyRuntimeSettings` (unmoved) writes the REAL field on every future
|
|
45511
|
+
* settings change, so `nodeLifecycle`'s `failoverPolicy` closure MUST keep
|
|
45512
|
+
* reading through `deps.failoverPolicy()` (never a local snapshot) to see
|
|
45513
|
+
* those later updates.
|
|
45514
|
+
*/
|
|
45515
|
+
/**
|
|
45184
45516
|
* Construct every orchestrator controller, wire the agent-lifecycle event
|
|
45185
45517
|
* subscriptions + readiness seed + detection-wiring subscriptions, and
|
|
45186
45518
|
* kick the eager reconcile — the ENTIRE former `onInitialize` body between
|
|
@@ -45339,6 +45671,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45339
45671
|
localNodeId: () => localNodeId,
|
|
45340
45672
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
45341
45673
|
getEligibleInferenceDevices: (nodeId) => deps.getEligibleInferenceDevices(nodeId),
|
|
45674
|
+
isNodeInferenceUsable: (nodeId) => deps.isNodeInferenceUsable(nodeId),
|
|
45342
45675
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
45343
45676
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
45344
45677
|
inferenceRotation,
|
|
@@ -45368,6 +45701,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45368
45701
|
listZones: async (deviceId) => await zonesProvider?.listZones({ deviceId }) ?? [],
|
|
45369
45702
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
45370
45703
|
getEligibleInferenceDevices: (nodeId) => deps.getEligibleInferenceDevices(nodeId),
|
|
45704
|
+
isNodeInferenceUsable: (nodeId) => deps.isNodeInferenceUsable(nodeId),
|
|
45371
45705
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
45372
45706
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
45373
45707
|
inferenceRotation,
|
|
@@ -45526,7 +45860,34 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45526
45860
|
thresholds: DEFAULT_WATCHDOG_THRESHOLDS
|
|
45527
45861
|
});
|
|
45528
45862
|
pipelineWatchdog.start(deps.watchdogIntervalMs);
|
|
45529
|
-
const
|
|
45863
|
+
const blindMonitor = new DetectionBlindMonitor();
|
|
45864
|
+
const pendingRetryTimer = setInterval(() => {
|
|
45865
|
+
reconcile.retryPendingDispatches();
|
|
45866
|
+
for (const blind of blindMonitor.sweep(Date.now())) {
|
|
45867
|
+
deps.ctx().logger.error("camera is being analysed and detecting NOTHING", {
|
|
45868
|
+
tags: { deviceId: blind.deviceId },
|
|
45869
|
+
meta: {
|
|
45870
|
+
blindForMs: blind.blindForMs,
|
|
45871
|
+
thresholdMs: blind.thresholdMs
|
|
45872
|
+
}
|
|
45873
|
+
});
|
|
45874
|
+
deps.ctx().eventBus.emit({
|
|
45875
|
+
id: (0, node_crypto.randomUUID)(),
|
|
45876
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
45877
|
+
source: {
|
|
45878
|
+
type: "device",
|
|
45879
|
+
id: blind.deviceId,
|
|
45880
|
+
deviceId: blind.deviceId
|
|
45881
|
+
},
|
|
45882
|
+
category: EventCategory.PipelineDetectionBlind,
|
|
45883
|
+
data: {
|
|
45884
|
+
deviceId: blind.deviceId,
|
|
45885
|
+
blindForMs: blind.blindForMs,
|
|
45886
|
+
thresholdMs: blind.thresholdMs
|
|
45887
|
+
}
|
|
45888
|
+
});
|
|
45889
|
+
}
|
|
45890
|
+
}, PENDING_RETRY_INTERVAL_MS);
|
|
45530
45891
|
const autoRebalanceTimer = setInterval(() => void placement.runAutoRebalance(), AUTO_REBALANCE_INTERVAL_MS);
|
|
45531
45892
|
deps.ctx().logger.info("Pipeline orchestrator detection-wiring subscriptions installed");
|
|
45532
45893
|
migrateLegacyFlagsToBindings({
|
|
@@ -45653,6 +46014,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45653
46014
|
loadShed,
|
|
45654
46015
|
nodeStress,
|
|
45655
46016
|
watchdog: () => pipelineWatchdog,
|
|
46017
|
+
blindMonitor: () => blindMonitor,
|
|
45656
46018
|
api: () => deps.ctx().api ?? null,
|
|
45657
46019
|
eventBus: deps.ctx().eventBus,
|
|
45658
46020
|
logger: deps.ctx().logger,
|
|
@@ -46230,6 +46592,48 @@ function deriveRuntimeSettings(config) {
|
|
|
46230
46592
|
//#endregion
|
|
46231
46593
|
//#region src/index.ts
|
|
46232
46594
|
/**
|
|
46595
|
+
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
46596
|
+
*
|
|
46597
|
+
* Owns the control plane for the distributed detection pipeline. Tracks
|
|
46598
|
+
* which `addon-pipeline-runner` instance is responsible for each camera,
|
|
46599
|
+
* queries each runner's capacity + hardware profile, and dispatches
|
|
46600
|
+
* `attachCamera` / `detachCamera` cap calls to the appropriate runner via
|
|
46601
|
+
* the Moleculer broker. Persists manual pins via the per-device settings
|
|
46602
|
+
* store so user overrides survive hub restarts.
|
|
46603
|
+
*
|
|
46604
|
+
* Strictly control plane — never sees a frame, never runs inference, never
|
|
46605
|
+
* drains queues. The runner does that work locally on each vision node.
|
|
46606
|
+
*
|
|
46607
|
+
* ## Dispatch entrypoints
|
|
46608
|
+
*
|
|
46609
|
+
* - **`dispatchCamera(runnerConfig)`** — in-process only (not in the cap
|
|
46610
|
+
* def). Called by `DetectionWiringService` with a freshly-built
|
|
46611
|
+
* `RunnerCameraConfig`. Runs the L1/L2/L3 balancer, picks an agent, then
|
|
46612
|
+
* issues `broker.call('pipeline-runner.pipeline-runner.attachCamera', ...)`.
|
|
46613
|
+
* Caches the config so future rebalance / pinning can re-dispatch without
|
|
46614
|
+
* the caller rebuilding it.
|
|
46615
|
+
*
|
|
46616
|
+
* - **`assignCamera({deviceId, agentNodeId})`** — cap method. Persists a
|
|
46617
|
+
* manual `preferredAgent` pin to per-device settings and re-dispatches
|
|
46618
|
+
* immediately if the camera is already attached.
|
|
46619
|
+
*
|
|
46620
|
+
* - **`unassignCamera({deviceId})`** — cap method. Clears the pin,
|
|
46621
|
+
* detaches the camera from its current agent.
|
|
46622
|
+
*
|
|
46623
|
+
* - **`rebalance()`** — cap method. Walks every cached camera config and
|
|
46624
|
+
* re-runs the balancer; migrates cameras whose current agent is no
|
|
46625
|
+
* longer optimal. Pinned cameras are never migrated.
|
|
46626
|
+
*
|
|
46627
|
+
* ## Runner dispatch
|
|
46628
|
+
*
|
|
46629
|
+
* All runner calls go through `this.ctx.api.pipelineRunner.*` (the generated
|
|
46630
|
+
* tRPC cap router). The router extracts `nodeId` from the input and
|
|
46631
|
+
* routes transparently to the local runner (direct in-process call) or
|
|
46632
|
+
* to a remote agent (via Moleculer). Node discovery is driven by
|
|
46633
|
+
* `EventCategory.AgentOnline/AgentOffline` events forwarded by
|
|
46634
|
+
* `agent-registry.service.ts`.
|
|
46635
|
+
*/
|
|
46636
|
+
/**
|
|
46233
46637
|
* The FULL action catalog, under the name the HUB HARVESTS.
|
|
46234
46638
|
*
|
|
46235
46639
|
* The hub's forked-addon harvest imports this entry module and reads the
|
|
@@ -46488,6 +46892,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
46488
46892
|
detachOn: (nodeId, deviceId) => this.detachOn(nodeId, deviceId),
|
|
46489
46893
|
readPipelinePin: (deviceId) => this.readPipelinePin(deviceId),
|
|
46490
46894
|
getEligibleInferenceDevices: (nodeId) => this.resolveEligibleInferenceDevices(nodeId),
|
|
46895
|
+
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
46491
46896
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
46492
46897
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
46493
46898
|
subscribeToAudioAnalyzerReadiness: (nodeId) => this.subscribeToAudioAnalyzerReadiness(nodeId),
|
|
@@ -47144,11 +47549,66 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
47144
47549
|
const stored = (await this.settingsStore.readAgentSettingsMap())[nodeId]?.inferenceDevices ?? {};
|
|
47145
47550
|
const { probed } = await this.probeNodeInferenceDevices(nodeId);
|
|
47146
47551
|
const catalog = await this.settingsStore.getCatalogForNode(nodeId);
|
|
47147
|
-
const
|
|
47552
|
+
const eligibility = resolveInferenceDeviceEligibility(probed, stored, catalog ? makeRootCapabilityGuard(catalog) : void 0);
|
|
47553
|
+
const { eligible, excluded } = eligibility;
|
|
47148
47554
|
this.logInferenceDeviceShrinkage(nodeId, eligible, excluded);
|
|
47555
|
+
this.noteNodeInferenceUsability(nodeId, resolveNodeInferenceUsability(eligibility));
|
|
47149
47556
|
return eligible;
|
|
47150
47557
|
}
|
|
47151
47558
|
/**
|
|
47559
|
+
* Fold one usability observation into the placement mirror and announce the
|
|
47560
|
+
* transition.
|
|
47561
|
+
*
|
|
47562
|
+
* This is called from the SAME read the dispatcher already makes, plus the
|
|
47563
|
+
* session controller's background refresher — so a node's usability is
|
|
47564
|
+
* re-observed on the cadence placement itself runs at, without a timer of
|
|
47565
|
+
* its own. The mirror is what makes the answer readable synchronously by
|
|
47566
|
+
* `detectionEligibleNodes`' callers; see its own docblock for why it refuses
|
|
47567
|
+
* to act on a single bad read.
|
|
47568
|
+
*
|
|
47569
|
+
* The system event is emitted ONLY on the transition into unusable — the
|
|
47570
|
+
* kind is `node-inference-unavailable` and it is authorable in a rule.
|
|
47571
|
+
* Recovery is logged and deliberately has no event: a notification saying
|
|
47572
|
+
* the broken thing is no longer broken is a second rule nobody asked for.
|
|
47573
|
+
*/
|
|
47574
|
+
noteNodeInferenceUsability(nodeId, usability) {
|
|
47575
|
+
const transition = this.nodeInferenceUsability.observe(nodeId, usability.usable);
|
|
47576
|
+
if (transition === null) return;
|
|
47577
|
+
if (transition === "recovered") {
|
|
47578
|
+
this.ctx.logger.info("node inference recovered — back in the placement candidate set", {
|
|
47579
|
+
tags: { nodeId },
|
|
47580
|
+
meta: { eligible: usability.eligibleKeys }
|
|
47581
|
+
});
|
|
47582
|
+
return;
|
|
47583
|
+
}
|
|
47584
|
+
this.ctx.logger.error("node has NO usable inference device — removed from the placement candidate set", {
|
|
47585
|
+
tags: { nodeId },
|
|
47586
|
+
meta: {
|
|
47587
|
+
unavailable: usability.unavailableKeys,
|
|
47588
|
+
eligible: usability.eligibleKeys
|
|
47589
|
+
}
|
|
47590
|
+
});
|
|
47591
|
+
this.ctx.eventBus.emit({
|
|
47592
|
+
id: (0, node_crypto.randomUUID)(),
|
|
47593
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
47594
|
+
source: {
|
|
47595
|
+
type: "addon",
|
|
47596
|
+
id: "pipeline-orchestrator",
|
|
47597
|
+
addonId: "pipeline-orchestrator"
|
|
47598
|
+
},
|
|
47599
|
+
category: EventCategory.PipelineNodeInferenceUnavailable,
|
|
47600
|
+
data: {
|
|
47601
|
+
nodeId,
|
|
47602
|
+
unavailableDeviceKeys: usability.unavailableKeys
|
|
47603
|
+
}
|
|
47604
|
+
});
|
|
47605
|
+
}
|
|
47606
|
+
/** Can this node be given cameras? Read synchronously by the two dispatch
|
|
47607
|
+
* paths before they hand `eligibleNodes` to the balancer. */
|
|
47608
|
+
isNodeInferenceUsable(nodeId) {
|
|
47609
|
+
return this.nodeInferenceUsability.isUsable(nodeId);
|
|
47610
|
+
}
|
|
47611
|
+
/**
|
|
47152
47612
|
* Per-node signature of the last logged eligible/excluded split, so
|
|
47153
47613
|
* {@link logInferenceDeviceShrinkage} fires on CHANGE only. This runs on
|
|
47154
47614
|
* every camera dispatch; without the gate it would be one line per dispatch
|
|
@@ -47156,6 +47616,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
47156
47616
|
*/
|
|
47157
47617
|
lastInferenceEligibilitySignature = /* @__PURE__ */ new Map();
|
|
47158
47618
|
/**
|
|
47619
|
+
* Which nodes can still infer. Refreshed by
|
|
47620
|
+
* {@link resolveEligibleInferenceDevices}; read synchronously by both
|
|
47621
|
+
* dispatch paths so a node whose every enabled accelerator vanished stops
|
|
47622
|
+
* receiving cameras. little-unraid was handed cameras for a whole day in
|
|
47623
|
+
* exactly that state.
|
|
47624
|
+
*/
|
|
47625
|
+
nodeInferenceUsability = new NodeInferenceUsabilityMirror();
|
|
47626
|
+
/**
|
|
47159
47627
|
* Say out loud which of a node's inference devices were dropped from the
|
|
47160
47628
|
* candidate set, and why.
|
|
47161
47629
|
*
|
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import fs from "node:fs";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
//#region ../types/dist/event-category-
|
|
5
|
+
//#region ../types/dist/event-category-Bxo5yJjt.mjs
|
|
5
6
|
var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
6
7
|
EventCategory["SystemBoot"] = "system.boot";
|
|
7
8
|
EventCategory["SystemAddonsReady"] = "system.addons-ready";
|
|
@@ -208,6 +209,33 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
|
|
|
208
209
|
EventCategory["PipelineCameraAssigned"] = "pipeline.camera-assigned";
|
|
209
210
|
EventCategory["PipelineCameraUnassigned"] = "pipeline.camera-unassigned";
|
|
210
211
|
/**
|
|
212
|
+
* A node the orchestrator would otherwise place cameras on has NO usable
|
|
213
|
+
* inference device: the operator enabled one or more accelerators there and
|
|
214
|
+
* the live probe reports every one of them unavailable. Emitted once per
|
|
215
|
+
* TRANSITION into that state (never per dispatch), and the node is dropped
|
|
216
|
+
* from the placement candidate set for as long as it holds.
|
|
217
|
+
*
|
|
218
|
+
* This exists because the state was previously invisible: little-unraid
|
|
219
|
+
* absorbed 283k inference errors in a day while still being handed cameras,
|
|
220
|
+
* and nothing in the system said so.
|
|
221
|
+
*
|
|
222
|
+
* A node with no accelerators configured at all is NOT this — its devices
|
|
223
|
+
* are `disabled`, not `unavailable`, and the runner's default CPU pool
|
|
224
|
+
* serves it exactly as before.
|
|
225
|
+
*/
|
|
226
|
+
EventCategory["PipelineNodeInferenceUnavailable"] = "pipeline.node-inference-unavailable";
|
|
227
|
+
/**
|
|
228
|
+
* A camera has an OPEN detection session and has produced no detection at
|
|
229
|
+
* all for longer than the blind threshold — the camera is being decoded and
|
|
230
|
+
* inferred and is returning nothing. Emitted once per transition into blind,
|
|
231
|
+
* per camera.
|
|
232
|
+
*
|
|
233
|
+
* The failure it reports: a 1h43 detection blackout on the entrance camera
|
|
234
|
+
* that nobody noticed, because "a camera that detects nothing" and "a quiet
|
|
235
|
+
* camera" produce byte-identical silence.
|
|
236
|
+
*/
|
|
237
|
+
EventCategory["PipelineDetectionBlind"] = "pipeline.detection-blind";
|
|
238
|
+
/**
|
|
211
239
|
* Per-camera pipeline config was mutated by the orchestrator
|
|
212
240
|
* (3-level settings change via `setAgentAddonDefaults` /
|
|
213
241
|
* `setCameraStepToggle` / `setCameraPipelineForAgent` or a
|
|
@@ -14899,6 +14927,8 @@ var NcSystemEventKindSchema = _enum([
|
|
|
14899
14927
|
"stream-offline",
|
|
14900
14928
|
"node-online",
|
|
14901
14929
|
"node-offline",
|
|
14930
|
+
"node-inference-unavailable",
|
|
14931
|
+
"detection-blind",
|
|
14902
14932
|
"addon-update-available",
|
|
14903
14933
|
"server-update-available",
|
|
14904
14934
|
"alarm-triggered",
|
|
@@ -15670,7 +15700,50 @@ var NcRuleInputSchema = object({
|
|
|
15670
15700
|
* tests every zone the track visited and a `crossing` condition can no longer
|
|
15671
15701
|
* be satisfied, because a closed track carries no crossing.
|
|
15672
15702
|
*/
|
|
15673
|
-
waitForEnhancement: boolean().optional()
|
|
15703
|
+
waitForEnhancement: boolean().optional(),
|
|
15704
|
+
/**
|
|
15705
|
+
* GROUP a burst of subjects into ONE notification that grows.
|
|
15706
|
+
*
|
|
15707
|
+
* Seconds of quiet after the last matching subject before the burst is
|
|
15708
|
+
* considered over. While it is open, the first subject enqueues immediately —
|
|
15709
|
+
* **exactly as today, with no added latency** — and every real growth (a new
|
|
15710
|
+
* subject, or a name confirmed on one already in it) REPLACES that
|
|
15711
|
+
* notification with an updated one naming everybody. The push carries the
|
|
15712
|
+
* group's own coalescing tag, so the phone replaces rather than stacks.
|
|
15713
|
+
*
|
|
15714
|
+
* `0` / absent = off, and off is today's behaviour byte for byte.
|
|
15715
|
+
*
|
|
15716
|
+
* ### Why an idle cutoff and not a window
|
|
15717
|
+
*
|
|
15718
|
+
* The measured seven-person arrival on device 590 spans 110 s with every
|
|
15719
|
+
* internal gap under 30 s. A 12 s fixed window cuts it into three groups; an
|
|
15720
|
+
* idle cutoff holds it as one and ends it when the arrival actually ends.
|
|
15721
|
+
* 30 is Frigate's shipped value for the same decision.
|
|
15722
|
+
*
|
|
15723
|
+
* ### What it replaces
|
|
15724
|
+
*
|
|
15725
|
+
* The blind cooldown, which collapses a burst by DISCARDING it. Measured on
|
|
15726
|
+
* device 615 / *Persona su Uscio* over six days: 116 qualifying tracks → 74
|
|
15727
|
+
* notifications, **44 (37.9%) suppressed outright**, 23 of them overlapping a
|
|
15728
|
+
* track that did fire and 7 carrying a confirmed identity nobody heard about.
|
|
15729
|
+
* A group collapses the same volume by MERGING, so the cooldown becomes a
|
|
15730
|
+
* budget over GROUPS — which is what it always meant — and a growth is never
|
|
15731
|
+
* throttled by the window its own first member spent.
|
|
15732
|
+
*
|
|
15733
|
+
* ### Interaction with {@link waitForEnhancement}
|
|
15734
|
+
*
|
|
15735
|
+
* They compose, and the order matters. `waitForEnhancement` defers the rule to
|
|
15736
|
+
* TRACK CLOSE, so with both set the group is opened by the first member to
|
|
15737
|
+
* CLOSE — already carrying its name — and grows as later members close. That
|
|
15738
|
+
* is later, and complete. With grouping alone the group opens on the first
|
|
15739
|
+
* object event and picks up names as they are confirmed, through the growth
|
|
15740
|
+
* path. Neither combination fires twice for one subject.
|
|
15741
|
+
*
|
|
15742
|
+
* `.optional()` and deliberately NOT `.default()`: a Zod default does not run
|
|
15743
|
+
* on the addon cap path, so absent must keep meaning what it meant before this
|
|
15744
|
+
* field existed.
|
|
15745
|
+
*/
|
|
15746
|
+
groupIdleSec: number().int().min(0).max(600).optional()
|
|
15674
15747
|
});
|
|
15675
15748
|
/**
|
|
15676
15749
|
* Partial patch for `updateRule` — any subset of the input fields, plus the
|
|
@@ -16596,7 +16669,7 @@ var TrackEnvelopeSchema = object({
|
|
|
16596
16669
|
* `snapshots[]` references — megabytes across a page of tracks. `slim`
|
|
16597
16670
|
* keeps every scalar the list surfaces actually render (ids, class(es),
|
|
16598
16671
|
* label / audioLabels / importance enrichment, firstSeen/lastSeen, state,
|
|
16599
|
-
* zonesVisited, bestEventId, envelope, hasFace) and returns `positions` /
|
|
16672
|
+
* zonesVisited, bestEventId, envelope, hasFace, hasRider) and returns `positions` /
|
|
16600
16673
|
* `snapshots` as EMPTY arrays — detail views re-fetch the full row via
|
|
16601
16674
|
* `getTrack`. Mirrors the event-store `projection` convention
|
|
16602
16675
|
* (`getObjectEvents` et al.).
|
|
@@ -16883,6 +16956,28 @@ var TrackSchema = object({
|
|
|
16883
16956
|
* `=== true` and render nothing otherwise, never infer "no face".
|
|
16884
16957
|
*/
|
|
16885
16958
|
hasFace: boolean().optional(),
|
|
16959
|
+
/**
|
|
16960
|
+
* This subject CONTAINS a folded rider — a person the rider-pairing step
|
|
16961
|
+
* ([D34](../decisions/adr-0034.md)) removed from the frame BEFORE the tracker,
|
|
16962
|
+
* so the passage is tracked once and as a VEHICLE.
|
|
16963
|
+
*
|
|
16964
|
+
* It exists because the fold's record was dishonest. D34 and the code both
|
|
16965
|
+
* said "the person is not lost — it is reported so both entities stay on the
|
|
16966
|
+
* record"; in fact the pair went into a per-processor RAM field behind an
|
|
16967
|
+
* accessor nobody called, and every durable surface said `vehicle`, full
|
|
16968
|
+
* stop. This is the composition note that makes the row true.
|
|
16969
|
+
*
|
|
16970
|
+
* A COMPOSITION, never a class and never a label. "This vehicle contains a
|
|
16971
|
+
* person" is not an answer to "what is this" — both label tiers would refuse
|
|
16972
|
+
* a macro token anyway (D89), and correctly. Nothing here changes what the
|
|
16973
|
+
* subject IS: a cyclist stays one vehicle track, occupancy still counts one,
|
|
16974
|
+
* and a `person` rule still does not fire for someone cycling past.
|
|
16975
|
+
*
|
|
16976
|
+
* **Absent ≠ false**, exactly like {@link hasFace}: every row written before
|
|
16977
|
+
* the column, and every hub that predates the field, omits it. Test
|
|
16978
|
+
* `=== true` and render nothing otherwise — never infer "no rider".
|
|
16979
|
+
*/
|
|
16980
|
+
hasRider: boolean().optional(),
|
|
16886
16981
|
...TrackFlagFields,
|
|
16887
16982
|
...TrackRetrainFields
|
|
16888
16983
|
});
|
|
@@ -34777,6 +34872,15 @@ function resolveInferenceDeviceEligibility(probed, stored, canRunRoot) {
|
|
|
34777
34872
|
excluded
|
|
34778
34873
|
};
|
|
34779
34874
|
}
|
|
34875
|
+
function resolveNodeInferenceUsability(eligibility) {
|
|
34876
|
+
const eligibleKeys = Object.keys(eligibility.eligible).toSorted();
|
|
34877
|
+
const unavailableKeys = eligibility.excluded.filter((e) => e.reason === "unavailable").map((e) => e.key).toSorted();
|
|
34878
|
+
return {
|
|
34879
|
+
usable: eligibleKeys.length > 0 || unavailableKeys.length === 0,
|
|
34880
|
+
unavailableKeys,
|
|
34881
|
+
eligibleKeys
|
|
34882
|
+
};
|
|
34883
|
+
}
|
|
34780
34884
|
/**
|
|
34781
34885
|
* Step-tree device jump (phase 1): the attach-payload roster of a node's
|
|
34782
34886
|
* enabled∧available inference devices with the balancer knobs (`weight`,
|
|
@@ -34795,6 +34899,67 @@ function buildInferenceDeviceRoster(eligible, caps) {
|
|
|
34795
34899
|
})).toSorted((a, b) => a.deviceKey < b.deviceKey ? -1 : a.deviceKey > b.deviceKey ? 1 : 0);
|
|
34796
34900
|
}
|
|
34797
34901
|
//#endregion
|
|
34902
|
+
//#region src/node-inference-usability-mirror.ts
|
|
34903
|
+
var NodeInferenceUsabilityMirror = class {
|
|
34904
|
+
state = /* @__PURE__ */ new Map();
|
|
34905
|
+
/**
|
|
34906
|
+
* Fold one observation in and report whether the caller should act.
|
|
34907
|
+
* Never throws.
|
|
34908
|
+
*/
|
|
34909
|
+
observe(nodeId, usable) {
|
|
34910
|
+
const prev = this.state.get(nodeId);
|
|
34911
|
+
if (usable) {
|
|
34912
|
+
this.state.set(nodeId, {
|
|
34913
|
+
usable: true,
|
|
34914
|
+
armed: false
|
|
34915
|
+
});
|
|
34916
|
+
return prev !== void 0 && !prev.usable ? "recovered" : null;
|
|
34917
|
+
}
|
|
34918
|
+
if (prev === void 0) {
|
|
34919
|
+
this.state.set(nodeId, {
|
|
34920
|
+
usable: true,
|
|
34921
|
+
armed: true
|
|
34922
|
+
});
|
|
34923
|
+
return null;
|
|
34924
|
+
}
|
|
34925
|
+
if (!prev.usable) {
|
|
34926
|
+
this.state.set(nodeId, {
|
|
34927
|
+
usable: false,
|
|
34928
|
+
armed: true
|
|
34929
|
+
});
|
|
34930
|
+
return null;
|
|
34931
|
+
}
|
|
34932
|
+
if (!prev.armed) {
|
|
34933
|
+
this.state.set(nodeId, {
|
|
34934
|
+
usable: true,
|
|
34935
|
+
armed: true
|
|
34936
|
+
});
|
|
34937
|
+
return null;
|
|
34938
|
+
}
|
|
34939
|
+
this.state.set(nodeId, {
|
|
34940
|
+
usable: false,
|
|
34941
|
+
armed: true
|
|
34942
|
+
});
|
|
34943
|
+
return "became-unusable";
|
|
34944
|
+
}
|
|
34945
|
+
/** Can this node be given cameras? Unknown nodes answer YES. */
|
|
34946
|
+
isUsable(nodeId) {
|
|
34947
|
+
return this.state.get(nodeId)?.usable ?? true;
|
|
34948
|
+
}
|
|
34949
|
+
/** Nodes currently excluded — for the placement log and diagnostics. */
|
|
34950
|
+
unusableNodeIds() {
|
|
34951
|
+
const out = [];
|
|
34952
|
+
for (const [nodeId, s] of this.state) if (!s.usable) out.push(nodeId);
|
|
34953
|
+
return out.toSorted();
|
|
34954
|
+
}
|
|
34955
|
+
forget(nodeId) {
|
|
34956
|
+
this.state.delete(nodeId);
|
|
34957
|
+
}
|
|
34958
|
+
reset() {
|
|
34959
|
+
this.state.clear();
|
|
34960
|
+
}
|
|
34961
|
+
};
|
|
34962
|
+
//#endregion
|
|
34798
34963
|
//#region src/orchestrator-types.ts
|
|
34799
34964
|
var PHASE_MODE_VALUES = new Set([
|
|
34800
34965
|
"disabled",
|
|
@@ -36305,6 +36470,69 @@ var LoadShedController = class LoadShedController {
|
|
|
36305
36470
|
}
|
|
36306
36471
|
}
|
|
36307
36472
|
};
|
|
36473
|
+
var DEFAULT_DETECTION_BLIND_CONFIG = {
|
|
36474
|
+
thresholdMs: 60 * 6e4,
|
|
36475
|
+
frameFreshnessMs: 5 * 6e4
|
|
36476
|
+
};
|
|
36477
|
+
var DetectionBlindMonitor = class {
|
|
36478
|
+
config;
|
|
36479
|
+
state = /* @__PURE__ */ new Map();
|
|
36480
|
+
constructor(config = DEFAULT_DETECTION_BLIND_CONFIG) {
|
|
36481
|
+
this.config = config;
|
|
36482
|
+
}
|
|
36483
|
+
/**
|
|
36484
|
+
* One inference frame arrived for a device. `hadDetection` is false for a
|
|
36485
|
+
* frame that returned nothing — which is the whole point: an empty frame
|
|
36486
|
+
* still proves the pipeline is alive, and it is the ONLY evidence that
|
|
36487
|
+
* distinguishes blind from detached.
|
|
36488
|
+
*/
|
|
36489
|
+
noteFrame(deviceId, hadDetection, now) {
|
|
36490
|
+
const prev = this.state.get(deviceId);
|
|
36491
|
+
if (hadDetection) {
|
|
36492
|
+
this.state.set(deviceId, {
|
|
36493
|
+
lastFrameAt: now,
|
|
36494
|
+
lastDetectionAt: now,
|
|
36495
|
+
reported: false
|
|
36496
|
+
});
|
|
36497
|
+
return;
|
|
36498
|
+
}
|
|
36499
|
+
this.state.set(deviceId, {
|
|
36500
|
+
lastFrameAt: now,
|
|
36501
|
+
lastDetectionAt: prev?.lastDetectionAt ?? now,
|
|
36502
|
+
reported: prev?.reported ?? false
|
|
36503
|
+
});
|
|
36504
|
+
}
|
|
36505
|
+
/**
|
|
36506
|
+
* Devices that have JUST crossed into blind. Call from an existing periodic
|
|
36507
|
+
* sweep; it allocates nothing on the healthy path.
|
|
36508
|
+
*/
|
|
36509
|
+
sweep(now) {
|
|
36510
|
+
const out = [];
|
|
36511
|
+
for (const [deviceId, s] of this.state) {
|
|
36512
|
+
if (s.reported) continue;
|
|
36513
|
+
if (now - s.lastFrameAt > this.config.frameFreshnessMs) continue;
|
|
36514
|
+
const blindForMs = now - s.lastDetectionAt;
|
|
36515
|
+
if (blindForMs < this.config.thresholdMs) continue;
|
|
36516
|
+
this.state.set(deviceId, {
|
|
36517
|
+
...s,
|
|
36518
|
+
reported: true
|
|
36519
|
+
});
|
|
36520
|
+
out.push({
|
|
36521
|
+
deviceId,
|
|
36522
|
+
blindForMs,
|
|
36523
|
+
thresholdMs: this.config.thresholdMs
|
|
36524
|
+
});
|
|
36525
|
+
}
|
|
36526
|
+
return out;
|
|
36527
|
+
}
|
|
36528
|
+
/** Drop a device (detach / unbind), so a re-attach starts a fresh clock. */
|
|
36529
|
+
forget(deviceId) {
|
|
36530
|
+
this.state.delete(deviceId);
|
|
36531
|
+
}
|
|
36532
|
+
reset() {
|
|
36533
|
+
this.state.clear();
|
|
36534
|
+
}
|
|
36535
|
+
};
|
|
36308
36536
|
//#endregion
|
|
36309
36537
|
//#region src/agent-load-service.ts
|
|
36310
36538
|
var AgentLoadService = class AgentLoadService {
|
|
@@ -38557,6 +38785,7 @@ var DetectionWiringController = class {
|
|
|
38557
38785
|
if (!this.deps.ctx()) return;
|
|
38558
38786
|
const { deviceId, frame, frameHandle, capturedAt } = payload;
|
|
38559
38787
|
this.deps.watchdog()?.noteSignal(deviceId, "detection");
|
|
38788
|
+
this.deps.blindMonitor()?.noteFrame(deviceId, frame.detections.length > 0, Date.now());
|
|
38560
38789
|
if (frame.detections.length === 0 && (frame.discarded?.length ?? 0) === 0) return;
|
|
38561
38790
|
this.deps.eventBus.emit({
|
|
38562
38791
|
id: `detection-${deviceId}-${Date.now()}`,
|
|
@@ -40344,6 +40573,34 @@ var PlacementService = class {
|
|
|
40344
40573
|
constructor(deps) {
|
|
40345
40574
|
this.deps = deps;
|
|
40346
40575
|
}
|
|
40576
|
+
/**
|
|
40577
|
+
* Drop nodes that cannot infer from a candidate set.
|
|
40578
|
+
*
|
|
40579
|
+
* Fails OPEN in one specific way that matters: if the filter would leave NO
|
|
40580
|
+
* candidate at all, the unfiltered set is used. Placing a camera on a node
|
|
40581
|
+
* that might fail its inference is bad; refusing to place it anywhere
|
|
40582
|
+
* because every node is currently mid-probe is worse, and that is the exact
|
|
40583
|
+
* shape a cold cluster start has.
|
|
40584
|
+
*/
|
|
40585
|
+
filterInferenceUsable(nodes, deviceId) {
|
|
40586
|
+
const usable = nodes.filter((nodeId) => this.deps.isNodeInferenceUsable(nodeId));
|
|
40587
|
+
if (usable.length === nodes.length) return nodes;
|
|
40588
|
+
if (usable.length === 0) {
|
|
40589
|
+
this.deps.ctx().logger.warn("every candidate node reports no usable inference device — placing anyway", {
|
|
40590
|
+
tags: { deviceId },
|
|
40591
|
+
meta: { nodes: [...nodes] }
|
|
40592
|
+
});
|
|
40593
|
+
return nodes;
|
|
40594
|
+
}
|
|
40595
|
+
this.deps.ctx().logger.info("nodes dropped from placement — no usable inference device", {
|
|
40596
|
+
tags: { deviceId },
|
|
40597
|
+
meta: {
|
|
40598
|
+
dropped: nodes.filter((n) => !usable.includes(n)),
|
|
40599
|
+
kept: usable
|
|
40600
|
+
}
|
|
40601
|
+
});
|
|
40602
|
+
return usable;
|
|
40603
|
+
}
|
|
40347
40604
|
async dispatchCamera(runnerConfig) {
|
|
40348
40605
|
if (!this.deps.ctx()) throw new Error("PipelineOrchestrator: dispatchCamera called before initialize");
|
|
40349
40606
|
if (this.deps.maintenancePaused?.() === true) return {
|
|
@@ -40358,7 +40615,7 @@ var PlacementService = class {
|
|
|
40358
40615
|
};
|
|
40359
40616
|
const preferredAgent = await this.deps.readPipelinePin(runnerConfig.deviceId);
|
|
40360
40617
|
const loads = await this.deps.loadService.collectAgentLoad({ onlyEnabled: true });
|
|
40361
|
-
const eligible = this.deps.topology.detectionEligibleNodes(runnerConfig.deviceId);
|
|
40618
|
+
const eligible = this.filterInferenceUsable(this.deps.topology.detectionEligibleNodes(runnerConfig.deviceId), runnerConfig.deviceId);
|
|
40362
40619
|
const nodeCaps = await this.deps.settingsStore.buildNodeCaps();
|
|
40363
40620
|
const motionRoleCandidate = pickMotionRolePlacement({
|
|
40364
40621
|
isAnalyzer: runnerConfig.motionSources.includes("analyzer"),
|
|
@@ -44231,7 +44488,10 @@ var SessionDispatchController = class {
|
|
|
44231
44488
|
await this.deps.reconcilePlacementFromRunners();
|
|
44232
44489
|
const preferredAgent = await this.deps.readPipelinePin(deviceId);
|
|
44233
44490
|
const loads = await this.agentLoads.get(CLUSTER_KEY);
|
|
44234
|
-
const
|
|
44491
|
+
const allEligible = this.deps.topology.detectionEligibleNodes(deviceId);
|
|
44492
|
+
const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
|
|
44493
|
+
const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
|
|
44494
|
+
if (usableEligible.length > 0 && usableEligible.length < allEligible.length) log.info("nodes dropped from session placement — no usable inference device", { meta: { dropped: allEligible.filter((n) => !usableEligible.includes(n)) } });
|
|
44235
44495
|
const decision = balance({
|
|
44236
44496
|
nodes: loads,
|
|
44237
44497
|
preferredAgent,
|
|
@@ -45153,6 +45413,78 @@ var ZonesProvider = class {
|
|
|
45153
45413
|
//#endregion
|
|
45154
45414
|
//#region src/orchestrator-bootstrap.ts
|
|
45155
45415
|
/**
|
|
45416
|
+
* `orchestrator-bootstrap.ts` — the controller-construction body of
|
|
45417
|
+
* `onInitialize` (S11 Task 4, last extraction of the S11 series).
|
|
45418
|
+
*
|
|
45419
|
+
* `buildOrchestratorControllers(deps)` constructs all ~16 controllers the
|
|
45420
|
+
* addon owns, in the EXACT order `onInitialize` built them in before this
|
|
45421
|
+
* extraction (ledger → topology → loadService → audio → settingsStore →
|
|
45422
|
+
* loadShed → placement → session → deviceConfig → cameraStatusService →
|
|
45423
|
+
* reconcile → [settings load + localNodeId resolve] → nodeLifecycle →
|
|
45424
|
+
* [event subs + readiness seed] → pipelineWatchdog → [timers + legacy
|
|
45425
|
+
* migration] → zoneRulesProvider → zonesProvider →
|
|
45426
|
+
* [wireOrchestratorSubscriptions + eager reconcile] → detectionWiring).
|
|
45427
|
+
* Every dep closure is carried over byte-for-byte — only `this.X` became
|
|
45428
|
+
* either a local variable (for a controller already constructed earlier in
|
|
45429
|
+
* THIS function) or a `deps.X` accessor (for addon-owned state/methods that
|
|
45430
|
+
* stay on `PipelineOrchestratorAddon`, or for a controller referenced
|
|
45431
|
+
* before its own construction point — a genuine forward ref, always via a
|
|
45432
|
+
* LAZY closure, exactly as it was in `onInitialize`).
|
|
45433
|
+
*
|
|
45434
|
+
* Three things could not simply become local variables, because an EXISTING
|
|
45435
|
+
* (unmoved) addon method reads the REAL class field synchronously, mid-
|
|
45436
|
+
* construction, not through a lazy closure:
|
|
45437
|
+
* - `applyRuntimeSettings` (stays on the addon) reads `this.topology!`
|
|
45438
|
+
* and `this.reconcile!` directly — called synchronously from the
|
|
45439
|
+
* settings-load block below. `deps.setTopology`/`deps.setReconcile`
|
|
45440
|
+
* progressively sync the REAL addon fields the instant each is built,
|
|
45441
|
+
* so `applyRuntimeSettings` sees them.
|
|
45442
|
+
* - `subscribeToAudioAnalyzerReadiness` (stays on the addon) reads
|
|
45443
|
+
* `this.nodeLifecycle!.readinessRegistry` AND `this.audio!` directly —
|
|
45444
|
+
* called synchronously right after `nodeLifecycle` is built (readiness
|
|
45445
|
+
* seed + the `AgentOnline` handler). `deps.setNodeLifecycle` /
|
|
45446
|
+
* `deps.setAudio` cover it the same way. This one bit LIVE: `audio` is
|
|
45447
|
+
* constructed early (fourth) but `subscribeToAudioAnalyzerReadiness`
|
|
45448
|
+
* wasn't called until much later (after `nodeLifecycle` exists), so the
|
|
45449
|
+
* old code never noticed `this.audio` was still unsynced — until
|
|
45450
|
+
* `ReadinessRegistry.onReadyState`'s `queueMicrotask` hydration replay
|
|
45451
|
+
* (an already-`ready` cap fires the handler once, asynchronously, on
|
|
45452
|
+
* subscribe) won the race against the `await buildOrchestratorControllers(...)`
|
|
45453
|
+
* continuation in `onInitialize` that assigns `this.audio` — both are
|
|
45454
|
+
* microtasks, and the replay's `queueMicrotask` was enqueued (during
|
|
45455
|
+
* this still-synchronous function body) strictly before this function's
|
|
45456
|
+
* `return` resolves the outer promise, so it always runs first. Fixed
|
|
45457
|
+
* by syncing `this.audio` the instant it's constructed, exactly like
|
|
45458
|
+
* `topology`/`reconcile`/`nodeLifecycle` — no lazy closure ever reads a
|
|
45459
|
+
* controller before this function has assigned its real field.
|
|
45460
|
+
* Every other controller-to-controller reference in this function — even
|
|
45461
|
+
* ones that look like a forward ref (e.g. `loadShed`'s `attach` closure
|
|
45462
|
+
* onto `placement`, built one step later) — is a LAZY closure that is
|
|
45463
|
+
* never invoked synchronously during construction (only later, off
|
|
45464
|
+
* event-bus subscriptions, timers, or RPC handlers), so a plain local
|
|
45465
|
+
* `let` variable captured by reference is sufficient and exactly
|
|
45466
|
+
* preserves the original semantics: see each field's inline comment in
|
|
45467
|
+
* `index.ts` for the original construction-order reasoning, unchanged
|
|
45468
|
+
* here.
|
|
45469
|
+
*
|
|
45470
|
+
* `globalSettings` is a purely local variable — read/written ONLY by
|
|
45471
|
+
* closures built inside this same function (the one exception,
|
|
45472
|
+
* `deviceConfig`'s `setGlobalSettings`, is ALSO built here, so the local
|
|
45473
|
+
* variable stays the single source of truth for every reader). No code
|
|
45474
|
+
* outside this closure network ever read the former `this.globalSettings`
|
|
45475
|
+
* class field, so it isn't part of the returned `OrchestratorControllers`
|
|
45476
|
+
* — the field was removed from the class entirely (S11 Task 4 cleanup).
|
|
45477
|
+
* `localNodeId` IS returned — `index.ts` still reads `this.localNodeId`
|
|
45478
|
+
* from a couple of unrelated cap methods (`dumpDiagnostics`,
|
|
45479
|
+
* `getDecoderAssignments`), so the class field is populated once, from the
|
|
45480
|
+
* returned `OrchestratorControllers`, after this function returns.
|
|
45481
|
+
* `failoverPolicy` is the opposite case from `globalSettings` —
|
|
45482
|
+
* `applyRuntimeSettings` (unmoved) writes the REAL field on every future
|
|
45483
|
+
* settings change, so `nodeLifecycle`'s `failoverPolicy` closure MUST keep
|
|
45484
|
+
* reading through `deps.failoverPolicy()` (never a local snapshot) to see
|
|
45485
|
+
* those later updates.
|
|
45486
|
+
*/
|
|
45487
|
+
/**
|
|
45156
45488
|
* Construct every orchestrator controller, wire the agent-lifecycle event
|
|
45157
45489
|
* subscriptions + readiness seed + detection-wiring subscriptions, and
|
|
45158
45490
|
* kick the eager reconcile — the ENTIRE former `onInitialize` body between
|
|
@@ -45311,6 +45643,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45311
45643
|
localNodeId: () => localNodeId,
|
|
45312
45644
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
45313
45645
|
getEligibleInferenceDevices: (nodeId) => deps.getEligibleInferenceDevices(nodeId),
|
|
45646
|
+
isNodeInferenceUsable: (nodeId) => deps.isNodeInferenceUsable(nodeId),
|
|
45314
45647
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
45315
45648
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
45316
45649
|
inferenceRotation,
|
|
@@ -45340,6 +45673,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45340
45673
|
listZones: async (deviceId) => await zonesProvider?.listZones({ deviceId }) ?? [],
|
|
45341
45674
|
readPipelinePin: (deviceId) => deps.readPipelinePin(deviceId),
|
|
45342
45675
|
getEligibleInferenceDevices: (nodeId) => deps.getEligibleInferenceDevices(nodeId),
|
|
45676
|
+
isNodeInferenceUsable: (nodeId) => deps.isNodeInferenceUsable(nodeId),
|
|
45343
45677
|
getInferenceDeviceCaps: (nodeId) => deps.getInferenceDeviceCaps(nodeId),
|
|
45344
45678
|
readPipelineDevicePin: (deviceId) => deps.readPipelineDevicePin(deviceId),
|
|
45345
45679
|
inferenceRotation,
|
|
@@ -45498,7 +45832,34 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45498
45832
|
thresholds: DEFAULT_WATCHDOG_THRESHOLDS
|
|
45499
45833
|
});
|
|
45500
45834
|
pipelineWatchdog.start(deps.watchdogIntervalMs);
|
|
45501
|
-
const
|
|
45835
|
+
const blindMonitor = new DetectionBlindMonitor();
|
|
45836
|
+
const pendingRetryTimer = setInterval(() => {
|
|
45837
|
+
reconcile.retryPendingDispatches();
|
|
45838
|
+
for (const blind of blindMonitor.sweep(Date.now())) {
|
|
45839
|
+
deps.ctx().logger.error("camera is being analysed and detecting NOTHING", {
|
|
45840
|
+
tags: { deviceId: blind.deviceId },
|
|
45841
|
+
meta: {
|
|
45842
|
+
blindForMs: blind.blindForMs,
|
|
45843
|
+
thresholdMs: blind.thresholdMs
|
|
45844
|
+
}
|
|
45845
|
+
});
|
|
45846
|
+
deps.ctx().eventBus.emit({
|
|
45847
|
+
id: randomUUID(),
|
|
45848
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
45849
|
+
source: {
|
|
45850
|
+
type: "device",
|
|
45851
|
+
id: blind.deviceId,
|
|
45852
|
+
deviceId: blind.deviceId
|
|
45853
|
+
},
|
|
45854
|
+
category: EventCategory.PipelineDetectionBlind,
|
|
45855
|
+
data: {
|
|
45856
|
+
deviceId: blind.deviceId,
|
|
45857
|
+
blindForMs: blind.blindForMs,
|
|
45858
|
+
thresholdMs: blind.thresholdMs
|
|
45859
|
+
}
|
|
45860
|
+
});
|
|
45861
|
+
}
|
|
45862
|
+
}, PENDING_RETRY_INTERVAL_MS);
|
|
45502
45863
|
const autoRebalanceTimer = setInterval(() => void placement.runAutoRebalance(), AUTO_REBALANCE_INTERVAL_MS);
|
|
45503
45864
|
deps.ctx().logger.info("Pipeline orchestrator detection-wiring subscriptions installed");
|
|
45504
45865
|
migrateLegacyFlagsToBindings({
|
|
@@ -45625,6 +45986,7 @@ async function buildOrchestratorControllers(deps) {
|
|
|
45625
45986
|
loadShed,
|
|
45626
45987
|
nodeStress,
|
|
45627
45988
|
watchdog: () => pipelineWatchdog,
|
|
45989
|
+
blindMonitor: () => blindMonitor,
|
|
45628
45990
|
api: () => deps.ctx().api ?? null,
|
|
45629
45991
|
eventBus: deps.ctx().eventBus,
|
|
45630
45992
|
logger: deps.ctx().logger,
|
|
@@ -46202,6 +46564,48 @@ function deriveRuntimeSettings(config) {
|
|
|
46202
46564
|
//#endregion
|
|
46203
46565
|
//#region src/index.ts
|
|
46204
46566
|
/**
|
|
46567
|
+
* addon-pipeline-orchestrator — hub-side camera-to-agent load balancer.
|
|
46568
|
+
*
|
|
46569
|
+
* Owns the control plane for the distributed detection pipeline. Tracks
|
|
46570
|
+
* which `addon-pipeline-runner` instance is responsible for each camera,
|
|
46571
|
+
* queries each runner's capacity + hardware profile, and dispatches
|
|
46572
|
+
* `attachCamera` / `detachCamera` cap calls to the appropriate runner via
|
|
46573
|
+
* the Moleculer broker. Persists manual pins via the per-device settings
|
|
46574
|
+
* store so user overrides survive hub restarts.
|
|
46575
|
+
*
|
|
46576
|
+
* Strictly control plane — never sees a frame, never runs inference, never
|
|
46577
|
+
* drains queues. The runner does that work locally on each vision node.
|
|
46578
|
+
*
|
|
46579
|
+
* ## Dispatch entrypoints
|
|
46580
|
+
*
|
|
46581
|
+
* - **`dispatchCamera(runnerConfig)`** — in-process only (not in the cap
|
|
46582
|
+
* def). Called by `DetectionWiringService` with a freshly-built
|
|
46583
|
+
* `RunnerCameraConfig`. Runs the L1/L2/L3 balancer, picks an agent, then
|
|
46584
|
+
* issues `broker.call('pipeline-runner.pipeline-runner.attachCamera', ...)`.
|
|
46585
|
+
* Caches the config so future rebalance / pinning can re-dispatch without
|
|
46586
|
+
* the caller rebuilding it.
|
|
46587
|
+
*
|
|
46588
|
+
* - **`assignCamera({deviceId, agentNodeId})`** — cap method. Persists a
|
|
46589
|
+
* manual `preferredAgent` pin to per-device settings and re-dispatches
|
|
46590
|
+
* immediately if the camera is already attached.
|
|
46591
|
+
*
|
|
46592
|
+
* - **`unassignCamera({deviceId})`** — cap method. Clears the pin,
|
|
46593
|
+
* detaches the camera from its current agent.
|
|
46594
|
+
*
|
|
46595
|
+
* - **`rebalance()`** — cap method. Walks every cached camera config and
|
|
46596
|
+
* re-runs the balancer; migrates cameras whose current agent is no
|
|
46597
|
+
* longer optimal. Pinned cameras are never migrated.
|
|
46598
|
+
*
|
|
46599
|
+
* ## Runner dispatch
|
|
46600
|
+
*
|
|
46601
|
+
* All runner calls go through `this.ctx.api.pipelineRunner.*` (the generated
|
|
46602
|
+
* tRPC cap router). The router extracts `nodeId` from the input and
|
|
46603
|
+
* routes transparently to the local runner (direct in-process call) or
|
|
46604
|
+
* to a remote agent (via Moleculer). Node discovery is driven by
|
|
46605
|
+
* `EventCategory.AgentOnline/AgentOffline` events forwarded by
|
|
46606
|
+
* `agent-registry.service.ts`.
|
|
46607
|
+
*/
|
|
46608
|
+
/**
|
|
46205
46609
|
* The FULL action catalog, under the name the HUB HARVESTS.
|
|
46206
46610
|
*
|
|
46207
46611
|
* The hub's forked-addon harvest imports this entry module and reads the
|
|
@@ -46460,6 +46864,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
46460
46864
|
detachOn: (nodeId, deviceId) => this.detachOn(nodeId, deviceId),
|
|
46461
46865
|
readPipelinePin: (deviceId) => this.readPipelinePin(deviceId),
|
|
46462
46866
|
getEligibleInferenceDevices: (nodeId) => this.resolveEligibleInferenceDevices(nodeId),
|
|
46867
|
+
isNodeInferenceUsable: (nodeId) => this.isNodeInferenceUsable(nodeId),
|
|
46463
46868
|
getInferenceDeviceCaps: (nodeId) => this.getInferenceDeviceCaps(nodeId),
|
|
46464
46869
|
readPipelineDevicePin: (deviceId) => this.readPipelineDevicePin(deviceId),
|
|
46465
46870
|
subscribeToAudioAnalyzerReadiness: (nodeId) => this.subscribeToAudioAnalyzerReadiness(nodeId),
|
|
@@ -47116,11 +47521,66 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
47116
47521
|
const stored = (await this.settingsStore.readAgentSettingsMap())[nodeId]?.inferenceDevices ?? {};
|
|
47117
47522
|
const { probed } = await this.probeNodeInferenceDevices(nodeId);
|
|
47118
47523
|
const catalog = await this.settingsStore.getCatalogForNode(nodeId);
|
|
47119
|
-
const
|
|
47524
|
+
const eligibility = resolveInferenceDeviceEligibility(probed, stored, catalog ? makeRootCapabilityGuard(catalog) : void 0);
|
|
47525
|
+
const { eligible, excluded } = eligibility;
|
|
47120
47526
|
this.logInferenceDeviceShrinkage(nodeId, eligible, excluded);
|
|
47527
|
+
this.noteNodeInferenceUsability(nodeId, resolveNodeInferenceUsability(eligibility));
|
|
47121
47528
|
return eligible;
|
|
47122
47529
|
}
|
|
47123
47530
|
/**
|
|
47531
|
+
* Fold one usability observation into the placement mirror and announce the
|
|
47532
|
+
* transition.
|
|
47533
|
+
*
|
|
47534
|
+
* This is called from the SAME read the dispatcher already makes, plus the
|
|
47535
|
+
* session controller's background refresher — so a node's usability is
|
|
47536
|
+
* re-observed on the cadence placement itself runs at, without a timer of
|
|
47537
|
+
* its own. The mirror is what makes the answer readable synchronously by
|
|
47538
|
+
* `detectionEligibleNodes`' callers; see its own docblock for why it refuses
|
|
47539
|
+
* to act on a single bad read.
|
|
47540
|
+
*
|
|
47541
|
+
* The system event is emitted ONLY on the transition into unusable — the
|
|
47542
|
+
* kind is `node-inference-unavailable` and it is authorable in a rule.
|
|
47543
|
+
* Recovery is logged and deliberately has no event: a notification saying
|
|
47544
|
+
* the broken thing is no longer broken is a second rule nobody asked for.
|
|
47545
|
+
*/
|
|
47546
|
+
noteNodeInferenceUsability(nodeId, usability) {
|
|
47547
|
+
const transition = this.nodeInferenceUsability.observe(nodeId, usability.usable);
|
|
47548
|
+
if (transition === null) return;
|
|
47549
|
+
if (transition === "recovered") {
|
|
47550
|
+
this.ctx.logger.info("node inference recovered — back in the placement candidate set", {
|
|
47551
|
+
tags: { nodeId },
|
|
47552
|
+
meta: { eligible: usability.eligibleKeys }
|
|
47553
|
+
});
|
|
47554
|
+
return;
|
|
47555
|
+
}
|
|
47556
|
+
this.ctx.logger.error("node has NO usable inference device — removed from the placement candidate set", {
|
|
47557
|
+
tags: { nodeId },
|
|
47558
|
+
meta: {
|
|
47559
|
+
unavailable: usability.unavailableKeys,
|
|
47560
|
+
eligible: usability.eligibleKeys
|
|
47561
|
+
}
|
|
47562
|
+
});
|
|
47563
|
+
this.ctx.eventBus.emit({
|
|
47564
|
+
id: randomUUID(),
|
|
47565
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
47566
|
+
source: {
|
|
47567
|
+
type: "addon",
|
|
47568
|
+
id: "pipeline-orchestrator",
|
|
47569
|
+
addonId: "pipeline-orchestrator"
|
|
47570
|
+
},
|
|
47571
|
+
category: EventCategory.PipelineNodeInferenceUnavailable,
|
|
47572
|
+
data: {
|
|
47573
|
+
nodeId,
|
|
47574
|
+
unavailableDeviceKeys: usability.unavailableKeys
|
|
47575
|
+
}
|
|
47576
|
+
});
|
|
47577
|
+
}
|
|
47578
|
+
/** Can this node be given cameras? Read synchronously by the two dispatch
|
|
47579
|
+
* paths before they hand `eligibleNodes` to the balancer. */
|
|
47580
|
+
isNodeInferenceUsable(nodeId) {
|
|
47581
|
+
return this.nodeInferenceUsability.isUsable(nodeId);
|
|
47582
|
+
}
|
|
47583
|
+
/**
|
|
47124
47584
|
* Per-node signature of the last logged eligible/excluded split, so
|
|
47125
47585
|
* {@link logInferenceDeviceShrinkage} fires on CHANGE only. This runs on
|
|
47126
47586
|
* every camera dispatch; without the gate it would be one line per dispatch
|
|
@@ -47128,6 +47588,14 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
|
|
|
47128
47588
|
*/
|
|
47129
47589
|
lastInferenceEligibilitySignature = /* @__PURE__ */ new Map();
|
|
47130
47590
|
/**
|
|
47591
|
+
* Which nodes can still infer. Refreshed by
|
|
47592
|
+
* {@link resolveEligibleInferenceDevices}; read synchronously by both
|
|
47593
|
+
* dispatch paths so a node whose every enabled accelerator vanished stops
|
|
47594
|
+
* receiving cameras. little-unraid was handed cameras for a whole day in
|
|
47595
|
+
* exactly that state.
|
|
47596
|
+
*/
|
|
47597
|
+
nodeInferenceUsability = new NodeInferenceUsabilityMirror();
|
|
47598
|
+
/**
|
|
47131
47599
|
* Say out loud which of a node's inference devices were dropped from the
|
|
47132
47600
|
* candidate set, and why.
|
|
47133
47601
|
*
|
package/dist/remoteEntry.js
CHANGED
|
@@ -30,7 +30,7 @@ async function d(e) {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
async function f() {
|
|
33
|
-
return l ||= d(() => import("./_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-
|
|
33
|
+
return l ||= d(() => import("./_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_orchestrator_widgets-CIgtalSC.mjs")).catch((e) => {
|
|
34
34
|
throw l = void 0, e;
|
|
35
35
|
}), l;
|
|
36
36
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@camstack/addon-pipeline-orchestrator",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.56",
|
|
4
4
|
"description": "Hub-side camera-to-agent load balancer — tracks runner capacity and dispatches attachCamera calls to the optimal pipeline-runner instance",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"camstack",
|