@trillboards/edge-sdk 0.2.225 → 0.2.226
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/windows/README.md +9 -4
- package/deploy/windows/ci/test-rollout-proofs.ps1 +300 -38
- package/deploy/windows/install-update.ps1 +5 -2
- package/deploy/windows/register-detector-task.ps1 +19 -1
- package/dist/AlertClips.d.ts +16 -1
- package/dist/AlertClips.d.ts.map +1 -1
- package/dist/AlertClips.js +35 -2
- package/dist/AlertClips.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +33 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +57 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/DurableVlmCandidateSpool.d.ts +17 -0
- package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
- package/dist/DurableVlmCandidateSpool.js +46 -0
- package/dist/DurableVlmCandidateSpool.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/VideoSegmentUploader.d.ts +28 -2
- package/dist/VideoSegmentUploader.d.ts.map +1 -1
- package/dist/VideoSegmentUploader.js +53 -6
- package/dist/VideoSegmentUploader.js.map +1 -1
- package/dist/autoSense.d.ts +11 -1
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +33 -2
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +5 -1
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +189 -38
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +333 -48
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/connectorUpdateApply.d.ts +23 -1
- package/dist/connectorUpdateApply.d.ts.map +1 -1
- package/dist/connectorUpdateApply.js +72 -1
- package/dist/connectorUpdateApply.js.map +1 -1
- package/dist/connectorUpdateApplyMac.d.ts +4 -0
- package/dist/connectorUpdateApplyMac.d.ts.map +1 -1
- package/dist/connectorUpdateApplyMac.js +14 -0
- package/dist/connectorUpdateApplyMac.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/localSensing.d.ts +37 -2
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +65 -2
- package/dist/localSensing.js.map +1 -1
- package/dist/sensingRunMode.d.ts +12 -0
- package/dist/sensingRunMode.d.ts.map +1 -1
- package/dist/sensingRunMode.js.map +1 -1
- package/package.json +3 -3
- package/retained/WebsiteAdmissionCaptureClient.cjs +31 -2
- package/retained/WebsiteAdmissionState.cjs +1 -1
- package/retained/__tests__/website-admission.test.cjs +27 -0
package/dist/connectorUpdate.js
CHANGED
|
@@ -76,7 +76,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
76
76
|
};
|
|
77
77
|
})();
|
|
78
78
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
79
|
-
exports.
|
|
79
|
+
exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS = exports.CONNECTOR_UPDATE_HOLD_GRACE_MS = exports.CONNECTOR_UPDATE_DRAIN_MAX_MS = exports.CONNECTOR_UPDATE_DRAIN_POLL_MS = exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS = exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS = exports.CONNECTOR_MANIFEST_RETRY_BASE_MS = exports.CONNECTOR_UPDATE_INITIAL_DELAY_MS = exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY = exports.CONNECTOR_UPDATE_FOLLOW_UP_HORIZON_MS = exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = exports.ConnectorManifestClient = exports.defaultManifestFetcher = exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS = exports.CONNECTOR_MODEL_REPORT_INTERVAL_MS = exports.CONNECTOR_RELEASE_DOORBELL_MIN_GAP_MS = exports.CONNECTOR_UPDATE_CHECK_INTERVAL_MS = exports.CONNECTOR_UPDATE_BASE_URL = void 0;
|
|
80
80
|
exports.parseConnectorVersion = parseConnectorVersion;
|
|
81
81
|
exports.isNewerConnectorVersion = isNewerConnectorVersion;
|
|
82
82
|
exports.manifestObjectForPlatform = manifestObjectForPlatform;
|
|
@@ -86,11 +86,15 @@ exports.parseConnectorReleaseAdvert = parseConnectorReleaseAdvert;
|
|
|
86
86
|
exports.onConnectorReleaseAdvert = onConnectorReleaseAdvert;
|
|
87
87
|
exports.noteConnectorReleaseAdvert = noteConnectorReleaseAdvert;
|
|
88
88
|
exports.parseVersionManifest = parseVersionManifest;
|
|
89
|
+
exports.parseSafePointPolicy = parseSafePointPolicy;
|
|
90
|
+
exports.safePointBlockers = safePointBlockers;
|
|
89
91
|
exports.isSafeToApplyUpdate = isSafeToApplyUpdate;
|
|
92
|
+
exports.parseAlertWork = parseAlertWork;
|
|
90
93
|
exports.busynessOf = busynessOf;
|
|
91
94
|
exports.connectorRuntimeFromHealth = connectorRuntimeFromHealth;
|
|
92
95
|
exports.isReleaseBakedEnough = isReleaseBakedEnough;
|
|
93
96
|
exports.checkForConnectorUpdate = checkForConnectorUpdate;
|
|
97
|
+
exports.describeHandover = describeHandover;
|
|
94
98
|
exports.startConnectorUpdateService = startConnectorUpdateService;
|
|
95
99
|
exports.connectorDeviceTokens = connectorDeviceTokens;
|
|
96
100
|
exports.startConnectorUpdates = startConnectorUpdates;
|
|
@@ -254,9 +258,10 @@ function parseConnectorReleaseAdvert(raw) {
|
|
|
254
258
|
const hold = o.minReleaseAgeMs;
|
|
255
259
|
const minReleaseAgeMs = typeof hold === 'number' && Number.isSafeInteger(hold) && hold >= 0
|
|
256
260
|
&& hold <= MAX_SERVER_RELEASE_HOLD_MS ? hold : null;
|
|
257
|
-
|
|
261
|
+
const safePoint = parseSafePointPolicy(o.safePoint);
|
|
262
|
+
if (!versions.windows && !versions.macos && minReleaseAgeMs === null && safePoint === null)
|
|
258
263
|
return null;
|
|
259
|
-
return { versions, minReleaseAgeMs };
|
|
264
|
+
return { versions, minReleaseAgeMs, safePoint };
|
|
260
265
|
}
|
|
261
266
|
const releaseAdvertListeners = new Set();
|
|
262
267
|
/** Subscribe to adverts; returns the unsubscribe. One per update loop. */
|
|
@@ -471,32 +476,115 @@ class ConnectorManifestClient {
|
|
|
471
476
|
}
|
|
472
477
|
exports.ConnectorManifestClient = ConnectorManifestClient;
|
|
473
478
|
/**
|
|
474
|
-
*
|
|
479
|
+
* How long to wait for a safe point before applying anyway: 30 minutes.
|
|
475
480
|
*
|
|
476
|
-
*
|
|
481
|
+
* Two hours through 0.2.225, sized for the quiet rule that a saturated box
|
|
482
|
+
* almost never meets. Under `alert_work` the deadline is only the backstop for
|
|
483
|
+
* alert work that does not clear. Measured on the busiest box: on 2026-11-09
|
|
484
|
+
* (00:00-21:40Z) Texas had alert-supporting looks pending in 972 stretches
|
|
485
|
+
* covering 31% of the day, p50 3.4 s, p90 82 s, longest 568 s; 30 minutes is
|
|
486
|
+
* three times the longest. Past it the update applies and says which branch it
|
|
487
|
+
* took. Server-settable: runtime_config CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS.
|
|
488
|
+
*/
|
|
489
|
+
exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = 30 * 60 * 1000;
|
|
490
|
+
/**
|
|
491
|
+
* A wait/recheck follow-up due within this long blocks the hand-over: 150 s.
|
|
492
|
+
*
|
|
493
|
+
* The follow-up is the look that confirms a wait/recheck alert; if it falls due
|
|
494
|
+
* while the box restarts, the alert waits for the restart. Measured on the
|
|
495
|
+
* Texas restart into 0.2.224 (2026-11-09 17:55Z): the cameras answered again
|
|
496
|
+
* 61-124 s after their last answer before it (first capture 46 s after the
|
|
497
|
+
* last, then the detector warmed up). The wait anchor itself survives a restart
|
|
498
|
+
* (the archive is durable), so a follow-up due later than this is safe.
|
|
499
|
+
* Server-settable: runtime_config CONNECTOR_UPDATE_SAFE_POINT_FOLLOW_UP_HORIZON_MS.
|
|
500
|
+
*/
|
|
501
|
+
exports.CONNECTOR_UPDATE_FOLLOW_UP_HORIZON_MS = 150000;
|
|
502
|
+
exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY = Object.freeze({
|
|
503
|
+
rule: 'alert_work',
|
|
504
|
+
deadlineMs: exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS,
|
|
505
|
+
followUpHorizonMs: exports.CONNECTOR_UPDATE_FOLLOW_UP_HORIZON_MS,
|
|
506
|
+
});
|
|
507
|
+
/** A deadline longer than a day, or a horizon longer than the longest wait rule (10 min), is a typo. */
|
|
508
|
+
const MAX_SAFE_POINT_DEADLINE_MS = 24 * 60 * 60 * 1000;
|
|
509
|
+
const MAX_FOLLOW_UP_HORIZON_MS = 10 * 60 * 1000;
|
|
510
|
+
/** Strictly read `connectorRelease.safePoint`; null when any part is unusable (the box keeps its last policy). */
|
|
511
|
+
function parseSafePointPolicy(raw) {
|
|
512
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
513
|
+
return null;
|
|
514
|
+
const o = raw;
|
|
515
|
+
if (o.rule !== 'alert_work' && o.rule !== 'all_work')
|
|
516
|
+
return null;
|
|
517
|
+
const ms = (v, max) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 && v <= max;
|
|
518
|
+
if (!ms(o.deadlineMs, MAX_SAFE_POINT_DEADLINE_MS) || !ms(o.followUpHorizonMs, MAX_FOLLOW_UP_HORIZON_MS))
|
|
519
|
+
return null;
|
|
520
|
+
return { rule: o.rule, deadlineMs: o.deadlineMs, followUpHorizonMs: o.followUpHorizonMs };
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* Why this is NOT a moment to stop and replace the connector; empty when it is.
|
|
524
|
+
*
|
|
525
|
+
* `all_work` (and `alert_work` on a manager that cannot count alert work)
|
|
526
|
+
* waits for the connector's two counters to read zero:
|
|
527
|
+
*
|
|
528
|
+
* - `inFlight` — a frame the model is answering, or a POST in flight;
|
|
529
|
+
* - `buffered` — frames held for a free slot. On the legacy (non-retained)
|
|
530
|
+
* path these are frames the motion cascade flagged, held in memory, and a
|
|
531
|
+
* stop drops them.
|
|
532
|
+
*
|
|
533
|
+
* `alert_work` waits only for what a restart could cost an alert:
|
|
477
534
|
*
|
|
478
|
-
* -
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
*
|
|
482
|
-
* -
|
|
483
|
-
*
|
|
484
|
-
* 30 s scan cadence the next look is a different event. `CloudSensingManager
|
|
485
|
-
* .stop()` calls `candidates.reset()`, so these die on ANY stop, graceful or
|
|
486
|
-
* not — stopping while the buffer is loaded is the dropped-alert failure, and
|
|
487
|
-
* it is a product failure, not an implementation detail.
|
|
535
|
+
* - an alert-supporting look the server has not acknowledged: its alert, or
|
|
536
|
+
* the first sighting of a wait/recheck episode, would wait out the restart;
|
|
537
|
+
* - alert video still owed: the receipt naming an alert's clip request, the
|
|
538
|
+
* after-window still being recorded, the cut, the upload;
|
|
539
|
+
* - a wait/recheck follow-up due within `followUpHorizonMs`: it would fall due
|
|
540
|
+
* while the box restarts.
|
|
488
541
|
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
493
|
-
*
|
|
494
|
-
* yet judged interesting, which is a different and much smaller loss than a
|
|
495
|
-
* flagged candidate, and the next tick captures a fresher frame. Widening the
|
|
496
|
-
* health struct to close it was judged not worth the blast radius.
|
|
542
|
+
* Routine looks and the frame backlog are not waited for: on a retained-review
|
|
543
|
+
* box every frame is in the durable archive and spool, and the schedule picks
|
|
544
|
+
* them up after the restart. Looks still in the model are drained by the
|
|
545
|
+
* update loop before it commits (see startConnectorUpdateService), so a look
|
|
546
|
+
* that turns out to support an alert is never cut off.
|
|
497
547
|
*/
|
|
498
|
-
function
|
|
499
|
-
|
|
548
|
+
function safePointBlockers(busyness, policy = exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY, nowMs = Date.now()) {
|
|
549
|
+
const reasons = [];
|
|
550
|
+
if (busyness.hold)
|
|
551
|
+
reasons.push('setup or reconfiguration in progress');
|
|
552
|
+
const alert = busyness.alert;
|
|
553
|
+
if (policy.rule === 'all_work' || !alert) {
|
|
554
|
+
if (busyness.inFlight > 0)
|
|
555
|
+
reasons.push(`${busyness.inFlight} analysis(es) in flight`);
|
|
556
|
+
if (busyness.buffered > 0)
|
|
557
|
+
reasons.push(`${busyness.buffered} frame(s) held`);
|
|
558
|
+
return reasons;
|
|
559
|
+
}
|
|
560
|
+
if (alert.looksAwaitingAnswer > 0) {
|
|
561
|
+
reasons.push(`${alert.looksAwaitingAnswer} alert look(s) awaiting the server's answer`);
|
|
562
|
+
}
|
|
563
|
+
if (alert.nextFollowUpDueAtMs !== null && Number.isFinite(alert.nextFollowUpDueAtMs)
|
|
564
|
+
&& alert.nextFollowUpDueAtMs - nowMs <= policy.followUpHorizonMs) {
|
|
565
|
+
const s = Math.round((alert.nextFollowUpDueAtMs - nowMs) / 1000);
|
|
566
|
+
reasons.push(s >= 0 ? `follow-up due in ${s} s` : `follow-up overdue by ${-s} s`);
|
|
567
|
+
}
|
|
568
|
+
if ((busyness.alertVideo ?? 0) > 0)
|
|
569
|
+
reasons.push(`${busyness.alertVideo} alert video item(s) pending`);
|
|
570
|
+
return reasons;
|
|
571
|
+
}
|
|
572
|
+
/** Is this a moment at which the connector may be stopped and replaced? See safePointBlockers. */
|
|
573
|
+
function isSafeToApplyUpdate(busyness, policy = exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY, nowMs = Date.now()) {
|
|
574
|
+
return safePointBlockers(busyness, policy, nowMs).length === 0;
|
|
575
|
+
}
|
|
576
|
+
/** Strictly read a manager's alert-work report; null for anything malformed (never a fabricated "no work"). */
|
|
577
|
+
function parseAlertWork(raw) {
|
|
578
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
579
|
+
return null;
|
|
580
|
+
const o = raw;
|
|
581
|
+
const count = (v) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 0;
|
|
582
|
+
if (!count(o.looksAwaitingAnswer) || !count(o.looksInModel))
|
|
583
|
+
return null;
|
|
584
|
+
const due = o.nextFollowUpDueAtMs;
|
|
585
|
+
if (due !== null && !(typeof due === 'number' && Number.isFinite(due)))
|
|
586
|
+
return null;
|
|
587
|
+
return { looksAwaitingAnswer: o.looksAwaitingAnswer, looksInModel: o.looksInModel, nextFollowUpDueAtMs: due };
|
|
500
588
|
}
|
|
501
589
|
/**
|
|
502
590
|
* Map a `getSensingHealth()` reading onto the two numbers that matter. Tolerant
|
|
@@ -650,18 +738,41 @@ exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS = 60 * 60 * 1000;
|
|
|
650
738
|
*/
|
|
651
739
|
const STAGE_OUTCOME_CLOCK_GRACE_MS = 5000;
|
|
652
740
|
/**
|
|
653
|
-
*
|
|
654
|
-
*
|
|
655
|
-
*
|
|
656
|
-
*
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
*
|
|
661
|
-
*
|
|
662
|
-
*
|
|
741
|
+
* The drain before a hand-over under `alert_work`: new looks are held while the
|
|
742
|
+
* looks already in the model finish, because one of them may come back
|
|
743
|
+
* supporting an alert. Polled this often while it runs, so looks pause for
|
|
744
|
+
* about one look's length (Texas: ~1 s detector looks), not a poll period.
|
|
745
|
+
*/
|
|
746
|
+
exports.CONNECTOR_UPDATE_DRAIN_POLL_MS = 250;
|
|
747
|
+
/**
|
|
748
|
+
* The longest a drain holds new looks while the live looks in the model finish
|
|
749
|
+
* (a detector look takes ~1 s, one that asks Gemma on a store CPU up to ~13 s;
|
|
750
|
+
* backlog reviews are history and not counted). A drain that reaches this with
|
|
751
|
+
* a live look still running does NOT commit: that look may support an alert. It
|
|
752
|
+
* lets the cameras be looked at again and tries again after
|
|
753
|
+
* CONNECTOR_UPDATE_DRAIN_RETRY_MS; the deadline is the backstop.
|
|
754
|
+
*/
|
|
755
|
+
exports.CONNECTOR_UPDATE_DRAIN_MAX_MS = 20000;
|
|
756
|
+
/**
|
|
757
|
+
* A setup or reconfiguration hold outlasts the safe-point deadline by up to
|
|
758
|
+
* this long: the deadline accepts losing alert work, never breaking an
|
|
759
|
+
* operator's camera setup half-way, and the cap keeps a setup session left open
|
|
760
|
+
* from holding a fix back for ever.
|
|
663
761
|
*/
|
|
664
|
-
exports.
|
|
762
|
+
exports.CONNECTOR_UPDATE_HOLD_GRACE_MS = 30 * 60 * 1000;
|
|
763
|
+
/**
|
|
764
|
+
* After a drain is cut short by alert work (a look in the model came back
|
|
765
|
+
* supporting an alert), the next drain waits this long: while an alert
|
|
766
|
+
* condition flickers, each drain would otherwise pause every camera's looks.
|
|
767
|
+
*/
|
|
768
|
+
exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS = 30000;
|
|
769
|
+
/** The `current` state's detail on the first reports of the version a hand-over installed. */
|
|
770
|
+
function describeHandover(record) {
|
|
771
|
+
const waited = Math.round(record.waitedMs / 1000);
|
|
772
|
+
const how = record.branch === 'safe_point' ? `at a safe point (rule ${record.rule})` : 'when the safe-point deadline passed';
|
|
773
|
+
return `took ${record.toVersion} from ${record.fromVersion} ${how} ${waited} s after it was staged ` +
|
|
774
|
+
`(staged ${record.stagedAt}, handed over ${record.handedOverAt})`;
|
|
775
|
+
}
|
|
665
776
|
/** The newest of a non-empty list of releases. */
|
|
666
777
|
function newestRelease(releases) {
|
|
667
778
|
return releases.reduce((a, b) => (isNewerConnectorVersion(b.connectorVersion, a.connectorVersion) ? b : a));
|
|
@@ -716,7 +827,6 @@ function startConnectorUpdateService(deps) {
|
|
|
716
827
|
const requestExit = deps.requestExit ?? (() => process.exit(0));
|
|
717
828
|
const pollIntervalMs = deps.pollIntervalMs ?? exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS;
|
|
718
829
|
const stageDeadlineMs = deps.stageDeadlineMs ?? exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS;
|
|
719
|
-
const safePointDeadlineMs = deps.safePointDeadlineMs ?? exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS;
|
|
720
830
|
// Where the loop stands, sent on every report. Every branch below that used to
|
|
721
831
|
// end in only a local log line also sets this, so a box that never updates
|
|
722
832
|
// tells the fleet why instead of looking exactly like one that is current.
|
|
@@ -744,6 +854,27 @@ function startConnectorUpdateService(deps) {
|
|
|
744
854
|
// stated value (runtime_config via `connectorRelease`), else none.
|
|
745
855
|
let serverHoldMs = null;
|
|
746
856
|
const releaseHoldMs = () => deps.minReleaseAgeMs ?? serverHoldMs ?? exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS;
|
|
857
|
+
// The safe-point policy in force: a test's fixed value, else the server's last
|
|
858
|
+
// stated value (runtime_config via `connectorRelease.safePoint`), else the
|
|
859
|
+
// default. Read at every look, so a corrected rule or deadline reaches a box
|
|
860
|
+
// that is already waiting within one profile poll.
|
|
861
|
+
let serverSafePoint = null;
|
|
862
|
+
const safePointPolicy = () => {
|
|
863
|
+
const policy = deps.safePointPolicy ?? serverSafePoint ?? exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY;
|
|
864
|
+
return deps.safePointDeadlineMs === undefined ? policy : { ...policy, deadlineMs: deps.safePointDeadlineMs };
|
|
865
|
+
};
|
|
866
|
+
// When the update being applied was staged and verified (or found verified).
|
|
867
|
+
let stagedAtMs = 0;
|
|
868
|
+
// The drain before a hand-over (alert_work): new looks held since drainStartedMs.
|
|
869
|
+
let drainStartedMs = null;
|
|
870
|
+
let drainTimer = null;
|
|
871
|
+
let nextDrainAtMs = 0;
|
|
872
|
+
// Consecutive drains cut off by a live look outlasting the cap: each doubles
|
|
873
|
+
// the wait before the next, so a hung model call does not pause looks for
|
|
874
|
+
// 20 s in every minute until it times out.
|
|
875
|
+
let drainCapOuts = 0;
|
|
876
|
+
// The blockers last logged, so a box waiting for alert work says why once per change.
|
|
877
|
+
let lastBlockers = null;
|
|
747
878
|
// The newest version the last good read of the channel listed. The doorbell
|
|
748
879
|
// reads the channel again only for something newer than this.
|
|
749
880
|
let lastSeenVersion = null;
|
|
@@ -754,14 +885,48 @@ function startConnectorUpdateService(deps) {
|
|
|
754
885
|
t.unref();
|
|
755
886
|
}
|
|
756
887
|
};
|
|
757
|
-
|
|
888
|
+
/** Stop the drain's fast poll; `release` also lets the manager start looks again. */
|
|
889
|
+
const endDrain = (release) => {
|
|
890
|
+
if (drainTimer) {
|
|
891
|
+
clearIntervalFn(drainTimer);
|
|
892
|
+
drainTimer = null;
|
|
893
|
+
}
|
|
894
|
+
if (drainStartedMs === null)
|
|
895
|
+
return;
|
|
896
|
+
drainStartedMs = null;
|
|
897
|
+
if (!release)
|
|
898
|
+
return;
|
|
899
|
+
try {
|
|
900
|
+
deps.holdReviews?.(null);
|
|
901
|
+
}
|
|
902
|
+
catch {
|
|
903
|
+
/* the hold expires on its own */
|
|
904
|
+
}
|
|
905
|
+
};
|
|
906
|
+
const endApply = (releaseHold = true) => {
|
|
758
907
|
phase = 'idle';
|
|
759
908
|
target = null;
|
|
909
|
+
lastBlockers = null;
|
|
910
|
+
endDrain(releaseHold);
|
|
760
911
|
if (pollTimer) {
|
|
761
912
|
clearIntervalFn(pollTimer);
|
|
762
913
|
pollTimer = null;
|
|
763
914
|
}
|
|
764
915
|
};
|
|
916
|
+
// What the previous process left when it handed this version the machine.
|
|
917
|
+
// Reported (as the `current` state's detail) until a report lands everywhere.
|
|
918
|
+
let handover = null;
|
|
919
|
+
let handoverReported = false;
|
|
920
|
+
try {
|
|
921
|
+
const left = applier?.readHandover?.() ?? null;
|
|
922
|
+
if (left && left.toVersion === deps.installedVersion)
|
|
923
|
+
handover = left;
|
|
924
|
+
else if (left)
|
|
925
|
+
applier?.clearHandover?.(); // a rolled-back update's record: not about this version
|
|
926
|
+
}
|
|
927
|
+
catch {
|
|
928
|
+
handover = null;
|
|
929
|
+
}
|
|
765
930
|
/**
|
|
766
931
|
* Report where this install stands, carrying any update outcome the applier
|
|
767
932
|
* left behind. The outcome is cleared only once the server has ACCEPTED it —
|
|
@@ -821,6 +986,18 @@ function startConnectorUpdateService(deps) {
|
|
|
821
986
|
if (outcome && result.attempted > 0 && result.accepted === result.attempted) {
|
|
822
987
|
applier?.clearOutcome();
|
|
823
988
|
}
|
|
989
|
+
// The handover record follows the same rule: forgotten on disk once every
|
|
990
|
+
// feed carried it, kept in memory as this process's `current` detail.
|
|
991
|
+
if (handover && !handoverReported && updater?.state === 'current'
|
|
992
|
+
&& result.attempted > 0 && result.accepted === result.attempted) {
|
|
993
|
+
handoverReported = true;
|
|
994
|
+
try {
|
|
995
|
+
applier?.clearHandover?.();
|
|
996
|
+
}
|
|
997
|
+
catch {
|
|
998
|
+
/* a re-report after a restart is harmless */
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
824
1001
|
};
|
|
825
1002
|
/** Advance the apply state machine. Called on a short timer while in flight. */
|
|
826
1003
|
const pollApply = () => {
|
|
@@ -832,6 +1009,7 @@ function startConnectorUpdateService(deps) {
|
|
|
832
1009
|
log(`[update] ${target.connectorVersion} is staged and verified — waiting for a safe point`);
|
|
833
1010
|
phase = 'awaiting-safe-point';
|
|
834
1011
|
phaseStartedMs = now();
|
|
1012
|
+
stagedAtMs = phaseStartedMs;
|
|
835
1013
|
elapsed = 0;
|
|
836
1014
|
setUpdater('awaiting_safe_point', target.connectorVersion);
|
|
837
1015
|
// Falls through: a connector that is already quiet hands over in this
|
|
@@ -860,26 +1038,125 @@ function startConnectorUpdateService(deps) {
|
|
|
860
1038
|
return;
|
|
861
1039
|
}
|
|
862
1040
|
// awaiting-safe-point
|
|
1041
|
+
const version = target.connectorVersion;
|
|
1042
|
+
const policy = safePointPolicy();
|
|
1043
|
+
const nowMs = now();
|
|
863
1044
|
const busyness = deps.busyness ? deps.busyness() : { inFlight: 0, buffered: 0 };
|
|
864
|
-
const
|
|
865
|
-
const
|
|
866
|
-
|
|
1045
|
+
const blockers = safePointBlockers(busyness, policy, nowMs);
|
|
1046
|
+
const alertRule = policy.rule === 'alert_work' && Boolean(busyness.alert);
|
|
1047
|
+
let branch = 'safe_point';
|
|
1048
|
+
if (busyness.hold && elapsed < policy.deadlineMs + exports.CONNECTOR_UPDATE_HOLD_GRACE_MS) {
|
|
1049
|
+
// An operator's camera setup or a reconfiguration is running: past the
|
|
1050
|
+
// deadline too, for a bounded grace.
|
|
1051
|
+
if (drainStartedMs !== null)
|
|
1052
|
+
endDrain(true);
|
|
1053
|
+
const kinds = 'setup or reconfiguration in progress';
|
|
1054
|
+
if (kinds !== lastBlockers)
|
|
1055
|
+
log(`[update] ${version} waits: ${kinds}`);
|
|
1056
|
+
lastBlockers = kinds;
|
|
867
1057
|
return;
|
|
868
|
-
|
|
1058
|
+
}
|
|
1059
|
+
if (elapsed >= policy.deadlineMs) {
|
|
1060
|
+
branch = 'deadline';
|
|
869
1061
|
log(`[update] no safe point in ${Math.round(elapsed / 60000)} min — applying anyway ` +
|
|
870
|
-
`(inFlight=${busyness.inFlight}, held candidates=${busyness.buffered}
|
|
1062
|
+
`(inFlight=${busyness.inFlight}, held candidates=${busyness.buffered}` +
|
|
1063
|
+
`${blockers.length ? `; ${blockers.join('; ')}` : ''})`);
|
|
871
1064
|
}
|
|
872
|
-
else {
|
|
873
|
-
|
|
1065
|
+
else if (blockers.length) {
|
|
1066
|
+
// Alert work appeared, or never cleared: a running drain lets the
|
|
1067
|
+
// cameras be looked at again, and the next one waits a little.
|
|
1068
|
+
if (drainStartedMs !== null) {
|
|
1069
|
+
endDrain(true);
|
|
1070
|
+
nextDrainAtMs = nowMs + exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS;
|
|
1071
|
+
}
|
|
1072
|
+
// Logged when the KIND of work changes, not every countdown second.
|
|
1073
|
+
const kinds = blockers.join('; ').replace(/\d+/g, '#');
|
|
1074
|
+
if (alertRule && kinds !== lastBlockers)
|
|
1075
|
+
log(`[update] ${version} waits for alert work: ${blockers.join('; ')}`);
|
|
1076
|
+
lastBlockers = kinds;
|
|
1077
|
+
return;
|
|
874
1078
|
}
|
|
875
|
-
|
|
876
|
-
|
|
1079
|
+
else if (alertRule && deps.holdReviews) {
|
|
1080
|
+
// No alert work is visible, but a look the model is answering may come
|
|
1081
|
+
// back supporting an alert. Hold new looks, let those finish, and hand
|
|
1082
|
+
// over only if nothing they answered supports an alert (the look above
|
|
1083
|
+
// then sees it as alert work and the drain ends).
|
|
1084
|
+
const inModel = busyness.alert.looksInModel;
|
|
1085
|
+
if (drainStartedMs === null) {
|
|
1086
|
+
if (nowMs < nextDrainAtMs)
|
|
1087
|
+
return;
|
|
1088
|
+
drainStartedMs = nowMs;
|
|
1089
|
+
try {
|
|
1090
|
+
// Self-expiring: if this loop died mid-drain, looks resume on their own.
|
|
1091
|
+
deps.holdReviews(nowMs + exports.CONNECTOR_UPDATE_DRAIN_MAX_MS + 5000);
|
|
1092
|
+
}
|
|
1093
|
+
catch (err) {
|
|
1094
|
+
log(`[update] could not hold new looks (non-fatal): ${String(err)}`);
|
|
1095
|
+
}
|
|
1096
|
+
if (inModel > 0) {
|
|
1097
|
+
log(`[update] no alert work — holding new looks while ${inModel} in progress finish before applying ${version}`);
|
|
1098
|
+
if (!drainTimer) {
|
|
1099
|
+
drainTimer = setIntervalFn(() => {
|
|
1100
|
+
try {
|
|
1101
|
+
pollApply();
|
|
1102
|
+
}
|
|
1103
|
+
catch (err) {
|
|
1104
|
+
log(`[update] apply step failed (non-fatal): ${String(err)}`);
|
|
1105
|
+
}
|
|
1106
|
+
}, exports.CONNECTOR_UPDATE_DRAIN_POLL_MS);
|
|
1107
|
+
unref(drainTimer);
|
|
1108
|
+
}
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
else if (inModel > 0) {
|
|
1113
|
+
if (nowMs - drainStartedMs < exports.CONNECTOR_UPDATE_DRAIN_MAX_MS)
|
|
1114
|
+
return;
|
|
1115
|
+
// A live look outlasted the drain: it may support an alert, so it is
|
|
1116
|
+
// not cut off. Look at the cameras again and try later.
|
|
1117
|
+
drainCapOuts += 1;
|
|
1118
|
+
const retryMs = Math.min(exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS * 2 ** (drainCapOuts - 1), 10 * 60 * 1000);
|
|
1119
|
+
log(`[update] ${inModel} look(s) still in the model after ${Math.round(exports.CONNECTOR_UPDATE_DRAIN_MAX_MS / 1000)} s — ` +
|
|
1120
|
+
`not applying ${version} over them; trying again in ${Math.round(retryMs / 1000)} s`);
|
|
1121
|
+
endDrain(true);
|
|
1122
|
+
nextDrainAtMs = nowMs + retryMs;
|
|
1123
|
+
return;
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
if (branch === 'safe_point') {
|
|
1127
|
+
if (alertRule) {
|
|
1128
|
+
log(`[update] safe point after ${Math.round(elapsed / 1000)} s (rule alert_work; not waited for: ` +
|
|
1129
|
+
`${busyness.inFlight} look(s) in flight, ${busyness.buffered} frame(s) held) — applying ${version}`);
|
|
1130
|
+
}
|
|
1131
|
+
else {
|
|
1132
|
+
log('[update] connector is idle — applying');
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
const handedOverMs = now();
|
|
1136
|
+
// The hold stays on through the exit: nothing new starts while the machine is handed over.
|
|
1137
|
+
endApply(false);
|
|
877
1138
|
if (!applier.commit({ version, currentVersion: deps.installedVersion })) {
|
|
1139
|
+
try {
|
|
1140
|
+
deps.holdReviews?.(null);
|
|
1141
|
+
}
|
|
1142
|
+
catch {
|
|
1143
|
+
/* expires on its own */
|
|
1144
|
+
}
|
|
878
1145
|
log('[update] could not launch the applier — staying on the current version');
|
|
879
1146
|
const why = applier.launchFailure?.() ?? null;
|
|
880
1147
|
setUpdater('apply_failed', why ? `${version}: ${why}` : version);
|
|
881
1148
|
return;
|
|
882
1149
|
}
|
|
1150
|
+
try {
|
|
1151
|
+
applier.recordHandover?.({
|
|
1152
|
+
version: 1, fromVersion: deps.installedVersion, toVersion: version, rule: policy.rule, branch,
|
|
1153
|
+
stagedAt: new Date(stagedAtMs || phaseStartedMs).toISOString(), handedOverAt: new Date(handedOverMs).toISOString(),
|
|
1154
|
+
waitedMs: Math.max(0, handedOverMs - (stagedAtMs || phaseStartedMs)),
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
catch (err) {
|
|
1158
|
+
log(`[update] could not record the hand-over (non-fatal): ${String(err)}`);
|
|
1159
|
+
}
|
|
883
1160
|
setUpdater('applying', version);
|
|
884
1161
|
// The applier owns the machine now. Exiting frees bin\node.exe for the swap;
|
|
885
1162
|
// it also means anything after this line does not run.
|
|
@@ -910,6 +1187,7 @@ function startConnectorUpdateService(deps) {
|
|
|
910
1187
|
phase = alreadyStaged ? 'awaiting-safe-point' : 'staging';
|
|
911
1188
|
target = manifest;
|
|
912
1189
|
phaseStartedMs = now();
|
|
1190
|
+
stagedAtMs = alreadyStaged ? phaseStartedMs : 0;
|
|
913
1191
|
if (!pollTimer) {
|
|
914
1192
|
pollTimer = setIntervalFn(() => {
|
|
915
1193
|
try {
|
|
@@ -936,7 +1214,8 @@ function startConnectorUpdateService(deps) {
|
|
|
936
1214
|
return null;
|
|
937
1215
|
}
|
|
938
1216
|
if (!check.availableVersion) {
|
|
939
|
-
|
|
1217
|
+
// The first reports of a version a hand-over installed say how it got here.
|
|
1218
|
+
setUpdater('current', handover ? describeHandover(handover) : null);
|
|
940
1219
|
return null;
|
|
941
1220
|
}
|
|
942
1221
|
const stageable = connectorReleaseChoices(check.manifest).filter(m => isNewerConnectorVersion(m.connectorVersion, deps.installedVersion) && applier.canStage(m));
|
|
@@ -1088,6 +1367,9 @@ function startConnectorUpdateService(deps) {
|
|
|
1088
1367
|
const holdChanged = advert.minReleaseAgeMs !== null && advert.minReleaseAgeMs !== serverHoldMs;
|
|
1089
1368
|
if (advert.minReleaseAgeMs !== null)
|
|
1090
1369
|
serverHoldMs = advert.minReleaseAgeMs;
|
|
1370
|
+
// Taken even mid-wait: the next look at the safe point uses it.
|
|
1371
|
+
if (advert.safePoint)
|
|
1372
|
+
serverSafePoint = advert.safePoint;
|
|
1091
1373
|
// Mid-stage, mid-wait, or backing off a failed read: those paths already
|
|
1092
1374
|
// own the next read.
|
|
1093
1375
|
if (phase !== 'idle' || failedReads > 0)
|
|
@@ -1181,6 +1463,8 @@ function startConnectorUpdateService(deps) {
|
|
|
1181
1463
|
clearIntervalFn(modelsTimer);
|
|
1182
1464
|
if (pollTimer)
|
|
1183
1465
|
clearIntervalFn(pollTimer);
|
|
1466
|
+
// A loop stopped mid-drain (a reconfigure rebuilds it) lets the cameras be looked at again.
|
|
1467
|
+
endDrain(true);
|
|
1184
1468
|
}
|
|
1185
1469
|
catch {
|
|
1186
1470
|
/* best-effort */
|
|
@@ -1294,6 +1578,7 @@ function startConnectorUpdates(opts) {
|
|
|
1294
1578
|
apiBaseUrl,
|
|
1295
1579
|
deviceTokens,
|
|
1296
1580
|
busyness: opts.busyness,
|
|
1581
|
+
...(opts.holdReviews ? { holdReviews: opts.holdReviews } : {}),
|
|
1297
1582
|
runtime: opts.runtime,
|
|
1298
1583
|
inventory: opts.inventory,
|
|
1299
1584
|
...(opts.videoBuffer ? { videoBuffer: opts.videoBuffer } : {}),
|