@trillboards/edge-sdk 0.2.225 → 0.2.227
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/windows/README.md +9 -4
- package/deploy/windows/ci/test-rollout-proofs.ps1 +300 -38
- package/deploy/windows/install-update.ps1 +5 -2
- package/deploy/windows/register-detector-task.ps1 +19 -1
- package/dist/AlertClips.d.ts +16 -1
- package/dist/AlertClips.d.ts.map +1 -1
- package/dist/AlertClips.js +35 -2
- package/dist/AlertClips.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +33 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +57 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/DurableVlmCandidateSpool.d.ts +17 -0
- package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
- package/dist/DurableVlmCandidateSpool.js +46 -0
- package/dist/DurableVlmCandidateSpool.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/VideoSegmentUploader.d.ts +28 -2
- package/dist/VideoSegmentUploader.d.ts.map +1 -1
- package/dist/VideoSegmentUploader.js +53 -6
- package/dist/VideoSegmentUploader.js.map +1 -1
- package/dist/autoSense.d.ts +11 -1
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +33 -2
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +5 -1
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +189 -38
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +337 -48
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/connectorUpdateApply.d.ts +23 -1
- package/dist/connectorUpdateApply.d.ts.map +1 -1
- package/dist/connectorUpdateApply.js +72 -1
- package/dist/connectorUpdateApply.js.map +1 -1
- package/dist/connectorUpdateApplyMac.d.ts +4 -0
- package/dist/connectorUpdateApplyMac.d.ts.map +1 -1
- package/dist/connectorUpdateApplyMac.js +14 -0
- package/dist/connectorUpdateApplyMac.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/localSensing.d.ts +37 -2
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +65 -2
- package/dist/localSensing.js.map +1 -1
- package/dist/sensingRunMode.d.ts +12 -0
- package/dist/sensingRunMode.d.ts.map +1 -1
- package/dist/sensingRunMode.js.map +1 -1
- package/package.json +3 -3
- package/retained/WebsiteAdmissionCaptureClient.cjs +31 -2
- package/retained/WebsiteAdmissionState.cjs +1 -1
- package/retained/__tests__/website-admission.test.cjs +27 -0
package/dist/connectorUpdate.js
CHANGED
|
@@ -76,7 +76,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
76
76
|
};
|
|
77
77
|
})();
|
|
78
78
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
79
|
-
exports.
|
|
79
|
+
exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS = exports.CONNECTOR_UPDATE_HOLD_GRACE_MS = exports.CONNECTOR_UPDATE_DRAIN_MAX_MS = exports.CONNECTOR_UPDATE_DRAIN_POLL_MS = exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS = exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS = exports.CONNECTOR_MANIFEST_RETRY_BASE_MS = exports.CONNECTOR_UPDATE_INITIAL_DELAY_MS = exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY = exports.CONNECTOR_UPDATE_FOLLOW_UP_HORIZON_MS = exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = exports.ConnectorManifestClient = exports.defaultManifestFetcher = exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS = exports.CONNECTOR_MODEL_REPORT_INTERVAL_MS = exports.CONNECTOR_RELEASE_DOORBELL_MIN_GAP_MS = exports.CONNECTOR_UPDATE_CHECK_INTERVAL_MS = exports.CONNECTOR_UPDATE_BASE_URL = void 0;
|
|
80
80
|
exports.parseConnectorVersion = parseConnectorVersion;
|
|
81
81
|
exports.isNewerConnectorVersion = isNewerConnectorVersion;
|
|
82
82
|
exports.manifestObjectForPlatform = manifestObjectForPlatform;
|
|
@@ -86,11 +86,15 @@ exports.parseConnectorReleaseAdvert = parseConnectorReleaseAdvert;
|
|
|
86
86
|
exports.onConnectorReleaseAdvert = onConnectorReleaseAdvert;
|
|
87
87
|
exports.noteConnectorReleaseAdvert = noteConnectorReleaseAdvert;
|
|
88
88
|
exports.parseVersionManifest = parseVersionManifest;
|
|
89
|
+
exports.parseSafePointPolicy = parseSafePointPolicy;
|
|
90
|
+
exports.safePointBlockers = safePointBlockers;
|
|
89
91
|
exports.isSafeToApplyUpdate = isSafeToApplyUpdate;
|
|
92
|
+
exports.parseAlertWork = parseAlertWork;
|
|
90
93
|
exports.busynessOf = busynessOf;
|
|
91
94
|
exports.connectorRuntimeFromHealth = connectorRuntimeFromHealth;
|
|
92
95
|
exports.isReleaseBakedEnough = isReleaseBakedEnough;
|
|
93
96
|
exports.checkForConnectorUpdate = checkForConnectorUpdate;
|
|
97
|
+
exports.describeHandover = describeHandover;
|
|
94
98
|
exports.startConnectorUpdateService = startConnectorUpdateService;
|
|
95
99
|
exports.connectorDeviceTokens = connectorDeviceTokens;
|
|
96
100
|
exports.startConnectorUpdates = startConnectorUpdates;
|
|
@@ -254,9 +258,10 @@ function parseConnectorReleaseAdvert(raw) {
|
|
|
254
258
|
const hold = o.minReleaseAgeMs;
|
|
255
259
|
const minReleaseAgeMs = typeof hold === 'number' && Number.isSafeInteger(hold) && hold >= 0
|
|
256
260
|
&& hold <= MAX_SERVER_RELEASE_HOLD_MS ? hold : null;
|
|
257
|
-
|
|
261
|
+
const safePoint = parseSafePointPolicy(o.safePoint);
|
|
262
|
+
if (!versions.windows && !versions.macos && minReleaseAgeMs === null && safePoint === null)
|
|
258
263
|
return null;
|
|
259
|
-
return { versions, minReleaseAgeMs };
|
|
264
|
+
return { versions, minReleaseAgeMs, safePoint };
|
|
260
265
|
}
|
|
261
266
|
const releaseAdvertListeners = new Set();
|
|
262
267
|
/** Subscribe to adverts; returns the unsubscribe. One per update loop. */
|
|
@@ -471,32 +476,115 @@ class ConnectorManifestClient {
|
|
|
471
476
|
}
|
|
472
477
|
exports.ConnectorManifestClient = ConnectorManifestClient;
|
|
473
478
|
/**
|
|
474
|
-
*
|
|
479
|
+
* How long to wait for a safe point before applying anyway: 30 minutes.
|
|
475
480
|
*
|
|
476
|
-
*
|
|
481
|
+
* Two hours through 0.2.225, sized for the quiet rule that a saturated box
|
|
482
|
+
* almost never meets. Under `alert_work` the deadline is only the backstop for
|
|
483
|
+
* alert work that does not clear. Measured on the busiest box: on 2026-11-09
|
|
484
|
+
* (00:00-21:40Z) Texas had alert-supporting looks pending in 972 stretches
|
|
485
|
+
* covering 31% of the day, p50 3.4 s, p90 82 s, longest 568 s; 30 minutes is
|
|
486
|
+
* three times the longest. Past it the update applies and says which branch it
|
|
487
|
+
* took. Server-settable: runtime_config CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS.
|
|
488
|
+
*/
|
|
489
|
+
exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = 30 * 60 * 1000;
|
|
490
|
+
/**
|
|
491
|
+
* A wait/recheck follow-up due within this long blocks the hand-over: 150 s.
|
|
492
|
+
*
|
|
493
|
+
* The follow-up is the look that confirms a wait/recheck alert; if it falls due
|
|
494
|
+
* while the box restarts, the alert waits for the restart. Measured on the
|
|
495
|
+
* Texas restart into 0.2.224 (2026-11-09 17:55Z): the cameras answered again
|
|
496
|
+
* 61-124 s after their last answer before it (first capture 46 s after the
|
|
497
|
+
* last, then the detector warmed up). The wait anchor itself survives a restart
|
|
498
|
+
* (the archive is durable), so a follow-up due later than this is safe.
|
|
499
|
+
* Server-settable: runtime_config CONNECTOR_UPDATE_SAFE_POINT_FOLLOW_UP_HORIZON_MS.
|
|
500
|
+
*/
|
|
501
|
+
exports.CONNECTOR_UPDATE_FOLLOW_UP_HORIZON_MS = 150000;
|
|
502
|
+
exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY = Object.freeze({
|
|
503
|
+
rule: 'alert_work',
|
|
504
|
+
deadlineMs: exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS,
|
|
505
|
+
followUpHorizonMs: exports.CONNECTOR_UPDATE_FOLLOW_UP_HORIZON_MS,
|
|
506
|
+
});
|
|
507
|
+
/** A deadline longer than a day, or a horizon longer than the longest wait rule (10 min), is a typo. */
|
|
508
|
+
const MAX_SAFE_POINT_DEADLINE_MS = 24 * 60 * 60 * 1000;
|
|
509
|
+
const MAX_FOLLOW_UP_HORIZON_MS = 10 * 60 * 1000;
|
|
510
|
+
/** Strictly read `connectorRelease.safePoint`; null when any part is unusable (the box keeps its last policy). */
|
|
511
|
+
function parseSafePointPolicy(raw) {
|
|
512
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
513
|
+
return null;
|
|
514
|
+
const o = raw;
|
|
515
|
+
if (o.rule !== 'alert_work' && o.rule !== 'all_work')
|
|
516
|
+
return null;
|
|
517
|
+
const ms = (v, max) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 && v <= max;
|
|
518
|
+
if (!ms(o.deadlineMs, MAX_SAFE_POINT_DEADLINE_MS) || !ms(o.followUpHorizonMs, MAX_FOLLOW_UP_HORIZON_MS))
|
|
519
|
+
return null;
|
|
520
|
+
return { rule: o.rule, deadlineMs: o.deadlineMs, followUpHorizonMs: o.followUpHorizonMs };
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* Why this is NOT a moment to stop and replace the connector; empty when it is.
|
|
524
|
+
*
|
|
525
|
+
* `all_work` (and `alert_work` on a manager that cannot count alert work)
|
|
526
|
+
* waits for the connector's two counters to read zero:
|
|
527
|
+
*
|
|
528
|
+
* - `inFlight` — a frame the model is answering, or a POST in flight;
|
|
529
|
+
* - `buffered` — frames held for a free slot. On the legacy (non-retained)
|
|
530
|
+
* path these are frames the motion cascade flagged, held in memory, and a
|
|
531
|
+
* stop drops them.
|
|
532
|
+
*
|
|
533
|
+
* `alert_work` waits only for what a restart could cost an alert:
|
|
477
534
|
*
|
|
478
|
-
* -
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
*
|
|
482
|
-
* -
|
|
483
|
-
*
|
|
484
|
-
* 30 s scan cadence the next look is a different event. `CloudSensingManager
|
|
485
|
-
* .stop()` calls `candidates.reset()`, so these die on ANY stop, graceful or
|
|
486
|
-
* not — stopping while the buffer is loaded is the dropped-alert failure, and
|
|
487
|
-
* it is a product failure, not an implementation detail.
|
|
535
|
+
* - an alert-supporting look the server has not acknowledged: its alert, or
|
|
536
|
+
* the first sighting of a wait/recheck episode, would wait out the restart;
|
|
537
|
+
* - alert video still owed: the receipt naming an alert's clip request, the
|
|
538
|
+
* after-window still being recorded, the cut, the upload;
|
|
539
|
+
* - a wait/recheck follow-up due within `followUpHorizonMs`: it would fall due
|
|
540
|
+
* while the box restarts.
|
|
488
541
|
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
493
|
-
*
|
|
494
|
-
* yet judged interesting, which is a different and much smaller loss than a
|
|
495
|
-
* flagged candidate, and the next tick captures a fresher frame. Widening the
|
|
496
|
-
* health struct to close it was judged not worth the blast radius.
|
|
542
|
+
* Routine looks and the frame backlog are not waited for: on a retained-review
|
|
543
|
+
* box every frame is in the durable archive and spool, and the schedule picks
|
|
544
|
+
* them up after the restart. Looks still in the model are drained by the
|
|
545
|
+
* update loop before it commits (see startConnectorUpdateService), so a look
|
|
546
|
+
* that turns out to support an alert is never cut off.
|
|
497
547
|
*/
|
|
498
|
-
function
|
|
499
|
-
|
|
548
|
+
function safePointBlockers(busyness, policy = exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY, nowMs = Date.now()) {
|
|
549
|
+
const reasons = [];
|
|
550
|
+
if (busyness.hold)
|
|
551
|
+
reasons.push('setup or reconfiguration in progress');
|
|
552
|
+
const alert = busyness.alert;
|
|
553
|
+
if (policy.rule === 'all_work' || !alert) {
|
|
554
|
+
if (busyness.inFlight > 0)
|
|
555
|
+
reasons.push(`${busyness.inFlight} analysis(es) in flight`);
|
|
556
|
+
if (busyness.buffered > 0)
|
|
557
|
+
reasons.push(`${busyness.buffered} frame(s) held`);
|
|
558
|
+
return reasons;
|
|
559
|
+
}
|
|
560
|
+
if (alert.looksAwaitingAnswer > 0) {
|
|
561
|
+
reasons.push(`${alert.looksAwaitingAnswer} alert look(s) awaiting the server's answer`);
|
|
562
|
+
}
|
|
563
|
+
if (alert.nextFollowUpDueAtMs !== null && Number.isFinite(alert.nextFollowUpDueAtMs)
|
|
564
|
+
&& alert.nextFollowUpDueAtMs - nowMs <= policy.followUpHorizonMs) {
|
|
565
|
+
const s = Math.round((alert.nextFollowUpDueAtMs - nowMs) / 1000);
|
|
566
|
+
reasons.push(s >= 0 ? `follow-up due in ${s} s` : `follow-up overdue by ${-s} s`);
|
|
567
|
+
}
|
|
568
|
+
if ((busyness.alertVideo ?? 0) > 0)
|
|
569
|
+
reasons.push(`${busyness.alertVideo} alert video item(s) pending`);
|
|
570
|
+
return reasons;
|
|
571
|
+
}
|
|
572
|
+
/** Is this a moment at which the connector may be stopped and replaced? See safePointBlockers. */
|
|
573
|
+
function isSafeToApplyUpdate(busyness, policy = exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY, nowMs = Date.now()) {
|
|
574
|
+
return safePointBlockers(busyness, policy, nowMs).length === 0;
|
|
575
|
+
}
|
|
576
|
+
/** Strictly read a manager's alert-work report; null for anything malformed (never a fabricated "no work"). */
|
|
577
|
+
function parseAlertWork(raw) {
|
|
578
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
579
|
+
return null;
|
|
580
|
+
const o = raw;
|
|
581
|
+
const count = (v) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 0;
|
|
582
|
+
if (!count(o.looksAwaitingAnswer) || !count(o.looksInModel))
|
|
583
|
+
return null;
|
|
584
|
+
const due = o.nextFollowUpDueAtMs;
|
|
585
|
+
if (due !== null && !(typeof due === 'number' && Number.isFinite(due)))
|
|
586
|
+
return null;
|
|
587
|
+
return { looksAwaitingAnswer: o.looksAwaitingAnswer, looksInModel: o.looksInModel, nextFollowUpDueAtMs: due };
|
|
500
588
|
}
|
|
501
589
|
/**
|
|
502
590
|
* Map a `getSensingHealth()` reading onto the two numbers that matter. Tolerant
|
|
@@ -650,18 +738,41 @@ exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS = 60 * 60 * 1000;
|
|
|
650
738
|
*/
|
|
651
739
|
const STAGE_OUTCOME_CLOCK_GRACE_MS = 5000;
|
|
652
740
|
/**
|
|
653
|
-
*
|
|
654
|
-
*
|
|
655
|
-
*
|
|
656
|
-
*
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
*
|
|
661
|
-
*
|
|
662
|
-
*
|
|
741
|
+
* The drain before a hand-over under `alert_work`: new looks are held while the
|
|
742
|
+
* looks already in the model finish, because one of them may come back
|
|
743
|
+
* supporting an alert. Polled this often while it runs, so looks pause for
|
|
744
|
+
* about one look's length (Texas: ~1 s detector looks), not a poll period.
|
|
745
|
+
*/
|
|
746
|
+
exports.CONNECTOR_UPDATE_DRAIN_POLL_MS = 250;
|
|
747
|
+
/**
|
|
748
|
+
* The longest a drain holds new looks while the live looks in the model finish
|
|
749
|
+
* (a detector look takes ~1 s, one that asks Gemma on a store CPU up to ~13 s;
|
|
750
|
+
* backlog reviews are history and not counted). A drain that reaches this with
|
|
751
|
+
* a live look still running does NOT commit: that look may support an alert. It
|
|
752
|
+
* lets the cameras be looked at again and tries again after
|
|
753
|
+
* CONNECTOR_UPDATE_DRAIN_RETRY_MS; the deadline is the backstop.
|
|
754
|
+
*/
|
|
755
|
+
exports.CONNECTOR_UPDATE_DRAIN_MAX_MS = 20000;
|
|
756
|
+
/**
|
|
757
|
+
* A setup or reconfiguration hold outlasts the safe-point deadline by up to
|
|
758
|
+
* this long: the deadline accepts losing alert work, never breaking an
|
|
759
|
+
* operator's camera setup half-way, and the cap keeps a setup session left open
|
|
760
|
+
* from holding a fix back for ever.
|
|
663
761
|
*/
|
|
664
|
-
exports.
|
|
762
|
+
exports.CONNECTOR_UPDATE_HOLD_GRACE_MS = 30 * 60 * 1000;
|
|
763
|
+
/**
|
|
764
|
+
* After a drain is cut short by alert work (a look in the model came back
|
|
765
|
+
* supporting an alert), the next drain waits this long: while an alert
|
|
766
|
+
* condition flickers, each drain would otherwise pause every camera's looks.
|
|
767
|
+
*/
|
|
768
|
+
exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS = 30000;
|
|
769
|
+
/** The `current` state's detail on the first reports of the version a hand-over installed. */
|
|
770
|
+
function describeHandover(record) {
|
|
771
|
+
const waited = Math.round(record.waitedMs / 1000);
|
|
772
|
+
const how = record.branch === 'safe_point' ? `at a safe point (rule ${record.rule})` : 'when the safe-point deadline passed';
|
|
773
|
+
return `took ${record.toVersion} from ${record.fromVersion} ${how} ${waited} s after it was staged ` +
|
|
774
|
+
`(staged ${record.stagedAt}, handed over ${record.handedOverAt})`;
|
|
775
|
+
}
|
|
665
776
|
/** The newest of a non-empty list of releases. */
|
|
666
777
|
function newestRelease(releases) {
|
|
667
778
|
return releases.reduce((a, b) => (isNewerConnectorVersion(b.connectorVersion, a.connectorVersion) ? b : a));
|
|
@@ -716,7 +827,6 @@ function startConnectorUpdateService(deps) {
|
|
|
716
827
|
const requestExit = deps.requestExit ?? (() => process.exit(0));
|
|
717
828
|
const pollIntervalMs = deps.pollIntervalMs ?? exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS;
|
|
718
829
|
const stageDeadlineMs = deps.stageDeadlineMs ?? exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS;
|
|
719
|
-
const safePointDeadlineMs = deps.safePointDeadlineMs ?? exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS;
|
|
720
830
|
// Where the loop stands, sent on every report. Every branch below that used to
|
|
721
831
|
// end in only a local log line also sets this, so a box that never updates
|
|
722
832
|
// tells the fleet why instead of looking exactly like one that is current.
|
|
@@ -744,6 +854,27 @@ function startConnectorUpdateService(deps) {
|
|
|
744
854
|
// stated value (runtime_config via `connectorRelease`), else none.
|
|
745
855
|
let serverHoldMs = null;
|
|
746
856
|
const releaseHoldMs = () => deps.minReleaseAgeMs ?? serverHoldMs ?? exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS;
|
|
857
|
+
// The safe-point policy in force: a test's fixed value, else the server's last
|
|
858
|
+
// stated value (runtime_config via `connectorRelease.safePoint`), else the
|
|
859
|
+
// default. Read at every look, so a corrected rule or deadline reaches a box
|
|
860
|
+
// that is already waiting within one profile poll.
|
|
861
|
+
let serverSafePoint = null;
|
|
862
|
+
const safePointPolicy = () => {
|
|
863
|
+
const policy = deps.safePointPolicy ?? serverSafePoint ?? exports.DEFAULT_CONNECTOR_SAFE_POINT_POLICY;
|
|
864
|
+
return deps.safePointDeadlineMs === undefined ? policy : { ...policy, deadlineMs: deps.safePointDeadlineMs };
|
|
865
|
+
};
|
|
866
|
+
// When the update being applied was staged and verified (or found verified).
|
|
867
|
+
let stagedAtMs = 0;
|
|
868
|
+
// The drain before a hand-over (alert_work): new looks held since drainStartedMs.
|
|
869
|
+
let drainStartedMs = null;
|
|
870
|
+
let drainTimer = null;
|
|
871
|
+
let nextDrainAtMs = 0;
|
|
872
|
+
// Consecutive drains cut off by a live look outlasting the cap: each doubles
|
|
873
|
+
// the wait before the next, so a hung model call does not pause looks for
|
|
874
|
+
// 20 s in every minute until it times out.
|
|
875
|
+
let drainCapOuts = 0;
|
|
876
|
+
// The blockers last logged, so a box waiting for alert work says why once per change.
|
|
877
|
+
let lastBlockers = null;
|
|
747
878
|
// The newest version the last good read of the channel listed. The doorbell
|
|
748
879
|
// reads the channel again only for something newer than this.
|
|
749
880
|
let lastSeenVersion = null;
|
|
@@ -754,14 +885,49 @@ function startConnectorUpdateService(deps) {
|
|
|
754
885
|
t.unref();
|
|
755
886
|
}
|
|
756
887
|
};
|
|
757
|
-
|
|
888
|
+
/** Stop the drain's fast poll; `release` also lets the manager start looks again. */
|
|
889
|
+
const endDrain = (release) => {
|
|
890
|
+
if (drainTimer) {
|
|
891
|
+
clearIntervalFn(drainTimer);
|
|
892
|
+
drainTimer = null;
|
|
893
|
+
}
|
|
894
|
+
if (drainStartedMs === null)
|
|
895
|
+
return;
|
|
896
|
+
drainStartedMs = null;
|
|
897
|
+
if (!release)
|
|
898
|
+
return;
|
|
899
|
+
try {
|
|
900
|
+
deps.holdReviews?.(null);
|
|
901
|
+
}
|
|
902
|
+
catch {
|
|
903
|
+
/* the hold expires on its own */
|
|
904
|
+
}
|
|
905
|
+
};
|
|
906
|
+
const endApply = (releaseHold = true) => {
|
|
758
907
|
phase = 'idle';
|
|
759
908
|
target = null;
|
|
909
|
+
lastBlockers = null;
|
|
910
|
+
drainCapOuts = 0;
|
|
911
|
+
endDrain(releaseHold);
|
|
760
912
|
if (pollTimer) {
|
|
761
913
|
clearIntervalFn(pollTimer);
|
|
762
914
|
pollTimer = null;
|
|
763
915
|
}
|
|
764
916
|
};
|
|
917
|
+
// What the previous process left when it handed this version the machine.
|
|
918
|
+
// Reported (as the `current` state's detail) until a report lands everywhere.
|
|
919
|
+
let handover = null;
|
|
920
|
+
let handoverReported = false;
|
|
921
|
+
try {
|
|
922
|
+
const left = applier?.readHandover?.() ?? null;
|
|
923
|
+
if (left && left.toVersion === deps.installedVersion)
|
|
924
|
+
handover = left;
|
|
925
|
+
else if (left)
|
|
926
|
+
applier?.clearHandover?.(); // a rolled-back update's record: not about this version
|
|
927
|
+
}
|
|
928
|
+
catch {
|
|
929
|
+
handover = null;
|
|
930
|
+
}
|
|
765
931
|
/**
|
|
766
932
|
* Report where this install stands, carrying any update outcome the applier
|
|
767
933
|
* left behind. The outcome is cleared only once the server has ACCEPTED it —
|
|
@@ -821,6 +987,18 @@ function startConnectorUpdateService(deps) {
|
|
|
821
987
|
if (outcome && result.attempted > 0 && result.accepted === result.attempted) {
|
|
822
988
|
applier?.clearOutcome();
|
|
823
989
|
}
|
|
990
|
+
// The handover record follows the same rule: forgotten on disk once every
|
|
991
|
+
// feed carried it, kept in memory as this process's `current` detail.
|
|
992
|
+
if (handover && !handoverReported && updater?.state === 'current'
|
|
993
|
+
&& result.attempted > 0 && result.accepted === result.attempted) {
|
|
994
|
+
handoverReported = true;
|
|
995
|
+
try {
|
|
996
|
+
applier?.clearHandover?.();
|
|
997
|
+
}
|
|
998
|
+
catch {
|
|
999
|
+
/* a re-report after a restart is harmless */
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
824
1002
|
};
|
|
825
1003
|
/** Advance the apply state machine. Called on a short timer while in flight. */
|
|
826
1004
|
const pollApply = () => {
|
|
@@ -832,6 +1010,7 @@ function startConnectorUpdateService(deps) {
|
|
|
832
1010
|
log(`[update] ${target.connectorVersion} is staged and verified — waiting for a safe point`);
|
|
833
1011
|
phase = 'awaiting-safe-point';
|
|
834
1012
|
phaseStartedMs = now();
|
|
1013
|
+
stagedAtMs = phaseStartedMs;
|
|
835
1014
|
elapsed = 0;
|
|
836
1015
|
setUpdater('awaiting_safe_point', target.connectorVersion);
|
|
837
1016
|
// Falls through: a connector that is already quiet hands over in this
|
|
@@ -860,26 +1039,128 @@ function startConnectorUpdateService(deps) {
|
|
|
860
1039
|
return;
|
|
861
1040
|
}
|
|
862
1041
|
// awaiting-safe-point
|
|
1042
|
+
const version = target.connectorVersion;
|
|
1043
|
+
const policy = safePointPolicy();
|
|
1044
|
+
const nowMs = now();
|
|
863
1045
|
const busyness = deps.busyness ? deps.busyness() : { inFlight: 0, buffered: 0 };
|
|
864
|
-
const
|
|
865
|
-
const
|
|
866
|
-
|
|
1046
|
+
const blockers = safePointBlockers(busyness, policy, nowMs);
|
|
1047
|
+
const alertRule = policy.rule === 'alert_work' && Boolean(busyness.alert);
|
|
1048
|
+
let branch = 'safe_point';
|
|
1049
|
+
if (busyness.hold && elapsed < policy.deadlineMs + exports.CONNECTOR_UPDATE_HOLD_GRACE_MS) {
|
|
1050
|
+
// An operator's camera setup or a reconfiguration is running: past the
|
|
1051
|
+
// deadline too, for a bounded grace.
|
|
1052
|
+
if (drainStartedMs !== null)
|
|
1053
|
+
endDrain(true);
|
|
1054
|
+
const kinds = 'setup or reconfiguration in progress';
|
|
1055
|
+
if (kinds !== lastBlockers)
|
|
1056
|
+
log(`[update] ${version} waits: ${kinds}`);
|
|
1057
|
+
lastBlockers = kinds;
|
|
867
1058
|
return;
|
|
868
|
-
|
|
1059
|
+
}
|
|
1060
|
+
if (elapsed >= policy.deadlineMs) {
|
|
1061
|
+
branch = 'deadline';
|
|
869
1062
|
log(`[update] no safe point in ${Math.round(elapsed / 60000)} min — applying anyway ` +
|
|
870
|
-
`(inFlight=${busyness.inFlight}, held candidates=${busyness.buffered}
|
|
1063
|
+
`(inFlight=${busyness.inFlight}, held candidates=${busyness.buffered}` +
|
|
1064
|
+
`${blockers.length ? `; ${blockers.join('; ')}` : ''})`);
|
|
871
1065
|
}
|
|
872
|
-
else {
|
|
873
|
-
|
|
1066
|
+
else if (blockers.length) {
|
|
1067
|
+
// Alert work appeared, or never cleared: a running drain lets the
|
|
1068
|
+
// cameras be looked at again, and the next one waits a little.
|
|
1069
|
+
if (drainStartedMs !== null) {
|
|
1070
|
+
endDrain(true);
|
|
1071
|
+
nextDrainAtMs = nowMs + exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS;
|
|
1072
|
+
// Ended by alert work, not by a look outlasting the cap: the cap-out
|
|
1073
|
+
// backoff counts CONSECUTIVE cap-outs only.
|
|
1074
|
+
drainCapOuts = 0;
|
|
1075
|
+
}
|
|
1076
|
+
// Logged when the KIND of work changes, not every countdown second.
|
|
1077
|
+
const kinds = blockers.join('; ').replace(/\d+/g, '#');
|
|
1078
|
+
if (alertRule && kinds !== lastBlockers)
|
|
1079
|
+
log(`[update] ${version} waits for alert work: ${blockers.join('; ')}`);
|
|
1080
|
+
lastBlockers = kinds;
|
|
1081
|
+
return;
|
|
874
1082
|
}
|
|
875
|
-
|
|
876
|
-
|
|
1083
|
+
else if (alertRule && deps.holdReviews) {
|
|
1084
|
+
// No alert work is visible, but a look the model is answering may come
|
|
1085
|
+
// back supporting an alert. Hold new looks, let those finish, and hand
|
|
1086
|
+
// over only if nothing they answered supports an alert (the look above
|
|
1087
|
+
// then sees it as alert work and the drain ends).
|
|
1088
|
+
const inModel = busyness.alert.looksInModel;
|
|
1089
|
+
if (drainStartedMs === null) {
|
|
1090
|
+
if (nowMs < nextDrainAtMs)
|
|
1091
|
+
return;
|
|
1092
|
+
drainStartedMs = nowMs;
|
|
1093
|
+
try {
|
|
1094
|
+
// Self-expiring: if this loop died mid-drain, looks resume on their own.
|
|
1095
|
+
deps.holdReviews(nowMs + exports.CONNECTOR_UPDATE_DRAIN_MAX_MS + 5000);
|
|
1096
|
+
}
|
|
1097
|
+
catch (err) {
|
|
1098
|
+
log(`[update] could not hold new looks (non-fatal): ${String(err)}`);
|
|
1099
|
+
}
|
|
1100
|
+
if (inModel > 0) {
|
|
1101
|
+
log(`[update] no alert work — holding new looks while ${inModel} in progress finish before applying ${version}`);
|
|
1102
|
+
if (!drainTimer) {
|
|
1103
|
+
drainTimer = setIntervalFn(() => {
|
|
1104
|
+
try {
|
|
1105
|
+
pollApply();
|
|
1106
|
+
}
|
|
1107
|
+
catch (err) {
|
|
1108
|
+
log(`[update] apply step failed (non-fatal): ${String(err)}`);
|
|
1109
|
+
}
|
|
1110
|
+
}, exports.CONNECTOR_UPDATE_DRAIN_POLL_MS);
|
|
1111
|
+
unref(drainTimer);
|
|
1112
|
+
}
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
else if (inModel > 0) {
|
|
1117
|
+
if (nowMs - drainStartedMs < exports.CONNECTOR_UPDATE_DRAIN_MAX_MS)
|
|
1118
|
+
return;
|
|
1119
|
+
// A live look outlasted the drain: it may support an alert, so it is
|
|
1120
|
+
// not cut off. Look at the cameras again and try later.
|
|
1121
|
+
drainCapOuts += 1;
|
|
1122
|
+
const retryMs = Math.min(exports.CONNECTOR_UPDATE_DRAIN_RETRY_MS * 2 ** (drainCapOuts - 1), 10 * 60 * 1000);
|
|
1123
|
+
log(`[update] ${inModel} look(s) still in the model after ${Math.round(exports.CONNECTOR_UPDATE_DRAIN_MAX_MS / 1000)} s — ` +
|
|
1124
|
+
`not applying ${version} over them; trying again in ${Math.round(retryMs / 1000)} s`);
|
|
1125
|
+
endDrain(true);
|
|
1126
|
+
nextDrainAtMs = nowMs + retryMs;
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
if (branch === 'safe_point') {
|
|
1131
|
+
if (alertRule) {
|
|
1132
|
+
log(`[update] safe point after ${Math.round(elapsed / 1000)} s (rule alert_work; not waited for: ` +
|
|
1133
|
+
`${busyness.inFlight} look(s) in flight, ${busyness.buffered} frame(s) held) — applying ${version}`);
|
|
1134
|
+
}
|
|
1135
|
+
else {
|
|
1136
|
+
log('[update] connector is idle — applying');
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
const handedOverMs = now();
|
|
1140
|
+
// The hold stays on through the exit: nothing new starts while the machine is handed over.
|
|
1141
|
+
endApply(false);
|
|
877
1142
|
if (!applier.commit({ version, currentVersion: deps.installedVersion })) {
|
|
1143
|
+
try {
|
|
1144
|
+
deps.holdReviews?.(null);
|
|
1145
|
+
}
|
|
1146
|
+
catch {
|
|
1147
|
+
/* expires on its own */
|
|
1148
|
+
}
|
|
878
1149
|
log('[update] could not launch the applier — staying on the current version');
|
|
879
1150
|
const why = applier.launchFailure?.() ?? null;
|
|
880
1151
|
setUpdater('apply_failed', why ? `${version}: ${why}` : version);
|
|
881
1152
|
return;
|
|
882
1153
|
}
|
|
1154
|
+
try {
|
|
1155
|
+
applier.recordHandover?.({
|
|
1156
|
+
version: 1, fromVersion: deps.installedVersion, toVersion: version, rule: policy.rule, branch,
|
|
1157
|
+
stagedAt: new Date(stagedAtMs || phaseStartedMs).toISOString(), handedOverAt: new Date(handedOverMs).toISOString(),
|
|
1158
|
+
waitedMs: Math.max(0, handedOverMs - (stagedAtMs || phaseStartedMs)),
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
catch (err) {
|
|
1162
|
+
log(`[update] could not record the hand-over (non-fatal): ${String(err)}`);
|
|
1163
|
+
}
|
|
883
1164
|
setUpdater('applying', version);
|
|
884
1165
|
// The applier owns the machine now. Exiting frees bin\node.exe for the swap;
|
|
885
1166
|
// it also means anything after this line does not run.
|
|
@@ -910,6 +1191,7 @@ function startConnectorUpdateService(deps) {
|
|
|
910
1191
|
phase = alreadyStaged ? 'awaiting-safe-point' : 'staging';
|
|
911
1192
|
target = manifest;
|
|
912
1193
|
phaseStartedMs = now();
|
|
1194
|
+
stagedAtMs = alreadyStaged ? phaseStartedMs : 0;
|
|
913
1195
|
if (!pollTimer) {
|
|
914
1196
|
pollTimer = setIntervalFn(() => {
|
|
915
1197
|
try {
|
|
@@ -936,7 +1218,8 @@ function startConnectorUpdateService(deps) {
|
|
|
936
1218
|
return null;
|
|
937
1219
|
}
|
|
938
1220
|
if (!check.availableVersion) {
|
|
939
|
-
|
|
1221
|
+
// The first reports of a version a hand-over installed say how it got here.
|
|
1222
|
+
setUpdater('current', handover ? describeHandover(handover) : null);
|
|
940
1223
|
return null;
|
|
941
1224
|
}
|
|
942
1225
|
const stageable = connectorReleaseChoices(check.manifest).filter(m => isNewerConnectorVersion(m.connectorVersion, deps.installedVersion) && applier.canStage(m));
|
|
@@ -1088,6 +1371,9 @@ function startConnectorUpdateService(deps) {
|
|
|
1088
1371
|
const holdChanged = advert.minReleaseAgeMs !== null && advert.minReleaseAgeMs !== serverHoldMs;
|
|
1089
1372
|
if (advert.minReleaseAgeMs !== null)
|
|
1090
1373
|
serverHoldMs = advert.minReleaseAgeMs;
|
|
1374
|
+
// Taken even mid-wait: the next look at the safe point uses it.
|
|
1375
|
+
if (advert.safePoint)
|
|
1376
|
+
serverSafePoint = advert.safePoint;
|
|
1091
1377
|
// Mid-stage, mid-wait, or backing off a failed read: those paths already
|
|
1092
1378
|
// own the next read.
|
|
1093
1379
|
if (phase !== 'idle' || failedReads > 0)
|
|
@@ -1181,6 +1467,8 @@ function startConnectorUpdateService(deps) {
|
|
|
1181
1467
|
clearIntervalFn(modelsTimer);
|
|
1182
1468
|
if (pollTimer)
|
|
1183
1469
|
clearIntervalFn(pollTimer);
|
|
1470
|
+
// A loop stopped mid-drain (a reconfigure rebuilds it) lets the cameras be looked at again.
|
|
1471
|
+
endDrain(true);
|
|
1184
1472
|
}
|
|
1185
1473
|
catch {
|
|
1186
1474
|
/* best-effort */
|
|
@@ -1294,6 +1582,7 @@ function startConnectorUpdates(opts) {
|
|
|
1294
1582
|
apiBaseUrl,
|
|
1295
1583
|
deviceTokens,
|
|
1296
1584
|
busyness: opts.busyness,
|
|
1585
|
+
...(opts.holdReviews ? { holdReviews: opts.holdReviews } : {}),
|
|
1297
1586
|
runtime: opts.runtime,
|
|
1298
1587
|
inventory: opts.inventory,
|
|
1299
1588
|
...(opts.videoBuffer ? { videoBuffer: opts.videoBuffer } : {}),
|