@trillboards/edge-sdk 0.2.219 → 0.2.221
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/macos/README.md +4 -2
- package/deploy/windows/README.md +12 -3
- package/deploy/windows/apply-update.ps1 +58 -0
- package/deploy/windows/ci/test-field-self-update.ps1 +35 -13
- package/deploy/windows/ci/test-published-install.ps1 +55 -7
- package/deploy/windows/ci/test-rollout-proofs.ps1 +538 -0
- package/deploy/windows/install-update.ps1 +147 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/PhoneHeldInHand.d.ts +20 -1
- package/dist/PhoneHeldInHand.d.ts.map +1 -1
- package/dist/PhoneHeldInHand.js +10 -7
- package/dist/PhoneHeldInHand.js.map +1 -1
- package/dist/ScopedVlmReview.d.ts.map +1 -1
- package/dist/ScopedVlmReview.js +2 -1
- package/dist/ScopedVlmReview.js.map +1 -1
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +5 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +5 -0
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +3 -1
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +113 -28
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +289 -46
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/connectorUpdateApply.d.ts +37 -0
- package/dist/connectorUpdateApply.d.ts.map +1 -1
- package/dist/connectorUpdateApply.js +70 -1
- package/dist/connectorUpdateApply.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/package.json +1 -1
package/dist/connectorUpdate.js
CHANGED
|
@@ -76,11 +76,15 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
76
76
|
};
|
|
77
77
|
})();
|
|
78
78
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
79
|
-
exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS = exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS = exports.CONNECTOR_MANIFEST_RETRY_BASE_MS = exports.CONNECTOR_UPDATE_INITIAL_DELAY_MS = exports.ConnectorManifestClient = exports.defaultManifestFetcher = exports.
|
|
79
|
+
exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = exports.CONNECTOR_UPDATE_STAGE_DEADLINE_MS = exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS = exports.CONNECTOR_MANIFEST_RETRY_BASE_MS = exports.CONNECTOR_UPDATE_INITIAL_DELAY_MS = exports.ConnectorManifestClient = exports.defaultManifestFetcher = exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS = exports.CONNECTOR_MODEL_REPORT_INTERVAL_MS = exports.CONNECTOR_RELEASE_DOORBELL_MIN_GAP_MS = exports.CONNECTOR_UPDATE_CHECK_INTERVAL_MS = exports.CONNECTOR_UPDATE_BASE_URL = void 0;
|
|
80
80
|
exports.parseConnectorVersion = parseConnectorVersion;
|
|
81
81
|
exports.isNewerConnectorVersion = isNewerConnectorVersion;
|
|
82
82
|
exports.manifestObjectForPlatform = manifestObjectForPlatform;
|
|
83
83
|
exports.manifestUrlForPlatform = manifestUrlForPlatform;
|
|
84
|
+
exports.releaseAdvertKeyForPlatform = releaseAdvertKeyForPlatform;
|
|
85
|
+
exports.parseConnectorReleaseAdvert = parseConnectorReleaseAdvert;
|
|
86
|
+
exports.onConnectorReleaseAdvert = onConnectorReleaseAdvert;
|
|
87
|
+
exports.noteConnectorReleaseAdvert = noteConnectorReleaseAdvert;
|
|
84
88
|
exports.parseVersionManifest = parseVersionManifest;
|
|
85
89
|
exports.isSafeToApplyUpdate = isSafeToApplyUpdate;
|
|
86
90
|
exports.busynessOf = busynessOf;
|
|
@@ -97,27 +101,39 @@ const hostSpecs_1 = require("./hostSpecs");
|
|
|
97
101
|
/** Where the release workflows publish every platform's manifest. */
|
|
98
102
|
exports.CONNECTOR_UPDATE_BASE_URL = 'https://screen.trillboards.com/connector';
|
|
99
103
|
/**
|
|
100
|
-
* How often an install
|
|
104
|
+
* How often an install reads the release channel ON ITS OWN. One hour, and it is
|
|
105
|
+
* the BACKSTOP, not how a box learns of a release.
|
|
101
106
|
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
107
|
+
* Discovery rides a poll every box already makes: each RTSP feed's video
|
|
108
|
+
* recorder GETs `/v2/earner/cameras/sensing-profile` every five minutes, and the
|
|
109
|
+
* API adds `connectorRelease` — the newest version each platform's channel lists
|
|
110
|
+
* — to that response. A version newer than this install rings the doorbell
|
|
111
|
+
* (`noteConnectorReleaseAdvert`) and the update loop reads the channel at once.
|
|
112
|
+
* This interval only decides how long a box goes without looking when the API
|
|
113
|
+
* cannot say (the API's own read of the channel failed, or an entrypoint polls
|
|
114
|
+
* no profile).
|
|
115
|
+
*
|
|
116
|
+
* It was six hours, and together with a six-hour release hold it put a merged
|
|
117
|
+
* fix on a store PC 6-12 h later (2026-11-08: fixes published 22:08-23:39Z were
|
|
118
|
+
* due on the Texas store at ~08:50Z the next morning).
|
|
119
|
+
*/
|
|
120
|
+
exports.CONNECTOR_UPDATE_CHECK_INTERVAL_MS = 60 * 60 * 1000;
|
|
121
|
+
/**
|
|
122
|
+
* Fewest milliseconds between two doorbell-triggered reads of the channel. Every
|
|
123
|
+
* feed rings it (32 on the Texas PC, every five minutes, and every five SECONDS
|
|
124
|
+
* on a feed with a live website admission hint), so this keeps one release from
|
|
125
|
+
* becoming 32 CDN reads.
|
|
109
126
|
*/
|
|
110
|
-
exports.
|
|
127
|
+
exports.CONNECTOR_RELEASE_DOORBELL_MIN_GAP_MS = 60000;
|
|
111
128
|
/**
|
|
112
129
|
* How often the DETECTOR MODEL INVENTORY is reported. Five minutes, and it is
|
|
113
130
|
* not a taste: `checkFleetHealth` counts only rows `reported_at > NOW() -
|
|
114
131
|
* INTERVAL '10 minutes'`, so a beacon slower than that window leaves the table
|
|
115
132
|
* empty whenever anyone looks, and the fleet reads `insufficient_data` for ever.
|
|
116
133
|
*
|
|
117
|
-
* SEPARATE FROM THE UPDATE CHECK, which is
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
* feed its own consumer. Two questions, two clocks.
|
|
134
|
+
* SEPARATE FROM THE UPDATE CHECK, which is an hourly backstop. Piggybacking the
|
|
135
|
+
* inventory on that clock produced a producer that could not feed its own
|
|
136
|
+
* consumer. Two questions, two clocks.
|
|
121
137
|
*
|
|
122
138
|
* The report is cheap on purpose — one loopback GET to the detector plus one
|
|
123
139
|
* small POST — so the interval is set by the consumer's window rather than by
|
|
@@ -125,19 +141,35 @@ exports.CONNECTOR_UPDATE_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
|
125
141
|
*/
|
|
126
142
|
exports.CONNECTOR_MODEL_REPORT_INTERVAL_MS = 5 * 60 * 1000;
|
|
127
143
|
/**
|
|
128
|
-
* How old a release must be before an install
|
|
144
|
+
* How old a release must be before an install takes it, when the server states
|
|
145
|
+
* no hold: ZERO. A release is taken the moment it is published.
|
|
146
|
+
*
|
|
147
|
+
* This was a six-hour "bake window", on the theory that waiting is what stands
|
|
148
|
+
* between a merged regression and every store. Waiting proves nothing: a build
|
|
149
|
+
* that cannot start on a store PC fails exactly the same way after six hours.
|
|
150
|
+
* The checks that do prove something now stand there instead:
|
|
151
|
+
*
|
|
152
|
+
* - before publication, the release workflow installs the signed package on a
|
|
153
|
+
* fresh Windows runner and runs real local AI on it, and nothing publishes
|
|
154
|
+
* until that passes;
|
|
155
|
+
* - after installation, `install-update.ps1` starts the new version and waits
|
|
156
|
+
* for it to report itself healthy (the marker `markHealthy` writes once
|
|
157
|
+
* startup completes and the update loop is running). A version that does not
|
|
158
|
+
* is rolled back to the previous one on the spot, recorded in
|
|
159
|
+
* `failed-versions.json`, and never taken again on that machine.
|
|
129
160
|
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
* "every store stopped sensing", and it costs a fix at most one extra check.
|
|
161
|
+
* If ops ever needs a hold (an incident, a staged rollout), it is a server
|
|
162
|
+
* value: runtime_config `CONNECTOR_UPDATE_MIN_RELEASE_AGE_MS`, default 0, sent
|
|
163
|
+
* as `connectorRelease.minReleaseAgeMs` on every sensing-profile response. It
|
|
164
|
+
* reaches every box within one profile poll and needs no release.
|
|
135
165
|
*
|
|
136
166
|
* `publishedAt` comes from the manifest, i.e. from the same channel as the
|
|
137
|
-
* payload, so
|
|
138
|
-
* security control is the Authenticode verification in the applier.
|
|
167
|
+
* payload, so a hold is a release-hygiene control and NOT a security control.
|
|
168
|
+
* The security control is the Authenticode verification in the applier.
|
|
139
169
|
*/
|
|
140
|
-
exports.
|
|
170
|
+
exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS = 0;
|
|
171
|
+
/** A hold longer than this is a typo, not a policy; it is ignored. */
|
|
172
|
+
const MAX_SERVER_RELEASE_HOLD_MS = 30 * 24 * 60 * 60 * 1000;
|
|
141
173
|
/** Hard cap on the manifest body. It is ~401 bytes; anything near this is hostile. */
|
|
142
174
|
const MANIFEST_MAX_BYTES = 64 * 1024;
|
|
143
175
|
/** Manifest fetch timeout. */
|
|
@@ -197,6 +229,64 @@ function manifestUrlForPlatform(platform, baseUrl = exports.CONNECTOR_UPDATE_BAS
|
|
|
197
229
|
function isNonEmptyString(v) {
|
|
198
230
|
return typeof v === 'string' && v.length > 0;
|
|
199
231
|
}
|
|
232
|
+
/** The advert key for a platform, or null where no channel is published. */
|
|
233
|
+
function releaseAdvertKeyForPlatform(platform) {
|
|
234
|
+
if (platform === 'win32')
|
|
235
|
+
return 'windows';
|
|
236
|
+
if (platform === 'darwin')
|
|
237
|
+
return 'macos';
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
/** Strictly read a `connectorRelease` body; null when it says nothing usable. */
|
|
241
|
+
function parseConnectorReleaseAdvert(raw) {
|
|
242
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
243
|
+
return null;
|
|
244
|
+
const o = raw;
|
|
245
|
+
const versions = {};
|
|
246
|
+
for (const key of ['windows', 'macos']) {
|
|
247
|
+
const entry = o[key];
|
|
248
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry))
|
|
249
|
+
continue;
|
|
250
|
+
const version = entry.version;
|
|
251
|
+
if (typeof version === 'string' && parseConnectorVersion(version))
|
|
252
|
+
versions[key] = version;
|
|
253
|
+
}
|
|
254
|
+
const hold = o.minReleaseAgeMs;
|
|
255
|
+
const minReleaseAgeMs = typeof hold === 'number' && Number.isSafeInteger(hold) && hold >= 0
|
|
256
|
+
&& hold <= MAX_SERVER_RELEASE_HOLD_MS ? hold : null;
|
|
257
|
+
if (!versions.windows && !versions.macos && minReleaseAgeMs === null)
|
|
258
|
+
return null;
|
|
259
|
+
return { versions, minReleaseAgeMs };
|
|
260
|
+
}
|
|
261
|
+
const releaseAdvertListeners = new Set();
|
|
262
|
+
/** Subscribe to adverts; returns the unsubscribe. One per update loop. */
|
|
263
|
+
function onConnectorReleaseAdvert(listener) {
|
|
264
|
+
releaseAdvertListeners.add(listener);
|
|
265
|
+
return () => {
|
|
266
|
+
releaseAdvertListeners.delete(listener);
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Hand a sensing-profile response's `connectorRelease` to the update loop.
|
|
271
|
+
* Called by `SensingProfileClient.fetchProfile` on every parsed body. Never
|
|
272
|
+
* throws and costs nothing when no loop listens: a profile fetch must not be
|
|
273
|
+
* able to fail because of the updater.
|
|
274
|
+
*/
|
|
275
|
+
function noteConnectorReleaseAdvert(raw) {
|
|
276
|
+
if (raw === undefined || raw === null || releaseAdvertListeners.size === 0)
|
|
277
|
+
return;
|
|
278
|
+
const advert = parseConnectorReleaseAdvert(raw);
|
|
279
|
+
if (!advert)
|
|
280
|
+
return;
|
|
281
|
+
for (const listener of [...releaseAdvertListeners]) {
|
|
282
|
+
try {
|
|
283
|
+
listener(advert);
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
/* the profile's caller never pays for the updater */
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
200
290
|
/**
|
|
201
291
|
* Strictly validate a manifest body. Returns null when ANY required field is
|
|
202
292
|
* absent or malformed — including when the body is the CDN's HTML SPA fallback,
|
|
@@ -477,13 +567,21 @@ function connectorRuntimeFromHealth(health) {
|
|
|
477
567
|
analyzeSamples: Number.isFinite(samples) && samples >= 0 ? samples : 0,
|
|
478
568
|
};
|
|
479
569
|
}
|
|
480
|
-
/**
|
|
481
|
-
|
|
570
|
+
/**
|
|
571
|
+
* Is this release clear of the server's hold?
|
|
572
|
+
*
|
|
573
|
+
* With no hold (the default) the answer is yes without consulting a clock at
|
|
574
|
+
* all, so a store PC whose clock runs behind the release runner never refuses a
|
|
575
|
+
* fresh build. Only a positive hold reads `publishedAt`.
|
|
576
|
+
*/
|
|
577
|
+
function isReleaseBakedEnough(manifest, nowMs, minAgeMs = exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS) {
|
|
578
|
+
if (!(minAgeMs > 0))
|
|
579
|
+
return true;
|
|
482
580
|
const published = Date.parse(manifest.publishedAt);
|
|
483
581
|
if (Number.isNaN(published))
|
|
484
582
|
return false;
|
|
485
|
-
//
|
|
486
|
-
// age. Refusing is the
|
|
583
|
+
// Under a hold, a publishedAt in the FUTURE (clock skew either side) is not
|
|
584
|
+
// evidence of age. Refusing is the direction the hold asked for.
|
|
487
585
|
return nowMs - published >= minAgeMs;
|
|
488
586
|
}
|
|
489
587
|
/** Only the current fetched publication authorizes a release; no local candidate cache. */
|
|
@@ -523,19 +621,21 @@ async function checkForConnectorUpdate(installedVersion, client) {
|
|
|
523
621
|
* likely to be stale. The check is tiny and non-blocking, so schedule it on the
|
|
524
622
|
* next event-loop turn: every paired connector checks at launch without
|
|
525
623
|
* delaying sensing.
|
|
526
|
-
*
|
|
527
|
-
* A fresh release still cannot be applied on that first check: the bake window
|
|
528
|
-
* is enforced separately, on `publishedAt`.
|
|
529
624
|
*/
|
|
530
625
|
exports.CONNECTOR_UPDATE_INITIAL_DELAY_MS = 0;
|
|
531
626
|
/**
|
|
532
627
|
* First retry after a manifest read fails; each further failure doubles it, up
|
|
533
628
|
* to the ordinary check interval. A ~16 KB GET every few minutes while a box
|
|
534
|
-
* cannot read is nothing;
|
|
629
|
+
* cannot read is nothing; an hour blind per failed attempt is not.
|
|
535
630
|
*/
|
|
536
631
|
exports.CONNECTOR_MANIFEST_RETRY_BASE_MS = 5 * 60 * 1000;
|
|
537
|
-
/**
|
|
538
|
-
|
|
632
|
+
/**
|
|
633
|
+
* How often the apply state machine looks at the world while an update is in
|
|
634
|
+
* flight. Ten seconds: each look is a stat of staged.json and two counters, so
|
|
635
|
+
* a shorter period costs nothing, and at sixty seconds every update waited up
|
|
636
|
+
* to two whole periods (staged, then the safe point) after the bytes were ready.
|
|
637
|
+
*/
|
|
638
|
+
exports.CONNECTOR_UPDATE_POLL_INTERVAL_MS = 10000;
|
|
539
639
|
/**
|
|
540
640
|
* How long to wait for staging to finish before giving up on this attempt.
|
|
541
641
|
*
|
|
@@ -562,6 +662,10 @@ const STAGE_OUTCOME_CLOCK_GRACE_MS = 5000;
|
|
|
562
662
|
* loss after two hours and record which branch was taken.
|
|
563
663
|
*/
|
|
564
664
|
exports.CONNECTOR_UPDATE_SAFE_POINT_DEADLINE_MS = 2 * 60 * 60 * 1000;
|
|
665
|
+
/** The newest of a non-empty list of releases. */
|
|
666
|
+
function newestRelease(releases) {
|
|
667
|
+
return releases.reduce((a, b) => (isNewerConnectorVersion(b.connectorVersion, a.connectorVersion) ? b : a));
|
|
668
|
+
}
|
|
565
669
|
/**
|
|
566
670
|
* The whole update loop: check the manifest, tell the server where this install
|
|
567
671
|
* stands, and — with a verified payload and a quiet moment — apply.
|
|
@@ -634,6 +738,17 @@ function startConnectorUpdateService(deps) {
|
|
|
634
738
|
let checkTimer = null;
|
|
635
739
|
let modelsTimer = null;
|
|
636
740
|
let initialTimer = null;
|
|
741
|
+
// A one-shot re-read for the moment a held release clears the server's hold.
|
|
742
|
+
let holdTimer = null;
|
|
743
|
+
// The release hold in force: a test's fixed value, else the server's last
|
|
744
|
+
// stated value (runtime_config via `connectorRelease`), else none.
|
|
745
|
+
let serverHoldMs = null;
|
|
746
|
+
const releaseHoldMs = () => deps.minReleaseAgeMs ?? serverHoldMs ?? exports.CONNECTOR_UPDATE_DEFAULT_MIN_RELEASE_AGE_MS;
|
|
747
|
+
// The newest version the last good read of the channel listed. The doorbell
|
|
748
|
+
// reads the channel again only for something newer than this.
|
|
749
|
+
let lastSeenVersion = null;
|
|
750
|
+
let lastDoorbellMs = Number.NEGATIVE_INFINITY;
|
|
751
|
+
const advertKey = releaseAdvertKeyForPlatform(deps.platform);
|
|
637
752
|
const unref = (t) => {
|
|
638
753
|
if (t && typeof t.unref === 'function') {
|
|
639
754
|
t.unref();
|
|
@@ -711,15 +826,19 @@ function startConnectorUpdateService(deps) {
|
|
|
711
826
|
const pollApply = () => {
|
|
712
827
|
if (stopped || phase === 'idle' || !target || !applier)
|
|
713
828
|
return;
|
|
714
|
-
|
|
829
|
+
let elapsed = now() - phaseStartedMs;
|
|
715
830
|
if (phase === 'staging') {
|
|
716
831
|
if (applier.stagedVersion() === target.connectorVersion) {
|
|
717
832
|
log(`[update] ${target.connectorVersion} is staged and verified — waiting for a safe point`);
|
|
718
833
|
phase = 'awaiting-safe-point';
|
|
719
834
|
phaseStartedMs = now();
|
|
835
|
+
elapsed = 0;
|
|
720
836
|
setUpdater('awaiting_safe_point', target.connectorVersion);
|
|
721
|
-
|
|
837
|
+
// Falls through: a connector that is already quiet hands over in this
|
|
838
|
+
// same look rather than one period later.
|
|
722
839
|
}
|
|
840
|
+
}
|
|
841
|
+
if (phase === 'staging') {
|
|
723
842
|
// A helper that refused or failed this stage leaves outcome.json behind.
|
|
724
843
|
// Waiting out the stage deadline would turn its reason into "not staged
|
|
725
844
|
// within 60 min" for an hour. Only an outcome written after this stage
|
|
@@ -767,7 +886,8 @@ function startConnectorUpdateService(deps) {
|
|
|
767
886
|
requestExit();
|
|
768
887
|
};
|
|
769
888
|
const beginApply = (manifest) => {
|
|
770
|
-
|
|
889
|
+
// Never a second stage over one already in flight.
|
|
890
|
+
if (!applier || phase !== 'idle')
|
|
771
891
|
return;
|
|
772
892
|
// A verified tree for THIS version may already be on disk: the connector
|
|
773
893
|
// restarted between staging and the safe point, or the safe point never
|
|
@@ -819,19 +939,53 @@ function startConnectorUpdateService(deps) {
|
|
|
819
939
|
setUpdater('current');
|
|
820
940
|
return null;
|
|
821
941
|
}
|
|
822
|
-
const
|
|
823
|
-
if (
|
|
942
|
+
const stageable = connectorReleaseChoices(check.manifest).filter(m => isNewerConnectorVersion(m.connectorVersion, deps.installedVersion) && applier.canStage(m));
|
|
943
|
+
if (stageable.length === 0) {
|
|
824
944
|
log('[update] the published manifest carries no usable update payload — reported only');
|
|
825
945
|
setUpdater('no_payload', check.availableVersion);
|
|
826
946
|
return null;
|
|
827
947
|
}
|
|
828
|
-
|
|
948
|
+
// A version that failed its post-install health check HERE was rolled back
|
|
949
|
+
// by install-update.ps1 and recorded. Taking it again would repeat the same
|
|
950
|
+
// outage on every check, so it waits for a newer release instead.
|
|
951
|
+
let failedHere;
|
|
952
|
+
try {
|
|
953
|
+
failedHere = applier.failedVersions?.() ?? new Set();
|
|
954
|
+
}
|
|
955
|
+
catch {
|
|
956
|
+
failedHere = new Set();
|
|
957
|
+
}
|
|
958
|
+
const usable = stageable.filter(m => !failedHere.has(m.connectorVersion));
|
|
959
|
+
if (usable.length === 0) {
|
|
960
|
+
const failed = newestRelease(stageable).connectorVersion;
|
|
961
|
+
log(`[update] ${failed} failed its health check on this machine and was rolled back — waiting for a newer release`);
|
|
962
|
+
setUpdater('apply_failed', `${failed} failed its health check on this machine and was rolled back; waiting for a newer release`);
|
|
963
|
+
return null;
|
|
964
|
+
}
|
|
965
|
+
const holdMs = releaseHoldMs();
|
|
966
|
+
const mature = usable.filter(m => isReleaseBakedEnough(m, now(), holdMs));
|
|
829
967
|
if (mature.length === 0) {
|
|
830
|
-
log(`[update] ${check.availableVersion} is
|
|
968
|
+
log(`[update] ${check.availableVersion} is inside the server's ${Math.round(holdMs / 60000)}-min release hold`);
|
|
831
969
|
setUpdater('baking', check.availableVersion);
|
|
970
|
+
// Read again the moment the first held release clears the hold, rather
|
|
971
|
+
// than at the next hourly backstop.
|
|
972
|
+
const clearsAt = Math.min(...usable.map(m => Date.parse(m.publishedAt) + holdMs));
|
|
973
|
+
scheduleHoldRecheck(clearsAt - now());
|
|
832
974
|
return null;
|
|
833
975
|
}
|
|
834
|
-
return mature
|
|
976
|
+
return newestRelease(mature);
|
|
977
|
+
};
|
|
978
|
+
const scheduleHoldRecheck = (delayMs) => {
|
|
979
|
+
if (holdTimer) {
|
|
980
|
+
clearTimeoutFn(holdTimer);
|
|
981
|
+
holdTimer = null;
|
|
982
|
+
}
|
|
983
|
+
// 2^31-1 ms is the longest delay a timer holds; the hourly backstop covers
|
|
984
|
+
// anything further out, and the parser already refuses a hold past 30 days.
|
|
985
|
+
if (stopped || !Number.isFinite(delayMs) || delayMs + 1000 > 0x7fffffff)
|
|
986
|
+
return;
|
|
987
|
+
holdTimer = setTimeoutFn(() => { holdTimer = null; safeRunCheck(); }, Math.max(1000, delayMs + 1000));
|
|
988
|
+
unref(holdTimer);
|
|
835
989
|
};
|
|
836
990
|
const runCheck = async () => {
|
|
837
991
|
if (stopped)
|
|
@@ -856,6 +1010,8 @@ function startConnectorUpdateService(deps) {
|
|
|
856
1010
|
else if (check.availableVersion) {
|
|
857
1011
|
log(`[update] connector ${check.installedVersion} is behind published ${check.availableVersion}`);
|
|
858
1012
|
}
|
|
1013
|
+
if (check.publishedVersion)
|
|
1014
|
+
lastSeenVersion = check.publishedVersion;
|
|
859
1015
|
// Decided BEFORE the report so the report carries this check's conclusion;
|
|
860
1016
|
// staging still starts only after the version beacon has gone out.
|
|
861
1017
|
const selected = chooseUpdate(check);
|
|
@@ -865,11 +1021,11 @@ function startConnectorUpdateService(deps) {
|
|
|
865
1021
|
scheduleRetry(!check.manifest);
|
|
866
1022
|
};
|
|
867
1023
|
/**
|
|
868
|
-
* A failed read is retried within minutes, not at the next
|
|
1024
|
+
* A failed read is retried within minutes, not at the next backstop check.
|
|
869
1025
|
* Measured 2026-11-07: a check whose GET is in flight while the event loop is
|
|
870
1026
|
* stalled for more than the 15 s socket timeout fails, and a stall at process
|
|
871
1027
|
* start (archive load, trust-store and proxy subprocesses) or under load would
|
|
872
|
-
* otherwise leave a box blind to fixes for
|
|
1028
|
+
* otherwise leave a box blind to fixes for a whole interval per attempt.
|
|
873
1029
|
*/
|
|
874
1030
|
let failedReads = 0;
|
|
875
1031
|
let retryTimer = null;
|
|
@@ -885,15 +1041,76 @@ function startConnectorUpdateService(deps) {
|
|
|
885
1041
|
retryTimer = setTimeoutFn(safeRunCheck, delay);
|
|
886
1042
|
unref(retryTimer);
|
|
887
1043
|
};
|
|
1044
|
+
/**
|
|
1045
|
+
* One channel read at a time. Five things can ask for one (launch, the hourly
|
|
1046
|
+
* backstop, a retry, a hold clearing, the doorbell), and at startup the
|
|
1047
|
+
* doorbell (the first profile poll) routinely lands while the launch check is
|
|
1048
|
+
* still reading: two overlapping checks would both choose the same release
|
|
1049
|
+
* and launch two stage helpers into the same directory. A request made while
|
|
1050
|
+
* a read is in flight runs once, after it, and sees whatever that read began.
|
|
1051
|
+
*/
|
|
1052
|
+
let checkInFlight = false;
|
|
1053
|
+
let checkRequested = false;
|
|
888
1054
|
const safeRunCheck = () => {
|
|
889
|
-
|
|
1055
|
+
if (checkInFlight) {
|
|
1056
|
+
checkRequested = true;
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
1059
|
+
checkInFlight = true;
|
|
1060
|
+
runCheck()
|
|
1061
|
+
.catch((err) => log(`[update] check failed (non-fatal): ${String(err)}`))
|
|
1062
|
+
.finally(() => {
|
|
1063
|
+
checkInFlight = false;
|
|
1064
|
+
if (checkRequested && !stopped) {
|
|
1065
|
+
checkRequested = false;
|
|
1066
|
+
safeRunCheck();
|
|
1067
|
+
}
|
|
1068
|
+
});
|
|
890
1069
|
};
|
|
891
1070
|
initialTimer = setTimeoutFn(safeRunCheck, deps.initialDelayMs ?? exports.CONNECTOR_UPDATE_INITIAL_DELAY_MS);
|
|
892
1071
|
unref(initialTimer);
|
|
893
1072
|
checkTimer = setIntervalFn(safeRunCheck, deps.intervalMs ?? exports.CONNECTOR_UPDATE_CHECK_INTERVAL_MS);
|
|
894
1073
|
unref(checkTimer);
|
|
1074
|
+
/**
|
|
1075
|
+
* THE DOORBELL. Every sensing-profile response carries `connectorRelease`;
|
|
1076
|
+
* a version newer than both this install and the last read of the channel
|
|
1077
|
+
* reads the channel now, so a box learns of a release within one profile poll
|
|
1078
|
+
* (five minutes) instead of at the next backstop check. The advert never
|
|
1079
|
+
* decides what is staged — the channel and the signed catalog do.
|
|
1080
|
+
*
|
|
1081
|
+
* The same response carries the server's release hold. A changed hold re-reads
|
|
1082
|
+
* the channel when this install is holding a release back, so lifting a hold
|
|
1083
|
+
* takes effect within one poll too.
|
|
1084
|
+
*/
|
|
1085
|
+
const onAdvert = (advert) => {
|
|
1086
|
+
if (stopped)
|
|
1087
|
+
return;
|
|
1088
|
+
const holdChanged = advert.minReleaseAgeMs !== null && advert.minReleaseAgeMs !== serverHoldMs;
|
|
1089
|
+
if (advert.minReleaseAgeMs !== null)
|
|
1090
|
+
serverHoldMs = advert.minReleaseAgeMs;
|
|
1091
|
+
// Mid-stage, mid-wait, or backing off a failed read: those paths already
|
|
1092
|
+
// own the next read.
|
|
1093
|
+
if (phase !== 'idle' || failedReads > 0)
|
|
1094
|
+
return;
|
|
1095
|
+
const advertised = advertKey ? advert.versions[advertKey] : undefined;
|
|
1096
|
+
const newer = Boolean(advertised && isNewerConnectorVersion(advertised, deps.installedVersion)
|
|
1097
|
+
&& (!lastSeenVersion || isNewerConnectorVersion(advertised, lastSeenVersion)));
|
|
1098
|
+
const holdLifted = holdChanged && updater?.state === 'baking';
|
|
1099
|
+
if (!newer && !holdLifted)
|
|
1100
|
+
return;
|
|
1101
|
+
if (now() - lastDoorbellMs < exports.CONNECTOR_RELEASE_DOORBELL_MIN_GAP_MS)
|
|
1102
|
+
return;
|
|
1103
|
+
lastDoorbellMs = now();
|
|
1104
|
+
log(newer
|
|
1105
|
+
? `[update] the API advertises connector ${advertised} — reading the release channel now`
|
|
1106
|
+
: `[update] the server changed its release hold to ${Math.round(releaseHoldMs() / 60000)} min — reading the release channel again`);
|
|
1107
|
+
safeRunCheck();
|
|
1108
|
+
};
|
|
1109
|
+
const unsubscribeAdverts = hasManifest && applier
|
|
1110
|
+
? (deps.subscribeReleaseAdverts ?? onConnectorReleaseAdvert)(onAdvert)
|
|
1111
|
+
: null;
|
|
895
1112
|
// Reuse the existing five-minute telemetry rail for actual saved-work stages.
|
|
896
|
-
// The
|
|
1113
|
+
// The hourly release check cannot keep a queue report fresh. Runtime-only
|
|
897
1114
|
// entrypoints also use this rail; missing stages never manufacture an empty queue.
|
|
898
1115
|
if (deps.inventory || deps.runtime || deps.videoBuffer) {
|
|
899
1116
|
const reportModels = async () => {
|
|
@@ -926,7 +1143,7 @@ function startConnectorUpdateService(deps) {
|
|
|
926
1143
|
connectorVersion: deps.installedVersion,
|
|
927
1144
|
...(models ? { models, ...(inv ? { detectorBox: inv.box } : {}) } : {}),
|
|
928
1145
|
...(runtime && workload ? { runtime: { ...runtime, workload } } : {}),
|
|
929
|
-
// Staging and the safe-point wait move between
|
|
1146
|
+
// Staging and the safe-point wait move between channel reads.
|
|
930
1147
|
...(updater ? { updater } : {}),
|
|
931
1148
|
},
|
|
932
1149
|
...(videoBuffers.size ? {
|
|
@@ -944,16 +1161,20 @@ function startConnectorUpdateService(deps) {
|
|
|
944
1161
|
modelsTimer = setIntervalFn(safeReportModels, deps.modelsIntervalMs ?? exports.CONNECTOR_MODEL_REPORT_INTERVAL_MS);
|
|
945
1162
|
unref(modelsTimer);
|
|
946
1163
|
}
|
|
1164
|
+
let markedHealthy = false;
|
|
947
1165
|
return {
|
|
948
1166
|
stop() {
|
|
949
1167
|
if (stopped)
|
|
950
1168
|
return;
|
|
951
1169
|
stopped = true;
|
|
952
1170
|
try {
|
|
1171
|
+
unsubscribeAdverts?.();
|
|
953
1172
|
if (initialTimer)
|
|
954
1173
|
clearTimeoutFn(initialTimer);
|
|
955
1174
|
if (retryTimer)
|
|
956
1175
|
clearTimeoutFn(retryTimer);
|
|
1176
|
+
if (holdTimer)
|
|
1177
|
+
clearTimeoutFn(holdTimer);
|
|
957
1178
|
if (checkTimer)
|
|
958
1179
|
clearIntervalFn(checkTimer);
|
|
959
1180
|
if (modelsTimer)
|
|
@@ -966,9 +1187,31 @@ function startConnectorUpdateService(deps) {
|
|
|
966
1187
|
}
|
|
967
1188
|
initialTimer = null;
|
|
968
1189
|
retryTimer = null;
|
|
1190
|
+
holdTimer = null;
|
|
969
1191
|
checkTimer = null;
|
|
970
1192
|
pollTimer = null;
|
|
971
1193
|
},
|
|
1194
|
+
markHealthy(opts) {
|
|
1195
|
+
// Once per process: the installer reads the FIRST start of the version it
|
|
1196
|
+
// installed, and a later reconfigure rebuilds this loop.
|
|
1197
|
+
if (markedHealthy || stopped)
|
|
1198
|
+
return;
|
|
1199
|
+
markedHealthy = true;
|
|
1200
|
+
// Only a platform whose installer waits for the marker writes one.
|
|
1201
|
+
if (!applier?.markHealthy)
|
|
1202
|
+
return;
|
|
1203
|
+
try {
|
|
1204
|
+
applier.markHealthy({
|
|
1205
|
+
version: deps.installedVersion,
|
|
1206
|
+
pid: deps.pid ?? process.pid,
|
|
1207
|
+
...(opts?.noFeeds ? { noFeeds: true } : {}),
|
|
1208
|
+
});
|
|
1209
|
+
log(`[update] connector ${deps.installedVersion} started; health marker written`);
|
|
1210
|
+
}
|
|
1211
|
+
catch (err) {
|
|
1212
|
+
log(`[update] could not write the health marker (non-fatal): ${String(err)}`);
|
|
1213
|
+
}
|
|
1214
|
+
},
|
|
972
1215
|
};
|
|
973
1216
|
}
|
|
974
1217
|
/**
|