@catalyst-cloud/cli 0.16.41 → 0.16.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +1 -1
- package/dist/ask.js +26 -10
- package/dist/cli.js +1 -1
- package/dist/runner-command.js +50 -9
- package/dist/runner-update-fleet.js +242 -35
- package/dist/runner-update.js +427 -46
- package/dist/update.js +14 -9
- package/package.json +1 -1
- package/skills/catalyst-github/SKILL.md +1 -1
- package/skills/catalyst-linear/SKILL.md +1 -1
- package/skills/catalyst-onboard/SKILL.md +1 -1
- package/skills/run-this-project/SKILL.md +1 -1
- package/skills/unstick/SKILL.md +1 -1
- package/skills/what-needs-me/SKILL.md +1 -1
- package/skills/what-this-repo-needs/SKILL.md +1 -1
- package/skills/whats-happening/SKILL.md +1 -1
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { existsSync, readFileSync, unlinkSync } from "node:fs";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
import { readManifest } from "./config.js";
|
|
4
5
|
import { dockerRunnerEngine, prepareRunnerImages, spawnDocker } from "./onboard-runner.js";
|
|
5
6
|
import { defaultRun } from "./runner-auto-update.js";
|
|
6
|
-
import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseState, releaseLock, takeLock, writePrivateAtomic, writeReleaseState, } from "./runner-update.js";
|
|
7
|
+
import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseState, releaseLock, takeLock, updateReport, writePrivateAtomic, writeReleaseState, } from "./runner-update.js";
|
|
7
8
|
// `catalyst runner update --fleet <dir>`: the same approved release, applied to a multi-team
|
|
8
9
|
// supervisor fleet (CTC-6530). One supervisor container serves several scopes from a Compose file
|
|
9
10
|
// written as JSON (`compose.json`), under one operator enrollment, and every host credential on the
|
|
@@ -23,23 +24,45 @@ import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseSt
|
|
|
23
24
|
// back and the supervisor is recreated on it. That scope set is saved before the recreate and removed
|
|
24
25
|
// once it qualifies or rolls back, so a run killed in between is finished by the next one.
|
|
25
26
|
//
|
|
27
|
+
// A busy host re-adopts its live sessions before it qualifies, which can take longer than four minutes
|
|
28
|
+
// (CTC-8163). Each recreated supervisor, the new release's or the restored previous one's, gets four
|
|
29
|
+
// minutes, extended to 150 seconds past each new step it logs: fleet tenure, then a saved scope
|
|
30
|
+
// re-adopted, then that scope ready. No supervisor gets more than seven minutes. A restart, another
|
|
31
|
+
// image or `host_not_ready` still ends the wait at the next poll. At the deadline each saved scope it
|
|
32
|
+
// has not qualified is reported, with the gate condition its latest log line shows failing.
|
|
33
|
+
//
|
|
34
|
+
// A team can stop being delegated while an update is in flight (CTC-7569): no supervisor, new or old,
|
|
35
|
+
// starts its scope again. The previous release, restored, is the control. When it misses exactly the
|
|
36
|
+
// scopes the new supervisor missed and qualifies the rest, those scopes are dropped from the saved set
|
|
37
|
+
// and the new release is applied again against what is left. A later run that finds a saved set beside
|
|
38
|
+
// the previous release's running supervisor drops the saved scopes that supervisor no longer qualifies.
|
|
39
|
+
// Either way the result names each dropped scope; the new release's own log never drops one.
|
|
40
|
+
//
|
|
26
41
|
// A release can qualify and then stop receiving work minutes later (CTC-6564). A successful apply
|
|
27
42
|
// arms a check on the host, outside the calling process, for 10 minutes later: it reads the supervisor's
|
|
28
43
|
// log since the roll began and, when the release was granted no work and the fleet's acquires got no
|
|
29
44
|
// answer, restores the saved previous `compose.json`. `runner update --fleet --post-roll-check` is
|
|
30
45
|
// that check; only the armed timer runs it.
|
|
46
|
+
// A check that is due while an update holds the lock judges nothing; that update arms it again
|
|
47
|
+
// for 5 minutes after it ends (CTC-7570).
|
|
31
48
|
export const FLEET_COMPOSE_FILE = "compose.json";
|
|
32
49
|
export const PREVIOUS_COMPOSE_FILE = "compose.release-previous.json";
|
|
33
50
|
const CANDIDATE_COMPOSE_FILE = "compose.release-candidate.json";
|
|
34
51
|
/** The scopes the supervisor served before a recreate, kept until that recreate is qualified or rolled
|
|
35
52
|
* back. A run that was killed in between finds it and qualifies against it, never against the new
|
|
36
|
-
* container's own log, which would read an unqualified supervisor as serving.
|
|
53
|
+
* container's own log, which would read an unqualified supervisor as serving. Only the previous
|
|
54
|
+
* release's supervisor can take a scope out of it (CTC-7569). */
|
|
37
55
|
export const IN_FLIGHT_FILE = "fleet-update-in-flight.json";
|
|
38
56
|
/** How far back a qualification counts as current before the update. The supervisor repeats it
|
|
39
57
|
* on every loop while a scope is ready, so a ready scope logs it many times in this window. */
|
|
40
58
|
const QUALIFIED_LOOKBACK_MS = 5 * 60_000;
|
|
41
59
|
const QUALIFY_MIN_MS = 90_000;
|
|
60
|
+
/** How long a recreated supervisor that shows no progress has to qualify every saved scope. */
|
|
42
61
|
const QUALIFY_DEADLINE_MS = 240_000;
|
|
62
|
+
/** How long past each new step of progress a recreated supervisor has to qualify (CTC-8163). */
|
|
63
|
+
const QUALIFY_PROGRESS_MS = 150_000;
|
|
64
|
+
/** The most one recreated supervisor is given, however it progresses (CTC-8163). */
|
|
65
|
+
const QUALIFY_CEILING_MS = 420_000;
|
|
43
66
|
const QUALIFY_POLL_MS = 10_000;
|
|
44
67
|
const CHILD_NAME_PREFIX = "catalyst-pa_";
|
|
45
68
|
/** What a successful apply leaves for the check: the bytes it wrote and when the roll began. */
|
|
@@ -47,6 +70,8 @@ export const POST_ROLL_ARMED_FILE = "post-roll-armed.json";
|
|
|
47
70
|
/** The check's last verdict, beside `release-state.json`. */
|
|
48
71
|
export const POST_ROLL_RESULT_FILE = "post-roll-result.json";
|
|
49
72
|
export const POST_ROLL_DELAY_MS = 10 * 60_000;
|
|
73
|
+
/** How long after an update run lets go of the lock a check it kept from judging runs again (CTC-7570). */
|
|
74
|
+
export const POST_ROLL_RETRY_DELAY_MS = 5 * 60_000;
|
|
50
75
|
const POST_ROLL_UNIT = "catalyst-fleet-post-roll";
|
|
51
76
|
/** No-response acquire retries, with no grants, that count a release as failing. */
|
|
52
77
|
const POST_ROLL_MAX_NO_RESPONSE = 3;
|
|
@@ -91,14 +116,83 @@ export function qualifiedScopes(text) {
|
|
|
91
116
|
/** A `host_not_ready` line for one of `scopes`, or one that names no scope. Other scopes may be
|
|
92
117
|
* unready for their own reasons (a team with no repository yet) and do not stop an update. */
|
|
93
118
|
export function notReadyScope(text, scopes) {
|
|
119
|
+
return readSupervisorLog(text, scopes).notReady;
|
|
120
|
+
}
|
|
121
|
+
const GATE_CODES = [
|
|
122
|
+
["resourcesReady", "resources_ready"],
|
|
123
|
+
["scopeEligible", "scope_eligible"],
|
|
124
|
+
["readyUntilValid", "ready_until"],
|
|
125
|
+
["sessionsReadopted", "sessions_readopted"],
|
|
126
|
+
];
|
|
127
|
+
/** One pass over the log text for everything qualify() reads. Lines for other scopes, and lines that
|
|
128
|
+
* are not JSON or lack a field, are skipped. */
|
|
129
|
+
function readSupervisorLog(text, scopes) {
|
|
130
|
+
const log = { notReady: null, qualified: new Set(), tenure: false, readopted: new Set(), ready: new Set(), gates: new Map() };
|
|
94
131
|
for (const event of logEvents(text)) {
|
|
95
|
-
if (event.event !== "host_not_ready")
|
|
96
|
-
continue;
|
|
97
132
|
const key = scopeKey(event);
|
|
98
|
-
if (key === null || scopes.has(key))
|
|
99
|
-
|
|
133
|
+
if (event.event === "host_not_ready" && log.notReady === null && (key === null || scopes.has(key)))
|
|
134
|
+
log.notReady = key ?? "unscoped";
|
|
135
|
+
if (event.event === "supervisor_fleet_tenure")
|
|
136
|
+
log.tenure = true;
|
|
137
|
+
if (key === null || !scopes.has(key))
|
|
138
|
+
continue;
|
|
139
|
+
if (event.event === "supervisor_scope_qualified")
|
|
140
|
+
log.qualified.add(key);
|
|
141
|
+
else if (event.event === "supervisor_readopted")
|
|
142
|
+
log.readopted.add(key);
|
|
143
|
+
else if (event.event === "host_ready")
|
|
144
|
+
log.ready.add(key);
|
|
145
|
+
else if (event.event === "supervisor_scope_qualification_state") {
|
|
146
|
+
const { resourcesReady, scopeEligible, readyUntilValid, sessionsReadopted } = event;
|
|
147
|
+
if (typeof resourcesReady === "boolean" && typeof scopeEligible === "boolean" && typeof readyUntilValid === "boolean" && typeof sessionsReadopted === "boolean")
|
|
148
|
+
log.gates.set(key, { resourcesReady, scopeEligible, readyUntilValid, sessionsReadopted });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return log;
|
|
152
|
+
}
|
|
153
|
+
/** Count the steps `log` shows that `budget` has not counted yet: fleet tenure, a scope re-adopted, and
|
|
154
|
+
* a re-adopted scope ready. A scope ready before its re-adoption shows counts once that does. Any new
|
|
155
|
+
* step moves the deadline to `QUALIFY_PROGRESS_MS` past `atMs`, never earlier and never past the ceiling. */
|
|
156
|
+
function countProgress(budget, log, atMs) {
|
|
157
|
+
let progressed = false;
|
|
158
|
+
if (log.tenure && !budget.tenure) {
|
|
159
|
+
budget.tenure = true;
|
|
160
|
+
progressed = true;
|
|
161
|
+
}
|
|
162
|
+
for (const scope of log.readopted) {
|
|
163
|
+
if (budget.readopted.has(scope))
|
|
164
|
+
continue;
|
|
165
|
+
budget.readopted.add(scope);
|
|
166
|
+
progressed = true;
|
|
167
|
+
}
|
|
168
|
+
for (const scope of log.ready) {
|
|
169
|
+
if (!budget.readopted.has(scope) || budget.ready.has(scope))
|
|
170
|
+
continue;
|
|
171
|
+
budget.ready.add(scope);
|
|
172
|
+
progressed = true;
|
|
100
173
|
}
|
|
101
|
-
|
|
174
|
+
if (progressed)
|
|
175
|
+
budget.deadline = Math.min(budget.ceiling, Math.max(budget.deadline, atMs + QUALIFY_PROGRESS_MS));
|
|
176
|
+
}
|
|
177
|
+
/** The longest one recreated supervisor is given to qualify, however it progresses. */
|
|
178
|
+
const qualifyCeilingMs = (options) => Math.max(QUALIFY_CEILING_MS, options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS);
|
|
179
|
+
/** A scope as printed: anything outside printable ASCII, and the backslash that starts an escape,
|
|
180
|
+
* escaped as `\uXXXX`, and at most 120 characters. */
|
|
181
|
+
const printableScope = (scope) => {
|
|
182
|
+
const escaped = scope.replace(/[^\x21-\x5b\x5d-\x7e]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`);
|
|
183
|
+
return escaped.length > 120 ? `${escaped.slice(0, 120)}...` : escaped;
|
|
184
|
+
};
|
|
185
|
+
/** The gate conditions a missing scope's snapshot shows failing, sorted; `diagnostic_unavailable`
|
|
186
|
+
* without one, and `qualification_event_missing` when every condition held. */
|
|
187
|
+
function failedGates(snapshot) {
|
|
188
|
+
if (snapshot === undefined)
|
|
189
|
+
return ["diagnostic_unavailable"];
|
|
190
|
+
const failed = GATE_CODES.filter(([field]) => !snapshot[field]).map(([, code]) => code).sort();
|
|
191
|
+
return failed.length > 0 ? failed : ["qualification_event_missing"];
|
|
192
|
+
}
|
|
193
|
+
/** One line per scope in `missing`, sorted by scope. `gates` null: the log could not be read. */
|
|
194
|
+
function qualificationPendingLines(roll, missing, gates) {
|
|
195
|
+
return [...missing].sort().map((scope) => `fleet qualification pending: roll=${roll} scope=${printableScope(scope)} failed=${failedGates(gates?.get(scope)).join(",")}`);
|
|
102
196
|
}
|
|
103
197
|
function readInFlight(dir) {
|
|
104
198
|
try {
|
|
@@ -112,6 +206,10 @@ function readInFlight(dir) {
|
|
|
112
206
|
return null;
|
|
113
207
|
}
|
|
114
208
|
}
|
|
209
|
+
/** Save the scopes the next recreate has to qualify, replacing any saved before. */
|
|
210
|
+
function saveInFlight(dir, scopes, atMs) {
|
|
211
|
+
writePrivateAtomic(join(dir, IN_FLIGHT_FILE), `${JSON.stringify({ version: 1, scopes: [...scopes].sort(), atMs })}\n`);
|
|
212
|
+
}
|
|
115
213
|
function clearInFlight(dir) {
|
|
116
214
|
try {
|
|
117
215
|
unlinkSync(join(dir, IN_FLIGHT_FILE));
|
|
@@ -120,6 +218,17 @@ function clearInFlight(dir) {
|
|
|
120
218
|
// Already gone.
|
|
121
219
|
}
|
|
122
220
|
}
|
|
221
|
+
/** The images a saved `compose.json` names, or null when there is none or it is not a fleet layout. */
|
|
222
|
+
function savedImages(path) {
|
|
223
|
+
try {
|
|
224
|
+
return fleetImages(JSON.parse(readFileSync(path, "utf8")));
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/** The detail token naming scopes dropped from the requirement, sorted; empty when there are none. */
|
|
231
|
+
const droppedDetail = (scopes) => scopes.length > 0 ? `;dropped_scope:${[...scopes].sort().join(",")}` : "";
|
|
123
232
|
const removeFile = (path) => {
|
|
124
233
|
try {
|
|
125
234
|
unlinkSync(path);
|
|
@@ -226,6 +335,23 @@ async function armPostRollCheck(engine, dir, armed, signal) {
|
|
|
226
335
|
return ";post_roll_check_not_armed";
|
|
227
336
|
}
|
|
228
337
|
}
|
|
338
|
+
/** The armed check runs once, from a one-shot timer. When it was due while an update held the lock, or
|
|
339
|
+
* Docker could not be read, it judged nothing and left the armed file; a reboot loses the timer the same
|
|
340
|
+
* way (CTC-7570). The update holding the lock arms it again for a few minutes later, before it lets go:
|
|
341
|
+
* while the lock is held no check is part-way through a judgment, so the arm's stop of the check's unit
|
|
342
|
+
* interrupts nothing. The check cannot arm itself: that stop would end its own unit. A check this run
|
|
343
|
+
* armed is not due yet and is left alone. */
|
|
344
|
+
async function rearmOverdueCheck(engine, dir, now, signal) {
|
|
345
|
+
const armed = readPostRollArmed(dir);
|
|
346
|
+
if (armed === null || now() < armed.armedAtMs + POST_ROLL_DELAY_MS)
|
|
347
|
+
return;
|
|
348
|
+
try {
|
|
349
|
+
await engine.armPostRollCheck(dir, POST_ROLL_RETRY_DELAY_MS, signal);
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
// The next run tries again.
|
|
353
|
+
}
|
|
354
|
+
}
|
|
229
355
|
export async function runFleetUpdate(engine, options) {
|
|
230
356
|
const { dir } = options;
|
|
231
357
|
const now = options.now ?? Date.now;
|
|
@@ -243,8 +369,10 @@ export async function runFleetUpdate(engine, options) {
|
|
|
243
369
|
return result;
|
|
244
370
|
}
|
|
245
371
|
finally {
|
|
246
|
-
if (lock !== null)
|
|
372
|
+
if (lock !== null) {
|
|
373
|
+
await rearmOverdueCheck(engine, dir, now, options.signal);
|
|
247
374
|
releaseLock(dir, lock);
|
|
375
|
+
}
|
|
248
376
|
}
|
|
249
377
|
}
|
|
250
378
|
async function update(engine, options, now) {
|
|
@@ -264,7 +392,7 @@ async function update(engine, options, now) {
|
|
|
264
392
|
const project = isRecord(compose) && typeof compose.name === "string" ? compose.name : "";
|
|
265
393
|
if (!isRecord(compose) || current === null || !/^[a-z0-9][a-z0-9_-]*$/.test(project))
|
|
266
394
|
return done("failed", "fleet_compose_unrecognized", { exitCode: 1 });
|
|
267
|
-
const read = await engine.desiredRelease(dir, compose, signal);
|
|
395
|
+
const read = await engine.desiredRelease(dir, compose, signal, options.check ? undefined : updateReport(state, readManifest().version));
|
|
268
396
|
const decision = decideRelease(state.generation, read);
|
|
269
397
|
if (decision.action === "retain")
|
|
270
398
|
return decision.reason === "not_newer"
|
|
@@ -291,6 +419,8 @@ async function update(engine, options, now) {
|
|
|
291
419
|
// The scopes the running supervisor qualifies are what the new one has to qualify again, unless an
|
|
292
420
|
// earlier run stopped mid-recreate: then the scopes it saved are the baseline.
|
|
293
421
|
let scopes = inFlight;
|
|
422
|
+
/** Saved scopes the previous release's supervisor no longer qualifies, named in the result (CTC-7569). */
|
|
423
|
+
let dropped = [];
|
|
294
424
|
if (scopes === null) {
|
|
295
425
|
if (!live?.running)
|
|
296
426
|
return done("retained", "supervisor_not_running", { ...tagged, exitCode: 1 });
|
|
@@ -301,6 +431,25 @@ async function update(engine, options, now) {
|
|
|
301
431
|
if (scopes.size === 0)
|
|
302
432
|
return done("retained", "no_qualified_scope", { ...tagged, exitCode: 1 });
|
|
303
433
|
}
|
|
434
|
+
else {
|
|
435
|
+
// An earlier run that ended back on the previous release, its rollback unqualified, left its scopes
|
|
436
|
+
// saved. Only when compose.json and the running supervisor are that previous release, up long enough
|
|
437
|
+
// to have qualified every scope it serves, does its log trim the saved set: a saved scope it no
|
|
438
|
+
// longer qualifies stopped being delegated. A `host_not_ready` for a saved scope is a scope still
|
|
439
|
+
// started, not one dropped. Anything less keeps the saved set whole. Long enough is the most any
|
|
440
|
+
// recreated supervisor is given, since this run cannot see how it progressed (CTC-8163).
|
|
441
|
+
const previous = savedImages(join(dir, PREVIOUS_COMPOSE_FILE));
|
|
442
|
+
const onPrevious = previous !== null && current.runner === previous.runner && current.supervisor === previous.supervisor && runs(previous);
|
|
443
|
+
if (live && onPrevious && now() - Date.parse(live.startedAt) >= qualifyCeilingMs(options)) {
|
|
444
|
+
const recent = await engine.logs(live.id, new Date(now() - QUALIFIED_LOOKBACK_MS).toISOString(), signal);
|
|
445
|
+
const serving = recent === null || notReadyScope(recent, scopes) !== null ? new Set() : qualifiedScopes(recent);
|
|
446
|
+
const kept = [...scopes].filter((scope) => serving.has(scope));
|
|
447
|
+
if (kept.length > 0 && kept.length < scopes.size) {
|
|
448
|
+
dropped = [...scopes].filter((scope) => !serving.has(scope));
|
|
449
|
+
scopes = new Set(kept);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
304
453
|
const record = (outcome, detail, previous, exitCode = 0) => {
|
|
305
454
|
writeReleaseState(dir, {
|
|
306
455
|
...state,
|
|
@@ -353,9 +502,15 @@ async function update(engine, options, now) {
|
|
|
353
502
|
const rollStartMs = now();
|
|
354
503
|
if (changed.length > 0)
|
|
355
504
|
writePrivateAtomic(join(dir, PREVIOUS_COMPOSE_FILE), composeText);
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
505
|
+
saveInFlight(dir, scopes, now());
|
|
506
|
+
/** Write `text` as compose.json, recreate the supervisor alone on it and qualify it against `required`. */
|
|
507
|
+
const roll = async (which, text, images, required) => {
|
|
508
|
+
writePrivateAtomic(composePath, text);
|
|
509
|
+
if (!(await engine.recreate(dir, project, signal)))
|
|
510
|
+
return { ok: false, reason: "recreate_failed" };
|
|
511
|
+
return qualify(engine, dir, project, images, required, options, now, which);
|
|
512
|
+
};
|
|
513
|
+
const succeed = async (droppedScopes) => {
|
|
359
514
|
clearInFlight(dir);
|
|
360
515
|
if (changed.length > 0)
|
|
361
516
|
await engine.removeImages(changed.map((c) => current[c]), signal);
|
|
@@ -371,20 +526,34 @@ async function update(engine, options, now) {
|
|
|
371
526
|
version: 1, sinceMs: rollStartMs, composeSha256: sha256(candidateText), generation: release.generation,
|
|
372
527
|
releaseId: release.releaseId, fromGeneration: state.generation, armedAtMs: now(),
|
|
373
528
|
};
|
|
374
|
-
const detail = (changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled") +
|
|
529
|
+
const detail = (changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled") + droppedDetail(droppedScopes) +
|
|
530
|
+
await armPostRollCheck(engine, dir, armed, signal);
|
|
375
531
|
const previous = changed.length > 0 ? current : null;
|
|
376
532
|
return disturbed === 0
|
|
377
533
|
? record("updated", detail, previous)
|
|
378
534
|
: record("updated", `${detail};${disturbed < 0 ? "children_unreadable" : `children_changed:${disturbed}`}`, previous, 1);
|
|
379
|
-
}
|
|
535
|
+
};
|
|
536
|
+
const candidate = await roll("candidate", candidateText, target, scopes);
|
|
537
|
+
if (candidate.ok)
|
|
538
|
+
return succeed(dropped);
|
|
380
539
|
if (rollbackText === null || rollbackImages === null)
|
|
381
|
-
return done("failed",
|
|
382
|
-
|
|
383
|
-
|
|
540
|
+
return done("failed", `new_supervisor_not_qualified_no_previous${droppedDetail(dropped)}`, { ...tagged, exitCode: 1 });
|
|
541
|
+
let restored = await roll("rollback", rollbackText, rollbackImages, scopes);
|
|
542
|
+
// The previous release is the control (CTC-7569). Missing exactly the scopes the candidate missed while
|
|
543
|
+
// qualifying the rest, it shows those scopes stopped being delegated, not that the candidate broke them:
|
|
544
|
+
// the candidate is applied again without them, and recorded only once it qualifies what is left.
|
|
545
|
+
const gone = lostDelegation(candidate, restored, scopes);
|
|
546
|
+
if (gone !== null) {
|
|
547
|
+
const kept = new Set([...scopes].filter((scope) => !gone.has(scope)));
|
|
548
|
+
saveInFlight(dir, kept, now());
|
|
549
|
+
if ((await roll("candidate", candidateText, target, kept)).ok)
|
|
550
|
+
return succeed([...dropped, ...gone]);
|
|
551
|
+
restored = await roll("rollback", rollbackText, rollbackImages, kept);
|
|
552
|
+
}
|
|
384
553
|
// A rollback that did not qualify keeps the baseline for the next run.
|
|
385
|
-
if (restored)
|
|
554
|
+
if (restored.ok)
|
|
386
555
|
clearInFlight(dir);
|
|
387
|
-
return done(restored ? "rolled_back" : "failed", restored ? "new_supervisor_not_qualified" : "rollback_not_qualified"
|
|
556
|
+
return done(restored.ok ? "rolled_back" : "failed", `${restored.ok ? "new_supervisor_not_qualified" : "rollback_not_qualified"}${droppedDetail(dropped)}`, { ...tagged, exitCode: 1 });
|
|
388
557
|
}
|
|
389
558
|
/** Put `text` back as `compose.json` and recreate the supervisor alone on it. */
|
|
390
559
|
async function restoreCompose(engine, dir, project, text, signal) {
|
|
@@ -461,6 +630,11 @@ export async function runFleetPostRollCheck(engine, options) {
|
|
|
461
630
|
}
|
|
462
631
|
if (previous === null)
|
|
463
632
|
return refuse("no_previous_compose");
|
|
633
|
+
// An update that started from this release and did not finish saved this release's own file as the
|
|
634
|
+
// previous one (CTC-7570). Restoring it would move nothing and record a rollback that never happened.
|
|
635
|
+
const current = fleetImages(compose);
|
|
636
|
+
if (current !== null && previous.runner === current.runner && previous.supervisor === current.supervisor)
|
|
637
|
+
return refuse("previous_compose_is_this_release");
|
|
464
638
|
// The apply removed the images it replaced, so the previous ones may need pulling again.
|
|
465
639
|
const unprepared = await engine.prepare([previous.runner, previous.supervisor], signal);
|
|
466
640
|
if (unprepared !== null)
|
|
@@ -478,37 +652,69 @@ export async function runFleetPostRollCheck(engine, options) {
|
|
|
478
652
|
}
|
|
479
653
|
}
|
|
480
654
|
/** The recreated supervisor runs `images`, stays up with no restart, logs no `host_not_ready` for a
|
|
481
|
-
* scope it must serve, and qualifies every one of `scopes`, held for the minimum window.
|
|
482
|
-
|
|
655
|
+
* scope it must serve, and qualifies every one of `scopes`, held for the minimum window. Its deadline
|
|
656
|
+
* moves with the progress it logs (CTC-8163); the restart, image and `host_not_ready` exits come first. */
|
|
657
|
+
async function qualify(engine, dir, project, images, scopes, options, now, roll) {
|
|
483
658
|
const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
484
659
|
const start = now();
|
|
485
|
-
const
|
|
660
|
+
const budget = {
|
|
661
|
+
deadline: start + (options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS),
|
|
662
|
+
ceiling: start + qualifyCeilingMs(options),
|
|
663
|
+
tenure: false,
|
|
664
|
+
readopted: new Set(),
|
|
665
|
+
ready: new Set(),
|
|
666
|
+
};
|
|
486
667
|
const minimum = start + (options.qualifyMinMs ?? QUALIFY_MIN_MS);
|
|
487
668
|
let id = null;
|
|
488
669
|
for (;;) {
|
|
489
670
|
if (options.signal?.aborted)
|
|
490
|
-
return false;
|
|
671
|
+
return { ok: false, reason: "aborted" };
|
|
491
672
|
const live = await engine.supervisor(dir, project, options.signal);
|
|
673
|
+
// What this poll's log has not qualified yet, and its gate lines; null when the supervisor or its
|
|
674
|
+
// log could not be read.
|
|
675
|
+
let missing = null;
|
|
676
|
+
let gates = null;
|
|
492
677
|
if (live) {
|
|
493
678
|
id ??= live.id;
|
|
494
679
|
if (live.id !== id || !live.running || live.restartCount !== 0)
|
|
495
|
-
return false;
|
|
680
|
+
return { ok: false, reason: "restarted" };
|
|
496
681
|
if (live.image !== images.supervisor || live.runnerImage !== images.runner)
|
|
497
|
-
return false;
|
|
682
|
+
return { ok: false, reason: "wrong_image" };
|
|
498
683
|
const text = await engine.logs(live.id, live.startedAt, options.signal);
|
|
499
684
|
if (text !== null) {
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
685
|
+
const log = readSupervisorLog(text, scopes);
|
|
686
|
+
if (log.notReady !== null)
|
|
687
|
+
return { ok: false, reason: "host_not_ready" };
|
|
688
|
+
missing = new Set([...scopes].filter((scope) => !log.qualified.has(scope)));
|
|
689
|
+
if (now() >= minimum && missing.size === 0)
|
|
690
|
+
return { ok: true };
|
|
691
|
+
countProgress(budget, log, now());
|
|
692
|
+
gates = log.gates;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
if (now() >= budget.deadline) {
|
|
696
|
+
for (const line of qualificationPendingLines(roll, missing ?? scopes, gates)) {
|
|
697
|
+
try {
|
|
698
|
+
options.onQualificationPending?.(line);
|
|
699
|
+
}
|
|
700
|
+
catch {
|
|
701
|
+
// A line that could not be written never holds up the other lines or the restore.
|
|
702
|
+
}
|
|
505
703
|
}
|
|
704
|
+
return missing !== null && missing.size > 0 ? { ok: false, reason: "missing", missing } : { ok: false, reason: "deadline" };
|
|
506
705
|
}
|
|
507
|
-
if (now() >= deadline)
|
|
508
|
-
return false;
|
|
509
706
|
await sleep(QUALIFY_POLL_MS);
|
|
510
707
|
}
|
|
511
708
|
}
|
|
709
|
+
/** The scopes to drop when the candidate and the previous release, its control, both failed only by
|
|
710
|
+
* missing scopes, the same ones, and the control qualified at least one other `required` scope. Null
|
|
711
|
+
* otherwise: then the candidate is what failed, or nothing shows which. */
|
|
712
|
+
function lostDelegation(candidate, control, required) {
|
|
713
|
+
if (candidate.ok || control.ok || candidate.reason !== "missing" || control.reason !== "missing")
|
|
714
|
+
return null;
|
|
715
|
+
const same = candidate.missing.size === control.missing.size && [...candidate.missing].every((scope) => control.missing.has(scope));
|
|
716
|
+
return same && control.missing.size < required.size ? control.missing : null;
|
|
717
|
+
}
|
|
512
718
|
const CONTAINER_ID = /^[0-9a-f]{12,64}$/;
|
|
513
719
|
/** The one volume or bind in `service.volumes` that holds `file`, as a read-only `--mount` value. */
|
|
514
720
|
export function credentialMount(compose, service, file) {
|
|
@@ -559,8 +765,8 @@ export function dockerFleetUpdateEngine(deps = {}) {
|
|
|
559
765
|
const compose = (dir, project, file = join(dir, FLEET_COMPOSE_FILE)) => ["compose", "--project-name", project, "--file", file];
|
|
560
766
|
return {
|
|
561
767
|
// Not `compose run`: that would mount the Docker socket and every scope's state. This container
|
|
562
|
-
// gets the credential alone, read-only,
|
|
563
|
-
async desiredRelease(_dir, file, signal) {
|
|
768
|
+
// gets the credential alone, read-only, the script's configuration and an optional update report.
|
|
769
|
+
async desiredRelease(_dir, file, signal, report) {
|
|
564
770
|
const service = isRecord(file.services) && isRecord(file.services.supervisor) ? file.services.supervisor : null;
|
|
565
771
|
const environment = service && isRecord(service.environment) ? service.environment : null;
|
|
566
772
|
const credential = environment?.CATALYST_HOST_CREDENTIAL_FILE;
|
|
@@ -571,9 +777,10 @@ export function dockerFleetUpdateEngine(deps = {}) {
|
|
|
571
777
|
if (!mount)
|
|
572
778
|
return { kind: "no_credential" };
|
|
573
779
|
const user = typeof service.user === "string" && /^[0-9]+(:[0-9]+)?$/.test(service.user) ? ["--user", service.user] : [];
|
|
780
|
+
const reportEnv = report ? ["--env", `CATALYST_UPDATE_REPORT=${report}`] : [];
|
|
574
781
|
const read = await docker(["run", "--rm", "--network", "bridge", "--read-only", "--tmpfs", "/tmp", "--cap-drop", "ALL",
|
|
575
782
|
"--security-opt", "no-new-privileges", ...user, "--mount", mount,
|
|
576
|
-
"--env", `CATALYST_HOST_CREDENTIAL_FILE=${credential}`, "--env", `CATALYST_MIRROR_URL=${mirror}`,
|
|
783
|
+
"--env", `CATALYST_HOST_CREDENTIAL_FILE=${credential}`, "--env", `CATALYST_MIRROR_URL=${mirror}`, ...reportEnv,
|
|
577
784
|
"--entrypoint", "bun", service.image, "-e", DESIRED_RELEASE_SCRIPT], { signal, timeoutMs: 120_000 });
|
|
578
785
|
if (read.code !== 0)
|
|
579
786
|
return { kind: "no_credential" };
|