@catalyst-cloud/cli 0.16.41 → 0.16.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { existsSync, readFileSync, unlinkSync } from "node:fs";
3
3
  import { dirname, join, resolve } from "node:path";
4
+ import { readManifest } from "./config.js";
4
5
  import { dockerRunnerEngine, prepareRunnerImages, spawnDocker } from "./onboard-runner.js";
5
6
  import { defaultRun } from "./runner-auto-update.js";
6
- import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseState, releaseLock, takeLock, writePrivateAtomic, writeReleaseState, } from "./runner-update.js";
7
+ import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseState, releaseLock, takeLock, updateReport, writePrivateAtomic, writeReleaseState, } from "./runner-update.js";
7
8
  // `catalyst runner update --fleet <dir>`: the same approved release, applied to a multi-team
8
9
  // supervisor fleet (CTC-6530). One supervisor container serves several scopes from a Compose file
9
10
  // written as JSON (`compose.json`), under one operator enrollment, and every host credential on the
@@ -23,23 +24,45 @@ import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseSt
23
24
  // back and the supervisor is recreated on it. That scope set is saved before the recreate and removed
24
25
  // once it qualifies or rolls back, so a run killed in between is finished by the next one.
25
26
  //
27
+ // A busy host re-adopts its live sessions before it qualifies, which can take longer than four minutes
28
+ // (CTC-8163). Each recreated supervisor, the new release's or the restored previous one's, gets four
29
+ // minutes, extended to 150 seconds past each new step it logs: fleet tenure, then a saved scope
30
+ // re-adopted, then that scope ready. No supervisor gets more than seven minutes. A restart, another
31
+ // image or `host_not_ready` still ends the wait at the next poll. At the deadline each saved scope it
32
+ // has not qualified is reported, with the gate condition its latest log line shows failing.
33
+ //
34
+ // A team can stop being delegated while an update is in flight (CTC-7569): no supervisor, new or old,
35
+ // starts its scope again. The previous release, restored, is the control. When it misses exactly the
36
+ // scopes the new supervisor missed and qualifies the rest, those scopes are dropped from the saved set
37
+ // and the new release is applied again against what is left. A later run that finds a saved set beside
38
+ // the previous release's running supervisor drops the saved scopes that supervisor no longer qualifies.
39
+ // Either way the result names each dropped scope; the new release's own log never drops one.
40
+ //
26
41
  // A release can qualify and then stop receiving work minutes later (CTC-6564). A successful apply
27
42
  // arms a check on the host, outside the calling process, for 10 minutes later: it reads the supervisor's
28
43
  // log since the roll began and, when the release was granted no work and the fleet's acquires got no
29
44
  // answer, restores the saved previous `compose.json`. `runner update --fleet --post-roll-check` is
30
45
  // that check; only the armed timer runs it.
46
+ // A check that is due while an update holds the lock judges nothing; that update arms it again
47
+ // for 5 minutes after it ends (CTC-7570).
31
48
  export const FLEET_COMPOSE_FILE = "compose.json";
32
49
  export const PREVIOUS_COMPOSE_FILE = "compose.release-previous.json";
33
50
  const CANDIDATE_COMPOSE_FILE = "compose.release-candidate.json";
34
51
  /** The scopes the supervisor served before a recreate, kept until that recreate is qualified or rolled
35
52
  * back. A run that was killed in between finds it and qualifies against it, never against the new
36
- * container's own log, which would read an unqualified supervisor as serving. */
53
+ * container's own log, which would read an unqualified supervisor as serving. Only the previous
54
+ * release's supervisor can take a scope out of it (CTC-7569). */
37
55
  export const IN_FLIGHT_FILE = "fleet-update-in-flight.json";
38
56
  /** How far back a qualification counts as current before the update. The supervisor repeats it
39
57
  * on every loop while a scope is ready, so a ready scope logs it many times in this window. */
40
58
  const QUALIFIED_LOOKBACK_MS = 5 * 60_000;
41
59
  const QUALIFY_MIN_MS = 90_000;
60
+ /** How long a recreated supervisor that shows no progress has to qualify every saved scope. */
42
61
  const QUALIFY_DEADLINE_MS = 240_000;
62
+ /** How long past each new step of progress a recreated supervisor has to qualify (CTC-8163). */
63
+ const QUALIFY_PROGRESS_MS = 150_000;
64
+ /** The most one recreated supervisor is given, however it progresses (CTC-8163). */
65
+ const QUALIFY_CEILING_MS = 420_000;
43
66
  const QUALIFY_POLL_MS = 10_000;
44
67
  const CHILD_NAME_PREFIX = "catalyst-pa_";
45
68
  /** What a successful apply leaves for the check: the bytes it wrote and when the roll began. */
@@ -47,6 +70,8 @@ export const POST_ROLL_ARMED_FILE = "post-roll-armed.json";
47
70
  /** The check's last verdict, beside `release-state.json`. */
48
71
  export const POST_ROLL_RESULT_FILE = "post-roll-result.json";
49
72
  export const POST_ROLL_DELAY_MS = 10 * 60_000;
73
+ /** How long after an update run lets go of the lock a check it kept from judging runs again (CTC-7570). */
74
+ export const POST_ROLL_RETRY_DELAY_MS = 5 * 60_000;
50
75
  const POST_ROLL_UNIT = "catalyst-fleet-post-roll";
51
76
  /** No-response acquire retries, with no grants, that count a release as failing. */
52
77
  const POST_ROLL_MAX_NO_RESPONSE = 3;
@@ -91,14 +116,83 @@ export function qualifiedScopes(text) {
91
116
  /** A `host_not_ready` line for one of `scopes`, or one that names no scope. Other scopes may be
92
117
  * unready for their own reasons (a team with no repository yet) and do not stop an update. */
93
118
  export function notReadyScope(text, scopes) {
119
+ return readSupervisorLog(text, scopes).notReady;
120
+ }
121
+ const GATE_CODES = [
122
+ ["resourcesReady", "resources_ready"],
123
+ ["scopeEligible", "scope_eligible"],
124
+ ["readyUntilValid", "ready_until"],
125
+ ["sessionsReadopted", "sessions_readopted"],
126
+ ];
127
+ /** One pass over the log text for everything qualify() reads. Lines for other scopes, and lines that
128
+ * are not JSON or lack a field, are skipped. */
129
+ function readSupervisorLog(text, scopes) {
130
+ const log = { notReady: null, qualified: new Set(), tenure: false, readopted: new Set(), ready: new Set(), gates: new Map() };
94
131
  for (const event of logEvents(text)) {
95
- if (event.event !== "host_not_ready")
96
- continue;
97
132
  const key = scopeKey(event);
98
- if (key === null || scopes.has(key))
99
- return key ?? "unscoped";
133
+ if (event.event === "host_not_ready" && log.notReady === null && (key === null || scopes.has(key)))
134
+ log.notReady = key ?? "unscoped";
135
+ if (event.event === "supervisor_fleet_tenure")
136
+ log.tenure = true;
137
+ if (key === null || !scopes.has(key))
138
+ continue;
139
+ if (event.event === "supervisor_scope_qualified")
140
+ log.qualified.add(key);
141
+ else if (event.event === "supervisor_readopted")
142
+ log.readopted.add(key);
143
+ else if (event.event === "host_ready")
144
+ log.ready.add(key);
145
+ else if (event.event === "supervisor_scope_qualification_state") {
146
+ const { resourcesReady, scopeEligible, readyUntilValid, sessionsReadopted } = event;
147
+ if (typeof resourcesReady === "boolean" && typeof scopeEligible === "boolean" && typeof readyUntilValid === "boolean" && typeof sessionsReadopted === "boolean")
148
+ log.gates.set(key, { resourcesReady, scopeEligible, readyUntilValid, sessionsReadopted });
149
+ }
150
+ }
151
+ return log;
152
+ }
153
+ /** Count the steps `log` shows that `budget` has not counted yet: fleet tenure, a scope re-adopted, and
154
+ * a re-adopted scope ready. A scope ready before its re-adoption shows counts once that does. Any new
155
+ * step moves the deadline to `QUALIFY_PROGRESS_MS` past `atMs`, never earlier and never past the ceiling. */
156
+ function countProgress(budget, log, atMs) {
157
+ let progressed = false;
158
+ if (log.tenure && !budget.tenure) {
159
+ budget.tenure = true;
160
+ progressed = true;
161
+ }
162
+ for (const scope of log.readopted) {
163
+ if (budget.readopted.has(scope))
164
+ continue;
165
+ budget.readopted.add(scope);
166
+ progressed = true;
167
+ }
168
+ for (const scope of log.ready) {
169
+ if (!budget.readopted.has(scope) || budget.ready.has(scope))
170
+ continue;
171
+ budget.ready.add(scope);
172
+ progressed = true;
100
173
  }
101
- return null;
174
+ if (progressed)
175
+ budget.deadline = Math.min(budget.ceiling, Math.max(budget.deadline, atMs + QUALIFY_PROGRESS_MS));
176
+ }
177
+ /** The longest one recreated supervisor is given to qualify, however it progresses. */
178
+ const qualifyCeilingMs = (options) => Math.max(QUALIFY_CEILING_MS, options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS);
179
+ /** A scope as printed: anything outside printable ASCII, and the backslash that starts an escape,
180
+ * escaped as `\uXXXX`, and at most 120 characters. */
181
+ const printableScope = (scope) => {
182
+ const escaped = scope.replace(/[^\x21-\x5b\x5d-\x7e]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`);
183
+ return escaped.length > 120 ? `${escaped.slice(0, 120)}...` : escaped;
184
+ };
185
+ /** The gate conditions a missing scope's snapshot shows failing, sorted; `diagnostic_unavailable`
186
+ * without one, and `qualification_event_missing` when every condition held. */
187
+ function failedGates(snapshot) {
188
+ if (snapshot === undefined)
189
+ return ["diagnostic_unavailable"];
190
+ const failed = GATE_CODES.filter(([field]) => !snapshot[field]).map(([, code]) => code).sort();
191
+ return failed.length > 0 ? failed : ["qualification_event_missing"];
192
+ }
193
+ /** One line per scope in `missing`, sorted by scope. `gates` null: the log could not be read. */
194
+ function qualificationPendingLines(roll, missing, gates) {
195
+ return [...missing].sort().map((scope) => `fleet qualification pending: roll=${roll} scope=${printableScope(scope)} failed=${failedGates(gates?.get(scope)).join(",")}`);
102
196
  }
103
197
  function readInFlight(dir) {
104
198
  try {
@@ -112,6 +206,10 @@ function readInFlight(dir) {
112
206
  return null;
113
207
  }
114
208
  }
209
+ /** Save the scopes the next recreate has to qualify, replacing any saved before. */
210
+ function saveInFlight(dir, scopes, atMs) {
211
+ writePrivateAtomic(join(dir, IN_FLIGHT_FILE), `${JSON.stringify({ version: 1, scopes: [...scopes].sort(), atMs })}\n`);
212
+ }
115
213
  function clearInFlight(dir) {
116
214
  try {
117
215
  unlinkSync(join(dir, IN_FLIGHT_FILE));
@@ -120,6 +218,17 @@ function clearInFlight(dir) {
120
218
  // Already gone.
121
219
  }
122
220
  }
221
+ /** The images a saved `compose.json` names, or null when there is none or it is not a fleet layout. */
222
+ function savedImages(path) {
223
+ try {
224
+ return fleetImages(JSON.parse(readFileSync(path, "utf8")));
225
+ }
226
+ catch {
227
+ return null;
228
+ }
229
+ }
230
+ /** The detail token naming scopes dropped from the requirement, sorted; empty when there are none. */
231
+ const droppedDetail = (scopes) => scopes.length > 0 ? `;dropped_scope:${[...scopes].sort().join(",")}` : "";
123
232
  const removeFile = (path) => {
124
233
  try {
125
234
  unlinkSync(path);
@@ -226,6 +335,23 @@ async function armPostRollCheck(engine, dir, armed, signal) {
226
335
  return ";post_roll_check_not_armed";
227
336
  }
228
337
  }
338
+ /** The armed check runs once, from a one-shot timer. When it was due while an update held the lock, or
339
+ * Docker could not be read, it judged nothing and left the armed file; a reboot loses the timer the same
340
+ * way (CTC-7570). The update holding the lock arms it again for a few minutes later, before it lets go:
341
+ * while the lock is held no check is part-way through a judgment, so the arm's stop of the check's unit
342
+ * interrupts nothing. The check cannot arm itself: that stop would end its own unit. A check this run
343
+ * armed is not due yet and is left alone. */
344
+ async function rearmOverdueCheck(engine, dir, now, signal) {
345
+ const armed = readPostRollArmed(dir);
346
+ if (armed === null || now() < armed.armedAtMs + POST_ROLL_DELAY_MS)
347
+ return;
348
+ try {
349
+ await engine.armPostRollCheck(dir, POST_ROLL_RETRY_DELAY_MS, signal);
350
+ }
351
+ catch {
352
+ // The next run tries again.
353
+ }
354
+ }
229
355
  export async function runFleetUpdate(engine, options) {
230
356
  const { dir } = options;
231
357
  const now = options.now ?? Date.now;
@@ -243,8 +369,10 @@ export async function runFleetUpdate(engine, options) {
243
369
  return result;
244
370
  }
245
371
  finally {
246
- if (lock !== null)
372
+ if (lock !== null) {
373
+ await rearmOverdueCheck(engine, dir, now, options.signal);
247
374
  releaseLock(dir, lock);
375
+ }
248
376
  }
249
377
  }
250
378
  async function update(engine, options, now) {
@@ -264,7 +392,7 @@ async function update(engine, options, now) {
264
392
  const project = isRecord(compose) && typeof compose.name === "string" ? compose.name : "";
265
393
  if (!isRecord(compose) || current === null || !/^[a-z0-9][a-z0-9_-]*$/.test(project))
266
394
  return done("failed", "fleet_compose_unrecognized", { exitCode: 1 });
267
- const read = await engine.desiredRelease(dir, compose, signal);
395
+ const read = await engine.desiredRelease(dir, compose, signal, options.check ? undefined : updateReport(state, readManifest().version));
268
396
  const decision = decideRelease(state.generation, read);
269
397
  if (decision.action === "retain")
270
398
  return decision.reason === "not_newer"
@@ -291,6 +419,8 @@ async function update(engine, options, now) {
291
419
  // The scopes the running supervisor qualifies are what the new one has to qualify again, unless an
292
420
  // earlier run stopped mid-recreate: then the scopes it saved are the baseline.
293
421
  let scopes = inFlight;
422
+ /** Saved scopes the previous release's supervisor no longer qualifies, named in the result (CTC-7569). */
423
+ let dropped = [];
294
424
  if (scopes === null) {
295
425
  if (!live?.running)
296
426
  return done("retained", "supervisor_not_running", { ...tagged, exitCode: 1 });
@@ -301,6 +431,25 @@ async function update(engine, options, now) {
301
431
  if (scopes.size === 0)
302
432
  return done("retained", "no_qualified_scope", { ...tagged, exitCode: 1 });
303
433
  }
434
+ else {
435
+ // An earlier run that ended back on the previous release, its rollback unqualified, left its scopes
436
+ // saved. Only when compose.json and the running supervisor are that previous release, up long enough
437
+ // to have qualified every scope it serves, does its log trim the saved set: a saved scope it no
438
+ // longer qualifies stopped being delegated. A `host_not_ready` for a saved scope is a scope still
439
+ // started, not one dropped. Anything less keeps the saved set whole. Long enough is the most any
440
+ // recreated supervisor is given, since this run cannot see how it progressed (CTC-8163).
441
+ const previous = savedImages(join(dir, PREVIOUS_COMPOSE_FILE));
442
+ const onPrevious = previous !== null && current.runner === previous.runner && current.supervisor === previous.supervisor && runs(previous);
443
+ if (live && onPrevious && now() - Date.parse(live.startedAt) >= qualifyCeilingMs(options)) {
444
+ const recent = await engine.logs(live.id, new Date(now() - QUALIFIED_LOOKBACK_MS).toISOString(), signal);
445
+ const serving = recent === null || notReadyScope(recent, scopes) !== null ? new Set() : qualifiedScopes(recent);
446
+ const kept = [...scopes].filter((scope) => serving.has(scope));
447
+ if (kept.length > 0 && kept.length < scopes.size) {
448
+ dropped = [...scopes].filter((scope) => !serving.has(scope));
449
+ scopes = new Set(kept);
450
+ }
451
+ }
452
+ }
304
453
  const record = (outcome, detail, previous, exitCode = 0) => {
305
454
  writeReleaseState(dir, {
306
455
  ...state,
@@ -353,9 +502,15 @@ async function update(engine, options, now) {
353
502
  const rollStartMs = now();
354
503
  if (changed.length > 0)
355
504
  writePrivateAtomic(join(dir, PREVIOUS_COMPOSE_FILE), composeText);
356
- writePrivateAtomic(join(dir, IN_FLIGHT_FILE), `${JSON.stringify({ version: 1, scopes: [...scopes].sort(), atMs: now() })}\n`);
357
- writePrivateAtomic(composePath, candidateText);
358
- if ((await engine.recreate(dir, project, signal)) && (await qualify(engine, dir, project, target, scopes, options, now))) {
505
+ saveInFlight(dir, scopes, now());
506
+ /** Write `text` as compose.json, recreate the supervisor alone on it and qualify it against `required`. */
507
+ const roll = async (which, text, images, required) => {
508
+ writePrivateAtomic(composePath, text);
509
+ if (!(await engine.recreate(dir, project, signal)))
510
+ return { ok: false, reason: "recreate_failed" };
511
+ return qualify(engine, dir, project, images, required, options, now, which);
512
+ };
513
+ const succeed = async (droppedScopes) => {
359
514
  clearInFlight(dir);
360
515
  if (changed.length > 0)
361
516
  await engine.removeImages(changed.map((c) => current[c]), signal);
@@ -371,20 +526,34 @@ async function update(engine, options, now) {
371
526
  version: 1, sinceMs: rollStartMs, composeSha256: sha256(candidateText), generation: release.generation,
372
527
  releaseId: release.releaseId, fromGeneration: state.generation, armedAtMs: now(),
373
528
  };
374
- const detail = (changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled") + await armPostRollCheck(engine, dir, armed, signal);
529
+ const detail = (changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled") + droppedDetail(droppedScopes) +
530
+ await armPostRollCheck(engine, dir, armed, signal);
375
531
  const previous = changed.length > 0 ? current : null;
376
532
  return disturbed === 0
377
533
  ? record("updated", detail, previous)
378
534
  : record("updated", `${detail};${disturbed < 0 ? "children_unreadable" : `children_changed:${disturbed}`}`, previous, 1);
379
- }
535
+ };
536
+ const candidate = await roll("candidate", candidateText, target, scopes);
537
+ if (candidate.ok)
538
+ return succeed(dropped);
380
539
  if (rollbackText === null || rollbackImages === null)
381
- return done("failed", "new_supervisor_not_qualified_no_previous", { ...tagged, exitCode: 1 });
382
- const restored = (await restoreCompose(engine, dir, project, rollbackText, signal)) &&
383
- (await qualify(engine, dir, project, rollbackImages, scopes, options, now));
540
+ return done("failed", `new_supervisor_not_qualified_no_previous${droppedDetail(dropped)}`, { ...tagged, exitCode: 1 });
541
+ let restored = await roll("rollback", rollbackText, rollbackImages, scopes);
542
+ // The previous release is the control (CTC-7569). Missing exactly the scopes the candidate missed while
543
+ // qualifying the rest, it shows those scopes stopped being delegated, not that the candidate broke them:
544
+ // the candidate is applied again without them, and recorded only once it qualifies what is left.
545
+ const gone = lostDelegation(candidate, restored, scopes);
546
+ if (gone !== null) {
547
+ const kept = new Set([...scopes].filter((scope) => !gone.has(scope)));
548
+ saveInFlight(dir, kept, now());
549
+ if ((await roll("candidate", candidateText, target, kept)).ok)
550
+ return succeed([...dropped, ...gone]);
551
+ restored = await roll("rollback", rollbackText, rollbackImages, kept);
552
+ }
384
553
  // A rollback that did not qualify keeps the baseline for the next run.
385
- if (restored)
554
+ if (restored.ok)
386
555
  clearInFlight(dir);
387
- return done(restored ? "rolled_back" : "failed", restored ? "new_supervisor_not_qualified" : "rollback_not_qualified", { ...tagged, exitCode: 1 });
556
+ return done(restored.ok ? "rolled_back" : "failed", `${restored.ok ? "new_supervisor_not_qualified" : "rollback_not_qualified"}${droppedDetail(dropped)}`, { ...tagged, exitCode: 1 });
388
557
  }
389
558
  /** Put `text` back as `compose.json` and recreate the supervisor alone on it. */
390
559
  async function restoreCompose(engine, dir, project, text, signal) {
@@ -461,6 +630,11 @@ export async function runFleetPostRollCheck(engine, options) {
461
630
  }
462
631
  if (previous === null)
463
632
  return refuse("no_previous_compose");
633
+ // An update that started from this release and did not finish saved this release's own file as the
634
+ // previous one (CTC-7570). Restoring it would move nothing and record a rollback that never happened.
635
+ const current = fleetImages(compose);
636
+ if (current !== null && previous.runner === current.runner && previous.supervisor === current.supervisor)
637
+ return refuse("previous_compose_is_this_release");
464
638
  // The apply removed the images it replaced, so the previous ones may need pulling again.
465
639
  const unprepared = await engine.prepare([previous.runner, previous.supervisor], signal);
466
640
  if (unprepared !== null)
@@ -478,37 +652,69 @@ export async function runFleetPostRollCheck(engine, options) {
478
652
  }
479
653
  }
480
654
  /** The recreated supervisor runs `images`, stays up with no restart, logs no `host_not_ready` for a
481
- * scope it must serve, and qualifies every one of `scopes`, held for the minimum window. */
482
- async function qualify(engine, dir, project, images, scopes, options, now) {
655
+ * scope it must serve, and qualifies every one of `scopes`, held for the minimum window. Its deadline
656
+ * moves with the progress it logs (CTC-8163); the restart, image and `host_not_ready` exits come first. */
657
+ async function qualify(engine, dir, project, images, scopes, options, now, roll) {
483
658
  const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
484
659
  const start = now();
485
- const deadline = start + (options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS);
660
+ const budget = {
661
+ deadline: start + (options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS),
662
+ ceiling: start + qualifyCeilingMs(options),
663
+ tenure: false,
664
+ readopted: new Set(),
665
+ ready: new Set(),
666
+ };
486
667
  const minimum = start + (options.qualifyMinMs ?? QUALIFY_MIN_MS);
487
668
  let id = null;
488
669
  for (;;) {
489
670
  if (options.signal?.aborted)
490
- return false;
671
+ return { ok: false, reason: "aborted" };
491
672
  const live = await engine.supervisor(dir, project, options.signal);
673
+ // What this poll's log has not qualified yet, and its gate lines; null when the supervisor or its
674
+ // log could not be read.
675
+ let missing = null;
676
+ let gates = null;
492
677
  if (live) {
493
678
  id ??= live.id;
494
679
  if (live.id !== id || !live.running || live.restartCount !== 0)
495
- return false;
680
+ return { ok: false, reason: "restarted" };
496
681
  if (live.image !== images.supervisor || live.runnerImage !== images.runner)
497
- return false;
682
+ return { ok: false, reason: "wrong_image" };
498
683
  const text = await engine.logs(live.id, live.startedAt, options.signal);
499
684
  if (text !== null) {
500
- if (notReadyScope(text, scopes) !== null)
501
- return false;
502
- const seen = qualifiedScopes(text);
503
- if (now() >= minimum && [...scopes].every((scope) => seen.has(scope)))
504
- return true;
685
+ const log = readSupervisorLog(text, scopes);
686
+ if (log.notReady !== null)
687
+ return { ok: false, reason: "host_not_ready" };
688
+ missing = new Set([...scopes].filter((scope) => !log.qualified.has(scope)));
689
+ if (now() >= minimum && missing.size === 0)
690
+ return { ok: true };
691
+ countProgress(budget, log, now());
692
+ gates = log.gates;
693
+ }
694
+ }
695
+ if (now() >= budget.deadline) {
696
+ for (const line of qualificationPendingLines(roll, missing ?? scopes, gates)) {
697
+ try {
698
+ options.onQualificationPending?.(line);
699
+ }
700
+ catch {
701
+ // A line that could not be written never holds up the other lines or the restore.
702
+ }
505
703
  }
704
+ return missing !== null && missing.size > 0 ? { ok: false, reason: "missing", missing } : { ok: false, reason: "deadline" };
506
705
  }
507
- if (now() >= deadline)
508
- return false;
509
706
  await sleep(QUALIFY_POLL_MS);
510
707
  }
511
708
  }
709
+ /** The scopes to drop when the candidate and the previous release, its control, both failed only by
710
+ * missing scopes, the same ones, and the control qualified at least one other `required` scope. Null
711
+ * otherwise: then the candidate is what failed, or nothing shows which. */
712
+ function lostDelegation(candidate, control, required) {
713
+ if (candidate.ok || control.ok || candidate.reason !== "missing" || control.reason !== "missing")
714
+ return null;
715
+ const same = candidate.missing.size === control.missing.size && [...candidate.missing].every((scope) => control.missing.has(scope));
716
+ return same && control.missing.size < required.size ? control.missing : null;
717
+ }
512
718
  const CONTAINER_ID = /^[0-9a-f]{12,64}$/;
513
719
  /** The one volume or bind in `service.volumes` that holds `file`, as a read-only `--mount` value. */
514
720
  export function credentialMount(compose, service, file) {
@@ -559,8 +765,8 @@ export function dockerFleetUpdateEngine(deps = {}) {
559
765
  const compose = (dir, project, file = join(dir, FLEET_COMPOSE_FILE)) => ["compose", "--project-name", project, "--file", file];
560
766
  return {
561
767
  // Not `compose run`: that would mount the Docker socket and every scope's state. This container
562
- // gets the credential alone, read-only, and the two variables the script reads.
563
- async desiredRelease(_dir, file, signal) {
768
+ // gets the credential alone, read-only, the script's configuration and an optional update report.
769
+ async desiredRelease(_dir, file, signal, report) {
564
770
  const service = isRecord(file.services) && isRecord(file.services.supervisor) ? file.services.supervisor : null;
565
771
  const environment = service && isRecord(service.environment) ? service.environment : null;
566
772
  const credential = environment?.CATALYST_HOST_CREDENTIAL_FILE;
@@ -571,9 +777,10 @@ export function dockerFleetUpdateEngine(deps = {}) {
571
777
  if (!mount)
572
778
  return { kind: "no_credential" };
573
779
  const user = typeof service.user === "string" && /^[0-9]+(:[0-9]+)?$/.test(service.user) ? ["--user", service.user] : [];
780
+ const reportEnv = report ? ["--env", `CATALYST_UPDATE_REPORT=${report}`] : [];
574
781
  const read = await docker(["run", "--rm", "--network", "bridge", "--read-only", "--tmpfs", "/tmp", "--cap-drop", "ALL",
575
782
  "--security-opt", "no-new-privileges", ...user, "--mount", mount,
576
- "--env", `CATALYST_HOST_CREDENTIAL_FILE=${credential}`, "--env", `CATALYST_MIRROR_URL=${mirror}`,
783
+ "--env", `CATALYST_HOST_CREDENTIAL_FILE=${credential}`, "--env", `CATALYST_MIRROR_URL=${mirror}`, ...reportEnv,
577
784
  "--entrypoint", "bun", service.image, "-e", DESIRED_RELEASE_SCRIPT], { signal, timeoutMs: 120_000 });
578
785
  if (read.code !== 0)
579
786
  return { kind: "no_credential" };