@catalyst-cloud/cli 0.16.41 → 0.16.42

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.16.42
+
+A busy machine now takes a waiting release after one day. Before, a busy machine could keep putting off a release that cannot take over from running work in place. Now the next check asks Catalyst to send the machine no new work, waits for the running work to finish, then moves to the release and takes work again. `catalyst runner update --check` says when a machine is waiting like this, and `catalyst runner auto-update off` ends the wait. If the update cannot restore the machine's private thoughts storage, the machine keeps taking no new work, and `catalyst runner update` tells you what to do. This release also makes updates on Catalyst's own machines more reliable.
+
 ## 0.16.41
 
 A runner's automatic update schedule now applies a newly approved runner release at its next check, instead of waiting for one of four set times of day. An owner who wants set times can still choose them with `catalyst update policy windows <HH:MM,...>`, and `catalyst update policy windows default` goes back to applying at the next check. `catalyst runner update` still applies a release at once, and `catalyst update policy` shows which setting is in effect.

package/README.md

@@ -346,7 +346,7 @@ The skill files themselves are written by whichever install command you ran, in
 
 The new CLI is installed beside the old one, and your `catalyst` command runs it at the same path. A running local replica restarts on it and keeps its synced data; a stopped one stays stopped. CLI, SDK and image updates keep their existing policy against moving to an older version. Ordinary CLI startup never replaces skills with its bundled copy.
 
-On a machine checking for releases every 15 minutes (`catalyst runner auto-update on`), that check applies the same component policies. Otherwise components move when you run `catalyst update`. That check moves the runner images to a newer release as soon as it finds one, with no set times of day. An update does not stop work that is already running: that work finishes on the image it started with. If you want updates only at set times, `catalyst update policy windows 06:00,18:00 --time-zone America/Chicago` sets your own times, `catalyst update policy windows default` goes back to none, and `catalyst runner update` or `catalyst update` applies a waiting release at once. Each component is on by default; turn one off with `catalyst update policy <component> off` (for example `cli`), and `catalyst update policy` lists every setting. If you turned the installer's daily update off, the CLI and both skills packs start held, and `catalyst update policy cli on` turns the CLI back on.
+On a machine checking for releases every 15 minutes (`catalyst runner auto-update on`), that check applies the same component policies. Otherwise components move when you run `catalyst update`. That check moves the runner images to a newer release as soon as it finds one, with no set times of day. An update does not stop work that is already running: that work finishes on the image it started with. When work is running and the new release cannot take over from it in place, the check leaves the release waiting. Once a release has waited a day, the next check asks Catalyst to send the machine no new work. It waits for the running work to finish, however long that takes, then moves to the release and takes work again. `catalyst runner update --check` says when a machine is waiting like this, and `catalyst runner auto-update off` ends the wait. If you want updates only at set times, `catalyst update policy windows 06:00,18:00 --time-zone America/Chicago` sets your own times, `catalyst update policy windows default` goes back to none, and `catalyst runner update` or `catalyst update` applies a waiting release at once. Each component is on by default; turn one off with `catalyst update policy <component> off` (for example `cli`), and `catalyst update policy` lists every setting. If you turned the installer's daily update off, the CLI and both skills packs start held, and `catalyst update policy cli on` turns the CLI back on.
 
 The installer's daily job runs the same release check, so it moves the CLI and the Cloud skills only to the approved release and keeps your holds. Until the development skills join the release, it refreshes them from their repository.
 

package/dist/runner-command.js

@@ -1,11 +1,11 @@
 import { spawnSync } from "node:child_process";
-import { realpathSync } from "node:fs";
+import { existsSync, realpathSync } from "node:fs";
 import { isAbsolute, resolve } from "node:path";
 import { flagBool, flagString, positionals } from "./args.js";
 import { CliError, UsageError } from "./errors.js";
 import { autoUpdateStatus, custodyNode, disableAutoUpdate, enableAutoUpdate, scheduledPath, } from "./runner-auto-update.js";
-import { dockerRunnerUpdateEngine, readReleaseState, RELEASE_COMPONENTS, runnerDir, runRunnerUpdate, updateResultLines, lastProblem, lastProblemLines, } from "./runner-update.js";
-import { dockerFleetUpdateEngine, postRollLines, readPostRollResult, runFleetPostRollCheck, runFleetUpdate } from "./runner-update-fleet.js";
+import { dockerRunnerUpdateEngine, readReleaseState, RELEASE_COMPONENTS, runnerDir, runRunnerUpdate, updateResultLines, lastProblem, lastProblemLines, drainLines, custodyUnattested, CUSTODY_UNATTESTED_LINE, releaseUpdateDrain, } from "./runner-update.js";
+import { dockerFleetUpdateEngine, postRollLines, readPostRollResult, runFleetPostRollCheck, runFleetUpdate, } from "./runner-update-fleet.js";
 import { runUpdate, sharedReleaseRead, updateReportJson, updateReportLines } from "./update.js";
 import { effectivePolicy, readUpdatePolicy, setComponentPolicy } from "./update-policy.js";
 import { resolveClaude } from "./update-plugin.js";
@@ -69,6 +69,18 @@ function statusLine(status) {
         return "Automatic runner updates need your systemd user manager, which is not running for you. Run: sudo loginctl enable-linger $USER, then catalyst runner auto-update on";
     return `Automatic runner updates are not available on this machine (${status.detail}). Run catalyst runner update yourself.`;
 }
+function autoLine(result, at) {
+    return `${at.toISOString()} ${result.outcome} ${result.detail} generation=${result.generation ?? "none"}`;
+}
+function drainReleaseLine(result) {
+    if (result === "released")
+        return "Catalyst can send this machine new phases again: it had stopped sending them so the machine could update.";
+    if (result === "pending")
+        return "The update that was waiting for running phases ends within a few seconds, and Catalyst can then send this machine new phases again.";
+    if (result === "failed")
+        return "Catalyst may still be holding new phases back from this machine. Run catalyst runner auto-update off again to retry.";
+    return null;
+}
 /** Why this machine's images must not move now: an owner's hold, or settings that cannot be read
  *  (which never count as "everything on"). Null when every image may move. */
 function imageHold(ctx) {
@@ -145,7 +157,9 @@ export async function cmdRunner(args, ctx, deps = {}) {
             if (args.json)
                 ctx.stdout(JSON.stringify(extra ? { ...result, components: extra.rows } : result));
             else if (auto) {
-                ctx.stdout(`${ctx.now().toISOString()} ${result.outcome} ${result.detail} generation=${result.generation ?? "none"}`);
+                ctx.stdout(autoLine(result, ctx.now()));
+                if (custodyUnattested(result))
+                    ctx.stdout(CUSTODY_UNATTESTED_LINE);
                 if (extra)
                     ctx.stdout(extra.line);
             }
@@ -160,13 +174,27 @@ export async function cmdRunner(args, ctx, deps = {}) {
         // windows; then only in a window. An owner's own run applies at once (CTC-7465, update-windows.ts).
         if (hold !== null) {
             const retained = { outcome: "retained", detail: hold, generation: readReleaseState(dir()).generation, exitCode: hold === "owner_setting" ? 0 : 1 };
+            if (hold === "owner_setting" && !fleet) {
+                const released = await releaseUpdateDrain(engine, dir(), { cloud: false });
+                if (released === "failed" && !auto && !args.json)
+                    ctx.stdout(drainReleaseLine(released));
+            }
             // Unreadable settings move nothing; a hold leaves the other components to the release check.
             return print(retained, hold === "owner_setting" ? await components(undefined) : null);
         }
         const ownerWindows = auto && !check && !fleet ? readUpdatePolicy(ctx.home).windows : undefined;
+        // A fleet supervisor that misses its qualification deadline names each scope it had not qualified,
+        // and why, on stderr before the restore, so stdout keeps its one line or JSON object (CTC-8163).
         const result = fleet
-            ? await runFleetUpdate(deps.fleetEngine ?? dockerFleetUpdateEngine({ env: ctx.env }), { dir: dir(), check })
-            : await runRunnerUpdate(engine, { dir: dir(), check, ...(ownerWindows ? { window: windowBounds(ctx.now().getTime(), ownerWindows) } : {}) });
+            ? await runFleetUpdate(deps.fleetEngine ?? dockerFleetUpdateEngine({ env: ctx.env }), {
+                dir: dir(), check, ...deps.fleetClock,
+                onQualificationPending: (line) => ctx.stderr(`catalyst: ${line}`),
+            })
+            : await runRunnerUpdate(engine, {
+                dir: dir(), check,
+                ...(ownerWindows ? { window: windowBounds(ctx.now().getTime(), ownerWindows) } : {}),
+                ...(auto && !check ? { drain: true, held: () => imageHold(ctx), ...(args.json ? {} : { onPass: (pass) => ctx.stdout(autoLine(pass, ctx.now())) }) } : {}),
+            });
         // `runner update --check` also lists every installed component against the approved release, as
         // `catalyst update --check` does. The image fields stay where scripts already read them.
         if (check && !fleet && !auto) {
@@ -190,10 +218,10 @@ export async function cmdRunner(args, ctx, deps = {}) {
             const state = readReleaseState(dir());
             if (args.json) {
                 const { schemaVersion, release, components, projects } = updateReportJson(report);
-                ctx.stdout(JSON.stringify({ ...result, schemaVersion, release, components, projects, lastProblem: lastProblem(state) }));
+                ctx.stdout(JSON.stringify({ ...result, schemaVersion, release, components, projects, lastProblem: lastProblem(state), drain: state.drain === null ? null : { generation: state.drain.generation, acknowledgedAtMs: state.drain.acknowledgedAtMs }, deferral: state.deferral }));
             }
             else
-                for (const line of [...updateResultLines(result), ...lastProblemLines(state), ...updateReportLines(report)])
+                for (const line of [...updateResultLines(result), ...lastProblemLines(state), ...drainLines(state), ...updateReportLines(report)])
                     ctx.stdout(line);
             return result.exitCode;
         }
@@ -229,12 +257,25 @@ export async function cmdRunner(args, ctx, deps = {}) {
         if (mode === "off" || (mode === "on" && status.state === "on"))
             for (const id of RELEASE_COMPONENTS)
                 setComponentPolicy(ctx.home, id, mode === "on");
+        let updateDrain = "none";
+        if (mode === "off" && !fleet) {
+            let runner = null;
+            try {
+                runner = dir();
+            }
+            catch { /* No runner directory to release. */ }
+            if (runner !== null && existsSync(resolve(runner, ".env")) && existsSync(resolve(runner, "compose.yaml")))
+                updateDrain = await releaseUpdateDrain(deps.engine ?? dockerRunnerUpdateEngine({ env: ctx.env }), runner, { cloud: true });
+        }
         if (args.json) {
             const state = mode === "status" && dirFlag !== undefined ? readReleaseState(dirFlag) : undefined;
-            ctx.stdout(JSON.stringify({ ...status, ...(state ? { release: state } : {}) }));
+            ctx.stdout(JSON.stringify({ ...status, ...(state ? { release: state } : {}), ...(mode === "off" ? { updateDrain } : {}) }));
         }
         else {
             ctx.stdout(statusLine(status));
+            const line = mode === "off" ? drainReleaseLine(updateDrain) : null;
+            if (line !== null)
+                ctx.stdout(line);
             // A single runner's schedule prints what it always has; a fleet's says so, and for which directory.
             if (status.state === "on" && status.fleet)
                 ctx.stdout(`This is a supervisor fleet's schedule: it updates the compose.json in ${status.dir}.`);

package/dist/runner-update-fleet.js

@@ -23,23 +23,45 @@ import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseSt
 // back and the supervisor is recreated on it. That scope set is saved before the recreate and removed
 // once it qualifies or rolls back, so a run killed in between is finished by the next one.
 //
+// A busy host re-adopts its live sessions before it qualifies, which can take longer than four minutes
+// (CTC-8163). Each recreated supervisor, the new release's or the restored previous one's, gets four
+// minutes, extended to 150 seconds past each new step it logs: fleet tenure, then a saved scope
+// re-adopted, then that scope ready. No supervisor gets more than seven minutes. A restart, another
+// image or `host_not_ready` still ends the wait at the next poll. At the deadline each saved scope it
+// has not qualified is reported, with the gate condition its latest log line shows failing.
+//
+// A team can stop being delegated while an update is in flight (CTC-7569): no supervisor, new or old,
+// starts its scope again. The previous release, restored, is the control. When it misses exactly the
+// scopes the new supervisor missed and qualifies the rest, those scopes are dropped from the saved set
+// and the new release is applied again against what is left. A later run that finds a saved set beside
+// the previous release's running supervisor drops the saved scopes that supervisor no longer qualifies.
+// Either way the result names each dropped scope; the new release's own log never drops one.
+//
 // A release can qualify and then stop receiving work minutes later (CTC-6564). A successful apply
 // arms a check on the host, outside the calling process, for 10 minutes later: it reads the supervisor's
 // log since the roll began and, when the release was granted no work and the fleet's acquires got no
 // answer, restores the saved previous `compose.json`. `runner update --fleet --post-roll-check` is
 // that check; only the armed timer runs it.
+// A check that is due while an update holds the lock judges nothing; that update arms it again
+// for 5 minutes after it ends (CTC-7570).
 export const FLEET_COMPOSE_FILE = "compose.json";
 export const PREVIOUS_COMPOSE_FILE = "compose.release-previous.json";
 const CANDIDATE_COMPOSE_FILE = "compose.release-candidate.json";
 /** The scopes the supervisor served before a recreate, kept until that recreate is qualified or rolled
  *  back. A run that was killed in between finds it and qualifies against it, never against the new
- *  container's own log, which would read an unqualified supervisor as serving. */
+ *  container's own log, which would read an unqualified supervisor as serving. Only the previous
+ *  release's supervisor can take a scope out of it (CTC-7569). */
 export const IN_FLIGHT_FILE = "fleet-update-in-flight.json";
 /** How far back a qualification counts as current before the update. The supervisor repeats it
  *  on every loop while a scope is ready, so a ready scope logs it many times in this window. */
 const QUALIFIED_LOOKBACK_MS = 5 * 60_000;
 const QUALIFY_MIN_MS = 90_000;
+/** How long a recreated supervisor that shows no progress has to qualify every saved scope. */
 const QUALIFY_DEADLINE_MS = 240_000;
+/** How long past each new step of progress a recreated supervisor has to qualify (CTC-8163). */
+const QUALIFY_PROGRESS_MS = 150_000;
+/** The most one recreated supervisor is given, however it progresses (CTC-8163). */
+const QUALIFY_CEILING_MS = 420_000;
 const QUALIFY_POLL_MS = 10_000;
 const CHILD_NAME_PREFIX = "catalyst-pa_";
 /** What a successful apply leaves for the check: the bytes it wrote and when the roll began. */
@@ -47,6 +69,8 @@ export const POST_ROLL_ARMED_FILE = "post-roll-armed.json";
 /** The check's last verdict, beside `release-state.json`. */
 export const POST_ROLL_RESULT_FILE = "post-roll-result.json";
 export const POST_ROLL_DELAY_MS = 10 * 60_000;
+/** How long after an update run lets go of the lock a check it kept from judging runs again (CTC-7570). */
+export const POST_ROLL_RETRY_DELAY_MS = 5 * 60_000;
 const POST_ROLL_UNIT = "catalyst-fleet-post-roll";
 /** No-response acquire retries, with no grants, that count a release as failing. */
 const POST_ROLL_MAX_NO_RESPONSE = 3;
@@ -91,14 +115,83 @@ export function qualifiedScopes(text) {
 /** A `host_not_ready` line for one of `scopes`, or one that names no scope. Other scopes may be
  *  unready for their own reasons (a team with no repository yet) and do not stop an update. */
 export function notReadyScope(text, scopes) {
+    return readSupervisorLog(text, scopes).notReady;
+}
+const GATE_CODES = [
+    ["resourcesReady", "resources_ready"],
+    ["scopeEligible", "scope_eligible"],
+    ["readyUntilValid", "ready_until"],
+    ["sessionsReadopted", "sessions_readopted"],
+];
+/** One pass over the log text for everything qualify() reads. Lines for other scopes, and lines that
+ *  are not JSON or lack a field, are skipped. */
+function readSupervisorLog(text, scopes) {
+    const log = { notReady: null, qualified: new Set(), tenure: false, readopted: new Set(), ready: new Set(), gates: new Map() };
     for (const event of logEvents(text)) {
-        if (event.event !== "host_not_ready")
-            continue;
         const key = scopeKey(event);
-        if (key === null || scopes.has(key))
-            return key ?? "unscoped";
+        if (event.event === "host_not_ready" && log.notReady === null && (key === null || scopes.has(key)))
+            log.notReady = key ?? "unscoped";
+        if (event.event === "supervisor_fleet_tenure")
+            log.tenure = true;
+        if (key === null || !scopes.has(key))
+            continue;
+        if (event.event === "supervisor_scope_qualified")
+            log.qualified.add(key);
+        else if (event.event === "supervisor_readopted")
+            log.readopted.add(key);
+        else if (event.event === "host_ready")
+            log.ready.add(key);
+        else if (event.event === "supervisor_scope_qualification_state") {
+            const { resourcesReady, scopeEligible, readyUntilValid, sessionsReadopted } = event;
+            if (typeof resourcesReady === "boolean" && typeof scopeEligible === "boolean" && typeof readyUntilValid === "boolean" && typeof sessionsReadopted === "boolean")
+                log.gates.set(key, { resourcesReady, scopeEligible, readyUntilValid, sessionsReadopted });
+        }
+    }
+    return log;
+}
+/** Count the steps `log` shows that `budget` has not counted yet: fleet tenure, a scope re-adopted, and
+ *  a re-adopted scope ready. A scope ready before its re-adoption shows counts once that does. Any new
+ *  step moves the deadline to `QUALIFY_PROGRESS_MS` past `atMs`, never earlier and never past the ceiling. */
+function countProgress(budget, log, atMs) {
+    let progressed = false;
+    if (log.tenure && !budget.tenure) {
+        budget.tenure = true;
+        progressed = true;
+    }
+    for (const scope of log.readopted) {
+        if (budget.readopted.has(scope))
+            continue;
+        budget.readopted.add(scope);
+        progressed = true;
+    }
+    for (const scope of log.ready) {
+        if (!budget.readopted.has(scope) || budget.ready.has(scope))
+            continue;
+        budget.ready.add(scope);
+        progressed = true;
     }
-    return null;
+    if (progressed)
+        budget.deadline = Math.min(budget.ceiling, Math.max(budget.deadline, atMs + QUALIFY_PROGRESS_MS));
+}
+/** The longest one recreated supervisor is given to qualify, however it progresses. */
+const qualifyCeilingMs = (options) => Math.max(QUALIFY_CEILING_MS, options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS);
+/** A scope as printed: anything outside printable ASCII, and the backslash that starts an escape,
+ *  escaped as `\uXXXX`, and at most 120 characters. */
+const printableScope = (scope) => {
+    const escaped = scope.replace(/[^\x21-\x5b\x5d-\x7e]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`);
+    return escaped.length > 120 ? `${escaped.slice(0, 120)}...` : escaped;
+};
+/** The gate conditions a missing scope's snapshot shows failing, sorted; `diagnostic_unavailable`
+ *  without one, and `qualification_event_missing` when every condition held. */
+function failedGates(snapshot) {
+    if (snapshot === undefined)
+        return ["diagnostic_unavailable"];
+    const failed = GATE_CODES.filter(([field]) => !snapshot[field]).map(([, code]) => code).sort();
+    return failed.length > 0 ? failed : ["qualification_event_missing"];
+}
+/** One line per scope in `missing`, sorted by scope. `gates` null: the log could not be read. */
+function qualificationPendingLines(roll, missing, gates) {
+    return [...missing].sort().map((scope) => `fleet qualification pending: roll=${roll} scope=${printableScope(scope)} failed=${failedGates(gates?.get(scope)).join(",")}`);
 }
 function readInFlight(dir) {
     try {
@@ -112,6 +205,10 @@ function readInFlight(dir) {
         return null;
     }
 }
+/** Save the scopes the next recreate has to qualify, replacing any saved before. */
+function saveInFlight(dir, scopes, atMs) {
+    writePrivateAtomic(join(dir, IN_FLIGHT_FILE), `${JSON.stringify({ version: 1, scopes: [...scopes].sort(), atMs })}\n`);
+}
 function clearInFlight(dir) {
     try {
         unlinkSync(join(dir, IN_FLIGHT_FILE));
@@ -120,6 +217,17 @@ function clearInFlight(dir) {
         // Already gone.
     }
 }
+/** The images a saved `compose.json` names, or null when there is none or it is not a fleet layout. */
+function savedImages(path) {
+    try {
+        return fleetImages(JSON.parse(readFileSync(path, "utf8")));
+    }
+    catch {
+        return null;
+    }
+}
+/** The detail token naming scopes dropped from the requirement, sorted; empty when there are none. */
+const droppedDetail = (scopes) => scopes.length > 0 ? `;dropped_scope:${[...scopes].sort().join(",")}` : "";
 const removeFile = (path) => {
     try {
         unlinkSync(path);
@@ -226,6 +334,23 @@ async function armPostRollCheck(engine, dir, armed, signal) {
         return ";post_roll_check_not_armed";
     }
 }
+/** The armed check runs once, from a one-shot timer. When it was due while an update held the lock, or
+ *  Docker could not be read, it judged nothing and left the armed file; a reboot loses the timer the same
+ *  way (CTC-7570). The update holding the lock arms it again for a few minutes later, before it lets go:
+ *  while the lock is held no check is part-way through a judgment, so the arm's stop of the check's unit
+ *  interrupts nothing. The check cannot arm itself: that stop would end its own unit. A check this run
+ *  armed is not due yet and is left alone. */
+async function rearmOverdueCheck(engine, dir, now, signal) {
+    const armed = readPostRollArmed(dir);
+    if (armed === null || now() < armed.armedAtMs + POST_ROLL_DELAY_MS)
+        return;
+    try {
+        await engine.armPostRollCheck(dir, POST_ROLL_RETRY_DELAY_MS, signal);
+    }
+    catch {
+        // The next run tries again.
+    }
+}
 export async function runFleetUpdate(engine, options) {
     const { dir } = options;
     const now = options.now ?? Date.now;
@@ -243,8 +368,10 @@ export async function runFleetUpdate(engine, options) {
         return result;
     }
     finally {
-        if (lock !== null)
+        if (lock !== null) {
+            await rearmOverdueCheck(engine, dir, now, options.signal);
             releaseLock(dir, lock);
+        }
     }
 }
 async function update(engine, options, now) {
@@ -291,6 +418,8 @@ async function update(engine, options, now) {
     // The scopes the running supervisor qualifies are what the new one has to qualify again, unless an
     // earlier run stopped mid-recreate: then the scopes it saved are the baseline.
     let scopes = inFlight;
+    /** Saved scopes the previous release's supervisor no longer qualifies, named in the result (CTC-7569). */
+    let dropped = [];
     if (scopes === null) {
         if (!live?.running)
             return done("retained", "supervisor_not_running", { ...tagged, exitCode: 1 });
@@ -301,6 +430,25 @@ async function update(engine, options, now) {
         if (scopes.size === 0)
             return done("retained", "no_qualified_scope", { ...tagged, exitCode: 1 });
     }
+    else {
+        // An earlier run that ended back on the previous release, its rollback unqualified, left its scopes
+        // saved. Only when compose.json and the running supervisor are that previous release, up long enough
+        // to have qualified every scope it serves, does its log trim the saved set: a saved scope it no
+        // longer qualifies stopped being delegated. A `host_not_ready` for a saved scope is a scope still
+        // started, not one dropped. Anything less keeps the saved set whole. Long enough is the most any
+        // recreated supervisor is given, since this run cannot see how it progressed (CTC-8163).
+        const previous = savedImages(join(dir, PREVIOUS_COMPOSE_FILE));
+        const onPrevious = previous !== null && current.runner === previous.runner && current.supervisor === previous.supervisor && runs(previous);
+        if (live && onPrevious && now() - Date.parse(live.startedAt) >= qualifyCeilingMs(options)) {
+            const recent = await engine.logs(live.id, new Date(now() - QUALIFIED_LOOKBACK_MS).toISOString(), signal);
+            const serving = recent === null || notReadyScope(recent, scopes) !== null ? new Set() : qualifiedScopes(recent);
+            const kept = [...scopes].filter((scope) => serving.has(scope));
+            if (kept.length > 0 && kept.length < scopes.size) {
+                dropped = [...scopes].filter((scope) => !serving.has(scope));
+                scopes = new Set(kept);
+            }
+        }
+    }
     const record = (outcome, detail, previous, exitCode = 0) => {
         writeReleaseState(dir, {
             ...state,
@@ -353,9 +501,15 @@ async function update(engine, options, now) {
     const rollStartMs = now();
     if (changed.length > 0)
         writePrivateAtomic(join(dir, PREVIOUS_COMPOSE_FILE), composeText);
-    writePrivateAtomic(join(dir, IN_FLIGHT_FILE), `${JSON.stringify({ version: 1, scopes: [...scopes].sort(), atMs: now() })}\n`);
-    writePrivateAtomic(composePath, candidateText);
-    if ((await engine.recreate(dir, project, signal)) && (await qualify(engine, dir, project, target, scopes, options, now))) {
+    saveInFlight(dir, scopes, now());
+    /** Write `text` as compose.json, recreate the supervisor alone on it and qualify it against `required`. */
+    const roll = async (which, text, images, required) => {
+        writePrivateAtomic(composePath, text);
+        if (!(await engine.recreate(dir, project, signal)))
+            return { ok: false, reason: "recreate_failed" };
+        return qualify(engine, dir, project, images, required, options, now, which);
+    };
+    const succeed = async (droppedScopes) => {
         clearInFlight(dir);
         if (changed.length > 0)
             await engine.removeImages(changed.map((c) => current[c]), signal);
@@ -371,20 +525,34 @@ async function update(engine, options, now) {
             version: 1, sinceMs: rollStartMs, composeSha256: sha256(candidateText), generation: release.generation,
             releaseId: release.releaseId, fromGeneration: state.generation, armedAtMs: now(),
         };
-        const detail = (changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled") + await armPostRollCheck(engine, dir, armed, signal);
+        const detail = (changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled") + droppedDetail(droppedScopes) +
+            await armPostRollCheck(engine, dir, armed, signal);
         const previous = changed.length > 0 ? current : null;
         return disturbed === 0
             ? record("updated", detail, previous)
             : record("updated", `${detail};${disturbed < 0 ? "children_unreadable" : `children_changed:${disturbed}`}`, previous, 1);
-    }
+    };
+    const candidate = await roll("candidate", candidateText, target, scopes);
+    if (candidate.ok)
+        return succeed(dropped);
     if (rollbackText === null || rollbackImages === null)
-        return done("failed", "new_supervisor_not_qualified_no_previous", { ...tagged, exitCode: 1 });
-    const restored = (await restoreCompose(engine, dir, project, rollbackText, signal)) &&
-        (await qualify(engine, dir, project, rollbackImages, scopes, options, now));
+        return done("failed", `new_supervisor_not_qualified_no_previous${droppedDetail(dropped)}`, { ...tagged, exitCode: 1 });
+    let restored = await roll("rollback", rollbackText, rollbackImages, scopes);
+    // The previous release is the control (CTC-7569). Missing exactly the scopes the candidate missed while
+    // qualifying the rest, it shows those scopes stopped being delegated, not that the candidate broke them:
+    // the candidate is applied again without them, and recorded only once it qualifies what is left.
+    const gone = lostDelegation(candidate, restored, scopes);
+    if (gone !== null) {
+        const kept = new Set([...scopes].filter((scope) => !gone.has(scope)));
+        saveInFlight(dir, kept, now());
+        if ((await roll("candidate", candidateText, target, kept)).ok)
+            return succeed([...dropped, ...gone]);
+        restored = await roll("rollback", rollbackText, rollbackImages, kept);
+    }
     // A rollback that did not qualify keeps the baseline for the next run.
-    if (restored)
+    if (restored.ok)
         clearInFlight(dir);
-    return done(restored ? "rolled_back" : "failed", restored ? "new_supervisor_not_qualified" : "rollback_not_qualified", { ...tagged, exitCode: 1 });
+    return done(restored.ok ? "rolled_back" : "failed", `${restored.ok ? "new_supervisor_not_qualified" : "rollback_not_qualified"}${droppedDetail(dropped)}`, { ...tagged, exitCode: 1 });
 }
 /** Put `text` back as `compose.json` and recreate the supervisor alone on it. */
 async function restoreCompose(engine, dir, project, text, signal) {
@@ -461,6 +629,11 @@ export async function runFleetPostRollCheck(engine, options) {
         }
         if (previous === null)
             return refuse("no_previous_compose");
+        // An update that started from this release and did not finish saved this release's own file as the
+        // previous one (CTC-7570). Restoring it would move nothing and record a rollback that never happened.
+        const current = fleetImages(compose);
+        if (current !== null && previous.runner === current.runner && previous.supervisor === current.supervisor)
+            return refuse("previous_compose_is_this_release");
         // The apply removed the images it replaced, so the previous ones may need pulling again.
         const unprepared = await engine.prepare([previous.runner, previous.supervisor], signal);
         if (unprepared !== null)
@@ -478,37 +651,69 @@ export async function runFleetPostRollCheck(engine, options) {
     }
 }
 /** The recreated supervisor runs `images`, stays up with no restart, logs no `host_not_ready` for a
- *  scope it must serve, and qualifies every one of `scopes`, held for the minimum window. */
-async function qualify(engine, dir, project, images, scopes, options, now) {
+ *  scope it must serve, and qualifies every one of `scopes`, held for the minimum window. Its deadline
+ *  moves with the progress it logs (CTC-8163); the restart, image and `host_not_ready` exits come first. */
+async function qualify(engine, dir, project, images, scopes, options, now, roll) {
     const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
     const start = now();
-    const deadline = start + (options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS);
+    const budget = {
+        deadline: start + (options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS),
+        ceiling: start + qualifyCeilingMs(options),
+        tenure: false,
+        readopted: new Set(),
+        ready: new Set(),
+    };
     const minimum = start + (options.qualifyMinMs ?? QUALIFY_MIN_MS);
     let id = null;
     for (;;) {
         if (options.signal?.aborted)
-            return false;
+            return { ok: false, reason: "aborted" };
         const live = await engine.supervisor(dir, project, options.signal);
+        // What this poll's log has not qualified yet, and its gate lines; null when the supervisor or its
+        // log could not be read.
+        let missing = null;
+        let gates = null;
         if (live) {
             id ??= live.id;
             if (live.id !== id || !live.running || live.restartCount !== 0)
-                return false;
+                return { ok: false, reason: "restarted" };
             if (live.image !== images.supervisor || live.runnerImage !== images.runner)
-                return false;
+                return { ok: false, reason: "wrong_image" };
             const text = await engine.logs(live.id, live.startedAt, options.signal);
             if (text !== null) {
-                if (notReadyScope(text, scopes) !== null)
-                    return false;
-                const seen = qualifiedScopes(text);
-                if (now() >= minimum && [...scopes].every((scope) => seen.has(scope)))
-                    return true;
+                const log = readSupervisorLog(text, scopes);
+                if (log.notReady !== null)
+                    return { ok: false, reason: "host_not_ready" };
+                missing = new Set([...scopes].filter((scope) => !log.qualified.has(scope)));
+                if (now() >= minimum && missing.size === 0)
+                    return { ok: true };
+                countProgress(budget, log, now());
+                gates = log.gates;
+            }
+        }
+        if (now() >= budget.deadline) {
+            for (const line of qualificationPendingLines(roll, missing ?? scopes, gates)) {
+                try {
+                    options.onQualificationPending?.(line);
+                }
+                catch {
+                    // A line that could not be written never holds up the other lines or the restore.
+                }
             }
+            return missing !== null && missing.size > 0 ? { ok: false, reason: "missing", missing } : { ok: false, reason: "deadline" };
         }
-        if (now() >= deadline)
-            return false;
         await sleep(QUALIFY_POLL_MS);
     }
 }
+/** The scopes to drop when the candidate and the previous release, its control, both failed only by
+ *  missing scopes, the same ones, and the control qualified at least one other `required` scope. Null
+ *  otherwise: then the candidate is what failed, or nothing shows which. */
+function lostDelegation(candidate, control, required) {
+    if (candidate.ok || control.ok || candidate.reason !== "missing" || control.reason !== "missing")
+        return null;
+    const same = candidate.missing.size === control.missing.size && [...candidate.missing].every((scope) => control.missing.has(scope));
+    return same && control.missing.size < required.size ? control.missing : null;
+}
 const CONTAINER_ID = /^[0-9a-f]{12,64}$/;
 /** The one volume or bind in `service.volumes` that holds `file`, as a read-only `--mount` value. */
 export function credentialMount(compose, service, file) {