@caelo-cms/provisioning 0.10.55 → 0.10.57

package/dist/upgrader-job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upgrader-job.d.ts","sourceRoot":"","sources":["../src/upgrader-job.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAmB,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAEpF,OAAO,EACL,KAAK,WAAW,EAKjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,EAAiB,KAAK,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAEpE,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAC;IACxC,mFAAmF;IACnF,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAC;CACrC;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IACpC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IACtC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,2BAA2B;AAC3B,MAAM,MAAM,eAAe,GACvB,WAAW,GACX,aAAa,GACb,QAAQ,GACR,SAAS,GACT,mBAAmB,CAAC;AAExB,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,sEAAsE;IACtE,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;CACpC;AAED,qFAAqF;AACrF,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,KAAK,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAC/D,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,wBAAwB,EAC/B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D;;;OAGG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,+DAA+D;IAC/D,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC5E;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,MAAM,IAAI,CAAC;IACrC,qEAAqE;IACrE,QAAQ,CAAC,WAAW,EAAE,MAAM,eAAe,GAAG,IAAI,CAAC;IACnD,QAAQ,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IACvD,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,EAAE,CAChB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE,QAAQ,CAAC,WAAW,EAAE,CACpB,IAAI,EAAE,WAAW,KACd,OAAO,CACR;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;KAAE,GAClD;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CACjD,CAAC;IACF,QAAQ,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;CACzC;AASD,6EAA6E;AAC7E,wBAAsB,cAAc,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA+QtF;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,SAAS,EACd,GAAG,EAAE,MAAM,EACX,KAAK,GAAE,KAAoB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAclB"}
+{"version":3,"file":"upgrader-job.d.ts","sourceRoot":"","sources":["../src/upgrader-job.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAmB,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAEpF,OAAO,EACL,KAAK,WAAW,EAKjB,MAAM,uBAAuB,CAAC;AAO/B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,EACL,KAAK,SAAS,EAKf,MAAM,6BAA6B,CAAC;AAMrC,OAAO,EAAiB,KAAK,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAEpE,4EAA4E;AAC5E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAC;IACxC,mFAAmF;IACnF,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAC;CACrC;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IACpC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IACtC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,2BAA2B;AAC3B,MAAM,MAAM,eAAe,GACvB,WAAW,GACX,aAAa,GACb,QAAQ,GACR,SAAS,GACT,mBAAmB,CAAC;AAExB,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,sEAAsE;IACtE,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;CACpC;AAED,qFAAqF;AACrF,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,KAAK,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAC/D,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,wBAAwB,EAC/B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D;;;OAGG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC;IACxB,+DAA+D;IAC/D,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC5E;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACvE,qFAAqF;IACrF,QAAQ,CAAC,WAAW,EAAE,MAAM,eAAe,GAAG,IAAI,CAAC;IACnD,QAAQ,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IACvD,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,EAAE,CAChB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE,QAAQ,CAAC,WAAW,EAAE,CACpB,IAAI,EAAE,WAAW,KACd,OAAO,CACR;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;KAAE,GAClD;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CACjD,CAAC;IACF,QAAQ,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;CACzC;AAyBD,6EAA6E;AAC7E,wBAAsB,cAAc,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAgZtF;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,SAAS,EACd,GAAG,EAAE,MAAM,EACX,KAAK,GAAE,KAAoB,GAC1B,OAAO,CAAC,OAAO,CAAC,CAclB"}

package/dist/upgrader-job.js

@@ -3,11 +3,19 @@ import { gcpNamePrefix, upgraderJobName } from "./gcp-names.js";
 import { StateLockHeldError } from "./state-store.js";
 import { executeAppRoll, planAppRoll, } from "./upgrade/app-roll.js";
 import { restoreCommand, takeSqlBackup } from "./upgrade/backup.js";
-import { findServingImageDigest, resolveGcpResourceName } from "./upgrade/gcp-lookup.js";
+import { findServingImage, resolveGcpResourceName, servingImageDigests, } from "./upgrade/gcp-lookup.js";
 import { releaseImageRef } from "./upgrade/release-images.js";
 import { assessUpgradeRequest } from "./upgrader-assess.js";
+import { preflightErrorReason, preflightMissingReason, runIamPreflight, upgraderAccessChecks, } from "./upgrader-iam-preflight.js";
+import { findServingRelease, recordMatchesServing, } from "./upgrader-records.js";
 import { eventReporter } from "./upgrader-reporter.js";
 const ROLLED_PHASES = new Set(["roll", "health"]);
+/**
+ * Lock outcomes that mean another run is changing the install right now
+ * (#815): a verdict judged from serving images read without the lock may
+ * describe that run's half-finished roll, so it is not reported.
+ */
+const STALE_VERDICT = new Set(["state-locked", "install-state-changed"]);
 /** Attempts at recording a request's end state before the run fails. */
 const FINISH_ATTEMPTS = 3;
 const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -58,27 +66,98 @@ export async function runUpgraderJob(deps) {
     try {
         r.phase("start");
         r.info(`Upgrader started${deps.execution ? ` (run ${deps.execution})` : ""}: installing Caelo ${request.targetVersion}.`);
-        // Exclusive against the installer's own state writers for the whole run.
-        try {
-            releaseLock = deps.lockState();
-        }
-        catch (e) {
-            if (e instanceof StateLockHeldError) {
-                return stopWith("failed", ["state-locked"], `Nothing was changed: another install operation is running on this site (${errorText(e)}). Try again when it has finished.`);
+        // #815 — the state lock is taken only right before the first write
+        // (lockBeforeWrite below), never before the decision: any lock problem
+        // must not hide the verdict (needs Cloud Shell, refused), and a run
+        // that changes nothing has no business blocking another.
+        const lockBeforeWrite = () => {
+            try {
+                releaseLock = deps.lockState((message) => r.info(message));
+                return { ok: true };
             }
-            // Not contention: the job cannot reach its own ops bucket. Retrying
-            // will not help; a Cloud Shell upgrade re-converges the job's grants.
-            return stopWith("needs_cloud_shell", ["state-access"], `Nothing was changed: the upgrader could not take the install's state lock (${errorText(e)}). Run the upgrade once with Cloud Shell; it repairs the upgrader's access.`);
-        }
+            catch (e) {
+                if (e instanceof StateLockHeldError) {
+                    return {
+                        ok: false,
+                        status: "failed",
+                        codes: ["state-locked"],
+                        reason: `Nothing was changed: another install operation is running on this site (${errorText(e)}). Try again when it has finished.`,
+                    };
+                }
+                // Not contention: the job cannot reach its own ops bucket or ask
+                // Cloud Run about the holder. Retrying will not help; a Cloud Shell
+                // upgrade re-converges the job's grants.
+                return {
+                    ok: false,
+                    status: "needs_cloud_shell",
+                    codes: ["state-access"],
+                    reason: `Nothing was changed: the upgrader could not take the install's state lock (${errorText(e)}). Run the upgrade once with Cloud Shell; it repairs the upgrader's access.`,
+                };
+            }
+        };
         // 2. What the install runs — from its own state, never from the admin.
         r.phase("verify");
-        const meta = deps.readInstall();
-        if (!meta || meta.projectId !== deps.projectId) {
+        const read = deps.readInstall();
+        if (!read || read.projectId !== deps.projectId) {
             return stopWith("needs_cloud_shell", ["install-state"], "The install's records (install.json) are not in its own cloud project yet, so the upgrader cannot tell what it runs. Run the upgrade once with Cloud Shell; it moves them there.");
         }
-        if (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") {
-            return stopWith("refused", ["provider"], `The upgrader job only upgrades Google Cloud installs; this one is ${meta.provider}.`);
+        if (read.provider !== "gcp" && read.provider !== "gcp-firebase") {
+            return stopWith("refused", ["provider"], `The upgrader job only upgrades Google Cloud installs; this one is ${read.provider}.`);
         }
+        // Rebound when the records are refreshed below; `provider` is narrowed.
+        let meta = {
+            ...read,
+            provider: read.provider,
+        };
+        /** The corrected records (#860), when install.json is stale; written under the lock. */
+        let heal = null;
+        /**
+         * Take the lock, confirm install.json is still what was judged, and
+         * write the pending correction. Returns null when all of that held.
+         */
+        const lockAndHeal = () => {
+            const locked = lockBeforeWrite();
+            if (!locked.ok)
+                return locked;
+            // Another writer may have changed the records between our read and
+            // the lock; everything above was judged against the old ones.
+            // A failed re-read is a Blocked like any other, never a throw: on the
+            // stop-verdict path a throw here would replace the verdict with
+            // failed/internal.
+            let now;
+            try {
+                now = deps.readInstall();
+            }
+            catch (e) {
+                return {
+                    status: "failed",
+                    codes: ["install-state-read"],
+                    reason: `Nothing was changed: the install's records (install.json) could not be read again once the lock was held (${errorText(e)}). Try again.`,
+                };
+            }
+            if (JSON.stringify(now) !== JSON.stringify(read)) {
+                return {
+                    status: "failed",
+                    codes: ["install-state-changed"],
+                    reason: "Nothing was changed: another install operation changed the install's records (install.json) while this update was being checked. Try again.",
+                };
+            }
+            if (!heal)
+                return null;
+            const v = heal.releaseVersion;
+            try {
+                deps.writeInstall(heal);
+            }
+            catch (e) {
+                return {
+                    status: "failed",
+                    codes: ["install-state-write"],
+                    reason: `Nothing was changed: the install's records (install.json) name another release than the one that serves (v${v}), and they could not be corrected (${errorText(e)}).`,
+                };
+            }
+            r.success(`Caelo refreshed its install records: v${v} serves.`);
+            return null;
+        };
         // install.json must describe what actually serves: if a previous run
         // rolled but could not record it, the recorded version is stale and
         // the downgrade / minFromVersion checks would judge against it.
@@ -86,11 +165,35 @@ export async function runUpgraderJob(deps) {
         if (!serving.ok) {
             return stopWith("failed", ["lookup"], `Nothing was changed: ${serving.error}.`);
         }
-        const recorded = meta.imageDigests;
-        if (!recorded ||
-            recorded.admin !== serving.digests.admin ||
-            recorded.gateway !== serving.digests.gateway) {
-            return stopWith("needs_cloud_shell", ["install-state-drift"], "The release that serves now is not the one the install's records (install.json) name, so the upgrader cannot tell safely what it runs. Run the upgrade once with Cloud Shell; it records the serving release again.");
+        if (!recordMatchesServing(meta.imageDigests, serving.digests)) {
+            // #860 — re-record the serving release ourselves, but only one whose
+            // admin AND gateway images a signed release manifest names.
+            const found = await findServingRelease(meta, serving.digests, deps);
+            if (!found.ok) {
+                if (found.retryable)
+                    return stopWith("failed", ["registry"], found.reason);
+                // #815 — the serving images were read without the lock: a run that
+                // holds it may be half-way through its roll, and that is not drift.
+                // Only once this run holds the lock (or cannot take it at all) is
+                // the mismatch the install's real state.
+                const blocked = lockAndHeal();
+                if (blocked && STALE_VERDICT.has(blocked.codes[0] ?? "")) {
+                    return stopWith(blocked.status, blocked.codes, blocked.reason);
+                }
+                return stopWith("needs_cloud_shell", ["install-state-drift"], found.reason);
+            }
+            // The manifest's digests, not the serving ones: the signed form is the
+            // one every later comparison and deploy uses (gcp-lookup.ts ServingImage).
+            meta = {
+                ...meta,
+                imageDigests: {
+                    admin: found.manifest.digests.admin,
+                    gateway: found.manifest.digests.gateway,
+                },
+                releaseVersion: found.manifest.version,
+            };
+            // Judged against now, written under the lock once decided (#815).
+            heal = meta;
         }
         // 3 + 4. The signed manifest decides.
         const manifest = await deps.fetchManifest(request.targetVersion);
@@ -111,12 +214,38 @@ export async function runUpgraderJob(deps) {
         }
         if (decision.kind === "stop") {
             r.phase("assess");
+            if (heal) {
+                // The records still get corrected, but the verdict is the answer:
+                // a lock or write problem here must not replace it (#815).
+                // Except when another run holds the lock or changed the records:
+                // then the drift this verdict was judged on may be that run's
+                // half-finished work, and "try again once it has finished" is the
+                // true answer.
+                const blocked = lockAndHeal();
+                if (blocked && STALE_VERDICT.has(blocked.codes[0] ?? "")) {
+                    return stopWith(blocked.status, blocked.codes, blocked.reason);
+                }
+                if (blocked) {
+                    r.warn(`The install's records could not be corrected this time: ${blocked.reason}`);
+                }
+            }
             return stopWith(decision.status, decision.codes, decision.reason);
         }
         r.phase("assess");
         for (const w of decision.warnings)
             r.warn(w);
         r.info(`${meta.releaseVersion} → ${decision.manifest.version}: no cloud-setup change needed; installing.`);
+        // #818 — decided to run: ask IAM for every permission the rest needs
+        // before the lock, the backup or the roll. A grant that looks right but
+        // cannot match (#799) ends the run here, all of them named, not halfway.
+        const access = await runIamPreflight(upgraderAccessChecks({ projectId: deps.projectId, region: deps.region, env: deps.env }), deps.iam);
+        if (access.kind === "missing") {
+            return stopWith("needs_cloud_shell", ["iam-preflight"], preflightMissingReason(access.missing));
+        }
+        if (access.kind === "error") {
+            return stopWith("failed", ["iam-preflight"], preflightErrorReason(access.errors));
+        }
+        r.info("The upgrader has every permission this update needs.");
         const m = decision.manifest;
         const images = {
             admin: { digest: m.digests.admin, imageRef: releaseImageRef("admin", m.digests.admin) },
@@ -138,6 +267,11 @@ export async function runUpgraderJob(deps) {
         }
         for (const w of planned.plan.warnings)
             r.warn(w);
+        // Decided, planned: the backup (or the records correction) is the first
+        // write. Held through the roll, install.json and the self-update.
+        const blocked = lockAndHeal();
+        if (blocked)
+            return stopWith(blocked.status, blocked.codes, blocked.reason);
         // 5. No backup, no change.
         r.phase("backup");
         const sBak = r.step("Backing up the database before anything changes...");
@@ -153,6 +287,7 @@ export async function runUpgraderJob(deps) {
             return stopWith("failed", ["health-check"], `The health check could not be prepared, so nothing was changed: ${probe.error}`);
         }
         // 6 + 7. The roll; it rolls back by itself on failure.
+        let recordError = null;
         const rolled = await executeAppRoll({
             plan: planned.plan,
             backup: backup.backup,
@@ -164,6 +299,9 @@ export async function runUpgraderJob(deps) {
             reporter: r,
             migrate: deps.migrate,
             health: probe.probe,
+            // The roll already succeeded, so a failed write must not undo it;
+            // it is held here and ends the run as failed below (no fallbacks:
+            // never "succeeded" while the records name the previous release).
             recordDigests: async (digests) => {
                 r.phase("record");
                 try {
@@ -172,7 +310,8 @@ export async function runUpgraderJob(deps) {
                     r.info(`Recorded ${m.version} as this install's release.`);
                 }
                 catch (e) {
-                    r.warn(`The new release serves, but install.json could not be updated (${errorText(e)}). The next update will stop and ask for one upgrade with Cloud Shell, which records it.`);
+                    recordError = errorText(e);
+                    r.error(`The new release serves, but install.json could not be updated (${recordError}).`);
                 }
             },
         });
@@ -185,6 +324,9 @@ export async function runUpgraderJob(deps) {
             }
             return stopWith("failed", [rolled.phase], `Stopped during ${rolled.phase === "migrations" ? "the database migrations" : "the pre-check"}; the previous release keeps serving.${restore}`);
         }
+        if (recordError !== null) {
+            return stopWith("failed", ["record"], `Caelo ${m.version} serves and passed its checks, but the install's records (install.json) could not be updated (${recordError}), so they still name the previous release. The next update corrects them by itself once the upgrader can write its ops bucket again; check the upgrader's access to it.`);
+        }
         // 8. Self-update, last: the next run uses this release's upgrader.
         r.phase("self-update");
         const job = upgraderJobName(deps.env);
@@ -233,18 +375,18 @@ export async function recordUpdateEnd(queue, requestId, end, log, sleep = defaul
     }
     return false;
 }
-/** The digests the revisions serving traffic run, per service. */
+/** Every digest the revisions serving traffic are known by, per service. */
 async function servingDigests(deps) {
     const digests = {};
     for (const slug of ["admin", "gateway"]) {
         const prefix = `${gcpNamePrefix(deps.env)}-${slug}`;
         const service = await resolveGcpResourceName(deps.run, "run-service", prefix, deps.projectId, deps.region);
-        const digest = service
-            ? await findServingImageDigest(deps.run, deps.projectId, deps.region, service)
+        const image = service
+            ? await findServingImage(deps.run, deps.projectId, deps.region, service)
             : null;
-        if (!digest)
+        if (!image)
             return { ok: false, error: `could not read which image the ${slug} serves` };
-        digests[slug] = digest;
+        digests[slug] = servingImageDigests(image);
     }
     return { ok: true, digests: digests };
 }

package/dist/upgrader-job.js.map

@@ -1 +1 @@
-{"version":3,"file":"upgrader-job.js","sourceRoot":"","sources":["../src/upgrader-job.ts"],"names":[],"mappings":"AAAA,mCAAmC;AA0CnC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAEL,cAAc,EACd,WAAW,GAGZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAEzF,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAoB,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAe,MAAM,wBAAwB,CAAC;AAqGpE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElD,wEAAwE;AACxE,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,YAAY,GAAU,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEtF,6EAA6E;AAC7E,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAqB;IACxD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5B,IAAI,OAA6B,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,mCAAmC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,GAAG,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,0CAA0C,EAAE,CAAC,CAAC;QAC/E,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC;IACxB,MAAM,CAAC,GAAG,aAAa,CACrB,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EACzE,GAAG,EACH,OAAO,CAAC,EAAE,CACX,CAAC;IACF,IAAI,WAAW,GAAwB,IAAI,CAAC;IAC5C,MAAM,MAAM,GAAG,KAAK,EAAE,GAAc,EAA8B,EAAE;QAClE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,CAAC,CAAC,OAAO,CAAC,SAAS,OAAO,CAAC,aAAa,gBAAgB,CAAC,CAAC;;YACrF,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,WAAW,EAAE,CAAC;YAChB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,aAAa,IAAI,CAAC,QAAQ,CAAC;QACpF,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IACjF,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,CACf,MAAuB,EACvB,KAAwC,EACxC,MAAc,EACd,EAAE,CACF,MAAM,CAAC;QACL,MAAM;QACN,MAAM;QACN,KAAK;QACL,gBAAgB,EACd,OAAO,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,aAAa,CAAC;KAClF,CAAC,CAAC;IAEL,IAAI,CAAC;QACH,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,IAAI,CACJ,mBAAmB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,sBAAsB,OAAO,CAAC,aAAa,GAAG,CAClH,CAAC;QAEF,yEAAyE;QACzE,IAAI,CAAC;YACH,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,kBAAkB,EAAE,CAAC;gBACpC,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,cAAc,CAAC,EAChB,2EAA2E,SAAS,CAAC,CAAC,CAAC,oCAAoC,CAC5H,CAAC;YACJ,CAAC;YACD,oEAAoE;YACpE,sEAAsE;YACtE,OAAO,QAAQ,CACb,mBAAmB,EACnB,CAAC,cAAc,CAAC,EAChB,8EAA8E,SAAS,CAAC,CAAC,CAAC,6EAA6E,CACxK,CAAC;QACJ,CAAC;QAED,uEAAuE;QACvE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YAC/C,OAAO,QAAQ,CACb,mBAAmB,EACnB,CAAC,eAAe,CAAC,EACjB,kLAAkL,CACnL,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;YAChE,OAAO,QAAQ,CACb,SAAS,EACT,CAAC,UAAU,CAAC,EACZ,qEAAqE,IAAI,CAAC,QAAQ,GAAG,CACtF,CAAC;QACJ,CAAC;QAED,qEAAqE;QACrE,oEAAoE;QACpE,gEAAgE;QAChE,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,wBAAwB,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;QACnC,IACE,CAAC,QAAQ;YACT,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,OAAO,CAAC,KAAK;YACxC,QAAQ,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,OAAO,EAC5C,CAAC;YACD,OAAO,QAAQ,CACb,mBAAmB,EACnB,CAAC,qBAAqB,CAAC,EACvB,qNAAqN,CACtN,CAAC;QACJ,CAAC;QAED,sCAAsC;QACtC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,OAAO;YACP,QAAQ;YACR,SAAS,EAAE;gBACT,OAAO,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;gBACpC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,IAAI,CAAC;aACrD;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;SAChB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACnC,IAAI,QAAQ,EAAE,CAAC;YACb,CAAC,CAAC,OAAO,CACP,0BAA0B,QAAQ,CAAC,OAAO,gDAAgD,CAC3F,CAAC;YACF,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClB,OAAO,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpE,CAAC;QACD,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ;YAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,IAAI,CACJ,GAAG,IAAI,CAAC,cAAc,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,6CAA6C,CACnG,CAAC;QAEF,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,GAAmC;YAC7C,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACvF,OAAO,EAAE;gBACP,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;gBACzB,QAAQ,EAAE,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;aACxD;SACF,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,WAAW,CAC/B;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,aAAa,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE;SAC5D,EACD,MAAM,EACN,IAAI,CAAC,GAAG,CACT,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,wBAAwB,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ;YAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEjD,2BAA2B;QAC3B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,MAAM,aAAa,CAChC,IAAI,CAAC,GAAG,EACR,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAC5C,kBAAkB,CAAC,CAAC,OAAO,EAAE,EAC7B,IAAI,CAAC,GAAG,CACT,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,kBAAkB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9C,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,QAAQ,CAAC,EACV,mEAAmE,MAAM,CAAC,KAAK,qBAAqB,CACrG,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,MAAM,CAAC,QAAQ,SAAS,CAAC,CAAC;QACrD,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,cAAc,CAAC,EAChB,mEAAmE,KAAK,CAAC,KAAK,EAAE,CACjF,CAAC;QACJ,CAAC;QAED,uDAAuD;QACvD,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC;YACE,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa,EAAE,CAAC,CAAC,OAAO;YACxB,kBAAkB,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ;SAC1C,EACD,EAAE,EACF;YACE,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,KAAK,CAAC,KAAK;YACnB,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAClB,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC;oBACzC,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBAClF,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,6BAA6B,CAAC,CAAC;gBAC7D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,CAAC,CAAC,IAAI,CACJ,kEAAkE,SAAS,CAAC,CAAC,CAAC,0FAA0F,CACzK,CAAC;gBACJ,CAAC;YACH,CAAC;SACF,CACF,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ;gBAC7B,CAAC,CAAC,wBAAwB,MAAM,CAAC,MAAM,CAAC,QAAQ,6BAA6B,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;gBAC9G,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpC,OAAO,QAAQ,CACb,aAAa,EACb,CAAC,MAAM,CAAC,KAAK,CAAC,EACd,SAAS,CAAC,CAAC,OAAO,kEAAkE,OAAO,EAAE,CAC9F,CAAC;YACJ,CAAC;YACD,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,MAAM,CAAC,KAAK,CAAC,EACd,kBAAkB,MAAM,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,eAAe,wCAAwC,OAAO,EAAE,CAC/I,CAAC;QACJ,CAAC;QAED,mEAAmE;QACnE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACvB,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;YAC1B,KAAK;YACL,MAAM;YACN,QAAQ;YACR,GAAG;YACH,WAAW,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAClD,YAAY,IAAI,CAAC,MAAM,EAAE;YACzB,aAAa,IAAI,CAAC,SAAS,EAAE;YAC7B,SAAS;SACV,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,EAAE;YAAE,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;aAC7D,CAAC;YACJ,CAAC,CAAC,IAAI,CACJ,yCAAyC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,2DAA2D,CACrI,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,UAAU,CAAC,EACZ,sCAAsC,SAAS,CAAC,CAAC,CAAC,uEAAuE,CAC1H,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAkB,EAClB,SAAiB,EACjB,GAAc,EACd,GAAW,EACX,QAAe,YAAY;IAE3B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,eAAe,EAAE,OAAO,EAAE,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,GAAG,CAAC;gBACF,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,uCAAuC,SAAS,aAAa,OAAO,IAAI,eAAe,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;aACrH,CAAC,CAAC;YACH,IAAI,OAAO,GAAG,eAAe;gBAAE,MAAM,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,cAAc,CAC3B,IAAqB;IAErB,MAAM,OAAO,GAAsC,EAAE,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,SAAS,CAAU,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,sBAAsB,CAC1C,IAAI,CAAC,GAAG,EACR,aAAa,EACb,MAAM,EACN,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,MAAM,MAAM,GAAG,OAAO;YACpB,CAAC,CAAC,MAAM,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;YAC9E,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,IAAI,SAAS,EAAE,CAAC;QAC1F,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAmC,EAAE,CAAC;AACpE,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAAkB,EAClB,SAAiB,EACjB,CAAkB,EAClB,IAAqB;IAErB,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE;QAC5B,gBAAgB,EAAE,CAAC,CAAC,QAAQ;QAC5B,kBAAkB,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ;QACzC,qBAAqB,EAAE,CAAC,CAAC,oBAAoB;QAC7C,sBAAsB,EAAE,IAAI,CAAC,oBAAoB,IAAI,CAAC;QACtD,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK;QAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;QAChC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG;KACzB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IAC3B,OAAO,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC"}
+{"version":3,"file":"upgrader-job.js","sourceRoot":"","sources":["../src/upgrader-job.ts"],"names":[],"mappings":"AAAA,mCAAmC;AA0DnC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAEL,cAAc,EACd,WAAW,GAGZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAoB,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAEL,oBAAoB,EACpB,sBAAsB,EACtB,eAAe,EACf,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,kBAAkB,EAClB,oBAAoB,GAErB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,aAAa,EAAe,MAAM,wBAAwB,CAAC;AAqHpE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElD;;;;GAIG;AACH,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC,CAAC;AAE9F,wEAAwE;AACxE,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,YAAY,GAAU,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEtF,6EAA6E;AAC7E,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAqB;IACxD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5B,IAAI,OAA6B,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,mCAAmC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,GAAG,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,0CAA0C,EAAE,CAAC,CAAC;QAC/E,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC;IACxB,MAAM,CAAC,GAAG,aAAa,CACrB,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EACzE,GAAG,EACH,OAAO,CAAC,EAAE,CACX,CAAC;IACF,IAAI,WAAW,GAAwB,IAAI,CAAC;IAC5C,MAAM,MAAM,GAAG,KAAK,EAAE,GAAc,EAA8B,EAAE;QAClE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,CAAC,CAAC,OAAO,CAAC,SAAS,OAAO,CAAC,aAAa,gBAAgB,CAAC,CAAC;;YACrF,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,WAAW,EAAE,CAAC;YAChB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,aAAa,IAAI,CAAC,QAAQ,CAAC;QACpF,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IACjF,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,CACf,MAAuB,EACvB,KAAwC,EACxC,MAAc,EACd,EAAE,CACF,MAAM,CAAC;QACL,MAAM;QACN,MAAM;QACN,KAAK;QACL,gBAAgB,EACd,OAAO,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,aAAa,CAAC;KAClF,CAAC,CAAC;IAEL,IAAI,CAAC;QACH,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,IAAI,CACJ,mBAAmB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,sBAAsB,OAAO,CAAC,aAAa,GAAG,CAClH,CAAC;QAEF,mEAAmE;QACnE,uEAAuE;QACvE,oEAAoE;QACpE,yDAAyD;QACzD,MAAM,eAAe,GAAG,GAAgB,EAAE;YACxC,IAAI,CAAC;gBACH,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3D,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YACtB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,kBAAkB,EAAE,CAAC;oBACpC,OAAO;wBACL,EAAE,EAAE,KAAK;wBACT,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,CAAC,cAAc,CAAC;wBACvB,MAAM,EAAE,2EAA2E,SAAS,CAAC,CAAC,CAAC,oCAAoC;qBACpI,CAAC;gBACJ,CAAC;gBACD,iEAAiE;gBACjE,oEAAoE;gBACpE,yCAAyC;gBACzC,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,MAAM,EAAE,mBAAmB;oBAC3B,KAAK,EAAE,CAAC,cAAc,CAAC;oBACvB,MAAM,EAAE,8EAA8E,SAAS,CAAC,CAAC,CAAC,6EAA6E;iBAChL,CAAC;YACJ,CAAC;QACH,CAAC,CAAC;QAEF,uEAAuE;QACvE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YAC/C,OAAO,QAAQ,CACb,mBAAmB,EACnB,CAAC,eAAe,CAAC,EACjB,kLAAkL,CACnL,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;YAChE,OAAO,QAAQ,CACb,SAAS,EACT,CAAC,UAAU,CAAC,EACZ,qEAAqE,IAAI,CAAC,QAAQ,GAAG,CACtF,CAAC;QACJ,CAAC;QACD,wEAAwE;QACxE,IAAI,IAAI,GAA2D;YACjE,GAAG,IAAI;YACP,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC;QACF,wFAAwF;QACxF,IAAI,IAAI,GAA2B,IAAI,CAAC;QAExC;;;WAGG;QACH,MAAM,WAAW,GAAG,GAAmB,EAAE;YACvC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,EAAE;gBAAE,OAAO,MAAM,CAAC;YAC9B,mEAAmE;YACnE,8DAA8D;YAC9D,sEAAsE;YACtE,gEAAgE;YAChE,mBAAmB;YACnB,IAAI,GAA2B,CAAC;YAChC,IAAI,CAAC;gBACH,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO;oBACL,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,CAAC,oBAAoB,CAAC;oBAC7B,MAAM,EAAE,6GAA6G,SAAS,CAAC,CAAC,CAAC,eAAe;iBACjJ,CAAC;YACJ,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjD,OAAO;oBACL,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,CAAC,uBAAuB,CAAC;oBAChC,MAAM,EACJ,6IAA6I;iBAChJ,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YACvB,MAAM,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC;YAC9B,IAAI,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO;oBACL,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,CAAC,qBAAqB,CAAC;oBAC9B,MAAM,EAAE,6GAA6G,CAAC,uCAAuC,SAAS,CAAC,CAAC,CAAC,IAAI;iBAC9K,CAAC;YACJ,CAAC;YACD,CAAC,CAAC,OAAO,CAAC,yCAAyC,CAAC,UAAU,CAAC,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QAEF,qEAAqE;QACrE,oEAAoE;QACpE,gEAAgE;QAChE,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,wBAAwB,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9D,qEAAqE;YACrE,4DAA4D;YAC5D,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACpE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;gBACd,IAAI,KAAK,CAAC,SAAS;oBAAE,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC3E,mEAAmE;gBACnE,oEAAoE;gBACpE,kEAAkE;gBAClE,yCAAyC;gBACzC,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC9B,IAAI,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;oBACzD,OAAO,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;gBACjE,CAAC;gBACD,OAAO,QAAQ,CAAC,mBAAmB,EAAE,CAAC,qBAAqB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9E,CAAC;YACD,uEAAuE;YACvE,2EAA2E;YAC3E,IAAI,GAAG;gBACL,GAAG,IAAI;gBACP,YAAY,EAAE;oBACZ,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK;oBACnC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO;iBACxC;gBACD,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;aACvC,CAAC;YACF,kEAAkE;YAClE,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;QAED,sCAAsC;QACtC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,OAAO;YACP,QAAQ;YACR,SAAS,EAAE;gBACT,OAAO,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;gBACpC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,IAAI,CAAC;aACrD;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;SAChB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACnC,IAAI,QAAQ,EAAE,CAAC;YACb,CAAC,CAAC,OAAO,CACP,0BAA0B,QAAQ,CAAC,OAAO,gDAAgD,CAC3F,CAAC;YACF,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClB,IAAI,IAAI,EAAE,CAAC;gBACT,kEAAkE;gBAClE,2DAA2D;gBAC3D,iEAAiE;gBACjE,8DAA8D;gBAC9D,kEAAkE;gBAClE,eAAe;gBACf,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC9B,IAAI,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;oBACzD,OAAO,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;gBACjE,CAAC;gBACD,IAAI,OAAO,EAAE,CAAC;oBACZ,CAAC,CAAC,IAAI,CAAC,2DAA2D,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;YACD,OAAO,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpE,CAAC;QACD,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ;YAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,IAAI,CACJ,GAAG,IAAI,CAAC,cAAc,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,6CAA6C,CACnG,CAAC;QAEF,qEAAqE;QACrE,wEAAwE;QACxE,yEAAyE;QACzE,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,oBAAoB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EACvF,IAAI,CAAC,GAAG,CACT,CAAC;QACF,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,QAAQ,CACb,mBAAmB,EACnB,CAAC,eAAe,CAAC,EACjB,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC,eAAe,CAAC,EAAE,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QACD,CAAC,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;QAE/D,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,GAAmC;YAC7C,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACvF,OAAO,EAAE;gBACP,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;gBACzB,QAAQ,EAAE,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;aACxD;SACF,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,WAAW,CAC/B;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,aAAa,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE;SAC5D,EACD,MAAM,EACN,IAAI,CAAC,GAAG,CACT,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,wBAAwB,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ;YAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEjD,wEAAwE;QACxE,kEAAkE;QAClE,MAAM,OAAO,GAAG,WAAW,EAAE,CAAC;QAC9B,IAAI,OAAO;YAAE,OAAO,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE5E,2BAA2B;QAC3B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,MAAM,aAAa,CAChC,IAAI,CAAC,GAAG,EACR,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAC5C,kBAAkB,CAAC,CAAC,OAAO,EAAE,EAC7B,IAAI,CAAC,GAAG,CACT,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,kBAAkB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9C,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,QAAQ,CAAC,EACV,mEAAmE,MAAM,CAAC,KAAK,qBAAqB,CACrG,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,MAAM,CAAC,QAAQ,SAAS,CAAC,CAAC;QACrD,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,cAAc,CAAC,EAChB,mEAAmE,KAAK,CAAC,KAAK,EAAE,CACjF,CAAC;QACJ,CAAC;QAED,uDAAuD;QACvD,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC;YACE,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa,EAAE,CAAC,CAAC,OAAO;YACxB,kBAAkB,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ;SAC1C,EACD,EAAE,EACF;YACE,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,KAAK,CAAC,KAAK;YACnB,kEAAkE;YAClE,kEAAkE;YAClE,kEAAkE;YAClE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/B,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAClB,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC;oBACzC,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBAClF,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,6BAA6B,CAAC,CAAC;gBAC7D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;oBAC3B,CAAC,CAAC,KAAK,CACL,kEAAkE,WAAW,IAAI,CAClF,CAAC;gBACJ,CAAC;YACH,CAAC;SACF,CACF,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ;gBAC7B,CAAC,CAAC,wBAAwB,MAAM,CAAC,MAAM,CAAC,QAAQ,6BAA6B,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;gBAC9G,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpC,OAAO,QAAQ,CACb,aAAa,EACb,CAAC,MAAM,CAAC,KAAK,CAAC,EACd,SAAS,CAAC,CAAC,OAAO,kEAAkE,OAAO,EAAE,CAC9F,CAAC;YACJ,CAAC;YACD,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,MAAM,CAAC,KAAK,CAAC,EACd,kBAAkB,MAAM,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,eAAe,wCAAwC,OAAO,EAAE,CAC/I,CAAC;QACJ,CAAC;QACD,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,QAAQ,CAAC,EACV,SAAS,CAAC,CAAC,OAAO,iGAAiG,WAAW,0KAA0K,CACzS,CAAC;QACJ,CAAC;QAED,mEAAmE;QACnE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACvB,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC;YAC1B,KAAK;YACL,MAAM;YACN,QAAQ;YACR,GAAG;YACH,WAAW,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAClD,YAAY,IAAI,CAAC,MAAM,EAAE;YACzB,aAAa,IAAI,CAAC,SAAS,EAAE;YAC7B,SAAS;SACV,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,EAAE;YAAE,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;aAC7D,CAAC;YACJ,CAAC,CAAC,IAAI,CACJ,yCAAyC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,2DAA2D,CACrI,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,QAAQ,CACb,QAAQ,EACR,CAAC,UAAU,CAAC,EACZ,sCAAsC,SAAS,CAAC,CAAC,CAAC,uEAAuE,CAC1H,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAkC,EAClC,SAAiB,EACjB,GAAc,EACd,GAAW,EACX,QAAe,YAAY;IAE3B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,eAAe,EAAE,OAAO,EAAE,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,GAAG,CAAC;gBACF,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,uCAAuC,SAAS,aAAa,OAAO,IAAI,eAAe,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;aACrH,CAAC,CAAC;YACH,IAAI,OAAO,GAAG,eAAe;gBAAE,MAAM,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4EAA4E;AAC5E,KAAK,UAAU,cAAc,CAC3B,IAAqB;IAErB,MAAM,OAAO,GAAiD,EAAE,CAAC;IACjE,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,SAAS,CAAU,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,sBAAsB,CAC1C,IAAI,CAAC,GAAG,EACR,aAAa,EACb,MAAM,EACN,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,MAAM,KAAK,GAAG,OAAO;YACnB,CAAC,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,kCAAkC,IAAI,SAAS,EAAE,CAAC;QACzF,OAAO,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAyB,EAAE,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAAkB,EAClB,SAAiB,EACjB,CAAkB,EAClB,IAAqB;IAErB,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE;QAC5B,gBAAgB,EAAE,CAAC,CAAC,QAAQ;QAC5B,kBAAkB,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ;QACzC,qBAAqB,EAAE,CAAC,CAAC,oBAAoB;QAC7C,sBAAsB,EAAE,IAAI,CAAC,oBAAoB,IAAI,CAAC;QACtD,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK;QAC5B,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;QAChC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG;KACzB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IAC3B,OAAO,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC"}

package/dist/upgrader-records.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Does install.json describe what serves, and if not, which signed release
+ * does (#860)?
+ *
+ * The upgrader judges a request (downgrade, minFromVersion, infra contract)
+ * against the version install.json records, so it must first know the
+ * record is true. When the recorded digests differ from the serving ones,
+ * the only repair a human could make is "record the release that serves",
+ * and the job can do that itself without widening its trust: it records a
+ * release only when BOTH serving images (admin + gateway) are the images of
+ * one release manifest that verified against Caelo's release workflow
+ * (fetchVerifiedReleaseManifest, the same cosign check every install takes).
+ * Images that belong to no signed release are a deployment nobody can
+ * vouch for, so the run stops and says so.
+ *
+ * Which releases to try comes from untrusted hints — the version
+ * install.json names, and the release tags pointing at the serving digests
+ * in the fixed mirror (releaseVersionForDigest). A wrong hint costs one
+ * manifest fetch; only the signed manifest decides.
+ */
+import type { GcloudRunner } from "./gcloud-retry.js";
+import type { ImageDigests, InstallMetadata } from "./install-state.js";
+import type { ReleaseManifest, ReleaseManifestResult } from "./release-manifest.js";
+import type { RollSlug } from "./upgrade/app-roll.js";
+/** Every digest each serving image is known by (gcp-lookup.ts servingImageDigests). */
+export type ServingDigests = Readonly<Record<RollSlug, readonly string[]>>;
+/** Whether `recorded` names the images that serve, in any form they are known by. */
+export declare function recordMatchesServing(recorded: ImageDigests | undefined, serving: ServingDigests): boolean;
+/** What {@link findServingRelease} learned. */
+export type ServingRelease = {
+    readonly ok: true;
+    readonly manifest: ReleaseManifest;
+} | {
+    readonly ok: false;
+    /**
+     * True when no release matched but a candidate's manifest could not be
+     * fetched because the registry was unreachable or failing (5xx, 429,
+     * network): an outage, not a verdict, so the run must end `failed` and
+     * be retried, not send the operator to Cloud Shell. A manifest the
+     * registry says does not exist (404) never appears on retry, so it
+     * counts as rejected like a bad signature.
+     */
+    readonly retryable: boolean;
+    readonly reason: string;
+};
+export interface ServingReleaseDeps {
+    readonly run: GcloudRunner;
+    /** fetchVerifiedReleaseManifest against the fixed registry. */
+    readonly fetchManifest: (version: string) => Promise<ReleaseManifestResult>;
+}
+/**
+ * The signed release whose admin AND gateway images both serve, or why there
+ * is none (an operator-facing sentence naming the serving digests).
+ *
+ * @param meta - install.json as read; its `releaseVersion` is the first hint.
+ */
+export declare function findServingRelease(meta: InstallMetadata, serving: ServingDigests, deps: ServingReleaseDeps): Promise<ServingRelease>;
+//# sourceMappingURL=upgrader-records.d.ts.map

package/dist/upgrader-records.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upgrader-records.d.ts","sourceRoot":"","sources":["../src/upgrader-records.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACpF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtD,uFAAuF;AACvF,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;AAI3E,qFAAqF;AACrF,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,YAAY,GAAG,SAAS,EAClC,OAAO,EAAE,cAAc,GACtB,OAAO,CAGT;AAED,+CAA+C;AAC/C,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,GACzD;IACE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IACnB;;;;;;;OAOG;IACH,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEN,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,+DAA+D;IAC/D,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC7E;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,cAAc,CAAC,CAiDzB"}

package/dist/upgrader-records.js

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: MPL-2.0
+import { releaseVersionForDigest } from "./upgrade/release-images.js";
+const SLUGS = ["admin", "gateway"];
+/** Whether `recorded` names the images that serve, in any form they are known by. */
+export function recordMatchesServing(recorded, serving) {
+    if (!recorded)
+        return false;
+    return SLUGS.every((slug) => serving[slug].includes(recorded[slug]));
+}
+/**
+ * The signed release whose admin AND gateway images both serve, or why there
+ * is none (an operator-facing sentence naming the serving digests).
+ *
+ * @param meta - install.json as read; its `releaseVersion` is the first hint.
+ */
+export async function findServingRelease(meta, serving, deps) {
+    const partial = [];
+    const unverified = [];
+    const unreachable = [];
+    const tried = [];
+    for await (const version of candidateVersions(meta, serving, deps.run)) {
+        if (tried.includes(version))
+            continue;
+        tried.push(version);
+        const fetched = await deps.fetchManifest(version);
+        if (!fetched.ok) {
+            if (fetched.error.code === "registry" && fetched.error.transient) {
+                unreachable.push(`v${version} (${fetched.error.message.slice(0, 200)})`);
+                continue;
+            }
+            unverified.push(`v${version} (${fetched.error.message.slice(0, 200)})`);
+            continue;
+        }
+        const m = fetched.manifest;
+        const matching = SLUGS.filter((slug) => serving[slug].includes(m.digests[slug]));
+        if (matching.length === SLUGS.length)
+            return { ok: true, manifest: m };
+        if (matching.length > 0) {
+            const other = SLUGS.filter((s) => !matching.includes(s));
+            partial.push(`the ${matching.join(", ")} serves v${m.version}'s image, the ${other.join(", ")} does not`);
+        }
+    }
+    const servingText = SLUGS.map((s) => `${s} ${serving[s][0] ?? "(unknown)"}`).join(", ");
+    if (unreachable.length > 0) {
+        // A release we could not read may be the serving one, so this is no
+        // verdict on the images: retry later rather than escalate.
+        return {
+            ok: false,
+            retryable: true,
+            reason: `Nothing was changed: the images that serve now (${servingText}) are not the ones install.json names, and the Caelo release registry could not be reached to check which release they belong to (${unreachable.join("; ")}). Try again later.`,
+        };
+    }
+    return {
+        ok: false,
+        retryable: false,
+        reason: [
+            `The images that serve now (${servingText}) are not the ones install.json names, and they do not belong to one signed Caelo release${tried.length === 0 ? " (no release tag could be found for them)" : ""}, so the upgrader cannot tell what this install runs.`,
+            ...(partial.length > 0 ? [`${capitalize(partial.join("; "))}.`] : []),
+            ...(unverified.length > 0
+                ? [`These release records could not be verified: ${unverified.join("; ")}.`]
+                : []),
+            "Nothing was changed. Check who deployed these images (the Cloud Run revision history of both services); an upgrade with Cloud Shell puts a signed release back.",
+        ].join(" "),
+    };
+}
+/**
+ * Release versions worth checking, most likely first. Lazy: the version
+ * install.json names usually is the serving one, and then no tag listing
+ * runs at all.
+ */
+async function* candidateVersions(meta, serving, run) {
+    if (meta.releaseVersion)
+        yield meta.releaseVersion;
+    // Release tags point at the index digest a deploy pins (the deployed form,
+    // listed first); the platform digest Cloud Run resolved has no tag.
+    for (const slug of SLUGS) {
+        for (const digest of serving[slug]) {
+            const version = await releaseVersionForDigest(run, slug, digest);
+            if (version)
+                yield version;
+        }
+    }
+}
+function capitalize(s) {
+    return s.charAt(0).toUpperCase() + s.slice(1);
+}
+//# sourceMappingURL=upgrader-records.js.map

package/dist/upgrader-records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upgrader-records.js","sourceRoot":"","sources":["../src/upgrader-records.ts"],"names":[],"mappings":"AAAA,mCAAmC;AA2BnC,OAAO,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAKtE,MAAM,KAAK,GAAwB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAExD,qFAAqF;AACrF,MAAM,UAAU,oBAAoB,CAClC,QAAkC,EAClC,OAAuB;IAEvB,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAyBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAqB,EACrB,OAAuB,EACvB,IAAwB;IAExB,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACvE,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAS;QACtC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACjE,WAAW,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzE,SAAS;YACX,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QACD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjF,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACvE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACzD,OAAO,CAAC,IAAI,CACV,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,iBAAiB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAC5F,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,oEAAoE;QACpE,2DAA2D;QAC3D,OAAO;YACL,EAAE,EAAE,KAAK;YACT,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,mDAAmD,WAAW,qIAAqI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB;SACvP,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,KAAK;QACT,SAAS,EAAE,KAAK;QAChB,MAAM,EAAE;YACN,8BAA8B,WAAW,4FAA4F,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,EAAE,uDAAuD;YACjQ,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,CAAC,gDAAgD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBAC5E,CAAC,CAAC,EAAE,CAAC;YACP,iKAAiK;SAClK,CAAC,IAAI,CAAC,GAAG,CAAC;KACZ,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,IAAqB,EACrB,OAAuB,EACvB,GAAiB;IAEjB,IAAI,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,CAAC,cAAc,CAAC;IACnD,2EAA2E;IAC3E,oEAAoE;IACpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YACjE,IAAI,OAAO;gBAAE,MAAM,OAAO,CAAC;QAC7B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC"}

package/dist/upgrader-reporter.d.ts

@@ -26,8 +26,9 @@ export interface EventReporter extends UpgradeReporter {
     flush(): Promise<void>;
 }
 /**
- * The phase an app-roll step belongs to, from its label (app-roll.ts names
- * its steps). Unknown labels keep the current phase.
+ * The phase an upgrade step belongs to, from its label (app-roll.ts,
+ * converge.ts and upgrade.ts name their steps). Unknown labels keep the
+ * current phase.
  */
 export declare function phaseOfStep(label: string): PlatformUpdatePhase | null;
 /** Build the reporter for one request. */

package/dist/upgrader-reporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upgrader-reporter.d.ts","sourceRoot":"","sources":["../src/upgrader-reporter.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;GAUG;AAEH,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,mBAAmB,EACzB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAc,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAEzE,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,CACtB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,wBAAwB,EAC/B,OAAO,EAAE,MAAM,KACZ,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,yCAAyC;AACzC,MAAM,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CACnC,KAAK,IAAI,CAAC;AAEX,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,sEAAsE;IACtE,KAAK,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACxC,gDAAgD;IAChD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAMrE;AASD,0CAA0C;AAC1C,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CA0C5F"}
+{"version":3,"file":"upgrader-reporter.d.ts","sourceRoot":"","sources":["../src/upgrader-reporter.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;GAUG;AAEH,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,mBAAmB,EACzB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAc,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAEzE,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,CACtB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,wBAAwB,EAC/B,OAAO,EAAE,MAAM,KACZ,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,yCAAyC;AACzC,MAAM,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CACnC,KAAK,IAAI,CAAC;AAEX,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,sEAAsE;IACtE,KAAK,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACxC,gDAAgD;IAChD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAUrE;AASD,0CAA0C;AAC1C,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CA0C5F"}

package/dist/upgrader-reporter.js

@@ -12,10 +12,18 @@
  */
 import { PLATFORM_UPDATE_EVENT_MAX, } from "@caelo-cms/shared";
 /**
- * The phase an app-roll step belongs to, from its label (app-roll.ts names
- * its steps). Unknown labels keep the current phase.
+ * The phase an upgrade step belongs to, from its label (app-roll.ts,
+ * converge.ts and upgrade.ts name their steps). Unknown labels keep the
+ * current phase.
  */
 export function phaseOfStep(label) {
+    // The full upgrade (upgrade/upgrade.ts) the converge job runs (#819).
+    if (label.startsWith("Pre-flight"))
+        return "verify";
+    if (label.startsWith("Taking a pre-upgrade"))
+        return "backup";
+    if (/^(Ensuring|Preparing|Removing) /.test(label))
+        return "converge";
     if (label.startsWith("Checking the health check"))
         return "health-baseline";
     if (label.startsWith("Applying DB migrations"))

package/dist/upgrader-reporter.js.map

@@ -1 +1 @@
-{"version":3,"file":"upgrader-reporter.js","sourceRoot":"","sources":["../src/upgrader-reporter.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;GAUG;AAEH,OAAO,EACL,yBAAyB,GAG1B,MAAM,mBAAmB,CAAC;AAwB3B;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,CAAC,UAAU,CAAC,2BAA2B,CAAC;QAAE,OAAO,iBAAiB,CAAC;IAC5E,IAAI,KAAK,CAAC,UAAU,CAAC,wBAAwB,CAAC;QAAE,OAAO,YAAY,CAAC;IACpE,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,MAAM,CAAC;IAChD,IAAI,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,QAAQ,GAAmE;IAC/E,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,MAAM;IACf,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,OAAO;CACf,CAAC;AAEF,0CAA0C;AAC1C,MAAM,UAAU,aAAa,CAAC,IAAe,EAAE,GAAW,EAAE,SAAiB;IAC3E,IAAI,OAAO,GAAwB,OAAO,CAAC;IAC3C,IAAI,KAAK,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,CAAC,KAA+B,EAAE,OAAe,EAAE,KAAK,GAAG,OAAO,EAAE,EAAE;QACjF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,CAAC,IAAI,cAAc,CAAC;QAClF,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CACtB,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;YAC5C,GAAG,CAAC;gBACF,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,sCAAsC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAC3F,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CACH,CAAC;IACJ,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,CAAC;YACL,OAAO,GAAG,CAAC,CAAC;QACd,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,KAAK;YACR,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,CAAC;gBAAE,OAAO,GAAG,CAAC,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACpB,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;gBAC/B,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/B,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;aAC/B,CAAC;QACJ,CAAC;QACD,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;YACV,iEAAiE;YACjE,IAAI,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC;gBAAE,OAAO,GAAG,UAAU,CAAC;YACvD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9B,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;KACnC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"upgrader-reporter.js","sourceRoot":"","sources":["../src/upgrader-reporter.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;GAUG;AAEH,OAAO,EACL,yBAAyB,GAG1B,MAAM,mBAAmB,CAAC;AAwB3B;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,sEAAsE;IACtE,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,QAAQ,CAAC;IACpD,IAAI,KAAK,CAAC,UAAU,CAAC,sBAAsB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9D,IAAI,iCAAiC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC;IACrE,IAAI,KAAK,CAAC,UAAU,CAAC,2BAA2B,CAAC;QAAE,OAAO,iBAAiB,CAAC;IAC5E,IAAI,KAAK,CAAC,UAAU,CAAC,wBAAwB,CAAC;QAAE,OAAO,YAAY,CAAC;IACpE,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,MAAM,CAAC;IAChD,IAAI,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,QAAQ,GAAmE;IAC/E,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,MAAM;IACf,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,OAAO;CACf,CAAC;AAEF,0CAA0C;AAC1C,MAAM,UAAU,aAAa,CAAC,IAAe,EAAE,GAAW,EAAE,SAAiB;IAC3E,IAAI,OAAO,GAAwB,OAAO,CAAC;IAC3C,IAAI,KAAK,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,CAAC,KAA+B,EAAE,OAAe,EAAE,KAAK,GAAG,OAAO,EAAE,EAAE;QACjF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,CAAC,IAAI,cAAc,CAAC;QAClF,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CACtB,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;YAC5C,GAAG,CAAC;gBACF,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,sCAAsC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAC3F,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CACH,CAAC;IACJ,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,CAAC;YACL,OAAO,GAAG,CAAC,CAAC;QACd,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,KAAK;YACR,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,CAAC;gBAAE,OAAO,GAAG,CAAC,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACpB,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;gBAC/B,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/B,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;aAC/B,CAAC;QACJ,CAAC;QACD,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;YACV,iEAAiE;YACjE,IAAI,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC;gBAAE,OAAO,GAAG,UAAU,CAAC;YACvD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9B,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;KACnC,CAAC;AACJ,CAAC"}