@caelo-cms/provisioning 0.10.46 → 0.10.48

package/src/agent/run.test.ts

@@ -0,0 +1,382 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * One poll of the self-hosted host agent (agent/run.ts) against fakes:
+ * heartbeat, claim, the refusal matrix (nothing of the target runs before
+ * its signed manifest passed), the hand-over to the target release's CLI,
+ * and what happens when that process records nothing.
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { StoredUpdatePolicy } from "@caelo-cms/shared";
+import type { ReleaseManifest, ReleaseManifestResult } from "../release-manifest.js";
+import type { SelfHostedRelease } from "../self-hosted-release.js";
+import type { ClaimedUpdate, UpdateEnd, UpdateFacts } from "../upgrader-job.js";
+import type { AgentHeartbeat, AgentQueue } from "./queue.js";
+import { type AgentApplyRequest, type AgentRunDeps, runAgentOnce } from "./run.js";
+
+const D = (c: string) => `sha256:${c.repeat(64)}`;
+
+const MANIFEST: ReleaseManifest = {
+  manifestVersion: 1,
+  version: "0.11.0",
+  channel: "stable",
+  digests: { admin: D("a"), gateway: D("b"), ops: D("c") },
+  minFromVersion: "0.10.0",
+  // A raised cloud contract: irrelevant on self-hosted.
+  infraContractVersion: 7,
+  migrations: { contract: false, lastContractVersion: null, head: "0270_platform_update_agent" },
+  security: false,
+  advisories: [],
+};
+
+const POLICY: StoredUpdatePolicy = {
+  mode: "notify",
+  autoSecurity: false,
+  autoPatch: false,
+  autoMinor: false,
+  channel: "stable",
+  windowStart: null,
+  windowEnd: null,
+  windowTimezone: "UTC",
+  soakHours: 24,
+  paused: false,
+  pausedReason: null,
+  pausedAt: null,
+  updatedAt: "2026-10-13T00:00:00Z",
+};
+
+const REQUEST: ClaimedUpdate = {
+  id: "11111111-1111-4111-8111-111111111111",
+  targetVersion: "0.11.0",
+  trigger: "manual",
+  policy: POLICY,
+};
+
+const INSTALLED: SelfHostedRelease = {
+  version: "0.10.47",
+  images: {
+    admin: `ghcr.io/caelo-cms/admin@${D("1")}`,
+    gateway: `ghcr.io/caelo-cms/gateway@${D("2")}`,
+  },
+  installedAt: "2026-10-01T00:00:00Z",
+  history: [],
+};
+
+interface Memory {
+  queue: AgentQueue;
+  beats: AgentHeartbeat[];
+  claims: (string | null)[];
+  events: { phase: string; level: string; message: string }[];
+  facts: UpdateFacts[];
+  ends: UpdateEnd[];
+  /** Whether the request is still running (finish only ends a running one). */
+  running: boolean;
+}
+
+function memoryQueue(request: ClaimedUpdate | null, opts: { heartbeatFails?: boolean } = {}) {
+  const m: Memory = {
+    beats: [],
+    claims: [],
+    events: [],
+    facts: [],
+    ends: [],
+    running: false,
+    queue: undefined as unknown as AgentQueue,
+  };
+  m.queue = {
+    heartbeat: async (beat) => {
+      if (opts.heartbeatFails) throw new Error("password authentication failed");
+      m.beats.push(beat);
+    },
+    claim: async (execution) => {
+      m.claims.push(execution);
+      m.running = request !== null;
+      return request;
+    },
+    event: async (_id, phase, level, message) => {
+      m.events.push({ phase, level, message });
+    },
+    record: async (_id, facts) => {
+      m.facts.push(facts);
+    },
+    finish: async (_id, end) => {
+      if (!m.running) return false;
+      m.running = false;
+      m.ends.push(end);
+      return true;
+    },
+  };
+  return m;
+}
+
+function setup(over: Partial<AgentRunDeps> = {}, request: ClaimedUpdate | null = REQUEST) {
+  const q = memoryQueue(request);
+  const applied: AgentApplyRequest[] = [];
+  const locks = { taken: 0, released: 0 };
+  const deps: AgentRunDeps = {
+    queue: q.queue,
+    agentVersion: "0.10.47",
+    host: "box-1",
+    preflight: async () => null,
+    readInstalled: () => INSTALLED,
+    fetchManifest: async (): Promise<ReleaseManifestResult> => ({ ok: true, manifest: MANIFEST }),
+    lock: () => {
+      locks.taken += 1;
+      return () => {
+        locks.released += 1;
+      };
+    },
+    verifyTargetCli: async (version) => ({
+      ok: true,
+      dir: `/srv/.caelo/agent/cli/caelo-${version}`,
+      cliPath: `/srv/.caelo/agent/cli/caelo-${version}/node_modules/@caelo-cms/provisioning/dist/cli.js`,
+    }),
+    applyTarget: async (req) => {
+      applied.push(req);
+      // The target's CLI records the end itself.
+      await q.queue.finish(req.requestId, {
+        status: "succeeded",
+        reason: null,
+        codes: [],
+        pauseAutoUpdates: false,
+      });
+      return { exitCode: 0, output: "" };
+    },
+    now: () => new Date("2026-10-13T12:00:00Z"),
+    log: () => {},
+    sleep: async () => {},
+    ...over,
+  };
+  return { deps, q, applied, locks };
+}
+
+describe("runAgentOnce", () => {
+  it("checks in and does nothing more without a pending request", async () => {
+    const { deps, q, applied } = setup({ preflight: async () => "cosign is missing" }, null);
+    expect(await runAgentOnce(deps)).toEqual({ exitCode: 0, requestId: null, status: null });
+    expect(q.beats).toEqual([
+      { agentVersion: "0.10.47", host: "box-1", problem: "cosign is missing" },
+    ]);
+    expect(q.claims).toEqual(["agent@box-1"]);
+    expect(applied).toEqual([]);
+  });
+
+  it("fails the poll (exit 1) when it cannot check in, and claims nothing", async () => {
+    const q = memoryQueue(REQUEST, { heartbeatFails: true });
+    const { deps } = setup({ queue: q.queue });
+    expect((await runAgentOnce(deps)).exitCode).toBe(1);
+    expect(q.claims).toEqual([]);
+  });
+
+  it("verifies, decides, then hands the request to the target release's CLI", async () => {
+    const { deps, q, applied, locks } = setup();
+    const r = await runAgentOnce(deps);
+    expect(r).toEqual({ exitCode: 0, requestId: REQUEST.id, status: null });
+    expect(applied).toEqual([
+      {
+        requestId: REQUEST.id,
+        targetVersion: "0.11.0",
+        trigger: "manual",
+        digests: { admin: D("a"), gateway: D("b") },
+        cliPath:
+          "/srv/.caelo/agent/cli/caelo-0.11.0/node_modules/@caelo-cms/provisioning/dist/cli.js",
+      },
+    ]);
+    expect(q.facts[0]).toMatchObject({ adminDigest: D("a"), manifestSecurity: false });
+    expect([...new Set(q.events.map((e) => e.phase))]).toEqual(["start", "verify", "assess"]);
+    expect(q.events.some((e) => /built by Caelo's release workflow/.test(e.message))).toBe(true);
+    expect(q.events.some((e) => /signed by Caelo's release workflow/.test(e.message))).toBe(true);
+    expect(q.ends).toEqual([
+      { status: "succeeded", reason: null, codes: [], pauseAutoUpdates: false },
+    ]);
+    expect(locks).toEqual({ taken: 1, released: 1 });
+  });
+
+  interface RefusalRow {
+    readonly name: string;
+    readonly over: Partial<AgentRunDeps>;
+    readonly request?: ClaimedUpdate;
+    /** `<status>/<codes>`. */
+    readonly expect: string;
+    readonly reason?: RegExp;
+  }
+  const refusals: readonly RefusalRow[] = [
+    {
+      name: "a host problem (preflight)",
+      over: { preflight: async () => "Docker is not reachable for the agent's user." },
+      expect: "failed/preflight",
+      reason: /Nothing was changed: Docker is not reachable/,
+    },
+    {
+      name: "another upgrade holds the lock",
+      over: {
+        lock: () => {
+          throw new Error("`cms-provision upgrade` is running on this install (pid 42).");
+        },
+      },
+      expect: "failed/state-locked",
+    },
+    {
+      name: "a malformed release.json",
+      over: {
+        readInstalled: () => {
+          throw new Error("release.json is malformed");
+        },
+      },
+      expect: "failed/install-state",
+    },
+    {
+      name: "a bad signature",
+      over: {
+        fetchManifest: async () => ({ ok: false, error: { code: "signature", message: "no" } }),
+      },
+      expect: "refused/signature",
+    },
+    {
+      name: "the registry is unreachable",
+      over: {
+        fetchManifest: async () => ({ ok: false, error: { code: "registry", message: "dns" } }),
+      },
+      expect: "failed/registry",
+    },
+    {
+      name: "a downgrade",
+      over: { readInstalled: () => ({ ...INSTALLED, version: "0.12.0" }) },
+      expect: "refused/not-newer",
+    },
+    {
+      name: "below minFromVersion",
+      over: {
+        fetchManifest: async () => ({
+          ok: true,
+          manifest: { ...MANIFEST, minFromVersion: "0.10.50" },
+        }),
+      },
+      expect: "refused/min-from-version",
+    },
+    {
+      name: "no recorded release (an install from before prebuilt images)",
+      over: { readInstalled: () => null },
+      expect: "needs_cloud_shell/install-version-unknown",
+      reason: /upgrade command on the server/,
+    },
+    {
+      name: "contract migrations on an automatic run",
+      over: {
+        fetchManifest: async () => ({
+          ok: true,
+          manifest: { ...MANIFEST, migrations: { ...MANIFEST.migrations, contract: true } },
+        }),
+      },
+      request: {
+        ...REQUEST,
+        trigger: "auto",
+        policy: { ...POLICY, mode: "auto", autoMinor: true },
+      },
+      expect: "refused/contract-migrations",
+    },
+    {
+      name: "a target CLI whose npm provenance does not check out",
+      over: {
+        verifyTargetCli: async () => ({
+          ok: false,
+          error: { code: "provenance", message: "built by https://github.com/evil/caelo-cms" },
+        }),
+      },
+      expect: "refused/cli-provenance",
+      reason: /not the one Caelo's release workflow published/,
+    },
+    {
+      name: "npm missing on the server",
+      over: {
+        verifyTargetCli: async () => ({
+          ok: false,
+          error: { code: "npm-missing", message: "npm is not installed" },
+        }),
+      },
+      expect: "failed/cli-npm-missing",
+    },
+    {
+      name: "an automatic run the policy does not allow",
+      over: {},
+      request: { ...REQUEST, trigger: "auto" },
+      expect: "refused/policy",
+    },
+  ];
+
+  it.each(refusals.map((r) => [r.name, r] as const))(
+    "stops before any target code runs: %s",
+    async (_name, row) => {
+      const { deps, q, applied, locks } = setup(row.over, row.request ?? REQUEST);
+      const r = await runAgentOnce(deps);
+      expect(applied).toEqual([]);
+      const end = q.ends[0];
+      expect(`${end?.status}/${end?.codes.join(",")}`).toBe(row.expect);
+      if (row.reason) expect(end?.reason ?? "").toMatch(row.reason);
+      expect(r.status).toBe(end?.status ?? null);
+      expect(r.exitCode).toBe(end?.status === "failed" ? 1 : 0);
+      // Whatever the lock did, it is not left held.
+      expect(locks.released).toBe(locks.taken);
+      expect(q.events.at(-1)?.phase).toBe("finish");
+    },
+  );
+
+  it("ends the request as failed with the output when the target's CLI recorded nothing", async () => {
+    const { deps, q } = setup({
+      applyTarget: async () => ({ exitCode: 1, output: "error: package not found" }),
+    });
+    const r = await runAgentOnce(deps);
+    expect(r).toEqual({ exitCode: 1, requestId: REQUEST.id, status: "failed" });
+    expect(q.ends[0]?.status).toBe("failed");
+    expect(q.ends[0]?.codes).toEqual(["apply"]);
+    expect(q.ends[0]?.reason).toContain("error: package not found");
+    expect(q.events.at(-1)).toMatchObject({ phase: "finish", level: "error" });
+  });
+
+  it("leaves a rolled-back end the target's CLI recorded alone", async () => {
+    const { deps, q } = setup();
+    const rolled: AgentRunDeps = {
+      ...deps,
+      applyTarget: async (req) => {
+        await q.queue.finish(req.requestId, {
+          status: "rolled_back",
+          reason: "unhealthy",
+          codes: ["roll-admin"],
+          pauseAutoUpdates: false,
+        });
+        return { exitCode: 1, output: "" };
+      },
+    };
+    const r = await runAgentOnce(rolled);
+    expect(r).toEqual({ exitCode: 1, requestId: REQUEST.id, status: null });
+    expect(q.ends.map((e) => e.status)).toEqual(["rolled_back"]);
+    expect(q.events.some((e) => e.phase === "finish")).toBe(false);
+  });
+
+  it("keeps checking in while the target's CLI runs", async () => {
+    const { deps, q } = setup();
+    const slow: AgentRunDeps = {
+      ...deps,
+      heartbeatEveryMs: 5,
+      applyTarget: async (req) => {
+        await new Promise((resolve) => setTimeout(resolve, 40));
+        return deps.applyTarget(req);
+      },
+    };
+    await runAgentOnce(slow);
+    const during = q.beats.length;
+    expect(during).toBeGreaterThan(2);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    expect(q.beats.length).toBe(during);
+  });
+
+  it("pauses automatic installs when an automatic run fails", async () => {
+    const { deps, q } = setup(
+      { preflight: async () => "cosign is missing." },
+      { ...REQUEST, trigger: "auto" },
+    );
+    await runAgentOnce(deps);
+    expect(q.ends[0]).toMatchObject({ status: "failed", pauseAutoUpdates: true });
+  });
+});

package/src/agent/run.ts

@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * One poll of the self-hosted host agent (#696 Phase 4c) — what the
+ * `caelo-agent-<domain>-<hash>` timer runs every minute:
+ *
+ *   1. preflight (Docker reachable, cosign present, a recorded release) and
+ *      a heartbeat with the result, so Settings → Updates knows the agent is
+ *      alive and what would stop an install before the Owner clicks;
+ *   2. claim the oldest pending request (ending a run that died mid-way) —
+ *      the admin only ever writes the request row;
+ *   3. verify the target's signed release manifest (cosign, fixed registry
+ *      and signer — nothing from the request but the version) and decide
+ *      with the upgrader's own refusal matrix (upgrader-assess.ts): bad
+ *      signature, downgrade, below minFromVersion, contract migrations or a
+ *      policy refusal on an automatic run. Self-hosted has no separately
+ *      converged infrastructure, so the infra-contract row never applies;
+ *   4. install the TARGET release's own CLI from npm and verify its registry
+ *      signatures + provenance (agent/cli-provenance.ts, the checks of the
+ *      Cloud Shell runner) — refused outright when they fail;
+ *   5. only then run that CLI's `agent apply` (agent/apply.ts), which
+ *      performs the same backup → pull → migrate → roll → health → rollback
+ *      as a hand-run `upgrade` and narrates it into the request's timeline.
+ *      The target's CLI renders the target's Compose file, so a release that
+ *      changes the stack needs no extra step. No code of the target runs
+ *      before its manifest verified and passed the checks above;
+ *   6. if that process ended without recording an end state, end the
+ *      request as failed with its output.
+ *
+ * The install's upgrade lock is held for the whole run, so a hand-run
+ * `upgrade` and the agent never roll at the same time.
+ */
+
+import { PLATFORM_UPDATE_AGENT_POLL_SECONDS, type PlatformUpdateTrigger } from "@caelo-cms/shared";
+import type { ReleaseManifest, ReleaseManifestResult } from "../release-manifest.js";
+import type { SelfHostedRelease } from "../self-hosted-release.js";
+import { assessUpgradeRequest } from "../upgrader-assess.js";
+import type { UpdateEnd, UpdateEndStatus } from "../upgrader-job.js";
+import { recordUpdateEnd } from "../upgrader-job.js";
+import { eventReporter, type JobLog } from "../upgrader-reporter.js";
+import type { VerifiedCli } from "./cli-provenance.js";
+import type { AgentQueue } from "./queue.js";
+
+/** What the target release's `agent apply` is told (argv of the child). */
+export interface AgentApplyRequest {
+  readonly requestId: string;
+  readonly targetVersion: string;
+  readonly trigger: PlatformUpdateTrigger;
+  /** From the verified manifest: the images the target must resolve to. */
+  readonly digests: { readonly admin: string; readonly gateway: string };
+  /** The target's CLI, installed and provenance-verified (agent/cli-provenance.ts). */
+  readonly cliPath: string;
+}
+
+/** How the target's `agent apply` process ended. */
+export interface AgentApplyOutcome {
+  readonly exitCode: number;
+  /** The tail of its output, for the timeline when it recorded nothing. */
+  readonly output: string;
+}
+
+export interface AgentRunDeps {
+  readonly queue: AgentQueue;
+  /** This agent's own release (CAELO_VERSION). */
+  readonly agentVersion: string;
+  readonly host: string;
+  /** Why an install would fail on this host now, or null. */
+  readonly preflight: () => Promise<string | null>;
+  /** `.caelo/release.json` (null when none is recorded; throws when malformed). */
+  readonly readInstalled: () => SelfHostedRelease | null;
+  /** fetchVerifiedReleaseManifest against the fixed registry. */
+  readonly fetchManifest: (version: string) => Promise<ReleaseManifestResult>;
+  /** Take the install's upgrade lock (agent/lock.ts); throws when held. */
+  readonly lock: () => () => void;
+  /**
+   * Install the target release's CLI and verify its npm signatures +
+   * provenance (agent/cli-provenance.ts). Nothing of it runs unless this
+   * passes.
+   */
+  readonly verifyTargetCli: (version: string) => Promise<VerifiedCli>;
+  /** Run the target release's `agent apply` and wait for it. */
+  readonly applyTarget: (request: AgentApplyRequest) => Promise<AgentApplyOutcome>;
+  readonly now: () => Date;
+  readonly log: JobLog;
+  readonly sleep?: (ms: number) => Promise<void>;
+  /**
+   * How often the agent keeps checking in while the target's CLI runs (an
+   * update takes minutes; the admin would otherwise think the agent gone).
+   * Default: every poll interval.
+   */
+  readonly heartbeatEveryMs?: number;
+}
+
+/** What one poll did. */
+export interface AgentRunResult {
+  readonly exitCode: number;
+  readonly requestId: string | null;
+  /** The end the agent itself recorded (null when the target's CLI did, or nothing ran). */
+  readonly status: UpdateEndStatus | null;
+}
+
+/** One poll. Never throws; see the file comment for the sequence. */
+export async function runAgentOnce(deps: AgentRunDeps): Promise<AgentRunResult> {
+  const { queue, log } = deps;
+  const problem = await deps.preflight().catch((e: unknown) => `preflight failed: ${errorText(e)}`);
+  try {
+    await queue.heartbeat({ agentVersion: deps.agentVersion, host: deps.host, problem });
+  } catch (e) {
+    log({ severity: "ERROR", message: `could not check in with the database: ${errorText(e)}` });
+    return { exitCode: 1, requestId: null, status: null };
+  }
+
+  let claimed: Awaited<ReturnType<AgentQueue["claim"]>>;
+  try {
+    claimed = await queue.claim(`agent@${deps.host}`.slice(0, 200));
+  } catch (e) {
+    log({ severity: "ERROR", message: `could not read update requests: ${errorText(e)}` });
+    return { exitCode: 1, requestId: null, status: null };
+  }
+  if (!claimed) return { exitCode: 0, requestId: null, status: null };
+  const request = claimed;
+  const r = eventReporter(
+    (phase, level, message) => queue.event(request.id, phase, level, message),
+    log,
+    request.id,
+  );
+  let release: (() => void) | null = null;
+  const unlock = () => {
+    try {
+      release?.();
+    } catch (e) {
+      log({ severity: "WARNING", message: `could not release the upgrade lock: ${errorText(e)}` });
+    }
+    release = null;
+  };
+  const stopWith = async (
+    status: UpdateEndStatus,
+    codes: readonly string[],
+    reason: string,
+  ): Promise<AgentRunResult> => {
+    r.phase("finish");
+    r.error(reason);
+    await r.flush();
+    unlock();
+    const end: UpdateEnd = {
+      status,
+      reason,
+      codes,
+      pauseAutoUpdates:
+        request.trigger === "auto" && (status === "failed" || status === "rolled_back"),
+    };
+    const recorded = await recordUpdateEnd(queue, request.id, end, log, deps.sleep);
+    const failed = status === "failed" || !recorded;
+    return { exitCode: failed ? 1 : 0, requestId: request.id, status };
+  };
+
+  try {
+    r.phase("start");
+    r.info(
+      `The update agent on ${deps.host} (Caelo ${deps.agentVersion}) picked up the request to install Caelo ${request.targetVersion}.`,
+    );
+    if (problem) {
+      return await stopWith(
+        "failed",
+        ["preflight"],
+        `Nothing was changed: ${problem} Fix this on the server, then try again.`,
+      );
+    }
+    try {
+      release = deps.lock();
+    } catch (e) {
+      return await stopWith(
+        "failed",
+        ["state-locked"],
+        `Nothing was changed: ${errorText(e)} Try again when it has finished.`,
+      );
+    }
+
+    r.phase("verify");
+    let installed: SelfHostedRelease | null;
+    try {
+      installed = deps.readInstalled();
+    } catch (e) {
+      return await stopWith("failed", ["install-state"], `Nothing was changed: ${errorText(e)}`);
+    }
+    const manifest = await deps.fetchManifest(request.targetVersion);
+    const decision = assessUpgradeRequest({
+      request,
+      manifest,
+      installed: { version: installed?.version ?? null, infraContractVersion: null },
+      policy: request.policy,
+      now: deps.now(),
+      manualPath: "server",
+    });
+    if (decision.manifest) {
+      r.success(
+        `Release information of ${decision.manifest.version} verified: signed by Caelo's release workflow.`,
+      );
+      await recordFacts(queue, request.id, decision.manifest);
+    }
+    r.phase("assess");
+    if (decision.kind === "stop") {
+      return await stopWith(decision.status, decision.codes, decision.reason);
+    }
+    for (const w of decision.warnings) r.warn(w);
+    const m = decision.manifest;
+    const step = r.step(
+      `Fetching the Caelo ${m.version} installer and verifying its npm provenance...`,
+    );
+    const cli = await deps.verifyTargetCli(m.version);
+    if (!cli.ok) {
+      step.failed(cli.error.message);
+      // A package that is not the release's is refused; a missing npm or a
+      // failed download is a failure to retry.
+      const refused = ["audit-signatures", "version", "provenance"].includes(cli.error.code);
+      return await stopWith(
+        refused ? "refused" : "failed",
+        [`cli-${cli.error.code}`],
+        `Nothing was changed: the installer of Caelo ${m.version} ${refused ? "is not the one Caelo's release workflow published" : "could not be fetched"} (${cli.error.message}).`,
+      );
+    }
+    step.done(
+      `The Caelo ${m.version} installer is signed and was built by Caelo's release workflow.`,
+    );
+    r.info(
+      `${installed?.version} → ${m.version}: handing over to the Caelo ${m.version} installer on this server.`,
+    );
+    await r.flush();
+
+    const beat = setInterval(
+      () => {
+        queue
+          .heartbeat({ agentVersion: deps.agentVersion, host: deps.host, problem: null })
+          .catch((e: unknown) => {
+            log({ severity: "WARNING", message: `could not check in: ${errorText(e)}` });
+          });
+      },
+      deps.heartbeatEveryMs ?? PLATFORM_UPDATE_AGENT_POLL_SECONDS * 1000,
+    );
+    let outcome: AgentApplyOutcome;
+    try {
+      outcome = await deps.applyTarget({
+        requestId: request.id,
+        targetVersion: m.version,
+        trigger: request.trigger,
+        digests: { admin: m.digests.admin, gateway: m.digests.gateway },
+        cliPath: cli.cliPath,
+      });
+    } finally {
+      clearInterval(beat);
+    }
+    unlock();
+    if (outcome.exitCode === 0) return { exitCode: 0, requestId: request.id, status: null };
+    // The target's CLI ends the request itself; only if it could not (it
+    // never started, or crashed) is the request still running here.
+    const reason = `The Caelo ${m.version} installer stopped (exit ${outcome.exitCode}) before it recorded how the update ended. If the site misbehaves, check \`docker compose ps\` on the server.${outcome.output ? `\n${outcome.output}` : ""}`;
+    const end: UpdateEnd = {
+      status: "failed",
+      reason: reason.slice(0, 4000),
+      codes: ["apply"],
+      pauseAutoUpdates: request.trigger === "auto",
+    };
+    let ended = false;
+    try {
+      ended = await queue.finish(request.id, end);
+    } catch (e) {
+      log({ severity: "ERROR", message: `could not end request ${request.id}: ${errorText(e)}` });
+      return { exitCode: 1, requestId: request.id, status: null };
+    }
+    if (!ended) return { exitCode: 1, requestId: request.id, status: null };
+    r.phase("finish");
+    r.error(reason);
+    await r.flush();
+    return { exitCode: 1, requestId: request.id, status: "failed" };
+  } catch (e) {
+    return await stopWith(
+      "failed",
+      ["internal"],
+      `The update agent stopped unexpectedly: ${errorText(e)}. If the site misbehaves, check \`docker compose ps\` on the server.`,
+    );
+  }
+}
+
+async function recordFacts(queue: AgentQueue, requestId: string, m: ReleaseManifest) {
+  await queue.record(requestId, {
+    manifestSecurity: m.security,
+    contractMigrations: m.migrations.contract,
+    adminDigest: m.digests.admin,
+    gatewayDigest: m.digests.gateway,
+  });
+}
+
+function errorText(e: unknown): string {
+  return e instanceof Error ? e.message : String(e);
+}