@intentius/chant-lexicon-fly 0.90.0 → 0.91.0

package/src/op/activities/machine-release.integration.test.ts

@@ -0,0 +1,113 @@
+import { describe, test, expect, beforeAll, afterAll } from "vitest";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { flapsUp, flapsDown } from "./flaps";
+import { defaultFlyHttp, deleteApp } from "./fly-apply";
+import { findMachine } from "./machine-release";
+import { readMachineRelease } from "../../release-metadata";
+import { createFlyReleaseCapability, createFlyRollbackCapability } from "../../components/fly-release";
+import { memoryMachineConfigStore } from "../../release-store";
+import { describeResources } from "../../describe-resources";
+import type { EffectReceiptRef } from "@intentius/chant/op/receipt-store";
+
+// A release, a migration, a second release and a rollback against a live
+// mudflaps container (#2736): the `fly-release` / `fly-rollback` capabilities
+// over the real Machines wire protocol, then describeResources reading the
+// release back the way `chant components status --live` does. Docker is
+// required; the suite skips in CI unless a job opts in with FLY_IT=1, like
+// ./fly-apply.integration.test.ts. No Fly credentials: mudflaps ignores tokens.
+
+const APP = "chant-release-it";
+const CONTAINER = "chant-mudflaps-release-it";
+const PORT = 4284;
+const WAIT = { intervalMs: 50, timeoutSecs: 5, deadlineMs: 30_000 };
+const http = defaultFlyHttp();
+
+let endpoint = "";
+let available = false;
+let tmp = "";
+
+beforeAll(async () => {
+  if (process.env.CI && !process.env.FLY_IT) return;
+  tmp = mkdtempSync(join(tmpdir(), "fly-release-it-"));
+  try {
+    endpoint = (await flapsUp({ name: CONTAINER, port: PORT, timeoutMs: 30_000 })).endpoint;
+    available = true;
+  } catch {
+    available = false;
+  }
+}, 60_000);
+
+afterAll(async () => {
+  if (available) {
+    await deleteApp({ base: endpoint }, APP, http).catch(() => undefined);
+    await flapsDown({ name: CONTAINER });
+  }
+  if (tmp) rmSync(tmp, { recursive: true, force: true });
+});
+
+describe("fly-release against mudflaps", () => {
+  test("release, migrate, release again, roll back; the Machine's metadata names each release", async (ctx) => {
+    if (!available) ctx.skip();
+    const planPath = join(tmp, "fly.json");
+    writeFileSync(
+      planPath,
+      JSON.stringify({
+        app: { endpoint: "/v1/apps", method: "POST", body: { app_name: APP, org_slug: "personal" } },
+        web: {
+          endpoint: `/v1/apps/${APP}/machines`,
+          method: "POST",
+          body: { name: "web", region: "iad", config: { image: "nginx:1", metadata: { "managed-by": "chant" } } },
+        },
+      }),
+    );
+    const configs = memoryMachineConfigStore();
+    const receipts = new Map<string, string>();
+    const deps = {
+      http,
+      configStore: () => configs,
+      receiptStore: () => ({
+        read: async (r: EffectReceiptRef) => receipts.get(r.name),
+        write: async (r: EffectReceiptRef, v: string) => void receipts.set(r.name, v),
+      }),
+      headCommit: async () => "1111111111111111111111111111111111111111",
+      log: () => undefined,
+    };
+    const release = createFlyReleaseCapability(deps);
+    const rollback = createFlyRollbackCapability(deps);
+    const c = { env: "it", component: "web" };
+    const common = { plan: planPath, endpoint, wait: WAIT, verify: { intervalMs: 100, timeoutMs: 10_000 } };
+
+    const one = await release.run(c, { ...common, digest: "sha256:one", image: "nginx:1", migrations: [{ name: "001.sql", command: "echo migrate 001" }] });
+    expect(one.migrations).toEqual([{ name: "001.sql", fired: true }]);
+    const two = await release.run(c, { ...common, digest: "sha256:two", image: "nginx:2", migrations: [{ name: "001.sql", command: "echo migrate 001" }] });
+    expect(two.migrations).toEqual([{ name: "001.sql", fired: false }]);
+    expect(two.previous?.digest).toBe("sha256:one");
+
+    let m = await findMachine({ base: endpoint }, APP, "web", http);
+    expect(readMachineRelease(m?.config?.metadata)).toMatchObject({ digest: "sha256:two", previousDigest: "sha256:one" });
+    expect(m?.config?.image).toBe("nginx:2");
+
+    const observed = await describeResources(
+      {
+        environment: "it",
+        buildOutput: "",
+        entityNames: ["app", "web"],
+        entities: new Map([
+          ["app", { entityType: "Fly::Machines::App", props: { name: APP } }],
+          ["web", { entityType: "Fly::Machines::Machine", props: { name: "web" } }],
+        ]),
+        endpoint,
+      },
+      http,
+    );
+    expect((observed.resources.web?.attributes as { digest?: string }).digest).toBe("sha256:two");
+
+    const back = await rollback.run(c, { plan: planPath, endpoint, wait: WAIT, verify: { intervalMs: 100, timeoutMs: 10_000 } });
+    expect(back.digest).toBe("sha256:one");
+    m = await findMachine({ base: endpoint }, APP, "web", http);
+    expect(m?.config?.image).toBe("nginx:1");
+    expect(readMachineRelease(m?.config?.metadata)?.digest).toBe("sha256:one");
+  }, 120_000);
+});

package/src/op/activities/machine-release.test.ts

@@ -0,0 +1,138 @@
+import { describe, test, expect } from "vitest";
+import {
+  RELEASE_METADATA_KEYS,
+  flyMachineExec,
+  flyMachineRelease,
+  flyMachineRestart,
+  flyMachineRestore,
+  flyMachineStop,
+  flyMachineVerify,
+  readMachineRelease,
+  releaseTarget,
+  withReleaseMetadata,
+} from "./machine-release";
+import { createMachinesFake } from "./machines-fake";
+import type { FlyPlan } from "./fly-apply";
+
+const ENDPOINT = "http://flaps.test";
+const NO_WAIT = { intervalMs: 0, deadlineMs: 2_000 };
+
+const PLAN: FlyPlan = {
+  shop: { endpoint: "/v1/apps", method: "POST", body: { app_name: "shop", org_slug: "personal" } },
+  web: {
+    endpoint: "/v1/apps/shop/machines",
+    method: "POST",
+    body: { name: "web", config: { image: "shop:1", env: { PORT: "8080" }, metadata: { "managed-by": "chant" } } },
+  },
+};
+
+describe("release metadata (pure)", () => {
+  test("withReleaseMetadata stamps the release and keeps other metadata", () => {
+    const config = withReleaseMetadata({ image: "x", metadata: { "managed-by": "chant", [RELEASE_METADATA_KEYS.release]: "old" } }, {
+      digest: "sha256:a",
+      gitSha: "abc1234",
+    });
+    expect(config.metadata).toEqual({
+      "managed-by": "chant",
+      [RELEASE_METADATA_KEYS.digest]: "sha256:a",
+      [RELEASE_METADATA_KEYS.gitSha]: "abc1234",
+    });
+    expect(readMachineRelease(config.metadata)).toEqual({ digest: "sha256:a", gitSha: "abc1234" });
+  });
+
+  test("readMachineRelease names no release without a digest", () => {
+    expect(readMachineRelease({ "managed-by": "chant" })).toBeUndefined();
+    expect(readMachineRelease(undefined)).toBeUndefined();
+  });
+
+  test("releaseTarget picks the plan's one Machine, or the one named", () => {
+    expect(releaseTarget(PLAN)).toMatchObject({ app: "shop", entity: "web", name: "web" });
+    expect(() => releaseTarget(PLAN, "worker")).toThrow(/no Machine named "worker"/);
+    const two: FlyPlan = { ...PLAN, worker: { endpoint: "/v1/apps/shop/machines", method: "POST", body: { name: "worker", config: {} } } };
+    expect(() => releaseTarget(two)).toThrow(/declares 2 Machines/);
+    expect(releaseTarget(two, "worker").name).toBe("worker");
+  });
+});
+
+describe("Machines activities against the fake", () => {
+  test("a first release creates the App and Machine with the release in its metadata", async () => {
+    const fake = createMachinesFake();
+    const r = await flyMachineRelease(
+      { plan: PLAN, endpoint: ENDPOINT, release: { digest: "sha256:one", gitSha: "1111111" }, image: "shop@sha256:img1", env: { MODE: "prod" }, wait: NO_WAIT },
+      undefined,
+      fake.http,
+    );
+    expect(r).toMatchObject({ app: "shop", action: "created", previous: null, release: { digest: "sha256:one", gitSha: "1111111" } });
+    const m = fake.machine("shop", "web")!;
+    expect(m.config.image).toBe("shop@sha256:img1");
+    expect(m.config.env).toEqual({ PORT: "8080", MODE: "prod" });
+    expect(readMachineRelease(m.config.metadata)).toEqual({ digest: "sha256:one", gitSha: "1111111" });
+    expect(m.config.metadata?.["managed-by"]).toBe("chant");
+  });
+
+  test("a second release records the one it replaced, and re-running it keeps that", async () => {
+    const fake = createMachinesFake();
+    const args = { plan: PLAN, endpoint: ENDPOINT, wait: NO_WAIT };
+    await flyMachineRelease({ ...args, release: { digest: "sha256:one" } }, undefined, fake.http);
+    const two = await flyMachineRelease({ ...args, release: { digest: "sha256:two" } }, undefined, fake.http);
+    expect(two.action).toBe("updated");
+    expect(two.release.previousDigest).toBe("sha256:one");
+    expect(two.previous?.release?.digest).toBe("sha256:one");
+    expect(readMachineRelease(two.previous?.config.metadata)?.digest).toBe("sha256:one");
+    const again = await flyMachineRelease({ ...args, release: { digest: "sha256:two" } }, undefined, fake.http);
+    expect(again.action).toBe("noop");
+    expect(again.release.previousDigest).toBe("sha256:one");
+  });
+
+  test("exec runs a command in the Machine and fails on a non-zero exit", async () => {
+    const fake = createMachinesFake({ exec: (_app, _m, cmd) => (cmd.join(" ").includes("bad") ? { exit_code: 3, stderr: "no table" } : { exit_code: 0, stdout: "ok" }) });
+    await flyMachineRelease({ plan: PLAN, endpoint: ENDPOINT, release: { digest: "sha256:one" }, wait: NO_WAIT }, undefined, fake.http);
+    const ok = await flyMachineExec({ app: "shop", machine: "web", command: "migrate 001", endpoint: ENDPOINT }, undefined, fake.http);
+    expect(ok).toEqual({ exitCode: 0, stdout: "ok", stderr: "" });
+    expect(fake.execs[0].command).toEqual(["sh", "-c", "migrate 001"]);
+    await expect(flyMachineExec({ app: "shop", machine: "web", command: ["bad"], endpoint: ENDPOINT }, undefined, fake.http)).rejects.toThrow(
+      /exited 3.*no table/,
+    );
+  });
+
+  test("stop and restart settle the Machine under a lease", async () => {
+    const fake = createMachinesFake();
+    await flyMachineRelease({ plan: PLAN, endpoint: ENDPOINT, release: { digest: "sha256:one" }, wait: NO_WAIT }, undefined, fake.http);
+    await flyMachineStop({ app: "shop", machine: "web", endpoint: ENDPOINT, wait: NO_WAIT }, undefined, fake.http);
+    expect(fake.machine("shop", "web")!.state).toBe("stopped");
+    await flyMachineRestart({ app: "shop", machine: "web", endpoint: ENDPOINT, wait: NO_WAIT }, undefined, fake.http);
+    expect(fake.machine("shop", "web")!.state).toBe("started");
+    expect(fake.calls.filter((c) => c.endsWith("/lease"))).toHaveLength(4);
+  });
+
+  test("verify: started with the release, and the health endpoint reports it", async () => {
+    const fake = createMachinesFake();
+    await flyMachineRelease(
+      { plan: PLAN, endpoint: ENDPOINT, release: { digest: "sha256:one", gitSha: "1111111" }, wait: NO_WAIT },
+      undefined,
+      fake.http,
+    );
+    const health = (revision: string) => (async () => new Response(JSON.stringify({ status: "healthy", revision }))) as unknown as typeof fetch;
+    const base = { app: "shop", machine: "web", endpoint: ENDPOINT, intervalMs: 1, timeoutMs: 20 };
+    await expect(flyMachineVerify({ ...base, digest: "sha256:one", url: "https://shop.test" }, undefined, fake.http, health("1111111"))).resolves.toMatchObject({
+      release: { digest: "sha256:one" },
+    });
+    await expect(flyMachineVerify({ ...base, digest: "sha256:two" }, undefined, fake.http)).rejects.toThrow(/serves sha256:one, expected sha256:two/);
+    await expect(flyMachineVerify({ ...base, digest: "sha256:one", url: "https://shop.test" }, undefined, fake.http, health("2222222"))).rejects.toThrow(
+      /reports 2222222/,
+    );
+    await flyMachineStop({ app: "shop", machine: "web", endpoint: ENDPOINT, wait: NO_WAIT }, undefined, fake.http);
+    await expect(flyMachineVerify({ ...base, digest: "sha256:one" }, undefined, fake.http)).rejects.toThrow(/is stopped/);
+  });
+
+  test("restore puts a recorded Machine config back", async () => {
+    const fake = createMachinesFake();
+    const args = { plan: PLAN, endpoint: ENDPOINT, wait: NO_WAIT };
+    const one = await flyMachineRelease({ ...args, release: { digest: "sha256:one" }, image: "shop:1" }, undefined, fake.http);
+    await flyMachineRelease({ ...args, release: { digest: "sha256:two" }, image: "shop:2" }, undefined, fake.http);
+    expect(fake.machine("shop", "web")!.config.image).toBe("shop:2");
+    const restored = await flyMachineRestore({ app: "shop", machine: "web", config: one.config, endpoint: ENDPOINT, wait: NO_WAIT }, undefined, fake.http);
+    expect(restored.release?.digest).toBe("sha256:one");
+    expect(fake.machine("shop", "web")!.config).toEqual(one.config);
+  });
+});

package/src/op/activities/machine-release.ts

@@ -0,0 +1,407 @@
+/**
+ * The release a Fly Machine serves, and the Machines activities that put one
+ * there (#2736, ws-056).
+ *
+ * chud's `Chud::FlySite` ran a site as one App served by one Machine and kept
+ * the release in the Machine's metadata. That moves here: the App and the
+ * Machine are this lexicon's own resources, and a release is the declared
+ * Machine with {@link RELEASE_METADATA_KEYS} stamped into its
+ * `config.metadata`. Because the record is on the Machine itself, any checkout
+ * can read which release is serving (`describeResources` reports it, and
+ * `chant components status --live` compares it with the release ledger).
+ *
+ * The site steps are activities over the Machines API, each usable from an
+ * Op by name and composed by the `fly-release` and `fly-rollback` capabilities
+ * (../../components/fly-release.ts):
+ *
+ * - {@link flyMachineRelease}: upload and start. Apply the declared plan with
+ *   the Machine serving the release (its image or env overridden when the
+ *   release says so), and report the release and config it replaced.
+ * - {@link flyMachineExec}: run a command inside the Machine (a migration).
+ * - {@link flyMachineRestart}, {@link flyMachineStop}: under a lease, then
+ *   wait for the Machine to settle.
+ * - {@link flyMachineVerify}: the Machine is started with this release in its
+ *   metadata, and, given a URL, its health endpoint answers.
+ * - {@link flyMachineRestore}: put a recorded Machine config back (restore on
+ *   a failed release, and rollback).
+ *
+ * Every activity takes the flaps endpoint and token the way `flyApply` does
+ * (`FLY_FLAPS_BASE_URL`, `FLY_API_TOKEN`), so the same code reaches mudflaps
+ * or a real Fly org, and an injectable `http` for tests (./machines-fake.ts).
+ */
+
+import {
+  appNameFromRequest,
+  defaultFlyHttp,
+  flyApply,
+  isAppRequest,
+  isMachineRequest,
+  listMachines,
+  machineAppSegment,
+  resolveApp,
+  resolveEndpoint,
+  waitForMachine,
+  withLease,
+  LEASE_NONCE_HEADER,
+  type ApplyCtx,
+  type FlapsMachine,
+  type FlapsRequest,
+  type FlyHttp,
+  type FlyPlan,
+  type WaitOpts,
+} from "./fly-apply";
+import { parsePlan } from "./fly-apply";
+import { readFileSync } from "node:fs";
+import { readMachineRelease, withReleaseMetadata, type MachineConfig, type MachineRelease } from "../../release-metadata";
+
+export { RELEASE_METADATA_KEYS, readMachineRelease, withReleaseMetadata } from "../../release-metadata";
+export type { MachineRelease } from "../../release-metadata";
+
+/** The App and Machine a release targets in a plan. */
+export interface ReleaseTarget {
+  app: string;
+  /** The plan entity that declares the Machine. */
+  entity: string;
+  /** The Machine's name on Fly (its `name`, else the entity name). */
+  name: string;
+  request: FlapsRequest;
+}
+
+/**
+ * The Machine a release targets: the plan's one Machine, or the one `machine`
+ * names (an entity name or a Machine name). Pure.
+ */
+export function releaseTarget(plan: FlyPlan, machine?: string): ReleaseTarget {
+  const appNames = Object.values(plan).filter(isAppRequest).map(appNameFromRequest);
+  const soleApp = appNames.length === 1 ? appNames[0] : undefined;
+  const machines = Object.entries(plan)
+    .filter(([, req]) => isMachineRequest(req))
+    .map(([entity, request]) => ({
+      entity,
+      request,
+      name: typeof request.body.name === "string" && request.body.name ? request.body.name : entity,
+    }));
+  const picked = machine ? machines.filter((m) => m.entity === machine || m.name === machine) : machines;
+  if (picked.length !== 1) {
+    throw new Error(
+      machine
+        ? `the fly plan declares no Machine named "${machine}" (declared: ${machines.map((m) => m.entity).join(", ") || "none"})`
+        : `the fly plan declares ${machines.length} Machines (${machines.map((m) => m.entity).join(", ")}): name the one that serves the release`,
+    );
+  }
+  const [m] = picked;
+  return { app: resolveApp(machineAppSegment(m.request.endpoint), soleApp), entity: m.entity, name: m.name, request: m.request };
+}
+
+const TERMINAL = new Set(["destroyed", "destroying"]);
+
+/** The live Machine named `name` (or with id `name`) in `app`; undefined when there is none. */
+export async function findMachine(
+  ctx: ApplyCtx,
+  app: string,
+  name: string,
+  http: FlyHttp,
+  signal?: AbortSignal,
+): Promise<FlapsMachine | undefined> {
+  const live = (await listMachines(ctx, app, http, signal)).filter((m) => !TERMINAL.has(m.state));
+  return live.find((m) => m.name === name) ?? live.find((m) => m.id === name);
+}
+
+async function mustFindMachine(ctx: ApplyCtx, app: string, name: string, http: FlyHttp, signal?: AbortSignal): Promise<FlapsMachine> {
+  const m = await findMachine(ctx, app, name, http, signal);
+  if (!m) throw new Error(`no machine ${name} on app ${app}`);
+  return m;
+}
+
+const machinePath = (ctx: ApplyCtx, app: string, id: string) =>
+  `${ctx.base}/v1/apps/${encodeURIComponent(app)}/machines/${encodeURIComponent(id)}`;
+
+/** Where to reach flaps, as every activity here takes it. */
+export interface FlapsTarget {
+  /** flaps endpoint override. Default: `FLY_FLAPS_BASE_URL`, else real Fly. */
+  endpoint?: string;
+  /** Bearer token for real Fly. Default: `FLY_API_TOKEN`. */
+  token?: string;
+}
+
+// ── flyMachineRelease ─────────────────────────────────────────────────────────
+
+export interface FlyMachineReleaseArgs extends FlapsTarget {
+  /** Path to the fly build output (`chant build --lexicon fly -o <path>`). */
+  planPath?: string;
+  /** The plan itself, in place of `planPath`. */
+  plan?: FlyPlan;
+  /** The Machine that serves the release, when the plan declares more than one. */
+  machine?: string;
+  /** The release to serve. `previousDigest` is filled in from the Machine it replaces. */
+  release: MachineRelease;
+  /** An image to run in place of the declared one (a digest-pinned image a publish step produced). */
+  image?: string;
+  /** Env to add to the declared Machine's env. */
+  env?: Record<string, string>;
+  wait?: WaitOpts;
+}
+
+export interface FlyMachineReleaseResult {
+  app: string;
+  machine: { id: string; name: string };
+  /** What the apply did to the Machine. */
+  action: "created" | "updated" | "noop";
+  /** The release the Machine serves now. */
+  release: MachineRelease;
+  /** The Machine config applied: what a later rollback to this release puts back. */
+  config: MachineConfig;
+  /** What the Machine served before, when it existed: its release (if it named one) and its config. */
+  previous: { release?: MachineRelease; config: MachineConfig } | null;
+}
+
+/**
+ * Upload and start: apply the declared plan (the App, its Volumes, IPs,
+ * Secrets) with the Machine serving `release`. The Machine is updated in place
+ * under a lease, so the release serves from here on. A Machine that already
+ * serves this release keeps the `previousDigest` it shipped with, so running a
+ * release twice never makes it its own predecessor.
+ */
+export async function flyMachineRelease(
+  args: FlyMachineReleaseArgs,
+  signal?: AbortSignal,
+  http: FlyHttp = defaultFlyHttp(args.token),
+): Promise<FlyMachineReleaseResult> {
+  const plan = args.plan ?? parsePlan(readFileSync(requirePlanPath(args.planPath), "utf8"));
+  const target = releaseTarget(plan, args.machine);
+  const ctx: ApplyCtx = { base: resolveEndpoint(args) };
+
+  const live = await findMachine(ctx, target.app, target.name, http, signal).catch(() => undefined);
+  const liveConfig = live?.config as MachineConfig | undefined;
+  const liveRelease = readMachineRelease(liveConfig?.metadata);
+  const previousDigest =
+    liveRelease && liveRelease.digest !== args.release.digest ? liveRelease.digest : liveRelease?.previousDigest ?? args.release.previousDigest;
+  const release: MachineRelease = { ...args.release, ...(previousDigest ? { previousDigest } : {}) };
+  if (!previousDigest) delete release.previousDigest;
+
+  const declared = (target.request.body.config ?? {}) as MachineConfig;
+  const config = withReleaseMetadata(
+    {
+      ...declared,
+      ...(args.image ? { image: args.image } : {}),
+      ...(args.env ? { env: { ...(declared.env ?? {}), ...args.env } } : {}),
+    },
+    release,
+  );
+  const request: FlapsRequest = { ...target.request, body: { ...target.request.body, config } };
+  const applied = await flyApply(
+    { plan: { ...plan, [target.entity]: request }, endpoint: args.endpoint, token: args.token, wait: args.wait },
+    signal,
+    http,
+  );
+  const action = applied.machines.find((m) => m.app === target.app && m.name === target.name)?.action ?? "noop";
+  const now = await mustFindMachine(ctx, target.app, target.name, http, signal);
+  return {
+    app: target.app,
+    machine: { id: now.id, name: target.name },
+    action,
+    release,
+    config,
+    previous: live ? { ...(liveRelease ? { release: liveRelease } : {}), config: liveConfig ?? {} } : null,
+  };
+}
+
+function requirePlanPath(planPath: string | undefined): string {
+  if (!planPath) throw new Error("pass planPath (the fly build output) or plan");
+  return planPath;
+}
+
+// ── flyMachineExec ────────────────────────────────────────────────────────────
+
+export interface FlyMachineExecArgs extends FlapsTarget {
+  app: string;
+  /** The Machine's name or id. */
+  machine: string;
+  /** The command: an argv, or a string run with `sh -c`. */
+  command: string[] | string;
+  /** Seconds the Machines API lets the command run. Default 120. */
+  timeoutSecs?: number;
+}
+
+export interface FlyMachineExecResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+/**
+ * Run a command inside the Machine (the Machines API's exec). A non-zero exit
+ * throws, with the command's output, so the step fails rather than going on
+ * over a half-migrated database.
+ */
+export async function flyMachineExec(
+  args: FlyMachineExecArgs,
+  signal?: AbortSignal,
+  http: FlyHttp = defaultFlyHttp(args.token),
+): Promise<FlyMachineExecResult> {
+  const ctx: ApplyCtx = { base: resolveEndpoint(args) };
+  const m = await mustFindMachine(ctx, args.app, args.machine, http, signal);
+  const command = typeof args.command === "string" ? ["sh", "-c", args.command] : args.command;
+  const res = await http("POST", `${machinePath(ctx, args.app, m.id)}/exec`, { command, timeout: args.timeoutSecs ?? 120 }, undefined, signal);
+  if (res.status >= 300) throw new Error(`exec on ${args.app}/${m.id} failed (${res.status}): ${res.text}`);
+  let out: { exit_code?: number; stdout?: string; stderr?: string } = {};
+  try {
+    out = JSON.parse(res.text) as typeof out;
+  } catch {
+    throw new Error(`exec on ${args.app}/${m.id} answered no exec result: ${res.text}`);
+  }
+  const result = { exitCode: out.exit_code ?? 0, stdout: out.stdout ?? "", stderr: out.stderr ?? "" };
+  if (result.exitCode !== 0) {
+    const said = [result.stderr, result.stdout].filter(Boolean).join("\n").trim().slice(-2000);
+    throw new Error(`${command.join(" ")} exited ${result.exitCode} on ${args.app}/${m.id}${said ? `: ${said}` : ""}`);
+  }
+  return result;
+}
+
+// ── flyMachineRestart / flyMachineStop ────────────────────────────────────────
+
+export interface FlyMachineStateArgs extends FlapsTarget {
+  app: string;
+  /** The Machine's name or id. */
+  machine: string;
+  wait?: WaitOpts;
+}
+
+async function transition(
+  args: FlyMachineStateArgs,
+  verb: "restart" | "stop",
+  signal: AbortSignal | undefined,
+  http: FlyHttp,
+): Promise<{ id: string; state: string }> {
+  const ctx: ApplyCtx = { base: resolveEndpoint(args) };
+  const m = await mustFindMachine(ctx, args.app, args.machine, http, signal);
+  const state = verb === "stop" ? "stopped" : "started";
+  if (verb === "stop" && m.state !== "started") return { id: m.id, state: m.state };
+  const res = await withLease(ctx, args.app, m.id, http, signal, (nonce) =>
+    http("POST", `${machinePath(ctx, args.app, m.id)}/${verb}`, undefined, { [LEASE_NONCE_HEADER]: nonce }, signal),
+  );
+  if (res.status >= 300) throw new Error(`${verb} of machine ${args.app}/${m.id} failed (${res.status}): ${res.text}`);
+  await waitForMachine(ctx, args.app, m.id, "", http, signal, { ...(args.wait ?? {}), state });
+  return { id: m.id, state };
+}
+
+/** Restart the Machine under a lease (the app starts again on migrated data), and wait for it to be started. */
+export async function flyMachineRestart(
+  args: FlyMachineStateArgs,
+  signal?: AbortSignal,
+  http: FlyHttp = defaultFlyHttp(args.token),
+): Promise<{ id: string; state: string }> {
+  return transition(args, "restart", signal, http);
+}
+
+/** Stop the Machine under a lease, and wait for it to be stopped. A Machine that is not started is left alone. */
+export async function flyMachineStop(
+  args: FlyMachineStateArgs,
+  signal?: AbortSignal,
+  http: FlyHttp = defaultFlyHttp(args.token),
+): Promise<{ id: string; state: string }> {
+  return transition(args, "stop", signal, http);
+}
+
+// ── flyMachineVerify ──────────────────────────────────────────────────────────
+
+export interface FlyMachineVerifyArgs extends FlapsTarget {
+  app: string;
+  machine: string;
+  /** The release digest the Machine must name. */
+  digest: string;
+  /** The app's public URL; with it, its health endpoint must answer. */
+  url?: string;
+  /** Path of the health endpoint under `url`. Default `/health`. */
+  healthPath?: string;
+  /** How long to keep asking. Default 120s. */
+  timeoutMs?: number;
+  /** Delay between asks. Default 1s. */
+  intervalMs?: number;
+}
+
+/**
+ * The Machine is started and its metadata names release `digest`. Given a
+ * `url`, its health endpoint answers 2xx too, and when that answer is JSON
+ * naming a `revision` or `digest`, it is this release's (the commit or the
+ * digest). Retries until `timeoutMs`, then throws the last reason.
+ */
+export async function flyMachineVerify(
+  args: FlyMachineVerifyArgs,
+  signal?: AbortSignal,
+  http: FlyHttp = defaultFlyHttp(args.token),
+  fetchImpl: typeof fetch = fetch,
+): Promise<{ id: string; release: MachineRelease }> {
+  const ctx: ApplyCtx = { base: resolveEndpoint(args) };
+  const deadline = Date.now() + (args.timeoutMs ?? 120_000);
+  const interval = args.intervalMs ?? 1_000;
+  let why = "";
+  for (;;) {
+    if (signal?.aborted) throw new Error("flyMachineVerify aborted");
+    try {
+      const m = await mustFindMachine(ctx, args.app, args.machine, http, signal);
+      if (m.state !== "started") throw new Error(`machine ${args.app}/${m.id} is ${m.state}`);
+      const release = readMachineRelease(m.config?.metadata);
+      if (release?.digest !== args.digest) {
+        throw new Error(`machine ${args.app}/${m.id} serves ${release?.digest ?? "no release"}, expected ${args.digest}`);
+      }
+      if (args.url) {
+        const url = `${args.url.replace(/\/$/, "")}${args.healthPath ?? "/health"}`;
+        const res = await fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
+        if (!res.ok) throw new Error(`${url} answered ${res.status}`);
+        let body: { revision?: unknown; digest?: unknown } | undefined;
+        try {
+          body = (await res.json()) as typeof body;
+        } catch {
+          body = undefined;
+        }
+        const reported = typeof body?.revision === "string" ? body.revision : typeof body?.digest === "string" ? body.digest : undefined;
+        if (reported !== undefined && reported !== release.gitSha && reported !== release.digest) {
+          throw new Error(`${url} reports ${reported}, expected ${release.gitSha ?? release.digest}`);
+        }
+      }
+      return { id: m.id, release };
+    } catch (err) {
+      why = err instanceof Error ? err.message : String(err);
+    }
+    if (Date.now() + interval > deadline) throw new Error(`verify ${args.app}/${args.machine}: ${why}`);
+    await new Promise((r) => setTimeout(r, interval));
+  }
+}
+
+// ── flyMachineRestore ─────────────────────────────────────────────────────────
+
+export interface FlyMachineRestoreArgs extends FlapsTarget {
+  app: string;
+  machine: string;
+  /** The Machine config to put back, as a release applied it (its metadata names that release). */
+  config: MachineConfig;
+  wait?: WaitOpts;
+}
+
+/**
+ * Put a recorded Machine config back: update the Machine to exactly `config`
+ * under a lease and wait for it to be started. Used on a failed release (the
+ * config it replaced) and by a rollback (the config an earlier release applied).
+ */
+export async function flyMachineRestore(
+  args: FlyMachineRestoreArgs,
+  signal?: AbortSignal,
+  http: FlyHttp = defaultFlyHttp(args.token),
+): Promise<{ id: string; release?: MachineRelease }> {
+  const ctx: ApplyCtx = { base: resolveEndpoint(args) };
+  const m = await mustFindMachine(ctx, args.app, args.machine, http, signal);
+  const res = await withLease(ctx, args.app, m.id, http, signal, (nonce) =>
+    http("POST", machinePath(ctx, args.app, m.id), { name: m.name, config: args.config }, { [LEASE_NONCE_HEADER]: nonce }, signal),
+  );
+  if (res.status >= 300) throw new Error(`restore of machine ${args.app}/${m.id} failed (${res.status}): ${res.text}`);
+  let instance = "";
+  try {
+    instance = (JSON.parse(res.text) as { instance_id?: string }).instance_id ?? "";
+  } catch {
+    instance = "";
+  }
+  await waitForMachine(ctx, args.app, m.id, instance, http, signal, args.wait ?? {});
+  const release = readMachineRelease(args.config.metadata);
+  return { id: m.id, ...(release ? { release } : {}) };
+}

package/src/op/activities/machines-contract.docker.integration.test.ts

@@ -1,6 +1,6 @@
 import { describe, test, expect, beforeAll, afterAll } from "vitest";
 import { flapsUp, flapsDown } from "./flaps";
-import { MACHINES_CONTRACT, normalizeEndpoint, contractKeys } from "./machines-contract";
+import { MACHINES_CONTRACT, MACHINE_RELEASE_CONTRACT, normalizeEndpoint, contractKeys } from "./machines-contract";
 
 // Fidelity check: every flaps endpoint the flyApply applier depends on
 // (MACHINES_CONTRACT) must be served by the pinned mudflaps image — the twin of
@@ -69,4 +69,13 @@ describe("Machines contract ⊆ mudflaps implemented paths", () => {
       expect(roadmap.has(key), `${e.op} → ${e.method} ${e.path} is a mudflaps roadmap endpoint`).toBe(false);
     }
   });
+
+  test("the pinned mudflaps serves every endpoint the release activities depend on (#2736)", async (ctx) => {
+    if (!available) ctx.skip();
+    const res = await fetch(`${endpoint}/_mudflaps/health`);
+    const health = (await res.json()) as { implemented?: string[] };
+    const served = new Set((health.implemented ?? []).map(normalizeImplemented));
+    const missing = MACHINE_RELEASE_CONTRACT.map((e) => normalizeEndpoint(e.method, e.path)).filter((k) => !served.has(k));
+    expect(missing, `mudflaps is missing release endpoints: ${missing.join(", ")}`).toEqual([]);
+  });
 });