@intentius/chant-lexicon-fly 0.124.67 → 0.124.68

package/dist/op/activities/machine-release.js

@@ -19,12 +19,16 @@
  *   release says so), and report the release and config it replaced. A
  *   scheduled Machine (`config.schedule`, #4696) is updated without starting:
  *   it runs on its schedule. The plan's other Machines can take images of
- *   their own (`images`).
+ *   their own (`images`). Given `checks` (#4719), each Machine it starts must
+ *   pass its Fly checks before the next is updated, so a group of serving
+ *   Machines rolls one at a time.
  * - {@link flyMachineExec}: run a command inside the Machine (a migration).
  * - {@link flyMachineRestart}, {@link flyMachineStop}: under a lease, then
  *   wait for the Machine to settle.
  * - {@link flyMachineVerify}: the Machine is started with this release in its
- *   metadata, and, given a URL, its health endpoint answers.
+ *   metadata, and, given a URL, its health endpoint answers. With `checks`,
+ *   its Fly checks pass too.
+ * - {@link flyMachineChecks}: the Machine is started and its Fly checks pass.
  * - {@link flyMachineRestore}: put a recorded Machine config back (restore on
  *   a failed release, and rollback).
  *
@@ -32,7 +36,7 @@
  * (`FLY_FLAPS_BASE_URL`, `FLY_API_TOKEN`), so the same code reaches mudflaps
  * or a real Fly org, and an injectable `http` for tests (./machines-fake.ts).
  */
-import { appNameFromRequest, defaultFlyHttp, flyApply, isAppRequest, isMachineRequest, listMachines, machineAppSegment, resolveApp, resolveEndpoint, skipsLaunch, waitForMachine, waitForUnlaunched, withLease, LEASE_NONCE_HEADER, } from "./fly-apply.js";
+import { appNameFromRequest, configEqual, defaultFlyHttp, flyApply, isAppRequest, isMachineRequest, listMachines, machineAppSegment, resolveApp, resolveEndpoint, resolveMounts, skipsLaunch, waitForMachine, waitForUnlaunched, withLease, LEASE_NONCE_HEADER, } from "./fly-apply.js";
 import { parsePlan } from "./fly-apply.js";
 import { readFileSync } from "node:fs";
 import { readMachineRelease, withReleaseMetadata } from "../../release-metadata.js";
@@ -77,6 +81,66 @@ async function mustFindMachine(ctx, app, name, http, signal) {
     return m;
 }
 const machinePath = (ctx, app, id) => `${ctx.base}/v1/apps/${encodeURIComponent(app)}/machines/${encodeURIComponent(id)}`;
+/** Five minutes, as `fly deploy`'s `--wait-timeout`. */
+export const DEFAULT_CHECKS_TIMEOUT_MS = 300000;
+/**
+ * True when a Machine config declares a Fly check: one under `checks`, or
+ * one on any of its `services`. Pure.
+ */
+export function declaresChecks(config) {
+    const c = (config ?? {});
+    if (c.checks && typeof c.checks === "object" && Object.keys(c.checks).length > 0)
+        return true;
+    const services = Array.isArray(c.services) ? c.services : [];
+    return services.some((svc) => Array.isArray(svc?.checks) && svc.checks.length > 0);
+}
+/**
+ * Why a Machine's Fly checks are not all passing, or undefined when they are
+ * or it has none. A Machine whose config declares checks that have not
+ * reported yet is not passing.
+ *
+ * Given `since`, the Machine's own `updated_at` as the call that changed it
+ * answered it, a check counts as passing only when its `updated_at` is at or
+ * after that time: a `passing` left over from before the change is stale.
+ * Both times come from flaps, so the local clock plays no part. A check
+ * that reports no `updated_at` cannot be told stale and counts on its status.
+ * Pure.
+ */
+export function checksPending(m, since) {
+    const checks = m.checks ?? [];
+    if (checks.length === 0)
+        return declaresChecks(m.config) ? "no check has reported yet" : undefined;
+    const from = since ? Date.parse(since) : NaN;
+    const stale = (c) => {
+        const at = c.updated_at ? Date.parse(c.updated_at) : NaN;
+        return !Number.isNaN(from) && !Number.isNaN(at) && at < from;
+    };
+    const failing = checks.filter((c) => c.status !== "passing" || stale(c));
+    if (failing.length === 0)
+        return undefined;
+    return failing
+        .map((c) => {
+        if (c.status === "passing")
+            return `check ${c.name} has not reported since the update at ${since} (last at ${c.updated_at})`;
+        const output = (c.output ?? "").trim().slice(0, 200);
+        return `check ${c.name} is ${c.status || "unknown"}${output ? ` (${output})` : ""}`;
+    })
+        .join("; ");
+}
+/**
+ * A release that failed partway through its apply (#4719): a Machine that
+ * did not settle or whose checks did not pass. `applied` holds the Machines
+ * it applied before the failure, the failing one included when its update
+ * went through, so the caller can put each back.
+ */
+export class MachineReleaseError extends Error {
+    constructor(reason, applied) {
+        super(reason instanceof Error ? reason.message : String(reason));
+        this.name = "MachineReleaseError";
+        this.applied = applied;
+        this.reason = reason;
+    }
+}
 /**
  * Upload and start: apply the declared plan (the App, its Volumes, IPs,
  * Secrets) with the Machine serving `release`. The Machine is updated in place
@@ -88,6 +152,11 @@ const machinePath = (ctx, app, id) => `${ctx.base}/v1/apps/${encodeURIComponent(
  * another in the plan, is created or updated with `skip_launch` and left
  * unstarted: Fly starts it on its schedule. Fly counts the interval from the
  * update, so each release pushes the next run back by up to one interval.
+ *
+ * Given `checks` (#4719), each Machine the apply starts must pass its Fly
+ * checks, reported since its update, before the next one is updated. With or
+ * without them, a failure partway through the apply throws a
+ * {@link MachineReleaseError} naming the Machines it changed so far.
  */
 export async function flyMachineRelease(args, signal, http = defaultFlyHttp(args.token)) {
     const plan = args.plan ?? parsePlan(readFileSync(requirePlanPath(args.planPath), "utf8"));
@@ -134,7 +203,36 @@ export async function flyMachineRelease(args, signal, http = defaultFlyHttp(args
         const found = await findMachine(ctx, m.app, m.name, http, signal).catch(() => undefined);
         before.set(m.entity, found ? (found.config ?? {}) : null);
     }
-    const applied = await flyApply({ plan: { ...plan, ...overrides, [target.entity]: request }, endpoint: args.endpoint, token: args.token, wait: args.wait }, signal, http);
+    const rolled = [];
+    const afterMachine = async (m) => {
+        const serving = m.entity === target.entity;
+        const previous = serving ? (live ? (liveConfig ?? {}) : null) : (before.get(m.entity) ?? null);
+        rolled.push({ app: m.app, name: m.name, id: m.id, action: m.action, unlaunched: m.unlaunched, previous, serving, ...(m.updatedAt ? { updatedAt: m.updatedAt } : {}) });
+        if (args.checks && m.action !== "noop" && !m.unlaunched) {
+            await waitForChecks(ctx, m.app, m.name, http, signal, args.checks, m.updatedAt);
+        }
+    };
+    let applied;
+    try {
+        applied = await flyApply({ plan: { ...plan, ...overrides, [target.entity]: request }, endpoint: args.endpoint, token: args.token, wait: args.wait, afterMachine }, signal, http);
+    }
+    catch (err) {
+        // A Machine whose update went through but which never settled is not in
+        // `rolled` yet: find it by its config, so it is put back too.
+        const reached = new Set(rolled.map((m) => `${m.app}/${m.name}`));
+        for (const m of planMachines(plan)) {
+            if (reached.has(`${m.app}/${m.name}`))
+                continue;
+            const serving = m.entity === target.entity;
+            const was = serving ? (live ? (liveConfig ?? {}) : null) : (before.get(m.entity) ?? null);
+            const now = await findMachine(ctx, m.app, m.name, http, signal).catch(() => undefined);
+            if (!now || (was && configEqual(now.config, was)))
+                continue;
+            const unlaunched = skipsLaunch((serving ? request : (overrides[m.entity] ?? m.request)).body);
+            rolled.push({ app: m.app, name: m.name, id: now.id, action: was ? "updated" : "created", unlaunched, previous: was, serving });
+        }
+        throw new MachineReleaseError(err, rolled);
+    }
     const actionOf = (app, name) => applied.machines.find((m) => m.app === app && m.name === name)?.action ?? "noop";
     const now = await mustFindMachine(ctx, target.app, target.name, http, signal);
     return {
@@ -152,6 +250,7 @@ export async function flyMachineRelease(args, signal, http = defaultFlyHttp(args
             unlaunched: skipsLaunch((overrides[m.entity] ?? m.request).body),
             previous: before.get(m.entity) ?? null,
         })),
+        applied: rolled,
     };
 }
 function requirePlanPath(planPath) {
@@ -230,6 +329,11 @@ export async function flyMachineVerify(args, signal, http = defaultFlyHttp(args.
             if (release?.digest !== args.digest) {
                 throw new Error(`machine ${args.app}/${m.id} serves ${release?.digest ?? "no release"}, expected ${args.digest}`);
             }
+            if (args.checks && m.state === "started") {
+                const pending = checksPending(m, args.checksSince);
+                if (pending)
+                    throw new Error(`machine ${args.app}/${args.machine} (${m.id}): ${pending}`);
+            }
             if (args.url) {
                 const url = `${args.url.replace(/\/$/, "")}${args.healthPath ?? "/health"}`;
                 const res = await fetchImpl(url, { signal: AbortSignal.timeout(5000) });
@@ -262,27 +366,62 @@ export async function flyMachineVerify(args, signal, http = defaultFlyHttp(args.
  * under a lease and wait for it to be started. Used on a failed release (the
  * config it replaced) and by a rollback (the config an earlier release applied).
  * A scheduled config (#4696) goes back with `skip_launch`, and the wait is for
- * the Machine to rest unstarted.
+ * the Machine to rest unstarted. A mount that names a volume goes back with
+ * the volume's id, looked up as `flyApply` looks it up (#4717); a mount with an
+ * id (a config read from the live Machine) goes back as it is.
  */
 export async function flyMachineRestore(args, signal, http = defaultFlyHttp(args.token)) {
     const ctx = { base: resolveEndpoint(args) };
     const m = await mustFindMachine(ctx, args.app, args.machine, http, signal);
     const unlaunched = skipsLaunch({ config: args.config });
-    const body = { name: m.name, config: args.config, ...(unlaunched ? { skip_launch: true } : {}) };
+    const config = await resolveMounts(ctx, args.app, args.config, { machineId: m.id, region: m.region }, http, signal);
+    const body = { name: m.name, config, ...(unlaunched ? { skip_launch: true } : {}) };
     const res = await withLease(ctx, args.app, m.id, http, signal, (nonce) => http("POST", machinePath(ctx, args.app, m.id), body, { [LEASE_NONCE_HEADER]: nonce }, signal));
     if (res.status >= 300)
         throw new Error(`restore of machine ${args.app}/${m.id} failed (${res.status}): ${res.text}`);
     let instance = "";
+    let updatedAt;
     try {
-        instance = JSON.parse(res.text).instance_id ?? "";
+        const answered = JSON.parse(res.text);
+        instance = answered.instance_id ?? "";
+        updatedAt = answered.updated_at;
     }
     catch {
         instance = "";
     }
     if (unlaunched)
         await waitForUnlaunched(ctx, args.app, m.id, instance, http, signal, args.wait ?? {});
-    else
+    else {
         await waitForMachine(ctx, args.app, m.id, instance, http, signal, args.wait ?? {});
+        if (args.checks)
+            await waitForChecks(ctx, args.app, args.machine, http, signal, args.checks, updatedAt);
+    }
     const release = readMachineRelease(args.config.metadata);
     return { id: m.id, ...(release ? { release } : {}) };
 }
+/**
+ * Wait until the Machine is started and every one of its Fly checks
+ * (`checks[].status` on the Machine) is `passing`, and, given `since`, has
+ * reported since then. A Machine with no checks is done once started. Throws after `timeoutMs`, naming the Machine and each
+ * check that is not passing.
+ */
+export async function flyMachineChecks(args, signal, http = defaultFlyHttp(args.token)) {
+    return waitForChecks({ base: resolveEndpoint(args) }, args.app, args.machine, http, signal, args, args.since);
+}
+async function waitForChecks(ctx, app, name, http, signal, opts, since) {
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_CHECKS_TIMEOUT_MS;
+    const interval = opts.intervalMs ?? 1000;
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        if (signal?.aborted)
+            throw new Error(`waiting for the checks of ${app}/${name} aborted`);
+        const m = await mustFindMachine(ctx, app, name, http, signal);
+        const why = m.state === "started" ? checksPending(m, since) : `it is ${m.state}`;
+        if (!why)
+            return { id: m.id };
+        if (Date.now() + interval > deadline) {
+            throw new Error(`machine ${app}/${name} (${m.id}) did not pass its Fly checks within ${timeoutMs >= 1000 ? `${Math.round(timeoutMs / 1000)}s` : `${timeoutMs}ms`}: ${why}`);
+        }
+        await new Promise((r) => setTimeout(r, interval));
+    }
+}

package/dist/op/activities/machines-fake.d.ts

@@ -22,6 +22,18 @@ export interface FakeMachine {
     config: Record<string, unknown> & {
         metadata?: Record<string, string>;
     };
+    /** When the machine was last created or updated (RFC 3339), as flaps reports it. */
+    updated_at?: string;
+}
+/** One volume the fake holds, as flaps lists it. */
+export interface FakeVolume {
+    id: string;
+    name: string;
+    region?: string;
+    size_gb?: number;
+    state: string;
+    /** The Machine whose config mounts it by id; null when none does. */
+    attached_machine_id: string | null;
 }
 /** What an `exec` answers: the Machines API's exec envelope. */
 export interface FakeExecResult {
@@ -33,6 +45,8 @@ export interface MachinesFake {
     http: FlyHttp;
     apps: Set<string>;
     machines: Map<string, FakeMachine[]>;
+    /** Each app's volumes, in creation order, with ids `vol_...` (#4717). */
+    volumes: Map<string, FakeVolume[]>;
     /** Each app's certificate hostnames, in creation order. */
     certs: Map<string, string[]>;
     /** Every exec, in order: the app, the machine id and the command. */
@@ -51,6 +65,22 @@ export interface MachinesFakeOptions {
     exec?: (app: string, machine: FakeMachine, command: string[]) => FakeExecResult | Promise<FakeExecResult>;
     /** The state a created or updated machine settles in. Default `started`. */
     settle?: (machine: FakeMachine) => string;
+    /** The clock a create or update stamps `updated_at` with. Default: the wall clock. */
+    now?: () => Date;
+    /**
+     * The Fly checks a machine reports each time it is read (#4719), as the
+     * Machines API's `checks`: asked on every GET of the machine or its list.
+     * Default: none.
+     */
+    checks?: (app: string, machine: FakeMachine) => FakeCheck[] | undefined;
+}
+/** One Fly check's status on a machine, as the Machines API reports it. */
+export interface FakeCheck {
+    name: string;
+    status: "passing" | "warning" | "critical";
+    output?: string;
+    /** When the check last reported (RFC 3339). */
+    updated_at?: string;
 }
 /** Fly's certificate-list page size: 25 when no `limit` is given, 500 at most. */
 export declare const CERT_PAGE_DEFAULT = 25;

package/dist/op/activities/machines-fake.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"machines-fake.d.ts","sourceRoot":"","sources":["../../../src/op/activities/machines-fake.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE3C,kCAAkC;AAClC,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,gEAAgE;AAChE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IACrC,2DAA2D;IAC3D,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7B,qEAAqE;IACrE,KAAK,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IAC7D,wDAAwD;IACxD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,sDAAsD;IACtD,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7D;AAED,MAAM,WAAW,mBAAmB;IAClC,gIAAgI;IAChI,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1G,4EAA4E;IAC5E,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;CAC3C;AAID,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,aAAa,MAAM,CAAC;AAWjC;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,GAAE,mBAAwB,GAAG,YAAY,CAkKlF"}
+{"version":3,"file":"machines-fake.d.ts","sourceRoot":"","sources":["../../../src/op/activities/machines-fake.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE3C,kCAAkC;AAClC,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;IACxE,oFAAoF;IACpF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,oDAAoD;AACpD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,qEAAqE;IACrE,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,gEAAgE;AAChE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IACrC,yEAAyE;IACzE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACnC,2DAA2D;IAC3D,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7B,qEAAqE;IACrE,KAAK,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IAC7D,wDAAwD;IACxD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,sDAAsD;IACtD,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7D;AAED,MAAM,WAAW,mBAAmB;IAClC,gIAAgI;IAChI,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1G,4EAA4E;IAC5E,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;IAC1C,sFAAsF;IACtF,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB;;;;OAIG;IACH,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,SAAS,EAAE,GAAG,SAAS,CAAC;CACzE;AAED,2EAA2E;AAC3E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,aAAa,MAAM,CAAC;AAWjC;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,GAAE,mBAAwB,GAAG,YAAY,CAwNlF"}

package/dist/op/activities/machines-fake.js

@@ -32,12 +32,30 @@ const nextId = (prefix) => `${prefix}${(++seq).toString(16).padStart(10, "0")}`;
 export function createMachinesFake(options = {}) {
     const apps = new Set();
     const machines = new Map();
+    const volumes = new Map();
     const certs = new Map();
     const execs = [];
     const calls = [];
     const settle = options.settle ?? (() => "started");
+    const stamp = () => (options.now?.() ?? new Date()).toISOString();
     const list = (app) => machines.get(app) ?? [];
     const find = (app, id) => list(app).find((m) => m.id === id);
+    const withChecks = (app, m) => {
+        const checks = options.checks?.(app, m);
+        return checks ? { ...m, checks } : m;
+    };
+    const vols = (app) => volumes.get(app) ?? [];
+    // A volume is attached to the Machine whose config mounts its id, as Fly
+    // attaches it. A mount naming no volume id attaches nothing.
+    const attach = (app, m, id) => {
+        const mounted = new Set((m?.config.mounts ?? []).map((x) => x.volume).filter((v) => typeof v === "string"));
+        for (const v of vols(app)) {
+            if (v.attached_machine_id === id && !mounted.has(v.id))
+                v.attached_machine_id = null;
+            if (m && mounted.has(v.id))
+                v.attached_machine_id = id;
+        }
+    };
     const http = async (method, url, body) => {
         const { pathname, searchParams } = new URL(url);
         calls.push(`${method} ${pathname}`);
@@ -63,6 +81,7 @@ export function createMachinesFake(options = {}) {
             if (method === "DELETE") {
                 apps.delete(app);
                 machines.delete(app);
+                volumes.delete(app);
                 certs.delete(app);
                 return json(202, undefined);
             }
@@ -75,7 +94,7 @@ export function createMachinesFake(options = {}) {
             const action = seg[5];
             if (!id) {
                 if (method === "GET")
-                    return json(200, list(app));
+                    return json(200, list(app).map((m) => withChecks(app, m)));
                 if (method === "POST") {
                     const m = {
                         id: nextId("m"),
@@ -83,10 +102,12 @@ export function createMachinesFake(options = {}) {
                         state: "created",
                         instance_id: nextId("I"),
                         config: structuredClone((b.config ?? {})),
+                        updated_at: stamp(),
                     };
                     // skip_launch: the Machine is created and not booted (#4696).
                     m.state = b.skip_launch === true ? "created" : settle(m);
                     machines.set(app, [...list(app), m]);
+                    attach(app, m, m.id);
                     return json(200, m);
                 }
             }
@@ -95,10 +116,12 @@ export function createMachinesFake(options = {}) {
                 return action === "wait" ? json(200, { ok: true }) : json(404, { error: "machine not found" });
             if (!action) {
                 if (method === "GET")
-                    return json(200, m);
+                    return json(200, withChecks(app, m));
                 if (method === "POST") {
                     m.config = structuredClone((b.config ?? {}));
                     m.instance_id = nextId("I");
+                    m.updated_at = stamp();
+                    attach(app, m, m.id);
                     // skip_launch: the new config isn't started (#4696). A Machine that
                     // never ran stays created; one that ran is stopped.
                     m.state = b.skip_launch === true ? (m.state === "created" ? "created" : "stopped") : settle(m);
@@ -113,6 +136,7 @@ export function createMachinesFake(options = {}) {
                         });
                     }
                     machines.set(app, list(app).filter((x) => x !== m));
+                    attach(app, undefined, m.id);
                     return json(200, { ok: true });
                 }
             }
@@ -144,10 +168,42 @@ export function createMachinesFake(options = {}) {
             return method === "GET" ? json(200, { ips: [] }) : json(200, {});
         if (kind === "certificates")
             return certificates(app, method, seg.slice(4), b, searchParams);
-        if (kind === "volumes" || kind === "secrets")
+        if (kind === "volumes")
+            return volumeCall(app, method, seg[4], b);
+        if (kind === "secrets")
             return method === "GET" ? json(200, []) : json(200, {});
         return json(404, { error: "not found" });
     };
+    // Volumes (#4717): created with a `vol_` id, listed with their region and
+    // the Machine they are attached to, read and deleted by id.
+    function volumeCall(app, method, id, b) {
+        if (!id) {
+            if (method === "GET")
+                return json(200, vols(app));
+            if (method === "POST") {
+                const v = {
+                    id: nextId("vol_"),
+                    name: String(b.name ?? ""),
+                    ...(typeof b.region === "string" ? { region: b.region } : {}),
+                    ...(typeof b.size_gb === "number" ? { size_gb: b.size_gb } : {}),
+                    state: "created",
+                    attached_machine_id: null,
+                };
+                volumes.set(app, [...vols(app), v]);
+                return json(200, v);
+            }
+        }
+        const v = vols(app).find((x) => x.id === id);
+        if (!v)
+            return json(404, { error: "volume not found" });
+        if (method === "GET")
+            return json(200, v);
+        if (method === "DELETE") {
+            volumes.set(app, vols(app).filter((x) => x !== v));
+            return json(200, { ...v, state: "destroyed" });
+        }
+        return json(404, { error: `no ${method} on a volume` });
+    }
     // Certificates, as the real API routes them (#3114): listed at
     // `.../certificates`, created only at `.../certificates/acme`, read and
     // deleted at `.../certificates/{hostname}`. A POST to the bare list path is
@@ -198,6 +254,7 @@ export function createMachinesFake(options = {}) {
         http,
         apps,
         machines,
+        volumes,
         certs,
         execs,
         calls,

package/dist/rules/fly011-mount-references-declared-volume.ts

@@ -6,17 +6,22 @@
  * apply time, so catch it at synth. This is a whole-stack check: it reads all
  * entities in the build (`ctx.entities`), so the mount and the Volume can be
  * declared in different files.
+ *
+ * A mount given a volume id (`vol_...`) names a volume that already exists on
+ * Fly, which the stack need not declare, so it passes (#4717). `flyApply`
+ * sends it as it is, and looks a name up to send its id.
  */
 
 import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
 import { readProps, entityTypeOf } from "./fly-helpers";
+import { isVolumeId } from "../../volume-id";
 
 const MACHINE_ENTITY_TYPE = "Fly::Machines::Machine";
 const VOLUME_ENTITY_TYPE = "Fly::Machines::Volume";
 
 export const fly011: PostSynthCheck = {
   id: "FLY011",
-  description: "A machine mount's volume must resolve to a Volume declared in the stack",
+  description: "A machine mount's volume must resolve to a Volume declared in the stack, or be a volume id (vol_...)",
 
   check(ctx: PostSynthContext): PostSynthDiagnostic[] {
     const diagnostics: PostSynthDiagnostic[] = [];
@@ -44,11 +49,13 @@ export const fly011: PostSynthCheck = {
         // reference to a Volume that exists in the stack by construction, so it
         // never dangles — skip it to avoid false positives.
         if (typeof volume !== "string" || volume.length === 0) continue;
+        // A volume id names a volume that exists outside the stack.
+        if (isVolumeId(volume)) continue;
         if (!declaredVolumes.has(volume)) {
           diagnostics.push({
             checkId: "FLY011",
             severity: "error",
-            message: `Machine "${machineName}" mounts volume "${volume}", which is not declared as a Volume in the stack.`,
+            message: `Machine "${machineName}" mounts volume "${volume}", which is not declared as a Volume in the stack. Declare the Volume, or mount an existing volume by its id (vol_...).`,
             entity: machineName,
             lexicon: "fly",
           });

package/dist/skills/chant-fly-ops.md

@@ -89,13 +89,37 @@ export const web: Component = {
 };
 ```
 
-The step updates the Machine in place with the release in its `config.metadata`: `chant-release-digest`, `chant-release-git-sha` (the commit, `git rev-parse HEAD` unless given), and `chant-release-previous-digest` (the release it replaced). Each migration then runs inside the Machine through the Machines API's exec, once per environment: its receipt is kept on `chant/lifecycle` at `<env>/receipts/`, and a migration fires again only when its `sha` (or command) changes. After a migration fires, the Machine restarts, and the step checks that it is started with this release and, given a `url`, that its health endpoint answers with this commit or digest. If anything fails after the Machine changed, the step puts back the config the Machine served before (on a first release, it stops the Machine) and fails.
+The step updates the Machine in place with the release in its `config.metadata`: `chant-release-digest`, `chant-release-git-sha` (the commit, `git rev-parse HEAD` unless given), and `chant-release-previous-digest` (the release it replaced). Each migration then runs inside the Machine through the Machines API's exec, once per environment: its receipt is kept on `chant/lifecycle` at `<env>/receipts/`, and a migration fires again only when its `sha` (or command) changes. After a migration fires, the Machine restarts, and the step checks that it is started with this release and, given a `url`, that its health endpoint answers with this commit or digest. If anything fails after the Machine changed, the step puts back the config the Machine served before (on a first release, it stops the Machine) and fails. That includes an apply that fails partway: the Machines it had already updated go back too.
 
 The step's output carries `uri` and `digest`, so `chant run --components web --env prod` records the release in the ledger. `chant components status prod --live` reads the Machine's metadata back and compares its digest with the ledger's: the row is `reconciled` when they agree and `drifted` when the Machine serves something the ledger does not name. The component joins the Machine by name: name the component after the Machine entity, or list it in `liveNames`.
 
 Each release's Machine config is kept on `chant/lifecycle` at `<env>/fly/<app>/<machine>/<digest>.json`. `fly-rollback` puts back the config of the release the serving one replaced (or the digest given as `to`), checks it, and outputs that digest so the ledger records it again. `fly-release`'s own saga compensation does the same.
 
-The steps are also Op activities, for an Op that composes them itself: `flyMachineRelease`, `flyMachineExec`, `flyMachineRestart`, `flyMachineStop`, `flyMachineVerify` and `flyMachineRestore`. Wrap a migration's `flyMachineExec` in `effect()` so it fires once.
+### Several serving Machines with Fly checks
+
+An app served by more than one Machine, such as two Machines behind Flycast with no public address, declares them all in the plan. `fly-release` releases the one named by `machine` and gives the others their image through `images`:
+
+```ts
+{
+  kind: "fly-release",
+  plan: "dist/fly.json",
+  machine: "platform-1",
+  digest: "@Publish.digest",
+  image: "@Publish.uri",
+  images: { "platform-2": "@Publish.uri" },
+  checks: { timeoutMs: 300_000 },
+}
+```
+
+Without `checks`, the apply updates the Machines in plan order and waits for each to be `started` only, and verify reads the serving Machine alone.
+
+With `checks`, the apply updates the Machines one at a time, as `fly deploy` does. Once a Machine is started, the step reads its Fly checks from the Machines API (`checks[].status` on the Machine) until every one is `passing`, and only then updates the next Machine. A check counts only when its `updated_at` is at or after the Machine's own `updated_at` as the update call answered it, so a `passing` left over from before the update does not count. Both times are flaps' own, so the local clock plays no part; a check with no `updated_at` counts on its status. A Machine whose config declares checks that have not reported yet is still waiting. `checks.timeoutMs` (default 300000, five minutes) bounds the wait for each Machine, and `checks.intervalMs` (default 1000) is how often it looks; `checks: true` takes both defaults. A Machine with no checks configured is done once it is started, and a scheduled Machine is not waited for. The wait comes right after each Machine's update, before migrations run, so a release whose new code can't pass its checks until a migration has run should leave `checks` off.
+
+Under `checks`, a Machine whose checks do not pass in time fails the release with an error naming the Machine and each check that is not passing, for example `machine platform/platform-2 (e784...) did not pass its Fly checks within 300s: check servicecheck-00-http-8080 is critical (503 Service Unavailable)`. The Machines after it are left alone. Every Machine the release changed goes back to its old config, the last one changed first, and each restore waits for its Machine to be started with its checks passing before the next one. On a first release the serving Machine is stopped instead.
+
+Under `checks`, verify then covers every Machine the release started: the serving one is started with this release and its checks passing (and its health URL answering, when given), and each other one is started with its checks passing, within `verify.timeoutMs`. A failure there restores them all in the same way.
+
+The steps are also Op activities, for an Op that composes them itself: `flyMachineRelease`, `flyMachineExec`, `flyMachineRestart`, `flyMachineStop`, `flyMachineVerify` and `flyMachineRestore`. Wrap a migration's `flyMachineExec` in `effect()` so it fires once. `flyMachineRelease` and `flyMachineRestore` wait for Fly checks when given `checks`, and `flyMachineVerify` reads them with `checks: true`; without them they wait for `started` only.
 
 ### A source tree instead of an image
 

package/dist/skills/chant-fly-patterns.md

@@ -10,9 +10,11 @@ Beyond the App and Machine covered in `chant-fly`, the lexicon models `Volume`,
 
 ## Volumes and mounts
 
-`flyApply` applies volumes before machines, because a machine's `config.mounts[]` references a volume by name, so the volume must exist first. A volume is created if absent (idempotent by name); a re-apply of an existing volume is a no-op.
+`flyApply` applies volumes before machines. A volume is created if absent (idempotent by name); a re-apply of an existing volume is a no-op.
 
-The FLY011 build check enforces the link statically: every machine mount must reference a `Volume` declared in the stack, checked across files. A mount that points at an undeclared volume fails `chant build` before anything reaches the API.
+A mount can name its volume. The Machines API takes the volume's id in `config.mounts[].volume`, so before it creates or updates a Machine, `flyApply` looks the name up among the app's live volumes and sends `{ volume: "vol_...", name: "data", path }`, as flyctl does. A volume declared in the same apply is created first, so its id is there to look up. Fly lets several volumes share a name (one per Machine), so the lookup takes the one already attached to the Machine, else the one in the Machine's region that no Machine has. No volume of that name, or more than one that fits, fails the apply naming the candidates; give the mount the id to settle it. A mount given an id (`vol_...`) is sent as it is. `flyMachineRestore` and the `fly-release` capability do the same lookup. The build output and a release's recorded config keep the name.
+
+The FLY011 build check enforces the link statically: every machine mount must reference a `Volume` declared in the stack, checked across files, or give a volume id. A mount that points at an undeclared volume fails `chant build` before anything reaches the API.
 
 ```ts
 import { App, Machine, MachineConfig, MachineGuest, Volume, Fly } from "@intentius/chant-lexicon-fly";

package/dist/skills/chant-fly.md

@@ -72,7 +72,7 @@ Build synthesizes the flaps plan and runs the lint rules before anything reaches
 | FLY002 | Guest sizing (`cpu_kind` / `cpus` / `memory_mb`) is not a valid combination |
 | FLY004 | A secret value written inline in machine config |
 | FLY010 | A machine config with no `image` |
-| FLY011 | A machine mount that references a `Volume` not declared in the stack (checked across files) |
+| FLY011 | A machine mount that references a `Volume` not declared in the stack (checked across files); a volume id (`vol_...`) passes |
 | FLY012 | A machine `schedule` other than `hourly`, `daily`, `weekly` or `monthly` |
 
 Fix every reported violation before applying. Secret values belong in a `Secret` or a reference, never inline (FLY004).

package/dist/volume-id.d.ts

@@ -0,0 +1,9 @@
+/**
+ * A Fly volume's id, as the Machines API gives it (`vol_r68lmz2doox9myd4`).
+ * A Machine's `config.mounts[].volume` takes this id; chant also lets it name
+ * a volume, and `flyApply` looks the name up (#4717). Kept apart from the
+ * applier so the FLY011 lint check can use it without loading the applier.
+ */
+/** True when `value` is a volume id (`vol_...`), not a volume's name. Pure. */
+export declare function isVolumeId(value: string): boolean;
+//# sourceMappingURL=volume-id.d.ts.map

package/dist/volume-id.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"volume-id.d.ts","sourceRoot":"","sources":["../src/volume-id.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEjD"}

package/dist/volume-id.js

@@ -0,0 +1,10 @@
+/**
+ * A Fly volume's id, as the Machines API gives it (`vol_r68lmz2doox9myd4`).
+ * A Machine's `config.mounts[].volume` takes this id; chant also lets it name
+ * a volume, and `flyApply` looks the name up (#4717). Kept apart from the
+ * applier so the FLY011 lint check can use it without loading the applier.
+ */
+/** True when `value` is a volume id (`vol_...`), not a volume's name. Pure. */
+export function isVolumeId(value) {
+    return /^vol_[A-Za-z0-9]+$/.test(value);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@intentius/chant-lexicon-fly",
-  "version": "0.124.67",
+  "version": "0.124.68",
   "type": "module",
   "files": [
     "src/",
@@ -58,7 +58,7 @@
     "build": "../../scripts/tsc7.sh -p tsconfig.build.json --sourceMap false && tsc-alias -p tsconfig.build.json"
   },
   "dependencies": {
-    "@intentius/chant-lexicon-otel": "^0.124.67",
+    "@intentius/chant-lexicon-otel": "^0.124.68",
     "ws": "^8.18.0"
   },
   "devDependencies": {
@@ -69,7 +69,7 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@intentius/chant": "^0.124.67",
+    "@intentius/chant": "^0.124.68",
     "zod": "^4.3.6"
   },
   "description": "Fly.io Machines lexicon for chant — declarative IaC in TypeScript",