@intentius/chant-lexicon-fly 0.100.0 → 0.102.0

package/src/op/activities/fly-apply.ts

@@ -26,6 +26,7 @@ import { readFileSync } from "node:fs";
 import { sleep } from "@intentius/chant/op";
 import { hasOwnershipMarker } from "@intentius/chant/ownership";
 import { FLY_METADATA_OWNERSHIP_KEYS } from "../../ownership";
+import { logProgress } from "./progress";
 import {
   applyResult,
   type ApplyResult,
@@ -50,8 +51,16 @@ export interface FlapsRequest {
    * POSTs these and excludes them from any drift/diff.
    */
   applyOnly?: boolean;
+  /**
+   * A Secret declared without a value: set outside chant. `flyApply` never
+   * POSTs it; it checks the app has the secret and fails naming it if not.
+   */
+  mustExist?: boolean;
 }
 
+// Progress lines go to stderr (#2516); the helper is shared with the Sprites activities.
+export { logProgress } from "./progress";
+
 /** The serializer's whole output: entity name → flaps create request. */
 export type FlyPlan = Record<string, FlapsRequest>;
 
@@ -111,9 +120,13 @@ export function isIpRequest(req: FlapsRequest): boolean {
   return /^\/v1\/apps\/[^/]+\/ip_assignments\/?$/.test(req.endpoint);
 }
 
-/** True when a request creates a Certificate (`.../certificates`). Pure. */
+/**
+ * True when a request creates a Certificate: `.../certificates/acme`, or the
+ * bare `.../certificates` that plans built before #3114 carry. Both apply the
+ * same way, through {@link applyCert}. Pure.
+ */
 export function isCertRequest(req: FlapsRequest): boolean {
-  return /^\/v1\/apps\/[^/]+\/certificates\/?$/.test(req.endpoint);
+  return /^\/v1\/apps\/[^/]+\/certificates(\/acme)?\/?$/.test(req.endpoint);
 }
 
 /** True when a request sets a Secret (`.../secrets/{name}`). Pure. */
@@ -235,6 +248,9 @@ const ipUrl = (base: string, app: string, ip: string): string => `${ipsUrl(base,
 const certsUrl = (base: string, app: string): string => `${appUrl(base, app)}/certificates`;
 const certUrl = (base: string, app: string, hostname: string): string =>
   `${certsUrl(base, app)}/${encodeURIComponent(hostname)}`;
+// Certificates are listed at `.../certificates` but created at `.../acme`:
+// flaps has no POST on the list path and answers one with 404 (#3114).
+const acmeCertUrl = (base: string, app: string): string => `${certsUrl(base, app)}/acme`;
 const secretsUrl = (base: string, app: string): string => `${appUrl(base, app)}/secrets`;
 const secretUrl = (base: string, app: string, secretName: string): string =>
   `${secretsUrl(base, app)}/${encodeURIComponent(secretName)}`;
@@ -487,22 +503,103 @@ export async function applyMachine(
   return { action: "updated", id: m.id, name };
 }
 
-/** Lease → destroy → wait for the machine to be reaped. */
+/** How {@link destroyMachine} treats a machine that is still running. */
+export interface DestroyOpts extends WaitOpts {
+  /**
+   * Destroy with `?force=true`, which kills a running machine at once, instead
+   * of stopping it first. Default false: stop, wait for `stopped`, then delete.
+   */
+  force?: boolean;
+  /** The machine's last listed state. It decides whether a stop is needed first; unknown means try the delete. */
+  state?: string;
+  /** The machine's name, for error messages. */
+  name?: string;
+}
+
+/**
+ * States flaps destroys without `force`. A started machine answers a bare
+ * DELETE with 412 failed_precondition (#3115).
+ */
+const DESTROYABLE_STATES = new Set(["stopped", "suspended", "failed", "created"]);
+
+/** True when a destroy response is flaps refusing a machine that is not stopped. Pure. */
+export function isNotStoppedRefusal(status: number, text: string): boolean {
+  return status === 412 || (status >= 400 && /failed_precondition/.test(text));
+}
+
+/**
+ * Stop a machine under a lease, then wait for `stopped` with no lease held (a
+ * slow stop can outlive the lease TTL). A machine already `stopping` is only
+ * waited on. A wait that runs out fails naming the machine.
+ */
+async function stopForDestroy(
+  ctx: ApplyCtx,
+  app: string,
+  id: string,
+  http: FlyHttp,
+  signal: AbortSignal | undefined,
+  opts: DestroyOpts,
+  state: string | undefined,
+): Promise<void> {
+  const label = opts.name ? `${app}/${opts.name} (${id})` : `${app}/${id}`;
+  if (state !== "stopping") {
+    const res = await withLease(ctx, app, id, http, signal, (nonce) =>
+      http("POST", `${machineUrl(ctx.base, app, id)}/stop`, undefined, { [LEASE_NONCE_HEADER]: nonce }, signal),
+    );
+    if (res.status === 404) return;
+    if (res.status >= 300) throw new Error(`machine ${label} stop before destroy failed (${res.status}): ${res.text}`);
+  }
+  const { timeoutSecs, intervalMs, deadlineMs } = opts;
+  try {
+    await waitForMachine(ctx, app, id, "", http, signal, { timeoutSecs, intervalMs, deadlineMs, state: "stopped" });
+  } catch (err) {
+    const why = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `machine ${label} did not stop, so it was not destroyed (${why}). ` +
+        `Check the machine, or destroy it with force: true to kill it without a stop.`,
+    );
+  }
+}
+
+/**
+ * Destroy a machine and wait for it to be reaped. flaps refuses a bare DELETE
+ * on a running machine (412, #3115), so by default a machine that is not
+ * stopped is stopped first (each mutation under a lease), and a 412 from the
+ * delete (a machine that started after it was listed) gets one stop and retry.
+ * With `force`, the delete carries `?force=true` and no stop is sent.
+ */
 export async function destroyMachine(
   ctx: ApplyCtx,
   app: string,
   id: string,
   http: FlyHttp,
   signal?: AbortSignal,
-  opts: WaitOpts = {},
+  opts: DestroyOpts = {},
 ): Promise<void> {
-  const res = await withLease(ctx, app, id, http, signal, (nonce) =>
-    http("DELETE", machineUrl(ctx.base, app, id), undefined, { [LEASE_NONCE_HEADER]: nonce }, signal),
-  );
-  if (res.status >= 300 && res.status !== 404) {
-    throw new Error(`machine ${app}/${id} destroy failed (${res.status}): ${res.text}`);
+  const { force, state } = opts;
+  const label = opts.name ? `${app}/${opts.name} (${id})` : `${app}/${id}`;
+  const waitOpts = { timeoutSecs: opts.timeoutSecs, intervalMs: opts.intervalMs, deadlineMs: opts.deadlineMs };
+  const gone = state === "destroyed" || state === "destroying";
+
+  if (!gone) {
+    if (!force && state !== undefined && !DESTROYABLE_STATES.has(state)) {
+      await stopForDestroy(ctx, app, id, http, signal, opts, state);
+    }
+    const url = force ? `${machineUrl(ctx.base, app, id)}?force=true` : machineUrl(ctx.base, app, id);
+    const del = () =>
+      withLease(ctx, app, id, http, signal, (nonce) =>
+        http("DELETE", url, undefined, { [LEASE_NONCE_HEADER]: nonce }, signal),
+      );
+    let res = await del();
+    if (!force && isNotStoppedRefusal(res.status, res.text)) {
+      await stopForDestroy(ctx, app, id, http, signal, opts, undefined);
+      res = await del();
+    }
+    if (res.status >= 300 && res.status !== 404) {
+      throw new Error(`machine ${label} destroy failed (${res.status}): ${res.text}`);
+    }
   }
-  await waitForMachine(ctx, app, id, "", http, signal, { ...opts, state: "destroyed" });
+  await waitForMachine(ctx, app, id, "", http, signal, { ...waitOpts, state: "destroyed" });
 }
 
 /** Delete an app (idempotent; a 404 means it is already gone). */
@@ -522,7 +619,8 @@ export async function deleteApp(
  * Owned-only prune (D2): for one app, destroy the chant-owned machines whose
  * name is not in `keep`. An unmarked machine (no `managed-by: chant`) is never
  * touched, so it survives an apply that would otherwise delete it. Machines
- * already tearing down are skipped.
+ * already tearing down are skipped. A running machine is stopped before it is
+ * destroyed unless `opts.force` is set (see {@link destroyMachine}).
  */
 export async function pruneMachines(
   ctx: ApplyCtx,
@@ -530,14 +628,14 @@ export async function pruneMachines(
   keep: Set<string>,
   http: FlyHttp,
   signal?: AbortSignal,
-  opts: WaitOpts = {},
+  opts: WaitOpts & { force?: boolean } = {},
 ): Promise<Array<{ app: string; name: string; id: string }>> {
   const pruned: Array<{ app: string; name: string; id: string }> = [];
   for (const m of await listMachines(ctx, app, http, signal)) {
     if (!isChantOwned(m.config?.metadata) || keep.has(m.name)) continue;
     if (m.state === "destroyed" || m.state === "destroying") continue;
-    await destroyMachine(ctx, app, m.id, http, signal, opts);
-    console.log(`pruned: ${app}/${m.name} (${ctx.base})`);
+    await destroyMachine(ctx, app, m.id, http, signal, { ...opts, state: m.state, name: m.name });
+    logProgress(`pruned: ${app}/${m.name} (${ctx.base})`);
     pruned.push({ app, name: m.name, id: m.id });
   }
   return pruned;
@@ -627,13 +725,49 @@ export async function listIps(ctx: ApplyCtx, app: string, http: FlyHttp, signal?
   return body?.ips ?? [];
 }
 
-/** List an app's live certificates (`{ certificates: [...] }`). */
+/**
+ * The page size asked of the certificate list. Fly pages it whether or not a
+ * `limit` is given (default 25, max 500), so asking for the max keeps the
+ * common case to one request.
+ */
+export const CERT_PAGE_LIMIT = 500;
+
+/**
+ * The most certificate pages {@link listCerts} reads before it gives up. At
+ * {@link CERT_PAGE_LIMIT} per page that is 100,000 certificates, far past any
+ * real app, so hitting it means the API never stopped handing out a cursor.
+ */
+export const MAX_CERT_PAGES = 200;
+
+/**
+ * List an app's live certificates, every page of them (#3224). Fly answers
+ * `{ certificates, next_cursor?, total_count }` and pages the list even
+ * without a `limit`; this follows `next_cursor` until it is absent. A cursor
+ * the API already handed out, or more than {@link MAX_CERT_PAGES} pages,
+ * throws rather than looping or returning a partial list, because callers
+ * read "not listed" as "absent" (re-apply creates it, prune skips it).
+ */
 export async function listCerts(ctx: ApplyCtx, app: string, http: FlyHttp, signal?: AbortSignal): Promise<FlapsCert[]> {
-  const res = await http("GET", certsUrl(ctx.base, app), undefined, undefined, signal);
-  if (res.status === 404) return [];
-  if (res.status >= 300) throw new Error(`certificate list failed for ${app} (${res.status}): ${res.text}`);
-  const body = parseJson(res.text) as { certificates?: FlapsCert[] } | undefined;
-  return body?.certificates ?? [];
+  const certs: FlapsCert[] = [];
+  const seen = new Set<string>();
+  let cursor: string | undefined;
+  for (let page = 0; page < MAX_CERT_PAGES; page++) {
+    const q = new URLSearchParams({ limit: String(CERT_PAGE_LIMIT) });
+    if (cursor !== undefined) q.set("cursor", cursor);
+    const res = await http("GET", `${certsUrl(ctx.base, app)}?${q.toString()}`, undefined, undefined, signal);
+    // A 404 on the first page is an absent app; on a later one the list is
+    // half-read, so it is an error like any other.
+    if (res.status === 404 && page === 0) return [];
+    if (res.status >= 300) throw new Error(`certificate list failed for ${app} (${res.status}): ${res.text}`);
+    const body = parseJson(res.text) as { certificates?: FlapsCert[]; next_cursor?: string } | undefined;
+    certs.push(...(body?.certificates ?? []));
+    const next = body?.next_cursor;
+    if (!next) return certs;
+    if (seen.has(next)) throw new Error(`certificate list for ${app} repeated cursor ${JSON.stringify(next)} after ${page + 1} page(s)`);
+    seen.add(next);
+    cursor = next;
+  }
+  throw new Error(`certificate list for ${app} still had a next_cursor after ${MAX_CERT_PAGES} pages`);
 }
 
 /** List an app's live secrets (`{ secrets: [...] }`; digests only, never values). */
@@ -679,7 +813,11 @@ export async function applyIp(
   return { action: "created", type };
 }
 
-/** Create a certificate if absent (idempotent by hostname). */
+/**
+ * Create an ACME certificate if absent (idempotent by hostname). Always POSTs
+ * `{ hostname }` to `.../certificates/acme`, whatever endpoint the plan entry
+ * names, so a plan built before #3114 applies correctly too.
+ */
 export async function applyCert(
   ctx: ApplyCtx,
   app: string,
@@ -692,11 +830,42 @@ export async function applyCert(
   if ((await listCerts(ctx, app, http, signal)).some((c) => c.hostname === hostname)) {
     return { action: "noop", hostname };
   }
-  const res = await http("POST", certsUrl(ctx.base, app), req.body, undefined, signal);
+  const res = await http("POST", acmeCertUrl(ctx.base, app), { hostname }, undefined, signal);
   if (res.status >= 300) throw new Error(`certificate ${app}/${hostname} create failed (${res.status}): ${res.text}`);
   return { action: "created", hostname };
 }
 
+/**
+ * True when a plan entry is a Secret with no value, which apply must never
+ * POST. Plans built before `mustExist` existed mark it with an empty body.
+ * Pure.
+ */
+export function isValuelessSecret(req: FlapsRequest): boolean {
+  return isSecretRequest(req) && (req.mustExist === true || req.body.value === undefined);
+}
+
+/**
+ * Check a value-less secret (set outside chant) exists on the app. Never
+ * POSTs. Throws naming the secret when the app does not have it.
+ */
+export async function checkSecretExists(
+  ctx: ApplyCtx,
+  app: string,
+  name: string,
+  http: FlyHttp,
+  signal?: AbortSignal,
+): Promise<{ action: "exists"; name: string }> {
+  const live = await listSecrets(ctx, app, http, signal);
+  if (!live.some((s) => s.name === name)) {
+    throw new Error(
+      `secret ${app}/${name} is declared without a value, so it must already exist, ` +
+        `and app ${app} does not have it. Set it with \`fly secrets set ${name}=...\` ` +
+        `or give the Secret a value.`,
+    );
+  }
+  return { action: "exists", name };
+}
+
 /**
  * Set a secret (D7, apply-only): always POST, never read back for a diff. flaps
  * returns only a digest, so there is nothing to compare — every apply re-sets it.
@@ -727,7 +896,7 @@ export async function pruneVolumes(
     if (keep.has(v.name)) continue;
     const res = await http("DELETE", volumeUrl(ctx.base, app, v.id), undefined, undefined, signal);
     if (res.status >= 300 && res.status !== 404) throw new Error(`volume ${app}/${v.name} delete failed (${res.status}): ${res.text}`);
-    console.log(`pruned: volume/${app}/${v.name} (${ctx.base})`);
+    logProgress(`pruned: volume/${app}/${v.name} (${ctx.base})`);
     pruned.push({ app, name: v.name, id: v.id });
   }
   return pruned;
@@ -746,7 +915,7 @@ export async function pruneIps(
     if (keep.has(ipType(ip.shared, ip.ip))) continue;
     const res = await http("DELETE", ipUrl(ctx.base, app, ip.ip), undefined, undefined, signal);
     if (res.status >= 300 && res.status !== 404) throw new Error(`ip ${app}/${ip.ip} delete failed (${res.status}): ${res.text}`);
-    console.log(`pruned: ip/${app}/${ip.ip} (${ctx.base})`);
+    logProgress(`pruned: ip/${app}/${ip.ip} (${ctx.base})`);
     pruned.push({ app, address: ip.ip });
   }
   return pruned;
@@ -765,7 +934,7 @@ export async function pruneCerts(
     if (keep.has(c.hostname)) continue;
     const res = await http("DELETE", certUrl(ctx.base, app, c.hostname), undefined, undefined, signal);
     if (res.status >= 300 && res.status !== 404) throw new Error(`certificate ${app}/${c.hostname} delete failed (${res.status}): ${res.text}`);
-    console.log(`pruned: certificate/${app}/${c.hostname} (${ctx.base})`);
+    logProgress(`pruned: certificate/${app}/${c.hostname} (${ctx.base})`);
     pruned.push({ app, hostname: c.hostname });
   }
   return pruned;
@@ -787,7 +956,7 @@ export async function pruneSecrets(
     if (keep.has(s.name)) continue;
     const res = await http("DELETE", secretUrl(ctx.base, app, s.name), undefined, undefined, signal);
     if (res.status >= 300 && res.status !== 404) throw new Error(`secret ${app}/${s.name} delete failed (${res.status}): ${res.text}`);
-    console.log(`pruned: secret/${app}/${s.name} (${ctx.base})`);
+    logProgress(`pruned: secret/${app}/${s.name} (${ctx.base})`);
     pruned.push({ app, name: s.name });
   }
   return pruned;
@@ -817,6 +986,12 @@ export interface FlyApplyArgs {
    * default.
    */
   prune?: boolean;
+  /**
+   * Destroy machines (prune, `flyDelete`) with `?force=true`, killing a running
+   * machine at once. Default false: stop it, wait for `stopped`, then destroy,
+   * so it can shut down on its configured kill signal and timeout.
+   */
+  force?: boolean;
   /** Wait-loop tuning (mainly for tests). */
   wait?: WaitOpts;
 }
@@ -853,7 +1028,7 @@ export function toApplyResult(result: {
   volumes: Array<{ app: string; name: string; action: "created" | "noop" }>;
   ips: Array<{ app: string; type: string; action: "created" | "noop" }>;
   certs: Array<{ app: string; hostname: string; action: "created" | "noop" }>;
-  secrets: Array<{ app: string; name: string }>;
+  secrets: Array<{ app: string; name: string; action?: "set" | "exists" }>;
   pruned: Array<{ app: string; name: string; id: string }>;
   prunedVolumes: Array<{ app: string; name: string; id: string }>;
   prunedIps: Array<{ app: string; address: string }>;
@@ -873,9 +1048,14 @@ export function toApplyResult(result: {
     ...result.volumes.map((x) => ({ kind: "volume", name: x.name, action: act(x.action) })),
     ...result.ips.map((x) => ({ kind: "ip", name: x.type, action: act(x.action) })),
     ...result.certs.map((x) => ({ kind: "cert", name: x.hostname, action: act(x.action) })),
-    // Secrets are apply-only (D7) — POSTed every run, never read back, so there
-    // is no unchanged to report.
-    ...result.secrets.map((x) => ({ kind: "secret", name: x.name, action: "updated" as const })),
+    // Secrets are apply-only (D7) — POSTed every run, never read back, so a
+    // set secret reports updated. A value-less secret is only checked, so it
+    // reports unchanged.
+    ...result.secrets.map((x) => ({
+      kind: "secret",
+      name: x.name,
+      action: (x.action === "exists" ? "unchanged" : "updated") as AppliedResource["action"],
+    })),
   ];
   const pruned: PrunedResource[] = [
     ...result.pruned.map((x) => ({ kind: "machine", name: x.name, deleted: true })),
@@ -897,7 +1077,7 @@ export async function flyApply(
   volumes: Array<{ app: string; name: string; action: "created" | "noop" }>;
   ips: Array<{ app: string; type: string; action: "created" | "noop" }>;
   certs: Array<{ app: string; hostname: string; action: "created" | "noop" }>;
-  secrets: Array<{ app: string; name: string }>;
+  secrets: Array<{ app: string; name: string; action?: "set" | "exists" }>;
   pruned: Array<{ app: string; name: string; id: string }>;
   prunedVolumes: Array<{ app: string; name: string; id: string }>;
   prunedIps: Array<{ app: string; address: string }>;
@@ -946,7 +1126,7 @@ export async function flyApply(
   const apps: Array<{ app: string; created: boolean }> = [];
   for (const req of appReqs) {
     const result = await applyApp(ctx, req, http, signal);
-    console.log(`${result.created ? "created" : "unchanged"}: app/${result.app} (${ctx.base})`);
+    logProgress(`${result.created ? "created" : "unchanged"}: app/${result.app} (${ctx.base})`);
     apps.push(result);
   }
 
@@ -978,7 +1158,7 @@ export async function flyApply(
     const app = resolveApp(resourceAppSegment(req.endpoint), soleApp);
     const result = await applyVolume(ctx, app, entityName, req, http, signal);
     track(keepVolumes, app, result.name);
-    console.log(`${result.action}: volume/${app}/${result.name} (${ctx.base})`);
+    logProgress(`${result.action}: volume/${app}/${result.name} (${ctx.base})`);
     volumes.push({ app, name: result.name, action: result.action });
   }
 
@@ -988,7 +1168,7 @@ export async function flyApply(
     const name = typeof req.body.name === "string" && req.body.name ? req.body.name : entityName;
     track(keepMachines, app, name);
     const result = await applyMachine(ctx, app, entityName, req, http, signal, opts);
-    console.log(`${result.action}: machine/${app}/${result.name} (${ctx.base})`);
+    logProgress(`${result.action}: machine/${app}/${result.name} (${ctx.base})`);
     machines.push({ app, name: result.name, action: result.action });
   }
 
@@ -998,7 +1178,7 @@ export async function flyApply(
     const app = resolveApp(resourceAppSegment(req.endpoint), soleApp);
     const result = await applyIp(ctx, app, req, http, signal);
     track(keepIps, app, result.type);
-    console.log(`${result.action}: ip/${app}/${result.type} (${ctx.base})`);
+    logProgress(`${result.action}: ip/${app}/${result.type} (${ctx.base})`);
     ips.push({ app, type: result.type, action: result.action });
   }
 
@@ -1007,19 +1187,22 @@ export async function flyApply(
     const app = resolveApp(resourceAppSegment(req.endpoint), soleApp);
     const result = await applyCert(ctx, app, req, http, signal);
     track(keepCerts, app, result.hostname);
-    console.log(`${result.action}: certificate/${app}/${result.hostname} (${ctx.base})`);
+    logProgress(`${result.action}: certificate/${app}/${result.hostname} (${ctx.base})`);
     certs.push({ app, hostname: result.hostname, action: result.action });
   }
 
-  // Secrets are apply-only (D7): always set, never read back for a diff.
-  const secrets: Array<{ app: string; name: string }> = [];
+  // Secrets are apply-only (D7): a secret with a value is set every run and
+  // never read back for a diff. A value-less one is only checked for.
+  const secrets: Array<{ app: string; name: string; action: "set" | "exists" }> = [];
   for (const [, req] of secretReqs) {
     const app = resolveApp(resourceAppSegment(req.endpoint), soleApp);
     const name = secretNameSegment(req.endpoint);
     track(keepSecrets, app, name);
-    const result = await applySecret(ctx, app, name, req, http, signal);
-    console.log(`set: secret/${app}/${result.name} (${ctx.base})`);
-    secrets.push({ app, name: result.name });
+    const result = isValuelessSecret(req)
+      ? await checkSecretExists(ctx, app, name, http, signal)
+      : await applySecret(ctx, app, name, req, http, signal);
+    logProgress(`${result.action}: secret/${app}/${result.name} (${ctx.base})`);
+    secrets.push({ app, name: result.name, action: result.action });
   }
 
   const pruned: Array<{ app: string; name: string; id: string }> = [];
@@ -1029,7 +1212,7 @@ export async function flyApply(
   const prunedSecrets: Array<{ app: string; name: string }> = [];
   if (args.prune) {
     for (const [app, keep] of keepMachines) {
-      pruned.push(...(await pruneMachines(ctx, app, keep, http, signal, opts)));
+      pruned.push(...(await pruneMachines(ctx, app, keep, http, signal, { ...opts, force: args.force })));
     }
     for (const [app, keep] of keepVolumes) {
       prunedVolumes.push(...(await pruneVolumes(ctx, app, keep, http, signal)));
@@ -1050,7 +1233,8 @@ export async function flyApply(
 
 /**
  * The inverse of {@link flyApply}: destroy the machines the plan declares, then
- * delete the apps (dependents before their app). Idempotent — already-absent
+ * delete the apps (dependents before their app). A running machine is stopped
+ * first unless `args.force` is set. Idempotent — already-absent
  * resources are a no-op. `http` is injectable for tests.
  */
 export async function flyDelete(
@@ -1076,14 +1260,14 @@ export async function flyDelete(
     const name = typeof req.body.name === "string" && req.body.name ? req.body.name : entityName;
     const live = (await listMachines(ctx, app, http, signal)).find((m) => m.name === name);
     if (!live) continue;
-    await destroyMachine(ctx, app, live.id, http, signal, opts);
+    await destroyMachine(ctx, app, live.id, http, signal, { ...opts, force: args.force, state: live.state, name });
     machines.push({ app, name });
   }
 
   const apps: Array<{ app: string; deleted: boolean }> = [];
   for (const req of appReqs) {
     const result = await deleteApp(ctx, appNameFromRequest(req), http, signal);
-    console.log(`${result.deleted ? "deleted" : "absent"}: app/${result.app} (${ctx.base})`);
+    logProgress(`${result.deleted ? "deleted" : "absent"}: app/${result.app} (${ctx.base})`);
     apps.push(result);
   }
 

package/src/op/activities/index.ts

@@ -14,6 +14,11 @@ export {
   destroyMachine,
   deleteApp,
   listMachines,
+  listVolumes,
+  listIps,
+  listCerts,
+  listSecrets,
+  checkSecretExists,
   pruneMachines,
   waitForMachine,
   acquireLease,
@@ -23,7 +28,16 @@ export {
   parsePlan,
   isAppRequest,
   isMachineRequest,
+  isVolumeRequest,
+  isIpRequest,
+  isCertRequest,
+  isSecretRequest,
+  isValuelessSecret,
   machineAppSegment,
+  resourceAppSegment,
+  secretNameSegment,
+  ipType,
+  declaredIpType,
   resolveApp,
   appNameFromRequest,
   isChantOwned,
@@ -33,7 +47,19 @@ export {
   DEFAULT_FLAPS_BASE_URL,
   LEASE_NONCE_HEADER,
 } from "./fly-apply";
-export type { FlyApplyArgs, FlyPlan, FlapsRequest, FlapsMachine, FlyHttp, WaitOpts, ApplyCtx } from "./fly-apply";
+export type {
+  FlyApplyArgs,
+  FlyPlan,
+  FlapsRequest,
+  FlapsMachine,
+  FlapsVolume,
+  FlapsIp,
+  FlapsCert,
+  FlapsSecret,
+  FlyHttp,
+  WaitOpts,
+  ApplyCtx,
+} from "./fly-apply";
 
 // The release a Machine serves and the site steps over the Machines API
 // (#2736, ws-056): upload and start, exec (migrations), restart, stop, verify,

package/src/op/activities/machines-contract.docker.integration.test.ts

@@ -16,6 +16,17 @@ const PORT = 4283;
 let available = false;
 let endpoint = "";
 
+/**
+ * Contract endpoints the pinned mudflaps is known not to serve, each with the
+ * upstream issue. The applier calls the real flaps path; the docker-gated
+ * apply test shims these. A gap the pin has since closed fails the last test
+ * below, so the entry and the shim get removed with the pin bump.
+ */
+const MUDFLAPS_KNOWN_GAPS = new Map<string, string>([
+  // ACME certificate create (#3114); mudflaps only serves POST .../certificates.
+  [normalizeEndpoint("POST", "/v1/apps/{app}/certificates/acme"), "INTENTIUS/mudflaps#69"],
+]);
+
 /** Parse a mudflaps health `implemented` entry ("METHOD path (note)") to a normalized key. */
 function normalizeImplemented(entry: string): string {
   const stripped = entry.replace(/\s*\(.*\)\s*$/, "").trim();
@@ -52,7 +63,7 @@ describe("Machines contract ⊆ mudflaps implemented paths", () => {
 
     const served = new Set((health.implemented ?? []).map(normalizeImplemented));
 
-    const missing = [...contractKeys()].filter((k) => !served.has(k));
+    const missing = [...contractKeys()].filter((k) => !served.has(k) && !MUDFLAPS_KNOWN_GAPS.has(k));
     // A non-empty list means flyApply calls something mudflaps can't serve — a
     // real fidelity gap between the applier and the pinned emulator.
     expect(missing, `mudflaps is missing contract endpoints: ${missing.join(", ")}`).toEqual([]);
@@ -78,4 +89,14 @@ describe("Machines contract ⊆ mudflaps implemented paths", () => {
     const missing = MACHINE_RELEASE_CONTRACT.map((e) => normalizeEndpoint(e.method, e.path)).filter((k) => !served.has(k));
     expect(missing, `mudflaps is missing release endpoints: ${missing.join(", ")}`).toEqual([]);
   });
+
+  test("every known mudflaps gap is still a gap (drop the entry when the pin closes it)", async (ctx) => {
+    if (!available) ctx.skip();
+    const res = await fetch(`${endpoint}/_mudflaps/health`);
+    const health = (await res.json()) as { implemented?: string[] };
+    const served = new Set((health.implemented ?? []).map(normalizeImplemented));
+    for (const [key, issue] of MUDFLAPS_KNOWN_GAPS) {
+      expect(served.has(key), `${key} is served now (${issue}): remove it from MUDFLAPS_KNOWN_GAPS and the apply-test shim`).toBe(false);
+    }
+  });
 });

package/src/op/activities/machines-contract.ts

@@ -39,6 +39,9 @@ export const MACHINES_CONTRACT: readonly MachinesEndpoint[] = [
   { method: "GET", path: "/v1/apps/{app}/machines", op: "listMachines" },
   { method: "POST", path: "/v1/apps/{app}/machines", op: "createMachine" },
   { method: "POST", path: "/v1/apps/{app}/machines/{id}", op: "updateMachine" },
+  // A running machine is stopped before the destroy (#3115); `?force=true` is a
+  // query, so it stays out of the path template.
+  { method: "POST", path: "/v1/apps/{app}/machines/{id}/stop", op: "stopForDestroy" },
   { method: "DELETE", path: "/v1/apps/{app}/machines/{id}", op: "destroyMachine" },
   { method: "GET", path: "/v1/apps/{app}/machines/{id}/wait", op: "waitForMachine" },
   // Leases
@@ -54,7 +57,8 @@ export const MACHINES_CONTRACT: readonly MachinesEndpoint[] = [
   { method: "DELETE", path: "/v1/apps/{app}/ip_assignments/{ip}", op: "releaseIp" },
   // Certificates
   { method: "GET", path: "/v1/apps/{app}/certificates", op: "listCerts" },
-  { method: "POST", path: "/v1/apps/{app}/certificates", op: "addCert" },
+  // Created at /acme: flaps has no POST on the list path (#3114).
+  { method: "POST", path: "/v1/apps/{app}/certificates/acme", op: "addCert" },
   { method: "DELETE", path: "/v1/apps/{app}/certificates/{hostname}", op: "deleteCert" },
   // Secrets
   { method: "GET", path: "/v1/apps/{app}/secrets", op: "listSecrets" },