@intentius/chant-lexicon-fly 0.100.0 → 0.102.0

package/src/op/activities/machines-fake.ts

@@ -35,6 +35,8 @@ export interface MachinesFake {
   http: FlyHttp;
   apps: Set<string>;
   machines: Map<string, FakeMachine[]>;
+  /** Each app's certificate hostnames, in creation order. */
+  certs: Map<string, string[]>;
   /** Every exec, in order: the app, the machine id and the command. */
   execs: Array<{ app: string; id: string; command: string[] }>;
   /** Every call, as `METHOD path` (no base, no query). */
@@ -52,13 +54,28 @@ export interface MachinesFakeOptions {
 
 const json = (status: number, body: unknown) => ({ status, text: body === undefined ? "" : JSON.stringify(body) });
 
+/** Fly's certificate-list page size: 25 when no `limit` is given, 500 at most. */
+export const CERT_PAGE_DEFAULT = 25;
+export const CERT_PAGE_MAX = 500;
+// The fake's cursor is an offset in disguise; callers must treat it as opaque.
+const offsetCursor = (offset: number): string => Buffer.from(`certs:${offset}`).toString("base64url");
+const cursorOffset = (cursor: string): number | undefined => {
+  const m = /^certs:(\d+)$/.exec(Buffer.from(cursor, "base64url").toString());
+  return m ? Number(m[1]) : undefined;
+};
+
 let seq = 0;
 const nextId = (prefix: string) => `${prefix}${(++seq).toString(16).padStart(10, "0")}`;
 
-/** A fresh in-memory flaps. */
+/**
+ * A fresh in-memory flaps. Like the real API, a bare DELETE on a machine that
+ * is not stopped, suspended, failed or created answers 412; `?force=true`
+ * destroys it in any state.
+ */
 export function createMachinesFake(options: MachinesFakeOptions = {}): MachinesFake {
   const apps = new Set<string>();
   const machines = new Map<string, FakeMachine[]>();
+  const certs = new Map<string, string[]>();
   const execs: MachinesFake["execs"] = [];
   const calls: string[] = [];
   const settle = options.settle ?? (() => "started");
@@ -88,6 +105,7 @@ export function createMachinesFake(options: MachinesFakeOptions = {}): MachinesF
       if (method === "DELETE") {
         apps.delete(app);
         machines.delete(app);
+        certs.delete(app);
         return json(202, undefined);
       }
     }
@@ -123,6 +141,13 @@ export function createMachinesFake(options: MachinesFakeOptions = {}): MachinesF
           return json(200, m);
         }
         if (method === "DELETE") {
+          // Real flaps destroys a running machine only with ?force=true and
+          // answers a bare DELETE with 412 (#3115). mudflaps does not (yet).
+          if (searchParams.get("force") !== "true" && !["stopped", "suspended", "failed", "created"].includes(m.state)) {
+            return json(412, {
+              error: "failed_precondition: unable to destroy machine, not currently stopped, suspended, failed or created",
+            });
+          }
           machines.set(app, list(app).filter((x) => x !== m));
           return json(200, { ok: true });
         }
@@ -151,15 +176,57 @@ export function createMachinesFake(options: MachinesFakeOptions = {}): MachinesF
       return json(404, { error: `no ${method} ${action}` });
     }
     if (kind === "ip_assignments") return method === "GET" ? json(200, { ips: [] }) : json(200, {});
-    if (kind === "certificates") return method === "GET" ? json(200, { certificates: [] }) : json(200, {});
+    if (kind === "certificates") return certificates(app, method, seg.slice(4), b, searchParams);
     if (kind === "volumes" || kind === "secrets") return method === "GET" ? json(200, []) : json(200, {});
     return json(404, { error: "not found" });
   };
 
+  // Certificates, as the real API routes them (#3114): listed at
+  // `.../certificates`, created only at `.../certificates/acme`, read and
+  // deleted at `.../certificates/{hostname}`. A POST to the bare list path is
+  // the 404 flaps gives, so an applier that posts there fails here too.
+  // The list pages like Fly's (#3224): `limit` defaults to 25 and is capped at
+  // 500, and `next_cursor` is present while certificates remain. mudflaps
+  // returns the whole list in one page (INTENTIUS/mudflaps#71).
+  function certificates(app: string, method: string, rest: string[], b: Record<string, unknown>, q: URLSearchParams) {
+    const hostnames = certs.get(app) ?? [];
+    const detail = (hostname: string) => ({ hostname, acme_requested: true, configured: false, status: "pending" });
+    if (rest.length === 0) {
+      if (method !== "GET") return { status: 404, text: "404 page not found" };
+      const limit = Math.min(CERT_PAGE_MAX, Math.max(1, Number(q.get("limit")) || CERT_PAGE_DEFAULT));
+      const cursor = q.get("cursor");
+      const from = cursor === null ? 0 : cursorOffset(cursor);
+      if (from === undefined) return json(400, { error: `invalid cursor ${cursor}` });
+      const page = hostnames.slice(from, from + limit);
+      const more = from + limit < hostnames.length;
+      return json(200, {
+        certificates: page.map(detail),
+        ...(more ? { next_cursor: offsetCursor(from + limit) } : {}),
+        total_count: hostnames.length,
+      });
+    }
+    if (rest.length === 1 && rest[0] === "acme" && method === "POST") {
+      const hostname = typeof b.hostname === "string" ? b.hostname : "";
+      if (!hostname) return json(422, { error: "hostname is required" });
+      if (hostnames.includes(hostname)) return json(422, { error: `certificate ${hostname} already exists` });
+      certs.set(app, [...hostnames, hostname]);
+      return json(201, detail(hostname));
+    }
+    if (rest.length === 1 && (method === "GET" || method === "DELETE")) {
+      const hostname = rest[0];
+      if (!hostnames.includes(hostname)) return json(404, { error: "certificate not found" });
+      if (method === "GET") return json(200, detail(hostname));
+      certs.set(app, hostnames.filter((h) => h !== hostname));
+      return json(204, undefined);
+    }
+    return { status: 404, text: "404 page not found" };
+  }
+
   return {
     http,
     apps,
     machines,
+    certs,
     execs,
     calls,
     machine: (app, name) => list(app).find((m) => m.name === name && !["destroyed", "destroying"].includes(m.state)),

package/src/op/activities/machines-local-cli.ts

@@ -51,6 +51,9 @@ async function main(): Promise<void> {
     console.error("usage: machines-local-cli.ts [--listen <port>] [--root <dir>] [--publish <port> | --publish <name>=<port>]... [--log <file>]");
     process.exit(2);
   }
+  // A standalone server, not an Op activity: its stdout is its output. The flaps URL and the
+  // event lines (which name each Machine's published port) stay there for the person or
+  // script that started it; errors and usage go to stderr.
   const served = await serveLocalMachines({ ...args, handleSignals: false, onEvent: (line) => console.log(`machines-local: ${line}`) });
   console.log(`machines-local: flaps on ${served.url}${args.root ? `, guest paths under ${args.root}` : ""}`);
   let closing = false;

package/src/op/activities/machines-local.test.ts

@@ -109,11 +109,12 @@ describe("the running mode of the local Machines API (#2831)", () => {
     expect(existsSync(join(root, "srv/app/page.txt"))).toBe(false);
     expect(readFileSync(join(root, "data/state"), "utf8")).toBe("migrated\n");
 
-    // Start, then delete: running again, then gone.
+    // Start, then a forced delete: running again, then gone. A bare delete of a
+    // started Machine is refused, as Fly refuses it (#3115).
     await call("POST", `/v1/apps/shop/machines/${id}/start`);
     const [fourth] = machines.processes();
     expect(await get(url!)).toBe("B migrated");
-    await call("DELETE", `/v1/apps/shop/machines/${id}`);
+    await call("DELETE", `/v1/apps/shop/machines/${id}?force=true`);
     expect(machines.processes()).toEqual([]);
     expect(alive(fourth.pid!)).toBe(false);
   });
@@ -165,6 +166,30 @@ describe("the running mode of the local Machines API (#2831)", () => {
     expect(JSON.parse((await machines.http("GET", `http://f/v1/apps/shop/machines/${id}/wait?state=started`)).text)).toEqual({ ok: false });
   });
 
+  test("an exec that times out while its command is forking leaves nothing behind", { timeout: 30_000 }, async () => {
+    const machines = createLocalMachines({ root: tmp(), log: join(tmp(), "log") });
+    open.push(machines);
+    await machines.http("POST", "http://f/v1/apps", { app_name: "shop" });
+    const r = await machines.http("POST", "http://f/v1/apps/shop/machines", { name: "web", config: {} });
+    const id = JSON.parse(r.text).id as string;
+    // Forks a long sleep and kills it, over and over: a child that one group kill misses survives (#3202),
+    // and it holds the exec's output pipes, so the exec would not answer until it ended.
+    const marker = `299.${process.pid}${Math.floor(Math.random() * 1e6)}`;
+    try {
+      const out = await machines.http("POST", `http://f/v1/apps/shop/machines/${id}/exec`, {
+        command: ["sh", "-c", `while :; do sleep ${marker} & kill $!; done`],
+        timeout: 1,
+      });
+      expect(JSON.parse(out.text).exit_code).not.toBe(0);
+      const deadline = Date.now() + 10_000;
+      const left = () => spawnSync("pgrep", ["-f", `sleep ${marker}`], { encoding: "utf8" }).stdout.trim();
+      while (left() && Date.now() < deadline) await new Promise((ok) => setTimeout(ok, 50));
+      expect(left()).toBe("");
+    } finally {
+      spawnSync("pkill", ["-KILL", "-f", `sleep ${marker}`]);
+    }
+  });
+
   test("with no root, guest paths are host paths", async () => {
     const dir = tmp();
     const machines = createLocalMachines({ log: join(dir, "log") });
@@ -177,6 +202,10 @@ describe("the running mode of the local Machines API (#2831)", () => {
     expect(readFileSync(join(dir, "a/b.txt"), "utf8")).toBe("hi");
   });
 
+  /** Forks children for a while, then becomes one: the host is ended while it is still forking. */
+  const FORKS = "i=0; while [ $i -lt 100 ]; do sleep 300 & i=$((i+1)); done; exec sleep 301";
+  /** The live (not zombie) members of a process group, as pgrep lists them. */
+  const group = (pgid: number) => spawnSync("pgrep", ["-g", String(pgid)], { encoding: "utf8" }).stdout.trim();
   for (const how of ["SIGTERM", "SIGINT", "exit"] as const) {
     test(`a Machine's process does not outlive its host process (${how})`, { timeout: 60_000 }, async () => {
       const root = tmp();
@@ -186,7 +215,7 @@ describe("the running mode of the local Machines API (#2831)", () => {
         `import { createLocalMachines } from ${JSON.stringify(join(here, "machines-local.ts"))};
 const m = createLocalMachines({ root: ${JSON.stringify(root)}, log: ${JSON.stringify(join(root, "log"))} });
 await m.http("POST", "http://f/v1/apps", { app_name: "shop" });
-await m.http("POST", "http://f/v1/apps/shop/machines", { name: "web", config: { init: { cmd: ["sh", "-c", "sleep 300 & exec sleep 301"] } } });
+await m.http("POST", "http://f/v1/apps/shop/machines", { name: "web", config: { init: { cmd: ["sh", "-c", ${JSON.stringify(FORKS)}] } } });
 console.log(JSON.stringify(m.processes()[0].pid));
 ${how === "exit" ? "setTimeout(() => process.exit(0), 100);" : "setInterval(() => undefined, 1000);"}
 `,
@@ -197,15 +226,22 @@ ${how === "exit" ? "setTimeout(() => process.exit(0), 100);" : "setInterval(() =
         env: { ...process.env, TSX_DISABLE_CACHE: "1" },
       });
       const pid = await new Promise<number>((ok) => host.stdout!.once("data", (d) => ok(Number(String(d).trim()))));
-      expect(alive(pid)).toBe(true);
-      const exited = new Promise((ok) => host.once("exit", ok));
-      if (how !== "exit") host.kill(how);
-      await exited;
-      for (let i = 0; i < 50 && alive(pid); i++) await new Promise((ok) => setTimeout(ok, 20));
-      expect(alive(pid)).toBe(false);
-      // Nor does anything the command started in its group.
-      const left = spawnSync("pgrep", ["-g", String(pid)], { encoding: "utf8" }).stdout.trim();
-      expect(left).toBe("");
+      try {
+        expect(alive(pid)).toBe(true);
+        const exited = new Promise((ok) => host.once("exit", ok));
+        // Sent while the command is still forking, which is when a single group kill can miss a child (#3202).
+        if (how !== "exit") host.kill(how);
+        await exited;
+        // Killed processes still take a moment to go (and the leader to be reaped), more so on a loaded host, so poll.
+        // Anything left after the deadline was never killed: a missed child sleeps for 300s.
+        const deadline = Date.now() + 10_000;
+        while ((alive(pid) || group(pid)) && Date.now() < deadline) await new Promise((ok) => setTimeout(ok, 50));
+        expect(alive(pid)).toBe(false);
+        // Nor does anything the command started in its group.
+        expect(group(pid)).toBe("");
+      } finally {
+        spawnSync("pkill", ["-KILL", "-g", String(pid)]);
+      }
     });
   }
 });

package/src/op/activities/machines-local.ts

@@ -130,6 +130,27 @@ function killGroup(child: ChildProcess | undefined, signal: NodeJS.Signals): voi
   }
 }
 
+/**
+ * SIGKILL a process group until nothing in it is left to kill. One kill(-pgid)
+ * can miss a child that a member is forking at that moment: the child joins the
+ * group after the signal went out, and then outlives the rest (seen on macOS,
+ * #3202). Sending again once the forker is dead catches it. Synchronous, so it
+ * works in an "exit" handler; it stops as soon as the group is empty (or only
+ * zombies), and gives up after about 100ms.
+ */
+function killGroupNow(child: ChildProcess | undefined): void {
+  if (!child?.pid) return;
+  const pause = new Int32Array(new SharedArrayBuffer(4));
+  for (let i = 0; i < 20; i++) {
+    try {
+      process.kill(-child.pid, "SIGKILL");
+    } catch {
+      return;
+    }
+    Atomics.wait(pause, 0, 0, 5);
+  }
+}
+
 async function freePort(): Promise<number> {
   const srv = createNetServer();
   await new Promise<void>((ok, fail) => srv.once("error", fail).listen(0, "127.0.0.1", ok));
@@ -225,9 +246,10 @@ export function createLocalMachines(options: LocalMachinesOptions = {}): LocalMa
       // Wait for the leader to exit and the rest of its group to go.
       await Promise.race([exited, new Promise((ok) => setTimeout(ok, stopTimeoutMs))]);
       while (groupAlive(child.pid) && Date.now() < deadline) await new Promise((ok) => setTimeout(ok, 25));
-      if (groupAlive(child.pid)) {
+      // Sent again each round: one group kill can miss a child being forked as it lands (see killGroupNow).
+      while (groupAlive(child.pid)) {
         killGroup(child, "SIGKILL");
-        while (groupAlive(child.pid)) await new Promise((ok) => setTimeout(ok, 25));
+        await new Promise((ok) => setTimeout(ok, 25));
       }
     }
     for (const path of r.files) rmSync(path, { force: true });
@@ -266,7 +288,7 @@ export function createLocalMachines(options: LocalMachinesOptions = {}): LocalMa
       if (r.stopping) return;
       say(`${key} instance ${m.instance_id} exited ${signal ?? code}`);
       // A process that ends on its own leaves its Machine stopped, and its group is killed with it.
-      killGroup(child, "SIGKILL");
+      killGroupNow(child);
       if (m.instance_id === r.instance && m.state === "started") m.state = "stopped";
     });
     say(`started ${key} instance ${m.instance_id}: ${files.length} files, ${argv.join(" ")}${port ? `, on 127.0.0.1:${port}` : ""}`);
@@ -287,7 +309,8 @@ export function createLocalMachines(options: LocalMachinesOptions = {}): LocalMa
       let stderr = "";
       child.stdout!.on("data", (d) => (stdout += d));
       child.stderr!.on("data", (d) => (stderr += d));
-      const timer = setTimeout(() => killGroup(child, "SIGKILL"), timeoutSecs * 1000);
+      // killGroupNow: a child missed by one group kill would hold the output pipes, and "close" would wait for it.
+      const timer = setTimeout(() => killGroupNow(child), timeoutSecs * 1000);
       child.once("error", (error) => {
         clearTimeout(timer);
         done({ exit_code: 127, stdout, stderr: `${stderr}${error.message}` });
@@ -347,7 +370,7 @@ export function createLocalMachines(options: LocalMachinesOptions = {}): LocalMa
   const killAll = () => {
     for (const r of running.values()) {
       r.stopping = true;
-      killGroup(r.child, "SIGKILL");
+      killGroupNow(r.child);
     }
   };
   const onSignal = (sig: NodeJS.Signals) => {

package/src/op/activities/progress.ts

@@ -0,0 +1,10 @@
+/**
+ * Progress output for the fly Op activities (#2516, #3200).
+ *
+ * An Op step's stdout is its output, so activities write their progress lines
+ * ("created: sprite/x", "pruned: ...") to stderr and keep stdout free for
+ * machine-readable results.
+ */
+export function logProgress(line: string): void {
+  process.stderr.write(`${line}\n`);
+}

package/src/op/activities/sprite-config.ts

@@ -27,6 +27,7 @@ import { promisify } from "node:util";
 import { expandServiceCommand, inStartOrder, boxServices, type BoxServiceDeclaration } from "./box-services";
 import { findSpriteEnv } from "./sprite-service-converge";
 import { resolveSpritesEndpoint, defaultSpritesHttp, type SpritesHttp } from "./sprites";
+import { logProgress } from "./progress";
 
 const execFileAsync = promisify(execFile);
 
@@ -101,13 +102,13 @@ export async function spriteApplyNetworkPolicy(
   if (cur.status >= 300) throw new Error(`sprite ${args.id} get policy failed (${cur.status}): ${cur.text}`);
   const live = (safeJson(cur.text) as { rules?: NetworkRule[] } | undefined)?.rules ?? [];
   if (networkRulesEqual(live, args.rules)) {
-    console.log(`policy: sprite/${args.id} already converged (${args.rules.length} rules)`);
+    logProgress(`policy: sprite/${args.id} already converged (${args.rules.length} rules)`);
     return { changed: false };
   }
 
   const res = await http("POST", url, { rules: args.rules }, undefined, signal);
   if (res.status >= 300) throw new Error(`sprite ${args.id} set policy failed (${res.status}): ${res.text}`);
-  console.log(`policy: sprite/${args.id} applied ${args.rules.length} rules (${base})`);
+  logProgress(`policy: sprite/${args.id} applied ${args.rules.length} rules (${base})`);
   return { changed: true };
 }
 
@@ -276,7 +277,7 @@ async function applyThroughApi(
     if (res.status >= 300) throw new Error(`sprite ${args.id} apply service ${s.name} failed (${res.status}): ${res.text}`);
     applied.push(s.name);
   }
-  console.log(`services: sprite/${args.id} applied ${applied.length}/${args.services.length} (${base})`);
+  logProgress(`services: sprite/${args.id} applied ${applied.length}/${args.services.length} (${base})`);
 
   const started: string[] = [];
   if (args.start) {
@@ -286,7 +287,7 @@ async function applyThroughApi(
       if (res.status >= 300) throw new Error(`sprite ${args.id} start service ${name} failed (${res.status}): ${res.text}`);
       started.push(name);
     }
-    console.log(`services: sprite/${args.id} started ${started.length} in dependency order`);
+    logProgress(`services: sprite/${args.id} started ${started.length} in dependency order`);
   }
 
   return { applied, started };
@@ -419,7 +420,7 @@ async function applyThroughSpriteEnv(args: SpriteApplyServicesArgs, signal?: Abo
     const cur = live.get(s.name);
     if (!cur) {
       await spriteEnv(spriteEnvCreateArgs(s, cmd));
-      console.log(`services: created ${s.name}`);
+      logProgress(`services: created ${s.name}`);
       services.push({ name: s.name, action: "created" });
       continue;
     }
@@ -427,13 +428,13 @@ async function applyThroughSpriteEnv(args: SpriteApplyServicesArgs, signal?: Abo
     if (why) {
       await spriteEnv(["services", "delete", s.name]);
       await spriteEnv(spriteEnvCreateArgs(s, cmd));
-      console.log(`services: replaced ${s.name} (${why})`);
+      logProgress(`services: replaced ${s.name} (${why})`);
       services.push({ name: s.name, action: "replaced" });
       continue;
     }
     if (args.restart) {
       await spriteEnv(["services", "restart", s.name]);
-      console.log(`services: restarted ${s.name}`);
+      logProgress(`services: restarted ${s.name}`);
       services.push({ name: s.name, action: "restarted" });
       continue;
     }
@@ -445,12 +446,12 @@ async function applyThroughSpriteEnv(args: SpriteApplyServicesArgs, signal?: Abo
     for (const entry of services) {
       if (entry.action !== "left" || UP.has(live.get(entry.name)?.status ?? "")) continue;
       await spriteEnv(["services", "start", entry.name]);
-      console.log(`services: started ${entry.name}`);
+      logProgress(`services: started ${entry.name}`);
       entry.action = "started";
       started.push(entry.name);
     }
   }
   const applied = services.filter((e) => e.action === "created" || e.action === "replaced").map((e) => e.name);
-  console.log(`services: applied ${applied.length}/${targets.length} through sprite-env, started ${started.length}`);
+  logProgress(`services: applied ${applied.length}/${targets.length} through sprite-env, started ${started.length}`);
   return { applied, started, services };
 }

package/src/op/activities/sprite-fs.ts

@@ -17,6 +17,7 @@
  */
 
 import { resolveSpritesEndpoint, resolveSpritesToken } from "./sprites";
+import { logProgress } from "./progress";
 
 function safeJson(text: string): unknown {
   try {
@@ -166,7 +167,7 @@ export async function spriteWriteFile(
   if (res.status >= 300) {
     throw new Error(`sprite ${args.id} write ${args.path} failed (${res.status}): ${res.text}`);
   }
-  console.log(`wrote: sprite/${args.id}:${args.path} (${args.content.length}b)`);
+  logProgress(`wrote: sprite/${args.id}:${args.path} (${args.content.length}b)`);
   return {};
 }
 
@@ -221,6 +222,6 @@ export async function spriteRemove(
   if (res.status >= 300 && res.status !== 404) {
     throw new Error(`sprite ${args.id} remove ${args.path} failed (${res.status}): ${res.text}`);
   }
-  console.log(`removed: sprite/${args.id}:${args.path}`);
+  logProgress(`removed: sprite/${args.id}:${args.path}`);
   return {};
 }

package/src/op/activities/sprite-progress.test.ts

@@ -0,0 +1,52 @@
+import { describe, test, expect, vi, afterEach } from "vitest";
+import { spriteCreate, type SpritesHttp } from "./sprites";
+import { spriteApplyServices } from "./sprite-config";
+
+/**
+ * #3200: an Op step's stdout is its output, so the Sprites activities write
+ * progress to stderr (as the Machines applier does since #2516) and leave
+ * stdout clean.
+ */
+describe("Sprites activities keep stdout clean (#3200)", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  test("create and services apply write progress to stderr only", async () => {
+    const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+    const stderr: string[] = [];
+    vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => {
+      stderr.push(String(chunk));
+      return true;
+    });
+    const http: SpritesHttp = async (method, url) => {
+      if (method === "POST" && url.endsWith("/v1/sprites")) return { status: 200, text: JSON.stringify({ id: "box-1", url: "https://box-1.sprites.app" }) };
+      if (method === "GET" && url.endsWith("/services")) return { status: 200, text: "[]" };
+      return { status: 200, text: "{}" };
+    };
+    const endpoint = "http://localhost:4290";
+
+    const created = await spriteCreate({ name: "box-1", endpoint }, undefined, http);
+    const applied = await spriteApplyServices(
+      {
+        id: "box-1",
+        endpoint,
+        start: true,
+        services: [
+          { name: "db", cmd: "postgres" },
+          { name: "web", cmd: "node server.js", needs: ["db"] },
+        ],
+      },
+      undefined,
+      http,
+    );
+
+    expect(created.id).toBe("box-1");
+    expect(applied.started).toEqual(["db", "web"]);
+    expect(stdout).not.toHaveBeenCalled();
+    expect(log).not.toHaveBeenCalled();
+    const err = stderr.join("");
+    expect(err).toMatch(/created: sprite\/box-1/);
+    expect(err).toMatch(/services: sprite\/box-1 applied 2\/2/);
+    expect(err).toMatch(/services: sprite\/box-1 started 2 in dependency order/);
+  });
+});

package/src/op/activities/sprite-service-converge.ts

@@ -35,6 +35,7 @@ import { boxServices } from "./box-services";
 import { spriteServiceList, spriteServiceStart, spriteServiceStop } from "./sprite-services";
 import type { SpritesHttp } from "./sprites";
 import { defaultSpritesHttp } from "./sprites";
+import { logProgress } from "./progress";
 
 const execFileAsync = promisify(execFile);
 
@@ -284,7 +285,7 @@ export async function spriteServiceRestart(
     if (!bin) throw new Error("no sprite-env on PATH or in /.sprite/bin, and no sprite id to reach the Sprites API with");
     await execFileAsync(bin, ["services", "restart", name], { signal, timeout: 120_000 });
   }
-  console.log(`restarted: ${name}`);
+  logProgress(`restarted: ${name}`);
   if (!health) return { name, healthy: null };
 
   const until = Date.now() + (args.waitMs ?? 60_000);

package/src/op/activities/sprite-services.ts

@@ -28,6 +28,7 @@
  */
 
 import { resolveSpritesEndpoint, defaultSpritesHttp, type SpritesHttp } from "./sprites";
+import { logProgress } from "./progress";
 
 // ── URL building (pure) ─────────────────────────────────────────────────────
 
@@ -197,7 +198,7 @@ export async function spriteServiceCreate(
   if (svc.state.status === "failed") {
     throw new Error(`sprite ${args.id} service ${args.name} failed to start: ${svc.state.error ?? "unknown error"}`);
   }
-  console.log(`service: sprite/${args.id}/${args.name} ${svc.state.status} (${base})`);
+  logProgress(`service: sprite/${args.id}/${args.name} ${svc.state.status} (${base})`);
   return svc;
 }
 
@@ -249,7 +250,7 @@ export async function spriteServiceStart(
   if (svc.state.status === "failed") {
     throw new Error(`sprite ${args.id} service ${args.name} failed to start: ${svc.state.error ?? "unknown error"}`);
   }
-  console.log(`service: sprite/${args.id}/${args.name} ${svc.state.status} (${base})`);
+  logProgress(`service: sprite/${args.id}/${args.name} ${svc.state.status} (${base})`);
   return svc;
 }
 
@@ -270,7 +271,7 @@ export async function spriteServiceStop(
     throw new Error(`sprite ${args.id} service ${args.name} stop failed (${res.status}): ${res.text}`);
   }
   const svc = await spriteServiceGet({ id: args.id, name: args.name, endpoint: args.endpoint, token: args.token }, signal, http);
-  console.log(`service: sprite/${args.id}/${args.name} ${svc.state.status} (${base})`);
+  logProgress(`service: sprite/${args.id}/${args.name} ${svc.state.status} (${base})`);
   return svc;
 }
 
@@ -288,7 +289,7 @@ export async function spriteServiceDelete(
   if (res.status >= 300 && res.status !== 404) {
     throw new Error(`sprite ${args.id} service ${args.name} delete failed (${res.status}): ${res.text}`);
   }
-  console.log(`deleted: sprite/${args.id}/${args.name}`);
+  logProgress(`deleted: sprite/${args.id}/${args.name}`);
   return {};
 }
 

package/src/op/activities/sprites.ts

@@ -35,6 +35,7 @@
 // every project that imports the lexicon and needs the run fallback (#2613).
 
 import { sleep } from "@intentius/chant/op";
+import { logProgress } from "./progress";
 
 export const DEFAULT_SPRITES_BASE_URL = "https://api.sprites.dev";
 
@@ -420,7 +421,7 @@ export async function spriteCreate(
   const res = await http("POST", spritesUrl(base), spriteCreateBody(args), undefined, signal);
   if (res.status >= 300) throw new Error(`sprite ${args.name} create failed (${res.status}): ${res.text}`);
   const result = parseCreateResponse(res.text, args.name);
-  console.log(`created: sprite/${result.id} (${base})`);
+  logProgress(`created: sprite/${result.id} (${base})`);
   return result;
 }
 
@@ -524,7 +525,7 @@ export async function spriteCheckpoint(
   const res = await http("POST", spriteCheckpointUrl(base, args.id), body, undefined, signal);
   if (res.status >= 300) throw new Error(`sprite ${args.id} checkpoint failed (${res.status}): ${res.text}`);
   const result = parseCheckpointNdjson(res.text);
-  console.log(`checkpoint: sprite/${args.id} @${result.checkpointId} (${base})`);
+  logProgress(`checkpoint: sprite/${args.id} @${result.checkpointId} (${base})`);
   return result;
 }
 
@@ -572,7 +573,7 @@ export async function spriteRestore(
   if (res.status >= 300) {
     throw new Error(`sprite ${args.id} restore to "${target}" failed (${res.status}): ${res.text}`);
   }
-  console.log(`restored: sprite/${args.id} to ${target} (${base})`);
+  logProgress(`restored: sprite/${args.id} to ${target} (${base})`);
   return {};
 }
 
@@ -587,7 +588,7 @@ export async function spriteDestroy(
   if (res.status >= 300 && res.status !== 404) {
     throw new Error(`sprite ${args.id} destroy failed (${res.status}): ${res.text}`);
   }
-  console.log(`destroyed: sprite/${args.id} (${base})`);
+  logProgress(`destroyed: sprite/${args.id} (${base})`);
   return {};
 }
 
@@ -656,7 +657,7 @@ export async function spriteUrl(
         const check = await http("GET", target, undefined, undefined, signal);
         const ok = args.status !== undefined ? check.status === args.status : check.status >= 200 && check.status < 300;
         if (ok) {
-          console.log(`url: sprite/${args.id} answers on ${target} (${check.status})`);
+          logProgress(`url: sprite/${args.id} answers on ${target} (${check.status})`);
           return { url };
         }
         lastErr = `status ${check.status} (want ${args.status ?? "2xx"})`;

package/src/serializer.test.ts

@@ -146,14 +146,14 @@ describe("fly serializer", () => {
     });
   });
 
-  it("emits the Certificate create body (createCertificateRequest: hostname) under the app", () => {
+  it("emits the Certificate create body (createAcmeCertificateRequest: hostname) at /certificates/acme (#3114)", () => {
     const entities = stack(
       ["app", new App({ name: "my-app" })],
       ["cert", new Certificate({ hostname: "example.com" })],
     );
     const out = JSON.parse(flySerializer.serialize(entities) as string);
     expect(out.cert).toEqual({
-      endpoint: "/v1/apps/my-app/certificates",
+      endpoint: "/v1/apps/my-app/certificates/acme",
       method: "POST",
       body: { hostname: "example.com" },
     });
@@ -172,6 +172,21 @@ describe("fly serializer", () => {
       applyOnly: true,
     });
   });
+
+  it("marks a value-less Secret mustExist and takes its name from `name` (#2516)", () => {
+    const entities = stack(
+      ["app", new App({ name: "my-app" })],
+      ["dbUrl", new Secret({ name: "DATABASE_URL" })],
+    );
+    const out = JSON.parse(flySerializer.serialize(entities) as string);
+    expect(out.dbUrl).toEqual({
+      endpoint: "/v1/apps/my-app/secrets/DATABASE_URL",
+      method: "POST",
+      body: {},
+      applyOnly: true,
+      mustExist: true,
+    });
+  });
 });
 
 // ── Pseudo-parameters (Fly.Region / Fly.OrgSlug) ────────────────────────────