@plexus-ms/cli-ci 0.109.0 → 0.111.0

package/dist/plx-ci.js

@@ -7395,7 +7395,7 @@ function realIo(prompt) {
     stdout: process.stdout,
     stderr: process.stderr,
     prompt: prompt && isInteractive() ? prompt : void 0,
-    exec: (command, args, options = {}) => new Promise((resolve4, reject) => {
+    exec: (command, args, options = {}) => new Promise((resolve5, reject) => {
       const child = spawn(command, args, {
         cwd: options.cwd,
         env: options.env,
@@ -7418,7 +7418,7 @@ function realIo(prompt) {
       child.on("error", reject);
       child.on("close", (code) => {
         if (code === 0)
-          resolve4(stdout);
+          resolve5(stdout);
         else
           reject(new ExecError(render(command, args), code, stderr));
       });
@@ -7437,7 +7437,7 @@ function quote(arg) {
   return /^[\w./:@%+=,-]+$/.test(arg) ? arg : `'${arg.replaceAll("'", "'\\''")}'`;
 }
 
-// ../lib/dist/plan.js
+// ../lib/dist/effects.js
 var run = (command, args = [], options = {}) => ({
   kind: "run",
   command,
@@ -7599,6 +7599,7 @@ var import_yaml3 = __toESM(require_dist(), 1);
 
 // ../lib/dist/deployTargets.js
 var import_yaml4 = __toESM(require_dist(), 1);
+import { join } from "node:path";
 
 // ../lib/dist/train.js
 var PRIMITIVES = "primitives";
@@ -7613,6 +7614,72 @@ var PHASES = [[PRIMITIVES], TRAIN_REPOS.filter((r) => r !== PRIMITIVES)];
 var UsageError = class extends Error {
 };
 
+// ../lib/dist/deployTargets.js
+var VARS_DIRS = ["group_vars", "host_vars"];
+var DEFAULT_ENVIRONMENT = "production";
+function parseInventoryFile(path2, text) {
+  const data = (0, import_yaml4.parse)(text);
+  const vars = data !== null && typeof data === "object" ? data : {};
+  const target = typeof vars.plexus_deploy_target === "string" ? vars.plexus_deploy_target : void 0;
+  const records = Array.isArray(vars.plexus_apps) ? vars.plexus_apps : [];
+  const apps = records.flatMap((r) => {
+    if (r === null || typeof r !== "object")
+      return [];
+    const { name, environment } = r;
+    if (typeof name !== "string")
+      return [];
+    return [{ app: name, environment: typeof environment === "string" ? environment : DEFAULT_ENVIRONMENT }];
+  });
+  return { path: path2, target, apps };
+}
+function selectDeployTargets(files) {
+  const targets = {};
+  const declaredIn = {};
+  for (const file of [...files].sort((a, b) => a.path.localeCompare(b.path))) {
+    if (file.target === void 0)
+      continue;
+    for (const { app, environment } of file.apps) {
+      const key = `${app}@${environment}`;
+      if (key in declaredIn)
+        throw new Error(`app "${app}" in ${environment} is declared with a deploy target in both ${declaredIn[key]} and ${file.path}`);
+      const envs = targets[app] ?? {};
+      envs[environment] = file.target;
+      targets[app] = envs;
+      declaredIn[key] = file.path;
+    }
+  }
+  return Object.fromEntries(Object.entries(targets).sort(([a], [b]) => a.localeCompare(b)).map(([app, envs]) => [app, Object.fromEntries(Object.entries(envs).sort(([a], [b]) => a.localeCompare(b)))]));
+}
+function targetsIn(targets, environment) {
+  const flat = {};
+  for (const [app, envs] of Object.entries(targets)) {
+    const target = envs[environment];
+    if (target !== void 0)
+      flat[app] = target;
+  }
+  return flat;
+}
+async function readDeployTargets(ctx, inventory) {
+  const files = [];
+  let found = false;
+  for (const sub of VARS_DIRS) {
+    const dir = join(inventory, sub);
+    const entries = await ctx.readDir(dir).catch(() => void 0);
+    if (entries === void 0)
+      continue;
+    found = true;
+    for (const e of entries) {
+      if (e.directory || !/\.ya?ml$/.test(e.name))
+        continue;
+      const path2 = join(dir, e.name);
+      files.push(parseInventoryFile(path2, await ctx.readFile(path2)));
+    }
+  }
+  if (!found && await ctx.readDir(inventory).catch(() => void 0) === void 0)
+    throw new UsageError(`no inventory directory at ${inventory}`);
+  return selectDeployTargets(files);
+}
+
 // ../lib/dist/ship.js
 var DEFAULT_WAIT = { interval: 15e3, timeout: 20 * 6e4 };
 
@@ -10997,7 +11064,7 @@ import { resolve } from "node:path";
 // src/core/build.ts
 import { randomBytes } from "node:crypto";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { join as join2 } from "node:path";
 
 // src/core/exoscale.ts
 var Exo = class {
@@ -11044,7 +11111,7 @@ var Exo = class {
   async securityGroupId(name) {
     const data = await this.read("list-security-groups");
     const g = (data["security-groups"] ?? []).find((g2) => g2.name === name);
-    if (!g) throw new Error(`no security group "${name}"; it is created once, at setup (ci.md)`);
+    if (!g) throw new Error(`no security group "${name}"; it is created once, at setup`);
     return g.id;
   }
   createInstance(request) {
@@ -11061,8 +11128,11 @@ var Exo = class {
     };
     return this.effect("create-instance", [], JSON.stringify(body2));
   }
-  /** Poll an operation to its end; the create completes within seconds. */
-  async waitOperation(id, attempts = 30) {
+  /**
+   * Poll an operation to its end, for three minutes: a create completes
+   * within seconds as a rule and has taken more than one minute under load.
+   */
+  async waitOperation(id, attempts = 90) {
     let state = "pending";
     for (let i = 0; i < attempts && state === "pending"; i++) {
       if (i > 0) await sleep(2e3);
@@ -11076,33 +11146,9 @@ var Exo = class {
   stopInstance(id) {
     return this.effect("stop-instance", [id]);
   }
-  startInstance(id) {
-    return this.effect("start-instance", [id]);
-  }
-  rebootInstance(id) {
-    return this.effect("reboot-instance", [id]);
-  }
-};
-var Storage = class {
-  ctx;
-  env;
-  constructor(ctx, accessKey, secretKey) {
-    this.ctx = ctx;
-    this.env = { EXOSCALE_API_KEY: accessKey, EXOSCALE_API_SECRET: secretKey };
-  }
-  async bucketExists(bucket) {
-    try {
-      await this.ctx.read("exo", ["storage", "ls", `sos://${bucket}`], { env: { ...process.env, ...this.env } });
-      return true;
-    } catch {
-      return false;
-    }
-  }
-  makeBucket(bucket, zone) {
-    return this.ctx.apply([run("exo", ["storage", "mb", `sos://${bucket}`, "-z", zone], { env: this.env })]);
-  }
-  removeBucket(bucket) {
-    return this.ctx.apply([run("exo", ["storage", "rb", "-r", "-f", `sos://${bucket}`], { env: this.env })]);
+  /** Rename and relabel: the whole label map is sent, since the API replaces it. */
+  updateInstance(id, name, labels) {
+    return this.effect("update-instance", [id], JSON.stringify({ name, labels }));
   }
 };
 function instanceOf(raw) {
@@ -11117,6 +11163,118 @@ function instanceOf(raw) {
 }
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
+// src/core/s3.ts
+import { createHash } from "node:crypto";
+var sha256 = (text) => createHash("sha256").update(text).digest("hex");
+var md5 = (text) => createHash("md5").update(text).digest("base64");
+var unescapeXml = (text) => text.replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"').replaceAll("&apos;", "'").replaceAll("&amp;", "&");
+var escapeXml = (text) => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
+function elements(xml, tag) {
+  return [...xml.matchAll(new RegExp(`<${tag}>([^<]*)</${tag}>`, "g"))].map((m) => unescapeXml(m[1] ?? ""));
+}
+function uploads(xml) {
+  return [...xml.matchAll(/<Upload>(.*?)<\/Upload>/gs)].flatMap((m) => {
+    const key = elements(m[1] ?? "", "Key")[0];
+    const id = elements(m[1] ?? "", "UploadId")[0];
+    return key !== void 0 && id !== void 0 ? [{ key, id }] : [];
+  });
+}
+function deleteBody(keys) {
+  return `<Delete><Quiet>true</Quiet>${keys.map((k) => `<Object><Key>${escapeXml(k)}</Key></Object>`).join("")}</Delete>`;
+}
+var S3 = class {
+  ctx;
+  storage;
+  constructor(ctx, storage) {
+    this.ctx = ctx;
+    this.storage = storage;
+  }
+  /** The curl config of one call: the URL, the signature's inputs, the body, and the status after the answer. */
+  config(call) {
+    const { endpoint, region, accessKey, secretKey } = this.storage;
+    const url = `${endpoint.replace(/\/$/, "")}/${call.path}${call.query ? `?${call.query}` : ""}`;
+    const body2 = call.body ?? "";
+    const lines = [
+      `url = ${JSON.stringify(url)}`,
+      call.method === "HEAD" ? "head" : `request = "${call.method}"`,
+      "silent",
+      "show-error",
+      "max-time = 60",
+      `user = ${JSON.stringify(`${accessKey}:${secretKey}`)}`,
+      `aws-sigv4 = "aws:amz:${region}:s3"`,
+      // Stated, not left to curl: its versions differ in whether they send it, and S3 wants it signed.
+      `header = "x-amz-content-sha256: ${sha256(body2)}"`,
+      ...(call.headers ?? []).map((h) => `header = ${JSON.stringify(h)}`),
+      ...call.body === void 0 ? [] : [`data-binary = ${JSON.stringify(body2)}`],
+      'write-out = "\\n%{http_code}"'
+    ];
+    return `${lines.join("\n")}
+`;
+  }
+  answer(out) {
+    const at = out.lastIndexOf("\n");
+    return { status: Number(out.slice(at + 1)), body: out.slice(0, at) };
+  }
+  async read(call) {
+    return this.answer(await this.ctx.read("curl", ["-K", "-"], { input: this.config(call) }));
+  }
+  /** An effect: its answer, or undefined under --dry-run. A status outside 2xx is an error. */
+  async effect(call, what) {
+    const out = await this.ctx.effect(run("curl", ["-K", "-"], { input: this.config(call) }));
+    if (out === void 0) return void 0;
+    const answer = this.answer(out);
+    if (answer.status < 200 || answer.status >= 300)
+      throw new Error(`${what} answered ${answer.status}: ${answer.body.slice(0, 300)}`);
+    return answer;
+  }
+  async bucketExists(bucket) {
+    const { status } = await this.read({ method: "HEAD", path: bucket });
+    if (status === 200) return true;
+    if (status === 404) return false;
+    throw new Error(`looking up the bucket ${bucket} answered ${status}`);
+  }
+  async makeBucket(bucket) {
+    await this.effect({ method: "PUT", path: bucket }, `creating the bucket ${bucket}`);
+  }
+  /**
+   * Delete a bucket whole: its objects a thousand at a time, the uploads
+   * something left unfinished, then the bucket, which S3 refuses while
+   * either remains. A dry run lists once and says what would go.
+   */
+  async removeBucket(bucket) {
+    let removed = 0;
+    for (; ; ) {
+      const page = await this.read({ method: "GET", path: bucket, query: "list-type=2&max-keys=1000" });
+      if (page.status !== 200) throw new Error(`listing the bucket ${bucket} answered ${page.status}`);
+      const keys = elements(page.body, "Key");
+      if (keys.length === 0) break;
+      const body2 = deleteBody(keys);
+      const answer = await this.effect(
+        { method: "POST", path: bucket, query: "delete=", body: body2, headers: [`Content-MD5: ${md5(body2)}`] },
+        `deleting ${keys.length} objects of ${bucket}`
+      );
+      if (answer === void 0) {
+        this.ctx.say(`  (the objects of ${bucket}, ${keys.length} in the first listing, then the bucket)`);
+        return;
+      }
+      const failed = elements(answer.body, "Message");
+      if (answer.body.includes("<Error>"))
+        throw new Error(`deleting objects of ${bucket} failed: ${failed[0] ?? answer.body.slice(0, 300)}`);
+      removed += keys.length;
+    }
+    const pending = await this.read({ method: "GET", path: bucket, query: "uploads=" });
+    for (const upload of pending.status === 200 ? uploads(pending.body) : []) {
+      const key = upload.key.split("/").map(encodeURIComponent).join("/");
+      await this.effect(
+        { method: "DELETE", path: `${bucket}/${key}`, query: `uploadId=${encodeURIComponent(upload.id)}` },
+        `aborting an unfinished upload in ${bucket}`
+      );
+    }
+    await this.effect({ method: "DELETE", path: bucket }, `deleting the bucket ${bucket}`);
+    this.ctx.say(`removed the bucket ${bucket} and its ${removed} object(s)`);
+  }
+};
+
 // src/core/time.ts
 function iso(at) {
   return at.toISOString().replace(/\.\d{3}Z$/, "Z");
@@ -11134,6 +11292,10 @@ function registryBuckets(owner, now) {
 }
 
 // src/core/registry.ts
+var imageName = (prefix, app) => `${prefix.toLowerCase().replace(/\/$/, "")}/${app}`;
+function registryOf(kind, prefix) {
+  return kind === "own" ? { kind } : { kind, prefix };
+}
 var CURRENT_PORT = 5e3;
 var PREVIOUS_PORT = 5001;
 var REGISTRY_IMAGE = "registry:2";
@@ -11173,7 +11335,10 @@ function sidecarArgs(name, port, bucket, storage, readOnly) {
     "-e",
     "REGISTRY_STORAGE_S3_FORCEPATHSTYLE=true",
     "-e",
-    readOnly ? "REGISTRY_STORAGE_MAINTENANCE_READONLY_ENABLED=true" : "REGISTRY_STORAGE_DELETE_ENABLED=true",
+    // registry:2 panics on the read-only flag as one nested variable
+    // (…_READONLY_ENABLED=true): the map it builds from it has the wrong
+    // type. The whole key as one YAML value is read as from a config file.
+    readOnly ? 'REGISTRY_STORAGE_MAINTENANCE_READONLY={"enabled":true}' : "REGISTRY_STORAGE_DELETE_ENABLED=true",
     REGISTRY_IMAGE
   ];
 }
@@ -11185,12 +11350,19 @@ async function answers(ctx, port) {
     return false;
   }
 }
-async function startSidecars(ctx, storage, owner, zone, now = /* @__PURE__ */ new Date()) {
+async function startSidecars(ctx, storage, owner, now = /* @__PURE__ */ new Date()) {
   const buckets = registryBuckets(owner, now);
-  const sos = new Storage(ctx, storage.accessKey, storage.secretKey);
-  if (!await sos.bucketExists(buckets.current)) {
-    ctx.say(`\u2192 creating the bucket ${buckets.current} in ${zone}`);
-    await sos.makeBucket(buckets.current, zone);
+  if (!ctx.dryRun && await answers(ctx, CURRENT_PORT)) {
+    const previousAnswers2 = await answers(ctx, PREVIOUS_PORT);
+    ctx.say(`registry: :${CURRENT_PORT} answers already, the host's pair; nothing started`);
+    return { current: buckets.current, previous: buckets.previous, previousAnswers: previousAnswers2 };
+  }
+  if (!storage)
+    throw new Error("REGISTRY_ENDPOINT, REGISTRY_REGION, REGISTRY_ACCESS_KEY and REGISTRY_SECRET_KEY are needed");
+  const s3 = new S3(ctx, storage);
+  if (!await s3.bucketExists(buckets.current)) {
+    ctx.say(`\u2192 creating the bucket ${buckets.current}`);
+    await s3.makeBucket(buckets.current);
   }
   await ctx.apply([
     run("docker", sidecarArgs("registry", CURRENT_PORT, buckets.current, storage, false)),
@@ -11257,14 +11429,247 @@ async function layers(ctx, app, tag, port = CURRENT_PORT) {
     return void 0;
   }
 }
+function candidateRef(registry, app, tag) {
+  return registry.kind === "own" ? `localhost:${CURRENT_PORT}/${app}:${tag}` : `${imageName(registry.prefix, app)}:${tag}`;
+}
+async function craneDigest(ctx, ref) {
+  try {
+    return (await ctx.read("crane", ["digest", ref])).trim() || void 0;
+  } catch {
+    return void 0;
+  }
+}
+async function findTag(ctx, registry, app, tag) {
+  if (registry.kind === "own") {
+    const port = await tagExists(ctx, app, tag);
+    return port === void 0 ? void 0 : `localhost:${port}/${app}:${tag}`;
+  }
+  const ref = candidateRef(registry, app, tag);
+  return await craneDigest(ctx, ref) === void 0 ? void 0 : ref;
+}
+function digestOf(ctx, registry, app, tag) {
+  return registry.kind === "own" ? digest(ctx, app, tag) : craneDigest(ctx, candidateRef(registry, app, tag));
+}
+async function layersOf(ctx, registry, app, tag) {
+  if (registry.kind === "own") return layers(ctx, app, tag);
+  try {
+    const data = JSON.parse(await ctx.read("crane", ["manifest", candidateRef(registry, app, tag)]));
+    return (data.layers ?? []).map((l) => ({ digest: l.digest, size: l.size }));
+  } catch {
+    return void 0;
+  }
+}
+async function loggedIn(ctx, host, env = process.env) {
+  const file = `${env.DOCKER_CONFIG || `${env.HOME}/.docker`}/config.json`;
+  try {
+    const config = JSON.parse(await ctx.readFile(file));
+    return Boolean(config.auths?.[host] || config.credHelpers?.[host] || config.credsStore);
+  } catch {
+    return false;
+  }
+}
+async function rotateBuckets(ctx, storage, owner, now = /* @__PURE__ */ new Date()) {
+  const buckets = registryBuckets(owner, now);
+  const s3 = new S3(ctx, storage);
+  const result = {};
+  if (!await s3.bucketExists(buckets.current)) {
+    ctx.say(`rotate: creating the bucket ${buckets.current}`);
+    await s3.makeBucket(buckets.current);
+    result.created = buckets.current;
+  }
+  if (await s3.bucketExists(buckets.expired)) {
+    ctx.say(`rotate: the registry bucket ${buckets.expired} is two months old`);
+    await s3.removeBucket(buckets.expired);
+    result.removed = buckets.expired;
+  }
+  return result;
+}
+
+// src/core/strategy.ts
+var STRATEGIES = ["github", "on-prem", "hybrid"];
+var REGISTRIES = ["ghcr", "own"];
+var OVERLAYS = ["none", "netbird"];
+function oneOf(name, value, allowed) {
+  if (!allowed.includes(value))
+    throw new UsageError(`${name} is "${value}"; it is one of ${allowed.join(", ")}`);
+  return value;
+}
+function settingsOf(env = process.env, flag) {
+  const strategy = oneOf("the strategy", flag || env.CI_STRATEGY || "github", STRATEGIES);
+  const registry = oneOf("CI_REGISTRY", env.CI_REGISTRY || (strategy === "github" ? "ghcr" : "own"), REGISTRIES);
+  const overlay = oneOf("CI_OVERLAY", env.CI_OVERLAY || "none", OVERLAYS);
+  return { strategy, registry, overlay };
+}
+var sizeLabel = (size) => `ci-${size}`;
+function runnerOf(strategy, size, instance) {
+  if (strategy === "github") return "ubuntu-latest";
+  if (strategy === "hybrid" && size === "large") {
+    if (!instance) throw new Error("a large job under hybrid needs its instance\u2019s label");
+    return instance;
+  }
+  return sizeLabel(size);
+}
+
+// src/core/plan.ts
+var NONEXISTENT = "<NONEXISTENT>";
+function hashesOf(dryRun, packages) {
+  const data = JSON.parse(dryRun);
+  const tasks = (data.tasks ?? []).filter((t) => t.command !== NONEXISTENT);
+  return Object.entries(packages).flatMap(([app, name]) => {
+    const build2 = tasks.find((t) => t.package === name && t.taskId.endsWith("#build"));
+    if (!build2) return [];
+    const test2 = tasks.find((t) => t.package === name && t.taskId.endsWith("#test"));
+    return [{ app, hash: build2.hash, ...test2 ? { test: test2.hash } : {} }];
+  });
+}
+async function packageName(ctx, dir, app) {
+  const pkg = JSON.parse(await ctx.readFile(`${dir}/apps/${app}/package.json`));
+  if (!pkg.name) throw new UsageError(`apps/${app}/package.json has no name`);
+  return pkg.name;
+}
+async function turboOf(ctx, dir) {
+  const pkg = JSON.parse(await ctx.readFile(`${dir}/package.json`));
+  return `turbo@${pkg.devDependencies?.turbo ?? "latest"}`;
+}
+async function readHashes(ctx, dir, apps) {
+  const packages = {};
+  for (const app of apps) packages[app] = await packageName(ctx, dir, app);
+  const out = await ctx.read("npx", ["--yes", await turboOf(ctx, dir), "run", "build", "test", "--dry-run=json"], {
+    cwd: dir
+  });
+  return hashesOf(out, packages);
+}
+function legsOf(apps, hashes, mode, strategy, run2) {
+  return apps.flatMap((app) => {
+    const hash = hashes.find((h) => h.app === app)?.hash;
+    if (!hash) throw new UsageError(`no build hash for ${app}; is it an app package of the workspace?`);
+    const copies = mode === "repro" ? 2 : 1;
+    return Array.from({ length: copies }, (_, i) => {
+      const n = i + 1;
+      const base = `exo-build-${run2.runId}-${app}-${run2.attempt}`;
+      return { app, hash, runner: runnerOf(strategy, "large", mode === "repro" ? `${base}-${n}` : base), n };
+    });
+  });
+}
+function testLegsOf(apps, hashes, strategy, run2) {
+  return apps.flatMap((app) => {
+    const h = hashes.find((x) => x.app === app);
+    if (!h?.test) return [];
+    const runner = runnerOf(strategy, "large", `exo-test-${run2.runId}-${app}-${run2.attempt}`);
+    return [{ app, hash: h.test, build: h.hash, runner }];
+  });
+}
+function environmentOf(event, ref, given = "") {
+  if (given) return given;
+  if (event !== "push" && event !== "workflow_dispatch") return "";
+  return ref === "main" ? "production" : ref === "develop" ? "staging" : "";
+}
+function deployLegsOf(apps, targets, prefix, sha) {
+  const legs = [];
+  const without = [];
+  for (const app of apps) {
+    const host = targets[app];
+    if (host === void 0) without.push(app);
+    else legs.push({ app, host, image: `${imageName(prefix, app)}:${sha}` });
+  }
+  return { legs, without };
+}
+async function plan(ctx, options) {
+  const { apps, hashes, mode, strategy, registry, run: run2 } = options;
+  const result = {
+    legs: [],
+    tests: [],
+    apps: apps.map((app) => {
+      const h = hashes.find((x) => x.app === app);
+      if (!h) throw new UsageError(`no build hash for ${app}; is it an app package of the workspace?`);
+      return h;
+    }),
+    environment: options.deploy?.environment ?? "",
+    deploys: [],
+    skipped: [],
+    skippedTests: []
+  };
+  for (const leg of legsOf(apps, hashes, mode, strategy, run2)) {
+    if (mode === "build") {
+      const ref = await findTag(ctx, registry, leg.app, `build-${leg.hash}`);
+      if (ref !== void 0) {
+        ctx.say(`${leg.app}: ${ref} exists, nothing to build`);
+        result.skipped.push({ app: leg.app, hash: leg.hash, ref });
+        continue;
+      }
+    }
+    result.legs.push(leg);
+  }
+  if (options.tests && mode !== "repro") {
+    for (const leg of testLegsOf(apps, hashes, strategy, run2)) {
+      if (mode === "build") {
+        const ref = await findTag(ctx, registry, leg.app, `test-${leg.hash}`);
+        if (ref !== void 0) {
+          ctx.say(`${leg.app}: ${ref} exists, nothing to test`);
+          result.skippedTests.push({ app: leg.app, hash: leg.hash, ref });
+          continue;
+        }
+      }
+      result.tests.push(leg);
+    }
+  }
+  if (options.deploy?.environment) {
+    const { environment, targets, prefix, sha } = options.deploy;
+    const { legs, without } = deployLegsOf(apps, targets, prefix, sha);
+    result.deploys = legs;
+    for (const app of without) ctx.say(`${app} has no deployment in ${environment}; nothing to deploy`);
+  }
+  return result;
+}
+async function report(ctx, registry, legs, mode, runId) {
+  const apps = [...new Set(legs.map((l) => l.app))].sort();
+  const out = [];
+  for (const app of apps) {
+    const hash = legs.find((l) => l.app === app)?.hash ?? "";
+    const c = { app, hash, digest: await digestOf(ctx, registry, app, `build-${hash}`) };
+    ctx.say(`${app} build-${hash}: ${c.digest ?? "missing"}`);
+    if (mode === "repro") {
+      const [one, two] = await Promise.all([
+        digestOf(ctx, registry, app, `repro-${runId}-1`),
+        digestOf(ctx, registry, app, `repro-${runId}-2`)
+      ]);
+      const identical = one !== void 0 && one === two;
+      const [l1, l2] = identical ? [[], []] : await Promise.all([
+        layersOf(ctx, registry, app, `repro-${runId}-1`),
+        layersOf(ctx, registry, app, `repro-${runId}-2`)
+      ]);
+      const differing = [];
+      const a = l1 ?? [];
+      const b = l2 ?? [];
+      for (let i = 0; i < Math.max(a.length, b.length); i++) {
+        if (a[i]?.digest !== b[i]?.digest)
+          differing.push(
+            `layer ${i + 1}: ${a[i]?.digest ?? "-"} (${a[i]?.size ?? "-"}) vs ${b[i]?.digest ?? "-"} (${b[i]?.size ?? "-"})`
+          );
+      }
+      c.repro = { one, two, identical, layers: a.length, same: a.length - differing.length, differing };
+      if (identical) ctx.say(`  reproducible: both builds are ${one}`);
+      else {
+        ctx.say(
+          `  not identical: ${one ?? "-"} vs ${two ?? "-"}; layers: ${a.length} in each, ${a.length - differing.length} identical`
+        );
+        for (const d of differing) ctx.say(`  ${d}`);
+      }
+    }
+    out.push(c);
+  }
+  return out;
+}
 
 // src/core/build.ts
-var REGISTRY = `localhost:${CURRENT_PORT}`;
 var PREVIOUS_REGISTRY = `localhost:${PREVIOUS_PORT}`;
+function tagsOf(options) {
+  return [`build-${options.hash}`, ...options.registry.kind === "own" ? [options.sha] : [], ...options.tags ?? []];
+}
 function buildArgs(options, context, metadataFile, previousAnswers) {
-  const { app, hash, sha } = options;
-  const ref = (tag) => `${REGISTRY}/${app}:${tag}`;
-  const tags = [`build-${hash}`, sha, ...options.tags ?? []].flatMap((t) => ["--tag", ref(t)]);
+  const { app } = options;
+  const ref = (tag) => candidateRef(options.registry, app, tag);
+  const tags = tagsOf(options).flatMap((t) => ["--tag", ref(t)]);
   const cache = options.noCache ? ["--no-cache"] : [
     "--cache-from",
     `type=registry,ref=${ref("cache")}`,
@@ -11283,7 +11688,7 @@ function buildArgs(options, context, metadataFile, previousAnswers) {
     "--build-arg",
     `APP=${app}`,
     "--build-arg",
-    `BUILD_HASH=${hash}`,
+    `BUILD_HASH=${options.hash}`,
     "--secret",
     "id=next-server-actions-encryption-key,env=NEXT_SERVER_ACTIONS_ENCRYPTION_KEY",
     ...cache,
@@ -11300,159 +11705,71 @@ function buildArgs(options, context, metadataFile, previousAnswers) {
     "plain"
   ];
 }
+async function ensureBuilder(ctx, name) {
+  if (!ctx.dryRun) {
+    try {
+      await ctx.read("docker", ["buildx", "inspect", name]);
+      return;
+    } catch {
+    }
+  }
+  await ctx.apply([
+    run("docker", [
+      "buildx",
+      "create",
+      "--name",
+      name,
+      "--driver",
+      "docker-container",
+      "--driver-opt",
+      "network=host",
+      "--bootstrap"
+    ])
+  ]);
+}
 async function build(ctx, options) {
   if (!options.encryptionKey) throw new UsageError("NEXT_SERVER_ACTIONS_ENCRYPTION_KEY is empty");
   const dir = options.dir ?? ".";
-  const out = join(tmpdir(), `plx-ci-build-${randomBytes(4).toString("hex")}`);
-  const context = join(out, "full");
+  const out = join2(tmpdir(), `plx-ci-build-${randomBytes(4).toString("hex")}`);
+  const context = join2(out, "full");
+  const builder = options.builder ?? "ci";
   const env = {
     NEXT_SERVER_ACTIONS_ENCRYPTION_KEY: options.encryptionKey,
-    BUILDX_BUILDER: options.builder ?? "ci",
+    BUILDX_BUILDER: builder,
     SOURCE_DATE_EPOCH: "0"
   };
-  const pkg = JSON.parse(await ctx.readFile(join(dir, "package.json")));
-  const turbo = pkg.devDependencies?.turbo ?? "latest";
+  const name = await packageName(ctx, dir, options.app);
+  await ensureBuilder(ctx, builder);
   let t = Date.now();
   await ctx.apply([
-    run("npx", ["--yes", `turbo@${turbo}`, "prune", `${options.scope}/${options.app}`, "--docker", "--out-dir", out], {
+    run("npx", ["--yes", await turboOf(ctx, dir), "prune", name, "--docker", "--out-dir", out], {
       cwd: dir
     }),
-    run("cp", [join(out, "pnpm-lock.yaml"), join(context, "pnpm-lock.yaml")]),
-    run("cp", [join(dir, ".dockerignore"), join(context, ".dockerignore")]),
-    run("mkdir", ["-p", join(context, "packages")])
+    run("cp", [join2(out, "pnpm-lock.yaml"), join2(context, "pnpm-lock.yaml")]),
+    run("cp", [join2(dir, ".dockerignore"), join2(context, ".dockerignore")]),
+    run("mkdir", ["-p", join2(context, "packages")])
   ]);
   const pruneSeconds = Math.round((Date.now() - t) / 1e3);
   ctx.say(`build: prune ${pruneSeconds}s`);
-  const previousAnswers = ctx.dryRun ? true : await answers(ctx, PREVIOUS_PORT);
-  if (!previousAnswers) ctx.say(`build: no previous-month registry on ${PREVIOUS_REGISTRY}`);
-  const metadataFile = join(out, "metadata.json");
+  const previousAnswers = options.registry.kind === "own" && (ctx.dryRun ? true : await answers(ctx, PREVIOUS_PORT));
+  if (options.registry.kind === "own" && !previousAnswers)
+    ctx.say(`build: no previous-month registry on ${PREVIOUS_REGISTRY}`);
+  const metadataFile = join2(out, "metadata.json");
   t = Date.now();
   await ctx.apply([run("docker", buildArgs(options, context, metadataFile, previousAnswers), { env })]);
   const buildSeconds = Math.round((Date.now() - t) / 1e3);
   ctx.say(`build: build${options.noCache ? " no-cache" : ""} ${buildSeconds}s`);
-  const tags = [`build-${options.hash}`, options.sha, ...options.tags ?? []];
+  const tags = tagsOf(options);
   const result = { app: options.app, tags, pruneSeconds, buildSeconds };
   if (!ctx.dryRun) {
     const meta = JSON.parse(await ctx.readFile(metadataFile));
     result.digest = meta["containerimage.digest"];
-    ctx.say(`build: ${options.app} build-${options.hash} ${options.sha} -> ${result.digest ?? "<no digest>"}`);
+    ctx.say(`build: ${options.app} ${tags.join(" ")} -> ${result.digest ?? "<no digest>"}`);
   }
   await ctx.apply([run("rm", ["-rf", out])]);
   return result;
 }
 
-// src/commands/build.ts
-function register(program2) {
-  program2.command("build").description("build one app\u2019s image from its pruned context against the registry sidecars and push it").argument("<app>", "the app, apps/<app> in the checkout").argument("<hash>", "Turbo\u2019s hash of the app\u2019s build task").argument("<sha>", "the commit, a second tag").argument("[tags...]", "further tags on the same image").option("--scope <scope>", "the npm scope of the workspace packages", "@damage-ms").option("-C, --dir <dir>", "the checkout to read", ".").action(
-    async (app, hash, sha, tags, opts, command) => {
-      const ctx = contextOf(command);
-      const result = await build(ctx, {
-        app,
-        hash,
-        sha,
-        tags,
-        encryptionKey: process.env.NEXT_SERVER_ACTIONS_ENCRYPTION_KEY ?? "",
-        noCache: process.env.NO_CACHE === "1",
-        dir: resolve(opts.dir),
-        scope: opts.scope,
-        builder: process.env.BUILDX_BUILDER
-      });
-      ctx.result(result, (r) => r.digest ?? "");
-      if (process.env.GITHUB_OUTPUT && result.digest)
-        appendFileSync(process.env.GITHUB_OUTPUT, `digest=${result.digest}
-`);
-    }
-  );
-}
-
-// src/core/inspect.ts
-import { join as join2 } from "node:path";
-var LOGS = [
-  ["journal-previous-boot.log", "sudo journalctl -b -1 --no-pager -o short-iso"],
-  [
-    "journal-previous-boot-network.log",
-    'sudo journalctl -b -1 --no-pager -o short-iso -u systemd-networkd -u cloud-init-local -u cloud-init -u "ifup@*" -u networking -k'
-  ],
-  ["cloud-init.log", "sudo cat /var/log/cloud-init.log"],
-  ["cloud-init-output.log", "sudo cat /var/log/cloud-init-output.log"],
-  [
-    "system.txt",
-    "sudo journalctl --list-boots --no-pager; echo; cat /etc/cloud/cloud.cfg.d/*.cfg 2>/dev/null; echo; ls -la /etc/network/interfaces.d/ /etc/systemd/network/ /run/systemd/network/ 2>/dev/null; echo; cat /etc/network/interfaces.d/* /etc/systemd/network/* /run/systemd/network/* 2>/dev/null; echo; ip -br link; ip -br addr"
-  ]
-];
-function sshArgs(key, ip, command) {
-  return [
-    "-i",
-    key,
-    "-o",
-    "BatchMode=yes",
-    "-o",
-    "ConnectTimeout=5",
-    "-o",
-    "StrictHostKeyChecking=no",
-    "-o",
-    "UserKnownHostsFile=/dev/null",
-    "-o",
-    "LogLevel=ERROR",
-    `debian@${ip}`,
-    command
-  ];
-}
-async function inspect(ctx, exo, options) {
-  const instance = await exo.instance(options.id);
-  if (!instance) throw new UsageError(`no instance ${options.id} in ${exo.zone}`);
-  if (!instance.publicIp) throw new UsageError(`instance ${options.id} has no public address`);
-  const ip = instance.publicIp;
-  const out = options.out ?? `inspect-${options.id}`;
-  const ssh = (command) => ctx.read("ssh", sshArgs(options.sshKey, ip, command));
-  ctx.say(`inspect: ${options.id} at ${ip}, state ${instance.state}`);
-  if (instance.state === "stopped") {
-    ctx.say("inspect: starting (the launch stopped it when it kept it)");
-    await exo.startInstance(options.id);
-  } else {
-    const before = await ssh("true").then(
-      () => true,
-      () => false
-    );
-    ctx.say(
-      before ? "  SSH answers before the reboot (not a stall?)" : "  no SSH before the reboot, as expected of a stalled boot"
-    );
-    ctx.say("inspect: rebooting");
-    await exo.rebootInstance(options.id);
-  }
-  if (ctx.dryRun) {
-    ctx.say(`  (then: wait for SSH, mask plexus-ci-runner and the timers, copy the previous boot's logs to ${out}/)`);
-    return out;
-  }
-  let reached = false;
-  for (let i = 0; i < 60 && !reached; i++) {
-    await sleep(2e3);
-    reached = await ssh("true").then(
-      () => true,
-      () => false
-    );
-  }
-  if (!reached) throw new Error("no SSH after the reboot");
-  ctx.say("inspect: in, masking the units");
-  await ssh(
-    "sudo systemctl mask --runtime --now plexus-ci-runner.service plexus-ci-no-job.timer plexus-ci-death.timer plexus-ci-poweroff.timer 2>&1 | tail -2; sudo systemctl stop plexus-ci-runner.service 2>/dev/null || true"
-  );
-  ctx.say("inspect: copying the logs");
-  await ctx.apply([run("mkdir", ["-p", out])]);
-  for (const [file, command] of LOGS) {
-    let text;
-    try {
-      text = await ssh(command);
-    } catch (error) {
-      text = error instanceof ExecError ? `${error.message}
-` : String(error);
-    }
-    await ctx.apply([write(join2(out, file), text)]);
-  }
-  ctx.say(`inspect: done; the instance ${options.id} is still running, delete it when finished`);
-  return out;
-}
-
 // src/core/github.ts
 var GitHubRunners = class {
   ctx;
@@ -11536,60 +11853,145 @@ function githubOf(ctx, env = process.env) {
   return env.RUNNER_TOKEN ? new GitHubRunners(ctx, need("GITHUB_REPOSITORY", env), env.RUNNER_TOKEN) : void 0;
 }
 var ownerOf = (repo) => repo.split("/")[0] ?? repo;
-
-// src/commands/inspect.ts
-function register2(program2) {
-  program2.command("inspect").description("start or reboot a kept instance, mask its runner and timers, and copy the previous boot\u2019s logs here").argument("<id>", "the instance id").option("--ssh-key <path>", "the private half of the key the instances get", `${process.env.HOME}/.ssh/deploy`).option("-o, --out <dir>", "where the logs land (default: ./inspect-<id>)").action(async (id, opts, command) => {
-    const ctx = contextOf(command);
-    const out = await inspect(ctx, exoOf(ctx), { id, sshKey: opts.sshKey, out: opts.out });
-    ctx.result(out);
-  });
+function withStrategy(command) {
+  return command.option("--strategy <strategy>", "github, on-prem or hybrid (default: CI_STRATEGY, else github)");
+}
+function settings(opts, env = process.env) {
+  return settingsOf(env, opts.strategy);
+}
+function imagePrefix(env = process.env) {
+  return env.IMAGE_PREFIX || `ghcr.io/${ownerOf(need("GITHUB_REPOSITORY", env))}`;
+}
+function registryFor(s, env = process.env) {
+  return registryOf(s.registry, s.registry === "ghcr" ? imagePrefix(env) : "");
 }
 
-// src/commands/launch.ts
-import { appendFileSync as appendFileSync2 } from "node:fs";
+// src/commands/build.ts
+function register(program2) {
+  withStrategy(program2.command("build")).description("build one app\u2019s image from its pruned context against the registry and push it").argument("<app>", "the app, apps/<app> in the checkout").argument("<hash>", "Turbo\u2019s hash of the app\u2019s build task").argument("<sha>", "the commit, a second tag").argument("[tags...]", "further tags on the same image").option("-C, --dir <dir>", "the checkout to read", ".").action(
+    async (app, hash, sha, tags, opts, command) => {
+      const ctx = contextOf(command);
+      const result = await build(ctx, {
+        app,
+        hash,
+        sha,
+        tags,
+        encryptionKey: process.env.NEXT_SERVER_ACTIONS_ENCRYPTION_KEY ?? "",
+        noCache: process.env.NO_CACHE === "1",
+        dir: resolve(opts.dir),
+        registry: registryFor(settings(opts)),
+        builder: process.env.BUILDX_BUILDER
+      });
+      ctx.result(result, (r) => r.digest ?? "");
+      if (process.env.GITHUB_OUTPUT && result.digest)
+        appendFileSync(process.env.GITHUB_OUTPUT, `digest=${result.digest}
+`);
+    }
+  );
+}
 
-// src/core/launch.ts
+// src/core/createInstances.ts
 import { gzipSync } from "node:zlib";
 
-// src/core/teardown.ts
+// src/core/destroyInstances.ts
 var repoLabel = (repo) => repo.replace("/", "-");
-function selectRunInstances(instances, run2, keep2 = []) {
+var KEPT_LABEL = "kept";
+function selectRunInstances(instances, run2) {
   return instances.filter(
-    (i) => i.labels["plexus-ci"] === "true" && i.labels.repo === repoLabel(run2.repo) && i.labels.run === run2.runId && i.labels.attempt === run2.attempt && !keep2.includes(i.id)
+    (i) => i.labels["plexus-ci"] === "true" && i.labels.repo === repoLabel(run2.repo) && i.labels.run === run2.runId && i.labels.attempt === run2.attempt && i.labels[KEPT_LABEL] === void 0
   );
 }
 function selectRunRegistrations(runners, run2) {
   const pattern = new RegExp(`^exo-[a-z]+-${run2.runId}(-[^-]+)*-${run2.attempt}(-[a-z0-9]+)?$`);
   return runners.filter((r) => !r.busy && pattern.test(r.name));
 }
-async function teardown(ctx, exo, github, run2, keep2 = []) {
+async function destroyInstances(ctx, exo, github, run2, { retries = 12, wait = sleep } = {}) {
   const all = await exo.listInstances();
-  const mine = selectRunInstances(all, run2, keep2);
-  const kept = all.filter((i) => keep2.includes(i.id)).map((i) => i.name);
-  for (const name of kept) ctx.say(`${name}: kept for inspection`);
-  for (const i of mine) {
-    ctx.say(`${i.name} (${i.id}): ${i.state}`);
-    await exo.deleteInstance(i.id);
-  }
-  ctx.say(`teardown: ${mine.length} instance(s) of run ${run2.runId} attempt ${run2.attempt}`);
-  const registrations = [];
-  if (github) {
+  const mine = selectRunInstances(all, run2);
+  const kept = all.filter(
+    (i) => i.labels["plexus-ci"] === "true" && i.labels[KEPT_LABEL] !== void 0 && i.labels.repo === repoLabel(run2.repo) && i.labels.run === run2.runId
+  ).map((i) => i.name);
+  for (const name of kept) ctx.say(`${name}: kept after a missed boot, left alone`);
+  let left = mine;
+  for (let round = 0; left.length > 0 && round <= retries; round++) {
+    if (round > 0) await wait(1e4);
+    const refused = [];
+    for (const i of left) {
+      if (round === 0) ctx.say(`${i.name} (${i.id}): ${i.state}`);
+      try {
+        await exo.deleteInstance(i.id);
+      } catch (error) {
+        if (round === retries) ctx.say(`\u2717 ${i.name} (${i.id}) is not deleted: ${error.message}`);
+        refused.push(i);
+      }
+    }
+    left = refused;
+  }
+  ctx.say(
+    `destroy-instances: ${mine.length - left.length} of ${mine.length} instance(s) of run ${run2.runId} attempt ${run2.attempt} deleted`
+  );
+  const registrations = [];
+  if (github) {
     for (const r of selectRunRegistrations(await github.list(), run2)) {
       ctx.say(`runner ${r.name} (${r.id}): ${r.status}`);
       await github.deleteRunner(r.id);
       registrations.push(r.name);
     }
-    ctx.say(`teardown: ${registrations.length} registration(s) removed`);
+    ctx.say(`destroy-instances: ${registrations.length} registration(s) removed`);
   }
+  if (left.length > 0)
+    throw new Error(
+      `${left.length} instance(s) could not be deleted: ${left.map((i) => i.name).join(", ")}; they power off on their own, and the sweep deletes them`
+    );
   return { instances: mine.map((i) => i.name), kept, registrations };
 }
 
+// src/core/mail.ts
+import { tmpdir as tmpdir2 } from "node:os";
+import { join as join3 } from "node:path";
+function smtpFromEnv(env = process.env) {
+  const { SMTP_HOST, SMTP_PORT, SMTP_USER, SMTP_PASSWORD, MAIL_FROM, MAIL_TO } = env;
+  if (!SMTP_HOST || !SMTP_USER || !SMTP_PASSWORD || !MAIL_FROM || !MAIL_TO) return void 0;
+  return {
+    host: SMTP_HOST,
+    port: SMTP_PORT || "587",
+    user: SMTP_USER,
+    password: SMTP_PASSWORD,
+    from: MAIL_FROM,
+    to: MAIL_TO
+  };
+}
+function message(smtp, subject, text, date) {
+  const body2 = text.replaceAll("\r\n", "\n").replaceAll("\n", "\r\n");
+  return `From: plexus-ci <${smtp.from}>\r
+To: ${smtp.to}\r
+Subject: ${subject}\r
+Date: ${date.toUTCString()}\r
+\r
+${body2}\r
+`;
+}
+async function sendMail(ctx, smtp, subject, text, date = /* @__PURE__ */ new Date()) {
+  const scheme = smtp.port === "465" || smtp.port === "2465" ? "smtps" : "smtp";
+  const file = join3(tmpdir2(), `plx-ci-mail-${process.pid}.eml`);
+  const config = [
+    `url = "${scheme}://${smtp.host}:${smtp.port}"`,
+    "ssl-reqd",
+    "silent",
+    "show-error",
+    `mail-from = "${smtp.from}"`,
+    `mail-rcpt = "${smtp.to}"`,
+    `user = ${JSON.stringify(`${smtp.user}:${smtp.password}`)}`,
+    `upload-file = ${JSON.stringify(file)}`
+  ].join("\n");
+  await ctx.apply([write(file, message(smtp, subject, text, date)), run("curl", ["-K", "-"], { input: config })]);
+}
+
 // src/core/userData.ts
 var RUNNER_USER_DATA = `#cloud-config
-# One throwaway CI runner (ci.md, "Instances"). Rendered by launch.sh, which
-# fills the placeholders. Every teardown layer is armed here, before the
-# runner exists:
+# One throwaway CI runner (PLX-Delivery, \xA7 8). Rendered by plx-ci hybrid
+# create-instances, which fills the placeholders. Every teardown layer is
+# armed here, before the runner exists:
 #
 #   layer 1   the runner exits after its one job, the instance deletes itself
 #   layer 1b  noJobTimeout, @@NO_JOB_MINUTES@@ minutes after boot with no job ever started,
@@ -11605,7 +12007,7 @@ var RUNNER_USER_DATA = `#cloud-config
 #
 # The self-delete key and the registry bucket's write key sit in
 # /etc/plexus-ci/env, readable by root; the job that runs here could read
-# them anyway (ci.md, "Secrets").
+# them anyway (\xA7 8 PLX-Delivery).
 hostname: @@LABEL@@
 
 write_files:
@@ -11644,7 +12046,7 @@ write_files:
   # The env file is sourced exported (set -a): exo reads the key from the
   # environment, and a plain source would leave it shell-local. A timer's
   # service has no HOME, and exo refuses to run without one (layers 1b and
-  # 2b failed on it on 2026-11-04, ci.md); the runner's unit has a user.
+  # 2b failed on it on 2026-11-04); the runner's unit has a user.
   - path: /usr/local/sbin/plexus-ci-exo
     permissions: "0755"
     content: |
@@ -11688,7 +12090,7 @@ write_files:
       set -u
       set -a; . /etc/plexus-ci/env; set +a
       # SELF_LAYERS=off (the launch's SELF_LAYERS, while the boot stall is
-      # investigated, ci.md): layers 1, 1b and 2b log and do nothing; the
+      # investigated): layers 1, 1b and 2b log and do nothing; the
       # power-off, the teardown job and the sweep remain.
       [ "\${SELF_LAYERS:-on}" != off ] || { logger -t plexus-ci "self-delete ($1): layers off, not deleting"; exit 0; }
       id=$(curl -fsS --max-time 10 http://metadata.exoscale.com/latest/meta-data/instance-id) || id=
@@ -11702,7 +12104,7 @@ write_files:
       #!/bin/sh
       # Layer 1b: the runner spawns a Worker process for a job and nothing
       # else, and logs it as _diag/Worker_*.log; _work itself exists empty
-      # from the runner's start (seen 2026-11-04, ci.md). No Worker log by
+      # from the runner's start (seen 2026-11-04). No Worker log by
       # now means no job ever came, and nothing will.
       set -u
       if ! ls /opt/runner/_diag/Worker_*.log >/dev/null 2>&1; then
@@ -11771,6 +12173,32 @@ write_files:
       [Install]
       WantedBy=timers.target
 
+  # Frees the dpkg lock for the run commands. The image is weeks old, so
+  # apt-daily-upgrade's persistent timer catches up seconds into the boot, and
+  # unattended-upgrades then holds the lock for about a minute: every apt-get
+  # here failed at once, there was no docker and no runner user, and the
+  # launch kept the instance as a stalled boot. Waiting on the lock costs that
+  # minute, so the instance, which lives under an hour, masks the apt timers
+  # and services (they never start after this), kills whatever of them
+  # already runs, dpkg children included (systemctl kill takes the unit's
+  # whole cgroup), and lets dpkg finish what a kill interrupted. Masking alone
+  # is not enough: the catch-up can start before any run command does, and
+  # bootcmd is out (see above).
+  - path: /usr/local/sbin/plexus-ci-apt-free
+    permissions: "0755"
+    content: |
+      #!/bin/sh
+      units='apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service unattended-upgrades.service'
+      systemctl mask $units >/dev/null 2>&1
+      systemctl kill --signal=KILL apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null
+      i=0
+      while pgrep -x 'unattended-upgr|apt\\.systemd\\.dai|apt-get|apt|dpkg' >/dev/null && [ "$i" -lt 10 ]; do
+        sleep 1; i=$((i + 1))
+      done
+      systemctl stop $units 2>/dev/null
+      dpkg --configure -a
+      logger -t plexus-ci "apt-free: apt timers masked, dpkg lock free after \${i}s"
+
   # The runner, one job, then layer 1: ExecStopPost runs as root (the +) when
   # the runner exits for any reason.
   - path: /etc/systemd/system/plexus-ci-runner.service
@@ -11793,45 +12221,21 @@ runcmd:
   # Layers 1b, 2b and 2a first, before anything that can hang.
   - systemctl daemon-reload
   - systemctl start plexus-ci-no-job.timer plexus-ci-death.timer plexus-ci-poweroff.timer
+  - plexus-ci-apt-free
   - export DEBIAN_FRONTEND=noninteractive; apt-get update -q && apt-get install -yq --no-install-recommends ca-certificates curl jq git tar unzip
   - curl -fsSL https://get.docker.com | sh
   - useradd -m -s /bin/bash -G docker runner && echo 'runner ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/runner
   - cp /etc/plexus-ci/jitconfig /etc/plexus-ci/jitconfig.runner && chown runner:runner /etc/plexus-ci/jitconfig.runner && mv /etc/plexus-ci/jitconfig.runner /etc/plexus-ci/jitconfig
   - . /etc/plexus-ci/env && mkdir -p /opt/runner && cd /opt/runner && curl -fsSLo runner.tgz "https://github.com/actions/runner/releases/download/v\${RUNNER_VERSION}/actions-runner-linux-x64-\${RUNNER_VERSION}.tar.gz" && echo "\${RUNNER_SHA256}  runner.tgz" | sha256sum -c - && tar -xzf runner.tgz && rm runner.tgz && ./bin/installdependencies.sh >/dev/null && chown -R runner:runner /opt/runner
   - plexus-ci-exo version >/dev/null
-  # The registry sidecars, when the launch named a bucket (ci.md, "The registry"): :5000 on this month's bucket, read-write; :5001 on the previous month's, read-only. The same two containers as .github/ci/registry.sh starts on a GitHub runner.
+  # The registry sidecars, when the launch named a bucket (\xA7 8 PLX-Delivery): :5000 on this month's bucket, read-write; :5001 on the previous month's, read-only. The same two containers as .github/ci/registry.sh starts on a GitHub runner.
   - . /etc/plexus-ci/env && if [ -n "$REGISTRY_ENDPOINT" ]; then docker run -d --name registry --restart unless-stopped -p 127.0.0.1:5000:5000 -e REGISTRY_STORAGE=s3 -e REGISTRY_STORAGE_S3_REGIONENDPOINT="$REGISTRY_ENDPOINT" -e REGISTRY_STORAGE_S3_REGION="$REGISTRY_REGION" -e REGISTRY_STORAGE_S3_BUCKET="$REGISTRY_BUCKET" -e REGISTRY_STORAGE_S3_ACCESSKEY="$REGISTRY_ACCESS_KEY" -e REGISTRY_STORAGE_S3_SECRETKEY="$REGISTRY_SECRET_KEY" -e REGISTRY_STORAGE_S3_FORCEPATHSTYLE=true -e REGISTRY_STORAGE_DELETE_ENABLED=true registry:2 >/dev/null; fi
-  - . /etc/plexus-ci/env && if [ -n "$REGISTRY_ENDPOINT" ]; then docker run -d --name registry-previous --restart on-failure:3 -p 127.0.0.1:5001:5000 -e REGISTRY_STORAGE=s3 -e REGISTRY_STORAGE_S3_REGIONENDPOINT="$REGISTRY_ENDPOINT" -e REGISTRY_STORAGE_S3_REGION="$REGISTRY_REGION" -e REGISTRY_STORAGE_S3_BUCKET="$REGISTRY_PREVIOUS_BUCKET" -e REGISTRY_STORAGE_S3_ACCESSKEY="$REGISTRY_ACCESS_KEY" -e REGISTRY_STORAGE_S3_SECRETKEY="$REGISTRY_SECRET_KEY" -e REGISTRY_STORAGE_S3_FORCEPATHSTYLE=true -e REGISTRY_STORAGE_MAINTENANCE_READONLY_ENABLED=true registry:2 >/dev/null; fi
+  - . /etc/plexus-ci/env && if [ -n "$REGISTRY_ENDPOINT" ]; then docker run -d --name registry-previous --restart on-failure:3 -p 127.0.0.1:5001:5000 -e REGISTRY_STORAGE=s3 -e REGISTRY_STORAGE_S3_REGIONENDPOINT="$REGISTRY_ENDPOINT" -e REGISTRY_STORAGE_S3_REGION="$REGISTRY_REGION" -e REGISTRY_STORAGE_S3_BUCKET="$REGISTRY_PREVIOUS_BUCKET" -e REGISTRY_STORAGE_S3_ACCESSKEY="$REGISTRY_ACCESS_KEY" -e REGISTRY_STORAGE_S3_SECRETKEY="$REGISTRY_SECRET_KEY" -e REGISTRY_STORAGE_S3_FORCEPATHSTYLE=true -e 'REGISTRY_STORAGE_MAINTENANCE_READONLY={"enabled":true}' registry:2 >/dev/null; fi
   # A builder: the docker driver cannot export a registry cache, so BuildKit runs in its own container on the host's network, where localhost:5000 is the sidecar; bootstrapped now so that the build does not wait for it.
   - . /etc/plexus-ci/env && if [ "$KIND" = build ]; then su -s /bin/bash runner -c 'docker buildx create --name ci --driver docker-container --driver-opt network=host --bootstrap >/dev/null' & fi
   - . /etc/plexus-ci/env && if [ "$KIND" = test ]; then docker pull -q postgres:18.6 >/dev/null & docker pull -q rustfs/rustfs:1.0.4 >/dev/null & fi
   - systemctl start plexus-ci-runner.service
 `;
-var BARE_USER_DATA = `#cloud-config
-# The bisect probe (ci-dummy with break = bare): nothing but the power-off
-# timer, so that a boot with this and a boot with user-data.yaml tell apart
-# the image from the scripts. The sweep removes a powered-off instance.
-write_files:
-  - path: /etc/systemd/system/plexus-ci-poweroff.service
-    content: |
-      [Unit]
-      Description=plexus-ci: power off at the hard deadline
-      [Service]
-      Type=oneshot
-      ExecStart=/usr/bin/systemctl poweroff
-  - path: /etc/systemd/system/plexus-ci-poweroff.timer
-    content: |
-      [Unit]
-      Description=plexus-ci: layer 2a
-      [Timer]
-      OnBootSec=@@POWEROFF_MINUTES@@min
-      AccuracySec=10s
-      [Install]
-      WantedBy=timers.target
-runcmd:
-  - systemctl daemon-reload
-  - systemctl start plexus-ci-poweroff.timer
-`;
 function placeholders(template) {
   return [...new Set([...template.matchAll(/@@([A-Z0-9_]+)@@/g)].map((m) => m[1] ?? ""))].filter((n) => n !== "");
 }
@@ -11842,7 +12246,7 @@ function render2(template, values) {
   return text;
 }
 
-// src/core/launch.ts
+// src/core/createInstances.ts
 var DEFAULTS = {
   zone: "de-fra-1",
   instanceType: "cpu.extra-large",
@@ -11859,8 +12263,6 @@ var DEFAULTS = {
   userDataLimit: 32768
 };
 function renderUserData(options, jitconfig, now) {
-  if (options.kind === "bare")
-    return render2(BARE_USER_DATA, { POWEROFF_MINUTES: String(options.deadlines.poweroffMinutes) });
   const owner = options.run.repo.split("/")[0] ?? options.run.repo;
   const buckets = registryBuckets(owner, now);
   return render2(RUNNER_USER_DATA, {
@@ -11914,44 +12316,49 @@ function instanceLabels(options, noJobDeadline, deathDeadline) {
 }
 async function launch(ctx, exo, github, options) {
   const now = options.now ?? (() => /* @__PURE__ */ new Date());
-  const { label, kind } = options;
+  const { label } = options;
   if (options.deadlines.poweroffMinutes <= options.deadlines.deathMinutes)
     throw new UsageError("the power-off must come after the death deadline");
   const t0 = now();
   const noJobDeadline = deadline(t0, options.deadlines.noJobMinutes);
   const deathDeadline = deadline(t0, options.deadlines.deathMinutes);
-  const registration = kind === "bare" ? void 0 : await github.generateJitConfig(label, label);
-  const jitconfig = registration?.encodedJitConfig ?? "<jitconfig>";
-  const unregister = async () => {
-    if (!registration) return;
-    ctx.say(`launch failed, removing the runner registration ${label} (${registration.runnerId})`);
-    await github.deleteRunner(registration.runnerId).catch(() => void 0);
+  const labels = instanceLabels(options, noJobDeadline, deathDeadline);
+  const [templateId, instanceTypeId, securityGroupId] = await Promise.all([
+    exo.templateId(options.template),
+    exo.instanceTypeId(options.instanceType),
+    exo.securityGroupId(options.securityGroup)
+  ]);
+  const result = {
+    id: "<instance-id>",
+    label,
+    createdAt: iso(t0),
+    noJobDeadline,
+    deathDeadline,
+    tries: 0,
+    kept: []
   };
-  try {
-    const userData = encodeUserData(renderUserData(options, jitconfig, t0));
-    const [templateId, instanceTypeId, securityGroupId] = await Promise.all([
-      exo.templateId(options.template),
-      exo.instanceTypeId(options.instanceType),
-      exo.securityGroupId(options.securityGroup)
-    ]);
-    const request = {
-      name: label,
-      templateId,
-      instanceTypeId,
-      securityGroupId,
-      sshKey: options.sshKey,
-      diskSize: options.diskSize,
-      userDataBase64: userData,
-      labels: instanceLabels(options, noJobDeadline, deathDeadline)
+  ctx.say(`\u2192 creating ${label} (${options.instanceType}, ${options.zone})`);
+  for (let attempt = 1; attempt <= options.tries; attempt++) {
+    const registration = await github.generateJitConfig(label, label);
+    const unregister = async () => {
+      if (registration) await github.deleteRunner(registration.runnerId).catch(() => void 0);
     };
-    const result = { id: "<instance-id>", label, createdAt: iso(t0), noJobDeadline, deathDeadline, tries: 0 };
-    ctx.say(`\u2192 creating ${label} (${options.instanceType}, ${options.zone})`);
-    for (let attempt = 1; attempt <= options.tries; attempt++) {
+    try {
+      const request = {
+        name: label,
+        templateId,
+        instanceTypeId,
+        securityGroupId,
+        sshKey: options.sshKey,
+        diskSize: options.diskSize,
+        userDataBase64: encodeUserData(renderUserData(options, registration?.encodedJitConfig ?? "<jitconfig>", t0)),
+        labels
+      };
       const operation = await exo.createInstance(request);
       result.tries = attempt;
       if (operation === void 0) {
         ctx.say(
-          `  (then: wait up to ${options.onlineSeconds}s for the runner to show online, replace the instance on a miss)`
+          `  (then: wait up to ${options.onlineSeconds}s for the runner to show online; on a miss keep the instance stopped as ${label}-kept, mail, and create one replacement)`
         );
         return result;
       }
@@ -11961,118 +12368,135 @@ async function launch(ctx, exo, github, options) {
       const state = await exo.waitOperation(operation.id);
       if (state !== "success") throw new Error(`create-instance ended in state ${state}`);
       ctx.say(`created ${label} (${result.id}) in ${options.zone}, try ${attempt}`);
-      if (kind === "bare") return result;
-      const waited = await waitOnline(github, label, options.onlineSeconds);
-      if (waited !== void 0) {
-        ctx.say(`runner ${label} online after ${waited}s`);
-        ctx.say(
-          `launched ${label} (${result.id}): no job by ${noJobDeadline}, dies at ${deathDeadline}, powers off ${options.deadlines.poweroffMinutes} minutes after creation`
-        );
-        return result;
-      }
-      ctx.say(`\u2717 ${label} (${result.id}) did not come online within ${options.onlineSeconds}s`);
-      if (options.diagSshKey) await diagnose2(ctx, exo, result.id, options.diagSshKey);
-      if (options.keepMissed && await keep(ctx, options.keepMissed, result.id)) {
-        await exo.stopInstance(result.id).catch(() => ctx.say("  stop-instance refused; the instance may delete itself"));
-        ctx.say(`kept ${label} (${result.id}) for inspection, stopped; run plx-ci inspect ${result.id}`);
-        result.kept = result.id;
-      } else {
-        ctx.say(`deleting ${label} (${result.id})`);
-        await exo.deleteInstance(result.id).catch(() => ctx.say(`  delete of ${result.id} failed`));
-      }
+    } catch (error) {
+      ctx.say(`launch failed, removing the runner registration ${label}`);
+      await unregister();
+      throw error;
+    }
+    const waited = await waitOnline(github, label, options.onlineSeconds, options.sleep ?? sleep);
+    if (waited !== void 0) {
+      ctx.say(`runner ${label} online after ${waited}s`);
+      ctx.say(
+        `launched ${label} (${result.id}): no job by ${noJobDeadline}, dies at ${deathDeadline}, powers off ${options.deadlines.poweroffMinutes} minutes after creation`
+      );
+      return result;
     }
+    ctx.say(`\u2717 ${label} (${result.id}) did not come online within ${options.onlineSeconds}s`);
+    await keep(ctx, exo, options, result.id, labels, now());
+    result.kept.push(result.id);
     await unregister();
-    throw new Error(`${options.tries} instance(s) of ${label} failed to boot, giving up`);
-  } catch (error) {
-    if (!(error instanceof Error && error.message.endsWith("giving up"))) await unregister();
-    throw error;
   }
+  throw new Error(`${options.tries} instance(s) of ${label} failed to come online and are kept, giving up`);
 }
-async function waitOnline(github, label, seconds) {
+async function waitOnline(github, label, seconds, wait) {
   for (let waited = 0; waited < seconds; waited += 5) {
-    await sleep(5e3);
+    await wait(5e3);
     if (await github.status(label) === "online") return waited + 5;
   }
   return void 0;
 }
-async function keep(ctx, file, id) {
-  try {
-    await ctx.stage(run("sh", ["-c", 'set -C && echo "$1" > "$2"', "keep", id, file]));
-    return true;
-  } catch (error) {
-    if (error instanceof ExecError) return false;
-    throw error;
-  }
+function keptMail(options, id) {
+  const { label, run: run2 } = options;
+  return {
+    subject: `plexus-ci: ${label} did not come online and is kept`,
+    text: [
+      `The instance ${label} (${options.kind}, app ${options.app ?? "none"}) of ${run2.repo} run ${run2.runId} attempt ${run2.attempt} did not register its runner within ${options.onlineSeconds} seconds.`,
+      "",
+      `It is stopped and kept as ${label}-kept:`,
+      `  id    ${id}`,
+      `  zone  ${options.zone}`,
+      "",
+      "Its disk holds the boot it stalled in. Snapshot it to read it; the daily sweep mails again after seven days and deletes the instance after fourteen.",
+      ""
+    ].join("\n")
+  };
 }
-async function diagnose2(ctx, exo, id, sshKey) {
-  const ip = (await exo.instance(id))?.publicIp;
-  ctx.say(`diagnosis of ${id} at ${ip ?? "no ip"}:`);
-  if (!ip) return;
-  try {
-    const out = await ctx.read("ssh", [
-      "-i",
-      sshKey,
-      "-o",
-      "BatchMode=yes",
-      "-o",
-      "ConnectTimeout=10",
-      "-o",
-      "StrictHostKeyChecking=no",
-      "-o",
-      "UserKnownHostsFile=/dev/null",
-      "-o",
-      "LogLevel=ERROR",
-      `debian@${ip}`,
-      'sudo cloud-init status --long 2>&1 | head -5; echo "-- runner unit"; sudo systemctl is-active plexus-ci-runner; sudo journalctl -u plexus-ci-runner --no-pager -o short-iso | tail -15; echo "-- plexus-ci"; sudo journalctl -t plexus-ci --no-pager -o short-iso | tail -5'
-    ]);
-    ctx.say(out.replace(/^/gm, "    "));
-  } catch (error) {
-    ctx.say(`    no SSH to ${ip}: ${error instanceof ExecError ? error.stderr.trim() : String(error)}`);
+async function keep(ctx, exo, options, id, labels, now) {
+  const { label } = options;
+  const stopped = await exo.stopInstance(id).then(
+    () => true,
+    () => false
+  );
+  if (!stopped) ctx.say(`  stop-instance refused for ${id}; the instance may delete itself`);
+  const marked = await exo.updateInstance(id, `${label}-kept`, { ...labels, [KEPT_LABEL]: iso(now) }).then(
+    () => true,
+    () => false
+  );
+  if (!marked) ctx.say(`  update-instance refused for ${id}; unmarked, the cleanup will delete it`);
+  ctx.say(`kept ${label} (${id}) as ${label}-kept${stopped ? ", stopped" : ""}`);
+  if (!options.smtp) return;
+  const mail = keptMail(options, id);
+  await sendMail(ctx, options.smtp, mail.subject, mail.text, now).catch(
+    () => ctx.say(`  the mail about ${id} could not be sent`)
+  );
+}
+async function createInstances(ctx, exo, github, legs, shared) {
+  const launches = await Promise.allSettled(legs.map((leg) => launch(ctx, exo, github, { ...shared, ...leg })));
+  const failed = launches.filter((l) => l.status === "rejected");
+  if (failed.length > 0) {
+    for (const f of failed) ctx.say(`\u2717 ${f.reason instanceof Error ? f.reason.message : String(f.reason)}`);
+    throw new Error(`${failed.length} launch(es) failed`);
   }
+  return launches.flatMap((l) => l.status === "fulfilled" ? [l.value] : []);
 }
 
-// src/core/mail.ts
-import { tmpdir as tmpdir2 } from "node:os";
-import { join as join3 } from "node:path";
-function smtpFromEnv(env = process.env) {
-  const { SMTP_HOST, SMTP_PORT, SMTP_USER, SMTP_PASSWORD, MAIL_FROM, MAIL_TO } = env;
-  if (!SMTP_HOST || !SMTP_USER || !SMTP_PASSWORD || !MAIL_FROM || !MAIL_TO) return void 0;
-  return {
-    host: SMTP_HOST,
-    port: SMTP_PORT || "587",
-    user: SMTP_USER,
-    password: SMTP_PASSWORD,
-    from: MAIL_FROM,
-    to: MAIL_TO
-  };
+// src/core/sweepInstances.ts
+var REMIND_AFTER_DAYS = 7;
+var DELETE_KEPT_AFTER_DAYS = 14;
+var DAY = 864e5;
+function verdictOn(instance, now) {
+  const kept = instance.labels[KEPT_LABEL];
+  if (kept !== void 0) {
+    const days = (now.getTime() - Date.parse(kept)) / DAY;
+    if (days >= DELETE_KEPT_AFTER_DAYS)
+      return { action: "remove", reason: `kept since ${kept}, ${DELETE_KEPT_AFTER_DAYS} days` };
+    if (days >= REMIND_AFTER_DAYS && days < REMIND_AFTER_DAYS + 1) return { action: "remind" };
+    return { action: "stay" };
+  }
+  const reasons = [];
+  if (instance.state === "stopped") reasons.push("stopped");
+  const deadline2 = instance.labels["death-deadline"];
+  if (deadline2 && deadline2 < iso(now)) reasons.push(`past its deadline ${deadline2}`);
+  return reasons.length > 0 ? { action: "remove", reason: reasons.join(", ") } : { action: "stay" };
 }
-function message(smtp, subject, text, date) {
-  const body2 = text.replaceAll("\r\n", "\n").replaceAll("\n", "\r\n");
-  return `From: plexus-ci <${smtp.from}>\r
-To: ${smtp.to}\r
-Subject: ${subject}\r
-Date: ${date.toUTCString()}\r
-\r
-${body2}\r
+async function sweepInstances(ctx, exo, options = {}) {
+  const now = options.now ?? /* @__PURE__ */ new Date();
+  const result = { removed: [], reminded: [], stayed: [] };
+  for (const i of await exo.listInstances()) {
+    if (i.labels["plexus-ci"] !== "true") continue;
+    const verdict = verdictOn(i, now);
+    if (verdict.action === "remove") {
+      ctx.say(`${i.name} (${i.id}): ${verdict.reason}`);
+      await exo.deleteInstance(i.id);
+      result.removed.push({ name: i.name, reason: verdict.reason });
+      continue;
+    }
+    const kept = i.labels[KEPT_LABEL];
+    ctx.say(
+      `${i.name} (${i.id}): ${i.state}, ${kept ? `kept since ${kept}` : `until ${i.labels["death-deadline"] ?? "-"}`}, stays`
+    );
+    result.stayed.push(i.name);
+    if (verdict.action === "remind") result.reminded.push(`${i.name} (${i.id}), kept since ${kept}`);
+  }
+  if (options.smtp && result.removed.length > 0) {
+    const text = `The daily sweep removed ${result.removed.length} plexus-ci instance(s) in ${exo.zone} that the other layers left behind:
+
+${result.removed.map((r) => `${r.name} (${r.reason})`).join("\n")}
 `;
-}
-async function sendMail(ctx, smtp, subject, text, date = /* @__PURE__ */ new Date()) {
-  const scheme = smtp.port === "465" || smtp.port === "2465" ? "smtps" : "smtp";
-  const file = join3(tmpdir2(), `plx-ci-mail-${process.pid}.eml`);
-  const config = [
-    `url = "${scheme}://${smtp.host}:${smtp.port}"`,
-    "ssl-reqd",
-    "silent",
-    "show-error",
-    `mail-from = "${smtp.from}"`,
-    `mail-rcpt = "${smtp.to}"`,
-    `user = ${JSON.stringify(`${smtp.user}:${smtp.password}`)}`,
-    `upload-file = ${JSON.stringify(file)}`
-  ].join("\n");
-  await ctx.apply([write(file, message(smtp, subject, text, date)), run("curl", ["-K", "-"], { input: config })]);
+    await sendMail(ctx, options.smtp, "plexus-ci: the sweep removed instances", text, now);
+  }
+  if (options.smtp && result.reminded.length > 0) {
+    const text = `These plexus-ci instance(s) in ${exo.zone} have been kept for ${REMIND_AFTER_DAYS} days after a missed boot, stopped, their disks billed. The sweep deletes each ${DELETE_KEPT_AFTER_DAYS} days after it was kept:
+
+${result.reminded.join("\n")}
+`;
+    await sendMail(ctx, options.smtp, "plexus-ci: kept instances are still there", text, now);
+  }
+  ctx.say(`sweep-instances: ${result.removed.length} removed, ${result.reminded.length} reminded of`);
+  return result;
 }
 
-// src/commands/launch.ts
+// src/commands/hybrid.ts
 function launchOptionsFromEnv(env = process.env) {
   const minutes = (name) => Number(need(name, env));
   return {
@@ -12094,444 +12518,302 @@ function launchOptionsFromEnv(env = process.env) {
     securityGroup: DEFAULTS.securityGroup,
     sshKey: DEFAULTS.sshKey,
     onlineSeconds: Number(env.ONLINE_SECONDS || DEFAULTS.onlineSeconds),
-    tries: Number(env.LAUNCH_TRIES || DEFAULTS.tries),
-    keepMissed: env.KEEP_MISSED || void 0,
-    diagSshKey: env.DIAG_SSH_KEY || void 0
+    tries: DEFAULTS.tries
   };
 }
-function register3(program2) {
-  program2.command("launch").description("launch one throwaway runner for one job: JIT config, instance, wait until the runner is online").argument("<kind>", "build, test, dummy, or bare (the probe\u2019s user-data)").argument("<label>", "the one runner label the job asks for").argument("[app]", "the app, for the instance\u2019s labels").action(async (kind, label, app, _opts, command) => {
+function instanceLegs(matrix, tests = "[]") {
+  const builds = JSON.parse(matrix);
+  const testLegs = JSON.parse(tests);
+  return [
+    ...builds.map((l) => ({ kind: "build", label: l.runner, app: l.app })),
+    ...testLegs.map((l) => ({ kind: "test", label: l.runner, app: l.app }))
+  ];
+}
+function register2(program2) {
+  const hybrid = program2.command("hybrid").description("the hybrid strategy\u2019s own: the throwaway instances its large jobs run on");
+  hybrid.command("create-instances").description(
+    "create one instance per build and test leg of the plan and wait for each runner; a miss is kept, stopped, mailed about and replaced once"
+  ).argument("<matrix>", "the plan\u2019s build matrix, JSON").argument("[tests]", "the plan\u2019s test matrix, JSON").action(async (matrix, tests, _opts, command) => {
     const ctx = contextOf(command);
     const github = githubOf(ctx);
     if (!github) throw new Error("RUNNER_TOKEN is empty");
-    const result = await launch(ctx, exoOf(ctx), github, { ...launchOptionsFromEnv(), kind, label, app });
-    ctx.result(result, (r) => `${r.id} ${r.label} ${r.createdAt}`);
-    if (process.env.GITHUB_OUTPUT)
-      appendFileSync2(
-        process.env.GITHUB_OUTPUT,
-        `id=${result.id}
-label=${result.label}
-created=${result.createdAt}
-`
-      );
+    const legs = instanceLegs(matrix, tests);
+    const result = await createInstances(ctx, exoOf(ctx), github, legs, launchOptionsFromEnv());
+    ctx.result(result, (r) => r.map((l) => `${l.id} ${l.label} ${l.createdAt}`).join("\n"));
+  });
+  hybrid.command("destroy-instances").description(
+    "delete every instance this run created, kept ones excepted, and the runner registrations it never used"
+  ).action(async (_opts, command) => {
+    const ctx = contextOf(command);
+    const result = await destroyInstances(ctx, exoOf(ctx), githubOf(ctx), runIdentity());
+    ctx.result(result, (r) => [...r.instances, ...r.registrations.map((n) => `registration ${n}`)].join("\n"));
+  });
+  hybrid.command("sweep-instances").description(
+    "delete every instance that is stopped or past its deadline and mail about it; a kept one goes after fourteen days"
+  ).action(async (_opts, command) => {
+    const ctx = contextOf(command);
+    const result = await sweepInstances(ctx, exoOf(ctx), { smtp: smtpFromEnv() });
+    ctx.result(result, (r) => r.removed.map((x) => `${x.name} (${x.reason})`).join("\n"));
   });
 }
 
 // src/commands/plan.ts
-import { appendFileSync as appendFileSync3 } from "node:fs";
+import { appendFileSync as appendFileSync2 } from "node:fs";
 import { resolve as resolve2 } from "node:path";
-
-// src/core/plan.ts
-var NONEXISTENT = "<NONEXISTENT>";
-function hashesOf(dryRun, scope) {
-  const data = JSON.parse(dryRun);
-  const tasks = (data.tasks ?? []).filter((t) => t.package.startsWith(`${scope}/`) && t.command !== NONEXISTENT);
-  const app = (t) => t.package.slice(scope.length + 1);
-  return tasks.filter((t) => t.taskId.endsWith("#build")).map((b) => {
-    const test2 = tasks.find((t) => t.package === b.package && t.taskId.endsWith("#test"));
-    return { app: app(b), hash: b.hash, ...test2 ? { test: test2.hash } : {} };
-  });
-}
-async function readHashes(ctx, dir, scope) {
-  const pkg = JSON.parse(await ctx.readFile(`${dir}/package.json`));
-  const turbo = pkg.devDependencies?.turbo ?? "latest";
-  const out = await ctx.read("npx", ["--yes", `turbo@${turbo}`, "run", "build", "test", "--dry-run=json"], {
-    cwd: dir
-  });
-  return hashesOf(out, scope);
-}
-function legsOf(apps, hashes, mode, runId, attempt) {
-  return apps.flatMap((app) => {
-    const hash = hashes.find((h) => h.app === app)?.hash;
-    if (!hash) throw new UsageError(`no build hash for ${app}; is it an app package of the workspace?`);
-    const copies = mode === "repro" ? 2 : 1;
-    return Array.from({ length: copies }, (_, i) => {
-      const n = i + 1;
-      const base = `exo-build-${runId}-${app}-${attempt}`;
-      return { app, hash, label: mode === "repro" ? `${base}-${n}` : base, n };
+function register3(program2) {
+  withStrategy(program2.command("plan")).description(
+    "compute the build and test hashes, look them up in the registry, and print the builds, tests and deploys the run needs"
+  ).argument("[apps...]", "the apps (default: every directory under apps/)").option(
+    "--mode <mode>",
+    "build (skip what the registry holds), force (build all), repro (two no-cache builds per app)",
+    "build"
+  ).option("--test", "plan the tests too, for the apps with a test task").option("--deploy", "plan the deploys too: the environment and each app\u2019s target in it").option("--environment <name>", "the environment to release into (default: from the branch, main or develop)").option("--inventory <dir>", "the platform repo\u2019s inventory, for --deploy", "platform/inventory").option("-C, --dir <dir>", "the checkout to read", ".").action(async (apps, opts, command) => {
+    const ctx = contextOf(command);
+    const s = settings(opts);
+    const dir = resolve2(opts.dir);
+    const env = process.env;
+    const all = apps.length > 0 ? apps : (await ctx.readDir(`${dir}/apps`)).filter((e) => e.directory).map((e) => e.name).sort();
+    const environment = opts.deploy ? environmentOf(env.GITHUB_EVENT_NAME ?? "", env.GITHUB_REF_NAME ?? "", opts.environment) : "";
+    const result = await plan(ctx, {
+      apps: all,
+      mode: opts.mode,
+      tests: opts.test ?? false,
+      hashes: await readHashes(ctx, dir, all),
+      strategy: s.strategy,
+      registry: registryFor(s),
+      // Only hybrid names anything after the run: its instances.
+      run: s.strategy === "hybrid" ? { runId: need("GITHUB_RUN_ID"), attempt: env.GITHUB_RUN_ATTEMPT || "1" } : { runId: env.GITHUB_RUN_ID || "0", attempt: env.GITHUB_RUN_ATTEMPT || "1" },
+      deploy: environment ? {
+        environment,
+        targets: targetsIn(await readDeployTargets(ctx, resolve2(opts.inventory)), environment),
+        prefix: imagePrefix(),
+        sha: need("GITHUB_SHA")
+      } : void 0
     });
+    ctx.result(
+      result,
+      (r) => [
+        ...r.legs.map((l) => `build ${l.app} ${l.hash} ${l.runner}`),
+        ...r.tests.map((l) => `test ${l.app} ${l.hash} ${l.runner}`),
+        ...r.deploys.map((l) => `deploy ${l.app} ${l.host} ${l.image}`)
+      ].join("\n")
+    );
+    if (env.GITHUB_OUTPUT)
+      appendFileSync2(
+        env.GITHUB_OUTPUT,
+        [
+          `matrix=${JSON.stringify(result.legs)}`,
+          `tests=${JSON.stringify(result.tests)}`,
+          `apps=${JSON.stringify(result.apps)}`,
+          `deploys=${JSON.stringify(result.deploys)}`,
+          `environment=${result.environment}`,
+          ""
+        ].join("\n")
+      );
   });
 }
-function testLegsOf(apps, hashes, runId, attempt) {
-  return apps.flatMap((app) => {
-    const h = hashes.find((x) => x.app === app);
-    if (!h?.test) return [];
-    return [{ app, hash: h.test, build: h.hash, label: `exo-test-${runId}-${app}-${attempt}` }];
-  });
-}
-async function plan(ctx, exo, github, options) {
-  const { runId, attempt } = options.launch.run;
-  const result = {
-    legs: [],
-    tests: [],
-    apps: options.apps.map((app) => {
-      const h = options.hashes.find((x) => x.app === app);
-      if (!h) throw new UsageError(`no build hash for ${app}; is it an app package of the workspace?`);
-      return h;
-    }),
-    skipped: [],
-    skippedTests: []
-  };
-  const builds = [];
-  for (const leg of legsOf(options.apps, options.hashes, options.mode, runId, attempt)) {
-    if (options.mode === "build") {
-      const port = await tagExists(ctx, leg.app, `build-${leg.hash}`);
-      if (port !== void 0) {
-        ctx.say(`${leg.app}: build-${leg.hash} exists in the registry on :${port}, nothing to build`);
-        result.skipped.push({ app: leg.app, hash: leg.hash, port });
-        continue;
-      }
-    }
-    builds.push(leg);
-  }
-  const tests = [];
-  if (options.tests && options.mode !== "repro") {
-    for (const leg of testLegsOf(options.apps, options.hashes, runId, attempt)) {
-      if (options.mode === "build") {
-        const port = await tagExists(ctx, leg.app, `test-${leg.hash}`);
-        if (port !== void 0) {
-          ctx.say(`${leg.app}: test-${leg.hash} exists in the registry on :${port}, nothing to test`);
-          result.skippedTests.push({ app: leg.app, hash: leg.hash, port });
-          continue;
-        }
-      }
-      tests.push(leg);
-    }
-  }
-  const launches = await Promise.allSettled([
-    ...builds.map(
-      (leg) => launch(ctx, exo, github, { ...options.launch, kind: "build", label: leg.label, app: leg.app })
-    ),
-    ...tests.map(
-      (leg) => launch(ctx, exo, github, { ...options.launch, kind: "test", label: leg.label, app: leg.app })
-    )
-  ]);
-  const failed = launches.filter((l) => l.status === "rejected");
-  for (const l of launches) if (l.status === "fulfilled" && l.value.kept) result.kept = l.value.kept;
-  if (failed.length > 0) {
-    for (const f of failed) ctx.say(`\u2717 ${f.reason instanceof Error ? f.reason.message : String(f.reason)}`);
-    throw new Error(`${failed.length} launch(es) failed`);
-  }
-  result.legs = builds;
-  result.tests = tests;
-  return result;
-}
-async function report(ctx, legs, mode, runId) {
-  const apps = [...new Set(legs.map((l) => l.app))].sort();
-  const out = [];
-  for (const app of apps) {
-    const hash = legs.find((l) => l.app === app)?.hash ?? "";
-    const c = { app, hash, digest: await digest(ctx, app, `build-${hash}`) };
-    ctx.say(`${app} build-${hash}: ${c.digest ?? "missing"}`);
-    if (mode === "repro") {
-      const [one, two] = await Promise.all([
-        digest(ctx, app, `repro-${runId}-1`),
-        digest(ctx, app, `repro-${runId}-2`)
-      ]);
-      const identical = one !== void 0 && one === two;
-      const [l1, l2] = identical ? [[], []] : await Promise.all([layers(ctx, app, `repro-${runId}-1`), layers(ctx, app, `repro-${runId}-2`)]);
-      const differing = [];
-      const a = l1 ?? [];
-      const b = l2 ?? [];
-      for (let i = 0; i < Math.max(a.length, b.length); i++) {
-        if (a[i]?.digest !== b[i]?.digest)
-          differing.push(
-            `layer ${i + 1}: ${a[i]?.digest ?? "-"} (${a[i]?.size ?? "-"}) vs ${b[i]?.digest ?? "-"} (${b[i]?.size ?? "-"})`
-          );
-      }
-      c.repro = { one, two, identical, layers: a.length, same: a.length - differing.length, differing };
-      if (identical) ctx.say(`  reproducible: both builds are ${one}`);
-      else {
-        ctx.say(
-          `  not identical: ${one ?? "-"} vs ${two ?? "-"}; layers: ${a.length} in each, ${a.length - differing.length} identical`
-        );
-        for (const d of differing) ctx.say(`  ${d}`);
-      }
-    }
-    out.push(c);
-  }
-  return out;
+
+// src/commands/publish.ts
+import { appendFileSync as appendFileSync3 } from "node:fs";
+
+// src/core/publish.ts
+async function publish(ctx, options) {
+  const { app, build: build2, test: test2, sha, registry } = options;
+  const image = imageName(options.prefix, app);
+  const target = `${image}:build-${build2}`;
+  const source = test2 ? `test-${test2}` : `build-${build2}`;
+  const missing = `${app}:${source} is in no registry here; ${test2 ? "the tests did not pass on this build" : "the build never ran"}`;
+  if (registry.kind === "ghcr") {
+    const ref = `${image}:${source}`;
+    if (await craneDigest(ctx, ref) === void 0 && !ctx.dryRun) throw new Error(missing);
+    ctx.say(`publish: ${ref} as ${sha}`);
+    await ctx.apply([run("crane", ["tag", ref, sha])]);
+    return { app, image, copied: false, tags: [`build-${build2}`, ...test2 ? [`test-${test2}`] : [], sha] };
+  }
+  const copied = await craneDigest(ctx, target) === void 0;
+  if (copied) {
+    const from = await findTag(ctx, registry, app, source);
+    if (from === void 0 && !ctx.dryRun) throw new Error(missing);
+    ctx.say(`publish: ${app}:${source} -> ${target}`);
+    await ctx.apply([run("crane", ["copy", "--insecure", from ?? `localhost:5000/${app}:${source}`, target])]);
+  } else ctx.say(`publish: ${target} is there already, tagging only`);
+  const tags = [sha, ...test2 ? [`test-${test2}`] : []];
+  await ctx.apply(tags.map((t) => run("crane", ["tag", target, t])));
+  return { app, image, copied, tags: [`build-${build2}`, ...tags] };
 }
 
-// src/commands/plan.ts
+// src/commands/publish.ts
 function register4(program2) {
-  program2.command("plan").description(
-    "compute the build and test hashes, look them up in the registry, and launch one runner per image to build or test"
-  ).argument("[apps...]", "the apps (default: every directory under apps/)").requiredOption(
-    "--mode <mode>",
-    "build (skip what the registry holds), force (build all), repro (two no-cache builds per app)"
-  ).option("--tests", "plan test runners too, for the apps with a test task").option("--scope <scope>", "the npm scope of the workspace packages", "@damage-ms").option("-C, --dir <dir>", "the checkout to read", ".").action(
-    async (apps, opts, command) => {
+  withStrategy(program2.command("publish")).description("publish one app\u2019s tested image as <sha> in the forge\u2019s registry, beside build-<hash> and test-<hash>").argument("<app>", "the app").argument("<build>", "Turbo\u2019s hash of the app\u2019s build task").argument("<sha>", "the commit, the tag a deploy takes").option("--test <hash>", "Turbo\u2019s hash of the app\u2019s test task; the image is read from test-<hash>").action(
+    async (app, build2, sha, opts, command) => {
       const ctx = contextOf(command);
-      const dir = resolve2(opts.dir);
-      const github = githubOf(ctx);
-      if (!github) throw new Error("RUNNER_TOKEN is empty");
-      const all = apps.length > 0 ? apps : (await ctx.readDir(`${dir}/apps`)).filter((e) => e.directory).map((e) => e.name).sort();
-      const result = await plan(ctx, exoOf(ctx), github, {
-        apps: all,
-        mode: opts.mode,
-        tests: opts.tests ?? false,
-        hashes: await readHashes(ctx, dir, opts.scope),
-        launch: launchOptionsFromEnv()
+      const result = await publish(ctx, {
+        app,
+        build: build2,
+        test: opts.test || void 0,
+        sha,
+        prefix: imagePrefix(),
+        registry: registryFor(settings(opts))
       });
-      ctx.result(
-        result,
-        (r) => [
-          ...r.legs.map((l) => `build ${l.app} ${l.hash} ${l.label}`),
-          ...r.tests.map((l) => `test ${l.app} ${l.hash} ${l.label}`)
-        ].join("\n")
-      );
-      if (process.env.GITHUB_OUTPUT)
-        appendFileSync3(
-          process.env.GITHUB_OUTPUT,
-          `matrix=${JSON.stringify(result.legs)}
-tests=${JSON.stringify(result.tests)}
-apps=${JSON.stringify(result.apps)}
-kept=${result.kept ?? ""}
-`
-        );
+      ctx.result(result, (r) => `${r.image}:${sha}${r.copied ? "" : " (was there)"}`);
+      if (process.env.GITHUB_OUTPUT) appendFileSync3(process.env.GITHUB_OUTPUT, `image=${result.image}:${sha}
+`);
     }
   );
 }
 
-// src/core/probe.ts
-async function probe(ctx, exo, options) {
-  const now = options.now ?? (() => /* @__PURE__ */ new Date());
-  const [templateId, instanceTypeId, securityGroupId] = await Promise.all([
-    exo.templateId(options.template),
-    exo.instanceTypeId(options.instanceType),
-    exo.securityGroupId(DEFAULTS.securityGroup)
-  ]);
-  const text = render2(BARE_USER_DATA, { POWEROFF_MINUTES: "60" });
-  const userData = options.plain ? Buffer.from(text).toString("base64") : encodeUserData(text);
-  const boots = [];
-  const started = now();
-  for (let i = 1; i <= options.boots; i++) {
-    const name = `probe-${iso(now()).slice(11, 19).replaceAll(":", "")}-${i}`;
-    const request = {
-      name,
-      templateId,
-      instanceTypeId,
-      securityGroupId,
-      sshKey: DEFAULTS.sshKey,
-      diskSize: 50,
-      userDataBase64: userData,
-      labels: { "plexus-ci": "true", kind: "probe", "death-deadline": deadline(now(), 90) }
-    };
-    const op = await exo.createInstance(request);
-    boots.push({ name, id: op?.reference, result: op ? "pending" : "dry run", ok: false });
-  }
-  if (ctx.dryRun) return boots;
-  const pending = () => boots.filter((b) => b.result === "pending");
-  while (pending().length > 0 && (now().getTime() - started.getTime()) / 1e3 < options.waitSeconds) {
-    const instances = await exo.listInstances();
-    for (const b of pending()) {
-      const i = instances.find((x) => x.id === b.id);
-      if (!i?.publicIp) continue;
-      b.ip = i.publicIp;
-      const ok2 = await ctx.read("ssh", sshArgs(options.sshKey, i.publicIp, "true")).then(
-        () => true,
-        () => false
-      );
-      if (ok2) {
-        b.result = `ok ${Math.round((now().getTime() - started.getTime()) / 1e3)}s`;
-        b.ok = true;
-      }
+// src/commands/registry.ts
+import { resolve as resolve3 } from "node:path";
+
+// src/core/ghcr.ts
+var PRUNE_AFTER_DAYS = 30;
+var CANDIDATE_TAG = /^(build|test)-[0-9a-f]+$/;
+function prunable(versions, now, days = PRUNE_AFTER_DAYS) {
+  const before = now.getTime() - days * 864e5;
+  return versions.filter(
+    (v) => v.tags.length > 0 && v.tags.every((t) => CANDIDATE_TAG.test(t)) && Date.parse(v.createdAt) < before
+  );
+}
+var Packages = class {
+  ctx;
+  org;
+  token;
+  constructor(ctx, org, token) {
+    this.ctx = ctx;
+    this.org = org;
+    this.token = token;
+  }
+  config(method, path2) {
+    return `${[
+      `url = "https://api.github.com/orgs/${this.org}/packages${path2}"`,
+      `request = "${method}"`,
+      "fail-with-body",
+      "silent",
+      "show-error",
+      `header = "Authorization: Bearer ${this.token}"`,
+      'header = "Accept: application/vnd.github+json"',
+      'header = "X-GitHub-Api-Version: 2022-11-28"'
+    ].join("\n")}
+`;
+  }
+  /** Every page of a list, a hundred entries each. */
+  async pages(path2) {
+    const all = [];
+    for (let page = 1; ; page++) {
+      const sep = path2.includes("?") ? "&" : "?";
+      const out = await this.ctx.read("curl", ["-K", "-"], {
+        input: this.config("GET", `${path2}${sep}per_page=100&page=${page}`)
+      });
+      const items = JSON.parse(out);
+      all.push(...items);
+      if (items.length < 100) return all;
     }
-    if (pending().length > 0) await sleep(1e4);
   }
-  for (const b of boots) {
-    if (b.result === "pending") b.result = b.ip ? "unreachable" : "no ip";
-    ctx.say(
-      `${options.template} ${options.instanceType} boot ${b.name} (${b.id ?? "no id"}, ${b.ip ?? "no ip"}): ${b.result}`
+  async versions(name) {
+    const raw = await this.pages(`/container/${encodeURIComponent(name)}/versions`);
+    return raw.map((v) => ({ id: v.id, createdAt: v.created_at, tags: v.metadata?.container?.tags ?? [] }));
+  }
+  async deleteVersion(name, id) {
+    await this.ctx.effect(
+      run("curl", ["-K", "-"], {
+        input: this.config("DELETE", `/container/${encodeURIComponent(name)}/versions/${id}`)
+      })
     );
-    if (b.id && (b.ok || !options.keep)) await exo.deleteInstance(b.id);
-    else if (b.id) ctx.say(`  kept ${b.name} for inspection`);
   }
-  const ok = boots.filter((b) => b.ok).length;
+};
+function packageOf(prefix, app) {
+  const path2 = prefix.toLowerCase().replace(/\/$/, "").split("/").slice(2);
+  return [...path2, app].join("/");
+}
+async function prune(ctx, packages, names, now = /* @__PURE__ */ new Date()) {
+  const pruned = [];
+  for (const name of names) {
+    let versions;
+    try {
+      versions = await packages.versions(name);
+    } catch (error) {
+      if (!(error instanceof ExecError)) throw error;
+      ctx.say(`rotate: no package ${name} in ghcr.io/${packages.org}`);
+      continue;
+    }
+    for (const v of prunable(versions, now)) {
+      ctx.say(`rotate: ${name} ${v.tags.join(" ")} (${v.createdAt}) is older than ${PRUNE_AFTER_DAYS} days`);
+      await packages.deleteVersion(name, v.id);
+      pruned.push({ package: name, id: v.id, tags: v.tags });
+    }
+  }
   ctx.say(
-    `== ${options.template} on ${options.instanceType}: ${ok} ok, ${boots.length - ok} failed of ${boots.length}`
+    `rotate: ${pruned.length} candidate version(s) pruned from ${names.length} package(s) of ghcr.io/${packages.org}`
   );
-  return boots;
-}
-
-// src/commands/probe.ts
-function register5(program2) {
-  program2.command("probe").description("boot bare instances and judge each by an SSH login; a failed one is kept with --keep").option("--boots <n>", "instances to boot at once", "10").option("--template <name>", "the template", DEFAULTS.template).option("--type <type>", "the instance type", DEFAULTS.instanceType).option("--wait <seconds>", "seconds to wait for the verdicts", "180").option("--ssh-key <path>", "the private half of the key the instances get", `${process.env.HOME}/.ssh/deploy`).option("--keep", "keep a failed instance for inspection").option("--plain", "send the user-data as plain base64 instead of gzip").action(async (opts, command) => {
-    const ctx = contextOf(command);
-    const boots = await probe(ctx, exoOf(ctx), {
-      template: opts.template,
-      instanceType: opts.type,
-      zone: zoneOf(),
-      boots: Number(opts.boots),
-      waitSeconds: Number(opts.wait),
-      sshKey: opts.sshKey,
-      keep: opts.keep,
-      plain: opts.plain
-    });
-    ctx.result(boots, (b) => b.map((x) => `${x.name}: ${x.result}`).join("\n"));
-    if (boots.some((b) => !b.ok)) process.exitCode = 1;
-  });
+  return pruned;
 }
 
 // src/commands/registry.ts
-function register6(program2) {
-  const registry = program2.command("registry").description(
-    "start the registry sidecars: :5000 on this month\u2019s bucket read-write, :5001 on the previous month\u2019s read-only"
-  ).action(async (_opts, command) => {
+function register5(program2) {
+  const registry = withStrategy(program2.command("registry")).description(
+    "make the candidates\u2019 registry reachable here: the pair on localhost (started unless :5000 answers), or a login check for the forge\u2019s"
+  ).action(async (opts, command) => {
     const ctx = contextOf(command);
-    const storage = storageFromEnv();
-    if (!storage)
-      throw new Error("REGISTRY_ENDPOINT, REGISTRY_REGION, REGISTRY_ACCESS_KEY and REGISTRY_SECRET_KEY are needed");
-    const result = await startSidecars(ctx, storage, ownerOf(need("GITHUB_REPOSITORY")), zoneOf());
+    const s = settings(opts);
+    if (s.registry === "ghcr") {
+      const host = imagePrefix().split("/")[0] ?? "ghcr.io";
+      if (!await loggedIn(ctx, host) && !ctx.dryRun)
+        throw new Error(`no login for ${host} here; the job logs in before it asks for the registry`);
+      ctx.result(`registry: ${imagePrefix()}, logged in`);
+      return;
+    }
+    const result = await startSidecars(ctx, storageFromEnv(), ownerOf(need("GITHUB_REPOSITORY")));
     ctx.result(
       result,
       (r) => `:5000 ${r.current}
 :5001 ${r.previous}${r.previousAnswers ? "" : " (not answering)"}`
     );
   });
-  registry.command("exists").description("exit 0 when either sidecar holds the tag").argument("<app>").argument("<tag>").action(async (app, tag, _opts, command) => {
+  withStrategy(registry.command("exists")).description("exit 0 when the registry holds the tag").argument("<app>").argument("<tag>").action(async (app, tag, opts, command) => {
     const ctx = contextOf(command);
-    const port = await tagExists(ctx, app, tag);
-    if (port === void 0) {
+    const ref = await findTag(ctx, registryFor(settings(opts)), app, tag);
+    if (ref === void 0) {
       ctx.say(`${app}:${tag} is in no registry here`);
       process.exitCode = 1;
-    } else ctx.result(`${app}:${tag} in the registry on :${port}`);
+    } else ctx.result(ref);
   });
-}
-
-// src/commands/release.ts
-import { appendFileSync as appendFileSync4 } from "node:fs";
-
-// src/core/release.ts
-var imageName = (prefix, app) => `${prefix.toLowerCase().replace(/\/$/, "")}/${app}`;
-async function exists(ctx, ref) {
-  try {
-    await ctx.read("crane", ["digest", ref]);
-    return true;
-  } catch {
-    return false;
-  }
-}
-async function release(ctx, options) {
-  const { app, build: build2, test: test2, sha } = options;
-  const image = imageName(options.prefix, app);
-  const target = `${image}:build-${build2}`;
-  const copied = !await exists(ctx, target);
-  if (copied) {
-    const source = test2 ? `test-${test2}` : `build-${build2}`;
-    const port = await tagExists(ctx, app, source);
-    if (port === void 0 && !ctx.dryRun)
-      throw new Error(
-        `${app}:${source} is in no registry here; ${test2 ? "the tests did not pass on this build" : "the build never ran"}`
-      );
-    ctx.say(`release: ${app}:${source} -> ${target}`);
-    await ctx.apply([run("crane", ["copy", "--insecure", `localhost:${port ?? 5e3}/${app}:${source}`, target])]);
-  } else ctx.say(`release: ${target} is there already, tagging only`);
-  const tags = [sha, ...test2 ? [`test-${test2}`] : []];
-  await ctx.apply(tags.map((t) => run("crane", ["tag", target, t])));
-  return { app, image, copied, tags: [`build-${build2}`, ...tags] };
-}
-
-// src/commands/release.ts
-function register7(program2) {
-  program2.command("release").description("copy one app\u2019s tested image from the sidecar to GHCR as build-<hash>, test-<hash> and <sha>").argument("<app>", "the app").argument("<build>", "Turbo\u2019s hash of the app\u2019s build task").argument("<sha>", "the commit, a tag on the released image").option("--test <hash>", "Turbo\u2019s hash of the app\u2019s test task; the image is read from test-<hash>").action(async (app, build2, sha, opts, command) => {
-    const ctx = contextOf(command);
-    const prefix = process.env.IMAGE_PREFIX || `ghcr.io/${ownerOf(need("GITHUB_REPOSITORY"))}`;
-    const result = await release(ctx, { app, build: build2, test: opts.test || void 0, sha, prefix });
-    ctx.result(result, (r) => `${r.image}:${sha}${r.copied ? "" : " (was there)"}`);
-    if (process.env.GITHUB_OUTPUT) appendFileSync4(process.env.GITHUB_OUTPUT, `image=${result.image}:${sha}
-`);
-  });
-}
-
-// src/commands/report.ts
-function register8(program2) {
-  program2.command("report").description("the digests the registry holds for a run\u2019s matrix, and in repro mode the layer-by-layer comparison").argument("<matrix>", "the plan\u2019s matrix, JSON").requiredOption("--mode <mode>", "build, force or repro").action(async (matrix, opts, command) => {
+  withStrategy(registry.command("rotate")).description(
+    "the retention: this month\u2019s bucket exists and the one from two months ago goes; in the forge\u2019s registry, candidates older than thirty days go"
+  ).argument("[apps...]", "under ghcr: the apps whose packages are pruned (default: every directory under apps/)").option("-C, --dir <dir>", "the checkout to read", ".").action(async (apps, opts, command) => {
     const ctx = contextOf(command);
-    const legs = JSON.parse(matrix);
-    const result = await report(ctx, legs, opts.mode, need("GITHUB_RUN_ID"));
-    ctx.result(result, () => "");
-  });
-}
-
-// src/core/sweep.ts
-function reasonToRemove(instance, now) {
-  const reasons = [];
-  if (instance.state === "stopped") reasons.push("stopped");
-  const deadline2 = instance.labels["death-deadline"];
-  if (deadline2 && deadline2 < now) reasons.push(`past its deadline ${deadline2}`);
-  return reasons.length > 0 ? reasons.join(", ") : void 0;
-}
-async function sweep(ctx, exo, options = {}) {
-  const now = options.now ?? /* @__PURE__ */ new Date();
-  const result = { removed: [], kept: [] };
-  for (const i of await exo.listInstances()) {
-    if (i.labels["plexus-ci"] !== "true") continue;
-    const reason = reasonToRemove(i, iso(now));
-    if (reason === void 0) {
-      ctx.say(`${i.name} (${i.id}): ${i.state}, until ${i.labels["death-deadline"] ?? "-"}, kept`);
-      result.kept.push(i.name);
-      continue;
-    }
-    ctx.say(`${i.name} (${i.id}): ${reason}`);
-    await exo.deleteInstance(i.id);
-    result.removed.push({ name: i.name, reason });
-  }
-  if (result.removed.length > 0 && options.smtp) {
-    const text = `The daily sweep removed ${result.removed.length} plexus-ci instance(s) in ${exo.zone} that the other layers left behind:
-
-${result.removed.map((r) => `${r.name} (${r.reason})`).join("\n")}
-`;
-    await sendMail(ctx, options.smtp, "plexus-ci: the sweep removed instances", text, now);
-  }
-  ctx.say(`sweep: ${result.removed.length} removed`);
-  if (options.storage && options.owner) {
-    const { expired } = registryBuckets(options.owner, now);
-    const sos = new Storage(ctx, options.storage.accessKey, options.storage.secretKey);
-    if (await sos.bucketExists(expired)) {
-      ctx.say(`sweep: the registry bucket ${expired} is two months old`);
-      await sos.removeBucket(expired);
-      result.bucket = expired;
+    const s = settings(opts);
+    const owner = ownerOf(need("GITHUB_REPOSITORY"));
+    if (s.registry === "ghcr") {
+      const all = apps.length > 0 ? apps : (await ctx.readDir(`${resolve3(opts.dir)}/apps`)).filter((e) => e.directory).map((e) => e.name).sort();
+      const names = all.map((app) => packageOf(imagePrefix(), app));
+      const result2 = await prune(ctx, new Packages(ctx, owner, need("GITHUB_TOKEN")), names);
+      ctx.result(result2, (r) => r.map((v) => `${v.package} ${v.tags.join(" ")}`).join("\n"));
+      return;
     }
-  }
-  return result;
-}
-
-// src/commands/sweep.ts
-function register9(program2) {
-  program2.command("sweep").description(
-    "delete every plexus-ci instance that is stopped or past its deadline, mail about it, rotate the registry buckets"
-  ).action(async (_opts, command) => {
-    const ctx = contextOf(command);
-    const repo = process.env.GITHUB_REPOSITORY;
-    const result = await sweep(ctx, exoOf(ctx), {
-      smtp: smtpFromEnv(),
-      storage: storageFromEnv(),
-      owner: repo ? ownerOf(repo) : void 0
-    });
+    const storage = storageFromEnv();
+    if (!storage)
+      throw new Error("REGISTRY_ENDPOINT, REGISTRY_REGION, REGISTRY_ACCESS_KEY and REGISTRY_SECRET_KEY are needed");
+    const result = await rotateBuckets(ctx, storage, owner);
     ctx.result(
       result,
-      (r) => [...r.removed.map((x) => `${x.name} (${x.reason})`), ...r.bucket ? [`bucket ${r.bucket}`] : []].join("\n")
+      (r) => [...r.created ? [`created ${r.created}`] : [], ...r.removed ? [`removed ${r.removed}`] : []].join("\n")
     );
   });
 }
 
-// src/commands/teardown.ts
-function register10(program2) {
-  program2.command("teardown").description(
-    "delete every instance this run launched, kept ones excepted, and the runner registrations it never used"
-  ).action(async (_opts, command) => {
+// src/commands/report.ts
+function register6(program2) {
+  withStrategy(program2.command("report")).description("the digests the registry holds for a run\u2019s matrix, and in repro mode the layer-by-layer comparison").argument("<matrix>", "the plan\u2019s matrix, JSON").requiredOption("--mode <mode>", "build, force or repro").action(async (matrix, opts, command) => {
     const ctx = contextOf(command);
-    const keep2 = (process.env.KEEP_IDS ?? "").split(/\s+/).filter(Boolean);
-    const result = await teardown(ctx, exoOf(ctx), githubOf(ctx), runIdentity(), keep2);
-    ctx.result(result, (r) => [...r.instances, ...r.registrations.map((n) => `registration ${n}`)].join("\n"));
+    const legs = JSON.parse(matrix);
+    const result = await report(ctx, registryFor(settings(opts)), legs, opts.mode, need("GITHUB_RUN_ID"));
+    ctx.result(result, () => "");
   });
 }
 
 // src/commands/test.ts
-import { appendFileSync as appendFileSync5 } from "node:fs";
-import { resolve as resolve3 } from "node:path";
+import { appendFileSync as appendFileSync4 } from "node:fs";
+import { resolve as resolve4 } from "node:path";
 
 // src/core/test.ts
 import { join as join4 } from "node:path";
@@ -12557,16 +12839,15 @@ function specEnv(port, fromOverride) {
   };
 }
 async function test(ctx, options) {
-  const { app, hash, build: build2 } = options;
+  const { app, hash, build: build2, registry } = options;
   const dir = options.dir ?? ".";
   const port = options.port ?? 3e3;
   const appDir = join4(dir, "apps", app);
   const composeDir = join4(appDir, "docker");
   const project = `ci-${app}`;
-  const registryPort = await tagExists(ctx, app, `build-${build2}`) ?? (ctx.dryRun ? CURRENT_PORT : void 0);
-  if (registryPort === void 0)
+  const image = await findTag(ctx, registry, app, `build-${build2}`) ?? (ctx.dryRun ? candidateRef(registry, app, `build-${build2}`) : void 0);
+  if (image === void 0)
     throw new Error(`${app}:build-${build2} is in no registry here; the build failed or never ran`);
-  const image = `localhost:${registryPort}/${app}:build-${build2}`;
   const tag = `test-${hash}`;
   const fromOverride = overrideEnv(await ctx.readFile(join4(composeDir, OVERRIDE)));
   const composeEnv = { PLEXUS_DEPLOYMENT_IMAGE: image, PLEXUS_LOOPBACK_PORT: String(port) };
@@ -12601,48 +12882,39 @@ ${logs}`);
     await ctx.apply([compose(["--profile", "migrate", "down", "--volumes", "--remove-orphans", "--timeout", "10"])]);
   }
   await ctx.apply([
-    run("docker", ["buildx", "imagetools", "create", "--tag", `localhost:${CURRENT_PORT}/${app}:${tag}`, image])
+    run("docker", ["buildx", "imagetools", "create", "--tag", candidateRef(registry, app, tag), image])
   ]);
   const result = { app, hash, image, tag, stackSeconds, testSeconds };
   if (!ctx.dryRun) {
-    result.digest = await digest(ctx, app, tag);
+    result.digest = await digestOf(ctx, registry, app, tag);
     ctx.say(`test: ${app} ${tag} -> ${result.digest ?? "<no digest>"}`);
   }
   return result;
 }
 
 // src/commands/test.ts
-function register11(program2) {
-  program2.command("test").description("run one app\u2019s tests against its compose stack on the candidate image, and tag the image tested").argument("<app>", "the app, apps/<app> in the checkout").argument("<build>", "Turbo\u2019s hash of the app\u2019s build task, the candidate\u2019s tag").argument("<hash>", "Turbo\u2019s hash of the app\u2019s test task, the tag on success").option("-C, --dir <dir>", "the checkout to read", ".").action(async (app, build2, hash, opts, command) => {
-    const ctx = contextOf(command);
-    const result = await test(ctx, {
-      app,
-      build: build2,
-      hash,
-      dir: resolve3(opts.dir),
-      port: process.env.PLEXUS_LOOPBACK_PORT ? Number(process.env.PLEXUS_LOOPBACK_PORT) : void 0
-    });
-    ctx.result(result, (r) => r.digest ?? "");
-    if (process.env.GITHUB_OUTPUT && result.digest)
-      appendFileSync5(process.env.GITHUB_OUTPUT, `digest=${result.digest}
+function register7(program2) {
+  withStrategy(program2.command("test")).description("run one app\u2019s tests against its compose stack on the candidate image, and tag the image tested").argument("<app>", "the app, apps/<app> in the checkout").argument("<build>", "Turbo\u2019s hash of the app\u2019s build task, the candidate\u2019s tag").argument("<hash>", "Turbo\u2019s hash of the app\u2019s test task, the tag on success").option("-C, --dir <dir>", "the checkout to read", ".").action(
+    async (app, build2, hash, opts, command) => {
+      const ctx = contextOf(command);
+      const result = await test(ctx, {
+        app,
+        build: build2,
+        hash,
+        dir: resolve4(opts.dir),
+        registry: registryFor(settings(opts)),
+        port: process.env.PLEXUS_LOOPBACK_PORT ? Number(process.env.PLEXUS_LOOPBACK_PORT) : void 0
+      });
+      ctx.result(result, (r) => r.digest ?? "");
+      if (process.env.GITHUB_OUTPUT && result.digest)
+        appendFileSync4(process.env.GITHUB_OUTPUT, `digest=${result.digest}
 `);
-  });
+    }
+  );
 }
 
 // src/commands/index.ts
-var commands = [
-  register4,
-  register3,
-  register,
-  register11,
-  register7,
-  register8,
-  register10,
-  register9,
-  register6,
-  register2,
-  register5
-];
+var commands = [register5, register3, register, register7, register4, register6, register2];
 
 // src/program.ts
 var { version, description } = createRequire(import.meta.url)("../package.json");
@@ -12651,7 +12923,7 @@ function buildProgram() {
   program2.addOption(new Option("-n, --dry-run", "print every effect this run would have, perform none").env("PLX_DRY_RUN")).addOption(
     new Option("-v, --verbose", "say more: once for reads and inputs, twice for command output").argParser((_value, previous) => previous + 1).default(0)
   ).addOption(new Option("-q, --quiet", "say nothing but results and errors").env("PLX_QUIET")).addOption(new Option("-y, --yes", "answer every confirmation with yes (needed off a terminal)").env("PLX_YES")).addOption(new Option("--json", "print results as JSON").env("PLX_JSON"));
-  for (const register12 of commands) register12(program2);
+  for (const register8 of commands) register8(program2);
   return program2;
 }
 function contextOf(command) {