@alchemy.run/fly 0.0.0 → 2.0.0-beta.85

package/src/replicas.ts

@@ -0,0 +1,1480 @@
+import type {
+  FlyMachineConfig,
+  FlyMachineMount,
+  FlyMachineService,
+  FlyMachineServiceCheck,
+  FlyStopConfig,
+  ImageRef as FlyImageRef,
+  Machine as FlyMachine,
+  Volume as FlyVolume,
+} from "@distilled.cloud/fly-io/machines";
+import * as machines from "@distilled.cloud/fly-io/machines";
+import * as Retry from "@distilled.cloud/fly-io/Retry";
+import { deepEqual } from "alchemy/Diff";
+import * as Data from "effect/Data";
+import * as Effect from "effect/Effect";
+import * as Result from "effect/Result";
+import * as Schedule from "effect/Schedule";
+import { listOwnedApps } from "./App.ts";
+import { reconcileBlueGreen, setRouting } from "./bluegreen.ts";
+import { validateDeployment, type DeploymentPolicy, type MachineCheck } from "./Deployment.ts";
+import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts";
+import { usingMachineLeases, type MachineLeases } from "./leases.ts";
+import type {
+  MachineGuest,
+  MachineImageRef,
+  MachineService,
+  MachineServiceCheck,
+} from "./Machine.ts";
+import {
+  alchemyMetadataKeys,
+  createMachineMetadata,
+  isAlchemyOwnedMetadata,
+  sanitizeFlyAppName,
+  type FlyAlchemyType,
+} from "./Metadata.ts";
+import type { DiskSpec, MountedDisk } from "./MountVolume.ts";
+import { regionOfReplica } from "./Region.ts";
+import { deleteVolume, ensureVolumeGroup, getVolumeById, volumeGroupName } from "./Volume.ts";
+
+const WAIT_TIMEOUT_SECONDS = 8;
+const waitBackoff = Schedule.exponential("500 millis");
+const SERVICE_CHECK_NAME_PREFIX = "servicecheck-";
+
+export class ReplicaNotCreated extends Data.TaggedError("Fly.ReplicaNotCreated")<{
+  resourceName: string;
+  appName: string;
+}> {}
+
+export class ReplicaChecksNotPassing extends Data.TaggedError("Fly.ReplicaChecksNotPassing")<{
+  appName: string;
+  machineId: string;
+  checks: ReadonlyArray<{
+    name: string | undefined;
+    status: string | undefined;
+    output: string | undefined;
+  }>;
+}> {
+  get message() {
+    const checks = this.checks.map(
+      (check) =>
+        `${check.name ?? "unnamed"}: ${check.status ?? "unknown"}${check.output ? ` (${check.output})` : ""}`,
+    );
+    return `Service checks did not pass for ${this.appName}/${this.machineId}: ${checks.join("; ") || "no service check results"}`;
+  }
+}
+
+export interface Replica {
+  machineId: string;
+  name: string;
+  baseName?: string;
+  region: string;
+  state: string;
+  instanceId: string | undefined;
+  privateIp: string | undefined;
+  imageRef: MachineImageRef | undefined;
+  guest: MachineGuest | undefined;
+  mounts: MountedDisk[];
+}
+
+export interface ReplicaSet {
+  rolloutPending?: boolean;
+  appName: string;
+  machineId: string;
+  machineIds: string[];
+  name: string;
+  baseName?: string;
+  /** Region of replica 0. */
+  region: string;
+  /** Regions the replicas run in, in replica order. */
+  regions: string[];
+  state: string;
+  instanceId: string | undefined;
+  privateIp: string | undefined;
+  imageRef: MachineImageRef | undefined;
+  guest: MachineGuest | undefined;
+  /** Observed proxy services published by replica 0. */
+  services: FlyMachineService[] | undefined;
+  count: number;
+  mounts: MountedDisk[];
+  replicas: Replica[];
+}
+
+const compactRecord = (
+  record: Record<string, string | undefined> | null | undefined,
+): Record<string, string> =>
+  Object.fromEntries(
+    Object.entries(record ?? {}).flatMap(([key, value]) =>
+      value === undefined ? [] : [[key, value]],
+    ),
+  );
+
+export const gone = (machine: FlyMachine | undefined) =>
+  machine === undefined || machine.state === "destroyed";
+
+export const getMachineById = (appName: string, machineId: string) =>
+  machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
+    Effect.map((machine) => (gone(machine) ? undefined : machine)),
+    Effect.catchTag("NotFound", () => Effect.undefined),
+  );
+
+/**
+ * Reads a Machine we hold a lease on. Fly's Machines API is not
+ * read-after-write consistent: a freshly created Machine can accept a lease
+ * (201) and then answer `GET /machines/{id}` with 404 for several seconds.
+ * Holding the lease blocks other writers, so a 404 here is lag rather than
+ * deletion; retry it (bounded) before surfacing `NotFound`.
+ */
+const getLeasedMachine = (appName: string, machineId: string) =>
+  machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
+    Effect.retry({
+      times: 10,
+      schedule: Schedule.spaced("1 second"),
+      while: (error) => error._tag === "NotFound",
+    }),
+  );
+
+const getLeasedMachineById = (appName: string, machineId: string) =>
+  getLeasedMachine(appName, machineId).pipe(
+    Effect.map((machine) => (gone(machine) ? undefined : machine)),
+    Effect.catchTag("NotFound", () => Effect.undefined),
+  );
+
+export const listMachinesByApp = (appName: string) =>
+  machines.listMachines({ app_name: appName }).pipe(
+    // A just-created App can answer 500 "failed to get app: sql: no rows in
+    // result set" until the Machines API sees it; safe to replay a read.
+    Effect.retry({
+      while: (error) => error._tag === "InternalServerError",
+      schedule: Schedule.spaced("2 seconds"),
+      times: 8,
+    }),
+    Effect.map((machines) => machines.filter((machine) => !gone(machine))),
+    Effect.catchTag("NotFound", () => Effect.succeed([])),
+  );
+
+export const resolveCount = (count: number | undefined) => Math.max(1, Math.floor(count ?? 1));
+
+export const replicaMachineName = (base: string, index: number, count: number) => {
+  if (count <= 1 && index === 0) return base;
+  const suffix = `-${index}`;
+  const room = 30 - suffix.length;
+  const clipped = base.slice(0, Math.max(1, room)).replace(/-+$/g, "");
+  return sanitizeFlyAppName(`${clipped}${suffix}`);
+};
+
+export const replicaIndexOf = (machine: FlyMachine): number => {
+  const raw = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.replica];
+  const parsed = raw === undefined ? 0 : Number(raw);
+  return Number.isFinite(parsed) ? parsed : 0;
+};
+
+export const alchemyIdOf = (machine: FlyMachine): string | undefined => {
+  const id = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.id];
+  return id !== undefined && id.length > 0 ? id : undefined;
+};
+
+export const isOwnedType = (machine: FlyMachine, type: FlyAlchemyType) => {
+  const metadata = compactRecord(machine.config?.metadata);
+  return isAlchemyOwnedMetadata(metadata) && metadata[alchemyMetadataKeys.type] === type;
+};
+
+export const toImageRef = (ref: FlyImageRef | undefined): MachineImageRef | undefined => {
+  if (ref === undefined) return undefined;
+  const imageRef: MachineImageRef = {
+    registry: ref.registry,
+    repository: ref.repository,
+    tag: ref.tag,
+    digest: ref.digest,
+  };
+  return imageRef.registry === undefined &&
+    imageRef.repository === undefined &&
+    imageRef.tag === undefined &&
+    imageRef.digest === undefined
+    ? undefined
+    : imageRef;
+};
+
+export const toGuestAttrs = (
+  guest:
+    | { cpu_kind?: string; cpus?: number; memory_mb?: number; gpu_kind?: string; gpus?: number }
+    | undefined,
+): MachineGuest | undefined => {
+  if (guest === undefined) return undefined;
+  return {
+    cpuKind: guest.cpu_kind,
+    cpus: guest.cpus,
+    memoryMb: guest.memory_mb,
+    gpuKind: guest.gpu_kind,
+    gpus: guest.gpus,
+  };
+};
+
+export const toFlyServiceCheck = (check: MachineServiceCheck): FlyMachineServiceCheck => ({
+  type: check.type,
+  port: check.port,
+  interval: check.interval,
+  timeout: check.timeout,
+  grace_period: check.gracePeriod,
+  method: check.method,
+  path: check.path,
+  protocol: check.protocol,
+  headers: check.headers?.map((header) => ({ name: header.name, values: header.values })),
+  tls_server_name: check.tlsServerName,
+  tls_skip_verify: check.tlsSkipVerify,
+});
+
+export const toFlyService = (service: MachineService): FlyMachineService => ({
+  protocol: service.protocol,
+  internal_port: service.internalPort,
+  autostart: service.autostart,
+  autostop:
+    typeof service.autostop === "boolean" ? (service.autostop ? "stop" : "off") : service.autostop,
+  min_machines_running: service.minMachinesRunning,
+  ports: service.ports?.map((port) => ({
+    port: port.port,
+    handlers: port.handlers,
+    force_https: port.forceHttps,
+    start_port: port.startPort,
+    end_port: port.endPort,
+  })),
+  checks: service.checks?.map((check) => ({
+    ...toFlyServiceCheck(check),
+    // Fly clamps service-check intervals; named Machine checks retain longer intervals.
+    interval:
+      (durationNanoseconds(check.interval) ?? 0n) > 60_000_000_000n ? "60s" : check.interval,
+  })),
+});
+
+export const autostopMode = (value: string | boolean | undefined) =>
+  value === true ? "stop" : value === false || value === undefined ? "off" : value;
+
+const normalizedCheckDuration = (value: string | undefined) => {
+  const nanos = durationNanoseconds(value);
+  return nanos === undefined ? value : `${nanos}ns`;
+};
+
+const normalizedCheck = (check: machines.FlyMachineCheck | FlyMachineServiceCheck | undefined) =>
+  check === undefined
+    ? undefined
+    : {
+        ...check,
+        grace_period: normalizedCheckDuration(check.grace_period),
+        interval: normalizedCheckDuration(check.interval),
+        timeout: normalizedCheckDuration(check.timeout),
+      };
+
+export const sameChecks = (
+  observed: FlyMachineConfig["checks"],
+  desired: FlyMachineConfig["checks"],
+) => {
+  const normalize = (checks: FlyMachineConfig["checks"]) =>
+    Object.fromEntries(
+      Object.entries(checks ?? {}).map(([name, check]) => [name, normalizedCheck(check)]),
+    );
+  return deepEqual(normalize(observed), normalize(desired), { stripNullish: true });
+};
+
+export const normalizedServices = (services: FlyMachineService[] | undefined) =>
+  (services ?? []).map((service) => ({
+    ...service,
+    autostop: autostopMode(service.autostop),
+    checks: service.checks?.map(normalizedCheck),
+  }));
+
+export const sameServices = (
+  observed: FlyMachineService[] | undefined,
+  desired: FlyMachineService[] | undefined,
+) => deepEqual(normalizedServices(observed), normalizedServices(desired), { stripNullish: true });
+
+export const hasPublishedService = (services: FlyMachineService[] | undefined) =>
+  (services ?? []).some((service) =>
+    (service.ports ?? []).some((port) => port.port !== undefined || port.start_port !== undefined),
+  );
+
+export const waitStarted = (appName: string, machineId: string) =>
+  machines
+    .waitMachine({
+      app_name: appName,
+      machine_id: machineId,
+      state: "started",
+      timeout: WAIT_TIMEOUT_SECONDS,
+    })
+    .pipe(
+      Retry.none,
+      Effect.retry({
+        times: 6,
+        schedule: waitBackoff,
+        while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
+      }),
+      Effect.timeout("50 seconds"),
+    );
+
+export const configuredCheckNames = (config: FlyMachineConfig | undefined) => [
+  ...Object.keys(config?.checks ?? {}),
+  ...(config?.services ?? []).flatMap((service) =>
+    (service.checks ?? []).map(
+      (check, checkIndex) =>
+        `${SERVICE_CHECK_NAME_PREFIX}${String(checkIndex).padStart(2, "0")}-${check.type ?? "tcp"}-${service.internal_port}`,
+    ),
+  ),
+];
+
+export const checksPassing = (machine: FlyMachine, config: FlyMachineConfig | undefined) => {
+  const expected = configuredCheckNames(config);
+  const checks = machine.checks ?? [];
+  // Fly may also report compatibility mirrors; they do not replace service reports.
+  return (
+    machine.state === "started" &&
+    new Set(expected).size === expected.length &&
+    checks.every(
+      (check) =>
+        check.name !== undefined &&
+        check.status === "passing" &&
+        (expected.includes(check.name) ||
+          expected.some(
+            (name) =>
+              name.startsWith(SERVICE_CHECK_NAME_PREFIX) &&
+              check.name === name.replace(SERVICE_CHECK_NAME_PREFIX, "bg_deployments_compat-"),
+          )),
+    ) &&
+    new Set(checks.map((check) => check.name)).size === checks.length &&
+    expected.every((name) => {
+      const reports = checks.filter((check) => check.name === name);
+      return reports.length === 1 && reports[0]?.status === "passing";
+    })
+  );
+};
+
+const TRANSIENT_GET_TAGS = [
+  "TooManyRequests",
+  "InternalServerError",
+  "BadGateway",
+  "ServiceUnavailable",
+  "GatewayTimeout",
+] as const;
+
+/**
+ * After the Machine is started, wait until Fly reports every service
+ * check as passing. No configured checks is a no-op. Empty live
+ * `servicecheck-*` results keep polling — they are not success.
+ * `warning` / `unknown` during grace keep polling.
+ */
+export const waitHealthy = Effect.fn(function* (
+  appName: string,
+  machine: FlyMachine,
+  healthTimeoutMs = 60_000,
+  config: FlyMachineConfig | undefined = machine.config,
+) {
+  const machineId = machine.id;
+  const named = Object.keys(config?.checks ?? {});
+  const expected = (config?.services ?? []).flatMap((service) => service.checks ?? []).length;
+  if (machineId === undefined || (expected === 0 && named.length === 0)) return machine;
+
+  let observed = machine;
+  const notPassing = () =>
+    new ReplicaChecksNotPassing({
+      appName,
+      machineId,
+      checks: (observed.checks ?? []).map((check) => ({
+        name: check.name,
+        status: check.status,
+        output: undefined,
+      })),
+    });
+  const passing = yield* getMachineById(appName, machineId).pipe(
+    Retry.none,
+    Effect.map((current) => {
+      if (current === undefined) return false;
+      observed = current;
+      return (
+        current.instance_id !== undefined &&
+        current.instance_id === machine.instance_id &&
+        checksPassing(current, config)
+      );
+    }),
+    Effect.catchTag(TRANSIENT_GET_TAGS, () => Effect.succeed(false)),
+    Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)),
+    Effect.repeat({
+      schedule: Schedule.spaced(healthTimeoutMs / 10),
+      until: (passing) => passing,
+      times: 10,
+    }),
+    Effect.timeoutOrElse({ duration: healthTimeoutMs, orElse: () => Effect.fail(notPassing()) }),
+  );
+  if (!passing) return yield* notPassing();
+  return observed;
+});
+
+export const waitDestroyed = (appName: string, machineId: string) =>
+  machines
+    .waitMachine({
+      app_name: appName,
+      machine_id: machineId,
+      state: "destroyed",
+      timeout: WAIT_TIMEOUT_SECONDS,
+    })
+    .pipe(
+      Retry.none,
+      Effect.as(undefined),
+      Effect.catchTag("NotFound", () => Effect.void),
+      Effect.retry({
+        times: 6,
+        schedule: waitBackoff,
+        while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
+      }),
+      Effect.timeout("50 seconds"),
+    );
+
+export const ensureStarted = (
+  appName: string,
+  machine: FlyMachine,
+  skipLaunch: boolean,
+  healthTimeoutMs = 60_000,
+  expectedConfig?: FlyMachineConfig,
+  existingLeases?: MachineLeases,
+) =>
+  usingMachineLeases(appName, existingLeases, (leases) =>
+    ensureLeasedStarted(appName, machine, skipLaunch, healthTimeoutMs, expectedConfig, leases),
+  );
+
+const ensureLeasedStarted = Effect.fn(function* (
+  appName: string,
+  machine: FlyMachine,
+  skipLaunch: boolean,
+  healthTimeoutMs: number,
+  expectedConfig: FlyMachineConfig | undefined,
+  leases: MachineLeases,
+) {
+  const machineId = machine.id;
+  if (machineId === undefined || skipLaunch) return machine;
+  yield* leases.acquire([machineId]);
+  const started = yield* Effect.gen(function* () {
+    // Create/update responses can lag Fly's automatic launch.
+    const current = yield* getLeasedMachine(appName, machineId);
+    if (!sameOwnership(current, machine)) {
+      return yield* new ReplicaOwnershipChanged({ appName, machineId });
+    }
+    yield* Effect.logDebug("Fly machine startup", { appName, machineId, state: current.state });
+    if (
+      current.state === "stopped" ||
+      current.state === "suspended" ||
+      current.state === "failed"
+    ) {
+      yield* leases.mutate(machineId, (lease_nonce) =>
+        machines.startMachine({ app_name: appName, machine_id: machineId, lease_nonce }),
+      );
+    }
+    // Re-observe state between waits instead of retrying the wait in the SDK.
+    yield* machines
+      .waitMachine({
+        app_name: appName,
+        machine_id: machineId,
+        state: "started",
+        instance_id: current.instance_id,
+        timeout: WAIT_TIMEOUT_SECONDS,
+      })
+      .pipe(Retry.none);
+    return yield* getLeasedMachine(appName, machineId);
+  }).pipe(
+    Effect.retry({
+      times: 6,
+      schedule: waitBackoff,
+      while: (error) =>
+        error._tag === "MachineStartFromCreatedState" ||
+        error._tag === "MachineReplacing" ||
+        error._tag === "MachineWaitTimeout" ||
+        error._tag === "Conflict" ||
+        error._tag === "GatewayTimeout",
+    }),
+    Effect.timeout("180 seconds"),
+  );
+  return yield* waitHealthy(appName, started, healthTimeoutMs, expectedConfig ?? machine.config);
+});
+
+export const deleteMachine = (appName: string, machineId: string, existingLeases?: MachineLeases) =>
+  usingMachineLeases(appName, existingLeases, (leases) =>
+    Effect.gen(function* () {
+      if (appName.length === 0 || machineId.length === 0) return;
+      yield* leases.acquire([machineId]);
+      yield* leases.remove(machineId, (lease_nonce) =>
+        machines
+          .deleteMachine({ app_name: appName, machine_id: machineId, force: true, lease_nonce })
+          .pipe(Effect.timeout("30 seconds")),
+      );
+      yield* waitDestroyed(appName, machineId);
+    }).pipe(Effect.catchTag("NotFound", () => leases.forget(machineId))),
+  );
+
+const durationNanoseconds = (value: string | undefined) => {
+  if (value === undefined || value === "") return undefined;
+  const negative = value.startsWith("-");
+  const duration = /^[+-]/.test(value) ? value.slice(1) : value;
+  if (duration === "0") return 0n;
+  const limit = negative ? 1n << 63n : (1n << 63n) - 1n;
+  const units: Record<string, bigint> = {
+    ns: 1n,
+    us: 1_000n,
+    µs: 1_000n,
+    μs: 1_000n,
+    ms: 1_000_000n,
+    s: 1_000_000_000n,
+    m: 60_000_000_000n,
+    h: 3_600_000_000_000n,
+  };
+  let total = 0n;
+  let consumed = 0;
+  for (const part of duration.matchAll(/(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|μs|ms|s|m|h)/gy)) {
+    const [integer = "", fraction = ""] = part[1]!.split(".");
+    const whole = integer.replace(/^0+/, "") || "0";
+    if (whole.length > 19) return undefined;
+    const unit = units[part[2]!]!;
+    let fractionalNanos = 0n;
+    // Truncate each component to nanoseconds without unbounded BigInt operands.
+    for (let index = fraction.length - 1; index >= 0; index--) {
+      fractionalNanos = (BigInt(fraction[index]!) * unit + fractionalNanos) / 10n;
+    }
+    total += BigInt(whole) * unit + fractionalNanos;
+    if (total > limit) return undefined;
+    consumed += part[0].length;
+  }
+  if (consumed === 0 || consumed !== duration.length) return undefined;
+  return negative ? -total : total;
+};
+
+const stopTimeoutMillis = (timeout: string | undefined) => {
+  if (timeout === "") return 0;
+  // Shutdown policies retain their unsigned millisecond-or-larger syntax.
+  if (timeout === undefined || !/^(?:\d+(?:\.\d+)?(?:ms|s|m|h))+$/.test(timeout)) {
+    return undefined;
+  }
+  const nanos = durationNanoseconds(timeout);
+  return nanos === undefined ? undefined : Number(nanos) / 1_000_000;
+};
+
+export const sameStopConfig = (
+  observed: FlyStopConfig | undefined,
+  desired: FlyStopConfig | undefined,
+) =>
+  observed?.signal === desired?.signal &&
+  stopTimeoutMillis(observed?.timeout) === stopTimeoutMillis(desired?.timeout);
+
+export class ShutdownPolicyMismatch extends Data.TaggedError("Fly.ShutdownPolicyMismatch")<{
+  machineId: string;
+  message: string;
+}> {}
+
+export const predecessorShutdown = Effect.fn(function* (machine: FlyMachine) {
+  const persisted = machine.config?.stop_config;
+  const injected = machine.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS;
+  const timeoutMs = stopTimeoutMillis(persisted?.timeout);
+  if (injected !== undefined) {
+    const managedTimeout = Number(injected);
+    const signal = persisted?.signal ?? "SIGTERM";
+    if (
+      !/^\d+$/.test(injected) ||
+      !Number.isSafeInteger(managedTimeout) ||
+      managedTimeout <= 0 ||
+      managedTimeout > 300_000 ||
+      (signal !== "SIGTERM" && signal !== "SIGINT") ||
+      (persisted?.timeout !== undefined && timeoutMs !== managedTimeout)
+    ) {
+      return yield* new ShutdownPolicyMismatch({
+        machineId: machine.id ?? "",
+        message:
+          "The predecessor's managed shutdown environment and stop_config disagree; reconcile its shutdown policy before migration.",
+      });
+    }
+    return {
+      signal,
+      timeout: persisted?.timeout ?? `${managedTimeout}ms`,
+      timeoutMs: managedTimeout,
+    };
+  }
+  if (persisted?.timeout !== undefined && (!timeoutMs || timeoutMs > 300_000)) {
+    return yield* new ShutdownPolicyMismatch({
+      machineId: machine.id ?? "",
+      message:
+        "The predecessor has an invalid stop_config timeout; refusing to shorten its shutdown window.",
+    });
+  }
+  // Unspecified raw-image overrides must remain unspecified for Fly's defaults.
+  return {
+    signal: persisted?.signal,
+    timeout: persisted?.timeout,
+    timeoutMs: timeoutMs ?? 300_000,
+  };
+});
+
+export class ReplicaOwnershipChanged extends Data.TaggedError("Fly.ReplicaOwnershipChanged")<{
+  appName: string;
+  machineId: string;
+}> {}
+
+export class ReplicaRetirementIncomplete extends Data.TaggedError(
+  "Fly.ReplicaRetirementIncomplete",
+)<{ appName: string; residuals: Array<{ machineId: string; stage: string }> }> {}
+
+const retirementStep =
+  (appName: string, machineId: string, stage: string) =>
+  <A, E extends { readonly _tag: string }, R>(effect: Effect.Effect<A, E, R>) =>
+    effect.pipe(
+      Effect.mapError((error) =>
+        error._tag === "NotFound"
+          ? error
+          : new ReplicaRetirementIncomplete({
+              appName,
+              residuals: [{ machineId, stage: `${stage}: ${error._tag}` }],
+            }),
+      ),
+    );
+
+const sameOwnership = (observed: FlyMachine, snapshot: FlyMachine) =>
+  observed.id === snapshot.id &&
+  [
+    alchemyMetadataKeys.stack,
+    alchemyMetadataKeys.stage,
+    alchemyMetadataKeys.id,
+    alchemyMetadataKeys.type,
+    alchemyMetadataKeys.instance,
+    alchemyMetadataKeys.fqn,
+    alchemyMetadataKeys.generation,
+  ].every((key) => observed.config?.metadata?.[key] === snapshot.config?.metadata?.[key]);
+
+export const leaseSnapshot = Effect.fn(function* (
+  appName: string,
+  snapshot: FlyMachine[],
+  leases: MachineLeases,
+) {
+  const ordered = [...snapshot].sort((a, b) => (a.id ?? "").localeCompare(b.id ?? ""));
+  const leased = new Set<string>();
+  yield* Effect.forEach(
+    ordered,
+    (machine) =>
+      Effect.gen(function* () {
+        if (!machine.id || machine.host_status === "unreachable") return;
+        const id = machine.id;
+        yield* leases.acquire([id]).pipe(
+          Effect.tap(() => Effect.sync(() => leased.add(id))),
+          // A just-created Machine can 404 on the lease API while it already
+          // lists; a Machine that is really gone is dropped by the GET below.
+          Effect.retry({
+            while: (error) => error._tag === "NotFound",
+            schedule: Schedule.spaced("1 second"),
+            times: 5,
+          }),
+          Effect.catchTag("NotFound", () => Effect.void),
+        );
+      }),
+    { concurrency: 1 },
+  ).pipe(Effect.timeout("30 seconds"));
+  const observed: FlyMachine[] = [];
+  for (const machine of ordered) {
+    if (!machine.id) continue;
+    if (machine.host_status === "unreachable") {
+      observed.push(machine);
+      continue;
+    }
+    const current = yield* leased.has(machine.id)
+      ? getLeasedMachineById(appName, machine.id)
+      : getMachineById(appName, machine.id);
+    if (!current) {
+      yield* leases.forget(machine.id);
+      continue;
+    }
+    if (!sameOwnership(current, machine))
+      return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
+    yield* leases.checkTarget(machine.id);
+    observed.push(current);
+  }
+  yield* leases.check;
+  return observed;
+});
+
+export const retireMachines = (
+  appName: string,
+  snapshot: FlyMachine[],
+  replacementReady = false,
+  existingLeases?: MachineLeases,
+) =>
+  usingMachineLeases(appName, existingLeases, (leases) =>
+    retireLeasedMachines(appName, snapshot, replacementReady, leases),
+  );
+
+const retireLeasedMachines = Effect.fn(function* (
+  appName: string,
+  snapshot: FlyMachine[],
+  replacementReady: boolean,
+  leases: MachineLeases,
+) {
+  const targets = yield* leaseSnapshot(appName, snapshot, leases);
+  const results = yield* Effect.forEach(
+    targets,
+    (machine) =>
+      Effect.gen(function* () {
+        yield* leases.check;
+        const result = yield* Effect.gen(function* () {
+          if (!machine.id) return;
+          if (replacementReady && machine.host_status === "unreachable") {
+            const metadata = machine.config?.metadata;
+            if (
+              !machine.config ||
+              machine.incomplete_config ||
+              (machine.config.mounts?.length ?? 0) > 0 ||
+              !metadata?.[alchemyMetadataKeys.instance] ||
+              !metadata[alchemyMetadataKeys.fqn]
+            ) {
+              return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
+            }
+            yield* Effect.logWarning(
+              "Force-retiring owned stateless Fly Machine on unreachable host; work was not proven drained",
+              { appName, machineId: machine.id },
+            );
+            const machineId = machine.id;
+            const heldNonce = yield* leases.nonceIfHeld(machineId);
+            yield* Effect.gen(function* () {
+              const remove = (lease_nonce?: string) =>
+                machines
+                  .deleteMachine({
+                    app_name: appName,
+                    machine_id: machineId,
+                    force: true,
+                    lease_nonce,
+                  })
+                  .pipe(Retry.none, Effect.timeout("30 seconds"));
+              if (heldNonce !== undefined) yield* leases.remove(machineId, remove);
+              else yield* leases.guard(remove());
+            }).pipe(
+              Effect.catchTag("NotFound", () => Effect.void),
+              retirementStep(appName, machineId, "unreachable force-delete"),
+            );
+            yield* leases.forget(machine.id);
+            return yield* waitDestroyed(appName, machine.id);
+          }
+          yield* retireMachine(appName, machine.id, machine, leases);
+        }).pipe(Effect.result);
+        yield* leases.check;
+        return Result.isFailure(result)
+          ? result.failure._tag === "Fly.ReplicaRetirementIncomplete"
+            ? result.failure.residuals
+            : [{ machineId: machine.id ?? "", stage: result.failure._tag }]
+          : [];
+      }),
+    { concurrency: 4 },
+  );
+  const residuals = results.flat();
+  if (residuals.length) return yield* new ReplicaRetirementIncomplete({ appName, residuals });
+});
+
+export const retireMachine = (
+  appName: string,
+  machineId: string,
+  snapshot?: FlyMachine,
+  existingLeases?: MachineLeases,
+) =>
+  usingMachineLeases(appName, existingLeases, (leases) =>
+    retireLeasedMachine(appName, machineId, snapshot, leases).pipe(
+      Effect.tap(() => leases.forget(machineId)),
+    ),
+  );
+
+const retireLeasedMachine = Effect.fn(
+  function* (
+    appName: string,
+    machineId: string,
+    snapshot: FlyMachine | undefined,
+    leases: MachineLeases,
+  ) {
+    yield* leases.acquire([machineId]);
+    const current = yield* getMachineById(appName, machineId);
+    if (current === undefined) return;
+    if (snapshot && !sameOwnership(current, snapshot)) {
+      return yield* new ReplicaOwnershipChanged({ appName, machineId });
+    }
+    const stop = yield* predecessorShutdown(current);
+    if (
+      current.cordoned === true &&
+      current.config?.metadata?.[alchemyMetadataKeys.phase] === "candidate"
+    )
+      return yield* deleteMachine(appName, machineId, leases);
+    if (current.state !== "stopped" && current.state !== "created") {
+      yield* leases
+        .mutate(
+          machineId,
+          (lease_nonce) =>
+            machines
+              .cordonMachine({ app_name: appName, machine_id: machineId, lease_nonce })
+              .pipe(Effect.timeout("30 seconds")),
+          { idempotent: true },
+        )
+        .pipe(retirementStep(appName, machineId, "cordon"));
+      if (hasPublishedService(current.config?.services)) yield* Effect.sleep("10 seconds");
+      yield* leases
+        .mutate(
+          machineId,
+          (lease_nonce) =>
+            machines
+              .stopMachine({
+                app_name: appName,
+                machine_id: machineId,
+                lease_nonce,
+                signal: stop.signal,
+                timeout: stop.timeout,
+              })
+              .pipe(Effect.timeout(stop.timeoutMs + 15_000)),
+          { timeoutMs: stop.timeoutMs + 120_000 },
+        )
+        .pipe(retirementStep(appName, machineId, "stop"));
+      yield* machines
+        .waitMachine({
+          app_name: appName,
+          machine_id: machineId,
+          state: "stopped",
+          instance_id: current.instance_id,
+          timeout: WAIT_TIMEOUT_SECONDS,
+        })
+        .pipe(
+          Retry.none,
+          Effect.retry({
+            times: 10,
+            schedule: Schedule.spaced(Math.max(500, stop.timeoutMs / 10)),
+            while: (error) =>
+              error._tag === "MachineWaitTimeout" || error._tag === "GatewayTimeout",
+          }),
+          Effect.timeout(stop.timeoutMs + 15_000),
+          retirementStep(appName, machineId, "wait-stopped"),
+        );
+    }
+    yield* leases
+      .remove(machineId, (lease_nonce) =>
+        machines
+          .deleteMachine({ app_name: appName, machine_id: machineId, force: false, lease_nonce })
+          .pipe(Effect.timeout("30 seconds")),
+      )
+      .pipe(
+        Effect.catchTag("NotFound", () => Effect.void),
+        retirementStep(appName, machineId, "delete"),
+      );
+    yield* waitDestroyed(appName, machineId).pipe(
+      retirementStep(appName, machineId, "verify-absent"),
+    );
+  },
+  Effect.catchTag("NotFound", () => Effect.void),
+);
+
+const mountedDisksOf = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): MountedDisk[] =>
+  (machine.config?.mounts ?? []).flatMap((mount) => {
+    const volumeId = mount.volume;
+    const path = mount.path;
+    if (volumeId === undefined || path === undefined) return [];
+    const volume = volumesById.get(volumeId);
+    return [{ path, volumeId, sizeGb: volume?.size_gb ?? 0, name: volume?.name ?? "" }];
+  });
+
+export const toReplica = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): Replica => ({
+  machineId: machine.id ?? "",
+  name: machine.name ?? "",
+  region: machine.region ?? "",
+  state: machine.state ?? "",
+  instanceId: machine.instance_id,
+  privateIp: machine.private_ip,
+  imageRef: toImageRef(machine.image_ref),
+  guest: toGuestAttrs(machine.config?.guest),
+  mounts: mountedDisksOf(machine, volumesById),
+});
+
+export const toReplicaSet = (
+  replicas: Replica[],
+  appName: string,
+  baseName: string,
+  services?: FlyMachineService[],
+): ReplicaSet => {
+  const primary = replicas[0];
+  return {
+    appName,
+    machineId: primary?.machineId ?? "",
+    machineIds: replicas.map((replica) => replica.machineId),
+    name: primary?.name ?? baseName,
+    baseName,
+    region: primary?.region ?? "",
+    regions: [
+      ...new Set(replicas.map((replica) => replica.region).filter((region) => region.length > 0)),
+    ],
+    state: primary?.state ?? "",
+    instanceId: primary?.instanceId,
+    privateIp: primary?.privateIp,
+    imageRef: primary?.imageRef,
+    guest: primary?.guest,
+    services,
+    count: replicas.length,
+    mounts: primary?.mounts ?? [],
+    replicas,
+  };
+};
+
+export const ownedReplicas = (
+  listed: FlyMachine[],
+  input: {
+    metadata: Record<string, string>;
+    resourceInstanceId: string;
+    fqn: string;
+    baseName?: string;
+    machineIds?: readonly string[];
+  },
+) =>
+  listed.filter((machine) => {
+    const metadata = machine.config?.metadata ?? {};
+    if (!Object.entries(input.metadata).every(([key, value]) => metadata[key] === value))
+      return false;
+    if (metadata[alchemyMetadataKeys.instance] !== undefined) {
+      return (
+        metadata[alchemyMetadataKeys.instance] === input.resourceInstanceId &&
+        metadata[alchemyMetadataKeys.fqn] === input.fqn
+      );
+    }
+    return (
+      (machine.id !== undefined && input.machineIds?.includes(machine.id)) ||
+      (input.baseName !== undefined &&
+        (machine.name === input.baseName ||
+          machine.name === replicaMachineName(input.baseName, replicaIndexOf(machine), 2)))
+    );
+  });
+
+export const listReplicas = Effect.fn(function* (input: {
+  appName: string;
+  id: string;
+  type: FlyAlchemyType;
+}) {
+  const machines = yield* listMachinesByApp(input.appName);
+  return machines
+    .filter((machine) => isOwnedType(machine, input.type) && alchemyIdOf(machine) === input.id)
+    .sort((left, right) => replicaIndexOf(left) - replicaIndexOf(right));
+});
+
+export const listReplicaSets = Effect.fn(function* (type: FlyAlchemyType) {
+  const apps = yield* listOwnedApps();
+  const groups = yield* Effect.forEach(
+    apps,
+    (app) =>
+      listMachinesByApp(app.appName).pipe(
+        Effect.map((machines) => {
+          const owned = machines.filter((machine) => isOwnedType(machine, type));
+          const byId = new Map<string, FlyMachine[]>();
+          for (const machine of owned) {
+            const id = alchemyIdOf(machine);
+            if (id === undefined) continue;
+            const metadata = machine.config?.metadata;
+            const key = JSON.stringify([
+              metadata?.[alchemyMetadataKeys.stack],
+              metadata?.[alchemyMetadataKeys.stage],
+              metadata?.[alchemyMetadataKeys.fqn] ?? id,
+              metadata?.[alchemyMetadataKeys.instance],
+              metadata?.[alchemyMetadataKeys.generation],
+            ]);
+            const group = byId.get(key) ?? [];
+            group.push(machine);
+            byId.set(key, group);
+          }
+          return [...byId.values()].map((group) => {
+            const sorted = [...group].sort(
+              (left, right) => replicaIndexOf(left) - replicaIndexOf(right),
+            );
+            const replicas = sorted.map((machine) => toReplica(machine, new Map()));
+            return toReplicaSet(
+              replicas,
+              app.appName,
+              replicas[0]?.name ?? "",
+              sorted[0]?.config?.services,
+            );
+          });
+        }),
+      ),
+    { concurrency: 8 },
+  );
+  return groups.flat();
+});
+
+const pickVolume = (
+  group: FlyVolume[],
+  used: Set<string>,
+  preferId: string | undefined,
+): FlyVolume | undefined => {
+  if (preferId !== undefined && !used.has(preferId)) {
+    const preferred = group.find((volume) => volume.id === preferId);
+    if (preferred !== undefined) return preferred;
+  }
+  return group.find((volume) => volume.id !== undefined && !used.has(volume.id));
+};
+
+export interface ReconcileReplicasInput {
+  fqn: string;
+  resourceInstanceId: string;
+  policy: DeploymentPolicy;
+  checks?: Record<string, MachineCheck>;
+  id: string;
+  type: FlyAlchemyType;
+  appName: string;
+  baseName: string;
+  /**
+   * Regions to run in. `count` is the total replica count; replica `i`
+   * runs in `regions[i % regions.length]`.
+   */
+  regions: readonly string[];
+  count: number;
+  disks: DiskSpec[];
+  skipLaunch?: boolean;
+  minSecretsVersion?: number;
+  outputMachineIds?: readonly string[];
+  preferVolumeIds?: ReadonlyArray<ReadonlyArray<string>>;
+  configDrifted: (
+    machine: FlyMachine,
+    desired: { mounts: FlyMachineMount[]; metadata: Record<string, string> },
+  ) => boolean;
+  buildConfig: (replica: {
+    index: number;
+    mounts: FlyMachineMount[];
+    metadata: Record<string, string>;
+  }) => FlyMachineConfig;
+}
+
+export const reconcileReplicas = (input: ReconcileReplicasInput) =>
+  usingMachineLeases(input.appName, undefined, (leases) => reconcileLeasedReplicas(input, leases));
+
+const reconcileLeasedReplicas = Effect.fn(function* (
+  input: ReconcileReplicasInput,
+  leases: MachineLeases,
+) {
+  const ownership = yield* createMachineMetadata(input.id, input.type);
+  const alchemy = {
+    ...ownership,
+    [alchemyMetadataKeys.instance]: input.resourceInstanceId,
+    [alchemyMetadataKeys.fqn]: input.fqn,
+    [alchemyMetadataKeys.baseName]: input.baseName,
+  };
+  const buildConfig: ReconcileReplicasInput["buildConfig"] = (replica) => ({
+    ...input.buildConfig(replica),
+    checks:
+      input.checks === undefined
+        ? undefined
+        : Object.fromEntries(
+            Object.entries(input.checks).map(([name, check]) => [name, toFlyServiceCheck(check)]),
+          ),
+    stop_config:
+      input.policy.shutdown === undefined
+        ? undefined
+        : { signal: input.policy.shutdown.signal, timeout: input.policy.shutdown.timeout },
+  });
+  const config = buildConfig({ index: 0, mounts: [], metadata: alchemy });
+  yield* validateDeployment(input.policy, config, input.disks.length > 0, input.skipLaunch);
+  if (input.policy.bluegreen)
+    return yield* reconcileBlueGreen({ ...input, buildConfig }, ownership, alchemy, leases);
+  const desiredNames = new Set(
+    Array.from({ length: input.count }, (_, index) =>
+      replicaMachineName(input.baseName, index, input.count),
+    ),
+  );
+  const listed = yield* listMachinesByApp(input.appName);
+  const owned = yield* leaseSnapshot(
+    input.appName,
+    ownedReplicas(listed, { ...input, metadata: ownership, machineIds: input.outputMachineIds }),
+    leases,
+  );
+  const byIndex = new Map<number, FlyMachine>();
+  const preferIds = new Set((input.outputMachineIds ?? []).filter((id) => id.length > 0));
+  for (const machine of owned) {
+    const id = machine.id;
+    if (id !== undefined && preferIds.has(id)) {
+      byIndex.set(replicaIndexOf(machine), machine);
+    }
+  }
+  for (const machine of owned) {
+    const name = machine.name;
+    if (
+      name === undefined ||
+      (!desiredNames.has(name) &&
+        machine.config?.metadata?.[alchemyMetadataKeys.instance] !== input.resourceInstanceId)
+    )
+      continue;
+    const index = replicaIndexOf(machine);
+    if (!byIndex.has(index)) byIndex.set(index, machine);
+  }
+
+  const scaleDown = [...byIndex].filter(([index]) => index >= input.count);
+  yield* retireMachines(
+    input.appName,
+    scaleDown.map(([, machine]) => machine),
+    false,
+    leases,
+  );
+  for (const [index] of scaleDown) byIndex.delete(index);
+
+  // Volumes are regional: one group per disk and region, sized to the
+  // replicas that run there.
+  const replicasIn = (region: string) =>
+    Array.from({ length: input.count }, (_, index) => index).filter(
+      (index) => regionOfReplica(input.regions, index) === region,
+    ).length;
+  const groups: Array<{
+    disk: DiskSpec;
+    name: string;
+    byRegion: Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>;
+  }> = [];
+  for (const [diskIndex, disk] of input.disks.entries()) {
+    const name = yield* volumeGroupName(input.id, disk);
+    const preferIds = (input.preferVolumeIds ?? [])
+      .map((replica) => replica[diskIndex])
+      .filter((id): id is string => id !== undefined && id.length > 0);
+    const byRegion = new Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>();
+    for (const region of input.regions) {
+      byRegion.set(
+        region,
+        yield* ensureVolumeGroup({
+          appName: input.appName,
+          name,
+          region,
+          count: replicasIn(region),
+          disk,
+          preferIds,
+        }),
+      );
+    }
+    groups.push({ disk, name, byRegion });
+  }
+
+  const usedVolumeIds = new Set<string>();
+  const live: FlyMachine[] = [];
+  for (let index = 0; index < input.count; index++) {
+    const name = replicaMachineName(input.baseName, index, input.count);
+    const region = regionOfReplica(input.regions, index);
+    const metadata = {
+      ...alchemy,
+      [alchemyMetadataKeys.replica]: String(index),
+      ...(input.minSecretsVersion === undefined
+        ? {}
+        : { [alchemyMetadataKeys.secretsVersion]: String(input.minSecretsVersion) }),
+    };
+    const prefer = input.preferVolumeIds?.[index] ?? [];
+    const mounts: FlyMachineMount[] = [];
+    for (const [diskIndex, group] of groups.entries()) {
+      const volume = pickVolume(
+        group.byRegion.get(region)?.volumes ?? [],
+        usedVolumeIds,
+        prefer[diskIndex],
+      );
+      const volumeId = volume?.id;
+      if (volumeId === undefined) {
+        return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
+      }
+      usedVolumeIds.add(volumeId);
+      mounts.push({ volume: volumeId, path: group.disk.path });
+    }
+    const config = buildConfig({ index, mounts, metadata });
+    let current = byIndex.get(index);
+    // Fly cannot move a Machine between regions. Machine names are unique
+    // per App, so retire the old replica before creating it in its region.
+    if (current !== undefined && current.region !== region) {
+      yield* retireMachines(input.appName, [current], false, leases);
+      byIndex.delete(index);
+      current = undefined;
+    }
+    if (current === undefined) {
+      const created = yield* machines
+        .createMachine({
+          app_name: input.appName,
+          name,
+          region,
+          config,
+          skip_launch: input.skipLaunch === true ? true : undefined,
+          min_secrets_version: input.minSecretsVersion,
+        })
+        .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
+      current =
+        created ??
+        (yield* listMachinesByApp(input.appName).pipe(
+          Effect.map((machines) =>
+            ownedReplicas(machines, {
+              ...input,
+              metadata: ownership,
+              machineIds: input.outputMachineIds,
+            }).find((machine) => machine.name === name),
+          ),
+        ));
+      if (current === undefined || current.id === undefined) {
+        return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
+      }
+      const observed = (yield* leaseSnapshot(input.appName, [current], leases))[0];
+      if (!observed || input.configDrifted(observed, { mounts, metadata })) {
+        return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
+      }
+      current = observed;
+    } else if (
+      !sameChecks(current.config?.checks, config.checks) ||
+      !sameStopConfig(current.config?.stop_config, config.stop_config) ||
+      input.configDrifted(current, { mounts, metadata })
+    ) {
+      const machineId = current.id!;
+      const currentVersion = current.instance_id;
+      const previous = current;
+      const expected = { ...previous, config };
+      yield* leases.mutate(machineId, (lease_nonce) =>
+        machines
+          .updateMachine({
+            app_name: input.appName,
+            machine_id: machineId,
+            current_version: currentVersion,
+            config,
+            skip_launch: input.skipLaunch === true ? true : undefined,
+            min_secrets_version: input.minSecretsVersion,
+            lease_nonce,
+          })
+          .pipe(Effect.timeout("30 seconds")),
+      );
+      // Rolling opt-out removes generation metadata; observe that transition before startup.
+      current = yield* leases.guard(
+        Effect.gen(function* () {
+          const observed = yield* machines
+            .getMachine({ app_name: input.appName, machine_id: machineId })
+            .pipe(Retry.none);
+          if (!sameOwnership(observed, expected) && !sameOwnership(observed, previous))
+            return yield* new ReplicaOwnershipChanged({ appName: input.appName, machineId });
+          if (
+            !sameOwnership(observed, expected) ||
+            input.configDrifted(observed, { mounts, metadata }) ||
+            !sameChecks(observed.config?.checks, config.checks) ||
+            !sameStopConfig(observed.config?.stop_config, config.stop_config)
+          )
+            return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
+          return observed;
+        }).pipe(
+          Effect.retry({
+            times: 8,
+            schedule: waitBackoff,
+            while: (error) =>
+              error._tag === "Fly.ReplicaNotCreated" ||
+              TRANSIENT_GET_TAGS.some((tag) => tag === error._tag),
+          }),
+          Effect.timeout("60 seconds"),
+        ),
+      );
+    }
+
+    current = yield* ensureStarted(
+      input.appName,
+      current,
+      input.skipLaunch === true,
+      input.policy.healthTimeoutMs,
+      config,
+      leases,
+    );
+    if (current.cordoned === true && !input.skipLaunch) {
+      yield* setRouting(input.appName, current.id!, false, leases);
+      if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds");
+      current = yield* waitHealthy(input.appName, current, input.policy.healthTimeoutMs, config);
+    }
+    live.push(current);
+  }
+
+  const liveIds = new Set(live.map((machine) => machine.id));
+  yield* retireMachines(
+    input.appName,
+    owned.filter((machine) => !liveIds.has(machine.id)),
+    false,
+    leases,
+  );
+
+  for (const group of groups) {
+    for (const extra of [...group.byRegion.values()].flatMap((placed) => placed.extras)) {
+      const volumeId = extra.id;
+      if (volumeId === undefined || usedVolumeIds.has(volumeId)) continue;
+      yield* deleteVolume(input.appName, volumeId);
+    }
+  }
+
+  const volumesById = new Map<string, FlyVolume>();
+  for (const group of groups) {
+    for (const volume of [...group.byRegion.values()].flatMap((placed) => placed.volumes)) {
+      if (volume.id !== undefined) volumesById.set(volume.id, volume);
+    }
+  }
+  const fresh = yield* Effect.forEach(
+    live,
+    (machine) =>
+      machine.id === undefined
+        ? Effect.succeed(machine)
+        : getMachineById(input.appName, machine.id).pipe(Effect.map((next) => next ?? machine)),
+    { concurrency: 4 },
+  );
+  const replicas = fresh.map((machine) => toReplica(machine, volumesById));
+  return toReplicaSet(replicas, input.appName, input.baseName, fresh[0]?.config?.services);
+});
+
+export const deleteReplicaSet = Effect.fn(function* (input: {
+  appName: string;
+  id: string;
+  type: FlyAlchemyType;
+  fqn: string;
+  resourceInstanceId: string;
+  force?: boolean;
+  machineIds: readonly string[];
+  volumeIds: readonly string[];
+}) {
+  return yield* usingMachineLeases(input.appName, undefined, (leases) =>
+    Effect.gen(function* () {
+      const owned = yield* leaseSnapshot(
+        input.appName,
+        ownedReplicas(yield* listMachinesByApp(input.appName), {
+          ...input,
+          metadata: yield* createMachineMetadata(input.id, input.type),
+        }),
+        leases,
+      );
+      if (input.force) {
+        yield* Effect.forEach(
+          owned,
+          (machine) =>
+            Effect.gen(function* () {
+              if (!machine.id) return;
+              const current = yield* getMachineById(input.appName, machine.id);
+              if (!current) return;
+              if (!sameOwnership(current, machine))
+                return yield* new ReplicaOwnershipChanged({
+                  appName: input.appName,
+                  machineId: machine.id,
+                });
+              yield* deleteMachine(input.appName, machine.id, leases);
+            }),
+          { concurrency: 4 },
+        );
+      } else yield* retireMachines(input.appName, owned, false, leases);
+      yield* Effect.forEach(
+        [...new Set(input.volumeIds)].filter((id) => id.length > 0),
+        (volumeId) => deleteVolume(input.appName, volumeId),
+        { concurrency: 4 },
+      );
+    }),
+  );
+});
+
+export const volumeIdsOf = (set: {
+  mounts?: readonly MountedDisk[];
+  replicas?: readonly Replica[];
+}): string[] => {
+  const ids = new Set<string>();
+  for (const mount of set.mounts ?? []) ids.add(mount.volumeId);
+  for (const replica of set.replicas ?? []) {
+    for (const mount of replica.mounts) ids.add(mount.volumeId);
+  }
+  return [...ids];
+};
+
+export const groupGenerations = (machines: FlyMachine[]) => {
+  const groups = new Map<string | undefined, FlyMachine[]>();
+  for (const machine of machines) {
+    const generation = machine.config?.metadata?.[alchemyMetadataKeys.generation];
+    const group = groups.get(generation) ?? [];
+    group.push(machine);
+    groups.set(generation, group);
+  }
+  return groups;
+};
+
+export const observeReplicaSet = Effect.fn(function* (input: {
+  appName?: string;
+  fqn: string;
+  resourceInstanceId: string;
+  id: string;
+  type: FlyAlchemyType;
+  machineIds?: readonly string[];
+  baseName?: string;
+}) {
+  if (input.appName === undefined) return undefined;
+  const owned = ownedReplicas(yield* listMachinesByApp(input.appName), {
+    ...input,
+    metadata: yield* createMachineMetadata(input.id, input.type),
+  });
+  const groups = groupGenerations(owned);
+  const committed = [...groups.entries()]
+    .filter(([generation, group]) => {
+      if (generation === undefined) return false;
+      const count = Number(group[0]?.config?.metadata?.[alchemyMetadataKeys.count]);
+      if (group.some((machine) => classifyDeploymentState(machine).protocol === "invalid"))
+        return false;
+      const protocol2 = group.some(
+        (machine) => machine.config?.metadata?.[alchemyMetadataKeys.protocol] === "2",
+      );
+      if (protocol2 && !validProtocol2Generation(group)) return false;
+      return (
+        Number.isSafeInteger(count) &&
+        count > 0 &&
+        group.length === count &&
+        new Set(group.map(replicaIndexOf)).size === count &&
+        group.every((machine) => {
+          const metadata = machine.config?.metadata;
+          const index = replicaIndexOf(machine);
+          return (
+            index >= 0 &&
+            index < count &&
+            Number(metadata?.[alchemyMetadataKeys.count]) === count &&
+            metadata?.[alchemyMetadataKeys.phase] === "active" &&
+            machine.cordoned === false &&
+            (metadata[alchemyMetadataKeys.protocol] === undefined ||
+              ((metadata[alchemyMetadataKeys.protocol] === "1" ||
+                metadata[alchemyMetadataKeys.protocol] === "2") &&
+                metadata[alchemyMetadataKeys.restored] === "true" &&
+                (metadata[alchemyMetadataKeys.role] === "idle" ||
+                  (machine.instance_id !== undefined &&
+                    metadata[alchemyMetadataKeys.checkedInstance] === machine.instance_id))))
+          );
+        })
+      );
+    })
+    .sort(
+      ([, a], [, b]) =>
+        Number(b[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0) -
+        Number(a[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0),
+    );
+  const legacy = (groups.get(undefined) ?? []).filter((machine) => {
+    const { protocol } = classifyDeploymentState(machine);
+    return protocol === "legacy" || protocol === "1";
+  });
+  const listed = (committed[0]?.[1] ?? legacy).sort(
+    (a, b) => replicaIndexOf(a) - replicaIndexOf(b),
+  );
+  const rolloutPending = owned.some((machine) => !listed.includes(machine));
+  if (listed.length > 0) {
+    const volumesById = new Map<string, FlyVolume>();
+    for (const machine of listed) {
+      for (const mount of machine.config?.mounts ?? []) {
+        const volumeId = mount.volume;
+        if (volumeId === undefined || volumesById.has(volumeId)) continue;
+        const volume = yield* getVolumeById(input.appName, volumeId);
+        if (volume !== undefined) volumesById.set(volumeId, volume);
+      }
+    }
+    const replicas = listed.map((machine) => toReplica(machine, volumesById));
+    return {
+      ...toReplicaSet(
+        replicas,
+        input.appName,
+        listed[0]?.config?.metadata?.[alchemyMetadataKeys.baseName] ??
+          input.baseName ??
+          replicas[0]?.name ??
+          "",
+        listed[0]?.config?.services,
+      ),
+      rolloutPending,
+    };
+  }
+  if (owned.length)
+    return {
+      ...toReplicaSet([], input.appName, input.baseName ?? ""),
+      region: owned[0]?.region ?? "",
+      regions: [
+        ...new Set(
+          owned.map((machine) => machine.region ?? "").filter((region) => region.length > 0),
+        ),
+      ],
+      rolloutPending: true,
+    };
+  return undefined;
+});