@alchemy.run/fly 0.0.0 → 2.0.0-beta.85

package/lib/replicas.js

@@ -0,0 +1,969 @@
+import * as machines from "@distilled.cloud/fly-io/machines";
+import * as Retry from "@distilled.cloud/fly-io/Retry";
+import { deepEqual } from "alchemy/Diff";
+import * as Data from "effect/Data";
+import * as Effect from "effect/Effect";
+import * as Result from "effect/Result";
+import * as Schedule from "effect/Schedule";
+import { listOwnedApps } from "./App.js";
+import { reconcileBlueGreen, setRouting } from "./bluegreen.js";
+import { validateDeployment } from "./Deployment.js";
+import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.js";
+import { usingMachineLeases } from "./leases.js";
+import { alchemyMetadataKeys, createMachineMetadata, isAlchemyOwnedMetadata, sanitizeFlyAppName, } from "./Metadata.js";
+import { regionOfReplica } from "./Region.js";
+import { deleteVolume, ensureVolumeGroup, getVolumeById, volumeGroupName } from "./Volume.js";
+const WAIT_TIMEOUT_SECONDS = 8;
+const waitBackoff = Schedule.exponential("500 millis");
+const SERVICE_CHECK_NAME_PREFIX = "servicecheck-";
+export class ReplicaNotCreated extends Data.TaggedError("Fly.ReplicaNotCreated") {
+}
+export class ReplicaChecksNotPassing extends Data.TaggedError("Fly.ReplicaChecksNotPassing") {
+    get message() {
+        const checks = this.checks.map((check) => `${check.name ?? "unnamed"}: ${check.status ?? "unknown"}${check.output ? ` (${check.output})` : ""}`);
+        return `Service checks did not pass for ${this.appName}/${this.machineId}: ${checks.join("; ") || "no service check results"}`;
+    }
+}
+const compactRecord = (record) => Object.fromEntries(Object.entries(record ?? {}).flatMap(([key, value]) => value === undefined ? [] : [[key, value]]));
+export const gone = (machine) => machine === undefined || machine.state === "destroyed";
+export const getMachineById = (appName, machineId) => machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(Effect.map((machine) => (gone(machine) ? undefined : machine)), Effect.catchTag("NotFound", () => Effect.undefined));
+/**
+ * Reads a Machine we hold a lease on. Fly's Machines API is not
+ * read-after-write consistent: a freshly created Machine can accept a lease
+ * (201) and then answer `GET /machines/{id}` with 404 for several seconds.
+ * Holding the lease blocks other writers, so a 404 here is lag rather than
+ * deletion; retry it (bounded) before surfacing `NotFound`.
+ */
+const getLeasedMachine = (appName, machineId) => machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(Effect.retry({
+    times: 10,
+    schedule: Schedule.spaced("1 second"),
+    while: (error) => error._tag === "NotFound",
+}));
+const getLeasedMachineById = (appName, machineId) => getLeasedMachine(appName, machineId).pipe(Effect.map((machine) => (gone(machine) ? undefined : machine)), Effect.catchTag("NotFound", () => Effect.undefined));
+export const listMachinesByApp = (appName) => machines.listMachines({ app_name: appName }).pipe(
+// A just-created App can answer 500 "failed to get app: sql: no rows in
+// result set" until the Machines API sees it; safe to replay a read.
+Effect.retry({
+    while: (error) => error._tag === "InternalServerError",
+    schedule: Schedule.spaced("2 seconds"),
+    times: 8,
+}), Effect.map((machines) => machines.filter((machine) => !gone(machine))), Effect.catchTag("NotFound", () => Effect.succeed([])));
+export const resolveCount = (count) => Math.max(1, Math.floor(count ?? 1));
+export const replicaMachineName = (base, index, count) => {
+    if (count <= 1 && index === 0)
+        return base;
+    const suffix = `-${index}`;
+    const room = 30 - suffix.length;
+    const clipped = base.slice(0, Math.max(1, room)).replace(/-+$/g, "");
+    return sanitizeFlyAppName(`${clipped}${suffix}`);
+};
+export const replicaIndexOf = (machine) => {
+    const raw = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.replica];
+    const parsed = raw === undefined ? 0 : Number(raw);
+    return Number.isFinite(parsed) ? parsed : 0;
+};
+export const alchemyIdOf = (machine) => {
+    const id = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.id];
+    return id !== undefined && id.length > 0 ? id : undefined;
+};
+export const isOwnedType = (machine, type) => {
+    const metadata = compactRecord(machine.config?.metadata);
+    return isAlchemyOwnedMetadata(metadata) && metadata[alchemyMetadataKeys.type] === type;
+};
+export const toImageRef = (ref) => {
+    if (ref === undefined)
+        return undefined;
+    const imageRef = {
+        registry: ref.registry,
+        repository: ref.repository,
+        tag: ref.tag,
+        digest: ref.digest,
+    };
+    return imageRef.registry === undefined &&
+        imageRef.repository === undefined &&
+        imageRef.tag === undefined &&
+        imageRef.digest === undefined
+        ? undefined
+        : imageRef;
+};
+export const toGuestAttrs = (guest) => {
+    if (guest === undefined)
+        return undefined;
+    return {
+        cpuKind: guest.cpu_kind,
+        cpus: guest.cpus,
+        memoryMb: guest.memory_mb,
+        gpuKind: guest.gpu_kind,
+        gpus: guest.gpus,
+    };
+};
+export const toFlyServiceCheck = (check) => ({
+    type: check.type,
+    port: check.port,
+    interval: check.interval,
+    timeout: check.timeout,
+    grace_period: check.gracePeriod,
+    method: check.method,
+    path: check.path,
+    protocol: check.protocol,
+    headers: check.headers?.map((header) => ({ name: header.name, values: header.values })),
+    tls_server_name: check.tlsServerName,
+    tls_skip_verify: check.tlsSkipVerify,
+});
+export const toFlyService = (service) => ({
+    protocol: service.protocol,
+    internal_port: service.internalPort,
+    autostart: service.autostart,
+    autostop: typeof service.autostop === "boolean" ? (service.autostop ? "stop" : "off") : service.autostop,
+    min_machines_running: service.minMachinesRunning,
+    ports: service.ports?.map((port) => ({
+        port: port.port,
+        handlers: port.handlers,
+        force_https: port.forceHttps,
+        start_port: port.startPort,
+        end_port: port.endPort,
+    })),
+    checks: service.checks?.map((check) => ({
+        ...toFlyServiceCheck(check),
+        // Fly clamps service-check intervals; named Machine checks retain longer intervals.
+        interval: (durationNanoseconds(check.interval) ?? 0n) > 60000000000n ? "60s" : check.interval,
+    })),
+});
+export const autostopMode = (value) => value === true ? "stop" : value === false || value === undefined ? "off" : value;
+const normalizedCheckDuration = (value) => {
+    const nanos = durationNanoseconds(value);
+    return nanos === undefined ? value : `${nanos}ns`;
+};
+const normalizedCheck = (check) => check === undefined
+    ? undefined
+    : {
+        ...check,
+        grace_period: normalizedCheckDuration(check.grace_period),
+        interval: normalizedCheckDuration(check.interval),
+        timeout: normalizedCheckDuration(check.timeout),
+    };
+export const sameChecks = (observed, desired) => {
+    const normalize = (checks) => Object.fromEntries(Object.entries(checks ?? {}).map(([name, check]) => [name, normalizedCheck(check)]));
+    return deepEqual(normalize(observed), normalize(desired), { stripNullish: true });
+};
+export const normalizedServices = (services) => (services ?? []).map((service) => ({
+    ...service,
+    autostop: autostopMode(service.autostop),
+    checks: service.checks?.map(normalizedCheck),
+}));
+export const sameServices = (observed, desired) => deepEqual(normalizedServices(observed), normalizedServices(desired), { stripNullish: true });
+export const hasPublishedService = (services) => (services ?? []).some((service) => (service.ports ?? []).some((port) => port.port !== undefined || port.start_port !== undefined));
+export const waitStarted = (appName, machineId) => machines
+    .waitMachine({
+    app_name: appName,
+    machine_id: machineId,
+    state: "started",
+    timeout: WAIT_TIMEOUT_SECONDS,
+})
+    .pipe(Retry.none, Effect.retry({
+    times: 6,
+    schedule: waitBackoff,
+    while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
+}), Effect.timeout("50 seconds"));
+export const configuredCheckNames = (config) => [
+    ...Object.keys(config?.checks ?? {}),
+    ...(config?.services ?? []).flatMap((service) => (service.checks ?? []).map((check, checkIndex) => `${SERVICE_CHECK_NAME_PREFIX}${String(checkIndex).padStart(2, "0")}-${check.type ?? "tcp"}-${service.internal_port}`)),
+];
+export const checksPassing = (machine, config) => {
+    const expected = configuredCheckNames(config);
+    const checks = machine.checks ?? [];
+    // Fly may also report compatibility mirrors; they do not replace service reports.
+    return (machine.state === "started" &&
+        new Set(expected).size === expected.length &&
+        checks.every((check) => check.name !== undefined &&
+            check.status === "passing" &&
+            (expected.includes(check.name) ||
+                expected.some((name) => name.startsWith(SERVICE_CHECK_NAME_PREFIX) &&
+                    check.name === name.replace(SERVICE_CHECK_NAME_PREFIX, "bg_deployments_compat-")))) &&
+        new Set(checks.map((check) => check.name)).size === checks.length &&
+        expected.every((name) => {
+            const reports = checks.filter((check) => check.name === name);
+            return reports.length === 1 && reports[0]?.status === "passing";
+        }));
+};
+const TRANSIENT_GET_TAGS = [
+    "TooManyRequests",
+    "InternalServerError",
+    "BadGateway",
+    "ServiceUnavailable",
+    "GatewayTimeout",
+];
+/**
+ * After the Machine is started, wait until Fly reports every service
+ * check as passing. No configured checks is a no-op. Empty live
+ * `servicecheck-*` results keep polling — they are not success.
+ * `warning` / `unknown` during grace keep polling.
+ */
+export const waitHealthy = Effect.fn(function* (appName, machine, healthTimeoutMs = 60_000, config = machine.config) {
+    const machineId = machine.id;
+    const named = Object.keys(config?.checks ?? {});
+    const expected = (config?.services ?? []).flatMap((service) => service.checks ?? []).length;
+    if (machineId === undefined || (expected === 0 && named.length === 0))
+        return machine;
+    let observed = machine;
+    const notPassing = () => new ReplicaChecksNotPassing({
+        appName,
+        machineId,
+        checks: (observed.checks ?? []).map((check) => ({
+            name: check.name,
+            status: check.status,
+            output: undefined,
+        })),
+    });
+    const passing = yield* getMachineById(appName, machineId).pipe(Retry.none, Effect.map((current) => {
+        if (current === undefined)
+            return false;
+        observed = current;
+        return (current.instance_id !== undefined &&
+            current.instance_id === machine.instance_id &&
+            checksPassing(current, config));
+    }), Effect.catchTag(TRANSIENT_GET_TAGS, () => Effect.succeed(false)), Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)), Effect.repeat({
+        schedule: Schedule.spaced(healthTimeoutMs / 10),
+        until: (passing) => passing,
+        times: 10,
+    }), Effect.timeoutOrElse({ duration: healthTimeoutMs, orElse: () => Effect.fail(notPassing()) }));
+    if (!passing)
+        return yield* notPassing();
+    return observed;
+});
+export const waitDestroyed = (appName, machineId) => machines
+    .waitMachine({
+    app_name: appName,
+    machine_id: machineId,
+    state: "destroyed",
+    timeout: WAIT_TIMEOUT_SECONDS,
+})
+    .pipe(Retry.none, Effect.as(undefined), Effect.catchTag("NotFound", () => Effect.void), Effect.retry({
+    times: 6,
+    schedule: waitBackoff,
+    while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
+}), Effect.timeout("50 seconds"));
+export const ensureStarted = (appName, machine, skipLaunch, healthTimeoutMs = 60_000, expectedConfig, existingLeases) => usingMachineLeases(appName, existingLeases, (leases) => ensureLeasedStarted(appName, machine, skipLaunch, healthTimeoutMs, expectedConfig, leases));
+const ensureLeasedStarted = Effect.fn(function* (appName, machine, skipLaunch, healthTimeoutMs, expectedConfig, leases) {
+    const machineId = machine.id;
+    if (machineId === undefined || skipLaunch)
+        return machine;
+    yield* leases.acquire([machineId]);
+    const started = yield* Effect.gen(function* () {
+        // Create/update responses can lag Fly's automatic launch.
+        const current = yield* getLeasedMachine(appName, machineId);
+        if (!sameOwnership(current, machine)) {
+            return yield* new ReplicaOwnershipChanged({ appName, machineId });
+        }
+        yield* Effect.logDebug("Fly machine startup", { appName, machineId, state: current.state });
+        if (current.state === "stopped" ||
+            current.state === "suspended" ||
+            current.state === "failed") {
+            yield* leases.mutate(machineId, (lease_nonce) => machines.startMachine({ app_name: appName, machine_id: machineId, lease_nonce }));
+        }
+        // Re-observe state between waits instead of retrying the wait in the SDK.
+        yield* machines
+            .waitMachine({
+            app_name: appName,
+            machine_id: machineId,
+            state: "started",
+            instance_id: current.instance_id,
+            timeout: WAIT_TIMEOUT_SECONDS,
+        })
+            .pipe(Retry.none);
+        return yield* getLeasedMachine(appName, machineId);
+    }).pipe(Effect.retry({
+        times: 6,
+        schedule: waitBackoff,
+        while: (error) => error._tag === "MachineStartFromCreatedState" ||
+            error._tag === "MachineReplacing" ||
+            error._tag === "MachineWaitTimeout" ||
+            error._tag === "Conflict" ||
+            error._tag === "GatewayTimeout",
+    }), Effect.timeout("180 seconds"));
+    return yield* waitHealthy(appName, started, healthTimeoutMs, expectedConfig ?? machine.config);
+});
+export const deleteMachine = (appName, machineId, existingLeases) => usingMachineLeases(appName, existingLeases, (leases) => Effect.gen(function* () {
+    if (appName.length === 0 || machineId.length === 0)
+        return;
+    yield* leases.acquire([machineId]);
+    yield* leases.remove(machineId, (lease_nonce) => machines
+        .deleteMachine({ app_name: appName, machine_id: machineId, force: true, lease_nonce })
+        .pipe(Effect.timeout("30 seconds")));
+    yield* waitDestroyed(appName, machineId);
+}).pipe(Effect.catchTag("NotFound", () => leases.forget(machineId))));
+const durationNanoseconds = (value) => {
+    if (value === undefined || value === "")
+        return undefined;
+    const negative = value.startsWith("-");
+    const duration = /^[+-]/.test(value) ? value.slice(1) : value;
+    if (duration === "0")
+        return 0n;
+    const limit = negative ? 1n << 63n : (1n << 63n) - 1n;
+    const units = {
+        ns: 1n,
+        us: 1000n,
+        µs: 1000n,
+        μs: 1000n,
+        ms: 1000000n,
+        s: 1000000000n,
+        m: 60000000000n,
+        h: 3600000000000n,
+    };
+    let total = 0n;
+    let consumed = 0;
+    for (const part of duration.matchAll(/(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|μs|ms|s|m|h)/gy)) {
+        const [integer = "", fraction = ""] = part[1].split(".");
+        const whole = integer.replace(/^0+/, "") || "0";
+        if (whole.length > 19)
+            return undefined;
+        const unit = units[part[2]];
+        let fractionalNanos = 0n;
+        // Truncate each component to nanoseconds without unbounded BigInt operands.
+        for (let index = fraction.length - 1; index >= 0; index--) {
+            fractionalNanos = (BigInt(fraction[index]) * unit + fractionalNanos) / 10n;
+        }
+        total += BigInt(whole) * unit + fractionalNanos;
+        if (total > limit)
+            return undefined;
+        consumed += part[0].length;
+    }
+    if (consumed === 0 || consumed !== duration.length)
+        return undefined;
+    return negative ? -total : total;
+};
+const stopTimeoutMillis = (timeout) => {
+    if (timeout === "")
+        return 0;
+    // Shutdown policies retain their unsigned millisecond-or-larger syntax.
+    if (timeout === undefined || !/^(?:\d+(?:\.\d+)?(?:ms|s|m|h))+$/.test(timeout)) {
+        return undefined;
+    }
+    const nanos = durationNanoseconds(timeout);
+    return nanos === undefined ? undefined : Number(nanos) / 1_000_000;
+};
+export const sameStopConfig = (observed, desired) => observed?.signal === desired?.signal &&
+    stopTimeoutMillis(observed?.timeout) === stopTimeoutMillis(desired?.timeout);
+export class ShutdownPolicyMismatch extends Data.TaggedError("Fly.ShutdownPolicyMismatch") {
+}
+export const predecessorShutdown = Effect.fn(function* (machine) {
+    const persisted = machine.config?.stop_config;
+    const injected = machine.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS;
+    const timeoutMs = stopTimeoutMillis(persisted?.timeout);
+    if (injected !== undefined) {
+        const managedTimeout = Number(injected);
+        const signal = persisted?.signal ?? "SIGTERM";
+        if (!/^\d+$/.test(injected) ||
+            !Number.isSafeInteger(managedTimeout) ||
+            managedTimeout <= 0 ||
+            managedTimeout > 300_000 ||
+            (signal !== "SIGTERM" && signal !== "SIGINT") ||
+            (persisted?.timeout !== undefined && timeoutMs !== managedTimeout)) {
+            return yield* new ShutdownPolicyMismatch({
+                machineId: machine.id ?? "",
+                message: "The predecessor's managed shutdown environment and stop_config disagree; reconcile its shutdown policy before migration.",
+            });
+        }
+        return {
+            signal,
+            timeout: persisted?.timeout ?? `${managedTimeout}ms`,
+            timeoutMs: managedTimeout,
+        };
+    }
+    if (persisted?.timeout !== undefined && (!timeoutMs || timeoutMs > 300_000)) {
+        return yield* new ShutdownPolicyMismatch({
+            machineId: machine.id ?? "",
+            message: "The predecessor has an invalid stop_config timeout; refusing to shorten its shutdown window.",
+        });
+    }
+    // Unspecified raw-image overrides must remain unspecified for Fly's defaults.
+    return {
+        signal: persisted?.signal,
+        timeout: persisted?.timeout,
+        timeoutMs: timeoutMs ?? 300_000,
+    };
+});
+export class ReplicaOwnershipChanged extends Data.TaggedError("Fly.ReplicaOwnershipChanged") {
+}
+export class ReplicaRetirementIncomplete extends Data.TaggedError("Fly.ReplicaRetirementIncomplete") {
+}
+const retirementStep = (appName, machineId, stage) => (effect) => effect.pipe(Effect.mapError((error) => error._tag === "NotFound"
+    ? error
+    : new ReplicaRetirementIncomplete({
+        appName,
+        residuals: [{ machineId, stage: `${stage}: ${error._tag}` }],
+    })));
+const sameOwnership = (observed, snapshot) => observed.id === snapshot.id &&
+    [
+        alchemyMetadataKeys.stack,
+        alchemyMetadataKeys.stage,
+        alchemyMetadataKeys.id,
+        alchemyMetadataKeys.type,
+        alchemyMetadataKeys.instance,
+        alchemyMetadataKeys.fqn,
+        alchemyMetadataKeys.generation,
+    ].every((key) => observed.config?.metadata?.[key] === snapshot.config?.metadata?.[key]);
+export const leaseSnapshot = Effect.fn(function* (appName, snapshot, leases) {
+    const ordered = [...snapshot].sort((a, b) => (a.id ?? "").localeCompare(b.id ?? ""));
+    const leased = new Set();
+    yield* Effect.forEach(ordered, (machine) => Effect.gen(function* () {
+        if (!machine.id || machine.host_status === "unreachable")
+            return;
+        const id = machine.id;
+        yield* leases.acquire([id]).pipe(Effect.tap(() => Effect.sync(() => leased.add(id))), 
+        // A just-created Machine can 404 on the lease API while it already
+        // lists; a Machine that is really gone is dropped by the GET below.
+        Effect.retry({
+            while: (error) => error._tag === "NotFound",
+            schedule: Schedule.spaced("1 second"),
+            times: 5,
+        }), Effect.catchTag("NotFound", () => Effect.void));
+    }), { concurrency: 1 }).pipe(Effect.timeout("30 seconds"));
+    const observed = [];
+    for (const machine of ordered) {
+        if (!machine.id)
+            continue;
+        if (machine.host_status === "unreachable") {
+            observed.push(machine);
+            continue;
+        }
+        const current = yield* leased.has(machine.id)
+            ? getLeasedMachineById(appName, machine.id)
+            : getMachineById(appName, machine.id);
+        if (!current) {
+            yield* leases.forget(machine.id);
+            continue;
+        }
+        if (!sameOwnership(current, machine))
+            return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
+        yield* leases.checkTarget(machine.id);
+        observed.push(current);
+    }
+    yield* leases.check;
+    return observed;
+});
+export const retireMachines = (appName, snapshot, replacementReady = false, existingLeases) => usingMachineLeases(appName, existingLeases, (leases) => retireLeasedMachines(appName, snapshot, replacementReady, leases));
+const retireLeasedMachines = Effect.fn(function* (appName, snapshot, replacementReady, leases) {
+    const targets = yield* leaseSnapshot(appName, snapshot, leases);
+    const results = yield* Effect.forEach(targets, (machine) => Effect.gen(function* () {
+        yield* leases.check;
+        const result = yield* Effect.gen(function* () {
+            if (!machine.id)
+                return;
+            if (replacementReady && machine.host_status === "unreachable") {
+                const metadata = machine.config?.metadata;
+                if (!machine.config ||
+                    machine.incomplete_config ||
+                    (machine.config.mounts?.length ?? 0) > 0 ||
+                    !metadata?.[alchemyMetadataKeys.instance] ||
+                    !metadata[alchemyMetadataKeys.fqn]) {
+                    return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
+                }
+                yield* Effect.logWarning("Force-retiring owned stateless Fly Machine on unreachable host; work was not proven drained", { appName, machineId: machine.id });
+                const machineId = machine.id;
+                const heldNonce = yield* leases.nonceIfHeld(machineId);
+                yield* Effect.gen(function* () {
+                    const remove = (lease_nonce) => machines
+                        .deleteMachine({
+                        app_name: appName,
+                        machine_id: machineId,
+                        force: true,
+                        lease_nonce,
+                    })
+                        .pipe(Retry.none, Effect.timeout("30 seconds"));
+                    if (heldNonce !== undefined)
+                        yield* leases.remove(machineId, remove);
+                    else
+                        yield* leases.guard(remove());
+                }).pipe(Effect.catchTag("NotFound", () => Effect.void), retirementStep(appName, machineId, "unreachable force-delete"));
+                yield* leases.forget(machine.id);
+                return yield* waitDestroyed(appName, machine.id);
+            }
+            yield* retireMachine(appName, machine.id, machine, leases);
+        }).pipe(Effect.result);
+        yield* leases.check;
+        return Result.isFailure(result)
+            ? result.failure._tag === "Fly.ReplicaRetirementIncomplete"
+                ? result.failure.residuals
+                : [{ machineId: machine.id ?? "", stage: result.failure._tag }]
+            : [];
+    }), { concurrency: 4 });
+    const residuals = results.flat();
+    if (residuals.length)
+        return yield* new ReplicaRetirementIncomplete({ appName, residuals });
+});
+export const retireMachine = (appName, machineId, snapshot, existingLeases) => usingMachineLeases(appName, existingLeases, (leases) => retireLeasedMachine(appName, machineId, snapshot, leases).pipe(Effect.tap(() => leases.forget(machineId))));
+const retireLeasedMachine = Effect.fn(function* (appName, machineId, snapshot, leases) {
+    yield* leases.acquire([machineId]);
+    const current = yield* getMachineById(appName, machineId);
+    if (current === undefined)
+        return;
+    if (snapshot && !sameOwnership(current, snapshot)) {
+        return yield* new ReplicaOwnershipChanged({ appName, machineId });
+    }
+    const stop = yield* predecessorShutdown(current);
+    if (current.cordoned === true &&
+        current.config?.metadata?.[alchemyMetadataKeys.phase] === "candidate")
+        return yield* deleteMachine(appName, machineId, leases);
+    if (current.state !== "stopped" && current.state !== "created") {
+        yield* leases
+            .mutate(machineId, (lease_nonce) => machines
+            .cordonMachine({ app_name: appName, machine_id: machineId, lease_nonce })
+            .pipe(Effect.timeout("30 seconds")), { idempotent: true })
+            .pipe(retirementStep(appName, machineId, "cordon"));
+        if (hasPublishedService(current.config?.services))
+            yield* Effect.sleep("10 seconds");
+        yield* leases
+            .mutate(machineId, (lease_nonce) => machines
+            .stopMachine({
+            app_name: appName,
+            machine_id: machineId,
+            lease_nonce,
+            signal: stop.signal,
+            timeout: stop.timeout,
+        })
+            .pipe(Effect.timeout(stop.timeoutMs + 15_000)), { timeoutMs: stop.timeoutMs + 120_000 })
+            .pipe(retirementStep(appName, machineId, "stop"));
+        yield* machines
+            .waitMachine({
+            app_name: appName,
+            machine_id: machineId,
+            state: "stopped",
+            instance_id: current.instance_id,
+            timeout: WAIT_TIMEOUT_SECONDS,
+        })
+            .pipe(Retry.none, Effect.retry({
+            times: 10,
+            schedule: Schedule.spaced(Math.max(500, stop.timeoutMs / 10)),
+            while: (error) => error._tag === "MachineWaitTimeout" || error._tag === "GatewayTimeout",
+        }), Effect.timeout(stop.timeoutMs + 15_000), retirementStep(appName, machineId, "wait-stopped"));
+    }
+    yield* leases
+        .remove(machineId, (lease_nonce) => machines
+        .deleteMachine({ app_name: appName, machine_id: machineId, force: false, lease_nonce })
+        .pipe(Effect.timeout("30 seconds")))
+        .pipe(Effect.catchTag("NotFound", () => Effect.void), retirementStep(appName, machineId, "delete"));
+    yield* waitDestroyed(appName, machineId).pipe(retirementStep(appName, machineId, "verify-absent"));
+}, Effect.catchTag("NotFound", () => Effect.void));
+const mountedDisksOf = (machine, volumesById) => (machine.config?.mounts ?? []).flatMap((mount) => {
+    const volumeId = mount.volume;
+    const path = mount.path;
+    if (volumeId === undefined || path === undefined)
+        return [];
+    const volume = volumesById.get(volumeId);
+    return [{ path, volumeId, sizeGb: volume?.size_gb ?? 0, name: volume?.name ?? "" }];
+});
+export const toReplica = (machine, volumesById) => ({
+    machineId: machine.id ?? "",
+    name: machine.name ?? "",
+    region: machine.region ?? "",
+    state: machine.state ?? "",
+    instanceId: machine.instance_id,
+    privateIp: machine.private_ip,
+    imageRef: toImageRef(machine.image_ref),
+    guest: toGuestAttrs(machine.config?.guest),
+    mounts: mountedDisksOf(machine, volumesById),
+});
+export const toReplicaSet = (replicas, appName, baseName, services) => {
+    const primary = replicas[0];
+    return {
+        appName,
+        machineId: primary?.machineId ?? "",
+        machineIds: replicas.map((replica) => replica.machineId),
+        name: primary?.name ?? baseName,
+        baseName,
+        region: primary?.region ?? "",
+        regions: [
+            ...new Set(replicas.map((replica) => replica.region).filter((region) => region.length > 0)),
+        ],
+        state: primary?.state ?? "",
+        instanceId: primary?.instanceId,
+        privateIp: primary?.privateIp,
+        imageRef: primary?.imageRef,
+        guest: primary?.guest,
+        services,
+        count: replicas.length,
+        mounts: primary?.mounts ?? [],
+        replicas,
+    };
+};
+export const ownedReplicas = (listed, input) => listed.filter((machine) => {
+    const metadata = machine.config?.metadata ?? {};
+    if (!Object.entries(input.metadata).every(([key, value]) => metadata[key] === value))
+        return false;
+    if (metadata[alchemyMetadataKeys.instance] !== undefined) {
+        return (metadata[alchemyMetadataKeys.instance] === input.resourceInstanceId &&
+            metadata[alchemyMetadataKeys.fqn] === input.fqn);
+    }
+    return ((machine.id !== undefined && input.machineIds?.includes(machine.id)) ||
+        (input.baseName !== undefined &&
+            (machine.name === input.baseName ||
+                machine.name === replicaMachineName(input.baseName, replicaIndexOf(machine), 2))));
+});
+export const listReplicas = Effect.fn(function* (input) {
+    const machines = yield* listMachinesByApp(input.appName);
+    return machines
+        .filter((machine) => isOwnedType(machine, input.type) && alchemyIdOf(machine) === input.id)
+        .sort((left, right) => replicaIndexOf(left) - replicaIndexOf(right));
+});
+export const listReplicaSets = Effect.fn(function* (type) {
+    const apps = yield* listOwnedApps();
+    const groups = yield* Effect.forEach(apps, (app) => listMachinesByApp(app.appName).pipe(Effect.map((machines) => {
+        const owned = machines.filter((machine) => isOwnedType(machine, type));
+        const byId = new Map();
+        for (const machine of owned) {
+            const id = alchemyIdOf(machine);
+            if (id === undefined)
+                continue;
+            const metadata = machine.config?.metadata;
+            const key = JSON.stringify([
+                metadata?.[alchemyMetadataKeys.stack],
+                metadata?.[alchemyMetadataKeys.stage],
+                metadata?.[alchemyMetadataKeys.fqn] ?? id,
+                metadata?.[alchemyMetadataKeys.instance],
+                metadata?.[alchemyMetadataKeys.generation],
+            ]);
+            const group = byId.get(key) ?? [];
+            group.push(machine);
+            byId.set(key, group);
+        }
+        return [...byId.values()].map((group) => {
+            const sorted = [...group].sort((left, right) => replicaIndexOf(left) - replicaIndexOf(right));
+            const replicas = sorted.map((machine) => toReplica(machine, new Map()));
+            return toReplicaSet(replicas, app.appName, replicas[0]?.name ?? "", sorted[0]?.config?.services);
+        });
+    })), { concurrency: 8 });
+    return groups.flat();
+});
+const pickVolume = (group, used, preferId) => {
+    if (preferId !== undefined && !used.has(preferId)) {
+        const preferred = group.find((volume) => volume.id === preferId);
+        if (preferred !== undefined)
+            return preferred;
+    }
+    return group.find((volume) => volume.id !== undefined && !used.has(volume.id));
+};
+export const reconcileReplicas = (input) => usingMachineLeases(input.appName, undefined, (leases) => reconcileLeasedReplicas(input, leases));
+const reconcileLeasedReplicas = Effect.fn(function* (input, leases) {
+    const ownership = yield* createMachineMetadata(input.id, input.type);
+    const alchemy = {
+        ...ownership,
+        [alchemyMetadataKeys.instance]: input.resourceInstanceId,
+        [alchemyMetadataKeys.fqn]: input.fqn,
+        [alchemyMetadataKeys.baseName]: input.baseName,
+    };
+    const buildConfig = (replica) => ({
+        ...input.buildConfig(replica),
+        checks: input.checks === undefined
+            ? undefined
+            : Object.fromEntries(Object.entries(input.checks).map(([name, check]) => [name, toFlyServiceCheck(check)])),
+        stop_config: input.policy.shutdown === undefined
+            ? undefined
+            : { signal: input.policy.shutdown.signal, timeout: input.policy.shutdown.timeout },
+    });
+    const config = buildConfig({ index: 0, mounts: [], metadata: alchemy });
+    yield* validateDeployment(input.policy, config, input.disks.length > 0, input.skipLaunch);
+    if (input.policy.bluegreen)
+        return yield* reconcileBlueGreen({ ...input, buildConfig }, ownership, alchemy, leases);
+    const desiredNames = new Set(Array.from({ length: input.count }, (_, index) => replicaMachineName(input.baseName, index, input.count)));
+    const listed = yield* listMachinesByApp(input.appName);
+    const owned = yield* leaseSnapshot(input.appName, ownedReplicas(listed, { ...input, metadata: ownership, machineIds: input.outputMachineIds }), leases);
+    const byIndex = new Map();
+    const preferIds = new Set((input.outputMachineIds ?? []).filter((id) => id.length > 0));
+    for (const machine of owned) {
+        const id = machine.id;
+        if (id !== undefined && preferIds.has(id)) {
+            byIndex.set(replicaIndexOf(machine), machine);
+        }
+    }
+    for (const machine of owned) {
+        const name = machine.name;
+        if (name === undefined ||
+            (!desiredNames.has(name) &&
+                machine.config?.metadata?.[alchemyMetadataKeys.instance] !== input.resourceInstanceId))
+            continue;
+        const index = replicaIndexOf(machine);
+        if (!byIndex.has(index))
+            byIndex.set(index, machine);
+    }
+    const scaleDown = [...byIndex].filter(([index]) => index >= input.count);
+    yield* retireMachines(input.appName, scaleDown.map(([, machine]) => machine), false, leases);
+    for (const [index] of scaleDown)
+        byIndex.delete(index);
+    // Volumes are regional: one group per disk and region, sized to the
+    // replicas that run there.
+    const replicasIn = (region) => Array.from({ length: input.count }, (_, index) => index).filter((index) => regionOfReplica(input.regions, index) === region).length;
+    const groups = [];
+    for (const [diskIndex, disk] of input.disks.entries()) {
+        const name = yield* volumeGroupName(input.id, disk);
+        const preferIds = (input.preferVolumeIds ?? [])
+            .map((replica) => replica[diskIndex])
+            .filter((id) => id !== undefined && id.length > 0);
+        const byRegion = new Map();
+        for (const region of input.regions) {
+            byRegion.set(region, yield* ensureVolumeGroup({
+                appName: input.appName,
+                name,
+                region,
+                count: replicasIn(region),
+                disk,
+                preferIds,
+            }));
+        }
+        groups.push({ disk, name, byRegion });
+    }
+    const usedVolumeIds = new Set();
+    const live = [];
+    for (let index = 0; index < input.count; index++) {
+        const name = replicaMachineName(input.baseName, index, input.count);
+        const region = regionOfReplica(input.regions, index);
+        const metadata = {
+            ...alchemy,
+            [alchemyMetadataKeys.replica]: String(index),
+            ...(input.minSecretsVersion === undefined
+                ? {}
+                : { [alchemyMetadataKeys.secretsVersion]: String(input.minSecretsVersion) }),
+        };
+        const prefer = input.preferVolumeIds?.[index] ?? [];
+        const mounts = [];
+        for (const [diskIndex, group] of groups.entries()) {
+            const volume = pickVolume(group.byRegion.get(region)?.volumes ?? [], usedVolumeIds, prefer[diskIndex]);
+            const volumeId = volume?.id;
+            if (volumeId === undefined) {
+                return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
+            }
+            usedVolumeIds.add(volumeId);
+            mounts.push({ volume: volumeId, path: group.disk.path });
+        }
+        const config = buildConfig({ index, mounts, metadata });
+        let current = byIndex.get(index);
+        // Fly cannot move a Machine between regions. Machine names are unique
+        // per App, so retire the old replica before creating it in its region.
+        if (current !== undefined && current.region !== region) {
+            yield* retireMachines(input.appName, [current], false, leases);
+            byIndex.delete(index);
+            current = undefined;
+        }
+        if (current === undefined) {
+            const created = yield* machines
+                .createMachine({
+                app_name: input.appName,
+                name,
+                region,
+                config,
+                skip_launch: input.skipLaunch === true ? true : undefined,
+                min_secrets_version: input.minSecretsVersion,
+            })
+                .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
+            current =
+                created ??
+                    (yield* listMachinesByApp(input.appName).pipe(Effect.map((machines) => ownedReplicas(machines, {
+                        ...input,
+                        metadata: ownership,
+                        machineIds: input.outputMachineIds,
+                    }).find((machine) => machine.name === name))));
+            if (current === undefined || current.id === undefined) {
+                return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
+            }
+            const observed = (yield* leaseSnapshot(input.appName, [current], leases))[0];
+            if (!observed || input.configDrifted(observed, { mounts, metadata })) {
+                return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
+            }
+            current = observed;
+        }
+        else if (!sameChecks(current.config?.checks, config.checks) ||
+            !sameStopConfig(current.config?.stop_config, config.stop_config) ||
+            input.configDrifted(current, { mounts, metadata })) {
+            const machineId = current.id;
+            const currentVersion = current.instance_id;
+            const previous = current;
+            const expected = { ...previous, config };
+            yield* leases.mutate(machineId, (lease_nonce) => machines
+                .updateMachine({
+                app_name: input.appName,
+                machine_id: machineId,
+                current_version: currentVersion,
+                config,
+                skip_launch: input.skipLaunch === true ? true : undefined,
+                min_secrets_version: input.minSecretsVersion,
+                lease_nonce,
+            })
+                .pipe(Effect.timeout("30 seconds")));
+            // Rolling opt-out removes generation metadata; observe that transition before startup.
+            current = yield* leases.guard(Effect.gen(function* () {
+                const observed = yield* machines
+                    .getMachine({ app_name: input.appName, machine_id: machineId })
+                    .pipe(Retry.none);
+                if (!sameOwnership(observed, expected) && !sameOwnership(observed, previous))
+                    return yield* new ReplicaOwnershipChanged({ appName: input.appName, machineId });
+                if (!sameOwnership(observed, expected) ||
+                    input.configDrifted(observed, { mounts, metadata }) ||
+                    !sameChecks(observed.config?.checks, config.checks) ||
+                    !sameStopConfig(observed.config?.stop_config, config.stop_config))
+                    return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
+                return observed;
+            }).pipe(Effect.retry({
+                times: 8,
+                schedule: waitBackoff,
+                while: (error) => error._tag === "Fly.ReplicaNotCreated" ||
+                    TRANSIENT_GET_TAGS.some((tag) => tag === error._tag),
+            }), Effect.timeout("60 seconds")));
+        }
+        current = yield* ensureStarted(input.appName, current, input.skipLaunch === true, input.policy.healthTimeoutMs, config, leases);
+        if (current.cordoned === true && !input.skipLaunch) {
+            yield* setRouting(input.appName, current.id, false, leases);
+            if (hasPublishedService(config.services))
+                yield* Effect.sleep("10 seconds");
+            current = yield* waitHealthy(input.appName, current, input.policy.healthTimeoutMs, config);
+        }
+        live.push(current);
+    }
+    const liveIds = new Set(live.map((machine) => machine.id));
+    yield* retireMachines(input.appName, owned.filter((machine) => !liveIds.has(machine.id)), false, leases);
+    for (const group of groups) {
+        for (const extra of [...group.byRegion.values()].flatMap((placed) => placed.extras)) {
+            const volumeId = extra.id;
+            if (volumeId === undefined || usedVolumeIds.has(volumeId))
+                continue;
+            yield* deleteVolume(input.appName, volumeId);
+        }
+    }
+    const volumesById = new Map();
+    for (const group of groups) {
+        for (const volume of [...group.byRegion.values()].flatMap((placed) => placed.volumes)) {
+            if (volume.id !== undefined)
+                volumesById.set(volume.id, volume);
+        }
+    }
+    const fresh = yield* Effect.forEach(live, (machine) => machine.id === undefined
+        ? Effect.succeed(machine)
+        : getMachineById(input.appName, machine.id).pipe(Effect.map((next) => next ?? machine)), { concurrency: 4 });
+    const replicas = fresh.map((machine) => toReplica(machine, volumesById));
+    return toReplicaSet(replicas, input.appName, input.baseName, fresh[0]?.config?.services);
+});
+export const deleteReplicaSet = Effect.fn(function* (input) {
+    return yield* usingMachineLeases(input.appName, undefined, (leases) => Effect.gen(function* () {
+        const owned = yield* leaseSnapshot(input.appName, ownedReplicas(yield* listMachinesByApp(input.appName), {
+            ...input,
+            metadata: yield* createMachineMetadata(input.id, input.type),
+        }), leases);
+        if (input.force) {
+            yield* Effect.forEach(owned, (machine) => Effect.gen(function* () {
+                if (!machine.id)
+                    return;
+                const current = yield* getMachineById(input.appName, machine.id);
+                if (!current)
+                    return;
+                if (!sameOwnership(current, machine))
+                    return yield* new ReplicaOwnershipChanged({
+                        appName: input.appName,
+                        machineId: machine.id,
+                    });
+                yield* deleteMachine(input.appName, machine.id, leases);
+            }), { concurrency: 4 });
+        }
+        else
+            yield* retireMachines(input.appName, owned, false, leases);
+        yield* Effect.forEach([...new Set(input.volumeIds)].filter((id) => id.length > 0), (volumeId) => deleteVolume(input.appName, volumeId), { concurrency: 4 });
+    }));
+});
+export const volumeIdsOf = (set) => {
+    const ids = new Set();
+    for (const mount of set.mounts ?? [])
+        ids.add(mount.volumeId);
+    for (const replica of set.replicas ?? []) {
+        for (const mount of replica.mounts)
+            ids.add(mount.volumeId);
+    }
+    return [...ids];
+};
+export const groupGenerations = (machines) => {
+    const groups = new Map();
+    for (const machine of machines) {
+        const generation = machine.config?.metadata?.[alchemyMetadataKeys.generation];
+        const group = groups.get(generation) ?? [];
+        group.push(machine);
+        groups.set(generation, group);
+    }
+    return groups;
+};
+export const observeReplicaSet = Effect.fn(function* (input) {
+    if (input.appName === undefined)
+        return undefined;
+    const owned = ownedReplicas(yield* listMachinesByApp(input.appName), {
+        ...input,
+        metadata: yield* createMachineMetadata(input.id, input.type),
+    });
+    const groups = groupGenerations(owned);
+    const committed = [...groups.entries()]
+        .filter(([generation, group]) => {
+        if (generation === undefined)
+            return false;
+        const count = Number(group[0]?.config?.metadata?.[alchemyMetadataKeys.count]);
+        if (group.some((machine) => classifyDeploymentState(machine).protocol === "invalid"))
+            return false;
+        const protocol2 = group.some((machine) => machine.config?.metadata?.[alchemyMetadataKeys.protocol] === "2");
+        if (protocol2 && !validProtocol2Generation(group))
+            return false;
+        return (Number.isSafeInteger(count) &&
+            count > 0 &&
+            group.length === count &&
+            new Set(group.map(replicaIndexOf)).size === count &&
+            group.every((machine) => {
+                const metadata = machine.config?.metadata;
+                const index = replicaIndexOf(machine);
+                return (index >= 0 &&
+                    index < count &&
+                    Number(metadata?.[alchemyMetadataKeys.count]) === count &&
+                    metadata?.[alchemyMetadataKeys.phase] === "active" &&
+                    machine.cordoned === false &&
+                    (metadata[alchemyMetadataKeys.protocol] === undefined ||
+                        ((metadata[alchemyMetadataKeys.protocol] === "1" ||
+                            metadata[alchemyMetadataKeys.protocol] === "2") &&
+                            metadata[alchemyMetadataKeys.restored] === "true" &&
+                            (metadata[alchemyMetadataKeys.role] === "idle" ||
+                                (machine.instance_id !== undefined &&
+                                    metadata[alchemyMetadataKeys.checkedInstance] === machine.instance_id)))));
+            }));
+    })
+        .sort(([, a], [, b]) => Number(b[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0) -
+        Number(a[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0));
+    const legacy = (groups.get(undefined) ?? []).filter((machine) => {
+        const { protocol } = classifyDeploymentState(machine);
+        return protocol === "legacy" || protocol === "1";
+    });
+    const listed = (committed[0]?.[1] ?? legacy).sort((a, b) => replicaIndexOf(a) - replicaIndexOf(b));
+    const rolloutPending = owned.some((machine) => !listed.includes(machine));
+    if (listed.length > 0) {
+        const volumesById = new Map();
+        for (const machine of listed) {
+            for (const mount of machine.config?.mounts ?? []) {
+                const volumeId = mount.volume;
+                if (volumeId === undefined || volumesById.has(volumeId))
+                    continue;
+                const volume = yield* getVolumeById(input.appName, volumeId);
+                if (volume !== undefined)
+                    volumesById.set(volumeId, volume);
+            }
+        }
+        const replicas = listed.map((machine) => toReplica(machine, volumesById));
+        return {
+            ...toReplicaSet(replicas, input.appName, listed[0]?.config?.metadata?.[alchemyMetadataKeys.baseName] ??
+                input.baseName ??
+                replicas[0]?.name ??
+                "", listed[0]?.config?.services),
+            rolloutPending,
+        };
+    }
+    if (owned.length)
+        return {
+            ...toReplicaSet([], input.appName, input.baseName ?? ""),
+            region: owned[0]?.region ?? "",
+            regions: [
+                ...new Set(owned.map((machine) => machine.region ?? "").filter((region) => region.length > 0)),
+            ],
+            rolloutPending: true,
+        };
+    return undefined;
+});
+//# sourceMappingURL=replicas.js.map