@alchemy.run/fly 0.0.0 → 2.0.0-beta.85

package/src/leases.ts

@@ -0,0 +1,437 @@
+import {
+  BadGateway,
+  GatewayTimeout,
+  InternalServerError,
+  ServiceUnavailable,
+  TooManyRequests,
+} from "@distilled.cloud/fly-io/Errors";
+import * as machines from "@distilled.cloud/fly-io/machines";
+import * as Retry from "@distilled.cloud/fly-io/Retry";
+import * as Clock from "effect/Clock";
+import * as Data from "effect/Data";
+import * as Deferred from "effect/Deferred";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as HttpClient from "effect/http/HttpClient";
+import * as HttpClientRequest from "effect/http/HttpClientRequest";
+import * as Option from "effect/Option";
+import * as Schedule from "effect/Schedule";
+
+const TTL_SECONDS = 120;
+const EXPIRY_MARGIN_MS = 15_000;
+
+const retrySchedule = <E>(): Schedule.Schedule<Duration.Duration, E> =>
+  Schedule.exponential("500 millis").pipe(
+    Schedule.modifyDelay<Duration.Duration, E>(({ input, duration }) =>
+      Effect.succeed(
+        (TooManyRequests.is(input) ||
+          InternalServerError.is(input) ||
+          BadGateway.is(input) ||
+          ServiceUnavailable.is(input) ||
+          GatewayTimeout.is(input)) &&
+          input.retryAfter
+          ? Math.min(Duration.toMillis(input.retryAfter), 60_000)
+          : Math.min(Duration.toMillis(duration), 5_000),
+      ),
+    ),
+    Schedule.addDelay<Duration.Duration, E>(() =>
+      Effect.sync(() => Duration.millis(Math.random() * 50)),
+    ),
+  );
+
+export class MachineLeaseBusy extends Data.TaggedError("Fly.MachineLeaseBusy")<{
+  appName: string;
+  machineId: string;
+}> {}
+
+export class MachineLeaseLost extends Data.TaggedError("Fly.MachineLeaseLost")<{
+  appName: string;
+  machineId: string;
+  reason: string;
+}> {
+  get message() {
+    return `Machine lease authority lost for ${this.appName}/${this.machineId}: ${this.reason}`;
+  }
+}
+
+export class MachineMutationUncertain extends Data.TaggedError("Fly.MachineMutationUncertain")<{
+  appName: string;
+  machineId: string;
+}> {}
+
+interface HeldLease {
+  nonce: string;
+  deadline: number;
+  renewAt: number;
+}
+
+export const makeMachineLeases = Effect.fn(function* (appName: string) {
+  const scope = yield* Effect.scope;
+  const held = new Map<string, HeldLease>();
+  const deleting = new Set<string>();
+  const renewals = new Set<Fiber.Fiber<void, never>>();
+  const uncertain = new Set<string>();
+  const lost = yield* Deferred.make<never, MachineLeaseLost>();
+  let failure: MachineLeaseLost | undefined;
+  const lose = (machineId: string, reason: string) =>
+    Effect.gen(function* () {
+      failure ??= new MachineLeaseLost({ appName, machineId, reason });
+      yield* Deferred.fail(lost, failure);
+      return yield* failure;
+    });
+  const check = Effect.gen(function* () {
+    if (failure) return yield* failure;
+    const now = yield* Clock.currentTimeMillis;
+    for (const [machineId, lease] of held) {
+      if (now >= lease.deadline) return yield* lose(machineId, "lease authority expired");
+    }
+  });
+  const guard = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+    check.pipe(Effect.andThen(Effect.raceFirst(effect, Deferred.await(lost))));
+  const record = (machineId: string, response: machines.MachineLease, requestedAt: number) =>
+    Effect.gen(function* () {
+      const lease = response.data;
+      const now = yield* Clock.currentTimeMillis;
+      const deadline =
+        Math.min((lease?.expires_at ?? 0) * 1000, requestedAt + TTL_SECONDS * 1000) -
+        EXPIRY_MARGIN_MS;
+      const captured = lease?.nonce
+        ? { nonce: lease.nonce, deadline, renewAt: requestedAt + 25_000 }
+        : undefined;
+      if (captured) held.set(machineId, captured);
+      if (!captured || !Number.isFinite(deadline) || deadline <= now) {
+        return yield* lose(machineId, "invalid or expired lease response");
+      }
+      return captured;
+    });
+  const acquire = (machineIds: readonly string[]) =>
+    Effect.gen(function* () {
+      for (const machineId of [...new Set(machineIds)].sort()) {
+        yield* check;
+        if (held.has(machineId)) continue;
+        // Save a returned nonce atomically; uncertain interrupted requests expire at Fly.
+        yield* Effect.uninterruptibleMask((restore) =>
+          Effect.gen(function* () {
+            const requestedAt = yield* Clock.currentTimeMillis;
+            const response = yield* restore(
+              machines
+                .createMachineLease({
+                  app_name: appName,
+                  machine_id: machineId,
+                  ttl: TTL_SECONDS,
+                  description: "Alchemy Machine lifecycle",
+                })
+                .pipe(
+                  Retry.none,
+                  Effect.retry({
+                    times: 8,
+                    schedule: retrySchedule<machines.CreateMachineLeaseError>(),
+                    while: (error) => error._tag === "Conflict" || error._tag === "TooManyRequests",
+                  }),
+                  Effect.timeout("30 seconds"),
+                  Effect.catchTag("Conflict", () =>
+                    Effect.fail(new MachineLeaseBusy({ appName, machineId })),
+                  ),
+                  Effect.catchTag(["HttpClientError", "GatewayTimeout", "TimeoutError"], (error) =>
+                    lose(machineId, `acquisition uncertain: ${error._tag}`),
+                  ),
+                ),
+            );
+            const lease = yield* record(machineId, response, requestedAt);
+            const renewal = yield* heartbeat(machineId, lease).pipe(
+              Effect.interruptible,
+              Effect.forkIn(scope),
+            );
+            renewals.add(renewal);
+          }),
+        );
+        yield* check;
+      }
+    });
+  const nonce = (machineId: string) =>
+    Effect.gen(function* () {
+      yield* check;
+      const lease = held.get(machineId);
+      if (!lease) return yield* lose(machineId, "target is outside the acquired lease set");
+      return lease.nonce;
+    });
+  const mutate = <A, E extends { readonly _tag: string }, R>(
+    machineId: string,
+    operation: (nonce: string) => Effect.Effect<A, E, R>,
+    options: { idempotent?: boolean; timeoutMs?: number } = {},
+  ) =>
+    guard(
+      Effect.gen(function* () {
+        const value = yield* nonce(machineId);
+        return yield* operation(value).pipe(Retry.none);
+      }).pipe(
+        Effect.retry({
+          times: 8,
+          schedule: retrySchedule<E | MachineLeaseLost>(),
+          // Only explicit throttling is safe to replay for update/stop/delete.
+          while: (error) =>
+            error._tag === "TooManyRequests" ||
+            (options.idempotent === true &&
+              (error._tag === "InternalServerError" ||
+                error._tag === "BadGateway" ||
+                error._tag === "ServiceUnavailable" ||
+                error._tag === "GatewayTimeout")),
+        }),
+        Effect.timeout(options.timeoutMs ?? 120_000),
+        Effect.catch((error) =>
+          Effect.gen(function* () {
+            if (error._tag === "Conflict" || error._tag === "Forbidden") {
+              return yield* lose(machineId, `leased mutation rejected: ${error._tag}`);
+            }
+            // Transport errors retain the request, including its private nonce header.
+            if (
+              error._tag === "HttpClientError" ||
+              error._tag === "GatewayTimeout" ||
+              error._tag === "TimeoutError"
+            ) {
+              return yield* new MachineMutationUncertain({
+                appName,
+                machineId,
+              });
+            }
+            return yield* Effect.fail(error);
+          }),
+        ),
+      ),
+    );
+  const forget = (machineId: string) =>
+    Effect.sync(() => {
+      held.delete(machineId);
+    });
+  const confirmRemoved = (machineId: string) =>
+    guard(
+      machines
+        .getMachine({
+          app_name: appName,
+          machine_id: machineId,
+        })
+        .pipe(
+          Retry.none,
+          Effect.map((machine) => machine.id === machineId && machine.state === "destroyed"),
+          Effect.catchTag("NotFound", () => Effect.succeed(true)),
+          Effect.catchTag(
+            [
+              "TooManyRequests",
+              "InternalServerError",
+              "BadGateway",
+              "ServiceUnavailable",
+              "GatewayTimeout",
+            ],
+            () => Effect.succeed(false),
+          ),
+          Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)),
+          Effect.repeat({
+            times: 8,
+            schedule: Schedule.spaced("2 seconds"),
+            until: (absent) => absent,
+          }),
+          Effect.timeoutOrElse({ duration: "30 seconds", orElse: () => Effect.succeed(false) }),
+          Effect.catch((error) =>
+            Effect.gen(function* () {
+              if (!held.has(machineId)) return true;
+              if (error._tag === "Forbidden") {
+                return yield* lose(machineId, "removal observation forbidden");
+              }
+              return yield* error;
+            }),
+          ),
+          // Another observer's confirmed removal wins over a late uncertain response.
+          Effect.map((absent) => absent || !held.has(machineId)),
+          Effect.tap((absent) => (absent ? forget(machineId) : Effect.void)),
+        ),
+    );
+  const remove = <A, E extends { readonly _tag: string }, R>(
+    machineId: string,
+    operation: (nonce: string) => Effect.Effect<A, E, R>,
+  ) =>
+    Effect.gen(function* () {
+      const http = yield* HttpClient.HttpClient;
+      const fetchOptions = yield* Effect.serviceOption(FetchHttpClient.RequestInit);
+      yield* Effect.sync(() => {
+        deleting.add(machineId);
+      });
+      yield* mutate(machineId, (nonce) =>
+        operation(nonce).pipe(
+          // Bun can replay DELETE on a reused socket below the SDK retry policy.
+          Effect.provideService(
+            HttpClient.HttpClient,
+            HttpClient.mapRequest(http, HttpClientRequest.setHeader("connection", "close")),
+          ),
+          Effect.provideService(FetchHttpClient.RequestInit, {
+            ...Option.getOrUndefined(fetchOptions),
+            keepalive: false,
+            redirect: "error",
+          }),
+        ),
+      ).pipe(
+        Effect.asVoid,
+        Effect.catch((error) =>
+          Effect.gen(function* () {
+            if (error._tag === "NotFound") return yield* forget(machineId);
+            if (
+              error._tag === "Fly.MachineMutationUncertain" ||
+              error._tag === "InternalServerError" ||
+              error._tag === "BadGateway" ||
+              error._tag === "ServiceUnavailable"
+            ) {
+              // Keep renewal aware of the removal until its exact target has been observed.
+              if (yield* confirmRemoved(machineId)) return;
+            }
+            return yield* Effect.fail(error);
+          }),
+        ),
+      );
+      yield* forget(machineId);
+    }).pipe(
+      Effect.ensuring(
+        Effect.sync(() => {
+          deleting.delete(machineId);
+        }),
+      ),
+    );
+  const nonceIfHeld = (machineId: string) =>
+    check.pipe(Effect.map(() => held.get(machineId)?.nonce));
+
+  yield* Effect.addFinalizer(() =>
+    Effect.gen(function* () {
+      // A late refresh can recreate a lease after a successful release.
+      yield* Fiber.interruptAll(renewals);
+      yield* Effect.forEach(
+        [...held.entries()]
+          .filter(([machineId]) => !uncertain.has(machineId))
+          .sort(([a], [b]) => a.localeCompare(b)),
+        ([machineId, lease]) =>
+          machines
+            .machinesReleaseLease({
+              app_name: appName,
+              machine_id: machineId,
+              lease_nonce: lease.nonce,
+            })
+            .pipe(
+              Retry.none,
+              Effect.timeout("5 seconds"),
+              Effect.interruptible,
+              Effect.catchTag("NotFound", () => Effect.void),
+              Effect.catch((error) =>
+                Effect.logWarning(
+                  "Fly lease release was not confirmed; expiry remains the safety bound",
+                  { appName, machineId, error: error._tag },
+                ),
+              ),
+              Effect.asVoid,
+            ),
+        { concurrency: 4 },
+      );
+      if (uncertain.size > 0)
+        yield* Effect.logWarning(
+          "Fly lease renewal was not confirmed; expiry remains the safety bound",
+          { appName, machineIds: [...uncertain] },
+        );
+    }),
+  );
+
+  const refresh = (machineId: string, lease: HeldLease) =>
+    Effect.gen(function* () {
+      yield* check;
+      if (held.get(machineId) !== lease) return;
+      const requestedAt = yield* Clock.currentTimeMillis;
+      const response = yield* machines
+        .createMachineLease({
+          app_name: appName,
+          machine_id: machineId,
+          ttl: TTL_SECONDS,
+          lease_nonce: lease.nonce,
+          description: "Alchemy Machine lifecycle",
+        })
+        .pipe(Retry.none);
+      if (held.get(machineId) !== lease) return;
+      if (response.data?.nonce !== lease.nonce) {
+        return yield* lose(machineId, "refresh returned a different nonce");
+      }
+      return yield* record(machineId, response, requestedAt);
+    }).pipe(
+      Effect.retry({
+        times: 8,
+        schedule: retrySchedule<machines.CreateMachineLeaseError | MachineLeaseLost>(),
+        while: (error) => error._tag === "TooManyRequests",
+      }),
+      Effect.timeout("75 seconds"),
+      Effect.catch((error) =>
+        Effect.gen(function* () {
+          if (held.get(machineId) !== lease) return;
+          uncertain.add(machineId);
+          // A missing lease during removal is not proof that the Machine is gone.
+          if (error._tag === "NotFound" && deleting.has(machineId)) {
+            const absent = yield* confirmRemoved(machineId).pipe(
+              Effect.catch((observation) =>
+                held.get(machineId) === lease
+                  ? lose(machineId, `removal observation uncertain: ${observation._tag}`)
+                  : Effect.succeed(true),
+              ),
+            );
+            if (absent || held.get(machineId) !== lease) return;
+            return yield* lose(machineId, "lease disappeared before removal was confirmed");
+          }
+          return yield* lose(machineId, `refresh uncertain: ${error._tag}`);
+        }),
+      ),
+      (operation) =>
+        Effect.uninterruptible(
+          Effect.gen(function* () {
+            const pending = yield* operation.pipe(Effect.interruptible, Effect.forkChild);
+            return yield* Fiber.join(pending);
+          }),
+        ),
+    );
+  // Each target renews serially on its own cadence within the operation scope.
+  const heartbeat = (machineId: string, acquired: HeldLease) =>
+    Effect.gen(function* () {
+      let lease: HeldLease | undefined = acquired;
+      while (lease && held.get(machineId) === lease) {
+        const now = yield* Clock.currentTimeMillis;
+        yield* Effect.sleep(Math.max(0, lease.renewAt - now));
+        lease = yield* refresh(machineId, lease);
+      }
+    }).pipe(Effect.ignore);
+  yield* Effect.sleep("1 second").pipe(
+    Effect.andThen(check),
+    Effect.forever,
+    Effect.ignore,
+    Effect.forkScoped,
+  );
+  const checkTarget = (machineId: string) => nonce(machineId).pipe(Effect.asVoid);
+  return {
+    acquire,
+    check,
+    checkTarget,
+    guard,
+    mutate,
+    remove,
+    forget,
+    nonceIfHeld,
+  };
+});
+
+export type MachineLeases = Effect.Success<ReturnType<typeof makeMachineLeases>>;
+
+export const usingMachineLeases = <A, E, R>(
+  appName: string,
+  existing: MachineLeases | undefined,
+  use: (leases: MachineLeases) => Effect.Effect<A, E, R>,
+) =>
+  Effect.gen(function* () {
+    if (existing) return yield* existing.guard(Effect.suspend(() => use(existing)));
+    return yield* Effect.scoped(
+      Effect.gen(function* () {
+        const leases = yield* makeMachineLeases(appName);
+        return yield* leases.guard(Effect.suspend(() => use(leases)));
+      }),
+    );
+  });

package/src/ports.ts

@@ -0,0 +1,177 @@
+import type { FlyMachineService } from "@distilled.cloud/fly-io/machines";
+import * as Data from "effect/Data";
+import type { MachineService } from "./Machine.ts";
+
+/** One port, or an inclusive port range, published on Fly's proxy. */
+export interface PublishedPort {
+  protocol: string;
+  start: number;
+  end: number;
+  handlers: string[];
+  forceHttps: boolean;
+}
+
+export class ServicePortConflict extends Data.TaggedError("Fly.ServicePortConflict")<{
+  appName: string;
+  port: number;
+  protocol: string;
+  /** The two publishers of the port: Service logical ids or Machine ids. */
+  publishers: [string, string];
+}> {
+  get message() {
+    return `Port ${this.port}/${this.protocol} in Fly App ${this.appName} is published by both ${this.publishers[0]} and ${this.publishers[1]}. Fly's proxy routes an App's traffic by port only, so give each Service in the App its own ports (\`services\`, or \`bindingPort\` for the plain-HTTP port Alchemy adds for bound callers).`;
+  }
+}
+
+const normalize = (entry: {
+  protocol?: string;
+  port?: number;
+  startPort?: number;
+  endPort?: number;
+  handlers?: string[];
+  forceHttps?: boolean;
+}): PublishedPort[] => {
+  const start = entry.port ?? entry.startPort;
+  const end = entry.port ?? entry.endPort ?? entry.startPort;
+  if (start === undefined || end === undefined) return [];
+  return [
+    {
+      protocol: entry.protocol ?? "tcp",
+      start,
+      end,
+      handlers: entry.handlers ?? [],
+      forceHttps: entry.forceHttps === true,
+    },
+  ];
+};
+
+/** Published ports of observed or desired Fly Machine services. */
+export const portsOfFly = (
+  services: readonly FlyMachineService[] | null | undefined,
+): PublishedPort[] =>
+  (services ?? []).flatMap((service) =>
+    (service.ports ?? []).flatMap((port) =>
+      normalize({
+        protocol: service.protocol,
+        port: port.port,
+        startPort: port.start_port,
+        endPort: port.end_port,
+        handlers: port.handlers,
+        forceHttps: port.force_https,
+      }),
+    ),
+  );
+
+/**
+ * Published ports of a Service's `services` prop, falling back to the
+ * defaults: HTTP 80 (redirecting to HTTPS) and HTTPS 443 when public,
+ * plain HTTP 80 when private.
+ */
+export const portsOfProps = (
+  services: readonly MachineService[] | undefined,
+  isPublic: boolean,
+): PublishedPort[] =>
+  services !== undefined
+    ? services.flatMap((service) =>
+        (service.ports ?? []).flatMap((port) => normalize({ protocol: service.protocol, ...port })),
+      )
+    : isPublic
+      ? [
+          {
+            protocol: "tcp",
+            start: 80,
+            end: 80,
+            handlers: ["http"],
+            forceHttps: true,
+          },
+          {
+            protocol: "tcp",
+            start: 443,
+            end: 443,
+            handlers: ["tls", "http"],
+            forceHttps: false,
+          },
+        ]
+      : [
+          {
+            protocol: "tcp",
+            start: 80,
+            end: 80,
+            handlers: ["http"],
+            forceHttps: false,
+          },
+        ];
+
+const isPlainHttp = (port: PublishedPort) =>
+  port.start === port.end &&
+  port.protocol === "tcp" &&
+  port.handlers.includes("http") &&
+  !port.handlers.includes("tls") &&
+  !port.forceHttps;
+
+/**
+ * The port bound callers use: the first plain-HTTP port, since Fly issues
+ * no certificate for `.flycast`. When the Service publishes ports but none
+ * of them is plain HTTP, Alchemy publishes `bindingPort` for bindings.
+ */
+export const bindingPortOf = (
+  ports: readonly PublishedPort[],
+  bindingPort: number,
+): { port: number; added: boolean } | undefined => {
+  if (ports.length === 0) return undefined;
+  const others = ports.filter((port) => !(isPlainHttp(port) && port.start === bindingPort));
+  const plain = others.find(isPlainHttp);
+  if (plain !== undefined) return { port: plain.start, added: false };
+  return { port: bindingPort, added: true };
+};
+
+/** `ports` plus the binding port Alchemy adds, when it adds one. */
+export const withBindingPort = (
+  ports: readonly PublishedPort[],
+  bindingPort: number,
+): PublishedPort[] => {
+  const binding = bindingPortOf(ports, bindingPort);
+  return binding?.added === true &&
+    !ports.some((port) => isPlainHttp(port) && port.start === bindingPort)
+    ? [
+        ...ports,
+        {
+          protocol: "tcp",
+          start: bindingPort,
+          end: bindingPort,
+          handlers: ["http"],
+          forceHttps: false,
+        },
+      ]
+    : [...ports];
+};
+
+const overlaps = (left: PublishedPort, right: PublishedPort) =>
+  left.protocol === right.protocol && left.start <= right.end && right.start <= left.end;
+
+/**
+ * The first port two publishers share, if any. A publisher that lists the
+ * same port twice also conflicts with itself.
+ */
+export const findPortConflict = (
+  appName: string,
+  publishers: ReadonlyArray<{ id: string; ports: readonly PublishedPort[] }>,
+): ServicePortConflict | undefined => {
+  const flat = publishers.flatMap((publisher) =>
+    publisher.ports.map((port) => ({ id: publisher.id, port })),
+  );
+  for (let i = 0; i < flat.length; i++) {
+    for (let j = i + 1; j < flat.length; j++) {
+      const left = flat[i]!;
+      const right = flat[j]!;
+      if (!overlaps(left.port, right.port)) continue;
+      return new ServicePortConflict({
+        appName,
+        port: Math.max(left.port.start, right.port.start),
+        protocol: left.port.protocol,
+        publishers: [left.id, right.id],
+      });
+    }
+  }
+  return undefined;
+};