@alchemy.run/fly 2.0.0-beta.84 → 2.0.0-beta.86

package/src/bluegreen.ts

@@ -102,7 +102,7 @@ export const setRouting = (
       yield* getMachineById(appName, machineId).pipe(
         Effect.filterOrFail(
           (machine) => machine !== undefined && machine.cordoned === cordoned,
-          () => new ReplicaNotCreated({ appName, name: machineId }),
+          () => new ReplicaNotCreated({ appName, resourceName: machineId }),
         ),
         Effect.retry({
           times: 8,
@@ -293,7 +293,9 @@ export const reconcileBlueGreen = Effect.fn(function* (
     reusable?.[0] ??
     (yield* sha256Object({
       workload,
-      predecessors: owned.map((machine) => machine.id).sort(),
+      predecessors: owned
+        .map((machine) => machine.id)
+        .sort((a, b) => ((a ?? "") < (b ?? "") ? -1 : (a ?? "") > (b ?? "") ? 1 : 0)),
     })).slice(0, 20);
   const desired = reusable?.[1] ?? [];
   if (
@@ -417,7 +419,7 @@ export const reconcileBlueGreen = Effect.fn(function* (
           ),
           Effect.filterOrFail(
             (machine) => machine !== undefined,
-            () => new ReplicaNotCreated({ appName: input.appName, name }),
+            () => new ReplicaNotCreated({ appName: input.appName, resourceName: name }),
           ),
           Effect.retry({
             times: 8,
@@ -442,11 +444,15 @@ export const reconcileBlueGreen = Effect.fn(function* (
           .pipe(
             Retry.none,
             Effect.timeout("30 seconds"),
-            Effect.catchTag(["Conflict", "GatewayTimeout", "TimeoutError"], () => readback),
+            Effect.catchTag(
+              ["Conflict", "GatewayTimeout", "TimeoutError", "StorageHostCapacity"],
+              () => readback,
+            ),
             Effect.catchReason("HttpClientError", "TransportError", () => readback),
           );
       }
-      if (!current?.id) return yield* new ReplicaNotCreated({ appName: input.appName, name });
+      if (!current?.id)
+        return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
       const leased = (yield* leaseSnapshot(input.appName, [current], leases))[0];
       if (!leased?.id)
         return yield* ambiguous(

package/src/hosted.ts

@@ -9,9 +9,12 @@ import {
   type PackageInstall,
 } from "alchemy/Bundle/InstalledPackages";
 import { findCwdForBundle, getStableContextDir, resolveMainPath } from "alchemy/Bundle/TempRoot";
+import { hashDirectory } from "alchemy/Command/Memo";
 import { safeHttpEffect } from "alchemy/Http";
+import * as Output from "alchemy/Output";
 import type { ResourceBinding } from "alchemy/Resource";
 import { createContainerRuntimeContext, type HostRuntimeContext } from "alchemy/Runtime/Process";
+import { packEnvValueKeepRedacted } from "alchemy/RuntimeContext";
 import {
   copyExtraFiles,
   contextRootOf,
@@ -30,6 +33,7 @@ import * as Path from "effect/Path";
 import * as Redacted from "effect/Redacted";
 import * as Schedule from "effect/Schedule";
 import type * as rolldown from "rolldown";
+import { flyHostBundle, flyOidcToken } from "./internal/Oidc.ts";
 import { buildOnRemoteBuilder } from "./internal/RemoteBuilder.ts";
 import type { BoundTarget, DiskSpec, ServiceBinding } from "./MountVolume.ts";
 import { rpcMethodsOf, serveFlyRpc } from "./rpc.ts";
@@ -59,7 +63,20 @@ export const createFlyHostRuntimeContext =
         boots ? { shape: { ...shape, fetch: shape?.fetch ?? wrapped } } : options,
       ) as Effect.Effect<void, never, any>;
     };
-    return Object.assign(base, { serve });
+    // Keep the `Redacted` wrapper outside the packed string so the Service
+    // reconcile can store those values as App secrets instead of Machine env.
+    const set: HostRuntimeContext["set"] = (bindingId, output) =>
+      Effect.sync(() => {
+        const key = bindingId.replaceAll(/[^a-zA-Z0-9]/g, "_");
+        base.env[key] = output.pipe(Output.map(packEnvValueKeepRedacted));
+        return key;
+      });
+    // A Service runs on Fly Machines, which mint OIDC tokens for their
+    // bindings to exchange with AWS and GCP (see `internal/Oidc.ts`).
+    return Object.assign(
+      base,
+      type === "Fly.Service" ? { serve, set, oidcToken: flyOidcToken } : { serve, set },
+    );
   };
 
 export const FLY_REGISTRY = "registry.fly.io";
@@ -70,13 +87,12 @@ export const MACHINE_PLATFORM = "linux/amd64";
 export interface FlyBuildOptions extends Bundle.BundleConfig {
   /**
    * Native or Node-only packages to install into the Machine image with
-   * `npm install` instead of bundling them. `pg` is CommonJS: Rolldown's
-   * interop turns `Client` into a namespace (`The superclass is not a
-   * constructor`). Same `build.install` shape as Lambda.
+   * `npm install` instead of bundling them. Same `build.install` shape as
+   * Lambda.
    *
    * @example
    * ```typescript
-   * build: { install: ["pg"] }
+   * build: { install: ["sharp"] }
    * ```
    */
   readonly install?: PackageInstall;
@@ -190,10 +206,14 @@ export const collectBindingState = (bindings: readonly ResourceBinding<ServiceBi
   const active = bindings.filter(
     (binding: ResourceBinding<ServiceBinding> & { action?: string }) => binding.action !== "delete",
   );
-  const env = toEnvRecord(
-    active
-      .map((binding) => binding?.data?.env)
-      .reduce<Record<string, any>>((acc, value) => ({ ...acc, ...value }), {}),
+  const merged = active
+    .map((binding) => binding?.data?.env)
+    .reduce<Record<string, any>>((acc, value) => ({ ...acc, ...value }), {});
+  const env = toEnvRecord(merged);
+  // Names of bound `Config` values and Redacted outputs. Only the names:
+  // the values stay in `env` until the Service moves them into App secrets.
+  const secretNames = Object.keys(toEnvRecord(merged)).filter((key) =>
+    Redacted.isRedacted(merged[key]),
   );
   const mounts: DiskSpec[] = [];
   const seen = new Set<string>();
@@ -201,9 +221,12 @@ export const collectBindingState = (bindings: readonly ResourceBinding<ServiceBi
   const buckets: { name: string; id?: string; values?: Record<string, string> }[] = [];
   const postgres: { clusterId: string; variableName?: string }[] = [];
   const targets: BoundTarget[] = [];
+  const callers = new Set<string>();
   for (const binding of active) {
     const target = binding?.data?.target;
     if (target !== undefined) targets.push(target);
+    const caller = coerceBindingName(binding?.data?.caller?.appName);
+    if (caller !== undefined) callers.add(caller);
     for (const mount of binding?.data?.mounts ?? []) {
       if (seen.has(mount.path)) continue;
       seen.add(mount.path);
@@ -257,7 +280,16 @@ export const collectBindingState = (bindings: readonly ResourceBinding<ServiceBi
       });
     }
   }
-  return { env, mounts, redis, buckets, postgres, targets };
+  return {
+    env,
+    secretNames,
+    mounts,
+    redis,
+    buckets,
+    postgres,
+    targets,
+    callers: [...callers].sort(),
+  };
 };
 
 /**
@@ -378,7 +410,7 @@ export const createFlyHostedSupport = ({
       plugins?: rolldown.RolldownPluginOption,
     ) {
       return yield* Bundle.build(
-        {
+        flyHostBundle({
           ...props.build?.input,
           input: entry,
           cwd,
@@ -395,7 +427,7 @@ export const createFlyHostedSupport = ({
             ...props.build?.input?.resolve,
           },
           plugins: [props.build?.input?.plugins, plugins],
-        },
+        }),
         {
           ...props.build?.output,
           format: "esm",
@@ -540,10 +572,10 @@ export const createFlyHostedSupport = ({
         );
         const entryRel = posixRelUnder(root, realMain, path) ?? path.basename(realMain);
         const dest = path.join(contextDir, entryRel);
-        if (!(yield* fs.exists(dest).pipe(Effect.orElseSucceed(() => false)))) {
-          yield* fs.makeDirectory(path.dirname(dest), { recursive: true });
-          yield* fs.copy(realMain, dest, { overwrite: true });
-        }
+        // The context directory is reused across deploys: always copy the
+        // entry, or a changed entry ships the previous deploy's copy.
+        yield* fs.makeDirectory(path.dirname(dest), { recursive: true });
+        yield* fs.copy(realMain, dest, { overwrite: true });
       }
       return contextDir;
     });
@@ -584,7 +616,79 @@ export const createFlyHostedSupport = ({
     return codeHash;
   });
 
-  return { alchemyEnv, bundleProgram, computeCodeHash, resolveImage, hash };
+  /** A Docker build context's content hash: its image changes when any file in it does. */
+  const contextHash = Effect.fn(function* (context: string) {
+    return (yield* hashDirectory({ cwd: context })).slice(0, 16);
+  });
+
+  /**
+   * Build the image a `context` directory's `Dockerfile` describes, as it
+   * is, and push it to `registry.fly.io/{appName}:{logicalId}-{hash}`.
+   * When `previousHash` matches, skip build and push.
+   */
+  const resolveContextImage = Effect.fn(function* (input: {
+    id: string;
+    appName: string;
+    context: string;
+    remote: boolean | undefined;
+    previousHash?: string;
+    session?: { note: (message: string) => Effect.Effect<void> };
+  }) {
+    const note = input.session?.note ?? ((_message: string) => Effect.void);
+    const codeHash = yield* contextHash(input.context);
+    const imageRef = `${FLY_REGISTRY}/${input.appName}:${sanitizeImageRepo(input.id)}-${codeHash}`;
+    if (input.previousHash === codeHash) return { imageRef, codeHash };
+
+    const minted = yield* machines.createAppDeployToken({ app_name: input.appName });
+    const token =
+      minted.token === undefined || Redacted.isRedacted(minted.token)
+        ? minted.token
+        : Redacted.make(minted.token);
+    if (token === undefined || Redacted.value(token).length === 0) {
+      return yield* new DeployTokenMissing({ appName: input.appName });
+    }
+    const registry = { server: FLY_REGISTRY, username: "x", password: token };
+
+    if (input.remote !== false) {
+      yield* note(`Building ${imageRef} on a Fly remote builder...`);
+      yield* buildOnRemoteBuilder({
+        docker,
+        appName: input.appName,
+        contextDir: input.context,
+        imageRef,
+        platform: MACHINE_PLATFORM,
+        registry,
+        note,
+      });
+      yield* note(`Pushed ${imageRef}`);
+      return { imageRef, codeHash };
+    }
+
+    if (!(yield* imageExists(imageRef))) {
+      yield* note(`Building container image ${imageRef}...`);
+      yield* docker.image.build({
+        context: input.context,
+        tag: imageRef,
+        platform: MACHINE_PLATFORM,
+      });
+    }
+    yield* note(`Pushing ${imageRef}...`);
+    yield* docker.image
+      .push(imageRef, registry)
+      .pipe(Effect.retry({ times: 3, schedule: pushBackoff }));
+    yield* note(`Pushed ${imageRef}`);
+    return { imageRef, codeHash };
+  });
+
+  return {
+    alchemyEnv,
+    bundleProgram,
+    computeCodeHash,
+    resolveImage,
+    hash,
+    contextHash,
+    resolveContextImage,
+  };
 };
 
 /**

package/src/internal/Oidc.ts

@@ -0,0 +1,99 @@
+import * as http from "node:http";
+import { OidcTokenError } from "alchemy/RuntimeContext";
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import type * as rolldown from "rolldown";
+
+/**
+ * Fly Machine OIDC tokens. Every Machine serves part of the Machines API on
+ * the `/.fly/api` socket, authenticated as that Machine, so a process can
+ * mint a token without holding one. A token is signed by
+ * `https://oidc.fly.io/<org>`, lives 10 minutes, and names its caller as
+ * `sub = <org>:<app>:<machine>`. AWS and GCP trust it through an OIDC
+ * provider, so a Fly host reaches them without a key Alchemy created.
+ *
+ * See https://fly.io/docs/security/openid-connect/.
+ */
+
+/** The socket a Machine serves its Machines API on. */
+const FLY_API_SOCKET = "/.fly/api";
+
+/** Fly's OIDC issuer for an organization. */
+export const flyOidcIssuer = (orgSlug: string) => `https://oidc.fly.io/${orgSlug}`;
+
+/**
+ * Bundle options for a Fly Service: defines `__ALCHEMY_HOST_IDENTITY__` as
+ * `"Fly.OIDC"` so bindings keep only the code that exchanges the Machine's
+ * OIDC token, and drop the created-key path.
+ */
+export const flyHostBundle = (input: rolldown.InputOptions): rolldown.InputOptions => ({
+  ...input,
+  transform: {
+    ...input.transform,
+    define: {
+      ...input.transform?.define,
+      "globalThis.__ALCHEMY_HOST_IDENTITY__": JSON.stringify("Fly.OIDC"),
+    },
+  },
+});
+
+/** Mint an OIDC token for `audience` from the Machine's `/.fly/api` socket. */
+export const flyOidcToken = (
+  audience: string,
+): Effect.Effect<Redacted.Redacted<string>, OidcTokenError> =>
+  Effect.callback<Redacted.Redacted<string>, OidcTokenError>((resume) => {
+    const body = JSON.stringify({ aud: audience });
+    const request = http.request(
+      {
+        socketPath: FLY_API_SOCKET,
+        path: "/v1/tokens/oidc",
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          "content-length": Buffer.byteLength(body),
+        },
+      },
+      (response) => {
+        let text = "";
+        response.setEncoding("utf8");
+        response.on("data", (chunk: string) => {
+          text += chunk;
+        });
+        response.on("end", () => {
+          if (response.statusCode === 200 && text.trim().length > 0) {
+            resume(Effect.succeed(Redacted.make(text.trim())));
+          } else {
+            resume(
+              Effect.fail(
+                new OidcTokenError({
+                  message: `${FLY_API_SOCKET} returned HTTP ${response.statusCode} for an OIDC token: ${text.slice(0, 200)}`,
+                }),
+              ),
+            );
+          }
+        });
+        response.on("error", (cause) =>
+          resume(
+            Effect.fail(
+              new OidcTokenError({
+                message: `Reading the OIDC token from ${FLY_API_SOCKET}`,
+                cause,
+              }),
+            ),
+          ),
+        );
+      },
+    );
+    request.on("error", (cause) =>
+      resume(
+        Effect.fail(
+          new OidcTokenError({
+            message: `Cannot reach ${FLY_API_SOCKET} for an OIDC token. It is served only on Fly Machines, to root.`,
+            cause,
+          }),
+        ),
+      ),
+    );
+    request.end(body);
+    return Effect.sync(() => request.destroy());
+  });

package/src/internal/RemoteBuilder.ts

@@ -37,7 +37,9 @@ const HEALTH_INTERVAL = Duration.seconds(5);
 const MAX_PENDING_POLLS = 240;
 /** How often, and how long, to wait for the builder's push to reach `registry.fly.io`. */
 const UPSTREAM_POLL = Duration.seconds(2);
-const MAX_UPSTREAM_POLLS = 90;
+const MAX_UPSTREAM_POLLS = 30;
+/** Pushes of one image before giving up on it reaching the registry. */
+const MAX_PUSHES = 3;
 
 export class RemoteBuilderError extends Data.TaggedError("Fly.RemoteBuilderError")<{
   operation: string;
@@ -170,7 +172,13 @@ export const buildOnRemoteBuilder = Effect.fn(function* (input: {
       () => input.docker.run(["buildx", "rm", builder]).pipe(Effect.ignore),
     );
 
-    yield* input.docker.image
+    // The builder pushes through its host's registry mirror, which forwards
+    // to registry.fly.io. Machines pull from there, so confirm the tag landed
+    // (usually within seconds) while the builder is still held. The mirror
+    // occasionally drops a push without an error, so a tag that hasn't
+    // landed is pushed again: the builder has every layer cached, so a
+    // repeat build only re-sends the image.
+    const build = input.docker.image
       .build(
         {
           context: input.contextDir,
@@ -193,25 +201,35 @@ export const buildOnRemoteBuilder = Effect.fn(function* (input: {
           }),
         }),
       );
-
-    // The builder pushes through its host's registry mirror, which forwards
-    // to registry.fly.io. Machines pull from there, so confirm the tag landed
-    // (usually within seconds) while the builder is still held.
-    const started = yield* Clock.currentTimeMillis;
-    yield* input.docker.run(["buildx", "imagetools", "inspect", "--raw", input.imageRef]).pipe(
-      Effect.provideService(DockerEnvironment, {
-        DOCKER_AUTH_CONFIG: JSON.stringify({
-          auths: { [input.registry.server]: { auth: basicAuth(input.registry) } },
-        }),
-      }),
-      Effect.retry({ schedule: Schedule.spaced(UPSTREAM_POLL), times: MAX_UPSTREAM_POLLS }),
-      Effect.mapError(
-        () =>
-          new RemoteBuilderError({
-            operation: "upload",
-            message: `${input.imageRef} did not reach ${input.registry.server}`,
+    const landed = input.docker
+      .run(["buildx", "imagetools", "inspect", "--raw", input.imageRef])
+      .pipe(
+        Effect.provideService(DockerEnvironment, {
+          DOCKER_AUTH_CONFIG: JSON.stringify({
+            auths: { [input.registry.server]: { auth: basicAuth(input.registry) } },
           }),
+        }),
+        Effect.retry({ schedule: Schedule.spaced(UPSTREAM_POLL), times: MAX_UPSTREAM_POLLS }),
+        Effect.mapError(
+          () =>
+            new RemoteBuilderError({
+              operation: "upload",
+              message: `${input.imageRef} did not reach ${input.registry.server} after ${MAX_PUSHES} pushes`,
+            }),
+        ),
+      );
+    const started = yield* Clock.currentTimeMillis;
+    yield* build.pipe(
+      Effect.andThen(landed),
+      Effect.tapError((error) =>
+        error._tag === "Fly.RemoteBuilderError" && error.operation === "upload"
+          ? note(`${input.imageRef} hasn't reached ${input.registry.server} yet`)
+          : Effect.void,
       ),
+      Effect.retry({
+        while: (error) => error._tag === "Fly.RemoteBuilderError" && error.operation === "upload",
+        times: MAX_PUSHES - 1,
+      }),
     );
     const waited = (yield* Clock.currentTimeMillis) - started;
     yield* note(`${input.imageRef} available on ${input.registry.server} after ${waited}ms`);

package/src/leases.ts

@@ -17,6 +17,7 @@ import * as FetchHttpClient from "effect/http/FetchHttpClient";
 import * as HttpClient from "effect/http/HttpClient";
 import * as HttpClientRequest from "effect/http/HttpClientRequest";
 import * as Option from "effect/Option";
+import * as Random from "effect/Random";
 import * as Schedule from "effect/Schedule";
 
 const TTL_SECONDS = 120;
@@ -37,14 +38,20 @@ const retrySchedule = <E>(): Schedule.Schedule<Duration.Duration, E> =>
       ),
     ),
     Schedule.addDelay<Duration.Duration, E>(() =>
-      Effect.sync(() => Duration.millis(Math.random() * 50)),
+      Effect.map(Random.next, (random) => Duration.millis(random * 50)),
     ),
   );
 
 export class MachineLeaseBusy extends Data.TaggedError("Fly.MachineLeaseBusy")<{
   appName: string;
   machineId: string;
-}> {}
+  /** Fly's conflict message: the lease holder and its expiry. */
+  reason?: string;
+}> {
+  get message() {
+    return `Machine lease for ${this.appName}/${this.machineId} is held by another operation${this.reason ? `: ${this.reason}` : ""}`;
+  }
+}
 
 export class MachineLeaseLost extends Data.TaggedError("Fly.MachineLeaseLost")<{
   appName: string;
@@ -71,6 +78,9 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
   const scope = yield* Effect.scope;
   const held = new Map<string, HeldLease>();
   const deleting = new Set<string>();
+  // Machines this lease set deleted; a later snapshot of the same operation
+  // must not wait for them to accept a lease.
+  const removed = new Set<string>();
   const renewals = new Set<Fiber.Fiber<void, never>>();
   const uncertain = new Set<string>();
   const lost = yield* Deferred.make<never, MachineLeaseLost>();
@@ -131,8 +141,10 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
                     while: (error) => error._tag === "Conflict" || error._tag === "TooManyRequests",
                   }),
                   Effect.timeout("30 seconds"),
-                  Effect.catchTag("Conflict", () =>
-                    Effect.fail(new MachineLeaseBusy({ appName, machineId })),
+                  Effect.catchTag("Conflict", (error) =>
+                    Effect.fail(
+                      new MachineLeaseBusy({ appName, machineId, reason: error.message }),
+                    ),
                   ),
                   Effect.catchTag(["HttpClientError", "GatewayTimeout", "TimeoutError"], (error) =>
                     lose(machineId, `acquisition uncertain: ${error._tag}`),
@@ -274,7 +286,10 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
         Effect.asVoid,
         Effect.catch((error) =>
           Effect.gen(function* () {
-            if (error._tag === "NotFound") return yield* forget(machineId);
+            if (error._tag === "NotFound") {
+              removed.add(machineId);
+              return yield* forget(machineId);
+            }
             if (
               error._tag === "Fly.MachineMutationUncertain" ||
               error._tag === "InternalServerError" ||
@@ -282,12 +297,16 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
               error._tag === "ServiceUnavailable"
             ) {
               // Keep renewal aware of the removal until its exact target has been observed.
-              if (yield* confirmRemoved(machineId)) return;
+              if (yield* confirmRemoved(machineId)) {
+                removed.add(machineId);
+                return;
+              }
             }
             return yield* Effect.fail(error);
           }),
         ),
       );
+      removed.add(machineId);
       yield* forget(machineId);
     }).pipe(
       Effect.ensuring(
@@ -360,7 +379,14 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
       Effect.retry({
         times: 8,
         schedule: retrySchedule<machines.CreateMachineLeaseError | MachineLeaseLost>(),
-        while: (error) => error._tag === "TooManyRequests",
+        // A renewal re-sends the nonce we hold, so replaying it after a
+        // transient 5xx extends the same lease instead of taking a new one.
+        while: (error) =>
+          error._tag === "TooManyRequests" ||
+          error._tag === "InternalServerError" ||
+          error._tag === "BadGateway" ||
+          error._tag === "ServiceUnavailable" ||
+          error._tag === "GatewayTimeout",
       }),
       Effect.timeout("75 seconds"),
       Effect.catch((error) =>
@@ -407,6 +433,7 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
     Effect.forkScoped,
   );
   const checkTarget = (machineId: string) => nonce(machineId).pipe(Effect.asVoid);
+  const wasRemoved = (machineId: string) => removed.has(machineId);
   return {
     acquire,
     check,
@@ -416,6 +443,7 @@ export const makeMachineLeases = Effect.fn(function* (appName: string) {
     remove,
     forget,
     nonceIfHeld,
+    wasRemoved,
   };
 });