@alchemy.run/effect-cluster 0.0.0 → 2.0.0-beta.87

package/src/Fly/Database.ts

@@ -0,0 +1,102 @@
+import * as Fly from "@alchemy.run/fly";
+import * as Output from "alchemy/Output";
+import { Random } from "alchemy/Random";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import { clusterShardGroups, Database as ClusterDatabase } from "../Database.ts";
+import type { Providers } from "../Providers.ts";
+
+export interface DatabaseProps {
+  /**
+   * Fly region of the database.
+   * @default "iad"
+   */
+  region?: string;
+  /**
+   * Postgres image.
+   * @default "postgres:17-alpine"
+   */
+  image?: string;
+  /**
+   * Size of the database's volume, in GB.
+   * @default 10
+   */
+  diskSize?: number;
+  /**
+   * Machine size of the database.
+   * @default { cpus: 1, memoryMb: 1024 }
+   */
+  guest?: Fly.MachineGuest;
+  /**
+   * The Fly private network the database and runners join.
+   * @default a network for the stack and stage
+   */
+  network?: string;
+}
+
+/** Where Postgres keeps its data, below the volume's mount point. */
+const DATA = "/var/lib/postgresql/data";
+
+/**
+ * The database an Effect Cluster coordinates through, on Fly.io, as a
+ * `Cluster.Database` Layer: Postgres on a Fly Machine with a volume, in an
+ * App on a private network that the pool's runners join. Nothing outside the
+ * network reaches it.
+ *
+ * One Postgres on one volume is meant for development and small clusters;
+ * for Fly's Managed Postgres, pass its URL to `Cluster.Database.fromUrl`.
+ *
+ * ### Running the database on Fly
+ * **Example:** Postgres in Ashburn
+ * ```typescript
+ * export const Database = ClusterFly.Database("Cluster");
+ * ```
+ *
+ * @primitive
+ */
+export const Database = (id: string, props: DatabaseProps = {}) =>
+  Layer.effect(
+    ClusterDatabase,
+    Effect.gen(function* () {
+      if (globalThis.__ALCHEMY_RUNTIME__) {
+        // Runners read their database from their own environment at runtime.
+        return { url: Redacted.make(""), shardGroups: undefined as never };
+      }
+      const network = props.network ?? (yield* Fly.stackNetwork);
+      const password = yield* Random(`${id}Password`, { bytes: 24 }) as unknown as Effect.Effect<
+        Random,
+        never,
+        Providers
+      >;
+      const app = yield* Fly.App(`${id}App`, { network });
+      const machine = yield* Fly.Machine(`${id}Postgres`, {
+        app,
+        region: props.region ?? "iad",
+        image: props.image ?? "postgres:17-alpine",
+        guest: props.guest ?? { cpus: 1, memoryMb: 1024 },
+        mounts: [{ path: DATA, sizeGb: props.diskSize ?? 10 }],
+        env: password.text.pipe(
+          Output.map((text) => ({
+            POSTGRES_PASSWORD: Redacted.value(text),
+            POSTGRES_DB: "cluster",
+            // A fresh volume holds `lost+found`: keep the data below it.
+            PGDATA: `${DATA}/pgdata`,
+          })),
+        ) as unknown as Record<string, string>,
+        restart: { policy: "always" },
+      });
+      return {
+        // Ordered after the Machine: runners connect once it exists.
+        url: Output.all(app.appName, machine.machineId, password.text).pipe(
+          Output.map(([appName, , text]) =>
+            Redacted.make(
+              `postgres://postgres:${encodeURIComponent(text.pipe(Redacted.value))}@${appName}.internal:5432/cluster`,
+            ),
+          ),
+        ),
+        network: { kind: "fly" as const, network },
+        shardGroups: yield* clusterShardGroups(id),
+      };
+    }),
+  );

package/src/Fly/MachinePool.ts

@@ -0,0 +1,72 @@
+import * as Fly from "@alchemy.run/fly";
+import * as Output from "alchemy/Output";
+import * as Effect from "effect/Effect";
+import { GATEWAY_PORT } from "../internal/Gateway.ts";
+import { makePoolLayer, type PoolSettings } from "../internal/PoolHost.ts";
+
+export interface MachinePoolOptions extends PoolSettings {
+  /**
+   * Number of runners.
+   * @default 2
+   */
+  replicas?: number;
+  /**
+   * Fly region of the pool.
+   * @default "iad"
+   */
+  region?: string;
+  /**
+   * Machine size of each runner.
+   * @default { cpus: 1, memoryMb: 1024 }
+   */
+  guest?: Fly.MachineGuest;
+}
+
+/**
+ * Run a pool of Effect Cluster runners on Fly.io: a Fly Service whose Machines
+ * host every class given this pool, on the database's private
+ * network. Runners talk to each other and the database over the network;
+ * the Service serves their gateway at `https://{name}.fly.dev` to callers
+ * anywhere. Requires a database on a Fly network — `ClusterFly.Database`.
+ *
+ * ### Running on Fly
+ * **Example:** Two runners in Ashburn
+ * ```typescript
+ * export const Database = ClusterFly.Database("Cluster");
+ * export const Default = ClusterFly.MachinePool("Default", { replicas: 2 }).pipe(
+ *   Layer.provide(Database),
+ * );
+ * ```
+ *
+ * @primitive
+ */
+export const MachinePool = (id: string, options: MachinePoolOptions = {}) =>
+  makePoolLayer({
+    label: "ClusterFly.MachinePool",
+    id,
+    settings: options,
+    expected: "a Fly private network (ClusterFly.Database)",
+    // Fly App names are at most 30 characters.
+    nameLength: 30,
+    network: (database) => (database.network?.kind === "fly" ? database.network : undefined),
+    // The Service's App is named after the pool.
+    gateway: (name) => Effect.succeed({ url: `https://${name}.fly.dev` }),
+    nodes: ({ network, name, program, environment }) =>
+      Fly.Service(`${id}Pool`, {
+        name,
+        network: network.network as string,
+        main: program.pipe(Output.map((p) => `${p.dir}/runner.mjs`)) as unknown as string,
+        region: options.region ?? "iad",
+        count: options.replicas ?? 2,
+        guest: options.guest ?? { cpus: 1, memoryMb: 1024 },
+        port: GATEWAY_PORT,
+        public: true,
+        env: environment.pipe(
+          // Runners listen on the private network, which is IPv6.
+          Output.map((env) => ({
+            ...env,
+            CLUSTER_LISTEN_HOST: "::",
+          })),
+        ) as unknown as Record<string, string>,
+      }).pipe(Effect.asVoid),
+  });

package/src/Fly/index.ts

@@ -0,0 +1,6 @@
+/**
+ * @alchemy.run/effect-cluster/Fly — run an Effect Cluster's database and runners
+ * on Fly.io. Requires `@alchemy.run/fly`.
+ */
+export * from "./Database.ts";
+export * from "./MachinePool.ts";

package/src/GCP/ComputeEnginePool.ts

@@ -0,0 +1,203 @@
+import * as Compute from "@alchemy.run/gcp/Compute";
+import * as IAM from "@alchemy.run/gcp/IAM";
+import * as Storage from "@alchemy.run/gcp/Storage";
+import * as Output from "alchemy/Output";
+import * as Effect from "effect/Effect";
+import { GATEWAY_PORT, RUNNER_PORT } from "../internal/Gateway.ts";
+import { makePoolLayer, type PoolSettings } from "../internal/PoolHost.ts";
+import { COS_IMAGE } from "./Database.ts";
+
+const NODE_IMAGE = "node:26-slim";
+const APP_PATH = "/var/lib/cluster-runner";
+const ENV_KEY = "cluster-runner-env";
+const HASH_KEY = "cluster-runner-hash";
+
+export interface ComputeEnginePoolOptions extends PoolSettings {
+  /**
+   * Number of runners, one instance each, spread across zones.
+   * @default 2
+   */
+  replicas?: number;
+  /**
+   * Machine type of each runner.
+   * @default "e2-small"
+   */
+  machineType?: string;
+  /**
+   * Zones the pool is spread across, in the database's region.
+   * @default the region's `-a`, `-b` and `-c` zones
+   */
+  zones?: string[];
+}
+
+/**
+ * Fetch the runner program from the bucket and run it on the host network.
+ * When the instance's metadata changes (a new program, new environment), run
+ * this script again, so every runner picks the change up.
+ */
+const startupScript = (options: { readonly bucket: string }) => `#!/bin/bash
+set -uo pipefail
+MD=http://metadata.google.internal/computeMetadata/v1
+mkdir -p ${APP_PATH}
+# Container-Optimized OS drops inbound traffic except SSH: open the runner's ports.
+for PORT in ${RUNNER_PORT} ${GATEWAY_PORT}; do
+  iptables -C INPUT -p tcp --dport $PORT -j ACCEPT 2>/dev/null || iptables -A INPUT -p tcp --dport $PORT -j ACCEPT
+done
+fetch() {
+  TOKEN="$(curl -sf -H 'Metadata-Flavor: Google' "$MD/instance/service-accounts/default/token" | sed -E 's/.*"access_token":"([^"]+)".*/\\1/')"
+  curl -sf -H "Authorization: Bearer $TOKEN" -o ${APP_PATH}/$1.next \\
+    "https://storage.googleapis.com/storage/v1/b/${options.bucket}/o/$1?alt=media" \\
+    && mv ${APP_PATH}/$1.next ${APP_PATH}/$1
+}
+start() {
+  fetch runner.mjs
+  curl -sf -H 'Metadata-Flavor: Google' "$MD/instance/attributes/${ENV_KEY}" > ${APP_PATH}/env.list
+  docker rm -f cluster-runner >/dev/null 2>&1 || true
+  docker run -d --name cluster-runner --restart always --network host \\
+    --env-file ${APP_PATH}/env.list -v ${APP_PATH}:/app:ro -w /app ${NODE_IMAGE} node /app/runner.mjs
+}
+etag() {
+  curl -sf -H 'Metadata-Flavor: Google' -D - -o /dev/null "$MD/instance/attributes/?recursive=true$1" \\
+    | grep -i '^etag:' | awk '{print $2}' | tr -d '\\r'
+}
+start
+(
+  ETAG="$(etag "")"
+  while true; do
+    NEXT="$(etag "&wait_for_change=true&timeout_sec=3600&last_etag=$ETAG")"
+    if [ -n "$NEXT" ] && [ "$NEXT" != "$ETAG" ]; then
+      curl -sf -H 'Metadata-Flavor: Google' "$MD/instance/attributes/startup-script" > ${APP_PATH}/startup.sh \\
+        && exec setsid bash ${APP_PATH}/startup.sh
+    fi
+    sleep 5
+  done
+) >/dev/null 2>&1 < /dev/null &
+# The runner's logs on the serial console (\`gcloud compute instances get-serial-port-output\`).
+pgrep -f "docker logs -f cluster-runner" >/dev/null || setsid sh -c 'while true; do docker logs -f cluster-runner 2>&1 | sed -u "s/^/cluster-runner: /"; sleep 2; done > /dev/ttyS0' >/dev/null 2>&1 < /dev/null &
+`;
+
+/**
+ * Run a pool of Effect Cluster runners on Compute Engine: instances in the
+ * database's VPC, spread across zones, each running every class given these
+ * runners from a Cloud Storage bucket, behind a regional load balancer on a
+ * static address that serves their gateway to callers anywhere. Runners
+ * reach each other and the database through the cluster's network tag.
+ * Requires a database on a GCP network — `ClusterGCP.Database`.
+ *
+ * Cloud Run can't host runners: runners take connections from each other.
+ *
+ * ### Running on Compute Engine
+ * **Example:** Two runners
+ * ```typescript
+ * export const Database = ClusterGCP.Database("Cluster");
+ * export const Default = ClusterGCP.ComputeEnginePool("Default", { replicas: 2 }).pipe(
+ *   Layer.provide(Database),
+ * );
+ * ```
+ *
+ * @primitive
+ */
+export const ComputeEnginePool = (id: string, options: ComputeEnginePoolOptions = {}) =>
+  makePoolLayer({
+    label: "ClusterGCP.ComputeEnginePool",
+    id,
+    settings: options,
+    expected: "a GCP network (ClusterGCP.Database)",
+    network: (database) => (database.network?.kind === "gcp" ? database.network : undefined),
+    // A static address, before the instances, so the URL never depends on it.
+    gateway: (_name, network) =>
+      Effect.gen(function* () {
+        const address = yield* Compute.Address(`${id}GatewayAddress`, { region: network.region });
+        return {
+          url: Output.interpolate`http://${address.address}:${GATEWAY_PORT}`,
+          address: address.address as Output.Output<string>,
+        };
+      }),
+    nodes: ({ network, gateway, program, environment }) =>
+      Effect.gen(function* () {
+        const zones = options.zones ?? [
+          `${network.region}-a`,
+          `${network.region}-b`,
+          `${network.region}-c`,
+        ];
+        // The gateway checks every call's token; the passthrough load
+        // balancer keeps the caller's address.
+        yield* Compute.Firewall(`${id}GatewayFirewall`, {
+          network: network.network,
+          direction: "INGRESS",
+          allowed: [{ protocol: "tcp", ports: [String(GATEWAY_PORT)] }],
+          sourceRanges: ["0.0.0.0/0"],
+          targetTags: [network.tag],
+        });
+        const bucket = yield* Storage.Bucket(`${id}Program`, {
+          location: network.region,
+          forceDestroy: true,
+          uniformBucketLevelAccess: true,
+        });
+        const runner = yield* Storage.Object(`${id}RunnerObject`, {
+          bucketName: bucket.bucketName,
+          key: "runner.mjs",
+          file: program.pipe(Output.map((p) => `${p.dir}/runner.mjs`)) as unknown as string,
+          contentType: "text/javascript",
+        });
+        const account = yield* IAM.ServiceAccount(`${id}RunnerAccount`, {
+          displayName: "Effect Cluster runners",
+        });
+        const grant = yield* IAM.Member(`${id}ProgramGrant`, {
+          kind: "storage.bucket",
+          name: bucket.bucketName,
+          role: "roles/storage.objectViewer",
+          member: Output.interpolate`serviceAccount:${account.email}`,
+        });
+        // A Docker env file: one `NAME=value` per line. Each runner finds its
+        // own address in the subnetwork's range.
+        const envFile = environment.pipe(
+          Output.map((env) =>
+            Object.entries({ ...env, CLUSTER_RUNNER_NETWORK: network.cidr })
+              .map(([name, value]) => `${name}=${String(value).replaceAll("\n", " ")}`)
+              .join("\n"),
+          ),
+        );
+        const metadata = Output.all(bucket.bucketName, runner.md5Hash, envFile, grant.role).pipe(
+          Output.map(([bucketName, hash, env]) => ({
+            "startup-script": startupScript({ bucket: bucketName }),
+            [HASH_KEY]: hash ?? "",
+            [ENV_KEY]: env,
+          })),
+        );
+        const instances = [];
+        for (let i = 0; i < (options.replicas ?? 2); i++) {
+          instances.push(
+            yield* Compute.Instance(`${id}Runner${i}`, {
+              zone: zones[i % zones.length]!,
+              machineType: options.machineType ?? "e2-small",
+              sourceImage: COS_IMAGE,
+              network: Output.interpolate`global/networks/${network.network}`,
+              subnetwork: Output.interpolate`regions/${network.region}/subnetworks/${network.subnetwork}`,
+              associatePublicIp: true,
+              tags: [network.tag],
+              serviceAccount: account.email,
+              oauthScopes: ["https://www.googleapis.com/auth/cloud-platform"],
+              metadata: metadata as unknown as Record<string, string>,
+            }),
+          );
+        }
+        const health = yield* Compute.HttpHealthCheck(`${id}GatewayHealth`, {
+          port: GATEWAY_PORT,
+          requestPath: "/health",
+        });
+        const pool = yield* Compute.TargetPool(`${id}Gateway`, {
+          region: network.region,
+          healthChecks: [health.selfLink as Output.Output<string>],
+          instances: instances.map((instance) => instance.selfLink as Output.Output<string>),
+        });
+        yield* Compute.ForwardingRule(`${id}GatewayRule`, {
+          region: network.region,
+          ipAddress: gateway.address,
+          ipProtocol: "TCP",
+          portRange: String(GATEWAY_PORT),
+          target: pool.selfLink as Output.Output<string>,
+          loadBalancingScheme: "EXTERNAL",
+        });
+      }),
+  });

package/src/GCP/Database.ts

@@ -0,0 +1,168 @@
+import * as Compute from "@alchemy.run/gcp/Compute";
+import { GcpEnvironment } from "@alchemy.run/gcp/Environment";
+import * as Output from "alchemy/Output";
+import { Random } from "alchemy/Random";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import { clusterShardGroups, Database as ClusterDatabase } from "../Database.ts";
+import { RUNNER_PORT } from "../internal/Gateway.ts";
+import { stackDnsLabel } from "../internal/Naming.ts";
+import type { Providers } from "../Providers.ts";
+
+export const COS_IMAGE = "projects/cos-cloud/global/images/family/cos-stable";
+const DISK_DEVICE = "cluster-data";
+const MOUNT_PATH = "/mnt/disks/postgres";
+
+export interface DatabaseProps {
+  /**
+   * Zone of the database instance.
+   * @default the region's `-a` zone
+   */
+  zone?: string;
+  /**
+   * Machine type of the database instance.
+   * @default "e2-small"
+   */
+  machineType?: string;
+  /**
+   * Size of the database's disk, in GB.
+   * @default 20
+   */
+  diskSize?: number;
+  /**
+   * Postgres image.
+   * @default "postgres:17-alpine"
+   */
+  image?: string;
+  /**
+   * IPv4 range of the cluster's subnetwork.
+   * @default "10.47.0.0/20"
+   */
+  cidrBlock?: string;
+}
+
+const shellQuote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`;
+
+/** Mount the data disk, then run Postgres on the host network. */
+const startupScript = (options: { image: string; password: string }) => `#!/bin/bash
+set -euo pipefail
+DEVICE=/dev/disk/by-id/google-${DISK_DEVICE}
+blkid "$DEVICE" >/dev/null || mkfs.ext4 -F "$DEVICE"
+mkdir -p ${MOUNT_PATH}
+mountpoint -q ${MOUNT_PATH} || mount "$DEVICE" ${MOUNT_PATH}
+# Container-Optimized OS drops inbound traffic except SSH: open Postgres.
+iptables -C INPUT -p tcp --dport 5432 -j ACCEPT 2>/dev/null || iptables -A INPUT -p tcp --dport 5432 -j ACCEPT
+docker rm -f postgres >/dev/null 2>&1 || true
+docker run -d --name postgres --restart always --network host \\
+  -v ${MOUNT_PATH}:/var/lib/postgresql/data \\
+  -e PGDATA=/var/lib/postgresql/data/pgdata \\
+  -e POSTGRES_DB=cluster \\
+  -e POSTGRES_PASSWORD=${shellQuote(options.password)} \\
+  ${shellQuote(options.image)}
+`;
+
+/**
+ * The database an Effect Cluster coordinates through, on Compute Engine,
+ * as a `Cluster.Database` Layer: Postgres on an instance with a persistent
+ * disk, in a VPC of its own that the pool's runners join. Only the runners reach
+ * it; runners reach each other on the runner port.
+ *
+ * ### Running the database on GCP
+ * **Example:** Postgres on Compute Engine
+ * ```typescript
+ * export const Database = ClusterGCP.Database("Cluster");
+ * ```
+ *
+ * @primitive
+ */
+export const Database = (id: string, props: DatabaseProps = {}) =>
+  Layer.effect(
+    ClusterDatabase,
+    Effect.gen(function* () {
+      if (globalThis.__ALCHEMY_RUNTIME__) {
+        // Runners read their database from their own environment at runtime.
+        return { url: Redacted.make(""), shardGroups: undefined as never };
+      }
+      const region = yield* GcpEnvironment.current.pipe(
+        Effect.map((env) => env.region),
+        Effect.orDie,
+      );
+      const zone = props.zone ?? `${region}-a`;
+      const cidr = props.cidrBlock ?? "10.47.0.0/20";
+      const poolTag = yield* stackDnsLabel(`${id}-pool`, 63);
+      const databaseTag = yield* stackDnsLabel(`${id}-db`, 63);
+      const network = yield* Compute.Network(`${id}Network`, { autoCreateSubnetworks: false });
+      const subnet = yield* Compute.Subnetwork(`${id}Subnet`, {
+        network: network.networkName,
+        region,
+        ipCidrRange: cidr,
+      });
+      yield* Compute.Firewall(`${id}DatabaseFirewall`, {
+        network: network.networkName,
+        direction: "INGRESS",
+        allowed: [{ protocol: "tcp", ports: ["5432"] }],
+        sourceTags: [poolTag],
+        targetTags: [databaseTag],
+      });
+      // Runners send each other messages for the shards they own.
+      yield* Compute.Firewall(`${id}RunnerFirewall`, {
+        network: network.networkName,
+        direction: "INGRESS",
+        allowed: [{ protocol: "tcp", ports: [String(RUNNER_PORT)] }],
+        sourceTags: [poolTag],
+        targetTags: [poolTag],
+      });
+      const password = yield* Random(`${id}Password`, { bytes: 24 }) as unknown as Effect.Effect<
+        Random,
+        never,
+        Providers
+      >;
+      const disk = yield* Compute.Disk(`${id}Disk`, {
+        zone,
+        type: "pd-balanced",
+        sizeGb: props.diskSize ?? 20,
+      });
+      const instance = yield* Compute.Instance(`${id}Postgres`, {
+        zone,
+        machineType: props.machineType ?? "e2-small",
+        sourceImage: COS_IMAGE,
+        network: Output.interpolate`global/networks/${network.networkName}`,
+        subnetwork: Output.interpolate`regions/${region}/subnetworks/${subnet.subnetworkName}`,
+        // Pulls the Postgres image.
+        associatePublicIp: true,
+        tags: [databaseTag],
+        attachedDisks: [
+          { source: disk.selfLink as Output.Output<string>, deviceName: DISK_DEVICE },
+        ],
+        metadata: {
+          "startup-script": password.text.pipe(
+            Output.map((text) =>
+              startupScript({
+                image: props.image ?? "postgres:17-alpine",
+                password: Redacted.value(text),
+              }),
+            ),
+          ) as unknown as string,
+        },
+      });
+      return {
+        url: Output.all(instance.networkIP, password.text).pipe(
+          Output.map(([ip, text]) =>
+            Redacted.make(
+              `postgres://postgres:${encodeURIComponent(text.pipe(Redacted.value))}@${ip}:5432/cluster`,
+            ),
+          ),
+        ),
+        network: {
+          kind: "gcp" as const,
+          network: network.networkName,
+          subnetwork: subnet.subnetworkName,
+          region,
+          cidr,
+          tag: poolTag,
+        },
+        shardGroups: yield* clusterShardGroups(id),
+      };
+    }),
+  );

package/src/GCP/index.ts

@@ -0,0 +1,6 @@
+/**
+ * @alchemy.run/effect-cluster/GCP — run an Effect Cluster's database and runners
+ * on Compute Engine. Requires `@alchemy.run/gcp`.
+ */
+export * from "./ComputeEnginePool.ts";
+export * from "./Database.ts";