@alchemy.run/effect-cluster 0.0.0 → 2.0.0-beta.87

package/src/Database.ts

@@ -0,0 +1,116 @@
+import type { Input } from "alchemy/Input";
+import * as Config from "effect/Config";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import type { Providers } from "./Providers.ts";
+import { ShardGroups } from "./ShardGroups.ts";
+
+/**
+ * The private network runners must join to reach the database and each
+ * other. A pool Layer matches on `kind` and refuses a network it cannot
+ * join.
+ */
+export type ClusterNetwork =
+  | {
+      /** A Docker network; runner containers join it by name. */
+      readonly kind: "docker";
+      readonly name: Input<string>;
+    }
+  | {
+      /** An AWS VPC; runners run in its subnets, in the pool's security group. */
+      readonly kind: "aws";
+      readonly vpcId: Input<string>;
+      readonly subnetIds: Input<string>[];
+      /** The group every runner joins: it reaches the database and the other runners. */
+      readonly securityGroupId: Input<string>;
+    }
+  | {
+      /** A Fly private network; runners' Apps join it by name. */
+      readonly kind: "fly";
+      readonly network: Input<string>;
+    }
+  | {
+      /** A Hetzner private Network; runners' Servers attach to it. */
+      readonly kind: "hetzner";
+      readonly networkId: Input<number>;
+      /** Its IPv4 range, where each runner finds its own address. */
+      readonly ipRange: string;
+      readonly location: string;
+    }
+  | {
+      /** A GCP VPC; runners' instances join its subnetwork. */
+      readonly kind: "gcp";
+      readonly network: Input<string>;
+      readonly subnetwork: Input<string>;
+      readonly region: string;
+      /** The subnetwork's IPv4 range, where each runner finds its own address. */
+      readonly cidr: string;
+      /** The network tag every runner carries: it reaches the database and the other runners. */
+      readonly tag: string;
+    }
+  | {
+      /** A Kubernetes cluster; runners run as a Deployment in its namespace. */
+      readonly kind: "kubernetes";
+      /** The cluster, as `@alchemy.run/kubernetes` takes it. */
+      readonly cluster: unknown;
+      readonly namespace: string;
+    };
+
+/** What every database Layer gives the pool it is provided to. */
+export interface DatabaseShape {
+  /**
+   * The Postgres connection URL runners use for runner registration,
+   * shard locks and persisted messages.
+   */
+  readonly url: Input<Redacted.Redacted<string>>;
+  /** The network runners join to reach it, when it is private. */
+  readonly network?: ClusterNetwork;
+  /** Every shard group in the cluster, which each pool registers with. */
+  readonly shardGroups: ShardGroups;
+}
+
+/**
+ * The database an Effect Cluster coordinates through: runners register in
+ * it, lock their shards in it, and keep persisted messages in it. Each
+ * host's `Database` Layer provides it.
+ */
+export class Database extends Context.Service<Database, DatabaseShape>()("Cluster.Database") {}
+
+export interface DatabaseProps {
+  /** A Postgres connection URL. */
+  url: Redacted.Redacted<string> | Config.Config<Redacted.Redacted<string>>;
+}
+
+/**
+ * A Postgres database you already run, as an Effect Cluster `Database`
+ * Layer: Neon, PlanetScale, RDS, or any Postgres the pool can reach.
+ *
+ * ### Using an existing database
+ * **Example:** A Postgres URL from config
+ * ```typescript
+ * export const Database = Cluster.Database.fromUrl({
+ *   url: Config.Redacted("CLUSTER_DATABASE_URL"),
+ * });
+ * ```
+ *
+ * @primitive
+ */
+export const fromUrl = (
+  props: DatabaseProps,
+): Layer.Layer<Database, Config.ConfigError, Providers> =>
+  Layer.effect(
+    Database,
+    Effect.gen(function* () {
+      if (globalThis.__ALCHEMY_RUNTIME__) {
+        // Runners read their database from their own environment at runtime.
+        return { url: Redacted.make(""), shardGroups: undefined as never };
+      }
+      const url = Config.isConfig(props.url) ? yield* props.url : props.url;
+      return { url, shardGroups: yield* clusterShardGroups("Database") };
+    }),
+  );
+
+/** The shard groups resource every database Layer declares for its cluster. @internal */
+export const clusterShardGroups = (id: string) => ShardGroups(`${id}ShardGroups`, {});

package/src/Docker/Database.ts

@@ -0,0 +1,94 @@
+import * as Docker from "@alchemy.run/docker";
+import * as Output from "alchemy/Output";
+import { Random } from "alchemy/Random";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Redacted from "effect/Redacted";
+import { clusterShardGroups, Database as ClusterDatabase } from "../Database.ts";
+import type { Providers } from "../Providers.ts";
+
+/** The Postgres release `ClusterDocker.Database` runs by default. */
+export const DEFAULT_POSTGRES_IMAGE = "postgres:17-alpine";
+
+export interface DatabaseProps {
+  /**
+   * Postgres image.
+   * @default "postgres:17-alpine"
+   */
+  image?: string;
+  /**
+   * Publish Postgres on this host port, to inspect it from your machine.
+   * @default not published
+   */
+  port?: number;
+  /**
+   * The Docker engine to run on: a context name, a `Docker.Context`, or a
+   * `Docker.Swarm`.
+   * @default the current Docker context
+   */
+  context?: Docker.Docker.EngineRef;
+}
+
+/**
+ * The database an Effect Cluster coordinates through, on a Docker swarm, as
+ * an `Cluster.Database` Layer: Postgres as a one-replica swarm service
+ * with its data on a named volume, on an overlay network the pool's runners join.
+ * The Docker engine must be in swarm mode (`docker swarm init`).
+ *
+ * One Postgres on one disk is meant for development and tests.
+ *
+ * ### Running a local database
+ * **Example:** Postgres for local entities
+ * ```typescript
+ * export const Database = ClusterDocker.Database("Cluster");
+ * ```
+ *
+ * @primitive
+ */
+export const Database = (
+  id: string,
+  props: DatabaseProps = {},
+): Layer.Layer<ClusterDatabase, never, Providers | Docker.Providers> =>
+  Layer.effect(
+    ClusterDatabase,
+    Effect.gen(function* () {
+      if (globalThis.__ALCHEMY_RUNTIME__) {
+        // Runners read their database from their own environment at runtime.
+        return { url: Redacted.make(""), shardGroups: undefined as never };
+      }
+      const context = props.context === undefined ? {} : { context: props.context };
+      const password = yield* Random(`${id}Password`, { bytes: 24 }) as unknown as Effect.Effect<
+        Random,
+        never,
+        Providers
+      >;
+      const network = yield* Docker.Network(`${id}Network`, { driver: "overlay", ...context });
+      const volumeContext = Docker.dockerEngineContextName(props.context);
+      const data = yield* Docker.Volume(
+        `${id}Data`,
+        volumeContext === undefined ? {} : { context: volumeContext },
+      );
+      const alias = "cluster-db";
+      const db = yield* Docker.Service(`${id}Postgres`, {
+        image: props.image ?? DEFAULT_POSTGRES_IMAGE,
+        ...context,
+        environment: { POSTGRES_PASSWORD: password.text, POSTGRES_DB: "cluster" },
+        ports: props.port === undefined ? [] : [{ external: props.port, internal: 5432 }],
+        volumes: [{ hostPath: data.name, containerPath: "/var/lib/postgresql/data" }],
+        networks: [{ name: network.name, aliases: [alias] }],
+        replicas: 1,
+      });
+      return {
+        // Ordered after the service: runners connect once it exists.
+        url: Output.all(db.name, password.text).pipe(
+          Output.map(([, text]) =>
+            Redacted.make(
+              `postgres://postgres:${encodeURIComponent(text.pipe(Redacted.value))}@${alias}:5432/cluster`,
+            ),
+          ),
+        ),
+        network: { kind: "docker" as const, name: network.name },
+        shardGroups: yield* clusterShardGroups(id),
+      };
+    }),
+  );

package/src/Docker/SwarmPool.ts

@@ -0,0 +1,83 @@
+import * as Docker from "@alchemy.run/docker";
+import * as Output from "alchemy/Output";
+import type { Stack } from "alchemy/Stack";
+import * as Effect from "effect/Effect";
+import type * as Layer from "effect/Layer";
+import type { Database } from "../Database.ts";
+import { GATEWAY_PORT } from "../internal/Gateway.ts";
+import { makePoolLayer, type PoolSettings } from "../internal/PoolHost.ts";
+import type { PoolHost } from "../internal/PoolHostService.ts";
+import type { Providers } from "../Providers.ts";
+
+export interface SwarmPoolOptions extends PoolSettings {
+  /**
+   * Number of runners.
+   * @default 1
+   */
+  replicas?: number;
+  /**
+   * Publish the gateway on this host port, so code outside the swarm (a
+   * test, a script) can call the entities at `http://localhost:<port>`.
+   * @default not published
+   */
+  gatewayPort?: number;
+}
+
+/**
+ * Run a pool of Effect Cluster runners on a Docker swarm: a swarm service whose
+ * tasks host every `Cluster.Entity` given this pool, on the
+ * database's overlay network. Runners find each other through the
+ * database and talk on the network; the gateway is served on the same
+ * network. Requires an `Cluster.Database` on a Docker network —
+ * `ClusterDocker.Database`.
+ *
+ * Every entity given the same pool shares it: declare the pool once
+ * and provide them to each class.
+ *
+ * ### Running entities on Docker
+ * **Example:** Two runners on the local swarm
+ * ```typescript
+ * export const Database = ClusterDocker.Database("Cluster");
+ * export const Default = ClusterDocker.SwarmPool("Default", { replicas: 2 }).pipe(
+ *   Layer.provide(Database),
+ * );
+ * ```
+ *
+ * @primitive
+ */
+export const SwarmPool = (
+  id: string,
+  options: SwarmPoolOptions = {},
+): Layer.Layer<PoolHost, never, Database | Stack | Providers | Docker.Providers> =>
+  makePoolLayer({
+    label: "ClusterDocker.SwarmPool",
+    id,
+    settings: options,
+    expected: "a Docker network (ClusterDocker.Database)",
+    network: (database) => (database.network?.kind === "docker" ? database.network : undefined),
+    // Callers on the swarm reach the gateway by the service's name.
+    gateway: (name) => Effect.succeed({ url: `http://${name}:${GATEWAY_PORT}` }),
+    nodes: ({ network, name, program, environment }) =>
+      Effect.gen(function* () {
+        // Tagged with the program's hash, so a changed program is a new image.
+        const image = yield* Docker.Image(`${id}RunnerImage`, {
+          build: { context: program.pipe(Output.map((p) => p.dir)) as unknown as string },
+          tag: program.pipe(Output.map((p) => p.hash)) as unknown as string,
+          skipPush: true,
+        });
+        yield* Docker.Service(`${id}Pool`, {
+          name,
+          image: image.imageRef,
+          replicas: options.replicas ?? 1,
+          networks: [{ name: network.name }],
+          ports:
+            options.gatewayPort === undefined
+              ? []
+              : [{ external: options.gatewayPort, internal: GATEWAY_PORT }],
+          // Each runner finds its own address among the service's tasks.
+          environment: environment.pipe(
+            Output.map((env) => ({ ...env, CLUSTER_PEERS: `tasks.${name}` })),
+          ) as unknown as Record<string, string>,
+        });
+      }),
+  });

package/src/Docker/index.ts

@@ -0,0 +1,6 @@
+/**
+ * @alchemy.run/effect-cluster/Docker — run an Effect Cluster's database and
+ * runners on a Docker swarm. Requires `@alchemy.run/docker`.
+ */
+export * from "./Database.ts";
+export * from "./SwarmPool.ts";