@alchemy.run/effect-cluster 0.0.0 → 2.0.0-beta.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +101 -4
  3. package/lib/AWS/Database.d.ts +54 -0
  4. package/lib/AWS/Database.d.ts.map +1 -0
  5. package/lib/AWS/Database.js +96 -0
  6. package/lib/AWS/Database.js.map +1 -0
  7. package/lib/AWS/EcsPool.d.ts +56 -0
  8. package/lib/AWS/EcsPool.d.ts.map +1 -0
  9. package/lib/AWS/EcsPool.js +87 -0
  10. package/lib/AWS/EcsPool.js.map +1 -0
  11. package/lib/AWS/index.d.ts +7 -0
  12. package/lib/AWS/index.d.ts.map +1 -0
  13. package/lib/AWS/index.js +7 -0
  14. package/lib/AWS/index.js.map +1 -0
  15. package/lib/AWS/internal/Gateway.d.ts +23 -0
  16. package/lib/AWS/internal/Gateway.d.ts.map +1 -0
  17. package/lib/AWS/internal/Gateway.js +70 -0
  18. package/lib/AWS/internal/Gateway.js.map +1 -0
  19. package/lib/Cloudflare/DurableObjectPool.d.ts +42 -0
  20. package/lib/Cloudflare/DurableObjectPool.d.ts.map +1 -0
  21. package/lib/Cloudflare/DurableObjectPool.js +136 -0
  22. package/lib/Cloudflare/DurableObjectPool.js.map +1 -0
  23. package/lib/Cloudflare/PoolWorker.d.ts +186 -0
  24. package/lib/Cloudflare/PoolWorker.d.ts.map +1 -0
  25. package/lib/Cloudflare/PoolWorker.js +75 -0
  26. package/lib/Cloudflare/PoolWorker.js.map +1 -0
  27. package/lib/Cloudflare/index.d.ts +7 -0
  28. package/lib/Cloudflare/index.d.ts.map +1 -0
  29. package/lib/Cloudflare/index.js +7 -0
  30. package/lib/Cloudflare/index.js.map +1 -0
  31. package/lib/Cloudflare/internal/Objects.d.ts +38 -0
  32. package/lib/Cloudflare/internal/Objects.d.ts.map +1 -0
  33. package/lib/Cloudflare/internal/Objects.js +102 -0
  34. package/lib/Cloudflare/internal/Objects.js.map +1 -0
  35. package/lib/Cron.d.ts +143 -0
  36. package/lib/Cron.d.ts.map +1 -0
  37. package/lib/Cron.js +83 -0
  38. package/lib/Cron.js.map +1 -0
  39. package/lib/Database.d.ts +95 -0
  40. package/lib/Database.d.ts.map +1 -0
  41. package/lib/Database.js +38 -0
  42. package/lib/Database.js.map +1 -0
  43. package/lib/Docker/Database.d.ts +42 -0
  44. package/lib/Docker/Database.d.ts.map +1 -0
  45. package/lib/Docker/Database.js +53 -0
  46. package/lib/Docker/Database.js.map +1 -0
  47. package/lib/Docker/SwarmPool.d.ts +44 -0
  48. package/lib/Docker/SwarmPool.d.ts.map +1 -0
  49. package/lib/Docker/SwarmPool.js +56 -0
  50. package/lib/Docker/SwarmPool.js.map +1 -0
  51. package/lib/Docker/index.d.ts +7 -0
  52. package/lib/Docker/index.d.ts.map +1 -0
  53. package/lib/Docker/index.js +7 -0
  54. package/lib/Docker/index.js.map +1 -0
  55. package/lib/Entity.d.ts +348 -0
  56. package/lib/Entity.d.ts.map +1 -0
  57. package/lib/Entity.js +308 -0
  58. package/lib/Entity.js.map +1 -0
  59. package/lib/Fly/Database.d.ts +50 -0
  60. package/lib/Fly/Database.d.ts.map +1 -0
  61. package/lib/Fly/Database.js +56 -0
  62. package/lib/Fly/Database.js.map +1 -0
  63. package/lib/Fly/MachinePool.d.ts +39 -0
  64. package/lib/Fly/MachinePool.d.ts.map +1 -0
  65. package/lib/Fly/MachinePool.js +51 -0
  66. package/lib/Fly/MachinePool.js.map +1 -0
  67. package/lib/Fly/index.d.ts +7 -0
  68. package/lib/Fly/index.d.ts.map +1 -0
  69. package/lib/Fly/index.js +7 -0
  70. package/lib/Fly/index.js.map +1 -0
  71. package/lib/GCP/ComputeEnginePool.d.ts +41 -0
  72. package/lib/GCP/ComputeEnginePool.d.ts.map +1 -0
  73. package/lib/GCP/ComputeEnginePool.js +172 -0
  74. package/lib/GCP/ComputeEnginePool.js.map +1 -0
  75. package/lib/GCP/Database.d.ts +48 -0
  76. package/lib/GCP/Database.d.ts.map +1 -0
  77. package/lib/GCP/Database.js +115 -0
  78. package/lib/GCP/Database.js.map +1 -0
  79. package/lib/GCP/index.d.ts +7 -0
  80. package/lib/GCP/index.d.ts.map +1 -0
  81. package/lib/GCP/index.js +7 -0
  82. package/lib/GCP/index.js.map +1 -0
  83. package/lib/Hetzner/Database.d.ts +48 -0
  84. package/lib/Hetzner/Database.d.ts.map +1 -0
  85. package/lib/Hetzner/Database.js +91 -0
  86. package/lib/Hetzner/Database.js.map +1 -0
  87. package/lib/Hetzner/ServerPool.d.ts +35 -0
  88. package/lib/Hetzner/ServerPool.d.ts.map +1 -0
  89. package/lib/Hetzner/ServerPool.js +98 -0
  90. package/lib/Hetzner/ServerPool.js.map +1 -0
  91. package/lib/Hetzner/index.d.ts +7 -0
  92. package/lib/Hetzner/index.d.ts.map +1 -0
  93. package/lib/Hetzner/index.js +7 -0
  94. package/lib/Hetzner/index.js.map +1 -0
  95. package/lib/Host.d.ts +60 -0
  96. package/lib/Host.d.ts.map +1 -0
  97. package/lib/Host.js +31 -0
  98. package/lib/Host.js.map +1 -0
  99. package/lib/Kubernetes/Database.d.ts +40 -0
  100. package/lib/Kubernetes/Database.d.ts.map +1 -0
  101. package/lib/Kubernetes/Database.js +56 -0
  102. package/lib/Kubernetes/Database.js.map +1 -0
  103. package/lib/Kubernetes/DeploymentPool.d.ts +54 -0
  104. package/lib/Kubernetes/DeploymentPool.d.ts.map +1 -0
  105. package/lib/Kubernetes/DeploymentPool.js +49 -0
  106. package/lib/Kubernetes/DeploymentPool.js.map +1 -0
  107. package/lib/Kubernetes/index.d.ts +7 -0
  108. package/lib/Kubernetes/index.d.ts.map +1 -0
  109. package/lib/Kubernetes/index.js +7 -0
  110. package/lib/Kubernetes/index.js.map +1 -0
  111. package/lib/Pool.d.ts +66 -0
  112. package/lib/Pool.d.ts.map +1 -0
  113. package/lib/Pool.js +47 -0
  114. package/lib/Pool.js.map +1 -0
  115. package/lib/Providers.d.ts +19 -0
  116. package/lib/Providers.d.ts.map +1 -0
  117. package/lib/Providers.js +31 -0
  118. package/lib/Providers.js.map +1 -0
  119. package/lib/ShardGroups.d.ts +24 -0
  120. package/lib/ShardGroups.d.ts.map +1 -0
  121. package/lib/ShardGroups.js +19 -0
  122. package/lib/ShardGroups.js.map +1 -0
  123. package/lib/Singleton.d.ts +110 -0
  124. package/lib/Singleton.d.ts.map +1 -0
  125. package/lib/Singleton.js +60 -0
  126. package/lib/Singleton.js.map +1 -0
  127. package/lib/Workflow.d.ts +251 -0
  128. package/lib/Workflow.d.ts.map +1 -0
  129. package/lib/Workflow.js +288 -0
  130. package/lib/Workflow.js.map +1 -0
  131. package/lib/index.d.ts +12 -0
  132. package/lib/index.d.ts.map +1 -0
  133. package/lib/index.js +12 -0
  134. package/lib/index.js.map +1 -0
  135. package/lib/internal/Bootstrap.d.ts +28 -0
  136. package/lib/internal/Bootstrap.d.ts.map +1 -0
  137. package/lib/internal/Bootstrap.js +257 -0
  138. package/lib/internal/Bootstrap.js.map +1 -0
  139. package/lib/internal/EntityRuntime.d.ts +40 -0
  140. package/lib/internal/EntityRuntime.d.ts.map +1 -0
  141. package/lib/internal/EntityRuntime.js +698 -0
  142. package/lib/internal/EntityRuntime.js.map +1 -0
  143. package/lib/internal/Gateway.d.ts +82 -0
  144. package/lib/internal/Gateway.d.ts.map +1 -0
  145. package/lib/internal/Gateway.js +219 -0
  146. package/lib/internal/Gateway.js.map +1 -0
  147. package/lib/internal/HostedPlatform.d.ts +36 -0
  148. package/lib/internal/HostedPlatform.d.ts.map +1 -0
  149. package/lib/internal/HostedPlatform.js +36 -0
  150. package/lib/internal/HostedPlatform.js.map +1 -0
  151. package/lib/internal/HostedRuntime.d.ts +58 -0
  152. package/lib/internal/HostedRuntime.d.ts.map +1 -0
  153. package/lib/internal/HostedRuntime.js +72 -0
  154. package/lib/internal/HostedRuntime.js.map +1 -0
  155. package/lib/internal/Naming.d.ts +12 -0
  156. package/lib/internal/Naming.d.ts.map +1 -0
  157. package/lib/internal/Naming.js +40 -0
  158. package/lib/internal/Naming.js.map +1 -0
  159. package/lib/internal/ObjectSharding.d.ts +141 -0
  160. package/lib/internal/ObjectSharding.d.ts.map +1 -0
  161. package/lib/internal/ObjectSharding.js +337 -0
  162. package/lib/internal/ObjectSharding.js.map +1 -0
  163. package/lib/internal/PoolHost.d.ts +80 -0
  164. package/lib/internal/PoolHost.d.ts.map +1 -0
  165. package/lib/internal/PoolHost.js +203 -0
  166. package/lib/internal/PoolHost.js.map +1 -0
  167. package/lib/internal/PoolHostService.d.ts +25 -0
  168. package/lib/internal/PoolHostService.d.ts.map +1 -0
  169. package/lib/internal/PoolHostService.js +10 -0
  170. package/lib/internal/PoolHostService.js.map +1 -0
  171. package/lib/internal/Wire.d.ts +60 -0
  172. package/lib/internal/Wire.d.ts.map +1 -0
  173. package/lib/internal/Wire.js +36 -0
  174. package/lib/internal/Wire.js.map +1 -0
  175. package/lib/internal/WorkflowGateway.d.ts +36 -0
  176. package/lib/internal/WorkflowGateway.d.ts.map +1 -0
  177. package/lib/internal/WorkflowGateway.js +124 -0
  178. package/lib/internal/WorkflowGateway.js.map +1 -0
  179. package/lib/internal/external-runner.d.ts +2 -0
  180. package/lib/internal/external-runner.d.ts.map +1 -0
  181. package/lib/internal/external-runner.js +7 -0
  182. package/lib/internal/external-runner.js.map +1 -0
  183. package/lib/internal/runner.d.ts +2 -0
  184. package/lib/internal/runner.d.ts.map +1 -0
  185. package/lib/internal/runner.js +5 -0
  186. package/lib/internal/runner.js.map +1 -0
  187. package/package.json +128 -3
  188. package/src/AWS/Database.ts +139 -0
  189. package/src/AWS/EcsPool.ts +118 -0
  190. package/src/AWS/index.ts +6 -0
  191. package/src/AWS/internal/Gateway.ts +82 -0
  192. package/src/Cloudflare/DurableObjectPool.ts +198 -0
  193. package/src/Cloudflare/PoolWorker.ts +124 -0
  194. package/src/Cloudflare/index.ts +6 -0
  195. package/src/Cloudflare/internal/Objects.ts +160 -0
  196. package/src/Cron.ts +202 -0
  197. package/src/Database.ts +116 -0
  198. package/src/Docker/Database.ts +94 -0
  199. package/src/Docker/SwarmPool.ts +83 -0
  200. package/src/Docker/index.ts +6 -0
  201. package/src/Entity.ts +621 -0
  202. package/src/Fly/Database.ts +102 -0
  203. package/src/Fly/MachinePool.ts +72 -0
  204. package/src/Fly/index.ts +6 -0
  205. package/src/GCP/ComputeEnginePool.ts +203 -0
  206. package/src/GCP/Database.ts +168 -0
  207. package/src/GCP/index.ts +6 -0
  208. package/src/Hetzner/Database.ts +142 -0
  209. package/src/Hetzner/ServerPool.ts +116 -0
  210. package/src/Hetzner/index.ts +6 -0
  211. package/src/Host.ts +68 -0
  212. package/src/Kubernetes/Database.ts +89 -0
  213. package/src/Kubernetes/DeploymentPool.ts +77 -0
  214. package/src/Kubernetes/index.ts +6 -0
  215. package/src/Pool.ts +104 -0
  216. package/src/Providers.ts +49 -0
  217. package/src/ShardGroups.ts +41 -0
  218. package/src/Singleton.ts +153 -0
  219. package/src/Workflow.ts +556 -0
  220. package/src/index.ts +11 -0
  221. package/src/internal/Bootstrap.ts +368 -0
  222. package/src/internal/EntityRuntime.ts +996 -0
  223. package/src/internal/Gateway.ts +367 -0
  224. package/src/internal/HostedPlatform.ts +83 -0
  225. package/src/internal/HostedRuntime.ts +118 -0
  226. package/src/internal/Naming.ts +44 -0
  227. package/src/internal/ObjectSharding.ts +578 -0
  228. package/src/internal/PoolHost.ts +330 -0
  229. package/src/internal/PoolHostService.ts +25 -0
  230. package/src/internal/Wire.ts +83 -0
  231. package/src/internal/WorkflowGateway.ts +188 -0
  232. package/src/internal/external-runner.ts +10 -0
  233. package/src/internal/runner.ts +5 -0
@@ -0,0 +1,142 @@
1
+ import * as Hetzner from "@alchemy.run/hetzner";
2
+ import * as Output from "alchemy/Output";
3
+ import { Random } from "alchemy/Random";
4
+ import * as Effect from "effect/Effect";
5
+ import * as Layer from "effect/Layer";
6
+ import * as Redacted from "effect/Redacted";
7
+ import { clusterShardGroups, Database as ClusterDatabase } from "../Database.ts";
8
+ import type { Providers } from "../Providers.ts";
9
+
10
+ export interface DatabaseProps {
11
+ /**
12
+ * Hetzner location of the database and the cluster's Network.
13
+ * @default "nbg1"
14
+ */
15
+ location?: string;
16
+ /**
17
+ * Server type of the database.
18
+ * @default "cx23"
19
+ */
20
+ serverType?: string;
21
+ /**
22
+ * Size of the database's Volume, in GB (at least 10).
23
+ * @default 10
24
+ */
25
+ diskSize?: number;
26
+ /**
27
+ * Postgres image.
28
+ * @default "postgres:17-alpine"
29
+ */
30
+ image?: string;
31
+ /**
32
+ * IPv4 range of the cluster's private Network.
33
+ * @default "10.46.0.0/16"
34
+ */
35
+ ipRange?: string;
36
+ }
37
+
38
+ /** The network zone a location belongs to. */
39
+ const networkZoneOf = (location: string) =>
40
+ location.startsWith("ash")
41
+ ? "us-east"
42
+ : location.startsWith("hil")
43
+ ? "us-west"
44
+ : location.startsWith("sin")
45
+ ? "ap-southeast"
46
+ : "eu-central";
47
+
48
+ const shellQuote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`;
49
+
50
+ /** Mount the Volume and run Postgres on it, on the Server's own network. */
51
+ const userData = (options: { volumeId: number; image: string; password: string }) => `#!/bin/bash
52
+ set -euo pipefail
53
+ DEVICE=/dev/disk/by-id/scsi-0HC_Volume_${options.volumeId}
54
+ for _ in $(seq 1 60); do [ -e "$DEVICE" ] && break; sleep 2; done
55
+ blkid "$DEVICE" >/dev/null || mkfs.ext4 -F "$DEVICE"
56
+ mkdir -p /mnt/postgres
57
+ grep -q " /mnt/postgres " /etc/fstab || echo "$DEVICE /mnt/postgres ext4 discard,nofail,defaults 0 0" >> /etc/fstab
58
+ mountpoint -q /mnt/postgres || mount /mnt/postgres
59
+ until docker info >/dev/null 2>&1; do sleep 2; done
60
+ docker rm -f postgres >/dev/null 2>&1 || true
61
+ docker run -d --name postgres --restart always --network host \\
62
+ -v /mnt/postgres:/var/lib/postgresql/data \\
63
+ -e PGDATA=/var/lib/postgresql/data/pgdata \\
64
+ -e POSTGRES_DB=cluster \\
65
+ -e POSTGRES_PASSWORD=${shellQuote(options.password)} \\
66
+ ${shellQuote(options.image)}
67
+ `;
68
+
69
+ /**
70
+ * The database an Effect Cluster coordinates through, on Hetzner Cloud, as
71
+ * a `Cluster.Database` Layer: Postgres on a Server with a Volume, on a
72
+ * private Network that the pool's Servers attach to. A firewall drops every
73
+ * inbound connection on the Server's public address; runners reach it
74
+ * over the Network.
75
+ *
76
+ * ### Running the database on Hetzner
77
+ * **Example:** Postgres in Nuremberg
78
+ * ```typescript
79
+ * export const Database = ClusterHetzner.Database("Cluster");
80
+ * ```
81
+ *
82
+ * @primitive
83
+ */
84
+ export const Database = (id: string, props: DatabaseProps = {}) =>
85
+ Layer.effect(
86
+ ClusterDatabase,
87
+ Effect.gen(function* () {
88
+ if (globalThis.__ALCHEMY_RUNTIME__) {
89
+ // Runners read their database from their own environment at runtime.
90
+ return { url: Redacted.make(""), shardGroups: undefined as never };
91
+ }
92
+ const location = props.location ?? "nbg1";
93
+ const ipRange = props.ipRange ?? "10.46.0.0/16";
94
+ const password = yield* Random(`${id}Password`, { bytes: 24 }) as unknown as Effect.Effect<
95
+ Random,
96
+ never,
97
+ Providers
98
+ >;
99
+ const network = yield* Hetzner.Network(`${id}Network`, {
100
+ ipRange,
101
+ subnets: [{ type: "cloud", networkZone: networkZoneOf(location) }],
102
+ });
103
+ // No inbound rules: only the Network reaches Postgres.
104
+ const firewall = yield* Hetzner.Firewall(`${id}Firewall`, {});
105
+ const volume = yield* Hetzner.Volume(`${id}Data`, {
106
+ size: props.diskSize ?? 10,
107
+ format: "ext4",
108
+ location,
109
+ });
110
+ const server = yield* Hetzner.Server(`${id}Postgres`, {
111
+ serverType: props.serverType ?? "cx23",
112
+ // Hetzner's Ubuntu image with Docker preinstalled.
113
+ image: "docker-ce",
114
+ location,
115
+ // IPv4 pulls the Postgres image; IPv6 would only spend an address.
116
+ enableIpv6: false,
117
+ networks: [network],
118
+ firewalls: [firewall],
119
+ volumes: [volume],
120
+ userData: Output.all(volume.id, password.text).pipe(
121
+ Output.map(([volumeId, text]) =>
122
+ userData({
123
+ volumeId,
124
+ image: props.image ?? "postgres:17-alpine",
125
+ password: Redacted.value(text),
126
+ }),
127
+ ),
128
+ ) as unknown as string,
129
+ });
130
+ return {
131
+ url: Output.all(server.privateIps, password.text).pipe(
132
+ Output.map(([ips, text]) =>
133
+ Redacted.make(
134
+ `postgres://postgres:${encodeURIComponent(text.pipe(Redacted.value))}@${ips[0]}:5432/cluster`,
135
+ ),
136
+ ),
137
+ ),
138
+ network: { kind: "hetzner" as const, networkId: network.networkId, ipRange, location },
139
+ shardGroups: yield* clusterShardGroups(id),
140
+ };
141
+ }),
142
+ );
@@ -0,0 +1,116 @@
1
+ import * as Hetzner from "@alchemy.run/hetzner";
2
+ import * as Output from "alchemy/Output";
3
+ import * as Effect from "effect/Effect";
4
+ import { GATEWAY_PORT } from "../internal/Gateway.ts";
5
+ import { makePoolLayer, type PoolSettings } from "../internal/PoolHost.ts";
6
+
7
+ export interface ServerPoolOptions extends PoolSettings {
8
+ /**
9
+ * Number of runners, one Server each.
10
+ * @default 2
11
+ */
12
+ replicas?: number;
13
+ /**
14
+ * Server type of each runner.
15
+ * @default "cx23"
16
+ */
17
+ serverType?: string;
18
+ }
19
+
20
+ /** The label the gateway's Load Balancer finds the pool's Servers by. */
21
+ const POOL_LABEL = "cluster-pool";
22
+
23
+ /**
24
+ * Run a pool of Effect Cluster runners on Hetzner Cloud: Servers on the
25
+ * database's private Network, each running every class given these
26
+ * runners as a `Hetzner.Service`, behind a Load Balancer that serves their
27
+ * gateway to callers anywhere. Runners talk to each other and the database
28
+ * over the Network; their firewall only lets SSH in, which deploys them.
29
+ * Requires a database on a Hetzner Network — `ClusterHetzner.Database`.
30
+ *
31
+ * ### Running on Hetzner
32
+ * **Example:** Two runners
33
+ * ```typescript
34
+ * export const Database = ClusterHetzner.Database("Cluster");
35
+ * export const Default = ClusterHetzner.ServerPool("Default", { replicas: 2 }).pipe(
36
+ * Layer.provide(Database),
37
+ * );
38
+ * ```
39
+ *
40
+ * @primitive
41
+ */
42
+ export const ServerPool = (id: string, options: ServerPoolOptions = {}) =>
43
+ makePoolLayer({
44
+ label: "ClusterHetzner.ServerPool",
45
+ id,
46
+ settings: options,
47
+ expected: "a Hetzner Network (ClusterHetzner.Database)",
48
+ network: (database) => (database.network?.kind === "hetzner" ? database.network : undefined),
49
+ // The Load Balancer finds the Servers by label, so it needs none of them.
50
+ gateway: (name, network) =>
51
+ Effect.gen(function* () {
52
+ const balancer = yield* Hetzner.LoadBalancer(`${id}Gateway`, {
53
+ loadBalancerType: "lb11",
54
+ location: network.location,
55
+ // A stub of the database's Network, by id.
56
+ networks: [{ networkId: network.networkId } as unknown as Hetzner.Network],
57
+ services: [
58
+ {
59
+ protocol: "tcp",
60
+ listenPort: 80,
61
+ destinationPort: GATEWAY_PORT,
62
+ healthCheck: {
63
+ protocol: "http",
64
+ port: GATEWAY_PORT,
65
+ interval: 10,
66
+ timeout: 5,
67
+ retries: 3,
68
+ http: { path: "/health" },
69
+ },
70
+ },
71
+ ],
72
+ targets: [
73
+ { type: "label_selector", selector: `${POOL_LABEL}=${name}`, usePrivateIp: true },
74
+ ],
75
+ });
76
+ return { url: Output.interpolate`http://${balancer.ipv4}` };
77
+ }),
78
+ nodes: ({ network, name, program, environment }) =>
79
+ Effect.gen(function* () {
80
+ const firewall = yield* Hetzner.Firewall(`${id}Firewall`, {
81
+ rules: [
82
+ {
83
+ description: "SSH, which deploys the pool",
84
+ direction: "in",
85
+ protocol: "tcp",
86
+ port: "22",
87
+ sourceIps: ["0.0.0.0/0", "::/0"],
88
+ },
89
+ ],
90
+ });
91
+ for (let i = 0; i < (options.replicas ?? 2); i++) {
92
+ const server = yield* Hetzner.Server(`${id}Runner${i}`, {
93
+ serverType: options.serverType ?? "cx23",
94
+ image: "ubuntu-24.04",
95
+ location: network.location,
96
+ // IPv4 installs Node and deploys over SSH; IPv6 would only spend an address.
97
+ enableIpv6: false,
98
+ networks: [{ networkId: network.networkId as number }],
99
+ firewalls: [firewall],
100
+ labels: { [POOL_LABEL]: name },
101
+ });
102
+ yield* Hetzner.Service(`${id}Runner${i}Service`, {
103
+ server,
104
+ main: program.pipe(Output.map((p) => `${p.dir}/runner.mjs`)) as unknown as string,
105
+ port: GATEWAY_PORT,
106
+ env: environment.pipe(
107
+ // Each runner finds its own address in the Network's range.
108
+ Output.map((env) => ({
109
+ ...env,
110
+ CLUSTER_RUNNER_NETWORK: network.ipRange,
111
+ })),
112
+ ) as unknown as Record<string, string>,
113
+ });
114
+ }
115
+ }),
116
+ });
@@ -0,0 +1,6 @@
1
+ /**
2
+ * @alchemy.run/effect-cluster/Hetzner — run an Effect Cluster's database and
3
+ * runners on Hetzner Cloud. Requires `@alchemy.run/hetzner`.
4
+ */
5
+ export * from "./Database.ts";
6
+ export * from "./ServerPool.ts";
package/src/Host.ts ADDED
@@ -0,0 +1,68 @@
1
+ /**
2
+ * What a platform built on Effect Cluster entities, such as
3
+ * `@alchemy.run/domain/EffectCluster`, needs from the runner it is built on
4
+ * and from the gateway its callers reach entities through.
5
+ *
6
+ * @module
7
+ */
8
+ import * as Context from "effect/Context";
9
+ import type * as Effect from "effect/Effect";
10
+ import type * as Layer from "effect/Layer";
11
+ import type { HostedEntity } from "./Entity.ts";
12
+
13
+ export { type GatewayTarget, makeNamespace } from "./internal/Gateway.ts";
14
+ export { HasNoPool } from "./internal/HostedRuntime.ts";
15
+
16
+ /**
17
+ * The runner a Layer is being built on. A runner's bootstrap provides it
18
+ * while it builds its entrypoints, so a Layer that hosts an entity itself
19
+ * (a Domain platform's, built by the model's Layer rather than exported as
20
+ * a class) hands the runner that entity, and the Layer to launch with it
21
+ * inside the cluster (with Sharding and the workflow engine). Anywhere
22
+ * else, the service is absent.
23
+ */
24
+ export class RunnerHost extends Context.Service<
25
+ RunnerHost,
26
+ {
27
+ /** Host an entity on this runner, launching `layer` in the cluster beside it. */
28
+ readonly host: (
29
+ entity: HostedEntity,
30
+ layer: Layer.Layer<never, never, any>,
31
+ ) => Effect.Effect<void>;
32
+ }
33
+ >()("Cluster.RunnerHost") {}
34
+
35
+ /** Options for {@link InstanceStorage}'s `list`. Keys come back in order. */
36
+ export interface InstanceStorageListOptions {
37
+ /** Only keys that start with this prefix. */
38
+ readonly prefix?: string;
39
+ /** Only keys at or after this key. */
40
+ readonly start?: string;
41
+ /** Only keys after this key. */
42
+ readonly startAfter?: string;
43
+ /** Only keys before this key. */
44
+ readonly end?: string;
45
+ /** At most this many entries. */
46
+ readonly limit?: number;
47
+ }
48
+
49
+ /**
50
+ * The key-value storage of the object an entity instance runs on, on pools
51
+ * that give every instance an object of its own
52
+ * (`ClusterCloudflare.DurableObjectPool`: the instance's Durable Object).
53
+ * Values are structured-cloneable; writes inside `transaction` commit
54
+ * together, or not at all. The pool provides it to every instance it runs;
55
+ * on pools that share one database, the service is absent.
56
+ */
57
+ export class InstanceStorage extends Context.Service<
58
+ InstanceStorage,
59
+ {
60
+ get<T>(key: string): Effect.Effect<T | undefined>;
61
+ put<T>(key: string, value: T): Effect.Effect<void>;
62
+ put<T>(entries: Record<string, T>): Effect.Effect<void>;
63
+ delete(key: string): Effect.Effect<boolean>;
64
+ delete(keys: ReadonlyArray<string>): Effect.Effect<number>;
65
+ list<T>(options?: InstanceStorageListOptions): Effect.Effect<Map<string, T>>;
66
+ transaction<A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R>;
67
+ }
68
+ >()("Cluster.InstanceStorage") {}
@@ -0,0 +1,89 @@
1
+ import * as Kubernetes from "@alchemy.run/kubernetes";
2
+ import type { Input } from "alchemy/Input";
3
+ import * as Output from "alchemy/Output";
4
+ import { Random } from "alchemy/Random";
5
+ import * as Effect from "effect/Effect";
6
+ import * as Layer from "effect/Layer";
7
+ import * as Redacted from "effect/Redacted";
8
+ import { clusterShardGroups, Database as ClusterDatabase } from "../Database.ts";
9
+ import { stackDnsLabel } from "../internal/Naming.ts";
10
+ import type { Providers } from "../Providers.ts";
11
+
12
+ export interface DatabaseProps {
13
+ /** The cluster the database and runners run on. */
14
+ cluster: Kubernetes.ClusterLike | Effect.Effect<Input<Kubernetes.ClusterLike>, never, any>;
15
+ /**
16
+ * Kubernetes namespace of the database and runners.
17
+ * @default "default"
18
+ */
19
+ namespace?: string;
20
+ /**
21
+ * Postgres image.
22
+ * @default "postgres:17-alpine"
23
+ */
24
+ image?: string;
25
+ }
26
+
27
+ /**
28
+ * The database an Effect Cluster coordinates through, on Kubernetes, as a
29
+ * `Cluster.Database` Layer: Postgres as a Deployment behind a ClusterIP
30
+ * Service, on the cluster the runners run on. Nothing outside the cluster
31
+ * reaches it.
32
+ *
33
+ * Its data lives in the pod, so it is meant for development and tests; for
34
+ * a managed Postgres (RDS, Cloud SQL, Neon), pass its URL to
35
+ * `Cluster.Database.fromUrl` and give the pool its cluster.
36
+ *
37
+ * ### Running the database on Kubernetes
38
+ * **Example:** Postgres on a local kind cluster
39
+ * ```typescript
40
+ * export const Kind = Kubernetes.LocalCluster("Kind", { name: "cluster" });
41
+ * export const Database = ClusterKubernetes.Database("Cluster", { cluster: Kind });
42
+ * ```
43
+ *
44
+ * @primitive
45
+ */
46
+ export const Database = (id: string, props: DatabaseProps) =>
47
+ Layer.effect(
48
+ ClusterDatabase,
49
+ Effect.gen(function* () {
50
+ if (globalThis.__ALCHEMY_RUNTIME__) {
51
+ // Runners read their database from their own environment at runtime.
52
+ return { url: Redacted.make(""), shardGroups: undefined as never };
53
+ }
54
+ const namespace = props.namespace ?? "default";
55
+ const cluster = Effect.isEffect(props.cluster) ? yield* props.cluster : props.cluster;
56
+ const name = yield* stackDnsLabel(`${id}-db`, 50);
57
+ const password = yield* Random(`${id}Password`, { bytes: 24 }) as unknown as Effect.Effect<
58
+ Random,
59
+ never,
60
+ Providers
61
+ >;
62
+ const db = yield* Kubernetes.Deployment(`${id}Postgres`, {
63
+ cluster,
64
+ name,
65
+ namespace,
66
+ image: props.image ?? "postgres:17-alpine",
67
+ port: 5432,
68
+ serviceType: "ClusterIP",
69
+ env: password.text.pipe(
70
+ Output.map((text) => ({
71
+ POSTGRES_PASSWORD: Redacted.value(text),
72
+ POSTGRES_DB: "cluster",
73
+ })),
74
+ ) as unknown as Record<string, string>,
75
+ });
76
+ return {
77
+ // Ordered after the Deployment: runners connect once it exists.
78
+ url: Output.all(db.deploymentName, password.text).pipe(
79
+ Output.map(([, text]) =>
80
+ Redacted.make(
81
+ `postgres://postgres:${encodeURIComponent(text.pipe(Redacted.value))}@${name}.${namespace}.svc.cluster.local:5432/cluster`,
82
+ ),
83
+ ),
84
+ ),
85
+ network: { kind: "kubernetes" as const, cluster, namespace },
86
+ shardGroups: yield* clusterShardGroups(id),
87
+ };
88
+ }),
89
+ );
@@ -0,0 +1,77 @@
1
+ import * as Kubernetes from "@alchemy.run/kubernetes";
2
+ import * as Output from "alchemy/Output";
3
+ import * as Effect from "effect/Effect";
4
+ import { GATEWAY_PORT } from "../internal/Gateway.ts";
5
+ import { makePoolLayer, type PoolSettings } from "../internal/PoolHost.ts";
6
+
7
+ export interface DeploymentPoolOptions extends PoolSettings {
8
+ /**
9
+ * Number of runners.
10
+ * @default 2
11
+ */
12
+ replicas?: number;
13
+ /**
14
+ * How the gateway's Service is exposed. `ClusterIP` serves callers in the
15
+ * cluster; `LoadBalancer` serves callers anywhere, where the cluster
16
+ * provisions load balancers.
17
+ * @default "ClusterIP"
18
+ */
19
+ serviceType?: "ClusterIP" | "NodePort" | "LoadBalancer";
20
+ /** Container CPU/memory requests and limits. */
21
+ resources?: {
22
+ requests?: { cpu?: string; memory?: string };
23
+ limits?: { cpu?: string; memory?: string };
24
+ };
25
+ /**
26
+ * CPU architecture of the runner image, built on the deploying machine.
27
+ * @default the cluster's
28
+ */
29
+ architecture?: "amd64" | "arm64";
30
+ }
31
+
32
+ /**
33
+ * Run a pool of Effect Cluster runners on Kubernetes: a Deployment whose pods
34
+ * host every class given this pool, in the database's cluster and
35
+ * namespace. Pods reach each other at their own addresses; the gateway is
36
+ * a Service of the Deployment, at
37
+ * `http://{name}.{namespace}.svc.cluster.local:34500` for callers in the
38
+ * cluster. Requires a database on a Kubernetes cluster —
39
+ * `ClusterKubernetes.Database`.
40
+ *
41
+ * ### Running on Kubernetes
42
+ * **Example:** Two runners
43
+ * ```typescript
44
+ * export const Database = ClusterKubernetes.Database("Cluster", { cluster: Kind });
45
+ * export const Default = ClusterKubernetes.DeploymentPool("Default", { replicas: 2 }).pipe(
46
+ * Layer.provide(Database),
47
+ * );
48
+ * ```
49
+ *
50
+ * @primitive
51
+ */
52
+ export const DeploymentPool = (id: string, options: DeploymentPoolOptions = {}) =>
53
+ makePoolLayer({
54
+ label: "ClusterKubernetes.DeploymentPool",
55
+ id,
56
+ settings: options,
57
+ expected: "a Kubernetes cluster (ClusterKubernetes.Database)",
58
+ network: (database) => (database.network?.kind === "kubernetes" ? database.network : undefined),
59
+ // The Deployment's Service is named after the pool.
60
+ gateway: (name, network) =>
61
+ Effect.succeed({
62
+ url: `http://${name}.${network.namespace}.svc.cluster.local:${GATEWAY_PORT}`,
63
+ }),
64
+ nodes: ({ network, name, program, environment }) =>
65
+ Kubernetes.Deployment(`${id}Pool`, {
66
+ cluster: network.cluster as Kubernetes.ClusterLike,
67
+ name,
68
+ namespace: network.namespace,
69
+ context: program.pipe(Output.map((p) => p.dir)) as unknown as string,
70
+ replicas: options.replicas ?? 2,
71
+ port: GATEWAY_PORT,
72
+ serviceType: options.serviceType ?? "ClusterIP",
73
+ ...(options.resources === undefined ? {} : { resources: options.resources }),
74
+ ...(options.architecture === undefined ? {} : { architecture: options.architecture }),
75
+ env: environment as unknown as Record<string, string>,
76
+ }).pipe(Effect.asVoid),
77
+ });
@@ -0,0 +1,6 @@
1
+ /**
2
+ * @alchemy.run/effect-cluster/Kubernetes — run an Effect Cluster's database and
3
+ * runners on Kubernetes. Requires `@alchemy.run/kubernetes`.
4
+ */
5
+ export * from "./Database.ts";
6
+ export * from "./DeploymentPool.ts";
package/src/Pool.ts ADDED
@@ -0,0 +1,104 @@
1
+ import * as Provider from "alchemy/Provider";
2
+ import { Resource } from "alchemy/Resource";
3
+ import * as Data from "effect/Data";
4
+ import * as Effect from "effect/Effect";
5
+ import type * as Layer from "effect/Layer";
6
+ import type { PoolHost } from "./internal/PoolHostService.ts";
7
+ import type { Providers } from "./Providers.ts";
8
+
9
+ /**
10
+ * A pool Layer, as every host's pool returns it: provide it to an
11
+ * implementation to deploy it to that pool.
12
+ */
13
+ export type PoolLayer<R = never> = Layer.Layer<PoolHost, never, R>;
14
+
15
+ /** An entity class a pool is built with. */
16
+ export interface PoolEntity {
17
+ /** The entity type: the class's logical id. */
18
+ name: string;
19
+ /** The module that exports the class. */
20
+ module: string;
21
+ /** The export of {@link module} that holds it. */
22
+ handler: string;
23
+ /**
24
+ * An existing Effect Cluster program that hosts the entity, run as it is:
25
+ * an external `Cluster.Entity`. It owns every runner in its set.
26
+ */
27
+ program?: string;
28
+ }
29
+
30
+ /** Data the entities a pool hosts bind onto it. */
31
+ export interface PoolBindingContract {
32
+ entities?: PoolEntity[];
33
+ env?: Record<string, unknown>;
34
+ }
35
+
36
+ export interface PoolProps {
37
+ /** The pool's name, used in its service names. */
38
+ name: string;
39
+ }
40
+
41
+ export interface PoolAttributes {
42
+ /** The pool's name. */
43
+ name: string;
44
+ /** The entity classes the pool hosts, sorted by type. */
45
+ entities: PoolEntity[];
46
+ /** Environment values the hosted classes need, merged. */
47
+ env: Record<string, unknown>;
48
+ }
49
+
50
+ export interface Pool extends Resource<
51
+ "Cluster.Pool",
52
+ PoolProps,
53
+ PoolAttributes,
54
+ PoolBindingContract,
55
+ Providers
56
+ > {}
57
+
58
+ /**
59
+ * The set of entity classes one pool hosts, collected from the
60
+ * entities that were given it. Pool Layers (`ClusterDocker.SwarmPool`,
61
+ * …) declare one and build their program from it; it owns nothing in the
62
+ * cloud.
63
+ *
64
+ * @internal
65
+ */
66
+ export const Pool = Resource<Pool>("Cluster.Pool");
67
+
68
+ /**
69
+ * A pool was given an external `Cluster.Entity` and something
70
+ * else hosted another way: an external program owns every runner it runs on.
71
+ */
72
+ export class PoolProgramConflict extends Data.TaggedError("Cluster.PoolProgramConflict")<{
73
+ readonly message: string;
74
+ }> {}
75
+
76
+ export const PoolProvider = () =>
77
+ Provider.succeed(Pool, {
78
+ stables: ["name"],
79
+ reconcile: ({ id, news, bindings }) =>
80
+ Effect.gen(function* () {
81
+ const entities = new Map<string, PoolEntity>();
82
+ const env: Record<string, unknown> = {};
83
+ for (const { data } of bindings) {
84
+ for (const entity of data.entities ?? []) entities.set(entity.name, entity);
85
+ Object.assign(env, data.env);
86
+ }
87
+ const all = [...entities.values()];
88
+ const programs = new Set(all.flatMap((entity) => (entity.program ? [entity.program] : [])));
89
+ if (programs.size > 1 || (programs.size === 1 && all.some((entity) => !entity.program))) {
90
+ return yield* new PoolProgramConflict({
91
+ message:
92
+ `Runners '${id}' were given ${all.map((entity) => `'${entity.name}'`).join(", ")}, ` +
93
+ `but an external program runs on every runner it is given: give each program ` +
94
+ `a pool of its own, and host Alchemy entities in another pool.`,
95
+ });
96
+ }
97
+ return {
98
+ name: news.name,
99
+ entities: [...entities.values()].sort((a, b) => a.name.localeCompare(b.name)),
100
+ env,
101
+ };
102
+ }),
103
+ delete: () => Effect.void,
104
+ });
@@ -0,0 +1,49 @@
1
+ import * as Provider from "alchemy/Provider";
2
+ import { Random, RandomProvider } from "alchemy/Random";
3
+ import * as Layer from "effect/Layer";
4
+ import { CronProvider, CronResource } from "./Cron.ts";
5
+ import { EntityProvider, EntityResource } from "./Entity.ts";
6
+ import { Pool, PoolProvider } from "./Pool.ts";
7
+ import { ShardGroups, ShardGroupsProvider } from "./ShardGroups.ts";
8
+ import { SingletonProvider, SingletonResource } from "./Singleton.ts";
9
+ import { WorkflowProvider, WorkflowResource } from "./Workflow.ts";
10
+
11
+ /** Service tag bundling every Effect Cluster provider. */
12
+ export class Providers extends Provider.ProviderCollection<Providers>()("Cluster") {}
13
+
14
+ export type ProviderRequirements = Layer.Services<ReturnType<typeof providers>>;
15
+
16
+ /**
17
+ * The Effect Cluster providers Layer. Add it next to the providers of the
18
+ * cloud your database and runners run on.
19
+ *
20
+ * @example
21
+ * ```ts
22
+ * Alchemy.Stack("App", { providers: Layer.mergeAll(Docker.providers(), Cluster.providers()) }, ...)
23
+ * ```
24
+ */
25
+ export const providers = () =>
26
+ Layer.effect(
27
+ Providers,
28
+ Provider.collection([
29
+ EntityResource,
30
+ WorkflowResource,
31
+ SingletonResource,
32
+ CronResource,
33
+ Pool,
34
+ ShardGroups,
35
+ Random,
36
+ ]),
37
+ ).pipe(
38
+ Layer.provide(
39
+ Layer.mergeAll(
40
+ EntityProvider(),
41
+ WorkflowProvider(),
42
+ SingletonProvider(),
43
+ CronProvider(),
44
+ PoolProvider(),
45
+ ShardGroupsProvider(),
46
+ RandomProvider(),
47
+ ),
48
+ ),
49
+ );