alasio 4.1.3 → 4.2.0

package/dist/cli/src/cluster/docker.js

@@ -1,228 +1,323 @@
 /**
- * Docker's Engine API, over the Unix socket Docker listens on: as much of it as the local
- * cluster (./local.ts) needs, its images, network, volumes and containers and commands
- * run in them, called with node:http, so neither the Docker CLI nor a client library is
- * needed. Paths are those of API version 1.44 (Docker 25), the oldest it is written for.
+ * The cluster in Docker: k3s in Docker's containers, made through its Engine API
+ * (./docker-engine.ts) from alasio's node image (cluster/node), which is k3s with gVisor
+ * registered as containerd's `runsc` runtime and an entrypoint that readies its container
+ * for k3s.
  *
- * The socket is the one `DOCKER_HOST` names (`unix://PATH`), /var/run/docker.sock when
- * it names none.
+ * A cluster NAME is a bridge network NAME, a server container NAME-server-0 and agent
+ * containers NAME-agent-I, each with named volumes for what k3s, kubelet and the CNI
+ * keep (NAME-server-0-k3s and so on), so a node made anew, as when its image or settings
+ * change, keeps its state. All of it is labelled as the cluster's, and nothing that is
+ * not is changed. The nodes have fixed addresses in the network's subnet: the server the
+ * second, after the gateway, and the agents those after it. Docker gives fixed addresses
+ * only in a subnet chosen for the network, so one is chosen when none is given: the first
+ * 172.30.N.0/24 no other network overlaps.
+ *
+ * The registries the nodes pull from are k3s's registries.yaml, which the node image's
+ * entrypoint writes from the variable each node is made with, so a node is made anew when
+ * they change, as when its other settings do.
+ *
+ * The API server is published on the loopback only. What is done in the cluster itself,
+ * waiting for it and configuring it, is done with the kubectl k3s carries in the server
+ * node, so this machine needs none of Kubernetes' tools.
  */
-import { request as httpRequest, STATUS_CODES } from "node:http";
-import { buffer } from "node:stream/consumers";
-import { Context, Effect, Layer, Result, Schedule, Schema } from "effect";
-const API_VERSION = "v1.44";
-const DEFAULT_SOCKET = "/var/run/docker.sock";
-/** Docker refused a call, or could not be reached: the status it answered with, when it answered. */
-export class DockerError extends Schema.TaggedError()("DockerError", {
-    /** The call, as its method and path. */
-    call: Schema.String,
-    /** Docker's HTTP status (404 for absent, 409 for a conflict), when it answered. */
-    status: Schema.optional(Schema.Number),
-    /** What Docker answered, or why it could not be reached. */
+import { createHash } from "node:crypto";
+import { Context, Effect, FileSystem, Layer, Schedule, Schema } from "effect";
+import { imageReference } from "../images.js";
+import { NODE_IMAGE } from "../release.js";
+import { DockerEngine } from "./docker-engine.js";
+import { awaitReady as waitFor, clusterKubeconfig, GVISOR_RUNTIME_CLASS, KUBELET_ARGS, nodeHostsWith, NotYet, readySince, registriesYaml, WAITS, writeKubeconfig, } from "./k3s.js";
+/** The cluster cannot be made as asked: something of its name is not its own, or its settings cannot hold. */
+export class ClusterUnusable extends Schema.TaggedError()("ClusterUnusable", {
+    cluster: Schema.String,
     reason: Schema.String,
 }) {
     get message() {
-        return `Docker ${this.status === undefined ? "failed" : `answered ${this.status} to`} ${this.call}: ${this.reason}`;
+        return `cluster ${this.cluster} cannot be made: ${this.reason}`;
     }
 }
-/** `DOCKER_HOST` names a Docker that is not reached over a Unix socket. */
-export class DockerHostUnsupported extends Schema.TaggedError()("DockerHostUnsupported", {
-    host: Schema.String,
+/** A command run in a node exited with another code than 0. */
+export class NodeCommandFailed extends Schema.TaggedError()("NodeCommandFailed", {
+    node: Schema.String,
+    command: Schema.String,
+    exitCode: Schema.Number,
+    stderr: Schema.String,
 }) {
     get message() {
-        return `DOCKER_HOST is ${this.host}, but alasio reaches Docker only over a Unix socket (unix://PATH)`;
+        return `${this.command} in ${this.node} exited with ${this.exitCode}${this.stderr.trim() ? `: ${this.stderr.trim()}` : ""}`;
     }
 }
-/** Whether `error` is Docker's answer `status`. */
-export const hasStatus = (status) => (error) => error.status === status;
-/**
- * Whether `error` is Docker's answer to removing a container it killed but did not see
- * exit within the time it waits: the kill stands, and the container exits once its
- * processes do, which those waiting on a FUSE mount whose client dies with them, as
- * JuiceFS's on a node are, can take longer to.
- */
-const killedNotYetExited = (error) => error.status === 500 && error.reason.includes("did not receive an exit event");
-/** The socket of the Docker `env` names, by `DOCKER_HOST`. */
-export function dockerSocket(env) {
-    const host = env["DOCKER_HOST"];
-    if (!host)
-        return Result.succeed(DEFAULT_SOCKET);
-    if (host.startsWith("unix://"))
-        return Result.succeed(host.slice("unix://".length));
-    return Result.fail(new DockerHostUnsupported({ host }));
+/** The cluster in Docker of the options its layer was given. */
+export class DockerCluster extends Context.Service()("alasio/cluster/DockerCluster") {
+    static layer = (options) => Layer.effect(DockerCluster, makeDockerCluster(options));
+}
+const CLUSTER_LABEL = "alasio.cluster";
+const ROLE_LABEL = "alasio.role";
+/** The label of a node's container that says what it was made from: specHash's. */
+const SPEC_LABEL = "alasio.spec";
+/** What a node keeps, each in a volume of its own: the volume's suffix, and where it is mounted. */
+const NODE_VOLUMES = [
+    ["k3s", "/var/lib/rancher/k3s"],
+    ["kubelet", "/var/lib/kubelet"],
+    ["cni", "/var/lib/cni"],
+    ["log", "/var/log"],
+];
+/** The API server's port in the server node. */
+const K3S_PORT = 6443;
+const API_PORT = `${K3S_PORT}/tcp`;
+const K3S_KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
+/** The token the server made on its first start, which agents join with. */
+const K3S_TOKEN = "/var/lib/rancher/k3s/server/token";
+/** The variable the node image's entrypoint writes k3s's registries.yaml from, before k3s starts. */
+const REGISTRIES_ENV = "ALASIO_REGISTRIES";
+/** The `index`th address of the IPv4 subnet `cidr` (a.b.c.d/n), its network address the 0th; null when it has no such host address. */
+export function subnetAddress(cidr, index) {
+    const match = /^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/u.exec(cidr);
+    if (!match)
+        return null;
+    const octets = match.slice(1, 5).map(Number);
+    const prefix = Number(match[5]);
+    if (octets.some((octet) => octet > 255) || prefix > 30)
+        return null;
+    // The last address is the subnet's broadcast address.
+    if (index < 1 || index >= 2 ** (32 - prefix) - 1)
+        return null;
+    const network = octets.reduce((value, octet) => value * 256 + octet, 0);
+    const address = network - (network % 2 ** (32 - prefix)) + index;
+    return [24, 16, 8, 0].map((shift) => Math.floor(address / 2 ** shift) % 256).join(".");
 }
-/** Docker's Engine API, as the local cluster calls it. Objects are addressed by name. */
-export class DockerEngine extends Context.Service()("alasio/cluster/DockerEngine") {
-    /** The Docker `env` names (dockerSocket). */
-    static layer = (env = process.env) => Layer.effect(DockerEngine, Effect.map(Effect.fromResult(dockerSocket(env)), makeDockerEngine));
+/** The first and last addresses of the IPv4 subnet `cidr`, as numbers; null when it is none. */
+function subnetRange(cidr) {
+    const match = /^(\d+)\.(\d+)\.(\d+)\.(\d+)\/(\d+)$/u.exec(cidr);
+    if (!match)
+        return null;
+    const octets = match.slice(1, 5).map(Number);
+    const prefix = Number(match[5]);
+    if (octets.some((octet) => octet > 255) || prefix > 32)
+        return null;
+    const size = 2 ** (32 - prefix);
+    const first = octets.reduce((value, octet) => value * 256 + octet, 0);
+    return [first - (first % size), first - (first % size) + size - 1];
 }
-const describe = (call) => `${call.method} ${call.path}`;
-const unreachable = (call, cause) => new DockerError({ call: describe(call), reason: cause instanceof Error ? cause.message : String(cause) });
-/** A label filter of a list call, in the JSON Docker's `filters` parameter takes. */
-const labelFilter = (labels) => JSON.stringify({ label: Object.entries(labels).map(([key, value]) => `${key}=${value}`) });
-/** The content of the first regular file in `tar`, a tar archive as Docker's archive endpoint sends one; null when it holds none. */
-export function firstFile(tar) {
-    const BLOCK = 512;
-    for (let offset = 0; offset + BLOCK <= tar.length;) {
-        const header = tar.subarray(offset, offset + BLOCK);
-        // Two zero blocks end an archive.
-        if (header.every((byte) => byte === 0))
-            return null;
-        const size = Number.parseInt(header.toString("latin1", 124, 136).replace(/\0.*$/su, "").trim() || "0", 8);
-        const type = header[156];
-        const start = offset + BLOCK;
-        // "0", or NUL in archives older than POSIX's.
-        if (type === 0x30 || type === 0)
-            return tar.subarray(start, start + size);
-        offset = start + Math.ceil(size / BLOCK) * BLOCK;
+/** The first 172.30.N.0/24 that overlaps none of the `taken` subnets (others than IPv4 are not in the way), or null when all do. Pure, for tests. */
+export function freeSubnet(taken) {
+    const ranges = taken.flatMap((cidr) => {
+        const range = subnetRange(cidr);
+        return range ? [range] : [];
+    });
+    for (let third = 0; third < 256; third++) {
+        const candidate = `172.30.${third}.0/24`;
+        // A /24 written as such always has a range.
+        const [first, last] = subnetRange(candidate);
+        if (!ranges.some(([start, end]) => start <= last && first <= end))
+            return candidate;
     }
     return null;
 }
 /**
- * The output of a command whose exec ran without a terminal: Docker sends it in frames of
- * an 8-byte header (the stream, 1 for stdout or 2 for stderr, and the size, big-endian, at
- * byte 4) and what was written.
+ * Aborts the FUSE connections of what is mounted in a node, as JuiceFS's workspaces are: a
+ * process waiting on one whose client is gone, as a node's are once they are killed, can
+ * itself be killed, and its node removed, only once it is. The machine's connections are
+ * all in /sys/fs/fuse/connections, each named after the minor of its mount's device, and
+ * only the node's own are aborted.
  */
-export function demultiplex(raw) {
-    const stdout = [];
-    const stderr = [];
-    for (let offset = 0; offset + 8 <= raw.length;) {
-        const size = raw.readUInt32BE(offset + 4);
-        const payload = raw.subarray(offset + 8, offset + 8 + size);
-        (raw[offset] === 2 ? stderr : stdout).push(payload);
-        offset += 8 + size;
-    }
-    return { stdout: Buffer.concat(stdout), stderr: Buffer.concat(stderr) };
-}
-/** The Engine API on the Unix socket `socketPath`. */
-function makeDockerEngine(socketPath) {
-    const send = (call, headers = {}) => {
-        const query = new URLSearchParams();
-        for (const [key, value] of Object.entries(call.query ?? {})) {
-            for (const each of typeof value === "string" ? [value] : value)
-                query.append(key, each);
+const ABORT_FUSE = [
+    "grep -q ' /sys/fs/fuse/connections ' /proc/self/mounts || mount -t fusectl fusectl /sys/fs/fuse/connections",
+    "for device in $(awk '/ - fuse/ { print $3 }' /proc/self/mountinfo | sort -u); do",
+    '  abort="/sys/fs/fuse/connections/${device#*:}/abort"',
+    '  if [ -e "$abort" ]; then echo 1 >"$abort"; fi',
+    "done",
+].join("\n");
+/** kubelet's settings, as k3s's flags give them. */
+const kubeletArgs = KUBELET_ARGS.map((arg) => `--kubelet-arg=${arg}`);
+/** A digest of `spec` that two specs share only when they are the same. */
+const specHash = (spec) => createHash("sha256").update(JSON.stringify(spec)).digest("hex");
+const makeDockerCluster = Effect.fnUntraced(function* ({ name: cluster, apiPort, storagePath, image = imageReference(NODE_IMAGE), subnet, hostAliases = [], mounts = [], agents = 0, registries, readyTimeout = WAITS.readyTimeout, poll = WAITS.poll, }) {
+    const docker = yield* DockerEngine;
+    const fs = yield* FileSystem.FileSystem;
+    const labels = { [CLUSTER_LABEL]: cluster };
+    const server = `${cluster}-server-0`;
+    const nodes = [server, ...Array.from({ length: agents }, (_, index) => `${cluster}-agent-${index}`)];
+    const unusable = (reason) => new ClusterUnusable({ cluster, reason });
+    /**
+     * The container of `node`, with what every node has: privileged, as k3s in Docker needs,
+     * its volumes, the storage directory and the host mounts, and the registries.
+     */
+    const nodeContainer = (node, role, address, command, env) => ({
+        Image: image,
+        Hostname: node,
+        Cmd: command,
+        Env: [...env, ...(registries ? [`${REGISTRIES_ENV}=${registriesYaml(registries)}`] : [])],
+        Labels: { ...labels, [ROLE_LABEL]: role },
+        ...(role === "server" ? { ExposedPorts: { [API_PORT]: {} } } : {}),
+        HostConfig: {
+            Privileged: true,
+            Init: true,
+            CgroupnsMode: "private",
+            RestartPolicy: { Name: "unless-stopped" },
+            SecurityOpt: ["label=disable"],
+            Tmpfs: { "/run": "", "/var/run": "" },
+            Mounts: [
+                ...NODE_VOLUMES.map(([suffix, target]) => ({ Type: "volume", Source: `${node}-${suffix}`, Target: target })),
+                { Type: "bind", Source: storagePath, Target: storagePath },
+                ...mounts.map(({ source, target, readOnly = false }) => ({ Type: "bind", Source: source, Target: target, ReadOnly: readOnly })),
+            ],
+            ExtraHosts: hostAliases.flatMap(({ ip, hostnames }) => hostnames.map((hostname) => `${hostname}:${ip}`)),
+            ...(role === "server" ? { PortBindings: { [API_PORT]: [{ HostIp: "127.0.0.1", HostPort: String(apiPort) }] } } : {}),
+        },
+        NetworkingConfig: { EndpointsConfig: { [cluster]: { IPAMConfig: { IPv4Address: address } } } },
+    });
+    /** The cluster's network, made unless it exists; its subnet, which must be `subnet` when that is given. */
+    const ensureNetwork = Effect.fnUntraced(function* () {
+        const existing = yield* docker.inspectNetwork(cluster);
+        if (existing && existing.Labels?.[CLUSTER_LABEL] !== cluster)
+            return yield* unusable(`a network named ${cluster} is not its own`);
+        let network = existing;
+        if (!network) {
+            const chosen = subnet ?? freeSubnet((yield* docker.listNetworks).flatMap(({ IPAM }) => (IPAM.Config ?? []).flatMap(({ Subnet }) => Subnet ?? [])));
+            if (!chosen)
+                return yield* unusable("every 172.30.N.0/24 is another network's: give the cluster a subnet");
+            yield* Effect.logInfo(`making network ${cluster} (${chosen})`);
+            yield* docker.createNetwork({ Name: cluster, Driver: "bridge", IPAM: { Config: [{ Subnet: chosen }] }, Labels: labels });
+            network = yield* docker.inspectNetwork(cluster);
         }
-        const body = call.body === undefined ? undefined : JSON.stringify(call.body);
-        const request = httpRequest({
-            socketPath,
-            method: call.method,
-            path: `/${API_VERSION}${call.path}${query.size ? `?${query}` : ""}`,
-            headers: body === undefined ? headers : { ...headers, "Content-Type": "application/json", "Content-Length": Buffer.byteLength(body) },
-        });
-        request.end(body);
-        return request;
+        const actual = network?.IPAM.Config?.[0]?.Subnet;
+        if (!actual)
+            return yield* unusable(`network ${cluster} has no subnet`);
+        if (subnet && actual !== subnet)
+            return yield* unusable(`network ${cluster} has the subnet ${actual}, not ${subnet}`);
+        return actual;
+    });
+    /** The `index`th node's address in `networkSubnet`. */
+    const nodeAddress = (networkSubnet, index) => {
+        const address = subnetAddress(networkSubnet, 2 + index);
+        return address ? Effect.succeed(address) : Effect.fail(unusable(`its subnet ${networkSubnet} has no room for ${nodes.length} nodes`));
     };
-    const readAll = (call, stream) => Effect.tryPromise({ try: () => buffer(stream), catch: (cause) => unreachable(call, cause) });
-    /** Docker's refusal of `call`: the status of `response`, and the message its body holds. */
-    const refusal = (call, response) => readAll(call, response).pipe(Effect.flatMap((body) => {
-        const status = response.statusCode ?? 0;
-        const text = body.toString("utf8").trim();
-        let reason = text || STATUS_CODES[status] || "no message";
-        try {
-            const parsed = JSON.parse(text);
-            if (typeof parsed === "object" && parsed !== null && "message" in parsed && typeof parsed.message === "string")
-                reason = parsed.message;
+    /** `node`'s container made as `spec` says, anew when it was made otherwise, and running. */
+    const ensureNode = Effect.fnUntraced(function* (node, spec) {
+        const hash = specHash(spec);
+        const existing = yield* docker.inspectContainer(node);
+        if (existing && existing.Config.Labels?.[CLUSTER_LABEL] !== cluster)
+            return yield* unusable(`a container named ${node} is not one of its nodes`);
+        const current = existing !== null && existing.Config.Labels?.[SPEC_LABEL] === hash;
+        if (existing && !current) {
+            yield* Effect.logInfo(`making node ${node} anew, from its current image and settings`);
+            yield* docker.stopContainer(node);
+            yield* docker.removeContainer(node);
+        }
+        if (!current) {
+            yield* Effect.logInfo(`making node ${node}`);
+            yield* Effect.forEach(NODE_VOLUMES, ([suffix]) => docker.createVolume({ Name: `${node}-${suffix}`, Labels: labels }), { discard: true });
+            yield* docker.createContainer(node, { ...spec, Labels: { ...spec.Labels, [SPEC_LABEL]: hash } });
         }
-        catch {
-            // Not JSON: the text is the message.
+        if (!current || !existing.State.Running) {
+            yield* Effect.logInfo(`starting node ${node}`);
+            yield* docker.startContainer(node);
         }
-        return Effect.fail(new DockerError({ call: describe(call), status, reason }));
+    });
+    /** The output of `command` in `node`, which fails unless it exits with 0. */
+    const inNode = (node, command, options) => docker.exec(node, command, options).pipe(Effect.flatMap(({ exitCode, stdout, stderr }) => exitCode === 0 ? Effect.succeed(stdout) : Effect.fail(new NodeCommandFailed({ node, command: command.join(" "), exitCode, stderr }))));
+    const kubectl = (args, options) => inNode(server, ["kubectl", ...args], options);
+    const awaitReady = waitFor(`cluster ${cluster}`, { readyTimeout, poll });
+    /** When each node's container last started, in milliseconds since the epoch. */
+    const startTimes = Effect.forEach(nodes, (node) => Effect.map(docker.inspectContainer(node), (container) => [node, Date.parse(container?.State.StartedAt ?? "")])).pipe(Effect.map((entries) => new Map(entries)));
+    /** Every node ready since its container started. */
+    const nodesReady = (started) => kubectl(["get", "nodes", "--output=json"]).pipe(Effect.flatMap((stdout) => {
+        const { items } = JSON.parse(stdout.toString("utf8"));
+        const ready = new Set(items.filter((node) => readySince(node, started.get(node.metadata?.name ?? "") ?? Infinity)).map((node) => node.metadata?.name));
+        const waiting = nodes.filter((node) => !ready.has(node));
+        return waiting.length === 0 ? Effect.void : Effect.fail(new NotYet({ message: `not reported ready since started: ${waiting.join(", ")}` }));
     }));
-    /** The response to `call`, open while the scope is; a status of 400 or over is its refusal (304, already so, is not). */
-    const open = (call) => Effect.acquireRelease(Effect.callback((resume) => {
-        const request = send(call);
-        request.on("response", (response) => resume(Effect.succeed(response)));
-        request.on("error", (cause) => resume(Effect.fail(unreachable(call, cause))));
-        return Effect.sync(() => request.destroy());
-    }), (response) => Effect.sync(() => response.destroy())).pipe(Effect.tap((response) => ((response.statusCode ?? 0) >= 400 ? refusal(call, response) : Effect.void)));
-    const body = (call) => Effect.scoped(Effect.flatMap(open(call), (response) => readAll(call, response)));
-    /** The JSON Docker answers `call` with, as the Engine API documents it. */
-    const json = (call) => Effect.map(body(call), (content) => JSON.parse(content.toString("utf8")));
-    const orNull = (effect) => effect.pipe(Effect.catchIf(hasStatus(404), () => Effect.succeed(null)));
-    const orGone = (effect) => effect.pipe(Effect.asVoid, Effect.catchIf(hasStatus(404), () => Effect.void));
-    const name = (value) => encodeURIComponent(value);
-    /**
-     * Starts exec `id` attached, `stdin` written to it and then closed, and collects what it
-     * writes until it ends. Docker upgrades the connection to a raw stream for this.
-     */
-    const attach = (id, stdin) => {
-        const call = { method: "POST", path: `/exec/${id}/start`, body: { Detach: false, Tty: false } };
-        return Effect.callback((resume) => {
-            const request = send(call, { Connection: "Upgrade", Upgrade: "tcp" });
-            let stream = null;
-            request.on("upgrade", (_response, socket, head) => {
-                stream = socket;
-                const chunks = [head];
-                socket.on("data", (chunk) => chunks.push(chunk));
-                socket.on("error", (cause) => resume(Effect.fail(unreachable(call, cause))));
-                socket.on("close", () => resume(Effect.succeed(Buffer.concat(chunks))));
-                if (stdin !== undefined)
-                    socket.end(stdin);
-            });
-            // A daemon that answers without upgrading refused.
-            request.on("response", (response) => resume(refusal(call, response)));
-            request.on("error", (cause) => resume(Effect.fail(unreachable(call, cause))));
-            return Effect.sync(() => {
-                request.destroy();
-                stream?.destroy();
-            });
-        });
-    };
-    const exec = Effect.fnUntraced(function* (container, command, { stdin } = {}) {
-        const { Id: id } = yield* json({
-            method: "POST",
-            path: `/containers/${name(container)}/exec`,
-            body: { AttachStdin: stdin !== undefined, AttachStdout: true, AttachStderr: true, Tty: false, Cmd: command },
-        });
-        const { stdout, stderr } = demultiplex(yield* attach(id, stdin));
-        // Docker records the exit code before it closes the exec's streams.
-        const inspect = { method: "GET", path: `/exec/${id}/json` };
-        const { ExitCode: exitCode } = yield* json(inspect);
-        if (exitCode === null)
-            return yield* new DockerError({ call: describe(inspect), reason: `${JSON.stringify(command[0])} has no exit code` });
-        return { exitCode, stdout, stderr: stderr.toString("utf8") };
+    /** CoreDNS's NodeHosts with the host aliases, patched only when they differ, and only over the version read. */
+    const applyHostAliases = Effect.gen(function* () {
+        const coredns = JSON.parse((yield* kubectl(["--namespace=kube-system", "get", "configmap", "coredns", "--output=json"])).toString("utf8"));
+        const nodeHosts = coredns.data?.["NodeHosts"] ?? "";
+        const desired = nodeHostsWith(nodeHosts, nodes, hostAliases);
+        if (desired === nodeHosts)
+            return;
+        const patch = { metadata: { resourceVersion: coredns.metadata?.resourceVersion }, data: { NodeHosts: desired } };
+        yield* kubectl(["--namespace=kube-system", "patch", "configmap", "coredns", "--type=merge", `--patch=${JSON.stringify(patch)}`]);
     });
-    const pullImage = Effect.fnUntraced(function* (reference) {
-        const call = { method: "POST", path: "/images/create", query: { fromImage: reference } };
-        // A pull that fails once it has begun still answers 200: its progress, one JSON object
-        // a line, ends with the error.
-        const progress = (yield* body(call)).toString("utf8").split("\n").filter((line) => line.trim());
-        for (const line of progress) {
-            const { error } = JSON.parse(line);
-            if (error)
-                return yield* new DockerError({ call: describe(call), reason: error });
-        }
+    /** The kubeconfig the server wrote, pointed at the API server's port on the loopback, written to `path`. */
+    const fetchKubeconfig = Effect.fnUntraced(function* (path) {
+        const content = yield* awaitReady("its kubeconfig", docker.readFile(server, K3S_KUBECONFIG).pipe(Effect.flatMap((written) => {
+            const local = clusterKubeconfig(written.toString("utf8"), cluster, `https://127.0.0.1:${apiPort}`);
+            return local ? Effect.succeed(local) : Effect.fail(new NotYet({ message: `${K3S_KUBECONFIG} is not yet a kubeconfig with a cluster and a user` }));
+        })));
+        yield* Effect.provideService(writeKubeconfig(path, content), FileSystem.FileSystem, fs);
+        yield* Effect.logInfo(`wrote the kubeconfig of cluster ${cluster} to ${path}`);
     });
-    const readFile = Effect.fnUntraced(function* (container, path) {
-        const call = { method: "GET", path: `/containers/${name(container)}/archive`, query: { path } };
-        const file = firstFile(yield* body(call));
-        if (!file)
-            return yield* new DockerError({ call: describe(call), reason: `${path} in ${container} is not a file` });
-        return file;
+    const up = Effect.fnUntraced(function* (kubeconfig) {
+        if (!(yield* docker.inspectImage(image))) {
+            yield* Effect.logInfo(`pulling ${image}`);
+            yield* docker.pullImage(image);
+        }
+        const networkSubnet = yield* ensureNetwork();
+        yield* fs.makeDirectory(storagePath, { recursive: true });
+        yield* ensureNode(server, nodeContainer(server, "server", yield* nodeAddress(networkSubnet, 0), [
+            "server",
+            "--disable=traefik",
+            // Where the kubeconfig reaches it.
+            "--tls-san=127.0.0.1",
+            `--default-local-storage-path=${storagePath}`,
+            ...kubeletArgs,
+        ], []));
+        yield* awaitReady("its API server", kubectl(["get", "--raw=/readyz"]));
+        if (agents > 0) {
+            const token = (yield* awaitReady("its token", docker.readFile(server, K3S_TOKEN))).toString("utf8").trim();
+            for (const [index, agent] of nodes.slice(1).entries()) {
+                yield* ensureNode(agent, nodeContainer(agent, "agent", yield* nodeAddress(networkSubnet, 1 + index), ["agent", ...kubeletArgs], [
+                    `K3S_URL=https://${server}:${K3S_PORT}`,
+                    `K3S_TOKEN=${token}`,
+                ]));
+            }
+        }
+        yield* awaitReady("its nodes", Effect.flatMap(startTimes, nodesReady));
+        yield* awaitReady("CoreDNS's NodeHosts", applyHostAliases);
+        yield* awaitReady("the gvisor RuntimeClass", kubectl(["apply", "--filename=-"], { stdin: JSON.stringify(GVISOR_RUNTIME_CLASS) }));
+        yield* fetchKubeconfig(kubeconfig);
+        yield* Effect.logInfo(`cluster ${cluster} is up`);
     });
-    return DockerEngine.of({
-        version: json({ method: "GET", path: "/version" }),
-        // A reference as it is: the route takes its slashes and colons.
-        inspectImage: (reference) => orNull(json({ method: "GET", path: `/images/${reference}/json` })),
-        pullImage,
-        inspectNetwork: (network) => orNull(json({ method: "GET", path: `/networks/${name(network)}` })),
-        listNetworks: json({ method: "GET", path: "/networks" }),
-        createNetwork: (network) => Effect.asVoid(body({ method: "POST", path: "/networks/create", body: network })),
-        removeNetwork: (network) => orGone(body({ method: "DELETE", path: `/networks/${name(network)}` })),
-        disconnectNetwork: (network, container) => Effect.asVoid(body({ method: "POST", path: `/networks/${name(network)}/disconnect`, body: { Container: container, Force: true } })),
-        createVolume: (volume) => Effect.asVoid(body({ method: "POST", path: "/volumes/create", body: volume })),
-        listVolumes: (labels) => json({ method: "GET", path: "/volumes", query: { filters: labelFilter(labels) } }).pipe(Effect.map(({ Volumes }) => Volumes ?? [])),
-        removeVolume: (volume) => orGone(body({ method: "DELETE", path: `/volumes/${name(volume)}` })),
-        listContainers: (labels) => json({ method: "GET", path: "/containers/json", query: { all: "true", filters: labelFilter(labels) } }),
-        inspectContainer: (container) => orNull(json({ method: "GET", path: `/containers/${name(container)}/json` })),
-        createContainer: (container, spec) => Effect.asVoid(body({ method: "POST", path: "/containers/create", query: { name: container }, body: spec })),
-        startContainer: (container) => Effect.asVoid(body({ method: "POST", path: `/containers/${name(container)}/start` })),
-        stopContainer: (container) => Effect.asVoid(body({ method: "POST", path: `/containers/${name(container)}/stop` })),
-        // Asked again while Docker has killed it but not seen it exit, for as long as a
-        // node's dying processes may take.
-        removeContainer: (container) => orGone(body({ method: "DELETE", path: `/containers/${name(container)}`, query: { force: "true" } }).pipe(Effect.retry({ while: killedNotYetExited, schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.during("2 minutes")]) }))),
-        exec,
-        readFile,
+    /** The cluster's nodes, the server first. */
+    const existingNodes = docker.listContainers(labels).pipe(Effect.map((found) => found
+        .map((container) => ({
+        name: container.Names[0]?.replace(/^\//u, "") ?? "",
+        role: container.Labels[ROLE_LABEL] === "server" ? "server" : "agent",
+        state: container.State,
+    }))
+        .sort((a, b) => (a.role === b.role ? a.name.localeCompare(b.name) : a.role === "server" ? -1 : 1))));
+    return DockerCluster.of({
+        up,
+        // The agents first, so none is left without its server.
+        down: existingNodes.pipe(Effect.flatMap((found) => Effect.forEach(found.toReversed(), ({ name }) => docker.stopContainer(name), { discard: true })), Effect.andThen(Effect.logInfo(`stopped cluster ${cluster}`))),
+        remove: ({ volumes = false, storage = false } = {}) => Effect.gen(function* () {
+            const found = yield* existingNodes;
+            if (storage && found.some(({ name }) => name === server)) {
+                yield* docker.startContainer(server);
+                // The pods still running write as it empties the directory, and one of its
+                // directories they write in as it is removed is not empty: it is emptied again.
+                yield* inNode(server, ["find", storagePath, "-mindepth", "1", "-delete"]).pipe(Effect.retry({ times: 9, schedule: Schedule.spaced("1 second") }));
+            }
+            for (const { name } of found.filter(({ state }) => state === "running"))
+                yield* Effect.ignore(inNode(name, ["sh", "-c", ABORT_FUSE]));
+            yield* Effect.forEach(found, ({ name }) => docker.removeContainer(name), { discard: true });
+            const network = yield* docker.inspectNetwork(cluster);
+            if (network?.Labels?.[CLUSTER_LABEL] === cluster) {
+                // Docker removes no network a container is on: others' (a collector given an
+                // address on it, say) are taken off it, and keep running.
+                yield* Effect.forEach(Object.values(network.Containers ?? {}), ({ Name }) => docker.disconnectNetwork(cluster, Name).pipe(Effect.andThen(Effect.logInfo(`took ${Name} off network ${cluster}`))), { discard: true });
+                yield* docker.removeNetwork(cluster);
+            }
+            if (volumes)
+                yield* Effect.forEach(yield* docker.listVolumes(labels), ({ Name }) => docker.removeVolume(Name), { discard: true });
+            if (storage)
+                yield* fs.remove(storagePath, { recursive: true, force: true });
+            yield* Effect.logInfo(`removed cluster ${cluster}${volumes ? " and its volumes" : ""}${storage ? `, and ${storagePath}` : ""}`);
+        }),
+        status: Effect.all({
+            docker: Effect.map(docker.version, ({ Version }) => Version),
+            nodes: existingNodes,
+        }),
     });
-}
+});
 //# sourceMappingURL=docker.js.map