@specific.dev/spectest 0.131.9 → 0.131.10

package/dist/components/k3s-join.d.ts

@@ -0,0 +1,2 @@
+export declare const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
+export declare function prepareK3sJoin(server: boolean, directory?: string): string;

package/dist/components/k3s-join.js

@@ -0,0 +1,13 @@
+export const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
+// The join volume is private to one cluster and is captured with its network.
+// Use the server's numeric address from the beginning: K3s otherwise demotes
+// the DNS fallback after discovery and closes in-flight bootstrap requests.
+export function prepareK3sJoin(server, directory = JOIN_DIRECTORY) {
+    if (server)
+        return ('SERVER="$(ip -4 route get 1.1.1.1 | awk \'{for (i=1;i<NF;i++) if ($i=="src") {print $(i+1); exit}}\')"; ' +
+            'case "$SERVER" in ""|*[!0-9.]*) echo "spectest: missing server IPv4 address" >&2; exit 1;; esac; ' +
+            `printf '%s\\n' "$SERVER" > ${directory}/server.tmp && mv ${directory}/server.tmp ${directory}/server || exit 1; ` +
+            `if [ ! -s ${directory}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${directory}/token) || exit 1; fi; `);
+    return (`while [ ! -s ${directory}/token ] || [ ! -s ${directory}/server ]; do sleep 0.1; done; ` +
+        `SERVER="$(cat ${directory}/server)"; `);
+}

package/dist/components/k3s.js

@@ -1,5 +1,6 @@
 import { isIP } from "node:net";
 import { waitForK3sNodes } from "./k3s-nodes.js";
+import { JOIN_DIRECTORY, prepareK3sJoin } from "./k3s-join.js";
 import { installPodTrust } from "./k3s-trust.js";
 import { k3sCoverage } from "./k3s-coverage.js";
 import { waitForAll } from "../harness/concurrency.js";
@@ -21,7 +22,6 @@ import { deepUnwrap, readTag, wrap } from "../inspect.js";
 import { recorderAnnotate, recorderRemove } from "../recorder.js";
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
-const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
 /** The socket of the throwaway daemon that prepares the store. */
@@ -1374,7 +1374,7 @@ function createK3sCluster(opts, count) {
             "--kubelet-arg=kubelet-cgroups=/spectest-$(hostname)/k3s",
             "--kubelet-arg=runtime-cgroups=/spectest-$(hostname)/k3s",
             `--token-file=${JOIN_DIRECTORY}/token`,
-            ...(server ? [] : [`--server=https://${cluster}.internal:6443`]),
+            ...(server ? [] : ['--server="https://$SERVER:6443"']),
             // CoreDNS / pod DNS upstream. Without this, k3s sees only the
             // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
             // /etc/resolv.conf, decides no usable nameserver exists, and writes a
@@ -1464,9 +1464,7 @@ function createK3sCluster(opts, count) {
             'sed "s/[^ ][^ ]*/+&/g" "/sys/fs/cgroup/$CG/cgroup.controllers" > "/sys/fs/cgroup/$CG/cgroup.subtree_control" || exit 1; ' +
             'else for C in /sys/fs/cgroup/*; do [ ! -d "$C" ] || mkdir -p "$C/$CG/k3s" || exit 1; done; fi; ' +
             storePrepareSh() +
-            (server
-                ? `if [ ! -s ${JOIN_DIRECTORY}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${JOIN_DIRECTORY}/token); fi; `
-                : `while [ ! -s ${JOIN_DIRECTORY}/token ]; do sleep 0.1; done; `) +
+            prepareK3sJoin(server) +
             // Sticky and environment-local stores use overlayfs.
             `exec ${nodeArgs} --snapshotter=$SNAP`;
         // Plain /readyz probe. On a warm zot cache the cluster's images are

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.131.9",
+  "version": "0.131.10",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/components/k3s-docker.test.ts

@@ -65,6 +65,13 @@ for (const count of [1, 3]) test.skipIf(!process.env.SPECTEST_TEST_K3S_DOCKER)(`
       await Bun.sleep(250);
     }
     await primary.setup!({ name, helpers, exec } as unknown as ServiceSetupContext<K3sHelpers>);
+    // Discovery must retain the bootstrap address. A DNS fallback is demoted
+    // to standby and closes active requests, which can kill initial Flannel
+    // authorization even though the API server remains healthy.
+    for (const node of names.filter(node => node !== name)) {
+      const config = JSON.parse(await checked(["docker", "exec", node, "cat", "/var/lib/rancher/k3s/agent/etc/k3s-agent-load-balancer.json"]));
+      expect(config.ServerAddresses).toContain(new URL(config.ServerURL).host);
+    }
     await helpers.apply(JSON.stringify({ apiVersion: "apps/v1", kind: "DaemonSet", metadata: { name: "web" }, spec: {
       selector: { matchLabels: { app: "web" } }, template: { metadata: { labels: { app: "web" } }, spec: {
         containers: [{ name: "web", image: "busybox:1.37", imagePullPolicy: "Never",

package/src/components/k3s-join.test.ts

@@ -0,0 +1,38 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { networkInterfaces, tmpdir } from "node:os";
+import { join } from "node:path";
+import { prepareK3sJoin } from "./k3s-join";
+
+test("concurrent agents join the server's real numeric identity and retain the private token on restart", async () => {
+  const directory = await mkdtemp(join(tmpdir(), "k3s-join-"));
+  const children: ReturnType<typeof Bun.spawn>[] = [];
+  const start = (server: boolean) => {
+    const child = Bun.spawn(["sh", "-c", prepareK3sJoin(server, directory) + 'printf "%s" "$SERVER"'], {
+      stdout: "pipe", stderr: "pipe",
+    });
+    children.push(child);
+    return child;
+  };
+  const finish = async (child: ReturnType<typeof start>) => {
+    expect(await child.exited).toBe(0);
+    return new Response(child.stdout).text();
+  };
+  try {
+    const agents = [start(false), start(false)];
+    const address = await finish(start(true));
+    const localAddresses = Object.values(networkInterfaces()).flat().filter(a => a?.family === "IPv4").map(a => a!.address);
+    expect(localAddresses).toContain(address);
+    expect(await Promise.all(agents.map(finish))).toEqual([address, address]);
+    expect((await readFile(join(directory, "server"), "utf8")).trim()).toBe(address);
+    const token = await readFile(join(directory, "token"), "utf8");
+    expect(token.trim()).toMatch(/^[0-9a-f-]{36}$/);
+    expect((await stat(join(directory, "token"))).mode & 0o777).toBe(0o600);
+    expect(await finish(start(true))).toBe(address);
+    expect(await readFile(join(directory, "token"), "utf8")).toBe(token);
+  } finally {
+    for (const child of children) if (child.exitCode === null) child.kill();
+    await Promise.all(children.map(child => child.exited));
+    await rm(directory, { recursive: true, force: true });
+  }
+}, 10_000);

package/src/components/k3s-join.ts

@@ -0,0 +1,17 @@
+export const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
+
+// The join volume is private to one cluster and is captured with its network.
+// Use the server's numeric address from the beginning: K3s otherwise demotes
+// the DNS fallback after discovery and closes in-flight bootstrap requests.
+export function prepareK3sJoin(server: boolean, directory = JOIN_DIRECTORY): string {
+  if (server) return (
+    'SERVER="$(ip -4 route get 1.1.1.1 | awk \'{for (i=1;i<NF;i++) if ($i=="src") {print $(i+1); exit}}\')"; ' +
+    'case "$SERVER" in ""|*[!0-9.]*) echo "spectest: missing server IPv4 address" >&2; exit 1;; esac; ' +
+    `printf '%s\\n' "$SERVER" > ${directory}/server.tmp && mv ${directory}/server.tmp ${directory}/server || exit 1; ` +
+    `if [ ! -s ${directory}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${directory}/token) || exit 1; fi; `
+  );
+  return (
+    `while [ ! -s ${directory}/token ] || [ ! -s ${directory}/server ]; do sleep 0.1; done; ` +
+    `SERVER="$(cat ${directory}/server)"; `
+  );
+}

package/src/components/k3s-nodes.test.ts

@@ -22,7 +22,7 @@ test("one and multiple nodes use the same topology and preserve primary override
   for (const name of ["cluster-node-2", "cluster-node-3"]) {
     const node = three.services[name];
     expect(node.command).toContain("exec k3s agent");
-    expect(node.command).toContain("--server=https://cluster.internal:6443");
+    expect(node.command).toContain('--server="https://$SERVER:6443"');
     expect(node.command).toContain("--kubelet-arg=cgroup-root=/spectest-$(hostname)");
     expect(node.dependsOn).toBeUndefined(); // joins while the server boots
     expect(node.imageBuilds).toHaveLength(0); // build once, import into every node

package/src/components/k3s.ts

@@ -1,5 +1,6 @@
 import { isIP } from "node:net";
 import { waitForK3sNodes } from "./k3s-nodes.js";
+import { JOIN_DIRECTORY, prepareK3sJoin } from "./k3s-join.js";
 import { installPodTrust } from "./k3s-trust.js";
 import { k3sCoverage } from "./k3s-coverage.js";
 import { waitForAll } from "../harness/concurrency.js";
@@ -189,7 +190,6 @@ export interface K3sOptions {
 
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
-const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
 
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
@@ -2008,7 +2008,7 @@ function createK3sCluster(opts: K3sOptions, count: number) {
       "--kubelet-arg=kubelet-cgroups=/spectest-$(hostname)/k3s",
       "--kubelet-arg=runtime-cgroups=/spectest-$(hostname)/k3s",
       `--token-file=${JOIN_DIRECTORY}/token`,
-      ...(server ? [] : [`--server=https://${cluster}.internal:6443`]),
+      ...(server ? [] : ['--server="https://$SERVER:6443"']),
       // CoreDNS / pod DNS upstream. Without this, k3s sees only the
       // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
       // /etc/resolv.conf, decides no usable nameserver exists, and writes a
@@ -2099,9 +2099,7 @@ function createK3sCluster(opts: K3sOptions, count: number) {
       'sed "s/[^ ][^ ]*/+&/g" "/sys/fs/cgroup/$CG/cgroup.controllers" > "/sys/fs/cgroup/$CG/cgroup.subtree_control" || exit 1; ' +
       'else for C in /sys/fs/cgroup/*; do [ ! -d "$C" ] || mkdir -p "$C/$CG/k3s" || exit 1; done; fi; ' +
       storePrepareSh() +
-      (server
-        ? `if [ ! -s ${JOIN_DIRECTORY}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${JOIN_DIRECTORY}/token); fi; `
-        : `while [ ! -s ${JOIN_DIRECTORY}/token ]; do sleep 0.1; done; `) +
+      prepareK3sJoin(server) +
       // Sticky and environment-local stores use overlayfs.
       `exec ${nodeArgs} --snapshotter=$SNAP`;
     // Plain /readyz probe. On a warm zot cache the cluster's images are