@mark1russell7/harness 0.11.16 → 0.11.17

package/build/schema.d.ts

@@ -1,4 +1,4 @@
-import { Y as Subject, i as ClientImage, r as Client, v as ImageSource } from "./cell-B6YHtKFb.js";
+import { Q as Subject, i as ClientImage, r as Client, x as ImageSource } from "./cell-BBPhdtav.js";
 import { z } from "zod";
 //#region src/schema.d.ts
 /**

package/build/schema.js

@@ -1,4 +1,4 @@
-import { S as subjectProblems, a as SHIPPED_NAMES, d as imageProblems, t as CLIENT_NAMES } from "./image-RM60Paqe.js";
+import { C as subjectProblems, a as SHIPPED_NAMES, f as imageProblems, t as CLIENT_NAMES } from "./image-67OW9r9u.js";
 import { z } from "zod";
 //#region src/schema.ts
 /** This function adds each problem that a check finds as an issue of a schema. */

package/image/amqp/Dockerfile

@@ -1,6 +1,6 @@
 # The harness's RabbitMQ client: amqplib 2.2.0 on Node 24.21.0. It runs as a container in the client's
 # network namespace, as the harness's own TCP client does, and prints the same JSON lines, with the
-# events of the connection beside them.
+# events of the connection beside them. The client of a cluster cell and the admin of its lab are here too.
 # lab.ts stages the build context: this folder, under harness/image/amqp. The lockfile is
 # npm-lock.json here, because npm leaves each package-lock.json out of a published tarball.
 FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1
@@ -8,4 +8,4 @@ WORKDIR /opt/harness/amqp
 COPY harness/image/amqp/package.json ./
 COPY harness/image/amqp/npm-lock.json ./package-lock.json
 RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund
-COPY harness/image/amqp/client-amqp.mjs ./
+COPY harness/image/amqp/client-amqp.mjs harness/image/amqp/client-cluster.mjs harness/image/amqp/admin-cluster.mjs ./

package/image/amqp/admin-cluster.mjs

@@ -0,0 +1,38 @@
+// admin-cluster HOST:PORT declare|holds QUEUE
+//
+// The admin of the lab of a cluster cell of RabbitMQ: amqplib 2.2.0, as the user guest. It prints one
+// JSON line with its result.
+// - declare: declares the quorum queue on the node HOST, durable. Its leader locator is client-local,
+//   thus the member on HOST is the first leader. It prints { ok: true }.
+// - holds: takes each message of the queue, and prints { held: [n, ...] }: the sequence numbers that
+//   the queue holds, in their order. The queue is empty after it.
+
+const [at, command, queue] = process.argv.slice(2);
+const print = (value) => process.stdout.write(`${JSON.stringify(value)}\n`);
+if (at === undefined || queue === undefined || !["declare", "holds"].includes(command)) {
+  print({ ok: false, why: "admin-cluster HOST:PORT declare|holds QUEUE" });
+  process.exit(1);
+}
+
+const { default: amqp } = await import("amqplib");
+const connection = await amqp.connect(`amqp://guest:guest@${at}/`);
+const channel = await connection.createChannel();
+try {
+  if (command === "declare") {
+    await channel.assertQueue(queue, { durable: true, arguments: { "x-queue-type": "quorum", "x-queue-leader-locator": "client-local" } });
+    print({ ok: true });
+  } else {
+    const held = [];
+    for (;;) {
+      const message = await channel.get(queue, { noAck: true });
+      if (message === false) break;
+      held.push(Number(message.content.toString()));
+    }
+    print({ held });
+  }
+} catch (error) {
+  print({ ok: false, why: error.message });
+  process.exitCode = 1;
+} finally {
+  await connection.close().catch(() => undefined);
+}

package/image/amqp/client-cluster.mjs

@@ -0,0 +1,188 @@
+// client-cluster HOSTS --queue NAME [--interval-ms N] [--max-s N] [--reconnect-ms N] [--options JSON]
+//
+// The client of a cluster cell of RabbitMQ: amqplib 2.2.0 on Node 24.21.0, as an application publishes
+// with it. It publishes a stream of sequence numbers to one quorum queue on a confirm channel, one at
+// a time, --interval-ms after the confirm of the one before. Each event is a JSON line with `ev` and
+// `t`, the time on CLOCK_MONOTONIC in nanoseconds (process.hrtime, which libuv reads from it). The
+// prober stamps the records of the kernel on the same clock, and the injector stamps each fault on it.
+//
+// HOSTS is host:port,host:port,... in the sequence of the connects: the first is the node of the
+// client. amqplib connects to one node and does not connect again by itself. When the connection ends,
+// this client connects to the next node of the list after --reconnect-ms (1000 ms), as an application
+// does. Then it publishes again the sequence number that has no confirm.
+//
+// The options (--options JSON): `heartbeat` in seconds goes in the URL. Without it, amqplib takes the
+// heartbeat that the broker proposes. `user` and `password` are the user of the broker: guest, as
+// amqplib gives it, when the options name none.
+//
+// The lines:
+// - start: the versions of amqplib and Node, the hosts and the options. clocks: the wall clock.
+// - connecting: each connect, with its node. socket, socket-end, socket-error: the socket of each
+//   connection, with its local address and its node. connected: that socket.
+// - connection, channel: each event of the connection (error, close, blocked, unblocked) and of the
+//   channel (error, close), with the error as data.
+// - ready: the first channel is open, and the stream starts.
+// - sent, ack, fail: each publish by its sequence number and the node that it went through. An ack is
+//   the confirm (basic.ack). A fail is a nack, or a channel that closed before the confirm.
+// - done, deadline: the end of the client.
+
+import { readFileSync } from "node:fs";
+
+const now = () => process.hrtime.bigint();
+const emit = (ev, fields = {}, at = now()) => process.stdout.write(`${JSON.stringify({ ev, t: Number(at), ...fields })}\n`);
+
+const argv = process.argv.slice(2);
+const flag = (name) => {
+  const at = argv.indexOf(name);
+  return at === -1 ? undefined : argv[at + 1];
+};
+const hosts = (argv[0] ?? "").split(",").filter((each) => each !== "");
+const queue = flag("--queue");
+if (hosts.length === 0 || queue === undefined) {
+  emit("fail", { why: "client-cluster HOST:PORT,... --queue NAME" });
+  process.exit(1);
+}
+const interval = Number(flag("--interval-ms") ?? 50);
+const max = Number(flag("--max-s") ?? 120) * 1000;
+const reconnect = Number(flag("--reconnect-ms") ?? 1000);
+const options = JSON.parse(flag("--options") ?? "{}");
+const { heartbeat, user = "guest", password = "guest" } = options;
+
+const { default: amqp } = await import("amqplib");
+const versionOf = (name) => JSON.parse(readFileSync(new URL(`./node_modules/${name}/package.json`, import.meta.url), "utf8")).version;
+const bootId = () => {
+  try {
+    return readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
+  } catch {
+    return null;
+  }
+};
+emit("start", { boot_id: bootId(), client: "amqp-cluster", driver: versionOf("amqplib"), node: process.versions.node, hosts, queue, options });
+// the wall clock beside the monotonic one: the run puts the log lines of the nodes on the monotonic clock
+const clocks = () => emit("clocks", { real: Date.now() });
+clocks();
+setInterval(clocks, 1000).unref();
+
+const urlOf = (at) => `amqp://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${at}/${heartbeat === undefined ? "" : `?heartbeat=${heartbeat}`}`;
+const failure = (error) => (error === undefined || error === null ? null : { name: error.name, code: error.code ?? null, message: error.message });
+const errnoOf = (error) => error?.code ?? error?.message ?? null;
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+let stopped = false;
+let connection = null;
+let channel = null;
+// the node of the connection that is open now, and the place of the next connect in the list
+let at = null;
+let next = 0;
+// this promise ends when the connection that is open now ends
+let ended = null;
+
+/** This function connects to the next node of the list and opens a confirm channel. It gives false when the connect fails. */
+const open = async () => {
+  const node = hosts[next % hosts.length];
+  next += 1;
+  emit("connecting", { at: node });
+  try {
+    connection = await amqp.connect(urlOf(node));
+  } catch (error) {
+    emit("connection", { event: "connect-failed", at: node, errno: errnoOf(error), failure: failure(error) });
+    return false;
+  }
+  const socket = connection.connection.stream;
+  const local = `${socket.localAddress}:${socket.localPort}`;
+  emit("socket", { role: "connection", local, server: node });
+  emit("connected", { local, at: node });
+  socket.once("close", (hadError) => emit("socket-end", { role: "connection", local, server: node, error: hadError }));
+  socket.once("error", (error) => emit("socket-error", { role: "connection", local, server: node, errno: errnoOf(error), message: error.message }));
+  let finish;
+  ended = new Promise((resolve) => (finish = resolve));
+  // amqplib emits each error once, then its close. Without a listener, Node throws the error
+  connection.on("error", (error) => emit("connection", { event: "error", at: node, failure: failure(error) }));
+  // a close of the broker with 320 (CONNECTION_FORCED) is not an error for amqplib: only its close tells
+  connection.on("close", (error) => {
+    emit("connection", { event: "close", at: node, failure: failure(error) });
+    finish();
+  });
+  connection.on("blocked", (reason) => emit("connection", { event: "blocked", at: node, reason }));
+  connection.on("unblocked", () => emit("connection", { event: "unblocked", at: node }));
+  try {
+    channel = await connection.createConfirmChannel();
+  } catch (error) {
+    emit("channel", { event: "open-failed", at: node, failure: failure(error) });
+    return false;
+  }
+  channel.on("error", (error) => emit("channel", { event: "error", at: node, failure: failure(error) }));
+  channel.on("close", () => emit("channel", { event: "close", at: node }));
+  at = node;
+  return true;
+};
+
+/** This function publishes n and waits for its confirm. It gives null for a confirm, or the error. */
+const publish = (n) =>
+  new Promise((resolve) => {
+    const through = at;
+    emit("sent", { n, at: through });
+    try {
+      channel.sendToQueue(queue, Buffer.from(String(n)), { persistent: true }, (error) => {
+        if (error === null || error === undefined) {
+          emit("ack", { n, at: through });
+          resolve(null);
+        } else {
+          emit("fail", { n, at: through, ...failure(error) });
+          resolve(error);
+        }
+      });
+    } catch (error) {
+      emit("fail", { n, at: through, ...failure(error) });
+      resolve(error);
+    }
+  });
+
+const stop = (code = 0) => {
+  if (stopped) return;
+  stopped = true;
+  emit("done");
+  const closing = connection === null ? Promise.resolve() : connection.close().catch(() => undefined);
+  // a dead connection does not answer its close: the client does not wait for it
+  Promise.race([closing, new Promise((resolve) => setTimeout(resolve, 1000))]).finally(() => setTimeout(() => process.exit(code), 50));
+};
+process.on("SIGTERM", () => stop(0));
+setTimeout(() => {
+  emit("deadline");
+  stop(0);
+}, max);
+
+let ready = false;
+let n = 1;
+while (!stopped) {
+  if (channel === null) {
+    if (!(await open())) {
+      channel = null;
+      await sleep(reconnect);
+      continue;
+    }
+    if (!ready) {
+      ready = true;
+      emit("ready", { at });
+    }
+  }
+  // a publish ends at its confirm, at a nack, or at the close of its channel
+  const failed = await publish(n);
+  if (failed === null) {
+    n += 1;
+    await sleep(interval);
+    continue;
+  }
+  // the same sequence number goes again: on the same channel after a nack, or on the next node after an end
+  if (failed.message === "message nacked") {
+    await sleep(interval);
+    continue;
+  }
+  // the channel closed: its connection ends with it, or after one more wait the client ends the connection
+  const closing = connection;
+  await Promise.race([ended, sleep(reconnect)]);
+  closing?.close().catch(() => undefined);
+  channel = null;
+  connection = null;
+  if (!stopped) await sleep(reconnect);
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@mark1russell7/harness",
-  "version": "0.11.16",
+  "version": "0.11.17",
   "description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, through a topology of routers and proxies, a replicated group under Sentinels or a replica set, injects a fault, measures the kernel's own records and judges a prediction. HTTP clients of Go, Node and undici race the keep-alive of a Node server, and the MongoDB driver meets the faults of a replica set. The WebSocket client of @procedures/client drops a silent connection on its own heartbeat, and the harness times that detection and the reconnect from the program's lines and the kernel's stamps.",
   "keywords": [
     "fault-injection",
@@ -74,6 +74,8 @@
     "image/hikari/jars.sha256",
     "image/amqp/Dockerfile",
     "image/amqp/client-amqp.mjs",
+    "image/amqp/client-cluster.mjs",
+    "image/amqp/admin-cluster.mjs",
     "image/amqp/package.json",
     "image/amqp/npm-lock.json",
     "image/procedures/Dockerfile",