lunibee 0.3.1 → 0.3.3

package/dist/sharding/index.js

@@ -2,7 +2,7 @@
 // packages/sharding/package.json
 var package_default = {
   name: "@lunibee/sharding",
-  version: "0.3.1",
+  version: "0.3.3",
   type: "module",
   module: "./src/index.ts",
   types: "./src/index.ts",
@@ -606,6 +606,7 @@ class WebSocketTransport {
         decoded = this.#decoder.push(event.data);
       } catch (error) {
         this.#handlers.onError(this.#asError(error));
+        this.#failDecoding(generation);
         return;
       }
       if (!(decoded instanceof Promise)) {
@@ -623,6 +624,7 @@ class WebSocketTransport {
         if (!this.#owns(generation))
           return;
         this.#handlers.onError(this.#asError(error));
+        this.#failDecoding(generation);
       });
     });
     socket.addEventListener("close", (event) => {
@@ -651,6 +653,10 @@ class WebSocketTransport {
       return false;
     }
   }
+  #failDecoding(generation) {
+    if (this.#owns(generation))
+      this.close(4000, "decoder failure");
+  }
   close(code = 1000, reason = "") {
     const socket = this.#socket;
     if (!socket)
@@ -993,6 +999,13 @@ class Gateway {
   constructor(options) {
     if (!options.token?.trim())
       throw new TypeError("A Gateway token is required.");
+    for (const key of [
+      "reconnectBaseDelay",
+      "reconnectMaxDelay",
+      "connectTimeout"
+    ])
+      if (options[key] !== undefined && !(options[key] > 0))
+        throw new RangeError(`Gateway ${key} must be greater than 0.`);
     const resolvedIntents = resolveGatewayIntents(options.intents);
     if (!Number.isInteger(resolvedIntents) || resolvedIntents < 0)
       throw new TypeError("Gateway intents must be a non-negative integer.");
@@ -1001,6 +1014,7 @@ class Gateway {
       shardCount: 1,
       reconnect: true,
       maxReconnectAttempts: Infinity,
+      connectTimeout: 30000,
       reconnectBaseDelay: 1000,
       reconnectMaxDelay: 30000,
       heartbeatAckTimeout: 1e4,
@@ -1076,7 +1090,7 @@ class Gateway {
     this.#reconnect.cancel();
     this.#closed = false;
     this.#setState("CONNECT" /* Connect */);
-    const connectURL = this.#options.compress ? url.includes("compress=") ? url : `${url}&compress=zlib-stream` : url;
+    const connectURL = this.#options.compress ? url.includes("compress=") ? url : `${url}${url.includes("?") ? "&" : "?"}compress=zlib-stream` : url;
     return this.#reconnect.attempt(() => this.#open(connectURL));
   }
   close() {
@@ -1162,6 +1176,8 @@ class Gateway {
     const count = this.#options.shardCount;
     if (count <= 1)
       return true;
+    if (!/^\d{1,20}$/.test(guildId))
+      return false;
     return Number((BigInt(guildId) >> 22n) % BigInt(count)) === this.#options.shardId;
   }
   requestGuildMembers(data) {
@@ -1177,8 +1193,16 @@ class Gateway {
       return;
     this.#token = this.#session.beginConnection();
     const result = this.#transport.connect(url);
-    if (result.ok)
+    if (result.ok) {
+      const token = this.#token;
+      clearTimeout(this.#connectTimer);
+      this.#connectTimer = setTimeout(() => {
+        if (token === this.#token && !this.#closed)
+          this.#transport.close(4000, "connect timeout");
+      }, this.#options.connectTimeout);
+      this.#connectTimer.unref?.();
       return;
+    }
     const failure = this.#normalizeError(result.error);
     this.#emitError(failure);
     if (this.#options.reconnect) {
@@ -1188,7 +1212,9 @@ class Gateway {
       this.#settleConnect(failure);
     }
   }
+  #connectTimer;
   #onOpen() {
+    clearTimeout(this.#connectTimer);
     this.#heartbeat.watch();
     this.#emit("open", undefined);
     this.#settleConnect();
@@ -1256,7 +1282,8 @@ class Gateway {
     this.#emit("RAW", { event: event ?? "UNKNOWN", data });
     if (this.#closed)
       return;
-    this.#emit(event ?? "dispatch", data);
+    if (event == null || /^[A-Z][A-Z0-9_]*$/.test(event))
+      this.#emit(event ?? "dispatch", data);
   }
   #handleHello(interval) {
     this.#setState(this.#session.canResume ? "RESUME" /* Resume */ : "HELLO" /* Hello */);
@@ -1319,6 +1346,7 @@ class Gateway {
     this.#reconnect.settle(error);
   }
   #clearTimers() {
+    clearTimeout(this.#connectTimer);
     this.#reconnect.cancel();
     this.#heartbeat.stop();
   }
@@ -1451,14 +1479,109 @@ function parseParentFrame(value) {
   }
 }
 
+// packages/sharding/src/thread.ts
+import { EventEmitter } from "events";
+import {
+  isMainThread,
+  parentPort,
+  Worker,
+  workerData
+} from "worker_threads";
+var THREAD_MARKER = "__lunibeeClusterThread";
+
+class ThreadChild extends EventEmitter {
+  pid;
+  exitCode = null;
+  signalCode = null;
+  connected = true;
+  #worker;
+  #signal;
+  constructor(worker) {
+    super();
+    this.#worker = worker;
+    this.pid = worker.threadId;
+    worker.on("message", (frame) => this.emit("message", frame));
+    worker.on("error", (error) => this.emit("error", error));
+    worker.once("exit", (code) => {
+      this.connected = false;
+      if (this.#signal) {
+        this.signalCode = this.#signal;
+        this.emit("exit", null, this.#signal);
+      } else {
+        this.exitCode = code;
+        this.emit("exit", code, null);
+      }
+    });
+  }
+  send(frame) {
+    if (!this.connected)
+      return false;
+    try {
+      this.#worker.postMessage(frame);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  kill(signal = "SIGTERM") {
+    if (!this.connected)
+      return false;
+    this.#signal ??= signal;
+    this.#worker.terminate();
+    return true;
+  }
+}
+function threadFork(options = {}) {
+  return (script, env) => {
+    const defined = {};
+    for (const [key, value] of Object.entries(env))
+      if (value !== undefined)
+        defined[key] = value;
+    const worker = new Worker(script, {
+      env: defined,
+      workerData: { [THREAD_MARKER]: true },
+      ...options.resourceLimits ? { resourceLimits: options.resourceLimits } : {}
+    });
+    return new ThreadChild(worker);
+  };
+}
+var endpoint;
+function threadEndpoint() {
+  const port = parentPort;
+  if (isMainThread || !port || !workerData?.[THREAD_MARKER])
+    return;
+  endpoint ??= portEndpoint(port);
+  return endpoint;
+}
+function portEndpoint(port) {
+  const wrapped = {
+    send: (message) => {
+      port.postMessage(message);
+      return true;
+    },
+    on: (event, listener) => {
+      if (event === "message")
+        port.on("message", listener);
+      return wrapped;
+    },
+    off: (event, listener) => {
+      if (event === "message")
+        port.off("message", listener);
+      return wrapped;
+    }
+  };
+  return wrapped;
+}
+
 // packages/sharding/src/worker.ts
 class ClusterWorker {
   static #current;
   static current() {
     if (!ClusterWorker.#current) {
       const context = readClusterContext();
-      if (context && typeof process.send === "function")
-        ClusterWorker.#current = new ClusterWorker(context, process);
+      const endpoint = typeof process.send === "function" ? process : threadEndpoint();
+      if (context && endpoint)
+        ClusterWorker.#current = new ClusterWorker(context, endpoint);
     }
     return ClusterWorker.#current;
   }
@@ -1474,13 +1597,19 @@ class ClusterWorker {
   #onDisconnect;
   #counter = 0;
   #closed = false;
-  constructor(context, endpoint = process) {
+  constructor(context, endpoint = process, options = {}) {
     if (typeof endpoint.send !== "function")
       throw new Error("ClusterWorker needs an IPC channel: run it in a process forked by ClusterManager.");
     this.context = context;
     this.#endpoint = endpoint;
     this.#onMessage = (frame) => this.#handle(frame);
-    this.#onDisconnect = () => this.#failAll("The parent went away.");
+    const exitOnLoss = options.exitOnParentLoss ?? endpoint === process;
+    const exit = options.exit ?? ((code) => process.exit(code));
+    this.#onDisconnect = () => {
+      this.#failAll("The parent went away.");
+      if (exitOnLoss && !this.#closed)
+        exit(1);
+    };
     endpoint.on("message", this.#onMessage);
     endpoint.on("disconnect", this.#onDisconnect);
   }
@@ -1591,6 +1720,7 @@ class BroadcastChannelTransport {
 }
 var BUS_FRAME = "__lunibeeBus";
 var PEER_DOWN_FRAME = "__lunibeePeerDown";
+var LOCAL = new WeakMap;
 
 class IpcTransport {
   #endpoint;
@@ -1611,9 +1741,18 @@ class IpcTransport {
         this.#down?.(record[PEER_DOWN_FRAME]);
     };
     endpoint.on("message", this.#listener);
+    let local = LOCAL.get(endpoint);
+    if (!local)
+      LOCAL.set(endpoint, local = new Set);
+    local.add(this);
   }
   post(message) {
     this.#endpoint.send({ [BUS_FRAME]: message });
+    for (const peer of LOCAL.get(this.#endpoint) ?? [])
+      if (peer !== this) {
+        const copy = structuredClone(message);
+        queueMicrotask(() => peer.#message?.(copy));
+      }
   }
   onMessage(listener) {
     this.#message = listener;
@@ -1623,11 +1762,17 @@ class IpcTransport {
   }
   close() {
     this.#endpoint.off("message", this.#listener);
+    LOCAL.get(this.#endpoint)?.delete(this);
   }
 }
 function defaultTransport(name) {
-  if (process.env.LUNIBEE_SHARD_BUS === "ipc" && typeof process.send === "function")
-    return new IpcTransport;
+  if (process.env.LUNIBEE_SHARD_BUS === "ipc") {
+    if (typeof process.send === "function")
+      return new IpcTransport;
+    const thread = threadEndpoint();
+    if (thread)
+      return new IpcTransport(thread);
+  }
   return new BroadcastChannelTransport(name);
 }
 
@@ -1645,6 +1790,7 @@ class ShardBus {
   #transport;
   #maxPending;
   #handlers = new Map;
+  #responders = new Map;
   #errorHandlers = new Set;
   #shardId;
   #namespace;
@@ -1675,7 +1821,13 @@ class ShardBus {
     return this;
   }
   off(type, handler) {
-    this.#handlers.get(type)?.delete(handler);
+    const handlers = this.#handlers.get(type);
+    handlers?.delete(handler);
+    const responder = this.#responders.get(type)?.get(handler);
+    if (responder) {
+      handlers?.delete(responder);
+      this.#responders.get(type).delete(handler);
+    }
     return this;
   }
   onError(handler) {
@@ -1697,7 +1849,7 @@ class ShardBus {
     return this.#publish(null, type, data);
   }
   respond(type, handler) {
-    return this.on(type, async (message) => {
+    const wrapper = async (message) => {
       if (!message.expectsReply)
         return;
       const reply = { requestId: message.id };
@@ -1707,7 +1859,12 @@ class ShardBus {
         reply.error = error instanceof Error ? error.message : String(error);
       }
       this.#publish(message.source, REPLY_TYPE, reply);
-    });
+    };
+    let byType = this.#responders.get(type);
+    if (!byType)
+      this.#responders.set(type, byType = new Map);
+    byType.set(handler, wrapper);
+    return this.on(type, wrapper);
   }
   async request(target, type, data, timeoutMs = 5000) {
     if (!Number.isInteger(target) || target < 0)
@@ -1868,7 +2025,7 @@ class ShardSupervisor {
     recent.push(now);
     this.#restarts.set(id, recent);
     const base = Math.min(maxDelay, restartDelay * backoff ** (recent.length - 1));
-    return Math.round(base * (1 + jitter * this.#random()));
+    return Math.min(maxDelay, Math.round(base * (1 + jitter * this.#random())));
   }
   reset(id) {
     if (id === undefined)
@@ -1896,6 +2053,7 @@ class IdentifyCoordinator {
   #remaining;
   #total;
   #resetAt = Number.POSITIVE_INFINITY;
+  #period = 24 * 60 * 60 * 1000;
   #timer;
   #timerAt = Number.POSITIVE_INFINITY;
   #closed = false;
@@ -1922,6 +2080,9 @@ class IdentifyCoordinator {
       this.#total = options.budget.total;
       this.#remaining = options.budget.remaining;
       this.#resetAt = this.#now() + options.budget.resetAfter;
+      this.#period = options.budget.period ?? this.#period;
+      if (!(this.#period > 0))
+        throw new RangeError("budget.period must be positive.");
     }
   }
   get remaining() {
@@ -1995,7 +2156,8 @@ class IdentifyCoordinator {
     const now = this.#now();
     if (this.#total !== undefined && now >= this.#resetAt) {
       this.#remaining = this.#total;
-      this.#resetAt = Number.POSITIVE_INFINITY;
+      while (now >= this.#resetAt)
+        this.#resetAt += this.#period;
     }
     let next = Number.POSITIVE_INFINITY;
     for (const key of this.#keys.values()) {
@@ -2007,11 +2169,6 @@ class IdentifyCoordinator {
       }
       if (this.#remaining !== undefined && this.#remaining <= 0) {
         const waiter = key.queue[0];
-        if (this.#resetAt === Number.POSITIVE_INFINITY) {
-          key.queue.shift();
-          waiter.fail(new Error("The session start limit is exhausted."));
-          continue;
-        }
         this.#onBudgetWait?.({
           shardId: waiter.shardId,
           waitMs: this.#resetAt - now
@@ -2062,7 +2219,7 @@ class IdentifyCoordinator {
 }
 
 // packages/sharding/src/remote.ts
-import { EventEmitter } from "events";
+import { EventEmitter as EventEmitter2 } from "events";
 import { randomBytes } from "crypto";
 
 // packages/sharding/src/remote-protocol.ts
@@ -2213,7 +2370,7 @@ function parseToCoordinator(text, maxBytes = MAX_REMOTE_FRAME_BYTES) {
 }
 
 // packages/sharding/src/remote.ts
-class RemoteChild extends EventEmitter {
+class RemoteChild extends EventEmitter2 {
   host;
   cluster;
   epoch;
@@ -2259,13 +2416,16 @@ class RemoteChild extends EventEmitter {
 class RemoteHosts {
   #options;
   #links = new Map;
+  #droppedAt = new Map;
   #waiters = new Set;
   #hooks;
   #server;
   #watchdog;
   #epoch = 0;
   #ping = 0;
-  #unauthenticated = 0;
+  #pending = new Set;
+  #pendingByAddress = new Map;
+  #rates = new Map;
   constructor(options, hooks) {
     if (typeof options.secret !== "string" || options.secret.length < 16)
       throw new RangeError("remote.secret must be a string of at least 16 characters.");
@@ -2294,6 +2454,13 @@ class RemoteHosts {
   connected() {
     return [...this.#links.keys()].sort();
   }
+  isConnected(hostId) {
+    return this.#links.has(hostId);
+  }
+  fencedAt(hostId) {
+    const dropped = this.#droppedAt.get(hostId);
+    return dropped === undefined || this.#links.has(hostId) ? 0 : dropped + this.#options.lease * 2;
+  }
   start() {
     if (this.#server)
       return;
@@ -2303,8 +2470,17 @@ class RemoteHosts {
       hostname: hostname ?? "127.0.0.1",
       ...tls ? { tls } : {},
       fetch: (request, server) => {
+        const peer = server.requestIP(request)?.address;
+        const address = this.#options.clientAddress?.(request, peer) ?? peer ?? "unknown";
+        if (!this.#allowConnection(address))
+          return new Response("Too many connections.", {
+            status: 429
+          });
         if (server.upgrade(request, {
-          data: { nonce: randomBytes(16).toString("hex") }
+          data: {
+            nonce: randomBytes(16).toString("hex"),
+            address
+          }
         }))
           return;
         return new Response("Lunibee cluster coordinator", {
@@ -2398,14 +2574,46 @@ class RemoteHosts {
       return false;
     }
   }
+  #allowConnection(address) {
+    const rate = this.#options.connectionsPerSecond ?? 10;
+    const now = Date.now();
+    let bucket = this.#rates.get(address);
+    if (!bucket) {
+      bucket = { tokens: rate, at: now };
+      this.#rates.set(address, bucket);
+      if (this.#rates.size > 1e4)
+        this.#rates.delete(this.#rates.keys().next().value);
+    }
+    bucket.tokens = Math.min(rate, bucket.tokens + (now - bucket.at) / 1000 * rate);
+    bucket.at = now;
+    if (bucket.tokens < 1)
+      return false;
+    bucket.tokens--;
+    return true;
+  }
+  #evict(victim) {
+    this.#uncount(victim);
+    clearTimeout(victim.data.helloTimer);
+    victim.close(RemoteClose.TooManyConnections, "Too many connections.");
+  }
   #open(ws) {
-    if (this.#unauthenticated >= (this.#options.maxUnauthenticated ?? 16)) {
-      ws.close(RemoteClose.TooManyConnections, "Too many connections.");
-      return;
+    const address = ws.data.address;
+    if ((this.#pendingByAddress.get(address) ?? 0) >= (this.#options.maxUnauthenticatedPerAddress ?? 4)) {
+      for (const other of this.#pending)
+        if (other.data.address === address) {
+          this.#evict(other);
+          break;
+        }
+    }
+    if (this.#pending.size >= (this.#options.maxUnauthenticated ?? 16)) {
+      const oldest = this.#pending.values().next().value;
+      if (oldest)
+        this.#evict(oldest);
     }
-    this.#unauthenticated++;
+    this.#pending.add(ws);
+    this.#pendingByAddress.set(address, (this.#pendingByAddress.get(address) ?? 0) + 1);
     ws.data.counted = true;
-    ws.data.helloTimer = setTimeout(() => ws.close(RemoteClose.HelloTimeout, "No hello."), this.#options.helloTimeout ?? 1e4);
+    ws.data.helloTimer = setTimeout(() => ws.close(RemoteClose.HelloTimeout, "No hello."), this.#options.helloTimeout ?? 3000);
     ws.send(JSON.stringify({
       t: "challenge",
       nonce: ws.data.nonce,
@@ -2485,6 +2693,7 @@ class RemoteHosts {
       children: new Map
     };
     this.#links.set(id, link);
+    this.#droppedAt.delete(id);
     const lease = this.#options.lease;
     this.#post(link, {
       t: "welcome",
@@ -2496,10 +2705,15 @@ class RemoteHosts {
       waiter.check();
   }
   #uncount(ws) {
-    if (ws.data.counted) {
-      ws.data.counted = false;
-      this.#unauthenticated--;
-    }
+    if (!ws.data.counted)
+      return;
+    ws.data.counted = false;
+    this.#pending.delete(ws);
+    const left = (this.#pendingByAddress.get(ws.data.address) ?? 1) - 1;
+    if (left > 0)
+      this.#pendingByAddress.set(ws.data.address, left);
+    else
+      this.#pendingByAddress.delete(ws.data.address);
   }
   #closed(ws, reason) {
     clearTimeout(ws.data.helloTimer);
@@ -2512,6 +2726,9 @@ class RemoteHosts {
     if (this.#links.get(link.id) !== link)
       return;
     this.#links.delete(link.id);
+    this.#droppedAt.set(link.id, Date.now());
+    if (this.#droppedAt.size > 1000)
+      this.#droppedAt.delete(this.#droppedAt.keys().next().value);
     if (code !== undefined)
       try {
         link.ws.close(code, reason.slice(0, 100));
@@ -2646,7 +2863,13 @@ class ClusterManager {
   #stopPromise;
   #wasStarted = false;
   #stopping = false;
+  #generation = 0;
+  #leading = false;
+  #renewTimer;
+  #lastRenew = 0;
+  #renewing = false;
   #restartTimers = new Set;
+  #fencing = new Map;
   #startTimers = new Map;
   #pauses = new Set;
   #readyWaiters = new Set;
@@ -2694,6 +2917,11 @@ class ClusterManager {
     if (options.shardIds && typeof options.shardCount === "number")
       validateShardIds(options.shardIds, options.shardCount);
     this.#auto = options.shardCount === "auto";
+    const ttl = options.leader?.ttl;
+    if (options.leader && (typeof options.leader.lock?.acquire !== "function" || typeof options.leader.lock.release !== "function"))
+      throw new TypeError("leader.lock must have acquire() and release().");
+    if (ttl !== undefined && (!Number.isInteger(ttl) || ttl < 100))
+      throw new RangeError("leader.ttl must be an integer of at least 100.");
     this.#options = { ...options };
     this.#supervisor = new ShardSupervisor({
       restartDelay: options.restartDelay,
@@ -2702,7 +2930,7 @@ class ClusterManager {
     this.#tracker = new RequestTracker("parent", options.maxPendingRequests);
     if (options.remote)
       this.#remote = new RemoteHosts(options.remote, {
-        join: (id) => this.#emit("hostJoin", id),
+        join: (id) => this.#hostJoined(id),
         leave: (id, reason) => this.#emit("hostLeave", id, reason),
         error: (error) => this.#emit("error", error)
       });
@@ -2776,8 +3004,15 @@ class ClusterManager {
     };
   }
   async#start() {
+    const generation = this.#generation;
+    const stale = () => this.#stopping || generation !== this.#generation;
+    if (this.#options.leader && !this.#leading) {
+      await this.#lead();
+      if (stale())
+        return;
+    }
     const { count, maxConcurrency, budget } = await this.#resolveGateway();
-    if (this.#stopping)
+    if (stale())
       return;
     const ids = this.#options.shardIds ?? Array.from({ length: count }, (_, i) => i);
     validateShardIds(ids, count);
@@ -2785,11 +3020,11 @@ class ClusterManager {
     const chunks = allocateShards(ids, clusterCount);
     assertCompleteAllocation(chunks, ids);
     const placement = await this.#placeHosts().catch((error) => {
-      if (this.#stopping)
+      if (stale())
         return;
       throw error;
     });
-    if (!placement || this.#stopping)
+    if (!placement || stale())
       return;
     this.shardCount = count;
     this.#wasStarted = true;
@@ -2815,7 +3050,7 @@ class ClusterManager {
     for (const chunk of active)
       this.#hostOf.set(chunk.id, placement[chunk.id % placement.length]);
     for (const [index, chunk] of active.entries()) {
-      if (this.#stopping)
+      if (stale())
         return;
       for (const shard of chunk.shards)
         this.#owner.set(shard, chunk.id);
@@ -2824,7 +3059,7 @@ class ClusterManager {
       if (index < active.length - 1 && gap > 0)
         await this.#pause(gap);
     }
-    if (this.#stopping)
+    if (stale())
       return;
     if (this.#options.waitForReady)
       await this.ready();
@@ -2834,6 +3069,104 @@ class ClusterManager {
       }, this.#options.autoScaleInterval);
     }
   }
+  get isLeader() {
+    return !this.#options.leader || this.#leading;
+  }
+  async#lead() {
+    const { lock, ttl = 1e4 } = this.#options.leader;
+    const generation = this.#generation;
+    while (!this.#stopping && generation === this.#generation) {
+      const started = Date.now();
+      let held = false;
+      try {
+        held = await withTimeout(lock.acquire(ttl), ttl / 2);
+      } catch (error) {
+        this.#emit("error", asError(error));
+      }
+      if (this.#stopping || generation !== this.#generation) {
+        if (held)
+          lock.release().catch(() => {
+            return;
+          });
+        return;
+      }
+      if (held) {
+        this.#leading = true;
+        this.#lastRenew = started;
+        this.#startRenewing(ttl);
+        this.#emit("leaderAcquired");
+        const lease = this.#options.remote ? this.#options.remote.lease ?? 15000 : 0;
+        if (lease > 0)
+          await this.#pause(lease * 2);
+        return;
+      }
+      await this.#pause(Math.floor(ttl / 2));
+    }
+  }
+  #startRenewing(ttl) {
+    this.#stopRenewing();
+    const lock = this.#options.leader.lock;
+    this.#renewTimer = setInterval(() => {
+      if (Date.now() - this.#lastRenew > ttl / 2) {
+        this.#standDown("The leader lock could not be renewed.");
+        return;
+      }
+      if (this.#renewing)
+        return;
+      this.#renewing = true;
+      const started = Date.now();
+      lock.acquire(ttl).then((held) => {
+        this.#renewing = false;
+        if (!this.#leading)
+          return;
+        if (held)
+          this.#lastRenew = started;
+        else
+          this.#standDown("Another coordinator holds the leader lock.");
+      }, (error) => {
+        this.#renewing = false;
+        this.#emit("error", asError(error));
+      });
+    }, Math.max(10, Math.floor(ttl / 4)));
+    this.#renewTimer.unref?.();
+  }
+  #stopRenewing() {
+    if (this.#renewTimer)
+      clearInterval(this.#renewTimer);
+    this.#renewTimer = undefined;
+  }
+  async#resign() {
+    this.#stopRenewing();
+    if (!this.#leading)
+      return;
+    this.#leading = false;
+    const { lock, ttl = 1e4 } = this.#options.leader;
+    try {
+      await withTimeout(lock.release(), ttl / 2);
+    } catch (error) {
+      this.#emit("error", asError(error));
+    }
+  }
+  #standDown(reason) {
+    if (!this.#leading)
+      return;
+    this.#stopRenewing();
+    this.#leading = false;
+    this.#stopping = true;
+    this.#halt();
+    for (const cluster of this.#infos.values()) {
+      cluster.state = "stopped";
+      try {
+        cluster.process.kill("SIGKILL");
+      } catch {}
+    }
+    this.clusters.clear();
+    this.#startPromise = undefined;
+    this.#remote?.stop();
+    this.#stopping = false;
+    this.#emit("leaderLost", reason);
+    this.spawn().catch((error) => this.#emit("error", asError(error)));
+  }
   async#placeHosts() {
     const remote = this.#remote;
     const options = this.#options.remote;
@@ -2901,6 +3234,9 @@ class ClusterManager {
     this.#infos.set(id, info);
     this.clusters.set(id, info);
     this.#allReady = false;
+    child.on("error", (error) => this.#emit("error", new Error(`Cluster ${id} process error: ${error.message}`, {
+      cause: error
+    })));
     child.on("message", (frame) => this.#onFrame(info, child, frame));
     child.once("exit", (code, signal) => this.#onExit(info, child, code, signal));
     const timeout = this.#options.startTimeout;
@@ -2942,7 +3278,7 @@ class ClusterManager {
     this.#announceDown(info.id, info.shards);
     if (this.#options.restartOnExit === false) {
       info.state = "failed";
-      this.#checkReady();
+      this.#rejectReady(info, "restartOnExit is false");
       return;
     }
     const delay = this.#supervisor.next(info.id);
@@ -2952,9 +3288,10 @@ class ClusterManager {
         this.#options.onGiveUp?.(info);
       } catch {}
       this.#emit("clusterGiveUp", info);
-      this.#checkReady();
       if (this.#options.reassignOnGiveUp)
         this.#reassign(info);
+      else
+        this.#rejectReady(info, "restarts exhausted");
       return;
     }
     info.state = "restarting";
@@ -2965,26 +3302,73 @@ class ClusterManager {
         attempt: info.restarts,
         delay
       });
+    this.#restartLater(info, delay);
+  }
+  #restartLater(info, delay) {
     const timer = setTimeout(() => {
       this.#restartTimers.delete(timer);
+      this.#fencing.delete(info.id);
       if (this.#stopping || info.state !== "restarting")
         return;
-      this.#launch(info.id, info.shards);
+      const wait = this.#relocate(info);
+      if (wait > 0)
+        this.#fencing.set(info.id, this.#restartLater(info, wait));
+      else
+        this.#launch(info.id, info.shards);
     }, delay);
     timer.unref?.();
     this.#restartTimers.add(timer);
+    return timer;
+  }
+  #hostJoined(hostId) {
+    this.#emit("hostJoin", hostId);
+    for (const [id, timer] of [...this.#fencing]) {
+      const info = this.#infos.get(id);
+      if (this.#hostOf.get(id) !== hostId || !info)
+        continue;
+      clearTimeout(timer);
+      this.#restartTimers.delete(timer);
+      this.#fencing.delete(id);
+      if (!this.#stopping && info.state === "restarting")
+        this.#launch(info.id, info.shards);
+    }
+  }
+  #relocate(info) {
+    const remote = this.#remote;
+    const host = this.#hostOf.get(info.id) ?? "local";
+    if (!remote || this.#options.remote?.moveClusters === false || host === "local" || remote.isConnected(host))
+      return 0;
+    const wait = remote.fencedAt(host) - Date.now();
+    if (wait > 0)
+      return wait;
+    const places = this.#options.remote?.local ? ["local", ...remote.connected()] : remote.connected();
+    if (places.length === 0)
+      return 0;
+    const load = new Map(places.map((place) => [place, 0]));
+    for (const [cluster, place] of this.#hostOf) {
+      const state = this.#infos.get(cluster)?.state;
+      if (load.has(place) && state !== "failed" && state !== "stopped")
+        load.set(place, load.get(place) + 1);
+    }
+    const target = places.reduce((best, place) => load.get(place) < load.get(best) ? place : best);
+    this.#hostOf.set(info.id, target);
+    info.host = target;
+    this.#emit("clusterMoved", info, { from: host, to: target });
+    return 0;
   }
   async#reassign(failed) {
     const shardIds = [...failed.shards];
     const target = [...this.#infos.values()].filter((cluster) => cluster.state === "ready").sort((a, b) => a.shards.length - b.shards.length)[0];
     if (!target) {
       this.#emit("error", new Error(`No ready cluster can take over shards [${shardIds}] of cluster ${failed.id}.`));
+      this.#rejectReady(failed, "no ready cluster to take them");
       return;
     }
     try {
       await this.requestCluster(target.id, REQUEST_ASSIGN, { shardIds }, { timeoutMs: this.#options.startTimeout ?? 120000 });
     } catch (error) {
       this.#emit("error", new Error(`Cluster ${target.id} could not take over shards [${shardIds}]: ${error instanceof Error ? error.message : String(error)}`));
+      this.#rejectReady(failed, "reassignment failed");
       return;
     }
     if (this.#stopping || failed.state !== "failed")
@@ -2998,6 +3382,7 @@ class ClusterManager {
       from: failed.id,
       to: target.id
     });
+    this.#checkReady();
   }
   #clearStartTimer(id) {
     const timer = this.#startTimers.get(id);
@@ -3133,9 +3518,20 @@ class ClusterManager {
       throw new RangeError(`${what} is larger than maxMessageBytes (${this.#maxBytes}).`);
   }
   #onBus(from, raw) {
-    const message = raw;
-    if (!message || typeof message !== "object" || typeof message.type !== "string" || typeof message.source !== "number" || message.target !== null && !Number.isInteger(message.target))
+    let message = raw;
+    if (!message || typeof message !== "object" || typeof message.type !== "string" || message.type.length === 0 || message.type.length > 256 || message.id === undefined || typeof message.id !== "string" || message.id.length > 256 || !Number.isInteger(message.source) || message.target !== null && !Number.isInteger(message.target))
       return;
+    if (!from.shards.includes(message.source))
+      return;
+    const { type, id, source, target, data, expectsReply } = message;
+    message = {
+      type,
+      id,
+      source,
+      target,
+      data,
+      ...expectsReply === undefined ? {} : { expectsReply }
+    };
     if (this.#tooBig(message.data)) {
       this.#emit("error", new Error(`Cluster ${from.id} sent a shard-bus message of more than ${this.#maxBytes} bytes; it was not relayed.`));
       return;
@@ -3279,12 +3675,21 @@ class ClusterManager {
     });
   }
   #isReady() {
-    if (this.#infos.size === 0)
-      return false;
-    for (const cluster of this.#infos.values())
-      if (cluster.state !== "ready")
+    let ready = 0;
+    for (const cluster of this.#infos.values()) {
+      if (cluster.state === "ready")
+        ready++;
+      else if (!(cluster.state === "failed" && cluster.shards.length === 0))
         return false;
-    return true;
+    }
+    return ready > 0;
+  }
+  #rejectReady(failed, why) {
+    if (failed.shards.length === 0)
+      return;
+    const error = new Error(`Cluster ${failed.id} failed and shards [${failed.shards}] have no owner (${why}).`);
+    for (const waiter of [...this.#readyWaiters])
+      waiter.reject(error);
   }
   #checkReady() {
     if (!this.#isReady())
@@ -3371,14 +3776,18 @@ class ClusterManager {
       cluster.state = "stopped";
     this.clusters.clear();
     this.#startPromise = undefined;
-    if (!keepRemote)
+    if (!keepRemote) {
       this.#remote?.stop();
+      await this.#resign();
+    }
     this.#stopping = false;
   }
   #halt() {
+    this.#generation++;
     for (const timer of this.#restartTimers)
       clearTimeout(timer);
     this.#restartTimers.clear();
+    this.#fencing.clear();
     for (const id of [...this.#startTimers.keys()])
       this.#clearStartTimer(id);
     for (const wake of [...this.#pauses])
@@ -3433,9 +3842,20 @@ class ClusterManager {
     this.clusters.clear();
     this.#startPromise = undefined;
     this.#remote?.stop();
+    this.#resign();
     this.#stopping = false;
   }
 }
+function withTimeout(promise, ms) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`The leader lock did not answer within ${ms} ms.`)), ms);
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+function asError(error) {
+  return error instanceof Error ? error : new Error(String(error));
+}
 // packages/sharding/src/host.ts
 import { fork as fork2 } from "child_process";
 class ClusterHost {
@@ -3443,11 +3863,16 @@ class ClusterHost {
   #running = new Map;
   #listeners = new Map;
   #socket;
+  #urls;
+  #urlIndex = 0;
+  #unreachable = 0;
+  #accepted = false;
   #lease = 15000;
   #lastRx = 0;
   #watchdog;
   #retry;
   #delay;
+  #busyAttempts = 0;
   #closed = true;
   #connected = false;
   #firstConnect;
@@ -3458,18 +3883,24 @@ class ClusterHost {
       throw new RangeError("secret must be at least 16 characters.");
     if (!options.script?.trim())
       throw new TypeError("A worker script path is required.");
-    let url;
-    try {
-      url = new URL(options.url);
-    } catch {
-      throw new TypeError("url must be an absolute ws:// or wss:// URL.");
+    const urls = Array.isArray(options.url) ? [...options.url] : [options.url];
+    if (urls.length === 0 || urls.length > 16)
+      throw new TypeError("url must list 1 to 16 coordinators.");
+    for (const raw of urls) {
+      let url;
+      try {
+        url = new URL(raw);
+      } catch {
+        throw new TypeError("url must be an absolute ws:// or wss:// URL.");
+      }
+      if (url.protocol !== "ws:" && url.protocol !== "wss:")
+        throw new TypeError("url must start with ws:// or wss://.");
+      if (url.username || url.password)
+        throw new TypeError("url must not contain credentials.");
+      if (url.protocol === "ws:" && !options.allowInsecure && !isLoopbackHost(url.hostname))
+        throw new TypeError("url must use wss:// to reach another machine (allowInsecure: true overrides this for a network you fully trust).");
     }
-    if (url.protocol !== "ws:" && url.protocol !== "wss:")
-      throw new TypeError("url must start with ws:// or wss://.");
-    if (url.username || url.password)
-      throw new TypeError("url must not contain credentials.");
-    if (url.protocol === "ws:" && !options.allowInsecure && !isLoopbackHost(url.hostname))
-      throw new TypeError("url must use wss:// to reach another machine (allowInsecure: true overrides this for a network you fully trust).");
+    this.#urls = urls;
     this.#options = options;
     this.#delay = options.reconnectDelay ?? 1000;
   }
@@ -3500,6 +3931,8 @@ class ClusterHost {
     if (!this.#closed)
       return Promise.resolve();
     this.#closed = false;
+    this.#busyAttempts = 0;
+    this.#unreachable = 0;
     return new Promise((resolve, reject) => {
       this.#firstConnect = { resolve, reject };
       this.#open();
@@ -3531,7 +3964,8 @@ class ClusterHost {
     });
   }
   #open() {
-    const socket = new WebSocket(this.#options.url);
+    this.#accepted = false;
+    const socket = new WebSocket(this.#urls[this.#urlIndex]);
     this.#socket = socket;
     socket.addEventListener("message", (event) => this.#message(socket, event.data));
     socket.addEventListener("error", () => {});
@@ -3550,12 +3984,31 @@ class ClusterHost {
       return false;
     }
   }
+  #refuseSpawn(socket, data) {
+    if (typeof data !== "string" || data.length > 65536)
+      return;
+    let frame;
+    try {
+      frame = JSON.parse(data);
+    } catch {
+      return;
+    }
+    if (frame?.t !== "spawn" || !Number.isInteger(frame.cluster) || !Number.isInteger(frame.epoch))
+      return;
+    this.#send(socket, {
+      t: "error",
+      cluster: frame.cluster,
+      epoch: frame.epoch,
+      message: "This host refused the spawn request (malformed or not allowed)."
+    });
+  }
   #message(socket, data) {
     if (this.#socket !== socket)
       return;
     const message = parseToHost(data);
     if (!message) {
       this.#emit("error", new Error("The coordinator sent a malformed frame."));
+      this.#refuseSpawn(socket, data);
       return;
     }
     this.#lastRx = Date.now();
@@ -3574,6 +4027,7 @@ class ClusterHost {
         return;
       case "welcome":
         this.#lease = message.leaseMs;
+        this.#accepted = true;
         this.#connected = true;
         this.#delay = this.#options.reconnectDelay ?? 1000;
         this.#startWatchdog();
@@ -3687,6 +4141,18 @@ class ClusterHost {
       return;
     const refused = code === RemoteClose.AuthFailed || code === RemoteClose.UnknownHost || code === RemoteClose.DuplicateHost || code === RemoteClose.VersionMismatch;
     const first = this.#firstConnect;
+    const busy = code === RemoteClose.TooManyConnections || code === RemoteClose.HelloTimeout;
+    const accepted = this.#accepted;
+    if (this.#urls.length > 1)
+      this.#urlIndex = (this.#urlIndex + 1) % this.#urls.length;
+    if (first && !refused && !busy && !accepted && ++this.#unreachable < this.#urls.length) {
+      this.#open();
+      return;
+    }
+    if (first && busy && ++this.#busyAttempts < (this.#options.connectAttempts ?? 10)) {
+      this.#retryLater();
+      return;
+    }
     if (first) {
       this.#firstConnect = undefined;
       this.#closed = true;
@@ -3695,6 +4161,9 @@ class ClusterHost {
     }
     if (refused)
       this.#emit("error", new Error(`The coordinator refused this host (${code}: ${reason}).`));
+    this.#retryLater();
+  }
+  #retryLater() {
     const wait = Math.round(this.#delay * (1 + Math.random() * 0.2));
     this.#delay = Math.min(this.#options.maxReconnectDelay ?? 30000, this.#delay * 2);
     this.#retry = setTimeout(() => {
@@ -3707,6 +4176,43 @@ class ClusterHost {
 function defaultFork(script, env) {
   return fork2(script, [], { env });
 }
+// packages/sharding/src/leader.ts
+var ACQUIRE = `local v = redis.call('GET', KEYS[1])
+if v == ARGV[1] then redis.call('PEXPIRE', KEYS[1], ARGV[2]) return 1 end
+if not v then redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2]) return 1 end
+return 0`;
+var RELEASE = `if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end
+return 0`;
+
+class RedisLeaderLock {
+  #client;
+  #key;
+  #token;
+  constructor(options) {
+    if (typeof options?.client?.send !== "function")
+      throw new TypeError("RedisLeaderLock needs a client with send().");
+    this.#client = options.client;
+    this.#key = options.key ?? "lunibee:cluster:leader";
+    this.#token = options.token ?? crypto.randomUUID();
+    if (!this.#key || !this.#token)
+      throw new TypeError("key and token must not be empty.");
+  }
+  async acquire(ttlMs) {
+    if (!Number.isInteger(ttlMs) || ttlMs < 1)
+      throw new RangeError("ttlMs must be a positive integer.");
+    const reply = await this.#client.send("EVAL", [
+      ACQUIRE,
+      "1",
+      this.#key,
+      this.#token,
+      String(ttlMs)
+    ]);
+    return Number(reply) === 1;
+  }
+  async release() {
+    await this.#client.send("EVAL", [RELEASE, "1", this.#key, this.#token]);
+  }
+}
 
 // packages/sharding/src/index.ts
 var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -3757,6 +4263,7 @@ class ShardManager {
   #resolved = false;
   #destroyed = false;
   #auto;
+  #fixedCount;
   #autoScaleTimer;
   #total;
   #listeners = new Map;
@@ -3799,6 +4306,7 @@ class ShardManager {
     if (options.handshakeTimeout !== undefined && (!Number.isFinite(options.handshakeTimeout) || options.handshakeTimeout < 0))
       throw new RangeError("handshakeTimeout must be a non-negative number of milliseconds.");
     this.#total = count;
+    this.#fixedCount = count;
     this.#auto = options.shardCount === "auto";
     this.#options = {
       spawnDelay: ShardManager.IDENTIFY_INTERVAL,
@@ -3830,7 +4338,7 @@ class ShardManager {
     }
     await this.#ensureInitialized();
     await this.#start([...this.shards].filter(([, gateway]) => needsConnection(gateway)).map(([id]) => id));
-    if (this.#options.autoScaleInterval && !this.#autoScaleTimer) {
+    if (this.#options.autoScaleInterval && this.#auto && !this.#autoScaleTimer) {
       this.#autoScaleTimer = setInterval(() => {
         this.checkAutoScale();
       }, this.#options.autoScaleInterval);
@@ -3905,6 +4413,8 @@ class ShardManager {
         this.#setStatus(id, "connecting");
       const results = await Promise.allSettled(round.map((id) => this.shards.get(id).connect()));
       const failed = results.findIndex((result) => result.status === "rejected");
+      if (limit)
+        limit.remaining = Math.max(0, limit.remaining - results.filter((r) => r.status === "fulfilled").length);
       if (failed !== -1) {
         if (destroyOnFailure)
           this.destroy();
@@ -4041,7 +4551,7 @@ class ShardManager {
       return;
     if (this.#destroyed)
       this.#destroyed = false;
-    const count = this.#options.shardCount === "auto" ? await this.fetchRecommendedShardCount() : this.#options.shardCount ?? 1;
+    const count = this.#options.shardCount === "auto" ? await this.fetchRecommendedShardCount() : this.#fixedCount;
     this.#initialize(count);
   }
   #initialize(count) {
@@ -4079,6 +4589,7 @@ class ShardManager {
       this.#setStatus(id, "unstable");
       this.#emit("shardUnstable", id, info);
     });
+    this.#emit("shardCreate", id, gateway);
     return gateway;
   }
 }
@@ -4092,6 +4603,7 @@ export {
   IpcTransport,
   MAX_GUILDS_PER_SHARD,
   REMOTE_PROTOCOL_VERSION,
+  RedisLeaderLock,
   RemoteClose,
   RequestTracker,
   ShardBus,
@@ -4105,7 +4617,9 @@ export {
   isClusterWorker,
   readClusterContext,
   shardsForGuilds,
+  threadEndpoint,
+  threadFork,
   validateShardIds
 };
 
-//# debugId=7D2FE3134CAB1F3364756E2164756E21
+//# debugId=D3CF0E421E8E307864756E2164756E21