lunibee 0.3.2 → 0.3.3

package/dist/sharding/index.js

@@ -2,7 +2,7 @@
 // packages/sharding/package.json
 var package_default = {
   name: "@lunibee/sharding",
-  version: "0.3.2",
+  version: "0.3.3",
   type: "module",
   module: "./src/index.ts",
   types: "./src/index.ts",
@@ -1479,14 +1479,109 @@ function parseParentFrame(value) {
   }
 }
 
+// packages/sharding/src/thread.ts
+import { EventEmitter } from "events";
+import {
+  isMainThread,
+  parentPort,
+  Worker,
+  workerData
+} from "worker_threads";
+var THREAD_MARKER = "__lunibeeClusterThread";
+
+class ThreadChild extends EventEmitter {
+  pid;
+  exitCode = null;
+  signalCode = null;
+  connected = true;
+  #worker;
+  #signal;
+  constructor(worker) {
+    super();
+    this.#worker = worker;
+    this.pid = worker.threadId;
+    worker.on("message", (frame) => this.emit("message", frame));
+    worker.on("error", (error) => this.emit("error", error));
+    worker.once("exit", (code) => {
+      this.connected = false;
+      if (this.#signal) {
+        this.signalCode = this.#signal;
+        this.emit("exit", null, this.#signal);
+      } else {
+        this.exitCode = code;
+        this.emit("exit", code, null);
+      }
+    });
+  }
+  send(frame) {
+    if (!this.connected)
+      return false;
+    try {
+      this.#worker.postMessage(frame);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  kill(signal = "SIGTERM") {
+    if (!this.connected)
+      return false;
+    this.#signal ??= signal;
+    this.#worker.terminate();
+    return true;
+  }
+}
+function threadFork(options = {}) {
+  return (script, env) => {
+    const defined = {};
+    for (const [key, value] of Object.entries(env))
+      if (value !== undefined)
+        defined[key] = value;
+    const worker = new Worker(script, {
+      env: defined,
+      workerData: { [THREAD_MARKER]: true },
+      ...options.resourceLimits ? { resourceLimits: options.resourceLimits } : {}
+    });
+    return new ThreadChild(worker);
+  };
+}
+var endpoint;
+function threadEndpoint() {
+  const port = parentPort;
+  if (isMainThread || !port || !workerData?.[THREAD_MARKER])
+    return;
+  endpoint ??= portEndpoint(port);
+  return endpoint;
+}
+function portEndpoint(port) {
+  const wrapped = {
+    send: (message) => {
+      port.postMessage(message);
+      return true;
+    },
+    on: (event, listener) => {
+      if (event === "message")
+        port.on("message", listener);
+      return wrapped;
+    },
+    off: (event, listener) => {
+      if (event === "message")
+        port.off("message", listener);
+      return wrapped;
+    }
+  };
+  return wrapped;
+}
+
 // packages/sharding/src/worker.ts
 class ClusterWorker {
   static #current;
   static current() {
     if (!ClusterWorker.#current) {
       const context = readClusterContext();
-      if (context && typeof process.send === "function")
-        ClusterWorker.#current = new ClusterWorker(context, process);
+      const endpoint = typeof process.send === "function" ? process : threadEndpoint();
+      if (context && endpoint)
+        ClusterWorker.#current = new ClusterWorker(context, endpoint);
     }
     return ClusterWorker.#current;
   }
@@ -1671,8 +1766,13 @@ class IpcTransport {
   }
 }
 function defaultTransport(name) {
-  if (process.env.LUNIBEE_SHARD_BUS === "ipc" && typeof process.send === "function")
-    return new IpcTransport;
+  if (process.env.LUNIBEE_SHARD_BUS === "ipc") {
+    if (typeof process.send === "function")
+      return new IpcTransport;
+    const thread = threadEndpoint();
+    if (thread)
+      return new IpcTransport(thread);
+  }
   return new BroadcastChannelTransport(name);
 }
 
@@ -2119,7 +2219,7 @@ class IdentifyCoordinator {
 }
 
 // packages/sharding/src/remote.ts
-import { EventEmitter } from "events";
+import { EventEmitter as EventEmitter2 } from "events";
 import { randomBytes } from "crypto";
 
 // packages/sharding/src/remote-protocol.ts
@@ -2270,7 +2370,7 @@ function parseToCoordinator(text, maxBytes = MAX_REMOTE_FRAME_BYTES) {
 }
 
 // packages/sharding/src/remote.ts
-class RemoteChild extends EventEmitter {
+class RemoteChild extends EventEmitter2 {
   host;
   cluster;
   epoch;
@@ -2316,6 +2416,7 @@ class RemoteChild extends EventEmitter {
 class RemoteHosts {
   #options;
   #links = new Map;
+  #droppedAt = new Map;
   #waiters = new Set;
   #hooks;
   #server;
@@ -2353,6 +2454,13 @@ class RemoteHosts {
   connected() {
     return [...this.#links.keys()].sort();
   }
+  isConnected(hostId) {
+    return this.#links.has(hostId);
+  }
+  fencedAt(hostId) {
+    const dropped = this.#droppedAt.get(hostId);
+    return dropped === undefined || this.#links.has(hostId) ? 0 : dropped + this.#options.lease * 2;
+  }
   start() {
     if (this.#server)
       return;
@@ -2585,6 +2693,7 @@ class RemoteHosts {
       children: new Map
     };
     this.#links.set(id, link);
+    this.#droppedAt.delete(id);
     const lease = this.#options.lease;
     this.#post(link, {
       t: "welcome",
@@ -2617,6 +2726,9 @@ class RemoteHosts {
     if (this.#links.get(link.id) !== link)
       return;
     this.#links.delete(link.id);
+    this.#droppedAt.set(link.id, Date.now());
+    if (this.#droppedAt.size > 1000)
+      this.#droppedAt.delete(this.#droppedAt.keys().next().value);
     if (code !== undefined)
       try {
         link.ws.close(code, reason.slice(0, 100));
@@ -2751,7 +2863,13 @@ class ClusterManager {
   #stopPromise;
   #wasStarted = false;
   #stopping = false;
+  #generation = 0;
+  #leading = false;
+  #renewTimer;
+  #lastRenew = 0;
+  #renewing = false;
   #restartTimers = new Set;
+  #fencing = new Map;
   #startTimers = new Map;
   #pauses = new Set;
   #readyWaiters = new Set;
@@ -2799,6 +2917,11 @@ class ClusterManager {
     if (options.shardIds && typeof options.shardCount === "number")
       validateShardIds(options.shardIds, options.shardCount);
     this.#auto = options.shardCount === "auto";
+    const ttl = options.leader?.ttl;
+    if (options.leader && (typeof options.leader.lock?.acquire !== "function" || typeof options.leader.lock.release !== "function"))
+      throw new TypeError("leader.lock must have acquire() and release().");
+    if (ttl !== undefined && (!Number.isInteger(ttl) || ttl < 100))
+      throw new RangeError("leader.ttl must be an integer of at least 100.");
     this.#options = { ...options };
     this.#supervisor = new ShardSupervisor({
       restartDelay: options.restartDelay,
@@ -2807,7 +2930,7 @@ class ClusterManager {
     this.#tracker = new RequestTracker("parent", options.maxPendingRequests);
     if (options.remote)
       this.#remote = new RemoteHosts(options.remote, {
-        join: (id) => this.#emit("hostJoin", id),
+        join: (id) => this.#hostJoined(id),
         leave: (id, reason) => this.#emit("hostLeave", id, reason),
         error: (error) => this.#emit("error", error)
       });
@@ -2881,8 +3004,15 @@ class ClusterManager {
     };
   }
   async#start() {
+    const generation = this.#generation;
+    const stale = () => this.#stopping || generation !== this.#generation;
+    if (this.#options.leader && !this.#leading) {
+      await this.#lead();
+      if (stale())
+        return;
+    }
     const { count, maxConcurrency, budget } = await this.#resolveGateway();
-    if (this.#stopping)
+    if (stale())
       return;
     const ids = this.#options.shardIds ?? Array.from({ length: count }, (_, i) => i);
     validateShardIds(ids, count);
@@ -2890,11 +3020,11 @@ class ClusterManager {
     const chunks = allocateShards(ids, clusterCount);
     assertCompleteAllocation(chunks, ids);
     const placement = await this.#placeHosts().catch((error) => {
-      if (this.#stopping)
+      if (stale())
         return;
       throw error;
     });
-    if (!placement || this.#stopping)
+    if (!placement || stale())
       return;
     this.shardCount = count;
     this.#wasStarted = true;
@@ -2920,7 +3050,7 @@ class ClusterManager {
     for (const chunk of active)
       this.#hostOf.set(chunk.id, placement[chunk.id % placement.length]);
     for (const [index, chunk] of active.entries()) {
-      if (this.#stopping)
+      if (stale())
         return;
       for (const shard of chunk.shards)
         this.#owner.set(shard, chunk.id);
@@ -2929,7 +3059,7 @@ class ClusterManager {
       if (index < active.length - 1 && gap > 0)
         await this.#pause(gap);
     }
-    if (this.#stopping)
+    if (stale())
       return;
     if (this.#options.waitForReady)
       await this.ready();
@@ -2939,6 +3069,104 @@ class ClusterManager {
       }, this.#options.autoScaleInterval);
     }
   }
+  get isLeader() {
+    return !this.#options.leader || this.#leading;
+  }
+  async#lead() {
+    const { lock, ttl = 1e4 } = this.#options.leader;
+    const generation = this.#generation;
+    while (!this.#stopping && generation === this.#generation) {
+      const started = Date.now();
+      let held = false;
+      try {
+        held = await withTimeout(lock.acquire(ttl), ttl / 2);
+      } catch (error) {
+        this.#emit("error", asError(error));
+      }
+      if (this.#stopping || generation !== this.#generation) {
+        if (held)
+          lock.release().catch(() => {
+            return;
+          });
+        return;
+      }
+      if (held) {
+        this.#leading = true;
+        this.#lastRenew = started;
+        this.#startRenewing(ttl);
+        this.#emit("leaderAcquired");
+        const lease = this.#options.remote ? this.#options.remote.lease ?? 15000 : 0;
+        if (lease > 0)
+          await this.#pause(lease * 2);
+        return;
+      }
+      await this.#pause(Math.floor(ttl / 2));
+    }
+  }
+  #startRenewing(ttl) {
+    this.#stopRenewing();
+    const lock = this.#options.leader.lock;
+    this.#renewTimer = setInterval(() => {
+      if (Date.now() - this.#lastRenew > ttl / 2) {
+        this.#standDown("The leader lock could not be renewed.");
+        return;
+      }
+      if (this.#renewing)
+        return;
+      this.#renewing = true;
+      const started = Date.now();
+      lock.acquire(ttl).then((held) => {
+        this.#renewing = false;
+        if (!this.#leading)
+          return;
+        if (held)
+          this.#lastRenew = started;
+        else
+          this.#standDown("Another coordinator holds the leader lock.");
+      }, (error) => {
+        this.#renewing = false;
+        this.#emit("error", asError(error));
+      });
+    }, Math.max(10, Math.floor(ttl / 4)));
+    this.#renewTimer.unref?.();
+  }
+  #stopRenewing() {
+    if (this.#renewTimer)
+      clearInterval(this.#renewTimer);
+    this.#renewTimer = undefined;
+  }
+  async#resign() {
+    this.#stopRenewing();
+    if (!this.#leading)
+      return;
+    this.#leading = false;
+    const { lock, ttl = 1e4 } = this.#options.leader;
+    try {
+      await withTimeout(lock.release(), ttl / 2);
+    } catch (error) {
+      this.#emit("error", asError(error));
+    }
+  }
+  #standDown(reason) {
+    if (!this.#leading)
+      return;
+    this.#stopRenewing();
+    this.#leading = false;
+    this.#stopping = true;
+    this.#halt();
+    for (const cluster of this.#infos.values()) {
+      cluster.state = "stopped";
+      try {
+        cluster.process.kill("SIGKILL");
+      } catch {}
+    }
+    this.clusters.clear();
+    this.#startPromise = undefined;
+    this.#remote?.stop();
+    this.#stopping = false;
+    this.#emit("leaderLost", reason);
+    this.spawn().catch((error) => this.#emit("error", asError(error)));
+  }
   async#placeHosts() {
     const remote = this.#remote;
     const options = this.#options.remote;
@@ -3074,14 +3302,59 @@ class ClusterManager {
         attempt: info.restarts,
         delay
       });
+    this.#restartLater(info, delay);
+  }
+  #restartLater(info, delay) {
     const timer = setTimeout(() => {
       this.#restartTimers.delete(timer);
+      this.#fencing.delete(info.id);
       if (this.#stopping || info.state !== "restarting")
         return;
-      this.#launch(info.id, info.shards);
+      const wait = this.#relocate(info);
+      if (wait > 0)
+        this.#fencing.set(info.id, this.#restartLater(info, wait));
+      else
+        this.#launch(info.id, info.shards);
     }, delay);
     timer.unref?.();
     this.#restartTimers.add(timer);
+    return timer;
+  }
+  #hostJoined(hostId) {
+    this.#emit("hostJoin", hostId);
+    for (const [id, timer] of [...this.#fencing]) {
+      const info = this.#infos.get(id);
+      if (this.#hostOf.get(id) !== hostId || !info)
+        continue;
+      clearTimeout(timer);
+      this.#restartTimers.delete(timer);
+      this.#fencing.delete(id);
+      if (!this.#stopping && info.state === "restarting")
+        this.#launch(info.id, info.shards);
+    }
+  }
+  #relocate(info) {
+    const remote = this.#remote;
+    const host = this.#hostOf.get(info.id) ?? "local";
+    if (!remote || this.#options.remote?.moveClusters === false || host === "local" || remote.isConnected(host))
+      return 0;
+    const wait = remote.fencedAt(host) - Date.now();
+    if (wait > 0)
+      return wait;
+    const places = this.#options.remote?.local ? ["local", ...remote.connected()] : remote.connected();
+    if (places.length === 0)
+      return 0;
+    const load = new Map(places.map((place) => [place, 0]));
+    for (const [cluster, place] of this.#hostOf) {
+      const state = this.#infos.get(cluster)?.state;
+      if (load.has(place) && state !== "failed" && state !== "stopped")
+        load.set(place, load.get(place) + 1);
+    }
+    const target = places.reduce((best, place) => load.get(place) < load.get(best) ? place : best);
+    this.#hostOf.set(info.id, target);
+    info.host = target;
+    this.#emit("clusterMoved", info, { from: host, to: target });
+    return 0;
   }
   async#reassign(failed) {
     const shardIds = [...failed.shards];
@@ -3503,14 +3776,18 @@ class ClusterManager {
       cluster.state = "stopped";
     this.clusters.clear();
     this.#startPromise = undefined;
-    if (!keepRemote)
+    if (!keepRemote) {
       this.#remote?.stop();
+      await this.#resign();
+    }
     this.#stopping = false;
   }
   #halt() {
+    this.#generation++;
     for (const timer of this.#restartTimers)
       clearTimeout(timer);
     this.#restartTimers.clear();
+    this.#fencing.clear();
     for (const id of [...this.#startTimers.keys()])
       this.#clearStartTimer(id);
     for (const wake of [...this.#pauses])
@@ -3565,9 +3842,20 @@ class ClusterManager {
     this.clusters.clear();
     this.#startPromise = undefined;
     this.#remote?.stop();
+    this.#resign();
     this.#stopping = false;
   }
 }
+function withTimeout(promise, ms) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`The leader lock did not answer within ${ms} ms.`)), ms);
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+function asError(error) {
+  return error instanceof Error ? error : new Error(String(error));
+}
 // packages/sharding/src/host.ts
 import { fork as fork2 } from "child_process";
 class ClusterHost {
@@ -3575,6 +3863,10 @@ class ClusterHost {
   #running = new Map;
   #listeners = new Map;
   #socket;
+  #urls;
+  #urlIndex = 0;
+  #unreachable = 0;
+  #accepted = false;
   #lease = 15000;
   #lastRx = 0;
   #watchdog;
@@ -3591,18 +3883,24 @@ class ClusterHost {
       throw new RangeError("secret must be at least 16 characters.");
     if (!options.script?.trim())
       throw new TypeError("A worker script path is required.");
-    let url;
-    try {
-      url = new URL(options.url);
-    } catch {
-      throw new TypeError("url must be an absolute ws:// or wss:// URL.");
+    const urls = Array.isArray(options.url) ? [...options.url] : [options.url];
+    if (urls.length === 0 || urls.length > 16)
+      throw new TypeError("url must list 1 to 16 coordinators.");
+    for (const raw of urls) {
+      let url;
+      try {
+        url = new URL(raw);
+      } catch {
+        throw new TypeError("url must be an absolute ws:// or wss:// URL.");
+      }
+      if (url.protocol !== "ws:" && url.protocol !== "wss:")
+        throw new TypeError("url must start with ws:// or wss://.");
+      if (url.username || url.password)
+        throw new TypeError("url must not contain credentials.");
+      if (url.protocol === "ws:" && !options.allowInsecure && !isLoopbackHost(url.hostname))
+        throw new TypeError("url must use wss:// to reach another machine (allowInsecure: true overrides this for a network you fully trust).");
     }
-    if (url.protocol !== "ws:" && url.protocol !== "wss:")
-      throw new TypeError("url must start with ws:// or wss://.");
-    if (url.username || url.password)
-      throw new TypeError("url must not contain credentials.");
-    if (url.protocol === "ws:" && !options.allowInsecure && !isLoopbackHost(url.hostname))
-      throw new TypeError("url must use wss:// to reach another machine (allowInsecure: true overrides this for a network you fully trust).");
+    this.#urls = urls;
     this.#options = options;
     this.#delay = options.reconnectDelay ?? 1000;
   }
@@ -3634,6 +3932,7 @@ class ClusterHost {
       return Promise.resolve();
     this.#closed = false;
     this.#busyAttempts = 0;
+    this.#unreachable = 0;
     return new Promise((resolve, reject) => {
       this.#firstConnect = { resolve, reject };
       this.#open();
@@ -3665,7 +3964,8 @@ class ClusterHost {
     });
   }
   #open() {
-    const socket = new WebSocket(this.#options.url);
+    this.#accepted = false;
+    const socket = new WebSocket(this.#urls[this.#urlIndex]);
     this.#socket = socket;
     socket.addEventListener("message", (event) => this.#message(socket, event.data));
     socket.addEventListener("error", () => {});
@@ -3727,6 +4027,7 @@ class ClusterHost {
         return;
       case "welcome":
         this.#lease = message.leaseMs;
+        this.#accepted = true;
         this.#connected = true;
         this.#delay = this.#options.reconnectDelay ?? 1000;
         this.#startWatchdog();
@@ -3841,6 +4142,13 @@ class ClusterHost {
     const refused = code === RemoteClose.AuthFailed || code === RemoteClose.UnknownHost || code === RemoteClose.DuplicateHost || code === RemoteClose.VersionMismatch;
     const first = this.#firstConnect;
     const busy = code === RemoteClose.TooManyConnections || code === RemoteClose.HelloTimeout;
+    const accepted = this.#accepted;
+    if (this.#urls.length > 1)
+      this.#urlIndex = (this.#urlIndex + 1) % this.#urls.length;
+    if (first && !refused && !busy && !accepted && ++this.#unreachable < this.#urls.length) {
+      this.#open();
+      return;
+    }
     if (first && busy && ++this.#busyAttempts < (this.#options.connectAttempts ?? 10)) {
       this.#retryLater();
       return;
@@ -3868,6 +4176,43 @@ class ClusterHost {
 function defaultFork(script, env) {
   return fork2(script, [], { env });
 }
+// packages/sharding/src/leader.ts
+var ACQUIRE = `local v = redis.call('GET', KEYS[1])
+if v == ARGV[1] then redis.call('PEXPIRE', KEYS[1], ARGV[2]) return 1 end
+if not v then redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2]) return 1 end
+return 0`;
+var RELEASE = `if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end
+return 0`;
+
+class RedisLeaderLock {
+  #client;
+  #key;
+  #token;
+  constructor(options) {
+    if (typeof options?.client?.send !== "function")
+      throw new TypeError("RedisLeaderLock needs a client with send().");
+    this.#client = options.client;
+    this.#key = options.key ?? "lunibee:cluster:leader";
+    this.#token = options.token ?? crypto.randomUUID();
+    if (!this.#key || !this.#token)
+      throw new TypeError("key and token must not be empty.");
+  }
+  async acquire(ttlMs) {
+    if (!Number.isInteger(ttlMs) || ttlMs < 1)
+      throw new RangeError("ttlMs must be a positive integer.");
+    const reply = await this.#client.send("EVAL", [
+      ACQUIRE,
+      "1",
+      this.#key,
+      this.#token,
+      String(ttlMs)
+    ]);
+    return Number(reply) === 1;
+  }
+  async release() {
+    await this.#client.send("EVAL", [RELEASE, "1", this.#key, this.#token]);
+  }
+}
 
 // packages/sharding/src/index.ts
 var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -4258,6 +4603,7 @@ export {
   IpcTransport,
   MAX_GUILDS_PER_SHARD,
   REMOTE_PROTOCOL_VERSION,
+  RedisLeaderLock,
   RemoteClose,
   RequestTracker,
   ShardBus,
@@ -4271,7 +4617,9 @@ export {
   isClusterWorker,
   readClusterContext,
   shardsForGuilds,
+  threadEndpoint,
+  threadFork,
   validateShardIds
 };
 
-//# debugId=8AD6CE4D87FD51CB64756E2164756E21
+//# debugId=D3CF0E421E8E307864756E2164756E21