actor-ts 0.9.0

package/dist/cluster/Cluster.js

@@ -0,0 +1,837 @@
+import { match } from 'ts-pattern';
+import { LogContext } from '../LogContext.js';
+import { metricsOf } from '../metrics/MetricsExtension.js';
+import { tracerOf } from '../tracing/TracingExtension.js';
+import { DEFAULT_GOSSIP_INTERVAL_MS, DEFAULT_SEED_RETRY_INTERVAL_MS, DEFAULT_TOMBSTONE_PRUNE_INTERVAL_MS, DEFAULT_TOMBSTONE_TTL_MS, } from '../util/Constants.js';
+import { none, some } from '../util/Option.js';
+import { LeaderChanged, MemberDown, MemberJoined, MemberLeft, MemberReachable, MemberRemoved, MemberUnreachable, MemberUp, MemberWeaklyUp, SelfRemoved, SelfUp, } from './ClusterEvents.js';
+import { defaultFailureDetectorSettings, FailureDetector, } from './FailureDetector.js';
+import { Member } from './Member.js';
+import { NodeAddress } from './NodeAddress.js';
+// `ClusterSharding` only imports `Cluster` as a type (erased at runtime),
+// so the value-import here doesn't create a runtime cycle — every
+// sharding file uses `import type { Cluster }`.
+import { ClusterSharding } from './sharding/ClusterSharding.js';
+import { decodeRefs, encodeRefs, parsePathSegments } from './RefCodec.js';
+import { InMemoryTransport, TcpTransport } from './Transport.js';
+/**
+ * Maximum allowed deviation between an incoming gossip version and the
+ * local wall-clock — 1 day.  Anything above is rejected as an exploit
+ * attempt (see {@link Cluster.mergeMember} for the full walkthrough).
+ *
+ * Tuned generous-but-finite: a legitimate node with a 23-hour clock
+ * skew still merges; a node claiming `version: Number.MAX_SAFE_INTEGER`
+ * (≈ 285 000 years above now) is rejected on the spot.
+ */
+const MAX_VERSION_SKEW_MS = 24 * 60 * 60 * 1_000;
+/**
+ * The Cluster is a single-instance "extension" attached to an ActorSystem.
+ * It owns a Transport, a gossip-based membership view, a failure detector
+ * and the plumbing that dispatches inbound envelope messages to local actors.
+ */
+export class Cluster {
+    selfAddress;
+    selfRoles;
+    system;
+    transport;
+    log;
+    members = new Map();
+    failureDetector;
+    gossipIntervalMs;
+    seedRetryIntervalMs;
+    seedAddrs = [];
+    tombstoneTtlMs;
+    tombstonePruneIntervalMs;
+    tombstoneMinRetentionMs;
+    heartbeatSeq = 0;
+    gossipTimer = null;
+    heartbeatTimer = null;
+    fdTimer = null;
+    seedTimer = null;
+    weaklyUpTimer = null;
+    tombstonePruneTimer = null;
+    currentLeader = none;
+    weaklyUpAfterMs;
+    envelopeHandler = null;
+    _envelopeHandlersByPath = new Map();
+    wireHandlers = new Map();
+    started = false;
+    downing;
+    /**
+     * Set of unreachable address keys observed at the last downing
+     * evaluation.  We only re-invoke the provider when the set
+     * actually changes — without this debounce a steady "always one
+     * unreachable peer" state would call `decide()` on every tick.
+     */
+    lastDownedView = null;
+    constructor(system, settings) {
+        this.system = system;
+        this.selfAddress = new NodeAddress(system.name, settings.host, settings.port);
+        this.selfRoles = new Set(settings.roles ?? []);
+        this.log = system.log.withSource(`cluster@${this.selfAddress}`);
+        this.transport = settings.transport ?? new TcpTransport(this.selfAddress, this.log);
+        const fdSettings = {
+            ...defaultFailureDetectorSettings,
+            ...(settings.failureDetector ?? {}),
+        };
+        this.failureDetector = new FailureDetector(fdSettings);
+        this.gossipIntervalMs = settings.gossipIntervalMs ?? DEFAULT_GOSSIP_INTERVAL_MS;
+        this.seedRetryIntervalMs = settings.seedRetryIntervalMs ?? DEFAULT_SEED_RETRY_INTERVAL_MS;
+        this.weaklyUpAfterMs = settings.weaklyUpAfterMs ?? 0;
+        this.downing = settings.downing ?? null;
+        this.tombstoneTtlMs = settings.tombstoneTtlMs ?? DEFAULT_TOMBSTONE_TTL_MS;
+        this.tombstonePruneIntervalMs = settings.tombstonePruneIntervalMs ?? DEFAULT_TOMBSTONE_PRUNE_INTERVAL_MS;
+        this.tombstoneMinRetentionMs =
+            settings.tombstoneMinRetentionMs ?? 6 * fdSettings.downAfterMs;
+    }
+    /** Entry point: start the cluster and attempt to contact seed nodes. */
+    static async join(system, settings) {
+        const cluster = new Cluster(system, settings);
+        await cluster._start(settings.seeds ?? []);
+        return cluster;
+    }
+    /**
+     * One-call bootstrap — creates the {@link ActorSystem}, joins the
+     * cluster, starts the Receptionist, waits for `SelfUp`, and wires
+     * SIGTERM/SIGINT shutdown.  The headline shape for the clustered
+     * case:
+     *
+     * ```ts
+     * const { system, cluster, shutdown } = await Cluster.bootstrap({ name: 'my-app' });
+     * ```
+     *
+     * For full configuration knobs see {@link ClusterBootstrapOptions}.
+     * For the power-user path (own `ActorSystem.create`, own
+     * `Cluster.join`, own signal wiring) keep using those directly —
+     * this is purely additive sugar.
+     *
+     * The helper lives in `ClusterBootstrap.ts` and is loaded lazily
+     * here to avoid a static import cycle (`ClusterBootstrap` ↔
+     * `Cluster.join`).
+     */
+    static async bootstrap(opts) {
+        const { bootstrapCluster } = await import('./ClusterBootstrap.js');
+        return bootstrapCluster(opts);
+    }
+    /**
+     * The cluster's sharding facade.  Lazily constructs (and memoises) a
+     * single `ClusterSharding` instance per `ActorSystem` so callers can
+     * start regions inline:
+     *
+     * ```ts
+     * const region = cluster.sharding.start('cart', CartActor, {
+     *   extractEntityId: (m) => m.entityId,
+     * });
+     * ```
+     *
+     * Equivalent to `ClusterSharding.get(cluster.system, cluster)` —
+     * which still works for callers that prefer the explicit form.
+     */
+    get sharding() {
+        return ClusterSharding.get(this.system, this);
+    }
+    /**
+     * Subscribe to membership events.  The listener is immediately replayed
+     * the current cluster state as a series of Member/SelfUp events so that
+     * late subscribers still see the world they joined.
+     */
+    subscribe(listener) {
+        this._listeners.push(listener);
+        // Replay current state.
+        for (const m of this.members.values()) {
+            try {
+                listener(new MemberJoined(m));
+            }
+            catch { /* ignore */ }
+            if (m.status === 'up') {
+                try {
+                    listener(new MemberUp(m));
+                }
+                catch { /* ignore */ }
+                if (m.address.equals(this.selfAddress)) {
+                    try {
+                        listener(new SelfUp(m));
+                    }
+                    catch { /* ignore */ }
+                }
+            }
+        }
+        if (this.currentLeader.isSome()) {
+            try {
+                listener(new LeaderChanged(this.currentLeader));
+            }
+            catch { /* ignore */ }
+        }
+        return () => {
+            const idx = this._listeners.indexOf(listener);
+            if (idx >= 0)
+                this._listeners.splice(idx, 1);
+        };
+    }
+    _listeners = [];
+    /**
+     * Current snapshot of known members.  `removed` entries are kept
+     * internally as tombstones (so stale gossip can't resurrect them
+     * via the merge path) but are filtered out here — the public
+     * contract is "members the cluster currently considers part of
+     * the topology".
+     */
+    getMembers() {
+        return Array.from(this.members.values()).filter((m) => m.status !== 'removed');
+    }
+    /** Members in the `up` state, ordered by address — the "active set". */
+    upMembers() {
+        return Array.from(this.members.values())
+            .filter(m => m.status === 'up')
+            .sort((a, b) => a.address.compareTo(b.address));
+    }
+    /** Reachable members (up + joining + leaving). */
+    reachableMembers() {
+        return Array.from(this.members.values()).filter(m => m.isReachable());
+    }
+    /** Up members that carry the given role tag. */
+    upMembersWithRole(role) {
+        return this.upMembers().filter(m => m.hasRole(role));
+    }
+    /** The oldest up-member is the cluster leader (deterministic across nodes). */
+    leader() {
+        const ups = this.upMembers();
+        return ups.length > 0 ? some(ups[0]) : none;
+    }
+    /** True if this node is currently the leader. */
+    isLeader() {
+        return this.leader().exists((l) => l.address.equals(this.selfAddress));
+    }
+    /**
+     * Register a handler for inbound user envelopes.  Kept for backward
+     * compatibility — prefer `_registerEnvelopeHandler(path, handler)` which
+     * allows multiple extensions (ClusterSharding, DistributedPubSub, …) to
+     * share the envelope pipeline.
+     */
+    _setEnvelopeHandler(handler) {
+        this.envelopeHandler = handler;
+    }
+    /** Route envelopes addressed to `path` to `handler`.  Returns unsubscribe. */
+    _registerEnvelopeHandler(path, handler) {
+        this._envelopeHandlersByPath.set(path, handler);
+        return () => this._envelopeHandlersByPath.delete(path);
+    }
+    /**
+     * Send an envelope to a remote node.  Used by RemoteActorRef and by the
+     * PubSub / Singleton extensions.  Any `ActorRef` embedded in the user
+     * payload is rewritten to a `WireActorRef` marker here — this is the
+     * single chokepoint where every cross-node message leaves, so hooking
+     * the encode step once covers all paths (sharding, pub-sub, singleton,
+     * direct remote-ref).  Receiving nodes decode in `handleEnvelope`.
+     */
+    _sendEnvelope(to, env) {
+        const encoded = { ...env, body: encodeRefs(env.body, this.selfAddress) };
+        this.transport.send(to, encoded);
+    }
+    /** Register a handler for a specific wire-message discriminator. */
+    _onWire(kind, handler) {
+        this.wireHandlers.set(kind, handler);
+        return () => this.wireHandlers.delete(kind);
+    }
+    /**
+     * Operator-initiated force-down of a remote peer (#56).  Mirrors the
+     * private `evaluateDowning` path: marks the peer `down`, emits the
+     * lifecycle events, tombstones with `removedAt` so stale gossip
+     * can't resurrect it, and tells the failure detector to forget it.
+     *
+     * Returns `true` if a member was found and downed, `false` if the
+     * address was unknown or already terminal (`down`/`removed`).
+     *
+     * Intended for operator tooling — the management HTTP endpoint
+     * `POST /cluster/down` calls this directly.  Don't use it as a
+     * replacement for the failure detector / downing provider in normal
+     * flow; it's a manual override.
+     */
+    down(addr) {
+        if (!this.started)
+            return false;
+        const key = typeof addr === 'string' ? addr : addr.toString();
+        const m = this.members.get(key);
+        if (!m)
+            return false;
+        if (m.status === 'down' || m.status === 'removed')
+            return false;
+        if (m.address.equals(this.selfAddress)) {
+            // Don't try to tombstone ourselves — that's `leave()`'s job.
+            return false;
+        }
+        const downed = m.withStatus('down');
+        this.updateMember(downed);
+        this.emit(new MemberDown(downed));
+        const removed = downed.withRemoved(Date.now());
+        this.members.set(key, removed);
+        this.failureDetector.forget(m.address);
+        this.emit(new MemberRemoved(removed));
+        this.log.info(`operator force-down: ${m.address}`);
+        return true;
+    }
+    /** Gracefully leave the cluster (broadcast `leave`, stop transport). */
+    async leave() {
+        if (!this.started)
+            return;
+        this.started = false;
+        const me = this.members.get(this.selfAddress.toString());
+        if (me) {
+            this.updateMember(me.withStatus('leaving'));
+        }
+        const leaveMsg = { t: 'leave', node: this.selfAddress.toJSON() };
+        const peers = this.reachableMembers().filter((m) => !m.address.equals(this.selfAddress));
+        this.log.debug(`leaving — sending leave to ${peers.length} reachable peer(s)`);
+        for (const m of peers)
+            this.transport.send(m.address, leaveMsg);
+        this.gossipTimer?.cancel();
+        this.heartbeatTimer?.cancel();
+        this.fdTimer?.cancel();
+        this.seedTimer?.cancel();
+        this.weaklyUpTimer?.cancel();
+        this.tombstonePruneTimer?.cancel();
+        await this.transport.shutdown();
+    }
+    /* ================================ Internal ================================ */
+    async _start(seeds) {
+        this.transport.setHandler((from, msg) => this.handleWire(from, msg));
+        await this.transport.start();
+        this.started = true;
+        // Self is "joining" initially; transitions to "up" once at least
+        // one peer has acknowledged us (or we are the seed).  We seed
+        // the version with a wall-clock epoch (`Date.now()`) instead of
+        // a constant `1` so that, when this address is restarted after
+        // a graceful leave, the new incarnation's gossip out-versions
+        // the `removed` tombstone surviving on peers — that's what
+        // makes rejoin-after-leave converge cleanly (see the
+        // `existing.removed && incoming.version > existing.version`
+        // branch in `mergeMember`).  Within a single incarnation
+        // version is still a monotonically-increasing logical clock;
+        // the epoch only ensures a fresh process starts above any
+        // version that previous incarnation could have reached.
+        const me = new Member(this.selfAddress, 'joining', Date.now(), this.selfRoles);
+        this.members.set(me.address.toString(), me);
+        this.emit(new MemberJoined(me));
+        this.log.debug(`self joining: epoch=v${me.version} roles=[${[...this.selfRoles].join(',')}]`);
+        for (const s of seeds) {
+            const a = NodeAddress.parse(s.includes('@') ? s : `${this.system.name}@${s}`);
+            if (!a.equals(this.selfAddress))
+                this.seedAddrs.push(a);
+        }
+        if (this.seedAddrs.length === 0) {
+            // No seeds — we are the first node. Become Up immediately.
+            this.log.debug('no seeds configured — self-electing as first cluster member');
+            this.updateMember(me.withStatus('up'));
+        }
+        else {
+            this.log.debug(`contacting ${this.seedAddrs.length} seed(s): [${this.seedAddrs.map((a) => a.toString()).join(',')}]`);
+            this.contactSeeds();
+            // Keep retrying seed contact until self has transitioned to up,
+            // covering the case where a seed hasn't started yet.
+            this.seedTimer = this.system.scheduler.scheduleAtFixedRateFn(this.seedRetryIntervalMs, this.seedRetryIntervalMs, () => {
+                const self = this.members.get(this.selfAddress.toString());
+                if (!self || self.status !== 'joining') {
+                    this.seedTimer?.cancel();
+                    this.seedTimer = null;
+                    return;
+                }
+                this.contactSeeds();
+            });
+        }
+        // Schedule automatic joining→weakly-up promotion if configured.
+        if (this.weaklyUpAfterMs > 0) {
+            this.weaklyUpTimer = this.system.scheduler.scheduleOnceFn(this.weaklyUpAfterMs, () => {
+                const me = this.members.get(this.selfAddress.toString());
+                if (me?.status === 'joining') {
+                    this.updateMember(me.withStatus('weakly-up'));
+                }
+                this.weaklyUpTimer = null;
+            });
+        }
+        this.gossipTimer = this.system.scheduler.scheduleAtFixedRateFn(this.gossipIntervalMs, this.gossipIntervalMs, () => this.gossipTick());
+        this.heartbeatTimer = this.system.scheduler.scheduleAtFixedRateFn(this.failureDetector.interval, this.failureDetector.interval, () => this.heartbeatTick());
+        this.fdTimer = this.system.scheduler.scheduleAtFixedRateFn(this.failureDetector.interval, this.failureDetector.interval, () => this.failureDetectionTick());
+        this.tombstonePruneTimer = this.system.scheduler.scheduleAtFixedRateFn(this.tombstonePruneIntervalMs, this.tombstonePruneIntervalMs, () => this.tombstonePruneTick());
+    }
+    contactSeeds() {
+        const me = this.members.get(this.selfAddress.toString());
+        if (!me)
+            return;
+        for (const seed of this.seedAddrs) {
+            this.failureDetector.register(seed);
+            const initialGossip = {
+                t: 'gossip',
+                from: this.selfAddress.toJSON(),
+                members: [me.toData()],
+            };
+            this.transport.send(seed, initialGossip);
+        }
+    }
+    handleWire(from, msg) {
+        this.failureDetector.heartbeat(from);
+        match(msg)
+            .with({ t: 'heartbeat' }, (m) => this.handleHeartbeat(from, m))
+            .with({ t: 'heartbeat-ack' }, () => { })
+            .with({ t: 'gossip' }, (m) => this.handleGossip(m))
+            .with({ t: 'envelope' }, (m) => this.handleEnvelope(from, m))
+            .with({ t: 'leave' }, (m) => this.handleLeave(m))
+            .otherwise(() => {
+            // 'shard-map' and any custom extension wire-msgs handled by the
+            // registry; we intentionally fall through when no handler is set.
+            const custom = this.wireHandlers.get(msg.t);
+            if (custom)
+                custom(msg, from);
+        });
+    }
+    handleHeartbeat(_from, msg) {
+        const peer = NodeAddress.fromJSON(msg.from);
+        this.failureDetector.heartbeat(peer);
+        // Reply isn't strictly needed because send() also bumps the detector,
+        // but it keeps symmetric latency information.
+        this.transport.send(peer, { t: 'heartbeat-ack', from: this.selfAddress.toJSON(), seq: msg.seq });
+        // If the peer was unreachable and we see traffic again, flip it back.
+        const existing = this.members.get(peer.toString());
+        if (existing && existing.status === 'unreachable') {
+            this.updateMember(existing.withStatus('up'));
+            this.emit(new MemberReachable(this.members.get(peer.toString())));
+        }
+    }
+    handleGossip(msg) {
+        const sender = NodeAddress.fromJSON(msg.from);
+        this.failureDetector.heartbeat(sender);
+        this.log.debug(`gossip from ${sender}: ${msg.members.length} member(s)`);
+        for (const data of msg.members) {
+            this.mergeMember(data);
+        }
+        // Ensure we know about the sender itself.
+        if (!this.members.has(sender.toString())) {
+            const m = new Member(sender, 'joining', 1);
+            this.members.set(sender.toString(), m);
+            this.emit(new MemberJoined(m));
+        }
+        // Leader promotes joining (and weakly-up) members to up.
+        if (this.isLeader()) {
+            for (const m of this.members.values()) {
+                if (m.status === 'joining' || m.status === 'weakly-up') {
+                    this.log.debug(`leader-promote: ${m.address} ${m.status}→up`);
+                    this.updateMember(m.withStatus('up'));
+                }
+            }
+        }
+    }
+    handleEnvelope(from, msg) {
+        // Re-install the originating MDC + active trace context for the
+        // duration of dispatch (#53, #10).  Local refs that the
+        // dispatcher subsequently `tell`s capture this same context onto
+        // the next envelope, so both trails keep flowing across hops.
+        // Empty / missing contexts skip the corresponding wrapper.
+        let dispatch = () => this.dispatchEnvelope(from, msg);
+        // Tracing: if the envelope carries a parent context, open a
+        // `cluster.envelope.received` span so the trace explicitly
+        // shows the network hop and downstream local-tells see this
+        // span as their active parent.
+        if (msg.trace) {
+            const tracer = tracerOf(this.system);
+            const parentCtx = tracer.extractContext(msg.trace);
+            if (parentCtx) {
+                const inner = dispatch;
+                dispatch = () => {
+                    const span = tracer.startSpan('cluster.envelope.received', {
+                        parent: parentCtx,
+                        kind: 'consumer',
+                        attributes: {
+                            'cluster.from': from.toString(),
+                            'cluster.to.path': msg.to,
+                        },
+                    });
+                    try {
+                        tracer.withActiveSpan(span, inner);
+                    }
+                    finally {
+                        span.end();
+                    }
+                };
+            }
+        }
+        if (msg.context && Object.keys(msg.context).length > 0) {
+            LogContext.run(msg.context, dispatch);
+        }
+        else {
+            dispatch();
+        }
+    }
+    dispatchEnvelope(from, msg) {
+        // Rehydrate any ActorRef markers embedded in the user payload before
+        // handing it off — downstream handlers (sharding, pubsub, …) just
+        // forward `env.body` and shouldn't each duplicate the decode step.
+        const decoded = { ...msg, body: decodeRefs(msg.body, this) };
+        // 1. Explicit per-path handler (pub-sub mediator, singleton manager,
+        //    sharding coordinator, …).
+        const perPath = this._envelopeHandlersByPath.get(msg.to);
+        if (perPath) {
+            perPath(decoded, from);
+            return;
+        }
+        // 2. Resolve the target path locally and deliver directly — covers the
+        //    case where a RemoteActorRef rebuilt from a WireActorRef targets an
+        //    arbitrary user-spawned actor (no extension routing).  This also
+        //    happens to be functionally identical to sharding's own
+        //    dispatchEnvelope for region paths (both end in `ref.tell(body)`).
+        const segs = parsePathSegments(decoded.to);
+        if (segs.length > 0) {
+            const refOpt = this.system._resolvePath(segs);
+            if (refOpt.isSome()) {
+                refOpt.value.tell(decoded.body);
+                return;
+            }
+        }
+        // 3. Catch-all — kept for backward-compat with legacy handlers.
+        if (this.envelopeHandler) {
+            this.envelopeHandler(decoded, from);
+        }
+        else {
+            this.log.warn(`no envelope handler registered, dropping message to ${msg.to}`);
+        }
+    }
+    handleLeave(msg) {
+        const peer = NodeAddress.fromJSON(msg.node);
+        const existing = this.members.get(peer.toString());
+        if (!existing)
+            return;
+        this.log.debug(`peer ${peer} sent leave — tombstoning (was ${existing.status} v${existing.version})`);
+        const leaving = existing.withStatus('leaving');
+        const removed = leaving.withRemoved(Date.now());
+        // Tombstone (don't delete) so a stale `up` gossip from a peer that
+        // hasn't seen the leave yet can't resurrect this address —
+        // `mergeMember`'s version check filters it out.  Public APIs
+        // (`getMembers`, `upMembers`, `reachableMembers`) all skip
+        // `removed` entries.  The `removedAt` stamp is what
+        // `tombstonePruneTick` uses to drop the entry once `tombstoneTtlMs`
+        // has elapsed (#75).
+        this.members.set(peer.toString(), removed);
+        this.failureDetector.forget(peer);
+        this.emit(new MemberLeft(leaving));
+        this.emit(new MemberRemoved(removed));
+        this.maybeEmitLeaderChange();
+    }
+    gossipTick() {
+        const targets = this.reachableMembers().filter(m => !m.address.equals(this.selfAddress));
+        if (targets.length === 0)
+            return;
+        // Push to one random reachable peer each tick — epidemic style.
+        const target = targets[Math.floor(Math.random() * targets.length)];
+        const gossip = {
+            t: 'gossip',
+            from: this.selfAddress.toJSON(),
+            members: Array.from(this.members.values()).map(m => m.toData()),
+        };
+        this.transport.send(target.address, gossip);
+        // Stock metric: gossip rounds count.
+        metricsOf(this.system).counter('cluster_gossip_rounds_total', {}, { help: 'Cumulative count of gossip-push rounds initiated by this node.' }).inc();
+    }
+    heartbeatTick() {
+        this.heartbeatSeq++;
+        const hb = {
+            t: 'heartbeat',
+            from: this.selfAddress.toJSON(),
+            seq: this.heartbeatSeq,
+            ts: Date.now(),
+        };
+        for (const m of this.reachableMembers()) {
+            if (m.address.equals(this.selfAddress))
+                continue;
+            this.transport.send(m.address, hb);
+        }
+    }
+    failureDetectionTick() {
+        for (const m of Array.from(this.members.values())) {
+            if (m.address.equals(this.selfAddress))
+                continue;
+            const decision = this.failureDetector.decide(m.address);
+            if (decision === 'unreachable' && m.status === 'up') {
+                this.log.debug(`FD: ${m.address} → unreachable (heartbeat timeout)`);
+                this.updateMember(m.withStatus('unreachable'));
+                this.emit(new MemberUnreachable(this.members.get(m.address.toString())));
+            }
+            else if (decision === 'down' && m.status !== 'down' && m.status !== 'removed') {
+                this.log.debug(`FD: ${m.address} → down (was ${m.status}); deleting from membership`);
+                const downed = m.withStatus('down');
+                this.updateMember(downed);
+                this.emit(new MemberDown(downed));
+                // Note: FD-driven downing is the *advisory* fallback when no
+                // `DowningProvider` is configured.  We delete here (rather
+                // than tombstone) so a partition followed by a heal can
+                // recover the peer — `partition+heal` semantics rely on
+                // this.  Definitive downing paths (`handleLeave`,
+                // `evaluateDowning` force-down) tombstone instead, which
+                // prevents stale gossip from resurrecting the address.
+                this.members.delete(m.address.toString());
+                this.failureDetector.forget(m.address);
+                // Transient `removed` Member only used for the event emit —
+                // not stored, so the missing `removedAt` here is intentional.
+                const removed = downed.withStatus('removed');
+                this.emit(new MemberRemoved(removed));
+            }
+        }
+        // Optional split-brain resolver — runs after the failure-detector
+        // pass so it sees the latest unreachable set.
+        if (this.downing)
+            this.evaluateDowning();
+    }
+    /**
+     * Build a `ClusterPartitionView` from the current member set and
+     * ask the configured `DowningProvider` to decide which addresses
+     * (if any) need to be force-downed.  Debounces by the JSON shape of
+     * the unreachable set + member view so a steady-state cluster
+     * doesn't re-invoke the provider on every tick.
+     */
+    evaluateDowning() {
+        if (!this.downing)
+            return;
+        const allMembers = Array.from(this.members.values());
+        const unreachable = new Set(allMembers
+            .filter((m) => m.status === 'unreachable')
+            .map((m) => m.address.toString()));
+        // Cheap fingerprint — re-evaluate only when membership or
+        // reachability shifts.  The fingerprint includes statuses so a
+        // change like "leaving → unreachable" also triggers a re-check.
+        const fingerprint = allMembers
+            .map((m) => `${m.address.toString()}:${m.status}`)
+            .sort()
+            .join('|');
+        // Debounce only when the LAST evaluation produced an applied
+        // decision.  Strategies that need multiple ticks to converge
+        // (e.g. `LeaseMajority` with an in-flight `acquire()`) will
+        // return an empty set on the first call and a real decision on
+        // a later call WITH THE SAME FINGERPRINT — we must keep
+        // re-asking them.  `lastDownedView === null` means "nothing
+        // committed yet", so we evaluate.
+        if (this.lastDownedView !== null && fingerprint === this.lastDownedView)
+            return;
+        const view = {
+            allMembers,
+            unreachable,
+            self: this.selfAddress,
+        };
+        let toDown;
+        try {
+            toDown = this.downing.decide(view);
+        }
+        catch (err) {
+            this.log.warn(`downing provider threw — treating as no decision`, err);
+            return;
+        }
+        if (toDown.size === 0)
+            return;
+        this.lastDownedView = fingerprint;
+        const selfKey = this.selfAddress.toString();
+        const downsSelf = toDown.has(selfKey);
+        for (const key of toDown) {
+            // Self gets handled via `leave()` below — it gossips a Leaving
+            // notice to peers + drains the transport cleanly, which is
+            // strictly better than just deleting ourselves out of our own
+            // member map.
+            if (key === selfKey)
+                continue;
+            const m = this.members.get(key);
+            if (!m)
+                continue;
+            if (m.status === 'down' || m.status === 'removed')
+                continue;
+            const downed = m.withStatus('down');
+            this.updateMember(downed);
+            this.emit(new MemberDown(downed));
+            // Tombstone (don't delete) so later gossip from peers that
+            // haven't seen the force-down can't resurrect this address.
+            // `removedAt` lets `tombstonePruneTick` reclaim the entry
+            // after `tombstoneTtlMs` (#75).
+            const removed = downed.withRemoved(Date.now());
+            this.members.set(key, removed);
+            this.failureDetector.forget(m.address);
+            this.emit(new MemberRemoved(removed));
+        }
+        if (downsSelf) {
+            void this.leave().catch((e) => this.log.warn(`self-leave after downing decision failed`, e));
+        }
+    }
+    mergeMember(data) {
+        const incoming = Member.fromData(data);
+        // Security: reject versions that are absurdly far in the future.
+        //
+        // **Exploit walkthrough (pre-fix).**  Versions are seeded from
+        // `Date.now()` and bumped by 1 on status transitions — in normal
+        // operation they stay within ~`Date.now() + small N`.  Nothing
+        // capped them before, so a malicious peer could gossip with
+        // `version: Number.MAX_SAFE_INTEGER` for any target.  The merge
+        // accepted it (higher than the existing version → win), and
+        // every legitimate update from that target was rejected
+        // afterward (`<= MAX_SAFE_INTEGER` always true).  Effect:
+        // permanent DoS — the target was pinned to whatever status the
+        // attacker chose, with no way to recover.  In JS,
+        // `MAX_SAFE_INTEGER + 1` rounds back to itself, so even fresh
+        // start-from-zero re-incarnation couldn't escape.
+        //
+        // Cap at `now + MAX_VERSION_SKEW_MS` (1 day) — generous enough
+        // for any real clock skew, tight enough that MAX_SAFE_INTEGER
+        // (≈ 285 000 years above now) is rejected immediately.
+        const maxAcceptableVersion = Date.now() + MAX_VERSION_SKEW_MS;
+        if (!Number.isFinite(incoming.version) || incoming.version > maxAcceptableVersion) {
+            this.log.warn(`merge: rejecting gossip from ${incoming.address} with implausible version ` +
+                `${incoming.version} (max acceptable ${maxAcceptableVersion}) — possible exploit`);
+            return;
+        }
+        // Reject expired tombstones from gossip — peers that haven't yet
+        // pruned a long-dead address would otherwise resurrect it on
+        // nodes that already pruned (#75).  Old nodes that pre-date the
+        // `removedAt` field gossip without it; we treat such tombstones
+        // as fresh (no age info ⇒ assume they need normal TTL) so a
+        // mixed-version cluster still converges.
+        if (incoming.status === 'removed'
+            && incoming.removedAt !== undefined
+            && Date.now() - incoming.removedAt >= this.tombstoneTtlMs) {
+            this.log.debug(`merge: dropping expired tombstone for ${incoming.address} ` +
+                `(age ${Date.now() - incoming.removedAt}ms ≥ ttl ${this.tombstoneTtlMs}ms)`);
+            return;
+        }
+        const existing = this.members.get(incoming.address.toString());
+        if (!existing) {
+            this.members.set(incoming.address.toString(), incoming);
+            this.failureDetector.register(incoming.address);
+            this.emit(new MemberJoined(incoming));
+            // If we first learn about the member already in a terminal or
+            // active state (common via gossip merging), also fire the matching
+            // status event so subscribers (ShardRegion, etc.) re-allocate.
+            if (incoming.status !== 'joining') {
+                this.emitStatusTransition(new Member(incoming.address, 'joining', 0), incoming);
+            }
+            return;
+        }
+        // Re-incarnation: a previously-removed (tombstoned) address can
+        // be revived by a fresh start on the same `host:port`.  We
+        // detect the new incarnation by its strictly higher base
+        // version: each `Cluster.join` seeds its self-member with a
+        // wall-clock-based epoch (see `_start`), so any state the new
+        // node reports — directly via its initial gossip OR indirectly
+        // via a peer that has already seen the rejoin — out-versions
+        // the tombstone left over from its previous run.  Drop the
+        // tombstone, treat the address as a fresh member, and emit the
+        // matching status event so subscribers (ShardRegion, etc.)
+        // re-allocate.  Without this branch, `mergeMember`'s strict
+        // version monotonicity below would pin the address to
+        // `removed` forever even though the higher version is the
+        // newer truth.
+        if (existing.status === 'removed' && incoming.version > existing.version) {
+            this.log.debug(`merge: ${incoming.address} re-incarnation (was removed v${existing.version}, now ${incoming.status} v${incoming.version})`);
+            this.members.set(incoming.address.toString(), incoming);
+            this.failureDetector.register(incoming.address);
+            this.emit(new MemberJoined(incoming));
+            if (incoming.status !== 'joining') {
+                this.emitStatusTransition(new Member(incoming.address, 'joining', 0), incoming);
+            }
+            return;
+        }
+        if (incoming.version <= existing.version)
+            return; // older or equal, ignore
+        if (existing.status !== incoming.status) {
+            this.log.debug(`merge: ${incoming.address} ${existing.status}→${incoming.status} (v${existing.version}→v${incoming.version})`);
+        }
+        this.members.set(incoming.address.toString(), incoming);
+        this.emitStatusTransition(existing, incoming);
+    }
+    updateMember(next) {
+        const key = next.address.toString();
+        const prev = this.members.get(key);
+        this.members.set(key, next);
+        if (prev)
+            this.emitStatusTransition(prev, next);
+        else
+            this.emit(new MemberJoined(next));
+        // Stock metric: members-up gauge.  Updated on every member-set
+        // mutation so all transitions (join, up, down, removed) keep it
+        // current.  Single value per node — labels are deliberately empty
+        // because the gauge is always the local view's snapshot.
+        metricsOf(this.system).gauge('cluster_members_up', {}, { help: 'Number of cluster members currently in `up` state.' }).set(this.upMembers().length);
+    }
+    emitStatusTransition(prev, next) {
+        if (prev.status === next.status)
+            return;
+        match(next.status)
+            .with('up', () => {
+            this.emit(new MemberUp(next));
+            if (next.address.equals(this.selfAddress))
+                this.emit(new SelfUp(next));
+        })
+            .with('weakly-up', () => this.emit(new MemberWeaklyUp(next)))
+            .with('unreachable', () => this.emit(new MemberUnreachable(next)))
+            .with('down', () => this.emit(new MemberDown(next)))
+            .with('leaving', () => this.emit(new MemberLeft(next)))
+            .with('removed', () => {
+            this.emit(new MemberRemoved(next));
+            if (next.address.equals(this.selfAddress))
+                this.emit(new SelfRemoved(next));
+        })
+            .with('joining', () => { })
+            .exhaustive();
+        this.maybeEmitLeaderChange();
+    }
+    maybeEmitLeaderChange() {
+        const newLeader = this.leader();
+        const prev = this.currentLeader;
+        const changed = prev.isSome() !== newLeader.isSome()
+            || (prev.isSome() && newLeader.isSome() && !prev.value.address.equals(newLeader.value.address));
+        if (changed) {
+            this.currentLeader = newLeader;
+            const prevStr = prev.fold(() => 'none', (m) => m.address.toString());
+            const nextStr = newLeader.fold(() => 'none', (m) => m.address.toString());
+            this.log.debug(`leader changed: ${prevStr} → ${nextStr}`);
+            this.emit(new LeaderChanged(newLeader));
+        }
+    }
+    emit(event) {
+        this.system.eventStream.publish(event);
+        for (const l of this._listeners) {
+            try {
+                l(event);
+            }
+            catch (e) {
+                this.log.warn('listener threw', e);
+            }
+        }
+    }
+    /**
+     * Drop tombstones whose `removedAt` exceeds `tombstoneTtlMs` (and
+     * the `tombstoneMinRetentionMs` floor).  Each peer makes the same
+     * decision on its local clock — with TTL ≫ gossip propagation lag,
+     * peers prune within seconds of each other and stale-gossip
+     * resurrection is filtered out by `mergeMember`'s "expired
+     * tombstone" guard.  See #75 for the full rationale.
+     *
+     * Tombstones lacking `removedAt` (older nodes pre-dating the field)
+     * are kept — we have no age info to compare against.  They drop
+     * out naturally when those nodes are upgraded or restart.
+     */
+    tombstonePruneTick() {
+        const now = Date.now();
+        const cutoff = Math.max(this.tombstoneTtlMs, this.tombstoneMinRetentionMs);
+        let pruned = 0;
+        for (const [key, m] of this.members) {
+            if (m.status !== 'removed')
+                continue;
+            if (m.removedAt === undefined)
+                continue;
+            if (now - m.removedAt < cutoff)
+                continue;
+            this.members.delete(key);
+            pruned++;
+        }
+        if (pruned > 0) {
+            this.log.debug(`tombstone prune: dropped ${pruned} expired entr${pruned === 1 ? 'y' : 'ies'} ` +
+                `(ttl=${this.tombstoneTtlMs}ms, minRetention=${this.tombstoneMinRetentionMs}ms)`);
+        }
+    }
+}
+/** Helper — creates an InMemoryTransport for tests. */
+export function inMemoryTransport(system, host, port) {
+    return new InMemoryTransport(new NodeAddress(system.name, host, port));
+}
+//# sourceMappingURL=Cluster.js.map