actor-ts 0.9.0

package/dist/cluster/sharding/ShardCoordinator.js

@@ -0,0 +1,719 @@
+import { match, P } from 'ts-pattern';
+import { Actor } from '../../Actor.js';
+import { LeaderChanged, MemberRemoved } from '../ClusterEvents.js';
+import { NodeAddress } from '../NodeAddress.js';
+import { RemoteActorRef } from '../RemoteActorRef.js';
+function isCoordinatorEvent(msg) {
+    if (!msg || typeof msg !== 'object')
+        return false;
+    const t = msg.t;
+    return t === 'reconcile' || t === 'lease-acquire-result'
+        || t === 'lease-lost' || t === 'acquire-retry';
+}
+function regionKey(node, path) {
+    return `${node}|${path}`;
+}
+/**
+ * Cluster-wide authoritative source of shard-to-region assignments.  Runs on
+ * every node but only responds to requests when the local node is the
+ * cluster leader.  Non-leader coordinators ignore incoming messages so that
+ * duplicate coordinators during a leader transition are harmless.
+ *
+ * State is reconstructed from Register messages: each region reports the
+ * shards it currently hosts, and the coordinator merges that with any new
+ * allocation requests.  This is deliberately lightweight — a production
+ * upgrade would snapshot state to a journal so the coordinator can recover
+ * across restarts without re-allocating every shard from scratch.
+ */
+export class ShardCoordinator extends Actor {
+    settings;
+    regions = new Map();
+    shardHome = new Map(); // shardId → regionPath
+    pending = new Map(); // waiting queries
+    rebalanceInProgress = new Map();
+    entitiesPerShard = new Map();
+    rebalanceTimer = null;
+    unsubscribeCluster = null;
+    unsubscribeLeaseLost = null;
+    acquireRetryTimer = null;
+    /**
+     * Lease lifecycle (only used when `settings.lease` is set).
+     * Drives the `isActive()` predicate — coordinator only processes
+     * shard messages while `isLeader() && leaseState === 'held'`.
+     */
+    leaseState = 'none';
+    /**
+     * Sharding messages received while we're the leader but waiting
+     * for the lease.  Drained on the `acquiring → held` transition so
+     * regions that asked early get an answer instead of having to
+     * wait for the next cluster event to retrigger their ask.
+     *
+     * Non-leader messages are NOT buffered — they're dropped because
+     * the regions on this node will retry against whichever node is
+     * the actual leader on their next attempt.
+     *
+     * Capped to avoid unbounded growth if the lease never resolves.
+     */
+    acquireBuffer = [];
+    static ACQUIRE_BUFFER_CAP = 1_000;
+    /**
+     * Promise chain over remembered-entity persistence.  Each new
+     * `EntityStarted` / `EntityStopped` chains its append onto the tail
+     * of this promise so writes serialise — `Journal.append`'s
+     * optimistic `expectedSeq` would otherwise race when two events
+     * fire in fast succession.  `.catch` on each link prevents a
+     * failed write from breaking the chain for subsequent writes.
+     */
+    rememberWriteChain = Promise.resolve();
+    /**
+     * Coalesced save state for `coordinatorStateStore`.  State
+     * mutations are bursty during rebalance; rather than fire one
+     * disk write per mutation we mark the state dirty and the
+     * in-flight save's `.finally` kicks off a follow-up if more
+     * changes accumulated meanwhile.  Same pattern as
+     * `DistributedData.scheduleDurableSave` from #40.
+     */
+    coordinatorStateInFlight = false;
+    coordinatorStateDirty = false;
+    constructor(settings) {
+        super();
+        this.settings = settings;
+    }
+    /** Path used by ClusterSharding to locate the coordinator on any node. */
+    static pathFor(typeName) {
+        return `actor-ts://SYSTEM/user/sharding-coordinator-${typeName}`;
+    }
+    async preStart() {
+        // 1. Replay the persisted remembered-entities log so the
+        //    in-memory map is populated BEFORE we accept any messages.
+        //    Without this, a fresh-cluster start would treat every
+        //    rememberEntities=true sharded type as empty and only
+        //    re-register entities lazily as messages arrive.
+        if (this.settings.rememberEntities && this.settings.rememberEntitiesStore) {
+            try {
+                const events = await this.settings.rememberEntitiesStore
+                    .load(this.settings.typeName);
+                for (const ev of events)
+                    this.applyRememberEvent(ev);
+            }
+            catch (err) {
+                this.system.log.warn(`[sharding] failed to load remembered entities for '${this.settings.typeName}'`, err);
+            }
+        }
+        this.unsubscribeCluster = this.settings.cluster.subscribe(evt => match(evt)
+            .with(P.instanceOf(MemberRemoved), (e) => this.onMemberRemoved(e.member.address))
+            .with(P.instanceOf(LeaderChanged), () => this.onLeaderChanged())
+            .otherwise(() => { }));
+        if (this.settings.lease) {
+            this.unsubscribeLeaseLost = this.settings.lease.onLost((reason) => {
+                this.self.tell({ t: 'lease-lost', reason });
+            });
+            // Kick the initial reconcile through the mailbox so the lease
+            // path serialises with subsequent cluster events.
+            this.self.tell({ t: 'reconcile' });
+        }
+        this.rebalanceTimer = this.system.scheduler.scheduleAtFixedRateFn(this.settings.rebalanceIntervalMs ?? 2_000, this.settings.rebalanceIntervalMs ?? 2_000, () => { if (this.isActive())
+            this.rebalanceTick(); });
+    }
+    /** Apply a single `RememberEvent` to the in-memory `entitiesPerShard`
+     *  map.  Used by both the preStart replay AND
+     *  `handleEntityStarted` / `handleEntityStopped` so the in-memory
+     *  derivation rule lives in exactly one place. */
+    applyRememberEvent(ev) {
+        if (ev.kind === 'started') {
+            const set = this.entitiesPerShard.get(ev.shardId) ?? new Set();
+            set.add(ev.entityId);
+            this.entitiesPerShard.set(ev.shardId, set);
+        }
+        else {
+            const set = this.entitiesPerShard.get(ev.shardId);
+            if (!set)
+                return;
+            set.delete(ev.entityId);
+            if (set.size === 0)
+                this.entitiesPerShard.delete(ev.shardId);
+        }
+    }
+    async postStop() {
+        this.unsubscribeCluster?.();
+        this.unsubscribeLeaseLost?.();
+        this.rebalanceTimer?.cancel();
+        this.acquireRetryTimer?.cancel();
+        for (const r of this.rebalanceInProgress.values())
+            r.timer.cancel();
+        if (this.settings.lease && this.leaseState === 'held') {
+            try {
+                await this.settings.lease.release();
+            }
+            catch { /* best-effort */ }
+            this.leaseState = 'none';
+        }
+    }
+    onReceive(msg) {
+        // Internal coordinator events drive the lease state machine — they
+        // run regardless of `isActive()` because they're how we transition
+        // INTO `isActive()` in the first place.
+        if (isCoordinatorEvent(msg)) {
+            this.handleCoordinatorEvent(msg);
+            return;
+        }
+        if (!this.isLeader())
+            return;
+        if (this.settings.lease && this.leaseState !== 'held') {
+            // Leader, but lease not yet held — buffer instead of drop so
+            // regions don't need to retry on the next cluster event.
+            if (this.acquireBuffer.length < ShardCoordinator.ACQUIRE_BUFFER_CAP) {
+                this.acquireBuffer.push(msg);
+            }
+            return;
+        }
+        this.dispatchShardingMessage(msg);
+    }
+    dispatchShardingMessage(msg) {
+        match(msg)
+            .with({ $t: 'sharding.Register' }, (m) => this.handleRegister(m))
+            .with({ $t: 'sharding.GetShardHome' }, (m) => this.handleGetShardHome(m))
+            .with({ $t: 'sharding.HandOffComplete' }, (m) => this.handleHandOffComplete(m))
+            .with({ $t: 'sharding.BeginHandOffAck' }, () => { })
+            .with({ $t: 'sharding.RegionTerminated' }, (m) => this.onRegionTerminated(m))
+            .with({ $t: 'sharding.EntityStarted' }, (m) => this.handleEntityStarted(m))
+            .with({ $t: 'sharding.EntityStopped' }, (m) => this.handleEntityStopped(m))
+            .otherwise(() => { });
+    }
+    isLeader() { return this.settings.cluster.isLeader(); }
+    /**
+     * True iff this coordinator is the authoritative one — i.e. should
+     * be processing shard messages.  Without a lease this is just
+     * `isLeader()`; with a lease it additionally requires that the
+     * lease be currently held by this replica.
+     */
+    isActive() {
+        if (!this.settings.lease)
+            return this.isLeader();
+        return this.isLeader() && this.leaseState === 'held';
+    }
+    /* --------------------------- Lease state machine ------------------------ */
+    handleCoordinatorEvent(evt) {
+        match(evt)
+            .with({ t: 'reconcile' }, () => this.reconcileLease())
+            .with({ t: 'lease-acquire-result' }, (m) => this.handleAcquireResult(m))
+            .with({ t: 'lease-lost' }, (m) => this.handleLeaseLost(m))
+            .with({ t: 'acquire-retry' }, () => this.reconcileLease())
+            .exhaustive();
+    }
+    reconcileLease() {
+        if (!this.settings.lease)
+            return;
+        const wantActive = this.isLeader();
+        if (wantActive) {
+            if (this.leaseState === 'held')
+                return; // already active
+            if (this.leaseState === 'acquiring')
+                return; // already trying
+            this.acquireRetryTimer?.cancel();
+            this.acquireRetryTimer = null;
+            this.leaseState = 'acquiring';
+            void this.runAcquire();
+        }
+        else {
+            if (this.leaseState === 'held') {
+                // Stepped down — release so a follower can pick up faster
+                // than waiting for the TTL to expire.
+                void this.settings.lease.release().catch((e) => this.system.log.warn(`[sharding] lease release failed`, e));
+                this.leaseState = 'none';
+                // Falling out of `held` already triggers our standard
+                // "not-leader" cleanup via onLeaderChanged below — no extra
+                // state reset needed here.
+            }
+            else if (this.leaseState === 'acquiring') {
+                // Let the in-flight acquire finish; handleAcquireResult will
+                // notice we no longer want it and release immediately.
+            }
+            else {
+                this.acquireRetryTimer?.cancel();
+                this.acquireRetryTimer = null;
+            }
+        }
+    }
+    async runAcquire() {
+        try {
+            const got = await this.settings.lease.acquire();
+            this.self.tell({ t: 'lease-acquire-result', got });
+        }
+        catch (error) {
+            this.self.tell({
+                t: 'lease-acquire-result', got: false, error: error,
+            });
+        }
+    }
+    handleAcquireResult(msg) {
+        if (this.leaseState !== 'acquiring') {
+            // Spurious result — release if we somehow got it.
+            if (msg.got)
+                void this.settings.lease.release().catch(() => { });
+            return;
+        }
+        if (!msg.got) {
+            if (msg.error)
+                this.system.log.warn(`[sharding] lease acquire failed`, msg.error);
+            this.leaseState = 'none';
+            this.scheduleAcquireRetry();
+            return;
+        }
+        if (!this.isLeader()) {
+            // Lost leadership during the acquire — release and let the
+            // new leader (if any) take over.
+            void this.settings.lease.release().catch(() => { });
+            this.leaseState = 'none';
+            return;
+        }
+        this.leaseState = 'held';
+        this.system.log.info(`[sharding] coordinator '${this.settings.typeName}' became active (lease acquired)`);
+        // Drain any messages that arrived while we were acquiring.
+        // Regions don't retry on a timer — they only re-ask on cluster
+        // events — so without this drain a region that asked during
+        // `acquiring` would sit forever on a buffered user message.
+        if (this.acquireBuffer.length > 0) {
+            const buffered = this.acquireBuffer;
+            this.acquireBuffer = [];
+            for (const m of buffered)
+                this.dispatchShardingMessage(m);
+        }
+    }
+    handleLeaseLost(msg) {
+        if (this.leaseState !== 'held')
+            return;
+        this.system.log.warn(`[sharding] coordinator '${this.settings.typeName}' lost lease — ${msg.reason}; stepping down`);
+        this.leaseState = 'none';
+        // Cancel any in-flight rebalance handoff timers — those would
+        // fire force-reallocations that we shouldn't be doing while
+        // passive.  Pending queries get dropped (regions retry once we
+        // become active again, or once cluster events flush their
+        // register loop).
+        for (const r of this.rebalanceInProgress.values())
+            r.timer.cancel();
+        this.rebalanceInProgress.clear();
+        this.pending.clear();
+        this.acquireBuffer = [];
+        // Deliberately do NOT clear `regions` or `shardHome` here.  We
+        // stay leader and likely re-acquire — keeping the cached view
+        // means subsequent re-acquires resume serving without waiting
+        // for every region to re-register.  If another node took the
+        // lease during our window and reallocated, our stale homes
+        // self-correct via the standard "remote send fails →
+        // MemberRemoved → invalidateHomesOnNode" flow on the regions.
+        // Re-enter the acquire loop in case we're still the leader.
+        this.self.tell({ t: 'reconcile' });
+    }
+    scheduleAcquireRetry() {
+        const interval = this.settings.acquireRetryIntervalMs ?? 5_000;
+        this.acquireRetryTimer?.cancel();
+        this.acquireRetryTimer = this.system.scheduler.scheduleOnceFn(interval, () => {
+            this.self.tell({ t: 'acquire-retry' });
+        });
+    }
+    candidates() {
+        const role = this.settings.role;
+        const activeRegions = Array.from(this.regions.values()).filter(r => !r.proxy);
+        const addrs = activeRegions.map(r => r.node);
+        if (!role)
+            return addrs;
+        return addrs.filter(a => {
+            const m = this.settings.cluster.getMembers().find(x => x.address.equals(a));
+            return m?.hasRole(role) ?? false;
+        });
+    }
+    currentShardCounts() {
+        // Keyed by node address string so AllocationStrategy can match against
+        // the NodeAddress it returned in `allocate`.
+        const out = new Map();
+        for (const info of this.regions.values()) {
+            if (info.proxy)
+                continue;
+            const addr = info.node.toString();
+            const existing = out.get(addr) ?? new Set();
+            for (const s of info.shards)
+                existing.add(s);
+            out.set(addr, existing);
+        }
+        return out;
+    }
+    /* ------------------------------- Handlers -------------------------------- */
+    handleRegister(msg) {
+        const node = NodeAddress.fromJSON(msg.node);
+        const key = regionKey(node, msg.region);
+        this.regions.set(key, {
+            node,
+            path: msg.region,
+            proxy: msg.proxy,
+            shards: new Set(msg.hostedShards),
+        });
+        for (const shardId of msg.hostedShards) {
+            this.shardHome.set(shardId, key);
+        }
+        const ack = {
+            $t: 'sharding.RegisterAck',
+            coordinator: this.self.path.toString(),
+        };
+        this.replyTo(msg.region, msg.node, ack);
+        for (const shardId of msg.hostedShards)
+            this.flushPending(shardId);
+        if (this.settings.rememberEntities) {
+            for (const shardId of msg.hostedShards)
+                this.shipRememberedEntities(shardId);
+        }
+        this.scheduleCoordinatorStateSave();
+    }
+    handleGetShardHome(msg) {
+        const home = this.shardHome.get(msg.shardId);
+        if (home && this.regions.has(home)) {
+            const info = this.regions.get(home);
+            this.replyTo(msg.requester, msg.requesterNode, {
+                $t: 'sharding.ShardHome',
+                shardId: msg.shardId,
+                region: info.path,
+                node: info.node.toJSON(),
+            });
+            return;
+        }
+        const list = this.pending.get(msg.shardId) ?? [];
+        list.push(msg);
+        this.pending.set(msg.shardId, list);
+        if (!this.rebalanceInProgress.has(msg.shardId))
+            this.tryAllocate(msg.shardId);
+    }
+    tryAllocate(shardId) {
+        const cs = this.candidates();
+        if (cs.length === 0)
+            return;
+        const owner = this.settings.allocationStrategy.allocate(shardId, cs, this.currentShardCounts());
+        const key = this.findRegionKey(owner);
+        if (!key)
+            return;
+        this.shardHome.set(shardId, key);
+        const info = this.regions.get(key);
+        info.shards.add(shardId);
+        // Proactively notify the new owner — they may not have asked, but need
+        // to know they are now responsible for the shard (and, when remembering
+        // entities, need that knowledge before RememberedEntities arrives).
+        this.sendToRegion(key, {
+            $t: 'sharding.ShardHome',
+            shardId,
+            region: info.path,
+            node: info.node.toJSON(),
+        });
+        this.flushPending(shardId);
+        if (this.settings.rememberEntities)
+            this.shipRememberedEntities(shardId);
+        this.scheduleCoordinatorStateSave();
+    }
+    handleHandOffComplete(msg) {
+        const shardId = msg.shardId;
+        const inProgress = this.rebalanceInProgress.get(shardId);
+        if (!inProgress)
+            return;
+        inProgress.timer.cancel();
+        this.rebalanceInProgress.delete(shardId);
+        // Remove ownership from old region.
+        const oldPath = inProgress.from;
+        const old = this.regions.get(oldPath);
+        if (old)
+            old.shards.delete(shardId);
+        this.shardHome.delete(shardId);
+        // Reallocate (the pending queries will get the new home).
+        // tryAllocate already calls scheduleCoordinatorStateSave, so a
+        // second save here would be redundant.
+        this.tryAllocate(shardId);
+    }
+    handleEntityStarted(msg) {
+        if (!this.settings.rememberEntities)
+            return;
+        this.applyRememberEvent({ kind: 'started', shardId: msg.shardId, entityId: msg.entityId });
+        this.persistRememberEvent({ kind: 'started', shardId: msg.shardId, entityId: msg.entityId });
+    }
+    handleEntityStopped(msg) {
+        if (!this.settings.rememberEntities) {
+            // Existing behaviour: tidy the in-memory map even when we're
+            // not remembering entities, so an unwise external trigger
+            // doesn't leave stale data in the map.
+            const set = this.entitiesPerShard.get(msg.shardId);
+            if (set) {
+                set.delete(msg.entityId);
+                if (set.size === 0)
+                    this.entitiesPerShard.delete(msg.shardId);
+            }
+            return;
+        }
+        this.applyRememberEvent({ kind: 'stopped', shardId: msg.shardId, entityId: msg.entityId });
+        this.persistRememberEvent({ kind: 'stopped', shardId: msg.shardId, entityId: msg.entityId });
+    }
+    /**
+     * Append a remembered-entity event to the persistent store.  Chains
+     * onto `rememberWriteChain` so two events fired in fast succession
+     * don't race the journal's optimistic-`expectedSeq` check — each
+     * append awaits the previous one.  Errors are caught + logged so a
+     * transient store failure doesn't break the chain for subsequent
+     * writes.
+     */
+    persistRememberEvent(event) {
+        const store = this.settings.rememberEntitiesStore;
+        if (!store)
+            return;
+        this.rememberWriteChain = this.rememberWriteChain
+            .catch(() => { })
+            .then(() => store.append(this.settings.typeName, event))
+            .catch((err) => {
+            this.system.log.warn(`[sharding] failed to persist remembered-entity event ${event.kind}/${event.shardId}/${event.entityId}`, err);
+        });
+    }
+    onRegionTerminated(msg) {
+        const addr = NodeAddress.fromJSON(msg.node);
+        const key = regionKey(addr, msg.region);
+        const info = this.regions.get(key);
+        if (!info)
+            return;
+        this.regions.delete(key);
+        for (const shardId of info.shards) {
+            this.shardHome.delete(shardId);
+            const inProg = this.rebalanceInProgress.get(shardId);
+            if (inProg) {
+                inProg.timer.cancel();
+                this.rebalanceInProgress.delete(shardId);
+            }
+            this.tryAllocate(shardId);
+        }
+        // tryAllocate already schedules saves for each re-allocation;
+        // an extra one here would be redundant.  But if `info.shards`
+        // was empty (region had no shards) we still removed it from the
+        // regions map and need to record that.
+        if (info.shards.size === 0)
+            this.scheduleCoordinatorStateSave();
+    }
+    onMemberRemoved(addr) {
+        for (const [_key, info] of Array.from(this.regions.entries())) {
+            if (info.node.equals(addr)) {
+                this.onRegionTerminated({
+                    $t: 'sharding.RegionTerminated',
+                    region: info.path,
+                    node: addr.toJSON(),
+                });
+            }
+        }
+    }
+    onLeaderChanged() {
+        if (!this.isLeader()) {
+            // No longer leader — drop the in-memory view; the new leader
+            // owns the canonical state now.
+            this.regions.clear();
+            this.shardHome.clear();
+            this.pending.clear();
+            this.acquireBuffer = [];
+            for (const r of this.rebalanceInProgress.values())
+                r.timer.cancel();
+            this.rebalanceInProgress.clear();
+        }
+        else {
+            // Just became leader (or re-elected).  If a state store is
+            // configured, try to seed `regions` + `shardHome` from the
+            // last known snapshot — saves the from-scratch reallocation
+            // storm of every shard re-registering through a fresh
+            // tryAllocate call.  Failure is tolerated: we fall back to
+            // the v1 rebuild-from-Register path when the load fails or
+            // returns nothing.
+            if (this.settings.coordinatorStateStore) {
+                void this.loadCoordinatorState();
+            }
+        }
+        // Lease-aware coordinators re-evaluate the acquire/release cycle
+        // any time the leader role flips — see `reconcileLease()`.  We
+        // route through the mailbox so the state machine serialises with
+        // any in-flight acquire result.
+        if (this.settings.lease) {
+            this.self.tell({ t: 'reconcile' });
+        }
+    }
+    /* ------------------- Coordinator-state persistence ------------------ */
+    /**
+     * Read the most recent snapshot from `coordinatorStateStore` and
+     * seed `regions` + `shardHome` from it.  Drops any region whose
+     * node has left the cluster between the snapshot and now —
+     * otherwise we'd happily route shards to dead pods.  Existing
+     * pending queries get a fresh allocation pass via the regular
+     * onMessage flow.
+     */
+    async loadCoordinatorState() {
+        const store = this.settings.coordinatorStateStore;
+        if (!store)
+            return;
+        let data;
+        try {
+            data = await store.load(this.settings.typeName);
+        }
+        catch (err) {
+            this.system.log.warn(`[sharding] coordinator-state load failed for '${this.settings.typeName}'`, err);
+            return;
+        }
+        if (!data)
+            return;
+        // If we already have local state (e.g. preStart already absorbed
+        // some Register messages), merge — keep what we know AND what
+        // the snapshot says.  The snapshot's `regions` may be stale (a
+        // node may have died), so we filter by current cluster membership.
+        const livingNodes = new Set(this.settings.cluster.upMembers().map((m) => m.address.toString()));
+        for (const r of data.regions) {
+            if (!livingNodes.has(r.node.systemName + '@' + r.node.host + ':' + r.node.port)) {
+                // Node dropped out of the cluster between snapshot and now
+                // — skip the entry; the dead region won't be re-resurrected.
+                continue;
+            }
+            const node = NodeAddress.fromJSON(r.node);
+            if (this.regions.has(r.key))
+                continue; // already known via Register
+            this.regions.set(r.key, {
+                node, path: r.path, proxy: r.proxy, shards: new Set(r.shards),
+            });
+        }
+        for (const [shardId, regionKey] of data.shardHome) {
+            // Only adopt the home if the region survived the filter above.
+            if (this.regions.has(regionKey) && !this.shardHome.has(shardId)) {
+                this.shardHome.set(shardId, regionKey);
+            }
+        }
+    }
+    /**
+     * Mark coordinator state dirty + schedule a save.  Called after
+     * every meaningful mutation — handleRegister, tryAllocate,
+     * handleHandOffComplete, onRegionTerminated.  Coalesces
+     * overlapping bursts into 1-2 store writes via the
+     * `inFlight + dirty` flag pair.
+     */
+    scheduleCoordinatorStateSave() {
+        const store = this.settings.coordinatorStateStore;
+        if (!store)
+            return;
+        if (!this.isLeader())
+            return;
+        if (this.coordinatorStateInFlight) {
+            this.coordinatorStateDirty = true;
+            return;
+        }
+        this.coordinatorStateInFlight = true;
+        const snapshot = this.snapshotCoordinatorState();
+        void store.save(this.settings.typeName, snapshot)
+            .catch((err) => {
+            this.system.log.warn(`[sharding] coordinator-state save failed for '${this.settings.typeName}'`, err);
+        })
+            .finally(() => {
+            this.coordinatorStateInFlight = false;
+            if (this.coordinatorStateDirty) {
+                this.coordinatorStateDirty = false;
+                this.scheduleCoordinatorStateSave();
+            }
+        });
+    }
+    snapshotCoordinatorState() {
+        const regions = [];
+        for (const [key, info] of this.regions) {
+            regions.push({
+                key,
+                node: info.node.toJSON(),
+                path: info.path,
+                proxy: info.proxy,
+                shards: Array.from(info.shards),
+            });
+        }
+        const shardHome = [];
+        for (const [shardId, regionKey] of this.shardHome) {
+            shardHome.push([shardId, regionKey]);
+        }
+        return {
+            leader: this.settings.cluster.selfAddress.toString(),
+            takenAt: Date.now(),
+            regions,
+            shardHome,
+        };
+    }
+    /* ------------------------------- Rebalance ------------------------------- */
+    rebalanceTick() {
+        const shardsToMove = this.settings.allocationStrategy.rebalance(this.currentShardCounts(), this.candidates(), new Set(this.rebalanceInProgress.keys()));
+        for (const shardId of shardsToMove)
+            this.beginHandOff(shardId);
+    }
+    beginHandOff(shardId) {
+        const ownerKey = this.shardHome.get(shardId);
+        if (!ownerKey)
+            return;
+        const owner = this.regions.get(ownerKey);
+        if (!owner)
+            return;
+        const timeout = this.settings.handOffTimeoutMs ?? 10_000;
+        const timer = this.system.scheduler.scheduleOnceFn(timeout, () => {
+            if (this.rebalanceInProgress.delete(shardId)) {
+                this.system.log.warn(`[sharding] handoff timeout for shard ${shardId}; forcing reallocate`);
+                this.shardHome.delete(shardId);
+                this.tryAllocate(shardId);
+            }
+        });
+        this.rebalanceInProgress.set(shardId, { from: ownerKey, timer });
+        this.sendToRegion(ownerKey, { $t: 'sharding.HandOff', shardId });
+    }
+    /* --------------------------------- Helpers ------------------------------- */
+    findRegionKey(node) {
+        for (const [key, info] of this.regions) {
+            if (!info.proxy && info.node.equals(node))
+                return key;
+        }
+        return null;
+    }
+    flushPending(shardId) {
+        const pending = this.pending.get(shardId);
+        if (!pending || pending.length === 0)
+            return;
+        const home = this.shardHome.get(shardId);
+        if (!home)
+            return;
+        const info = this.regions.get(home);
+        if (!info)
+            return;
+        for (const q of pending) {
+            this.replyTo(q.requester, q.requesterNode, {
+                $t: 'sharding.ShardHome',
+                shardId,
+                region: info.path,
+                node: info.node.toJSON(),
+            });
+        }
+        this.pending.delete(shardId);
+    }
+    shipRememberedEntities(shardId) {
+        const set = this.entitiesPerShard.get(shardId);
+        if (!set || set.size === 0)
+            return;
+        const home = this.shardHome.get(shardId);
+        if (!home)
+            return;
+        this.sendToRegion(home, {
+            $t: 'sharding.RememberedEntities',
+            shardId,
+            entityIds: Array.from(set),
+        });
+    }
+    sendToRegion(key, msg) {
+        const info = this.regions.get(key);
+        if (!info)
+            return;
+        this.replyTo(info.path, info.node.toJSON(), msg);
+    }
+    replyTo(path, nodeData, msg) {
+        const node = NodeAddress.fromJSON(nodeData);
+        if (node.equals(this.settings.cluster.selfAddress)) {
+            const ref = this.settings.localResolver(path);
+            if (ref)
+                ref.tell(msg);
+            return;
+        }
+        const remote = new RemoteActorRef(node, path, this.settings.cluster);
+        remote.tell(msg);
+    }
+}
+//# sourceMappingURL=ShardCoordinator.js.map