@interopio/bridge 0.0.1-alpha → 0.0.3-beta

package/src/gossip/Gossiper.ts

@@ -1,1133 +0,0 @@
-import getLogger from '../logging.ts';
-import {type Message, MessageFactory} from '../net/Message.ts';
-import type {DataInput, DataOutput, VersionedSerializer} from '../io/VersionedSerializer.ts';
-import {NoPayload, type VerbHandler, type VerbHandlerContext, Verbs} from '../net/Action.ts';
-import {type Address, addressAsString, compareAddress, fromIpAddress} from '../cluster/Address.ts';
-import {type Comparator, composeComparators, ObjectMap, ObjectSet} from '../utils/collections.ts';
-import {type MessageDelivery, MessagingService, type Messenger} from './Messenger.ts';
-import {encodedUtfLength} from '../io/util.ts';
-import {getByAddress} from '../utils/network.ts';
-import {computeUnsignedVIntSize} from '../utils/vint.ts';
-import {addressSerializer} from '../net/AddressSerializer.ts';
-import {EndpointState, EndpointStateSerializer} from './EndpointState.ts';
-import {EMPTY_VERSION, HeartbeatState} from './HeartbeatState.ts';
-import {globalClock} from '../utils/clock.ts';
-import {type ApplicationState} from './ApplicationState.ts';
-import {type VersionedValue, VersionedValueDelimiter} from './VersionedValue.ts';
-import {failureDetector} from './FailureDetector.ts';
-import type {RequestCallback} from '../net/RequestCallbacks.ts';
-import {convert} from '../config/Duration.ts';
-
-function getMaxEndpointStateVersion(state: EndpointState): number {
-    let maxVersion = state.heartbeat.version;
-    for (const [key, value] of state.states()) {
-        maxVersion = Math.max(maxVersion, value.version);
-    }
-    return maxVersion;
-}
-
-type GossipResult = { seed?: true };
-type GossipStatus = '' | 'LEFT' | 'removing' | 'removed' | 'hibernate' | 'shutdown';
-const DEAD_STATES = new Set<GossipStatus>(['removing', 'removed', 'hibernate', 'LEFT']);
-
-function getGossipStatus(state?: Readonly<EndpointState>): GossipStatus {
-    if (state === undefined) {
-        return '';
-    }
-
-    let versionedValue = state.getApplicationState('STATUS');
-    if (versionedValue === undefined) {
-        versionedValue = state.getApplicationState('_STATUS');
-        if (versionedValue === undefined) {
-            return '';
-        }
-    }
-    const value = versionedValue.value;
-    const values = value.split(VersionedValueDelimiter);
-    if (values.length === 0) {
-        throw new Error(`Invalid status value ${value}`);
-    }
-    return values[0] as GossipStatus;
-}
-
-const BOOTSTRAPPING_STATUS = new Set(['BOOT', 'BOOT_REPLACE']);
-
-function stateOrder(): Comparator<[endpoint: Address, state: EndpointState]> {
-    const cache = new class extends ObjectMap<Address, EndpointState> {
-        lookup([key, value]: [Address, EndpointState]) {
-            if (this.has(key)) {
-                return this.get(key);
-            }
-            this.set(key, value.copy());
-            return this.get(key);
-        }
-    }();
-
-
-    return composeComparators<[endpoint: Address, state: EndpointState]>(
-        (e1, e2) => {
-            const status1 = getGossipStatus(cache.lookup(e1));
-            const status2 = getGossipStatus(cache.lookup(e2));
-            if (status1 === status2 || BOOTSTRAPPING_STATUS.has(status1) && BOOTSTRAPPING_STATUS.has(status2)) {
-                return 0;
-            }
-            if (BOOTSTRAPPING_STATUS.has(status1)) {
-                return 1;
-            }
-            if (BOOTSTRAPPING_STATUS.has(status2)) {
-                return -1;
-            }
-            return 0;
-        },
-        (e1, e2) => {
-            const generation1 = cache.lookup(e1).heartbeat.generation;
-            const generation2 = cache.lookup(e2).heartbeat.generation;
-            return (generation1 === generation2) ? 0 : ((generation1 > generation2) ? 1 : -1);
-        },
-        ([a1], [a2]) => {
-            return compareAddress(a1, a2);
-        });
-}
-
-const logger = getLogger('gossiper');
-
-export type EndpointStateSubscriber = {
-    onJoin(endpoint: Address, state: EndpointState): void
-    onChange(endpoint: Address, state: ApplicationState, value: VersionedValue): void
-    onAlive(endpoint: Address, state: EndpointState): void
-    onDead(endpoint: Address, state: EndpointState): void
-    onRemove(endpoint: Address): void
-    onRestart(endpoint: Address, state: EndpointState): void
-}
-
-const MAX_GENERATION_DIFFERENCE = 86400 * 365;
-const QUARANTINE_DELAY = 30 * 1000 * 2;
-
-const VERY_LONG_TIME_MS = 259200000;
-const aVeryLongTime = BigInt(VERY_LONG_TIME_MS);
-
-function computeExpireTime() {
-    return globalClock.epochTime() + aVeryLongTime;
-}
-
-class Gossiper {
-
-    private static readonly disableEndpointRemoval = false;
-
-    private gossipTaskSetInterval: ReturnType<typeof setInterval>;
-
-    private readonly fatClientTimeout: bigint;
-
-    private readonly subscribers = new Array<EndpointStateSubscriber>();
-    private readonly liveEndpoints: Set<Address> = new ObjectSet<Address>();
-    private readonly unreachableEndpoints: Map<Address, bigint> = new ObjectMap<Address, bigint>();
-
-    private readonly seeds: Set<Address> = new ObjectSet<Address>();
-
-    // key address:port -> value EndpointState
-    private readonly endpointState: Map<Address, EndpointState> = new ObjectMap<Address, EndpointState>();
-
-    private readonly justRemovedEndpoints: Map<Address, bigint> = new ObjectMap();
-    private readonly expiredEndpoints: Map<Address, bigint> = new Map();
-    lastProcessedTimeMs: bigint = globalClock.epochTime();
-    firstSynSendAt?: bigint;
-    readonly messaging: Messenger;
-
-    private readonly config: { broadcastAddress: Address, clusterName: string };
-    private readonly messageFactory: MessageFactory;
-
-    constructor(config: { localAddress: Address, broadcastAddress: Address, clusterName: string }) {
-        const context = {
-            gossiper: () => this,
-            messaging: () => this.messaging,
-            messageFactory: () => this.messageFactory
-        };
-
-        this.fatClientTimeout = BigInt(QUARANTINE_DELAY / 3);
-        this.config = config;
-        this.messageFactory = new MessageFactory(config.broadcastAddress);
-        this.messaging = new MessagingService(config.localAddress, context);
-    }
-
-    get endpointStates(): ReadonlyMap<Address, EndpointState> {
-        return this.endpointState;
-    }
-
-    register(subscriber: EndpointStateSubscriber): void {
-        this.subscribers.push(subscriber);
-    }
-
-    examineGossiper(digests: ReadonlyArray<GossipDigest>): GossipDigestAck {
-        if (digests.length === 0) {
-            throw new Error(`examineGossiper: empty digests`);
-        }
-
-        const delta = {
-            digests: new Array<GossipDigest>(),
-            states: new ObjectMap<Address, EndpointState>()
-        };
-        for (const digest of digests) {
-            const {endpoint, generation: remoteGeneration, maxVersion: maxRemoteVersion} = digest;
-
-            const state = this.endpointState.get(endpoint);
-            if (state) {
-                const localGeneration = state.heartbeat.generation;
-                const maxLocalVersion = getMaxEndpointStateVersion(state);
-
-                if (remoteGeneration === localGeneration && maxRemoteVersion === maxLocalVersion) {
-                    continue;
-                }
-                // request all from the gossiper
-                if (remoteGeneration > localGeneration) {
-                    this.requestAll(digest, delta.digests, remoteGeneration);
-                } else if (remoteGeneration < localGeneration) {
-                    // send all data with local generation > -1
-                    this.sendAll(digest, delta.states, EMPTY_VERSION);
-                } else if (remoteGeneration === localGeneration) {
-                    if (maxRemoteVersion > maxLocalVersion) {
-                        delta.digests.push({endpoint, generation: remoteGeneration, maxVersion: maxLocalVersion});
-                    } else if (maxRemoteVersion < maxLocalVersion) {
-                        this.sendAll(digest, delta.states, maxRemoteVersion);
-                    }
-                }
-            } else {
-                this.requestAll(digest, delta.digests, remoteGeneration);
-            }
-        }
-        return delta;
-
-    }
-
-    applyStateLocally(states: ReadonlyMap<Address, EndpointState>): void {
-        for (const [endpoint, remoteState] of Array.from(states.entries()).sort(stateOrder())) {
-            // todo: skip if broadcast address
-
-            if (this.justRemovedEndpoints.has(endpoint)) {
-                continue;
-            }
-            const localState = this.endpointState.get(endpoint);
-
-            if (localState != null) {
-                const localGeneration = localState.heartbeat.generation;
-                const remoteGeneration = remoteState.heartbeat.generation;
-                const localTime = Number(globalClock.epochTime() / 1000n);
-                if (logger.enabledFor('debug')) {
-                    logger.debug(`${addressAsString(endpoint)} local generation ${localGeneration}, remote generation ${remoteGeneration}`);
-                }
-                if (remoteGeneration > localTime + MAX_GENERATION_DIFFERENCE) {
-                    logger.warn(`received an invalid gossip generation for peer ${addressAsString(endpoint)}; local time = ${localTime}, remote generation = ${remoteGeneration}`);
-                }
-                else if (remoteGeneration > localGeneration) {
-                    if (logger.enabledFor('debug')) {
-                        logger.debug(`Updating heartbeat state generation to ${remoteGeneration} from ${localGeneration} for ${addressAsString(endpoint)}`);
-                    }
-                    this.handleMajorStateChange(endpoint, remoteState);
-                } else if (remoteGeneration === localGeneration) {
-                    const localMaxVersion = getMaxEndpointStateVersion(localState);
-                    const remoteMaxVersion = getMaxEndpointStateVersion(remoteState);
-                    if (remoteMaxVersion > localMaxVersion) {
-                        this.applyNewStates(endpoint, localState, remoteState);
-                    } else if (logger.enabledFor('debug')) {
-                            logger.debug(`Ignoring remote version ${remoteMaxVersion} <= ${localMaxVersion} for ${addressAsString(endpoint)}`);
-                    }
-                    if (!localState.isAlive &&  !this.isDeadState(localState)) {
-                        this.markAlive(endpoint, localState);
-                    }
-                } else {
-                    if (logger.enabledFor('debug')) {
-                        logger.debug(`Ignoring remote generation ${remoteGeneration} < ${localGeneration} for ${addressAsString(endpoint)} `);
-                    }
-                }
-            }
-            else {
-                failureDetector.report(endpoint);
-                this.handleMajorStateChange(endpoint, remoteState);
-            }
-        }
-    }
-
-    private applyNewStates(endpoint: Address, localState: EndpointState, remoteState: EndpointState): void {
-        const oldVersion = localState.heartbeat.version;
-
-        localState.heartbeat = remoteState.heartbeat;
-        if (logger.enabledFor('debug')) {
-            logger.debug(`Updating heartbeat state version to ${localState.heartbeat.version} from ${oldVersion} for ${addressAsString(endpoint)} ...`);
-        }
-
-        const remoteStates = Array.from(remoteState.states());
-        if (remoteState.heartbeat.generation !== localState.heartbeat.generation) {
-            throw new Error();
-        }
-
-        const updatedStates =  remoteStates.filter(([key, value]) => {
-            const local = localState.getApplicationState(key);
-            return local === undefined || local.version < value.version;
-        });
-
-        if (logger.enabledFor('debug') && updatedStates.length > 0) {
-            for (const [key, value] of updatedStates) {
-                logger.debug(`Updating ${key} state version to ${value.version} for ${addressAsString(endpoint)}`);
-            }
-        }
-
-        localState.addApplicationStates(updatedStates);
-        // localState.remove
-
-        for (const [key, value] of updatedStates) {
-            switch (key) {
-                default:
-                    continue;
-                case '_STATUS':
-                    if (localState.hasApplicationState('STATUS')) {
-                        continue;
-                    }
-                case 'STATUS': {
-
-                }
-            }
-            this.doOnChangeNotification(endpoint, key, value);
-        }
-
-        for (const [key, value] of updatedStates) {
-            switch (key) {
-                case '_STATUS':
-                case 'STATUS':
-                    continue;
-            }
-
-            if ((key === '_INTERNAL_IP' && localState.hasApplicationState('INTERNAL_ADDRESS'))
-                || (key === '_RPC_ADDRESS' && localState.hasApplicationState('NATIVE_ADDRESS'))) {
-                continue;
-            }
-            this.doOnChangeNotification(endpoint, key, value);
-        }
-
-    }
-
-    public isGossipOnlyMember(endpoint: Address): boolean {
-        const state = this.endpointState.get(endpoint);
-        if (state === undefined) {
-            return false;
-        }
-        const status = getGossipStatus(state);
-        if (status === 'removing' || status === 'removed') {
-            return true;
-        }
-        return !this.isDeadState(state) && !(false/*todo check current metadata*/);
-    }
-
-    private async doStatusCheck() {
-        if (logger.enabledFor('trace')) {
-            logger.debug(`Performing status check ...`);
-        }
-
-        const now  = globalClock.epochTime();
-        const nowNs = globalClock.nanoTime();
-
-        for (const endpoint of this.endpointStates.keys()) {
-            if (compareAddress(this.config.broadcastAddress, endpoint) === 0) {
-                continue;
-            }
-            failureDetector.interpret(endpoint);
-            const state = this.endpointState.get(endpoint);
-            if (state !== undefined) {
-                if (this.isGossipOnlyMember(endpoint)
-                    && !this.justRemovedEndpoints.has(endpoint)
-                    && convert('milliseconds', nowNs - state.timestamp , 'nanoseconds') > this.fatClientTimeout) {
-                    logger.info(`FatClient ${addressAsString(endpoint)} has been silent for ${this.fatClientTimeout}ms, removing from gossip`);
-
-                    await new Promise<void>((resolve, reject) => {
-                        setImmediate(() => {
-                            try {
-                                if (!this.isGossipOnlyMember(endpoint)) {
-                                    logger.info(`Race condition marking ${addressAsString(endpoint)} as FatClient; ignoring`);
-                                } else {
-                                    this.removeEndpoint(endpoint);
-                                    this.evictFromMembership(endpoint);
-                                }
-                                resolve();
-                            } catch (e) {
-                                reject(e);
-                            }
-                        });
-                    });
-                }
-                const expireTime = this.getExpireTimeForEndpoint(endpoint);
-                if (!state.isAlive && (now > expireTime) && (!false /*metadata*/)) {
-                    if (logger.enabledFor('debug')) {
-                        logger.debug(`time is expiring for endpoint ${addressAsString(endpoint)}: ${expireTime}`);
-                    }
-                    await new Promise<void>((resolve, reject) => {
-                        setImmediate(() => {
-                            try {
-                                this.evictFromMembership(endpoint);
-                                resolve();
-                            } catch (e) {
-                                reject(e);
-                            }
-                        });
-                    });
-                }
-            }
-        }
-        if (this.justRemovedEndpoints.size !== 0) {
-            for (const [endpoint, timestamp] of this.justRemovedEndpoints) {
-                if ((now - timestamp) > QUARANTINE_DELAY) {
-                    logger.debug(`${QUARANTINE_DELAY} elapsed, ${addressAsString(endpoint)} quarantine over`);
-                    this.justRemovedEndpoints.delete(endpoint);
-                }
-            }
-        }
-    }
-
-    public doOnChangeNotification(endpoint: Address, state: ApplicationState, value: VersionedValue): void {
-        for (const subscriber of this.subscribers) {
-            subscriber.onChange(endpoint, state, value);
-        }
-
-    }
-    private handleMajorStateChange(endpoint: Address, remoteState: EndpointState): void {
-        const localState = this.endpointState.get(endpoint);
-        if (!this.isDeadState(remoteState)) {
-            if (localState !== undefined) {
-                logger.info(`Node ${addressAsString(endpoint)} has restarted, now UP`);
-            }
-            else {
-                logger.info(`Node ${addressAsString(endpoint)} is now part of the cluster`);
-            }
-        }
-        if (logger.enabledFor('trace')) {
-            logger.debug(`Adding endpoint state for ${addressAsString(endpoint)}`);
-        }
-        this.endpointState.set(endpoint, remoteState);
-
-        if (localState !== undefined) {
-            for (const subscriber of this.subscribers) {
-                subscriber.onRestart(endpoint, localState);
-            }
-        }
-
-        if (!this.isDeadState(remoteState)) {
-            this.markAlive(endpoint, remoteState);
-        }
-        else {
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Not marking ${addressAsString(endpoint)} alive due to dead state`);
-            }
-            this.markDead(endpoint, remoteState);
-        }
-
-        for (const subscriber of this.subscribers) {
-            subscriber.onJoin(endpoint, remoteState);
-        }
-
-        if (this.isShutdown(endpoint)) {
-            this.markAsShutdown(endpoint, remoteState);
-        }
-    }
-
-    private markAlive(endpoint: Address, localState: EndpointState): void {
-        localState.markAsDead();
-
-        const echo = this.messageFactory.out(Verbs.ECHO_REQ, NoPayload.instance);
-        if (logger.enabledFor('debug')) {
-            logger.debug(`Sending ECHO_REQ to ${addressAsString(endpoint)}`);
-        }
-
-        const handler: RequestCallback<unknown> = (msg) => {
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Received ECHO_RSP from ${addressAsString(endpoint)}`);
-            }
-            this.realMarkAlive(endpoint, localState);
-        }
-        this.messaging.sendWithCallback(echo, endpoint, handler);
-    }
-
-    private realMarkAlive(endpoint: Address, localState: EndpointState): void {
-        if (logger.enabledFor('trace')) {
-            logger.debug(`marking as alive ${addressAsString(endpoint)}`);
-        }
-        localState.markAsAlive();
-        localState.updateTimestamp();
-        this.liveEndpoints.add(endpoint);
-        this.unreachableEndpoints.delete(endpoint);
-        this.expiredEndpoints.delete(endpoint);
-        if (logger.enabledFor('debug')) {
-            logger.debug(`removing expire time for ${addressAsString(endpoint)}`);
-        }
-        logger.info(`Address ${addressAsString(endpoint)} is now UP`);
-        for (const subscriber of this.subscribers) {
-            subscriber.onAlive(endpoint, localState);
-        }
-        if (logger.enabledFor('trace')) {
-            logger.debug(`Notified {}`, this.subscribers);
-        }
-    }
-
-    markDead(endpoint: Address, localState: EndpointState): void {
-        if (logger.enabledFor('trace')) {
-            logger.debug(`marking as down ${addressAsString(endpoint)}`);
-        }
-        this.silentlyMarkDead(endpoint, localState);
-        logger.info(`Address ${addressAsString(endpoint)} is now DOWN`);
-        // todo: check if the node is registered
-        for (const subscriber of this.subscribers) {
-            subscriber.onDead(endpoint, localState);
-        }
-
-        if (logger.enabledFor('trace')) {
-            logger.debug(`Notified ${this.subscribers}`);
-        }
-    }
-
-    private silentlyMarkDead(endpoint: Address, localState: EndpointState): void {
-        localState.markAsDead();
-        if (!Gossiper.disableEndpointRemoval) {
-            this.liveEndpoints.delete(endpoint);
-            this.unreachableEndpoints.set(endpoint, globalClock.nanoTime());
-        }
-    }
-
-    private isShutdown(endpoint: Address) {
-        const state = this.endpointState.get(endpoint);
-        if (state === undefined) {
-            return false;
-        }
-        return getGossipStatus(state) === 'shutdown';
-    }
-
-    private requestAll(digest: GossipDigest, digests: GossipDigest[], remoteGeneration: number): void {
-        digests.push({endpoint: digest.endpoint, generation: remoteGeneration, maxVersion: 0});
-        if (logger.enabledFor('debug')) {
-            logger.debug(`requestAll for ${addressAsString(digest.endpoint)}`);
-        }
-    }
-
-    private sendAll(digest: GossipDigest, states: Map<Address, EndpointState>, maxRemoteVersion: number): void {
-        const localState: EndpointState = this.getStatesForVersionsBiggerThan(digest.endpoint, maxRemoteVersion);
-        if (localState) {
-            states.set(digest.endpoint, localState);
-        }
-    }
-
-    examineShadowState(): Map<Address, EndpointState> {
-        const map = new ObjectMap<Address, EndpointState>();
-        for (const [endpoint, value] of this.endpointState) {
-            const state = value.copy()
-            if (state.isEmptyWithoutStatus) {
-
-            }
-        }
-        return map;
-    }
-
-    private getExpireTimeForEndpoint(endpoint: Address): bigint {
-        const storedTime = this.expiredEndpoints.get(endpoint);
-        if (storedTime !== undefined) {
-            return storedTime;
-        }
-        return computeExpireTime();
-    }
-
-    getStatesForVersionsBiggerThan(endpoint: Address, version: number): EndpointState | undefined {
-        const state = this.endpointState.get(endpoint);
-        let requestedState: EndpointState | undefined;
-        if (state) {
-            const {generation: localHbGeneration, version: localHbVersion} = state.heartbeat;
-            if (localHbVersion > version) {
-                requestedState = new EndpointState(new HeartbeatState(localHbGeneration, localHbVersion));
-                if (logger.enabledFor('debug')) {
-                    logger.debug(`Local heartbeat version ${localHbVersion} greater than ${version}  for ${addressAsString(endpoint)}`);
-                }
-            }
-
-            const states = new Map<ApplicationState, VersionedValue>();
-            /* Accumulate all application states whose versions are greater than "version" variable */
-            for (const [key, value] of state.states()) {
-                if (value.version > version) {
-                    if (requestedState === undefined) {
-                        requestedState = new EndpointState(new HeartbeatState(localHbGeneration, localHbVersion));
-                    }
-                    logger.debug(`Adding state ${key} with version ${value.version} for val ${value.value}`);
-                    states.set(key, value);
-                }
-            }
-            if (requestedState) {
-                requestedState.addApplicationStates(Array.from(states.entries()));
-            }
-        }
-        return requestedState;
-    }
-
-    notifyFailureDetector(remoteState: ReadonlyMap<Address, EndpointState>): void {
-        for (const [endpoint, state] of remoteState) {
-            this.notifyFailureDetectorFor(endpoint, state);
-        }
-    }
-
-    private notifyFailureDetectorFor(endpoint: Address, remoteState?: EndpointState): void {
-        if (remoteState === undefined) {
-            return;
-        }
-
-        const localState = this.endpointState.get(endpoint);
-
-        if (localState !== undefined) {
-            const fd = failureDetector;
-
-            const localGeneration = localState.heartbeat.generation;
-            const remoteGeneration = remoteState.heartbeat.generation;
-            if (remoteGeneration > localGeneration) {
-                localState.updateTimestamp();
-                // this endpoint was dead and the generation changed, this indicates a re-boot
-                if (!localState.isAlive) {
-                    if (logger.enabledFor('debug')) {
-                        logger.debug(`Clearing interval times for ${addressAsString(endpoint)} due to generation change`);
-                    }
-                    fd.remove(endpoint);
-                }
-                fd.report(endpoint);
-                return;
-            }
-            if (remoteGeneration === localGeneration) {
-                const localVersion = getMaxEndpointStateVersion(localState);
-                const remoteVersion = remoteState.heartbeat.version;
-                if (remoteVersion > localVersion) {
-                    localState.updateTimestamp();
-                    // just a version change
-                    fd.report(endpoint);
-                }
-            }
-        }
-    }
-
-
-    start(generationNumber: number, mergeLocalStates = false) {
-        this.buildSeedList();
-        this.maybeInitializeLocalState(generationNumber);
-        this.gossipTaskSetInterval = setInterval(() => {
-            this.task();
-        }, 1000)
-    }
-
-    stop() {
-        if (this.gossipTaskSetInterval) {
-            clearInterval(this.gossipTaskSetInterval);
-            this.gossipTaskSetInterval = undefined;
-        }
-    }
-
-    get enabled() {
-        return this.gossipTaskSetInterval !== undefined;
-    }
-
-    private buildSeedList(): void {
-        this.seeds.add(fromIpAddress(getByAddress(undefined, 0x7f000001), 7000));
-    }
-
-    private maybeInitializeLocalState(generationNumber: number): void {
-        const heartbeat = new HeartbeatState(generationNumber);
-        const localState = new EndpointState(heartbeat);
-        localState.markAsAlive();
-        this.endpointState.set(this.config.broadcastAddress, localState);
-    }
-
-
-    async task() {
-        this.endpointState.get(this.config.broadcastAddress).heartbeat.updateHeartbeatVersion();
-        if (logger.enabledFor('trace')) {
-            logger.debug(`My heartbeat is now ${this.endpointState.get(this.config.broadcastAddress).heartbeat.version}`);
-        }
-        const digests: GossipDigest[] = [];
-        this.makeGossipDigest(digests);
-        if (digests.length > 0) {
-            const cluster = this.config.clusterName;
-            const partitioner = "org.apache.cassandra.dht.Murmur3Partitioner";
-            const digestSynMessage = {cluster, partitioner, digests};
-            const message = this.messageFactory.out(Verbs.GOSSIP_DIGEST_SYN, digestSynMessage)
-            let gossipedWith: GossipResult = this.doGossipToLiveMember(message);
-
-            this.maybeGossipToUnreachableMember(message);
-
-            if (!gossipedWith?.seed || this.liveEndpoints.size < this.seeds.size) {
-                gossipedWith = {...this.maybeGossipToSeed(message), ...gossipedWith};
-            }
-
-            await this.doStatusCheck();
-        }
-    }
-
-    markAsShutdown(endpoint: Address, remoteState: EndpointState): void {
-        const endpointState = this.endpointState.get(endpoint);
-        if (endpointState === undefined || endpointState.isStateEmpty) {
-            return;
-        }
-        const shutdown = remoteState.getApplicationState('STATUS');
-        if (shutdown === undefined) {
-            throw new Error(`assertion failed: remote shutdown sent but not with a shutdown status? ${remoteState}`);
-        }
-        remoteState.heartbeat.forceHighestPossibleVersion()
-        this.endpointState.set(endpoint, remoteState);
-        this.markDead(endpoint, remoteState);
-        failureDetector.forceConviction(endpoint);
-        for (const subscriber of this.subscribers) {
-            subscriber.onChange(endpoint, 'STATUS', shutdown);
-        }
-        if (logger.enabledFor('debug')) {
-            logger.debug(`Marked ${addressAsString(endpoint)} as shutdown`);
-        }
-    }
-
-    evictFromMembership(endpoint: Address): void {
-        this.unreachableEndpoints.delete(endpoint);
-        this.endpointState.delete(endpoint);
-        this.expiredEndpoints.delete(endpoint);
-        failureDetector.remove(endpoint);
-        this.quarantineEndpoint(endpoint);
-        if (logger.enabledFor('debug')) {
-            logger.debug(`Evicting ${addressAsString(endpoint)} from membership`);
-        }
-    }
-
-    removeEndpoint(endpoint: Address): void {
-
-        for (const subscriber of this.subscribers) {
-            subscriber.onRemove(endpoint);
-        }
-
-        if (this.seeds.has(endpoint)) {
-            this.buildSeedList();
-            this.seeds.delete(endpoint);
-            logger.info(`removed ${addressAsString(endpoint)} from seeds, updated seed list = ${this.seeds}`);
-            if (this.seeds.size === 0) {
-                logger.warn(`seeds list is empty`);
-            }
-        }
-
-        if (Gossiper.disableEndpointRemoval) {
-            return;
-        }
-
-        this.liveEndpoints.delete(endpoint);
-        this.unreachableEndpoints.delete(endpoint);
-        this.quarantineEndpoint(endpoint);
-        this.messaging.closeOutbound(endpoint);
-        this.messaging.removeInbound(endpoint);
-        logger.debug(`removing endpoint`);
-    }
-
-    private quarantineEndpoint(endpoint: Address, expirationMs?: bigint): void {
-        if (Gossiper.disableEndpointRemoval) {
-            return;
-        }
-
-        expirationMs ??= globalClock.epochTime();
-        this.justRemovedEndpoints.set(endpoint, expirationMs);
-
-
-    }
-
-    makeGossipDigest(digests: GossipDigest[]): void {
-        let generation: number;
-        let maxVersion: number;
-        for (const [endpoint, state] of this.endpointState) {
-            if (state) {
-                generation = state.heartbeat.generation;
-                maxVersion = getMaxEndpointStateVersion(state);
-            } else {
-                generation = 0;
-                maxVersion = 0;
-            }
-            digests.push({endpoint, generation, maxVersion});
-        }
-        if (logger.enabledFor('trace')) {
-            logger.debug(`Gossip Digests are ${JSON.stringify(digests)}`);
-        }
-    }
-
-    private doGossipToLiveMember(message: Message<GossipDigestSyn>): GossipResult {
-        const size = this.liveEndpoints.size;
-        if (size === 0) {
-            return {};
-        }
-        return this.sendGossip(message, this.liveEndpoints)
-    }
-
-    private isDeadState(endpoint?: Readonly<EndpointState>): boolean {
-        const status = getGossipStatus(endpoint)
-        if (status === '') {
-            return false;
-        }
-        return DEAD_STATES.has(status);
-    }
-
-    private maybeGossipToUnreachableMember(message: Message<GossipDigestSyn>): void {
-        const liveCount = this.liveEndpoints.size;
-        const unreachableCount = this.unreachableEndpoints.size;
-        if (unreachableCount > 0) {
-            const prob = unreachableCount / (liveCount + 1);
-            if (prob > Math.random()) {
-                const members = Array.from(this.unreachableEndpoints.keys())
-                    .filter(ep => !this.isDeadState(this.endpointState.get(ep)));
-                this.sendGossip(message, new ObjectSet<Address>(members));
-            }
-        }
-    }
-
-    private maybeGossipToSeed(message: Message<GossipDigestSyn>): GossipResult {
-        const size = this.seeds.size;
-        let result: GossipResult = {};
-        if (size > 0) {
-            if (size === 1 && this.seeds.has(this.config.broadcastAddress)) {
-                return result;
-            }
-
-            if (this.liveEndpoints.size == 0) {
-                result = this.sendGossip(message, this.seeds);
-            } else {
-                const prob = size / (this.liveEndpoints.size + this.unreachableEndpoints.size);
-                if (prob >= Math.random()) {
-                    result = this.sendGossip(message, this.seeds);
-                }
-            }
-        }
-    }
-
-    private sendGossip(message: Message<GossipDigestSyn>, epSet: Set<Address>): GossipResult {
-        const endpoints = Array.from(epSet);
-        const size = endpoints.length;
-        if (size < 1) {
-            return {};
-        }
-        const index = size === 1 ? 0 : Math.floor(Math.random() * size);
-        const to = endpoints[index];
-        if (logger.enabledFor('trace')) {
-            logger.debug(`Sending a GossipDigestSyn to ${addressAsString(to)} ...`);
-        }
-        if (this.firstSynSendAt === undefined) {
-            this.firstSynSendAt = globalClock.nanoTime();
-        }
-        this.messaging.send(message, to);
-        const result: GossipResult = {};
-        if (this.seeds.has(to)) {
-            result.seed = true;
-        }
-        return result;
-
-    }
-}
-
-export type {Gossiper}
-export type GossiperConfig = { localAddress: Address, broadcastAddress: Address, clusterName: string }
-
-export function create(config: GossiperConfig): Gossiper {
-    return new Gossiper(config);
-}
-
-
-export type GossipDigest = {
-    readonly endpoint: Address,
-    readonly generation: number,
-    readonly maxVersion: number,
-};
-
-const GossipDigestSerializer = new (class implements VersionedSerializer<GossipDigest> {
-    serializedSize(value: GossipDigest, version: number): number {
-        let size = addressSerializer.serializedSize(value.endpoint, version);
-        size += 4; // generation
-        size += 4; // maxVersion
-        return size;
-    }
-
-    serialize(value: GossipDigest, output: DataOutput, version?: number) {
-        addressSerializer.serialize(value.endpoint, output, version);
-        output.writeInt32(value.generation);
-        output.writeInt32(value.maxVersion);
-    }
-
-    deserialize(input: DataInput, version?: number): GossipDigest {
-        const endpoint = addressSerializer.deserialize(input, version);
-        const generation = input.readInt32();
-        const maxVersion = input.readInt32();
-        return {endpoint, generation, maxVersion};
-    }
-});
-
-function serializeGossipDigestList(digests: readonly GossipDigest[], output: DataOutput, version?: number): void {
-    output.writeInt32(digests.length);
-    for (const digest of digests) {
-        GossipDigestSerializer.serialize(digest, output, version);
-    }
-}
-
-function deserializeGossipDigestList(input: DataInput, version?: number): readonly GossipDigest[] {
-    const size = input.readInt32();
-    const digests = new Array<GossipDigest>(size);
-    for (let i = 0; i < size; i++) {
-        digests[i] = GossipDigestSerializer.deserialize(input, version);
-    }
-    return digests;
-}
-
-function serializedSizeGossipDigestList(digests: readonly GossipDigest[], version?: number): number {
-    let size = 4;
-    for (const digest of digests) {
-        size += GossipDigestSerializer.serializedSize(digest, version);
-    }
-    return size;
-}
-
-export type GossipDigestSyn = { readonly cluster: string, readonly digests: ReadonlyArray<GossipDigest> };
-
-export const GossipDigestSynSerializer = new (class implements VersionedSerializer<GossipDigestSyn> {
-    serialize(value: GossipDigestSyn, output: DataOutput, version?: number): void {
-        output.writeUtf8String(value.cluster);
-        output.writeUtf8String("org.apache.cassandra.dht.Murmur3Partitioner");
-        if (version >= 14) {
-            output.writeUnsignedVInt(0);
-        }
-        serializeGossipDigestList(value.digests, output, version);
-
-    }
-
-    deserialize(input: DataInput, version?: number): GossipDigestSyn {
-        const clusterId = input.readUtf8String();
-        const _ = input.readUtf8String();
-        const metadataId = (version >= 14) ? input.readUnsignedVInt32() : 0;
-        const digests = deserializeGossipDigestList(input, version);
-        return {cluster: clusterId, digests};
-    }
-
-    serializedSize(value: GossipDigestSyn, version?: number): number {
-        let size = 2 + encodedUtfLength(value.cluster);
-        size += 2 + encodedUtfLength("org.apache.cassandra.dht.Murmur3Partitioner");
-        if (version >= 14) {
-            size += computeUnsignedVIntSize(0);
-        }
-        size += serializedSizeGossipDigestList(value.digests, version);
-        return size;
-    }
-
-});
-
-class GossipVerbHandler<T> implements VerbHandler<T> {
-    protected readonly gossiper: Gossiper;
-
-    constructor(context: { gossiper: () => Gossiper }) {
-        this.gossiper = context.gossiper();
-    }
-
-    doVerb(message: Message<T>) {
-        this.gossiper.lastProcessedTimeMs = message.creationTime;
-    }
-}
-
-export function gossipDigestSynHandler(ctx: VerbHandlerContext): VerbHandler<GossipDigestSyn> {
-    return new class extends GossipVerbHandler<GossipDigestSyn> {
-        doVerb(message: Message<GossipDigestSyn>): void {
-            const from = message.from;
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Received a GossipDigestSyn from ${addressAsString(from)}`);
-            }
-            if (!this.gossiper.enabled) {
-                logger.debug(`Ignoring GossipDigestSyn because gossiper is disabled`);
-                return;
-            }
-            const {cluster, digests} = message.payload;
-
-            if (!this.gossiper.enabled) {
-                if (digests.length > 0) {
-                    return;
-                }
-                ctx.messaging().send(ctx.messageFactory().out(Verbs.GOSSIP_DIGEST_ACK, {
-                    digests: [],
-                    states: new Map()
-                }), from);
-                return;
-            }
-
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Gossip syn digests are: ${digests}`);
-            }
-
-            const gossipDigestAckMessage = (digests.length === 0)
-                ? this.createShadowReply()
-                : this.createReply(digests);
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Sending a GossipDigestAck to ${addressAsString(from)}`);
-            }
-            ctx.messaging().send(gossipDigestAckMessage, from);
-
-            super.doVerb(message);
-        }
-
-        createReply(digests: ReadonlyArray<GossipDigest>): Message<GossipDigestAck> {
-            const delta = ctx.gossiper().examineGossiper(digests);
-            logger.debug(`Sending ${delta.digests.length} digests and ${delta.states.size} deltas`);
-            return ctx.messageFactory().out(Verbs.GOSSIP_DIGEST_ACK, delta);
-        }
-
-        createShadowReply(): Message<GossipDigestAck> {
-            const deltas: Map<Address, EndpointState> = ctx.gossiper().examineShadowState();
-            logger.debug(`Sending 0 digests and ${deltas.size} deltas`);
-            return ctx.messageFactory().out(Verbs.GOSSIP_DIGEST_ACK, {digests: [], states: deltas});
-        }
-    }(ctx);
-}
-
-export type GossipDigestAck = Readonly<{
-    digests: ReadonlyArray<GossipDigest>,
-    states: ReadonlyMap<Address, EndpointState>,
-}>
-
-
-export const GossipDigestAckSerializer = new (class implements VersionedSerializer<GossipDigestAck> {
-    serialize(value: GossipDigestAck, output: DataOutput, version: number) {
-        serializeGossipDigestList(value.digests, output, version);
-        output.writeInt32(value.states.size);
-        for (const [address, state] of value.states) {
-            addressSerializer.serialize(address, output, version);
-            EndpointStateSerializer.serialize(state, output, version);
-        }
-    }
-
-    deserialize(input: DataInput, version: number): GossipDigestAck {
-        const digests = deserializeGossipDigestList(input, version);
-        const size = input.readInt32();
-        const states = new ObjectMap<Address, EndpointState>();
-        for (let i = 0; i < size; i++) {
-            const address = addressSerializer.deserialize(input, version);
-            const endpointState = EndpointStateSerializer.deserialize(input, version);
-            states.set(address, endpointState);
-        }
-        return {digests, states};
-    }
-
-    serializedSize(value: GossipDigestAck, version: number): number {
-        let size = serializedSizeGossipDigestList(value.digests, version);
-        size += 4; // size
-        for (const [address, state] of value.states) {
-            size += addressSerializer.serializedSize(address, version);
-            size += EndpointStateSerializer.serializedSize(state, version);
-        }
-        return size;
-    }
-});
-
-export function gossipDigestAckHandler(ctx: VerbHandlerContext): VerbHandler<GossipDigestAck> {
-    return new (class extends GossipVerbHandler<GossipDigestAck> {
-        private readonly messageFactory: MessageFactory;
-        private readonly messageDelivery: MessageDelivery;
-        constructor(ctx: VerbHandlerContext) {
-            super(ctx);
-            this.messageFactory = ctx.messageFactory();
-            this.messageDelivery = ctx.messaging();
-        }
-
-        doVerb(message: Message<GossipDigestAck>): void {
-            const from = message.from;
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Received a GossipDigestAck from ${addressAsString(from)}`);
-            }
-            if (!this.gossiper.enabled) {
-                logger.debug(`Ignoring GossipDigestAck because gossiper is disabled`);
-                return;
-            }
-
-            const {states, digests} = message.payload;
-            if (logger.enabledFor('debug')) {
-                logger.debug(`Received ack with ${digests.length} digests and ${states.size} states`);
-            }
-
-            if (states.size > 0) {
-
-                if (this.gossiper.firstSynSendAt === undefined || (globalClock.nanoTime() - this.gossiper.firstSynSendAt) < 0) {
-                    logger.debug(`Ignoring unrequested GossipDigestAck from ${addressAsString(from)}`);
-                    return;
-                }
-                this.gossiper.notifyFailureDetector(states);
-                this.gossiper.applyStateLocally(states);
-            }
-
-            const delta = new ObjectMap<Address, EndpointState>();
-            for (const digest of digests) {
-                const addr = digest.endpoint;
-                const localState = this.gossiper.getStatesForVersionsBiggerThan(addr, digest.maxVersion);
-                if (localState) {
-                    delta.set(addr, localState);
-                }
-            }
-
-            const ack2: Message<GossipDigestAck2> = this.messageFactory.out(Verbs.GOSSIP_DIGEST_ACK2, {states: delta});
-            logger.debug(`Sending a GossipDigestAck2 to ${addressAsString(from)}`);
-            this.messageDelivery.send(ack2, from);
-
-            super.doVerb(message);
-        }
-    })(ctx);
-}
-
-export type GossipDigestAck2 = Readonly<{ states: ReadonlyMap<Address, EndpointState> }>;
-export const GossipDigestAck2Serializer = new (class implements VersionedSerializer<GossipDigestAck2> {
-
-    serializedSize(value: GossipDigestAck2, version: number): number {
-        let size = 4;// size
-        for (const [address, state] of value.states) {
-            size += addressSerializer.serializedSize(address, version);
-            size += EndpointStateSerializer.serializedSize(state, version);
-        }
-        return size;
-    }
-
-    serialize(value: GossipDigestAck2, output: DataOutput, version: number): void {
-        output.writeInt32(value.states.size);
-        for (const [address, state] of value.states) {
-            addressSerializer.serialize(address, output, version);
-            EndpointStateSerializer.serialize(state, output, version);
-        }
-    }
-
-    deserialize(input: DataInput, version: number): GossipDigestAck2 {
-        const size = input.readInt32();
-        const states = new ObjectMap<Address, EndpointState>();
-        for (let i = 0; i < size; i++) {
-            const address = addressSerializer.deserialize(input, version);
-            const endpointState = EndpointStateSerializer.deserialize(input, version);
-            states.set(address, endpointState);
-        }
-        return {states};
-    }
-});
-
-export function gossipDigestAck2Handler(ctx: VerbHandlerContext): VerbHandler<GossipDigestAck2> {
-    return new (class extends GossipVerbHandler<GossipDigestAck2> {
-        doVerb(message: Message<GossipDigestAck2>) {
-            if (logger.enabledFor('debug')) {
-                const from = message.from;
-                logger.debug(`Received GossipDigestAck2Message with ${addressAsString(from)}`);
-            }
-            if (!this.gossiper.enabled) {
-                if (logger.enabledFor('debug')) {
-                    logger.debug('Ignoring GossipDigestAck2Message because gossiper is disabled');
-                }
-                return;
-            }
-            const states = message.payload.states;
-            this.gossiper.notifyFailureDetector(states);
-            this.gossiper.applyStateLocally(states);
-            super.doVerb(message);
-        }
-    })(ctx);
-}
-
-
-export function echoHandler(ctx: VerbHandlerContext) {
-    return new class implements VerbHandler<NoPayload> {
-        doVerb(message: Message<NoPayload>) {
-            if (logger.enabledFor('trace')) {
-                logger.debug(`Sending ECHO_RSP to ${addressAsString(message.from)}`);
-            }
-            ctx.messaging().send(ctx.messageFactory().emptyResponse(message), message.from);
-        }
-    };
-}