@interopio/bridge 0.0.1-alpha

package/src/gossip/FailureDetector.ts

@@ -0,0 +1,235 @@
+import {type Address, addressAsString} from '../cluster/Address.ts';
+import {ObjectMap} from '../utils/collections.ts';
+import {preciseClock} from '../utils/clock.ts';
+import {convert} from '../config/Duration.ts';
+import getLogger from '../logging.ts';
+
+export interface FailureDetector {
+
+    /**
+     * invoked by any entity (gossiper) wanting to interrogate the status of the endpoint.
+     *
+     * @param endpoint
+     */
+    interpret(endpoint: Address): void;
+
+    /**
+     * invoked by the receiver of the heartbeat (gossiper).
+     * @param endpoint endpoint being reported
+     */
+    report(endpoint: Address): void
+
+    /**
+     * remove an endpoint
+     */
+    remove(endpoint: Address): void
+
+    /**
+     * force conviction of the specified endpoint in the failure detector.
+     * @param endpoint endpoint to be convicted
+     */
+    forceConviction(endpoint: Address): void;
+
+
+}
+
+
+export interface IFailureDetectionEventListener {
+    /**
+     * Convict the specified endpoint
+     * @param endpoint endpoint to be convicted
+     * @param phi the value of phi with which the endpoint was convicted
+     */
+    convict(endpoint: Address, phi: number): void;
+}
+
+class ArrayBackedBoundedStats {
+    private readonly _arrivalIntervals: Array<bigint>;
+    private _sum: bigint = 0n;
+    private _index: number = 0;
+    private _full = false;
+    private _mean: number = 0.0;
+
+    constructor(size: number) {
+        this._arrivalIntervals = new Array<bigint>(size);
+    }
+
+    add(interval: bigint): void {
+        if (this._index === this._arrivalIntervals.length) {
+            this._full = true;
+            this._index = 0;
+        }
+        if (this._full) {
+            this._sum = this._sum - this._arrivalIntervals[this._index];
+        }
+
+        this._arrivalIntervals[this._index++] = interval;
+        this._sum += interval;
+
+        this._mean = Number(this._sum) / this.size();
+    }
+
+    private size(): number {
+        return this._full ? this._arrivalIntervals.length : this._index;
+    }
+
+    public get mean(): number {
+        return this._mean;
+    }
+
+    toString(): string {
+        return `${this._arrivalIntervals}`;
+    }
+
+}
+
+const logger = getLogger('gossip.fd');
+
+const INITIAL_VALUE_NS = convert('nanoseconds', 2 * 1000/*gossiper interval*/, 'milliseconds');
+
+
+
+class ArrivalWindow {
+    private static readonly MAX_INTERVAL_NS = ArrivalWindow.getMaxInterval();
+    private readonly _arrivalIntervals: ArrayBackedBoundedStats;
+    private _lastReportedPhi: number = Number.MIN_VALUE;
+    private _last: bigint = 0n;
+
+    constructor(size: number) {
+        this._arrivalIntervals = new ArrayBackedBoundedStats(size);
+    }
+
+    static getMaxInterval(): bigint {
+        return INITIAL_VALUE_NS;
+    }
+
+    add(now: bigint, endpoint: Address) {
+        if (this._last < 0n) {
+            throw new Error();
+        }
+        if (this._last > 0n) {
+            const interval = now - this._last;
+            if (interval <= ArrivalWindow.MAX_INTERVAL_NS) {
+                this._arrivalIntervals.add(interval);
+                logger.enabledFor(`trace`) && logger.debug(`Reporting Interval time for ${addressAsString(endpoint)}: ${interval}ns`);
+            }
+            else {
+                logger.enabledFor(`trace`) && logger.debug(`Ignoring Interval time for ${addressAsString(endpoint)}: ${interval}ns`);
+            }
+        }
+        else {
+            this._arrivalIntervals.add(INITIAL_VALUE_NS);
+        }
+        this._last = now;
+    }
+
+    get mean(): number {
+        return this._arrivalIntervals.mean;
+    }
+
+    get lastReportedPhi(): number {
+        return this._lastReportedPhi;
+    }
+
+    phi(now: bigint): number {
+        if (!(this._arrivalIntervals.mean > 0 && this._last > 0n)) {
+            throw new Error('mean and last should be > 0 (mean: ' + this._arrivalIntervals.mean + ', last: ' + this._last + ')');
+        }
+        const t = Number(now - this._last);
+        this._lastReportedPhi = t / this.mean;
+        return this._lastReportedPhi;
+    }
+}
+
+// "The Phi Accrual Failure Detector"
+export const failureDetector: FailureDetector = new (class implements FailureDetector {
+    private readonly SAMPLE_SIZE = 1000;
+
+
+    private readonly PHI_FACTOR = 1.0 / Math.log(10.0);
+    private readonly arrivalSamples: Map<Address, ArrivalWindow> = new ObjectMap<Address, ArrivalWindow>();
+    private readonly listeners = new Array<IFailureDetectionEventListener>()
+
+    private lastPause: bigint = 0n;
+    private lastInterpret = preciseClock.now();
+
+    report(endpoint: Address) {
+
+        const now = preciseClock.now();
+        let arrivalWindow = this.arrivalSamples.get(endpoint);
+        if (arrivalWindow === undefined) {
+            arrivalWindow = new ArrivalWindow(this.SAMPLE_SIZE);
+            arrivalWindow.add(now, endpoint);
+
+            this.arrivalSamples.set(endpoint, arrivalWindow);
+        }
+        else {
+            arrivalWindow.add(now, endpoint);
+        }
+        if (logger.enabledFor('trace')) {
+            logger.debug(`Average for ${addressAsString(endpoint)}: ${arrivalWindow.mean}ns`);
+        }
+    }
+
+    interpret(endpoint: Address) {
+        const arrivalWindow =  this.arrivalSamples.get(endpoint);
+        if (arrivalWindow === undefined) {
+            return;
+        }
+        const now = preciseClock.now();
+        const diff = now - this.lastInterpret;
+        this.lastInterpret = now;
+
+        if (diff > MAX_LOCAL_PAUSE_NS) {
+            logger.warn(`Not marking nodes down due to local pause of ${diff}ns > ${MAX_LOCAL_PAUSE_NS}ns`);
+            this.lastPause = now;
+            return;
+        }
+        if ((preciseClock.now() - this.lastPause) < MAX_LOCAL_PAUSE_NS) {
+            if (logger.enabledFor('debug')) {
+                logger.debug(`Still not marking nodes down due to local pause`);
+            }
+            return;
+        }
+
+        const phi = arrivalWindow.phi(now);
+        if (logger.enabledFor('trace')) {
+            logger.debug(`PHI for ${addressAsString(endpoint)}: ${phi}`);
+        }
+
+        if (this.PHI_FACTOR * phi > this.getPhiConvictThreshold()) {
+            if (logger.enabledFor('trace')) {
+                logger.debug(`Node ${addressAsString(endpoint)} phi ${this.PHI_FACTOR * phi} > ${this.getPhiConvictThreshold()}; intervals ${arrivalWindow}, mean: ${arrivalWindow.mean}ns`);
+            }
+            for (const listener of this.listeners) {
+                listener.convict(endpoint, phi);
+            }
+        }
+        else if (logger.enabledFor('debug') && (this.PHI_FACTOR * phi * DEBUG_PERCENTAGE /100.0 > this.getPhiConvictThreshold())) {
+            logger.debug(`PHI for ${addressAsString(endpoint)}: ${phi}`);
+        } else if (logger.enabledFor('trace')) {
+            logger.debug(`PHI for ${addressAsString(endpoint)}: ${phi}`);
+        }
+    }
+
+    forceConviction(endpoint: Address) {
+        if (logger.enabledFor('debug')) {
+            logger.debug(`Forcing conviction of ${addressAsString(endpoint)}`);
+        }
+        for (const listener of this.listeners) {
+            listener.convict(endpoint, this.getPhiConvictThreshold());
+        }
+    }
+
+    remove(endpoint: Address) {
+        this.arrivalSamples.delete(endpoint);
+    }
+
+    private getPhiConvictThreshold(): number {
+        return 8.0;
+    }
+
+
+});
+const MAX_LOCAL_PAUSE_NS = convert('nanoseconds', 5000, 'milliseconds');
+const DEBUG_PERCENTAGE = 80;