@nsshunt/stsappframework 3.1.214 → 3.1.216

package/src/tcpserver/appmaster.ts

@@ -1,503 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import fs from "fs"
-import crypto from 'crypto';
-import jayson from 'jayson'
-import net from 'node:net'
-import tls from 'node:tls'
-
-import { JSONObject } from '@nsshunt/stsutils';
-import { MasterProcessBase } from './../masterprocessbase'
-import { ProcessOptions } from './../commonTypes'
-
-import chalk from 'chalk';
-
-// Colour codes
-// ---------------------------------
-// Red - Error
-// Green - Vote YES
-// Yellow - Vote No
-// White - standard logging message
-// Magenta - node state change
-// Cyan - node term change
-
-export enum AppMasterRAFTState {
-    IDLE = "IDLE",
-    FOLLOWER = "FOLLOWER",
-    CANDIDATE = "CANDIDATE",
-    LEADER = "LEADER"
-}
-
-export interface IServiceDetail {
-    host: string
-    port: number
-}
-
-export interface ILogItem {
-    term: number
-    index: number
-    data: JSONObject | null
-}
-
-export class AppMaster extends MasterProcessBase
-{
-    #lastRequiredVotes = -1; // used for logging only
-    #lastVotes = -1; // used for logging only
-    #RAFTState: AppMasterRAFTState;
-    #minElectionTimeout: number;
-    #maxElectionTimeout: number;
-    #electionTimeout: number;
-    #heartBeatTimeout: number;
-    #startRAFTProtocolTimeout: number;
-    #votes: JSONObject; // keyed by service key (host:port)
-    #services: IServiceDetail[];
-    #thisService: IServiceDetail;
-    #canidateTimeout: NodeJS.Timeout | null = null;
-    #serverPort: number;
-    #votedLog: JSONObject = { }; // keyed by term number
-    #appendedEntries: JSONObject = { };
-    #leaderNode: IServiceDetail | null = null;
-
-    #currentTerm: number;
-    #log: ILogItem[];
-    #currentIndex = 0;
-
-    constructor(serviceConfig: ProcessOptions)
-    {
-        super(serviceConfig);
-        this.#RAFTState = AppMasterRAFTState.IDLE;
-        this.#currentTerm = 0;
-        this.#votes = { };
-        this.#services = [{
-            host: "stscore.stsmda.org",
-            port: 3006
-        }, {
-            host: "stscore.stsmda.org",
-            port: 3007
-        }, {
-            host: "stscore.stsmda.org",
-            port: 3008
-        }]
-
-        this.#thisService = this.#services[parseInt(process.env.SERVICE_INDEX as string)]; // set self service
-        this.#serverPort = this.#thisService.port;
-
-        //this.#thisService = this.#services[0]; // set self service
-        this.#log = [ ]; // Need to get from storage - volume, redis, etc.
-
-        // Set default config items
-        this.#minElectionTimeout = 1000; //@@ config
-        this.#maxElectionTimeout = 1200; //@@ config
-        this.#startRAFTProtocolTimeout = 2000; //@@ config
-        this.#electionTimeout = crypto.randomInt(this.#minElectionTimeout, this.#maxElectionTimeout);
-        this.#heartBeatTimeout = 750; //@@ config - also must be smaller than electionTimeout
-    }
-
-    #LogInfoMessage(message: any) {
-        this.options.logger.error(message);
-    }
-
-    #GetTLSOptions = (): JSONObject => {
-        return {
-            key: fs.readFileSync(this.options.httpsServerKeyPath),
-            cert: fs.readFileSync(this.options.httpsServerCertificatePath)
-        };
-    }
-
-    #GetLogPrefix = () => {
-        return `${this.#thisService.host}:${this.#thisService.port}: currentTerm: [${this.#currentTerm}] index: [${this.#currentIndex}]: `;
-    }
-
-    #VoteYes = (callback: any, term: number, message: string) => {
-        // Log the fact that I have now voted for this currentTerm
-        this.#votedLog[term] = true;
-        // Update my currentTerm
-        this.#currentTerm = term;
-        // Return my yes vote
-        this.#LogInfoMessage(chalk.green(`${message} - Vote Yes`));
-        callback(null, true); // YES vote
-    }
-
-    #VoteNo = (callback: any, term: number, message: string) => {
-        // Log the fact that I have now voted for this currentTerm
-        this.#votedLog[term] = true;
-        // Return my yes vote
-        this.#LogInfoMessage(chalk.yellow(`${message} - Vote No`));
-        callback(null, false); // NO vote
-    }
-
-    #FindService = (host: string, port: number): IServiceDetail | null => {
-        let retService = null;
-        this.#services.forEach((service, index) => {
-            if (service.host.localeCompare(host) === 0 && service.port === port) {
-                retService = service;
-                return;
-            }
-        });
-        return retService;
-    }
-
-    #CheckMoreRecentTerm = (term: number, lastLogItemTerm: number, lastLogItemIndex: number) => {
-        // Check if the request to vote is newer than my current term
-        let msg = '';
-        let retVal: boolean;
-        if (term > this.#currentTerm) {
-            
-            msg = `${this.#GetLogPrefix()}: canidate term: [${term}] greater than my currentTerm: [${this.#currentTerm}]`;
-            retVal = true;
-        } else {
-            // Check if the request to vote is older than my current term
-            if (term === this.#currentTerm) {
-                if (lastLogItemIndex > this.#currentIndex) {
-                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] greater than my index: [${this.#currentIndex}]`;
-                    retVal = true;
-                } else if (lastLogItemIndex === this.#currentIndex) {
-                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] the same as my index: [${this.#currentIndex}]`;
-                    retVal = true;
-                } else {
-                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] NOT greater than my index: [${this.#currentIndex}]`;
-                    retVal = false;
-                }
-            } else {
-                msg = `${this.#GetLogPrefix()}: canidate term: [${term}] less than my currentTerm: [${this.#currentTerm}]`;
-                retVal = false;
-            }
-        }
-        return {
-            msg,
-            retVal
-        }
-    }
-
-    #SetupJSONRPCServer = async (): Promise<jayson.TlsServer> => {
-        const jaysonServer = new jayson.server();
-
-        jaysonServer.method("appendEntries", async (args: any, callback: any) => {
-            // Just received a message - reset my electionTimeout
-            this.#ResetElectionTimeout();
-
-            // Do something with this new log entry
-
-            // Needs to check if current, otherwise get old entries and duplicate
-
-        });
-
-        jaysonServer.method("vote", async (args: any, callback: any) => {
-            // Just received a message - reset my electionTimeout
-            this.#ResetElectionTimeout();
-
-            const term = args[0];
-            const lastLogItemTerm = args[1];
-            const lastLogItemIndex = args[2];
-
-            // Now determine whether this requestror gets my vote
-
-            // Check already voted
-            if (this.#votedLog[term]) {
-                this.#VoteNo(callback, term, `${this.#GetLogPrefix()}: Already voted - vote NO`)
-                return;
-            }
-
-            // Check if the request to vote is newer than my current term
-            const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
-            if (retVal.retVal === true) {
-                this.#VoteYes(callback, term, retVal.msg);
-            } else {
-                this.#VoteNo(callback, term, retVal.msg);
-            }
-        });
-
-        jaysonServer.method("appendEntry", async (args: any, callback: any) => {
-            // Just received a message - reset my electionTimeout
-            this.#ResetElectionTimeout();
-
-            const host = args[0];
-            const port = args[1];
-            const term = args[2];
-            const lastLogItemTerm = args[3];
-            const lastLogItemIndex = args[4];
-
-            if (this.#RAFTState.localeCompare(AppMasterRAFTState.CANDIDATE) === 0) {
-                const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
-                if (retVal.retVal === true) {
-                    this.#LogInfoMessage(`${retVal.msg}`);
-                    this.#LogInfoMessage(chalk.magenta(`${this.#GetLogPrefix()}: New leader detected, Host: [${host}], Port: [${port}], while in CANDIDATE mode, reverting to FOLLOWER.`));
-                    // This sending node is the leader so revert to follower
-                    this.#currentTerm = term;
-                    this.#ChangeState(AppMasterRAFTState.FOLLOWER);
-                } else {
-                    this.#LogInfoMessage(`${retVal.msg}`);
-                    this.#LogInfoMessage(chalk.magenta(`${this.#GetLogPrefix()}: appendEntry called but log is older than my current state, Host: [${host}], Port: [${port}].`));
-                    // Reject this RPC and continue in the CANDIDATE state
-                    callback(null, false);
-                    return;
-                }
-            } else if (this.#RAFTState.localeCompare(AppMasterRAFTState.LEADER) === 0) {
-                this.#LogInfoMessage(chalk.red(`${this.#GetLogPrefix()}: appendEntry called while I am the current leader, rejecting RPC call.`));
-                // Reject this RPC and continue in the LEADER state
-                callback(null, false);
-                return;
-            } else {
-                // FOLLOWER or IDLE
-                if (this.#RAFTState.localeCompare(AppMasterRAFTState.IDLE) === 0) {
-                    this.#ChangeState(AppMasterRAFTState.FOLLOWER);
-                }
-                const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
-                if (retVal.retVal === true) {
-                    if (term > this.#currentTerm) {
-                        this.#LogInfoMessage(chalk.cyan(`${retVal.msg}`));
-                        this.#currentTerm = term;
-                    }
-                } else {
-                    this.#LogInfoMessage(`${retVal.msg}`);
-                    this.#LogInfoMessage(chalk.magenta(`${this.#GetLogPrefix()}: appendEntry called but log is older than my current state, Host: [${host}], Port: [${port}].`));
-                    // Reject this RPC and continue in the CANDIDATE state
-                    callback(null, false);
-                    return;
-                }
-            }
-
-            if (this.#leaderNode) {
-                const sentLeaderService = this.#FindService(host, port);
-                if (sentLeaderService) {
-                    if (sentLeaderService.host.localeCompare(this.#leaderNode.host) !== 0 || sentLeaderService.port !== this.#leaderNode.port) {
-                        this.#leaderNode = sentLeaderService;
-                        this.#LogInfoMessage(chalk.white(`${this.#GetLogPrefix()}: Leader Service: Host:[${this.#leaderNode.host}] Port: [${this.#leaderNode.port}]`));
-                    }
-                }
-            } else {
-                this.#leaderNode = this.#FindService(host, port);
-                if (this.#leaderNode) {
-                    this.#LogInfoMessage(chalk.white(`${this.#GetLogPrefix()}: Leader Service: Host:[${this.#leaderNode.host}] Port: [${this.#leaderNode.port}]`));
-                }
-            }
-
-            // Now update the log if only more recent
-            callback(null, true);
-        });
-
-        jaysonServer.method("ping", async (args: any, callback: any) => {
-            // do something, do nothing
-            this.#LogInfoMessage('server got ping');
-            callback();
-        });
-
-        const server = jaysonServer.tls(this.#GetTLSOptions());
-        //(server as tls.Server).on('secureConnection', this.#SetupRPCServer);
-        //server.listen(this.options.listenPort, () => { // 'stscore.stsmda.org'
-        server.listen(this.#serverPort, () => { // 'stscore.stsmda.org'
-            this.#LogInfoMessage('JSON RPC 2.0 Server is running on port ' + this.options.listenPort + '.');
-        }).on('listening', () =>
-        {
-            this.LogInfoMessage(`JSON RPC 2.0 live on ${this.options.endpoint}:${this.options.listenPort}${this.options.apiRoot}`);
-        });
-        return server;
-    }
-
-    #RequestVote = (service: IServiceDetail) => {
-        const options: jayson.TcpClientOptions = {
-            port: service.port,
-            host: service.host
-        };
-        
-        const client = jayson.client.tls(options);
-
-        let lastLogItem: ILogItem;
-        if (this.#log.length > 0) {
-            lastLogItem = this.#log[this.#log.length-1];
-        } else {
-            lastLogItem = {
-                term: 0,
-                index: 0,
-                data: null
-            }
-        }
-
-        client.request('vote', [ this.#currentTerm, lastLogItem.term, lastLogItem.index ], (error: any, response: any) => {
-            if (error) { 
-                //this.#LogInfoMessage(chalk.red(`vote attempt fail: [${JSON.stringify(options)}: ${error}]`));
-            } else {
-                // Log the result
-                if (response.result === true) {
-                    this.#votes[this.#GetServiceKey(service)] = 1;
-                }
-                this.#CheckMajority();
-            }
-        });
-    }
-
-    #AppendEntry = (service: IServiceDetail) => {
-        const options: jayson.TcpClientOptions = {
-            port: service.port,
-            host: service.host
-        };
-        
-        const client = jayson.client.tls(options);
-
-        let lastLogItem: ILogItem;
-        if (this.#log.length > 0) {
-            lastLogItem = this.#log[this.#log.length-1];
-        } else {
-            lastLogItem = {
-                term: 0,
-                index: 0,
-                data: null
-            }
-        }
-
-        client.request('appendEntry', [ this.#thisService.host, this.#thisService.port, this.#currentTerm, lastLogItem.term, lastLogItem.index ], (error: any, response: any) => {
-            if (error) { 
-                //this.#LogInfoMessage(chalk.red(`appendEntry attempt fail: [${JSON.stringify(options)}: ${error}]`));
-            } else {
-                this.#appendedEntries[this.#GetServiceKey(service)] = 1;
-                // Log the result
-                /*
-                if (response.result === true) {
-                    this.#votes[this.#GetServiceKey(service)] = 1;
-                }
-                */
-            }
-        });
-    }
-
-    #AppendEntries = () => {
-        // Reset all appendEntries for this iteration
-        this.#appendedEntries = { };
-
-        // Reset all votes
-        this.#services.forEach((service, index) => {
-            this.#appendedEntries[this.#GetServiceKey(service)] = 0;
-        });
-        
-        // Request vote for all other services - include my logs current item
-        this.#services.forEach((service, index) => {
-            if (service.host.localeCompare(this.#thisService.host) === 0 && service.port === this.#thisService.port) {
-                // Myself
-                this.#ResetElectionTimeout();
-                this.#appendedEntries[this.#GetSelfServiceKey()] = 1;
-            } else {
-                // Send vote request
-                this.#AppendEntry(service);
-            }
-        });
-    }
-
-    #GetServiceKey = (service: IServiceDetail) => {
-        return `${service.host}:${service.port}`;
-    }
-
-    #GetSelfServiceKey = () => {
-        return this.#GetServiceKey(this.#thisService);
-    }
-
-    // set interval ???
-    #StartAppendEntries = () => {
-        this.#AppendEntries();
-        setTimeout(() => {
-            // Now check if we still have majority
-            const majority: boolean = this.#CheckMajorityWithCollection(this.#appendedEntries);
-            if (majority) {
-                this.#StartAppendEntries();
-            } else {
-                this.#LogInfoMessage(chalk.red(`${this.#GetLogPrefix()}: Not enough response for valid cluster state.`));
-            }
-        }, this.#heartBeatTimeout).unref();
-    }
-
-    #Promote = () => {
-        this.#ChangeState(AppMasterRAFTState.LEADER);
-        this.#LogInfoMessage(chalk.green(`${this.#GetLogPrefix()}: New LEADER is elected: [${this.#GetSelfServiceKey()}]`));
-        // As leader, now start the regular pings
-        this.#StartAppendEntries();
-    }
-
-    #CheckMajority = () => {
-        if (this.#CheckMajorityWithCollection(this.#votes)) {
-            this.#Promote();
-        }
-    }
-
-    #CheckMajorityWithCollection = (col: JSONObject): boolean => {
-        let votes = 0;
-        for (const [key, val] of Object.entries(col)) {
-            votes += val;
-        }
-        const requiredVotes = Math.floor(this.#services.length / 2) + 1;
-
-        if (requiredVotes !== this.#lastRequiredVotes || votes !== this.#lastVotes) {
-            this.#LogInfoMessage(chalk.cyan(`${this.#GetLogPrefix()}votes: [${votes} need: [${requiredVotes}]`));
-            this.#lastRequiredVotes = requiredVotes;
-            this.#lastVotes = votes;
-        }
-
-        return votes >= requiredVotes;
-    }
-
-    #StartCanidateVote = () => {
-
-        // Get the election timeout going again
-        this.#StartElectionTimeout();
-
-        this.#LogInfoMessage(chalk.yellow(`${this.#GetLogPrefix()}: Start CANDIDATE vote`));
-        // Start in the CANDIDATE state
-        this.#ChangeState(AppMasterRAFTState.CANDIDATE);
-        // Reset votes
-        this.#votes = { };
-
-        // Reset all votes from all service instances
-        this.#services.forEach((service, index) => {
-            this.#votes[this.#GetServiceKey(service)] = 0;
-        });
-
-        // Start a new term
-        this.#currentTerm++;
-
-        // Request vote for all other services - include my logs current item
-        this.#services.forEach((service, index) => {
-            if (service.host.localeCompare(this.#thisService.host) === 0 && service.port === this.#thisService.port) {
-                // Log the fact that I have now voted for this term
-                this.#votedLog[this.#currentTerm] = true;
-                // Log the result - Vote for myself
-                this.#votes[this.#GetSelfServiceKey()] = 1;
-                // Return my yes vote
-                this.#LogInfoMessage(chalk.green(`${this.#GetLogPrefix()}: SELF VOTE: canidate currentTerm: [${this.#currentTerm}], vote YES`));
-                this.#CheckMajority();
-            } else {
-                // Send vote request
-                this.#RequestVote(service);
-            }
-        });
-    }
-
-    #ResetElectionTimeout = () => {
-        if (this.#canidateTimeout) {
-            clearTimeout(this.#canidateTimeout);
-        }
-        this.#StartElectionTimeout();
-    }
-
-    #StartElectionTimeout = () => {
-        if (this.#canidateTimeout) {
-            clearTimeout(this.#canidateTimeout);
-        }
-        this.#canidateTimeout = setTimeout(() => {
-            this.#StartCanidateVote();
-        }, this.#electionTimeout).unref();
-    }
-
-    #StartRAFTService = async () => {
-        this.LogInfoMessage(chalk.green(`---[ RAFT Protocol Starting ]---`));
-        await this.#SetupJSONRPCServer();
-        this.#StartElectionTimeout();
-    }
-
-    #ChangeState(newState: AppMasterRAFTState) {
-        this.LogInfoMessage(chalk.magenta(`${this.#GetLogPrefix()}: State Change, From: [${this.#RAFTState.toString()}] --> To: [${newState.toString()}]`));
-        this.#RAFTState = newState;
-    }
-
-    MasterStarted(): void {
-        setTimeout(() => {
-            this.#StartRAFTService();
-        }, this.#startRAFTProtocolTimeout).unref();
-    }
-}