@nsshunt/stsappframework 3.0.104 → 3.0.105

package/src_working/tcpserver/appmaster.ts

@@ -1,544 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import fs from "fs"
-import crypto from 'crypto';
-import jayson from 'jayson'
-import net from 'node:net'
-import tls from 'node:tls'
-
-import { JSONObject } from '@nsshunt/stsutils';
-import { MasterProcessBase } from './../masterprocessbase'
-import { ProcessOptions } from './../processoptions'
-
-// Colour codes
-// ---------------------------------
-// Red - Error
-// Green - Vote YES
-// Yellow - Vote No
-// White - standard logging message
-// Magenta - node state change
-// Cyan - node term change
-
-export enum AppMasterRAFTState {
-    IDLE = "IDLE",
-    FOLLOWER = "FOLLOWER",
-    CANDIDATE = "CANDIDATE",
-    LEADER = "LEADER"
-}
-
-export interface IServiceDetail {
-    host: string
-    port: number
-}
-
-export interface ILogItem {
-    term: number
-    index: number
-    data: JSONObject | null
-}
-
-export class AppMaster extends MasterProcessBase
-{
-    #lastRequiredVotes = -1; // used for logging only
-    #lastVotes = -1; // used for logging only
-    #RAFTState: AppMasterRAFTState;
-    #minElectionTimeout: number;
-    #maxElectionTimeout: number;
-    #electionTimeout: number;
-    #heartBeatTimeout: number;
-    #startRAFTProtocolTimeout: number;
-    #votes: JSONObject; // keyed by service key (host:port)
-    #services: IServiceDetail[];
-    #thisService: IServiceDetail;
-    #canidateTimeout: NodeJS.Timeout | null = null;
-    #appendEntriesTimeout: NodeJS.Timeout | null = null;
-    #voteRegister = 0;
-    #server?: jayson.TlsServer;
-    #serverPort: number;
-    #votedLog: JSONObject = { }; // keyed by term number
-    #appendedEntries: JSONObject = { };
-    #leaderNode: IServiceDetail | null = null;
-
-    #currentTerm: number;
-    #log: ILogItem[];
-    #currentIndex = 0;
-
-    constructor(serviceConfig: ProcessOptions)
-    {
-        super(serviceConfig);
-        this.#RAFTState = AppMasterRAFTState.IDLE;
-        this.#currentTerm = 0;
-        this.#votes = { };
-        this.#services = [{
-            host: "stscore.stsmda.org",
-            port: 3006
-        }, {
-            host: "stscore.stsmda.org",
-            port: 3007
-        }, {
-            host: "stscore.stsmda.org",
-            port: 3008
-        }]
-
-        this.#thisService = this.#services[parseInt(process.env.SERVICE_INDEX as string)]; // set self service
-        this.#serverPort = this.#thisService.port;
-
-        //this.#thisService = this.#services[0]; // set self service
-        this.#log = [ ]; // Need to get from storage - volume, redis, etc.
-
-        // Set default config items
-        this.#minElectionTimeout = 1000; //@@ config
-        this.#maxElectionTimeout = 1200; //@@ config
-        this.#startRAFTProtocolTimeout = 2000; //@@ config
-        this.#electionTimeout = crypto.randomInt(this.#minElectionTimeout, this.#maxElectionTimeout);
-        this.#heartBeatTimeout = 750; //@@ config - also must be smaller than electionTimeout
-    }
-
-    #GetTLSOptions = (): JSONObject => {
-        return {
-            key: fs.readFileSync(this.options.httpsServerKeyPath),
-            cert: fs.readFileSync(this.options.httpsServerCertificatePath)
-        };
-    }
-
-    #SetupRPCServer = async (socket: net.Socket): Promise<void> => {
-        console.log('CONNECTED: ' + socket.remoteAddress + ':' + socket.remotePort + ' ' + process.pid);
-
-        socket.on('close', function(data: any) {
-            console.log('CLOSED: ' + socket.remoteAddress + ' ' + socket.remotePort + ' ' + process.pid);
-        });
-
-        socket.on('data', function(data: any) {
-            console.log('DATA ' + socket.remoteAddress + ': ' + data);
-        });
-    }
-
-    #GetLogPrefix = () => {
-        return `${this.#thisService.host}:${this.#thisService.port}: currentTerm: [${this.#currentTerm}] index: [${this.#currentIndex}]: `;
-    }
-
-    #VoteYes = (callback: any, term: number, message: string) => {
-        // Log the fact that I have now voted for this currentTerm
-        this.#votedLog[term] = true;
-        // Update my currentTerm
-        this.#currentTerm = term;
-        // Return my yes vote
-        console.log(`${message} - Vote Yes`.green)
-        callback(null, true); // YES vote
-    }
-
-    #VoteNo = (callback: any, term: number, message: string) => {
-        // Log the fact that I have now voted for this currentTerm
-        this.#votedLog[term] = true;
-        // Return my yes vote
-        console.log(`${message} - Vote No`.yellow)
-        callback(null, false); // NO vote
-    }
-
-    #FindService = (host: string, port: number): IServiceDetail | null => {
-        let retService = null;
-        this.#services.forEach((service, index) => {
-            if (service.host.localeCompare(host) === 0 && service.port === port) {
-                retService = service;
-                return;
-            }
-        });
-        return retService;
-    }
-
-    #CheckMoreRecentTerm = (term: number, lastLogItemTerm: number, lastLogItemIndex: number) => {
-        // Check if the request to vote is newer than my current term
-        let msg = '';
-        let retVal: boolean;
-        if (term > this.#currentTerm) {
-            
-            msg = `${this.#GetLogPrefix()}: canidate term: [${term}] greater than my currentTerm: [${this.#currentTerm}]`;
-            retVal = true;
-        } else {
-            // Check if the request to vote is older than my current term
-            if (term === this.#currentTerm) {
-                if (lastLogItemIndex > this.#currentIndex) {
-                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] greater than my index: [${this.#currentIndex}]`;
-                    retVal = true;
-                } else if (lastLogItemIndex === this.#currentIndex) {
-                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] the same as my index: [${this.#currentIndex}]`;
-                    retVal = true;
-                } else {
-                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] NOT greater than my index: [${this.#currentIndex}]`;
-                    retVal = false;
-                }
-            } else {
-                msg = `${this.#GetLogPrefix()}: canidate term: [${term}] less than my currentTerm: [${this.#currentTerm}]`;
-                retVal = false;
-            }
-        }
-        return {
-            msg,
-            retVal
-        }
-    }
-
-    #SetupJSONRPCServer = async (): Promise<jayson.TlsServer> => {
-        const jaysonServer = new jayson.server();
-
-        jaysonServer.method("appendEntries", async (args: any, callback: any) => {
-            // Just received a message - reset my electionTimeout
-            this.#ResetElectionTimeout();
-
-            // Do something with this new log entry
-
-            // Needs to check if current, otherwise get old entries and duplicate
-
-        });
-
-        jaysonServer.method("vote", async (args: any, callback: any) => {
-            // Just received a message - reset my electionTimeout
-            this.#ResetElectionTimeout();
-
-            const term = args[0];
-            const lastLogItemTerm = args[1];
-            const lastLogItemIndex = args[2];
-
-            // Now determine whether this requestror gets my vote
-
-            // Check already voted
-            if (this.#votedLog[term]) {
-                this.#VoteNo(callback, term, `${this.#GetLogPrefix()}: Already voted - vote NO`)
-                return;
-            }
-
-            // Check if the request to vote is newer than my current term
-            const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
-            if (retVal.retVal === true) {
-                this.#VoteYes(callback, term, retVal.msg);
-            } else {
-                this.#VoteNo(callback, term, retVal.msg);
-            }
-        });
-
-        jaysonServer.method("appendEntry", async (args: any, callback: any) => {
-            // Just received a message - reset my electionTimeout
-            this.#ResetElectionTimeout();
-
-            const host = args[0];
-            const port = args[1];
-            const term = args[2];
-            const lastLogItemTerm = args[3];
-            const lastLogItemIndex = args[4];
-
-            if (this.#RAFTState.localeCompare(AppMasterRAFTState.CANDIDATE) === 0) {
-                const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
-                if (retVal.retVal === true) {
-                    console.log(`${retVal.msg}`);
-                    console.log(`${this.#GetLogPrefix()}: New leader detected, Host: [${host}], Port: [${port}], while in CANDIDATE mode, reverting to FOLLOWER.`.magenta);
-                    // This sending node is the leader so revert to follower
-                    this.#currentTerm = term;
-                    this.#ChangeState(AppMasterRAFTState.FOLLOWER);
-                } else {
-                    console.log(`${retVal.msg}`);
-                    console.log(`${this.#GetLogPrefix()}: appendEntry called but log is older than my current state, Host: [${host}], Port: [${port}].`.magenta);
-                    // Reject this RPC and continue in the CANDIDATE state
-                    callback(null, false);
-                    return;
-                }
-            } else if (this.#RAFTState.localeCompare(AppMasterRAFTState.LEADER) === 0) {
-                console.log(`${this.#GetLogPrefix()}: appendEntry called while I am the current leader, rejecting RPC call.`.red);
-                // Reject this RPC and continue in the LEADER state
-                callback(null, false);
-                return;
-            } else {
-                // FOLLOWER or IDLE
-                if (this.#RAFTState.localeCompare(AppMasterRAFTState.IDLE) === 0) {
-                    this.#ChangeState(AppMasterRAFTState.FOLLOWER);
-                }
-                const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
-                if (retVal.retVal === true) {
-                    if (term > this.#currentTerm) {
-                        console.log(`${retVal.msg}`.cyan);
-                        this.#currentTerm = term;
-                    }
-                } else {
-                    console.log(`${retVal.msg}`);
-                    console.log(`${this.#GetLogPrefix()}: appendEntry called but log is older than my current state, Host: [${host}], Port: [${port}].`.magenta);
-                    // Reject this RPC and continue in the CANDIDATE state
-                    callback(null, false);
-                    return;
-                }
-            }
-
-            if (this.#leaderNode) {
-                const sentLeaderService = this.#FindService(host, port);
-                if (sentLeaderService) {
-                    if (sentLeaderService.host.localeCompare(this.#leaderNode.host) !== 0 || sentLeaderService.port !== this.#leaderNode.port) {
-                        this.#leaderNode = sentLeaderService;
-                        console.log(`${this.#GetLogPrefix()}: Leader Service: Host:[${this.#leaderNode.host}] Port: [${this.#leaderNode.port}]`.white);
-                    }
-                }
-            } else {
-                this.#leaderNode = this.#FindService(host, port);
-                if (this.#leaderNode) {
-                    console.log(`${this.#GetLogPrefix()}: Leader Service: Host:[${this.#leaderNode.host}] Port: [${this.#leaderNode.port}]`.white);
-                }
-            }
-
-            // Now update the log if only more recent
-            callback(null, true);
-        });
-
-        jaysonServer.method("ping", async function(args: any, callback: any) {
-            // do something, do nothing
-            console.log('server got ping');
-            callback();
-        });
-
-        const server = jaysonServer.tls(this.#GetTLSOptions());
-        //(server as tls.Server).on('secureConnection', this.#SetupRPCServer);
-        //server.listen(this.options.listenPort, () => { // 'stscore.stsmda.org'
-        server.listen(this.#serverPort, () => { // 'stscore.stsmda.org'
-            console.log('JSON RPC 2.0 Server is running on port ' + this.options.listenPort + '.');
-        }).on('listening', () =>
-        {
-            this.LogEx(`JSON RPC 2.0 live on ${this.options.endpoint}:${this.options.listenPort}${this.options.apiRoot}`);
-        });
-        return server;
-    }
-
-    #RequestVote = (service: IServiceDetail) => {
-        const options: jayson.TcpClientOptions = {
-            port: service.port,
-            host: service.host
-        };
-        
-        const client = jayson.client.tls(options);
-        /*
-        client.on('tcp socket', (socket: tls.TLSSocket) => {
-            socket.setTimeout(200); //@@ config
-            socket.on('timeout', () => {
-                console.log('socket timeout');
-                socket.end();
-            });
-        });
-        client.on('tcp error', (error: any) => {
-            console.log(`ERROR: [${error}]`.red);
-        });
-        */
-
-        let lastLogItem: ILogItem;
-        if (this.#log.length > 0) {
-            lastLogItem = this.#log[this.#log.length-1];
-        } else {
-            lastLogItem = {
-                term: 0,
-                index: 0,
-                data: null
-            }
-        }
-
-        client.request('vote', [ this.#currentTerm, lastLogItem.term, lastLogItem.index ], (error: any, response: any) => {
-            if (error) { 
-                //console.log(`vote attempt fail: [${JSON.stringify(options)}: ${error}]`.red);
-            } else {
-                // Log the result
-                if (response.result === true) {
-                    this.#votes[this.#GetServiceKey(service)] = 1;
-                }
-                this.#CheckMajority();
-            }
-        });
-    }
-
-    #AppendEntry = (service: IServiceDetail) => {
-        const options: jayson.TcpClientOptions = {
-            port: service.port,
-            host: service.host
-        };
-        
-        const client = jayson.client.tls(options);
-        /*
-        client.on('tcp socket', (socket: tls.TLSSocket) => {
-            socket.setTimeout(200); //@@ config
-            socket.on('timeout', () => {
-                console.log('socket timeout');
-                socket.end();
-            });
-        });
-        client.on('tcp error', (error: any) => {
-            console.log(`ERROR: [${error}]`.red);
-        });
-        */
-
-        let lastLogItem: ILogItem;
-        if (this.#log.length > 0) {
-            lastLogItem = this.#log[this.#log.length-1];
-        } else {
-            lastLogItem = {
-                term: 0,
-                index: 0,
-                data: null
-            }
-        }
-
-        client.request('appendEntry', [ this.#thisService.host, this.#thisService.port, this.#currentTerm, lastLogItem.term, lastLogItem.index ], (error: any, response: any) => {
-            if (error) { 
-                //console.log(`appendEntry attempt fail: [${JSON.stringify(options)}: ${error}]`.red);
-            } else {
-                this.#appendedEntries[this.#GetServiceKey(service)] = 1;
-                // Log the result
-                /*
-                if (response.result === true) {
-                    this.#votes[this.#GetServiceKey(service)] = 1;
-                }
-                */
-            }
-        });
-    }
-
-    #AppendEntries = () => {
-        // Reset all appendEntries for this iteration
-        this.#appendedEntries = { };
-
-        // Reset all votes
-        this.#services.forEach((service, index) => {
-            this.#appendedEntries[this.#GetServiceKey(service)] = 0;
-        });
-        
-        // Request vote for all other services - include my logs current item
-        this.#services.forEach((service, index) => {
-            if (service.host.localeCompare(this.#thisService.host) === 0 && service.port === this.#thisService.port) {
-                // Myself
-                this.#ResetElectionTimeout();
-                this.#appendedEntries[this.#GetSelfServiceKey()] = 1;
-            } else {
-                // Send vote request
-                this.#AppendEntry(service);
-            }
-        });
-    }
-
-    #GetServiceKey = (service: IServiceDetail) => {
-        return `${service.host}:${service.port}`;
-    }
-
-    #GetSelfServiceKey = () => {
-        return this.#GetServiceKey(this.#thisService);
-    }
-
-    // set interval ???
-    #StartAppendEntries = () => {
-        this.#AppendEntries();
-        this.#appendEntriesTimeout = setTimeout(() => {
-            // Now check if we still have majority
-            const majority: boolean = this.#CheckMajorityWithCollection(this.#appendedEntries);
-            if (majority) {
-                this.#StartAppendEntries();
-            } else {
-                console.log(`${this.#GetLogPrefix()}: Not enough response for valid cluster state.`.red);
-            }
-        }, this.#heartBeatTimeout);
-    }
-
-    #Promote = () => {
-        this.#ChangeState(AppMasterRAFTState.LEADER);
-        console.log(`${this.#GetLogPrefix()}: New LEADER is elected: [${this.#GetSelfServiceKey()}]`.green);
-        // As leader, now start the regular pings
-        this.#StartAppendEntries();
-    }
-
-    #CheckMajority = () => {
-        if (this.#CheckMajorityWithCollection(this.#votes)) {
-            this.#Promote();
-        }
-    }
-
-    #CheckMajorityWithCollection = (col: JSONObject): boolean => {
-        let votes = 0;
-        for (const [key, val] of Object.entries(col)) {
-            votes += val;
-        }
-        const requiredVotes = Math.floor(this.#services.length / 2) + 1;
-
-        if (requiredVotes !== this.#lastRequiredVotes || votes !== this.#lastVotes) {
-            console.log(`${this.#GetLogPrefix()}votes: [${votes} need: [${requiredVotes}]`.cyan);
-            this.#lastRequiredVotes = requiredVotes;
-            this.#lastVotes = votes;
-        }
-
-        return votes >= requiredVotes;
-    }
-
-    #StartCanidateVote = () => {
-
-        // Get the election timeout going again
-        this.#StartElectionTimeout();
-
-        console.log(`${this.#GetLogPrefix()}: Start CANDIDATE vote`.yellow);
-        // Start in the CANDIDATE state
-        this.#ChangeState(AppMasterRAFTState.CANDIDATE);
-        // Reset votes
-        this.#votes = { };
-
-        // Reset all votes from all service instances
-        this.#services.forEach((service, index) => {
-            this.#votes[this.#GetServiceKey(service)] = 0;
-        });
-
-        // Start a new term
-        this.#currentTerm++;
-
-        // Request vote for all other services - include my logs current item
-        this.#services.forEach((service, index) => {
-            if (service.host.localeCompare(this.#thisService.host) === 0 && service.port === this.#thisService.port) {
-                // Log the fact that I have now voted for this term
-                this.#votedLog[this.#currentTerm] = true;
-                // Log the result - Vote for myself
-                this.#votes[this.#GetSelfServiceKey()] = 1;
-                // Return my yes vote
-                console.log(`${this.#GetLogPrefix()}: SELF VOTE: canidate currentTerm: [${this.#currentTerm}], vote YES`.green)
-                this.#CheckMajority();
-            } else {
-                // Send vote request
-                this.#RequestVote(service);
-            }
-        });
-    }
-
-    #ReceiveVoteRequest = () => {
-        // Check if the request term and meta-data is more recent than me
-
-        // If request more recent ...
-        this.#voteRegister = 1;
-        // Send vote back to requestor
-    }
-
-    #ResetElectionTimeout = () => {
-        if (this.#canidateTimeout) {
-            clearTimeout(this.#canidateTimeout);
-        }
-        this.#StartElectionTimeout();
-    }
-
-    #StartElectionTimeout = () => {
-        if (this.#canidateTimeout) {
-            clearTimeout(this.#canidateTimeout);
-        }
-        this.#canidateTimeout = setTimeout(() => {
-            this.#StartCanidateVote();
-        }, this.#electionTimeout).unref();
-    }
-
-    #StartRAFTService = async () => {
-        this.LogEx(`---[ RAFT Protocol Starting ]---`.green);
-        this.#server = await this.#SetupJSONRPCServer();
-        this.#StartElectionTimeout();
-    }
-
-    #ChangeState(newState: AppMasterRAFTState) {
-        this.LogEx(`${this.#GetLogPrefix()}: State Change, From: [${this.#RAFTState.toString()}] --> To: [${newState.toString()}]`.magenta);
-        this.#RAFTState = newState;
-    }
-
-    MasterStarted(): void {
-        setTimeout(() => {
-            this.#StartRAFTService();
-        }, this.#startRAFTProtocolTimeout).unref();
-    }
-}

package/src_working/validation/errors.ts

@@ -1,6 +0,0 @@
-export const AppFrameworkErrorCode = Object.freeze({
-    APPFRAMEWORK_MISSING_PERMISSION: {
-        code: 'STS_AF_0001',
-        description: 'STS_AF_0001: Missing Permission(s).'
-    },
-});

package/src_working/webworkertesting/app.ts

@@ -1,49 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import cluster from 'cluster';
-import { Worker, MessageChannel } from 'worker_threads';
-
-if (cluster.isPrimary) {
-    console.log(`Primary ${process.pid} is running`);
-  
-    // Fork workers.
-    cluster.fork();
-  
-    cluster.on('exit', (worker, code, signal) => {
-        console.log(`worker ${worker.process.pid} died`);
-    });
-
-    const fileName ='./dist/webworkertesting/worker.js';
-
-    const publishCollectorWebWorker = new Worker(fileName);
-    publishCollectorWebWorker.unref();
-
-    publishCollectorWebWorker.postMessage({cmd: 'text', message: 'Hello World'});
-
-    publishCollectorWebWorker.on('message', (data) => {
-        console.log(`cluster.primary (${process.pid}): message from worker = [${JSON.stringify(data)}]`);
-    });
-
-    const { port1, port2 } = new MessageChannel();
-
-    publishCollectorWebWorker.postMessage({cmd: 'portmessage', port: port2}, [ port2 ]);
-    port1.postMessage('sending to port1');
-    port1.on('message', (data) => {
-        console.log(`cluster.primary (${process.pid}): message from message port = [${data}]`);
-    });
-
-
-} else {
-    console.log(`Worker ${process.pid} started`);
-
-    const fileName ='./dist/webworkertesting/worker.js';
-
-    const clusterWorker = new Worker(fileName);
-    clusterWorker.unref();
-
-    clusterWorker.postMessage({cmd: 'text', message: 'Hello World'});
-
-    clusterWorker.on('message', (data: any) => {
-        console.log(`cluster.worker (${process.pid}): message from worker = [${JSON.stringify(data)}]`);
-    });
-
-}

package/src_working/webworkertesting/worker.ts

@@ -1,24 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import { matchesProperty } from 'lodash';
-import {
-    Worker,
-    isMainThread,
-    parentPort,
-    MessagePort
-} from 'worker_threads';
-
-let messagePort: MessagePort;
-
-parentPort?.on('message', (data: any) => {
-    if (data.cmd.localeCompare('portmessage') === 0) {
-        messagePort = data.port;
-        messagePort.on('message', (data) => {
-            console.log(`webWorker (${process.pid}): message from passed message port = [${data}]`);
-            messagePort.postMessage(data);
-        });
-    } else {
-        console.log(`webWorker (${process.pid}): message from parent = [${JSON.stringify(data)}]`);
-        parentPort?.postMessage(data);
-    }
-});
-

package/src_working/workerprocessbase.test.ts

@@ -1,47 +0,0 @@
-import { beforeAll, afterAll, test, describe, expect } from 'vitest';
-
-describe("Test Latency Controller", () =>
-{
-    beforeAll(async () => 
-    {
-        /*
-        process.env.AS_ENDPOINT = testHelper.authHost;
-        process.env.AS_HOST_PORT = testHelper.authPort;
-        process.env.AS_PORT = testHelper.authPort;
-
-        $ResetOptions();
-        //goptions = $Options()
-
-        //goptions.asendpoint = testHelper.authHost;
-        //goptions.asport = testHelper.authPort;
-
-        const appOptions = ServiceConfigOptions(true);
-        appOptions.processExitOnTerminate = false;
-        spb = new SingleProcessBase(appOptions)
-        await spb.SetupServer();
-        */
-    }, 60000);
-    
-    afterAll(async () =>
-    {
-        /*
-        await Sleep(500);
-        await spb?.TerminateApplication();
-        await Sleep(1000);
-        spb = null;
-
-        await testHelper.StopAuthService();
-        await testHelper.StopDatabase();
-        await testHelper.StopNetwork();
-
-        await Sleep(2000);
-        */
-    }, 15000);
-
-    test('Testing Module', async () => 
-    {
-        expect.assertions(1);
-        expect(1).toEqual(1);
-    });
-});
-