@nsshunt/stsappframework 3.0.94 → 3.0.96

package/src_working/tcpserver/appmaster.ts

@@ -0,0 +1,544 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+import fs from "fs"
+import crypto from 'crypto';
+import jayson from 'jayson'
+import net from 'node:net'
+import tls from 'node:tls'
+
+import { JSONObject } from '@nsshunt/stsutils';
+import { MasterProcessBase } from './../masterprocessbase'
+import { ProcessOptions } from './../processoptions'
+
+// Colour codes
+// ---------------------------------
+// Red - Error
+// Green - Vote YES
+// Yellow - Vote No
+// White - standard logging message
+// Magenta - node state change
+// Cyan - node term change
+
+export enum AppMasterRAFTState {
+    IDLE = "IDLE",
+    FOLLOWER = "FOLLOWER",
+    CANDIDATE = "CANDIDATE",
+    LEADER = "LEADER"
+}
+
+export interface IServiceDetail {
+    host: string
+    port: number
+}
+
+export interface ILogItem {
+    term: number
+    index: number
+    data: JSONObject | null
+}
+
+export class AppMaster extends MasterProcessBase
+{
+    #lastRequiredVotes = -1; // used for logging only
+    #lastVotes = -1; // used for logging only
+    #RAFTState: AppMasterRAFTState;
+    #minElectionTimeout: number;
+    #maxElectionTimeout: number;
+    #electionTimeout: number;
+    #heartBeatTimeout: number;
+    #startRAFTProtocolTimeout: number;
+    #votes: JSONObject; // keyed by service key (host:port)
+    #services: IServiceDetail[];
+    #thisService: IServiceDetail;
+    #canidateTimeout: NodeJS.Timeout | null = null;
+    #appendEntriesTimeout: NodeJS.Timeout | null = null;
+    #voteRegister = 0;
+    #server?: jayson.TlsServer;
+    #serverPort: number;
+    #votedLog: JSONObject = { }; // keyed by term number
+    #appendedEntries: JSONObject = { };
+    #leaderNode: IServiceDetail | null = null;
+
+    #currentTerm: number;
+    #log: ILogItem[];
+    #currentIndex = 0;
+
+    constructor(serviceConfig: ProcessOptions)
+    {
+        super(serviceConfig);
+        this.#RAFTState = AppMasterRAFTState.IDLE;
+        this.#currentTerm = 0;
+        this.#votes = { };
+        this.#services = [{
+            host: "stscore.stsmda.org",
+            port: 3006
+        }, {
+            host: "stscore.stsmda.org",
+            port: 3007
+        }, {
+            host: "stscore.stsmda.org",
+            port: 3008
+        }]
+
+        this.#thisService = this.#services[parseInt(process.env.SERVICE_INDEX as string)]; // set self service
+        this.#serverPort = this.#thisService.port;
+
+        //this.#thisService = this.#services[0]; // set self service
+        this.#log = [ ]; // Need to get from storage - volume, redis, etc.
+
+        // Set default config items
+        this.#minElectionTimeout = 1000; //@@ config
+        this.#maxElectionTimeout = 1200; //@@ config
+        this.#startRAFTProtocolTimeout = 2000; //@@ config
+        this.#electionTimeout = crypto.randomInt(this.#minElectionTimeout, this.#maxElectionTimeout);
+        this.#heartBeatTimeout = 750; //@@ config - also must be smaller than electionTimeout
+    }
+
+    #GetTLSOptions = (): JSONObject => {
+        return {
+            key: fs.readFileSync(this.options.httpsServerKeyPath),
+            cert: fs.readFileSync(this.options.httpsServerCertificatePath)
+        };
+    }
+
+    #SetupRPCServer = async (socket: net.Socket): Promise<void> => {
+        console.log('CONNECTED: ' + socket.remoteAddress + ':' + socket.remotePort + ' ' + process.pid);
+
+        socket.on('close', function(data: any) {
+            console.log('CLOSED: ' + socket.remoteAddress + ' ' + socket.remotePort + ' ' + process.pid);
+        });
+
+        socket.on('data', function(data: any) {
+            console.log('DATA ' + socket.remoteAddress + ': ' + data);
+        });
+    }
+
+    #GetLogPrefix = () => {
+        return `${this.#thisService.host}:${this.#thisService.port}: currentTerm: [${this.#currentTerm}] index: [${this.#currentIndex}]: `;
+    }
+
+    #VoteYes = (callback: any, term: number, message: string) => {
+        // Log the fact that I have now voted for this currentTerm
+        this.#votedLog[term] = true;
+        // Update my currentTerm
+        this.#currentTerm = term;
+        // Return my yes vote
+        console.log(`${message} - Vote Yes`.green)
+        callback(null, true); // YES vote
+    }
+
+    #VoteNo = (callback: any, term: number, message: string) => {
+        // Log the fact that I have now voted for this currentTerm
+        this.#votedLog[term] = true;
+        // Return my yes vote
+        console.log(`${message} - Vote No`.yellow)
+        callback(null, false); // NO vote
+    }
+
+    #FindService = (host: string, port: number): IServiceDetail | null => {
+        let retService = null;
+        this.#services.forEach((service, index) => {
+            if (service.host.localeCompare(host) === 0 && service.port === port) {
+                retService = service;
+                return;
+            }
+        });
+        return retService;
+    }
+
+    #CheckMoreRecentTerm = (term: number, lastLogItemTerm: number, lastLogItemIndex: number) => {
+        // Check if the request to vote is newer than my current term
+        let msg = '';
+        let retVal: boolean;
+        if (term > this.#currentTerm) {
+            
+            msg = `${this.#GetLogPrefix()}: canidate term: [${term}] greater than my currentTerm: [${this.#currentTerm}]`;
+            retVal = true;
+        } else {
+            // Check if the request to vote is older than my current term
+            if (term === this.#currentTerm) {
+                if (lastLogItemIndex > this.#currentIndex) {
+                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] greater than my index: [${this.#currentIndex}]`;
+                    retVal = true;
+                } else if (lastLogItemIndex === this.#currentIndex) {
+                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] the same as my index: [${this.#currentIndex}]`;
+                    retVal = true;
+                } else {
+                    msg = `${this.#GetLogPrefix()}: canidate term: [${term}] the same as my currentTerm: [${this.#currentTerm}]. Canidate index: [${lastLogItemIndex}] NOT greater than my index: [${this.#currentIndex}]`;
+                    retVal = false;
+                }
+            } else {
+                msg = `${this.#GetLogPrefix()}: canidate term: [${term}] less than my currentTerm: [${this.#currentTerm}]`;
+                retVal = false;
+            }
+        }
+        return {
+            msg,
+            retVal
+        }
+    }
+
+    #SetupJSONRPCServer = async (): Promise<jayson.TlsServer> => {
+        const jaysonServer = new jayson.server();
+
+        jaysonServer.method("appendEntries", async (args: any, callback: any) => {
+            // Just received a message - reset my electionTimeout
+            this.#ResetElectionTimeout();
+
+            // Do something with this new log entry
+
+            // Needs to check if current, otherwise get old entries and duplicate
+
+        });
+
+        jaysonServer.method("vote", async (args: any, callback: any) => {
+            // Just received a message - reset my electionTimeout
+            this.#ResetElectionTimeout();
+
+            const term = args[0];
+            const lastLogItemTerm = args[1];
+            const lastLogItemIndex = args[2];
+
+            // Now determine whether this requestror gets my vote
+
+            // Check already voted
+            if (this.#votedLog[term]) {
+                this.#VoteNo(callback, term, `${this.#GetLogPrefix()}: Already voted - vote NO`)
+                return;
+            }
+
+            // Check if the request to vote is newer than my current term
+            const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
+            if (retVal.retVal === true) {
+                this.#VoteYes(callback, term, retVal.msg);
+            } else {
+                this.#VoteNo(callback, term, retVal.msg);
+            }
+        });
+
+        jaysonServer.method("appendEntry", async (args: any, callback: any) => {
+            // Just received a message - reset my electionTimeout
+            this.#ResetElectionTimeout();
+
+            const host = args[0];
+            const port = args[1];
+            const term = args[2];
+            const lastLogItemTerm = args[3];
+            const lastLogItemIndex = args[4];
+
+            if (this.#RAFTState.localeCompare(AppMasterRAFTState.CANDIDATE) === 0) {
+                const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
+                if (retVal.retVal === true) {
+                    console.log(`${retVal.msg}`);
+                    console.log(`${this.#GetLogPrefix()}: New leader detected, Host: [${host}], Port: [${port}], while in CANDIDATE mode, reverting to FOLLOWER.`.magenta);
+                    // This sending node is the leader so revert to follower
+                    this.#currentTerm = term;
+                    this.#ChangeState(AppMasterRAFTState.FOLLOWER);
+                } else {
+                    console.log(`${retVal.msg}`);
+                    console.log(`${this.#GetLogPrefix()}: appendEntry called but log is older than my current state, Host: [${host}], Port: [${port}].`.magenta);
+                    // Reject this RPC and continue in the CANDIDATE state
+                    callback(null, false);
+                    return;
+                }
+            } else if (this.#RAFTState.localeCompare(AppMasterRAFTState.LEADER) === 0) {
+                console.log(`${this.#GetLogPrefix()}: appendEntry called while I am the current leader, rejecting RPC call.`.red);
+                // Reject this RPC and continue in the LEADER state
+                callback(null, false);
+                return;
+            } else {
+                // FOLLOWER or IDLE
+                if (this.#RAFTState.localeCompare(AppMasterRAFTState.IDLE) === 0) {
+                    this.#ChangeState(AppMasterRAFTState.FOLLOWER);
+                }
+                const retVal = this.#CheckMoreRecentTerm(term, lastLogItemTerm, lastLogItemIndex);
+                if (retVal.retVal === true) {
+                    if (term > this.#currentTerm) {
+                        console.log(`${retVal.msg}`.cyan);
+                        this.#currentTerm = term;
+                    }
+                } else {
+                    console.log(`${retVal.msg}`);
+                    console.log(`${this.#GetLogPrefix()}: appendEntry called but log is older than my current state, Host: [${host}], Port: [${port}].`.magenta);
+                    // Reject this RPC and continue in the CANDIDATE state
+                    callback(null, false);
+                    return;
+                }
+            }
+
+            if (this.#leaderNode) {
+                const sentLeaderService = this.#FindService(host, port);
+                if (sentLeaderService) {
+                    if (sentLeaderService.host.localeCompare(this.#leaderNode.host) !== 0 || sentLeaderService.port !== this.#leaderNode.port) {
+                        this.#leaderNode = sentLeaderService;
+                        console.log(`${this.#GetLogPrefix()}: Leader Service: Host:[${this.#leaderNode.host}] Port: [${this.#leaderNode.port}]`.white);
+                    }
+                }
+            } else {
+                this.#leaderNode = this.#FindService(host, port);
+                if (this.#leaderNode) {
+                    console.log(`${this.#GetLogPrefix()}: Leader Service: Host:[${this.#leaderNode.host}] Port: [${this.#leaderNode.port}]`.white);
+                }
+            }
+
+            // Now update the log if only more recent
+            callback(null, true);
+        });
+
+        jaysonServer.method("ping", async function(args: any, callback: any) {
+            // do something, do nothing
+            console.log('server got ping');
+            callback();
+        });
+
+        const server = jaysonServer.tls(this.#GetTLSOptions());
+        //(server as tls.Server).on('secureConnection', this.#SetupRPCServer);
+        //server.listen(this.options.listenPort, () => { // 'stscore.stsmda.org'
+        server.listen(this.#serverPort, () => { // 'stscore.stsmda.org'
+            console.log('JSON RPC 2.0 Server is running on port ' + this.options.listenPort + '.');
+        }).on('listening', () =>
+        {
+            this.LogEx(`JSON RPC 2.0 live on ${this.options.endpoint}:${this.options.listenPort}${this.options.apiRoot}`);
+        });
+        return server;
+    }
+
+    #RequestVote = (service: IServiceDetail) => {
+        const options: jayson.TcpClientOptions = {
+            port: service.port,
+            host: service.host
+        };
+        
+        const client = jayson.client.tls(options);
+        /*
+        client.on('tcp socket', (socket: tls.TLSSocket) => {
+            socket.setTimeout(200); //@@ config
+            socket.on('timeout', () => {
+                console.log('socket timeout');
+                socket.end();
+            });
+        });
+        client.on('tcp error', (error: any) => {
+            console.log(`ERROR: [${error}]`.red);
+        });
+        */
+
+        let lastLogItem: ILogItem;
+        if (this.#log.length > 0) {
+            lastLogItem = this.#log[this.#log.length-1];
+        } else {
+            lastLogItem = {
+                term: 0,
+                index: 0,
+                data: null
+            }
+        }
+
+        client.request('vote', [ this.#currentTerm, lastLogItem.term, lastLogItem.index ], (error: any, response: any) => {
+            if (error) { 
+                //console.log(`vote attempt fail: [${JSON.stringify(options)}: ${error}]`.red);
+            } else {
+                // Log the result
+                if (response.result === true) {
+                    this.#votes[this.#GetServiceKey(service)] = 1;
+                }
+                this.#CheckMajority();
+            }
+        });
+    }
+
+    #AppendEntry = (service: IServiceDetail) => {
+        const options: jayson.TcpClientOptions = {
+            port: service.port,
+            host: service.host
+        };
+        
+        const client = jayson.client.tls(options);
+        /*
+        client.on('tcp socket', (socket: tls.TLSSocket) => {
+            socket.setTimeout(200); //@@ config
+            socket.on('timeout', () => {
+                console.log('socket timeout');
+                socket.end();
+            });
+        });
+        client.on('tcp error', (error: any) => {
+            console.log(`ERROR: [${error}]`.red);
+        });
+        */
+
+        let lastLogItem: ILogItem;
+        if (this.#log.length > 0) {
+            lastLogItem = this.#log[this.#log.length-1];
+        } else {
+            lastLogItem = {
+                term: 0,
+                index: 0,
+                data: null
+            }
+        }
+
+        client.request('appendEntry', [ this.#thisService.host, this.#thisService.port, this.#currentTerm, lastLogItem.term, lastLogItem.index ], (error: any, response: any) => {
+            if (error) { 
+                //console.log(`appendEntry attempt fail: [${JSON.stringify(options)}: ${error}]`.red);
+            } else {
+                this.#appendedEntries[this.#GetServiceKey(service)] = 1;
+                // Log the result
+                /*
+                if (response.result === true) {
+                    this.#votes[this.#GetServiceKey(service)] = 1;
+                }
+                */
+            }
+        });
+    }
+
+    #AppendEntries = () => {
+        // Reset all appendEntries for this iteration
+        this.#appendedEntries = { };
+
+        // Reset all votes
+        this.#services.forEach((service, index) => {
+            this.#appendedEntries[this.#GetServiceKey(service)] = 0;
+        });
+        
+        // Request vote for all other services - include my logs current item
+        this.#services.forEach((service, index) => {
+            if (service.host.localeCompare(this.#thisService.host) === 0 && service.port === this.#thisService.port) {
+                // Myself
+                this.#ResetElectionTimeout();
+                this.#appendedEntries[this.#GetSelfServiceKey()] = 1;
+            } else {
+                // Send vote request
+                this.#AppendEntry(service);
+            }
+        });
+    }
+
+    #GetServiceKey = (service: IServiceDetail) => {
+        return `${service.host}:${service.port}`;
+    }
+
+    #GetSelfServiceKey = () => {
+        return this.#GetServiceKey(this.#thisService);
+    }
+
+    // set interval ???
+    #StartAppendEntries = () => {
+        this.#AppendEntries();
+        this.#appendEntriesTimeout = setTimeout(() => {
+            // Now check if we still have majority
+            const majority: boolean = this.#CheckMajorityWithCollection(this.#appendedEntries);
+            if (majority) {
+                this.#StartAppendEntries();
+            } else {
+                console.log(`${this.#GetLogPrefix()}: Not enough response for valid cluster state.`.red);
+            }
+        }, this.#heartBeatTimeout);
+    }
+
+    #Promote = () => {
+        this.#ChangeState(AppMasterRAFTState.LEADER);
+        console.log(`${this.#GetLogPrefix()}: New LEADER is elected: [${this.#GetSelfServiceKey()}]`.green);
+        // As leader, now start the regular pings
+        this.#StartAppendEntries();
+    }
+
+    #CheckMajority = () => {
+        if (this.#CheckMajorityWithCollection(this.#votes)) {
+            this.#Promote();
+        }
+    }
+
+    #CheckMajorityWithCollection = (col: JSONObject): boolean => {
+        let votes = 0;
+        for (const [key, val] of Object.entries(col)) {
+            votes += val;
+        }
+        const requiredVotes = Math.floor(this.#services.length / 2) + 1;
+
+        if (requiredVotes !== this.#lastRequiredVotes || votes !== this.#lastVotes) {
+            console.log(`${this.#GetLogPrefix()}votes: [${votes} need: [${requiredVotes}]`.cyan);
+            this.#lastRequiredVotes = requiredVotes;
+            this.#lastVotes = votes;
+        }
+
+        return votes >= requiredVotes;
+    }
+
+    #StartCanidateVote = () => {
+
+        // Get the election timeout going again
+        this.#StartElectionTimeout();
+
+        console.log(`${this.#GetLogPrefix()}: Start CANDIDATE vote`.yellow);
+        // Start in the CANDIDATE state
+        this.#ChangeState(AppMasterRAFTState.CANDIDATE);
+        // Reset votes
+        this.#votes = { };
+
+        // Reset all votes from all service instances
+        this.#services.forEach((service, index) => {
+            this.#votes[this.#GetServiceKey(service)] = 0;
+        });
+
+        // Start a new term
+        this.#currentTerm++;
+
+        // Request vote for all other services - include my logs current item
+        this.#services.forEach((service, index) => {
+            if (service.host.localeCompare(this.#thisService.host) === 0 && service.port === this.#thisService.port) {
+                // Log the fact that I have now voted for this term
+                this.#votedLog[this.#currentTerm] = true;
+                // Log the result - Vote for myself
+                this.#votes[this.#GetSelfServiceKey()] = 1;
+                // Return my yes vote
+                console.log(`${this.#GetLogPrefix()}: SELF VOTE: canidate currentTerm: [${this.#currentTerm}], vote YES`.green)
+                this.#CheckMajority();
+            } else {
+                // Send vote request
+                this.#RequestVote(service);
+            }
+        });
+    }
+
+    #ReceiveVoteRequest = () => {
+        // Check if the request term and meta-data is more recent than me
+
+        // If request more recent ...
+        this.#voteRegister = 1;
+        // Send vote back to requestor
+    }
+
+    #ResetElectionTimeout = () => {
+        if (this.#canidateTimeout) {
+            clearTimeout(this.#canidateTimeout);
+        }
+        this.#StartElectionTimeout();
+    }
+
+    #StartElectionTimeout = () => {
+        if (this.#canidateTimeout) {
+            clearTimeout(this.#canidateTimeout);
+        }
+        this.#canidateTimeout = setTimeout(() => {
+            this.#StartCanidateVote();
+        }, this.#electionTimeout).unref();
+    }
+
+    #StartRAFTService = async () => {
+        this.LogEx(`---[ RAFT Protocol Starting ]---`.green);
+        this.#server = await this.#SetupJSONRPCServer();
+        this.#StartElectionTimeout();
+    }
+
+    #ChangeState(newState: AppMasterRAFTState) {
+        this.LogEx(`${this.#GetLogPrefix()}: State Change, From: [${this.#RAFTState.toString()}] --> To: [${newState.toString()}]`.magenta);
+        this.#RAFTState = newState;
+    }
+
+    MasterStarted(): void {
+        setTimeout(() => {
+            this.#StartRAFTService();
+        }, this.#startRAFTProtocolTimeout).unref();
+    }
+}

package/src_working/validation/errors.ts

@@ -0,0 +1,6 @@
+export const AppFrameworkErrorCode = Object.freeze({
+    APPFRAMEWORK_MISSING_PERMISSION: {
+        code: 'STS_AF_0001',
+        description: 'STS_AF_0001: Missing Permission(s).'
+    },
+});

package/src_working/webworkertesting/app.ts

@@ -0,0 +1,49 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+import cluster from 'cluster';
+import { Worker, MessageChannel } from 'worker_threads';
+
+if (cluster.isPrimary) {
+    console.log(`Primary ${process.pid} is running`);
+  
+    // Fork workers.
+    cluster.fork();
+  
+    cluster.on('exit', (worker, code, signal) => {
+        console.log(`worker ${worker.process.pid} died`);
+    });
+
+    const fileName ='./dist/webworkertesting/worker.js';
+
+    const publishCollectorWebWorker = new Worker(fileName);
+    publishCollectorWebWorker.unref();
+
+    publishCollectorWebWorker.postMessage({cmd: 'text', message: 'Hello World'});
+
+    publishCollectorWebWorker.on('message', (data) => {
+        console.log(`cluster.primary (${process.pid}): message from worker = [${JSON.stringify(data)}]`);
+    });
+
+    const { port1, port2 } = new MessageChannel();
+
+    publishCollectorWebWorker.postMessage({cmd: 'portmessage', port: port2}, [ port2 ]);
+    port1.postMessage('sending to port1');
+    port1.on('message', (data) => {
+        console.log(`cluster.primary (${process.pid}): message from message port = [${data}]`);
+    });
+
+
+} else {
+    console.log(`Worker ${process.pid} started`);
+
+    const fileName ='./dist/webworkertesting/worker.js';
+
+    const clusterWorker = new Worker(fileName);
+    clusterWorker.unref();
+
+    clusterWorker.postMessage({cmd: 'text', message: 'Hello World'});
+
+    clusterWorker.on('message', (data: any) => {
+        console.log(`cluster.worker (${process.pid}): message from worker = [${JSON.stringify(data)}]`);
+    });
+
+}

package/src_working/webworkertesting/worker.ts

@@ -0,0 +1,24 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+import { matchesProperty } from 'lodash';
+import {
+    Worker,
+    isMainThread,
+    parentPort,
+    MessagePort
+} from 'worker_threads';
+
+let messagePort: MessagePort;
+
+parentPort?.on('message', (data: any) => {
+    if (data.cmd.localeCompare('portmessage') === 0) {
+        messagePort = data.port;
+        messagePort.on('message', (data) => {
+            console.log(`webWorker (${process.pid}): message from passed message port = [${data}]`);
+            messagePort.postMessage(data);
+        });
+    } else {
+        console.log(`webWorker (${process.pid}): message from parent = [${JSON.stringify(data)}]`);
+        parentPort?.postMessage(data);
+    }
+});
+

package/src_working/workerprocessbase.test.ts

@@ -0,0 +1,47 @@
+import { beforeAll, afterAll, test, describe, expect } from 'vitest';
+
+describe("Test Latency Controller", () =>
+{
+    beforeAll(async () => 
+    {
+        /*
+        process.env.AS_ENDPOINT = testHelper.authHost;
+        process.env.AS_HOST_PORT = testHelper.authPort;
+        process.env.AS_PORT = testHelper.authPort;
+
+        $ResetOptions();
+        //goptions = $Options()
+
+        //goptions.asendpoint = testHelper.authHost;
+        //goptions.asport = testHelper.authPort;
+
+        const appOptions = ServiceConfigOptions(true);
+        appOptions.processExitOnTerminate = false;
+        spb = new SingleProcessBase(appOptions)
+        await spb.SetupServer();
+        */
+    }, 60000);
+    
+    afterAll(async () =>
+    {
+        /*
+        await Sleep(500);
+        await spb?.TerminateApplication();
+        await Sleep(1000);
+        spb = null;
+
+        await testHelper.StopAuthService();
+        await testHelper.StopDatabase();
+        await testHelper.StopNetwork();
+
+        await Sleep(2000);
+        */
+    }, 15000);
+
+    test('Testing Module', async () => 
+    {
+        expect.assertions(1);
+        expect(1).toEqual(1);
+    });
+});
+