@nsshunt/stsappframework 3.0.94 → 3.0.96

package/src_working/masterprocessbase.ts

@@ -0,0 +1,598 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+import fs from "fs"
+import si from 'systeminformation' // https://systeminformation.io/
+import https from 'https'
+
+import axios from 'axios';
+
+import cluster, { Worker } from 'node:cluster'
+
+import os from 'os';
+
+import colors from 'colors'
+
+import express from 'express'
+
+import { setupPrimary } from '@socket.io/cluster-adapter'
+import { createServer as createServerHttps } from 'https'
+import { createServer } from 'http'
+import { AggregatorRegistry } from 'prom-client'
+
+import { $Options } from '@nsshunt/stsconfig'
+const goptions = $Options()
+
+import debugModule from 'debug'
+
+import { Gauge, GaugeTypes, InstrumentGaugeTelemetry, InstrumentGaugeOptions, InstrumentHistogramTelemetry } from '@nsshunt/stsinstrumentation'
+import { GetFirstNetworkInterface } from './network'
+
+import { ProcessOptions, STSServerType } from './processoptions'
+import { ProcessBase } from './processbase';
+import { IMasterProcessBase } from './commonTypes';
+
+import { InstrumentDefinitions } from '@nsshunt/stspublisherserver'
+import { IPCMessagePayload, IPCMessageCommand } from './commonTypes'
+import { REQUEST_HEADER_FIELDS_TOO_LARGE } from "http-status-codes";
+
+
+export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
+{
+    //static WORKER_MESSAGE_EVENT = 'sts_worker_message_event';
+    #masterProcessExitTime = goptions.masterProcessExitTime;
+    #childProcessExitTime = goptions.childProcessExitTime;
+    #siValueObject = {
+        currentLoad: 'currentLoad'
+    }
+    #httpServer: any = null; // Prometheus cluster server. See https://github.com/siimon/prom-client/blob/master/example/cluster.js
+    #metricsServer: any= null;
+    #aggregatorRegistry: any = null;
+    #httpsAgent: any;
+    #debug = debugModule(`proc:${process.pid}`);
+    #checkLatency: NodeJS.Timeout | null = null;
+    #killWorkers: Record<string, string> = { };
+    #workers = 1; // Start at 1 becuase of main thread
+    #shuttingDown = false;
+
+    constructor(options: ProcessOptions)
+    {
+        super(options);
+
+        this.#httpsAgent = new https.Agent({ 
+            // Use basic defaults. This agent will not use keep-alive.
+            timeout: goptions.timeout,
+            rejectUnauthorized: goptions.isProduction // Allows self-signed certificates if non-production
+        });
+    }
+
+    override CollectAdditionalTelemetry(): void {
+        si.get(this.#siValueObject).then(data => {
+            this.UpdateInstrument(Gauge.CPU_SYSTEM_LOAD_GAUGE, {
+                val: data.currentLoad.currentLoad
+            } as InstrumentGaugeTelemetry);
+        });
+    }
+
+    override GetAdditionalInstruments(): InstrumentDefinitions {
+        return [
+            [ Gauge.CPU_SYSTEM_LOAD_GAUGE, GaugeTypes.INSTRUMENT_GAUGE, {
+                interval: goptions.instrumentationObservationInterval, 
+                sampleSize: goptions.instrumentationTimeWindow
+            } as InstrumentGaugeOptions]
+        ]
+    }
+
+    /**
+	 * Note: msg removed from signature but will be passed at run-time.
+	 * @param {*} msg 
+	 */
+    // eslint-disable-next-line  @typescript-eslint/no-explicit-any, @typescript-eslint/no-unused-vars
+    WorkerMessageEvent(msg: any)
+    {
+        return null;
+    }
+
+    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+    #WorkerMessageEvent(msg: any)
+    {
+        if (msg.command) {
+            this.WorkerMessageEvent(msg);
+        }
+    }
+
+    #InitCluster = (LogEx: any) =>
+    {
+        cluster.on('listening', () => {
+            let allListening = true;
+            for (const worker of Object.values(cluster.workers as NodeJS.Dict<Worker>)) {
+                if (worker?.isConnected) {
+                    allListening = false;
+                    break;
+                }
+            }
+
+            if (allListening) {
+                LogEx(`Service instance started.`);
+            }
+        });
+    };
+
+    // https://github.com/siimon/prom-client/blob/master/example/cluster.js
+    #SetupPrometheusForMaster = (LogEx: any) => {
+        this.#metricsServer = express();
+        this.#aggregatorRegistry = new AggregatorRegistry();
+
+        switch (this.options.serverType) {
+        case STSServerType.EXPRESS_TLS : {
+            const options = {
+                key: fs.readFileSync(this.options.httpsServerKeyPath),
+                cert: fs.readFileSync(this.options.httpsServerCertificatePath)
+            };
+            this.#httpServer = createServerHttps(options, this.#metricsServer);
+        }
+            break;
+        case STSServerType.EXPRESS : {
+            this.#httpServer = createServer(this.#metricsServer);
+        }
+        }
+        //this.#httpServer.maxConnections = 50;
+
+        this.#metricsServer.get('/cluster_metrics', async (req: any, res: any) => {
+            try {
+                const metrics = await this.#aggregatorRegistry.clusterMetrics();
+                res.set('Content-Type', this.#aggregatorRegistry.contentType);
+                res.send(metrics);
+            } catch (ex: any) {
+                res.statusCode = 500;
+                res.send(ex.message);
+            }
+        });
+
+        //@@@ options wrong
+
+        try {
+            // https://stackoverflow.com/questions/21342828/node-express-unix-domain-socket-permissions
+            //@@httpServer.listen('/tmp/stsrest01.sock').on('listening', () =>
+            //@@httpServer.listen('/var/run/sts/stsrest01.sock').on('listening', () =>
+            //@@httpServer.listen('/var/lib/sts/stsrest01.sock').on('listening', () =>
+            this.#httpServer.listen(this.options.prometheusClusterPort, () => { 
+                //@@chmodSync(this.options.port, 511);
+            }).on('listening', () =>
+            {
+                LogEx(`Prometheus scrapes ready and live on ${this.options.endpoint}:${this.options.prometheusClusterPort}/metrics`);
+            });
+        } catch (error)
+        {
+            console.error(error);
+            throw error;
+        }
+    }
+
+    #CheckLatency = async () => {
+        const start = process.hrtime();
+        await this.#GetLatency();
+        this.#LatencyRequestCompleted(start);
+    }
+
+    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+    #GetLatency = async (): Promise<any> => {
+        // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+        let retVal: any = null;
+        // We use port rather than hostport becuase this test going outside the service and comes back in as a regular client
+        const url = `${this.options.endpoint}:${this.options.port}${this.options.apiRoot}/latency`;
+        const headers = {
+            'Content-Type': 'application/json'
+        };
+        try {
+            retVal = await axios({
+                url: url
+                ,method: 'get'
+                ,headers: headers
+                ,httpsAgent: this.#httpsAgent
+            });
+            if (retVal.status !== 200) {
+                this.#debug(`Error (MasterProcessBase:#GetLatency): Invalid response from server: [${retVal.status}]`.magenta);
+                return null;
+            }
+            return retVal.data.detail;
+        } catch (error: any) {
+            this.#debug(`Error (MasterProcessBase:#GetLatency:catch): [${error}]`.red);
+            this.#debug(`  url: [${url}]`.red);
+            if (error.response && error.response.data) {
+                this.#debug(`  Details: [${JSON.stringify(error.response.data)}]`.red);
+            }
+        }
+    }
+
+    #LatencyRequestCompleted = (start: any) => {
+        // Update request duration histo data
+        let timeInMs = 0;
+        const end = process.hrtime(start);
+        timeInMs = (end[0]* 1000000000 + end[1]) / 1000000;
+        timeInMs = parseFloat(timeInMs.toFixed(4));
+		
+        this.UpdateInstrument(Gauge.LATENCY_HISTOGRAM_GAUGE, {
+            val: timeInMs
+        } as InstrumentHistogramTelemetry);
+
+        this.UpdateInstrument(Gauge.LATENCY_GAUGE, {
+            val: timeInMs
+        } as InstrumentGaugeTelemetry);
+    };
+
+    SetupServer = async () =>
+    {
+        this.SetupInstrumentation();
+        setTimeout(() => {
+            this.SetupServerEx();
+        }, 100);
+    }
+
+    #_KillWorker = (id: string, signal: NodeJS.Signals, killProcess: boolean): boolean => {
+        if (cluster.workers && cluster.workers[id]) {
+            const worker = cluster.workers[id] as Worker;
+            if (worker.process) {
+                this.LogEx(`Sending terminate message `.grey + `(initiated by ${signal})`.yellow + ` for worker PID: ${worker.process.pid}`.grey);
+                const command: string = (killProcess ? 'TerminateAndKill' : 'Terminate');
+                worker.process.send( { command } );
+                return true;
+            } else {
+                this.LogEx(`Could not kill worker with id: [${id}]. The process does not exists`.red);
+                return false;
+            }
+        } else {
+            this.LogEx(`Could not kill worker with id: [${id}]. Worker does not exist within workers collection`.red);
+            return false;
+        }
+    }
+
+    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+    KillWorker = (id: string, signal: NodeJS.Signals = "SIGTERM", options: any, killProcess: boolean, allowKillAll: boolean): boolean => {
+        if (allowKillAll || Object.keys(cluster.workers as NodeJS.Dict<Worker>).length > 1) {
+            if (id.localeCompare('0') === 0) {
+                const keys = Object.keys(cluster.workers as NodeJS.Dict<Worker>);
+                for (let i=keys.length-1; i > 0; i--) {
+                    // Kill the last one added (assumed node keeps them in order)
+                    id = keys[i];
+                    if (!this.#killWorkers[id]) {
+                        this.#killWorkers[id] = id;
+                        // Allow some time for the worker to be terminated before clean-up of the killWorkers array
+                        setTimeout(() => {
+                            delete this.#killWorkers[id];
+                        }, 2000).unref(); //@@
+                        return this.#_KillWorker(id, signal, killProcess);
+                    }
+                }
+                return false;
+            } else {
+                return this.#_KillWorker(id, signal, killProcess);
+            }
+        } else {
+            this.LogEx(`Not allowed to kill the last worker process.`.yellow);
+            return false;
+        }
+    }
+
+    KillWorkers = (signal: NodeJS.Signals): void => {
+        try {
+            for (const id in cluster.workers) {
+                try {
+                    this.KillWorker(id, signal, null, false, true);
+                    //cluster.workers[id].process.kill(signal);
+
+                    // Using kill (below) does not fire the events in the worker processes.
+                    //cluster.workers[id].kill('SIGINT');
+                } catch (error) {
+                    this.LogEx(error);
+                }
+            }
+        } catch (error) {
+            this.LogEx(error);
+        }
+    }
+
+    #UpdateWorkersInstrument = (): void => {
+        setTimeout(() => {
+            this.UpdateInstrument(Gauge.CORE_COUNT_GAUGE, {
+                val: this.#workers
+            } as InstrumentGaugeTelemetry);
+        }, 2000);
+    }
+
+    IncWorkers = (): void => {
+        this.#workers++;
+        this.#debug(`  Inc Workers. Total thread count: [${this.#workers}]`);
+        this.#UpdateWorkersInstrument();
+    }
+
+    DecWorkers = (): void => {
+        this.#workers--;
+        this.#debug(`  Dec Workers. Total thread count: [${this.#workers}]`);
+        this.#UpdateWorkersInstrument();
+    }
+
+    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+    AddWorker = (options: any): number => {
+        const workerId: number = this.#SpawnWorker(options);
+        this.#debug(`  Spawned worker with id: [${workerId}]`.yellow);
+        if (options) {
+            this.#debug(`  Options: [${JSON.stringify(options)}]`.yellow);
+        }
+        return workerId;
+    }
+
+    #processIPCCommand = async (iPCMessagePayload: IPCMessagePayload): Promise<IPCMessagePayload> => {
+        this.#debug(`  Processing message command: [${iPCMessagePayload.command}]`.yellow);
+        switch (iPCMessagePayload.command) {
+        case IPCMessageCommand.AddWorker : {
+            const workerId = this.AddWorker(iPCMessagePayload.requestDetail?.options);
+            iPCMessagePayload.responseDetail = {
+                workerId
+            }
+            return iPCMessagePayload;
+        }
+        case IPCMessageCommand.DeleteWorker : {
+            const workerId = iPCMessagePayload.requestDetail?.workerId;
+            const workerKilled = this.KillWorker(workerId, "SIGTERM", iPCMessagePayload.requestDetail?.options, true, false);
+            this.#debug(`  Killed worker with id: [${workerId}]`.yellow);
+            iPCMessagePayload.responseDetail = {
+                workerId,
+                workerKilled
+            }
+            return iPCMessagePayload;
+        }
+        default : {
+            const errorMessage = `Could not process command: [${iPCMessagePayload.command}].`;
+            this.#debug(`  ${errorMessage}`.red);
+            throw new Error(errorMessage);
+        }
+        }
+    }
+
+    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+    #SpawnWorker = (spawnWorkerOptions?: any): number => {
+        const workerEnv: any = { };
+        workerEnv['STS_GSD_SII'] = JSON.stringify(this.options);
+        if (spawnWorkerOptions) {
+            workerEnv['STS_GSD_OPTIONS'] = JSON.stringify(spawnWorkerOptions);
+        }
+
+        // https://nodejs.org/api/cluster.html#clustersetupprimarysettings
+        if (this.options.workerExec) {
+            cluster.setupPrimary({
+                exec: this.options.workerExec,
+                silent: true,
+            });
+        }
+
+        const worker = cluster.fork(workerEnv);
+        this.IncWorkers();
+
+        worker.on('exit', (code, signal) => {
+            if (signal) {
+                this.LogEx(`Worker: ${worker.process.pid} was killed by signal: ${signal}`);
+            } else if (code !== 0) {
+                this.LogEx(`Worker: ${worker.process.pid} exited with error code: ${code}`);
+            } else {
+                this.LogEx(`Worker: ${worker.process.pid} exited successfully, code: ${code}, signal: ${signal}`);
+            }
+        });
+
+        worker.on('message', async (payload) => {
+            // Only handle request/response message types here ...
+            if (payload.requestResponse) {
+                const iPCMessagePayload: IPCMessagePayload = payload as IPCMessagePayload;
+                this.#debug(`Received message with id: [${iPCMessagePayload.id}] from worker: [${worker.process.pid}]. Details: [${JSON.stringify(iPCMessagePayload)}]`.yellow);
+                const response: IPCMessagePayload = await this.#processIPCCommand(iPCMessagePayload);
+                this.#debug(`Sending response message with id: [${iPCMessagePayload.id}] to worker: [${worker.process.pid}]. Details: [${JSON.stringify(response)}]`.green);
+                worker.send(response);
+            } else {
+                this.#WorkerMessageEvent(payload);
+            }
+        });
+
+        return worker.id;
+    };
+
+    MasterStarted(): void { // eslint-disable @typescript-eslint/no-empty-function
+
+    }
+
+    override get shuttingDown(): boolean {
+        return this.#shuttingDown;
+    }
+
+    ProcessTerminating = async (): Promise<void> => {
+        return;
+    }
+
+    SetupServerEx = async () =>
+    {
+        this.ProcessStartup();
+        const LogEx = this.LogEx;
+
+        LogEx(`Service instance starting. Instance Id: [${this.options.serviceInstanceId}]`);
+
+        this.LogSystemTelemetry();
+
+        // socket.io
+        // setup connections between the workers
+        if (this.options.wssServer === true) {
+            if (this.options.useRedisAdaptor) {
+                LogEx(`Using Redis for socket.io cluster management (master)`);
+            } else {
+                LogEx(`Using nodejs cluster mode for socket.io cluster management`);
+                setupPrimary();
+            }
+        }
+
+        if (this.options.prometheusSupport === true) {
+            this.#SetupPrometheusForMaster(LogEx);
+        }
+
+        const numCPUs = await this.GetNumCPUs();
+        for (let i=0; i < numCPUs; i++) {
+            this.#SpawnWorker();
+        }
+
+        this.#InitCluster(LogEx);
+
+        cluster.on('listening', (worker, address) =>
+        {
+            LogEx(`Worker process ${worker.process.pid} is listening at address: ${JSON.stringify(address)}`);
+        });
+
+        //Setting up lifecycle event listeners for worker processes
+        cluster.on('online', worker => 
+        {
+            LogEx(`Worker process ${worker.process.pid} is online`);
+        });
+
+        cluster.on('exit', (worker, code, signal) => 
+        {
+            if ((code !== null && code === 0) || (signal === 'SIGINT'))
+            {
+                LogEx(`Process ${worker.process.pid} terminated gracefully with code: ${code}, signal: ${signal}`.green);
+                this.DecWorkers();
+            } else if ((code !== null && code === 15) || (signal === 'SIGTERM'))
+            {
+                this.DecWorkers();
+                LogEx(`Process ${worker.process.pid} terminated with code: ${code}, signal: ${signal}`.red);
+            } else {
+                this.DecWorkers();
+                LogEx(`worker ${worker.process.pid} died`.red);
+                LogEx(`code: ${code}`.red);
+                LogEx(`signal: ${signal}`.red);
+                LogEx('process terminated in an error state'.red);
+                if (goptions.respawnOnFail === true)
+                {
+                    LogEx(`Attemping to respawn worker`);
+                    this.#SpawnWorker();
+                }
+            }
+        });
+
+        // Terminate in order;
+        // forked worker threads (send signal)
+        // De-Register service
+        // systeminformation observers
+        // instrument timers (gauge etc.)
+        // publisher
+        // terminate UI (if loaded)
+        const Terminate = async (signal: any) =>
+        {
+            if (this.#shuttingDown === false)
+            {
+                this.#shuttingDown = true;
+
+                if (this.#checkLatency) {
+                    clearInterval(this.#checkLatency);
+                    this.#checkLatency = null;
+                }
+
+                await this.ProcessTerminate();
+
+                await this.ProcessTerminating();
+
+                if (this.GetUIController() !== null)
+                {
+                    LogEx('Destroy the user interface controller.');
+                    this.GetUIController().DestroyUI();
+                }
+
+                if (signal) {
+                    LogEx(this.GetSignalColour(signal)(`Main Process (masterprocess): ${process.pid} received signal: ${signal}`));
+                } else {
+                    LogEx(this.GetSignalColour(null)(`Main Process (masterprocess): ${process.pid} received Terminate without signal.`));
+                }
+
+                LogEx(`De-Registering service.`);
+                //@@ De-register here ...
+
+                if (this.#httpServer !== null) {
+                    LogEx(`Closing httpServer.`);
+                    await this.#httpServer.close();
+                    this.#httpServer = null;
+                }
+				
+                LogEx(`Stopping instruments.`);
+                //@@StopInstruments(this.instruments);
+
+                //@@endpublish was here (and working ...)
+
+                LogEx('Killing Workers.');
+                this.KillWorkers(signal);
+
+                if (this.options.useDatabase) {
+                    LogEx(`Ending database connections and pools.`);
+                    await this.TerminateDatabase();
+                    //await this.accessLayer.enddatabase();
+                }
+
+                // Now allow some time for the workers to die and send any remaining messages ...
+                //if (this.InstrumentController && this.InstrumentController.Workers.length > 0) {
+                if (this.InstrumentController) {
+                    LogEx(`Ending publisher.`);
+                    setTimeout(() => {
+                        if (this.InstrumentController) {
+                            this.InstrumentController.InstrumentTerminate();
+                        }
+                    }, 100);
+                }
+
+                if (this.options.processExitOnTerminate && this.options.processExitOnTerminate === true) {
+                    setTimeout(() => {
+                        LogEx(`Performing process.exit(0).`);
+                        process.exit(0);
+                    }, this.#childProcessExitTime + this.#masterProcessExitTime); // Give the workers time to terminate gracefully
+                } else {
+                    LogEx(`Performing process.exit(0) - Immediate.`);
+                }
+            }
+        }
+
+        process.on('SIGINT', async () =>
+        {
+            await Terminate('SIGINT');
+        });
+
+        process.on('SIGTERM', async () =>
+        {
+            await Terminate('SIGTERM');
+        });
+
+        process.on('exit', (code) =>
+        {
+            if (code === 0)
+            {
+                LogEx(`Main Process: ${process.pid} terminated gracefully with code: ${code}`.green);
+            } else {
+                LogEx(`Main Process: ${process.pid} terminated with code: ${code}`.red);
+            }
+        });
+
+        if (this.options.useLatency) {
+            this.#checkLatency = setInterval(() => {
+                this.#CheckLatency();
+            }, 1000).unref();
+        }
+
+        this.MasterStarted();
+
+        LogEx(`Master process:${process.pid} started`.green);
+    }
+
+    BroadcastDataToWorkers = (command: any, data: any) => {
+        try {
+            for (const id in cluster.workers) {
+                try {
+                    //@@LogEx(`Sending message to worker PID: ${cluster.workers[id].process.pid}`.grey);
+                    (cluster.workers[id] as Worker).process.send( { command: command, data: data } );
+                } catch (error) {
+                    //@@LogEx(error);
+                }
+            }
+        } catch (error) {
+            //@@LogEx(error);
+        }
+    }
+}