@nsshunt/stsappframework 3.0.104 → 3.0.106

package/src_working/masterprocessbase.ts

@@ -1,598 +0,0 @@
-/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
-import fs from "fs"
-import si from 'systeminformation' // https://systeminformation.io/
-import https from 'https'
-
-import axios from 'axios';
-
-import cluster, { Worker } from 'node:cluster'
-
-import os from 'os';
-
-import colors from 'colors'
-
-import express from 'express'
-
-import { setupPrimary } from '@socket.io/cluster-adapter'
-import { createServer as createServerHttps } from 'https'
-import { createServer } from 'http'
-import { AggregatorRegistry } from 'prom-client'
-
-import { $Options } from '@nsshunt/stsconfig'
-const goptions = $Options()
-
-import debugModule from 'debug'
-
-import { Gauge, GaugeTypes, InstrumentGaugeTelemetry, InstrumentGaugeOptions, InstrumentHistogramTelemetry } from '@nsshunt/stsinstrumentation'
-import { GetFirstNetworkInterface } from './network'
-
-import { ProcessOptions, STSServerType } from './processoptions'
-import { ProcessBase } from './processbase';
-import { IMasterProcessBase } from './commonTypes';
-
-import { InstrumentDefinitions } from '@nsshunt/stspublisherserver'
-import { IPCMessagePayload, IPCMessageCommand } from './commonTypes'
-import { REQUEST_HEADER_FIELDS_TOO_LARGE } from "http-status-codes";
-
-
-export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
-{
-    //static WORKER_MESSAGE_EVENT = 'sts_worker_message_event';
-    #masterProcessExitTime = goptions.masterProcessExitTime;
-    #childProcessExitTime = goptions.childProcessExitTime;
-    #siValueObject = {
-        currentLoad: 'currentLoad'
-    }
-    #httpServer: any = null; // Prometheus cluster server. See https://github.com/siimon/prom-client/blob/master/example/cluster.js
-    #metricsServer: any= null;
-    #aggregatorRegistry: any = null;
-    #httpsAgent: any;
-    #debug = debugModule(`proc:${process.pid}`);
-    #checkLatency: NodeJS.Timeout | null = null;
-    #killWorkers: Record<string, string> = { };
-    #workers = 1; // Start at 1 becuase of main thread
-    #shuttingDown = false;
-
-    constructor(options: ProcessOptions)
-    {
-        super(options);
-
-        this.#httpsAgent = new https.Agent({ 
-            // Use basic defaults. This agent will not use keep-alive.
-            timeout: goptions.timeout,
-            rejectUnauthorized: goptions.isProduction // Allows self-signed certificates if non-production
-        });
-    }
-
-    override CollectAdditionalTelemetry(): void {
-        si.get(this.#siValueObject).then(data => {
-            this.UpdateInstrument(Gauge.CPU_SYSTEM_LOAD_GAUGE, {
-                val: data.currentLoad.currentLoad
-            } as InstrumentGaugeTelemetry);
-        });
-    }
-
-    override GetAdditionalInstruments(): InstrumentDefinitions {
-        return [
-            [ Gauge.CPU_SYSTEM_LOAD_GAUGE, GaugeTypes.INSTRUMENT_GAUGE, {
-                interval: goptions.instrumentationObservationInterval, 
-                sampleSize: goptions.instrumentationTimeWindow
-            } as InstrumentGaugeOptions]
-        ]
-    }
-
-    /**
-	 * Note: msg removed from signature but will be passed at run-time.
-	 * @param {*} msg 
-	 */
-    // eslint-disable-next-line  @typescript-eslint/no-explicit-any, @typescript-eslint/no-unused-vars
-    WorkerMessageEvent(msg: any)
-    {
-        return null;
-    }
-
-    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-    #WorkerMessageEvent(msg: any)
-    {
-        if (msg.command) {
-            this.WorkerMessageEvent(msg);
-        }
-    }
-
-    #InitCluster = (LogEx: any) =>
-    {
-        cluster.on('listening', () => {
-            let allListening = true;
-            for (const worker of Object.values(cluster.workers as NodeJS.Dict<Worker>)) {
-                if (worker?.isConnected) {
-                    allListening = false;
-                    break;
-                }
-            }
-
-            if (allListening) {
-                LogEx(`Service instance started.`);
-            }
-        });
-    };
-
-    // https://github.com/siimon/prom-client/blob/master/example/cluster.js
-    #SetupPrometheusForMaster = (LogEx: any) => {
-        this.#metricsServer = express();
-        this.#aggregatorRegistry = new AggregatorRegistry();
-
-        switch (this.options.serverType) {
-        case STSServerType.EXPRESS_TLS : {
-            const options = {
-                key: fs.readFileSync(this.options.httpsServerKeyPath),
-                cert: fs.readFileSync(this.options.httpsServerCertificatePath)
-            };
-            this.#httpServer = createServerHttps(options, this.#metricsServer);
-        }
-            break;
-        case STSServerType.EXPRESS : {
-            this.#httpServer = createServer(this.#metricsServer);
-        }
-        }
-        //this.#httpServer.maxConnections = 50;
-
-        this.#metricsServer.get('/cluster_metrics', async (req: any, res: any) => {
-            try {
-                const metrics = await this.#aggregatorRegistry.clusterMetrics();
-                res.set('Content-Type', this.#aggregatorRegistry.contentType);
-                res.send(metrics);
-            } catch (ex: any) {
-                res.statusCode = 500;
-                res.send(ex.message);
-            }
-        });
-
-        //@@@ options wrong
-
-        try {
-            // https://stackoverflow.com/questions/21342828/node-express-unix-domain-socket-permissions
-            //@@httpServer.listen('/tmp/stsrest01.sock').on('listening', () =>
-            //@@httpServer.listen('/var/run/sts/stsrest01.sock').on('listening', () =>
-            //@@httpServer.listen('/var/lib/sts/stsrest01.sock').on('listening', () =>
-            this.#httpServer.listen(this.options.prometheusClusterPort, () => { 
-                //@@chmodSync(this.options.port, 511);
-            }).on('listening', () =>
-            {
-                LogEx(`Prometheus scrapes ready and live on ${this.options.endpoint}:${this.options.prometheusClusterPort}/metrics`);
-            });
-        } catch (error)
-        {
-            console.error(error);
-            throw error;
-        }
-    }
-
-    #CheckLatency = async () => {
-        const start = process.hrtime();
-        await this.#GetLatency();
-        this.#LatencyRequestCompleted(start);
-    }
-
-    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-    #GetLatency = async (): Promise<any> => {
-        // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-        let retVal: any = null;
-        // We use port rather than hostport becuase this test going outside the service and comes back in as a regular client
-        const url = `${this.options.endpoint}:${this.options.port}${this.options.apiRoot}/latency`;
-        const headers = {
-            'Content-Type': 'application/json'
-        };
-        try {
-            retVal = await axios({
-                url: url
-                ,method: 'get'
-                ,headers: headers
-                ,httpsAgent: this.#httpsAgent
-            });
-            if (retVal.status !== 200) {
-                this.#debug(`Error (MasterProcessBase:#GetLatency): Invalid response from server: [${retVal.status}]`.magenta);
-                return null;
-            }
-            return retVal.data.detail;
-        } catch (error: any) {
-            this.#debug(`Error (MasterProcessBase:#GetLatency:catch): [${error}]`.red);
-            this.#debug(`  url: [${url}]`.red);
-            if (error.response && error.response.data) {
-                this.#debug(`  Details: [${JSON.stringify(error.response.data)}]`.red);
-            }
-        }
-    }
-
-    #LatencyRequestCompleted = (start: any) => {
-        // Update request duration histo data
-        let timeInMs = 0;
-        const end = process.hrtime(start);
-        timeInMs = (end[0]* 1000000000 + end[1]) / 1000000;
-        timeInMs = parseFloat(timeInMs.toFixed(4));
-		
-        this.UpdateInstrument(Gauge.LATENCY_HISTOGRAM_GAUGE, {
-            val: timeInMs
-        } as InstrumentHistogramTelemetry);
-
-        this.UpdateInstrument(Gauge.LATENCY_GAUGE, {
-            val: timeInMs
-        } as InstrumentGaugeTelemetry);
-    };
-
-    SetupServer = async () =>
-    {
-        this.SetupInstrumentation();
-        setTimeout(() => {
-            this.SetupServerEx();
-        }, 100);
-    }
-
-    #_KillWorker = (id: string, signal: NodeJS.Signals, killProcess: boolean): boolean => {
-        if (cluster.workers && cluster.workers[id]) {
-            const worker = cluster.workers[id] as Worker;
-            if (worker.process) {
-                this.LogEx(`Sending terminate message `.grey + `(initiated by ${signal})`.yellow + ` for worker PID: ${worker.process.pid}`.grey);
-                const command: string = (killProcess ? 'TerminateAndKill' : 'Terminate');
-                worker.process.send( { command } );
-                return true;
-            } else {
-                this.LogEx(`Could not kill worker with id: [${id}]. The process does not exists`.red);
-                return false;
-            }
-        } else {
-            this.LogEx(`Could not kill worker with id: [${id}]. Worker does not exist within workers collection`.red);
-            return false;
-        }
-    }
-
-    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-    KillWorker = (id: string, signal: NodeJS.Signals = "SIGTERM", options: any, killProcess: boolean, allowKillAll: boolean): boolean => {
-        if (allowKillAll || Object.keys(cluster.workers as NodeJS.Dict<Worker>).length > 1) {
-            if (id.localeCompare('0') === 0) {
-                const keys = Object.keys(cluster.workers as NodeJS.Dict<Worker>);
-                for (let i=keys.length-1; i > 0; i--) {
-                    // Kill the last one added (assumed node keeps them in order)
-                    id = keys[i];
-                    if (!this.#killWorkers[id]) {
-                        this.#killWorkers[id] = id;
-                        // Allow some time for the worker to be terminated before clean-up of the killWorkers array
-                        setTimeout(() => {
-                            delete this.#killWorkers[id];
-                        }, 2000).unref(); //@@
-                        return this.#_KillWorker(id, signal, killProcess);
-                    }
-                }
-                return false;
-            } else {
-                return this.#_KillWorker(id, signal, killProcess);
-            }
-        } else {
-            this.LogEx(`Not allowed to kill the last worker process.`.yellow);
-            return false;
-        }
-    }
-
-    KillWorkers = (signal: NodeJS.Signals): void => {
-        try {
-            for (const id in cluster.workers) {
-                try {
-                    this.KillWorker(id, signal, null, false, true);
-                    //cluster.workers[id].process.kill(signal);
-
-                    // Using kill (below) does not fire the events in the worker processes.
-                    //cluster.workers[id].kill('SIGINT');
-                } catch (error) {
-                    this.LogEx(error);
-                }
-            }
-        } catch (error) {
-            this.LogEx(error);
-        }
-    }
-
-    #UpdateWorkersInstrument = (): void => {
-        setTimeout(() => {
-            this.UpdateInstrument(Gauge.CORE_COUNT_GAUGE, {
-                val: this.#workers
-            } as InstrumentGaugeTelemetry);
-        }, 2000);
-    }
-
-    IncWorkers = (): void => {
-        this.#workers++;
-        this.#debug(`  Inc Workers. Total thread count: [${this.#workers}]`);
-        this.#UpdateWorkersInstrument();
-    }
-
-    DecWorkers = (): void => {
-        this.#workers--;
-        this.#debug(`  Dec Workers. Total thread count: [${this.#workers}]`);
-        this.#UpdateWorkersInstrument();
-    }
-
-    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-    AddWorker = (options: any): number => {
-        const workerId: number = this.#SpawnWorker(options);
-        this.#debug(`  Spawned worker with id: [${workerId}]`.yellow);
-        if (options) {
-            this.#debug(`  Options: [${JSON.stringify(options)}]`.yellow);
-        }
-        return workerId;
-    }
-
-    #processIPCCommand = async (iPCMessagePayload: IPCMessagePayload): Promise<IPCMessagePayload> => {
-        this.#debug(`  Processing message command: [${iPCMessagePayload.command}]`.yellow);
-        switch (iPCMessagePayload.command) {
-        case IPCMessageCommand.AddWorker : {
-            const workerId = this.AddWorker(iPCMessagePayload.requestDetail?.options);
-            iPCMessagePayload.responseDetail = {
-                workerId
-            }
-            return iPCMessagePayload;
-        }
-        case IPCMessageCommand.DeleteWorker : {
-            const workerId = iPCMessagePayload.requestDetail?.workerId;
-            const workerKilled = this.KillWorker(workerId, "SIGTERM", iPCMessagePayload.requestDetail?.options, true, false);
-            this.#debug(`  Killed worker with id: [${workerId}]`.yellow);
-            iPCMessagePayload.responseDetail = {
-                workerId,
-                workerKilled
-            }
-            return iPCMessagePayload;
-        }
-        default : {
-            const errorMessage = `Could not process command: [${iPCMessagePayload.command}].`;
-            this.#debug(`  ${errorMessage}`.red);
-            throw new Error(errorMessage);
-        }
-        }
-    }
-
-    // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-    #SpawnWorker = (spawnWorkerOptions?: any): number => {
-        const workerEnv: any = { };
-        workerEnv['STS_GSD_SII'] = JSON.stringify(this.options);
-        if (spawnWorkerOptions) {
-            workerEnv['STS_GSD_OPTIONS'] = JSON.stringify(spawnWorkerOptions);
-        }
-
-        // https://nodejs.org/api/cluster.html#clustersetupprimarysettings
-        if (this.options.workerExec) {
-            cluster.setupPrimary({
-                exec: this.options.workerExec,
-                silent: true,
-            });
-        }
-
-        const worker = cluster.fork(workerEnv);
-        this.IncWorkers();
-
-        worker.on('exit', (code, signal) => {
-            if (signal) {
-                this.LogEx(`Worker: ${worker.process.pid} was killed by signal: ${signal}`);
-            } else if (code !== 0) {
-                this.LogEx(`Worker: ${worker.process.pid} exited with error code: ${code}`);
-            } else {
-                this.LogEx(`Worker: ${worker.process.pid} exited successfully, code: ${code}, signal: ${signal}`);
-            }
-        });
-
-        worker.on('message', async (payload) => {
-            // Only handle request/response message types here ...
-            if (payload.requestResponse) {
-                const iPCMessagePayload: IPCMessagePayload = payload as IPCMessagePayload;
-                this.#debug(`Received message with id: [${iPCMessagePayload.id}] from worker: [${worker.process.pid}]. Details: [${JSON.stringify(iPCMessagePayload)}]`.yellow);
-                const response: IPCMessagePayload = await this.#processIPCCommand(iPCMessagePayload);
-                this.#debug(`Sending response message with id: [${iPCMessagePayload.id}] to worker: [${worker.process.pid}]. Details: [${JSON.stringify(response)}]`.green);
-                worker.send(response);
-            } else {
-                this.#WorkerMessageEvent(payload);
-            }
-        });
-
-        return worker.id;
-    };
-
-    MasterStarted(): void { // eslint-disable @typescript-eslint/no-empty-function
-
-    }
-
-    override get shuttingDown(): boolean {
-        return this.#shuttingDown;
-    }
-
-    ProcessTerminating = async (): Promise<void> => {
-        return;
-    }
-
-    SetupServerEx = async () =>
-    {
-        this.ProcessStartup();
-        const LogEx = this.LogEx;
-
-        LogEx(`Service instance starting. Instance Id: [${this.options.serviceInstanceId}]`);
-
-        this.LogSystemTelemetry();
-
-        // socket.io
-        // setup connections between the workers
-        if (this.options.wssServer === true) {
-            if (this.options.useRedisAdaptor) {
-                LogEx(`Using Redis for socket.io cluster management (master)`);
-            } else {
-                LogEx(`Using nodejs cluster mode for socket.io cluster management`);
-                setupPrimary();
-            }
-        }
-
-        if (this.options.prometheusSupport === true) {
-            this.#SetupPrometheusForMaster(LogEx);
-        }
-
-        const numCPUs = await this.GetNumCPUs();
-        for (let i=0; i < numCPUs; i++) {
-            this.#SpawnWorker();
-        }
-
-        this.#InitCluster(LogEx);
-
-        cluster.on('listening', (worker, address) =>
-        {
-            LogEx(`Worker process ${worker.process.pid} is listening at address: ${JSON.stringify(address)}`);
-        });
-
-        //Setting up lifecycle event listeners for worker processes
-        cluster.on('online', worker => 
-        {
-            LogEx(`Worker process ${worker.process.pid} is online`);
-        });
-
-        cluster.on('exit', (worker, code, signal) => 
-        {
-            if ((code !== null && code === 0) || (signal === 'SIGINT'))
-            {
-                LogEx(`Process ${worker.process.pid} terminated gracefully with code: ${code}, signal: ${signal}`.green);
-                this.DecWorkers();
-            } else if ((code !== null && code === 15) || (signal === 'SIGTERM'))
-            {
-                this.DecWorkers();
-                LogEx(`Process ${worker.process.pid} terminated with code: ${code}, signal: ${signal}`.red);
-            } else {
-                this.DecWorkers();
-                LogEx(`worker ${worker.process.pid} died`.red);
-                LogEx(`code: ${code}`.red);
-                LogEx(`signal: ${signal}`.red);
-                LogEx('process terminated in an error state'.red);
-                if (goptions.respawnOnFail === true)
-                {
-                    LogEx(`Attemping to respawn worker`);
-                    this.#SpawnWorker();
-                }
-            }
-        });
-
-        // Terminate in order;
-        // forked worker threads (send signal)
-        // De-Register service
-        // systeminformation observers
-        // instrument timers (gauge etc.)
-        // publisher
-        // terminate UI (if loaded)
-        const Terminate = async (signal: any) =>
-        {
-            if (this.#shuttingDown === false)
-            {
-                this.#shuttingDown = true;
-
-                if (this.#checkLatency) {
-                    clearInterval(this.#checkLatency);
-                    this.#checkLatency = null;
-                }
-
-                await this.ProcessTerminate();
-
-                await this.ProcessTerminating();
-
-                if (this.GetUIController() !== null)
-                {
-                    LogEx('Destroy the user interface controller.');
-                    this.GetUIController().DestroyUI();
-                }
-
-                if (signal) {
-                    LogEx(this.GetSignalColour(signal)(`Main Process (masterprocess): ${process.pid} received signal: ${signal}`));
-                } else {
-                    LogEx(this.GetSignalColour(null)(`Main Process (masterprocess): ${process.pid} received Terminate without signal.`));
-                }
-
-                LogEx(`De-Registering service.`);
-                //@@ De-register here ...
-
-                if (this.#httpServer !== null) {
-                    LogEx(`Closing httpServer.`);
-                    await this.#httpServer.close();
-                    this.#httpServer = null;
-                }
-				
-                LogEx(`Stopping instruments.`);
-                //@@StopInstruments(this.instruments);
-
-                //@@endpublish was here (and working ...)
-
-                LogEx('Killing Workers.');
-                this.KillWorkers(signal);
-
-                if (this.options.useDatabase) {
-                    LogEx(`Ending database connections and pools.`);
-                    await this.TerminateDatabase();
-                    //await this.accessLayer.enddatabase();
-                }
-
-                // Now allow some time for the workers to die and send any remaining messages ...
-                //if (this.InstrumentController && this.InstrumentController.Workers.length > 0) {
-                if (this.InstrumentController) {
-                    LogEx(`Ending publisher.`);
-                    setTimeout(() => {
-                        if (this.InstrumentController) {
-                            this.InstrumentController.InstrumentTerminate();
-                        }
-                    }, 100);
-                }
-
-                if (this.options.processExitOnTerminate && this.options.processExitOnTerminate === true) {
-                    setTimeout(() => {
-                        LogEx(`Performing process.exit(0).`);
-                        process.exit(0);
-                    }, this.#childProcessExitTime + this.#masterProcessExitTime); // Give the workers time to terminate gracefully
-                } else {
-                    LogEx(`Performing process.exit(0) - Immediate.`);
-                }
-            }
-        }
-
-        process.on('SIGINT', async () =>
-        {
-            await Terminate('SIGINT');
-        });
-
-        process.on('SIGTERM', async () =>
-        {
-            await Terminate('SIGTERM');
-        });
-
-        process.on('exit', (code) =>
-        {
-            if (code === 0)
-            {
-                LogEx(`Main Process: ${process.pid} terminated gracefully with code: ${code}`.green);
-            } else {
-                LogEx(`Main Process: ${process.pid} terminated with code: ${code}`.red);
-            }
-        });
-
-        if (this.options.useLatency) {
-            this.#checkLatency = setInterval(() => {
-                this.#CheckLatency();
-            }, 1000).unref();
-        }
-
-        this.MasterStarted();
-
-        LogEx(`Master process:${process.pid} started`.green);
-    }
-
-    BroadcastDataToWorkers = (command: any, data: any) => {
-        try {
-            for (const id in cluster.workers) {
-                try {
-                    //@@LogEx(`Sending message to worker PID: ${cluster.workers[id].process.pid}`.grey);
-                    (cluster.workers[id] as Worker).process.send( { command: command, data: data } );
-                } catch (error) {
-                    //@@LogEx(error);
-                }
-            }
-        } catch (error) {
-            //@@LogEx(error);
-        }
-    }
-}