@nsshunt/stsappframework 2.19.205 → 2.19.206

package/runc2.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# openssl req -nodes -new -x509 -keyout server.key -out server.cert
+clear; \
+export STS_PROJ_ROOT=./..; \
+STSENVFILE=$STS_PROJ_ROOT/stsglobalresources/.env \
+DB_SCRIPT_FOLDER=$STS_PROJ_ROOT/stsglobalresources/db-scripts \
+REST01_PORT=3003 \
+REST01_HOST_PORT=3003 \
+REST01_SERVICE_NAME="STSRest01-3003" \
+REST01_API_IDENTIFIER="https://stsmda.com.au/stsrest01api/v1.0/" \
+REST01_ENDPOINT="https://stsrest.stsmda.org" \
+MAX_CPU=1 \
+AS_ENDPOINT=https://stscore.stsmda.org \
+AS_HOST_PORT=3002 \
+AS_PORT=3002 \
+DB_HOST=localhost \
+DB_PORT=5432 \
+DB_PASSWORD=postgres \
+HTTPS_SERVER_KEY_PATH=/etc/letsencrypt/live/stsmda.org/privkey.pem \
+HTTPS_SERVER_CERT_PATH=/etc/letsencrypt/live/stsmda.org/fullchain.pem \
+DEBUG=proc* \
+PUBLISH_DEBUG=false \
+UV_THREADPOOL_SIZE=64 \
+node dist/tcpclient/app2

package/src/commonTypes.ts

@@ -70,7 +70,7 @@ export interface ISingleProcessBase extends IProcessBase {
     get expressServer(): any // STSExpressServer - need to put interface in common types - circular reference
     SetupServer(): Promise<boolean>
     SetupServerEx: () => Promise<void>
-    Terminate(): Promise<void>
+    TerminateApplication(): Promise<void>
 }
 
 export interface IWorkerProcessBase extends IProcessBase {

package/src/masterprocessbase.ts

@@ -7,7 +7,6 @@ import axios from 'axios';
 import cluster, { Worker } from 'node:cluster'
 
 import os from 'os';
-let numCPUs = os.cpus().length;
 
 import colors from 'colors'
 
@@ -26,7 +25,7 @@ import debugModule from 'debug'
 import { Gauge, GaugeTypes, InstrumentGaugeTelemetry, InstrumentGaugeOptions, InstrumentHistogramTelemetry } from '@nsshunt/stsinstrumentation'
 import { GetFirstNetworkInterface } from './network'
 
-import { ProcessOptions } from './processoptions'
+import { ProcessOptions, STSServerType } from './processoptions'
 import { ProcessBase } from './processbase';
 import { IMasterProcessBase } from './commonTypes';
 
@@ -120,15 +119,19 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
         this.#metricsServer = express();
         this.#aggregatorRegistry = new AggregatorRegistry();
 
-        if (this.options.httpsServer === true) {
+        switch (this.options.serverType) {
+        case STSServerType.EXPRESS_TLS : {
             const options = {
                 key: fs.readFileSync(this.options.httpsServerKeyPath),
                 cert: fs.readFileSync(this.options.httpsServerCertificatePath)
             };
             this.#httpServer = createServerHttps(options, this.#metricsServer);
-        } else {
+        }
+            break;
+        case STSServerType.EXPRESS : {
             this.#httpServer = createServer(this.#metricsServer);
         }
+        }
         //this.#httpServer.maxConnections = 50;
 
         this.#metricsServer.get('/cluster_metrics', async (req: any, res: any) => {
@@ -388,49 +391,9 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
         this.ProcessStartup();
         const LogEx = this.LogEx;
 
-        /*
-		if (this.instruments !== null) {
-			this.instruments[Gauge.LOGGER].consoleLogging = this.options.consoleLogging;
-			this.instruments[Gauge.LOGGER].instrumentLogging = this.options.instrumentLogging;
-		}
-		*/
-
-        // https://systeminformation.io/
-        const valueObject = {
-            system: '*',
-            osInfo: '*',
-            cpu: '*',
-            mem: '*'
-        }
-		
-        const sysinfo = await si.get(valueObject);
-		
-        if (goptions.useCPUs > 0) {
-            if (goptions.useCPUs >= 1) {
-                numCPUs = goptions.useCPUs;
-            } else {
-                numCPUs = Math.round(sysinfo.cpu.cores * goptions.useCPUs);
-            }
-        } else {
-            numCPUs = sysinfo.cpu.physicalCores;
-        }
-
         LogEx(`Service instance starting. Instance Id: [${this.options.serviceInstanceId}]`);
 
-        const hostaddr = GetFirstNetworkInterface();
-        if (hostaddr !== null)
-        {
-            LogEx(`Host Address: ${hostaddr}`);
-        } else {
-            LogEx(`Unknown Host Address.`);
-        }
-        LogEx(`Server starting with ${numCPUs} Cores/Threads`);
-
-        LogEx(`Hostname: ${sysinfo.osInfo.hostname}`);
-        LogEx(`System: ${JSON.stringify(sysinfo.system)}`);
-        LogEx(`OS Info: ${JSON.stringify(sysinfo.osInfo)}`);
-        LogEx(`CPU: ${JSON.stringify(sysinfo.cpu)}`);
-        LogEx(`Memory: ${JSON.stringify(sysinfo.mem)}`);
+        this.LogSystemTelemetry();
 
         // socket.io
         // setup connections between the workers
@@ -447,18 +410,13 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
             this.#SetupPrometheusForMaster(LogEx);
         }
 
+        const numCPUs = await this.GetNumCPUs();
         for (let i=0; i < numCPUs; i++) {
             this.#SpawnWorker();
         }
 
         this.#InitCluster(LogEx);
 
-        /*
-		if (this.publishBroker !== null) {
-			this.publishBroker.StartPublish();
-		}
-		*/
-
         cluster.on('listening', (worker, address) =>
         {
             LogEx(`Worker process ${worker.process.pid} is listening at address: ${JSON.stringify(address)}`);
@@ -494,19 +452,6 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
             }
         });
 
-
-        const GetSignalColour = (signal: any) =>
-        {
-            let msgcolor = null;
-            if (signal && signal === 'SIGINT')
-            {
-                msgcolor = colors.yellow;
-            } else {
-                msgcolor = colors.red;
-            }
-            return msgcolor;
-        };
-
         // Terminate in order;
         // forked worker threads (send signal)
         // De-Register service
@@ -534,9 +479,9 @@ export class MasterProcessBase extends ProcessBase implements IMasterProcessBase
                 }
 
                 if (signal) {
-                    LogEx(GetSignalColour(signal)(`Main Process (masterprocess): ${process.pid} received signal: ${signal}`));
+                    LogEx(this.GetSignalColour(signal)(`Main Process (masterprocess): ${process.pid} received signal: ${signal}`));
                 } else {
-                    LogEx(GetSignalColour(null)(`Main Process (masterprocess): ${process.pid} received Terminate without signal.`));
+                    LogEx(this.GetSignalColour(null)(`Main Process (masterprocess): ${process.pid} received Terminate without signal.`));
                 }
 
                 LogEx(`De-Registering service.`);

package/src/processbase.ts

@@ -3,6 +3,8 @@ import { memoryUsage, exit } from 'process'
 import path from 'node:path';
 import cluster from 'node:cluster'
 import colors from 'colors'
+import si from 'systeminformation' // https://systeminformation.io/
+import { GetFirstNetworkInterface } from './network'
 
 import debugModule from 'debug'
 const debug = debugModule(`proc:${process.pid}:processBase`);
@@ -14,7 +16,7 @@ import { PGAccessLayer, PGPoolManager, PGPoolManagerEventName } from '@nsshunt/s
 import { $Options } from '@nsshunt/stsconfig'
 const goptions = $Options()
 
-import { ProcessOptions } from './processoptions'
+import { ProcessOptions, STSServerType } from './processoptions'
 import { STSOptionsBase, JSONObject } from '@nsshunt/stsutils'
 import { StatusCodes } from 'http-status-codes'
 
@@ -162,7 +164,7 @@ export abstract class ProcessBase extends STSOptionsBase implements IProcessBase
             payloadType: InstrumentPayloadType.service,
             consoleLogging: this.options.consoleLogging,
             instrumentLogging: this.options.instrumentLogging,
-            httpServer: this.options.httpServer,
+            httpServer: (this.options.serverType === STSServerType.EXPRESS),
             instrumentationObservationInterval: this.options.instrumentationObservationInterval,
             instrumentationTimeWindow: this.options.instrumentationTimeWindow,
             instrumentDefinitions: this.GetAdditionalInstruments(),
@@ -325,4 +327,62 @@ export abstract class ProcessBase extends STSOptionsBase implements IProcessBase
         }
         return null;
     }
+
+    GetNumCPUs = async (): Promise<number> => {
+        // https://systeminformation.io/
+        const valueObject = {
+            cpu: '*'
+        }
+
+        const sysinfo = await si.get(valueObject);
+        let numCPUs = 2;
+        if (goptions.useCPUs > 0) {
+            if (goptions.useCPUs >= 1) {
+                numCPUs = goptions.useCPUs;
+            } else {
+                numCPUs = Math.round(sysinfo.cpu.cores * goptions.useCPUs);
+            }
+        } else {
+            numCPUs = sysinfo.cpu.physicalCores;
+        }
+        return numCPUs;
+    }
+
+    LogSystemTelemetry = async () => {
+        // https://systeminformation.io/
+        const valueObject = {
+            system: '*',
+            osInfo: '*',
+            cpu: '*',
+            mem: '*'
+        }
+
+        const sysinfo = await si.get(valueObject);
+        const numCPUs = await this.GetNumCPUs();
+        const hostname = sysinfo.osInfo.hostname;
+
+        const hostaddr = GetFirstNetworkInterface();
+        if (hostaddr !== null) {
+            this.LogEx(`Host Address: ${hostaddr}`);
+        } else {
+            this.LogEx(`Unknown Host Address.`);
+        }
+        this.LogEx(`Server starting with ${numCPUs} Cores/Threads`);
+
+        this.LogEx(`Hostname: ${hostname}`);
+        this.LogEx(`System: ${JSON.stringify(sysinfo.system)}`);
+        this.LogEx(`OS Info: ${JSON.stringify(sysinfo.osInfo)}`);
+        this.LogEx(`CPU: ${JSON.stringify(sysinfo.cpu)}`);
+        this.LogEx(`Memory: ${JSON.stringify(sysinfo.mem)}`);
+    }
+
+    GetSignalColour = (signal: any) => {
+        let msgcolor = null;
+        if (signal === 'SIGINT') {
+            msgcolor = colors.yellow;
+        } else {
+            msgcolor = colors.red;
+        }
+        return msgcolor;
+    };
 }

package/src/processoptions.ts

@@ -13,6 +13,14 @@ export interface STSExpressServer {
  */
 export type ExpressRouteFactory = (app: express.Express, stsApp: IProcessBase) => void
 
+export enum STSServerType {
+    NONE = "NONE",
+    EXPRESS = "EXPRESS",
+    EXPRESS_TLS = "EXPRESS_TLS",
+	TCPRAW_TLS = "TCPRAW_TLS",
+    JSONRPC2_TLS = "JSONRPC_TLS"
+}
+
 //export interface ProcessOptions extends JSONObject {
 export interface ProcessOptions {
         /**
@@ -27,15 +35,7 @@ export interface ProcessOptions {
 
     // Service details
 
-    /**
-     * Does this service run an HTTP server.
-     */
-    httpServer: boolean
-
-    /**
-     * Does this service run an HTTPS server. Note that httpServer must also be true if this value is true.
-     */
-    httpsServer: boolean
+    serverType: STSServerType
 
 	/**
 	 * HTTPS service public key path.

package/src/serverprocessbase.ts

@@ -0,0 +1,390 @@
+import debugModule from 'debug'
+const debug = debugModule(`proc:${process.pid}`);
+
+import { $Options } from '@nsshunt/stsconfig'
+const goptions = $Options()
+
+import fs from "fs"
+import colors from 'colors'
+
+import { createAdapter as clusterCreateAdapter } from '@socket.io/cluster-adapter'
+import { createAdapter } from "@socket.io/redis-streams-adapter";
+
+import { Gauge, InstrumentGaugeTelemetry } from '@nsshunt/stsinstrumentation'
+import { JSONObject, Sleep } from '@nsshunt/stsutils'
+
+import { ProcessOptions, STSServerType } from './processoptions'
+import { ProcessBase } from './processbase';
+
+import { register, Counter, collectDefaultMetrics, AggregatorRegistry } from 'prom-client'
+
+import { createServer as createServerHttps } from 'https'
+import { createServer } from 'http'
+import tls from 'node:tls'
+import net from 'node:net'
+
+import { Server, ServerOptions } from "socket.io";
+
+import { v4 as uuidv4 } from 'uuid';
+
+import { IPCMessage, IPCMessages, IPCMessagePayload, IPCMessageCommand, IWorkerProcessBase, ISocketIoHelper } from './commonTypes'
+import { STSExpressServer } from './server';
+import { Express } from 'express'
+
+import { createClient, RedisClientType } from 'redis';
+
+import jayson from 'jayson'
+
+/**
+ * todo
+ * @typedef {Object} options - todo
+ * @property {boolean} [wssServer=false] - Create a web socket server on this worker instance
+ */
+export class ServerProcessBase extends ProcessBase
+{
+    #masterProcessExitTime = goptions.masterProcessExitTime;
+    #io: Server | null = null;
+    #redisClient: RedisClientType | null = null;
+    #httpServer: any = null;
+    #expressServer: STSExpressServer | null = null;
+    #sockets: net.Socket[] = [ ];
+    #shuttingDown = false;
+
+    constructor(options: ProcessOptions) {
+        super(options);
+    }
+
+    get httpServer() {
+        return this.#httpServer;
+    }
+
+    get io() {
+        return this.#io;
+    }
+
+    get expressServer(): STSExpressServer | null {
+        return this.#expressServer;
+    }
+    set expressServer(val: STSExpressServer | null) {
+        this.#expressServer = val;
+    }
+
+    // Setup server to Prometheus scrapes:
+    #SetupPrometheusEndPoints = (expressServer: Express) => {
+        // AggregatorRegistry is required here in the worker as well as the master in order for prom-client to work correctly.
+        new AggregatorRegistry();
+
+        const prefix = 'sts_';
+
+        collectDefaultMetrics({
+            labels: { NODE_APP_INSTANCE: process.pid },
+            prefix: prefix
+        });
+
+        const c = new Counter({
+            name: 'sts_test_counter',
+            help: 'Example of a counter',
+            labelNames: ['code'],
+        });
+
+        setInterval(() => {
+            c.inc({ code: 200 });
+        }, 1000).unref();
+
+        setInterval(() => {
+            c.inc({ code: 400 });
+            c.inc({ code: 'worker_' + process.pid });
+        }, 500).unref();
+
+        expressServer.get('/metrics', async (req: any, res: any) => {
+            try {
+                res.set('Content-Type', register.contentType);
+                res.end(await register.metrics());
+            } catch (ex) {
+                res.status(500).end(ex);
+            }
+        });
+
+        expressServer.get('/metrics/counter', async (req: any, res: any) => {
+            try {
+                res.set('Content-Type', register.contentType);
+                res.end(await register.getSingleMetricAsString('test_counter'));
+            } catch (ex) {
+                res.status(500).end(ex);
+            }
+        });
+    }
+
+    #SetupTLSServer = async (socket: net.Socket): Promise<void> => {
+        // Add a 'close' event handler to this instance of socket
+        console.log('CONNECTED: ' + socket.remoteAddress + ':' + socket.remotePort + ' ' + process.pid);
+        this.#sockets.push(socket);
+
+        const self = this;
+        socket.on('close', function(data: any) {
+            const index = self.#sockets.findIndex(function(o) {
+                return o.remoteAddress === socket.remoteAddress && o.remotePort === socket.remotePort;
+            })
+            if (index !== -1) self.#sockets.splice(index, 1);
+            console.log('CLOSED: ' + socket.remoteAddress + ' ' + socket.remotePort + ' ' + process.pid);
+        });
+
+        socket.on('data', function(data: any) {
+            console.log('DATA ' + socket.remoteAddress + ': ' + socket.remotePort + ': ' + data);
+            socket.write(socket.remoteAddress + ':' + socket.remotePort + " said " + data + '\n');
+
+            // Write the data back to all the connected, the client will receive it as data from the server
+            /*
+            self.#sockets.forEach(function(socket, index, array) {
+                socket.write(socket.remoteAddress + ':' + socket.remotePort + " said " + data + '\n');
+            });
+
+            */
+        });
+    }
+
+    #SetupRPCServer = async (socket: net.Socket): Promise<void> => {
+        console.log('CONNECTED: ' + socket.remoteAddress + ':' + socket.remotePort + ' ' + process.pid);
+
+        socket.on('close', function(data: any) {
+            console.log('CLOSED: ' + socket.remoteAddress + ' ' + socket.remotePort + ' ' + process.pid);
+        });
+
+        socket.on('data', function(data: any) {
+            console.log('DATA ' + socket.remoteAddress + ': ' + data);
+        });
+    }
+
+    #SetupWSSServer = async (clusterMode: boolean): Promise<void> => {
+        // socket.io
+        // WebSocket
+        const options: Partial<ServerOptions> = {
+            transports: [ "websocket" ] // or [ "websocket", "polling" ] (to use long-poolling. Note that the order matters)
+            // The default path is /socket.io
+            // This can be changed with the path option as shown below
+            //,path: '/zzz'
+        };
+
+        //this.#io = require("socket.io")(this.#httpServer, options);
+        this.#io = new Server(this.#httpServer, options);
+
+        if (clusterMode) {
+            if (this.options.useRedisAdaptor) {
+                this.LogEx(`Using Redis for socket.io cluster management (worker)`);
+                if (this.options.redisAdaptorUrl) {
+                    this.LogEx(`Redis url: [${this.options.redisAdaptorUrl}]`);
+                    this.#redisClient = createClient({url: this.options.redisAdaptorUrl});
+                } else {
+                    this.LogEx(`Redis url: [localhost]`);
+                    this.#redisClient = createClient();
+                }
+                await this.#redisClient.connect();
+                this.#io.adapter(createAdapter(this.#redisClient) as any);
+                this.LogEx(`Redis successfully connected.`);
+            } else {
+                this.#io.adapter(clusterCreateAdapter() as any);
+                this.LogEx(`Using nodejs cluster mode for socket.io cluster management`);
+            }
+        } else {
+            this.LogEx(`Not using any adaptors for socket.io cluster management.}`);
+        }
+
+        // To use a seperate socket server, the code below can be applied.
+        // this.#io = require("socket.io")(options);
+        // this.#io.adapter(createAdapter());
+        // this.#io.listen(3006);
+        // LogEx(`socket.io init`);
+
+        this.#io.engine.on("connection_error", (err) => {
+            this.LogEx(err.req); // the request object
+            this.LogEx(err.code); // the error code, for example 1
+            this.LogEx(err.message); // the error message, for example "Session ID unknown"
+            this.LogEx(err.context); // some additional error context
+        });
+    }
+
+    #GetTLSOptions = (): JSONObject => {
+        return {
+            key: fs.readFileSync(this.options.httpsServerKeyPath),
+            cert: fs.readFileSync(this.options.httpsServerCertificatePath)
+        };
+    }
+
+    #SetupExpressServer = async (useTls: boolean): Promise<void> => {
+        if (useTls) {
+            this.#httpServer = createServerHttps(this.#GetTLSOptions(), (this.#expressServer as STSExpressServer).App);
+        } else {
+            this.#httpServer = createServer((this.#expressServer as STSExpressServer).App);
+        }
+        if (this.options.prometheusSupport === true) {
+            this.#SetupPrometheusEndPoints((this.#expressServer as STSExpressServer).App);
+        }
+        if (this.options.wssServer === true) {
+            await this.#SetupWSSServer(true);
+        }
+        // https://stackoverflow.com/questions/21342828/node-express-unix-domain-socket-permissions
+        //@@httpServer.listen('/tmp/stsrest01.sock').on('listening', () =>
+        //@@httpServer.listen('/var/run/sts/stsrest01.sock').on('listening', () =>
+        this.#httpServer.listen(this.options.listenPort, () => { 
+            //@@chmodSync(this.options.port, 511);
+        }).on('listening', () =>
+        {
+            this.LogEx(`live on ${this.options.endpoint}:${this.options.listenPort}${this.options.apiRoot}`);
+        });
+    }
+
+    #SetupTCPRawServer = async (): Promise<void> => {
+        // The second parameter is the automatic listener for the secureConnection event from the tls.Server class
+        this.#httpServer = tls.createServer(this.#GetTLSOptions(), this.#SetupTLSServer);
+        this.#httpServer.listen(this.options.listenPort, 'stscore.stsmda.org', () => { 
+            console.log('TCP Server is running on port ' + this.options.listenPort + '.');
+        }).on('listening', () =>
+        {
+            this.LogEx(`TCP live on ${this.options.endpoint}:${this.options.listenPort}${this.options.apiRoot}`);
+        });
+    }
+
+    #SetupJSONRPCServer = async (): Promise<void> => {
+        const jaysonServer = new jayson.server();
+        // Supported methods here - move somewhere else ...
+        jaysonServer.method("add", function(args: any, callback: any) {
+            callback(null, args[0] + args[1]);
+        });
+        this.#httpServer = jaysonServer.tls(this.#GetTLSOptions());
+        (this.#httpServer as tls.Server).on('secureConnection', this.#SetupRPCServer);
+        this.#httpServer.listen(this.options.listenPort, 'stscore.stsmda.org', () => { 
+            console.log('JSON RPC 2.0 Server is running on port ' + this.options.listenPort + '.');
+        }).on('listening', () =>
+        {
+            this.LogEx(`JSON RPC 2.0 live on ${this.options.endpoint}:${this.options.listenPort}${this.options.apiRoot}`);
+        });
+    }
+
+    ProcessTerminating = async (): Promise<void> => {
+        return;
+    }
+
+    // Terminate in order;
+    // forked worker threads (send signal)
+    // De-Register service
+    // systeminformation observers
+    // instrument timers (gauge etc.)
+    // publisher
+    // terminate UI (if loaded)
+    Terminate = async (clusterMode: boolean, clusterPerformExit: boolean, signal?: any): Promise<void> => {
+        if (this.#shuttingDown === false) {
+            this.#shuttingDown = true;
+
+            await this.ProcessTerminate();
+
+            await this.ProcessTerminating();
+
+            if (!clusterMode) {
+                if (this.GetUIController() !== null)
+                {
+                    this.LogEx('Destroy the user interface controller.');
+                    this.GetUIController().DestroyUI();
+                }
+        
+                if (signal) {
+                    this.LogEx(this.GetSignalColour(signal)(`Main Process (singleprocess): ${process.pid} received signal: ${signal}`));
+                } else {
+                    this.LogEx(this.GetSignalColour(null)(`Main Process (singleprocess): ${process.pid} received Terminate without signal.`));
+                }
+            }
+
+            if (this.options.wssServer === true && this.#io !== null)
+            {
+                this.LogEx(`Disconnect Sockets.`);
+                if (this.socketIoHelper !== null) {
+                    this.socketIoHelper.DisconnectSockets();
+                } else {
+                    this.#io.disconnectSockets();
+                }
+                this.socketIoHelper = null;
+                this.#io = null;
+                // Note that this.#redisClient.disconnect() is not required becuase DisconnectSockets performs this action.
+            }
+
+            if (this.#httpServer) {
+                if (this.options.serverType === STSServerType.TCPRAW_TLS) {
+                    this.#sockets.forEach((socket: net.Socket, index, array) => {
+                        this.LogEx(`TCP Socket destroy, remote address: [${socket.remoteAddress}], remote port: [${socket.remotePort}]`.yellow);
+                        socket.destroy();
+                        //socket.end();
+                    });
+                }
+                this.LogEx(`Closing httpServer.`);
+                await this.#httpServer.close();
+            }
+            
+            if (this.options.useDatabase) {
+                this.LogEx(`Ending database connections and pools.`);
+                await this.TerminateDatabase();
+                //await this.accessLayer.enddatabase();
+            }
+
+            if (clusterMode) {
+                this.LogEx(`Performing exit value: [${clusterPerformExit}]`);
+                if (clusterPerformExit) {
+                    this.LogEx(`Process will self terminate with process.exit(0).`);
+                } else {
+                    this.LogEx(`Child process will not self terminate. Terminate will be handled by master process.`);
+                }
+            }
+
+            if (this.InstrumentController && this.InstrumentController.Workers.length > 0) {
+                this.LogEx(`Ending publisher.`);
+                setTimeout(() => {
+                    if (this.InstrumentController && this.InstrumentController.Workers.length > 0) {
+                        this.InstrumentController.InstrumentTerminate();
+                    }
+                }, 100);
+            }
+
+            //@@ always return here appears to always cleanly exit
+            // and cleanly exit from socket.io cluster adaptor
+            // without return here, socket.io cluster adaptor terminates in an error state
+            // as the implementation relies on cluster.on to send messages to worker threads
+            // but these have already been closed from the process.exit(0) below.
+
+            await Sleep(1000); // Allow socket.io time to clean-up
+
+            if (clusterMode) {
+                if (clusterPerformExit) {
+                    setTimeout(() => {
+                        process.exit(0);
+                    }, 0);
+                } 
+            } else {
+                if (this.options.processExitOnTerminate && this.options.processExitOnTerminate === true) {
+                    setTimeout(() => {
+                        this.LogEx(`Performing process.exit(0).`);
+                        process.exit(0);
+                    }, this.#masterProcessExitTime); // Give the workers time to terminate gracefully
+                } else {
+                    this.LogEx(`Performing process.exit(0) - Immediate.`);
+                }
+            }
+        } else {
+            this.LogEx(`Process already terminating.`);
+        }
+    }
+
+    SetupSTSServer = async(): Promise<void> => {
+        switch (this.options.serverType) {
+        case STSServerType.EXPRESS :
+            await this.#SetupExpressServer(false);
+            break;
+        case STSServerType.EXPRESS_TLS :
+            await this.#SetupExpressServer(true);
+            break;
+        case STSServerType.TCPRAW_TLS :
+            await this.#SetupTCPRawServer();
+            break;
+        case STSServerType.JSONRPC2_TLS :
+            await this.#SetupJSONRPCServer();
+            break;
+        }
+    }
+}