@nsshunt/stsappframework 3.1.241 → 3.2.1

package/src/process/workerprocessbase.ts

@@ -0,0 +1,224 @@
+/* eslint @typescript-eslint/no-explicit-any: 0 */  // --> OFF
+import chalk from 'chalk';
+
+import { Gauge, InstrumentGaugeTelemetry } from '@nsshunt/stsobservability'
+import { JSONObject } from '@nsshunt/stsutils'
+import { IPCMessage, IPCMessages, IPCMessagePayload, IPCMessageCommand, IWorkerProcessBase, ProcessOptions } from './../commonTypes'
+import { ServerProcessBase } from './serverprocessbase'
+
+import { v4 as uuidv4 } from 'uuid';
+
+import { goptions } from '@nsshunt/stsconfig'
+
+import si from 'systeminformation' // https://systeminformation.io/
+import { GetFirstNetworkInterface } from './../network'
+
+/**
+ * todo
+ * @typedef {Object} options - todo
+ * @property {boolean} [wssServer=false] - Create a web socket server on this worker instance
+ */
+export class WorkerProcessBase extends ServerProcessBase implements IWorkerProcessBase {
+    #inFlightMessage: IPCMessages = { }
+    #requestResponseMessageTimeout = 5000; //@@ config
+    #pingTimeout: NodeJS.Timeout | null = null;
+
+    constructor(options: ProcessOptions) {
+        super(options);
+    }
+
+    // eslint-disable-next-line  @typescript-eslint/no-unused-vars
+    ReceivedMessageFromMaster(msg: any) {
+        // Override in subclass if required
+    }
+
+    override CollectAdditionalTelemetry(): void {
+        this.httpServer.getConnections((error: any, count: any) => {
+            //@@this.instruments[Gauge.CONNECTION_COUNT_GAUGE].val = count;
+            this.UpdateInstrument(Gauge.CONNECTION_COUNT_GAUGE, {
+                val: count
+            } as InstrumentGaugeTelemetry);
+        });
+    }
+
+    #SendMessageToParentProcess = (message: IPCMessagePayload): Promise<JSONObject> => {
+        return new Promise((resolve, reject) => {
+            if (this.#inFlightMessage[message.id]) {
+                reject(`Message with id: [${message.id}] already exists within the Request/Response record structure`);
+            } else {
+                this.#inFlightMessage[message.id] = {
+                    iPCMessagePayload: { ...message },
+                    cb: () => {
+                        const detail: JSONObject = this.#inFlightMessage[message.id].iPCMessagePayload.responseDetail as JSONObject
+                        clearTimeout(this.#inFlightMessage[message.id].timeout);
+                        setTimeout(() => {
+                            delete this.#inFlightMessage[message.id];
+                        }, 0).unref();
+                        //this.#LogDebugMessage(chalk.green(`Resolving response message with id: [${message.id}] from parent process via IPC. Details: [${JSON.stringify(this.#inFlightMessage[message.id].iPCMessagePayload)}]`));
+                        resolve(detail);
+                    },
+                    timeout: setTimeout(() => {
+                        setTimeout(() => {
+                            delete this.#inFlightMessage[message.id];
+                        }, 0).unref();
+                        //this.#LogDebugMessage(chalk.red(`Timeout has occurred after: [${this.#requestResponseMessageTimeout}]ms with message id: [${message.id}]. Details: [${JSON.stringify(this.#inFlightMessage[message.id].iPCMessagePayload)}]`));
+                        reject('Did not receive response form parent process.');
+                    }, this.#requestResponseMessageTimeout) // max message timeout allowed
+                };
+                //this.#LogDebugMessage(chalk.yellow(`Sending message with id: [${message.id}] to parent process via IPC. Details: [${JSON.stringify(this.#inFlightMessage[message.id].iPCMessagePayload)}]`));
+                (process as any).send(message);
+            }
+        });
+    }
+
+     
+    AddWorker = async (options: any): Promise<string> => {
+        const workerResponse: JSONObject = await this.#SendMessageToParentProcess({
+            requestResponse: true,
+            id: uuidv4(),
+            command: IPCMessageCommand.AddWorker,
+            requestDetail: {
+                options
+            }
+        });
+        return workerResponse.workerId;
+    }
+     
+    DeleteWorker = async (workerId: string, options: any): Promise<JSONObject> => {
+        const workerResponse: JSONObject = await this.#SendMessageToParentProcess({
+            requestResponse: true,
+            id: uuidv4(),
+            command: IPCMessageCommand.DeleteWorker,
+            requestDetail: {
+                options,
+                workerId
+            }
+        });
+        return workerResponse;
+    }
+
+    GetMasterProcessOptions = async (): Promise<JSONObject> => {
+        const parentResponse: JSONObject = await this.#SendMessageToParentProcess({
+            requestResponse: true,
+            id: uuidv4(),
+            command: IPCMessageCommand.GetConfig,
+            requestDetail: { }
+        });
+        return parentResponse;
+    }
+
+    GetNumCPUs = async (): Promise<number> => {
+        // https://systeminformation.io/
+        const valueObject = {
+            cpu: '*'
+        }
+    
+        const sysinfo = await si.get(valueObject);
+        let numCPUs = 2;
+        if (goptions.useCPUs > 0) {
+            if (goptions.useCPUs >= 1) {
+                numCPUs = goptions.useCPUs;
+            } else {
+                numCPUs = Math.round(sysinfo.cpu.cores * goptions.useCPUs);
+            }
+        } else {
+            numCPUs = sysinfo.cpu.physicalCores;
+        }
+        return numCPUs;
+    }
+
+    GetSystemTelemetry = async () => {
+        // https://systeminformation.io/
+        const valueObject = {
+            system: '*',
+            osInfo: '*',
+            cpu: '*',
+            mem: '*',
+            dockerInfo: '*',
+            //dockerImages: '*',
+            dockerContainers: '*',
+        }
+    
+        const sysinfo = await si.get(valueObject);
+        const numCPUs = await this.GetNumCPUs();
+        const hostname = sysinfo.osInfo.hostname;
+        const hostaddr = GetFirstNetworkInterface();
+
+        const promArray: Promise<any>[] = [ ];
+    
+        sysinfo.dockerContainers.forEach((dc: { id: string; }) => {
+            const dcs = promArray.push(si.dockerContainerStats(dc.id));
+            console.log(dcs);
+        });
+        const dockerContainerStats = await Promise.all(promArray);
+    
+        const sysInfo = {
+            serviceInstanceId: this.options.serviceInstanceId,
+            serviceProcessContext: this.options.serviceProcessContext,
+            hostname,
+            numCPUs,
+            hostaddr,
+            system: sysinfo.system,
+            osInfo: sysinfo.osInfo,
+            cpu: sysinfo.cpu,
+            mem: sysinfo.mem,
+            dockerInfo: sysinfo.dockerInfo,
+            dockerContainers: sysinfo.dockerContainers,
+            dockerContainerStats
+        }
+
+        return sysInfo;
+    }
+
+    override SetupServerEx = async (): Promise<boolean> => {
+        await super.SetupServerEx();
+
+        process.on('message', async (msg: any) => {
+            if (msg.requestResponse) {
+                const iPCMessagePayload: IPCMessagePayload = msg as IPCMessagePayload;
+                if (iPCMessagePayload.id) {
+                    if (this.#inFlightMessage[iPCMessagePayload.id]) {
+                        const responseMessage: IPCMessage = this.#inFlightMessage[iPCMessagePayload.id];
+                        responseMessage.iPCMessagePayload.responseDetail = { ...iPCMessagePayload.responseDetail }
+                        responseMessage.cb();
+                    } else {
+                        throw new Error(`Could not find Request/Response message with id: [${iPCMessagePayload.id}]`);
+                    }
+                } else {
+                    throw new Error(`Message does not have id attribute. [${JSON.stringify(iPCMessagePayload)}]`);
+                }
+                return;
+            }
+            if (msg.command) //@@ constants
+            {
+                switch (msg.command)
+                {
+                case 'Terminate' :
+                    this.LogInfoMessage(chalk.grey(`Received ` + chalk.bold.italic(`Terminate`) + ` message from master thread`));
+                    await this.Terminate(true, false); // Don't kill the child process here, the master will take care of that ...
+                    break;
+                case 'TerminateAndKill' :
+                    this.LogInfoMessage(chalk.grey(`Received ` + chalk.bold.italic(`Terminate`) + ` message from master thread`));
+                    await this.Terminate(true, true);
+                    break;
+                case 'Message' :
+                    //this.LogInfoMessage(chalk.grey(`Received ` + chalk.bold.italic(`Message`) + ` message from master thread`));
+                    this.ReceivedMessageFromMaster(msg.data);
+                    break;
+                case 'Response' : // General response to a req/response interaction
+                    //msg.details
+                }
+            }
+        });
+
+        return true;
+    };
+
+    override ProcessTerminate(): Promise<void> {
+        //this.#redisMessageHandler?.off()
+        if (this.#pingTimeout) {
+            clearTimeout(this.#pingTimeout);
+        }
+        return super.ProcessTerminate();
+    }
+}

package/src/publishertransports/publishTransportUtils.ts

@@ -0,0 +1,53 @@
+import { ModelDelimeter } from '@nsshunt/stsutils'
+import { IServiceProcessContext } from './../commonTypes'
+
+export class PublishTransportUtils
+{
+    static GetServiceContext(nid: string): IServiceProcessContext {
+        // Return null if no context is required
+        //return null;
+        if (nid === ModelDelimeter.ROOT) {
+            return { root: ModelDelimeter.ROOT };
+        }
+        const context: IServiceProcessContext = { nid };
+        const idsplit = nid.split(ModelDelimeter.SEPERATOR);
+
+        const serviceId = idsplit[0];
+        const serviceInstanceId = idsplit[1].split(ModelDelimeter.NID_SEPERATOR)[0];
+        const serviceInstanceProcessId = idsplit[1].split(ModelDelimeter.NID_SEPERATOR)[1];
+
+        const serviceIdSplit = serviceId.split(ModelDelimeter.COMPONENT_SEPERATOR);
+        const serviceName =serviceIdSplit[0]; // Defined in .env or environment variable
+        const serviceVersion = serviceIdSplit[1]; // Defined in .env or environment variable
+
+        context.serviceId = serviceId; // Service name and the service version
+
+        if (typeof serviceInstanceId != 'undefined') {
+            const serviceInstanceIdSplit = serviceInstanceId.split(ModelDelimeter.COMPONENT_SEPERATOR);
+            const sid = serviceInstanceIdSplit[0];
+            const hostName = serviceInstanceIdSplit[1];
+
+            context.serviceInstanceId = serviceInstanceId; // Service unique id and the host name
+            context.sid = sid; // Service unique id only, does not include host
+            context.hostName = hostName; // Service Instance Host Name
+
+            if (typeof serviceInstanceProcessId != 'undefined') {
+                const serviceInstanceProcessIdSplit = serviceInstanceProcessId.split(ModelDelimeter.COMPONENT_SEPERATOR);
+                const pid = parseInt(serviceInstanceProcessIdSplit[0]);
+                const ppid = parseInt(serviceInstanceProcessIdSplit[1]);
+                const isMaster = (pid === ppid ? true : false);
+
+                context.serviceInstanceProcessId = serviceInstanceProcessId; // Service Instance OS Process ID (parent process id)
+                context.pid = pid; // Service Instance OS Process ID (parent process id)
+                context.ppid = ppid; // Service Instance OS Parent Process ID if worker. Will be pid if parent process.
+                context.isMaster = isMaster; // True if is Master Process
+                context.isWorker = !isMaster; // True if is Worker Process
+                context.serviceName = serviceName; // Defined in .env or environment variable
+                context.serviceVersion = serviceVersion; // Defined in .env or environment variable
+                context.serviceInstanceId = serviceInstanceId; // unique service id with host
+            }
+        }
+
+        return context;
+    }
+}

package/src/route/stslatencyroute.ts

@@ -0,0 +1,15 @@
+import { STSRouterBase } from './stsrouterbase'
+import { STSLatencyController } from './../controller/stslatencycontroller'
+import { IProcessBase } from './../commonTypes'
+
+export class STSLatencyRoute extends STSRouterBase {
+    constructor(stsApp: IProcessBase)
+    {
+        super(stsApp);
+
+        const latencyController = new STSLatencyController(stsApp);
+        const { stslatency } = latencyController;
+
+        this.router.get('/latency', stslatency.bind(latencyController)); // Create new test
+    }
+}

package/src/route/stsrouterbase.ts

@@ -0,0 +1,21 @@
+/* eslint @typescript-eslint/no-explicit-any: 0 */  // --> OFF
+import { Router } from 'express';
+import { IProcessBase } from './../commonTypes';
+
+export abstract class STSRouterBase {
+    #router: Router;
+    #stsApp: any;
+
+    constructor(stsApp: IProcessBase) {
+        this.#stsApp = stsApp;
+        this.#router = Router();
+    }
+
+    get router(): Router {
+        return this.#router;
+    }
+
+    get stsApp(): IProcessBase {
+        return this.#stsApp
+    }
+}

package/src/stsexpressserver.ts

@@ -0,0 +1,137 @@
+/* eslint @typescript-eslint/no-explicit-any: 0, @typescript-eslint/no-unused-vars: 0 */  // --> OFF
+import express, { Express, Request, Response, NextFunction } from 'express';
+
+import cors from 'cors';
+import cookieParser from 'cookie-parser';
+
+import { RequestLoggerMiddleware, RequestLoggerMiddlewareEventName } from '@nsshunt/stsobservability'
+
+import { goptions } from '@nsshunt/stsconfig'
+
+import { IProcessBase, ProcessOptions } from './commonTypes'
+import { Gauge, InstrumentGaugeTelemetry, InstrumentHistogramTelemetry, InstrumentVelocityTelemetry } from '@nsshunt/stsobservability'
+
+import { ISocketRecord, ServerNetworkMiddleware, ServerNetworkMiddlewareEventName } from './middleware/serverNetworkMiddleware'
+
+export class STSExpressServer {
+    #app: Express;
+
+    constructor(options: ProcessOptions, stsApp: IProcessBase)
+    {
+        // Standard RequestLoggerMiddleware for all STS services
+        const requestLoggerMiddleware = new RequestLoggerMiddleware({ 
+            ignoresocketio: goptions.ignoresocketio 
+        });
+
+        const serverNetworkMiddleware = new ServerNetworkMiddleware({ 
+            name: (stsApp.options as ProcessOptions).serviceName,
+            outputDebug: false,
+            logger: stsApp.options?.logger
+        });
+
+        serverNetworkMiddleware.on(ServerNetworkMiddlewareEventName.UpdateInstrument_SERVER_NET_VAL, (data: ISocketRecord) => {
+            //this.#LogInfoMessage(chalk.magenta(`serverNetworkMiddleware: [${JSON.stringify(data)}]`));
+            stsApp.UpdateInstrument(Gauge.NETWORK_RX_GAUGE, {
+                Inc: data.requestBytesRead
+            } as InstrumentGaugeTelemetry);
+
+            stsApp.UpdateInstrument(Gauge.NETWORK_TX_GAUGE, {
+                Inc: data.requestBytesWritten
+            } as InstrumentGaugeTelemetry);
+        });
+
+        requestLoggerMiddleware.on(RequestLoggerMiddlewareEventName.UpdateInstrument_AR_INC, () => {
+            stsApp.UpdateInstrument(Gauge.ACTIVE_REQUEST_GAUGE, {
+                Inc: 1
+            } as InstrumentGaugeTelemetry);
+        });
+
+        requestLoggerMiddleware.on(RequestLoggerMiddlewareEventName.UpdateInstrument_AR_DEC, () => {
+            stsApp.UpdateInstrument(Gauge.ACTIVE_REQUEST_GAUGE, {
+                Dec: 1
+            } as InstrumentGaugeTelemetry);
+        });
+
+        requestLoggerMiddleware.on(RequestLoggerMiddlewareEventName.UpdateInstrument_DH_VAL, (timeInMs) => {
+            stsApp.UpdateInstrument(Gauge.DURATION_HISTOGRAM_GAUGE, {
+                val: timeInMs
+            } as InstrumentHistogramTelemetry);
+        });
+
+        requestLoggerMiddleware.on(RequestLoggerMiddlewareEventName.UpdateInstrument_D_VAL, (timeInMs) => {
+            stsApp.UpdateInstrument(Gauge.DURATION_GAUGE, {
+                val: timeInMs
+            } as InstrumentHistogramTelemetry);
+        });
+
+        requestLoggerMiddleware.on(RequestLoggerMiddlewareEventName.UpdateInstrument_RC_INC, () => {
+            stsApp.UpdateInstrument(Gauge.REQUEST_COUNT_GAUGE, {
+                Inc: 1
+            } as InstrumentGaugeTelemetry);
+        });
+
+        requestLoggerMiddleware.on(RequestLoggerMiddlewareEventName.UpdateInstrument_V_INC, () => {
+            if (stsApp.InstrumentExists(Gauge.VELOCITY_GAUGE)) {
+                stsApp.UpdateInstrument(Gauge.VELOCITY_GAUGE, {
+                    Inc: 1
+                } as InstrumentVelocityTelemetry);
+            }
+        });
+
+        const app = express();
+        this.#app = app;
+
+        // https://stackoverflow.com/questions/19743396/cors-cannot-use-wildcard-in-access-control-allow-origin-when-credentials-flag-i
+        // See answer from Christoph Hansen is thread above ...
+        // https://expressjs.com/en/resources/middleware/cors.html
+        const corsConfig = {
+            credentials: true,
+            origin: true,
+        };
+
+        app.use(cors(corsConfig));
+
+        app.use(cookieParser())
+
+        app.use(express.urlencoded({ extended: false, limit: goptions.maxPayloadSize }));
+		
+        app.use(express.json({limit: goptions.maxPayloadSize}));
+
+        // Allow the static SPA to also be servered by this app
+        //app.use('/static', express.static(path.join(__dirname, 'public')))
+        if (options.expressServerRouteStaticFactory) {
+            options.expressServerRouteStaticFactory(app, stsApp);
+        }
+
+        app.use(requestLoggerMiddleware.Middleware.bind(requestLoggerMiddleware));
+
+        app.use(serverNetworkMiddleware.Middleware.bind(serverNetworkMiddleware));
+
+        if (options.expressServerRouteFactory) {
+            options.expressServerRouteFactory(app, stsApp);
+        }
+
+        // Express error handling fall-back
+        app.use(function(err: any, req: Request, res: Response, next: NextFunction) {
+            //@@ add to errors metric here - perhaps break down by type
+
+            if (err) {
+                //@@this.#LogInfoMessage(err);
+                res.status(err.status).send(err);
+            } else {
+                next();
+            }
+            /*
+			if(err.name === 'UnauthorizedError') {
+				res.status(401).send(err);
+			} else {
+				next();
+			}
+			*/
+        });
+    }
+
+    get App(): Express {
+        return this.#app;
+    }
+}

package/src/validation/errors.ts

@@ -0,0 +1,6 @@
+export const AppFrameworkErrorCode = Object.freeze({
+    APPFRAMEWORK_MISSING_PERMISSION: {
+        code: 'STS_AF_0001',
+        description: 'STS_AF_0001: Missing Permission(s).'
+    },
+});

package/src/vitesttesting/appConfig.ts

@@ -0,0 +1,111 @@
+import { v4 as uuidv4 } from 'uuid';
+import express from 'express';
+
+import { goptions } from '@nsshunt/stsconfig'
+
+import { ProcessOptions, IProcessBase } from './..'
+
+import { STSExpressRouteFactory } from './server'
+
+import winston from 'winston'
+
+/*
+// nid: `${goptions.rest01servicename} @ ${goptions.rest01serviceversion}  |  ${this.options.globalServiceData.serviceInstanceId} @ ${os.hostname()}   ^   ${process.pid} @ ${(cluster.isMaster ? process.pid : process.ppid)}`,
+//                        <serviceId>                                                                 <serviceInstanceId>                                                       <serviceInstanceProcessId>
+//              <serviceName>                    <serviceVersion>                                  <sid>                              <hostName>               <pid>                                     <ppid>
+//                                                  << ............... Static Nid ............... >>                                                                    << ............... Dynamic Nid ............... >>
+// Note: The final nid will NOT contain the NID_SEPERATOR character. This will be replaced with the SEPERATOR character.
+const Context = (isMaster, serviceInstanceId) => {
+    return {
+        nid: `\
+${goptions.rest01servicename}${ModelDelimeter.COMPONENT_SEPERATOR}${goptions.rest01serviceversion}\
+${ModelDelimeter.SEPERATOR}\
+${serviceInstanceId}${ModelDelimeter.COMPONENT_SEPERATOR}${os.hostname()}\
+${ModelDelimeter.NID_SEPERATOR}\
+${process.pid}${ModelDelimeter.COMPONENT_SEPERATOR}${(isMaster ? process.pid : process.ppid)}`
+    }
+}
+*/
+
+export function ServiceConfigOptions(clusterMode: boolean, isMaster: boolean): ProcessOptions {
+    winston.format.combine(
+        winston.format.colorize(),
+        winston.format.simple()
+    );
+
+    const logger = winston.createLogger({
+        level: 'debug',
+        format: winston.format.combine(
+            winston.format.colorize(),
+            winston.format.simple()
+        ),
+        transports: [
+            new winston.transports.Console()
+        ]
+    });
+
+    const publisherLogger = winston.createLogger({
+        level: 'info',
+        format: winston.format.combine(
+            winston.format.colorize(),
+            winston.format.simple()
+        ),
+        transports: [
+            new winston.transports.Console()
+        ]
+    });
+
+    if (isMaster === true) {
+        const serviceInstanceId = uuidv4();
+        const data: ProcessOptions = {
+            clusterMode: clusterMode,
+            wssServer: true,
+            useLatency: true,
+            httpsServerKeyPath: goptions.httpsserverkeypath,
+            httpsServerCertificatePath: goptions.httpsservercertpath,
+            processExitOnTerminate: true,
+            serviceInstanceId: serviceInstanceId,
+            useDatabase: false,
+
+            isMaster: isMaster,
+            endpoint: goptions.rest01endpoint,
+            apiRoot: goptions.rest01apiroot,
+            listenPort: goptions.rest01hostport,
+            port: goptions.rest01port,
+            prometheusSupport: goptions.rest01prometheussupport,
+            prometheusClusterPort: goptions.rest01prometheusclusterport,
+            serviceName: goptions.rest01servicename,
+            serviceVersion: goptions.rest01serviceversion,
+            consoleLogging: false,  //@@
+            instrumentLogging: true,
+
+            instrumentationObservationInterval: goptions.instrumentationObservationInterval,
+            instrumentationTimeWindow: goptions.instrumentationTimeWindow,
+
+            useSocketIoRedisAdaptor: true,
+            socketIoRedisAdaptorUrl: goptions.socketIoRedisAdaptorUrl,
+            //socketIoRedisAdaptorUrl: 'redis://192.168.50.5:6379',
+            //workerExec: './dist/testing/app.js',
+
+            logger: logger,
+            publisherLogger: publisherLogger,
+            publishInterval: goptions.publishinterval
+        }
+
+        if (!clusterMode) {
+            data.expressServerRouteFactory = (app: express.Express, stsApp: IProcessBase) => {
+                return new STSExpressRouteFactory(app, stsApp);
+            }
+        }
+    
+        return data;
+    } else {
+        const data: ProcessOptions = JSON.parse(process.env['STS_GSD_SII'] as string) as ProcessOptions;
+        data.expressServerRouteFactory = (app: express.Express, stsApp: IProcessBase) => {
+            return new STSExpressRouteFactory(app, stsApp);
+        }
+        data.logger = logger;
+        data.publisherLogger = publisherLogger;
+        return data;
+    }
+}