@nsshunt/stsappframework 3.1.130 → 3.1.132

package/src/ipcMessageProcessorWorker.ts

@@ -39,7 +39,8 @@ export class IPCMessageProcessorWorker {
             },
             messageReceiverStop: (options: any) => {
                 process.off('message', (payload) => this.#ipcMessageManager.ProcessMessage(payload, options));
-            }
+            },
+            groups: [ ]
         }
         this.#ipcMessageManager = new IPCMessageManager(ipcMessageManagerOptions);
     }

package/src/redisMessageHandler.test.ts

@@ -39,7 +39,8 @@ describe.skip("Redis Message Handler Test", () =>
                 logger: defaultLogger,
                 role: 'SERVER',
                 redisUrl: ioRedisMessageProcessorUrl, // goptions.imRedisMessageProcessorUrl,
-                namespace: 'redistestingstsframework'
+                namespace: 'redistestingstsframework',
+                groups: [ ]
             });
             this.#r1.Start();
 
@@ -87,7 +88,8 @@ describe.skip("Redis Message Handler Test", () =>
                 logger: defaultLogger,
                 role: 'CLIENT',
                 redisUrl: ioRedisMessageProcessorUrl, // goptions.imRedisMessageProcessorUrl,
-                namespace: 'redistestingstsframework'
+                namespace: 'redistestingstsframework',
+                groups: [ ]
             });
             this.#r1.Start();
     

package/src/redisMessageHandler.ts

@@ -8,6 +8,7 @@ import { IPCMessageManager, IPCMessageManagerOptions } from './ipcMessageManager
 import { Redis, RedisOptions } from "ioredis";
 
 import chalk from 'chalk';
+import { AggregateSteps } from "redis";
 
 const REQUEST_CHANNEL = '__STS__SVC_stsappframework_request'
 const RESPONSE_CHANNEL = '__STS__SVC_stsappframework_response'
@@ -17,6 +18,7 @@ export interface IRedisAdminManagerOptions {
     logger: ISTSLogger
     role: 'SERVER' | 'CLIENT'
     namespace: string
+    groups: string[]
 }
 
 export interface IClientRecord {
@@ -24,6 +26,7 @@ export interface IClientRecord {
     clientConnected: Date
     pingCount: number
     timeout: NodeJS.Timeout
+    groups: string[]
 }
 
 export interface IEventPayload {
@@ -78,19 +81,32 @@ export class RedisMessageHandler extends TinyEmitter {
         if (this.#options.role.localeCompare('CLIENT') === 0) {
             const ping = () => {
                 this.#pingTimeout = setTimeout(() => {
-                    this.emit('ping', this.#ipcMessageManager?.id, (response: any) => { });
+                    let group = '';
+                    if (process.pid % 2 === 0) {
+                        group = 'even';
+                    } else {
+                        group = 'odd';
+                    }
+                    this.emit('ping', {
+                        id: this.#ipcMessageManager?.id,
+                        groups: [
+                            group
+                        ]
+                    }, (response: any) => { });
                     ping();
                 }, 1000).unref();
             }
             ping();
         } else {
-            this.on('ping', (id: string, callback: any) => {
+            this.on('ping', (pingData: JSONObject, callback: any) => {
+                const { id, groups } = pingData;
                 if (this.#clients[id]) {
                     clearTimeout(this.#clients[id].timeout);
                     this.#clients[id].pingCount++;
                     this.#clients[id].timeout = setTimeout(() => {
                         delete this.#clients[id];
                     }, 2000);
+                    this.#clients[id].groups = groups;
                 } else {
                     this.#clients[id] = {
                         id,
@@ -98,7 +114,8 @@ export class RedisMessageHandler extends TinyEmitter {
                         pingCount: 0,
                         timeout: setTimeout(() => {
                             delete this.#clients[id];
-                        }, 2000)
+                        }, 2000),
+                        groups
                     }
                 }
                 callback('ok');
@@ -122,10 +139,11 @@ export class RedisMessageHandler extends TinyEmitter {
     SetupPrimary = () => {
         const ipcMessageManagerOptions: IPCMessageManagerOptions = {
             logger: this.#options.logger,
-            requestResponseMessageTimeout: 2000,
+            requestResponseMessageTimeout: 5000,
             namespace: this.#options.namespace,
             role: this.#options.role,
             messageSender: this.#messageSender,
+            groups: this.#options.groups,
             // This method is used to calculate if all responses have been received from multiple clients (broadcast)
             // returns true/false.
             ProcessResponseMessage: this.#ProcessResponseMessage,
@@ -157,26 +175,55 @@ export class RedisMessageHandler extends TinyEmitter {
         // Now check if we have all responses ...
         let allFound = false;
 
-        Object.keys(responses).forEach(r => {
-            if (responses[r].senderRole.localeCompare('CLIENT') === 0) {
+        for (const [responseId, response] of Object.entries(responses)) {
+            if (response.senderRole.localeCompare('CLIENT') === 0) {
                 allFound = true;
+                break;
             }
-        });
+        }
         if (allFound) {
             return allFound;
         }
 
         let found = true;
-        Object.values(this.#clients).map(c => {
-            if (!responses[c.id]) {
-                found = false;
+
+        // Sender role here is SERVER
+        let requestGroup = null;
+        for (const [responseId, response] of Object.entries(responses)) {
+            if (response.requestPayload.args.length > 0 && response.requestPayload.args[0].group) {
+                requestGroup = response.requestPayload.args[0].group;
+                break;
             }
-        });
+        }
+
+        if (requestGroup) {
+            const clientsInGroup = Object.values(this.#clients).filter(c => {
+                if (c.groups.indexOf(requestGroup) === -1) {
+                    return false;
+                }
+                return true;
+            });
 
-        if (found) {
-            return true;
+            // Now make sure that all clients are in the responses
+            found = true;
+            clientsInGroup.forEach(c => {
+                if (!responses[c.id]) {
+                    found = false;
+                }
+            })
+        } else {
+            const clientsInGroup = Object.values(this.#clients)
+
+            // Now make sure that all clients are in the responses
+            found = true;
+            clientsInGroup.forEach(c => {
+                if (!responses[c.id]) {
+                    found = false;
+                }
+            })
         }
-        return false;
+
+        return found;
     }
 
     #processPayload = (payload: IIPCMessageProcessorIPCPayload, options: any): Promise<JSONObject> => {
@@ -244,13 +291,24 @@ export class RedisMessageHandler extends TinyEmitter {
 
     override emit(event: string, ...args: any[]): this {
         (async () => {
-            const retVal = await this.#ipcMessageManager?.SendMessage({
-                __eventName: event,
-                args: args.slice(0, args.length-1)
-            } as IEventPayload);            
-            // Invoke the response callback
-            args[args.length-1](retVal);
+            try {
+                const retVal = await this.#ipcMessageManager?.SendMessage({
+                    __eventName: event,
+                    args: args.slice(0, args.length-1)
+                } as IEventPayload);            
+                // Invoke the response callback
+                args[args.length-1](retVal);
+            } catch (error) {
+                this.#options.logger.error(chalk.red(`emit(): Error: [${JSON.stringify(error)}]`));
+            }
         })();
         return this;
     }
+
+    emitex = async(event: string, ...args: any[]): Promise<JSONObject> => {
+        return (this.#ipcMessageManager as IPCMessageManager).SendMessage({
+            __eventName: event,
+            args
+        } as IEventPayload);            
+    }
 }

package/src/testing/app.ts

@@ -20,6 +20,9 @@ import { v4 as uuidv4 } from 'uuid';
 import { RedisMessageHandler } from './../redisMessageHandler'
 import { goptions } from '@nsshunt/stsconfig'
 
+import si from 'systeminformation' // https://systeminformation.io/
+import { GetFirstNetworkInterface } from './../network'
+
 const sleepVal = 0;
 const maxLoop = 100;
 
@@ -105,7 +108,7 @@ const StartTestWorker = () => {
     const w1 = new IPCMessageProcessorWorker({
         logger: defaultLogger,
         namespace: 'test1',
-        requestResponseMessageTimeout: 2000,
+        requestResponseMessageTimeout: 5000,
         processPayload: async (payload) => {
             return {
                 msg1: `From WORKER: [${process.pid}]`,
@@ -118,7 +121,7 @@ const StartTestWorker = () => {
     const w2 = new IPCMessageProcessorWorker({
         logger: defaultLogger,
         namespace: 'test2',
-        requestResponseMessageTimeout: 2000,
+        requestResponseMessageTimeout: 5000,
         processPayload: async (payload) => {
             return {
                 msg2: `From WORKER: [${process.pid}]`,
@@ -131,7 +134,7 @@ const StartTestWorker = () => {
     const w3 = new IPCMessageProcessorWorker({
         logger: defaultLogger,
         namespace: 'zzz',
-        requestResponseMessageTimeout: 2000,
+        requestResponseMessageTimeout: 5000,
         processPayload: async (payload) => {
             switch (payload.requestPayload.cmd) {
             case 'cmd1' : return {
@@ -177,13 +180,97 @@ const StartTestWorker = () => {
     }, 1000);
 }
 
-const iterations = 100;
+const iterations = 0;
 const delay = 100;
 
+const GetNumCPUs = async (): Promise<number> => {
+    // https://systeminformation.io/
+    const valueObject = {
+        cpu: '*'
+    }
+
+    const sysinfo = await si.get(valueObject);
+    let numCPUs = 2;
+    if (goptions.useCPUs > 0) {
+        if (goptions.useCPUs >= 1) {
+            numCPUs = goptions.useCPUs;
+        } else {
+            numCPUs = Math.round(sysinfo.cpu.cores * goptions.useCPUs);
+        }
+    } else {
+        numCPUs = sysinfo.cpu.physicalCores;
+    }
+    return numCPUs;
+}
+
+const LogInfoMessage = (message: string) => {
+    console.log(message);
+}
+
+const LogSystemTelemetry = async () => {
+    // https://systeminformation.io/
+    const valueObject = {
+        system: '*',
+        osInfo: '*',
+        cpu: '*',
+        mem: '*',
+        dockerInfo: '*',
+        //dockerImages: '*',
+        dockerContainers: '*',
+    }
+
+    const sysinfo = await si.get(valueObject);
+    const numCPUs = await GetNumCPUs();
+    const hostname = sysinfo.osInfo.hostname;
+
+    const hostaddr = GetFirstNetworkInterface();
+    if (hostaddr !== null) {
+        LogInfoMessage(`Host Address: ${hostaddr}`);
+    } else {
+        LogInfoMessage(`Unknown Host Address.`);
+    }
+    LogInfoMessage(`Server starting with ${numCPUs} Cores/Threads`);
+
+    LogInfoMessage(`Hostname: ${hostname}`);
+    LogInfoMessage(`System: ${JSON.stringify(sysinfo.system)}`);
+    LogInfoMessage(`OS Info: ${JSON.stringify(sysinfo.osInfo)}`);
+    LogInfoMessage(`CPU: ${JSON.stringify(sysinfo.cpu)}`);
+    LogInfoMessage(`Memory: ${JSON.stringify(sysinfo.mem)}`);
+
+    const promArray: Promise<any>[] = [ ];
+
+    sysinfo.dockerContainers.forEach((dc: { id: string; }) => {
+        LogInfoMessage(dc.id);
+        const dcs = promArray.push(si.dockerContainerStats(dc.id));
+        console.log(dcs);
+    });
+    const dockerContainerStats = await Promise.all(promArray);
+
+    const sysInfo = {
+        hostname,
+        numCPUs,
+        hostaddr,
+        system: sysinfo.system,
+        osInfo: sysinfo.osInfo,
+        cpu: sysinfo.cpu,
+        mem: sysinfo.mem,
+        dockerInfo: sysinfo.dockerInfo,
+        //dockerImages: sysinfo.dockerImages,
+        dockerContainers: sysinfo.dockerContainers,
+        dockerContainerStats
+    }
+
+    console.log(sysInfo);
+
+    console.log(JSON.stringify(sysInfo));
+}
+
 if (cluster.isPrimary) {
     new MasterProcessBase(ServiceConfigOptions(true, cluster.isPrimary)).SetupServer();
     //StartTestPrimary();
 
+    LogSystemTelemetry();
+
     setTimeout(() => {
         const p: Record<string, IPCMessageHandler> = { };
         /*
@@ -206,7 +293,94 @@ if (cluster.isPrimary) {
         }
         */
 
+        /*
+        const stsServiceControl: RedisMessageHandler = new RedisMessageHandler({
+            logger: defaultLogger,
+            role: 'SERVER',
+            redisUrl: goptions.imRedisMessageProcessorUrl,
+            namespace: 'stsServiceControl',
+            groups: [ ]
+        });
+        stsServiceControl.Start();
+
+        stsServiceControl.on('ping_ex', (pingData: JSONObject, callback: any) => {
+            console.log(chalk.yellow(`${process.pid}: stsServiceControl.on(ping): [${JSON.stringify(pingData)}]`));
+            callback({pingData, dateTime: new Date().getTime()})
+        });
+
+        setInterval(() => {
+            stsServiceControl.emit('GET_SYSTEM_TELEMETRY', { }, (response: any) => {
+                console.log(chalk.green(`  ==>> ${process.pid}: Response(GET_SYSTEM_TELEMETRY): [${JSON.stringify(response)}]`));
+            });
+        }, 5000);
+        */
+
+
+
+        const r1: RedisMessageHandler = new RedisMessageHandler({
+            logger: defaultLogger,
+            role: 'SERVER',
+            redisUrl: goptions.imRedisMessageProcessorUrl,
+            namespace: 'mytestapp',
+            groups: [ ]
+        });
+        r1.Start();
+
+        const RunTest = async (eventName: string, emitobj: JSONObject) => {
+            const start = performance.now();
+            try {
+                const response = await r1.emitex(eventName, emitobj);
+                const duration = performance.now() - start;
+                console.log(chalk.cyan(`${process.pid}: Duration: ${duration} Test: [test1] Response(${chalk.bold(eventName)}): [${JSON.stringify(response)}]`));
+            } catch (error) {
+                const duration = performance.now() - start;
+                console.log(chalk.cyan(`${process.pid}: Duration: ${duration} Test: [test1] Response(${chalk.bold(eventName)}): Error: [${JSON.stringify(error)}]`));
+            }
+        }
+
+        let i = 0;
+        let group = '';
+        const maxIterations = 20;
+        const xxx = () => {
+            setTimeout(() => {
+                i++;
+
+                if (i > maxIterations) {
+                    return;
+                }
+
+                console.log(chalk.yellow(`Iteration: [${i}]`));
+
+                if (i % 2 === 0) {
+                    group = 'even'
+                } else {
+                    group = 'odd'
+                }
+
+                const promArray: Promise<void>[] = [ ];
+                const start = performance.now();
+                promArray.push(
+                    RunTest('fromprimaryredis', { group, i: i, a: 'a', b: 'b' }),
+                    RunTest('fromprimaryredis', { group: '2', i: i, a: 'a', b: 'b' }),
+                    RunTest('fromprimaryredis', { group: '4', i: i, a: 'a', b: 'b' }),
+                    RunTest('globalmessage', { i: i, a: 'a', b: 'b' })
+                );
+                (async () => {
+                    await Promise.all(promArray);
+                    const duration = performance.now() - start;
+                    console.log(chalk.green(`${process.pid}: Duration: ${duration} Test: [All Tests Run]`));
+                    console.log();
+                    xxx();
+                })();
+            }, 250);
+        }
+        setTimeout(() => xxx(), 1000);
+
+
+
+        
             
+        /*
         const r1: RedisMessageHandler = new RedisMessageHandler({
             logger: defaultLogger,
             role: 'SERVER',
@@ -224,16 +398,6 @@ if (cluster.isPrimary) {
 
         setTimeout(async () => {
             for (let i=0; i < iterations; i++) {
-                /*
-                Object.keys(p).forEach(async (pKey) => {
-                    const emitobj = { i: i, a: 'a', b: 'b', worker: pKey };
-                    console.log(chalk.green(`${process.pid}: emit event to worker: [${pKey}] fromprimary: [${JSON.stringify(emitobj)}`));
-                    p[pKey].emit('fromprimary', emitobj, (response: any) => {
-                        console.log(chalk.green(`${process.pid}: ${JSON.stringify(response)}`));
-                    });
-                });
-                */
-
 
                 const emitobj = { i: i, a: 'a', b: 'b' };
                 console.log(chalk.cyan(`${process.pid}: emit event fromprimaryredis fromprimary: [${JSON.stringify(emitobj)}`));
@@ -245,12 +409,49 @@ if (cluster.isPrimary) {
             }
 
         }, 1000);
-    }, 1000);
+        */
+    }, 2000);
 } else {
     new WorkerProcess(ServiceConfigOptions(true, cluster.isPrimary)).SetupServer();
     //StartTestWorker();
 
     setTimeout(async () => {
+
+        let group = '';
+        if (process.pid % 2 === 0) {
+            group = 'even'
+        } else {
+            group = 'odd'
+        }
+
+        let workerId = '0';
+        if (cluster.worker) {
+            console.log(chalk.blue(`Worker: [${cluster.worker?.id}]`));
+            workerId = cluster.worker?.id.toString();
+        }
+
+        const r1: RedisMessageHandler = new RedisMessageHandler({
+            logger: defaultLogger,
+            role: 'CLIENT',
+            redisUrl: goptions.imRedisMessageProcessorUrl,
+            namespace: 'mytestapp',
+            groups: [ group, workerId ]
+        });
+        r1.Start();
+
+        r1.on('fromprimaryredis', (arg1: JSONObject, callback: any) => {
+            callback( {
+                status: `PID: [${process.pid}]: response message from event = fromprimaryredis with args ${JSON.stringify(arg1)}`
+            })
+        });
+
+        r1.on('globalmessage', (arg1: JSONObject, callback: any) => {
+            callback( {
+                status: `PID: [${process.pid}]: response message from event = globalmessage with args ${JSON.stringify(arg1)}`
+            })
+        });
+
+
         /*
         const w1: IPCMessageHandler = new IPCMessageHandler({
             logger: defaultLogger,
@@ -265,6 +466,7 @@ if (cluster.isPrimary) {
         });
         */
 
+        /*
         const r1: RedisMessageHandler = new RedisMessageHandler({
             logger: defaultLogger,
             role: 'CLIENT',
@@ -282,12 +484,6 @@ if (cluster.isPrimary) {
         setTimeout(async () => {
             for (let i=0; i < iterations; i++) {
                 const id = uuidv4();
-                /*
-                console.log(chalk.yellow(`${process.pid}: emit event fromworker: [${id} ${i} Hello]`));
-                w1.emit('fromworker', id, i, 'Hello', (response: any) => {
-                    console.log(chalk.yellow(`${process.pid}: ${JSON.stringify(response)}`));
-                });
-                */
 
                 console.log(chalk.green(`${process.pid}: emit event fromworkerredis: [${id} ${i} Hello]`));
                 r1.emit('fromworkerredis', id, i, 'Hello', (response: any) => {
@@ -296,5 +492,6 @@ if (cluster.isPrimary) {
                 await Sleep(delay);
             }
         }, 1000);
-    }, 1000);
+        */
+    }, 2000);
 }

package/src/workerprocessbase.ts

@@ -12,6 +12,13 @@ import { v4 as uuidv4 } from 'uuid';
 
 import { STSTransportLoggerWinston } from './stsTransportLoggerWinston'
 
+import { RedisMessageHandler } from './redisMessageHandler';
+
+import { goptions } from '@nsshunt/stsconfig'
+
+import si from 'systeminformation' // https://systeminformation.io/
+import { GetFirstNetworkInterface } from './network'
+
 /**
  * todo
  * @typedef {Object} options - todo
@@ -20,7 +27,11 @@ import { STSTransportLoggerWinston } from './stsTransportLoggerWinston'
 export class WorkerProcessBase extends ServerProcessBase implements IWorkerProcessBase
 {
     #inFlightMessage: IPCMessages = { }
-    #requestResponseMessageTimeout = 2000; //@@ config
+    #requestResponseMessageTimeout = 5000; //@@ config
+    #redisMessageHandler: RedisMessageHandler | null = null;
+    #redisMessageHandlerPing: number = 1000;
+    #pingTimeout: NodeJS.Timeout | null = null;
+    #instanceId: string = uuidv4();
 
     constructor(options: ProcessOptions) {
         super(options);
@@ -120,14 +131,77 @@ export class WorkerProcessBase extends ServerProcessBase implements IWorkerProce
         return parentResponse;
     }
 
-    SetupServer = async () =>
-    {
+    SetupServer = async () => {
         this.SetupInstrumentation();
         setTimeout(() => {
             this.SetupServerEx();
         }, 100);
     }
 
+    GetNumCPUs = async (): Promise<number> => {
+        // https://systeminformation.io/
+        const valueObject = {
+            cpu: '*'
+        }
+    
+        const sysinfo = await si.get(valueObject);
+        let numCPUs = 2;
+        if (goptions.useCPUs > 0) {
+            if (goptions.useCPUs >= 1) {
+                numCPUs = goptions.useCPUs;
+            } else {
+                numCPUs = Math.round(sysinfo.cpu.cores * goptions.useCPUs);
+            }
+        } else {
+            numCPUs = sysinfo.cpu.physicalCores;
+        }
+        return numCPUs;
+    }
+
+    GetSystemTelemetry = async () => {
+        
+
+        // https://systeminformation.io/
+        const valueObject = {
+            system: '*',
+            osInfo: '*',
+            cpu: '*',
+            mem: '*',
+            dockerInfo: '*',
+            //dockerImages: '*',
+            dockerContainers: '*',
+        }
+    
+        const sysinfo = await si.get(valueObject);
+        const numCPUs = await this.GetNumCPUs();
+        const hostname = sysinfo.osInfo.hostname;
+        const hostaddr = GetFirstNetworkInterface();
+
+        const promArray: Promise<any>[] = [ ];
+    
+        sysinfo.dockerContainers.forEach((dc: { id: string; }) => {
+            const dcs = promArray.push(si.dockerContainerStats(dc.id));
+            console.log(dcs);
+        });
+        const dockerContainerStats = await Promise.all(promArray);
+    
+        const sysInfo = {
+            instanceId: this.#instanceId,
+            hostname,
+            numCPUs,
+            hostaddr,
+            system: sysinfo.system,
+            osInfo: sysinfo.osInfo,
+            cpu: sysinfo.cpu,
+            mem: sysinfo.mem,
+            dockerInfo: sysinfo.dockerInfo,
+            dockerContainers: sysinfo.dockerContainers,
+            dockerContainerStats
+        }
+
+        return sysInfo;
+    }
+
     SetupServerEx = async () => {
         this.ProcessStartup();
 		
@@ -192,8 +266,55 @@ export class WorkerProcessBase extends ServerProcessBase implements IWorkerProce
 
         await this.SetupSTSServer();
 
+        /*
+        this.#redisMessageHandler = new RedisMessageHandler({
+            logger: this.options.logger,
+            role: 'CLIENT',
+            redisUrl: goptions.imRedisMessageProcessorUrl,
+            namespace: 'stsServiceControl',
+            groups: [ ]
+        });
+        this.#redisMessageHandler.Start();
+
+        const ping = () => {
+            this.#pingTimeout = setTimeout(() => {
+                const pingData = {
+                    instanceId: this.#instanceId
+                }
+                if (this.#redisMessageHandler) {
+                    this.#redisMessageHandler.emit('ping_ex', pingData, (response: any) => {
+                        console.log(chalk.cyan(`  ==>> ${process.pid}: Response(ping): [${JSON.stringify(response)}]`));
+                    });
+                }
+                ping();
+            }, this.#redisMessageHandlerPing).unref();
+        }
+        ping();
+        */
+
+        //        this.#redisMessageHandler.on('GET_SYSTEM_TELEMETRY', async (getSystemTelemetryCommand: JSONObject, callback: any) => {
+        //          callback({ instanceId: this.#instanceId, process: process.pid, systemTelemetry: getSystemTelemetryCommand });
+        /*
+            if (getSystemTelemetryCommand.instanceId.localeCompare(this.#instanceId) === 0) {
+                const systemTelemetry = await this.GetSystemTelemetry();
+                callback({ systemTelemetry });
+            } else {
+                callback({ systemTelemetry: null });
+            }
+                */
+        //        });
+
         this.WorkerStarted();
 
         this.LogInfoMessage(chalk.green(`Worker process:${process.pid} started`));
     };
+
+    override ProcessTerminate(): Promise<void> {
+        //this.#redisMessageHandler?.off()
+        if (this.#pingTimeout) {
+            clearTimeout(this.#pingTimeout);
+        }
+        this.#redisMessageHandler?.Stop();
+        return super.ProcessTerminate();
+    }
 }