cry-ebus2 2.2.5 → 3.0.0

package/src/run/workers.ts

@@ -1,14 +1,50 @@
-import { DelayWorker } from "../delayWorker";
-import { EchoWorker } from "../echoWorker";
-import { ErrorWorker } from "../errorWorker";
-import { RejectWorker } from "../rejectWorker";
 
-import { config } from "dotenv"
-import { Sub } from "../sub";
-config()
+import { objStrToJson } from "cry-helpers";
+import { config } from "dotenv";
+import { Client } from "../client";
+import { program } from "commander"
+import { EbusRequestOptions } from "../EbusRequestOptions";
+import { exit } from "process";
 
-new EchoWorker().start()
-new DelayWorker().start()
-new RejectWorker().start()
-new ErrorWorker().start()
+config();
 
+program.version('0.0.2');
+
+let client = new Client()
+
+function exitProgram(message: string) {
+    console.error(message)
+    exit(2)
+}
+
+program
+    .description('calls a worker')
+    .option('-w, --workers', 'get workers (default)', true)
+    .option('-s, --services', 'get services instead of workers')
+    .option('-r, --restart <whom>', 'restart all workers')
+    .action(async (params) => {
+
+        try {
+
+            if (program.restart) {
+                console.log(`request restart of \x1b[32m${program.restart}\x1b[0m workers`)
+                await client.restartAllWorkers(program.restart);
+            }
+            else if (program.services) {
+                let res = await client.services()
+                console.dir(res)
+            } else {
+                let res = await client.workers()
+                console.dir(res)
+            }
+
+        } catch (err: any) {
+            console.error(`\x1b[31munexpected error\x1b[0m  ${err.message}`)
+            console.log(err)
+            process.exit(1)
+        }
+
+        process.exit(0)
+    })
+
+program.parse(process.argv);

package/src/serialize.ts

@@ -1,27 +1,26 @@
+
 import { serialize } from "cry-helpers";
-import { deflate, inflate } from "pako";
+import { deflateRaw, inflateRaw } from "pako";
 import { EbusRequestOptions } from "./EbusRequestOptions";
-
+import notepack from "notepack.io"
 
 export function pack(data: any, opts: Partial<EbusRequestOptions>)
 {
     if (opts.prepacked) return data;
     if (!opts.raw) {
-        data = serialize.pack(data)
-        if (opts.zip) data = deflate(data, { to: "string" })
-    } else {
-        if (opts.zip) data = deflate( Uint8Array.from(data) )
+        data = serialize.encode(data);
+        data = notepack.encode(data);
     }
+    if (opts.zip) data = deflateRaw(data)
     return data
 }
 
 export function unpack<T>(data: any, opts: Partial<EbusRequestOptions>): T | undefined {
     if (data === undefined) return undefined;
+    if (opts?.zip) data = inflateRaw(data)
     if (!opts.raw) {
-        if (opts.zip) data = inflate(data, { to: "string" })
-        data = serialize.unpack(data)
-    } else {
-        if (opts.zip) data = Buffer.from(inflate(data))
+        data = notepack.decode(Buffer.from(data))
+        data = serialize.decode(data)
     }
     return data as T
 }

package/src/src/notepack.d.ts

@@ -0,0 +1,2 @@
+
+declare module "notepack.io"

package/src/test/echo.ts

@@ -7,7 +7,7 @@ import { Broker, DelayWorker, unpack, pack } from "../index";
 
 describe('echo', async function () {
 
-    this.timeout(4000)
+    this.timeout(60000)
 
     new Broker().start()
     new EchoWorker().start()
@@ -15,14 +15,17 @@ describe('echo', async function () {
 
     let client = new Client()
 
+    console.log('started');
+    
+
     it('should return passed-in object', async () => {
-        let rep = await client.request<{ delay: number }>("delay", { delay: 500 })
-        expect(rep?.delay).to.eq(500)
+        let rep = await client.request<{ delay: number }>("delay", { delay: 50 })
+        expect(rep?.delay).to.eq(50)
     });
 
     it('should return passed-in object zipped', async () => {
-        let rep = await client.request<{ delay: number }>("delay", { delay: 500 }, { zip: true })
-        expect(rep?.delay).to.eq(500)
+        let rep = await client.request<{ delay: number }>("delay", { delay: 50 }, { zip: true })
+        expect(rep?.delay).to.eq(50)
     });
 
 
@@ -35,24 +38,24 @@ describe('echo', async function () {
     it('should return passed-in object raw and zipped', async () => {
         let load = Buffer.from("ABC")
         let rep = await client.request<{ delay: number }>("echo", load, { raw: true, zip: true })
-        expect(rep?.toString()).to.eq("ABC")
+        expect(Buffer.from(rep as any).toString()).to.eq("ABC")
     });
 
+
     it('should return unpacked object', async () => {
         let load = Buffer.from("ABC")
         let res = (await client.requestUpacked<{ delay: number }>("echo", load, { raw: true, zip: true, })) as unknown as EbusUnpackedResult<string>
-        let rep = unpack<string>(res.data,res.opts)
-        expect(rep?.toString()).to.eq("ABC")
+        let rep = unpack<string>(res.data, res.opts)
+        expect(Buffer.from(rep as any).toString()).to.eq("ABC")
     });
 
-
     it('should accept externally prepacked object', async () => {
         let load = { a: { b: 2 } }
         let loadPrepacked = pack(load,{ zip: true })
         let rep = (await client.request<{ a: { b:2 } }>("echo", loadPrepacked, { prepacked:true, zip: true,  receiveTimeout:30000 }))
         expect(rep!.a.b).to.eq(2)
     });
-
+    
     it('should accept externally prepacked object', async () => {
         let load = { a: { b: 2 } }
         let loadPrepacked = pack(load, { zip: true })
@@ -60,15 +63,9 @@ describe('echo', async function () {
         let rep = unpack < { a: { b: 2 } }>(res.data, res.opts)
         expect(rep!.a.b).to.eq(2)
     });
+    
 
-    it('should accept externally prepacked raw object', async () => {
-        let load = { a: { b: 2 } }
-        let loadPrepacked = pack(load, { zip: true })
-        let res = (await client.requestUpacked<{ a: { b: 2 } }>("echo", loadPrepacked, { prepacked: true, zip: true, receiveTimeout: 30000 })) as unknown as EbusUnpackedResult<{ a: { b: 2 } }>
-        let rep = unpack<{ a: { b: 2 } }>(res.data, res.opts)
-        expect(rep!.a.b).to.eq(2)
-    });
-
+    
     it('exists', () => {
         process.exit()
     })

package/src/test/pack.ts

@@ -1,9 +1,8 @@
 
 import { expect } from "chai";
-import { EbusRequestOptions } from "../EbusRequestOptions";
 import { unpack, pack } from "../index";
 
-describe('stress', async function () {
+describe('pack', async function () {
 
     let o = () => [{ o: 1, a: ["ABC"] }];
 
@@ -16,20 +15,23 @@ describe('stress', async function () {
         expect(o1[0].a[0]).to.eq("ABC")
         expect(o2[0].a[0]).to.eq("ABC")
     });
-
     it('should unpack serialized, unzipped', async () => {
-        let opts = {}
-        expect(unpack<any>(pack(o(), opts), opts)[0].a[0]).to.eq("ABC")
+        let opts = { zip: true }
+        let packed = pack(o(), opts)
+        let unpacked = unpack<any>(packed, opts)
+        expect(unpacked[0].a[0]).to.eq("ABC")
     });
     it('should unpack serialized, zipped', async () => {
         let opts = { zip: true }
-        expect(unpack<any>(pack(o(), opts), opts)[0].a[0]).to.eq("ABC")
+        let packed = pack(o(), opts)
+        let unpacked = unpack<any>(packed, opts)
+        expect(unpacked[0].a[0]).to.eq("ABC")
     });
     it('should unpack serialized, zipped', async () => {
         let opts = { zip: true, raw: true }
         let buff1 = Buffer.from( JSON.stringify(o()) )
         let buff2 = Buffer.from( JSON.stringify(o()) )
-        let res = unpack<any>(pack(buff1, opts), opts)
+        let res = Buffer.from(unpack<any>(pack(buff1, opts), opts))
         expect(res.toString()).to.equal(buff2.toString())
     });
     it('should unpack unserialized, zipped', async () => {
@@ -37,7 +39,6 @@ describe('stress', async function () {
         let buff1 = Buffer.from(JSON.stringify(o()))
         let buff2 = Buffer.from(JSON.stringify(o()))
         let res = unpack<any>(pack(buff1, opts), opts)
-        
         for (let i = 0; i < buff2.length; i++)
            expect(res[i]).to.equal(buff2[i])
 

package/src/test/performance.ts

@@ -4,16 +4,16 @@ import { EbusRequestOptions } from "../EbusRequestOptions";
 import { EchoWorker } from "../echoWorker";
 import { Broker, DelayWorker, unpack, pack } from "../index";
 
-describe('stress', async function () {
-
-    this.timeout(10000)
+describe('performance', async function () {
 
+    this.timeout(160000)
     new Broker().start()
-    new DelayWorker().start()
     new EchoWorker().start()
+    new DelayWorker().start()
+
     let client = new Client()
 
-    let requestSizes = [1, 10, 100, 1000, 10000]
+    let requestSizes = [1, 10, 100, 1000, 10000, 100000]
 
     function sizedRq(size: number) {
         let bigRequest = Array(size)
@@ -29,7 +29,8 @@ describe('stress', async function () {
     
     async function measure(name: string, opts: Partial<EbusRequestOptions>) {
         let sizedRequests = requestSizes.map(r => sizedRq(r))
-        let line  : any[] = [ name ]
+        let line: any[] = [name]
+        
         for (let req of sizedRequests) {
             let start = new Date().valueOf()
             let rep = await client.request<any>("delay", { delay: 10, req }, opts)
@@ -41,13 +42,13 @@ describe('stress', async function () {
        printLine(line)
     }   
     
-    it('should measue raw', async () => {
-        
+    it('should measure raw', async () => {
+
         printLine([ 'opts', ...requestSizes ])
-        //await measure("-raw -zipped",{ raw:false, zip: false, receiveTimeout: 4000 })
-        await measure("+raw -zipped",{ raw:true, zip: false, receiveTimeout: 4000 })
-        //await measure("-raw +zipped",{ raw:false, zip: true, receiveTimeout: 4000 })
-        await measure("+raw +zipped",{ raw:true, zip: true, receiveTimeout: 4000 })
+        await measure("-raw -zipped",{ raw:false, zip: false, receiveTimeout: 4000 })
+        // await measure("+raw -zipped",{ raw:true, zip: false, receiveTimeout: 4000 })
+        await measure("-raw +zipped",{ raw:false, zip: true, receiveTimeout: 4000 })
+        // await measure("+raw +zipped",{ raw:true, zip: true, receiveTimeout: 4000 })
 
     });
 

package/src/test/pub.ts

@@ -1,9 +1,7 @@
-import { expect } from "chai"
-import { Worker, worker } from "cluster";
 import { Client } from "../client";
 import { EchoWorker } from "../echoWorker";
 import { ErrorWorker } from "../errorWorker";
-import { Broker, DelayWorker, unpack, pack } from "../index";
+import { Broker, DelayWorker } from "../index";
 import { RejectWorker } from "../rejectWorker";
 import { Sub } from "../sub";
 

package/src/types.ts

@@ -19,4 +19,6 @@ export enum Message {
     Subscribe = "\x12",
     Publish = "\x13",
     Shutdown = "\x14",
+    ShutdownAndRestart = "\x15",
+    RestartAllWorkers = "\x16",
 }

package/src/worker.ts

@@ -1,5 +1,5 @@
 import { Log , serialize} from "cry-helpers"
-import { Dealer, Request } from "zeromq"
+import { Dealer } from "zeromq"
 import { EbusConfiguration } from "./EbusConfiguration"
 import { EbusRequestOptions } from "./EbusRequestOptions"
 import { loadConfiguration } from "./loadConfiguration"
@@ -97,27 +97,39 @@ export class Worker {
 
                 try {
                     
-                    if (type.toString() === Message.Shutdown) {
-                      
-                        log.status(`worker for ${this.service} received broker shutdown signal`)
-                        break;
-
-                    } else 
-               
-                        if (type.toString() === Message.Heartbeat) {
-                        log.trace(`worker for ${this.service} received heartbeat from broker`)
-                  
-                    } else if (type.toString() === Message.Request) {
-                            
-                        this.processing=true
-                        await this.processRequest(reqId, optBuff, type, req, client)
-                        this.processing=false
+                    switch (type.toString()) {
+                        case Message.Shutdown:
+
+                            log.status(`worker for ${this.service} received broker shutdown signal`)
+                            break;
+                        
+                        case Message.Heartbeat:
+
+                            log.trace(`worker for ${this.service} received heartbeat from broker`)
+                            break;
+                         
+                        case Message.ShutdownAndRestart:
+        
+                            log.status(`worker for ${this.service} received ShutdownAndRestart from broker`)
+                            process.exit(1)
+                            break;
+        
+                        case Message.Request:
+                                   
+                            this.processing = true
+                            await this.processRequest(reqId, optBuff, type, req, client)
+                            this.processing = false
+                            break;
+                               
+                        default:
+
+                            log.error(`worker for ${this.service} cannot process message type ${type}`, [blank1, header, type, client, blank2, reqId, optBuff, req])
+                            log.error([blank1, header, type, client, blank2, reqId, optBuff, req])
                         
-                    } else {
-                            
-                        log.error(`worker for ${this.service} cannot process  message type ${type}`)
                     }
 
+                    
+
                 } catch (err) {
                     console.error(`error in worker for '${this.service}'`)
                     console.error(err)
@@ -151,14 +163,14 @@ export class Worker {
     }
 
     // to be implemented at the actual worker
-    public async onError(err:any) 
+    public async onError(err:any, req: any, opts: any) 
     {
         console.log('worker/onError - should override');
         
     }
 
     private async processRequest(reqId: Buffer, optBuff: Buffer, type: Buffer, req: Buffer, client: Buffer) {
-        
+    
         let opts = serialize.unpack(optBuff) as Partial<EbusRequestOptions>
         log.debug(`worker for ${this.service} received request ${type} with opts`, opts)
 
@@ -176,11 +188,13 @@ export class Worker {
             rep = await Promise.race(promises)
             if (rep instanceof EbusRejectError && rep.reason === "timeout") {
                 log.warn(`request expired at worker for '${this.service}' after ${timeout} ms with opts ${serialize.pack(opts)}`)
+                console.log('opts',opts);
+                
                 log.warn(unpack(req, opts))
                 responseMessageType = Message.Timeout
-                this.onError("timeout");
+                this.onError("timeout", unpack(req, opts), opts);
             }
-        } catch (err) {
+        } catch (err:any) {
             rep = err
             if (!(rep instanceof EbusRejectError)) {
                 rep = new EbusRejectError("error", `error in worker for '${this.service}' process method: '${err.message}'`, err);
@@ -193,7 +207,7 @@ export class Worker {
             if (rep instanceof EbusRejectError && rep.reason === "error")
                 responseMessageType = Message.Error
             
-            this.onError(err);
+            this.onError(err, unpack(req, opts), opts );
         }
         
         let data = rep;