@hahnpro/flow-sdk 4.15.0 → 4.15.1

package/dist/FlowApplication.d.ts

@@ -15,10 +15,13 @@ export declare class FlowApplication {
     private elements;
     private logger;
     private outputStreamMap;
+    private outputQueueMetrics;
     private performanceMap;
     private properties;
     private _rpcClient;
     constructor(modules: ClassType<any>[], flow: Flow, logger?: Logger, amqpConnection?: AmqpConnection, skipApi?: boolean);
+    private setQueueMetrics;
+    private updateMetrics;
     subscribe: (streamId: string, observer: PartialObserver<FlowEvent>) => import("rxjs").Subscription;
     emit: (event: FlowEvent) => void;
     emitPartial: (completeEvent: FlowEvent, partialEvent: FlowEvent) => void;

package/dist/FlowApplication.js

@@ -14,7 +14,9 @@ const api_1 = require("./api");
 const utils_1 = require("./utils");
 const FlowLogger_1 = require("./FlowLogger");
 const RpcClient_1 = require("./RpcClient");
-const MAX_EVENT_SIZE_BYTES = 512 * 1024;
+const MAX_EVENT_SIZE_BYTES = +process.env.MAX_EVENT_SIZE_BYTES || 512 * 1024;
+const WARN_EVENT_PROCESSING_SEC = +process.env.WARN_EVENT_PROCESSING_SEC || 60;
+const WARN_EVENT_QUEUE_SIZE = +process.env.WARN_EVENT_QUEUE_SIZE || 100;
 class FlowApplication {
     constructor(modules, flow, logger, amqpConnection, skipApi) {
         var _a, _b, _c;
@@ -22,7 +24,31 @@ class FlowApplication {
         this.declarations = {};
         this.elements = {};
         this.outputStreamMap = new Map();
+        this.outputQueueMetrics = new Map();
         this.performanceMap = new Map();
+        this.setQueueMetrics = (id) => {
+            const metrics = this.outputQueueMetrics.get(id) || { size: 0, lastAdd: 0, lastRemove: Date.now(), warnings: 0 };
+            const secsProcessing = Math.round((metrics.lastAdd - metrics.lastRemove) / 1000);
+            metrics.size++;
+            metrics.lastAdd = Date.now();
+            if (secsProcessing >= WARN_EVENT_PROCESSING_SEC * (metrics.warnings + 1)) {
+                this.logger.warn(`Input stream "${id}" has ${metrics.size} queued events and the last event has been processing for ${secsProcessing}s`);
+                metrics.warnings++;
+            }
+            else if (metrics.size % WARN_EVENT_QUEUE_SIZE === 0) {
+                this.logger.warn(`Input stream "${id}" has ${metrics.size} queued events`);
+            }
+            this.outputQueueMetrics.set(id, metrics);
+        };
+        this.updateMetrics = (id) => {
+            const metrics = this.outputQueueMetrics.get(id);
+            if (metrics) {
+                metrics.size = metrics.size > 0 ? metrics.size - 1 : 0;
+                metrics.lastRemove = Date.now();
+                metrics.warnings = 0;
+                this.outputQueueMetrics.set(id, metrics);
+            }
+        };
         this.subscribe = (streamId, observer) => this.getOutputStream(streamId).subscribe(observer);
         this.emit = (event) => {
             if (event) {
@@ -188,7 +214,8 @@ class FlowApplication {
                 if (!source || !target) {
                     continue;
                 }
-                const streamId = `${source}.${sourceStream}`;
+                const sourceStreamId = `${source}.${sourceStream}`;
+                const targetStreamId = `${target}.${targetStream}`;
                 const element = this.elements[target];
                 if (!element || !element.constructor) {
                     throw new Error(target + ' has not been initialized');
@@ -199,9 +226,9 @@ class FlowApplication {
                 }
                 const streamOptions = Reflect.getMetadata(`stream:options:${targetStream}`, element.constructor) || {};
                 const concurrent = streamOptions.concurrent || 1;
-                const outputStream = this.getOutputStream(streamId);
+                const outputStream = this.getOutputStream(sourceStreamId);
                 outputStream
-                    .pipe((0, operators_1.mergeMap)(async (event) => {
+                    .pipe((0, operators_1.tap)(() => this.setQueueMetrics(targetStreamId)), (0, operators_1.mergeMap)(async (event) => {
                     this.performanceMap.set(event.getId(), perf_hooks_1.performance.eventLoopUtilization());
                     try {
                         await element[streamHandler](event);
@@ -216,12 +243,13 @@ class FlowApplication {
                     }
                     return event;
                 }, concurrent), (0, operators_1.tap)((event) => {
+                    this.updateMetrics(targetStreamId);
                     let elu = this.performanceMap.get(event.getId());
                     if (elu) {
                         this.performanceMap.delete(event.getId());
                         elu = perf_hooks_1.performance.eventLoopUtilization(elu);
                         if (elu.utilization > 0.7 && elu.active > 1000) {
-                            this.logger.warn(`High event loop utilization detected for ${target}.${targetStream} with event ${event.getId()}! Handler has been active for ${Number(elu.active).toFixed(2)}ms with a utilization of ${Number(elu.utilization * 100).toFixed(2)}%. Consider refactoring or move tasks to a worker thread.`);
+                            this.logger.warn(`High event loop utilization detected for ${targetStreamId} with event ${event.getId()}! Handler has been active for ${Number(elu.active).toFixed(2)}ms with a utilization of ${Number(elu.utilization * 100).toFixed(2)}%. Consider refactoring or move tasks to a worker thread.`);
                         }
                     }
                 }))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hahnpro/flow-sdk",
-  "version": "4.15.0",
+  "version": "4.15.1",
   "description": "SDK for building Flow Modules",
   "license": "MIT",
   "author": {
@@ -26,7 +26,7 @@
   "dependencies": {
     "amqp-connection-manager": "^3.7.0",
     "amqplib": "^0.8.0",
-    "axios": "^0.21.4",
+    "axios": "^0.22.0",
     "class-transformer": "0.3.1",
     "class-validator": "~0.12.2",
     "class-validator-jsonschema": "^2.2.0",
@@ -38,7 +38,7 @@
     "p-queue": "^6.6.2",
     "python-shell": "^3.0.0",
     "reflect-metadata": "^0.1.13",
-    "rxjs": "^7.3.0",
+    "rxjs": "^7.3.1",
     "string-interp": "^0.3.6",
     "uuid": "^8.3.2"
   },
@@ -47,10 +47,10 @@
     "@types/amqp-connection-manager": "^2.0.12",
     "@types/amqplib": "^0.8.2",
     "@types/jest": "^27.0.2",
-    "@types/lodash": "^4.14.174",
-    "@types/node": "^14.17.19",
+    "@types/lodash": "^4.14.175",
+    "@types/node": "^14.17.20",
     "axios-mock-adapter": "^1.20.0",
-    "jest": "^27.2.2",
+    "jest": "^27.2.4",
     "typescript": "^4.4.3"
   },
   "engines": {