@hahnpro/flow-sdk 4.14.5 → 4.17.0

package/dist/FlowApplication.d.ts

@@ -15,10 +15,13 @@ export declare class FlowApplication {
     private elements;
     private logger;
     private outputStreamMap;
+    private outputQueueMetrics;
     private performanceMap;
     private properties;
     private _rpcClient;
     constructor(modules: ClassType<any>[], flow: Flow, logger?: Logger, amqpConnection?: AmqpConnection, skipApi?: boolean);
+    private setQueueMetrics;
+    private updateMetrics;
     subscribe: (streamId: string, observer: PartialObserver<FlowEvent>) => import("rxjs").Subscription;
     emit: (event: FlowEvent) => void;
     emitPartial: (completeEvent: FlowEvent, partialEvent: FlowEvent) => void;

package/dist/FlowApplication.js

@@ -14,7 +14,9 @@ const api_1 = require("./api");
 const utils_1 = require("./utils");
 const FlowLogger_1 = require("./FlowLogger");
 const RpcClient_1 = require("./RpcClient");
-const MAX_EVENT_SIZE_BYTES = 512 * 1024;
+const MAX_EVENT_SIZE_BYTES = +process.env.MAX_EVENT_SIZE_BYTES || 512 * 1024;
+const WARN_EVENT_PROCESSING_SEC = +process.env.WARN_EVENT_PROCESSING_SEC || 60;
+const WARN_EVENT_QUEUE_SIZE = +process.env.WARN_EVENT_QUEUE_SIZE || 100;
 class FlowApplication {
     constructor(modules, flow, logger, amqpConnection, skipApi) {
         var _a, _b, _c;
@@ -22,7 +24,31 @@ class FlowApplication {
         this.declarations = {};
         this.elements = {};
         this.outputStreamMap = new Map();
+        this.outputQueueMetrics = new Map();
         this.performanceMap = new Map();
+        this.setQueueMetrics = (id) => {
+            const metrics = this.outputQueueMetrics.get(id) || { size: 0, lastAdd: 0, lastRemove: Date.now(), warnings: 0 };
+            const secsProcessing = Math.round((metrics.lastAdd - metrics.lastRemove) / 1000);
+            metrics.size++;
+            metrics.lastAdd = Date.now();
+            if (secsProcessing >= WARN_EVENT_PROCESSING_SEC * (metrics.warnings + 1)) {
+                this.logger.warn(`Input stream "${id}" has ${metrics.size} queued events and the last event has been processing for ${secsProcessing}s`);
+                metrics.warnings++;
+            }
+            else if (metrics.size % WARN_EVENT_QUEUE_SIZE === 0) {
+                this.logger.warn(`Input stream "${id}" has ${metrics.size} queued events`);
+            }
+            this.outputQueueMetrics.set(id, metrics);
+        };
+        this.updateMetrics = (id) => {
+            const metrics = this.outputQueueMetrics.get(id);
+            if (metrics) {
+                metrics.size = metrics.size > 0 ? metrics.size - 1 : 0;
+                metrics.lastRemove = Date.now();
+                metrics.warnings = 0;
+                this.outputQueueMetrics.set(id, metrics);
+            }
+        };
         this.subscribe = (streamId, observer) => this.getOutputStream(streamId).subscribe(observer);
         this.emit = (event) => {
             if (event) {
@@ -31,9 +57,6 @@ class FlowApplication {
                     if (this.outputStreamMap.has(event.getStreamId())) {
                         this.getOutputStream(event.getStreamId()).next(event);
                     }
-                    else {
-                        throw new Error('Output Stream ' + event.getStreamId() + ' is not defined');
-                    }
                 }
                 catch (err) {
                     this.logger.error(err);
@@ -42,13 +65,8 @@ class FlowApplication {
         };
         this.emitPartial = (completeEvent, partialEvent) => {
             try {
-                if (completeEvent) {
-                    if (this.outputStreamMap.has(completeEvent.getStreamId())) {
-                        this.getOutputStream(completeEvent.getStreamId()).next(completeEvent);
-                    }
-                    else {
-                        throw new Error('Output Stream ' + completeEvent.getStreamId() + ' is not defined');
-                    }
+                if (completeEvent && this.outputStreamMap.has(completeEvent.getStreamId())) {
+                    this.getOutputStream(completeEvent.getStreamId()).next(completeEvent);
                 }
                 if (partialEvent) {
                     this.publishEvent(partialEvent);
@@ -196,7 +214,8 @@ class FlowApplication {
                 if (!source || !target) {
                     continue;
                 }
-                const streamId = `${source}.${sourceStream}`;
+                const sourceStreamId = `${source}.${sourceStream}`;
+                const targetStreamId = `${target}.${targetStream}`;
                 const element = this.elements[target];
                 if (!element || !element.constructor) {
                     throw new Error(target + ' has not been initialized');
@@ -207,9 +226,9 @@ class FlowApplication {
                 }
                 const streamOptions = Reflect.getMetadata(`stream:options:${targetStream}`, element.constructor) || {};
                 const concurrent = streamOptions.concurrent || 1;
-                const outputStream = this.getOutputStream(streamId);
+                const outputStream = this.getOutputStream(sourceStreamId);
                 outputStream
-                    .pipe((0, operators_1.mergeMap)(async (event) => {
+                    .pipe((0, operators_1.tap)(() => this.setQueueMetrics(targetStreamId)), (0, operators_1.mergeMap)(async (event) => {
                     this.performanceMap.set(event.getId(), perf_hooks_1.performance.eventLoopUtilization());
                     try {
                         await element[streamHandler](event);
@@ -224,12 +243,13 @@ class FlowApplication {
                     }
                     return event;
                 }, concurrent), (0, operators_1.tap)((event) => {
+                    this.updateMetrics(targetStreamId);
                     let elu = this.performanceMap.get(event.getId());
                     if (elu) {
                         this.performanceMap.delete(event.getId());
                         elu = perf_hooks_1.performance.eventLoopUtilization(elu);
                         if (elu.utilization > 0.7 && elu.active > 1000) {
-                            this.logger.warn(`High event loop utilization detected for ${target}.${targetStream} with event ${event.getId()}! Handler has been active for ${Number(elu.active).toFixed(2)}ms with a utilization of ${Number(elu.utilization * 100).toFixed(2)}%. Consider refactoring or move tasks to a worker thread.`);
+                            this.logger.warn(`High event loop utilization detected for ${targetStreamId} with event ${event.getId()}! Handler has been active for ${Number(elu.active).toFixed(2)}ms with a utilization of ${Number(elu.utilization * 100).toFixed(2)}%. Consider refactoring or move tasks to a worker thread.`);
                         }
                     }
                 }))

package/dist/api/endpoint.interface.d.ts

@@ -9,6 +9,9 @@ export interface Endpoint {
         authToken: string;
         recipients?: string[];
     };
+    notificationCheck: number;
+    notificationCount: number;
+    notificationPause: number;
     readPermissions: string[];
     readWritePermissions: string[];
 }

package/dist/api/endpoint.service.d.ts

@@ -3,6 +3,6 @@ import { DataService } from './data.service';
 import { Endpoint, EndpointLog } from './endpoint.interface';
 export declare class EndpointService extends DataService<Endpoint> {
     constructor(httpClient: HttpClient);
-    sendNotification(endpointId: string, subject: string, message: string, group: string, eventLink?: string): Promise<void>;
+    sendNotification(endpointId: string, subject: string, message: string, group: string, eventLink?: string, assetLink?: string): Promise<void>;
     readLastLogByGroup(endpointId: string, group: string): Promise<EndpointLog>;
 }

package/dist/api/endpoint.service.js

@@ -6,8 +6,8 @@ class EndpointService extends data_service_1.DataService {
     constructor(httpClient) {
         super(httpClient, process.env.DEBUG_ENDPOINT_URL || 'api/notification/endpoints');
     }
-    sendNotification(endpointId, subject, message, group, eventLink) {
-        const body = Object.assign({ subject, message, group }, (eventLink && { eventLink }));
+    sendNotification(endpointId, subject, message, group, eventLink, assetLink) {
+        const body = Object.assign(Object.assign({ subject, message, group }, (eventLink && { eventLink })), (assetLink && { assetLink }));
         return this.httpClient.post(`${this.basePath}/${endpointId}`, body);
     }
     readLastLogByGroup(endpointId, group) {

package/dist/api/mock/api.mock.d.ts

@@ -101,6 +101,9 @@ export interface EndpointInit {
         authToken: string;
         recipients?: string[];
     };
+    notificationCheck?: number;
+    notificationCount?: number;
+    notificationPause?: number;
     readPermissions?: string[];
     readWritePermissions?: string[];
 }

package/dist/api/mock/api.mock.js

@@ -58,6 +58,9 @@ class MockAPI {
             description: value.description,
             status: value.status,
             config: value.config,
+            notificationCheck: value.notificationCheck,
+            notificationCount: value.notificationCount,
+            notificationPause: value.notificationPause,
             readPermissions: [],
             readWritePermissions: [],
         }));

package/dist/api/mock/endpoint.mock.service.d.ts

@@ -3,7 +3,7 @@ import { EndpointService } from '../endpoint.service';
 import { DataMockService } from './data.mock.service';
 export declare class EndpointMockService extends DataMockService<Endpoint> implements EndpointService {
     constructor(endpoints: Endpoint[]);
-    sendNotification(endpointId: string, subject: string, message: string, group: string, eventLink?: string): Promise<void>;
+    sendNotification(endpointId: string, subject: string, message: string, group: string, eventLink?: string, assetLink?: string): Promise<void>;
     readLastLogByGroup(endpointId: string, group: string): Promise<{
         id: string;
         endpoint: string;

package/dist/api/mock/endpoint.mock.service.js

@@ -7,7 +7,7 @@ class EndpointMockService extends data_mock_service_1.DataMockService {
         super();
         this.data = endpoints;
     }
-    sendNotification(endpointId, subject, message, group, eventLink) {
+    sendNotification(endpointId, subject, message, group, eventLink, assetLink) {
         return Promise.resolve();
     }
     readLastLogByGroup(endpointId, group) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hahnpro/flow-sdk",
-  "version": "4.14.5",
+  "version": "4.17.0",
   "description": "SDK for building Flow Modules",
   "license": "MIT",
   "author": {
@@ -26,7 +26,7 @@
   "dependencies": {
     "amqp-connection-manager": "^3.7.0",
     "amqplib": "^0.8.0",
-    "axios": "^0.21.4",
+    "axios": "^0.22.0",
     "class-transformer": "0.3.1",
     "class-validator": "~0.12.2",
     "class-validator-jsonschema": "^2.2.0",
@@ -38,19 +38,19 @@
     "p-queue": "^6.6.2",
     "python-shell": "^3.0.0",
     "reflect-metadata": "^0.1.13",
-    "rxjs": "^7.3.0",
+    "rxjs": "^7.4.0",
     "string-interp": "^0.3.6",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@golevelup/nestjs-rabbitmq": "^1.17.1",
+    "@golevelup/nestjs-rabbitmq": "^1.18.0",
     "@types/amqp-connection-manager": "^2.0.12",
     "@types/amqplib": "^0.8.2",
     "@types/jest": "^27.0.2",
-    "@types/lodash": "^4.14.174",
-    "@types/node": "^14.17.19",
+    "@types/lodash": "^4.14.175",
+    "@types/node": "^14.17.21",
     "axios-mock-adapter": "^1.20.0",
-    "jest": "^27.2.2",
+    "jest": "^27.2.4",
     "typescript": "^4.4.3"
   },
   "engines": {