@hahnpro/flow-sdk 4.20.11 → 4.21.0-2

package/dist/FlowApplication.d.ts

@@ -1,10 +1,10 @@
 import 'reflect-metadata';
-import type { CloudEvent } from 'cloudevents';
+import { CloudEvent } from 'cloudevents';
 import { PartialObserver } from 'rxjs';
 import { API } from '@hahnpro/hpc-api';
 import { AmqpConnection, Nack } from './amqp';
-import type { ClassType, Flow, FlowElementContext } from './flow.interface';
-import type { FlowEvent } from './FlowEvent';
+import { ClassType, Flow, FlowElementContext } from './flow.interface';
+import { FlowEvent } from './FlowEvent';
 import { Logger } from './FlowLogger';
 import { RpcClient } from './RpcClient';
 export declare class FlowApplication {
@@ -21,6 +21,7 @@ export declare class FlowApplication {
     private _rpcClient;
     constructor(modules: ClassType<any>[], flow: Flow, logger?: Logger, amqpConnection?: AmqpConnection, skipApi?: boolean);
     private init;
+    private publishLifecycleEvent;
     private setQueueMetrics;
     private updateMetrics;
     subscribe: (streamId: string, observer: PartialObserver<FlowEvent>) => import("rxjs").Subscription;
@@ -32,7 +33,7 @@ export declare class FlowApplication {
      * Publish a flow event to the amqp flowlogs exchange.
      * If the event size exceeds the limit it will be truncated
      */
-    publishEvent: (event: FlowEvent) => Promise<void>;
+    publishEvent: (event: FlowEvent) => void;
     rpcClient(): Promise<RpcClient>;
     /**
      * Calls onDestroy lifecycle method on all flow elements,

package/dist/FlowApplication.js

@@ -3,14 +3,16 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FlowApplication = void 0;
 const tslib_1 = require("tslib");
 require("reflect-metadata");
+const cloudevents_1 = require("cloudevents");
 const object_sizeof_1 = tslib_1.__importDefault(require("object-sizeof"));
 const perf_hooks_1 = require("perf_hooks");
 const rxjs_1 = require("rxjs");
 const operators_1 = require("rxjs/operators");
-const uuid_1 = require("uuid");
+const crypto_1 = require("crypto");
 const hpc_api_1 = require("@hahnpro/hpc-api");
 const amqp_1 = require("./amqp");
 const utils_1 = require("./utils");
+const flow_interface_1 = require("./flow.interface");
 const FlowLogger_1 = require("./FlowLogger");
 const RpcClient_1 = require("./RpcClient");
 const MAX_EVENT_SIZE_BYTES = +process.env.MAX_EVENT_SIZE_BYTES || 512 * 1024;
@@ -24,6 +26,27 @@ class FlowApplication {
         this.outputStreamMap = new Map();
         this.outputQueueMetrics = new Map();
         this.performanceMap = new Map();
+        this.publishLifecycleEvent = (element, flowEventId, eventType, data = {}) => {
+            if (!this.amqpConnection) {
+                return;
+            }
+            try {
+                const { flowId, deploymentId, id: elementId, functionFqn, inputStreamId } = element.getMetadata();
+                const event = new cloudevents_1.CloudEvent({
+                    source: `flows/${flowId}/deployments/${deploymentId}/elements/${elementId}`,
+                    type: eventType,
+                    data: Object.assign({ flowEventId,
+                        functionFqn,
+                        inputStreamId }, data),
+                    time: new Date().toISOString(),
+                });
+                const message = event.toJSON();
+                return this.amqpConnection.publish('flow', 'lifecycle', message);
+            }
+            catch (err) {
+                this.logger.error(err);
+            }
+        };
         this.setQueueMetrics = (id) => {
             const metrics = this.outputQueueMetrics.get(id) || { size: 0, lastAdd: 0, lastRemove: Date.now(), warnings: 0 };
             const secsProcessing = Math.round((metrics.lastAdd - metrics.lastRemove) / 1000);
@@ -103,24 +126,34 @@ class FlowApplication {
                         (_e = (_d = this.elements) === null || _d === void 0 ? void 0 : _d[element.id]) === null || _e === void 0 ? void 0 : _e.onPropertiesChanged(element.properties);
                     }
                     const statusEvent = {
-                        eventId: (0, uuid_1.v4)(),
+                        eventId: (0, crypto_1.randomUUID)(),
                         eventTime: new Date().toISOString(),
                         eventType: 'com.hahnpro.event.health',
                         contentType: 'application/json',
                         data: { deploymentId: this.context.deploymentId, status: 'updated' },
                     };
-                    (_f = this.amqpConnection) === null || _f === void 0 ? void 0 : _f.publish('deployment', 'health', statusEvent).catch((err) => this.logger.error(err));
+                    try {
+                        (_f = this.amqpConnection) === null || _f === void 0 ? void 0 : _f.publish('deployment', 'health', statusEvent);
+                    }
+                    catch (err) {
+                        this.logger.error(err);
+                    }
                 }
                 catch (err) {
                     this.logger.error(err);
                     const statusEvent = {
-                        eventId: (0, uuid_1.v4)(),
+                        eventId: (0, crypto_1.randomUUID)(),
                         eventTime: new Date().toISOString(),
                         eventType: 'com.hahnpro.event.health',
                         contentType: 'application/json',
                         data: { deploymentId: this.context.deploymentId, status: 'updating failed' },
                     };
-                    (_g = this.amqpConnection) === null || _g === void 0 ? void 0 : _g.publish('deployment', 'health', statusEvent).catch((err) => this.logger.error(err));
+                    try {
+                        (_g = this.amqpConnection) === null || _g === void 0 ? void 0 : _g.publish('deployment', 'health', statusEvent);
+                    }
+                    catch (e) {
+                        this.logger.error(e);
+                    }
                 }
             }
             else if (event.type === 'com.flowstudio.deployment.message') {
@@ -193,6 +226,7 @@ class FlowApplication {
             try {
                 await this.amqpConnection.managedChannel.assertExchange('deployment', 'direct', { durable: true });
                 await this.amqpConnection.managedChannel.assertExchange('flowlogs', 'fanout', { durable: true });
+                await this.amqpConnection.managedChannel.assertExchange('flow', 'direct', { durable: true });
             }
             catch (e) {
                 logErrorAndExit(`Could not assert exchanges: ${e}`);
@@ -203,7 +237,7 @@ class FlowApplication {
                     exchange: 'deployment',
                     routingKey: this.context.deploymentId,
                     queueOptions: { durable: false, exclusive: true },
-                });
+                }, 'FlowApplication.onMessage');
             }
             catch (err) {
                 logErrorAndExit(`Could not subscribe to deployment exchange: ${err}`);
@@ -259,11 +293,18 @@ class FlowApplication {
             const outputStream = this.getOutputStream(sourceStreamId);
             outputStream
                 .pipe((0, operators_1.tap)(() => this.setQueueMetrics(targetStreamId)), (0, operators_1.mergeMap)(async (event) => {
-                this.performanceMap.set(event.getId(), perf_hooks_1.performance.eventLoopUtilization());
+                const eventId = event.getId();
+                this.publishLifecycleEvent(element, eventId, flow_interface_1.LifecycleEvent.ACTIVATED);
+                this.performanceMap.set(eventId, perf_hooks_1.performance.eventLoopUtilization());
+                const start = perf_hooks_1.performance.now();
                 try {
                     await element[streamHandler](event);
+                    const duration = Math.ceil(perf_hooks_1.performance.now() - start);
+                    this.publishLifecycleEvent(element, eventId, flow_interface_1.LifecycleEvent.COMPLETED, { duration });
                 }
                 catch (err) {
+                    const duration = Math.ceil(perf_hooks_1.performance.now() - start);
+                    this.publishLifecycleEvent(element, eventId, flow_interface_1.LifecycleEvent.TERMINATED, { duration });
                     try {
                         element.handleApiError(err);
                     }
@@ -278,8 +319,8 @@ class FlowApplication {
                 if (elu) {
                     this.performanceMap.delete(event.getId());
                     elu = perf_hooks_1.performance.eventLoopUtilization(elu);
-                    if (elu.utilization > 0.7 && elu.active > 1000) {
-                        this.logger.warn(`High event loop utilization detected for ${targetStreamId} with event ${event.getId()}! Handler has been active for ${Number(elu.active).toFixed(2)}ms with a utilization of ${Number(elu.utilization * 100).toFixed(2)}%. Consider refactoring or move tasks to a worker thread.`);
+                    if (elu.utilization > 0.75 && elu.active > 2000) {
+                        this.logger.warn(`High event loop utilization detected for ${targetStreamId} with event ${event.getId()}! Handler was active for ${Number(elu.active).toFixed(2)}ms with a utilization of ${Number(elu.utilization * 100).toFixed(2)}%. Consider refactoring or move tasks to a worker thread.`);
                     }
                 }
             }))

package/dist/FlowElement.d.ts

@@ -22,6 +22,7 @@ export declare abstract class FlowElement<T = any> {
     onFlowPropertiesChanged?: (properties: Record<string, any>) => void;
     onContextChanged: (context: Partial<FlowContext>) => void;
     onPropertiesChanged: (properties: T) => void;
+    getMetadata: () => FlowElementContext;
     protected setProperties: (properties: T) => void;
     handleApiError: (error: any) => void;
     /**

package/dist/FlowElement.js

@@ -22,6 +22,7 @@ class FlowElement {
         this.onPropertiesChanged = (properties) => {
             this.setProperties(properties);
         };
+        this.getMetadata = () => this.metadata;
         this.setProperties = (properties) => {
             if (this.propertiesClassType) {
                 this.properties = this.validateProperties(this.propertiesClassType, properties, this.whitelist);
@@ -126,7 +127,7 @@ function InputStream(id = 'default', options) {
             if (!this.stopPropagateStream.has(id)) {
                 this.stopPropagateStream.set(id, (_a = options === null || options === void 0 ? void 0 : options.stopPropagation) !== null && _a !== void 0 ? _a : false);
             }
-            return method.call(this, new FlowEvent_1.FlowEvent(Object.assign(Object.assign({}, event.getMetadata()), { inputStreamId: id }), event.getData(), event.getType(), new Date(event.getTime())));
+            return method.call(this, new FlowEvent_1.FlowEvent(Object.assign(Object.assign({ id: event.getMetadata().elementId }, event.getMetadata()), { inputStreamId: id }), event.getData(), event.getType(), new Date(event.getTime())));
         };
     };
 }

package/dist/FlowEvent.d.ts

@@ -8,7 +8,13 @@ export declare class FlowEvent {
     getDataContentType: () => string;
     getDataschema: () => string;
     getId: () => string;
-    getMetadata: () => any;
+    getMetadata: () => {
+        deploymentId: string;
+        elementId: string;
+        flowId: string;
+        functionFqn: string;
+        inputStreamId: string;
+    };
     getSource: () => string;
     getStreamId: () => string;
     getSubject: () => string;

package/dist/FlowLogger.d.ts

@@ -16,7 +16,7 @@ export declare class FlowLogger implements Logger {
     private readonly logger;
     private readonly publishEvent?;
     private static getStackTrace;
-    constructor(metadata: FlowElementContext, logger?: Logger, publishEvent?: (event: FlowEvent) => Promise<void>);
+    constructor(metadata: FlowElementContext, logger?: Logger, publishEvent?: (event: FlowEvent) => void);
     debug: (message: any, options?: LoggerOptions) => void;
     error: (message: any, options?: LoggerOptions) => void;
     log: (message: any, options?: LoggerOptions) => void;

package/dist/FlowLogger.js

@@ -35,10 +35,9 @@ class FlowLogger {
         return stack.splice(1).join('\n');
     }
     publish(message, level, options) {
-        var _a;
         if (this.publishEvent) {
             const event = new FlowEvent_1.FlowEvent(this.metadata, message, `flow.log.${level}`);
-            (_a = this.publishEvent(event)) === null || _a === void 0 ? void 0 : _a.catch((err) => this.logger.error(err, this.metadata));
+            this.publishEvent(event);
         }
         switch (level) {
             case 'debug':

package/dist/RpcClient.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RpcClient = void 0;
-const uuid_1 = require("uuid");
+const crypto_1 = require("crypto");
 class RpcClient {
     constructor(amqpConnection) {
         this.amqpConnection = amqpConnection;
@@ -40,7 +40,7 @@ class RpcClient {
             const stack = new Error('test').stack;
             return new Promise((resolve, reject) => {
                 const call = { functionName, arguments: args };
-                const correlationId = (0, uuid_1.v4)();
+                const correlationId = (0, crypto_1.randomUUID)();
                 this.openRequests.set(correlationId, { resolve, reject, trace: RpcClient.formatTrace(stack) });
                 this.channel
                     .publish('rpc_direct_exchange', routingKey, call, { correlationId, replyTo: 'amq.rabbitmq.reply-to' })

package/dist/amqp.d.ts

@@ -5,8 +5,8 @@ export interface AmqpConnection {
     connection: Connection;
     managedChannel: ChannelWrapper;
     managedConnection: AmqpConnectionManager;
-    createSubscriber<T>(handler: (msg: T | undefined, rawMessage?: ConsumeMessage) => Promise<any | undefined>, msgOptions: MessageHandlerOptions): Promise<void>;
-    publish(exchange: string, routingKey: string, message: any, options?: Options.Publish): Promise<void>;
+    createSubscriber<T>(handler: (msg: T | undefined, rawMessage?: ConsumeMessage) => Promise<any | undefined | void>, msgOptions: MessageHandlerOptions, originalHandlerName: string): Promise<void>;
+    publish(exchange: string, routingKey: string, message: any, options?: Options.Publish): void;
 }
 export declare class Nack {
     private readonly _requeue;

package/dist/flow.interface.d.ts

@@ -38,4 +38,9 @@ export interface StreamOptions {
     concurrent?: number;
 }
 export declare type ClassType<T> = new (...args: any[]) => T;
+export declare enum LifecycleEvent {
+    ACTIVATED = "com.hahnpro.flow_function.activated",
+    COMPLETED = "com.hahnpro.flow_function.completed",
+    TERMINATED = "com.hahnpro.flow_function.terminated"
+}
 export {};

package/dist/flow.interface.js

@@ -1,2 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.LifecycleEvent = void 0;
+var LifecycleEvent;
+(function (LifecycleEvent) {
+    LifecycleEvent["ACTIVATED"] = "com.hahnpro.flow_function.activated";
+    LifecycleEvent["COMPLETED"] = "com.hahnpro.flow_function.completed";
+    LifecycleEvent["TERMINATED"] = "com.hahnpro.flow_function.terminated";
+})(LifecycleEvent = exports.LifecycleEvent || (exports.LifecycleEvent = {}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hahnpro/flow-sdk",
-  "version": "4.20.11",
+  "version": "4.21.0-2",
   "description": "SDK for building Flow Modules",
   "license": "MIT",
   "author": {
@@ -24,9 +24,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@hahnpro/hpc-api": "2.3.1",
+    "@hahnpro/hpc-api": "3.0.0",
     "amqp-connection-manager": "^3.9.0",
-    "amqplib": "^0.9.0",
+    "amqplib": "^0.9.1",
     "class-transformer": "0.5.1",
     "class-validator": "~0.13.2",
     "cloudevents": "^6.0.1",
@@ -35,20 +35,19 @@
     "python-shell": "^3.0.1",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.5",
-    "string-interp": "^0.3.6",
-    "uuid": "^8.3.2"
+    "string-interp": "^0.3.6"
   },
   "devDependencies": {
-    "@golevelup/nestjs-rabbitmq": "^2.3.0",
-    "@nestjs/common": "^8.4.4",
+    "@golevelup/nestjs-rabbitmq": "^2.4.0",
+    "@nestjs/common": "^8.4.6",
     "@types/amqp-connection-manager": "^2.0.12",
     "@types/amqplib": "^0.8.2",
-    "@types/jest": "^27.5.0",
+    "@types/jest": "^27.5.1",
     "@types/lodash": "^4.14.182",
-    "@types/node": "^16.11.34",
-    "class-validator-jsonschema": "^3.1.0",
-    "jest": "^27.5.1",
-    "typescript": "^4.6.4"
+    "@types/node": "^16.11.38",
+    "class-validator-jsonschema": "^3.1.1",
+    "jest": "^28.1.0",
+    "typescript": "^4.7.2"
   },
   "engines": {
     "node": "^14.13.1 || >=16.0.0"