@hahnpro/flow-sdk 4.21.1 → 4.22.0

package/dist/FlowApplication.d.ts

@@ -8,19 +8,23 @@ import { FlowEvent } from './FlowEvent';
 import { Logger } from './FlowLogger';
 import { RpcClient } from './RpcClient';
 export declare class FlowApplication {
+    private modules;
+    private flow;
     private amqpConnection?;
+    private skipApi;
     api: API;
     private context;
     private declarations;
     private elements;
-    private logger;
+    private readonly logger;
     private outputStreamMap;
     private outputQueueMetrics;
     private performanceMap;
     private properties;
     private _rpcClient;
-    constructor(modules: ClassType<any>[], flow: Flow, logger?: Logger, amqpConnection?: AmqpConnection, skipApi?: boolean);
-    private init;
+    private initialized;
+    constructor(modules: ClassType<any>[], flow: Flow, logger?: Logger, amqpConnection?: AmqpConnection, skipApi?: boolean, explicitInit?: boolean);
+    init(): Promise<void>;
     private publishLifecycleEvent;
     private setQueueMetrics;
     private updateMetrics;

package/dist/FlowApplication.js

@@ -19,13 +19,17 @@ const MAX_EVENT_SIZE_BYTES = +process.env.MAX_EVENT_SIZE_BYTES || 512 * 1024;
 const WARN_EVENT_PROCESSING_SEC = +process.env.WARN_EVENT_PROCESSING_SEC || 60;
 const WARN_EVENT_QUEUE_SIZE = +process.env.WARN_EVENT_QUEUE_SIZE || 100;
 class FlowApplication {
-    constructor(modules, flow, logger, amqpConnection, skipApi = false) {
+    constructor(modules, flow, logger, amqpConnection, skipApi = false, explicitInit = false) {
+        this.modules = modules;
+        this.flow = flow;
         this.amqpConnection = amqpConnection;
+        this.skipApi = skipApi;
         this.declarations = {};
         this.elements = {};
         this.outputStreamMap = new Map();
         this.outputQueueMetrics = new Map();
         this.performanceMap = new Map();
+        this.initialized = false;
         this.publishLifecycleEvent = (element, flowEventId, eventType, data = {}) => {
             if (!this.amqpConnection) {
                 return;
@@ -205,22 +209,26 @@ class FlowApplication {
             this.logger.log('Flow Deployment is terminating');
             this.destroy(0);
         });
-        this.init(flow, modules, logger, skipApi);
+        if (!explicitInit) {
+            this.init();
+        }
     }
-    async init(flow, modules, logger, skipApi) {
-        this.context = Object.assign({}, flow.context);
-        this.properties = flow.properties || {};
+    async init() {
+        if (this.initialized)
+            return;
+        this.context = Object.assign({}, this.flow.context);
+        this.properties = this.flow.properties || {};
         try {
-            if (!skipApi) {
+            if (!this.skipApi) {
                 this.api = new hpc_api_1.API();
             }
         }
         catch (err) {
             this.logger.error((err === null || err === void 0 ? void 0 : err.message) || err);
         }
-        const logErrorAndExit = (err) => {
+        const logErrorAndExit = async (err) => {
             this.logger.error(new Error(err));
-            this.destroy(1);
+            await this.destroy(1);
         };
         if (this.amqpConnection) {
             try {
@@ -229,7 +237,7 @@ class FlowApplication {
                 await this.amqpConnection.managedChannel.assertExchange('flow', 'direct', { durable: true });
             }
             catch (e) {
-                logErrorAndExit(`Could not assert exchanges: ${e}`);
+                await logErrorAndExit(`Could not assert exchanges: ${e}`);
                 return;
             }
             try {
@@ -240,38 +248,38 @@ class FlowApplication {
                 }, 'FlowApplication.onMessage');
             }
             catch (err) {
-                logErrorAndExit(`Could not subscribe to deployment exchange: ${err}`);
+                await logErrorAndExit(`Could not subscribe to deployment exchange: ${err}`);
                 return;
             }
         }
-        for (const module of modules) {
+        for (const module of this.modules) {
             const moduleName = Reflect.getMetadata('module:name', module);
             const moduleDeclarations = Reflect.getMetadata('module:declarations', module);
             if (!moduleName || !moduleDeclarations || !Array.isArray(moduleDeclarations)) {
-                logErrorAndExit(`FlowModule (${module.name}) metadata is missing or invalid`);
+                await logErrorAndExit(`FlowModule (${module.name}) metadata is missing or invalid`);
                 return;
             }
             for (const declaration of moduleDeclarations) {
                 const functionFqn = Reflect.getMetadata('element:functionFqn', declaration);
                 if (!functionFqn) {
-                    logErrorAndExit(`FlowFunction (${declaration.name}) metadata is missing or invalid`);
+                    await logErrorAndExit(`FlowFunction (${declaration.name}) metadata is missing or invalid`);
                     return;
                 }
                 this.declarations[`${moduleName}.${functionFqn}`] = declaration;
             }
         }
-        for (const element of flow.elements) {
+        for (const element of this.flow.elements) {
             const { id, name, properties, module, functionFqn } = element;
             try {
-                const context = Object.assign(Object.assign({}, this.context), { id, name, logger, app: this });
+                const context = Object.assign(Object.assign({}, this.context), { id, name, logger: this.logger, app: this });
                 this.elements[id] = new this.declarations[`${module}.${functionFqn}`](context, properties);
             }
             catch (err) {
-                logErrorAndExit(`Could not create FlowElement for ${module}.${functionFqn}`);
+                await logErrorAndExit(`Could not create FlowElement for ${module}.${functionFqn}`);
                 return;
             }
         }
-        for (const connection of flow.connections) {
+        for (const connection of this.flow.connections) {
             const { source, target, sourceStream = 'default', targetStream = 'default' } = connection;
             if (!source || !target) {
                 continue;
@@ -280,12 +288,12 @@ class FlowApplication {
             const targetStreamId = `${target}.${targetStream}`;
             const element = this.elements[target];
             if (!element || !element.constructor) {
-                logErrorAndExit(`${target} has not been initialized`);
+                await logErrorAndExit(`${target} has not been initialized`);
                 return;
             }
             const streamHandler = Reflect.getMetadata(`stream:${targetStream}`, element.constructor);
             if (!streamHandler || !element[streamHandler]) {
-                logErrorAndExit(`${target} does not implement a handler for ${targetStream}`);
+                await logErrorAndExit(`${target} does not implement a handler for ${targetStream}`);
                 return;
             }
             const streamOptions = Reflect.getMetadata(`stream:options:${targetStream}`, element.constructor) || {};
@@ -326,6 +334,7 @@ class FlowApplication {
             }))
                 .subscribe();
         }
+        this.initialized = true;
         this.logger.log('Flow Deployment is running');
     }
     getProperties() {

package/dist/FlowElement.d.ts

@@ -29,7 +29,7 @@ export declare abstract class FlowElement<T = any> {
      * @deprecated since version 4.8.0, will be removed in 5.0.0, use emitEvent(...) instead
      */
     protected emitOutput(data?: any, outputId?: string, time?: Date): FlowEvent;
-    protected emitEvent(data: any, inputEvent: FlowEvent, outputId?: string, time?: Date): FlowEvent;
+    protected emitEvent(data: Record<string, any>, inputEvent: FlowEvent, outputId?: string, time?: Date): FlowEvent;
     protected validateProperties<P>(classType: ClassType<P>, properties?: any, whitelist?: boolean): P;
     protected logValidationErrors(error: any, parent?: string): void;
     protected validateEventData<E>(classType: ClassType<E>, event: FlowEvent, whitelist?: boolean): E;

package/dist/FlowElement.js

@@ -133,7 +133,7 @@ function InputStream(id = 'default', options) {
 }
 exports.InputStream = InputStream;
 function FlowFunction(fqn) {
-    const fqnRegExp = new RegExp('^([a-zA-Z][a-zA-Z0-9]*[.-])*[a-zA-Z][a-zA-Z0-9]*$');
+    const fqnRegExp = /^([a-zA-Z][a-zA-Z\d]*[.-])*[a-zA-Z][a-zA-Z\d]*$/;
     if (!fqnRegExp.test(fqn)) {
         throw new Error(`Flow Function FQN (${fqn}) is not valid`);
     }

package/dist/TestModule.js

@@ -7,7 +7,7 @@ const FlowEvent_1 = require("./FlowEvent");
 const FlowModule_1 = require("./FlowModule");
 let TestTrigger = class TestTrigger extends FlowElement_1.FlowTask {
     async onDefault(event) {
-        return this.emitOutput(event.getData());
+        return this.emitEvent({}, event);
     }
 };
 tslib_1.__decorate([

package/dist/rpc_server.py

@@ -3,6 +3,7 @@ import json
 from asyncio import Future
 from functools import partial, wraps
 from aio_pika import IncomingMessage, Exchange, Message, connect_robust, ExchangeType
+from aio_pika.abc import AbstractRobustExchange
 import os
 
 user = os.getenv("RABBIT_USER", "guest")
@@ -12,22 +13,24 @@ port = os.getenv("RABBIT_PORT", "5672")
 vhost = os.getenv("RABBIT_VHOST", "")
 routingKey = os.getenv("RPC_ROUTING_KEY", "rpc")
 
-remoteProcedures = {}
+remote_procedures = {}
+flow_logs_exchange: AbstractRobustExchange
 
 
 def RemoteProcedure(func):
-    global remoteProcedures
+    global remote_procedures
 
     @wraps(func)
     def function_wrapper(*args, **kwargs):
         return func(*args, **kwargs)
 
-    remoteProcedures[func.__name__] = func
+    remote_procedures[func.__name__] = func
 
     return function_wrapper
 
 
-loop = asyncio.get_event_loop()
+loop = asyncio.new_event_loop()
+asyncio.set_event_loop(loop)
 
 
 async def on_message(exchange: Exchange, message: IncomingMessage):
@@ -39,36 +42,38 @@ async def on_message(exchange: Exchange, message: IncomingMessage):
             # print(traceback.format_list(traceback.extract_stack(err)))
             reply1 = {"type": "error", "message": str(err), "stack": "failed"}
 
-        asyncio.ensure_future(sendReply(exchange, reply1, message), loop=loop)
+        asyncio.ensure_future(send_reply(exchange, reply1, message), loop=loop)
 
-    with message.process():
-        request = json.loads(message.body.decode())
 
-        # call function
-        if remoteProcedures.keys().__contains__(request["functionName"]):
-            func = remoteProcedures.get(request["functionName"])
-            future = loop.run_in_executor(None, func, *request["arguments"])
-            future.add_done_callback(callback)
+    request = json.loads(message.body.decode())
 
-        else:
-            reply = {
-                "type": "error",
-                "message": request["functionName"] + " is not a function",
-            }
-            await sendReply(exchange, reply, originalMessage=message)
+    # call function
+    if remote_procedures.keys().__contains__(request["functionName"]):
+        func = remote_procedures.get(request["functionName"])
+        future = loop.run_in_executor(None, func, *request["arguments"])
+        future.add_done_callback(callback)
 
+    else:
+        reply = {
+            "type": "error",
+            "message": request["functionName"] + " is not a function",
+        }
+        await send_reply(exchange, reply, original_message=message)
 
-async def sendReply(exchange: Exchange, reply, originalMessage: Message):
+
+async def send_reply(exchange: Exchange, reply, original_message: Message):
     await exchange.publish(
         Message(
             body=json.dumps(reply).encode("utf-8"),
-            correlation_id=originalMessage.correlation_id
+            correlation_id=original_message.correlation_id
         ),
-        routing_key=originalMessage.reply_to,
+        routing_key=original_message.reply_to,
     )
 
 
 async def main(loop, routing_key):
+    global flow_logs_exchange
+
     url = "amqp://%s:%s@%s:%s/%s" % (user, password, host, port, vhost)
     connection = await connect_robust(
         url, loop=loop
@@ -77,6 +82,7 @@ async def main(loop, routing_key):
     channel = await connection.channel()
 
     dest_exchange = await channel.declare_exchange(name="rpc_direct_exchange", type=ExchangeType.DIRECT)
+    flow_logs_exchange = await channel.declare_exchange(name='flowlogs', type=ExchangeType.FANOUT, durable=True)
 
     queue = await channel.declare_queue("", exclusive=True)
 
@@ -90,3 +96,11 @@ async def main(loop, routing_key):
 def start_consumer(routing_key=routingKey):
     loop.create_task(main(loop, routing_key))
     loop.run_forever()
+
+def log(message):
+    global flow_logs_exchange
+
+    if flow_logs_exchange is not None:
+        flow_logs_exchange.publish(Message(body=json.dumps(message).encode("utf-8")), "")
+    else:
+        print("Connection not established. Call start_consumer first.")

package/dist/utils.js

@@ -61,7 +61,8 @@ function delay(ms) {
 exports.delay = delay;
 async function deleteFiles(dir, ...filenames) {
     for (const filename of filenames) {
-        await fs_1.promises.unlink((0, path_1.join)(dir, filename)).catch((err) => { });
+        await fs_1.promises.unlink((0, path_1.join)(dir, filename)).catch((err) => {
+        });
     }
 }
 exports.deleteFiles = deleteFiles;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hahnpro/flow-sdk",
-  "version": "4.21.1",
+  "version": "4.22.0",
   "description": "SDK for building Flow Modules",
   "license": "MIT",
   "author": {
@@ -24,30 +24,30 @@
     "access": "public"
   },
   "dependencies": {
-    "@hahnpro/hpc-api": "3.2.0",
+    "@hahnpro/hpc-api": "3.3.0",
     "amqp-connection-manager": "^3.9.0",
     "amqplib": "^0.10.0",
     "class-transformer": "0.5.1",
     "class-validator": "~0.13.2",
-    "cloudevents": "^6.0.1",
+    "cloudevents": "^6.0.2",
     "lodash": "^4.17.21",
     "object-sizeof": "^1.6.3",
     "python-shell": "^3.0.1",
     "reflect-metadata": "^0.1.13",
-    "rxjs": "^7.5.5",
+    "rxjs": "^7.5.6",
     "string-interp": "^0.3.6"
   },
   "devDependencies": {
-    "@golevelup/nestjs-rabbitmq": "^2.4.0",
+    "@golevelup/nestjs-rabbitmq": "^3.0.0",
     "@nestjs/common": "^8.4.7",
     "@types/amqp-connection-manager": "^2.0.12",
     "@types/amqplib": "^0.8.2",
-    "@types/jest": "^28.1.1",
+    "@types/jest": "^28.1.6",
     "@types/lodash": "^4.14.182",
-    "@types/node": "^16.11.40",
+    "@types/node": "^16.11.45",
     "class-validator-jsonschema": "^3.1.1",
-    "jest": "^28.1.1",
-    "typescript": "^4.7.3"
+    "jest": "^28.1.3",
+    "typescript": "^4.7.4"
   },
   "engines": {
     "node": "^14.13.1 || >=16.0.0"