@hahnpro/flow-sdk 4.23.8 → 4.24.0

package/dist/FlowApplication.d.ts

@@ -1,36 +1,47 @@
 import 'reflect-metadata';
+import { API } from '@hahnpro/hpc-api';
+import { AmqpConnectionManager } from 'amqp-connection-manager';
 import { CloudEvent } from 'cloudevents';
 import { PartialObserver } from 'rxjs';
-import { API } from '@hahnpro/hpc-api';
-import { AmqpConnection, Nack } from './amqp';
+import { AmqpConnection, AmqpConnectionConfig, Nack } from './amqp';
 import { ClassType, Flow, FlowElementContext } from './flow.interface';
 import { FlowEvent } from './FlowEvent';
 import { Logger } from './FlowLogger';
 import { RpcClient } from './RpcClient';
+interface FlowAppConfig {
+    logger?: Logger;
+    amqpConfig?: AmqpConnectionConfig;
+    amqpConnection?: AmqpConnectionManager;
+    skipApi?: boolean;
+    explicitInit?: boolean;
+}
 export declare class FlowApplication {
     private modules;
     private flow;
-    private readonly baseLogger?;
-    private amqpConnection?;
-    private skipApi;
-    api: API;
+    private _api;
+    private _rpcClient;
+    private amqpChannel;
+    private readonly amqpConnection;
+    private readonly baseLogger;
     private context;
     private declarations;
     private elements;
+    private initialized;
     private readonly logger;
     private outputStreamMap;
     private outputQueueMetrics;
     private performanceMap;
     private properties;
-    private _rpcClient;
-    private initialized;
+    private readonly skipApi;
+    constructor(modules: ClassType<any>[], flow: Flow, config?: FlowAppConfig);
     constructor(modules: ClassType<any>[], flow: Flow, baseLogger?: Logger, amqpConnection?: AmqpConnection, skipApi?: boolean, explicitInit?: boolean);
+    get rpcClient(): RpcClient;
+    get api(): API;
     init(): Promise<void>;
     private publishLifecycleEvent;
     private setQueueMetrics;
     private updateMetrics;
     subscribe: (streamId: string, observer: PartialObserver<FlowEvent>) => import("rxjs").Subscription;
-    rpcClient: () => RpcClient;
     emit: (event: FlowEvent) => void;
     emitPartial: (completeEvent: FlowEvent, partialEvent: FlowEvent) => void;
     getProperties(): Record<string, any>;
@@ -39,7 +50,7 @@ export declare class FlowApplication {
      * Publish a flow event to the amqp flowlogs exchange.
      * If the event size exceeds the limit it will be truncated
      */
-    publishEvent: (event: FlowEvent) => void;
+    publishEvent: (event: FlowEvent) => Promise<boolean>;
     /**
      * Calls onDestroy lifecycle method on all flow elements,
      * closes amqp connection after allowing logs to be processed and published
@@ -56,3 +67,4 @@ export interface Context extends FlowElementContext {
     app?: FlowApplication;
     logger?: Logger;
 }
+export {};

package/dist/FlowApplication.js

@@ -3,36 +3,33 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FlowApplication = void 0;
 const tslib_1 = require("tslib");
 require("reflect-metadata");
+const hpc_api_1 = require("@hahnpro/hpc-api");
 const cloudevents_1 = require("cloudevents");
+const crypto_1 = require("crypto");
 const object_sizeof_1 = tslib_1.__importDefault(require("object-sizeof"));
 const perf_hooks_1 = require("perf_hooks");
 const rxjs_1 = require("rxjs");
 const operators_1 = require("rxjs/operators");
-const crypto_1 = require("crypto");
-const hpc_api_1 = require("@hahnpro/hpc-api");
 const amqp_1 = require("./amqp");
-const utils_1 = require("./utils");
 const flow_interface_1 = require("./flow.interface");
 const FlowLogger_1 = require("./FlowLogger");
 const RpcClient_1 = require("./RpcClient");
+const utils_1 = require("./utils");
 const MAX_EVENT_SIZE_BYTES = +process.env.MAX_EVENT_SIZE_BYTES || 512 * 1024;
 const WARN_EVENT_PROCESSING_SEC = +process.env.WARN_EVENT_PROCESSING_SEC || 60;
 const WARN_EVENT_QUEUE_SIZE = +process.env.WARN_EVENT_QUEUE_SIZE || 100;
 class FlowApplication {
-    constructor(modules, flow, baseLogger, amqpConnection, skipApi = false, explicitInit = false) {
+    constructor(modules, flow, baseLoggerOrConfig, amqpConnection, skipApi, explicitInit) {
         this.modules = modules;
         this.flow = flow;
-        this.baseLogger = baseLogger;
-        this.amqpConnection = amqpConnection;
-        this.skipApi = skipApi;
         this.declarations = {};
         this.elements = {};
+        this.initialized = false;
         this.outputStreamMap = new Map();
         this.outputQueueMetrics = new Map();
         this.performanceMap = new Map();
-        this.initialized = false;
         this.publishLifecycleEvent = (element, flowEventId, eventType, data = {}) => {
-            if (!this.amqpConnection) {
+            if (!this.amqpChannel) {
                 return;
             }
             try {
@@ -46,7 +43,7 @@ class FlowApplication {
                     time: new Date().toISOString(),
                 });
                 const message = event.toJSON();
-                return this.amqpConnection.publish('flow', 'lifecycle', message);
+                return this.amqpChannel.publish('flow', 'lifecycle', message);
             }
             catch (err) {
                 this.logger.error(err);
@@ -76,7 +73,6 @@ class FlowApplication {
             }
         };
         this.subscribe = (streamId, observer) => this.getOutputStream(streamId).subscribe(observer);
-        this.rpcClient = () => this._rpcClient;
         this.emit = (event) => {
             if (event) {
                 try {
@@ -139,7 +135,7 @@ class FlowApplication {
                         data: { deploymentId: this.context.deploymentId, status: 'updated' },
                     };
                     try {
-                        (_f = this.amqpConnection) === null || _f === void 0 ? void 0 : _f.publish('deployment', 'health', statusEvent);
+                        (_f = this.amqpChannel) === null || _f === void 0 ? void 0 : _f.publish('deployment', 'health', statusEvent);
                     }
                     catch (err) {
                         this.logger.error(err);
@@ -155,7 +151,7 @@ class FlowApplication {
                         data: { deploymentId: this.context.deploymentId, status: 'updating failed' },
                     };
                     try {
-                        (_g = this.amqpConnection) === null || _g === void 0 ? void 0 : _g.publish('deployment', 'health', statusEvent);
+                        (_g = this.amqpChannel) === null || _g === void 0 ? void 0 : _g.publish('deployment', 'health', statusEvent);
                     }
                     catch (e) {
                         this.logger.error(e);
@@ -183,7 +179,7 @@ class FlowApplication {
             return undefined;
         };
         this.publishEvent = (event) => {
-            if (!this.amqpConnection) {
+            if (!this.amqpChannel) {
                 return;
             }
             try {
@@ -191,13 +187,26 @@ class FlowApplication {
                 if ((0, object_sizeof_1.default)(message) > MAX_EVENT_SIZE_BYTES) {
                     message.data = (0, utils_1.truncate)(message.data);
                 }
-                return this.amqpConnection.publish('flowlogs', '', message);
+                return this.amqpChannel.publish('flowlogs', '', message);
             }
             catch (err) {
                 this.logger.error(err);
             }
         };
-        this.logger = new FlowLogger_1.FlowLogger(Object.assign({ id: 'none', functionFqn: 'FlowApplication' }, flow === null || flow === void 0 ? void 0 : flow.context), baseLogger || undefined, this.publishEvent);
+        if (baseLoggerOrConfig && !baseLoggerOrConfig.log) {
+            const config = baseLoggerOrConfig;
+            this.baseLogger = config.logger;
+            this.amqpConnection = config.amqpConnection || (0, amqp_1.createAmqpConnection)(config.amqpConfig);
+            this.skipApi = config.skipApi || false;
+            explicitInit = config.explicitInit || false;
+        }
+        else {
+            this.baseLogger = baseLoggerOrConfig;
+            this.amqpConnection = amqpConnection === null || amqpConnection === void 0 ? void 0 : amqpConnection.managedConnection;
+            this.skipApi = skipApi || false;
+            explicitInit = explicitInit || false;
+        }
+        this.logger = new FlowLogger_1.FlowLogger(Object.assign({ id: 'none', functionFqn: 'FlowApplication' }, flow === null || flow === void 0 ? void 0 : flow.context), this.baseLogger || undefined, this.publishEvent);
         process.once('uncaughtException', (err) => {
             this.logger.error('Uncaught exception!');
             this.logger.error(err);
@@ -215,14 +224,24 @@ class FlowApplication {
             this.init();
         }
     }
+    get rpcClient() {
+        if (!this._rpcClient && this.amqpConnection) {
+            this._rpcClient = new RpcClient_1.RpcClient(this.amqpConnection, this.logger);
+        }
+        return this._rpcClient;
+    }
+    get api() {
+        return this._api;
+    }
     async init() {
+        var _a;
         if (this.initialized)
             return;
         this.context = Object.assign({}, this.flow.context);
         this.properties = this.flow.properties || {};
         try {
             if (!this.skipApi) {
-                this.api = new hpc_api_1.API();
+                this._api = new hpc_api_1.API();
             }
         }
         catch (err) {
@@ -232,29 +251,28 @@ class FlowApplication {
             this.logger.error(new Error(err));
             await this.destroy(1);
         };
-        if (this.amqpConnection) {
-            try {
-                await this.amqpConnection.managedChannel.assertExchange('deployment', 'direct', { durable: true });
-                await this.amqpConnection.managedChannel.assertExchange('flowlogs', 'fanout', { durable: true });
-                await this.amqpConnection.managedChannel.assertExchange('flow', 'direct', { durable: true });
-            }
-            catch (e) {
-                await logErrorAndExit(`Could not assert exchanges: ${e}`);
-                return;
-            }
-            try {
-                await this.amqpConnection.createSubscriber((msg) => this.onMessage(msg), {
-                    exchange: 'deployment',
-                    routingKey: this.context.deploymentId,
-                    queueOptions: { durable: false, exclusive: true },
-                }, 'FlowApplication.onMessage');
-            }
-            catch (err) {
-                await logErrorAndExit(`Could not subscribe to deployment exchange: ${err}`);
-                return;
-            }
-            this._rpcClient = new RpcClient_1.RpcClient(this.amqpConnection);
-        }
+        this.amqpChannel = (_a = this.amqpConnection) === null || _a === void 0 ? void 0 : _a.createChannel({
+            json: true,
+            setup: async (channel) => {
+                try {
+                    await channel.assertExchange('deployment', 'direct', { durable: true });
+                    await channel.assertExchange('flowlogs', 'fanout', { durable: true });
+                    await channel.assertExchange('flow', 'direct', { durable: true });
+                }
+                catch (e) {
+                    await logErrorAndExit(`Could not assert exchanges: ${e}`);
+                }
+                try {
+                    const queue = await channel.assertQueue(null, { durable: false, exclusive: true });
+                    await channel.bindQueue(queue.queue, 'deployment', this.context.deploymentId);
+                    await channel.consume(queue.queue, (msg) => this.onMessage(msg));
+                }
+                catch (err) {
+                    await logErrorAndExit(`Could not subscribe to deployment exchange: ${err}`);
+                }
+            },
+        });
+        this.amqpChannel && (await this.amqpChannel.waitForConnect());
         for (const module of this.modules) {
             const moduleName = Reflect.getMetadata('module:name', module);
             const moduleDeclarations = Reflect.getMetadata('module:declarations', module);
@@ -344,19 +362,19 @@ class FlowApplication {
         return this.properties;
     }
     async destroy(exitCode = 0) {
-        var _a, _b, _c, _d;
+        var _a;
         try {
             try {
                 for (const element of Object.values(this.elements)) {
                     (_a = element === null || element === void 0 ? void 0 : element.onDestroy) === null || _a === void 0 ? void 0 : _a.call(element);
                 }
-                await ((_b = this._rpcClient) === null || _b === void 0 ? void 0 : _b.close());
+                this._rpcClient && (await this._rpcClient.close());
             }
             catch (err) {
                 this.logger.error(err);
             }
             await (0, utils_1.delay)(250);
-            await ((_d = (_c = this.amqpConnection) === null || _c === void 0 ? void 0 : _c.managedConnection) === null || _d === void 0 ? void 0 : _d.close());
+            this.amqpConnection && (await this.amqpConnection.close());
         }
         catch (err) {
             console.error(err);

package/dist/FlowElement.js

@@ -93,7 +93,7 @@ class FlowElement {
     }
     async callRpcFunction(functionName, ...args) {
         var _a;
-        return (_a = this.app) === null || _a === void 0 ? void 0 : _a.rpcClient().callFunction(this.rpcRoutingKey, functionName, ...args);
+        return (_a = this.app) === null || _a === void 0 ? void 0 : _a.rpcClient.callFunction(this.rpcRoutingKey, functionName, ...args);
     }
     runPyRpcScript(scriptPath, ...args) {
         const options = {

package/dist/RpcClient.d.ts

@@ -1,9 +1,10 @@
-import { AmqpConnection } from './amqp';
+import type { AmqpConnectionManager } from 'amqp-connection-manager';
+import { FlowLogger } from './FlowLogger';
 export declare class RpcClient {
-    private amqpConnection;
-    private channel;
+    private readonly logger?;
+    private readonly channel;
     private openRequests;
-    constructor(amqpConnection: AmqpConnection);
+    constructor(amqpConnection: AmqpConnectionManager, logger?: FlowLogger);
     private onMessage;
     callFunction: (routingKey: string, functionName: string, ...args: any[]) => Promise<unknown>;
     declareFunction: (routingKey: string, name: string) => (...args: any[]) => Promise<unknown>;

package/dist/RpcClient.js

@@ -1,10 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RpcClient = void 0;
+const tslib_1 = require("tslib");
 const crypto_1 = require("crypto");
+const object_sizeof_1 = tslib_1.__importDefault(require("object-sizeof"));
+const MAX_MSG_SIZE = +process.env.MAX_RPC_MSG_SIZE_BYTES;
+const WARN_MSG_SIZE = +process.env.WARN_RPC_MSG_SIZE_BYTES;
 class RpcClient {
-    constructor(amqpConnection) {
-        this.amqpConnection = amqpConnection;
+    constructor(amqpConnection, logger) {
+        this.logger = logger;
         this.openRequests = new Map();
         this.onMessage = (msg) => {
             if (this.openRequests.has(msg.properties.correlationId)) {
@@ -39,6 +43,16 @@ class RpcClient {
         this.callFunction = (routingKey, functionName, ...args) => {
             const stack = new Error('test').stack;
             return new Promise((resolve, reject) => {
+                var _a;
+                if (MAX_MSG_SIZE || WARN_MSG_SIZE) {
+                    const messageSize = (0, object_sizeof_1.default)(args);
+                    if (messageSize > MAX_MSG_SIZE) {
+                        throw new Error(`Max RPC message size exceeded: ${messageSize} bytes / ${MAX_MSG_SIZE} bytes`);
+                    }
+                    if (messageSize > WARN_MSG_SIZE) {
+                        (_a = this.logger) === null || _a === void 0 ? void 0 : _a.warn(`Large RPC message size detected: ${messageSize} bytes`);
+                    }
+                }
                 const call = { functionName, arguments: args };
                 const correlationId = (0, crypto_1.randomUUID)();
                 this.openRequests.set(correlationId, { resolve, reject, trace: RpcClient.formatTrace(stack) });
@@ -53,7 +67,7 @@ class RpcClient {
         if (!amqpConnection) {
             throw new Error('currently no amqp connection available');
         }
-        this.channel = this.amqpConnection.managedConnection.createChannel({
+        this.channel = amqpConnection.createChannel({
             json: true,
             setup: async (channel) => {
                 await channel.assertExchange('rpc_direct_exchange', 'direct', { durable: false });

package/dist/amqp.d.ts

@@ -1,37 +1,19 @@
-import type { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
-import type { Channel, Connection, ConsumeMessage, Options } from 'amqplib';
-interface SubscriptionResult {
-    consumerTag: string;
-}
+import { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
 export interface AmqpConnection {
-    channel: Channel;
-    connection: Connection;
     managedChannel: ChannelWrapper;
     managedConnection: AmqpConnectionManager;
-    createSubscriber<T>(handler: (msg: T | undefined, rawMessage?: ConsumeMessage) => Promise<any | undefined | void>, msgOptions: MessageHandlerOptions, originalHandlerName: string): Promise<SubscriptionResult>;
-    publish(exchange: string, routingKey: string, message: any, options?: Options.Publish): void;
 }
+export interface AmqpConnectionConfig {
+    protocol?: string;
+    hostname?: string;
+    vhost?: string;
+    user?: string;
+    password?: string;
+    port?: number;
+}
+export declare function createAmqpConnection(config: AmqpConnectionConfig): AmqpConnectionManager;
 export declare class Nack {
     private readonly _requeue;
     constructor(_requeue?: boolean);
     get requeue(): boolean;
 }
-export interface MessageHandlerOptions {
-    exchange: string;
-    routingKey: string | string[];
-    queue?: string;
-    queueOptions?: QueueOptions;
-}
-export interface QueueOptions {
-    durable?: boolean;
-    exclusive?: boolean;
-    autoDelete?: boolean;
-    arguments?: any;
-    messageTtl?: number;
-    expires?: number;
-    deadLetterExchange?: string;
-    deadLetterRoutingKey?: string;
-    maxLength?: number;
-    maxPriority?: number;
-}
-export {};

package/dist/amqp.js

@@ -1,6 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Nack = void 0;
+exports.Nack = exports.createAmqpConnection = void 0;
+const amqp_connection_manager_1 = require("amqp-connection-manager");
+function createAmqpConnection(config) {
+    if (!config) {
+        return;
+    }
+    const { protocol = process.env.RABBIT_PROTOCOL || 'amqp', hostname = process.env.RABBIT_HOST || 'localhost', port = +process.env.RABBIT_PORT || 5672, user = process.env.RABBIT_USER || 'guest', password = process.env.RABBIT_PASSWORD || 'guest', vhost = process.env.RABBIT_VHOST || '', } = config;
+    const uri = `${protocol}://${user}:${password}@${hostname}:${port}${vhost ? '/' + vhost : ''}`;
+    return (0, amqp_connection_manager_1.connect)(uri);
+}
+exports.createAmqpConnection = createAmqpConnection;
 class Nack {
     constructor(_requeue = false) {
         this._requeue = _requeue;

package/dist/rpc_server.py

@@ -12,6 +12,7 @@ host = os.getenv("RABBIT_HOST", "localhost")
 port = os.getenv("RABBIT_PORT", "5672")
 vhost = os.getenv("RABBIT_VHOST", "")
 routingKey = os.getenv("RPC_ROUTING_KEY", "rpc")
+max_msg_size = int(os.getenv("MAX_RPC_MSG_SIZE_BYTES", "0"))
 
 remote_procedures = {}
 flow_logs_exchange: AbstractRobustExchange
@@ -64,11 +65,18 @@ async def on_message(exchange: Exchange, message: IncomingMessage):
 
 
 async def send_reply(exchange: Exchange, reply, original_message: Message):
+    body = json.dumps(reply).encode("utf-8")
+
+    if max_msg_size > 0 and len(body) > max_msg_size:
+        body = json.dumps(
+            {
+                "type": "error",
+                "message": "Max RPC message size exceeded: " + str(len(body)) + " bytes / " + max_msg_size + " bytes",
+            }
+        ).encode("utf-8")
+
     await exchange.publish(
-        Message(
-            body=json.dumps(reply).encode("utf-8"),
-            correlation_id=original_message.correlation_id
-        ),
+        Message(body=body, correlation_id=original_message.correlation_id),
         routing_key=original_message.reply_to,
     )
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hahnpro/flow-sdk",
-  "version": "4.23.8",
+  "version": "4.24.0",
   "description": "SDK for building Flow Modules",
   "license": "MIT",
   "author": {
@@ -24,7 +24,7 @@
     "access": "public"
   },
   "dependencies": {
-    "amqp-connection-manager": "^3.9.0",
+    "amqp-connection-manager": "^4.1.11",
     "amqplib": "^0.10.3",
     "class-transformer": "0.5.1",
     "class-validator": "~0.14.0",
@@ -38,9 +38,6 @@
     "@hahnpro/hpc-api": "3.7.1"
   },
   "devDependencies": {
-    "@golevelup/nestjs-rabbitmq": "^3.5.0",
-    "@nestjs/common": "^9.3.9",
-    "@types/amqp-connection-manager": "^2.0.12",
     "@types/amqplib": "^0.10.1",
     "@types/jest": "^29.4.2",
     "@types/lodash": "^4.14.191",