@hahnpro/flow-sdk 8.0.12 → 9.0.0

package/dist/FlowApplication.d.ts

@@ -1,8 +1,8 @@
 import 'reflect-metadata';
 import { API, HttpClient, MockAPI } from '@hahnpro/hpc-api';
 import { NatsConnection, ConnectionOptions as NatsConnectionOptions } from '@nats-io/nats-core';
-import { ConsumeMessage } from 'amqplib';
 import { AmqpConnectionManager } from 'amqp-connection-manager';
+import { CloudEvent } from 'cloudevents';
 import { PartialObserver } from 'rxjs';
 import { AmqpConnection, AmqpConnectionConfig } from './amqp';
 import { ClassType, Flow, FlowElementContext } from './flow.interface';
@@ -42,6 +42,7 @@ export declare class FlowApplication {
     private readonly skipApi;
     private readonly apiClient?;
     private readonly contextManager;
+    private natsMessageIterator;
     constructor(modules: ClassType<any>[], flow: Flow, config?: FlowAppConfig);
     constructor(modules: ClassType<any>[], flow: Flow, baseLogger?: Logger, amqpConnection?: AmqpConnection, natsConnection?: NatsConnection, skipApi?: boolean, explicitInit?: boolean);
     get rpcClient(): RpcClient;
@@ -49,6 +50,7 @@ export declare class FlowApplication {
     get natsConnection(): NatsConnection;
     getContextManager(): ContextManager;
     getProperties(): Record<string, any>;
+    private consumeNatsMessagesOfConsumer;
     init(): Promise<void>;
     private publishLifecycleEvent;
     private setQueueMetrics;
@@ -56,12 +58,14 @@ export declare class FlowApplication {
     subscribe: (streamId: string, observer: PartialObserver<FlowEvent>) => import("rxjs").Subscription;
     emit: (event: FlowEvent) => void;
     emitPartial: (completeEvent: FlowEvent, partialEvent: FlowEvent) => void;
-    onMessage: (msg: ConsumeMessage) => Promise<void>;
+    onMessage: (cloudEvent: CloudEvent) => Promise<void>;
     /**
      * Publish a flow event to the amqp flowlogs exchange.
      * If the event size exceeds the limit it will be truncated
+     *
+     * TODO warum darf hier nicht false zurückgegeben werden? -> erzeugt loop
      */
-    publishEvent: (event: FlowEvent) => Promise<boolean>;
+    publishNatsEventFlowlogs: (event: FlowEvent) => Promise<boolean>;
     /**
      * Calls onDestroy lifecycle method on all flow elements,
      * closes amqp connection after allowing logs to be processed and published

package/dist/FlowApplication.js

@@ -5,7 +5,6 @@ const tslib_1 = require("tslib");
 require("reflect-metadata");
 const hpc_api_1 = require("@hahnpro/hpc-api");
 const cloudevents_1 = require("cloudevents");
-const crypto_1 = require("crypto");
 const lodash_1 = require("lodash");
 const object_sizeof_1 = tslib_1.__importDefault(require("object-sizeof"));
 const perf_hooks_1 = require("perf_hooks");
@@ -31,13 +30,13 @@ class FlowApplication {
         this.outputStreamMap = new Map();
         this.outputQueueMetrics = new Map();
         this.performanceMap = new Map();
-        this.publishLifecycleEvent = (element, flowEventId, eventType, data = {}) => {
+        this.publishLifecycleEvent = async (element, flowEventId, eventType, data = {}) => {
             if (!this.amqpChannel) {
                 return;
             }
             try {
                 const { flowId, deploymentId, id: elementId, functionFqn, inputStreamId } = element.getMetadata();
-                const event = new cloudevents_1.CloudEvent({
+                const natsEvent = {
                     source: `flows/${flowId}/deployments/${deploymentId}/elements/${elementId}`,
                     type: eventType,
                     data: {
@@ -46,10 +45,8 @@ class FlowApplication {
                         inputStreamId,
                         ...data,
                     },
-                    time: new Date().toISOString(),
-                });
-                const message = event.toJSON();
-                return this.amqpChannel.publish('flow', 'lifecycle', message);
+                };
+                await (0, nats_1.publishNatsEvent)(this.logger, this.natsConnection, natsEvent, `${nats_1.natsFlowsPrefixFlowDeployment}.flowlifecycle.${deploymentId}`);
             }
             catch (err) {
                 this.logger.error(err);
@@ -82,7 +79,7 @@ class FlowApplication {
         this.emit = (event) => {
             if (event) {
                 try {
-                    this.publishEvent(event);
+                    this.publishNatsEventFlowlogs(event);
                     if (this.outputStreamMap.has(event.getStreamId())) {
                         this.getOutputStream(event.getStreamId()).next(event);
                     }
@@ -98,27 +95,26 @@ class FlowApplication {
                     this.getOutputStream(completeEvent.getStreamId()).next(completeEvent);
                 }
                 if (partialEvent) {
-                    this.publishEvent(partialEvent);
+                    this.publishNatsEventFlowlogs(partialEvent);
                 }
             }
             catch (err) {
                 this.logger.error(err);
             }
         };
-        this.onMessage = async (msg) => {
-            let event;
-            try {
-                event = JSON.parse(msg.content.toString());
-            }
-            catch (err) {
-                this.logger.error(err);
-                return;
-            }
-            if (event.type === 'com.flowstudio.deployment.update') {
+        this.onMessage = async (cloudEvent) => {
+            if (cloudEvent.subject.endsWith('.update')) {
+                let event;
+                try {
+                    event = JSON.parse(cloudEvent.content.toString());
+                }
+                catch (err) {
+                    this.logger.error(err);
+                    return;
+                }
                 try {
                     const flow = event.data;
                     if (!flow) {
-                        this.amqpChannel.nack(msg, false, false);
                         return;
                     }
                     let context = {};
@@ -143,39 +139,33 @@ class FlowApplication {
                         this.elements?.[element.id]?.setPropertiesWithPlaceholders((0, lodash_1.cloneDeep)(element.properties));
                         this.elements?.[element.id]?.onPropertiesChanged(this.contextManager.replaceAllPlaceholderProperties(this.elements[element.id].getPropertiesWithPlaceholders()));
                     }
-                    const statusEvent = {
-                        eventId: (0, crypto_1.randomUUID)(),
-                        eventTime: new Date().toISOString(),
-                        eventType: 'com.hahnpro.event.health',
-                        contentType: 'application/json',
-                        data: { deploymentId: this.context.deploymentId, status: 'updated' },
+                    const natsEvent = {
+                        source: `hpc/flow-application`,
+                        type: `${nats_1.natsFlowsPrefixFlowDeployment}.health`,
+                        subject: `${this.context.deploymentId}`,
+                        data: {
+                            deploymentId: this.context.deploymentId,
+                            status: 'updated',
+                        },
                     };
-                    try {
-                        this.amqpChannel?.publish('deployment', 'health', statusEvent);
-                    }
-                    catch (err) {
-                        this.logger.error(err);
-                    }
+                    await (0, nats_1.publishNatsEvent)(this.logger, this.natsConnection, natsEvent);
                 }
                 catch (err) {
                     this.logger.error(err);
-                    const statusEvent = {
-                        eventId: (0, crypto_1.randomUUID)(),
-                        eventTime: new Date().toISOString(),
-                        eventType: 'com.hahnpro.event.health',
-                        contentType: 'application/json',
-                        data: { deploymentId: this.context.deploymentId, status: 'updating failed' },
+                    const natsEvent = {
+                        source: `hpc/flow-application`,
+                        type: `${nats_1.natsFlowsPrefixFlowDeployment}.health`,
+                        subject: `${this.context.deploymentId}`,
+                        data: {
+                            deploymentId: this.context.deploymentId,
+                            status: 'updating failed',
+                        },
                     };
-                    try {
-                        this.amqpChannel?.publish('deployment', 'health', statusEvent);
-                    }
-                    catch (e) {
-                        this.logger.error(e);
-                    }
+                    await (0, nats_1.publishNatsEvent)(this.logger, this.natsConnection, natsEvent);
                 }
             }
-            else if (event.type === 'com.flowstudio.deployment.message') {
-                const data = event.data;
+            else if (cloudEvent.subject.endsWith('.message')) {
+                const data = cloudEvent.data;
                 const elementId = data?.elementId;
                 if (elementId) {
                     this.elements?.[elementId]?.onMessage?.(data);
@@ -186,15 +176,12 @@ class FlowApplication {
                     }
                 }
             }
-            else if (event.type === 'com.flowstudio.deployment.destroy') {
+            else if (cloudEvent.subject.endsWith('.destroy')) {
                 this.destroy();
             }
-            else {
-                this.amqpChannel.nack(msg, false, false);
-            }
         };
-        this.publishEvent = (event) => {
-            if (!this.amqpChannel) {
+        this.publishNatsEventFlowlogs = async (event) => {
+            if (!this.natsConnection || this.natsConnection.isClosed()) {
                 return;
             }
             try {
@@ -202,7 +189,14 @@ class FlowApplication {
                 if ((0, object_sizeof_1.default)(message) > MAX_EVENT_SIZE_BYTES) {
                     message.data = (0, utils_1.truncate)(message.data);
                 }
-                return this.amqpChannel.publish('flowlogs', '', message);
+                const natsEvent = {
+                    source: `hpc/flow-application`,
+                    type: `${nats_1.natsFlowsPrefixFlowDeployment}.flowlogs`,
+                    subject: `${this.context.deploymentId}`,
+                    data: message,
+                };
+                await (0, nats_1.publishNatsEvent)(this.logger, this.natsConnection, natsEvent);
+                return true;
             }
             catch (err) {
                 this.logger.error(err);
@@ -227,7 +221,7 @@ class FlowApplication {
             explicitInit = explicitInit || false;
             this._api = mockApi || null;
         }
-        this.logger = new FlowLogger_1.FlowLogger({ id: 'none', functionFqn: 'FlowApplication', ...flow?.context }, this.baseLogger || undefined, this.publishEvent);
+        this.logger = new FlowLogger_1.FlowLogger({ id: 'none', functionFqn: 'FlowApplication', ...flow?.context }, this.baseLogger || undefined, this.publishNatsEventFlowlogs);
         this.contextManager = new ContextManager_1.ContextManager(this.logger, this.flow?.properties);
         process.once('uncaughtException', (err) => {
             this.logger.error('Uncaught exception!');
@@ -264,6 +258,33 @@ class FlowApplication {
     getProperties() {
         return this.contextManager.getProperties();
     }
+    async consumeNatsMessagesOfConsumer(consumer, consumerOptions) {
+        if (this.natsMessageIterator) {
+            await this.natsMessageIterator.close();
+        }
+        this.natsMessageIterator = await consumer.consume(consumerOptions);
+        for await (const msg of this.natsMessageIterator) {
+            try {
+                let event;
+                try {
+                    event = new cloudevents_1.CloudEvent(msg.json());
+                    event.validate();
+                }
+                catch (error) {
+                    this.logger.error('Message is not a valid CloudEvent and will be discarded');
+                    msg.ack();
+                    continue;
+                }
+                await this.onMessage(event);
+                msg.ack();
+            }
+            catch (error) {
+                this.logger.error('Error processing message');
+                this.logger.error(error);
+                msg.nak(1000);
+            }
+        }
+    }
     async init() {
         if (this.initialized)
             return;
@@ -294,25 +315,29 @@ class FlowApplication {
                 await logErrorAndExit(`Could not connect to the NATS-Servers: ${err}`);
             }
         }
+        if (this._natsConnection && !this._natsConnection.isClosed() && this.context?.deploymentId !== undefined) {
+            try {
+                const consumerOptions = {
+                    ...nats_1.defaultConsumerConfig,
+                    name: `flow-deployment-${this.context.deploymentId}`,
+                    filter_subject: `${nats_1.natsFlowsPrefixFlowDeployment}.${this.context.deploymentId}.*`,
+                };
+                const consumer = await (0, nats_1.getOrCreateConsumer)(this.logger, this._natsConnection, nats_1.FLOWS_STREAM_NAME, consumerOptions.name, consumerOptions);
+                this.consumeNatsMessagesOfConsumer(consumer, consumerOptions);
+            }
+            catch (e) {
+                await logErrorAndExit(`Could not set up consumer for deployment messages exchanges: ${e}`);
+            }
+        }
         this.amqpChannel = this.amqpConnection?.createChannel({
             json: true,
             setup: async (channel) => {
                 try {
-                    await channel.assertExchange('deployment', 'direct', { durable: true });
-                    await channel.assertExchange('flowlogs', 'fanout', { durable: true });
                     await channel.assertExchange('flow', 'direct', { durable: true });
                 }
                 catch (e) {
                     await logErrorAndExit(`Could not assert exchanges: ${e}`);
                 }
-                try {
-                    const queue = await channel.assertQueue(null, { durable: false, exclusive: true });
-                    await channel.bindQueue(queue.queue, 'deployment', this.context.deploymentId);
-                    await channel.consume(queue.queue, (msg) => this.onMessage(msg));
-                }
-                catch (err) {
-                    await logErrorAndExit(`Could not subscribe to deployment exchange: ${err}`);
-                }
             },
         });
         if (this.amqpChannel) {
@@ -421,6 +446,11 @@ class FlowApplication {
             if (this.amqpConnection) {
                 await this.amqpConnection.close();
             }
+            if (this.natsConnection && !this._natsConnection.isClosed()) {
+                await this._natsConnection?.drain();
+            }
+            await this.natsMessageIterator?.close();
+            await this._natsConnection?.close();
         }
         catch (err) {
             console.error(err);

package/dist/FlowElement.js

@@ -34,7 +34,7 @@ class FlowElement {
         this.app = app;
         this.api = this.app?.api;
         this.metadata = { ...metadata, functionFqn: this.functionFqn };
-        this.logger = new FlowLogger_1.FlowLogger(this.metadata, logger || undefined, this.app?.publishEvent);
+        this.logger = new FlowLogger_1.FlowLogger(this.metadata, logger || undefined, this.app?.publishNatsEventFlowlogs);
         this.rpcRoutingKey = (this.metadata.flowId || '') + (this.metadata.deploymentId || '') + this.metadata.id;
         if (properties) {
             this.setProperties(properties);

package/dist/nats.d.ts

@@ -1,2 +1,11 @@
 import { ConnectionOptions, NatsConnection } from '@nats-io/nats-core';
+import { CloudEvent } from 'cloudevents';
+import { Consumer, ConsumerConfig, PubAck } from '@nats-io/jetstream';
+import { Logger } from './FlowLogger';
+export type NatsEvent<T> = Pick<CloudEvent<T>, 'type' | 'source' | 'subject' | 'data' | 'datacontenttype' | 'time'>;
+export declare const natsFlowsPrefixFlowDeployment = "com.hahnpro.flows.flowdeployment";
+export declare const defaultConsumerConfig: ConsumerConfig;
+export declare const FLOWS_STREAM_NAME = "flows";
+export declare function getOrCreateConsumer(logger: Logger, natsConnection: NatsConnection, streamName: string, consumerName: string, options: Partial<ConsumerConfig>): Promise<Consumer>;
+export declare function publishNatsEvent<T>(logger: Logger, nc: NatsConnection, event: NatsEvent<T>, subject?: string): Promise<PubAck>;
 export declare function createNatsConnection(config: ConnectionOptions): Promise<NatsConnection>;

package/dist/nats.js

@@ -1,7 +1,72 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.FLOWS_STREAM_NAME = exports.defaultConsumerConfig = exports.natsFlowsPrefixFlowDeployment = void 0;
+exports.getOrCreateConsumer = getOrCreateConsumer;
+exports.publishNatsEvent = publishNatsEvent;
 exports.createNatsConnection = createNatsConnection;
 const transport_node_1 = require("@nats-io/transport-node");
+const cloudevents_1 = require("cloudevents");
+const jetstream_1 = require("@nats-io/jetstream");
+const lodash_1 = require("lodash");
+exports.natsFlowsPrefixFlowDeployment = `com.hahnpro.flows.flowdeployment`;
+exports.defaultConsumerConfig = {
+    ack_policy: jetstream_1.AckPolicy.Explicit,
+    ack_wait: 30_000_000_000,
+    deliver_policy: jetstream_1.DeliverPolicy.All,
+    max_ack_pending: 1000,
+    max_deliver: -1,
+    max_waiting: 512,
+    replay_policy: jetstream_1.ReplayPolicy.Instant,
+    num_replicas: 0,
+};
+exports.FLOWS_STREAM_NAME = 'flows';
+async function getOrCreateConsumer(logger, natsConnection, streamName, consumerName, options) {
+    if (!natsConnection || natsConnection.isClosed()) {
+        throw new Error('NATS connection is not available');
+    }
+    else if (!streamName) {
+        throw new Error('Stream name is not available');
+    }
+    else if (!consumerName) {
+        throw new Error('Consumer name is not available');
+    }
+    logger.debug(`Creating consumer ${consumerName} for stream ${streamName}`);
+    const jsm = await (0, jetstream_1.jetstreamManager)(natsConnection);
+    const consumerInfo = await jsm.consumers.info(streamName, consumerName).catch((err) => {
+        if (err.status !== 404) {
+            logger.error(`Could not get consumer info of stream ${streamName}`, err);
+            logger.error(err.message);
+            throw err;
+        }
+    });
+    const consumerConfig = { ...exports.defaultConsumerConfig, ...options };
+    if (consumerInfo) {
+        const compared = (0, lodash_1.omitBy)(consumerConfig, (value, key) => (0, lodash_1.isEqual)(value, consumerInfo.config[key]));
+        if (Object.keys(compared).length !== 0) {
+            await jsm.consumers.update(streamName, consumerName, consumerConfig);
+        }
+    }
+    else {
+        await jsm.consumers.add(streamName, { name: consumerName, ...consumerConfig });
+    }
+    return await (0, jetstream_1.jetstream)(natsConnection).consumers.get(streamName, consumerName);
+}
+async function publishNatsEvent(logger, nc, event, subject) {
+    if (!nc || nc.isClosed()) {
+        return;
+    }
+    const cloudEvent = new cloudevents_1.CloudEvent({ datacontenttype: 'application/json', ...event });
+    cloudEvent.validate();
+    const js = (0, jetstream_1.jetstream)(nc);
+    if (js) {
+        return js.publish(subject || `${cloudEvent.type}.${cloudEvent.subject}`, JSON.stringify(cloudEvent.toJSON()), {
+            msgID: cloudEvent.id,
+        });
+    }
+    else {
+        logger.error(`Could not publish nats event, because jetstream is unavailable / undefined`);
+    }
+}
 async function createNatsConnection(config) {
     const servers = config?.servers ?? process.env.NATS_SERVERS?.split(',') ?? [];
     const reconnect = config?.reconnect ?? (process.env.NATS_RECONNECT ?? 'true') === 'true';

package/dist/utils.js

@@ -9,6 +9,7 @@ exports.handleApiError = handleApiError;
 exports.runPyScript = runPyScript;
 exports.truncate = truncate;
 const tslib_1 = require("tslib");
+const axios_1 = require("axios");
 const fs_1 = require("fs");
 const isPlainObject_1 = tslib_1.__importDefault(require("lodash/isPlainObject"));
 const path_1 = require("path");
@@ -69,16 +70,23 @@ async function deleteFiles(dir, ...filenames) {
     }
 }
 function handleApiError(error, logger) {
-    if (error.isAxiosError) {
-        if (error.response && error.response.data) {
-            logger.error(error.response.data);
-        }
-        else {
-            logger.error(`Error ${error.code}`);
-            logger.error(error.config);
-            if (error.stack) {
-                logger.error(error.stack);
+    if ((0, axios_1.isAxiosError)(error)) {
+        const status = error.response?.status ?? error.code ?? '';
+        const statusText = error.response?.statusText ?? '';
+        const url = error.config?.url ?? '[Unknown URL]';
+        const method = error.config?.method?.toUpperCase() ?? '';
+        let errorText = error.response?.data ?? '';
+        if (typeof errorText !== 'string') {
+            try {
+                errorText = JSON.stringify(errorText, null, 2);
             }
+            catch (_) {
+                errorText = '[Unserializable error body]';
+            }
+        }
+        logger.error(`${status} ${statusText}: ${method} request to ${url} failed\n${errorText}`);
+        if (error.stack) {
+            logger.error(error.stack);
         }
     }
     else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hahnpro/flow-sdk",
-  "version": "8.0.12",
+  "version": "9.0.0",
   "description": "SDK for building Flow Modules",
   "license": "MIT",
   "author": {
@@ -24,13 +24,14 @@
     "access": "public"
   },
   "dependencies": {
-    "@hahnpro/hpc-api": "2025.2.12",
+    "@hahnpro/hpc-api": "2025.2.13",
+    "@nats-io/jetstream": "3.0.2",
     "@nats-io/nats-core": "3.0.2",
     "@nats-io/transport-node": "3.0.2",
     "amqp-connection-manager": "4.1.14",
-    "amqplib": "0.10.7",
+    "amqplib": "0.10.8",
     "class-transformer": "0.5.1",
-    "class-validator": "~0.14.1",
+    "class-validator": "~0.14.2",
     "cloudevents": "9.0.0",
     "lodash": "4.17.21",
     "object-sizeof": "~2.6.5",
@@ -42,16 +43,16 @@
   "devDependencies": {
     "@types/amqplib": "0.10.7",
     "@types/jest": "29.5.14",
-    "@types/lodash": "4.17.16",
-    "@types/node": "22.14.1",
+    "@types/lodash": "4.17.17",
+    "@types/node": "22.15.21",
     "class-validator-jsonschema": "5.0.2",
     "jest": "29.7.0",
     "typescript": "5.8.3"
   },
   "peerDependencies": {
-    "axios": "1.8.4",
+    "axios": "1.9.0",
     "class-transformer": "0.5.1",
-    "class-validator": "0.14.1",
+    "class-validator": "0.14.2",
     "lodash": "4.17.21",
     "python-shell": "5.x"
   },