@hahnpro/flow-sdk 2026.1.1 → 2026.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +7 -7
- package/src/lib/FlowApplication.d.ts +5 -3
- package/src/lib/FlowApplication.js +170 -68
- package/src/lib/FlowElement.js +1 -1
- package/src/lib/FlowLogger.d.ts +1 -7
- package/src/lib/FlowLogger.js +11 -25
- package/src/lib/nats.d.ts +2 -2
- package/src/lib/nats.js +4 -6
- package/src/lib/utils.d.ts +0 -1
- package/src/lib/utils.js +0 -13
package/CHANGELOG.md
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hahnpro/flow-sdk",
|
|
3
|
-
"version": "2026.1.
|
|
3
|
+
"version": "2026.1.2",
|
|
4
4
|
"description": "SDK for building Flow Modules",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -17,14 +17,14 @@
|
|
|
17
17
|
"access": "public"
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@hahnpro/hpc-api": "2026.1.
|
|
20
|
+
"@hahnpro/hpc-api": "2026.1.2",
|
|
21
21
|
"@nats-io/jetstream": "3.3.0",
|
|
22
22
|
"@nats-io/nats-core": "3.3.0",
|
|
23
23
|
"@nats-io/transport-node": "3.3.0",
|
|
24
24
|
"class-transformer": "0.5.1",
|
|
25
25
|
"class-validator": "0.14.3",
|
|
26
26
|
"cloudevents": "10.0.0",
|
|
27
|
-
"lodash": "4.17.
|
|
27
|
+
"lodash": "4.17.23",
|
|
28
28
|
"object-sizeof": "2.6.5",
|
|
29
29
|
"python-shell": "5.0.0",
|
|
30
30
|
"reflect-metadata": "0.2.2",
|
|
@@ -33,17 +33,17 @@
|
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
35
|
"@types/jest": "30.0.0",
|
|
36
|
-
"@types/lodash": "4.17.
|
|
37
|
-
"@types/node": "24.10.
|
|
36
|
+
"@types/lodash": "4.17.23",
|
|
37
|
+
"@types/node": "24.10.9",
|
|
38
38
|
"class-validator-jsonschema": "5.1.0",
|
|
39
39
|
"jest": "30.2.0",
|
|
40
40
|
"typescript": "5.9.3"
|
|
41
41
|
},
|
|
42
42
|
"peerDependencies": {
|
|
43
|
-
"axios": "1.13.
|
|
43
|
+
"axios": "1.13.4",
|
|
44
44
|
"class-transformer": "0.5.1",
|
|
45
45
|
"class-validator": "0.14.3",
|
|
46
|
-
"lodash": "4.17.
|
|
46
|
+
"lodash": "4.17.23"
|
|
47
47
|
},
|
|
48
48
|
"engines": {
|
|
49
49
|
"node": ">=v24"
|
|
@@ -35,13 +35,16 @@ export declare class FlowApplication {
|
|
|
35
35
|
private readonly apiClient?;
|
|
36
36
|
private readonly contextManager;
|
|
37
37
|
private natsMessageIterator;
|
|
38
|
+
private lastErrorBecauseFlowLogCouldNotBeSend;
|
|
38
39
|
constructor(modules: ClassType<any>[], flow: Flow, config?: FlowAppConfig);
|
|
39
40
|
constructor(modules: ClassType<any>[], flow: Flow, baseLogger?: Logger, natsConnection?: NatsConnection, skipApi?: boolean, explicitInit?: boolean);
|
|
40
41
|
get api(): API;
|
|
41
42
|
get natsConnection(): NatsConnection;
|
|
42
43
|
getContextManager(): ContextManager;
|
|
43
44
|
getProperties(): Record<string, any>;
|
|
45
|
+
private logErrorAndExit;
|
|
44
46
|
private consumeNatsMessagesOfConsumer;
|
|
47
|
+
private setupNatsConsumerForFlowDeployment;
|
|
45
48
|
init(): Promise<void>;
|
|
46
49
|
private publishLifecycleEvent;
|
|
47
50
|
private setQueueMetrics;
|
|
@@ -53,10 +56,9 @@ export declare class FlowApplication {
|
|
|
53
56
|
/**
|
|
54
57
|
* Publish a flow event to NATS
|
|
55
58
|
* If the event size exceeds the limit it will be truncated
|
|
56
|
-
*
|
|
57
|
-
* TODO warum darf hier nicht false zurückgegeben werden? -> erzeugt loop
|
|
58
59
|
*/
|
|
59
|
-
|
|
60
|
+
publishFlowEventsAsFlowLogsOverNats: (event: FlowEvent) => Promise<void>;
|
|
61
|
+
private tryToSendLastErrorOnPublishingNatsEvents;
|
|
60
62
|
/**
|
|
61
63
|
* Calls onDestroy lifecycle method on all flow elements,
|
|
62
64
|
* closes NATS connection after allowing logs to be processed and published
|
|
@@ -13,6 +13,7 @@ const rxjs_1 = require("rxjs");
|
|
|
13
13
|
const operators_1 = require("rxjs/operators");
|
|
14
14
|
const ContextManager_1 = require("./ContextManager");
|
|
15
15
|
const flow_interface_1 = require("./flow.interface");
|
|
16
|
+
const FlowEvent_1 = require("./FlowEvent");
|
|
16
17
|
const FlowLogger_1 = require("./FlowLogger");
|
|
17
18
|
const nats_1 = require("./nats");
|
|
18
19
|
const utils_1 = require("./utils");
|
|
@@ -29,6 +30,7 @@ class FlowApplication {
|
|
|
29
30
|
this.outputStreamMap = new Map();
|
|
30
31
|
this.outputQueueMetrics = new Map();
|
|
31
32
|
this.performanceMap = new Map();
|
|
33
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend = null;
|
|
32
34
|
this.publishLifecycleEvent = async (element, flowEventId, eventType, data = {}) => {
|
|
33
35
|
try {
|
|
34
36
|
const { flowId, deploymentId, id: elementId, functionFqn, inputStreamId } = element.getMetadata();
|
|
@@ -42,10 +44,10 @@ class FlowApplication {
|
|
|
42
44
|
...data,
|
|
43
45
|
},
|
|
44
46
|
};
|
|
45
|
-
await (0, nats_1.publishNatsEvent)(this.
|
|
47
|
+
await (0, nats_1.publishNatsEvent)(this._natsConnection, natsEvent, `${nats_1.natsFlowsPrefixFlowDeployment}.flowlifecycle.${deploymentId}`);
|
|
46
48
|
}
|
|
47
49
|
catch (err) {
|
|
48
|
-
this.logger.error(err);
|
|
50
|
+
this.logger.error(err, { doNotPublishAsNatsEvent: true });
|
|
49
51
|
}
|
|
50
52
|
};
|
|
51
53
|
this.setQueueMetrics = (id) => {
|
|
@@ -75,7 +77,7 @@ class FlowApplication {
|
|
|
75
77
|
this.emit = (event) => {
|
|
76
78
|
if (event) {
|
|
77
79
|
try {
|
|
78
|
-
this.
|
|
80
|
+
this.publishFlowEventsAsFlowLogsOverNats(event);
|
|
79
81
|
if (this.outputStreamMap.has(event.getStreamId())) {
|
|
80
82
|
this.getOutputStream(event.getStreamId()).next(event);
|
|
81
83
|
}
|
|
@@ -92,7 +94,7 @@ class FlowApplication {
|
|
|
92
94
|
this.getOutputStream(completeEvent.getStreamId()).next(completeEvent);
|
|
93
95
|
}
|
|
94
96
|
if (partialEvent) {
|
|
95
|
-
this.
|
|
97
|
+
this.publishFlowEventsAsFlowLogsOverNats(partialEvent);
|
|
96
98
|
}
|
|
97
99
|
}
|
|
98
100
|
catch (err) {
|
|
@@ -137,7 +139,9 @@ class FlowApplication {
|
|
|
137
139
|
status: 'updated',
|
|
138
140
|
},
|
|
139
141
|
};
|
|
140
|
-
await (0, nats_1.publishNatsEvent)(this.
|
|
142
|
+
await (0, nats_1.publishNatsEvent)(this._natsConnection, natsEvent).catch((err) => {
|
|
143
|
+
this.logger.error(`Could not publish health update after flow deployment update: ${err.message}`);
|
|
144
|
+
});
|
|
141
145
|
}
|
|
142
146
|
catch (err) {
|
|
143
147
|
this.logger.error(err);
|
|
@@ -150,7 +154,9 @@ class FlowApplication {
|
|
|
150
154
|
status: 'updating failed',
|
|
151
155
|
},
|
|
152
156
|
};
|
|
153
|
-
await (0, nats_1.publishNatsEvent)(this.
|
|
157
|
+
await (0, nats_1.publishNatsEvent)(this._natsConnection, natsEvent).catch((err) => {
|
|
158
|
+
this.logger.error(`Could not publish health update after flow deployment update failed: ${err.message}`);
|
|
159
|
+
});
|
|
154
160
|
}
|
|
155
161
|
}
|
|
156
162
|
else if (cloudEvent.subject.endsWith('.message')) {
|
|
@@ -169,30 +175,61 @@ class FlowApplication {
|
|
|
169
175
|
/**
|
|
170
176
|
* Publish a flow event to NATS
|
|
171
177
|
* If the event size exceeds the limit it will be truncated
|
|
172
|
-
*
|
|
173
|
-
* TODO warum darf hier nicht false zurückgegeben werden? -> erzeugt loop
|
|
174
178
|
*/
|
|
175
|
-
this.
|
|
176
|
-
|
|
177
|
-
|
|
179
|
+
this.publishFlowEventsAsFlowLogsOverNats = async (event) => {
|
|
180
|
+
const errorMessageTextWithoutReason = `Cannot publish flow event to NATS as flow logs, because `;
|
|
181
|
+
if (!this.context?.deploymentId) {
|
|
182
|
+
this.logger.error(`${errorMessageTextWithoutReason} the deploymentId is undefined!`, { doNotPublishAsNatsEvent: true });
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
if (!this._natsConnection) {
|
|
186
|
+
this.logger.error(`${errorMessageTextWithoutReason} the NATS connection is undefined!`, { doNotPublishAsNatsEvent: true });
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
if (this._natsConnection.isClosed()) {
|
|
190
|
+
this.logger.error(`${errorMessageTextWithoutReason} the NATS connection was already closed!`, { doNotPublishAsNatsEvent: true });
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
if (!this._natsConnection?.info?.jetstream) {
|
|
194
|
+
this.logger.error(`${errorMessageTextWithoutReason} Jetstream is not enabled!`, { doNotPublishAsNatsEvent: true });
|
|
195
|
+
return;
|
|
178
196
|
}
|
|
197
|
+
// try to format and create the event
|
|
198
|
+
let natsEvent;
|
|
179
199
|
try {
|
|
180
200
|
const formatedEvent = event.format();
|
|
181
201
|
if ((0, object_sizeof_1.default)(formatedEvent) > MAX_EVENT_SIZE_BYTES) {
|
|
182
202
|
formatedEvent.data = (0, utils_1.truncate)(formatedEvent.data);
|
|
183
203
|
}
|
|
184
|
-
|
|
204
|
+
natsEvent = {
|
|
185
205
|
source: `hpc/flow-application`,
|
|
186
206
|
type: `${nats_1.natsFlowsPrefixFlowDeployment}.flowlogs`,
|
|
187
207
|
subject: `${this.context.deploymentId}`,
|
|
188
208
|
data: formatedEvent,
|
|
189
209
|
};
|
|
190
|
-
await (0, nats_1.publishNatsEvent)(this.logger, this._natsConnection, natsEvent);
|
|
191
|
-
return true;
|
|
192
210
|
}
|
|
193
|
-
catch (
|
|
194
|
-
this.logger.error(
|
|
195
|
-
return
|
|
211
|
+
catch (error) {
|
|
212
|
+
this.logger.error(error);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
// Try to publish the event
|
|
216
|
+
try {
|
|
217
|
+
await this.tryToSendLastErrorOnPublishingNatsEvents();
|
|
218
|
+
await (0, nats_1.publishNatsEvent)(this._natsConnection, natsEvent);
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
// Setting doNotPublishAsNatsEvent to true to avoid
|
|
222
|
+
// - infinite logging loop and to stop sending
|
|
223
|
+
// - events if NATS is not available
|
|
224
|
+
this.logger.error(error, { doNotPublishAsNatsEvent: true });
|
|
225
|
+
const message = `${error.message ?? String(error)}`;
|
|
226
|
+
if (!this.lastErrorBecauseFlowLogCouldNotBeSend) {
|
|
227
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend = { error: message, count: 1 };
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend.error = message;
|
|
231
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend.count++;
|
|
232
|
+
}
|
|
196
233
|
}
|
|
197
234
|
};
|
|
198
235
|
if (baseLoggerOrConfig && !baseLoggerOrConfig.log) {
|
|
@@ -212,7 +249,7 @@ class FlowApplication {
|
|
|
212
249
|
explicitInit = explicitInit || false;
|
|
213
250
|
this._api = mockApi || null;
|
|
214
251
|
}
|
|
215
|
-
this.logger = new FlowLogger_1.FlowLogger({ id: 'none', functionFqn: 'FlowApplication', ...flow?.context }, this.baseLogger || undefined, this.
|
|
252
|
+
this.logger = new FlowLogger_1.FlowLogger({ id: 'none', functionFqn: 'FlowApplication', ...flow?.context }, this.baseLogger || undefined, this.publishFlowEventsAsFlowLogsOverNats);
|
|
216
253
|
this.contextManager = new ContextManager_1.ContextManager(this.logger, this.flow?.properties);
|
|
217
254
|
process.once('uncaughtException', (err) => {
|
|
218
255
|
this.logger.error('Uncaught exception!');
|
|
@@ -243,6 +280,10 @@ class FlowApplication {
|
|
|
243
280
|
getProperties() {
|
|
244
281
|
return this.contextManager.getProperties();
|
|
245
282
|
}
|
|
283
|
+
async logErrorAndExit(err) {
|
|
284
|
+
this.logger.error(new Error(err));
|
|
285
|
+
await this.destroy(1);
|
|
286
|
+
}
|
|
246
287
|
async consumeNatsMessagesOfConsumer(consumer, consumeOptions) {
|
|
247
288
|
if (this.natsMessageIterator) {
|
|
248
289
|
await this.natsMessageIterator.close();
|
|
@@ -270,76 +311,102 @@ class FlowApplication {
|
|
|
270
311
|
}
|
|
271
312
|
}
|
|
272
313
|
}
|
|
314
|
+
async setupNatsConsumerForFlowDeployment() {
|
|
315
|
+
if (!this._natsConnection || this._natsConnection.isClosed() || !this.context?.deploymentId) {
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (!this._natsConnection?.info?.jetstream) {
|
|
319
|
+
if (process.env.NODE_ENV !== 'test') {
|
|
320
|
+
await this.logErrorAndExit('NATS JetStream is not enabled on the connected NATS server(s). Terminating FlowApplication...');
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
else {
|
|
324
|
+
// Do not set up consumers in tests because no mock exits
|
|
325
|
+
// TODO mock nats jetstream
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
const natsSubject = `${nats_1.natsFlowsPrefixFlowDeployment}.${this.context.deploymentId}.*`;
|
|
330
|
+
const consumerOptions = {
|
|
331
|
+
...nats_1.defaultConsumerConfig,
|
|
332
|
+
name: `flow-deployment-${this.context.deploymentId}`,
|
|
333
|
+
filter_subject: natsSubject,
|
|
334
|
+
inactive_threshold: 10 * 60 * 1000000000, // 10 mins
|
|
335
|
+
deliver_policy: jetstream_1.DeliverPolicy.New,
|
|
336
|
+
};
|
|
337
|
+
let consumer;
|
|
338
|
+
try {
|
|
339
|
+
consumer = await (0, nats_1.getOrCreateConsumer)(this.logger, this._natsConnection, nats_1.FLOWS_STREAM_NAME, consumerOptions.name, consumerOptions);
|
|
340
|
+
}
|
|
341
|
+
catch (e) {
|
|
342
|
+
this.logger.error(`Could not set up the nats consumer 'flow-deployment-${this.context.deploymentId}' for this flow deployment.`);
|
|
343
|
+
throw e;
|
|
344
|
+
}
|
|
345
|
+
// NO AWAIT, listen for messages of the consumer asynchronously
|
|
346
|
+
this.consumeNatsMessagesOfConsumer(consumer, { expires: 10 * 1000000000 /* 10 seconds */ }).catch((error) => {
|
|
347
|
+
this.logger.error(`Could not consume messages for the nats consumer 'flow-deployment-${this.context.deploymentId}' for deployment messages on the nats subject ${natsSubject}.`);
|
|
348
|
+
this.logErrorAndExit(error.message ?? String(error));
|
|
349
|
+
});
|
|
350
|
+
(0, nats_1.natsEventListener)(this._natsConnection, this.logger, async () => {
|
|
351
|
+
try {
|
|
352
|
+
const recreatedConsumer = await (0, nats_1.getOrCreateConsumer)(this.logger, this._natsConnection, nats_1.FLOWS_STREAM_NAME, consumerOptions.name, consumerOptions);
|
|
353
|
+
this.consumeNatsMessagesOfConsumer(recreatedConsumer, { expires: 10 * 1000000000 /* 10 seconds */ }).catch((error) => {
|
|
354
|
+
this.logger.error(`Could not consume messages for the nats consumer 'flow-deployment-${this.context.deploymentId}' for deployment messages on the nats subject ${natsSubject}.`);
|
|
355
|
+
this.logErrorAndExit(error.message ?? String(error));
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
catch (error) {
|
|
359
|
+
this.logger.error('Could not re-create consumers after NATS reconnect');
|
|
360
|
+
this.logErrorAndExit(error.message ?? String(error));
|
|
361
|
+
}
|
|
362
|
+
}).catch((error) => {
|
|
363
|
+
this.logger.error(`Could not set up NATS status listener`);
|
|
364
|
+
this.logErrorAndExit(error.message ?? String(error));
|
|
365
|
+
});
|
|
366
|
+
}
|
|
273
367
|
async init() {
|
|
274
368
|
if (this.initialized) {
|
|
275
369
|
return;
|
|
276
370
|
}
|
|
277
371
|
this.context = { ...this.flow.context };
|
|
278
372
|
this.contextManager.overwriteAllProperties(this.flow.properties ?? {});
|
|
279
|
-
|
|
280
|
-
|
|
373
|
+
if (!this.skipApi && !(this._api instanceof hpc_api_1.MockAPI)) {
|
|
374
|
+
try {
|
|
281
375
|
const { owner } = this.context;
|
|
282
376
|
// only create real API if it should not be skipped and is not already a mock
|
|
283
377
|
this._api = new hpc_api_1.API(this.apiClient, { activeOrg: owner?.id }, { queueOptions: { concurrency: 1, timeout: 70000, throwOnTimeout: true } });
|
|
284
378
|
}
|
|
379
|
+
catch (err) {
|
|
380
|
+
this.logger.error(err?.message || err);
|
|
381
|
+
}
|
|
285
382
|
}
|
|
286
|
-
catch (err) {
|
|
287
|
-
this.logger.error(err?.message || err);
|
|
288
|
-
}
|
|
289
|
-
const logErrorAndExit = async (err) => {
|
|
290
|
-
this.logger.error(new Error(err));
|
|
291
|
-
await this.destroy(1);
|
|
292
|
-
};
|
|
293
383
|
if (!this._natsConnection && this.natsConnectionConfig) {
|
|
294
384
|
try {
|
|
295
385
|
this._natsConnection = await (0, nats_1.createNatsConnection)(this.natsConnectionConfig);
|
|
296
386
|
}
|
|
297
387
|
catch (err) {
|
|
298
|
-
await logErrorAndExit(`Could not connect to the NATS-Servers: ${err}`);
|
|
388
|
+
await this.logErrorAndExit(`Could not connect to the NATS-Servers: ${err}`);
|
|
389
|
+
return;
|
|
299
390
|
}
|
|
300
391
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
inactive_threshold: 10 * 60 * 1000000000, // 10 mins
|
|
308
|
-
deliver_policy: jetstream_1.DeliverPolicy.New,
|
|
309
|
-
};
|
|
310
|
-
const consumer = await (0, nats_1.getOrCreateConsumer)(this.logger, this._natsConnection, nats_1.FLOWS_STREAM_NAME, consumerOptions.name, consumerOptions);
|
|
311
|
-
// Recreate consumers on reconnects: NO AWAIT, listen asynchronously
|
|
312
|
-
const handleNatsStatus = async () => {
|
|
313
|
-
try {
|
|
314
|
-
this.logger.debug('ConsumerService: Reconnected to Nats and re-creating non-durable consumers');
|
|
315
|
-
await (0, nats_1.getOrCreateConsumer)(this.logger, this._natsConnection, nats_1.FLOWS_STREAM_NAME, consumerOptions.name, consumerOptions);
|
|
316
|
-
this.consumeNatsMessagesOfConsumer(consumer, { expires: 10 * 1000000000 /* 10 seconds */ });
|
|
317
|
-
}
|
|
318
|
-
catch (e) {
|
|
319
|
-
this.logger.error('NATS Status-AsyncIterator is not available, cannot listen. Due to error:');
|
|
320
|
-
this.logger.error(e);
|
|
321
|
-
(0, nats_1.natsEventListener)(this._natsConnection, this.logger, handleNatsStatus);
|
|
322
|
-
}
|
|
323
|
-
};
|
|
324
|
-
(0, nats_1.natsEventListener)(this._natsConnection, this.logger, handleNatsStatus);
|
|
325
|
-
// NO AWAIT, listen for messages of the consumer asynchronously
|
|
326
|
-
this.consumeNatsMessagesOfConsumer(consumer, { expires: 10 * 1000000000 /* 10 seconds */ });
|
|
327
|
-
}
|
|
328
|
-
catch (e) {
|
|
329
|
-
await logErrorAndExit(`Could not set up consumer for deployment messages exchanges: ${e}`);
|
|
330
|
-
}
|
|
392
|
+
try {
|
|
393
|
+
await this.setupNatsConsumerForFlowDeployment();
|
|
394
|
+
}
|
|
395
|
+
catch (error) {
|
|
396
|
+
await this.logErrorAndExit(error.message ?? String(error));
|
|
397
|
+
return;
|
|
331
398
|
}
|
|
332
399
|
for (const module of this.modules) {
|
|
333
400
|
const moduleName = Reflect.getMetadata('module:name', module);
|
|
334
401
|
const moduleDeclarations = Reflect.getMetadata('module:declarations', module);
|
|
335
402
|
if (!moduleName || !moduleDeclarations || !Array.isArray(moduleDeclarations)) {
|
|
336
|
-
await logErrorAndExit(`FlowModule (${module.name}) metadata is missing or invalid`);
|
|
403
|
+
await this.logErrorAndExit(`FlowModule (${module.name}) metadata is missing or invalid`);
|
|
337
404
|
return;
|
|
338
405
|
}
|
|
339
406
|
for (const declaration of moduleDeclarations) {
|
|
340
407
|
const functionFqn = Reflect.getMetadata('element:functionFqn', declaration);
|
|
341
408
|
if (!functionFqn) {
|
|
342
|
-
await logErrorAndExit(`FlowFunction (${declaration.name}) metadata is missing or invalid`);
|
|
409
|
+
await this.logErrorAndExit(`FlowFunction (${declaration.name}) metadata is missing or invalid`);
|
|
343
410
|
return;
|
|
344
411
|
}
|
|
345
412
|
this.declarations[`${moduleName}.${functionFqn}`] = declaration;
|
|
@@ -355,7 +422,7 @@ class FlowApplication {
|
|
|
355
422
|
this.elements[id].setPropertiesWithPlaceholders((0, lodash_1.cloneDeep)(properties));
|
|
356
423
|
}
|
|
357
424
|
catch (err) {
|
|
358
|
-
await logErrorAndExit(`Could not create FlowElement for ${module}.${functionFqn}`);
|
|
425
|
+
await this.logErrorAndExit(`Could not create FlowElement for ${module}.${functionFqn}`);
|
|
359
426
|
return;
|
|
360
427
|
}
|
|
361
428
|
}
|
|
@@ -368,12 +435,12 @@ class FlowApplication {
|
|
|
368
435
|
const targetStreamId = `${target}.${targetStream}`;
|
|
369
436
|
const element = this.elements[target];
|
|
370
437
|
if (!element || !element.constructor) {
|
|
371
|
-
await logErrorAndExit(`${target} has not been initialized`);
|
|
438
|
+
await this.logErrorAndExit(`${target} has not been initialized`);
|
|
372
439
|
return;
|
|
373
440
|
}
|
|
374
441
|
const streamHandler = Reflect.getMetadata(`stream:${targetStream}`, element.constructor);
|
|
375
442
|
if (!streamHandler || !element[streamHandler]) {
|
|
376
|
-
await logErrorAndExit(`${target} does not implement a handler for ${targetStream}`);
|
|
443
|
+
await this.logErrorAndExit(`${target} does not implement a handler for ${targetStream}`);
|
|
377
444
|
return;
|
|
378
445
|
}
|
|
379
446
|
const streamOptions = Reflect.getMetadata(`stream:options:${targetStream}`, element.constructor) || {};
|
|
@@ -417,12 +484,37 @@ class FlowApplication {
|
|
|
417
484
|
this.initialized = true;
|
|
418
485
|
this.logger.log('Flow Deployment is running');
|
|
419
486
|
}
|
|
487
|
+
async tryToSendLastErrorOnPublishingNatsEvents() {
|
|
488
|
+
if (!this.lastErrorBecauseFlowLogCouldNotBeSend) {
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
try {
|
|
492
|
+
await (0, nats_1.publishNatsEvent)(this._natsConnection, {
|
|
493
|
+
source: `hpc/flow-application`,
|
|
494
|
+
type: `${nats_1.natsFlowsPrefixFlowDeployment}.flowlogs`,
|
|
495
|
+
subject: `${this.context.deploymentId}`,
|
|
496
|
+
data: new FlowEvent_1.FlowEvent({ id: 'none', name: 'functionFqn', ...this.context }, {
|
|
497
|
+
message: `${this.lastErrorBecauseFlowLogCouldNotBeSend.count} Nats events could not be sent. Last occurred error: ${this.lastErrorBecauseFlowLogCouldNotBeSend.error}`,
|
|
498
|
+
}, 'flow.log.error'),
|
|
499
|
+
});
|
|
500
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend = null;
|
|
501
|
+
}
|
|
502
|
+
catch (error) {
|
|
503
|
+
const message = `${error.message ?? String(error)}`;
|
|
504
|
+
this.logger.error(`Retry: Could not publish NATS event containing reason while sending nats events failed previously, due to ${error.message ?? String(error)}`, {
|
|
505
|
+
doNotPublishAsNatsEvent: true,
|
|
506
|
+
});
|
|
507
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend.error = message;
|
|
508
|
+
this.lastErrorBecauseFlowLogCouldNotBeSend.count++;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
420
511
|
/**
|
|
421
512
|
* Calls onDestroy lifecycle method on all flow elements,
|
|
422
513
|
* closes NATS connection after allowing logs to be processed and published
|
|
423
514
|
* then exits process
|
|
424
515
|
*/
|
|
425
516
|
async destroy(exitCode = 0) {
|
|
517
|
+
this.logger.debug(`Destroying Flow Application with exitCode ${exitCode}...`);
|
|
426
518
|
try {
|
|
427
519
|
try {
|
|
428
520
|
for (const element of Object.values(this.elements)) {
|
|
@@ -443,6 +535,13 @@ class FlowApplication {
|
|
|
443
535
|
this.logger.error(`Error completing output stream ${id}: ${err.message}`);
|
|
444
536
|
}
|
|
445
537
|
}
|
|
538
|
+
// Publish any unpublished NATS flow events as flow logs
|
|
539
|
+
await this.tryToSendLastErrorOnPublishingNatsEvents();
|
|
540
|
+
if (this.lastErrorBecauseFlowLogCouldNotBeSend) {
|
|
541
|
+
this.logger.error(`${this.lastErrorBecauseFlowLogCouldNotBeSend.count} Nats events cloud not be sent. Last occurred error: ${this.lastErrorBecauseFlowLogCouldNotBeSend.error}`, {
|
|
542
|
+
doNotPublishAsNatsEvent: true,
|
|
543
|
+
});
|
|
544
|
+
}
|
|
446
545
|
// Nats: Delete consumer for flow deployment, stop message listening and close connection
|
|
447
546
|
try {
|
|
448
547
|
await this.natsMessageIterator?.close();
|
|
@@ -453,16 +552,20 @@ class FlowApplication {
|
|
|
453
552
|
jsm.consumers
|
|
454
553
|
.delete(nats_1.FLOWS_STREAM_NAME, `flow-deployment-${this.context?.deploymentId}`)
|
|
455
554
|
.then(() => {
|
|
456
|
-
this.logger.debug(`Deleted consumer for flow deployment ${this.context?.deploymentId}
|
|
555
|
+
this.logger.debug(`Deleted consumer for flow deployment ${this.context?.deploymentId}`, {
|
|
556
|
+
doNotPublishAsNatsEvent: true,
|
|
557
|
+
});
|
|
457
558
|
})
|
|
458
559
|
.catch((err) => {
|
|
459
|
-
this.logger.error(`Could not delete consumer for flow deployment ${this.context?.deploymentId}: ${err.message}
|
|
560
|
+
this.logger.error(`Could not delete consumer for flow deployment ${this.context?.deploymentId}: ${err.message}`, {
|
|
561
|
+
doNotPublishAsNatsEvent: true,
|
|
562
|
+
});
|
|
460
563
|
});
|
|
461
564
|
});
|
|
462
565
|
}
|
|
463
566
|
}
|
|
464
567
|
catch (err) {
|
|
465
|
-
this.logger.error(err);
|
|
568
|
+
this.logger.error(err, { doNotPublishAsNatsEvent: true });
|
|
466
569
|
}
|
|
467
570
|
// remove process listeners
|
|
468
571
|
process.removeAllListeners('SIGTERM');
|
|
@@ -470,8 +573,7 @@ class FlowApplication {
|
|
|
470
573
|
process.removeAllListeners('unhandledRejection');
|
|
471
574
|
}
|
|
472
575
|
catch (err) {
|
|
473
|
-
|
|
474
|
-
console.error(err);
|
|
576
|
+
this.logger.error(err, { doNotPublishAsNatsEvent: true });
|
|
475
577
|
}
|
|
476
578
|
finally {
|
|
477
579
|
if (process.env.NODE_ENV !== 'test') {
|
package/src/lib/FlowElement.js
CHANGED
|
@@ -33,7 +33,7 @@ class FlowElement {
|
|
|
33
33
|
this.app = app;
|
|
34
34
|
this.api = this.app?.api;
|
|
35
35
|
this.metadata = { ...metadata, functionFqn: this.functionFqn };
|
|
36
|
-
this.logger = new FlowLogger_1.FlowLogger(this.metadata, logger || undefined, this.app?.
|
|
36
|
+
this.logger = new FlowLogger_1.FlowLogger(this.metadata, logger || undefined, this.app?.publishFlowEventsAsFlowLogsOverNats);
|
|
37
37
|
if (properties) {
|
|
38
38
|
this.setProperties(properties);
|
|
39
39
|
}
|
package/src/lib/FlowLogger.d.ts
CHANGED
|
@@ -8,13 +8,8 @@ export interface Logger {
|
|
|
8
8
|
verbose(message: any, metadata?: any): void;
|
|
9
9
|
}
|
|
10
10
|
export declare const defaultLogger: Logger;
|
|
11
|
-
export declare enum STACK_TRACE {
|
|
12
|
-
FULL = "full",
|
|
13
|
-
ONLY_LOG_CALL = "only-log-call"
|
|
14
|
-
}
|
|
15
11
|
export interface LoggerOptions {
|
|
16
|
-
|
|
17
|
-
stackTrace?: STACK_TRACE;
|
|
12
|
+
doNotPublishAsNatsEvent?: boolean;
|
|
18
13
|
}
|
|
19
14
|
export declare class FlowLogger implements Logger {
|
|
20
15
|
private readonly metadata;
|
|
@@ -36,7 +31,6 @@ export declare class FlowLogger implements Logger {
|
|
|
36
31
|
*
|
|
37
32
|
* @param {any} message - The message to be logged. Can be a string, an object with a `message` property, or any other type.
|
|
38
33
|
* @param {string} level - The log level (e.g., 'error', 'debug', 'warn', 'verbose').
|
|
39
|
-
* @param {LoggerOptions} options - Additional options for logging, such as whether to include a stack trace.
|
|
40
34
|
* @returns {FlowLog} - An object containing the parsed log message and optional stack trace.
|
|
41
35
|
*/
|
|
42
36
|
private parseMessageToFlowLog;
|
package/src/lib/FlowLogger.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.FlowLogger = exports.
|
|
3
|
+
exports.FlowLogger = exports.defaultLogger = void 0;
|
|
4
4
|
const FlowEvent_1 = require("./FlowEvent");
|
|
5
5
|
/* eslint-disable no-console */
|
|
6
6
|
exports.defaultLogger = {
|
|
@@ -10,14 +10,8 @@ exports.defaultLogger = {
|
|
|
10
10
|
warn: (msg, metadata) => console.warn(msg),
|
|
11
11
|
verbose: (msg, metadata) => console.log(msg, metadata),
|
|
12
12
|
};
|
|
13
|
-
/* eslint-enable no-console */
|
|
14
|
-
var STACK_TRACE;
|
|
15
|
-
(function (STACK_TRACE) {
|
|
16
|
-
STACK_TRACE["FULL"] = "full";
|
|
17
|
-
STACK_TRACE["ONLY_LOG_CALL"] = "only-log-call";
|
|
18
|
-
})(STACK_TRACE || (exports.STACK_TRACE = STACK_TRACE = {}));
|
|
19
13
|
class FlowLogger {
|
|
20
|
-
static getStackTrace(
|
|
14
|
+
static getStackTrace() {
|
|
21
15
|
// get stacktrace without extra dependencies
|
|
22
16
|
let stack;
|
|
23
17
|
try {
|
|
@@ -27,17 +21,10 @@ class FlowLogger {
|
|
|
27
21
|
stack = error.stack || '';
|
|
28
22
|
}
|
|
29
23
|
// cleanup stacktrace and remove calls within this file
|
|
30
|
-
|
|
24
|
+
return stack
|
|
31
25
|
.split('\n')
|
|
32
26
|
.map((line) => line.trim())
|
|
33
|
-
.filter((value) => value.
|
|
34
|
-
if (stacktrace === STACK_TRACE.ONLY_LOG_CALL && stack.length > 0) {
|
|
35
|
-
stack = stack[0];
|
|
36
|
-
}
|
|
37
|
-
else {
|
|
38
|
-
stack = stack.splice(1).join('\n');
|
|
39
|
-
}
|
|
40
|
-
return stack;
|
|
27
|
+
.filter((value) => value.startsWith('at FlowLogger.FlowApplication') || value.startsWith('at FlowApplication'));
|
|
41
28
|
}
|
|
42
29
|
constructor(metadata, logger = exports.defaultLogger, publishEvent) {
|
|
43
30
|
this.metadata = metadata;
|
|
@@ -58,10 +45,9 @@ class FlowLogger {
|
|
|
58
45
|
*
|
|
59
46
|
* @param {any} message - The message to be logged. Can be a string, an object with a `message` property, or any other type.
|
|
60
47
|
* @param {string} level - The log level (e.g., 'error', 'debug', 'warn', 'verbose').
|
|
61
|
-
* @param {LoggerOptions} options - Additional options for logging, such as whether to include a stack trace.
|
|
62
48
|
* @returns {FlowLog} - An object containing the parsed log message and optional stack trace.
|
|
63
49
|
*/
|
|
64
|
-
parseMessageToFlowLog(message, level
|
|
50
|
+
parseMessageToFlowLog(message, level) {
|
|
65
51
|
let flowLogMessage;
|
|
66
52
|
if (!message) {
|
|
67
53
|
flowLogMessage = 'No message provided!';
|
|
@@ -81,18 +67,18 @@ class FlowLogger {
|
|
|
81
67
|
}
|
|
82
68
|
}
|
|
83
69
|
const flowLog = { message: flowLogMessage };
|
|
84
|
-
if (['error', 'debug', 'warn', 'verbose'].includes(level)
|
|
85
|
-
flowLog.stackTrace = FlowLogger.getStackTrace(
|
|
70
|
+
if (['error', 'debug', 'warn', 'verbose'].includes(level)) {
|
|
71
|
+
flowLog.stackTrace = FlowLogger.getStackTrace();
|
|
86
72
|
}
|
|
87
73
|
return flowLog;
|
|
88
74
|
}
|
|
89
75
|
publish(message, level, options) {
|
|
90
|
-
const
|
|
91
|
-
if (this.publishEvent) {
|
|
92
|
-
const event = new FlowEvent_1.FlowEvent(this.metadata,
|
|
76
|
+
const flowLog = this.parseMessageToFlowLog(message, level);
|
|
77
|
+
if (this.publishEvent && options?.doNotPublishAsNatsEvent !== true) {
|
|
78
|
+
const event = new FlowEvent_1.FlowEvent(this.metadata, flowLog, `flow.log.${level}`);
|
|
93
79
|
this.publishEvent(event);
|
|
94
80
|
}
|
|
95
|
-
const messageWithStackTrace =
|
|
81
|
+
const messageWithStackTrace = flowLog.stackTrace ? `${flowLog.message}\n${flowLog.stackTrace}` : flowLog.message;
|
|
96
82
|
switch (level) {
|
|
97
83
|
case 'debug':
|
|
98
84
|
return this.logger.debug(messageWithStackTrace, { ...this.metadata, ...options });
|
package/src/lib/nats.d.ts
CHANGED
|
@@ -7,6 +7,6 @@ export declare const natsFlowsPrefixFlowDeployment = "fs.flowdeployment";
|
|
|
7
7
|
export declare const defaultConsumerConfig: ConsumerConfig;
|
|
8
8
|
export declare const FLOWS_STREAM_NAME = "flows";
|
|
9
9
|
export declare function getOrCreateConsumer(logger: Logger, natsConnection: NatsConnection, streamName: string, consumerName: string, options: Partial<ConsumerConfig>): Promise<Consumer>;
|
|
10
|
-
export declare function natsEventListener(nc: NatsConnection, logger: Logger, reconnectHandler: () => void): Promise<void>;
|
|
11
|
-
export declare function publishNatsEvent<T>(
|
|
10
|
+
export declare function natsEventListener(nc: NatsConnection, logger: Logger, reconnectHandler: () => Promise<void>): Promise<void>;
|
|
11
|
+
export declare function publishNatsEvent<T>(nc: NatsConnection, event: NatsEvent<T>, subject?: string): Promise<PubAck>;
|
|
12
12
|
export declare function createNatsConnection(config: ConnectionOptions): Promise<NatsConnection>;
|
package/src/lib/nats.js
CHANGED
|
@@ -62,14 +62,13 @@ async function natsEventListener(nc, logger, reconnectHandler) {
|
|
|
62
62
|
for await (const status of statusAsyncIterator) {
|
|
63
63
|
// Handle reconnect: event is triggered when the NATS client reconnected to the server
|
|
64
64
|
if (status.type === 'reconnect') {
|
|
65
|
-
reconnectHandler();
|
|
65
|
+
await reconnectHandler();
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
|
-
async function publishNatsEvent(
|
|
69
|
+
async function publishNatsEvent(nc, event, subject) {
|
|
70
70
|
if (!nc || nc.isClosed()) {
|
|
71
|
-
|
|
72
|
-
return null;
|
|
71
|
+
throw new Error('NATS connection is not available, cannot publish event');
|
|
73
72
|
}
|
|
74
73
|
const cloudEvent = new cloudevents_1.CloudEvent({ datacontenttype: 'application/json', ...event });
|
|
75
74
|
cloudEvent.validate();
|
|
@@ -80,8 +79,7 @@ async function publishNatsEvent(logger, nc, event, subject) {
|
|
|
80
79
|
});
|
|
81
80
|
}
|
|
82
81
|
else {
|
|
83
|
-
|
|
84
|
-
return null;
|
|
82
|
+
throw new Error('NATS jetstream is not available, cannot publish event');
|
|
85
83
|
}
|
|
86
84
|
}
|
|
87
85
|
async function createNatsConnection(config) {
|
package/src/lib/utils.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { FlowLogger } from './FlowLogger';
|
|
2
2
|
export declare function fillTemplate(value: any, ...templateVariables: any): any;
|
|
3
|
-
export declare function getCircularReplacer(): (key: any, value: any) => any;
|
|
4
3
|
export declare function toArray(value?: string | string[]): string[];
|
|
5
4
|
export declare function delay(ms: number): Promise<void>;
|
|
6
5
|
/**
|
package/src/lib/utils.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.fillTemplate = fillTemplate;
|
|
4
|
-
exports.getCircularReplacer = getCircularReplacer;
|
|
5
4
|
exports.toArray = toArray;
|
|
6
5
|
exports.delay = delay;
|
|
7
6
|
exports.delayWithAbort = delayWithAbort;
|
|
@@ -47,18 +46,6 @@ function fillTemplate(value, ...templateVariables) {
|
|
|
47
46
|
return value;
|
|
48
47
|
}
|
|
49
48
|
}
|
|
50
|
-
function getCircularReplacer() {
|
|
51
|
-
const seen = new WeakSet();
|
|
52
|
-
return (key, value) => {
|
|
53
|
-
if (typeof value === 'object' && value !== null) {
|
|
54
|
-
if (seen.has(value)) {
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
seen.add(value);
|
|
58
|
-
}
|
|
59
|
-
return value;
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
49
|
function toArray(value = []) {
|
|
63
50
|
return Array.isArray(value) ? value : value.split(',').map((v) => v.trim());
|
|
64
51
|
}
|