@hahnpro/flow-sdk 2025.2.0-beta.1 → 2025.2.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -4
- package/src/{index.ts → index.d.ts} +0 -3
- package/src/index.js +18 -0
- package/src/lib/ContextManager.d.ts +40 -0
- package/src/lib/ContextManager.js +105 -0
- package/src/lib/FlowApplication.d.ts +85 -0
- package/src/lib/FlowApplication.js +528 -0
- package/src/lib/FlowElement.d.ts +67 -0
- package/src/lib/FlowElement.js +178 -0
- package/src/lib/FlowEvent.d.ts +25 -0
- package/src/lib/FlowEvent.js +72 -0
- package/src/lib/FlowLogger.d.ts +44 -0
- package/src/lib/FlowLogger.js +110 -0
- package/src/lib/FlowModule.d.ts +7 -0
- package/src/lib/FlowModule.js +14 -0
- package/src/lib/RpcClient.d.ts +13 -0
- package/src/lib/RpcClient.js +88 -0
- package/src/lib/TestModule.d.ts +2 -0
- package/src/lib/TestModule.js +27 -0
- package/src/lib/amqp.d.ts +14 -0
- package/src/lib/amqp.js +12 -0
- package/src/lib/extra-validators.d.ts +1 -0
- package/src/lib/extra-validators.js +53 -0
- package/src/lib/flow.interface.d.ts +48 -0
- package/src/lib/flow.interface.js +9 -0
- package/src/lib/{index.ts → index.d.ts} +0 -3
- package/src/lib/index.js +18 -0
- package/src/lib/nats.d.ts +12 -0
- package/src/lib/nats.js +115 -0
- package/src/lib/unit-decorators.d.ts +39 -0
- package/src/lib/unit-decorators.js +156 -0
- package/src/lib/unit-utils.d.ts +8 -0
- package/src/lib/unit-utils.js +144 -0
- package/src/lib/units.d.ts +31 -0
- package/src/lib/units.js +572 -0
- package/src/lib/utils.d.ts +51 -0
- package/src/lib/utils.js +178 -0
- package/jest.config.ts +0 -10
- package/project.json +0 -41
- package/src/lib/ContextManager.ts +0 -111
- package/src/lib/FlowApplication.ts +0 -659
- package/src/lib/FlowElement.ts +0 -220
- package/src/lib/FlowEvent.ts +0 -73
- package/src/lib/FlowLogger.ts +0 -131
- package/src/lib/FlowModule.ts +0 -18
- package/src/lib/RpcClient.ts +0 -99
- package/src/lib/TestModule.ts +0 -14
- package/src/lib/__pycache__/rpc_server.cpython-310.pyc +0 -0
- package/src/lib/amqp.ts +0 -32
- package/src/lib/extra-validators.ts +0 -62
- package/src/lib/flow.interface.ts +0 -56
- package/src/lib/nats.ts +0 -140
- package/src/lib/unit-decorators.ts +0 -156
- package/src/lib/unit-utils.ts +0 -163
- package/src/lib/units.ts +0 -587
- package/src/lib/utils.ts +0 -176
- package/test/context-manager-purpose.spec.ts +0 -248
- package/test/context-manager.spec.ts +0 -55
- package/test/context.spec.ts +0 -180
- package/test/event.spec.ts +0 -155
- package/test/extra-validators.spec.ts +0 -84
- package/test/flow-logger.spec.ts +0 -104
- package/test/flow.spec.ts +0 -508
- package/test/input-stream.decorator.spec.ts +0 -379
- package/test/long-rpc.test.py +0 -14
- package/test/long-running-rpc.spec.ts +0 -60
- package/test/message.spec.ts +0 -57
- package/test/mocks/logger.mock.ts +0 -7
- package/test/mocks/nats-connection.mock.ts +0 -135
- package/test/mocks/nats-prepare.reals-nats.ts +0 -15
- package/test/rpc.spec.ts +0 -198
- package/test/rpc.test.py +0 -45
- package/test/rx.spec.ts +0 -92
- package/test/unit-decorator.spec.ts +0 -57
- package/test/utils.spec.ts +0 -210
- package/test/validation.spec.ts +0 -174
- package/tsconfig.json +0 -13
- package/tsconfig.lib.json +0 -22
- package/tsconfig.spec.json +0 -8
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FlowDashboard = exports.FlowTrigger = exports.FlowTask = exports.FlowResource = exports.FlowElement = void 0;
|
|
4
|
+
exports.InputStream = InputStream;
|
|
5
|
+
exports.FlowFunction = FlowFunction;
|
|
6
|
+
const class_transformer_1 = require("class-transformer");
|
|
7
|
+
const class_validator_1 = require("class-validator");
|
|
8
|
+
const python_shell_1 = require("python-shell");
|
|
9
|
+
const FlowEvent_1 = require("./FlowEvent");
|
|
10
|
+
const FlowLogger_1 = require("./FlowLogger");
|
|
11
|
+
const utils_1 = require("./utils");
|
|
12
|
+
class FlowElement {
|
|
13
|
+
constructor({ app, logger, ...metadata }, properties, propertiesClassType, whitelist = false) {
|
|
14
|
+
this.propertiesClassType = propertiesClassType;
|
|
15
|
+
this.whitelist = whitelist;
|
|
16
|
+
this.stopPropagateStream = new Map();
|
|
17
|
+
this.onContextChanged = (context) => {
|
|
18
|
+
this.metadata = { ...this.metadata, ...context };
|
|
19
|
+
};
|
|
20
|
+
this.onPropertiesChanged = (properties) => {
|
|
21
|
+
this.setProperties(properties);
|
|
22
|
+
};
|
|
23
|
+
this.getMetadata = () => this.metadata;
|
|
24
|
+
this.setProperties = (properties) => {
|
|
25
|
+
if (this.propertiesClassType) {
|
|
26
|
+
this.properties = this.validateProperties(this.propertiesClassType, properties, this.whitelist);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
this.properties = properties;
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
this.handleApiError = (error) => (0, utils_1.handleApiError)(error, this.logger);
|
|
33
|
+
this.interpolate = (value, ...templateVariables) => (0, utils_1.fillTemplate)(value, ...templateVariables, { flow: this.app?.getContextManager?.().getProperties?.().flow ?? {} });
|
|
34
|
+
this.app = app;
|
|
35
|
+
this.api = this.app?.api;
|
|
36
|
+
this.metadata = { ...metadata, functionFqn: this.functionFqn };
|
|
37
|
+
this.logger = new FlowLogger_1.FlowLogger(this.metadata, logger || undefined, this.app?.publishNatsEventFlowlogs);
|
|
38
|
+
this.rpcRoutingKey = (this.metadata.flowId || '') + (this.metadata.deploymentId || '') + this.metadata.id;
|
|
39
|
+
if (properties) {
|
|
40
|
+
this.setProperties(properties);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Sets the placeholder properties for this flow element
|
|
45
|
+
* @param propertiesWithPlaceholders
|
|
46
|
+
*/
|
|
47
|
+
setPropertiesWithPlaceholders(propertiesWithPlaceholders) {
|
|
48
|
+
this.propertiesWithPlaceholders = propertiesWithPlaceholders;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Returns the placeholder properties for this flow element
|
|
52
|
+
*/
|
|
53
|
+
getPropertiesWithPlaceholders() {
|
|
54
|
+
return this.propertiesWithPlaceholders;
|
|
55
|
+
}
|
|
56
|
+
get flowProperties() {
|
|
57
|
+
return this.app?.getProperties?.() || {};
|
|
58
|
+
}
|
|
59
|
+
get natsConnection() {
|
|
60
|
+
return this.app?.natsConnection;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Replace all placeholder properties with their explicit updated value and set them as the properties of the element
|
|
64
|
+
*/
|
|
65
|
+
replacePlaceholderAndSetProperties() {
|
|
66
|
+
const placeholderProperties = this.propertiesWithPlaceholders;
|
|
67
|
+
if (this.propertiesWithPlaceholders) {
|
|
68
|
+
this.setProperties(this.app.getContextManager?.()?.replaceAllPlaceholderProperties(placeholderProperties));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* @deprecated since version 4.8.0, will be removed in 5.0.0, use emitEvent(...) instead
|
|
73
|
+
*/
|
|
74
|
+
emitOutput(data = {}, outputId = 'default', time = new Date()) {
|
|
75
|
+
return this.emitEvent(data, null, outputId, time);
|
|
76
|
+
}
|
|
77
|
+
emitEvent(data, inputEvent, outputId = 'default', time = new Date()) {
|
|
78
|
+
const partialEvent = new FlowEvent_1.FlowEvent(this.metadata, data, outputId, time);
|
|
79
|
+
const completeEvent = new FlowEvent_1.FlowEvent(this.metadata, { ...(inputEvent?.getData() || {}), ...data }, outputId, time);
|
|
80
|
+
const streamID = inputEvent?.getMetadata()?.inputStreamId || '';
|
|
81
|
+
if ((this.stopPropagateStream.has(streamID) && this.stopPropagateStream.get(streamID)) || !this.stopPropagateStream.has(streamID)) {
|
|
82
|
+
this.app?.emit(partialEvent);
|
|
83
|
+
return partialEvent;
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
this.app?.emitPartial(completeEvent, partialEvent);
|
|
87
|
+
return completeEvent;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
validateProperties(classType, properties = {}, whitelist = false) {
|
|
91
|
+
const props = (0, class_transformer_1.plainToInstance)(classType, properties);
|
|
92
|
+
const errors = (0, class_validator_1.validateSync)(props, { forbidUnknownValues: false, whitelist });
|
|
93
|
+
if (Array.isArray(errors) && errors.length > 0) {
|
|
94
|
+
for (const e of errors) {
|
|
95
|
+
this.logValidationErrors(e);
|
|
96
|
+
}
|
|
97
|
+
throw new Error('Properties Validation failed');
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
return props;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
logValidationErrors(error, parent) {
|
|
104
|
+
const { children, constraints, property, value } = error;
|
|
105
|
+
const name = parent ? parent + '.' + property : property;
|
|
106
|
+
if (constraints) {
|
|
107
|
+
this.logger.error(`Validation for property "${name}" failed:\n${JSON.stringify(constraints || {})}\nvalue: ${value}`);
|
|
108
|
+
}
|
|
109
|
+
else if (Array.isArray(children)) {
|
|
110
|
+
for (const child of children) {
|
|
111
|
+
this.logValidationErrors(child, name);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
validateEventData(classType, event, whitelist = false) {
|
|
116
|
+
return this.validateProperties(classType, event.getData(), whitelist);
|
|
117
|
+
}
|
|
118
|
+
async callRpcFunction(functionName, ...args) {
|
|
119
|
+
return this.app?.rpcClient.callFunction(this.rpcRoutingKey, functionName, ...args);
|
|
120
|
+
}
|
|
121
|
+
runPyRpcScript(scriptPath, ...args) {
|
|
122
|
+
const options = {
|
|
123
|
+
mode: 'text',
|
|
124
|
+
pythonOptions: ['-u'],
|
|
125
|
+
args: [__dirname, this.rpcRoutingKey, ...args.map((v) => v.toString())],
|
|
126
|
+
};
|
|
127
|
+
const pyshell = new python_shell_1.PythonShell(scriptPath, options);
|
|
128
|
+
pyshell.on('message', (message) => {
|
|
129
|
+
this.logger.debug(message);
|
|
130
|
+
});
|
|
131
|
+
pyshell.end((err) => {
|
|
132
|
+
if (err) {
|
|
133
|
+
this.logger.error(err);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
return pyshell;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
exports.FlowElement = FlowElement;
|
|
140
|
+
function InputStream(id = 'default', options) {
|
|
141
|
+
return (target, propertyKey, propertyDescriptor) => {
|
|
142
|
+
Reflect.defineMetadata(`stream:${id}`, propertyKey, target.constructor);
|
|
143
|
+
if (options) {
|
|
144
|
+
Reflect.defineMetadata(`stream:options:${id}`, options, target.constructor);
|
|
145
|
+
}
|
|
146
|
+
const method = propertyDescriptor.value;
|
|
147
|
+
propertyDescriptor.value = function (event) {
|
|
148
|
+
if (!this.stopPropagateStream.has(id)) {
|
|
149
|
+
this.stopPropagateStream.set(id, options?.stopPropagation ?? false);
|
|
150
|
+
}
|
|
151
|
+
// add input stream to data to later determine if data should be propagated
|
|
152
|
+
return method.call(this, new FlowEvent_1.FlowEvent({ id: event.getMetadata().elementId, ...event.getMetadata(), inputStreamId: id }, event.getData(), event.getType(), new Date(event.getTime())));
|
|
153
|
+
};
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
function FlowFunction(fqn) {
|
|
157
|
+
const fqnRegExp = /^([a-zA-Z][a-zA-Z\d]*[.-])*[a-zA-Z][a-zA-Z\d]*$/;
|
|
158
|
+
if (!fqnRegExp.test(fqn)) {
|
|
159
|
+
throw new Error(`Flow Function FQN (${fqn}) is not valid`);
|
|
160
|
+
}
|
|
161
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
162
|
+
return (target) => {
|
|
163
|
+
Reflect.defineMetadata('element:functionFqn', fqn, target);
|
|
164
|
+
target.prototype.functionFqn = fqn;
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
class FlowResource extends FlowElement {
|
|
168
|
+
}
|
|
169
|
+
exports.FlowResource = FlowResource;
|
|
170
|
+
class FlowTask extends FlowElement {
|
|
171
|
+
}
|
|
172
|
+
exports.FlowTask = FlowTask;
|
|
173
|
+
class FlowTrigger extends FlowElement {
|
|
174
|
+
}
|
|
175
|
+
exports.FlowTrigger = FlowTrigger;
|
|
176
|
+
class FlowDashboard extends FlowResource {
|
|
177
|
+
}
|
|
178
|
+
exports.FlowDashboard = FlowDashboard;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { FlowElementContext } from './flow.interface';
|
|
2
|
+
export declare class FlowEvent {
|
|
3
|
+
private event;
|
|
4
|
+
private metadata;
|
|
5
|
+
constructor(metadata: FlowElementContext, data: any, outputId?: string, time?: Date, dataType?: string);
|
|
6
|
+
format: () => any;
|
|
7
|
+
getData: () => any;
|
|
8
|
+
getDataContentType: () => string;
|
|
9
|
+
getDataschema: () => string;
|
|
10
|
+
getId: () => string;
|
|
11
|
+
getMetadata: () => {
|
|
12
|
+
deploymentId: string;
|
|
13
|
+
elementId: string;
|
|
14
|
+
flowId: string;
|
|
15
|
+
functionFqn: string;
|
|
16
|
+
inputStreamId: string;
|
|
17
|
+
};
|
|
18
|
+
getSource: () => string;
|
|
19
|
+
getStreamId: () => string;
|
|
20
|
+
getSubject: () => string;
|
|
21
|
+
getTime: () => Date;
|
|
22
|
+
getType: () => string;
|
|
23
|
+
toJSON: () => any;
|
|
24
|
+
toString: () => string;
|
|
25
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FlowEvent = void 0;
|
|
4
|
+
const cloudevents_1 = require("cloudevents");
|
|
5
|
+
const lodash_1 = require("lodash");
|
|
6
|
+
class FlowEvent {
|
|
7
|
+
constructor(metadata, data, outputId = 'default', time = new Date(), dataType) {
|
|
8
|
+
this.format = () => {
|
|
9
|
+
const event = this.event.toJSON();
|
|
10
|
+
if (event.datacontenttype === 'application/json') {
|
|
11
|
+
try {
|
|
12
|
+
event.data = JSON.parse(event.data);
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
/* ignore error */
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return event;
|
|
19
|
+
};
|
|
20
|
+
this.getData = () => (0, lodash_1.cloneDeep)(this.event.data || {});
|
|
21
|
+
this.getDataContentType = () => this.event.datacontenttype;
|
|
22
|
+
this.getDataschema = () => this.event.dataschema;
|
|
23
|
+
this.getId = () => this.event.id;
|
|
24
|
+
this.getMetadata = () => this.metadata;
|
|
25
|
+
this.getSource = () => this.event.source;
|
|
26
|
+
this.getStreamId = () => `${this.metadata.elementId}.${this.event.type}`;
|
|
27
|
+
this.getSubject = () => this.event.subject;
|
|
28
|
+
this.getTime = () => new Date(this.event.time);
|
|
29
|
+
this.getType = () => this.event.type;
|
|
30
|
+
this.toJSON = () => this.event.toJSON();
|
|
31
|
+
this.toString = () => this.event.toString();
|
|
32
|
+
const { id: elementId, deploymentId, flowId, functionFqn, inputStreamId } = metadata;
|
|
33
|
+
if (data instanceof Error) {
|
|
34
|
+
const error = { message: data.message, stack: data.stack };
|
|
35
|
+
data = error;
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
data = (0, lodash_1.cloneDeep)(data);
|
|
39
|
+
}
|
|
40
|
+
if (data == null) {
|
|
41
|
+
data = {};
|
|
42
|
+
}
|
|
43
|
+
if (dataType == null) {
|
|
44
|
+
if (typeof data === 'string') {
|
|
45
|
+
try {
|
|
46
|
+
JSON.parse(data);
|
|
47
|
+
dataType = 'application/json';
|
|
48
|
+
}
|
|
49
|
+
catch (err) {
|
|
50
|
+
dataType = 'text/plain';
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
else if (typeof data === 'object') {
|
|
54
|
+
dataType = 'application/json';
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
data = String(data);
|
|
58
|
+
dataType = 'text/plain';
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
this.metadata = { deploymentId, elementId, flowId, functionFqn, inputStreamId };
|
|
62
|
+
this.event = new cloudevents_1.CloudEvent({
|
|
63
|
+
source: `flows/${flowId}/deployments/${deploymentId}/elements/${elementId}`,
|
|
64
|
+
type: outputId,
|
|
65
|
+
subject: functionFqn,
|
|
66
|
+
datacontenttype: dataType,
|
|
67
|
+
data,
|
|
68
|
+
time: time.toISOString(),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
exports.FlowEvent = FlowEvent;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { FlowElementContext } from './flow.interface';
|
|
2
|
+
import { FlowEvent } from './FlowEvent';
|
|
3
|
+
export interface Logger {
|
|
4
|
+
debug(message: any, metadata?: any): void;
|
|
5
|
+
error(message: any, metadata?: any): void;
|
|
6
|
+
log(message: any, metadata?: any): void;
|
|
7
|
+
warn(message: any, metadata?: any): void;
|
|
8
|
+
verbose(message: any, metadata?: any): void;
|
|
9
|
+
}
|
|
10
|
+
export declare const defaultLogger: Logger;
|
|
11
|
+
export declare enum STACK_TRACE {
|
|
12
|
+
FULL = "full",
|
|
13
|
+
ONLY_LOG_CALL = "only-log-call"
|
|
14
|
+
}
|
|
15
|
+
export interface LoggerOptions {
|
|
16
|
+
truncate: boolean;
|
|
17
|
+
stackTrace?: STACK_TRACE;
|
|
18
|
+
}
|
|
19
|
+
export declare class FlowLogger implements Logger {
|
|
20
|
+
private readonly metadata;
|
|
21
|
+
private readonly logger;
|
|
22
|
+
private readonly publishEvent?;
|
|
23
|
+
private static getStackTrace;
|
|
24
|
+
constructor(metadata: FlowElementContext, logger?: Logger, publishEvent?: (event: FlowEvent) => void);
|
|
25
|
+
debug: (message: any, options?: LoggerOptions) => void;
|
|
26
|
+
error: (message: any, options?: LoggerOptions) => void;
|
|
27
|
+
log: (message: any, options?: LoggerOptions) => void;
|
|
28
|
+
warn: (message: any, options?: LoggerOptions) => void;
|
|
29
|
+
verbose: (message: any, options?: LoggerOptions) => void;
|
|
30
|
+
/**
|
|
31
|
+
* Parses a message into a FlowLog object, including optional stack trace information.
|
|
32
|
+
*
|
|
33
|
+
* @details Requirements for the output format of messages:
|
|
34
|
+
* - Necessary for consistent logging and event publishing, because the OpenSearch index expects a specific structure: flat_object.
|
|
35
|
+
* - The current UI expects a `message` property to be present, so we ensure it is always set.
|
|
36
|
+
*
|
|
37
|
+
* @param {any} message - The message to be logged. Can be a string, an object with a `message` property, or any other type.
|
|
38
|
+
* @param {string} level - The log level (e.g., 'error', 'debug', 'warn', 'verbose').
|
|
39
|
+
* @param {LoggerOptions} options - Additional options for logging, such as whether to include a stack trace.
|
|
40
|
+
* @returns {FlowLog} - An object containing the parsed log message and optional stack trace.
|
|
41
|
+
*/
|
|
42
|
+
private parseMessageToFlowLog;
|
|
43
|
+
private publish;
|
|
44
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FlowLogger = exports.STACK_TRACE = exports.defaultLogger = void 0;
|
|
4
|
+
const FlowEvent_1 = require("./FlowEvent");
|
|
5
|
+
/* eslint-disable no-console */
|
|
6
|
+
exports.defaultLogger = {
|
|
7
|
+
debug: (msg, metadata) => console.debug(msg),
|
|
8
|
+
error: (msg, metadata) => console.error(msg),
|
|
9
|
+
log: (msg, metadata) => console.log(msg),
|
|
10
|
+
warn: (msg, metadata) => console.warn(msg),
|
|
11
|
+
verbose: (msg, metadata) => console.log(msg, metadata),
|
|
12
|
+
};
|
|
13
|
+
/* eslint-enable no-console */
|
|
14
|
+
var STACK_TRACE;
|
|
15
|
+
(function (STACK_TRACE) {
|
|
16
|
+
STACK_TRACE["FULL"] = "full";
|
|
17
|
+
STACK_TRACE["ONLY_LOG_CALL"] = "only-log-call";
|
|
18
|
+
})(STACK_TRACE || (exports.STACK_TRACE = STACK_TRACE = {}));
|
|
19
|
+
class FlowLogger {
|
|
20
|
+
static getStackTrace(stacktrace = STACK_TRACE.FULL) {
|
|
21
|
+
// get stacktrace without extra dependencies
|
|
22
|
+
let stack;
|
|
23
|
+
try {
|
|
24
|
+
throw new Error('');
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
stack = error.stack || '';
|
|
28
|
+
}
|
|
29
|
+
// cleanup stacktrace and remove calls within this file
|
|
30
|
+
stack = stack
|
|
31
|
+
.split('\n')
|
|
32
|
+
.map((line) => line.trim())
|
|
33
|
+
.filter((value) => value.includes('at ') && !value.includes('Logger'));
|
|
34
|
+
if (stacktrace === STACK_TRACE.ONLY_LOG_CALL && stack.length > 0) {
|
|
35
|
+
stack = stack[0];
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
stack = stack.splice(1).join('\n');
|
|
39
|
+
}
|
|
40
|
+
return stack;
|
|
41
|
+
}
|
|
42
|
+
constructor(metadata, logger = exports.defaultLogger, publishEvent) {
|
|
43
|
+
this.metadata = metadata;
|
|
44
|
+
this.logger = logger;
|
|
45
|
+
this.publishEvent = publishEvent;
|
|
46
|
+
this.debug = (message, options) => this.publish(message, 'debug', options);
|
|
47
|
+
this.error = (message, options) => this.publish(message, 'error', options);
|
|
48
|
+
this.log = (message, options) => this.publish(message, 'info', options);
|
|
49
|
+
this.warn = (message, options) => this.publish(message, 'warn', options);
|
|
50
|
+
this.verbose = (message, options) => this.publish(message, 'verbose', options);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parses a message into a FlowLog object, including optional stack trace information.
|
|
54
|
+
*
|
|
55
|
+
* @details Requirements for the output format of messages:
|
|
56
|
+
* - Necessary for consistent logging and event publishing, because the OpenSearch index expects a specific structure: flat_object.
|
|
57
|
+
* - The current UI expects a `message` property to be present, so we ensure it is always set.
|
|
58
|
+
*
|
|
59
|
+
* @param {any} message - The message to be logged. Can be a string, an object with a `message` property, or any other type.
|
|
60
|
+
* @param {string} level - The log level (e.g., 'error', 'debug', 'warn', 'verbose').
|
|
61
|
+
* @param {LoggerOptions} options - Additional options for logging, such as whether to include a stack trace.
|
|
62
|
+
* @returns {FlowLog} - An object containing the parsed log message and optional stack trace.
|
|
63
|
+
*/
|
|
64
|
+
parseMessageToFlowLog(message, level, options) {
|
|
65
|
+
let flowLogMessage;
|
|
66
|
+
if (!message) {
|
|
67
|
+
flowLogMessage = 'No message provided!';
|
|
68
|
+
}
|
|
69
|
+
else if (typeof message.message === 'string') {
|
|
70
|
+
flowLogMessage = message.message;
|
|
71
|
+
}
|
|
72
|
+
else if (typeof message === 'string') {
|
|
73
|
+
flowLogMessage = message;
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
try {
|
|
77
|
+
flowLogMessage = JSON.stringify(message.message ?? message);
|
|
78
|
+
}
|
|
79
|
+
catch (e) {
|
|
80
|
+
flowLogMessage = 'Error: Could not stringify the message.';
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const flowLog = { message: flowLogMessage };
|
|
84
|
+
if (['error', 'debug', 'warn', 'verbose'].includes(level) || options?.stackTrace) {
|
|
85
|
+
flowLog.stackTrace = FlowLogger.getStackTrace(options?.stackTrace ?? STACK_TRACE.ONLY_LOG_CALL);
|
|
86
|
+
}
|
|
87
|
+
return flowLog;
|
|
88
|
+
}
|
|
89
|
+
publish(message, level, options) {
|
|
90
|
+
const flowLogData = this.parseMessageToFlowLog(message, level, options);
|
|
91
|
+
if (this.publishEvent) {
|
|
92
|
+
const event = new FlowEvent_1.FlowEvent(this.metadata, flowLogData, `flow.log.${level}`);
|
|
93
|
+
this.publishEvent(event);
|
|
94
|
+
}
|
|
95
|
+
const messageWithStackTrace = flowLogData.stackTrace ? `${flowLogData.message}\n${flowLogData.stackTrace}` : flowLogData.message;
|
|
96
|
+
switch (level) {
|
|
97
|
+
case 'debug':
|
|
98
|
+
return this.logger.debug(messageWithStackTrace, { ...this.metadata, ...options });
|
|
99
|
+
case 'error':
|
|
100
|
+
return this.logger.error(messageWithStackTrace, { ...this.metadata, ...options });
|
|
101
|
+
case 'warn':
|
|
102
|
+
return this.logger.warn(messageWithStackTrace, { ...this.metadata, ...options });
|
|
103
|
+
case 'verbose':
|
|
104
|
+
return this.logger.verbose(messageWithStackTrace, { ...this.metadata, ...options });
|
|
105
|
+
default:
|
|
106
|
+
this.logger.log(messageWithStackTrace, { ...this.metadata, ...options });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
exports.FlowLogger = FlowLogger;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FlowModule = FlowModule;
|
|
4
|
+
function FlowModule(metadata) {
|
|
5
|
+
const validateNameRegExp = new RegExp(/^(@[a-z][a-z0-9-]*\/)?[a-z][a-z0-9-]*$/);
|
|
6
|
+
if (!validateNameRegExp.test(metadata.name)) {
|
|
7
|
+
throw new Error(`Flow Module name (${metadata.name}) is not valid. Name must be all lowercase and not contain any special characters except for hyphens. Can optionally start with a scope "@scopename/"`);
|
|
8
|
+
}
|
|
9
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
10
|
+
return (target) => {
|
|
11
|
+
Reflect.defineMetadata('module:name', metadata.name, target);
|
|
12
|
+
Reflect.defineMetadata('module:declarations', metadata.declarations, target);
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { AmqpConnectionManager } from 'amqp-connection-manager';
|
|
2
|
+
import { FlowLogger } from './FlowLogger';
|
|
3
|
+
export declare class RpcClient {
|
|
4
|
+
private readonly logger?;
|
|
5
|
+
private readonly channel;
|
|
6
|
+
private openRequests;
|
|
7
|
+
constructor(amqpConnection: AmqpConnectionManager, logger?: FlowLogger);
|
|
8
|
+
private onMessage;
|
|
9
|
+
callFunction: (routingKey: string, functionName: string, ...args: any[]) => Promise<unknown>;
|
|
10
|
+
declareFunction: (routingKey: string, name: string) => (...args: any[]) => Promise<unknown>;
|
|
11
|
+
close(): Promise<void>;
|
|
12
|
+
static formatTrace(stack?: string): string;
|
|
13
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RpcClient = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const crypto_1 = require("crypto");
|
|
6
|
+
const object_sizeof_1 = tslib_1.__importDefault(require("object-sizeof"));
|
|
7
|
+
const MAX_MSG_SIZE = +process.env.MAX_RPC_MSG_SIZE_BYTES;
|
|
8
|
+
const WARN_MSG_SIZE = +process.env.WARN_RPC_MSG_SIZE_BYTES;
|
|
9
|
+
class RpcClient {
|
|
10
|
+
constructor(amqpConnection, logger) {
|
|
11
|
+
this.logger = logger;
|
|
12
|
+
this.openRequests = new Map();
|
|
13
|
+
this.onMessage = (msg) => {
|
|
14
|
+
if (this.openRequests.has(msg.properties.correlationId)) {
|
|
15
|
+
const { resolve, reject, trace } = this.openRequests.get(msg.properties.correlationId);
|
|
16
|
+
const response = JSON.parse(msg.content.toString());
|
|
17
|
+
switch (response.type) {
|
|
18
|
+
case 'reply':
|
|
19
|
+
resolve(response.value);
|
|
20
|
+
break;
|
|
21
|
+
case 'error': {
|
|
22
|
+
const err = new Error(response.message);
|
|
23
|
+
if (response.stack) {
|
|
24
|
+
const stack = RpcClient.formatTrace(response.stack);
|
|
25
|
+
err.stack = 'Remote Stack\n'.concat(stack, '\nLocal Stack\n', trace);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
err.stack = trace;
|
|
29
|
+
}
|
|
30
|
+
reject(err);
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
default:
|
|
34
|
+
reject(response);
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
const message = `received unexpected response correlationID: ${msg.properties.correlationId}`;
|
|
40
|
+
/* eslint-disable-next-line no-console */
|
|
41
|
+
console.warn(message);
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
this.callFunction = (routingKey, functionName, ...args) => {
|
|
45
|
+
// in case remote returns error add this to the trace
|
|
46
|
+
const stack = new Error('test').stack;
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
// save to correlationId-> resolve/reject map
|
|
49
|
+
// on return resolve or reject promise
|
|
50
|
+
if (MAX_MSG_SIZE || WARN_MSG_SIZE) {
|
|
51
|
+
const messageSize = (0, object_sizeof_1.default)(args);
|
|
52
|
+
if (messageSize > MAX_MSG_SIZE) {
|
|
53
|
+
throw new Error(`Max RPC message size exceeded: ${messageSize} bytes / ${MAX_MSG_SIZE} bytes`);
|
|
54
|
+
}
|
|
55
|
+
if (messageSize > WARN_MSG_SIZE) {
|
|
56
|
+
this.logger?.warn(`Large RPC message size detected: ${messageSize} bytes`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const call = { functionName, arguments: args };
|
|
60
|
+
const correlationId = (0, crypto_1.randomUUID)();
|
|
61
|
+
this.openRequests.set(correlationId, { resolve, reject, trace: RpcClient.formatTrace(stack) });
|
|
62
|
+
this.channel
|
|
63
|
+
.publish('rpc_direct_exchange', routingKey, call, { correlationId, replyTo: 'amq.rabbitmq.reply-to' })
|
|
64
|
+
.catch((err) => reject(err));
|
|
65
|
+
});
|
|
66
|
+
};
|
|
67
|
+
this.declareFunction = (routingKey, name) => {
|
|
68
|
+
return (...args) => this.callFunction(routingKey, name, ...args);
|
|
69
|
+
};
|
|
70
|
+
if (!amqpConnection) {
|
|
71
|
+
throw new Error('currently no amqp connection available');
|
|
72
|
+
}
|
|
73
|
+
this.channel = amqpConnection.createChannel({
|
|
74
|
+
json: true,
|
|
75
|
+
setup: async (channel) => {
|
|
76
|
+
await channel.assertExchange('rpc_direct_exchange', 'direct', { durable: false });
|
|
77
|
+
await channel.consume('amq.rabbitmq.reply-to', this.onMessage, { noAck: true });
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
close() {
|
|
82
|
+
return this.channel.close();
|
|
83
|
+
}
|
|
84
|
+
static formatTrace(stack = '') {
|
|
85
|
+
return stack.split('\n').splice(1).join('\n');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
exports.RpcClient = RpcClient;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TestModule = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const FlowElement_1 = require("./FlowElement");
|
|
6
|
+
const FlowEvent_1 = require("./FlowEvent");
|
|
7
|
+
const FlowModule_1 = require("./FlowModule");
|
|
8
|
+
let TestTrigger = class TestTrigger extends FlowElement_1.FlowTask {
|
|
9
|
+
async onDefault(event) {
|
|
10
|
+
return this.emitEvent({}, event);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
tslib_1.__decorate([
|
|
14
|
+
(0, FlowElement_1.InputStream)('default'),
|
|
15
|
+
tslib_1.__metadata("design:type", Function),
|
|
16
|
+
tslib_1.__metadata("design:paramtypes", [FlowEvent_1.FlowEvent]),
|
|
17
|
+
tslib_1.__metadata("design:returntype", Promise)
|
|
18
|
+
], TestTrigger.prototype, "onDefault", null);
|
|
19
|
+
TestTrigger = tslib_1.__decorate([
|
|
20
|
+
(0, FlowElement_1.FlowFunction)('test.task.Trigger')
|
|
21
|
+
], TestTrigger);
|
|
22
|
+
let TestModule = class TestModule {
|
|
23
|
+
};
|
|
24
|
+
exports.TestModule = TestModule;
|
|
25
|
+
exports.TestModule = TestModule = tslib_1.__decorate([
|
|
26
|
+
(0, FlowModule_1.FlowModule)({ name: 'test', declarations: [TestTrigger] })
|
|
27
|
+
], TestModule);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
|
|
2
|
+
export interface AmqpConnection {
|
|
3
|
+
managedChannel: ChannelWrapper;
|
|
4
|
+
managedConnection: AmqpConnectionManager;
|
|
5
|
+
}
|
|
6
|
+
export interface AmqpConnectionConfig {
|
|
7
|
+
protocol?: string;
|
|
8
|
+
hostname?: string;
|
|
9
|
+
vhost?: string;
|
|
10
|
+
user?: string;
|
|
11
|
+
password?: string;
|
|
12
|
+
port?: number;
|
|
13
|
+
}
|
|
14
|
+
export declare function createAmqpConnection(config: AmqpConnectionConfig): AmqpConnectionManager;
|
package/src/lib/amqp.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createAmqpConnection = createAmqpConnection;
|
|
4
|
+
const amqp_connection_manager_1 = require("amqp-connection-manager");
|
|
5
|
+
function createAmqpConnection(config) {
|
|
6
|
+
if (!config) {
|
|
7
|
+
return null;
|
|
8
|
+
}
|
|
9
|
+
const { protocol = process.env.RABBIT_PROTOCOL || 'amqp', hostname = process.env.RABBIT_HOST || 'localhost', port = +process.env.RABBIT_PORT || 5672, user = process.env.RABBIT_USER || 'guest', password = process.env.RABBIT_PASSWORD || 'guest', vhost = process.env.RABBIT_VHOST || '', } = config;
|
|
10
|
+
const uri = `${protocol}://${user}:${password}@${hostname}:${port}${vhost ? '/' + vhost : ''}`;
|
|
11
|
+
return (0, amqp_connection_manager_1.connect)(uri);
|
|
12
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function IncompatableWith(incompatibleSiblings: string[]): (target: any, key: string) => void;
|