@hahnpro/flow-sdk 9.6.4 → 2025.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +904 -0
- package/jest.config.ts +10 -0
- package/package.json +7 -20
- package/project.json +41 -0
- package/src/index.ts +15 -0
- package/src/lib/ContextManager.ts +111 -0
- package/src/lib/FlowApplication.ts +659 -0
- package/src/lib/FlowElement.ts +220 -0
- package/src/lib/FlowEvent.ts +73 -0
- package/src/lib/FlowLogger.ts +131 -0
- package/src/lib/FlowModule.ts +18 -0
- package/src/lib/RpcClient.ts +99 -0
- package/src/lib/TestModule.ts +14 -0
- package/src/lib/__pycache__/rpc_server.cpython-310.pyc +0 -0
- package/src/lib/amqp.ts +32 -0
- package/src/lib/extra-validators.ts +62 -0
- package/src/lib/flow.interface.ts +56 -0
- package/{dist/index.d.ts → src/lib/index.ts} +3 -0
- package/src/lib/nats.ts +140 -0
- package/src/lib/unit-decorators.ts +156 -0
- package/src/lib/unit-utils.ts +163 -0
- package/src/lib/units.ts +587 -0
- package/src/lib/utils.ts +176 -0
- package/test/context-manager-purpose.spec.ts +248 -0
- package/test/context-manager.spec.ts +55 -0
- package/test/context.spec.ts +180 -0
- package/test/event.spec.ts +155 -0
- package/test/extra-validators.spec.ts +84 -0
- package/test/flow-logger.spec.ts +104 -0
- package/test/flow.spec.ts +508 -0
- package/test/input-stream.decorator.spec.ts +379 -0
- package/test/long-rpc.test.py +14 -0
- package/test/long-running-rpc.spec.ts +60 -0
- package/test/message.spec.ts +57 -0
- package/test/mocks/logger.mock.ts +7 -0
- package/test/mocks/nats-connection.mock.ts +135 -0
- package/test/mocks/nats-prepare.reals-nats.ts +15 -0
- package/test/rpc.spec.ts +198 -0
- package/test/rpc.test.py +45 -0
- package/test/rx.spec.ts +92 -0
- package/test/unit-decorator.spec.ts +57 -0
- package/test/utils.spec.ts +210 -0
- package/test/validation.spec.ts +174 -0
- package/tsconfig.json +13 -0
- package/tsconfig.lib.json +22 -0
- package/tsconfig.spec.json +8 -0
- package/LICENSE +0 -21
- package/dist/ContextManager.d.ts +0 -40
- package/dist/ContextManager.js +0 -77
- package/dist/FlowApplication.d.ts +0 -85
- package/dist/FlowApplication.js +0 -500
- package/dist/FlowElement.d.ts +0 -67
- package/dist/FlowElement.js +0 -163
- package/dist/FlowEvent.d.ts +0 -25
- package/dist/FlowEvent.js +0 -71
- package/dist/FlowLogger.d.ts +0 -44
- package/dist/FlowLogger.js +0 -94
- package/dist/FlowModule.d.ts +0 -7
- package/dist/FlowModule.js +0 -13
- package/dist/RpcClient.d.ts +0 -13
- package/dist/RpcClient.js +0 -84
- package/dist/TestModule.d.ts +0 -2
- package/dist/TestModule.js +0 -27
- package/dist/amqp.d.ts +0 -14
- package/dist/amqp.js +0 -12
- package/dist/extra-validators.d.ts +0 -1
- package/dist/extra-validators.js +0 -51
- package/dist/flow.interface.d.ts +0 -48
- package/dist/flow.interface.js +0 -9
- package/dist/index.js +0 -18
- package/dist/nats.d.ts +0 -12
- package/dist/nats.js +0 -109
- package/dist/unit-decorators.d.ts +0 -39
- package/dist/unit-decorators.js +0 -156
- package/dist/unit-utils.d.ts +0 -8
- package/dist/unit-utils.js +0 -143
- package/dist/units.d.ts +0 -31
- package/dist/units.js +0 -570
- package/dist/utils.d.ts +0 -51
- package/dist/utils.js +0 -137
- /package/{dist → src/lib}/rpc_server.py +0 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import { API } from '@hahnpro/hpc-api';
|
|
2
|
+
import { plainToInstance } from 'class-transformer';
|
|
3
|
+
import { validateSync } from 'class-validator';
|
|
4
|
+
import { Options, PythonShell } from 'python-shell';
|
|
5
|
+
|
|
6
|
+
import { ClassType, DeploymentMessage, FlowContext, FlowElementContext } from './flow.interface';
|
|
7
|
+
import { Context, FlowApplication } from './FlowApplication';
|
|
8
|
+
import { FlowEvent } from './FlowEvent';
|
|
9
|
+
import { FlowLogger } from './FlowLogger';
|
|
10
|
+
import { fillTemplate, handleApiError } from './utils';
|
|
11
|
+
|
|
12
|
+
export abstract class FlowElement<T = any> {
|
|
13
|
+
public readonly functionFqn: string;
|
|
14
|
+
protected readonly api?: API;
|
|
15
|
+
protected readonly logger: FlowLogger;
|
|
16
|
+
protected metadata: FlowElementContext;
|
|
17
|
+
protected properties: T;
|
|
18
|
+
private propertiesWithPlaceholders: T;
|
|
19
|
+
private readonly app?: FlowApplication;
|
|
20
|
+
private readonly rpcRoutingKey: string;
|
|
21
|
+
|
|
22
|
+
private stopPropagateStream: Map<string, boolean> = new Map<string, boolean>();
|
|
23
|
+
|
|
24
|
+
constructor(
|
|
25
|
+
{ app, logger, ...metadata }: Context,
|
|
26
|
+
properties?: unknown,
|
|
27
|
+
private readonly propertiesClassType?: ClassType<T>,
|
|
28
|
+
private readonly whitelist = false,
|
|
29
|
+
) {
|
|
30
|
+
this.app = app;
|
|
31
|
+
this.api = this.app?.api;
|
|
32
|
+
this.metadata = { ...metadata, functionFqn: this.functionFqn };
|
|
33
|
+
this.logger = new FlowLogger(this.metadata, logger || undefined, this.app?.publishNatsEventFlowlogs);
|
|
34
|
+
this.rpcRoutingKey = (this.metadata.flowId || '') + (this.metadata.deploymentId || '') + this.metadata.id;
|
|
35
|
+
if (properties) {
|
|
36
|
+
this.setProperties(properties as T);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Sets the placeholder properties for this flow element
|
|
42
|
+
* @param propertiesWithPlaceholders
|
|
43
|
+
*/
|
|
44
|
+
public setPropertiesWithPlaceholders(propertiesWithPlaceholders: T) {
|
|
45
|
+
this.propertiesWithPlaceholders = propertiesWithPlaceholders;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Returns the placeholder properties for this flow element
|
|
50
|
+
*/
|
|
51
|
+
public getPropertiesWithPlaceholders() {
|
|
52
|
+
return this.propertiesWithPlaceholders;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
get flowProperties() {
|
|
56
|
+
return this.app?.getProperties?.() || {};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
get natsConnection() {
|
|
60
|
+
return this.app?.natsConnection;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
public onDestroy?: () => void;
|
|
64
|
+
|
|
65
|
+
public onMessage?: (message: DeploymentMessage) => void;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Replace all placeholder properties with their explicit updated value and set them as the properties of the element
|
|
69
|
+
*/
|
|
70
|
+
public replacePlaceholderAndSetProperties() {
|
|
71
|
+
const placeholderProperties = this.propertiesWithPlaceholders;
|
|
72
|
+
if (this.propertiesWithPlaceholders) {
|
|
73
|
+
this.setProperties(this.app.getContextManager?.()?.replaceAllPlaceholderProperties(placeholderProperties));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
public onFlowPropertiesChanged?: (properties: Record<string, any>) => void;
|
|
78
|
+
|
|
79
|
+
public onContextChanged = (context: Partial<FlowContext>): void => {
|
|
80
|
+
this.metadata = { ...this.metadata, ...context };
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
public onPropertiesChanged = (properties: T): void => {
|
|
84
|
+
this.setProperties(properties);
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
public getMetadata = () => this.metadata;
|
|
88
|
+
|
|
89
|
+
protected setProperties = (properties: T): void => {
|
|
90
|
+
if (this.propertiesClassType) {
|
|
91
|
+
this.properties = this.validateProperties(this.propertiesClassType, properties, this.whitelist);
|
|
92
|
+
} else {
|
|
93
|
+
this.properties = properties;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
public handleApiError = (error: any) => handleApiError(error, this.logger);
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* @deprecated since version 4.8.0, will be removed in 5.0.0, use emitEvent(...) instead
|
|
101
|
+
*/
|
|
102
|
+
protected emitOutput(data: any = {}, outputId = 'default', time = new Date()): FlowEvent {
|
|
103
|
+
return this.emitEvent(data, null, outputId, time);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
protected emitEvent(data: Record<string, any>, inputEvent: FlowEvent, outputId = 'default', time = new Date()): FlowEvent {
|
|
107
|
+
const partialEvent = new FlowEvent(this.metadata, data, outputId, time);
|
|
108
|
+
const completeEvent = new FlowEvent(this.metadata, { ...(inputEvent?.getData() || {}), ...data }, outputId, time);
|
|
109
|
+
|
|
110
|
+
const streamID = inputEvent?.getMetadata()?.inputStreamId || '';
|
|
111
|
+
if ((this.stopPropagateStream.has(streamID) && this.stopPropagateStream.get(streamID)) || !this.stopPropagateStream.has(streamID)) {
|
|
112
|
+
this.app?.emit(partialEvent);
|
|
113
|
+
return partialEvent;
|
|
114
|
+
} else {
|
|
115
|
+
this.app?.emitPartial(completeEvent, partialEvent);
|
|
116
|
+
return completeEvent;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
protected validateProperties<P>(classType: ClassType<P>, properties: any = {}, whitelist = false): P {
|
|
121
|
+
const props: P = plainToInstance<P, any>(classType, properties);
|
|
122
|
+
const errors = validateSync(props as any, { forbidUnknownValues: false, whitelist });
|
|
123
|
+
if (Array.isArray(errors) && errors.length > 0) {
|
|
124
|
+
for (const e of errors) {
|
|
125
|
+
this.logValidationErrors(e);
|
|
126
|
+
}
|
|
127
|
+
throw new Error('Properties Validation failed');
|
|
128
|
+
} else {
|
|
129
|
+
return props;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
protected logValidationErrors(error: any, parent?: string) {
|
|
134
|
+
const { children, constraints, property, value } = error;
|
|
135
|
+
const name = parent ? parent + '.' + property : property;
|
|
136
|
+
if (constraints) {
|
|
137
|
+
this.logger.error(`Validation for property "${name}" failed:\n${JSON.stringify(constraints || {})}\nvalue: ${value}`);
|
|
138
|
+
} else if (Array.isArray(children)) {
|
|
139
|
+
for (const child of children) {
|
|
140
|
+
this.logValidationErrors(child, name);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
protected validateEventData<E>(classType: ClassType<E>, event: FlowEvent, whitelist = false): E {
|
|
146
|
+
return this.validateProperties(classType, event.getData(), whitelist);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
protected interpolate = (value: any, ...templateVariables: any) =>
|
|
150
|
+
fillTemplate(value, ...templateVariables, { flow: this.app?.getContextManager?.().getProperties?.().flow ?? {} });
|
|
151
|
+
|
|
152
|
+
protected async callRpcFunction(functionName: string, ...args: any[]) {
|
|
153
|
+
return this.app?.rpcClient.callFunction(this.rpcRoutingKey, functionName, ...args);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
protected runPyRpcScript(scriptPath: string, ...args: (string | boolean | number)[]) {
|
|
157
|
+
const options: Options = {
|
|
158
|
+
mode: 'text',
|
|
159
|
+
pythonOptions: ['-u'],
|
|
160
|
+
args: [__dirname, this.rpcRoutingKey, ...args.map((v) => v.toString())],
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const pyshell = new PythonShell(scriptPath, options);
|
|
164
|
+
pyshell.on('message', (message) => {
|
|
165
|
+
this.logger.debug(message);
|
|
166
|
+
});
|
|
167
|
+
pyshell.end((err) => {
|
|
168
|
+
if (err) {
|
|
169
|
+
this.logger.error(err);
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
return pyshell;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function InputStream(id = 'default', options?: { concurrent?: number; stopPropagation?: boolean }): MethodDecorator {
|
|
177
|
+
return (target: any, propertyKey: string, propertyDescriptor: PropertyDescriptor) => {
|
|
178
|
+
Reflect.defineMetadata(`stream:${id}`, propertyKey, target.constructor);
|
|
179
|
+
if (options) {
|
|
180
|
+
Reflect.defineMetadata(`stream:options:${id}`, options, target.constructor);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const method = propertyDescriptor.value;
|
|
184
|
+
|
|
185
|
+
propertyDescriptor.value = function (event: FlowEvent) {
|
|
186
|
+
if (!this.stopPropagateStream.has(id)) {
|
|
187
|
+
this.stopPropagateStream.set(id, options?.stopPropagation ?? false);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// add input stream to data to later determine if data should be propagated
|
|
191
|
+
return method.call(
|
|
192
|
+
this,
|
|
193
|
+
new FlowEvent(
|
|
194
|
+
{ id: event.getMetadata().elementId, ...event.getMetadata(), inputStreamId: id },
|
|
195
|
+
event.getData(),
|
|
196
|
+
event.getType(),
|
|
197
|
+
new Date(event.getTime()),
|
|
198
|
+
),
|
|
199
|
+
);
|
|
200
|
+
};
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export function FlowFunction(fqn: string): ClassDecorator {
|
|
205
|
+
const fqnRegExp = /^([a-zA-Z][a-zA-Z\d]*[.-])*[a-zA-Z][a-zA-Z\d]*$/;
|
|
206
|
+
if (!fqnRegExp.test(fqn)) {
|
|
207
|
+
throw new Error(`Flow Function FQN (${fqn}) is not valid`);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
211
|
+
return <TFunction extends Function>(target: TFunction): TFunction | void => {
|
|
212
|
+
Reflect.defineMetadata('element:functionFqn', fqn, target);
|
|
213
|
+
target.prototype.functionFqn = fqn;
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export abstract class FlowResource<T = any> extends FlowElement<T> {}
|
|
218
|
+
export abstract class FlowTask<T = any> extends FlowElement<T> {}
|
|
219
|
+
export abstract class FlowTrigger<T = any> extends FlowElement<T> {}
|
|
220
|
+
export abstract class FlowDashboard<T = any> extends FlowResource<T> {}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { CloudEvent } from 'cloudevents';
|
|
2
|
+
import { cloneDeep } from 'lodash';
|
|
3
|
+
|
|
4
|
+
import { FlowElementContext } from './flow.interface';
|
|
5
|
+
|
|
6
|
+
export class FlowEvent {
|
|
7
|
+
private event: CloudEvent;
|
|
8
|
+
private metadata: { deploymentId: string; elementId: string; flowId: string; functionFqn: string; inputStreamId: string };
|
|
9
|
+
|
|
10
|
+
constructor(metadata: FlowElementContext, data: any, outputId = 'default', time = new Date(), dataType?: string) {
|
|
11
|
+
const { id: elementId, deploymentId, flowId, functionFqn, inputStreamId } = metadata;
|
|
12
|
+
if (data instanceof Error) {
|
|
13
|
+
const error = { message: data.message, stack: data.stack };
|
|
14
|
+
data = error;
|
|
15
|
+
} else {
|
|
16
|
+
data = cloneDeep(data);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
if (data == null) {
|
|
20
|
+
data = {};
|
|
21
|
+
}
|
|
22
|
+
if (dataType == null) {
|
|
23
|
+
if (typeof data === 'string') {
|
|
24
|
+
try {
|
|
25
|
+
JSON.parse(data);
|
|
26
|
+
dataType = 'application/json';
|
|
27
|
+
} catch (err) {
|
|
28
|
+
dataType = 'text/plain';
|
|
29
|
+
}
|
|
30
|
+
} else if (typeof data === 'object') {
|
|
31
|
+
dataType = 'application/json';
|
|
32
|
+
} else {
|
|
33
|
+
data = String(data);
|
|
34
|
+
dataType = 'text/plain';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
this.metadata = { deploymentId, elementId, flowId, functionFqn, inputStreamId };
|
|
39
|
+
this.event = new CloudEvent({
|
|
40
|
+
source: `flows/${flowId}/deployments/${deploymentId}/elements/${elementId}`,
|
|
41
|
+
type: outputId,
|
|
42
|
+
subject: functionFqn,
|
|
43
|
+
datacontenttype: dataType,
|
|
44
|
+
data,
|
|
45
|
+
time: time.toISOString(),
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
public format = (): any => {
|
|
50
|
+
const event = this.event.toJSON();
|
|
51
|
+
if (event.datacontenttype === 'application/json') {
|
|
52
|
+
try {
|
|
53
|
+
event.data = JSON.parse(event.data as string);
|
|
54
|
+
} catch (err) {
|
|
55
|
+
/* ignore error */
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return event;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
public getData = (): any => cloneDeep(this.event.data || {});
|
|
62
|
+
public getDataContentType = (): string => this.event.datacontenttype;
|
|
63
|
+
public getDataschema = (): string => this.event.dataschema;
|
|
64
|
+
public getId = (): string => this.event.id;
|
|
65
|
+
public getMetadata = () => this.metadata;
|
|
66
|
+
public getSource = (): string => this.event.source;
|
|
67
|
+
public getStreamId = (): string => `${this.metadata.elementId}.${this.event.type}`;
|
|
68
|
+
public getSubject = (): string => this.event.subject;
|
|
69
|
+
public getTime = (): Date => new Date(this.event.time);
|
|
70
|
+
public getType = (): string => this.event.type;
|
|
71
|
+
public toJSON = (): any => this.event.toJSON();
|
|
72
|
+
public toString = (): string => this.event.toString();
|
|
73
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { FlowElementContext } from './flow.interface';
|
|
2
|
+
import { FlowEvent } from './FlowEvent';
|
|
3
|
+
|
|
4
|
+
export interface Logger {
|
|
5
|
+
debug(message, metadata?): void;
|
|
6
|
+
error(message, metadata?): void;
|
|
7
|
+
log(message, metadata?): void;
|
|
8
|
+
warn(message, metadata?): void;
|
|
9
|
+
verbose(message, metadata?): void;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/* eslint-disable no-console */
|
|
13
|
+
export const defaultLogger: Logger = {
|
|
14
|
+
debug: (msg, metadata?) => console.debug(msg),
|
|
15
|
+
error: (msg, metadata?) => console.error(msg),
|
|
16
|
+
log: (msg, metadata?) => console.log(msg),
|
|
17
|
+
warn: (msg, metadata?) => console.warn(msg),
|
|
18
|
+
verbose: (msg, metadata?) => console.log(msg, metadata),
|
|
19
|
+
};
|
|
20
|
+
/* eslint-enable no-console */
|
|
21
|
+
|
|
22
|
+
export enum STACK_TRACE {
|
|
23
|
+
FULL = 'full',
|
|
24
|
+
ONLY_LOG_CALL = 'only-log-call',
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface LoggerOptions {
|
|
28
|
+
truncate: boolean;
|
|
29
|
+
stackTrace?: STACK_TRACE;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface FlowLog {
|
|
33
|
+
message: string;
|
|
34
|
+
stackTrace?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export class FlowLogger implements Logger {
|
|
38
|
+
private static getStackTrace(stacktrace: STACK_TRACE = STACK_TRACE.FULL) {
|
|
39
|
+
// get stacktrace without extra dependencies
|
|
40
|
+
let stack;
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
throw new Error('');
|
|
44
|
+
} catch (error) {
|
|
45
|
+
stack = error.stack || '';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// cleanup stacktrace and remove calls within this file
|
|
49
|
+
stack = stack
|
|
50
|
+
.split('\n')
|
|
51
|
+
.map((line) => line.trim())
|
|
52
|
+
.filter((value) => value.includes('at ') && !value.includes('Logger'));
|
|
53
|
+
|
|
54
|
+
if (stacktrace === STACK_TRACE.ONLY_LOG_CALL && stack.length > 0) {
|
|
55
|
+
stack = stack[0];
|
|
56
|
+
} else {
|
|
57
|
+
stack = stack.splice(1).join('\n');
|
|
58
|
+
}
|
|
59
|
+
return stack;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
constructor(
|
|
63
|
+
private readonly metadata: FlowElementContext,
|
|
64
|
+
private readonly logger: Logger = defaultLogger,
|
|
65
|
+
private readonly publishEvent?: (event: FlowEvent) => void,
|
|
66
|
+
) {}
|
|
67
|
+
|
|
68
|
+
public debug = (message, options?: LoggerOptions) => this.publish(message, 'debug', options);
|
|
69
|
+
public error = (message, options?: LoggerOptions) => this.publish(message, 'error', options);
|
|
70
|
+
public log = (message, options?: LoggerOptions) => this.publish(message, 'info', options);
|
|
71
|
+
public warn = (message, options?: LoggerOptions) => this.publish(message, 'warn', options);
|
|
72
|
+
public verbose = (message, options?: LoggerOptions) => this.publish(message, 'verbose', options);
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Parses a message into a FlowLog object, including optional stack trace information.
|
|
76
|
+
*
|
|
77
|
+
* @details Requirements for the output format of messages:
|
|
78
|
+
* - Necessary for consistent logging and event publishing, because the OpenSearch index expects a specific structure: flat_object.
|
|
79
|
+
* - The current UI expects a `message` property to be present, so we ensure it is always set.
|
|
80
|
+
*
|
|
81
|
+
* @param {any} message - The message to be logged. Can be a string, an object with a `message` property, or any other type.
|
|
82
|
+
* @param {string} level - The log level (e.g., 'error', 'debug', 'warn', 'verbose').
|
|
83
|
+
* @param {LoggerOptions} options - Additional options for logging, such as whether to include a stack trace.
|
|
84
|
+
* @returns {FlowLog} - An object containing the parsed log message and optional stack trace.
|
|
85
|
+
*/
|
|
86
|
+
private parseMessageToFlowLog(message: any, level: string, options: LoggerOptions): FlowLog {
|
|
87
|
+
let flowLogMessage: string;
|
|
88
|
+
if (!message) {
|
|
89
|
+
flowLogMessage = 'No message provided!';
|
|
90
|
+
} else if (typeof message.message === 'string') {
|
|
91
|
+
flowLogMessage = message.message;
|
|
92
|
+
} else if (typeof message === 'string') {
|
|
93
|
+
flowLogMessage = message;
|
|
94
|
+
} else {
|
|
95
|
+
try {
|
|
96
|
+
flowLogMessage = JSON.stringify(message.message ?? message);
|
|
97
|
+
} catch (e) {
|
|
98
|
+
flowLogMessage = 'Error: Could not stringify the message.';
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const flowLog: FlowLog = { message: flowLogMessage };
|
|
103
|
+
if (['error', 'debug', 'warn', 'verbose'].includes(level) || options?.stackTrace) {
|
|
104
|
+
flowLog.stackTrace = FlowLogger.getStackTrace(options?.stackTrace ?? STACK_TRACE.ONLY_LOG_CALL);
|
|
105
|
+
}
|
|
106
|
+
return flowLog;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
private publish(message, level: string, options: LoggerOptions) {
|
|
110
|
+
const flowLogData: FlowLog = this.parseMessageToFlowLog(message, level, options);
|
|
111
|
+
|
|
112
|
+
if (this.publishEvent) {
|
|
113
|
+
const event = new FlowEvent(this.metadata, flowLogData, `flow.log.${level}`);
|
|
114
|
+
this.publishEvent(event);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const messageWithStackTrace = flowLogData.stackTrace ? `${flowLogData.message}\n${flowLogData.stackTrace}` : flowLogData.message;
|
|
118
|
+
switch (level) {
|
|
119
|
+
case 'debug':
|
|
120
|
+
return this.logger.debug(messageWithStackTrace, { ...this.metadata, ...options });
|
|
121
|
+
case 'error':
|
|
122
|
+
return this.logger.error(messageWithStackTrace, { ...this.metadata, ...options });
|
|
123
|
+
case 'warn':
|
|
124
|
+
return this.logger.warn(messageWithStackTrace, { ...this.metadata, ...options });
|
|
125
|
+
case 'verbose':
|
|
126
|
+
return this.logger.verbose(messageWithStackTrace, { ...this.metadata, ...options });
|
|
127
|
+
default:
|
|
128
|
+
this.logger.log(messageWithStackTrace, { ...this.metadata, ...options });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { FlowElement } from './FlowElement';
|
|
2
|
+
|
|
3
|
+
export function FlowModule(metadata: { name: string; declarations: Array<ClassType<FlowElement>> }): ClassDecorator {
|
|
4
|
+
const validateNameRegExp = new RegExp(/^(@[a-z][a-z0-9-]*\/)?[a-z][a-z0-9-]*$/);
|
|
5
|
+
if (!validateNameRegExp.test(metadata.name)) {
|
|
6
|
+
throw new Error(
|
|
7
|
+
`Flow Module name (${metadata.name}) is not valid. Name must be all lowercase and not contain any special characters except for hyphens. Can optionally start with a scope "@scopename/"`,
|
|
8
|
+
);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
12
|
+
return <TFunction extends Function>(target: TFunction): TFunction | void => {
|
|
13
|
+
Reflect.defineMetadata('module:name', metadata.name, target);
|
|
14
|
+
Reflect.defineMetadata('module:declarations', metadata.declarations, target);
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
type ClassType<T> = new (...args: any[]) => T;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
2
|
+
|
|
3
|
+
import type { AmqpConnectionManager, Channel, ChannelWrapper } from 'amqp-connection-manager';
|
|
4
|
+
import type { ConsumeMessage } from 'amqplib';
|
|
5
|
+
import sizeof from 'object-sizeof';
|
|
6
|
+
|
|
7
|
+
import { FlowLogger } from './FlowLogger';
|
|
8
|
+
|
|
9
|
+
const MAX_MSG_SIZE = +process.env.MAX_RPC_MSG_SIZE_BYTES;
|
|
10
|
+
const WARN_MSG_SIZE = +process.env.WARN_RPC_MSG_SIZE_BYTES;
|
|
11
|
+
|
|
12
|
+
export class RpcClient {
|
|
13
|
+
private readonly channel: ChannelWrapper;
|
|
14
|
+
private openRequests: Map<string, { resolve; reject; trace: string }> = new Map<string, { resolve; reject; trace: string }>();
|
|
15
|
+
|
|
16
|
+
constructor(
|
|
17
|
+
amqpConnection: AmqpConnectionManager,
|
|
18
|
+
private readonly logger?: FlowLogger,
|
|
19
|
+
) {
|
|
20
|
+
if (!amqpConnection) {
|
|
21
|
+
throw new Error('currently no amqp connection available');
|
|
22
|
+
}
|
|
23
|
+
this.channel = amqpConnection.createChannel({
|
|
24
|
+
json: true,
|
|
25
|
+
setup: async (channel: Channel) => {
|
|
26
|
+
await channel.assertExchange('rpc_direct_exchange', 'direct', { durable: false });
|
|
27
|
+
await channel.consume('amq.rabbitmq.reply-to', this.onMessage, { noAck: true });
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private onMessage = (msg: ConsumeMessage) => {
|
|
33
|
+
if (this.openRequests.has(msg.properties.correlationId)) {
|
|
34
|
+
const { resolve, reject, trace } = this.openRequests.get(msg.properties.correlationId);
|
|
35
|
+
const response = JSON.parse(msg.content.toString());
|
|
36
|
+
switch (response.type) {
|
|
37
|
+
case 'reply':
|
|
38
|
+
resolve(response.value);
|
|
39
|
+
break;
|
|
40
|
+
case 'error': {
|
|
41
|
+
const err = new Error(response.message);
|
|
42
|
+
if (response.stack) {
|
|
43
|
+
const stack: string = RpcClient.formatTrace(response.stack);
|
|
44
|
+
err.stack = 'Remote Stack\n'.concat(stack, '\nLocal Stack\n', trace);
|
|
45
|
+
} else {
|
|
46
|
+
err.stack = trace;
|
|
47
|
+
}
|
|
48
|
+
reject(err);
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
default:
|
|
52
|
+
reject(response);
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
} else {
|
|
56
|
+
const message = `received unexpected response correlationID: ${msg.properties.correlationId}`;
|
|
57
|
+
/* eslint-disable-next-line no-console */
|
|
58
|
+
console.warn(message);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
public callFunction = (routingKey: string, functionName: string, ...args: any[]) => {
|
|
63
|
+
// in case remote returns error add this to the trace
|
|
64
|
+
const stack = new Error('test').stack;
|
|
65
|
+
return new Promise((resolve, reject) => {
|
|
66
|
+
// save to correlationId-> resolve/reject map
|
|
67
|
+
// on return resolve or reject promise
|
|
68
|
+
|
|
69
|
+
if (MAX_MSG_SIZE || WARN_MSG_SIZE) {
|
|
70
|
+
const messageSize = sizeof(args);
|
|
71
|
+
if (messageSize > MAX_MSG_SIZE) {
|
|
72
|
+
throw new Error(`Max RPC message size exceeded: ${messageSize} bytes / ${MAX_MSG_SIZE} bytes`);
|
|
73
|
+
}
|
|
74
|
+
if (messageSize > WARN_MSG_SIZE) {
|
|
75
|
+
this.logger?.warn(`Large RPC message size detected: ${messageSize} bytes`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const call = { functionName, arguments: args };
|
|
80
|
+
const correlationId = randomUUID();
|
|
81
|
+
this.openRequests.set(correlationId, { resolve, reject, trace: RpcClient.formatTrace(stack) });
|
|
82
|
+
this.channel
|
|
83
|
+
.publish('rpc_direct_exchange', routingKey, call, { correlationId, replyTo: 'amq.rabbitmq.reply-to' })
|
|
84
|
+
.catch((err) => reject(err));
|
|
85
|
+
});
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
public declareFunction = (routingKey: string, name: string) => {
|
|
89
|
+
return (...args) => this.callFunction(routingKey, name, ...args);
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
public close() {
|
|
93
|
+
return this.channel.close();
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
public static formatTrace(stack = '') {
|
|
97
|
+
return stack.split('\n').splice(1).join('\n');
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { FlowFunction, FlowTask, InputStream } from './FlowElement';
|
|
2
|
+
import { FlowEvent } from './FlowEvent';
|
|
3
|
+
import { FlowModule } from './FlowModule';
|
|
4
|
+
|
|
5
|
+
@FlowFunction('test.task.Trigger')
|
|
6
|
+
class TestTrigger extends FlowTask {
|
|
7
|
+
@InputStream('default')
|
|
8
|
+
public async onDefault(event: FlowEvent) {
|
|
9
|
+
return this.emitEvent({}, event);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
@FlowModule({ name: 'test', declarations: [TestTrigger] })
|
|
14
|
+
export class TestModule {}
|
|
Binary file
|
package/src/lib/amqp.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { AmqpConnectionManager, ChannelWrapper, connect } from 'amqp-connection-manager';
|
|
2
|
+
|
|
3
|
+
export interface AmqpConnection {
|
|
4
|
+
managedChannel: ChannelWrapper;
|
|
5
|
+
managedConnection: AmqpConnectionManager;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface AmqpConnectionConfig {
|
|
9
|
+
protocol?: string;
|
|
10
|
+
hostname?: string;
|
|
11
|
+
vhost?: string;
|
|
12
|
+
user?: string;
|
|
13
|
+
password?: string;
|
|
14
|
+
port?: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function createAmqpConnection(config: AmqpConnectionConfig): AmqpConnectionManager {
|
|
18
|
+
if (!config) {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const {
|
|
23
|
+
protocol = process.env.RABBIT_PROTOCOL || 'amqp',
|
|
24
|
+
hostname = process.env.RABBIT_HOST || 'localhost',
|
|
25
|
+
port = +process.env.RABBIT_PORT || 5672,
|
|
26
|
+
user = process.env.RABBIT_USER || 'guest',
|
|
27
|
+
password = process.env.RABBIT_PASSWORD || 'guest',
|
|
28
|
+
vhost = process.env.RABBIT_VHOST || '',
|
|
29
|
+
} = config;
|
|
30
|
+
const uri = `${protocol}://${user}:${password}@${hostname}:${port}${vhost ? '/' + vhost : ''}`;
|
|
31
|
+
return connect(uri);
|
|
32
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import {
|
|
2
|
+
isDefined,
|
|
3
|
+
registerDecorator,
|
|
4
|
+
ValidateIf,
|
|
5
|
+
ValidationArguments,
|
|
6
|
+
ValidationOptions,
|
|
7
|
+
ValidatorConstraint,
|
|
8
|
+
ValidatorConstraintInterface,
|
|
9
|
+
} from 'class-validator';
|
|
10
|
+
|
|
11
|
+
@ValidatorConstraint({ async: false })
|
|
12
|
+
class IsNotSiblingOfConstraint implements ValidatorConstraintInterface {
|
|
13
|
+
validate(value: any, args: ValidationArguments) {
|
|
14
|
+
if (isDefined(value)) {
|
|
15
|
+
return this.getFailedConstraints(args).length === 0;
|
|
16
|
+
} else {
|
|
17
|
+
return !args.constraints.every((prop) => !isDefined(args.object[prop]));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
defaultMessage(args: ValidationArguments) {
|
|
22
|
+
if (args.value) {
|
|
23
|
+
return `${args.property} cannot exist alongside the following defined properties: ${this.getFailedConstraints(args).join(', ')}`;
|
|
24
|
+
} else {
|
|
25
|
+
return `at least one of the properties must be defined`;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
getFailedConstraints(args: ValidationArguments) {
|
|
30
|
+
return args.constraints.filter((prop) => isDefined(args.object[prop]));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Create Decorator for the constraint that was just created
|
|
35
|
+
function IsNotSiblingOf(props: string[], validationOptions?: ValidationOptions) {
|
|
36
|
+
return (object: any, propertyName: string) => {
|
|
37
|
+
registerDecorator({
|
|
38
|
+
target: object.constructor,
|
|
39
|
+
propertyName,
|
|
40
|
+
options: validationOptions,
|
|
41
|
+
constraints: props,
|
|
42
|
+
validator: IsNotSiblingOfConstraint,
|
|
43
|
+
});
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Helper function for determining if a prop should be validated
|
|
48
|
+
function incompatibleSiblingsNotPresent(incompatibleSiblings: string[]) {
|
|
49
|
+
return (o, v) =>
|
|
50
|
+
Boolean(
|
|
51
|
+
isDefined(v) || incompatibleSiblings.every((prop) => !isDefined(o[prop])), // Validate if all incompatible siblings are not defined
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function IncompatableWith(incompatibleSiblings: string[]) {
|
|
56
|
+
const notSibling = IsNotSiblingOf(incompatibleSiblings);
|
|
57
|
+
const validateIf = ValidateIf(incompatibleSiblingsNotPresent(incompatibleSiblings));
|
|
58
|
+
return (target: any, key: string) => {
|
|
59
|
+
notSibling(target, key);
|
|
60
|
+
validateIf(target, key);
|
|
61
|
+
};
|
|
62
|
+
}
|