elasticio-sailor-nodejs 2.7.7 → 3.0.0-dev1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +12 -146
- package/CHANGELOG.md +0 -5
- package/config/local.json +19 -0
- package/lib/encryptor.js +1 -0
- package/lib/executor.js +0 -9
- package/lib/proxy-client.js +666 -0
- package/lib/sailor.js +67 -186
- package/lib/settings.js +13 -20
- package/lib/utils.js +8 -0
- package/package.json +8 -6
- package/run.js +1 -1
- package/run.local.js +14 -0
- package/tsconfig.json +23 -0
- package/lib/amqp.js +0 -624
- package/lib/messagesDB.js +0 -37
package/lib/sailor.js
CHANGED
|
@@ -1,28 +1,22 @@
|
|
|
1
1
|
const uuid = require('uuid');
|
|
2
2
|
const ComponentReader = require('./component_reader.js').ComponentReader;
|
|
3
|
-
const
|
|
3
|
+
const { ProxyClient, MESSAGE_PROCESSING_STATUS } = require('./proxy-client.js');
|
|
4
4
|
const TaskExec = require('./executor.js').TaskExec;
|
|
5
5
|
const log = require('./logging.js');
|
|
6
6
|
const _ = require('lodash');
|
|
7
7
|
const hooksData = require('./hooksData');
|
|
8
|
-
const Encryptor = require('../lib/encryptor');
|
|
9
8
|
const RestApiClient = require('elasticio-rest-node');
|
|
10
9
|
const assert = require('assert');
|
|
11
10
|
const co = require('co');
|
|
12
|
-
const pThrottle = require('p-throttle');
|
|
13
|
-
const { ObjectStorage } = require('@elastic.io/maester-client');
|
|
14
|
-
const { Readable } = require('stream');
|
|
15
|
-
const messagesDB = require('./messagesDB.js');
|
|
16
11
|
|
|
17
|
-
const AMQP_HEADER_META_PREFIX = 'x-eio-meta-';
|
|
18
12
|
const OBJECT_ID_HEADER = 'x-ipaas-object-storage-id';
|
|
19
13
|
|
|
20
|
-
function
|
|
21
|
-
return _.mapKeys(settings, (value, key) => _.
|
|
14
|
+
function convertSettingsToSnakeCase(settings) {
|
|
15
|
+
return _.mapKeys(settings, (value, key) => _.snakeCase(key));
|
|
22
16
|
}
|
|
23
17
|
|
|
24
18
|
function getAdditionalHeadersFromSettings(settings) {
|
|
25
|
-
return
|
|
19
|
+
return convertSettingsToSnakeCase(settings.additionalVars);
|
|
26
20
|
}
|
|
27
21
|
|
|
28
22
|
class Sailor {
|
|
@@ -32,11 +26,12 @@ class Sailor {
|
|
|
32
26
|
constructor(settings) {
|
|
33
27
|
this.settings = settings;
|
|
34
28
|
this.messagesCount = 0;
|
|
35
|
-
this.
|
|
29
|
+
this.proxyClient = new ProxyClient(settings);
|
|
36
30
|
this.componentReader = new ComponentReader();
|
|
37
31
|
this.snapshot = {};
|
|
38
32
|
this.stepData = {};
|
|
39
33
|
this.shutdownCallback = null;
|
|
34
|
+
// TODO move endpoint to proxy
|
|
40
35
|
//eslint-disable-next-line new-cap
|
|
41
36
|
this.apiClient = RestApiClient(
|
|
42
37
|
settings.API_USERNAME,
|
|
@@ -44,35 +39,16 @@ class Sailor {
|
|
|
44
39
|
{
|
|
45
40
|
retryCount: settings.API_REQUEST_RETRY_ATTEMPTS,
|
|
46
41
|
retryDelay: settings.API_REQUEST_RETRY_DELAY
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const objectStorage = new ObjectStorage({
|
|
50
|
-
uri: settings.OBJECT_STORAGE_URI,
|
|
51
|
-
jwtSecret: settings.OBJECT_STORAGE_TOKEN
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
const encryptor = new Encryptor(settings.MESSAGE_CRYPTO_PASSWORD, settings.MESSAGE_CRYPTO_IV);
|
|
55
|
-
this.objectStorage = objectStorage.use(
|
|
56
|
-
() => encryptor.createCipher(),
|
|
57
|
-
() => encryptor.createDecipher()
|
|
42
|
+
}
|
|
58
43
|
);
|
|
59
|
-
|
|
60
|
-
this.throttles = {
|
|
61
|
-
// 100 Messages per Second
|
|
62
|
-
data: pThrottle(() => Promise.resolve(true),
|
|
63
|
-
settings.DATA_RATE_LIMIT,
|
|
64
|
-
settings.RATE_INTERVAL),
|
|
65
|
-
error: pThrottle(() => Promise.resolve(true),
|
|
66
|
-
settings.ERROR_RATE_LIMIT,
|
|
67
|
-
settings.RATE_INTERVAL),
|
|
68
|
-
snapshot: pThrottle(() => Promise.resolve(true),
|
|
69
|
-
settings.SNAPSHOT_RATE_LIMIT,
|
|
70
|
-
settings.RATE_INTERVAL)
|
|
71
|
-
};
|
|
72
44
|
}
|
|
73
45
|
|
|
74
46
|
async connect() {
|
|
75
|
-
return this.
|
|
47
|
+
return this.proxyClient.connect();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async isConnected() {
|
|
51
|
+
return this.proxyClient.isConnected();
|
|
76
52
|
}
|
|
77
53
|
|
|
78
54
|
async prepare() {
|
|
@@ -102,10 +78,10 @@ class Sailor {
|
|
|
102
78
|
|
|
103
79
|
async disconnect() {
|
|
104
80
|
log.debug('Disconnecting, %s messages in processing', this.messagesCount);
|
|
105
|
-
return this.
|
|
81
|
+
return this.proxyClient.disconnect();
|
|
106
82
|
}
|
|
107
83
|
|
|
108
|
-
|
|
84
|
+
reportError(err) {
|
|
109
85
|
const headers = Object.assign({}, getAdditionalHeadersFromSettings(this.settings), {
|
|
110
86
|
execId: this.settings.EXEC_ID,
|
|
111
87
|
taskId: this.settings.FLOW_ID,
|
|
@@ -116,7 +92,7 @@ class Sailor {
|
|
|
116
92
|
compId: this.settings.COMP_ID,
|
|
117
93
|
function: this.settings.FUNCTION
|
|
118
94
|
});
|
|
119
|
-
return this.
|
|
95
|
+
return this.proxyClient.sendError(err, headers);
|
|
120
96
|
}
|
|
121
97
|
|
|
122
98
|
startup() {
|
|
@@ -184,17 +160,16 @@ class Sailor {
|
|
|
184
160
|
}
|
|
185
161
|
|
|
186
162
|
run() {
|
|
187
|
-
const incomingQueue = this.settings.LISTEN_MESSAGES_ON;
|
|
188
163
|
const handler = this.processMessageAndMaybeShutdownCallback.bind(this);
|
|
189
|
-
log.debug('Start listening for messages
|
|
190
|
-
return this.
|
|
164
|
+
log.debug('Start listening for messages');
|
|
165
|
+
return this.proxyClient.listenForMessages(handler);
|
|
191
166
|
}
|
|
192
167
|
|
|
193
|
-
async processMessageAndMaybeShutdownCallback(
|
|
168
|
+
async processMessageAndMaybeShutdownCallback(headers, body) {
|
|
194
169
|
try {
|
|
195
|
-
return await this.processMessage(
|
|
170
|
+
return await this.processMessage(headers, body);
|
|
196
171
|
} catch (e) {
|
|
197
|
-
log.error('Something very bad happened during message processing');
|
|
172
|
+
log.error(e, 'Something very bad happened during message processing');
|
|
198
173
|
} finally {
|
|
199
174
|
if (this.shutdownCallback) {
|
|
200
175
|
if (this.messagesCount === 0) {
|
|
@@ -216,7 +191,8 @@ class Sailor {
|
|
|
216
191
|
return new Promise(resolve => this.shutdownCallback = resolve);
|
|
217
192
|
}
|
|
218
193
|
|
|
219
|
-
|
|
194
|
+
// TODO: remove duplicate disconnect call (also in run.js)
|
|
195
|
+
await this.proxyClient.disconnect();
|
|
220
196
|
if (this.messagesCount === 0) {
|
|
221
197
|
// there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
|
|
222
198
|
log.debug('scheduleShutdown – about to shutdown immediately');
|
|
@@ -228,103 +204,23 @@ class Sailor {
|
|
|
228
204
|
return new Promise(resolve => this.shutdownCallback = resolve);
|
|
229
205
|
}
|
|
230
206
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
const { headers } = message.properties;
|
|
234
|
-
|
|
235
|
-
// Get meta headers
|
|
236
|
-
const metaHeaderNames = Object.keys(headers)
|
|
237
|
-
.filter(key => key.toLowerCase().startsWith(AMQP_HEADER_META_PREFIX));
|
|
238
|
-
|
|
239
|
-
const metaHeaders = _.pick(headers, metaHeaderNames);
|
|
240
|
-
const metaHeadersLowerCased = _.mapKeys(metaHeaders, (value, key) => key.toLowerCase());
|
|
241
|
-
|
|
242
|
-
const result = {
|
|
243
|
-
stepId: headers.stepId, // the only use is passthrough mechanism
|
|
244
|
-
...metaHeadersLowerCased,
|
|
245
|
-
threadId: headers.threadId || metaHeadersLowerCased['x-eio-meta-trace-id'],
|
|
246
|
-
messageId: headers.messageId,
|
|
247
|
-
parentMessageId: headers.parentMessageId
|
|
248
|
-
};
|
|
249
|
-
if (!result.threadId) {
|
|
250
|
-
const threadId = uuid.v4();
|
|
251
|
-
log.debug({ threadId }, 'Initiate new thread as it is not started ATM');
|
|
252
|
-
result.threadId = threadId;
|
|
253
|
-
}
|
|
254
|
-
if (headers.reply_to) {
|
|
255
|
-
result.reply_to = headers.reply_to;
|
|
256
|
-
}
|
|
257
|
-
return result;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
async fetchMessageBody(message, logger) {
|
|
261
|
-
const { body, headers } = message;
|
|
262
|
-
|
|
263
|
-
logger.info('Checking if incoming messages is lightweight...');
|
|
264
|
-
|
|
265
|
-
if (!headers) {
|
|
266
|
-
logger.info('Empty headers so not lightweight.');
|
|
267
|
-
return body;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
const { [OBJECT_ID_HEADER]: objectId } = headers;
|
|
271
|
-
|
|
272
|
-
if (!objectId) {
|
|
273
|
-
logger.trace('No object id header so not lightweight.');
|
|
274
|
-
return body;
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
logger.info('Object id header found, message is lightweight.', { objectId });
|
|
278
|
-
|
|
279
|
-
let object;
|
|
280
|
-
|
|
281
|
-
logger.info('Going to fetch message body.', { objectId });
|
|
282
|
-
|
|
283
|
-
try {
|
|
284
|
-
object = await this.objectStorage.getOne(
|
|
285
|
-
objectId,
|
|
286
|
-
{ jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
|
|
287
|
-
);
|
|
288
|
-
} catch (e) {
|
|
289
|
-
log.error(e);
|
|
290
|
-
throw new Error(`Failed to get message body with id=${objectId}`);
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
logger.info('Successfully obtained message body.', { objectId });
|
|
294
|
-
logger.trace('Message body object received');
|
|
295
|
-
|
|
296
|
-
return object.data;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
uploadMessageBody(bodyBuf) {
|
|
300
|
-
const stream = () => Readable.from(bodyBuf);
|
|
301
|
-
return this.objectStorage.add(
|
|
302
|
-
stream,
|
|
303
|
-
{ jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
|
|
304
|
-
);
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
async runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger) {
|
|
207
|
+
async runExec(module, payload, incomingMessageHeaders, outgoingMessageHeaders, stepData, timeStart, logger) {
|
|
208
|
+
log.debug({ incomingMessageHeaders }, 'runExec started');
|
|
308
209
|
const origPassthrough = _.cloneDeep(payload.passthrough) || {};
|
|
309
|
-
const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
|
|
310
|
-
const messageId = incomingMessageHeaders.messageId;
|
|
311
210
|
const settings = this.settings;
|
|
312
211
|
const cfg = _.cloneDeep(stepData.config) || {};
|
|
313
212
|
const snapshot = _.cloneDeep(this.snapshot);
|
|
314
|
-
const { deliveryTag } = message.fields;
|
|
315
213
|
|
|
316
214
|
const that = this;
|
|
317
215
|
|
|
318
216
|
await new Promise(resolve => {
|
|
319
217
|
let endWasEmitted;
|
|
320
218
|
|
|
321
|
-
|
|
322
219
|
const taskExec = new TaskExec({
|
|
323
220
|
loggerOptions: _.pick(incomingMessageHeaders, ['threadId', 'messageId', 'parentMessageId']),
|
|
324
221
|
variables: stepData.variables,
|
|
325
222
|
services: {
|
|
326
223
|
apiClient: this.apiClient,
|
|
327
|
-
amqp: this.amqpConnection,
|
|
328
224
|
config: this.settings
|
|
329
225
|
}
|
|
330
226
|
});
|
|
@@ -391,9 +287,9 @@ class Sailor {
|
|
|
391
287
|
let passthroughIds;
|
|
392
288
|
try {
|
|
393
289
|
[bodyId, ...passthroughIds] = await Promise.all([
|
|
394
|
-
that.uploadMessageBody(bodyBuf),
|
|
290
|
+
that.proxyClient.uploadMessageBody(bodyBuf),
|
|
395
291
|
...passthroughBufs.map(async ({ stepId, body, id }) => {
|
|
396
|
-
const bodyId = id || await that.uploadMessageBody(body);
|
|
292
|
+
const bodyId = id || await that.proxyClient.uploadMessageBody(body);
|
|
397
293
|
return { stepId, bodyId };
|
|
398
294
|
})
|
|
399
295
|
]);
|
|
@@ -402,7 +298,7 @@ class Sailor {
|
|
|
402
298
|
return onError(new Error('Lightweight message/passthrough body upload error'));
|
|
403
299
|
}
|
|
404
300
|
|
|
405
|
-
logger.info(
|
|
301
|
+
logger.info({ id: bodyId }, 'Message body uploaded');
|
|
406
302
|
const { headers } = data;
|
|
407
303
|
data.body = {};
|
|
408
304
|
data.headers = {
|
|
@@ -439,7 +335,7 @@ class Sailor {
|
|
|
439
335
|
logger.trace('Body is not empty.', { stepId });
|
|
440
336
|
return;
|
|
441
337
|
}
|
|
442
|
-
data.passthrough[stepId].body = await that.fetchMessageBody(
|
|
338
|
+
data.passthrough[stepId].body = await that.proxyClient.fetchMessageBody(
|
|
443
339
|
passthrough[stepId],
|
|
444
340
|
logger
|
|
445
341
|
);
|
|
@@ -458,7 +354,12 @@ class Sailor {
|
|
|
458
354
|
log.trace('Going to send outgoing message');
|
|
459
355
|
|
|
460
356
|
try {
|
|
461
|
-
await that.
|
|
357
|
+
await that.proxyClient.sendMessage({
|
|
358
|
+
incomingMessageId: incomingMessageHeaders.messageId,
|
|
359
|
+
data,
|
|
360
|
+
headers,
|
|
361
|
+
type: 'data'
|
|
362
|
+
});
|
|
462
363
|
log.trace('Outgoing message sent');
|
|
463
364
|
} catch (err) {
|
|
464
365
|
return onError(err);
|
|
@@ -471,7 +372,12 @@ class Sailor {
|
|
|
471
372
|
messageProcessingTime: Date.now() - timeStart
|
|
472
373
|
}, 'processMessage emit HttpReply');
|
|
473
374
|
|
|
474
|
-
return that.
|
|
375
|
+
return that.proxyClient.sendMessage({
|
|
376
|
+
incomingMessageId: incomingMessageHeaders.messageId,
|
|
377
|
+
data: reply,
|
|
378
|
+
headers,
|
|
379
|
+
type: 'http-reply'
|
|
380
|
+
});
|
|
475
381
|
}
|
|
476
382
|
|
|
477
383
|
async function onError(err) {
|
|
@@ -484,7 +390,7 @@ class Sailor {
|
|
|
484
390
|
messageProcessingTime: Date.now() - timeStart
|
|
485
391
|
}, 'processMessage emit error');
|
|
486
392
|
headers.end = new Date().getTime();
|
|
487
|
-
return that.
|
|
393
|
+
return that.proxyClient.sendError(err, headers, payload, incomingMessageHeaders);
|
|
488
394
|
}
|
|
489
395
|
|
|
490
396
|
async function onRebound(err) {
|
|
@@ -495,14 +401,14 @@ class Sailor {
|
|
|
495
401
|
messagesCount: that.messagesCount,
|
|
496
402
|
messageProcessingTime: Date.now() - timeStart
|
|
497
403
|
}, 'processMessage emit rebound');
|
|
498
|
-
return that.
|
|
404
|
+
return that.proxyClient.sendRebound(err, incomingMessageHeaders, outgoingHeaders);
|
|
499
405
|
}
|
|
500
406
|
|
|
501
407
|
async function onSnapshot(data) {
|
|
502
408
|
const headers = _.clone(outgoingMessageHeaders);
|
|
503
409
|
headers.snapshotEvent = 'snapshot';
|
|
504
410
|
that.snapshot = data; //replacing `local` snapshot
|
|
505
|
-
return that.
|
|
411
|
+
return that.proxyClient.sendSnapshot(data, headers);
|
|
506
412
|
}
|
|
507
413
|
|
|
508
414
|
async function onUpdateSnapshot(data) {
|
|
@@ -514,7 +420,7 @@ class Sailor {
|
|
|
514
420
|
return log.warn('ERROR: $set is not supported any more in `updateSnapshot` event');
|
|
515
421
|
}
|
|
516
422
|
_.extend(that.snapshot, data); //updating `local` snapshot
|
|
517
|
-
return that.
|
|
423
|
+
return that.proxyClient.sendSnapshot(data, headers);
|
|
518
424
|
} else {
|
|
519
425
|
log.error('You should pass an object to the `updateSnapshot` event');
|
|
520
426
|
}
|
|
@@ -527,14 +433,14 @@ class Sailor {
|
|
|
527
433
|
|
|
528
434
|
try {
|
|
529
435
|
await that.apiClient.accounts.update(cfg._account, { keys: keys });
|
|
530
|
-
logger.debug('Successfully updated keys
|
|
436
|
+
logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Successfully updated keys');
|
|
531
437
|
} catch (error) {
|
|
532
|
-
logger.
|
|
438
|
+
logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Failed to update keys');
|
|
533
439
|
await onError(error);
|
|
534
440
|
}
|
|
535
441
|
}
|
|
536
442
|
|
|
537
|
-
|
|
443
|
+
function onEnd() {
|
|
538
444
|
if (endWasEmitted) {
|
|
539
445
|
logger.warn({
|
|
540
446
|
messagesCount: that.messagesCount,
|
|
@@ -546,11 +452,12 @@ class Sailor {
|
|
|
546
452
|
|
|
547
453
|
endWasEmitted = true;
|
|
548
454
|
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
455
|
+
that.proxyClient.finishProcessing(
|
|
456
|
+
incomingMessageHeaders,
|
|
457
|
+
taskExec.errorCount > 0
|
|
458
|
+
? MESSAGE_PROCESSING_STATUS.ERROR
|
|
459
|
+
: MESSAGE_PROCESSING_STATUS.SUCCESS
|
|
460
|
+
);
|
|
554
461
|
that.messagesCount -= 1;
|
|
555
462
|
logger.trace({
|
|
556
463
|
messagesCount: that.messagesCount,
|
|
@@ -579,43 +486,21 @@ class Sailor {
|
|
|
579
486
|
}
|
|
580
487
|
}
|
|
581
488
|
|
|
582
|
-
async processMessage(
|
|
489
|
+
async processMessage(headers, payload) {
|
|
583
490
|
//eslint-disable-next-line consistent-this
|
|
584
491
|
const self = this;
|
|
585
492
|
const settings = this.settings;
|
|
586
|
-
const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
|
|
587
493
|
|
|
588
494
|
self.messagesCount += 1;
|
|
589
495
|
|
|
590
496
|
const timeStart = Date.now();
|
|
591
497
|
|
|
592
|
-
const messageId = incomingMessageHeaders.messageId;
|
|
593
498
|
const logger = log.child({
|
|
594
|
-
threadId:
|
|
595
|
-
messageId: messageId || 'unknown',
|
|
596
|
-
parentMessageId:
|
|
597
|
-
...message.fields
|
|
499
|
+
threadId: headers.threadId || 'unknown',
|
|
500
|
+
messageId: headers.messageId || 'unknown',
|
|
501
|
+
parentMessageId: headers.parentMessageId || 'unknown'
|
|
598
502
|
});
|
|
599
503
|
|
|
600
|
-
if (messageId) {
|
|
601
|
-
const alreadyExists = messagesDB.getMessageById(messageId);
|
|
602
|
-
// Add message to DB even if it already exists
|
|
603
|
-
messagesDB.addMessage(messageId, message);
|
|
604
|
-
if (alreadyExists) {
|
|
605
|
-
logger.warn({ messageId }, 'Duplicate message detected. This'
|
|
606
|
-
+ ' delivery will be ignored; the handler that first received'
|
|
607
|
-
+ ' this message will process it as part of deduplication.');
|
|
608
|
-
// If message was in messagesDB, it means that the connection was closed
|
|
609
|
-
// and this message was redelivered. In this case, the process for original
|
|
610
|
-
// message is waiting for this message to be added to DB and then ack or
|
|
611
|
-
// nack the new message, instead of the one that was delivered by closed
|
|
612
|
-
// channel
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
} else {
|
|
616
|
-
logger.warn('Message does not have messageId');
|
|
617
|
-
}
|
|
618
|
-
|
|
619
504
|
logger.trace({ messagesCount: this.messagesCount }, 'processMessage received');
|
|
620
505
|
|
|
621
506
|
const stepData = this.stepData;
|
|
@@ -623,10 +508,10 @@ class Sailor {
|
|
|
623
508
|
log.debug('Trigger or action: %s', settings.FUNCTION);
|
|
624
509
|
const outgoingMessageId = uuid.v4();
|
|
625
510
|
const outgoingMessageHeaders = {
|
|
626
|
-
...
|
|
511
|
+
...headers,
|
|
627
512
|
...getAdditionalHeadersFromSettings(settings),
|
|
628
|
-
parentMessageId:
|
|
629
|
-
threadId:
|
|
513
|
+
parentMessageId: headers.messageId,
|
|
514
|
+
threadId: headers.threadId,
|
|
630
515
|
messageId: outgoingMessageId,
|
|
631
516
|
execId: settings.EXEC_ID,
|
|
632
517
|
taskId: settings.FLOW_ID,
|
|
@@ -644,10 +529,8 @@ class Sailor {
|
|
|
644
529
|
} catch (e) {
|
|
645
530
|
log.error(e);
|
|
646
531
|
outgoingMessageHeaders.end = new Date().getTime();
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
self.amqpConnection.reject(messageId)
|
|
650
|
-
]);
|
|
532
|
+
self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
|
|
533
|
+
self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
|
|
651
534
|
return;
|
|
652
535
|
}
|
|
653
536
|
|
|
@@ -660,12 +543,12 @@ class Sailor {
|
|
|
660
543
|
await Promise.all([
|
|
661
544
|
(async () => {
|
|
662
545
|
logger.trace('Going to check if incoming message body is lightweight.');
|
|
663
|
-
payload.body = await this.fetchMessageBody(payload, logger);
|
|
546
|
+
payload.body = await this.proxyClient.fetchMessageBody(payload, logger);
|
|
664
547
|
})(),
|
|
665
548
|
...(passthrough
|
|
666
549
|
? Object.keys(passthrough).map(async stepId => {
|
|
667
550
|
logger.trace('Going to check if passthrough for step is lightweight.', { stepId });
|
|
668
|
-
payload.passthrough[stepId].body = await this.fetchMessageBody(
|
|
551
|
+
payload.passthrough[stepId].body = await this.proxyClient.fetchMessageBody(
|
|
669
552
|
payload.passthrough[stepId],
|
|
670
553
|
logger
|
|
671
554
|
);
|
|
@@ -675,15 +558,13 @@ class Sailor {
|
|
|
675
558
|
} catch (e) {
|
|
676
559
|
logger.error(e);
|
|
677
560
|
outgoingMessageHeaders.end = new Date().getTime();
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
self.amqpConnection.reject(messageId)
|
|
681
|
-
]);
|
|
561
|
+
self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
|
|
562
|
+
self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
|
|
682
563
|
return;
|
|
683
564
|
}
|
|
684
565
|
}
|
|
685
566
|
|
|
686
|
-
await this.runExec(module, payload,
|
|
567
|
+
await this.runExec(module, payload, headers, outgoingMessageHeaders, stepData, timeStart, logger);
|
|
687
568
|
}
|
|
688
569
|
}
|
|
689
570
|
|
package/lib/settings.js
CHANGED
|
@@ -4,14 +4,19 @@ const PREFIX = 'ELASTICIO_';
|
|
|
4
4
|
|
|
5
5
|
function getOptionalEnvVars(envVars) {
|
|
6
6
|
const optional = {
|
|
7
|
-
REBOUND_INITIAL_EXPIRATION: 15000,
|
|
8
|
-
REBOUND_LIMIT: 20,
|
|
9
7
|
COMPONENT_PATH: '',
|
|
10
|
-
|
|
8
|
+
PROXY_PREFETCH_SAILOR: 1,
|
|
11
9
|
STARTUP_REQUIRED: false,
|
|
12
10
|
HOOK_SHUTDOWN: false,
|
|
13
11
|
API_REQUEST_RETRY_ATTEMPTS: 3,
|
|
14
12
|
API_REQUEST_RETRY_DELAY: 100,
|
|
13
|
+
PROXY_RECONNECT_MAX_RETRIES: Infinity,
|
|
14
|
+
PROXY_RECONNECT_INITIAL_DELAY: 1000,
|
|
15
|
+
PROXY_RECONNECT_MAX_DELAY: 30 * 1000, // 30 seconds
|
|
16
|
+
PROXY_RECONNECT_BACKOFF_MULTIPLIER: 2,
|
|
17
|
+
PROXY_RECONNECT_JITTER_FACTOR: 0.3,
|
|
18
|
+
|
|
19
|
+
// TODO: Move to proxy?
|
|
15
20
|
DATA_RATE_LIMIT: 10, // 10 data events every 100ms
|
|
16
21
|
ERROR_RATE_LIMIT: 2, // 2 errors every 100ms
|
|
17
22
|
SNAPSHOT_RATE_LIMIT: 2, // 2 Snapshots every 100ms
|
|
@@ -20,19 +25,14 @@ function getOptionalEnvVars(envVars) {
|
|
|
20
25
|
AMQP_PUBLISH_RETRY_DELAY: 100, // 100ms
|
|
21
26
|
AMQP_PUBLISH_RETRY_ATTEMPTS: Infinity,
|
|
22
27
|
AMQP_PUBLISH_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
|
|
28
|
+
// Should be defaulted to true and moved to proxy
|
|
23
29
|
AMQP_PERSISTENT_MESSAGES: false,
|
|
30
|
+
|
|
24
31
|
OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,
|
|
25
32
|
NO_SELF_PASSTRHOUGH: false,
|
|
26
33
|
PROTOCOL_VERSION: 1,
|
|
27
|
-
NO_ERROR_REPLIES: false,
|
|
28
34
|
INPUT_FORMAT: 'default',
|
|
29
|
-
OBJECT_STORAGE_URI: null,
|
|
30
|
-
OBJECT_STORAGE_TOKEN: null,
|
|
31
|
-
OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
|
|
32
35
|
EMIT_LIGHTWEIGHT_MESSAGE: false,
|
|
33
|
-
AMQP_RECONNECT_ATTEMPTS: 3,
|
|
34
|
-
AMQP_RECONNECT_TIMEOUT: 100,
|
|
35
|
-
WAIT_MESSAGES_TIMEOUT: 50
|
|
36
36
|
};
|
|
37
37
|
|
|
38
38
|
const result = {};
|
|
@@ -87,16 +87,9 @@ function getMandatoryEnvVars(envVars) {
|
|
|
87
87
|
];
|
|
88
88
|
|
|
89
89
|
const requiredForMessageProcessing = [
|
|
90
|
-
'
|
|
91
|
-
'
|
|
92
|
-
'
|
|
93
|
-
|
|
94
|
-
'DATA_ROUTING_KEY',
|
|
95
|
-
'ERROR_ROUTING_KEY',
|
|
96
|
-
'REBOUND_ROUTING_KEY',
|
|
97
|
-
'SNAPSHOT_ROUTING_KEY',
|
|
98
|
-
'MESSAGE_CRYPTO_IV',
|
|
99
|
-
'MESSAGE_CRYPTO_PASSWORD'
|
|
90
|
+
'SAILOR_PROXY_URI',
|
|
91
|
+
'MESSAGE_CRYPTO_PASSWORD',
|
|
92
|
+
'MESSAGE_CRYPTO_IV'
|
|
100
93
|
];
|
|
101
94
|
|
|
102
95
|
const envVarsList = requiredAlways.slice(0);
|
package/lib/utils.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
function getJitteredDelay(baseDelay, jitterFactor) {
|
|
2
|
+
const minJitter = baseDelay * -jitterFactor;
|
|
3
|
+
const maxJitter = baseDelay * jitterFactor;
|
|
4
|
+
const jitter = Math.random() * (maxJitter - minJitter) + minJitter;
|
|
5
|
+
return baseDelay + jitter;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
module.exports.getJitteredDelay = getJitteredDelay;
|
package/package.json
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "elasticio-sailor-nodejs",
|
|
3
3
|
"description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "3.0.0-dev1",
|
|
5
5
|
"main": "run.js",
|
|
6
6
|
"scripts": {
|
|
7
|
+
"build": "tsc",
|
|
7
8
|
"audit": "better-npm-audit audit --level high --production",
|
|
8
|
-
"lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js",
|
|
9
9
|
"pretest": "npm run lint",
|
|
10
|
+
"lint": "eslint --ext .ts .",
|
|
10
11
|
"test": "npm run test:jasmine && npm run test:mocha",
|
|
11
12
|
"test:jasmine": "NODE_ENV=test jasmine-node spec",
|
|
12
13
|
"test:mocha": "NODE_ENV=test node_modules/.bin/mocha --recursive mocha_spec",
|
|
13
|
-
"postpublish": "./postpublish.js"
|
|
14
|
+
"postpublish": "./postpublish.js",
|
|
15
|
+
"dev:local": "node run.local.js | bunyan"
|
|
14
16
|
},
|
|
15
17
|
"engines": {
|
|
16
18
|
"node": ">=12.13.0"
|
|
17
19
|
},
|
|
18
20
|
"dependencies": {
|
|
19
|
-
"@elastic.io/maester-client": "6.0.0",
|
|
20
|
-
"amqplib": "0.8.0",
|
|
21
21
|
"bunyan": "1.8.10",
|
|
22
22
|
"co": "4.6.0",
|
|
23
23
|
"debug": "3.1.0",
|
|
@@ -45,7 +45,9 @@
|
|
|
45
45
|
"request": "2.88.0",
|
|
46
46
|
"request-promise-native": "1.0.5",
|
|
47
47
|
"sinon": "9.0.2",
|
|
48
|
-
"sinon-chai": "3.5.0"
|
|
48
|
+
"sinon-chai": "3.5.0",
|
|
49
|
+
"ts-node": "10.4.0",
|
|
50
|
+
"typescript": "4.4.4"
|
|
49
51
|
},
|
|
50
52
|
"repository": "elasticio/sailor-nodejs",
|
|
51
53
|
"license": "Apache-2.0"
|
package/run.js
CHANGED
|
@@ -98,7 +98,7 @@ async function run(settings, ipc) {
|
|
|
98
98
|
await putOutToSea(settings, ipc);
|
|
99
99
|
logger.info('Fully initialized and waiting for messages');
|
|
100
100
|
} catch (e) {
|
|
101
|
-
if (sailor && !sailor.
|
|
101
|
+
if (sailor && !sailor.isConnected()) {
|
|
102
102
|
await sailor.reportError(e);
|
|
103
103
|
}
|
|
104
104
|
logger.criticalErrorAndExit('putOutToSea.catch', e);
|
package/run.local.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
const config = require('./config/local.json');
|
|
2
|
+
|
|
3
|
+
function setEnvVars() {
|
|
4
|
+
for (const [key, value] of Object.entries(config)) {
|
|
5
|
+
process.env[key] = value;
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
setEnvVars();
|
|
9
|
+
|
|
10
|
+
const { IPC } = require('./lib/ipc');
|
|
11
|
+
const { run } = require('./run');
|
|
12
|
+
const settings = require('./lib/settings.js');
|
|
13
|
+
const ipc = new IPC();
|
|
14
|
+
run(settings.readFrom(process.env), ipc);
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"experimentalDecorators": true,
|
|
4
|
+
"emitDecoratorMetadata": true,
|
|
5
|
+
"lib": [
|
|
6
|
+
"es2020"
|
|
7
|
+
],
|
|
8
|
+
"types": ["node", "mocha"],
|
|
9
|
+
"module": "commonjs",
|
|
10
|
+
"esModuleInterop": true,
|
|
11
|
+
"target": "es2019",
|
|
12
|
+
"noImplicitAny": false,
|
|
13
|
+
"moduleResolution": "node",
|
|
14
|
+
"resolveJsonModule": true,
|
|
15
|
+
"sourceMap": true,
|
|
16
|
+
"outDir": "dist",
|
|
17
|
+
"skipLibCheck": true,
|
|
18
|
+
"typeRoots": ["./node_modules/@types", "./src/@types"]
|
|
19
|
+
},
|
|
20
|
+
"include": [
|
|
21
|
+
"src/**/*"
|
|
22
|
+
]
|
|
23
|
+
}
|