elasticio-sailor-nodejs 2.7.7 → 3.0.0-dev2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +12 -146
- package/CHANGELOG.md +0 -5
- package/config/local.json +19 -0
- package/lib/executor.js +0 -9
- package/lib/proxy-client.js +667 -0
- package/lib/sailor.js +68 -186
- package/lib/settings.js +13 -20
- package/lib/utils.js +8 -0
- package/package.json +8 -6
- package/run.js +3 -1
- package/run.local.js +14 -0
- package/tsconfig.json +23 -0
- package/lib/amqp.js +0 -624
- package/lib/messagesDB.js +0 -37
package/lib/sailor.js
CHANGED
|
@@ -1,28 +1,22 @@
|
|
|
1
1
|
const uuid = require('uuid');
|
|
2
2
|
const ComponentReader = require('./component_reader.js').ComponentReader;
|
|
3
|
-
const
|
|
3
|
+
const { ProxyClient, MESSAGE_PROCESSING_STATUS } = require('./proxy-client.js');
|
|
4
4
|
const TaskExec = require('./executor.js').TaskExec;
|
|
5
5
|
const log = require('./logging.js');
|
|
6
6
|
const _ = require('lodash');
|
|
7
7
|
const hooksData = require('./hooksData');
|
|
8
|
-
const Encryptor = require('../lib/encryptor');
|
|
9
8
|
const RestApiClient = require('elasticio-rest-node');
|
|
10
9
|
const assert = require('assert');
|
|
11
10
|
const co = require('co');
|
|
12
|
-
const pThrottle = require('p-throttle');
|
|
13
|
-
const { ObjectStorage } = require('@elastic.io/maester-client');
|
|
14
|
-
const { Readable } = require('stream');
|
|
15
|
-
const messagesDB = require('./messagesDB.js');
|
|
16
11
|
|
|
17
|
-
const AMQP_HEADER_META_PREFIX = 'x-eio-meta-';
|
|
18
12
|
const OBJECT_ID_HEADER = 'x-ipaas-object-storage-id';
|
|
19
13
|
|
|
20
|
-
function
|
|
21
|
-
return _.mapKeys(settings, (value, key) => _.
|
|
14
|
+
function convertSettingsToSnakeCase(settings) {
|
|
15
|
+
return _.mapKeys(settings, (value, key) => _.snakeCase(key));
|
|
22
16
|
}
|
|
23
17
|
|
|
24
18
|
function getAdditionalHeadersFromSettings(settings) {
|
|
25
|
-
return
|
|
19
|
+
return convertSettingsToSnakeCase(settings.additionalVars);
|
|
26
20
|
}
|
|
27
21
|
|
|
28
22
|
class Sailor {
|
|
@@ -32,11 +26,12 @@ class Sailor {
|
|
|
32
26
|
constructor(settings) {
|
|
33
27
|
this.settings = settings;
|
|
34
28
|
this.messagesCount = 0;
|
|
35
|
-
this.
|
|
29
|
+
this.proxyClient = new ProxyClient(settings);
|
|
36
30
|
this.componentReader = new ComponentReader();
|
|
37
31
|
this.snapshot = {};
|
|
38
32
|
this.stepData = {};
|
|
39
33
|
this.shutdownCallback = null;
|
|
34
|
+
// TODO move endpoint to proxy
|
|
40
35
|
//eslint-disable-next-line new-cap
|
|
41
36
|
this.apiClient = RestApiClient(
|
|
42
37
|
settings.API_USERNAME,
|
|
@@ -44,38 +39,20 @@ class Sailor {
|
|
|
44
39
|
{
|
|
45
40
|
retryCount: settings.API_REQUEST_RETRY_ATTEMPTS,
|
|
46
41
|
retryDelay: settings.API_REQUEST_RETRY_DELAY
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const objectStorage = new ObjectStorage({
|
|
50
|
-
uri: settings.OBJECT_STORAGE_URI,
|
|
51
|
-
jwtSecret: settings.OBJECT_STORAGE_TOKEN
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
const encryptor = new Encryptor(settings.MESSAGE_CRYPTO_PASSWORD, settings.MESSAGE_CRYPTO_IV);
|
|
55
|
-
this.objectStorage = objectStorage.use(
|
|
56
|
-
() => encryptor.createCipher(),
|
|
57
|
-
() => encryptor.createDecipher()
|
|
42
|
+
}
|
|
58
43
|
);
|
|
59
|
-
|
|
60
|
-
this.throttles = {
|
|
61
|
-
// 100 Messages per Second
|
|
62
|
-
data: pThrottle(() => Promise.resolve(true),
|
|
63
|
-
settings.DATA_RATE_LIMIT,
|
|
64
|
-
settings.RATE_INTERVAL),
|
|
65
|
-
error: pThrottle(() => Promise.resolve(true),
|
|
66
|
-
settings.ERROR_RATE_LIMIT,
|
|
67
|
-
settings.RATE_INTERVAL),
|
|
68
|
-
snapshot: pThrottle(() => Promise.resolve(true),
|
|
69
|
-
settings.SNAPSHOT_RATE_LIMIT,
|
|
70
|
-
settings.RATE_INTERVAL)
|
|
71
|
-
};
|
|
72
44
|
}
|
|
73
45
|
|
|
74
46
|
async connect() {
|
|
75
|
-
return this.
|
|
47
|
+
return this.proxyClient.connect();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async isConnected() {
|
|
51
|
+
return this.proxyClient.isConnected();
|
|
76
52
|
}
|
|
77
53
|
|
|
78
54
|
async prepare() {
|
|
55
|
+
log.trace('prepare sailor');
|
|
79
56
|
const {
|
|
80
57
|
settings: {
|
|
81
58
|
COMPONENT_PATH: compPath,
|
|
@@ -102,10 +79,10 @@ class Sailor {
|
|
|
102
79
|
|
|
103
80
|
async disconnect() {
|
|
104
81
|
log.debug('Disconnecting, %s messages in processing', this.messagesCount);
|
|
105
|
-
return this.
|
|
82
|
+
return this.proxyClient.disconnect();
|
|
106
83
|
}
|
|
107
84
|
|
|
108
|
-
|
|
85
|
+
reportError(err) {
|
|
109
86
|
const headers = Object.assign({}, getAdditionalHeadersFromSettings(this.settings), {
|
|
110
87
|
execId: this.settings.EXEC_ID,
|
|
111
88
|
taskId: this.settings.FLOW_ID,
|
|
@@ -116,7 +93,7 @@ class Sailor {
|
|
|
116
93
|
compId: this.settings.COMP_ID,
|
|
117
94
|
function: this.settings.FUNCTION
|
|
118
95
|
});
|
|
119
|
-
return this.
|
|
96
|
+
return this.proxyClient.sendError(err, headers);
|
|
120
97
|
}
|
|
121
98
|
|
|
122
99
|
startup() {
|
|
@@ -184,17 +161,16 @@ class Sailor {
|
|
|
184
161
|
}
|
|
185
162
|
|
|
186
163
|
run() {
|
|
187
|
-
const incomingQueue = this.settings.LISTEN_MESSAGES_ON;
|
|
188
164
|
const handler = this.processMessageAndMaybeShutdownCallback.bind(this);
|
|
189
|
-
log.debug('Start listening for messages
|
|
190
|
-
return this.
|
|
165
|
+
log.debug('Start listening for messages');
|
|
166
|
+
return this.proxyClient.listenForMessages(handler);
|
|
191
167
|
}
|
|
192
168
|
|
|
193
|
-
async processMessageAndMaybeShutdownCallback(
|
|
169
|
+
async processMessageAndMaybeShutdownCallback(headers, body) {
|
|
194
170
|
try {
|
|
195
|
-
return await this.processMessage(
|
|
171
|
+
return await this.processMessage(headers, body);
|
|
196
172
|
} catch (e) {
|
|
197
|
-
log.error('Something very bad happened during message processing');
|
|
173
|
+
log.error(e, 'Something very bad happened during message processing');
|
|
198
174
|
} finally {
|
|
199
175
|
if (this.shutdownCallback) {
|
|
200
176
|
if (this.messagesCount === 0) {
|
|
@@ -216,7 +192,8 @@ class Sailor {
|
|
|
216
192
|
return new Promise(resolve => this.shutdownCallback = resolve);
|
|
217
193
|
}
|
|
218
194
|
|
|
219
|
-
|
|
195
|
+
// TODO: remove duplicate disconnect call (also in run.js)
|
|
196
|
+
await this.proxyClient.disconnect();
|
|
220
197
|
if (this.messagesCount === 0) {
|
|
221
198
|
// there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
|
|
222
199
|
log.debug('scheduleShutdown – about to shutdown immediately');
|
|
@@ -228,103 +205,23 @@ class Sailor {
|
|
|
228
205
|
return new Promise(resolve => this.shutdownCallback = resolve);
|
|
229
206
|
}
|
|
230
207
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
const { headers } = message.properties;
|
|
234
|
-
|
|
235
|
-
// Get meta headers
|
|
236
|
-
const metaHeaderNames = Object.keys(headers)
|
|
237
|
-
.filter(key => key.toLowerCase().startsWith(AMQP_HEADER_META_PREFIX));
|
|
238
|
-
|
|
239
|
-
const metaHeaders = _.pick(headers, metaHeaderNames);
|
|
240
|
-
const metaHeadersLowerCased = _.mapKeys(metaHeaders, (value, key) => key.toLowerCase());
|
|
241
|
-
|
|
242
|
-
const result = {
|
|
243
|
-
stepId: headers.stepId, // the only use is passthrough mechanism
|
|
244
|
-
...metaHeadersLowerCased,
|
|
245
|
-
threadId: headers.threadId || metaHeadersLowerCased['x-eio-meta-trace-id'],
|
|
246
|
-
messageId: headers.messageId,
|
|
247
|
-
parentMessageId: headers.parentMessageId
|
|
248
|
-
};
|
|
249
|
-
if (!result.threadId) {
|
|
250
|
-
const threadId = uuid.v4();
|
|
251
|
-
log.debug({ threadId }, 'Initiate new thread as it is not started ATM');
|
|
252
|
-
result.threadId = threadId;
|
|
253
|
-
}
|
|
254
|
-
if (headers.reply_to) {
|
|
255
|
-
result.reply_to = headers.reply_to;
|
|
256
|
-
}
|
|
257
|
-
return result;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
async fetchMessageBody(message, logger) {
|
|
261
|
-
const { body, headers } = message;
|
|
262
|
-
|
|
263
|
-
logger.info('Checking if incoming messages is lightweight...');
|
|
264
|
-
|
|
265
|
-
if (!headers) {
|
|
266
|
-
logger.info('Empty headers so not lightweight.');
|
|
267
|
-
return body;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
const { [OBJECT_ID_HEADER]: objectId } = headers;
|
|
271
|
-
|
|
272
|
-
if (!objectId) {
|
|
273
|
-
logger.trace('No object id header so not lightweight.');
|
|
274
|
-
return body;
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
logger.info('Object id header found, message is lightweight.', { objectId });
|
|
278
|
-
|
|
279
|
-
let object;
|
|
280
|
-
|
|
281
|
-
logger.info('Going to fetch message body.', { objectId });
|
|
282
|
-
|
|
283
|
-
try {
|
|
284
|
-
object = await this.objectStorage.getOne(
|
|
285
|
-
objectId,
|
|
286
|
-
{ jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
|
|
287
|
-
);
|
|
288
|
-
} catch (e) {
|
|
289
|
-
log.error(e);
|
|
290
|
-
throw new Error(`Failed to get message body with id=${objectId}`);
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
logger.info('Successfully obtained message body.', { objectId });
|
|
294
|
-
logger.trace('Message body object received');
|
|
295
|
-
|
|
296
|
-
return object.data;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
uploadMessageBody(bodyBuf) {
|
|
300
|
-
const stream = () => Readable.from(bodyBuf);
|
|
301
|
-
return this.objectStorage.add(
|
|
302
|
-
stream,
|
|
303
|
-
{ jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
|
|
304
|
-
);
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
async runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger) {
|
|
208
|
+
async runExec(module, payload, incomingMessageHeaders, outgoingMessageHeaders, stepData, timeStart, logger) {
|
|
209
|
+
log.debug({ incomingMessageHeaders }, 'runExec started');
|
|
308
210
|
const origPassthrough = _.cloneDeep(payload.passthrough) || {};
|
|
309
|
-
const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
|
|
310
|
-
const messageId = incomingMessageHeaders.messageId;
|
|
311
211
|
const settings = this.settings;
|
|
312
212
|
const cfg = _.cloneDeep(stepData.config) || {};
|
|
313
213
|
const snapshot = _.cloneDeep(this.snapshot);
|
|
314
|
-
const { deliveryTag } = message.fields;
|
|
315
214
|
|
|
316
215
|
const that = this;
|
|
317
216
|
|
|
318
217
|
await new Promise(resolve => {
|
|
319
218
|
let endWasEmitted;
|
|
320
219
|
|
|
321
|
-
|
|
322
220
|
const taskExec = new TaskExec({
|
|
323
221
|
loggerOptions: _.pick(incomingMessageHeaders, ['threadId', 'messageId', 'parentMessageId']),
|
|
324
222
|
variables: stepData.variables,
|
|
325
223
|
services: {
|
|
326
224
|
apiClient: this.apiClient,
|
|
327
|
-
amqp: this.amqpConnection,
|
|
328
225
|
config: this.settings
|
|
329
226
|
}
|
|
330
227
|
});
|
|
@@ -391,9 +288,9 @@ class Sailor {
|
|
|
391
288
|
let passthroughIds;
|
|
392
289
|
try {
|
|
393
290
|
[bodyId, ...passthroughIds] = await Promise.all([
|
|
394
|
-
that.uploadMessageBody(bodyBuf),
|
|
291
|
+
that.proxyClient.uploadMessageBody(bodyBuf),
|
|
395
292
|
...passthroughBufs.map(async ({ stepId, body, id }) => {
|
|
396
|
-
const bodyId = id || await that.uploadMessageBody(body);
|
|
293
|
+
const bodyId = id || await that.proxyClient.uploadMessageBody(body);
|
|
397
294
|
return { stepId, bodyId };
|
|
398
295
|
})
|
|
399
296
|
]);
|
|
@@ -402,7 +299,7 @@ class Sailor {
|
|
|
402
299
|
return onError(new Error('Lightweight message/passthrough body upload error'));
|
|
403
300
|
}
|
|
404
301
|
|
|
405
|
-
logger.info(
|
|
302
|
+
logger.info({ id: bodyId }, 'Message body uploaded');
|
|
406
303
|
const { headers } = data;
|
|
407
304
|
data.body = {};
|
|
408
305
|
data.headers = {
|
|
@@ -439,7 +336,7 @@ class Sailor {
|
|
|
439
336
|
logger.trace('Body is not empty.', { stepId });
|
|
440
337
|
return;
|
|
441
338
|
}
|
|
442
|
-
data.passthrough[stepId].body = await that.fetchMessageBody(
|
|
339
|
+
data.passthrough[stepId].body = await that.proxyClient.fetchMessageBody(
|
|
443
340
|
passthrough[stepId],
|
|
444
341
|
logger
|
|
445
342
|
);
|
|
@@ -458,7 +355,12 @@ class Sailor {
|
|
|
458
355
|
log.trace('Going to send outgoing message');
|
|
459
356
|
|
|
460
357
|
try {
|
|
461
|
-
await that.
|
|
358
|
+
await that.proxyClient.sendMessage({
|
|
359
|
+
incomingMessageId: incomingMessageHeaders.messageId,
|
|
360
|
+
data,
|
|
361
|
+
headers,
|
|
362
|
+
type: 'data'
|
|
363
|
+
});
|
|
462
364
|
log.trace('Outgoing message sent');
|
|
463
365
|
} catch (err) {
|
|
464
366
|
return onError(err);
|
|
@@ -471,7 +373,12 @@ class Sailor {
|
|
|
471
373
|
messageProcessingTime: Date.now() - timeStart
|
|
472
374
|
}, 'processMessage emit HttpReply');
|
|
473
375
|
|
|
474
|
-
return that.
|
|
376
|
+
return that.proxyClient.sendMessage({
|
|
377
|
+
incomingMessageId: incomingMessageHeaders.messageId,
|
|
378
|
+
data: reply,
|
|
379
|
+
headers,
|
|
380
|
+
type: 'http-reply'
|
|
381
|
+
});
|
|
475
382
|
}
|
|
476
383
|
|
|
477
384
|
async function onError(err) {
|
|
@@ -484,7 +391,7 @@ class Sailor {
|
|
|
484
391
|
messageProcessingTime: Date.now() - timeStart
|
|
485
392
|
}, 'processMessage emit error');
|
|
486
393
|
headers.end = new Date().getTime();
|
|
487
|
-
return that.
|
|
394
|
+
return that.proxyClient.sendError(err, headers, payload, incomingMessageHeaders);
|
|
488
395
|
}
|
|
489
396
|
|
|
490
397
|
async function onRebound(err) {
|
|
@@ -495,14 +402,14 @@ class Sailor {
|
|
|
495
402
|
messagesCount: that.messagesCount,
|
|
496
403
|
messageProcessingTime: Date.now() - timeStart
|
|
497
404
|
}, 'processMessage emit rebound');
|
|
498
|
-
return that.
|
|
405
|
+
return that.proxyClient.sendRebound(err, incomingMessageHeaders, outgoingHeaders);
|
|
499
406
|
}
|
|
500
407
|
|
|
501
408
|
async function onSnapshot(data) {
|
|
502
409
|
const headers = _.clone(outgoingMessageHeaders);
|
|
503
410
|
headers.snapshotEvent = 'snapshot';
|
|
504
411
|
that.snapshot = data; //replacing `local` snapshot
|
|
505
|
-
return that.
|
|
412
|
+
return that.proxyClient.sendSnapshot(data, headers);
|
|
506
413
|
}
|
|
507
414
|
|
|
508
415
|
async function onUpdateSnapshot(data) {
|
|
@@ -514,7 +421,7 @@ class Sailor {
|
|
|
514
421
|
return log.warn('ERROR: $set is not supported any more in `updateSnapshot` event');
|
|
515
422
|
}
|
|
516
423
|
_.extend(that.snapshot, data); //updating `local` snapshot
|
|
517
|
-
return that.
|
|
424
|
+
return that.proxyClient.sendSnapshot(data, headers);
|
|
518
425
|
} else {
|
|
519
426
|
log.error('You should pass an object to the `updateSnapshot` event');
|
|
520
427
|
}
|
|
@@ -527,14 +434,14 @@ class Sailor {
|
|
|
527
434
|
|
|
528
435
|
try {
|
|
529
436
|
await that.apiClient.accounts.update(cfg._account, { keys: keys });
|
|
530
|
-
logger.debug('Successfully updated keys
|
|
437
|
+
logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Successfully updated keys');
|
|
531
438
|
} catch (error) {
|
|
532
|
-
logger.
|
|
439
|
+
logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Failed to update keys');
|
|
533
440
|
await onError(error);
|
|
534
441
|
}
|
|
535
442
|
}
|
|
536
443
|
|
|
537
|
-
|
|
444
|
+
function onEnd() {
|
|
538
445
|
if (endWasEmitted) {
|
|
539
446
|
logger.warn({
|
|
540
447
|
messagesCount: that.messagesCount,
|
|
@@ -546,11 +453,12 @@ class Sailor {
|
|
|
546
453
|
|
|
547
454
|
endWasEmitted = true;
|
|
548
455
|
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
456
|
+
that.proxyClient.finishProcessing(
|
|
457
|
+
incomingMessageHeaders,
|
|
458
|
+
taskExec.errorCount > 0
|
|
459
|
+
? MESSAGE_PROCESSING_STATUS.ERROR
|
|
460
|
+
: MESSAGE_PROCESSING_STATUS.SUCCESS
|
|
461
|
+
);
|
|
554
462
|
that.messagesCount -= 1;
|
|
555
463
|
logger.trace({
|
|
556
464
|
messagesCount: that.messagesCount,
|
|
@@ -579,43 +487,21 @@ class Sailor {
|
|
|
579
487
|
}
|
|
580
488
|
}
|
|
581
489
|
|
|
582
|
-
async processMessage(
|
|
490
|
+
async processMessage(headers, payload) {
|
|
583
491
|
//eslint-disable-next-line consistent-this
|
|
584
492
|
const self = this;
|
|
585
493
|
const settings = this.settings;
|
|
586
|
-
const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
|
|
587
494
|
|
|
588
495
|
self.messagesCount += 1;
|
|
589
496
|
|
|
590
497
|
const timeStart = Date.now();
|
|
591
498
|
|
|
592
|
-
const messageId = incomingMessageHeaders.messageId;
|
|
593
499
|
const logger = log.child({
|
|
594
|
-
threadId:
|
|
595
|
-
messageId: messageId || 'unknown',
|
|
596
|
-
parentMessageId:
|
|
597
|
-
...message.fields
|
|
500
|
+
threadId: headers.threadId || 'unknown',
|
|
501
|
+
messageId: headers.messageId || 'unknown',
|
|
502
|
+
parentMessageId: headers.parentMessageId || 'unknown'
|
|
598
503
|
});
|
|
599
504
|
|
|
600
|
-
if (messageId) {
|
|
601
|
-
const alreadyExists = messagesDB.getMessageById(messageId);
|
|
602
|
-
// Add message to DB even if it already exists
|
|
603
|
-
messagesDB.addMessage(messageId, message);
|
|
604
|
-
if (alreadyExists) {
|
|
605
|
-
logger.warn({ messageId }, 'Duplicate message detected. This'
|
|
606
|
-
+ ' delivery will be ignored; the handler that first received'
|
|
607
|
-
+ ' this message will process it as part of deduplication.');
|
|
608
|
-
// If message was in messagesDB, it means that the connection was closed
|
|
609
|
-
// and this message was redelivered. In this case, the process for original
|
|
610
|
-
// message is waiting for this message to be added to DB and then ack or
|
|
611
|
-
// nack the new message, instead of the one that was delivered by closed
|
|
612
|
-
// channel
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
} else {
|
|
616
|
-
logger.warn('Message does not have messageId');
|
|
617
|
-
}
|
|
618
|
-
|
|
619
505
|
logger.trace({ messagesCount: this.messagesCount }, 'processMessage received');
|
|
620
506
|
|
|
621
507
|
const stepData = this.stepData;
|
|
@@ -623,10 +509,10 @@ class Sailor {
|
|
|
623
509
|
log.debug('Trigger or action: %s', settings.FUNCTION);
|
|
624
510
|
const outgoingMessageId = uuid.v4();
|
|
625
511
|
const outgoingMessageHeaders = {
|
|
626
|
-
...
|
|
512
|
+
...headers,
|
|
627
513
|
...getAdditionalHeadersFromSettings(settings),
|
|
628
|
-
parentMessageId:
|
|
629
|
-
threadId:
|
|
514
|
+
parentMessageId: headers.messageId,
|
|
515
|
+
threadId: headers.threadId,
|
|
630
516
|
messageId: outgoingMessageId,
|
|
631
517
|
execId: settings.EXEC_ID,
|
|
632
518
|
taskId: settings.FLOW_ID,
|
|
@@ -644,10 +530,8 @@ class Sailor {
|
|
|
644
530
|
} catch (e) {
|
|
645
531
|
log.error(e);
|
|
646
532
|
outgoingMessageHeaders.end = new Date().getTime();
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
self.amqpConnection.reject(messageId)
|
|
650
|
-
]);
|
|
533
|
+
self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
|
|
534
|
+
self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
|
|
651
535
|
return;
|
|
652
536
|
}
|
|
653
537
|
|
|
@@ -660,12 +544,12 @@ class Sailor {
|
|
|
660
544
|
await Promise.all([
|
|
661
545
|
(async () => {
|
|
662
546
|
logger.trace('Going to check if incoming message body is lightweight.');
|
|
663
|
-
payload.body = await this.fetchMessageBody(payload, logger);
|
|
547
|
+
payload.body = await this.proxyClient.fetchMessageBody(payload, logger);
|
|
664
548
|
})(),
|
|
665
549
|
...(passthrough
|
|
666
550
|
? Object.keys(passthrough).map(async stepId => {
|
|
667
551
|
logger.trace('Going to check if passthrough for step is lightweight.', { stepId });
|
|
668
|
-
payload.passthrough[stepId].body = await this.fetchMessageBody(
|
|
552
|
+
payload.passthrough[stepId].body = await this.proxyClient.fetchMessageBody(
|
|
669
553
|
payload.passthrough[stepId],
|
|
670
554
|
logger
|
|
671
555
|
);
|
|
@@ -675,15 +559,13 @@ class Sailor {
|
|
|
675
559
|
} catch (e) {
|
|
676
560
|
logger.error(e);
|
|
677
561
|
outgoingMessageHeaders.end = new Date().getTime();
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
self.amqpConnection.reject(messageId)
|
|
681
|
-
]);
|
|
562
|
+
self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
|
|
563
|
+
self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
|
|
682
564
|
return;
|
|
683
565
|
}
|
|
684
566
|
}
|
|
685
567
|
|
|
686
|
-
await this.runExec(module, payload,
|
|
568
|
+
await this.runExec(module, payload, headers, outgoingMessageHeaders, stepData, timeStart, logger);
|
|
687
569
|
}
|
|
688
570
|
}
|
|
689
571
|
|
package/lib/settings.js
CHANGED
|
@@ -4,14 +4,19 @@ const PREFIX = 'ELASTICIO_';
|
|
|
4
4
|
|
|
5
5
|
function getOptionalEnvVars(envVars) {
|
|
6
6
|
const optional = {
|
|
7
|
-
REBOUND_INITIAL_EXPIRATION: 15000,
|
|
8
|
-
REBOUND_LIMIT: 20,
|
|
9
7
|
COMPONENT_PATH: '',
|
|
10
|
-
|
|
8
|
+
PROXY_PREFETCH_SAILOR: 1,
|
|
11
9
|
STARTUP_REQUIRED: false,
|
|
12
10
|
HOOK_SHUTDOWN: false,
|
|
13
11
|
API_REQUEST_RETRY_ATTEMPTS: 3,
|
|
14
12
|
API_REQUEST_RETRY_DELAY: 100,
|
|
13
|
+
PROXY_RECONNECT_MAX_RETRIES: Infinity,
|
|
14
|
+
PROXY_RECONNECT_INITIAL_DELAY: 1000,
|
|
15
|
+
PROXY_RECONNECT_MAX_DELAY: 30 * 1000, // 30 seconds
|
|
16
|
+
PROXY_RECONNECT_BACKOFF_MULTIPLIER: 2,
|
|
17
|
+
PROXY_RECONNECT_JITTER_FACTOR: 0.3,
|
|
18
|
+
|
|
19
|
+
// TODO: Move to proxy?
|
|
15
20
|
DATA_RATE_LIMIT: 10, // 10 data events every 100ms
|
|
16
21
|
ERROR_RATE_LIMIT: 2, // 2 errors every 100ms
|
|
17
22
|
SNAPSHOT_RATE_LIMIT: 2, // 2 Snapshots every 100ms
|
|
@@ -20,19 +25,14 @@ function getOptionalEnvVars(envVars) {
|
|
|
20
25
|
AMQP_PUBLISH_RETRY_DELAY: 100, // 100ms
|
|
21
26
|
AMQP_PUBLISH_RETRY_ATTEMPTS: Infinity,
|
|
22
27
|
AMQP_PUBLISH_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
|
|
28
|
+
// Should be defaulted to true and moved to proxy
|
|
23
29
|
AMQP_PERSISTENT_MESSAGES: false,
|
|
30
|
+
|
|
24
31
|
OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,
|
|
25
32
|
NO_SELF_PASSTRHOUGH: false,
|
|
26
33
|
PROTOCOL_VERSION: 1,
|
|
27
|
-
NO_ERROR_REPLIES: false,
|
|
28
34
|
INPUT_FORMAT: 'default',
|
|
29
|
-
OBJECT_STORAGE_URI: null,
|
|
30
|
-
OBJECT_STORAGE_TOKEN: null,
|
|
31
|
-
OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
|
|
32
35
|
EMIT_LIGHTWEIGHT_MESSAGE: false,
|
|
33
|
-
AMQP_RECONNECT_ATTEMPTS: 3,
|
|
34
|
-
AMQP_RECONNECT_TIMEOUT: 100,
|
|
35
|
-
WAIT_MESSAGES_TIMEOUT: 50
|
|
36
36
|
};
|
|
37
37
|
|
|
38
38
|
const result = {};
|
|
@@ -87,16 +87,9 @@ function getMandatoryEnvVars(envVars) {
|
|
|
87
87
|
];
|
|
88
88
|
|
|
89
89
|
const requiredForMessageProcessing = [
|
|
90
|
-
'
|
|
91
|
-
'
|
|
92
|
-
'
|
|
93
|
-
|
|
94
|
-
'DATA_ROUTING_KEY',
|
|
95
|
-
'ERROR_ROUTING_KEY',
|
|
96
|
-
'REBOUND_ROUTING_KEY',
|
|
97
|
-
'SNAPSHOT_ROUTING_KEY',
|
|
98
|
-
'MESSAGE_CRYPTO_IV',
|
|
99
|
-
'MESSAGE_CRYPTO_PASSWORD'
|
|
90
|
+
'SAILOR_PROXY_URI',
|
|
91
|
+
'MESSAGE_CRYPTO_PASSWORD',
|
|
92
|
+
'MESSAGE_CRYPTO_IV'
|
|
100
93
|
];
|
|
101
94
|
|
|
102
95
|
const envVarsList = requiredAlways.slice(0);
|
package/lib/utils.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
function getJitteredDelay(baseDelay, jitterFactor) {
|
|
2
|
+
const minJitter = baseDelay * -jitterFactor;
|
|
3
|
+
const maxJitter = baseDelay * jitterFactor;
|
|
4
|
+
const jitter = Math.random() * (maxJitter - minJitter) + minJitter;
|
|
5
|
+
return baseDelay + jitter;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
module.exports.getJitteredDelay = getJitteredDelay;
|
package/package.json
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "elasticio-sailor-nodejs",
|
|
3
3
|
"description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "3.0.0-dev2",
|
|
5
5
|
"main": "run.js",
|
|
6
6
|
"scripts": {
|
|
7
|
+
"build": "tsc",
|
|
7
8
|
"audit": "better-npm-audit audit --level high --production",
|
|
8
|
-
"lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js",
|
|
9
9
|
"pretest": "npm run lint",
|
|
10
|
+
"lint": "eslint --ext .ts .",
|
|
10
11
|
"test": "npm run test:jasmine && npm run test:mocha",
|
|
11
12
|
"test:jasmine": "NODE_ENV=test jasmine-node spec",
|
|
12
13
|
"test:mocha": "NODE_ENV=test node_modules/.bin/mocha --recursive mocha_spec",
|
|
13
|
-
"postpublish": "./postpublish.js"
|
|
14
|
+
"postpublish": "./postpublish.js",
|
|
15
|
+
"dev:local": "node run.local.js | bunyan"
|
|
14
16
|
},
|
|
15
17
|
"engines": {
|
|
16
18
|
"node": ">=12.13.0"
|
|
17
19
|
},
|
|
18
20
|
"dependencies": {
|
|
19
|
-
"@elastic.io/maester-client": "6.0.0",
|
|
20
|
-
"amqplib": "0.8.0",
|
|
21
21
|
"bunyan": "1.8.10",
|
|
22
22
|
"co": "4.6.0",
|
|
23
23
|
"debug": "3.1.0",
|
|
@@ -45,7 +45,9 @@
|
|
|
45
45
|
"request": "2.88.0",
|
|
46
46
|
"request-promise-native": "1.0.5",
|
|
47
47
|
"sinon": "9.0.2",
|
|
48
|
-
"sinon-chai": "3.5.0"
|
|
48
|
+
"sinon-chai": "3.5.0",
|
|
49
|
+
"ts-node": "10.4.0",
|
|
50
|
+
"typescript": "4.4.4"
|
|
49
51
|
},
|
|
50
52
|
"repository": "elasticio/sailor-nodejs",
|
|
51
53
|
"license": "Apache-2.0"
|
package/run.js
CHANGED
|
@@ -24,6 +24,7 @@ function prepareSandbox() {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
async function putOutToSea(settings, ipc) {
|
|
27
|
+
logger.trace({ settings }, 'putOutToSea called');
|
|
27
28
|
ipc.send('init:started');
|
|
28
29
|
const deferred = Q.defer();
|
|
29
30
|
sailorInit = deferred.promise;
|
|
@@ -31,6 +32,7 @@ async function putOutToSea(settings, ipc) {
|
|
|
31
32
|
|
|
32
33
|
//eslint-disable-next-line no-extra-boolean-cast
|
|
33
34
|
if (!!settings.HOOK_SHUTDOWN) {
|
|
35
|
+
logger.trace('Running hook shutdown');
|
|
34
36
|
disconnectRequired = false;
|
|
35
37
|
//eslint-disable-next-line no-empty-function
|
|
36
38
|
sailor.reportError = () => {
|
|
@@ -98,7 +100,7 @@ async function run(settings, ipc) {
|
|
|
98
100
|
await putOutToSea(settings, ipc);
|
|
99
101
|
logger.info('Fully initialized and waiting for messages');
|
|
100
102
|
} catch (e) {
|
|
101
|
-
if (sailor && !sailor.
|
|
103
|
+
if (sailor && !sailor.isConnected()) {
|
|
102
104
|
await sailor.reportError(e);
|
|
103
105
|
}
|
|
104
106
|
logger.criticalErrorAndExit('putOutToSea.catch', e);
|
package/run.local.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
const config = require('./config/local.json');
|
|
2
|
+
|
|
3
|
+
function setEnvVars() {
|
|
4
|
+
for (const [key, value] of Object.entries(config)) {
|
|
5
|
+
process.env[key] = value;
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
setEnvVars();
|
|
9
|
+
|
|
10
|
+
const { IPC } = require('./lib/ipc');
|
|
11
|
+
const { run } = require('./run');
|
|
12
|
+
const settings = require('./lib/settings.js');
|
|
13
|
+
const ipc = new IPC();
|
|
14
|
+
run(settings.readFrom(process.env), ipc);
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"experimentalDecorators": true,
|
|
4
|
+
"emitDecoratorMetadata": true,
|
|
5
|
+
"lib": [
|
|
6
|
+
"es2020"
|
|
7
|
+
],
|
|
8
|
+
"types": ["node", "mocha"],
|
|
9
|
+
"module": "commonjs",
|
|
10
|
+
"esModuleInterop": true,
|
|
11
|
+
"target": "es2019",
|
|
12
|
+
"noImplicitAny": false,
|
|
13
|
+
"moduleResolution": "node",
|
|
14
|
+
"resolveJsonModule": true,
|
|
15
|
+
"sourceMap": true,
|
|
16
|
+
"outDir": "dist",
|
|
17
|
+
"skipLibCheck": true,
|
|
18
|
+
"typeRoots": ["./node_modules/@types", "./src/@types"]
|
|
19
|
+
},
|
|
20
|
+
"include": [
|
|
21
|
+
"src/**/*"
|
|
22
|
+
]
|
|
23
|
+
}
|