elasticio-sailor-nodejs 3.0.0-dev6 → 3.0.0-dev7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,37 @@
1
+ // Simple map to store messages by their IDs
2
+ // This is useful when connection is re-established and we need to get the same
3
+ // message again, but now from the new connection
4
+ const EventEmitter = require('events');
5
+
6
+ const messagesDB = (() => {
7
+ const messagesById = new Map();
8
+ const emitter = new EventEmitter();
9
+
10
+ return {
11
+ getMessageById: function getMessageById(id) {
12
+ return messagesById.get(id);
13
+ },
14
+ addMessage: function addMessage(id, message) {
15
+ const existingMessage = messagesById.get(id);
16
+ messagesById.set(id, message);
17
+ if (existingMessage) {
18
+ emitter.emit('message-updated', id, message);
19
+ }
20
+ },
21
+ deleteMessage: function deleteMessage(id) {
22
+ messagesById.delete(id);
23
+ },
24
+ on: function on(event, listener) {
25
+ emitter.on(event, listener);
26
+ },
27
+ off: function off(event, listener) {
28
+ emitter.off(event, listener);
29
+ },
30
+ __reset__: function reset() {
31
+ messagesById.clear();
32
+ emitter.removeAllListeners();
33
+ }
34
+ };
35
+ })();
36
+
37
+ module.exports = messagesDB;
package/lib/sailor.js CHANGED
@@ -1,22 +1,28 @@
1
1
  const uuid = require('uuid');
2
2
  const ComponentReader = require('./component_reader.js').ComponentReader;
3
- const { ProxyClient, MESSAGE_PROCESSING_STATUS } = require('./proxy-client.js');
3
+ const amqp = require('./amqp.js');
4
4
  const TaskExec = require('./executor.js').TaskExec;
5
5
  const log = require('./logging.js');
6
6
  const _ = require('lodash');
7
7
  const hooksData = require('./hooksData');
8
+ const Encryptor = require('../lib/encryptor');
8
9
  const RestApiClient = require('elasticio-rest-node');
9
10
  const assert = require('assert');
10
11
  const co = require('co');
12
+ const pThrottle = require('p-throttle');
13
+ const { ObjectStorage } = require('@elastic.io/maester-client');
14
+ const { Readable } = require('stream');
15
+ const messagesDB = require('./messagesDB.js');
11
16
 
17
+ const AMQP_HEADER_META_PREFIX = 'x-eio-meta-';
12
18
  const OBJECT_ID_HEADER = 'x-ipaas-object-storage-id';
13
19
 
14
- function convertSettingsToSnakeCase(settings) {
15
- return _.mapKeys(settings, (value, key) => _.snakeCase(key));
20
+ function convertSettingsToCamelCase(settings) {
21
+ return _.mapKeys(settings, (value, key) => _.camelCase(key));
16
22
  }
17
23
 
18
24
  function getAdditionalHeadersFromSettings(settings) {
19
- return convertSettingsToSnakeCase(settings.additionalVars);
25
+ return convertSettingsToCamelCase(settings.additionalVars);
20
26
  }
21
27
 
22
28
  class Sailor {
@@ -26,35 +32,50 @@ class Sailor {
26
32
  constructor(settings) {
27
33
  this.settings = settings;
28
34
  this.messagesCount = 0;
29
- this.proxyClient = new ProxyClient(settings);
35
+ this.amqpConnection = new amqp.Amqp(settings);
30
36
  this.componentReader = new ComponentReader();
31
37
  this.snapshot = {};
32
38
  this.stepData = {};
33
39
  this.shutdownCallback = null;
34
- // TODO move endpoint to proxy
35
40
  //eslint-disable-next-line new-cap
36
41
  this.apiClient = RestApiClient(
37
42
  settings.API_USERNAME,
38
- // TODO find a way to make username and key consistent (without running api tests and looking up
39
- // correct values in MongoDB)
40
43
  settings.API_KEY,
41
44
  {
42
45
  retryCount: settings.API_REQUEST_RETRY_ATTEMPTS,
43
46
  retryDelay: settings.API_REQUEST_RETRY_DELAY
44
- }
47
+ });
48
+
49
+ const objectStorage = new ObjectStorage({
50
+ uri: settings.OBJECT_STORAGE_URI,
51
+ jwtSecret: settings.OBJECT_STORAGE_TOKEN
52
+ });
53
+
54
+ const encryptor = new Encryptor(settings.MESSAGE_CRYPTO_PASSWORD, settings.MESSAGE_CRYPTO_IV);
55
+ this.objectStorage = objectStorage.use(
56
+ () => encryptor.createCipher(),
57
+ () => encryptor.createDecipher()
45
58
  );
46
- }
47
59
 
48
- async connect() {
49
- return this.proxyClient.connect();
60
+ this.throttles = {
61
+ // 100 Messages per Second
62
+ data: pThrottle(() => Promise.resolve(true),
63
+ settings.DATA_RATE_LIMIT,
64
+ settings.RATE_INTERVAL),
65
+ error: pThrottle(() => Promise.resolve(true),
66
+ settings.ERROR_RATE_LIMIT,
67
+ settings.RATE_INTERVAL),
68
+ snapshot: pThrottle(() => Promise.resolve(true),
69
+ settings.SNAPSHOT_RATE_LIMIT,
70
+ settings.RATE_INTERVAL)
71
+ };
50
72
  }
51
73
 
52
- async isConnected() {
53
- return this.proxyClient.isConnected();
74
+ async connect() {
75
+ return this.amqpConnection.connect(this.settings.AMQP_URI);
54
76
  }
55
77
 
56
78
  async prepare() {
57
- log.trace('prepare sailor');
58
79
  const {
59
80
  settings: {
60
81
  COMPONENT_PATH: compPath,
@@ -80,12 +101,11 @@ class Sailor {
80
101
  }
81
102
 
82
103
  async disconnect() {
83
- // TODO: delete if not needed (currently used only in old tests)
84
104
  log.debug('Disconnecting, %s messages in processing', this.messagesCount);
85
- return this.proxyClient.disconnect();
105
+ return this.amqpConnection.disconnect();
86
106
  }
87
107
 
88
- reportError(err) {
108
+ async reportError(err) {
89
109
  const headers = Object.assign({}, getAdditionalHeadersFromSettings(this.settings), {
90
110
  execId: this.settings.EXEC_ID,
91
111
  taskId: this.settings.FLOW_ID,
@@ -96,7 +116,7 @@ class Sailor {
96
116
  compId: this.settings.COMP_ID,
97
117
  function: this.settings.FUNCTION
98
118
  });
99
- return this.proxyClient.sendError(err, headers);
119
+ return this.amqpConnection.sendError(err, headers);
100
120
  }
101
121
 
102
122
  startup() {
@@ -164,16 +184,17 @@ class Sailor {
164
184
  }
165
185
 
166
186
  run() {
187
+ const incomingQueue = this.settings.LISTEN_MESSAGES_ON;
167
188
  const handler = this.processMessageAndMaybeShutdownCallback.bind(this);
168
- log.debug('Start listening for messages');
169
- return this.proxyClient.listenForMessages(handler);
189
+ log.debug('Start listening for messages on %s', incomingQueue);
190
+ return this.amqpConnection.listenQueue(incomingQueue, handler);
170
191
  }
171
192
 
172
- async processMessageAndMaybeShutdownCallback(headers, body) {
193
+ async processMessageAndMaybeShutdownCallback(payload, message) {
173
194
  try {
174
- return await this.processMessage(headers, body);
195
+ return await this.processMessage(payload, message);
175
196
  } catch (e) {
176
- log.error(e, 'Something very bad happened during message processing');
197
+ log.error('Something very bad happened during message processing');
177
198
  } finally {
178
199
  if (this.shutdownCallback) {
179
200
  if (this.messagesCount === 0) {
@@ -195,7 +216,7 @@ class Sailor {
195
216
  return new Promise(resolve => this.shutdownCallback = resolve);
196
217
  }
197
218
 
198
- await this.proxyClient.disconnect();
219
+ await this.amqpConnection.stopConsume();
199
220
  if (this.messagesCount === 0) {
200
221
  // there is no unfinished processMessage invocation, let's just resolve scheduleShutdown now
201
222
  log.debug('scheduleShutdown – about to shutdown immediately');
@@ -207,23 +228,103 @@ class Sailor {
207
228
  return new Promise(resolve => this.shutdownCallback = resolve);
208
229
  }
209
230
 
210
- async runExec(module, payload, incomingMessageHeaders, outgoingMessageHeaders, stepData, timeStart, logger) {
211
- log.debug({ incomingMessageHeaders }, 'runExec started');
231
+
232
+ readIncomingMessageHeaders(message) {
233
+ const { headers } = message.properties;
234
+
235
+ // Get meta headers
236
+ const metaHeaderNames = Object.keys(headers)
237
+ .filter(key => key.toLowerCase().startsWith(AMQP_HEADER_META_PREFIX));
238
+
239
+ const metaHeaders = _.pick(headers, metaHeaderNames);
240
+ const metaHeadersLowerCased = _.mapKeys(metaHeaders, (value, key) => key.toLowerCase());
241
+
242
+ const result = {
243
+ stepId: headers.stepId, // the only use is passthrough mechanism
244
+ ...metaHeadersLowerCased,
245
+ threadId: headers.threadId || metaHeadersLowerCased['x-eio-meta-trace-id'],
246
+ messageId: headers.messageId,
247
+ parentMessageId: headers.parentMessageId
248
+ };
249
+ if (!result.threadId) {
250
+ const threadId = uuid.v4();
251
+ log.debug({ threadId }, 'Initiate new thread as it is not started ATM');
252
+ result.threadId = threadId;
253
+ }
254
+ if (headers.reply_to) {
255
+ result.reply_to = headers.reply_to;
256
+ }
257
+ return result;
258
+ }
259
+
260
+ async fetchMessageBody(message, logger) {
261
+ const { body, headers } = message;
262
+
263
+ logger.info('Checking if incoming messages is lightweight...');
264
+
265
+ if (!headers) {
266
+ logger.info('Empty headers so not lightweight.');
267
+ return body;
268
+ }
269
+
270
+ const { [OBJECT_ID_HEADER]: objectId } = headers;
271
+
272
+ if (!objectId) {
273
+ logger.trace('No object id header so not lightweight.');
274
+ return body;
275
+ }
276
+
277
+ logger.info('Object id header found, message is lightweight.', { objectId });
278
+
279
+ let object;
280
+
281
+ logger.info('Going to fetch message body.', { objectId });
282
+
283
+ try {
284
+ object = await this.objectStorage.getOne(
285
+ objectId,
286
+ { jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
287
+ );
288
+ } catch (e) {
289
+ log.error(e);
290
+ throw new Error(`Failed to get message body with id=${objectId}`);
291
+ }
292
+
293
+ logger.info('Successfully obtained message body.', { objectId });
294
+ logger.trace('Message body object received');
295
+
296
+ return object.data;
297
+ }
298
+
299
+ uploadMessageBody(bodyBuf) {
300
+ const stream = () => Readable.from(bodyBuf);
301
+ return this.objectStorage.add(
302
+ stream,
303
+ { jwtPayloadOrToken: this.settings.OBJECT_STORAGE_TOKEN }
304
+ );
305
+ }
306
+
307
+ async runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger) {
212
308
  const origPassthrough = _.cloneDeep(payload.passthrough) || {};
309
+ const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
310
+ const messageId = incomingMessageHeaders.messageId;
213
311
  const settings = this.settings;
214
312
  const cfg = _.cloneDeep(stepData.config) || {};
215
313
  const snapshot = _.cloneDeep(this.snapshot);
314
+ const { deliveryTag } = message.fields;
216
315
 
217
316
  const that = this;
218
317
 
219
318
  await new Promise(resolve => {
220
319
  let endWasEmitted;
221
320
 
321
+
222
322
  const taskExec = new TaskExec({
223
323
  loggerOptions: _.pick(incomingMessageHeaders, ['threadId', 'messageId', 'parentMessageId']),
224
324
  variables: stepData.variables,
225
325
  services: {
226
326
  apiClient: this.apiClient,
327
+ amqp: this.amqpConnection,
227
328
  config: this.settings
228
329
  }
229
330
  });
@@ -290,9 +391,9 @@ class Sailor {
290
391
  let passthroughIds;
291
392
  try {
292
393
  [bodyId, ...passthroughIds] = await Promise.all([
293
- that.proxyClient.uploadMessageBody(bodyBuf),
394
+ that.uploadMessageBody(bodyBuf),
294
395
  ...passthroughBufs.map(async ({ stepId, body, id }) => {
295
- const bodyId = id || await that.proxyClient.uploadMessageBody(body);
396
+ const bodyId = id || await that.uploadMessageBody(body);
296
397
  return { stepId, bodyId };
297
398
  })
298
399
  ]);
@@ -301,7 +402,7 @@ class Sailor {
301
402
  return onError(new Error('Lightweight message/passthrough body upload error'));
302
403
  }
303
404
 
304
- logger.info({ id: bodyId }, 'Message body uploaded');
405
+ logger.info('Message body uploaded', { id: bodyId });
305
406
  const { headers } = data;
306
407
  data.body = {};
307
408
  data.headers = {
@@ -338,7 +439,7 @@ class Sailor {
338
439
  logger.trace('Body is not empty.', { stepId });
339
440
  return;
340
441
  }
341
- data.passthrough[stepId].body = await that.proxyClient.fetchMessageBody(
442
+ data.passthrough[stepId].body = await that.fetchMessageBody(
342
443
  passthrough[stepId],
343
444
  logger
344
445
  );
@@ -357,12 +458,7 @@ class Sailor {
357
458
  log.trace('Going to send outgoing message');
358
459
 
359
460
  try {
360
- await that.proxyClient.sendMessage({
361
- incomingMessageId: incomingMessageHeaders.messageId,
362
- data,
363
- headers,
364
- type: 'data'
365
- });
461
+ await that.amqpConnection.sendData(data, headers, that.throttles.data);
366
462
  log.trace('Outgoing message sent');
367
463
  } catch (err) {
368
464
  return onError(err);
@@ -375,12 +471,7 @@ class Sailor {
375
471
  messageProcessingTime: Date.now() - timeStart
376
472
  }, 'processMessage emit HttpReply');
377
473
 
378
- return that.proxyClient.sendMessage({
379
- incomingMessageId: incomingMessageHeaders.messageId,
380
- data: reply,
381
- headers,
382
- type: 'http-reply'
383
- });
474
+ return that.amqpConnection.sendHttpReply(reply, headers);
384
475
  }
385
476
 
386
477
  async function onError(err) {
@@ -393,7 +484,7 @@ class Sailor {
393
484
  messageProcessingTime: Date.now() - timeStart
394
485
  }, 'processMessage emit error');
395
486
  headers.end = new Date().getTime();
396
- return that.proxyClient.sendError(err, headers, payload, incomingMessageHeaders);
487
+ return that.amqpConnection.sendError(err, headers, message, that.throttles.error);
397
488
  }
398
489
 
399
490
  async function onRebound(err) {
@@ -404,14 +495,14 @@ class Sailor {
404
495
  messagesCount: that.messagesCount,
405
496
  messageProcessingTime: Date.now() - timeStart
406
497
  }, 'processMessage emit rebound');
407
- return that.proxyClient.sendRebound(err, incomingMessageHeaders, outgoingHeaders);
498
+ return that.amqpConnection.sendRebound(err, message, outgoingHeaders);
408
499
  }
409
500
 
410
501
  async function onSnapshot(data) {
411
502
  const headers = _.clone(outgoingMessageHeaders);
412
503
  headers.snapshotEvent = 'snapshot';
413
504
  that.snapshot = data; //replacing `local` snapshot
414
- return that.proxyClient.sendSnapshot(data, headers);
505
+ return that.amqpConnection.sendSnapshot(data, headers, that.throttles.snapshot);
415
506
  }
416
507
 
417
508
  async function onUpdateSnapshot(data) {
@@ -423,7 +514,7 @@ class Sailor {
423
514
  return log.warn('ERROR: $set is not supported any more in `updateSnapshot` event');
424
515
  }
425
516
  _.extend(that.snapshot, data); //updating `local` snapshot
426
- return that.proxyClient.sendSnapshot(data, headers);
517
+ return that.amqpConnection.sendSnapshot(data, headers);
427
518
  } else {
428
519
  log.error('You should pass an object to the `updateSnapshot` event');
429
520
  }
@@ -436,14 +527,14 @@ class Sailor {
436
527
 
437
528
  try {
438
529
  await that.apiClient.accounts.update(cfg._account, { keys: keys });
439
- logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Successfully updated keys');
530
+ logger.debug('Successfully updated keys #%s', deliveryTag);
440
531
  } catch (error) {
441
- logger.debug({ messageId: incomingMessageHeaders.messageId }, 'Failed to update keys');
532
+ logger.error('Failed to updated keys #%s', deliveryTag);
442
533
  await onError(error);
443
534
  }
444
535
  }
445
536
 
446
- function onEnd() {
537
+ async function onEnd() {
447
538
  if (endWasEmitted) {
448
539
  logger.warn({
449
540
  messagesCount: that.messagesCount,
@@ -455,12 +546,11 @@ class Sailor {
455
546
 
456
547
  endWasEmitted = true;
457
548
 
458
- that.proxyClient.finishProcessing(
459
- incomingMessageHeaders,
460
- taskExec.errorCount > 0
461
- ? MESSAGE_PROCESSING_STATUS.ERROR
462
- : MESSAGE_PROCESSING_STATUS.SUCCESS
463
- );
549
+ if (taskExec.errorCount > 0) {
550
+ await that.amqpConnection.reject(messageId);
551
+ } else {
552
+ await that.amqpConnection.ack(messageId);
553
+ }
464
554
  that.messagesCount -= 1;
465
555
  logger.trace({
466
556
  messagesCount: that.messagesCount,
@@ -489,21 +579,43 @@ class Sailor {
489
579
  }
490
580
  }
491
581
 
492
- async processMessage(headers, payload) {
582
+ async processMessage(payload, message) {
493
583
  //eslint-disable-next-line consistent-this
494
584
  const self = this;
495
585
  const settings = this.settings;
586
+ const incomingMessageHeaders = this.readIncomingMessageHeaders(message);
496
587
 
497
588
  self.messagesCount += 1;
498
589
 
499
590
  const timeStart = Date.now();
500
591
 
592
+ const messageId = incomingMessageHeaders.messageId;
501
593
  const logger = log.child({
502
- threadId: headers.threadId || 'unknown',
503
- messageId: headers.messageId || 'unknown',
504
- parentMessageId: headers.parentMessageId || 'unknown'
594
+ threadId: incomingMessageHeaders.threadId || 'unknown',
595
+ messageId: messageId || 'unknown',
596
+ parentMessageId: incomingMessageHeaders.parentMessageId || 'unknown',
597
+ ...message.fields
505
598
  });
506
599
 
600
+ if (messageId) {
601
+ const alreadyExists = messagesDB.getMessageById(messageId);
602
+ // Add message to DB even if it already exists
603
+ messagesDB.addMessage(messageId, message);
604
+ if (alreadyExists) {
605
+ logger.warn({ messageId }, 'Duplicate message detected. This'
606
+ + ' delivery will be ignored; the handler that first received'
607
+ + ' this message will process it as part of deduplication.');
608
+ // If message was in messagesDB, it means that the connection was closed
609
+ // and this message was redelivered. In this case, the process for original
610
+ // message is waiting for this message to be added to DB and then ack or
611
+ // nack the new message, instead of the one that was delivered by closed
612
+ // channel
613
+ return;
614
+ }
615
+ } else {
616
+ logger.warn('Message does not have messageId');
617
+ }
618
+
507
619
  logger.trace({ messagesCount: this.messagesCount }, 'processMessage received');
508
620
 
509
621
  const stepData = this.stepData;
@@ -511,10 +623,10 @@ class Sailor {
511
623
  log.debug('Trigger or action: %s', settings.FUNCTION);
512
624
  const outgoingMessageId = uuid.v4();
513
625
  const outgoingMessageHeaders = {
514
- ...headers,
626
+ ...incomingMessageHeaders,
515
627
  ...getAdditionalHeadersFromSettings(settings),
516
- parentMessageId: headers.messageId,
517
- threadId: headers.threadId,
628
+ parentMessageId: incomingMessageHeaders.messageId,
629
+ threadId: incomingMessageHeaders.threadId,
518
630
  messageId: outgoingMessageId,
519
631
  execId: settings.EXEC_ID,
520
632
  taskId: settings.FLOW_ID,
@@ -532,8 +644,10 @@ class Sailor {
532
644
  } catch (e) {
533
645
  log.error(e);
534
646
  outgoingMessageHeaders.end = new Date().getTime();
535
- self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
536
- self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
647
+ await Promise.all([
648
+ self.amqpConnection.sendError(e, outgoingMessageHeaders, message),
649
+ self.amqpConnection.reject(messageId)
650
+ ]);
537
651
  return;
538
652
  }
539
653
 
@@ -546,12 +660,12 @@ class Sailor {
546
660
  await Promise.all([
547
661
  (async () => {
548
662
  logger.trace('Going to check if incoming message body is lightweight.');
549
- payload.body = await this.proxyClient.fetchMessageBody(payload, logger);
663
+ payload.body = await this.fetchMessageBody(payload, logger);
550
664
  })(),
551
665
  ...(passthrough
552
666
  ? Object.keys(passthrough).map(async stepId => {
553
667
  logger.trace('Going to check if passthrough for step is lightweight.', { stepId });
554
- payload.passthrough[stepId].body = await this.proxyClient.fetchMessageBody(
668
+ payload.passthrough[stepId].body = await this.fetchMessageBody(
555
669
  payload.passthrough[stepId],
556
670
  logger
557
671
  );
@@ -561,13 +675,15 @@ class Sailor {
561
675
  } catch (e) {
562
676
  logger.error(e);
563
677
  outgoingMessageHeaders.end = new Date().getTime();
564
- self.proxyClient.sendError(e, outgoingMessageHeaders, payload, headers);
565
- self.proxyClient.finishProcessing(headers, MESSAGE_PROCESSING_STATUS.ERROR);
678
+ await Promise.all([
679
+ self.amqpConnection.sendError(e, outgoingMessageHeaders, message),
680
+ self.amqpConnection.reject(messageId)
681
+ ]);
566
682
  return;
567
683
  }
568
684
  }
569
685
 
570
- await this.runExec(module, payload, headers, outgoingMessageHeaders, stepData, timeStart, logger);
686
+ await this.runExec(module, payload, message, outgoingMessageHeaders, stepData, timeStart, logger);
571
687
  }
572
688
  }
573
689
 
package/lib/settings.js CHANGED
@@ -4,22 +4,14 @@ const PREFIX = 'ELASTICIO_';
4
4
 
5
5
  function getOptionalEnvVars(envVars) {
6
6
  const optional = {
7
+ REBOUND_INITIAL_EXPIRATION: 15000,
8
+ REBOUND_LIMIT: 20,
7
9
  COMPONENT_PATH: '',
8
- PROXY_PREFETCH_SAILOR: 1,
10
+ RABBITMQ_PREFETCH_SAILOR: 1,
9
11
  STARTUP_REQUIRED: false,
10
12
  HOOK_SHUTDOWN: false,
11
13
  API_REQUEST_RETRY_ATTEMPTS: 3,
12
14
  API_REQUEST_RETRY_DELAY: 100,
13
- PROXY_RECONNECT_MAX_RETRIES: Infinity,
14
- PROXY_RECONNECT_INITIAL_DELAY: 1000,
15
- PROXY_RECONNECT_MAX_DELAY: 30 * 1000, // 30 seconds
16
- PROXY_RECONNECT_BACKOFF_MULTIPLIER: 2,
17
- PROXY_RECONNECT_JITTER_FACTOR: 0.3,
18
- PROXY_OBJECT_REQUEST_RETRY_ATTEMPTS: Infinity,
19
- PROXY_OBJECT_REQUEST_RETRY_DELAY: 100,
20
- PROXY_OBJECT_REQUEST_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
21
-
22
- // TODO: Move to proxy?
23
15
  DATA_RATE_LIMIT: 10, // 10 data events every 100ms
24
16
  ERROR_RATE_LIMIT: 2, // 2 errors every 100ms
25
17
  SNAPSHOT_RATE_LIMIT: 2, // 2 Snapshots every 100ms
@@ -28,15 +20,19 @@ function getOptionalEnvVars(envVars) {
28
20
  AMQP_PUBLISH_RETRY_DELAY: 100, // 100ms
29
21
  AMQP_PUBLISH_RETRY_ATTEMPTS: Infinity,
30
22
  AMQP_PUBLISH_MAX_RETRY_DELAY: 5 * 60 * 1000, // 5 mins
31
- // Should be defaulted to true and moved to proxy
32
23
  AMQP_PERSISTENT_MESSAGES: false,
33
-
34
- OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
35
24
  OUTGOING_MESSAGE_SIZE_LIMIT: 10485760,
36
25
  NO_SELF_PASSTRHOUGH: false,
37
26
  PROTOCOL_VERSION: 1,
27
+ NO_ERROR_REPLIES: false,
38
28
  INPUT_FORMAT: 'default',
29
+ OBJECT_STORAGE_URI: null,
30
+ OBJECT_STORAGE_TOKEN: null,
31
+ OBJECT_STORAGE_SIZE_THRESHOLD: 1048576,
39
32
  EMIT_LIGHTWEIGHT_MESSAGE: false,
33
+ AMQP_RECONNECT_ATTEMPTS: 3,
34
+ AMQP_RECONNECT_TIMEOUT: 100,
35
+ WAIT_MESSAGES_TIMEOUT: 50
40
36
  };
41
37
 
42
38
  const result = {};
@@ -91,9 +87,16 @@ function getMandatoryEnvVars(envVars) {
91
87
  ];
92
88
 
93
89
  const requiredForMessageProcessing = [
94
- 'SAILOR_PROXY_URI',
95
- 'MESSAGE_CRYPTO_PASSWORD',
96
- 'MESSAGE_CRYPTO_IV'
90
+ 'AMQP_URI',
91
+ 'LISTEN_MESSAGES_ON',
92
+ 'PUBLISH_MESSAGES_TO',
93
+
94
+ 'DATA_ROUTING_KEY',
95
+ 'ERROR_ROUTING_KEY',
96
+ 'REBOUND_ROUTING_KEY',
97
+ 'SNAPSHOT_ROUTING_KEY',
98
+ 'MESSAGE_CRYPTO_IV',
99
+ 'MESSAGE_CRYPTO_PASSWORD'
97
100
  ];
98
101
 
99
102
  const envVarsList = requiredAlways.slice(0);
package/package.json CHANGED
@@ -1,23 +1,23 @@
1
1
  {
2
2
  "name": "elasticio-sailor-nodejs",
3
3
  "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors",
4
- "version": "3.0.0-dev6",
4
+ "version": "3.0.0-dev7",
5
5
  "main": "run.js",
6
6
  "scripts": {
7
- "build": "tsc",
8
7
  "audit": "better-npm-audit audit --level high --production",
8
+ "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js",
9
9
  "pretest": "npm run lint",
10
- "lint": "eslint --ext .ts .",
11
10
  "test": "npm run test:jasmine && npm run test:mocha",
12
11
  "test:jasmine": "NODE_ENV=test jasmine-node spec",
13
12
  "test:mocha": "NODE_ENV=test node_modules/.bin/mocha --recursive mocha_spec",
14
- "postpublish": "./postpublish.js",
15
- "dev:local": "node run.local.js | bunyan"
13
+ "postpublish": "./postpublish.js"
16
14
  },
17
15
  "engines": {
18
16
  "node": ">=12.13.0"
19
17
  },
20
18
  "dependencies": {
19
+ "@elastic.io/maester-client": "6.0.0",
20
+ "amqplib": "0.8.0",
21
21
  "bunyan": "1.8.10",
22
22
  "co": "4.6.0",
23
23
  "debug": "3.1.0",
@@ -45,9 +45,7 @@
45
45
  "request": "2.88.0",
46
46
  "request-promise-native": "1.0.5",
47
47
  "sinon": "9.0.2",
48
- "sinon-chai": "3.5.0",
49
- "ts-node": "10.4.0",
50
- "typescript": "4.4.4"
48
+ "sinon-chai": "3.5.0"
51
49
  },
52
50
  "repository": "elasticio/sailor-nodejs",
53
51
  "license": "Apache-2.0"