@nomikos/module-comm 1.0.4

package/README.md

@@ -0,0 +1,53 @@
+
+# @nomikos/module-google
+
+Servicios de FCM y GCE
+
+# Sobre redundancia con varios workers pero haciendo de a una tarea
+
+One approach to handling failover in a case where you want redundant consumers but need to process messages in a specific order is to use the exclusive consumer option when setting up the bind to the queue, and to have two consumers who keep trying to bind even when they can't get the exclusive lock.
+
+The process is something like this:
+
+Consumer A starts first and binds to the queue as an exclusive consumer. Consumer A begins processing messages from the queue.
+Consumer B starts next and attempts to bind to the queue as an exclusive consumer, but is rejected because the queue already has an exclusive consumer.
+On a recurring basis, consumer B attempts to get an exclusive bind on the queue but is rejected.
+Process hosting consumer A crashes.
+Consumer B attempts to bind to the queue as an exclusive consumer, and succeeds this time. Consumer B starts processing messages from the queue.
+Consumer A is brought back online, and attempts an exclusive bind, but is rejected now.
+Consumer B continues to process messages in FIFO order.
+While this approach doesn't provide load sharing, it does provide redundancy.
+
+### Exclusive Queues
+An exclusive queue can only be used (consumed from, purged, deleted, etc) by its declaring connection. An attempt to use an exclusive queue from a different connection will result in a channel-level exception RESOURCE_LOCKED with an error message that says cannot obtain exclusive access to locked queue.
+
+Exclusive queues are deleted when their declaring connection is closed or gone (e.g. due to underlying TCP connection loss). They therefore are only suitable for client-specific transient state.
+
+It is common to make exclusive queues server-named.
+
+### Single Active Consumer
+Single active consumer allows to have only one consumer at a time consuming from a queue and to fail over to another registered consumer in case the active one is cancelled or dies. Consuming with only one consumer is useful when messages must be consumed and processed in the same order they arrive in the queue.
+
+A typical sequence of events would be the following:
+
+A queue is declared and some consumers register to it at roughly the same time.
+The very first registered consumer become the single active consumer: messages are dispatched to it and the other consumers are ignored.
+The single active consumer is cancelled for some reason or simply dies. One of the registered consumer becomes the new single active consumer and messages are now dispatched to it. In other terms, the queue fails over automatically to another consumer.
+Note that without the single active consumer feature enabled, messages would be dispatched to all consumers using round-robin.
+
+Single active consumer can be enabled when declaring a queue, with the x-single-active-consumer argument set to true, e.g. with the Java client:
+
+Channel ch = ...;
+Map<String, Object> arguments = new HashMap<String, Object>();
+arguments.put("x-single-active-consumer", true);
+ch.queueDeclare("my-queue", false, false, false, arguments);
+Compared to AMQP exclusive consumer, single active consumer puts less pressure on the application side to maintain consumption continuity. Consumers just need to be registered and failover is handled automatically, there's no need to detect the active consumer failure and to register a new consumer.
+
+The management UI and the CLI can report which consumer is the current active one on a queue where the feature is enabled.
+
+Please note the following about single active consumer:
+
+There's no guarantee on the selected active consumer, it is picked up randomly, even if consumer priorities are in use.
+Trying to register a consumer with the exclusive consume flag set to true will result in an error if single active consumer is enabled on the queue.
+Messages are always delivered to the active consumer, even if it is too busy at some point. This can happen when using manual acknowledgment and basic.qos, the consumer may be busy dealing with the maximum number of unacknowledged messages it requested with basic.qos. In this case, the other consumers are ignored and messages are enqueued.
+It is not possible to enable single active consumer with a policy. Here is the reason why. Policies in RabbitMQ are dynamic by nature, they can come and go, enabling and disabling the features they declare. Imagine suddenly disabling single active consumer on a queue: the broker would start sending messages to inactive consumers and messages would be processed in parallel, exactly the opposite of what single active consumer is trying to achieve. As the semantics of single active consumer do not play well with the dynamic nature of policies, this feature can be enabled only when declaring a queue, with queue arguments.

package/index.js

@@ -0,0 +1,11 @@
+const AmqpReceiver = require('./src/services/AmqpReceiver')
+const AmqpSender = require('./src/services/AmqpSender')
+const AmqpEncolador = require('./src/services/AmqpEncolador')
+const amqpConn = require('./src/lib/amqpServer')
+
+module.exports = {
+  AmqpReceiver,
+  AmqpSender,
+  AmqpEncolador,
+  amqpConn
+}

package/jest.config.js

@@ -0,0 +1,25 @@
+process.env.NODE_APP_PATH = __dirname
+module.exports = {
+  verbose: true,
+  silent: true,
+  // diabolico pq evita mostrar algunos logs cuando los analizo (en dev)
+  forceExit: false,
+  testEnvironment: 'node',
+  testMatch: [
+    '**/__tests__/**/*.test.js'
+  ],
+  collectCoverageFrom: [
+    'src/**/*.js'
+  ],
+  coveragePathIgnorePatterns: [
+    '/node_modules/',
+    '__tests__',
+    'src/bin'
+  ],
+  testPathIgnorePatterns: [
+    '/node_modules/'
+  ],
+  transform: {
+    '^.+\\.[t|j]sx?$': ['babel-jest', {rootMode: 'upward'}]
+  }
+}

package/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@nomikos/module-comm",
+  "version": "1.0.4",
+  "description": "Servicios externos de redis, amqp",
+  "main": "index.js",
+  "prettier": "@nomikos/prettierrc",
+  "scripts": {
+    "test": "NODE_ENV=test jest",
+    "lint": "eslint --fix src && prettier --write \"src/**/*.js\""
+  },
+  "author": "Igor Parra B.",
+  "license": "ISC",
+  "dependencies": {
+    "amqp-connection-manager": "^3.1.1",
+    "amqplib": "^0.5.5",
+    "awilix": "^4.2.6",
+    "yenv": "^2.1.1"
+  }
+}

package/src/lib/amqpServer.js

@@ -0,0 +1,57 @@
+const { env } = require('./env')
+const amqp = require('amqp-connection-manager')
+const amqpHost = env['amqp-host']
+
+let amqpConn = null
+
+function amqpClientConnect(loggerRoot) {
+  if (!loggerRoot) {
+    loggerRoot = console
+    // console.warn escribe a pm2 logs
+    loggerRoot.debug = console.warn
+    loggerRoot.fatal = console.warn
+  }
+
+  return new Promise((resolve, reject) => {
+    if (amqpConn !== null) {
+      if (amqpConn.isConnected()) {
+        loggerRoot.debug('Devolviendo conneccion amqp ya iniciada')
+        return resolve(amqpConn)
+      }
+    } else {
+      loggerRoot.debug('Iniciando nueva conneccion amqp')
+    }
+
+    // Devuelve singleton. Se conecta ahora.
+    amqpConn = amqp.connect(amqpHost)
+
+    // Emitted whenever we successfully connect to a broker.
+    amqpConn.on('connect', function (a, b) {
+      loggerRoot.debug('CLIENTE AMQP CONECTADO')
+      resolve(amqpConn)
+    })
+
+    // Emitted whenever we disconnect from a broker.
+    amqpConn.on('disconnect', function (params) {
+      const error = params.err
+      if (error.includes && error.includes('PRECONDITION_FAILED')) {
+        /**
+         * This error results in the channel that was used for the declaration being
+         * forcibly closed by RabbitMQ. If the program subsequently tries to
+         * communicate with RabbitMQ using the same channel without re-opening it then
+         * Bunny will raise a Bunny::ChannelAlreadyClosed error. In order to continue
+         * communications in the same program after such an error, a different channel
+         * would have to be used.
+         *
+         * TODO: Sacar cola de lista de colas para evitar siguientes errores en
+         * presente instancia de app.
+         * Luego habria quer reparar cola.
+         */
+        loggerRoot.fatal('Sacar cola de lista de colas !', error)
+      }
+      loggerRoot.fatal('CLIENTE AMQP DESCONECTADO, error.stack:', error.stack)
+    })
+  })
+}
+
+module.exports = amqpClientConnect

package/src/lib/env.js

@@ -0,0 +1,12 @@
+const yenv = require('yenv')
+process.env.NODE_ENV = process.env.NODE_ENV || 'local'
+
+/**
+ * We just export what `yenv()` returns.
+ * `keyblade` will make sure we don't rely on undefined values.
+ */
+exports.env = yenv('env.yaml', {
+  cwd: process.env.NODE_APP_PATH,
+  message: key => `[yenv] ${key} not found in the loaded environment`,
+  logBeforeThrow: message => console.warn({message})
+})

package/src/services/AmqpEncolador.js

@@ -0,0 +1,15 @@
+module.exports = {
+
+  send (queueName, payload, axiosEncolador, tracers, logger) {
+
+    const headers = {
+      'x-tracer-session-id': tracers.sessionId,
+      'x-tracer-user-id': tracers.userId,
+      'x-tracer-systems': tracers.systems
+    }
+
+    logger.info(`A ENCOLADOR: ${queueName}`, {payload, headers})
+
+    return axiosEncolador.post('queue', {queueName, payload, headers})
+  }
+}

package/src/services/AmqpReceiver.js

@@ -0,0 +1,65 @@
+const amqpClientConnect = require('../lib/amqpServer')
+let channelWrapper = null
+
+module.exports = {
+  async init(colas, amqpReceiverAdapter, loggerRoot = null) {
+    const amqpConn = await amqpClientConnect(loggerRoot)
+
+    if (!loggerRoot) {
+      loggerRoot = console
+      // console.warn escribe a pm2 logs
+      loggerRoot.debug = console.warn
+      loggerRoot.fatal = console.warn
+    }
+
+    await new Promise((resolve) => {
+      if (channelWrapper) {
+        loggerRoot.debug('FATAL: Solo usar un channel para recibir')
+        return
+      }
+
+      channelWrapper = amqpConn.createChannel({
+        name: 'channel-receiver',
+        setup: async (ch) => {
+          loggerRoot.debug('CREANDO CHANNEL PARA CONSUMIR')
+
+          await Promise.all(
+            Object.keys(colas).map(async (queueName) => {
+              // Configurar cola
+              const cola = colas[queueName]
+              const options = cola.options
+              const prefetch = cola.prefetch || 1
+
+              await new Promise((resolve) => {
+                ch.assertQueue(queueName, options)
+                  .then(() => {
+                    if (
+                      options.arguments &&
+                      options.arguments['x-dead-letter-routing-key']
+                    ) {
+                      // Crear su respectivo dead-letter
+                      loggerRoot.debug(`Creando dead letter: ${queueName}.dl`)
+                      ch.assertQueue(`${queueName}.dl`, { durable: true })
+                    }
+                  })
+                  .then(() => {
+                    ch.prefetch(prefetch)
+                    amqpReceiverAdapter(ch, queueName, cola)
+                  })
+                  .then(() => {
+                    loggerRoot.debug(`COLA CONSUMO ${queueName}`)
+                    resolve()
+                  })
+              })
+            })
+          ) // end Promise.all
+        }
+      })
+
+      channelWrapper.waitForConnect().then(function () {
+        loggerRoot.debug('CHANEL PARA CONSUMIR LISTO')
+        resolve()
+      })
+    })
+  }
+}

package/src/services/AmqpSender.js

@@ -0,0 +1,67 @@
+const { env } = require('../lib/env')
+const amqpClientConnect = require('../lib/amqpServer')
+let channelWrapper = null
+
+module.exports = {
+  sendToQueue: async (queueName, { payload, headers }, logger = null) => {
+    let data = {}
+    if (headers) {
+      data = { payload, headers }
+    } else {
+      data = { payload }
+    }
+
+    await channelWrapper
+      .sendToQueue(queueName, data, { persistent: true })
+      .then(() => {
+        if (logger) {
+          logger.info(`SENT OK A COLA AMQP: ${queueName}`, data)
+        }
+      })
+      .catch((e) => {
+        throw e
+      })
+  },
+
+  async init(colas, loggerRoot = null) {
+    const amqpConn = await amqpClientConnect(loggerRoot)
+    if (loggerRoot) {
+      loggerRoot.debug(
+        `Amqp write on ${env['amqp-host']} in ${env.NODE_ENV} mode.`
+      )
+    }
+    await new Promise((resolve) => {
+      if (channelWrapper) {
+        loggerRoot.debug('WARNING: Solo usar un channel para enviar')
+        resolve()
+        return
+      }
+
+      channelWrapper = amqpConn.createChannel({
+        name: 'channel-sender',
+        json: true,
+        setup: async (channel) => {
+          loggerRoot.debug('CREANDO CHANNEL PARA ENVIAR')
+
+          await Promise.all(
+            Object.keys(colas).map(async (queueName) => {
+              const q = colas[queueName]
+              const options = q.options
+              await new Promise((resolve) => {
+                channel.assertQueue(queueName, options).then(() => {
+                  loggerRoot.debug(`COLA ENVIO ${queueName}`)
+                  resolve()
+                })
+              })
+            })
+          ) // end Promise.all
+        }
+      })
+
+      channelWrapper.waitForConnect().then(function () {
+        loggerRoot.debug('CHANEL PARA ENVIAR LISTO')
+        resolve()
+      })
+    })
+  }
+}

package/src/services/__tests__/AmqpReceiver.test.js

@@ -0,0 +1,165 @@
+/* eslint-disable promise/always-return */
+import AmqpSender from '../AmqpSender'
+const { closeAmqpConn } = require('../../lib/amqpServer')
+const AmqpReceiver = require('../AmqpReceiver')
+
+const axios = require('axios')
+const MockAdapter = require('axios-mock-adapter')
+
+/**
+ * Aplicando mock a instancia common de axios!
+ * Se veran afectados todas las llamadas. Esto obliga a simular mocks en todos
+ * los tests.
+ */
+const mockAxios = new MockAdapter(axios)
+
+const EventEmitter = require('events').EventEmitter
+const controlEmitter = new EventEmitter()
+
+const uuidv1 = require('uuid/v1')
+
+const colas = {
+  'task.test.q': {
+    options: {
+      durable: true
+    },
+    forwardUrls: '/final-endpoint',
+    prefetch: 1
+  }
+}
+
+const bodyMessageToTest = {
+  queueName: 'task.test.q',
+  routingKey: 'task.test.q',
+  headers: {
+    'x-tracer-user-id': 3,
+    'x-tracer-session-id': uuidv1()
+  },
+  payload: {
+    foo: 'bar' // Aqui no testeamos contenido
+  }
+}
+
+/**
+ * Ver si hace ack, nack, requeue (esto siempre consume el mensaje,
+ * luego no se puede analizar que pasó con el mensaje, por que desaparece)
+ *
+ * Si hace requeue y es nack ver si quedó en dlx
+ * Si no hace requeue y es nack ver que no quedó en dlx
+ * Para analizar dlx hay que consumir ultimo mensaje en cola dead.letter.queue
+ *
+ * Es delicado testear colas en prod, ya que pueden causar loops infinitos
+ * al hacer caer la conn. Como tenemos autolevantamiento de conn se pueden
+ * causar estos loops infinitos.
+ */
+
+describe('Api de Desencolación', () => {
+  let receiver, sender
+
+  beforeAll(async () => {
+    // const desencolador = new Desencolador({axios})
+    // receiver = await new AmqpReceiver({desencolador})
+    // sender = await new AmqpSender()
+  })
+
+  afterAll(() => {
+    // Siempre cerrar, para evitar cuelgue de jest
+    closeAmqpConn()
+  })
+
+  it.only('Desencolar mensajes desde colas', async () => {
+    expect.assertions(1)
+
+    mockAxios.onPost('/final-endpoint').reply(200)
+
+    // Set promesa antes de todo para que escuche des de ya el evento "consuming"
+    const result = new Promise(function (resolve, reject) {
+      controlEmitter.on('consuming', ({ op }) => {
+        if (op === 'ack') {
+          return resolve(true)
+        }
+        reject(new Error())
+      })
+    })
+
+    const onMessage = (queuePayload) => {
+      const dataEnCola = JSON.parse(queuePayload.content.toString())
+      console.warn({ dataEnCola }) // Contiene {payload, headers}
+
+      const op = 'ack'
+      controlEmitter.emit('consuming', { op, queuePayload })
+    }
+
+    const queueName = 'task.test.q'
+
+    function receiverAdapter(ch, queueName, cola) {
+      console.warn({ cola })
+      return ch.consume(queueName, onMessage, {
+        noAck: false
+      })
+    }
+
+    await AmqpSender.init(colas)
+    await AmqpReceiver.init(colas, receiverAdapter)
+
+    // Enviamos mensaje a cola
+    await AmqpSender.sendToQueue(queueName, { testendo: 'esto' })
+
+    await result.then((r) => {
+      console.warn('Desencolado OK')
+      expect(r).toBe(true)
+    })
+  })
+
+  it('Mensajes rechazado en api remota (simulando status 400), ver si quedó en dlx.', async () => {
+    // expect.assertions(1)
+
+    mockAxios.onPost('/final-endpoint').reply(400)
+    const desencolador = new Desencolador({ axios })
+    receiver = await new AmqpReceiver({ desencolador })
+
+    // Set promesa antes de todo para que escuche desde ya
+    // evento consuming
+    const result = new Promise(function (resolve, reject) {
+      controlEmitter.on('consuming', ({ op }) => {
+        if (op === 'nack') {
+          return resolve(true)
+        }
+        reject(new Error())
+      })
+    })
+
+    await receiver.loadColas(colas)
+
+    // Esto es lo estamos testeando
+    await receiver.initListenig(controlEmitter)
+
+    const body = bodyMessageToTest
+
+    // Enviamos mensaje a cola
+    const config = new QueueConfig(body)
+    await sender.execute(config)
+
+    await result.then((r) => {
+      console.warn('Mensajes rechazado en api remota, ver si quedó en dlx.')
+      expect(r).toBe(true)
+    })
+  })
+
+  /**
+   * Si limpio la cola antes de testear puedo analizar siempre el ultimo mensaje,
+   * si no tengo que leer hasta encontrar un token dado
+   */
+})
+
+function setup() {
+  const axiosOK = {
+    post: jest.fn(() => {
+      return {
+        status: 200
+      }
+    })
+  }
+
+  return { axiosOK }
+}