@emartech/program-executor 3.11.1 → 3.12.1

package/src/index-pubsub-e2e.test.js

@@ -0,0 +1,209 @@
+'use strict';
+
+const ProgramExecutor = require('./index-pubsub');
+const ProgramsRepository = require('./repositories/programs');
+const e2eTestReuse = require('@emartech/pubsub-client-js/e2e/e2e-test-resuse.js');
+
+const knex = require('knex');
+const DbCleaner = require('./test-helper/db-cleaner');
+const { db } = require('./test-helper/get-test-db-config');
+
+const customerId = 1234;
+const topicName = 'program-executor';
+
+class SuccessfulJob {
+  static get name() {
+    return 'successful_job';
+  }
+
+  static create(programData) {
+    return new SuccessfulJob(programData);
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  constructor(programData) {
+    /// ... initialize member variables based on programData if needed
+  }
+
+  async execute(message, jobDataHandler) {
+    // eslint-disable-next-line no-unused-vars
+    const jobSpecificData = await jobDataHandler.get();
+  }
+}
+
+class ProblematicJob {
+  static get name() {
+    return 'problematic_job';
+  }
+
+  static create(programData) {
+    return new ProblematicJob(programData);
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  constructor(programData) {
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async execute(message, jobDataHandler) {
+    throw new Error('Some error happened in the job');
+  }
+}
+
+
+const testJobLibrary = {
+  firstJob: {},
+  secondJob: {},
+  successful_job: SuccessfulJob,
+  problematic_job: ProblematicJob
+};
+
+describe('ProgramExecutor', function () {
+  let config;
+  let configFlowControl;
+  let dbConnection;
+  let mockConstructorSpy;
+  let publisher;
+
+  beforeAll(async function () {
+    jest.clearAllMocks();
+    dbConnection = knex({
+      client: 'pg',
+      connection: db.connection.connString
+    });
+    await DbCleaner.create(dbConnection).tearDown();
+    ProgramsRepository.create(dbConnection, 'programs');
+  });
+
+  afterAll(async function () {
+    await DbCleaner.create(dbConnection).tearDown();
+    dbConnection.destroy();
+  });
+
+
+  afterEach(async () => {
+    await e2eTestReuse.afterEach(mockConstructorSpy, publisher);
+
+    publisher = undefined;
+    jest.clearAllMocks();
+  });
+
+
+  beforeEach(async function () {
+
+    config = {
+      knex: dbConnection,
+      tableName: 'programs',
+      topicName: topicName,
+      projectId: 'some-project-id'
+    };
+
+    configFlowControl = {
+      knex: dbConnection,
+      tableName: 'programs',
+      topicName: topicName,
+      projectId: 'some-project-id',
+      pubSubMaxExtensionMinutes: 20,
+      pubSubMinAckDeadline: 10,
+      pubSubMaxAckDeadline: {"millis":600000}
+    };
+    const result = await e2eTestReuse.beforeEach(topicName);
+    mockConstructorSpy = result.mockConstructorSpy;
+    publisher = result.publisher;
+
+  });
+
+  describe('#Program Executor', function () {
+    it('should execute program and maintain status in DB: two skipped jobs', async function () {
+      const runId = await ProgramExecutor.create(config).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['current_program', 'next_program']
+        }
+      );
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['current_program', 'next_program']);
+      expect(result.step).toEqual(1);
+      expect(result.errored_at).toEqual(null);
+      expect(result.finished_at).not.toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+
+        it('should execute program and maintain status in DB: two skipped jobs (with flow control config)', async function () {
+      const runId = await ProgramExecutor.create(configFlowControl).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['current_program', 'next_program']
+        }
+      );
+      await ProgramExecutor.create(configFlowControl).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['current_program', 'next_program']);
+      expect(result.step).toEqual(1);
+      expect(result.errored_at).toEqual(null);
+      expect(result.finished_at).not.toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+
+    it('should execute program and maintain status in DB: one executed job', async function () {
+      const runId = await ProgramExecutor.create(config).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['successful_job']
+        }
+      );
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['successful_job']);
+      expect(result.step).toEqual(0);
+      expect(result.errored_at).toEqual(null);
+      expect(result.finished_at).not.toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+
+    it('should execute program and maintain status in DB: one failed job', async function () {
+      const runId = await ProgramExecutor.create(config).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['problematic_job']
+        }
+      );
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['problematic_job']);
+      expect(result.step).toEqual(0);
+      expect(result.errored_at).not.toEqual(null);
+      expect(result.finished_at).toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+  });
+});

package/src/index-pubsub.js

@@ -0,0 +1,84 @@
+'use strict';
+
+const consumer = require('@emartech/pubsub-client-js').Consumer;
+const EventEmitter = require('events');
+
+const ProgramHandler = require('./program-handler');
+const ProgramsRepository = require('./repositories/programs');
+const QueueManager = require('./queue-manager-pubsub');
+
+class ProgramExecutor extends EventEmitter {
+  /**
+   * @param {object} config
+   * @param {object} config.knex - Connected Knex instance
+   * @param {string} config.projectId - GCP Project ID
+   * @param {string} config.tableName - Table name for bookkeeping
+   * @param {string} config.topicName - Topic name to publish to
+   * @param {string} config.pubSubMaxExtensionMinutes - Pubsub lease management: max extension minutes
+   * @param {string} config.pubSubMinAckDeadline - Pubsub lease management: Min Ack Dead line
+   * @param {string} config.pubSubMaxAckDeadline - Pubsub lease management: Max Ack Dead line
+   */
+  constructor(config) {
+    super();
+    this._config = config;
+    this._programsRepository = ProgramsRepository.create(config.knex, config.tableName);
+    this._queueManager = QueueManager.create(config.topicName, config.projectId);
+    this._programHandler = ProgramHandler.create(this._programsRepository, this._queueManager);
+  }
+
+  /**
+   * @param {object} data
+   * @param {object} data.programData
+   * @param {array} data.jobs
+   * @param {object} data.jobsData
+   */
+  createProgram(data) {
+    return this._programHandler.createProgram(data);
+  }
+
+  processPrograms(jobLibrary) {
+    const eventEmitter = this; // eslint-disable-line consistent-this
+
+    const programExecutorProcessor = require('./program-executor-processor').create(
+      this._programHandler,
+      this._queueManager,
+      jobLibrary
+    );
+
+    consumer
+      .create(this._config.topicName,
+        {
+          logger: `${this._config.topicName}-consumer`,
+          MaxStreams: 1,
+          MaxMessages: 1,
+          MaxExtensionMinutes:this._config.pubSubMaxExtensionMinutes || 60,
+          MinAckDeadline:this._config.pubSubMinAckDeadline || undefined,
+          MaxAckDeadline:this._config.pubSubMaxAckDeadline || undefined,      
+          onMessage: async (message) => {
+            try {
+              await programExecutorProcessor.process(message);
+            } catch (error) {
+              eventEmitter.emit('programError', { message, error });
+              error.message = error.message.substring(0, 255);
+              throw error;
+            }
+          }
+        },
+        { projectId: this._config.projectId }
+      )
+      .process();
+  }
+
+  /**
+   * @param {object} config
+   * @param {object} config.knex - Connected Knex instance
+   * @param {string} config.projectId - GCP Project ID
+   * @param {string} config.tableName - Table name for bookkeeping
+   * @param {string} config.topicName - Topic name to publish to
+   */
+  static create(config) {
+    return new ProgramExecutor(config);
+  }
+}
+
+module.exports = ProgramExecutor;

package/src/index-pubsub.test.js

@@ -0,0 +1,160 @@
+'use strict';
+
+const ProgramExecutor = require('./index-pubsub');
+const ProgramHandler = require('./program-handler');
+const ProgramsRepository = require('./repositories/programs');
+const QueueManager = require('./queue-manager-pubsub');
+const ProgramExecutorProcessor = require('./program-executor-processor');
+const Consumer = require('@emartech/pubsub-client-js').Consumer;
+
+const testJobLibrary = {
+  firstJob: {},
+  secondJob: {}
+};
+
+describe('ProgramExecutor', function () {
+  let config;
+
+  beforeEach(async function () {
+    config = {
+      knex: this.db,
+      tableName: 'programs',
+      topicName: 'program-executor',
+      projectId: 'amqp://guest:guest@localhost:9999',
+      pubSubMaxExtensionMinutes: 60,
+      pubSubMinAckDeadline: {'millis':600000},
+      pubSubMaxAckDeadline: {'millis':600000}
+    };
+
+
+    jest.spyOn(ProgramHandler, 'create');
+    ProgramHandler.prototype.createProgram = jest.fn().mockReturnValue(true);
+
+    jest.spyOn(ProgramsRepository, 'create');
+    jest.spyOn(QueueManager, 'create');
+    jest.spyOn(ProgramExecutorProcessor, 'create');
+
+    ProgramExecutorProcessor.prototype.process = jest.fn().mockReturnValue(true);
+    jest.spyOn(Consumer, 'create');
+    Consumer.prototype.process = jest.fn().mockReturnValue(true);
+
+
+  });
+
+  describe('#createProgram', function () {
+    it('should create program handler and call createProgram with given data', async function () {
+      await ProgramExecutor.create(config).createProgram({
+        jobs: ['current_program', 'next_program']
+      });
+
+      expect(ProgramHandler.create).toHaveBeenCalledWith(
+        expect.any(ProgramsRepository),
+        expect.any(QueueManager)
+      );
+      expect(ProgramsRepository.create).toHaveBeenCalledWith(config.knex, config.tableName);
+      expect(QueueManager.create).toHaveBeenCalledWith(config.topicName, config.projectId);
+      expect(ProgramHandler.prototype.createProgram).toHaveBeenCalledWith({
+        jobs: ['current_program', 'next_program']
+      });
+    });
+  });
+
+  describe('#processPrograms', function () {
+    it('should create program executor processor with job library', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      expect(ProgramExecutorProcessor.create).toHaveBeenCalledWith(
+        expect.any(Object), //TODO check
+        expect.any(QueueManager),
+        testJobLibrary
+      );
+
+      expect(ProgramsRepository.create).toHaveBeenCalledWith(config.knex, config.tableName);
+      expect(QueueManager.create).toHaveBeenCalledWith(config.topicName, config.projectId);
+    });
+
+    it('should create consumer with the given pubsub config', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      expect(Consumer.create.mock.calls[0][0]).toEqual(config.topicName);
+    });
+
+    it('should create consumer to consume the given topic', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      expect(Consumer.create.mock.calls[0][1]).toEqual(expect.objectContaining({
+        MaxMessages: 1,
+        MaxStreams: 1,
+        MaxExtensionMinutes: 60,
+        MinAckDeadline:{'millis':600000},
+        MaxAckDeadline:{'millis':600000},
+        logger: 'program-executor-consumer' }));
+    });
+
+    it('should create consumer with a logger based on the given topic name', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      expect(Consumer.create.mock.calls[0][1]).toEqual(
+        expect.objectContaining({ logger: `${config.topicName}-consumer` }));
+    });
+
+    it('should config consumer with given flow control settings', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      expect(Consumer.create.mock.calls[0][1]).toEqual(
+        expect.objectContaining({ MaxMessages: 1,
+          MaxStreams: 1 }));
+    });
+
+    it('should call the created executor when message callback fires', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      const onMessageFunction = Consumer.create.mock.calls[0][1].onMessage;
+
+      await onMessageFunction({ random: 'message' });
+
+      expect(ProgramExecutorProcessor.prototype.process).toHaveBeenCalledWith({ random: 'message' });
+    });
+
+    it('should truncate log message to avoid splitting by heroku', async function () {
+      const veryLongError = new Error('1'.repeat(100000));
+
+      ProgramExecutorProcessor.prototype.process = jest.fn().mockRejectedValue(veryLongError);
+
+      let caughtError;
+      try {
+        await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+        const onMessageFunction = Consumer.create.mock.calls[0][1].onMessage;
+        await onMessageFunction({ random: 'message' });
+      } catch (error) {
+        caughtError = error;
+      }
+
+      expect(caughtError).not.toBe(undefined);
+      expect(caughtError.message.length).toEqual(255);
+    });
+
+    it('should emit an error event', function (done) {
+      const sampleError = new Error('Error to be emitted');
+      ProgramExecutorProcessor.prototype.process = jest.fn().mockRejectedValue(sampleError);
+
+      const programExecutor = ProgramExecutor.create(config);
+
+      programExecutor.on('programError', function ({ error, message }) {
+        expect(message).toEqual({ random: 'message' });
+        expect(error.message).toEqual('Error to be emitted');
+        done();
+      });
+
+      programExecutor.processPrograms(testJobLibrary);
+      const onMessageFunction = Consumer.create.mock.calls[0][1].onMessage;
+      onMessageFunction({ random: 'message' }).catch(() => {});
+    });
+
+    it('should start processing', async function () {
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      expect(Consumer.prototype.process).toHaveBeenCalled();
+    });
+  });
+});

package/src/{index.spec.js → index.test.js}

@@ -1,6 +1,6 @@
 'use strict';
 
-const ProgramExecutor = require('./');
+const ProgramExecutor = require('.');
 const ProgramHandler = require('./program-handler');
 const ProgramsRepository = require('./repositories/programs');
 const QueueManager = require('./queue-manager');
@@ -23,125 +23,129 @@ describe('ProgramExecutor', function () {
       amqpUrl: 'amqp://guest:guest@localhost:9999'
     };
 
-    this.sandbox.spy(ProgramHandler, 'create');
-    this.sandbox.stub(ProgramHandler.prototype, 'createProgram');
 
-    this.sandbox.spy(ProgramsRepository, 'create');
-    this.sandbox.spy(QueueManager, 'create');
+    jest.spyOn(ProgramHandler, 'create');
+    ProgramHandler.prototype.createProgram = jest.fn().mockReturnValue(true);
+
+    jest.spyOn(ProgramsRepository, 'create');
+    jest.spyOn(QueueManager, 'create');
+    jest.spyOn(ProgramExecutorProcessor, 'create');
+
+    ProgramExecutorProcessor.prototype.process = jest.fn().mockReturnValue(true);
+    jest.spyOn(Consumer, 'create');
+    Consumer.prototype.process = jest.fn().mockReturnValue(true);
 
-    this.sandbox.spy(ProgramExecutorProcessor, 'create');
-    this.sandbox.stub(ProgramExecutorProcessor.prototype, 'process');
 
-    this.sandbox.spy(Consumer, 'create');
-    this.sandbox.stub(Consumer.prototype, 'process');
   });
 
-  describe('#createProgram', async function () {
+  describe('#createProgram', function () {
     it('should create program handler and call createProgram with given data', async function () {
       await ProgramExecutor.create(config).createProgram({
         jobs: ['current_program', 'next_program']
       });
 
-      expect(ProgramHandler.create).to.have.been.calledWith(
-        this.sinon.match.instanceOf(ProgramsRepository),
-        this.sinon.match.instanceOf(QueueManager)
+      expect(ProgramHandler.create).toHaveBeenCalledWith(
+        expect.any(ProgramsRepository),
+        expect.any(QueueManager)
       );
-      expect(ProgramsRepository.create).to.have.been.calledWith(config.knex, config.tableName);
-      expect(QueueManager.create).to.have.been.calledWith(config.amqpUrl, config.queueName);
-      expect(ProgramHandler.prototype.createProgram).to.have.been.calledWith({
+      expect(ProgramsRepository.create).toHaveBeenCalledWith(config.knex, config.tableName);
+      expect(QueueManager.create).toHaveBeenCalledWith(config.amqpUrl, config.queueName);
+      expect(ProgramHandler.prototype.createProgram).toHaveBeenCalledWith({
         jobs: ['current_program', 'next_program']
       });
     });
   });
 
-  describe('#processPrograms', async function () {
+  describe('#processPrograms', function () {
     it('should create program executor processor with job library', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      expect(ProgramExecutorProcessor.create).to.have.been.calledWith(
-        this.sinon.match.instanceOf(ProgramHandler),
-        this.sinon.match.instanceOf(QueueManager),
+      expect(ProgramExecutorProcessor.create).toHaveBeenCalledWith(
+        expect.any(Object), //TODO check
+        expect.any(QueueManager),
         testJobLibrary
       );
 
-      expect(ProgramsRepository.create).to.have.been.calledWith(config.knex, config.tableName);
-      expect(QueueManager.create).to.have.been.calledWith(config.amqpUrl, config.queueName);
+      expect(ProgramsRepository.create).toHaveBeenCalledWith(config.knex, config.tableName);
+      expect(QueueManager.create).toHaveBeenCalledWith(config.amqpUrl, config.queueName);
     });
 
     it('should create consumer with the given rabbitMq config', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      expect(Consumer.create.lastCall.args[0]).to.eql({ default: { url: config.amqpUrl } });
+      expect(Consumer.create.mock.calls[0][0]).toEqual({ default: { url: config.amqpUrl } });
     });
 
     it('should create consumer to consume the given queue', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      expect(Consumer.create.lastCall.args[1]).to.containSubset({ channel: config.queueName });
+      expect(Consumer.create.mock.calls[0][1]).toEqual(
+        expect.objectContaining({ channel: config.queueName }));
     });
 
     it('should create consumer with a logger based on the given queue name', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      expect(Consumer.create.lastCall.args[1]).to.containSubset({ logger: `${config.queueName}-consumer` });
+      expect(Consumer.create.mock.calls[0][1]).toEqual(
+        expect.objectContaining({ logger: `${config.queueName}-consumer` }));
     });
 
     it('should config consumer with prefecth count and a retry time', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      expect(Consumer.create.lastCall.args[1]).to.containSubset({ prefetchCount: 1, retryTime: 1000 });
+      expect(Consumer.create.mock.calls[0][1]).toEqual(
+        expect.objectContaining({ prefetchCount: 1, retryTime: 1000 }));
     });
 
     it('should call the created executor when message callback fires', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      const onMessageFunction = Consumer.create.lastCall.args[1].onMessage;
+      const onMessageFunction = Consumer.create.mock.calls[0][1].onMessage;
 
       await onMessageFunction({ random: 'message' });
 
-      expect(ProgramExecutorProcessor.prototype.process).to.have.been.calledWith({ random: 'message' });
+      expect(ProgramExecutorProcessor.prototype.process).toHaveBeenCalledWith({ random: 'message' });
     });
 
     it('should truncate log message to avoid splitting by heroku', async function () {
       const veryLongError = new Error('1'.repeat(100000));
 
-      ProgramExecutorProcessor.prototype.process.rejects(veryLongError);
+      ProgramExecutorProcessor.prototype.process = jest.fn().mockRejectedValue(veryLongError);
 
       let caughtError;
       try {
         await ProgramExecutor.create(config).processPrograms(testJobLibrary);
-        const onMessageFunction = Consumer.create.lastCall.args[1].onMessage;
+        const onMessageFunction = Consumer.create.mock.calls[0][1].onMessage;
         await onMessageFunction({ random: 'message' });
       } catch (error) {
         caughtError = error;
       }
 
-      expect(caughtError).not.to.be.undefined;
-      expect(caughtError.message.length).to.eql(255);
+      expect(caughtError).not.toBe(undefined);
+      expect(caughtError.message.length).toEqual(255);
     });
 
     it('should emit an error event', function (done) {
       const sampleError = new Error('Error to be emitted');
-
-      ProgramExecutorProcessor.prototype.process.rejects(sampleError);
+      ProgramExecutorProcessor.prototype.process = jest.fn().mockRejectedValue(sampleError);
 
       const programExecutor = ProgramExecutor.create(config);
 
       programExecutor.on('programError', function ({ error, message }) {
-        expect(message).to.eql({ random: 'message' });
-        expect(error.message).to.eql('Error to be emitted');
+        expect(message).toEqual({ random: 'message' });
+        expect(error.message).toEqual('Error to be emitted');
         done();
       });
 
       programExecutor.processPrograms(testJobLibrary);
-      const onMessageFunction = Consumer.create.lastCall.args[1].onMessage;
+      const onMessageFunction = Consumer.create.mock.calls[0][1].onMessage;
       onMessageFunction({ random: 'message' }).catch(() => {});
     });
 
     it('should start processing', async function () {
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      expect(Consumer.prototype.process).to.have.been.called;
+      expect(Consumer.prototype.process).toHaveBeenCalled();
     });
   });
 });