@emartech/program-executor 3.12.0 → 3.13.0

package/.github/workflows/dependabot-random-reviewer-poker.yml

@@ -0,0 +1,8 @@
+name: Dependabot Reviewer Poker
+on:
+  pull_request:
+    types: [opened, reopened]
+jobs:
+  assign-random-reviewer:
+    uses: Emartech/IntegrationCloud-DevOps-Workstream/.github/workflows/dependabot-random-review-poker.yml@main
+    secrets: inherit

package/package.json

@@ -19,8 +19,8 @@
   "author": "team-shopify@emarsys.com",
   "license": "ISC",
   "dependencies": {
-    "@emartech/json-logger": "3.4.0",
-    "@emartech/pubsub-client-js": "^1.2.5",
+    "@emartech/json-logger": "9.1.0",
+    "@emartech/pubsub-client-js": "^1.2.7",
     "@emartech/rabbitmq-client": "5.9.1",
     "camelcase-keys": "^6.2.2",
     "graphql": "^16.11.0"
@@ -39,5 +39,5 @@
     "pg": "^8.16.3",
     "semantic-release": "^24.2.0"
   },
-  "version": "3.12.0"
+  "version": "3.13.0"
 }

package/src/index-pubsub-e2e.test.js

@@ -11,18 +11,56 @@ const { db } = require('./test-helper/get-test-db-config');
 const customerId = 1234;
 const topicName = 'program-executor';
 
+class SuccessfulJob {
+  static get name() {
+    return 'successful_job';
+  }
+
+  static create(programData) {
+    return new SuccessfulJob(programData);
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  constructor(programData) {
+    /// ... initialize member variables based on programData if needed
+  }
+
+  async execute(message, jobDataHandler) {
+    // eslint-disable-next-line no-unused-vars
+    const jobSpecificData = await jobDataHandler.get();
+  }
+}
+
+class ProblematicJob {
+  static get name() {
+    return 'problematic_job';
+  }
+
+  static create(programData) {
+    return new ProblematicJob(programData);
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  constructor(programData) {
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async execute(message, jobDataHandler) {
+    throw new Error('Some error happened in the job');
+  }
+}
+
+
 const testJobLibrary = {
   firstJob: {},
-  secondJob: {}
-};
-
-const waitForMessages = (timeout) => {
-  let timeOutLength = timeout;
-  return new Promise((resolve) => setTimeout(resolve, timeOutLength));
+  secondJob: {},
+  successful_job: SuccessfulJob,
+  problematic_job: ProblematicJob
 };
 
 describe('ProgramExecutor', function () {
   let config;
+  let configFlowControl;
   let dbConnection;
   let mockConstructorSpy;
   let publisher;
@@ -48,7 +86,6 @@ describe('ProgramExecutor', function () {
 
     publisher = undefined;
     jest.clearAllMocks();
-    jest.resetModules();
   });
 
 
@@ -60,6 +97,16 @@ describe('ProgramExecutor', function () {
       topicName: topicName,
       projectId: 'some-project-id'
     };
+
+    configFlowControl = {
+      knex: dbConnection,
+      tableName: 'programs',
+      topicName: topicName,
+      projectId: 'some-project-id',
+      pubSubMaxExtensionMinutes: 20,
+      pubSubMinAckDeadline: 10,
+      pubSubMaxAckDeadline: {"millis":600000}
+    };
     const result = await e2eTestReuse.beforeEach(topicName);
     mockConstructorSpy = result.mockConstructorSpy;
     publisher = result.publisher;
@@ -67,7 +114,7 @@ describe('ProgramExecutor', function () {
   });
 
   describe('#Program Executor', function () {
-    it('should execute program and maintain status in DB', async function () {
+    it('should execute program and maintain status in DB: two skipped jobs', async function () {
       const runId = await ProgramExecutor.create(config).createProgram(
         {
           programData: {
@@ -78,7 +125,30 @@ describe('ProgramExecutor', function () {
       );
       await ProgramExecutor.create(config).processPrograms(testJobLibrary);
 
-      await waitForMessages(1000);
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['current_program', 'next_program']);
+      expect(result.step).toEqual(1);
+      expect(result.errored_at).toEqual(null);
+      expect(result.finished_at).not.toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+
+        it('should execute program and maintain status in DB: two skipped jobs (with flow control config)', async function () {
+      const runId = await ProgramExecutor.create(configFlowControl).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['current_program', 'next_program']
+        }
+      );
+      await ProgramExecutor.create(configFlowControl).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
 
       const result = await dbConnection('programs').where({ run_id: runId }).first();
 
@@ -89,6 +159,51 @@ describe('ProgramExecutor', function () {
       expect(result.finished_at).not.toEqual(null);
       expect(result.program_data).toEqual({ customerId: 1234 });
     });
-  });
 
+    it('should execute program and maintain status in DB: one executed job', async function () {
+      const runId = await ProgramExecutor.create(config).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['successful_job']
+        }
+      );
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['successful_job']);
+      expect(result.step).toEqual(0);
+      expect(result.errored_at).toEqual(null);
+      expect(result.finished_at).not.toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+
+    it('should execute program and maintain status in DB: one failed job', async function () {
+      const runId = await ProgramExecutor.create(config).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['problematic_job']
+        }
+      );
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      await e2eTestReuse.waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['problematic_job']);
+      expect(result.step).toEqual(0);
+      expect(result.errored_at).not.toEqual(null);
+      expect(result.finished_at).toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+  });
 });

package/src/index-pubsub.js

@@ -14,6 +14,9 @@ class ProgramExecutor extends EventEmitter {
    * @param {string} config.projectId - GCP Project ID
    * @param {string} config.tableName - Table name for bookkeeping
    * @param {string} config.topicName - Topic name to publish to
+   * @param {string} config.pubSubMaxExtensionMinutes - Pubsub lease management: max extension minutes
+   * @param {string} config.pubSubMinAckDeadline - Pubsub lease management: Min Ack Dead line
+   * @param {string} config.pubSubMaxAckDeadline - Pubsub lease management: Max Ack Dead line
    */
   constructor(config) {
     super();
@@ -48,6 +51,9 @@ class ProgramExecutor extends EventEmitter {
           logger: `${this._config.topicName}-consumer`,
           MaxStreams: 1,
           MaxMessages: 1,
+          MaxExtensionMinutes:this._config.pubSubMaxExtensionMinutes || 60,
+          MinAckDeadline:this._config.pubSubMinAckDeadline || undefined,
+          MaxAckDeadline:this._config.pubSubMaxAckDeadline || undefined,      
           onMessage: async (message) => {
             try {
               await programExecutorProcessor.process(message);

package/src/index-pubsub.test.js

@@ -20,7 +20,10 @@ describe('ProgramExecutor', function () {
       knex: this.db,
       tableName: 'programs',
       topicName: 'program-executor',
-      projectId: 'amqp://guest:guest@localhost:9999'
+      projectId: 'amqp://guest:guest@localhost:9999',
+      pubSubMaxExtensionMinutes: 60,
+      pubSubMinAckDeadline: {'millis':600000},
+      pubSubMaxAckDeadline: {'millis':600000}
     };
 
 
@@ -82,6 +85,9 @@ describe('ProgramExecutor', function () {
       expect(Consumer.create.mock.calls[0][1]).toEqual(expect.objectContaining({
         MaxMessages: 1,
         MaxStreams: 1,
+        MaxExtensionMinutes: 60,
+        MinAckDeadline:{'millis':600000},
+        MaxAckDeadline:{'millis':600000},
         logger: 'program-executor-consumer' }));
     });
 

package/src/program-executor-processor/index.js

@@ -1,7 +1,12 @@
 'use strict';
 
-const logger = require('@emartech/json-logger')('program-executor');
-const devLogger = require('@emartech/json-logger')('dev-program-executor');
+const { createLogger } = require('@emartech/json-logger');
+createLogger.configure({
+  outputFormat: 'legacy'
+});
+
+const logger = createLogger('program-executor');
+const devLogger = createLogger('dev-program-executor');
 const RetryableError = require('../retryable-error');
 const JobHandler = require('../job-data-handler');
 
@@ -19,9 +24,11 @@ class ProgramExecutorProcessor {
     try {
       await this._executeNextJob(message);
     } catch (error) {
-      devLogger.error('error in job', { run_id: runId, OSPID: process.pid,
+      devLogger.error('error in job', {
+        run_id: runId, OSPID: process.pid,
         error, errorMessage: error.message, errorCode: error.code,
-        errorRetryable: error.retryable, errorIgnorable: error.ignorable });
+        errorRetryable: error.retryable, errorIgnorable: error.ignorable
+      });
       if (error.retryable) {
         await this._programHandler.incrementStepRetryCount(runId);
         await this._programHandler.setJobRetriableErrorMessage(runId, error.message);
@@ -71,8 +78,10 @@ class ProgramExecutorProcessor {
 
   log(action, { programData, jobs, runId, currentJob, level = 'info' }) {
     logger[level](action,
-      { customer_id: programData.customerId, run_id: runId, OSPID: process.pid, msg: currentJob,
-        executor: { current_job: currentJob, program_data: programData, jobs } });
+      {
+        customer_id: programData.customerId, run_id: runId, OSPID: process.pid, msg: currentJob,
+        executor: { current_job: currentJob, program_data: programData, jobs }
+      });
   }
 
   static create(programHandler, queueManager, jobLibrary) {

package/src/program-handler/index.js

@@ -1,6 +1,11 @@
 'use strict';
 
-const devLogger = require('@emartech/json-logger')('dev-program-executor');
+const { createLogger } = require('@emartech/json-logger');
+createLogger.configure({
+  outputFormat: 'legacy'
+});
+
+const devLogger = createLogger('dev-program-executor');
 const RunIdGenerator = require('../runid-generator');
 
 class ProgramHandler {

package/src/queue-manager/index.js

@@ -1,7 +1,12 @@
 'use strict';
 
 const RabbitMq = require('@emartech/rabbitmq-client').RabbitMq;
-const logger = require('@emartech/json-logger')('program-executor-queue-manager');
+const { createLogger } = require('@emartech/json-logger');
+createLogger.configure({
+  outputFormat: 'legacy'
+});
+
+const logger = createLogger('program-executor-queue-manager');
 
 class QueueManager {
   constructor(amqpUrl, queueName) {

package/src/queue-manager-pubsub/index.js

@@ -1,7 +1,12 @@
 'use strict';
 
 const PubSubClient = require('@emartech/pubsub-client-js').PubSubClient;
-const logger = require('@emartech/json-logger')('program-executor-queue-manager-pubsub');
+const { createLogger } = require('@emartech/json-logger');
+createLogger.configure({
+  outputFormat: 'legacy'
+});
+
+const logger = createLogger('program-executor-queue-manager-pubsub');
 
 class QueueManager {
   constructor(topicName, projectId, gcpKeyFileName) {