@emartech/program-executor 3.11.0 → 3.12.0

package/.env.example

@@ -1,3 +1,11 @@
 DEBUG=program-executor*
 
-DATABASE_URL=postgres://developer:development_secret@localhost:5435/programexecutor
+POSTGRES_USER=developer
+POSTGRES_HOST=localhost
+POSTGRES_PASSWORD=development_secret
+POSTGRES_PORT=5435
+POSTGRES_DATABASE=programexecutor
+
+PUBSUB_EMULATOR_HOST=0.0.0.0:8085
+PUBSUB_PROJECT_ID=some-project-id
+PUBSUB_TIMEOUT=1000

package/.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # Each line is a file pattern followed by one or more owners.
 # Order is important. The last matching pattern has the most precedence.
 
-*.js   oliver.weisenburger@sap.com   nikolay.dimitrov@sap.com   ian.helmrich@sap.com   mauro.greco@sap.com
+*   @emartech/octopus-shopify

package/.github/dependabot.yml

@@ -19,13 +19,6 @@ updates:
       interval: "weekly"
       day: "monday"
       time: "09:00"
-    # Add assignees
-    reviewers:
-      - "IvanFroehlich"
-      - "MauroGreco"
-      - "oliverweisenburger"
-      - "dimirovn"
-      - "ianhelmrich"
     commit-message:
       # Prefix all commit messages with "npm: "
       prefix: "[dependabot]npm"

package/.github/workflows/main.yaml

@@ -11,8 +11,6 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
-    container: node:18-alpine3.16
-
     services:
       postgres:
         image: postgres:14-alpine
@@ -29,12 +27,19 @@ jobs:
           - 5432:5432
 
     steps:
+      - name: 'Install Cloud SDK'
+        uses: google-github-actions/setup-gcloud@v2.1.4
+        with:
+          install_components: 'beta,pubsub-emulator'
+      - name: 'start pubsub emulator'
+        run: |
+          gcloud beta emulators pubsub start --project=some-project-id --host-port=0.0.0.0:8085 &
       - name: Check out repository
         uses: actions/checkout@v4
-      - name: Use newest Node version
+      - name: Use Node 20.18.0 version
         uses: actions/setup-node@v3
         with:
-          node-version: "lts/*"
+          node-version: "20.18.0"
       - name: Set NPM token
         run: npm config set '//registry.npmjs.org/:_authToken' "${{ env.NPM_TOKEN }}"
       - name: npm dependencies
@@ -42,7 +47,14 @@ jobs:
       - name: npm test
         run: npm test
         env:
-          DATABASE_URL: 'postgres://developer:development_secret@postgres:5432/programexecutor'
+          POSTGRES_USER: developer
+          POSTGRES_HOST: localhost
+          POSTGRES_PASSWORD: development_secret
+          POSTGRES_PORT: 5432
+          POSTGRES_DATABASE: programexecutor
+          PUBSUB_EMULATOR_HOST: 0.0.0.0:8085
+          PUBSUB_PROJECT_ID: some-project-id
+          PUBSUB_TIMEOUT: 2000
 
   deploy:
     name: deploy

package/docker-compose.yml

@@ -7,3 +7,14 @@ services:
       - POSTGRES_DB=programexecutor
     ports:
       - 5435:5432
+  pub-sub-emulator:
+    image: google/cloud-sdk:526.0.0-emulators
+    command: ["gcloud", "beta", "emulators", "pubsub", "start", "--host-port=0.0.0.0:8085", "--project=some-project-id"]
+    ports:
+      - "8085:8085"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8085/v1/projects/test/schemas"]
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s

package/eslint.config.js

@@ -14,14 +14,10 @@ const compat = new FlatCompat({
 module.exports = [
   ...compat.extends('emarsys'),
   {
-    plugins: {
-      mocha
-    },
-
     languageOptions: {
       globals: {
         ...globals.node,
-        ...globals.mocha,
+        ...globals.jest,
         inject: true,
         onmessage: true,
         expect: true

package/jest.config.js

@@ -0,0 +1,13 @@
+'use strict';
+
+module.exports = {
+  testEnvironment: 'node',
+  rootDir: './',
+  testPathIgnorePatterns: ['/node_modules/'],
+  testRegex: '\\.(test|e2e)\\.js$',
+  clearMocks: true,
+  collectCoverage: false,
+  coverageReporters: ['json', 'html', 'text', 'lcov'],
+  globalSetup: "./src/testSetup.js",
+  globalTeardown: './src/testTeardown.js',
+};

package/package.json

@@ -3,7 +3,9 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "mocha --exit --reporter spec 'src/**/*.spec.js'",
+    "test": "npm run create-test-dbs && jest",
+    "jest-watch": "jest --watch",
+    "create-test-dbs": "NODE_ENV=test node ./src/create-test-databases.js",
     "code-style": "eslint '**/*.js' --ignore-pattern node_modules/",
     "semantic-release": "semantic-release --branches main"
   },
@@ -17,30 +19,25 @@
   "author": "team-shopify@emarsys.com",
   "license": "ISC",
   "dependencies": {
-    "@emartech/rabbitmq-client": "^3.5.2",
+    "@emartech/json-logger": "3.4.0",
+    "@emartech/pubsub-client-js": "^1.2.5",
+    "@emartech/rabbitmq-client": "5.9.1",
     "camelcase-keys": "^6.2.2",
-    "graphql": "^15.5.0"
+    "graphql": "^16.11.0"
   },
   "engines": {
     "node": "20.18.0"
   },
   "devDependencies": {
-    "chai": "^4.5.0",
-    "chai-as-promised": "7.1.1",
-    "chai-string": "1.5.0",
-    "chai-subset": "1.6.0",
-    "dotenv": "^16.4.7",
-    "eslint": "^9.17.0",
+    "dotenv": "^17.2.1",
+    "eslint": "^9.34.0",
     "eslint-config-emarsys": "5.1.0",
-    "eslint-plugin-mocha": "^10.5.0",
     "eslint-plugin-no-only-tests": "^3.3.0",
     "eslint-plugin-security": "^3.0.1",
+    "jest": "29.0.0",
     "knex": "^3.1.0",
-    "mocha": "^11.0.1",
-    "pg": "^8.13.1",
-    "semantic-release": "^24.2.0",
-    "sinon": "19.0.2",
-    "sinon-chai": "^3.7.0"
+    "pg": "^8.16.3",
+    "semantic-release": "^24.2.0"
   },
-  "version": "3.11.0"
+  "version": "3.12.0"
 }

package/src/create-test-databases.js

@@ -0,0 +1,84 @@
+'use strict';
+
+require('dotenv').config({ silent: true });
+
+const os = require('os');
+
+const { db } = require('./test-helper/get-test-db-config');
+
+const { Client } = require('pg');
+
+const cpuCount = os.cpus() ? os.cpus().length : 1;
+
+const defaultDbName = db.connection.database;
+
+const getConnection = function (databaseName) {
+  return {
+    ...db.connection,
+    database: databaseName
+  };
+};
+
+const recreateTestDbs = async () => {
+  const client = new Client(getConnection('template1'));
+
+  await client.connect();
+
+  for (let index = 0; index <= cpuCount; index++) {
+    console.log(`Creating test database ${index}`);
+    try {
+      const pgUser = process.env.POSTGRES_USER || 'developer';
+      await client.query(`DROP DATABASE IF EXISTS ${defaultDbName}_${index}`);
+      await client.query(`CREATE DATABASE ${defaultDbName}_${index} TEMPLATE ${defaultDbName}`);
+      await client.query(`ALTER DATABASE ${defaultDbName}_${index} OWNER TO ${pgUser}`);
+    } catch (error) {
+      console.log(error.message);
+      await client.end();
+      process.exit(1);
+    }
+  }
+  await client.end();
+};
+
+const getLatestMigrationId = async (databaseName) => {
+  const client = new Client(getConnection(databaseName));
+  await client.connect();
+
+  let result;
+
+  try {
+    result = await client.query('SELECT MAX(id) FROM knex_migrations');
+  } catch (error) {
+    console.log(`Migrations not found in ${databaseName ? databaseName : defaultDbName}...`);
+    await client.end();
+    throw error;
+  }
+
+  await client.end();
+  return result.rows[0].max;
+};
+
+const isRecreationNeeded = async () => {
+  try {
+    const [mainMigrationId, secondaryMigrationId] = await Promise.all([
+      getLatestMigrationId(defaultDbName),
+      getLatestMigrationId(`${defaultDbName}_${cpuCount}`)
+    ]);
+
+    return mainMigrationId !== secondaryMigrationId;
+  // eslint-disable-next-line no-unused-vars
+  } catch (error) {
+    return true;
+  }
+};
+
+(async () => {
+  const recreate = await isRecreationNeeded();
+
+  if (recreate) {
+    await recreateTestDbs();
+  } else {
+    console.log('Everything is fine, using previously generated dbs...');
+  }
+  process.exit(0);
+})();

package/src/execution-time-exceeded-error/{index.spec.js → index.test.js}

@@ -1,13 +1,13 @@
 'use strict';
 
-const ExecutionTimeExceededError = require('./');
+const ExecutionTimeExceededError = require('.');
 
 describe('ExecutionTimeExceededError', () => {
   it('should have a executionTimeExceeded property set to true', () => {
     try {
       throw new ExecutionTimeExceededError();
     } catch (error) {
-      expect(error.executionTimeExceeded).to.be.true;
+      expect(error.executionTimeExceeded).toBe(true);
     }
   });
 
@@ -15,7 +15,7 @@ describe('ExecutionTimeExceededError', () => {
     try {
       throw new ExecutionTimeExceededError('Something bad happened!');
     } catch (error) {
-      expect(error.message).to.eql('Something bad happened!');
+      expect(error.message).toEqual('Something bad happened!');
     }
   });
 });

package/src/graphql/{schema.spec.js → schema.test.js}

@@ -3,8 +3,28 @@
 const { graphql } = require('graphql');
 const ProgramsRepository = require('../repositories/programs');
 const schema = require('./schema');
+const knex = require('knex');
+const DbCleaner = require('../test-helper/db-cleaner');
+const { db } = require('../test-helper/get-test-db-config');
 
 describe('schema', function () {
+
+  let dbConnection;
+
+  beforeAll(async function () {
+    dbConnection = knex({
+      client: 'pg',
+      connection: db.connection.connString
+    });
+    await DbCleaner.create(dbConnection).tearDown();
+    ProgramsRepository.create(dbConnection, 'programs');
+  });
+
+    afterAll(async function () {
+    await DbCleaner.create(dbConnection).tearDown();
+    dbConnection.destroy();
+  });
+
   describe('with empty database', () => {
     it('returns with empty array when table does not exist (error code 42P01)', async function () {
       const query = `
@@ -14,8 +34,9 @@ describe('schema', function () {
         }
       }`;
 
-      const { data } = await graphql(schema, query, {}, { knex: this.db, tableName: 'programs' });
-      expect(data).to.eql({ programs: [] });
+      const { data } = await graphql({ schema, source: query, contextValue: { knex: dbConnection,
+        tableName: 'programs' } });
+      expect(data).toEqual({ programs: [] });
     });
   });
 
@@ -48,8 +69,11 @@ describe('schema', function () {
       }
     ];
 
+
+
     beforeEach(async function () {
-      const programsRepository = new ProgramsRepository(this.db, 'programs');
+      await DbCleaner.create(dbConnection).tearDown();
+      const programsRepository = ProgramsRepository.create(dbConnection, 'programs');
       for (const program of programs) {
         await programsRepository.save(program);
       }
@@ -73,33 +97,34 @@ describe('schema', function () {
         }
       }`;
 
-      const { data } = await graphql(schema, query, {}, { knex: this.db, tableName: 'programs' });
+      const { data } = await graphql({ schema, source: query, contextValue: { knex: dbConnection,
+        tableName: 'programs' } });
 
-      expect(data.programs[0]).to.containSubset({
+      expect(data.programs[0]).toEqual(expect.objectContaining({
         ...programs[0],
         jobData: JSON.stringify({}),
         programData: JSON.stringify(programs[0].programData)
-      });
-      expect(data.programs[0].createdAt).not.to.be.undefined;
-      expect(data.programs[0].updatedAt).not.to.be.undefined;
+      }));
+      expect(data.programs[0].createdAt).not.toBe(undefined);
+      expect(data.programs[0].updatedAt).not.toBe(undefined);
 
-      expect(data.programs[1]).to.containSubset({
+      expect(data.programs[1]).toEqual(expect.objectContaining({
         ...programs[1],
         jobData: JSON.stringify(programs[1].jobData),
         programData: JSON.stringify(programs[1].programData),
         erroredAt: programs[1].erroredAt.getTime().toString()
-      });
-      expect(data.programs[1].createdAt).not.to.be.undefined;
-      expect(data.programs[1].updatedAt).not.to.be.undefined;
+      }));
+      expect(data.programs[1].createdAt).not.toBe(undefined);
+      expect(data.programs[1].updatedAt).not.toBe(undefined);
 
-      expect(data.programs[2]).to.containSubset({
+      expect(data.programs[2]).toEqual(expect.objectContaining({
         ...programs[2],
         jobData: JSON.stringify(programs[2].jobData),
         programData: JSON.stringify(programs[2].programData),
         finishedAt: programs[2].finishedAt.getTime().toString()
-      });
-      expect(data.programs[2].createdAt).not.to.be.undefined;
-      expect(data.programs[2].updatedAt).not.to.be.undefined;
+      }));
+      expect(data.programs[2].createdAt).not.toBe(undefined);
+      expect(data.programs[2].updatedAt).not.toBe(undefined);
     });
 
     describe('accepts orderBy input for `id` field', () => {
@@ -110,10 +135,15 @@ describe('schema', function () {
           }
         }`;
 
-        const { data } = await graphql(schema, query, {}, { knex: this.db, tableName: 'programs' });
+        const { data } = await graphql({
+          schema,
+          source: query,
+          contextValue: { knex: dbConnection,
+            tableName: 'programs' }
+        });
         const ids = data.programs.map((program) => program.id);
 
-        expect(ids).to.eql(['1', '2', '3']);
+        expect(ids).toEqual(['1', '2', '3']);
       });
 
       it('supports ordering by id DESC', async function () {
@@ -123,10 +153,15 @@ describe('schema', function () {
           }
         }`;
 
-        const { data } = await graphql(schema, query, {}, { knex: this.db, tableName: 'programs' });
+        const { data } = await graphql({
+          schema,
+          source: query,
+          contextValue: { knex: dbConnection,
+            tableName: 'programs' }
+        });
         const ids = data.programs.map((program) => program.id);
 
-        expect(ids).to.eql(['3', '2', '1']);
+        expect(ids).toEqual(['3', '2', '1']);
       });
     });
 
@@ -138,10 +173,15 @@ describe('schema', function () {
           }
         }`;
 
-        const { data } = await graphql(schema, query, {}, { knex: this.db, tableName: 'programs' });
+        const { data } = await graphql({
+          schema,
+          source: query,
+          contextValue: { knex: dbConnection,
+            tableName: 'programs' }
+        });
         const ids = data.programs.map((program) => program.id);
 
-        expect(ids).to.eql(['1']);
+        expect(ids).toEqual(['1']);
       });
 
       it('supports filtering for programs with step retry count greater than a given input', async function () {
@@ -151,10 +191,15 @@ describe('schema', function () {
           }
         }`;
 
-        const { data } = await graphql(schema, query, {}, { knex: this.db, tableName: 'programs' });
+        const { data } = await graphql({
+          schema,
+          source: query,
+          contextValue: { knex: dbConnection,
+            tableName: 'programs' }
+        });
         const ids = data.programs.map((program) => program.id);
 
-        expect(ids).to.eql(['3']);
+        expect(ids).toEqual(['3']);
       });
     });
   });

package/src/ignorable-error/{index.spec.js → index.test.js}

@@ -1,13 +1,13 @@
 'use strict';
 
-const IgnorableError = require('./');
+const IgnorableError = require('.');
 
 describe('IgnorableError', () => {
   it('should have a retryable property set to true', () => {
     try {
       throw new IgnorableError();
     } catch (error) {
-      expect(error.ignorable).to.be.true;
+      expect(error.ignorable).toBe(true);
     }
   });
 
@@ -15,8 +15,8 @@ describe('IgnorableError', () => {
     try {
       throw new IgnorableError('Something bad happened!', 200);
     } catch (error) {
-      expect(error.message).to.eql('Something bad happened!');
-      expect(error.code).to.eql(200);
+      expect(error.message).toEqual('Something bad happened!');
+      expect(error.code).toEqual(200);
     }
   });
 
@@ -24,7 +24,7 @@ describe('IgnorableError', () => {
     try {
       throw new IgnorableError('Something bad happened!');
     } catch (error) {
-      expect(IgnorableError.isIgnorable(error)).to.eql(true);
+      expect(IgnorableError.isIgnorable(error)).toEqual(true);
     }
   });
 });

package/src/index-pubsub-e2e.test.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const ProgramExecutor = require('./index-pubsub');
+const ProgramsRepository = require('./repositories/programs');
+const e2eTestReuse = require('@emartech/pubsub-client-js/e2e/e2e-test-resuse.js');
+
+const knex = require('knex');
+const DbCleaner = require('./test-helper/db-cleaner');
+const { db } = require('./test-helper/get-test-db-config');
+
+const customerId = 1234;
+const topicName = 'program-executor';
+
+const testJobLibrary = {
+  firstJob: {},
+  secondJob: {}
+};
+
+const waitForMessages = (timeout) => {
+  let timeOutLength = timeout;
+  return new Promise((resolve) => setTimeout(resolve, timeOutLength));
+};
+
+describe('ProgramExecutor', function () {
+  let config;
+  let dbConnection;
+  let mockConstructorSpy;
+  let publisher;
+
+  beforeAll(async function () {
+    jest.clearAllMocks();
+    dbConnection = knex({
+      client: 'pg',
+      connection: db.connection.connString
+    });
+    await DbCleaner.create(dbConnection).tearDown();
+    ProgramsRepository.create(dbConnection, 'programs');
+  });
+
+  afterAll(async function () {
+    await DbCleaner.create(dbConnection).tearDown();
+    dbConnection.destroy();
+  });
+
+
+  afterEach(async () => {
+    await e2eTestReuse.afterEach(mockConstructorSpy, publisher);
+
+    publisher = undefined;
+    jest.clearAllMocks();
+    jest.resetModules();
+  });
+
+
+  beforeEach(async function () {
+
+    config = {
+      knex: dbConnection,
+      tableName: 'programs',
+      topicName: topicName,
+      projectId: 'some-project-id'
+    };
+    const result = await e2eTestReuse.beforeEach(topicName);
+    mockConstructorSpy = result.mockConstructorSpy;
+    publisher = result.publisher;
+
+  });
+
+  describe('#Program Executor', function () {
+    it('should execute program and maintain status in DB', async function () {
+      const runId = await ProgramExecutor.create(config).createProgram(
+        {
+          programData: {
+            customerId: customerId
+          },
+          jobs: ['current_program', 'next_program']
+        }
+      );
+      await ProgramExecutor.create(config).processPrograms(testJobLibrary);
+
+      await waitForMessages(1000);
+
+      const result = await dbConnection('programs').where({ run_id: runId }).first();
+
+      expect(result.run_id).toBe(runId);
+      expect(result.jobs).toEqual(['current_program', 'next_program']);
+      expect(result.step).toEqual(1);
+      expect(result.errored_at).toEqual(null);
+      expect(result.finished_at).not.toEqual(null);
+      expect(result.program_data).toEqual({ customerId: 1234 });
+    });
+  });
+
+});

package/src/index-pubsub.js

@@ -0,0 +1,78 @@
+'use strict';
+
+const consumer = require('@emartech/pubsub-client-js').Consumer;
+const EventEmitter = require('events');
+
+const ProgramHandler = require('./program-handler');
+const ProgramsRepository = require('./repositories/programs');
+const QueueManager = require('./queue-manager-pubsub');
+
+class ProgramExecutor extends EventEmitter {
+  /**
+   * @param {object} config
+   * @param {object} config.knex - Connected Knex instance
+   * @param {string} config.projectId - GCP Project ID
+   * @param {string} config.tableName - Table name for bookkeeping
+   * @param {string} config.topicName - Topic name to publish to
+   */
+  constructor(config) {
+    super();
+    this._config = config;
+    this._programsRepository = ProgramsRepository.create(config.knex, config.tableName);
+    this._queueManager = QueueManager.create(config.topicName, config.projectId);
+    this._programHandler = ProgramHandler.create(this._programsRepository, this._queueManager);
+  }
+
+  /**
+   * @param {object} data
+   * @param {object} data.programData
+   * @param {array} data.jobs
+   * @param {object} data.jobsData
+   */
+  createProgram(data) {
+    return this._programHandler.createProgram(data);
+  }
+
+  processPrograms(jobLibrary) {
+    const eventEmitter = this; // eslint-disable-line consistent-this
+
+    const programExecutorProcessor = require('./program-executor-processor').create(
+      this._programHandler,
+      this._queueManager,
+      jobLibrary
+    );
+
+    consumer
+      .create(this._config.topicName,
+        {
+          logger: `${this._config.topicName}-consumer`,
+          MaxStreams: 1,
+          MaxMessages: 1,
+          onMessage: async (message) => {
+            try {
+              await programExecutorProcessor.process(message);
+            } catch (error) {
+              eventEmitter.emit('programError', { message, error });
+              error.message = error.message.substring(0, 255);
+              throw error;
+            }
+          }
+        },
+        { projectId: this._config.projectId }
+      )
+      .process();
+  }
+
+  /**
+   * @param {object} config
+   * @param {object} config.knex - Connected Knex instance
+   * @param {string} config.projectId - GCP Project ID
+   * @param {string} config.tableName - Table name for bookkeeping
+   * @param {string} config.topicName - Topic name to publish to
+   */
+  static create(config) {
+    return new ProgramExecutor(config);
+  }
+}
+
+module.exports = ProgramExecutor;