@emartech/program-executor 3.11.1 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -1
- package/.github/CODEOWNERS +1 -1
- package/.github/dependabot.yml +0 -7
- package/.github/workflows/main.yaml +17 -5
- package/docker-compose.yml +11 -0
- package/eslint.config.js +1 -5
- package/jest.config.js +13 -0
- package/package.json +12 -16
- package/src/create-test-databases.js +84 -0
- package/src/execution-time-exceeded-error/{index.spec.js → index.test.js} +3 -3
- package/src/graphql/{schema.spec.js → schema.test.js} +53 -24
- package/src/ignorable-error/{index.spec.js → index.test.js} +5 -5
- package/src/index-pubsub-e2e.test.js +94 -0
- package/src/index-pubsub.js +78 -0
- package/src/index-pubsub.test.js +154 -0
- package/src/{index.spec.js → index.test.js} +42 -38
- package/src/job-data-handler/{index.spec.js → index.test.js} +15 -21
- package/src/program-executor-processor/{index.spec.js → index.test.js} +59 -48
- package/src/program-handler/{index.spec.js → index.test.js} +58 -34
- package/src/queue-manager/{index.spec.js → index.test.js} +16 -19
- package/src/queue-manager-pubsub/index.js +33 -0
- package/src/queue-manager-pubsub/index.test.js +44 -0
- package/src/repositories/{programs.spec.js → programs.test.js} +54 -37
- package/src/retryable-error/{index.spec.js → index.test.js} +5 -5
- package/src/runid-generator/{index.spec.js → index.test.js} +2 -2
- package/src/test-helper/get-test-db-config.js +29 -0
- package/src/testSetup.js +23 -0
- package/src/testTeardown.js +13 -0
- package/src/setup.spec.js +0 -50
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
const ProgramsRepository = require('./programs');
|
|
4
|
+
const knex = require('knex');
|
|
5
|
+
const DbCleaner = require('../test-helper/db-cleaner');
|
|
6
|
+
const { db } = require('../test-helper/get-test-db-config');
|
|
4
7
|
|
|
5
8
|
const programData = {
|
|
6
9
|
customerId: 1,
|
|
@@ -12,18 +15,31 @@ const runId = '9123434';
|
|
|
12
15
|
describe('ProgramsRepository', () => {
|
|
13
16
|
let programsRepository;
|
|
14
17
|
let resetUpdatedAt;
|
|
18
|
+
let dbConnection;
|
|
19
|
+
|
|
20
|
+
beforeAll(function () {
|
|
21
|
+
|
|
22
|
+
dbConnection = knex({
|
|
23
|
+
client: 'pg',
|
|
24
|
+
connection: db.connection.connString
|
|
25
|
+
});
|
|
15
26
|
|
|
16
|
-
before(function () {
|
|
17
27
|
resetUpdatedAt = async (runId) => {
|
|
18
28
|
const oldDate = new Date(0);
|
|
19
|
-
await
|
|
29
|
+
await dbConnection('programs').where({ run_id: runId }).update({ updated_at: oldDate });
|
|
20
30
|
|
|
21
31
|
return oldDate;
|
|
22
32
|
};
|
|
23
33
|
});
|
|
24
34
|
|
|
25
|
-
|
|
26
|
-
|
|
35
|
+
afterAll(async function () {
|
|
36
|
+
await DbCleaner.create(dbConnection).tearDown();
|
|
37
|
+
dbConnection.destroy();
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
beforeEach(async function () {
|
|
41
|
+
await DbCleaner.create(dbConnection).tearDown();
|
|
42
|
+
programsRepository = ProgramsRepository.create(dbConnection, 'programs');
|
|
27
43
|
});
|
|
28
44
|
|
|
29
45
|
describe('#save', () => {
|
|
@@ -48,7 +64,7 @@ describe('ProgramsRepository', () => {
|
|
|
48
64
|
|
|
49
65
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
50
66
|
|
|
51
|
-
expect(result).
|
|
67
|
+
expect(result).toEqual(expect.objectContaining({
|
|
52
68
|
runId,
|
|
53
69
|
programData,
|
|
54
70
|
jobs,
|
|
@@ -58,12 +74,12 @@ describe('ProgramsRepository', () => {
|
|
|
58
74
|
erroredAt,
|
|
59
75
|
errorMessage,
|
|
60
76
|
stepRetryCount
|
|
61
|
-
});
|
|
77
|
+
}));
|
|
62
78
|
});
|
|
63
79
|
|
|
64
80
|
it('creates table if not exists and saves program', async function () {
|
|
65
81
|
await programsRepository.save({ runId, programData, jobs: ['a'] });
|
|
66
|
-
await
|
|
82
|
+
await dbConnection.schema.dropTable('programs');
|
|
67
83
|
|
|
68
84
|
const jobs = ['a', 'b'];
|
|
69
85
|
const jobData = { product_sync: { page: 40 } };
|
|
@@ -71,13 +87,13 @@ describe('ProgramsRepository', () => {
|
|
|
71
87
|
|
|
72
88
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
73
89
|
|
|
74
|
-
expect(result).
|
|
90
|
+
expect(result).toEqual(expect.objectContaining({
|
|
75
91
|
runId,
|
|
76
92
|
jobs,
|
|
77
93
|
jobData,
|
|
78
94
|
programData,
|
|
79
95
|
step: 0
|
|
80
|
-
});
|
|
96
|
+
}));
|
|
81
97
|
});
|
|
82
98
|
});
|
|
83
99
|
|
|
@@ -91,9 +107,9 @@ describe('ProgramsRepository', () => {
|
|
|
91
107
|
await programsRepository.finishProgram(runId);
|
|
92
108
|
|
|
93
109
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
94
|
-
expect(result.finishedAt).not.
|
|
95
|
-
expect(result.stepRetryCount).
|
|
96
|
-
expect(result.updatedAt.toString()).not.
|
|
110
|
+
expect(result.finishedAt).not.toEqual(null);
|
|
111
|
+
expect(result.stepRetryCount).toEqual(0);
|
|
112
|
+
expect(result.updatedAt.toString()).not.toEqual(oldDate.toString());
|
|
97
113
|
});
|
|
98
114
|
});
|
|
99
115
|
|
|
@@ -106,10 +122,10 @@ describe('ProgramsRepository', () => {
|
|
|
106
122
|
await programsRepository.setProgramToError(runId, 'Something wrong happened!');
|
|
107
123
|
|
|
108
124
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
109
|
-
expect(result.finishedAt).
|
|
110
|
-
expect(result.erroredAt).not.
|
|
111
|
-
expect(result.errorMessage).
|
|
112
|
-
expect(result.updatedAt.toString()).not.
|
|
125
|
+
expect(result.finishedAt).toEqual(null);
|
|
126
|
+
expect(result.erroredAt).not.toEqual(null);
|
|
127
|
+
expect(result.errorMessage).toEqual('Something wrong happened!');
|
|
128
|
+
expect(result.updatedAt.toString()).not.toEqual(oldDate.toString());
|
|
113
129
|
});
|
|
114
130
|
|
|
115
131
|
it('should set and error_message and updated_at without setting finished_at or errored_at', async function () {
|
|
@@ -120,10 +136,10 @@ describe('ProgramsRepository', () => {
|
|
|
120
136
|
await programsRepository.setProgramToError(runId, 'Something wrong happened!', false);
|
|
121
137
|
|
|
122
138
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
123
|
-
expect(result.finishedAt).
|
|
124
|
-
expect(result.erroredAt).
|
|
125
|
-
expect(result.errorMessage).
|
|
126
|
-
expect(result.updatedAt.toString()).not.
|
|
139
|
+
expect(result.finishedAt).toEqual(null);
|
|
140
|
+
expect(result.erroredAt).toEqual(null);
|
|
141
|
+
expect(result.errorMessage).toEqual('Something wrong happened!');
|
|
142
|
+
expect(result.updatedAt.toString()).not.toEqual(oldDate.toString());
|
|
127
143
|
});
|
|
128
144
|
|
|
129
145
|
it('should trim error message', async function () {
|
|
@@ -138,7 +154,7 @@ describe('ProgramsRepository', () => {
|
|
|
138
154
|
errorThrown = error;
|
|
139
155
|
}
|
|
140
156
|
|
|
141
|
-
expect(errorThrown).
|
|
157
|
+
expect(errorThrown).toBe(undefined);
|
|
142
158
|
});
|
|
143
159
|
});
|
|
144
160
|
|
|
@@ -148,14 +164,14 @@ describe('ProgramsRepository', () => {
|
|
|
148
164
|
await programsRepository.incrementStepRetryCount(runId);
|
|
149
165
|
|
|
150
166
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
151
|
-
expect(result.step).
|
|
167
|
+
expect(result.step).toEqual(0);
|
|
152
168
|
|
|
153
169
|
const oldDate = await resetUpdatedAt(runId);
|
|
154
170
|
await programsRepository.incrementStep(runId);
|
|
155
171
|
const incrementedResult = await programsRepository.getProgramByRunId(runId);
|
|
156
|
-
expect(incrementedResult.step).
|
|
157
|
-
expect(incrementedResult.stepRetryCount).
|
|
158
|
-
expect(incrementedResult.updatedAt.toString()).not.
|
|
172
|
+
expect(incrementedResult.step).toEqual(1);
|
|
173
|
+
expect(incrementedResult.stepRetryCount).toEqual(0);
|
|
174
|
+
expect(incrementedResult.updatedAt.toString()).not.toEqual(oldDate.toString());
|
|
159
175
|
});
|
|
160
176
|
});
|
|
161
177
|
|
|
@@ -163,13 +179,13 @@ describe('ProgramsRepository', () => {
|
|
|
163
179
|
it('should increment step retry counter', async function () {
|
|
164
180
|
await programsRepository.save({ runId, programData, jobs: ['a'] });
|
|
165
181
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
166
|
-
expect(result.stepRetryCount).
|
|
182
|
+
expect(result.stepRetryCount).toBe(0);
|
|
167
183
|
|
|
168
184
|
const oldDate = await resetUpdatedAt(runId);
|
|
169
185
|
await programsRepository.incrementStepRetryCount(runId);
|
|
170
186
|
const incrementedResult = await programsRepository.getProgramByRunId(runId);
|
|
171
|
-
expect(incrementedResult.stepRetryCount).
|
|
172
|
-
expect(incrementedResult.updatedAt.toString()).not.
|
|
187
|
+
expect(incrementedResult.stepRetryCount).toBe(1);
|
|
188
|
+
expect(incrementedResult.updatedAt.toString()).not.toEqual(oldDate.toString());
|
|
173
189
|
});
|
|
174
190
|
});
|
|
175
191
|
|
|
@@ -180,17 +196,18 @@ describe('ProgramsRepository', () => {
|
|
|
180
196
|
|
|
181
197
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
182
198
|
|
|
183
|
-
expect(result).
|
|
199
|
+
expect(result).toEqual(expect.objectContaining({
|
|
184
200
|
runId,
|
|
185
201
|
jobs,
|
|
186
202
|
step: 0
|
|
187
|
-
});
|
|
203
|
+
}));
|
|
188
204
|
});
|
|
189
205
|
|
|
190
206
|
it('throws an error if program not found', async function () {
|
|
191
207
|
await programsRepository.save({ runId, programData, jobs: [] });
|
|
192
208
|
|
|
193
|
-
await expect(programsRepository.getProgramByRunId('NON_EXISTING_RUN_ID')).
|
|
209
|
+
await expect(programsRepository.getProgramByRunId('NON_EXISTING_RUN_ID')).rejects
|
|
210
|
+
.toThrow('Program not found for NON_EXISTING_RUN_ID.');
|
|
194
211
|
});
|
|
195
212
|
});
|
|
196
213
|
|
|
@@ -203,11 +220,11 @@ describe('ProgramsRepository', () => {
|
|
|
203
220
|
await programsRepository.setJobDataByRunId(runId, { product_sync: { page: 1 } });
|
|
204
221
|
const result = await programsRepository.getProgramByRunId(runId);
|
|
205
222
|
|
|
206
|
-
expect(result).
|
|
223
|
+
expect(result).toEqual(expect.objectContaining({
|
|
207
224
|
runId: runId,
|
|
208
225
|
jobData: { product_sync: { page: 1 } }
|
|
209
|
-
});
|
|
210
|
-
expect(result.updatedAt.toString()).not.
|
|
226
|
+
}));
|
|
227
|
+
expect(result.updatedAt.toString()).not.toEqual(oldDate.toString());
|
|
211
228
|
});
|
|
212
229
|
});
|
|
213
230
|
|
|
@@ -221,10 +238,10 @@ describe('ProgramsRepository', () => {
|
|
|
221
238
|
|
|
222
239
|
const result = await programsRepository.getUnfinishedPrograms();
|
|
223
240
|
|
|
224
|
-
expect(result.length).
|
|
225
|
-
expect(result[0]).
|
|
241
|
+
expect(result.length).toEqual(1);
|
|
242
|
+
expect(result[0]).toEqual(expect.objectContaining(({
|
|
226
243
|
runId: '2'
|
|
227
|
-
});
|
|
244
|
+
})));
|
|
228
245
|
});
|
|
229
246
|
});
|
|
230
247
|
});
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const RetryableError = require('
|
|
3
|
+
const RetryableError = require('.');
|
|
4
4
|
|
|
5
5
|
describe('RetryableError', () => {
|
|
6
6
|
it('should have a retryable property set to true', () => {
|
|
7
7
|
try {
|
|
8
8
|
throw new RetryableError();
|
|
9
9
|
} catch (error) {
|
|
10
|
-
expect(error.retryable).
|
|
10
|
+
expect(error.retryable).toBe(true);
|
|
11
11
|
}
|
|
12
12
|
});
|
|
13
13
|
|
|
@@ -15,8 +15,8 @@ describe('RetryableError', () => {
|
|
|
15
15
|
try {
|
|
16
16
|
throw new RetryableError('Something bad happened!', 200);
|
|
17
17
|
} catch (error) {
|
|
18
|
-
expect(error.message).
|
|
19
|
-
expect(error.code).
|
|
18
|
+
expect(error.message).toEqual('Something bad happened!');
|
|
19
|
+
expect(error.code).toEqual(200);
|
|
20
20
|
}
|
|
21
21
|
});
|
|
22
22
|
|
|
@@ -24,7 +24,7 @@ describe('RetryableError', () => {
|
|
|
24
24
|
try {
|
|
25
25
|
throw new RetryableError('Something bad happened!');
|
|
26
26
|
} catch (error) {
|
|
27
|
-
expect(RetryableError.isRetryable(error)).
|
|
27
|
+
expect(RetryableError.isRetryable(error)).toEqual(true);
|
|
28
28
|
}
|
|
29
29
|
});
|
|
30
30
|
});
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const RunIdGenerator = require('
|
|
3
|
+
const RunIdGenerator = require('.');
|
|
4
4
|
|
|
5
5
|
describe('RunId Generator', () => {
|
|
6
6
|
describe('generate', () => {
|
|
7
7
|
it('should return a string', () => {
|
|
8
8
|
const result = RunIdGenerator.generate();
|
|
9
|
-
expect(result).
|
|
9
|
+
expect(typeof result).toBe('string');
|
|
10
10
|
});
|
|
11
11
|
});
|
|
12
12
|
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const getDbConfig = function () {
|
|
4
|
+
let database = process.env.POSTGRES_DATABASE;
|
|
5
|
+
|
|
6
|
+
if (process.env.JEST_WORKER_ID) {
|
|
7
|
+
database = `${database}_${process.env.JEST_WORKER_ID}`;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
const port = process.env.POSTGRES_PORT;
|
|
11
|
+
|
|
12
|
+
return {
|
|
13
|
+
user: process.env.POSTGRES_USER,
|
|
14
|
+
host: process.env.POSTGRES_HOST,
|
|
15
|
+
database,
|
|
16
|
+
password: process.env.POSTGRES_PASSWORD,
|
|
17
|
+
port,
|
|
18
|
+
connString: `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}` +
|
|
19
|
+
`@${process.env.POSTGRES_HOST}:${port}/${database}`
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
module.exports = {
|
|
24
|
+
db: {
|
|
25
|
+
connection: {
|
|
26
|
+
...getDbConfig(process.env.NODE_ENV)
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
};
|
package/src/testSetup.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
require('dotenv').config({ silent: true });
|
|
4
|
+
|
|
5
|
+
const { PubSub } = require('@google-cloud/pubsub');
|
|
6
|
+
const { db } = require('./test-helper/get-test-db-config');
|
|
7
|
+
const knex = require('knex');
|
|
8
|
+
const DbCleaner = require('./test-helper/db-cleaner');
|
|
9
|
+
|
|
10
|
+
module.exports = async () => {
|
|
11
|
+
const dbConnection = knex({
|
|
12
|
+
client: 'pg',
|
|
13
|
+
connection: db.connection.connString
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
await DbCleaner.create(dbConnection).tearDown();
|
|
17
|
+
|
|
18
|
+
dbConnection.destroy();
|
|
19
|
+
|
|
20
|
+
const _pubsub = new PubSub();
|
|
21
|
+
await _pubsub.createTopic('program-executor');
|
|
22
|
+
await _pubsub.topic('program-executor').createSubscription('program-executor');
|
|
23
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
require('dotenv').config({ silent: true });
|
|
4
|
+
const { PubSub } = require('@google-cloud/pubsub');
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
module.exports = async () => {
|
|
8
|
+
console.log('global teardown');
|
|
9
|
+
|
|
10
|
+
const _pubsub = new PubSub();
|
|
11
|
+
await _pubsub.topic('program-executor').delete();
|
|
12
|
+
await _pubsub.subscription('program-executor').delete();
|
|
13
|
+
};
|
package/src/setup.spec.js
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
require('dotenv').config({ silent: true });
|
|
4
|
-
|
|
5
|
-
const knex = require('knex');
|
|
6
|
-
const sinon = require('sinon');
|
|
7
|
-
const chai = require('chai');
|
|
8
|
-
const sinonChai = require('sinon-chai');
|
|
9
|
-
const chaiSubset = require('chai-subset');
|
|
10
|
-
const chaiString = require('chai-string');
|
|
11
|
-
const chaiAsPromised = require('chai-as-promised');
|
|
12
|
-
const Logger = require('@emartech/json-logger').Logger;
|
|
13
|
-
|
|
14
|
-
const DbCleaner = require('./test-helper/db-cleaner');
|
|
15
|
-
|
|
16
|
-
chai.use(chaiSubset);
|
|
17
|
-
chai.use(sinonChai);
|
|
18
|
-
chai.use(chaiString);
|
|
19
|
-
chai.use(chaiAsPromised);
|
|
20
|
-
|
|
21
|
-
global.expect = chai.expect;
|
|
22
|
-
|
|
23
|
-
before(function () {
|
|
24
|
-
this.db = knex({
|
|
25
|
-
client: 'pg',
|
|
26
|
-
connection: process.env.DATABASE_URL || process.env.DATABASE_TEST_URL
|
|
27
|
-
});
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
beforeEach(async function () {
|
|
31
|
-
this.sinon = sinon;
|
|
32
|
-
this.sandbox = sinon.createSandbox();
|
|
33
|
-
|
|
34
|
-
this.loggerLog = this.sandbox.stub(Logger.prototype, 'error');
|
|
35
|
-
|
|
36
|
-
this.sandbox.stub(Logger.prototype, 'trace');
|
|
37
|
-
this.sandbox.stub(Logger.prototype, 'debug');
|
|
38
|
-
this.infoStub = this.sandbox.stub(Logger.prototype, 'info');
|
|
39
|
-
this.warnStub = this.sandbox.stub(Logger.prototype, 'warn');
|
|
40
|
-
this.formErrorStub = this.sandbox.stub(Logger.prototype, 'fromError');
|
|
41
|
-
this.sandbox.stub(Logger.prototype, 'warnFromError');
|
|
42
|
-
this.sandbox.stub(Logger.prototype, 'fatal');
|
|
43
|
-
|
|
44
|
-
await DbCleaner.create(this.db).tearDown();
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
afterEach(async function () {
|
|
48
|
-
this.sandbox.restore();
|
|
49
|
-
await DbCleaner.create(this.db).tearDown();
|
|
50
|
-
});
|