@squiz/job-runner-lib 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/lib/domains/job/constants/RetryConfig.constants.d.ts +6 -1
- package/lib/domains/job/constants/RuntimeImage.constants.d.ts +5 -0
- package/lib/domains/job/errors/RuntimeImageTaskDefinitionError.d.ts +7 -0
- package/lib/domains/job/errors/index.d.ts +1 -0
- package/lib/domains/job/manifest/JobManifestTypes.d.ts +13 -1
- package/lib/domains/job/model/Job/Job.d.ts +12 -0
- package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +1 -1
- package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +2 -2
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +1 -0
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -0
- package/lib/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.d.ts +6 -0
- package/lib/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.types.d.ts +9 -0
- package/lib/domains/job/utils/index.d.ts +2 -0
- package/lib/domains/manifest/index.d.ts +3 -2
- package/lib/domains/manifest/utils/defaultAutoRetry.d.ts +5 -0
- package/lib/domains/manifest/utils/defaultAutoRetry.spec.d.ts +5 -0
- package/lib/index.js +9 -9
- package/lib/index.js.map +4 -4
- package/package.json +2 -2
- package/src/domains/job/constants/RetryConfig.constants.ts +8 -2
- package/src/domains/job/constants/RuntimeImage.constants.ts +7 -1
- package/src/domains/job/errors/RuntimeImageTaskDefinitionError.ts +10 -0
- package/src/domains/job/errors/index.ts +1 -0
- package/src/domains/job/manifest/JobManifestTypes.ts +16 -1
- package/src/domains/job/manifest/mocks/JobManifest.mock.ts +1 -0
- package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -0
- package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +1 -0
- package/src/domains/job/manifest/mocks/JobManifestRequest.mock.ts +1 -0
- package/src/domains/job/model/Job/Job.ts +14 -0
- package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -0
- package/src/domains/job/model/Job/mocks/JobDto.mock.ts +4 -0
- package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +2 -0
- package/src/domains/job/model/Job/mocks/JobRequest.mock.ts +2 -0
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +56 -4
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +14 -4
- package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +36 -4
- package/src/domains/job/repository/JobRepository/JobRepository.ts +10 -5
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -0
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +53 -3
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +19 -8
- package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +9 -8
- package/src/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.ts +39 -0
- package/src/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.types.ts +10 -0
- package/src/domains/job/utils/index.ts +2 -0
- package/src/domains/manifest/index.ts +3 -2
- package/src/domains/manifest/utils/defaultAutoRetry.spec.ts +16 -0
- package/src/domains/manifest/utils/defaultAutoRetry.ts +6 -0
- package/src/domains/manifest/utils/validateRuntimeImage.spec.ts +11 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@squiz/job-runner-lib",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.8.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"private": false,
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"@aws-sdk/client-sqs": "3.1003.0",
|
|
36
36
|
"@aws-sdk/s3-request-presigner": "3.1003.0",
|
|
37
37
|
"@squiz/dx-common-lib": "1.66.1",
|
|
38
|
-
"@squiz/dx-json-schema-lib": "1.
|
|
38
|
+
"@squiz/dx-json-schema-lib": "1.86.0",
|
|
39
39
|
"@squiz/dx-logger-lib": "1.64.0",
|
|
40
40
|
"@squiz/optimization-logger": "1.11.2",
|
|
41
41
|
"axios": "1.16.1",
|
|
@@ -7,15 +7,21 @@
|
|
|
7
7
|
* Retry configuration with exponential backoff delays in seconds.
|
|
8
8
|
* Using SQS message timer limits (max 15 minutes = 900 seconds).
|
|
9
9
|
*
|
|
10
|
-
* Delays: 30s, 1min, 2min, 5min, 10min, 15min
|
|
10
|
+
* Delays: 30s, 1min, 2min, 5min, 10min, then 15min thereafter.
|
|
11
11
|
*/
|
|
12
|
-
export const RETRY_DELAYS_SECONDS: ReadonlyArray<number> = [30, 60, 120, 300, 600, 900];
|
|
12
|
+
export const RETRY_DELAYS_SECONDS: ReadonlyArray<number> = [30, 60, 120, 300, 600, 900, 900, 900, 900, 900];
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* Maximum number of retry attempts for container start failures.
|
|
16
16
|
*/
|
|
17
17
|
export const MAX_RETRY_ATTEMPTS = RETRY_DELAYS_SECONDS.length;
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Maximum number of total attempts for opted-in job code failures.
|
|
21
|
+
* This allows the initial attempt plus 3 retries.
|
|
22
|
+
*/
|
|
23
|
+
export const MAX_JOB_CODE_RETRY_ATTEMPTS = 4;
|
|
24
|
+
|
|
19
25
|
/**
|
|
20
26
|
* After a job has attempted to be started this number of times alerts will start to be sent.
|
|
21
27
|
*/
|
|
@@ -14,9 +14,15 @@ export const RECOMMENDED_RUNTIME_IMAGE = 'node:24';
|
|
|
14
14
|
export const DEPRECATED_RUNTIME_IMAGES: readonly string[] = [];
|
|
15
15
|
|
|
16
16
|
export const getRuntimeImageOrDefault = (image: string | null | undefined): string =>
|
|
17
|
-
image
|
|
17
|
+
image === null || image === undefined ? LEGACY_RUNTIME_IMAGE : image;
|
|
18
18
|
|
|
19
19
|
export const isSupportedRuntimeImage = (image: string): boolean =>
|
|
20
20
|
(SUPPORTED_RUNTIME_IMAGES as readonly string[]).includes(image);
|
|
21
21
|
|
|
22
22
|
export const isDeprecatedRuntimeImage = (image: string): boolean => DEPRECATED_RUNTIME_IMAGES.includes(image);
|
|
23
|
+
|
|
24
|
+
export const getRuntimeImagesMetadata = () => ({
|
|
25
|
+
supportedRuntimeImages: [...SUPPORTED_RUNTIME_IMAGES],
|
|
26
|
+
deprecatedRuntimeImages: [...DEPRECATED_RUNTIME_IMAGES],
|
|
27
|
+
recommendedRuntimeImage: RECOMMENDED_RUNTIME_IMAGE,
|
|
28
|
+
});
|
|
@@ -64,6 +64,13 @@ export interface JobManifest {
|
|
|
64
64
|
*/
|
|
65
65
|
image: string;
|
|
66
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Whether failed job code executions should be retried automatically.
|
|
69
|
+
* @type {boolean}
|
|
70
|
+
* @memberof JobManifest
|
|
71
|
+
*/
|
|
72
|
+
autoRetry: boolean;
|
|
73
|
+
|
|
67
74
|
/**
|
|
68
75
|
* The DynamoDB record type ('job').
|
|
69
76
|
* @type {EntityType}
|
|
@@ -142,6 +149,13 @@ export interface JobManifestRecord {
|
|
|
142
149
|
* @memberof JobManifestRecord
|
|
143
150
|
*/
|
|
144
151
|
image: string;
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Whether failed job code executions should be retried automatically.
|
|
155
|
+
* @type {boolean}
|
|
156
|
+
* @memberof JobManifestRecord
|
|
157
|
+
*/
|
|
158
|
+
autoRetry: boolean;
|
|
145
159
|
}
|
|
146
160
|
|
|
147
161
|
export type JobManifestDto = Record<keyof JobManifestRecord, AttributeValue>;
|
|
@@ -150,7 +164,8 @@ export type JobManifestDto = Record<keyof JobManifestRecord, AttributeValue>;
|
|
|
150
164
|
* The request body for creating a job (POST /job-manifest).
|
|
151
165
|
* @export
|
|
152
166
|
*/
|
|
153
|
-
export type CreateJobManifestRequest = Omit<JobManifest, 'type'
|
|
167
|
+
export type CreateJobManifestRequest = Omit<JobManifest, 'type' | 'autoRetry'> &
|
|
168
|
+
Partial<Pick<JobManifest, 'autoRetry'>>;
|
|
154
169
|
|
|
155
170
|
/**
|
|
156
171
|
* The request body for updating a job (PUT /job-manifest).
|
|
@@ -26,6 +26,7 @@ export const createMockJobManifest = (partialRecord: Partial<JobManifest> = {}):
|
|
|
26
26
|
version,
|
|
27
27
|
timeoutSeconds: faker.number.int({ max: 28800, min: 5 }),
|
|
28
28
|
image: LEGACY_RUNTIME_IMAGE,
|
|
29
|
+
autoRetry: false,
|
|
29
30
|
type: ENTITY_TYPES.manifest,
|
|
30
31
|
...partialRecord,
|
|
31
32
|
};
|
|
@@ -34,6 +34,13 @@ export interface Job {
|
|
|
34
34
|
*/
|
|
35
35
|
version: string;
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* The runtime image the job should execute against (e.g. `node:20`, `node:24`).
|
|
39
|
+
* @type {string}
|
|
40
|
+
* @memberof Job
|
|
41
|
+
*/
|
|
42
|
+
image: string;
|
|
43
|
+
|
|
37
44
|
/**
|
|
38
45
|
* The DynamoDB record type ('job').
|
|
39
46
|
* @type {EntityType}
|
|
@@ -82,6 +89,13 @@ export interface JobRecord {
|
|
|
82
89
|
* @memberof JobRecord
|
|
83
90
|
*/
|
|
84
91
|
version: string;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The runtime image the job should execute against (e.g. `node:20`, `node:24`).
|
|
95
|
+
* @type {string}
|
|
96
|
+
* @memberof JobRecord
|
|
97
|
+
*/
|
|
98
|
+
image: string;
|
|
85
99
|
}
|
|
86
100
|
|
|
87
101
|
export type JobDto = Record<keyof JobRecord, AttributeValue>;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
7
|
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
8
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
8
9
|
import { Job } from '../Job';
|
|
9
10
|
|
|
10
11
|
export const createMockJob = (partialRecord: Partial<Job> = {}): Job => {
|
|
@@ -16,6 +17,7 @@ export const createMockJob = (partialRecord: Partial<Job> = {}): Job => {
|
|
|
16
17
|
description: faker.lorem.sentence(),
|
|
17
18
|
type: ENTITY_TYPES.job,
|
|
18
19
|
version: version,
|
|
20
|
+
image: RECOMMENDED_RUNTIME_IMAGE,
|
|
19
21
|
...partialRecord,
|
|
20
22
|
};
|
|
21
23
|
};
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
7
|
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
8
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
8
9
|
import { JobDto, JobRecord } from '../Job';
|
|
9
10
|
|
|
10
11
|
export const createMockJobDto = (partialRecord: Partial<JobRecord> = {}): JobDto => {
|
|
@@ -29,6 +30,9 @@ export const createMockJobDto = (partialRecord: Partial<JobRecord> = {}): JobDto
|
|
|
29
30
|
version: {
|
|
30
31
|
S: partialRecord.version ?? version,
|
|
31
32
|
},
|
|
33
|
+
image: {
|
|
34
|
+
S: partialRecord.image ?? RECOMMENDED_RUNTIME_IMAGE,
|
|
35
|
+
},
|
|
32
36
|
};
|
|
33
37
|
};
|
|
34
38
|
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
7
|
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
8
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
8
9
|
import { JobRecord } from '../Job';
|
|
9
10
|
|
|
10
11
|
export const createMockJobRecord = (partialRecord: Partial<JobRecord> = {}): JobRecord => {
|
|
@@ -19,6 +20,7 @@ export const createMockJobRecord = (partialRecord: Partial<JobRecord> = {}): Job
|
|
|
19
20
|
name: faker.lorem.word(),
|
|
20
21
|
description: faker.lorem.sentence(),
|
|
21
22
|
version,
|
|
23
|
+
image: RECOMMENDED_RUNTIME_IMAGE,
|
|
22
24
|
...partialRecord,
|
|
23
25
|
};
|
|
24
26
|
};
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
7
8
|
import { CreateJobRequest } from '../Job';
|
|
8
9
|
|
|
9
10
|
export const createMockCreateJobRequest = (partialRecord: Partial<CreateJobRequest> = {}): CreateJobRequest => {
|
|
@@ -15,6 +16,7 @@ export const createMockCreateJobRequest = (partialRecord: Partial<CreateJobReque
|
|
|
15
16
|
name: faker.lorem.word(),
|
|
16
17
|
description: faker.lorem.sentence(),
|
|
17
18
|
version,
|
|
19
|
+
image: RECOMMENDED_RUNTIME_IMAGE,
|
|
18
20
|
...partialRecord,
|
|
19
21
|
};
|
|
20
22
|
};
|
|
@@ -12,21 +12,21 @@ import {
|
|
|
12
12
|
ScanCommandOutput,
|
|
13
13
|
} from '@aws-sdk/client-dynamodb';
|
|
14
14
|
import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
|
|
15
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
16
|
+
import { LEGACY_RUNTIME_IMAGE } from '../../constants/RuntimeImage.constants';
|
|
15
17
|
import {
|
|
18
|
+
createMockClientGetResponse,
|
|
16
19
|
createMockClientPutResponse,
|
|
17
20
|
createMockClientScanResponse,
|
|
18
|
-
createMockClientGetResponse,
|
|
19
21
|
} from '../../core/dynamoClient.mock';
|
|
20
|
-
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
21
22
|
import {
|
|
22
23
|
JobManifest,
|
|
24
|
+
createMockCreateJobManifestRequest,
|
|
23
25
|
createMockJobManifest,
|
|
24
26
|
createMockJobManifestDto,
|
|
25
27
|
createMockJobManifestDtos,
|
|
26
28
|
createMockJobManifestRecord,
|
|
27
|
-
createMockCreateJobManifestRequest,
|
|
28
29
|
} from '../../manifest';
|
|
29
|
-
import { LEGACY_RUNTIME_IMAGE } from '../../constants/RuntimeImage.constants';
|
|
30
30
|
import { JobManifestRepository } from './JobManifestRepository';
|
|
31
31
|
|
|
32
32
|
initIocContainer();
|
|
@@ -120,6 +120,7 @@ describe('JobManifestRepository', (): void => {
|
|
|
120
120
|
entryFunction: JSON.parse(mockJobManifestItem.entryFunction.S as string),
|
|
121
121
|
timeoutSeconds: parseInt(mockJobManifestItem.timeoutSeconds.N as string),
|
|
122
122
|
image: 'node:24',
|
|
123
|
+
autoRetry: false,
|
|
123
124
|
type: `manifest`,
|
|
124
125
|
} as JobManifest);
|
|
125
126
|
expect(mockSend).toBeCalledTimes(1);
|
|
@@ -143,6 +144,51 @@ describe('JobManifestRepository', (): void => {
|
|
|
143
144
|
expect(response).toEqual(
|
|
144
145
|
expect.objectContaining({
|
|
145
146
|
image: LEGACY_RUNTIME_IMAGE,
|
|
147
|
+
autoRetry: false,
|
|
148
|
+
}),
|
|
149
|
+
);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it(`should default missing autoRetry to false`, async (): Promise<void> => {
|
|
153
|
+
const mockJobManifestItem = createMockJobManifestDto();
|
|
154
|
+
delete (mockJobManifestItem as Partial<typeof mockJobManifestItem>).autoRetry;
|
|
155
|
+
|
|
156
|
+
mockSend.mockImplementationOnce(
|
|
157
|
+
(): GetItemCommandOutput =>
|
|
158
|
+
createMockClientGetResponse({
|
|
159
|
+
Item: mockJobManifestItem,
|
|
160
|
+
}),
|
|
161
|
+
);
|
|
162
|
+
|
|
163
|
+
const response = await jobManifestRepository.findOne({
|
|
164
|
+
name: mockJobManifestItem.name.S as string,
|
|
165
|
+
version: mockJobManifestItem.version.S as string,
|
|
166
|
+
});
|
|
167
|
+
expect(response).toEqual(
|
|
168
|
+
expect.objectContaining({
|
|
169
|
+
autoRetry: false,
|
|
170
|
+
}),
|
|
171
|
+
);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it(`should preserve an explicit unsupported runtime image`, async (): Promise<void> => {
|
|
175
|
+
const mockJobManifestItem = createMockJobManifestDto({ image: 'node:26' });
|
|
176
|
+
|
|
177
|
+
mockSend.mockImplementationOnce(
|
|
178
|
+
(): GetItemCommandOutput =>
|
|
179
|
+
createMockClientGetResponse({
|
|
180
|
+
Item: mockJobManifestItem,
|
|
181
|
+
}),
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
const response = await jobManifestRepository.findOne({
|
|
185
|
+
name: mockJobManifestItem.name.S as string,
|
|
186
|
+
version: mockJobManifestItem.version.S as string,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
expect(response).toEqual(
|
|
190
|
+
expect.objectContaining({
|
|
191
|
+
image: 'node:26',
|
|
146
192
|
}),
|
|
147
193
|
);
|
|
148
194
|
});
|
|
@@ -221,6 +267,12 @@ describe('JobManifestRepository', (): void => {
|
|
|
221
267
|
const dto = jobManifestRepository.convertToDynamoDbItem(manifest);
|
|
222
268
|
expect(dto.image).toEqual({ S: 'node:24' });
|
|
223
269
|
});
|
|
270
|
+
|
|
271
|
+
it(`should persist autoRetry on the DynamoDB item`, () => {
|
|
272
|
+
const manifest = createMockJobManifest({ autoRetry: true });
|
|
273
|
+
const dto = jobManifestRepository.convertToDynamoDbItem(manifest);
|
|
274
|
+
expect(dto.autoRetry).toEqual({ BOOL: true });
|
|
275
|
+
});
|
|
224
276
|
});
|
|
225
277
|
|
|
226
278
|
describe('delete', () => {
|
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
|
|
6
6
|
import { AttributeValue, DynamoDBClient, UpdateItemCommand, UpdateItemCommandInput } from '@aws-sdk/client-dynamodb';
|
|
7
7
|
import { inject } from 'inversify';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { defaultAutoRetry } from '../../../manifest/utils/defaultAutoRetry';
|
|
9
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
10
10
|
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
11
|
+
import { getRuntimeImageOrDefault } from '../../constants/RuntimeImage.constants';
|
|
11
12
|
import {
|
|
12
13
|
CreateJobManifestRequest,
|
|
13
14
|
JobManifest,
|
|
@@ -15,9 +16,9 @@ import {
|
|
|
15
16
|
JobManifestSk,
|
|
16
17
|
UpdateJobManifestRequest,
|
|
17
18
|
} from '../../manifest';
|
|
18
|
-
import { InjectTokens } from '../../constants/InjectTokens';
|
|
19
|
-
import { getRuntimeImageOrDefault } from '../../constants/RuntimeImage.constants';
|
|
20
19
|
import { parsePartitionKey } from '../../ParsePartitionKey';
|
|
20
|
+
import { provideTransient } from '../../transientProvider';
|
|
21
|
+
import { Repository } from '../AbstractRepository';
|
|
21
22
|
|
|
22
23
|
@provideTransient(JobManifestRepository)
|
|
23
24
|
export class JobManifestRepository extends Repository<JobManifest, JobManifestDto, JobManifestSk> {
|
|
@@ -32,6 +33,7 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
32
33
|
public async create(input: CreateJobManifestRequest): Promise<JobManifest> {
|
|
33
34
|
const newJobManifest: JobManifest = {
|
|
34
35
|
...input,
|
|
36
|
+
autoRetry: defaultAutoRetry(input.autoRetry),
|
|
35
37
|
type: ENTITY_TYPES.manifest,
|
|
36
38
|
};
|
|
37
39
|
|
|
@@ -74,6 +76,11 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
74
76
|
expressionAttributeValues[':image'] = { S: input.image };
|
|
75
77
|
}
|
|
76
78
|
|
|
79
|
+
if (input.autoRetry !== undefined) {
|
|
80
|
+
expressionAttributeNames['#autoRetry'] = 'autoRetry';
|
|
81
|
+
expressionAttributeValues[':autoRetry'] = { BOOL: input.autoRetry };
|
|
82
|
+
}
|
|
83
|
+
|
|
77
84
|
const updateExpressionPairs = Object.keys(input).map((key) => `#${key} = :${key}`);
|
|
78
85
|
const updateExpression = `SET ${updateExpressionPairs.join(', ')}`;
|
|
79
86
|
|
|
@@ -98,12 +105,14 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
98
105
|
sk: { S: `${item.name}~${item.version}` },
|
|
99
106
|
timeoutSeconds: { N: item.timeoutSeconds.toString() },
|
|
100
107
|
image: { S: item.image },
|
|
108
|
+
autoRetry: { BOOL: item.autoRetry },
|
|
101
109
|
};
|
|
102
110
|
}
|
|
103
111
|
|
|
104
112
|
convertToItem(dbItem: JobManifestDto): JobManifest {
|
|
105
113
|
const { pk, name, version, concurrency, entryFile, entryFunction, timeoutSeconds } = dbItem;
|
|
106
114
|
const image = (dbItem as Partial<JobManifestDto>).image?.S;
|
|
115
|
+
const autoRetry = (dbItem as Partial<JobManifestDto>).autoRetry?.BOOL;
|
|
107
116
|
return {
|
|
108
117
|
concurrency: parseInt(concurrency.N as string),
|
|
109
118
|
entryFile: entryFile.S as string,
|
|
@@ -112,6 +121,7 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
112
121
|
version: version.S as string,
|
|
113
122
|
timeoutSeconds: parseInt(timeoutSeconds.N as string),
|
|
114
123
|
image: getRuntimeImageOrDefault(image),
|
|
124
|
+
autoRetry: defaultAutoRetry(autoRetry),
|
|
115
125
|
type: parsePartitionKey(pk.S as string).entityType,
|
|
116
126
|
};
|
|
117
127
|
}
|
|
@@ -14,21 +14,22 @@ import {
|
|
|
14
14
|
ScanCommandOutput,
|
|
15
15
|
} from '@aws-sdk/client-dynamodb';
|
|
16
16
|
import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
|
|
17
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
18
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
19
|
+
import { LEGACY_RUNTIME_IMAGE } from '../../constants/RuntimeImage.constants';
|
|
17
20
|
import {
|
|
21
|
+
createMockClientGetResponse,
|
|
18
22
|
createMockClientPutResponse,
|
|
19
23
|
createMockClientScanResponse,
|
|
20
|
-
createMockClientGetResponse,
|
|
21
24
|
} from '../../core/dynamoClient.mock';
|
|
22
|
-
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
23
25
|
import { PaginationInfo } from '../../core/pagination/model/PaginationInfo';
|
|
24
26
|
import { PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/model/PaginationTypes';
|
|
25
27
|
import * as PaginationUtils from '../../core/pagination/utils/PaginationUtils';
|
|
28
|
+
import { createMockJob, createMockJobRecord } from '../../model/Job';
|
|
26
29
|
import { Job } from '../../model/Job/Job';
|
|
27
30
|
import { createMockJobDto, createMockJobDtos } from '../../model/Job/mocks/JobDto.mock';
|
|
28
31
|
import { createMockCreateJobRequest } from '../../model/Job/mocks/JobRequest.mock';
|
|
29
32
|
import { JobRepository } from './JobRepository';
|
|
30
|
-
import { createMockJob, createMockJobRecord } from '../../model/Job';
|
|
31
|
-
import { InjectTokens } from '../../constants/InjectTokens';
|
|
32
33
|
|
|
33
34
|
initIocContainer();
|
|
34
35
|
|
|
@@ -170,10 +171,33 @@ describe('JobRepository', (): void => {
|
|
|
170
171
|
type: `job`,
|
|
171
172
|
version: mockJobRecord.version.S as string,
|
|
172
173
|
description: mockJobRecord.description.S as string,
|
|
174
|
+
image: mockJobRecord.image.S as string,
|
|
173
175
|
} as Job);
|
|
174
176
|
expect(mockSend).toBeCalledTimes(1);
|
|
175
177
|
});
|
|
176
178
|
|
|
179
|
+
it(`should default missing image to legacy runtime`, async (): Promise<void> => {
|
|
180
|
+
const mockJobRecord = createMockJobDto();
|
|
181
|
+
delete (mockJobRecord as Partial<typeof mockJobRecord>).image;
|
|
182
|
+
mockSend.mockImplementationOnce(
|
|
183
|
+
(): GetItemCommandOutput =>
|
|
184
|
+
createMockClientGetResponse({
|
|
185
|
+
Item: mockJobRecord,
|
|
186
|
+
}),
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
const response = await jobRepository.findOne({
|
|
190
|
+
name: mockJobRecord.name.S as string,
|
|
191
|
+
version: mockJobRecord.version.S as string,
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
expect(response).toEqual(
|
|
195
|
+
expect.objectContaining({
|
|
196
|
+
image: LEGACY_RUNTIME_IMAGE,
|
|
197
|
+
}),
|
|
198
|
+
);
|
|
199
|
+
});
|
|
200
|
+
|
|
177
201
|
it(`should return null if job does not exist`, async (): Promise<void> => {
|
|
178
202
|
mockSend.mockImplementationOnce(
|
|
179
203
|
(): ScanCommandOutput =>
|
|
@@ -313,6 +337,14 @@ describe('JobRepository', (): void => {
|
|
|
313
337
|
});
|
|
314
338
|
});
|
|
315
339
|
|
|
340
|
+
describe('convertToDynamoDbItem', () => {
|
|
341
|
+
it(`should persist image on the DynamoDB item`, () => {
|
|
342
|
+
const job = createMockJob({ image: 'node:24' });
|
|
343
|
+
const dto = jobRepository.convertToDynamoDbItem(job);
|
|
344
|
+
expect(dto.image).toEqual({ S: 'node:24' });
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
|
|
316
348
|
describe('delete', () => {
|
|
317
349
|
it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
|
|
318
350
|
mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
|
|
@@ -11,18 +11,19 @@ import {
|
|
|
11
11
|
QueryCommandOutput,
|
|
12
12
|
} from '@aws-sdk/client-dynamodb';
|
|
13
13
|
import { inject } from 'inversify';
|
|
14
|
-
import {
|
|
15
|
-
import { Repository } from '../AbstractRepository';
|
|
14
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
16
15
|
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
17
|
-
import {
|
|
18
|
-
import { CreateJobRequest, Job, JobDto, JobSk, ListJobParameters } from '../../model/Job';
|
|
16
|
+
import { getRuntimeImageOrDefault } from '../../constants/RuntimeImage.constants';
|
|
19
17
|
import {
|
|
20
18
|
decodeTokenString,
|
|
21
19
|
PAGINATION_DIRECTION_TYPE_BEFORE,
|
|
22
20
|
PaginationInfo,
|
|
23
21
|
PaginationQueryResponse,
|
|
24
22
|
} from '../../core/pagination';
|
|
25
|
-
import {
|
|
23
|
+
import { CreateJobRequest, Job, JobDto, JobSk, ListJobParameters } from '../../model/Job';
|
|
24
|
+
import { parsePartitionKey } from '../../ParsePartitionKey';
|
|
25
|
+
import { provideTransient } from '../../transientProvider';
|
|
26
|
+
import { Repository } from '../AbstractRepository';
|
|
26
27
|
|
|
27
28
|
@provideTransient(JobRepository)
|
|
28
29
|
export class JobRepository extends Repository<Job, JobDto, JobSk> {
|
|
@@ -37,6 +38,7 @@ export class JobRepository extends Repository<Job, JobDto, JobSk> {
|
|
|
37
38
|
public async create(input: CreateJobRequest): Promise<Job> {
|
|
38
39
|
const newJob: Job = {
|
|
39
40
|
...input,
|
|
41
|
+
image: getRuntimeImageOrDefault(input.image),
|
|
40
42
|
type: ENTITY_TYPES.job,
|
|
41
43
|
};
|
|
42
44
|
|
|
@@ -157,11 +159,13 @@ export class JobRepository extends Repository<Job, JobDto, JobSk> {
|
|
|
157
159
|
name: { S: item.name },
|
|
158
160
|
description: item.description ? { S: item.description } : { NULL: true },
|
|
159
161
|
version: { S: item.version },
|
|
162
|
+
image: { S: item.image },
|
|
160
163
|
};
|
|
161
164
|
}
|
|
162
165
|
|
|
163
166
|
convertToItem(dbItem: JobDto): Job {
|
|
164
167
|
const { pk, name, version } = dbItem;
|
|
168
|
+
const image = (dbItem as Partial<JobDto>).image?.S;
|
|
165
169
|
let description;
|
|
166
170
|
// TODO https://jira.squiz.net/browse/CONN-609 remove this check if we tidy up
|
|
167
171
|
// jobs added before this field was implemented
|
|
@@ -175,6 +179,7 @@ export class JobRepository extends Repository<Job, JobDto, JobSk> {
|
|
|
175
179
|
description: description.S ? (description.S as string) : '',
|
|
176
180
|
type: parsePartitionKey(pk.S as string).entityType,
|
|
177
181
|
version: version.S as string,
|
|
182
|
+
image: getRuntimeImageOrDefault(image),
|
|
178
183
|
};
|
|
179
184
|
}
|
|
180
185
|
|
|
@@ -4,6 +4,7 @@ import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
|
|
|
4
4
|
import 'aws-sdk-client-mock-jest';
|
|
5
5
|
import { JobDetailsRepository, JobUploadsRepository } from '../../S3Repository';
|
|
6
6
|
import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
|
|
7
|
+
import { RuntimeImageTaskDefinitionError } from '../../errors';
|
|
7
8
|
import { createMockJobManifest } from '../../manifest';
|
|
8
9
|
import { createMockJobContext } from '../../model/JobContext';
|
|
9
10
|
import { createMockJobExecution } from '../../model/JobExecution';
|
|
@@ -30,7 +31,11 @@ describe('JobExecutionTaskService', () => {
|
|
|
30
31
|
shortRegion: 'au',
|
|
31
32
|
shortEnvironment: 'dev',
|
|
32
33
|
clusterArn: `clusterArn`,
|
|
33
|
-
taskDefinitionArn: `
|
|
34
|
+
taskDefinitionArn: `legacyTaskDefinitionArn`,
|
|
35
|
+
taskDefinitionArnsByRuntimeImage: {
|
|
36
|
+
'node:20': 'node20TaskDefinitionArn',
|
|
37
|
+
'node:24': 'node24TaskDefinitionArn',
|
|
38
|
+
},
|
|
34
39
|
tenantId: 'some_tenant',
|
|
35
40
|
};
|
|
36
41
|
|
|
@@ -104,7 +109,7 @@ describe('JobExecutionTaskService', () => {
|
|
|
104
109
|
const mockArn = `mock-arn`;
|
|
105
110
|
const mockTask = { taskArn: mockArn };
|
|
106
111
|
const jobContext = createMockJobContext();
|
|
107
|
-
const jobManifest = createMockJobManifest();
|
|
112
|
+
const jobManifest = createMockJobManifest({ image: 'node:24' });
|
|
108
113
|
const jobExecution = createMockJobExecution({
|
|
109
114
|
id: `20250917T005949395Z-9602417782`,
|
|
110
115
|
jobName: 'jobName',
|
|
@@ -173,7 +178,7 @@ describe('JobExecutionTaskService', () => {
|
|
|
173
178
|
{ key: 'map-migrated', value: 'migUJ1IX3DUXG' },
|
|
174
179
|
{ key: 'cmp:tenant', value: 'some_tenant' },
|
|
175
180
|
],
|
|
176
|
-
taskDefinition: defaultRunJobTaskParams.
|
|
181
|
+
taskDefinition: defaultRunJobTaskParams.taskDefinitionArnsByRuntimeImage['node:24'],
|
|
177
182
|
// sha1 of execution job name/id + tenant + retryCount.
|
|
178
183
|
clientToken: '4725755e08138edde031a177c36a269ecdbed5a3',
|
|
179
184
|
});
|
|
@@ -207,6 +212,51 @@ describe('JobExecutionTaskService', () => {
|
|
|
207
212
|
null,
|
|
208
213
|
defaultRunJobTaskParams.jobExecution,
|
|
209
214
|
);
|
|
215
|
+
expect(mockEcsClient).toHaveReceivedCommandWith(RunTaskCommand, {
|
|
216
|
+
taskDefinition: defaultRunJobTaskParams.taskDefinitionArnsByRuntimeImage['node:20'],
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
it('should use the legacy task definition ARN when the node:20 map entry is missing', async () => {
|
|
221
|
+
const mockArn = `mock-arn`;
|
|
222
|
+
|
|
223
|
+
mockEcsClient.on(RunTaskCommand).resolves({ tasks: [{ taskArn: mockArn }] });
|
|
224
|
+
mockJobContextService.findJobContext.mockResolvedValue(createMockJobContext());
|
|
225
|
+
mockJobManifestService.findJobManifest.mockResolvedValue(createMockJobManifest({ image: 'node:20' }));
|
|
226
|
+
|
|
227
|
+
await jobExecutionTaskService.runJobExecutionTask({
|
|
228
|
+
...defaultRunJobTaskParams,
|
|
229
|
+
taskDefinitionArnsByRuntimeImage: {},
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
expect(mockEcsClient).toHaveReceivedCommandWith(RunTaskCommand, {
|
|
233
|
+
taskDefinition: defaultRunJobTaskParams.taskDefinitionArn,
|
|
234
|
+
});
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
it('should throw before creating an ECS task when the runtime image is unsupported', async () => {
|
|
238
|
+
mockJobManifestService.findJobManifest.mockResolvedValue(createMockJobManifest({ image: 'node:26' }));
|
|
239
|
+
|
|
240
|
+
await expect(jobExecutionTaskService.runJobExecutionTask(defaultRunJobTaskParams)).rejects.toThrow(
|
|
241
|
+
RuntimeImageTaskDefinitionError,
|
|
242
|
+
);
|
|
243
|
+
expect(mockEcsClient).not.toHaveReceivedCommand(RunTaskCommand);
|
|
244
|
+
expect(mockJobDetailsRepository.createJobDetailsFile).not.toHaveBeenCalled();
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
it('should throw before creating an ECS task when the runtime image has no task definition ARN', async () => {
|
|
248
|
+
mockJobManifestService.findJobManifest.mockResolvedValue(createMockJobManifest({ image: 'node:24' }));
|
|
249
|
+
|
|
250
|
+
await expect(
|
|
251
|
+
jobExecutionTaskService.runJobExecutionTask({
|
|
252
|
+
...defaultRunJobTaskParams,
|
|
253
|
+
taskDefinitionArnsByRuntimeImage: {
|
|
254
|
+
'node:20': 'node20TaskDefinitionArn',
|
|
255
|
+
},
|
|
256
|
+
}),
|
|
257
|
+
).rejects.toThrow(RuntimeImageTaskDefinitionError);
|
|
258
|
+
expect(mockEcsClient).not.toHaveReceivedCommand(RunTaskCommand);
|
|
259
|
+
expect(mockJobDetailsRepository.createJobDetailsFile).not.toHaveBeenCalled();
|
|
210
260
|
});
|
|
211
261
|
|
|
212
262
|
it('should handle ConflictException with a previously started task arn', async () => {
|