@squiz/job-runner-lib 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/lib/domains/job/constants/QueueBacklog.constants.d.ts +11 -0
- package/lib/domains/job/constants/RetryConfig.constants.d.ts +6 -1
- package/lib/domains/job/constants/RuntimeImage.constants.d.ts +5 -0
- package/lib/domains/job/errors/RuntimeImageTaskDefinitionError.d.ts +7 -0
- package/lib/domains/job/errors/index.d.ts +1 -0
- package/lib/domains/job/index.d.ts +1 -0
- package/lib/domains/job/manifest/JobManifestTypes.d.ts +13 -1
- package/lib/domains/job/model/Job/Job.d.ts +12 -0
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +15 -0
- package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +1 -1
- package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +2 -2
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +1 -0
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -0
- package/lib/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.d.ts +6 -0
- package/lib/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.types.d.ts +9 -0
- package/lib/domains/job/utils/index.d.ts +2 -0
- package/lib/domains/manifest/index.d.ts +3 -2
- package/lib/domains/manifest/utils/defaultAutoRetry.d.ts +5 -0
- package/lib/domains/manifest/utils/defaultAutoRetry.spec.d.ts +5 -0
- package/lib/index.js +9 -9
- package/lib/index.js.map +4 -4
- package/package.json +2 -2
- package/src/domains/job/constants/QueueBacklog.constants.ts +12 -0
- package/src/domains/job/constants/RetryConfig.constants.ts +8 -2
- package/src/domains/job/constants/RuntimeImage.constants.ts +8 -2
- package/src/domains/job/errors/RuntimeImageTaskDefinitionError.ts +10 -0
- package/src/domains/job/errors/index.ts +1 -0
- package/src/domains/job/index.ts +1 -0
- package/src/domains/job/manifest/JobManifestTypes.ts +16 -1
- package/src/domains/job/manifest/mocks/JobManifest.mock.ts +1 -0
- package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -0
- package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +1 -0
- package/src/domains/job/manifest/mocks/JobManifestRequest.mock.ts +1 -0
- package/src/domains/job/model/Job/Job.ts +14 -0
- package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -0
- package/src/domains/job/model/Job/mocks/JobDto.mock.ts +4 -0
- package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +2 -0
- package/src/domains/job/model/Job/mocks/JobRequest.mock.ts +2 -0
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +65 -0
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +53 -0
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +56 -4
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +14 -4
- package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +36 -4
- package/src/domains/job/repository/JobRepository/JobRepository.ts +10 -5
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -0
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +53 -3
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +19 -8
- package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +9 -8
- package/src/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.ts +39 -0
- package/src/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.types.ts +10 -0
- package/src/domains/job/utils/index.ts +2 -0
- package/src/domains/manifest/index.ts +3 -2
- package/src/domains/manifest/utils/defaultAutoRetry.spec.ts +16 -0
- package/src/domains/manifest/utils/defaultAutoRetry.ts +6 -0
- package/src/domains/manifest/utils/validateRuntimeImage.spec.ts +11 -12
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@squiz/job-runner-lib",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.9.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"private": false,
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"@aws-sdk/client-sqs": "3.1003.0",
|
|
36
36
|
"@aws-sdk/s3-request-presigner": "3.1003.0",
|
|
37
37
|
"@squiz/dx-common-lib": "1.66.1",
|
|
38
|
-
"@squiz/dx-json-schema-lib": "1.
|
|
38
|
+
"@squiz/dx-json-schema-lib": "1.86.0",
|
|
39
39
|
"@squiz/dx-logger-lib": "1.64.0",
|
|
40
40
|
"@squiz/optimization-logger": "1.11.2",
|
|
41
41
|
"axios": "1.16.1",
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Backlog guard defaults. A job's `begin` is rejected with 422 once it has more than
|
|
8
|
+
* DEFAULT_MAX_QUEUED_EXECUTIONS queued AND its oldest queued execution is older than
|
|
9
|
+
* DEFAULT_MAX_QUEUED_AGE_HOURS. Both conditions are required (large AND stale).
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULT_MAX_QUEUED_EXECUTIONS = 500;
|
|
12
|
+
export const DEFAULT_MAX_QUEUED_AGE_HOURS = 2;
|
|
@@ -7,15 +7,21 @@
|
|
|
7
7
|
* Retry configuration with exponential backoff delays in seconds.
|
|
8
8
|
* Using SQS message timer limits (max 15 minutes = 900 seconds).
|
|
9
9
|
*
|
|
10
|
-
* Delays: 30s, 1min, 2min, 5min, 10min, 15min
|
|
10
|
+
* Delays: 30s, 1min, 2min, 5min, 10min, then 15min thereafter.
|
|
11
11
|
*/
|
|
12
|
-
export const RETRY_DELAYS_SECONDS: ReadonlyArray<number> = [30, 60, 120, 300, 600, 900];
|
|
12
|
+
export const RETRY_DELAYS_SECONDS: ReadonlyArray<number> = [30, 60, 120, 300, 600, 900, 900, 900, 900, 900];
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* Maximum number of retry attempts for container start failures.
|
|
16
16
|
*/
|
|
17
17
|
export const MAX_RETRY_ATTEMPTS = RETRY_DELAYS_SECONDS.length;
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Maximum number of total attempts for opted-in job code failures.
|
|
21
|
+
* This allows the initial attempt plus 3 retries.
|
|
22
|
+
*/
|
|
23
|
+
export const MAX_JOB_CODE_RETRY_ATTEMPTS = 4;
|
|
24
|
+
|
|
19
25
|
/**
|
|
20
26
|
* After a job has attempted to be started this number of times alerts will start to be sent.
|
|
21
27
|
*/
|
|
@@ -11,12 +11,18 @@ export const LEGACY_RUNTIME_IMAGE = 'node:20';
|
|
|
11
11
|
export const RECOMMENDED_RUNTIME_IMAGE = 'node:24';
|
|
12
12
|
|
|
13
13
|
// Still runnable but rejected on new uploads.
|
|
14
|
-
export const DEPRECATED_RUNTIME_IMAGES: readonly string[] = [];
|
|
14
|
+
export const DEPRECATED_RUNTIME_IMAGES: readonly string[] = ['node:20'];
|
|
15
15
|
|
|
16
16
|
export const getRuntimeImageOrDefault = (image: string | null | undefined): string =>
|
|
17
|
-
image
|
|
17
|
+
image === null || image === undefined ? LEGACY_RUNTIME_IMAGE : image;
|
|
18
18
|
|
|
19
19
|
export const isSupportedRuntimeImage = (image: string): boolean =>
|
|
20
20
|
(SUPPORTED_RUNTIME_IMAGES as readonly string[]).includes(image);
|
|
21
21
|
|
|
22
22
|
export const isDeprecatedRuntimeImage = (image: string): boolean => DEPRECATED_RUNTIME_IMAGES.includes(image);
|
|
23
|
+
|
|
24
|
+
export const getRuntimeImagesMetadata = () => ({
|
|
25
|
+
supportedRuntimeImages: [...SUPPORTED_RUNTIME_IMAGES],
|
|
26
|
+
deprecatedRuntimeImages: [...DEPRECATED_RUNTIME_IMAGES],
|
|
27
|
+
recommendedRuntimeImage: RECOMMENDED_RUNTIME_IMAGE,
|
|
28
|
+
});
|
package/src/domains/job/index.ts
CHANGED
|
@@ -3,6 +3,7 @@ export * from './core/pagination';
|
|
|
3
3
|
|
|
4
4
|
export * from './constants/InjectTokens';
|
|
5
5
|
export * from './constants/JobExecutionStatus.constants';
|
|
6
|
+
export * from './constants/QueueBacklog.constants';
|
|
6
7
|
export * from './constants/Repository.constants';
|
|
7
8
|
export * from './constants/RetryConfig.constants';
|
|
8
9
|
export * from './constants/RuntimeImage.constants';
|
|
@@ -64,6 +64,13 @@ export interface JobManifest {
|
|
|
64
64
|
*/
|
|
65
65
|
image: string;
|
|
66
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Whether failed job code executions should be retried automatically.
|
|
69
|
+
* @type {boolean}
|
|
70
|
+
* @memberof JobManifest
|
|
71
|
+
*/
|
|
72
|
+
autoRetry: boolean;
|
|
73
|
+
|
|
67
74
|
/**
|
|
68
75
|
* The DynamoDB record type ('job').
|
|
69
76
|
* @type {EntityType}
|
|
@@ -142,6 +149,13 @@ export interface JobManifestRecord {
|
|
|
142
149
|
* @memberof JobManifestRecord
|
|
143
150
|
*/
|
|
144
151
|
image: string;
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Whether failed job code executions should be retried automatically.
|
|
155
|
+
* @type {boolean}
|
|
156
|
+
* @memberof JobManifestRecord
|
|
157
|
+
*/
|
|
158
|
+
autoRetry: boolean;
|
|
145
159
|
}
|
|
146
160
|
|
|
147
161
|
export type JobManifestDto = Record<keyof JobManifestRecord, AttributeValue>;
|
|
@@ -150,7 +164,8 @@ export type JobManifestDto = Record<keyof JobManifestRecord, AttributeValue>;
|
|
|
150
164
|
* The request body for creating a job (POST /job-manifest).
|
|
151
165
|
* @export
|
|
152
166
|
*/
|
|
153
|
-
export type CreateJobManifestRequest = Omit<JobManifest, 'type'
|
|
167
|
+
export type CreateJobManifestRequest = Omit<JobManifest, 'type' | 'autoRetry'> &
|
|
168
|
+
Partial<Pick<JobManifest, 'autoRetry'>>;
|
|
154
169
|
|
|
155
170
|
/**
|
|
156
171
|
* The request body for updating a job (PUT /job-manifest).
|
|
@@ -26,6 +26,7 @@ export const createMockJobManifest = (partialRecord: Partial<JobManifest> = {}):
|
|
|
26
26
|
version,
|
|
27
27
|
timeoutSeconds: faker.number.int({ max: 28800, min: 5 }),
|
|
28
28
|
image: LEGACY_RUNTIME_IMAGE,
|
|
29
|
+
autoRetry: false,
|
|
29
30
|
type: ENTITY_TYPES.manifest,
|
|
30
31
|
...partialRecord,
|
|
31
32
|
};
|
|
@@ -34,6 +34,13 @@ export interface Job {
|
|
|
34
34
|
*/
|
|
35
35
|
version: string;
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* The runtime image the job should execute against (e.g. `node:20`, `node:24`).
|
|
39
|
+
* @type {string}
|
|
40
|
+
* @memberof Job
|
|
41
|
+
*/
|
|
42
|
+
image: string;
|
|
43
|
+
|
|
37
44
|
/**
|
|
38
45
|
* The DynamoDB record type ('job').
|
|
39
46
|
* @type {EntityType}
|
|
@@ -82,6 +89,13 @@ export interface JobRecord {
|
|
|
82
89
|
* @memberof JobRecord
|
|
83
90
|
*/
|
|
84
91
|
version: string;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The runtime image the job should execute against (e.g. `node:20`, `node:24`).
|
|
95
|
+
* @type {string}
|
|
96
|
+
* @memberof JobRecord
|
|
97
|
+
*/
|
|
98
|
+
image: string;
|
|
85
99
|
}
|
|
86
100
|
|
|
87
101
|
export type JobDto = Record<keyof JobRecord, AttributeValue>;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
7
|
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
8
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
8
9
|
import { Job } from '../Job';
|
|
9
10
|
|
|
10
11
|
export const createMockJob = (partialRecord: Partial<Job> = {}): Job => {
|
|
@@ -16,6 +17,7 @@ export const createMockJob = (partialRecord: Partial<Job> = {}): Job => {
|
|
|
16
17
|
description: faker.lorem.sentence(),
|
|
17
18
|
type: ENTITY_TYPES.job,
|
|
18
19
|
version: version,
|
|
20
|
+
image: RECOMMENDED_RUNTIME_IMAGE,
|
|
19
21
|
...partialRecord,
|
|
20
22
|
};
|
|
21
23
|
};
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
7
|
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
8
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
8
9
|
import { JobDto, JobRecord } from '../Job';
|
|
9
10
|
|
|
10
11
|
export const createMockJobDto = (partialRecord: Partial<JobRecord> = {}): JobDto => {
|
|
@@ -29,6 +30,9 @@ export const createMockJobDto = (partialRecord: Partial<JobRecord> = {}): JobDto
|
|
|
29
30
|
version: {
|
|
30
31
|
S: partialRecord.version ?? version,
|
|
31
32
|
},
|
|
33
|
+
image: {
|
|
34
|
+
S: partialRecord.image ?? RECOMMENDED_RUNTIME_IMAGE,
|
|
35
|
+
},
|
|
32
36
|
};
|
|
33
37
|
};
|
|
34
38
|
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
7
|
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
8
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
8
9
|
import { JobRecord } from '../Job';
|
|
9
10
|
|
|
10
11
|
export const createMockJobRecord = (partialRecord: Partial<JobRecord> = {}): JobRecord => {
|
|
@@ -19,6 +20,7 @@ export const createMockJobRecord = (partialRecord: Partial<JobRecord> = {}): Job
|
|
|
19
20
|
name: faker.lorem.word(),
|
|
20
21
|
description: faker.lorem.sentence(),
|
|
21
22
|
version,
|
|
23
|
+
image: RECOMMENDED_RUNTIME_IMAGE,
|
|
22
24
|
...partialRecord,
|
|
23
25
|
};
|
|
24
26
|
};
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
|
+
import { RECOMMENDED_RUNTIME_IMAGE } from '../../../constants/RuntimeImage.constants';
|
|
7
8
|
import { CreateJobRequest } from '../Job';
|
|
8
9
|
|
|
9
10
|
export const createMockCreateJobRequest = (partialRecord: Partial<CreateJobRequest> = {}): CreateJobRequest => {
|
|
@@ -15,6 +16,7 @@ export const createMockCreateJobRequest = (partialRecord: Partial<CreateJobReque
|
|
|
15
16
|
name: faker.lorem.word(),
|
|
16
17
|
description: faker.lorem.sentence(),
|
|
17
18
|
version,
|
|
19
|
+
image: RECOMMENDED_RUNTIME_IMAGE,
|
|
18
20
|
...partialRecord,
|
|
19
21
|
};
|
|
20
22
|
};
|
|
@@ -381,6 +381,71 @@ describe('JobExecutionRepository', (): void => {
|
|
|
381
381
|
});
|
|
382
382
|
});
|
|
383
383
|
|
|
384
|
+
describe('getOldestQueuedExecution', () => {
|
|
385
|
+
const jobName = 'backed-up-job';
|
|
386
|
+
|
|
387
|
+
it('queries lsi1 for the oldest queued execution (ascending, limit 1)', async () => {
|
|
388
|
+
const mockJobExecution = createMockJobExecution({ jobName, status: JOB_EXECUTION_STATUS.queued });
|
|
389
|
+
dynamoClient.on(QueryCommand).resolvesOnce({ Items: [createMockJobExecutionDto(mockJobExecution)] });
|
|
390
|
+
|
|
391
|
+
const result = await jobExecutionRepository.getOldestQueuedExecution(jobName);
|
|
392
|
+
|
|
393
|
+
expect(result).toEqual(mockJobExecution);
|
|
394
|
+
expect(dynamoClient).toHaveReceivedCommandWith(QueryCommand, {
|
|
395
|
+
TableName: tableName,
|
|
396
|
+
IndexName: 'lsi1',
|
|
397
|
+
KeyConditionExpression: '#pk = :pk and #lsi1sk = :lsi1sk',
|
|
398
|
+
ExpressionAttributeNames: { '#pk': 'pk', '#lsi1sk': 'lsi1sk' },
|
|
399
|
+
ExpressionAttributeValues: {
|
|
400
|
+
':pk': { S: `${ENTITY_TYPES.jobExecution}~${tenant}` },
|
|
401
|
+
':lsi1sk': { S: `${JOB_EXECUTION_STATUS.queued}~${jobName}` },
|
|
402
|
+
},
|
|
403
|
+
ScanIndexForward: true,
|
|
404
|
+
Limit: 1,
|
|
405
|
+
});
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
it('returns null when there are no queued executions', async () => {
|
|
409
|
+
dynamoClient.on(QueryCommand).resolvesOnce({ Items: [] });
|
|
410
|
+
|
|
411
|
+
const result = await jobExecutionRepository.getOldestQueuedExecution(jobName);
|
|
412
|
+
|
|
413
|
+
expect(result).toBeNull();
|
|
414
|
+
});
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
describe('countQueuedExecutions', () => {
|
|
418
|
+
const jobName = 'backed-up-job';
|
|
419
|
+
|
|
420
|
+
it('counts queued executions on lsi1 bounded by limit', async () => {
|
|
421
|
+
dynamoClient.on(QueryCommand).resolvesOnce({ Count: 501 });
|
|
422
|
+
|
|
423
|
+
const result = await jobExecutionRepository.countQueuedExecutions(jobName, 501);
|
|
424
|
+
|
|
425
|
+
expect(result).toBe(501);
|
|
426
|
+
expect(dynamoClient).toHaveReceivedCommandWith(QueryCommand, {
|
|
427
|
+
TableName: tableName,
|
|
428
|
+
IndexName: 'lsi1',
|
|
429
|
+
Select: 'COUNT',
|
|
430
|
+
KeyConditionExpression: '#pk = :pk and #lsi1sk = :lsi1sk',
|
|
431
|
+
ExpressionAttributeNames: { '#pk': 'pk', '#lsi1sk': 'lsi1sk' },
|
|
432
|
+
ExpressionAttributeValues: {
|
|
433
|
+
':pk': { S: `${ENTITY_TYPES.jobExecution}~${tenant}` },
|
|
434
|
+
':lsi1sk': { S: `${JOB_EXECUTION_STATUS.queued}~${jobName}` },
|
|
435
|
+
},
|
|
436
|
+
Limit: 501,
|
|
437
|
+
});
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
it('returns 0 when DynamoDB returns no count', async () => {
|
|
441
|
+
dynamoClient.on(QueryCommand).resolvesOnce({});
|
|
442
|
+
|
|
443
|
+
const result = await jobExecutionRepository.countQueuedExecutions(jobName, 501);
|
|
444
|
+
|
|
445
|
+
expect(result).toBe(0);
|
|
446
|
+
});
|
|
447
|
+
});
|
|
448
|
+
|
|
384
449
|
describe('update', () => {
|
|
385
450
|
it('Updates the record properties', async () => {
|
|
386
451
|
const jobExecution = createMockJobExecution({ status: JOB_EXECUTION_STATUS.queued });
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
QueryCommand,
|
|
10
10
|
QueryCommandInput,
|
|
11
11
|
QueryCommandOutput,
|
|
12
|
+
Select,
|
|
12
13
|
UpdateItemCommand,
|
|
13
14
|
UpdateItemCommandInput,
|
|
14
15
|
} from '@aws-sdk/client-dynamodb';
|
|
@@ -136,6 +137,58 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
136
137
|
return results.map((item) => this.convertToItem(item as JobExecutionDto));
|
|
137
138
|
}
|
|
138
139
|
|
|
140
|
+
/**
|
|
141
|
+
* Returns the single oldest queued execution for a job, or null if there are none.
|
|
142
|
+
*
|
|
143
|
+
* Queries the `lsi1` index (`lsi1sk = "<status>~<jobName>"`). All queued executions of one
|
|
144
|
+
* job share the same `lsi1sk`, so DynamoDB breaks ties by the base-table `sk`
|
|
145
|
+
* (`"<jobName>~<id>"`); `id` is prefixed with the ISO `timeQueued`, which sorts
|
|
146
|
+
* lexicographically, so ascending order is chronological and the first item is the oldest.
|
|
147
|
+
*/
|
|
148
|
+
async getOldestQueuedExecution(jobName: string): Promise<JobExecution | null> {
|
|
149
|
+
const res = await this.dynamoDbClient.send(
|
|
150
|
+
new QueryCommand({
|
|
151
|
+
TableName: this.tableName,
|
|
152
|
+
IndexName: 'lsi1',
|
|
153
|
+
KeyConditionExpression: '#pk = :pk and #lsi1sk = :lsi1sk',
|
|
154
|
+
ExpressionAttributeNames: { '#pk': 'pk', '#lsi1sk': 'lsi1sk' },
|
|
155
|
+
ExpressionAttributeValues: {
|
|
156
|
+
':pk': { S: this.pk },
|
|
157
|
+
':lsi1sk': { S: `${JOB_EXECUTION_STATUS.queued}~${jobName}` },
|
|
158
|
+
},
|
|
159
|
+
ScanIndexForward: true,
|
|
160
|
+
Limit: 1,
|
|
161
|
+
}),
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
const item = res.Items?.[0];
|
|
165
|
+
return item ? this.convertToItem(item as JobExecutionDto) : null;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Counts queued executions for a job, evaluating at most `limit` items (`Select: COUNT` so no
|
|
170
|
+
* item data is returned). Pass a `limit` just above the threshold you care about — queued
|
|
171
|
+
* items are small, so a single page covers the bounded count.
|
|
172
|
+
*/
|
|
173
|
+
async countQueuedExecutions(jobName: string, limit?: number): Promise<number> {
|
|
174
|
+
const res = await this.dynamoDbClient.send(
|
|
175
|
+
new QueryCommand({
|
|
176
|
+
TableName: this.tableName,
|
|
177
|
+
IndexName: 'lsi1',
|
|
178
|
+
Select: Select.COUNT,
|
|
179
|
+
KeyConditionExpression: '#pk = :pk and #lsi1sk = :lsi1sk',
|
|
180
|
+
ExpressionAttributeNames: { '#pk': 'pk', '#lsi1sk': 'lsi1sk' },
|
|
181
|
+
ExpressionAttributeValues: {
|
|
182
|
+
':pk': { S: this.pk },
|
|
183
|
+
':lsi1sk': { S: `${JOB_EXECUTION_STATUS.queued}~${jobName}` },
|
|
184
|
+
},
|
|
185
|
+
Limit: limit,
|
|
186
|
+
}),
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
return res.Count ?? 0;
|
|
190
|
+
}
|
|
191
|
+
|
|
139
192
|
/**
|
|
140
193
|
* Get list of records from database.
|
|
141
194
|
* @abstract
|
|
@@ -12,21 +12,21 @@ import {
|
|
|
12
12
|
ScanCommandOutput,
|
|
13
13
|
} from '@aws-sdk/client-dynamodb';
|
|
14
14
|
import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
|
|
15
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
16
|
+
import { LEGACY_RUNTIME_IMAGE } from '../../constants/RuntimeImage.constants';
|
|
15
17
|
import {
|
|
18
|
+
createMockClientGetResponse,
|
|
16
19
|
createMockClientPutResponse,
|
|
17
20
|
createMockClientScanResponse,
|
|
18
|
-
createMockClientGetResponse,
|
|
19
21
|
} from '../../core/dynamoClient.mock';
|
|
20
|
-
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
21
22
|
import {
|
|
22
23
|
JobManifest,
|
|
24
|
+
createMockCreateJobManifestRequest,
|
|
23
25
|
createMockJobManifest,
|
|
24
26
|
createMockJobManifestDto,
|
|
25
27
|
createMockJobManifestDtos,
|
|
26
28
|
createMockJobManifestRecord,
|
|
27
|
-
createMockCreateJobManifestRequest,
|
|
28
29
|
} from '../../manifest';
|
|
29
|
-
import { LEGACY_RUNTIME_IMAGE } from '../../constants/RuntimeImage.constants';
|
|
30
30
|
import { JobManifestRepository } from './JobManifestRepository';
|
|
31
31
|
|
|
32
32
|
initIocContainer();
|
|
@@ -120,6 +120,7 @@ describe('JobManifestRepository', (): void => {
|
|
|
120
120
|
entryFunction: JSON.parse(mockJobManifestItem.entryFunction.S as string),
|
|
121
121
|
timeoutSeconds: parseInt(mockJobManifestItem.timeoutSeconds.N as string),
|
|
122
122
|
image: 'node:24',
|
|
123
|
+
autoRetry: false,
|
|
123
124
|
type: `manifest`,
|
|
124
125
|
} as JobManifest);
|
|
125
126
|
expect(mockSend).toBeCalledTimes(1);
|
|
@@ -143,6 +144,51 @@ describe('JobManifestRepository', (): void => {
|
|
|
143
144
|
expect(response).toEqual(
|
|
144
145
|
expect.objectContaining({
|
|
145
146
|
image: LEGACY_RUNTIME_IMAGE,
|
|
147
|
+
autoRetry: false,
|
|
148
|
+
}),
|
|
149
|
+
);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it(`should default missing autoRetry to false`, async (): Promise<void> => {
|
|
153
|
+
const mockJobManifestItem = createMockJobManifestDto();
|
|
154
|
+
delete (mockJobManifestItem as Partial<typeof mockJobManifestItem>).autoRetry;
|
|
155
|
+
|
|
156
|
+
mockSend.mockImplementationOnce(
|
|
157
|
+
(): GetItemCommandOutput =>
|
|
158
|
+
createMockClientGetResponse({
|
|
159
|
+
Item: mockJobManifestItem,
|
|
160
|
+
}),
|
|
161
|
+
);
|
|
162
|
+
|
|
163
|
+
const response = await jobManifestRepository.findOne({
|
|
164
|
+
name: mockJobManifestItem.name.S as string,
|
|
165
|
+
version: mockJobManifestItem.version.S as string,
|
|
166
|
+
});
|
|
167
|
+
expect(response).toEqual(
|
|
168
|
+
expect.objectContaining({
|
|
169
|
+
autoRetry: false,
|
|
170
|
+
}),
|
|
171
|
+
);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it(`should preserve an explicit unsupported runtime image`, async (): Promise<void> => {
|
|
175
|
+
const mockJobManifestItem = createMockJobManifestDto({ image: 'node:26' });
|
|
176
|
+
|
|
177
|
+
mockSend.mockImplementationOnce(
|
|
178
|
+
(): GetItemCommandOutput =>
|
|
179
|
+
createMockClientGetResponse({
|
|
180
|
+
Item: mockJobManifestItem,
|
|
181
|
+
}),
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
const response = await jobManifestRepository.findOne({
|
|
185
|
+
name: mockJobManifestItem.name.S as string,
|
|
186
|
+
version: mockJobManifestItem.version.S as string,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
expect(response).toEqual(
|
|
190
|
+
expect.objectContaining({
|
|
191
|
+
image: 'node:26',
|
|
146
192
|
}),
|
|
147
193
|
);
|
|
148
194
|
});
|
|
@@ -221,6 +267,12 @@ describe('JobManifestRepository', (): void => {
|
|
|
221
267
|
const dto = jobManifestRepository.convertToDynamoDbItem(manifest);
|
|
222
268
|
expect(dto.image).toEqual({ S: 'node:24' });
|
|
223
269
|
});
|
|
270
|
+
|
|
271
|
+
it(`should persist autoRetry on the DynamoDB item`, () => {
|
|
272
|
+
const manifest = createMockJobManifest({ autoRetry: true });
|
|
273
|
+
const dto = jobManifestRepository.convertToDynamoDbItem(manifest);
|
|
274
|
+
expect(dto.autoRetry).toEqual({ BOOL: true });
|
|
275
|
+
});
|
|
224
276
|
});
|
|
225
277
|
|
|
226
278
|
describe('delete', () => {
|
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
|
|
6
6
|
import { AttributeValue, DynamoDBClient, UpdateItemCommand, UpdateItemCommandInput } from '@aws-sdk/client-dynamodb';
|
|
7
7
|
import { inject } from 'inversify';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { defaultAutoRetry } from '../../../manifest/utils/defaultAutoRetry';
|
|
9
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
10
10
|
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
11
|
+
import { getRuntimeImageOrDefault } from '../../constants/RuntimeImage.constants';
|
|
11
12
|
import {
|
|
12
13
|
CreateJobManifestRequest,
|
|
13
14
|
JobManifest,
|
|
@@ -15,9 +16,9 @@ import {
|
|
|
15
16
|
JobManifestSk,
|
|
16
17
|
UpdateJobManifestRequest,
|
|
17
18
|
} from '../../manifest';
|
|
18
|
-
import { InjectTokens } from '../../constants/InjectTokens';
|
|
19
|
-
import { getRuntimeImageOrDefault } from '../../constants/RuntimeImage.constants';
|
|
20
19
|
import { parsePartitionKey } from '../../ParsePartitionKey';
|
|
20
|
+
import { provideTransient } from '../../transientProvider';
|
|
21
|
+
import { Repository } from '../AbstractRepository';
|
|
21
22
|
|
|
22
23
|
@provideTransient(JobManifestRepository)
|
|
23
24
|
export class JobManifestRepository extends Repository<JobManifest, JobManifestDto, JobManifestSk> {
|
|
@@ -32,6 +33,7 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
32
33
|
public async create(input: CreateJobManifestRequest): Promise<JobManifest> {
|
|
33
34
|
const newJobManifest: JobManifest = {
|
|
34
35
|
...input,
|
|
36
|
+
autoRetry: defaultAutoRetry(input.autoRetry),
|
|
35
37
|
type: ENTITY_TYPES.manifest,
|
|
36
38
|
};
|
|
37
39
|
|
|
@@ -74,6 +76,11 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
74
76
|
expressionAttributeValues[':image'] = { S: input.image };
|
|
75
77
|
}
|
|
76
78
|
|
|
79
|
+
if (input.autoRetry !== undefined) {
|
|
80
|
+
expressionAttributeNames['#autoRetry'] = 'autoRetry';
|
|
81
|
+
expressionAttributeValues[':autoRetry'] = { BOOL: input.autoRetry };
|
|
82
|
+
}
|
|
83
|
+
|
|
77
84
|
const updateExpressionPairs = Object.keys(input).map((key) => `#${key} = :${key}`);
|
|
78
85
|
const updateExpression = `SET ${updateExpressionPairs.join(', ')}`;
|
|
79
86
|
|
|
@@ -98,12 +105,14 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
98
105
|
sk: { S: `${item.name}~${item.version}` },
|
|
99
106
|
timeoutSeconds: { N: item.timeoutSeconds.toString() },
|
|
100
107
|
image: { S: item.image },
|
|
108
|
+
autoRetry: { BOOL: item.autoRetry },
|
|
101
109
|
};
|
|
102
110
|
}
|
|
103
111
|
|
|
104
112
|
convertToItem(dbItem: JobManifestDto): JobManifest {
|
|
105
113
|
const { pk, name, version, concurrency, entryFile, entryFunction, timeoutSeconds } = dbItem;
|
|
106
114
|
const image = (dbItem as Partial<JobManifestDto>).image?.S;
|
|
115
|
+
const autoRetry = (dbItem as Partial<JobManifestDto>).autoRetry?.BOOL;
|
|
107
116
|
return {
|
|
108
117
|
concurrency: parseInt(concurrency.N as string),
|
|
109
118
|
entryFile: entryFile.S as string,
|
|
@@ -112,6 +121,7 @@ export class JobManifestRepository extends Repository<JobManifest, JobManifestDt
|
|
|
112
121
|
version: version.S as string,
|
|
113
122
|
timeoutSeconds: parseInt(timeoutSeconds.N as string),
|
|
114
123
|
image: getRuntimeImageOrDefault(image),
|
|
124
|
+
autoRetry: defaultAutoRetry(autoRetry),
|
|
115
125
|
type: parsePartitionKey(pk.S as string).entityType,
|
|
116
126
|
};
|
|
117
127
|
}
|
|
@@ -14,21 +14,22 @@ import {
|
|
|
14
14
|
ScanCommandOutput,
|
|
15
15
|
} from '@aws-sdk/client-dynamodb';
|
|
16
16
|
import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
|
|
17
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
18
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
19
|
+
import { LEGACY_RUNTIME_IMAGE } from '../../constants/RuntimeImage.constants';
|
|
17
20
|
import {
|
|
21
|
+
createMockClientGetResponse,
|
|
18
22
|
createMockClientPutResponse,
|
|
19
23
|
createMockClientScanResponse,
|
|
20
|
-
createMockClientGetResponse,
|
|
21
24
|
} from '../../core/dynamoClient.mock';
|
|
22
|
-
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
23
25
|
import { PaginationInfo } from '../../core/pagination/model/PaginationInfo';
|
|
24
26
|
import { PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/model/PaginationTypes';
|
|
25
27
|
import * as PaginationUtils from '../../core/pagination/utils/PaginationUtils';
|
|
28
|
+
import { createMockJob, createMockJobRecord } from '../../model/Job';
|
|
26
29
|
import { Job } from '../../model/Job/Job';
|
|
27
30
|
import { createMockJobDto, createMockJobDtos } from '../../model/Job/mocks/JobDto.mock';
|
|
28
31
|
import { createMockCreateJobRequest } from '../../model/Job/mocks/JobRequest.mock';
|
|
29
32
|
import { JobRepository } from './JobRepository';
|
|
30
|
-
import { createMockJob, createMockJobRecord } from '../../model/Job';
|
|
31
|
-
import { InjectTokens } from '../../constants/InjectTokens';
|
|
32
33
|
|
|
33
34
|
initIocContainer();
|
|
34
35
|
|
|
@@ -170,10 +171,33 @@ describe('JobRepository', (): void => {
|
|
|
170
171
|
type: `job`,
|
|
171
172
|
version: mockJobRecord.version.S as string,
|
|
172
173
|
description: mockJobRecord.description.S as string,
|
|
174
|
+
image: mockJobRecord.image.S as string,
|
|
173
175
|
} as Job);
|
|
174
176
|
expect(mockSend).toBeCalledTimes(1);
|
|
175
177
|
});
|
|
176
178
|
|
|
179
|
+
it(`should default missing image to legacy runtime`, async (): Promise<void> => {
|
|
180
|
+
const mockJobRecord = createMockJobDto();
|
|
181
|
+
delete (mockJobRecord as Partial<typeof mockJobRecord>).image;
|
|
182
|
+
mockSend.mockImplementationOnce(
|
|
183
|
+
(): GetItemCommandOutput =>
|
|
184
|
+
createMockClientGetResponse({
|
|
185
|
+
Item: mockJobRecord,
|
|
186
|
+
}),
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
const response = await jobRepository.findOne({
|
|
190
|
+
name: mockJobRecord.name.S as string,
|
|
191
|
+
version: mockJobRecord.version.S as string,
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
expect(response).toEqual(
|
|
195
|
+
expect.objectContaining({
|
|
196
|
+
image: LEGACY_RUNTIME_IMAGE,
|
|
197
|
+
}),
|
|
198
|
+
);
|
|
199
|
+
});
|
|
200
|
+
|
|
177
201
|
it(`should return null if job does not exist`, async (): Promise<void> => {
|
|
178
202
|
mockSend.mockImplementationOnce(
|
|
179
203
|
(): ScanCommandOutput =>
|
|
@@ -313,6 +337,14 @@ describe('JobRepository', (): void => {
|
|
|
313
337
|
});
|
|
314
338
|
});
|
|
315
339
|
|
|
340
|
+
describe('convertToDynamoDbItem', () => {
|
|
341
|
+
it(`should persist image on the DynamoDB item`, () => {
|
|
342
|
+
const job = createMockJob({ image: 'node:24' });
|
|
343
|
+
const dto = jobRepository.convertToDynamoDbItem(job);
|
|
344
|
+
expect(dto.image).toEqual({ S: 'node:24' });
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
|
|
316
348
|
describe('delete', () => {
|
|
317
349
|
it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
|
|
318
350
|
mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
|