@squiz/job-runner-lib 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
- package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
- package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
- package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
- package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
- package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
- package/lib/domains/job/S3Repository/index.d.ts +2 -1
- package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
- package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
- package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
- package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
- package/lib/domains/job/index.d.ts +6 -4
- package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
- package/lib/domains/job/model/Base/Base.d.ts +10 -5
- package/lib/domains/job/model/Job/Job.d.ts +9 -6
- package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
- package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
- package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
- package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
- package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
- package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
- package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
- package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
- package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
- package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
- package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
- package/lib/domains/job/repository/index.d.ts +1 -0
- package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -3
- package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
- package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
- package/lib/domains/job/services/index.d.ts +2 -1
- package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
- package/lib/index.d.ts +1 -0
- package/lib/index.js +13 -13
- package/lib/index.js.map +4 -4
- package/lib/jest-global-setup.d.ts +6 -1
- package/lib/mocks/mockIoc.d.ts +3 -3
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
- package/lib/utils/index.d.ts +2 -0
- package/package.json +12 -11
- package/src/domains/component/errors/JobError.spec.ts +30 -0
- package/src/domains/component/errors/JobError.ts +8 -1
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
- package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
- package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
- package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
- package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
- package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
- package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
- package/src/domains/job/S3Repository/index.ts +2 -1
- package/src/domains/job/constants/InjectTokens.ts +8 -0
- package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
- package/src/domains/job/constants/Repository.constants.ts +3 -3
- package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
- package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
- package/src/domains/job/index.ts +6 -4
- package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
- package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
- package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
- package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
- package/src/domains/job/model/Base/Base.ts +11 -5
- package/src/domains/job/model/Job/Job.ts +11 -6
- package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
- package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
- package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
- package/src/domains/job/model/JobContext/JobContext.ts +7 -5
- package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
- package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
- package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
- package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
- package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
- package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
- package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
- package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
- package/src/domains/job/model/LogPayload/index.ts +1 -0
- package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
- package/src/domains/job/repository/AbstractRepository.ts +91 -59
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
- package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
- package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
- package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
- package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
- package/src/domains/job/repository/index.ts +1 -0
- package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
- package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -3
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +113 -31
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +101 -49
- package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
- package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
- package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
- package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
- package/src/domains/job/services/index.ts +2 -1
- package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
- package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
- package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
- package/src/index.ts +1 -0
- package/src/jest-global-setup.ts +8 -4
- package/src/mocks/mockIoc.ts +18 -43
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
- package/src/utils/index.ts +2 -0
- package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
- package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
- package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
- package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
- package/src/custom-types.d.ts +0 -7
- package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
- package/src/domains/job/S3Repository/S3Repository.ts +0 -27
- package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
- package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
- package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
|
@@ -3,46 +3,57 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
|
|
7
|
-
import { PARTITION_KEYS } from '../../../constants/Repository.constants';
|
|
8
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
|
+
import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
|
|
8
|
+
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
9
9
|
import { JobExecutionDto, JobExecutionRecord } from '../JobExecution';
|
|
10
10
|
|
|
11
11
|
export const createMockJobExecutionDto = (partialRecord: Partial<JobExecutionRecord> = {}): JobExecutionDto => {
|
|
12
|
-
const
|
|
12
|
+
const tenant = faker.string.uuid();
|
|
13
|
+
const jobName = partialRecord.jobName ?? faker.lorem.word();
|
|
13
14
|
const timeQueued = faker.date.recent().toISOString();
|
|
14
15
|
const id = `${timeQueued}-${faker.string.uuid()}`;
|
|
15
16
|
const version = Array.from({ length: 3 })
|
|
16
17
|
.map(() => faker.number.int())
|
|
17
18
|
.join('.');
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
19
|
+
const status = partialRecord.status ?? faker.helpers.arrayElement(Object.values(JOB_EXECUTION_STATUS));
|
|
20
|
+
const input =
|
|
21
|
+
'input' in partialRecord
|
|
22
|
+
? partialRecord.input
|
|
23
|
+
: faker.helpers.arrayElement([
|
|
24
|
+
faker.datatype.boolean(),
|
|
25
|
+
faker.lorem.word(),
|
|
26
|
+
faker.number.int(),
|
|
27
|
+
{ [faker.lorem.word()]: faker.lorem.words },
|
|
28
|
+
null,
|
|
29
|
+
undefined,
|
|
30
|
+
]);
|
|
26
31
|
|
|
27
32
|
return {
|
|
28
33
|
context: {
|
|
29
34
|
S: partialRecord.context ?? faker.string.uuid(),
|
|
30
35
|
},
|
|
31
36
|
ecsTaskId: partialRecord.ecsTaskId ? { S: partialRecord.ecsTaskId } : { NULL: true },
|
|
37
|
+
gsi1pk: {
|
|
38
|
+
S: `${ENTITY_TYPES.jobExecution}~${status}`,
|
|
39
|
+
},
|
|
32
40
|
id: {
|
|
33
41
|
S: partialRecord.id ?? id,
|
|
34
42
|
},
|
|
35
43
|
jobName: {
|
|
36
|
-
S:
|
|
44
|
+
S: jobName,
|
|
45
|
+
},
|
|
46
|
+
lsi1sk: {
|
|
47
|
+
S: `${status}~${jobName}`,
|
|
37
48
|
},
|
|
38
49
|
pk: {
|
|
39
|
-
S: partialRecord.pk ?? `${
|
|
50
|
+
S: partialRecord.pk ?? `${ENTITY_TYPES.jobExecution}~${tenant}`,
|
|
40
51
|
},
|
|
41
52
|
sk: {
|
|
42
|
-
S: partialRecord.sk ?? `${
|
|
53
|
+
S: partialRecord.sk ?? `${jobName}~${id}`,
|
|
43
54
|
},
|
|
44
55
|
status: {
|
|
45
|
-
S:
|
|
56
|
+
S: status,
|
|
46
57
|
},
|
|
47
58
|
timeFinished: partialRecord.timeFinished ? { S: partialRecord.timeFinished } : { NULL: true },
|
|
48
59
|
timeQueued: {
|
|
@@ -52,7 +63,12 @@ export const createMockJobExecutionDto = (partialRecord: Partial<JobExecutionRec
|
|
|
52
63
|
version: {
|
|
53
64
|
S: partialRecord.version ?? version,
|
|
54
65
|
},
|
|
55
|
-
input: input ? {
|
|
66
|
+
input: input === undefined || input === null ? { NULL: true } : { S: JSON.stringify(input) },
|
|
67
|
+
retryCount: {
|
|
68
|
+
N: String(partialRecord.retryCount ?? 0),
|
|
69
|
+
},
|
|
70
|
+
lastFailureReason: partialRecord.lastFailureReason ? { S: partialRecord.lastFailureReason } : { NULL: true },
|
|
71
|
+
retryAfter: partialRecord.retryAfter ? { S: partialRecord.retryAfter } : { NULL: true },
|
|
56
72
|
};
|
|
57
73
|
};
|
|
58
74
|
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
|
|
7
|
-
import { PARTITION_KEYS } from '../../../constants/Repository.constants';
|
|
8
6
|
import { faker } from '@faker-js/faker/locale/en';
|
|
7
|
+
import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
|
|
8
|
+
import { ENTITY_TYPES } from '../../../constants/Repository.constants';
|
|
9
9
|
import { JobExecutionRecord } from '../JobExecution';
|
|
10
10
|
|
|
11
11
|
export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecutionRecord> = {}): JobExecutionRecord => {
|
|
@@ -15,6 +15,7 @@ export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecution
|
|
|
15
15
|
const version = Array.from({ length: 3 })
|
|
16
16
|
.map(() => faker.number.int())
|
|
17
17
|
.join('.');
|
|
18
|
+
const status = faker.helpers.objectValue(JOB_EXECUTION_STATUS);
|
|
18
19
|
const input = faker.helpers.arrayElement([
|
|
19
20
|
faker.datatype.boolean(),
|
|
20
21
|
faker.lorem.word(),
|
|
@@ -27,16 +28,21 @@ export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecution
|
|
|
27
28
|
return {
|
|
28
29
|
context: faker.string.uuid(),
|
|
29
30
|
ecsTaskId: null,
|
|
31
|
+
gsi1pk: `${ENTITY_TYPES.jobExecution}~${status}`,
|
|
30
32
|
id: id,
|
|
31
33
|
jobName: jobName,
|
|
32
|
-
|
|
34
|
+
lsi1sk: `${status}~${jobName}`,
|
|
35
|
+
pk: `${ENTITY_TYPES.jobExecution}-${jobName}`,
|
|
33
36
|
sk: `${timeQueued}-${id}`,
|
|
34
|
-
status:
|
|
37
|
+
status: status,
|
|
35
38
|
timeFinished: null,
|
|
36
39
|
timeQueued: timeQueued,
|
|
37
40
|
timeStarted: null,
|
|
38
41
|
version,
|
|
39
42
|
input,
|
|
43
|
+
retryCount: 0,
|
|
44
|
+
lastFailureReason: null,
|
|
45
|
+
retryAfter: null,
|
|
40
46
|
...partialRecord,
|
|
41
47
|
};
|
|
42
48
|
};
|
|
@@ -1,12 +1,33 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
'
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
export enum LogLevel {
|
|
4
|
+
AUDIT = 'AUDIT',
|
|
5
|
+
INFO = 'INFO',
|
|
6
|
+
WARNING = 'WARNING',
|
|
7
|
+
ERROR = 'ERROR',
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const LogPayloadSchema = z.object({
|
|
11
|
+
message: z.string(),
|
|
12
|
+
meta: z.object({
|
|
13
|
+
// Properties are nested under "meta" so the DXP log level does not conflict with the winston log level.
|
|
14
|
+
// Job runner specific attributes.
|
|
15
|
+
tenant: z.string(),
|
|
16
|
+
jobName: z.string(),
|
|
17
|
+
// Logging service attributes.
|
|
18
|
+
level: z.enum(LogLevel),
|
|
19
|
+
timestamp: z.iso.datetime(),
|
|
20
|
+
traceid: z.string(),
|
|
21
|
+
tags: z.array(z.string()).optional(),
|
|
22
|
+
details: z.record(z.string(), z.unknown()).optional(),
|
|
23
|
+
}),
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export type LogPayload = z.infer<typeof LogPayloadSchema>;
|
|
27
|
+
|
|
28
|
+
export const CONSOLE_LOG_METHOD_LEVELS: Partial<Record<keyof typeof console, LogLevel>> = {
|
|
29
|
+
log: LogLevel.INFO,
|
|
30
|
+
info: LogLevel.INFO,
|
|
31
|
+
warn: LogLevel.WARNING,
|
|
32
|
+
error: LogLevel.ERROR,
|
|
12
33
|
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { faker } from '@faker-js/faker';
|
|
2
|
+
import { LogLevel, LogPayload } from '../LogPayload';
|
|
3
|
+
|
|
4
|
+
type DeepPartial<T> = {
|
|
5
|
+
[K in keyof T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export const createMockLogPayload = (partialRecord: DeepPartial<LogPayload>): LogPayload => {
|
|
9
|
+
return {
|
|
10
|
+
message: faker.lorem.words(),
|
|
11
|
+
...partialRecord,
|
|
12
|
+
meta: {
|
|
13
|
+
traceid: faker.string.uuid(),
|
|
14
|
+
level: faker.helpers.objectValue(LogLevel),
|
|
15
|
+
timestamp: faker.date.recent().toISOString(),
|
|
16
|
+
tenant: faker.string.uuid(),
|
|
17
|
+
jobName: faker.string.uuid(),
|
|
18
|
+
...partialRecord.meta,
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
};
|
|
@@ -6,21 +6,19 @@ import { injectable } from 'inversify';
|
|
|
6
6
|
import 'reflect-metadata';
|
|
7
7
|
import {
|
|
8
8
|
AttributeValue,
|
|
9
|
+
BatchWriteItemCommand,
|
|
9
10
|
DeleteItemCommand,
|
|
10
11
|
DeleteItemCommandInput,
|
|
11
12
|
DynamoDBClient,
|
|
12
13
|
GetItemCommand,
|
|
13
14
|
GetItemCommandInput,
|
|
14
15
|
PutItemCommand,
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
ScanCommandOutput,
|
|
16
|
+
QueryCommand,
|
|
17
|
+
QueryCommandOutput,
|
|
18
18
|
UpdateItemCommand,
|
|
19
19
|
UpdateItemCommandInput,
|
|
20
20
|
} from '@aws-sdk/client-dynamodb';
|
|
21
|
-
import {
|
|
22
|
-
import { PartitionKey } from '../constants/Repository.constants';
|
|
23
|
-
import { BaseDto, BaseItem } from '../../job/model/Base/Base';
|
|
21
|
+
import { BaseDto, BaseItem, ListParameters } from '../../job/model/Base/Base';
|
|
24
22
|
|
|
25
23
|
/**
|
|
26
24
|
* Generic abstract class for representing repositories interacting with DynamoDB.
|
|
@@ -31,22 +29,12 @@ import { BaseDto, BaseItem } from '../../job/model/Base/Base';
|
|
|
31
29
|
* @template U The corresponding DB record attributes for the entity.
|
|
32
30
|
*/
|
|
33
31
|
@injectable()
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
// HACK: workaround to address inheritance issue in InversifyJS
|
|
41
|
-
// Reference: https://github.com/inversify/InversifyJS/blob/master/wiki/inheritance.md
|
|
42
|
-
|
|
43
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
44
|
-
// @ts-ignore @ts-2322
|
|
45
|
-
this.pk = null;
|
|
46
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
47
|
-
// @ts-ignore @ts-2322
|
|
48
|
-
this.tableName = null;
|
|
49
|
-
}
|
|
32
|
+
export abstract class Repository<T extends BaseItem, U extends BaseDto, SK extends Partial<T>> {
|
|
33
|
+
public constructor(
|
|
34
|
+
protected dynamoDbClient: DynamoDBClient,
|
|
35
|
+
protected tableName: string,
|
|
36
|
+
protected pk: string,
|
|
37
|
+
) {}
|
|
50
38
|
|
|
51
39
|
/**
|
|
52
40
|
* Save item into database.
|
|
@@ -55,7 +43,6 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
55
43
|
* @returns {Promise<T>} The item stored.
|
|
56
44
|
* @memberof Repository
|
|
57
45
|
*/
|
|
58
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
59
46
|
async create(input: T): Promise<T> {
|
|
60
47
|
const dynamoDbRecord = this.convertToDynamoDbItem(input);
|
|
61
48
|
|
|
@@ -76,11 +63,11 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
76
63
|
* @returns {Promise<T | null>} The retrieved record.
|
|
77
64
|
* @memberof Repository
|
|
78
65
|
*/
|
|
79
|
-
async findOne(sk:
|
|
66
|
+
async findOne(sk: SK): Promise<T | null> {
|
|
80
67
|
const params: GetItemCommandInput = {
|
|
81
68
|
Key: {
|
|
82
|
-
pk: { S:
|
|
83
|
-
sk: { S: sk },
|
|
69
|
+
pk: { S: this.pk },
|
|
70
|
+
sk: { S: this.buildSk(sk) },
|
|
84
71
|
},
|
|
85
72
|
TableName: this.tableName,
|
|
86
73
|
};
|
|
@@ -94,41 +81,41 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
94
81
|
* @returns {Promise<Array<T>>} The retrieved records.
|
|
95
82
|
* @memberof Repository
|
|
96
83
|
*/
|
|
97
|
-
async list(): Promise<Array<T>> {
|
|
98
|
-
const params: ScanCommandInput = {
|
|
99
|
-
ExpressionAttributeNames: {
|
|
100
|
-
'#pk': 'pk',
|
|
101
|
-
},
|
|
102
|
-
ExpressionAttributeValues: {
|
|
103
|
-
':pk': { S: this.pk },
|
|
104
|
-
},
|
|
105
|
-
FilterExpression: '#pk = :pk',
|
|
106
|
-
TableName: this.tableName,
|
|
107
|
-
};
|
|
108
|
-
|
|
84
|
+
async list(parameters: ListParameters<never> = {}): Promise<Array<T>> {
|
|
109
85
|
// Apply pagination for the scan.
|
|
110
86
|
// By default, the Scan operation processes data sequentially.
|
|
111
87
|
// DynamoDB returns data in increments of 1 MB increments.
|
|
112
88
|
|
|
113
|
-
let startKey; /* Indicator of where to paginate from. */
|
|
89
|
+
let startKey: Record<string, AttributeValue> | undefined = undefined; /* Indicator of where to paginate from. */
|
|
114
90
|
const results: Array<Record<string, AttributeValue>> = [];
|
|
115
91
|
|
|
116
92
|
// Iterate through pages until no more results are found.
|
|
117
|
-
|
|
118
|
-
const res:
|
|
119
|
-
new
|
|
93
|
+
do {
|
|
94
|
+
const res: QueryCommandOutput = await this.dynamoDbClient.send(
|
|
95
|
+
new QueryCommand({
|
|
96
|
+
ExpressionAttributeNames: {
|
|
97
|
+
'#pk': 'pk',
|
|
98
|
+
},
|
|
99
|
+
ExpressionAttributeValues: {
|
|
100
|
+
':pk': { S: this.pk },
|
|
101
|
+
},
|
|
102
|
+
KeyConditionExpression: '#pk = :pk',
|
|
103
|
+
TableName: this.tableName,
|
|
104
|
+
ExclusiveStartKey: startKey,
|
|
105
|
+
Limit: parameters.limit,
|
|
106
|
+
ConsistentRead: parameters.consistentRead,
|
|
107
|
+
}),
|
|
120
108
|
);
|
|
121
109
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
startKey = null;
|
|
125
|
-
} else {
|
|
126
|
-
results.push(...(res.Items ?? []));
|
|
110
|
+
results.push(...(res.Items ?? []));
|
|
111
|
+
startKey = res.LastEvaluatedKey;
|
|
127
112
|
|
|
128
|
-
|
|
129
|
-
|
|
113
|
+
if (parameters.limit !== undefined && results.length >= parameters.limit) {
|
|
114
|
+
// Truncate list to max requested items, stop querying.
|
|
115
|
+
results.length = parameters.limit;
|
|
116
|
+
break;
|
|
130
117
|
}
|
|
131
|
-
}
|
|
118
|
+
} while (startKey);
|
|
132
119
|
|
|
133
120
|
return results.length ? results.map((item) => this.convertToItem(item as U)) : [];
|
|
134
121
|
}
|
|
@@ -136,16 +123,16 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
136
123
|
/**
|
|
137
124
|
* Update record in database.
|
|
138
125
|
* @param {string} sk The sort key used to identify the record.
|
|
139
|
-
* @param {Partial<T>} input The (partial) record to update.
|
|
126
|
+
* @param {Partial<T>} input The (partial) record to update (properties used in the SK cannot be updated).
|
|
140
127
|
* @returns {Promise<T | null>} The updated record.
|
|
141
128
|
* @memberof Repository
|
|
142
129
|
*/
|
|
143
|
-
async update(sk:
|
|
130
|
+
async update(sk: SK, input: Partial<Omit<T, keyof SK>>): Promise<T> {
|
|
144
131
|
// Initialise update command
|
|
145
132
|
const updateParams: UpdateItemCommandInput = {
|
|
146
133
|
Key: {
|
|
147
134
|
pk: { S: this.pk },
|
|
148
|
-
sk: { S: sk },
|
|
135
|
+
sk: { S: this.buildSk(sk) },
|
|
149
136
|
},
|
|
150
137
|
ReturnValues: 'ALL_NEW',
|
|
151
138
|
TableName: this.tableName,
|
|
@@ -156,11 +143,11 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
156
143
|
|
|
157
144
|
// To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
|
|
158
145
|
for (const key of Object.keys(input)) {
|
|
159
|
-
if (input[key as keyof Partial<T
|
|
146
|
+
if (input[key as keyof Partial<Omit<T, keyof SK>>] === null) {
|
|
160
147
|
expressionAttributeNames[`#${key}`] = key;
|
|
161
148
|
expressionAttributeValues[`:${key}`] = { NULL: true };
|
|
162
149
|
} else {
|
|
163
|
-
const val: string = input[key as keyof Partial<T
|
|
150
|
+
const val: string = input[key as keyof Partial<Omit<T, keyof SK>>] as string;
|
|
164
151
|
expressionAttributeNames[`#${key}`] = key;
|
|
165
152
|
expressionAttributeValues[`:${key}`] = { S: val };
|
|
166
153
|
}
|
|
@@ -176,7 +163,7 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
176
163
|
const updateItemCommand = new UpdateItemCommand(updateParams);
|
|
177
164
|
const updatedRecord = await this.dynamoDbClient.send(updateItemCommand);
|
|
178
165
|
|
|
179
|
-
return
|
|
166
|
+
return this.convertToItem(updatedRecord.Attributes as U);
|
|
180
167
|
}
|
|
181
168
|
|
|
182
169
|
/**
|
|
@@ -186,11 +173,11 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
186
173
|
* @returns {void}
|
|
187
174
|
* @memberof Repository
|
|
188
175
|
*/
|
|
189
|
-
async delete(sk:
|
|
176
|
+
async delete(sk: T | SK): Promise<void> {
|
|
190
177
|
const deleteCommandParams: DeleteItemCommandInput = {
|
|
191
178
|
Key: {
|
|
192
179
|
pk: { S: this.pk },
|
|
193
|
-
sk: { S: sk },
|
|
180
|
+
sk: { S: this.buildSk(sk) },
|
|
194
181
|
},
|
|
195
182
|
TableName: this.tableName,
|
|
196
183
|
};
|
|
@@ -199,10 +186,47 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
199
186
|
return;
|
|
200
187
|
}
|
|
201
188
|
|
|
189
|
+
async batchDelete<TItem extends T | SK>(sk: Array<TItem>): Promise<{ unprocessedItems: Array<TItem> }> {
|
|
190
|
+
if (sk.length === 0) {
|
|
191
|
+
return { unprocessedItems: [] };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const skMap: Record<string, TItem> = {};
|
|
195
|
+
|
|
196
|
+
sk.forEach((sk) => {
|
|
197
|
+
skMap[this.buildSk(sk)] = sk;
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
const result = await this.dynamoDbClient.send(
|
|
201
|
+
new BatchWriteItemCommand({
|
|
202
|
+
RequestItems: {
|
|
203
|
+
[this.tableName]: Object.keys(skMap).map((sk) => ({
|
|
204
|
+
DeleteRequest: {
|
|
205
|
+
Key: {
|
|
206
|
+
pk: { S: this.pk },
|
|
207
|
+
sk: { S: sk },
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
})),
|
|
211
|
+
},
|
|
212
|
+
}),
|
|
213
|
+
);
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
unprocessedItems:
|
|
217
|
+
result.UnprocessedItems?.[this.tableName]?.map((item) => {
|
|
218
|
+
const sk = item.DeleteRequest?.Key?.sk.S;
|
|
219
|
+
|
|
220
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
221
|
+
return skMap[sk!];
|
|
222
|
+
}) || [],
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
202
226
|
/**
|
|
203
227
|
* Convert the DynanomDB attributes of the entity into a DynamoDB item.
|
|
204
228
|
* @abstract
|
|
205
|
-
* @param {
|
|
229
|
+
* @param {T} item The DynamoDB item attributes to convert.
|
|
206
230
|
* @returns {Record<keyof U, AttributeValue>} The DynamoDB record.
|
|
207
231
|
* @memberof Repository
|
|
208
232
|
*/
|
|
@@ -216,4 +240,12 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
|
|
|
216
240
|
* @memberof Repository
|
|
217
241
|
*/
|
|
218
242
|
protected abstract convertToItem(record: U): T;
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Converts a sort key to a string for querying/putting in Dynamo.
|
|
246
|
+
* @param {SK} sk
|
|
247
|
+
* @return {string}
|
|
248
|
+
* @protected
|
|
249
|
+
*/
|
|
250
|
+
protected abstract buildSk(sk: T | SK): string;
|
|
219
251
|
}
|
|
@@ -14,25 +14,26 @@ import {
|
|
|
14
14
|
ScanCommandOutput,
|
|
15
15
|
} from '@aws-sdk/client-dynamodb';
|
|
16
16
|
import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
|
|
17
|
-
import { testLogger } from '../../../../jest-global-setup';
|
|
18
17
|
import {
|
|
19
18
|
createMockClientPutResponse,
|
|
20
19
|
createMockClientScanResponse,
|
|
21
20
|
createMockClientGetResponse,
|
|
22
21
|
} from '../../core/dynamoClient.mock';
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
22
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
23
|
+
import {
|
|
24
|
+
JobContext,
|
|
25
|
+
createMockJobContext,
|
|
26
|
+
createMockJobContextDto,
|
|
27
|
+
createMockJobContextDtos,
|
|
28
|
+
createMockJobContextRecord,
|
|
29
|
+
createMockCreateJobContextRequest,
|
|
30
|
+
} from '../../model/JobContext';
|
|
29
31
|
import { JobContextRepository } from './JobContextRepository';
|
|
30
32
|
import * as PaginationUtils from '../../core/pagination/utils/PaginationUtils';
|
|
31
|
-
import { PaginationInfo } from '../../core/pagination/
|
|
32
|
-
import {
|
|
33
|
+
import { PaginationInfo, PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/';
|
|
34
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
initIocContainer(testLogger);
|
|
36
|
+
initIocContainer();
|
|
36
37
|
|
|
37
38
|
// Mock DynamoDB Client
|
|
38
39
|
const mockOptions = {} as DynamoDBClientConfig;
|
|
@@ -41,20 +42,20 @@ const mockSend = jest.fn();
|
|
|
41
42
|
mockClient.send = mockSend;
|
|
42
43
|
|
|
43
44
|
// Inject JobContextRepository with a mocked client
|
|
44
|
-
iocContainer.rebind
|
|
45
|
-
|
|
46
|
-
let jobContextRepository: JobContextRepository;
|
|
45
|
+
iocContainer.rebind(DynamoDBClient).toConstantValue(mockClient);
|
|
47
46
|
|
|
48
47
|
describe('JobContextRepository', (): void => {
|
|
48
|
+
let jobContextRepository: JobContextRepository;
|
|
49
|
+
let dynamoTableName: string;
|
|
50
|
+
let tenant: string;
|
|
51
|
+
|
|
49
52
|
beforeAll(() => {
|
|
50
53
|
jobContextRepository = iocContainer.get(JobContextRepository);
|
|
54
|
+
dynamoTableName = iocContainer.get(InjectTokens.DynamoTableName);
|
|
55
|
+
tenant = iocContainer.get(InjectTokens.Tenant);
|
|
51
56
|
});
|
|
52
57
|
|
|
53
58
|
describe('find', () => {
|
|
54
|
-
beforeEach(() => {
|
|
55
|
-
jest.resetAllMocks();
|
|
56
|
-
});
|
|
57
|
-
|
|
58
59
|
it(`should return an empty list`, async (): Promise<void> => {
|
|
59
60
|
mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
|
|
60
61
|
|
|
@@ -80,8 +81,8 @@ describe('JobContextRepository', (): void => {
|
|
|
80
81
|
});
|
|
81
82
|
|
|
82
83
|
it(`should paginate results to return a list of existing job contexts`, async (): Promise<void> => {
|
|
83
|
-
const mockContextItemsPage1 = createMockJobContextDtos();
|
|
84
|
-
const mockContextItemsPage2 = createMockJobContextDtos();
|
|
84
|
+
const mockContextItemsPage1 = createMockJobContextDtos(2);
|
|
85
|
+
const mockContextItemsPage2 = createMockJobContextDtos(2);
|
|
85
86
|
|
|
86
87
|
mockSend
|
|
87
88
|
.mockImplementationOnce(
|
|
@@ -99,17 +100,15 @@ describe('JobContextRepository', (): void => {
|
|
|
99
100
|
}),
|
|
100
101
|
);
|
|
101
102
|
|
|
102
|
-
const response = await jobContextRepository.list(
|
|
103
|
+
const response = await jobContextRepository.list({
|
|
104
|
+
limit: 3,
|
|
105
|
+
});
|
|
103
106
|
expect(mockSend).toBeCalledTimes(2);
|
|
104
|
-
expect(response).toHaveLength(
|
|
107
|
+
expect(response).toHaveLength(3);
|
|
105
108
|
});
|
|
106
109
|
});
|
|
107
110
|
|
|
108
111
|
describe('findOne', () => {
|
|
109
|
-
beforeEach(() => {
|
|
110
|
-
jest.resetAllMocks();
|
|
111
|
-
});
|
|
112
|
-
|
|
113
112
|
it(`should return job context by name`, async (): Promise<void> => {
|
|
114
113
|
const mockJobContextItem = createMockJobContextDto({
|
|
115
114
|
environment: { key: 'value' },
|
|
@@ -121,7 +120,7 @@ describe('JobContextRepository', (): void => {
|
|
|
121
120
|
}),
|
|
122
121
|
);
|
|
123
122
|
|
|
124
|
-
const response = await jobContextRepository.findOne(mockJobContextItem.sk.S as string);
|
|
123
|
+
const response = await jobContextRepository.findOne({ contextName: mockJobContextItem.sk.S as string });
|
|
125
124
|
expect(response).toStrictEqual({
|
|
126
125
|
contextName: mockJobContextItem.sk.S as string,
|
|
127
126
|
description: mockJobContextItem.description.S as string,
|
|
@@ -139,22 +138,18 @@ describe('JobContextRepository', (): void => {
|
|
|
139
138
|
}),
|
|
140
139
|
);
|
|
141
140
|
|
|
142
|
-
const response = await jobContextRepository.findOne(`null`);
|
|
141
|
+
const response = await jobContextRepository.findOne({ contextName: `null` });
|
|
143
142
|
expect(mockSend).toBeCalledTimes(1);
|
|
144
143
|
expect(response).toBe(null);
|
|
145
144
|
});
|
|
146
145
|
});
|
|
147
146
|
|
|
148
147
|
describe('create', () => {
|
|
149
|
-
beforeEach(() => {
|
|
150
|
-
jest.resetAllMocks();
|
|
151
|
-
});
|
|
152
|
-
|
|
153
148
|
it(`should create a new job context record and return job context in the response`, async (): Promise<void> => {
|
|
154
149
|
const newJobContextToCreate = createMockCreateJobContextRequest();
|
|
155
150
|
const mockResponse = createMockClientPutResponse({
|
|
156
151
|
Attributes: createMockJobContextDto({
|
|
157
|
-
pk:
|
|
152
|
+
pk: `${ENTITY_TYPES.context}~${tenant}`,
|
|
158
153
|
sk: newJobContextToCreate.contextName,
|
|
159
154
|
}),
|
|
160
155
|
});
|
|
@@ -165,16 +160,12 @@ describe('JobContextRepository', (): void => {
|
|
|
165
160
|
expect(mockSend).toBeCalledTimes(1);
|
|
166
161
|
expect(response).toStrictEqual({
|
|
167
162
|
...newJobContextToCreate,
|
|
168
|
-
type:
|
|
163
|
+
type: ENTITY_TYPES.context,
|
|
169
164
|
});
|
|
170
165
|
});
|
|
171
166
|
});
|
|
172
167
|
|
|
173
168
|
describe('update', () => {
|
|
174
|
-
beforeEach(() => {
|
|
175
|
-
jest.resetAllMocks();
|
|
176
|
-
});
|
|
177
|
-
|
|
178
169
|
it(`should update a job context record`, async (): Promise<void> => {
|
|
179
170
|
const jobContextToUpdate = createMockJobContext();
|
|
180
171
|
const updatedJobContext: JobContext = {
|
|
@@ -197,17 +188,16 @@ describe('JobContextRepository', (): void => {
|
|
|
197
188
|
|
|
198
189
|
mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
|
|
199
190
|
|
|
200
|
-
const response = await jobContextRepository.update(
|
|
191
|
+
const response = await jobContextRepository.update(
|
|
192
|
+
{ contextName: jobContextToUpdate.contextName },
|
|
193
|
+
updatedJobContextRecord,
|
|
194
|
+
);
|
|
201
195
|
expect(response).toStrictEqual(updatedJobContext);
|
|
202
196
|
expect(mockSend).toBeCalledTimes(1);
|
|
203
197
|
});
|
|
204
198
|
});
|
|
205
199
|
|
|
206
200
|
describe('delete', () => {
|
|
207
|
-
beforeEach(() => {
|
|
208
|
-
jest.resetAllMocks();
|
|
209
|
-
});
|
|
210
|
-
|
|
211
201
|
it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
|
|
212
202
|
mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
|
|
213
203
|
return {
|
|
@@ -215,7 +205,7 @@ describe('JobContextRepository', (): void => {
|
|
|
215
205
|
};
|
|
216
206
|
});
|
|
217
207
|
|
|
218
|
-
const response = await jobContextRepository.delete(`name`);
|
|
208
|
+
const response = await jobContextRepository.delete({ contextName: `name` });
|
|
219
209
|
expect(response).toBe(undefined);
|
|
220
210
|
expect(mockSend).toBeCalledTimes(1);
|
|
221
211
|
});
|
|
@@ -228,7 +218,7 @@ describe('JobContextRepository', (): void => {
|
|
|
228
218
|
|
|
229
219
|
beforeEach(() => {
|
|
230
220
|
mockExclusiveStartKey = {
|
|
231
|
-
pk: { S:
|
|
221
|
+
pk: { S: ENTITY_TYPES.context },
|
|
232
222
|
sk: { S: 'mockSk' },
|
|
233
223
|
};
|
|
234
224
|
mockPaginationInfo = {
|
|
@@ -240,11 +230,11 @@ describe('JobContextRepository', (): void => {
|
|
|
240
230
|
mockParams = {
|
|
241
231
|
ExclusiveStartKey: mockExclusiveStartKey,
|
|
242
232
|
ExpressionAttributeValues: {
|
|
243
|
-
':pk': { S:
|
|
233
|
+
':pk': { S: `${ENTITY_TYPES.context}~${tenant}` },
|
|
244
234
|
},
|
|
245
235
|
KeyConditionExpression: 'pk = :pk',
|
|
246
236
|
ScanIndexForward: false,
|
|
247
|
-
TableName:
|
|
237
|
+
TableName: dynamoTableName,
|
|
248
238
|
};
|
|
249
239
|
|
|
250
240
|
jest.spyOn(PaginationUtils, 'decodeTokenString').mockReturnValue(mockExclusiveStartKey);
|
|
@@ -271,10 +261,10 @@ describe('JobContextRepository', (): void => {
|
|
|
271
261
|
expect(result.lastEvaluatedKey).toBe(null);
|
|
272
262
|
});
|
|
273
263
|
|
|
274
|
-
it('should return an array of
|
|
264
|
+
it('should return an array of contexts if found', async () => {
|
|
275
265
|
const mockJobContextRecords = createMockJobContextDtos();
|
|
276
266
|
const mockLastEvaluatedKey = {
|
|
277
|
-
pk: { S:
|
|
267
|
+
pk: { S: ENTITY_TYPES.job },
|
|
278
268
|
sk: { S: 'mockSk' },
|
|
279
269
|
};
|
|
280
270
|
|