@squiz/job-runner-lib 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
  3. package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
  4. package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
  5. package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
  6. package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
  7. package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
  8. package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
  9. package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
  10. package/lib/domains/job/S3Repository/index.d.ts +2 -1
  11. package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
  12. package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
  13. package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
  14. package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
  15. package/lib/domains/job/index.d.ts +6 -4
  16. package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
  17. package/lib/domains/job/model/Base/Base.d.ts +10 -5
  18. package/lib/domains/job/model/Job/Job.d.ts +9 -6
  19. package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
  20. package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
  21. package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
  22. package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
  23. package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
  24. package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
  25. package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
  26. package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
  27. package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
  28. package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
  29. package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
  30. package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
  31. package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
  32. package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
  33. package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
  34. package/lib/domains/job/repository/index.d.ts +1 -0
  35. package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
  36. package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
  37. package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -3
  38. package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
  39. package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
  40. package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
  41. package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
  42. package/lib/domains/job/services/index.d.ts +2 -1
  43. package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
  44. package/lib/index.d.ts +1 -0
  45. package/lib/index.js +13 -13
  46. package/lib/index.js.map +4 -4
  47. package/lib/jest-global-setup.d.ts +6 -1
  48. package/lib/mocks/mockIoc.d.ts +3 -3
  49. package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
  50. package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
  51. package/lib/utils/index.d.ts +2 -0
  52. package/package.json +12 -11
  53. package/src/domains/component/errors/JobError.spec.ts +30 -0
  54. package/src/domains/component/errors/JobError.ts +8 -1
  55. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
  56. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
  57. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
  58. package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
  59. package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
  60. package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
  61. package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
  62. package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
  63. package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
  64. package/src/domains/job/S3Repository/index.ts +2 -1
  65. package/src/domains/job/constants/InjectTokens.ts +8 -0
  66. package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
  67. package/src/domains/job/constants/Repository.constants.ts +3 -3
  68. package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
  69. package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
  70. package/src/domains/job/index.ts +6 -4
  71. package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
  72. package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
  73. package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
  74. package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
  75. package/src/domains/job/model/Base/Base.ts +11 -5
  76. package/src/domains/job/model/Job/Job.ts +11 -6
  77. package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
  78. package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
  79. package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
  80. package/src/domains/job/model/JobContext/JobContext.ts +7 -5
  81. package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
  82. package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
  83. package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
  84. package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
  85. package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
  86. package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
  87. package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
  88. package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
  89. package/src/domains/job/model/LogPayload/index.ts +1 -0
  90. package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
  91. package/src/domains/job/repository/AbstractRepository.ts +91 -59
  92. package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
  93. package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
  94. package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
  95. package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
  96. package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
  97. package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
  98. package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
  99. package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
  100. package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
  101. package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
  102. package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
  103. package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
  104. package/src/domains/job/repository/index.ts +1 -0
  105. package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
  106. package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
  107. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -3
  108. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +113 -31
  109. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +101 -49
  110. package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
  111. package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
  112. package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
  113. package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
  114. package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
  115. package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
  116. package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
  117. package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
  118. package/src/domains/job/services/index.ts +2 -1
  119. package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
  120. package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
  121. package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
  122. package/src/index.ts +1 -0
  123. package/src/jest-global-setup.ts +8 -4
  124. package/src/mocks/mockIoc.ts +18 -43
  125. package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
  126. package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
  127. package/src/utils/index.ts +2 -0
  128. package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
  129. package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
  130. package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
  131. package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
  132. package/src/custom-types.d.ts +0 -7
  133. package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
  134. package/src/domains/job/S3Repository/S3Repository.ts +0 -27
  135. package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
  136. package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
  137. package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
@@ -3,46 +3,57 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
- import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
7
- import { PARTITION_KEYS } from '../../../constants/Repository.constants';
8
6
  import { faker } from '@faker-js/faker/locale/en';
7
+ import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
8
+ import { ENTITY_TYPES } from '../../../constants/Repository.constants';
9
9
  import { JobExecutionDto, JobExecutionRecord } from '../JobExecution';
10
10
 
11
11
  export const createMockJobExecutionDto = (partialRecord: Partial<JobExecutionRecord> = {}): JobExecutionDto => {
12
- const jobName = faker.lorem.word();
12
+ const tenant = faker.string.uuid();
13
+ const jobName = partialRecord.jobName ?? faker.lorem.word();
13
14
  const timeQueued = faker.date.recent().toISOString();
14
15
  const id = `${timeQueued}-${faker.string.uuid()}`;
15
16
  const version = Array.from({ length: 3 })
16
17
  .map(() => faker.number.int())
17
18
  .join('.');
18
- const input = faker.helpers.arrayElement([
19
- faker.datatype.boolean(),
20
- faker.lorem.word(),
21
- faker.number.int(),
22
- { [faker.lorem.word()]: faker.lorem.words },
23
- null,
24
- undefined,
25
- ]);
19
+ const status = partialRecord.status ?? faker.helpers.arrayElement(Object.values(JOB_EXECUTION_STATUS));
20
+ const input =
21
+ 'input' in partialRecord
22
+ ? partialRecord.input
23
+ : faker.helpers.arrayElement([
24
+ faker.datatype.boolean(),
25
+ faker.lorem.word(),
26
+ faker.number.int(),
27
+ { [faker.lorem.word()]: faker.lorem.words },
28
+ null,
29
+ undefined,
30
+ ]);
26
31
 
27
32
  return {
28
33
  context: {
29
34
  S: partialRecord.context ?? faker.string.uuid(),
30
35
  },
31
36
  ecsTaskId: partialRecord.ecsTaskId ? { S: partialRecord.ecsTaskId } : { NULL: true },
37
+ gsi1pk: {
38
+ S: `${ENTITY_TYPES.jobExecution}~${status}`,
39
+ },
32
40
  id: {
33
41
  S: partialRecord.id ?? id,
34
42
  },
35
43
  jobName: {
36
- S: partialRecord.jobName ?? jobName,
44
+ S: jobName,
45
+ },
46
+ lsi1sk: {
47
+ S: `${status}~${jobName}`,
37
48
  },
38
49
  pk: {
39
- S: partialRecord.pk ?? `${PARTITION_KEYS.jobExecution}-${partialRecord.jobName ?? jobName}`,
50
+ S: partialRecord.pk ?? `${ENTITY_TYPES.jobExecution}~${tenant}`,
40
51
  },
41
52
  sk: {
42
- S: partialRecord.sk ?? `${timeQueued}-${id}`,
53
+ S: partialRecord.sk ?? `${jobName}~${id}`,
43
54
  },
44
55
  status: {
45
- S: partialRecord.status ?? faker.helpers.arrayElement(Object.values(JOB_EXECUTION_STATUS)),
56
+ S: status,
46
57
  },
47
58
  timeFinished: partialRecord.timeFinished ? { S: partialRecord.timeFinished } : { NULL: true },
48
59
  timeQueued: {
@@ -52,7 +63,12 @@ export const createMockJobExecutionDto = (partialRecord: Partial<JobExecutionRec
52
63
  version: {
53
64
  S: partialRecord.version ?? version,
54
65
  },
55
- input: input ? { S: JSON.stringify(input) } : { NULL: true },
66
+ input: input === undefined || input === null ? { NULL: true } : { S: JSON.stringify(input) },
67
+ retryCount: {
68
+ N: String(partialRecord.retryCount ?? 0),
69
+ },
70
+ lastFailureReason: partialRecord.lastFailureReason ? { S: partialRecord.lastFailureReason } : { NULL: true },
71
+ retryAfter: partialRecord.retryAfter ? { S: partialRecord.retryAfter } : { NULL: true },
56
72
  };
57
73
  };
58
74
 
@@ -3,9 +3,9 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
- import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
7
- import { PARTITION_KEYS } from '../../../constants/Repository.constants';
8
6
  import { faker } from '@faker-js/faker/locale/en';
7
+ import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
8
+ import { ENTITY_TYPES } from '../../../constants/Repository.constants';
9
9
  import { JobExecutionRecord } from '../JobExecution';
10
10
 
11
11
  export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecutionRecord> = {}): JobExecutionRecord => {
@@ -15,6 +15,7 @@ export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecution
15
15
  const version = Array.from({ length: 3 })
16
16
  .map(() => faker.number.int())
17
17
  .join('.');
18
+ const status = faker.helpers.objectValue(JOB_EXECUTION_STATUS);
18
19
  const input = faker.helpers.arrayElement([
19
20
  faker.datatype.boolean(),
20
21
  faker.lorem.word(),
@@ -27,16 +28,21 @@ export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecution
27
28
  return {
28
29
  context: faker.string.uuid(),
29
30
  ecsTaskId: null,
31
+ gsi1pk: `${ENTITY_TYPES.jobExecution}~${status}`,
30
32
  id: id,
31
33
  jobName: jobName,
32
- pk: `${PARTITION_KEYS.jobExecution}-${jobName}`,
34
+ lsi1sk: `${status}~${jobName}`,
35
+ pk: `${ENTITY_TYPES.jobExecution}-${jobName}`,
33
36
  sk: `${timeQueued}-${id}`,
34
- status: faker.helpers.objectValue(JOB_EXECUTION_STATUS),
37
+ status: status,
35
38
  timeFinished: null,
36
39
  timeQueued: timeQueued,
37
40
  timeStarted: null,
38
41
  version,
39
42
  input,
43
+ retryCount: 0,
44
+ lastFailureReason: null,
45
+ retryAfter: null,
40
46
  ...partialRecord,
41
47
  };
42
48
  };
@@ -1,12 +1,33 @@
1
- export type LogPayload = {
2
- message: string;
3
- type?: string;
4
- area?: string;
5
- req_id?: string;
6
- 'log-command'?: string;
7
- level?: string;
8
- 'function-name'?: string;
9
- 'job-name'?: string;
10
- 'job-version'?: string;
11
- timestamp: string;
1
+ import { z } from 'zod';
2
+
3
+ export enum LogLevel {
4
+ AUDIT = 'AUDIT',
5
+ INFO = 'INFO',
6
+ WARNING = 'WARNING',
7
+ ERROR = 'ERROR',
8
+ }
9
+
10
+ export const LogPayloadSchema = z.object({
11
+ message: z.string(),
12
+ meta: z.object({
13
+ // Properties are nested under "meta" so the DXP log level does not conflict with the winston log level.
14
+ // Job runner specific attributes.
15
+ tenant: z.string(),
16
+ jobName: z.string(),
17
+ // Logging service attributes.
18
+ level: z.enum(LogLevel),
19
+ timestamp: z.iso.datetime(),
20
+ traceid: z.string(),
21
+ tags: z.array(z.string()).optional(),
22
+ details: z.record(z.string(), z.unknown()).optional(),
23
+ }),
24
+ });
25
+
26
+ export type LogPayload = z.infer<typeof LogPayloadSchema>;
27
+
28
+ export const CONSOLE_LOG_METHOD_LEVELS: Partial<Record<keyof typeof console, LogLevel>> = {
29
+ log: LogLevel.INFO,
30
+ info: LogLevel.INFO,
31
+ warn: LogLevel.WARNING,
32
+ error: LogLevel.ERROR,
12
33
  };
@@ -1 +1,2 @@
1
1
  export * from './LogPayload';
2
+ export * from './mocks/LogPayload.mock';
@@ -0,0 +1,21 @@
1
+ import { faker } from '@faker-js/faker';
2
+ import { LogLevel, LogPayload } from '../LogPayload';
3
+
4
+ type DeepPartial<T> = {
5
+ [K in keyof T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
6
+ };
7
+
8
+ export const createMockLogPayload = (partialRecord: DeepPartial<LogPayload>): LogPayload => {
9
+ return {
10
+ message: faker.lorem.words(),
11
+ ...partialRecord,
12
+ meta: {
13
+ traceid: faker.string.uuid(),
14
+ level: faker.helpers.objectValue(LogLevel),
15
+ timestamp: faker.date.recent().toISOString(),
16
+ tenant: faker.string.uuid(),
17
+ jobName: faker.string.uuid(),
18
+ ...partialRecord.meta,
19
+ },
20
+ };
21
+ };
@@ -6,21 +6,19 @@ import { injectable } from 'inversify';
6
6
  import 'reflect-metadata';
7
7
  import {
8
8
  AttributeValue,
9
+ BatchWriteItemCommand,
9
10
  DeleteItemCommand,
10
11
  DeleteItemCommandInput,
11
12
  DynamoDBClient,
12
13
  GetItemCommand,
13
14
  GetItemCommandInput,
14
15
  PutItemCommand,
15
- ScanCommand,
16
- ScanCommandInput,
17
- ScanCommandOutput,
16
+ QueryCommand,
17
+ QueryCommandOutput,
18
18
  UpdateItemCommand,
19
19
  UpdateItemCommandInput,
20
20
  } from '@aws-sdk/client-dynamodb';
21
- import { parsePartitionKey } from '../ParsePartitionKey';
22
- import { PartitionKey } from '../constants/Repository.constants';
23
- import { BaseDto, BaseItem } from '../../job/model/Base/Base';
21
+ import { BaseDto, BaseItem, ListParameters } from '../../job/model/Base/Base';
24
22
 
25
23
  /**
26
24
  * Generic abstract class for representing repositories interacting with DynamoDB.
@@ -31,22 +29,12 @@ import { BaseDto, BaseItem } from '../../job/model/Base/Base';
31
29
  * @template U The corresponding DB record attributes for the entity.
32
30
  */
33
31
  @injectable()
34
- /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
35
- export abstract class Repository<T extends BaseItem, U extends BaseDto> {
36
- protected pk: PartitionKey;
37
- protected tableName: string;
38
-
39
- public constructor(protected dynamoDbClient: DynamoDBClient) {
40
- // HACK: workaround to address inheritance issue in InversifyJS
41
- // Reference: https://github.com/inversify/InversifyJS/blob/master/wiki/inheritance.md
42
-
43
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
44
- // @ts-ignore @ts-2322
45
- this.pk = null;
46
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
47
- // @ts-ignore @ts-2322
48
- this.tableName = null;
49
- }
32
+ export abstract class Repository<T extends BaseItem, U extends BaseDto, SK extends Partial<T>> {
33
+ public constructor(
34
+ protected dynamoDbClient: DynamoDBClient,
35
+ protected tableName: string,
36
+ protected pk: string,
37
+ ) {}
50
38
 
51
39
  /**
52
40
  * Save item into database.
@@ -55,7 +43,6 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
55
43
  * @returns {Promise<T>} The item stored.
56
44
  * @memberof Repository
57
45
  */
58
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
59
46
  async create(input: T): Promise<T> {
60
47
  const dynamoDbRecord = this.convertToDynamoDbItem(input);
61
48
 
@@ -76,11 +63,11 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
76
63
  * @returns {Promise<T | null>} The retrieved record.
77
64
  * @memberof Repository
78
65
  */
79
- async findOne(sk: string): Promise<T | null> {
66
+ async findOne(sk: SK): Promise<T | null> {
80
67
  const params: GetItemCommandInput = {
81
68
  Key: {
82
- pk: { S: parsePartitionKey(this.pk) },
83
- sk: { S: sk },
69
+ pk: { S: this.pk },
70
+ sk: { S: this.buildSk(sk) },
84
71
  },
85
72
  TableName: this.tableName,
86
73
  };
@@ -94,41 +81,41 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
94
81
  * @returns {Promise<Array<T>>} The retrieved records.
95
82
  * @memberof Repository
96
83
  */
97
- async list(): Promise<Array<T>> {
98
- const params: ScanCommandInput = {
99
- ExpressionAttributeNames: {
100
- '#pk': 'pk',
101
- },
102
- ExpressionAttributeValues: {
103
- ':pk': { S: this.pk },
104
- },
105
- FilterExpression: '#pk = :pk',
106
- TableName: this.tableName,
107
- };
108
-
84
+ async list(parameters: ListParameters<never> = {}): Promise<Array<T>> {
109
85
  // Apply pagination for the scan.
110
86
  // By default, the Scan operation processes data sequentially.
111
87
  // DynamoDB returns data in increments of 1 MB increments.
112
88
 
113
- let startKey; /* Indicator of where to paginate from. */
89
+ let startKey: Record<string, AttributeValue> | undefined = undefined; /* Indicator of where to paginate from. */
114
90
  const results: Array<Record<string, AttributeValue>> = [];
115
91
 
116
92
  // Iterate through pages until no more results are found.
117
- while (startKey !== null) {
118
- const res: ScanCommandOutput = await this.dynamoDbClient.send(
119
- new ScanCommand({ ...params, ExclusiveStartKey: startKey }),
93
+ do {
94
+ const res: QueryCommandOutput = await this.dynamoDbClient.send(
95
+ new QueryCommand({
96
+ ExpressionAttributeNames: {
97
+ '#pk': 'pk',
98
+ },
99
+ ExpressionAttributeValues: {
100
+ ':pk': { S: this.pk },
101
+ },
102
+ KeyConditionExpression: '#pk = :pk',
103
+ TableName: this.tableName,
104
+ ExclusiveStartKey: startKey,
105
+ Limit: parameters.limit,
106
+ ConsistentRead: parameters.consistentRead,
107
+ }),
120
108
  );
121
109
 
122
- if (!res) {
123
- // Stop paginating if result is unexpected.
124
- startKey = null;
125
- } else {
126
- results.push(...(res.Items ?? []));
110
+ results.push(...(res.Items ?? []));
111
+ startKey = res.LastEvaluatedKey;
127
112
 
128
- // If no more results, set key to null to terminate loop
129
- startKey = res.LastEvaluatedKey ?? null;
113
+ if (parameters.limit !== undefined && results.length >= parameters.limit) {
114
+ // Truncate list to max requested items, stop querying.
115
+ results.length = parameters.limit;
116
+ break;
130
117
  }
131
- }
118
+ } while (startKey);
132
119
 
133
120
  return results.length ? results.map((item) => this.convertToItem(item as U)) : [];
134
121
  }
@@ -136,16 +123,16 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
136
123
  /**
137
124
  * Update record in database.
138
125
  * @param {string} sk The sort key used to identify the record.
139
- * @param {Partial<T>} input The (partial) record to update.
126
+ * @param {Partial<T>} input The (partial) record to update (properties used in the SK cannot be updated).
140
127
  * @returns {Promise<T | null>} The updated record.
141
128
  * @memberof Repository
142
129
  */
143
- async update(sk: string, input: Partial<T>): Promise<T | null> {
130
+ async update(sk: SK, input: Partial<Omit<T, keyof SK>>): Promise<T> {
144
131
  // Initialise update command
145
132
  const updateParams: UpdateItemCommandInput = {
146
133
  Key: {
147
134
  pk: { S: this.pk },
148
- sk: { S: sk },
135
+ sk: { S: this.buildSk(sk) },
149
136
  },
150
137
  ReturnValues: 'ALL_NEW',
151
138
  TableName: this.tableName,
@@ -156,11 +143,11 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
156
143
 
157
144
  // To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
158
145
  for (const key of Object.keys(input)) {
159
- if (input[key as keyof Partial<T>] === null) {
146
+ if (input[key as keyof Partial<Omit<T, keyof SK>>] === null) {
160
147
  expressionAttributeNames[`#${key}`] = key;
161
148
  expressionAttributeValues[`:${key}`] = { NULL: true };
162
149
  } else {
163
- const val: string = input[key as keyof Partial<T>] as string;
150
+ const val: string = input[key as keyof Partial<Omit<T, keyof SK>>] as string;
164
151
  expressionAttributeNames[`#${key}`] = key;
165
152
  expressionAttributeValues[`:${key}`] = { S: val };
166
153
  }
@@ -176,7 +163,7 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
176
163
  const updateItemCommand = new UpdateItemCommand(updateParams);
177
164
  const updatedRecord = await this.dynamoDbClient.send(updateItemCommand);
178
165
 
179
- return updatedRecord.Attributes ? this.convertToItem(updatedRecord.Attributes as U) : null;
166
+ return this.convertToItem(updatedRecord.Attributes as U);
180
167
  }
181
168
 
182
169
  /**
@@ -186,11 +173,11 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
186
173
  * @returns {void}
187
174
  * @memberof Repository
188
175
  */
189
- async delete(sk: string): Promise<void> {
176
+ async delete(sk: T | SK): Promise<void> {
190
177
  const deleteCommandParams: DeleteItemCommandInput = {
191
178
  Key: {
192
179
  pk: { S: this.pk },
193
- sk: { S: sk },
180
+ sk: { S: this.buildSk(sk) },
194
181
  },
195
182
  TableName: this.tableName,
196
183
  };
@@ -199,10 +186,47 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
199
186
  return;
200
187
  }
201
188
 
189
+ async batchDelete<TItem extends T | SK>(sk: Array<TItem>): Promise<{ unprocessedItems: Array<TItem> }> {
190
+ if (sk.length === 0) {
191
+ return { unprocessedItems: [] };
192
+ }
193
+
194
+ const skMap: Record<string, TItem> = {};
195
+
196
+ sk.forEach((sk) => {
197
+ skMap[this.buildSk(sk)] = sk;
198
+ });
199
+
200
+ const result = await this.dynamoDbClient.send(
201
+ new BatchWriteItemCommand({
202
+ RequestItems: {
203
+ [this.tableName]: Object.keys(skMap).map((sk) => ({
204
+ DeleteRequest: {
205
+ Key: {
206
+ pk: { S: this.pk },
207
+ sk: { S: sk },
208
+ },
209
+ },
210
+ })),
211
+ },
212
+ }),
213
+ );
214
+
215
+ return {
216
+ unprocessedItems:
217
+ result.UnprocessedItems?.[this.tableName]?.map((item) => {
218
+ const sk = item.DeleteRequest?.Key?.sk.S;
219
+
220
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
221
+ return skMap[sk!];
222
+ }) || [],
223
+ };
224
+ }
225
+
202
226
  /**
203
227
  * Convert the DynanomDB attributes of the entity into a DynamoDB item.
204
228
  * @abstract
205
- * @param {U} attributes The DynamoDB item attributes to convert.
229
+ * @param {T} item The DynamoDB item attributes to convert.
206
230
  * @returns {Record<keyof U, AttributeValue>} The DynamoDB record.
207
231
  * @memberof Repository
208
232
  */
@@ -216,4 +240,12 @@ export abstract class Repository<T extends BaseItem, U extends BaseDto> {
216
240
  * @memberof Repository
217
241
  */
218
242
  protected abstract convertToItem(record: U): T;
243
+
244
+ /**
245
+ * Converts a sort key to a string for querying/putting in Dynamo.
246
+ * @param {SK} sk
247
+ * @return {string}
248
+ * @protected
249
+ */
250
+ protected abstract buildSk(sk: T | SK): string;
219
251
  }
@@ -14,25 +14,26 @@ import {
14
14
  ScanCommandOutput,
15
15
  } from '@aws-sdk/client-dynamodb';
16
16
  import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
17
- import { testLogger } from '../../../../jest-global-setup';
18
17
  import {
19
18
  createMockClientPutResponse,
20
19
  createMockClientScanResponse,
21
20
  createMockClientGetResponse,
22
21
  } from '../../core/dynamoClient.mock';
23
- import { PARTITION_KEYS } from '../../constants/Repository.constants';
24
- import { JobContext } from '../../model/JobContext/JobContext';
25
- import { createMockJobContext } from '../../model/JobContext/mocks/JobContext.mock';
26
- import { createMockJobContextDto, createMockJobContextDtos } from '../../model/JobContext/mocks/JobContextDto.mock';
27
- import { createMockJobContextRecord } from '../../model/JobContext/mocks/JobContextRecord.mock';
28
- import { createMockCreateJobContextRequest } from '../../model/JobContext/mocks/JobContextRequest.mock';
22
+ import { ENTITY_TYPES } from '../../constants/Repository.constants';
23
+ import {
24
+ JobContext,
25
+ createMockJobContext,
26
+ createMockJobContextDto,
27
+ createMockJobContextDtos,
28
+ createMockJobContextRecord,
29
+ createMockCreateJobContextRequest,
30
+ } from '../../model/JobContext';
29
31
  import { JobContextRepository } from './JobContextRepository';
30
32
  import * as PaginationUtils from '../../core/pagination/utils/PaginationUtils';
31
- import { PaginationInfo } from '../../core/pagination/model/PaginationInfo';
32
- import { PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/model/PaginationTypes';
33
+ import { PaginationInfo, PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/';
34
+ import { InjectTokens } from '../../constants/InjectTokens';
33
35
 
34
- // Create a new container tsoa can use
35
- initIocContainer(testLogger);
36
+ initIocContainer();
36
37
 
37
38
  // Mock DynamoDB Client
38
39
  const mockOptions = {} as DynamoDBClientConfig;
@@ -41,20 +42,20 @@ const mockSend = jest.fn();
41
42
  mockClient.send = mockSend;
42
43
 
43
44
  // Inject JobContextRepository with a mocked client
44
- iocContainer.rebind<DynamoDBClient>(DynamoDBClient).toConstantValue(mockClient);
45
-
46
- let jobContextRepository: JobContextRepository;
45
+ iocContainer.rebind(DynamoDBClient).toConstantValue(mockClient);
47
46
 
48
47
  describe('JobContextRepository', (): void => {
48
+ let jobContextRepository: JobContextRepository;
49
+ let dynamoTableName: string;
50
+ let tenant: string;
51
+
49
52
  beforeAll(() => {
50
53
  jobContextRepository = iocContainer.get(JobContextRepository);
54
+ dynamoTableName = iocContainer.get(InjectTokens.DynamoTableName);
55
+ tenant = iocContainer.get(InjectTokens.Tenant);
51
56
  });
52
57
 
53
58
  describe('find', () => {
54
- beforeEach(() => {
55
- jest.resetAllMocks();
56
- });
57
-
58
59
  it(`should return an empty list`, async (): Promise<void> => {
59
60
  mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
60
61
 
@@ -80,8 +81,8 @@ describe('JobContextRepository', (): void => {
80
81
  });
81
82
 
82
83
  it(`should paginate results to return a list of existing job contexts`, async (): Promise<void> => {
83
- const mockContextItemsPage1 = createMockJobContextDtos();
84
- const mockContextItemsPage2 = createMockJobContextDtos();
84
+ const mockContextItemsPage1 = createMockJobContextDtos(2);
85
+ const mockContextItemsPage2 = createMockJobContextDtos(2);
85
86
 
86
87
  mockSend
87
88
  .mockImplementationOnce(
@@ -99,17 +100,15 @@ describe('JobContextRepository', (): void => {
99
100
  }),
100
101
  );
101
102
 
102
- const response = await jobContextRepository.list();
103
+ const response = await jobContextRepository.list({
104
+ limit: 3,
105
+ });
103
106
  expect(mockSend).toBeCalledTimes(2);
104
- expect(response).toHaveLength(mockContextItemsPage1.length + mockContextItemsPage2.length);
107
+ expect(response).toHaveLength(3);
105
108
  });
106
109
  });
107
110
 
108
111
  describe('findOne', () => {
109
- beforeEach(() => {
110
- jest.resetAllMocks();
111
- });
112
-
113
112
  it(`should return job context by name`, async (): Promise<void> => {
114
113
  const mockJobContextItem = createMockJobContextDto({
115
114
  environment: { key: 'value' },
@@ -121,7 +120,7 @@ describe('JobContextRepository', (): void => {
121
120
  }),
122
121
  );
123
122
 
124
- const response = await jobContextRepository.findOne(mockJobContextItem.sk.S as string);
123
+ const response = await jobContextRepository.findOne({ contextName: mockJobContextItem.sk.S as string });
125
124
  expect(response).toStrictEqual({
126
125
  contextName: mockJobContextItem.sk.S as string,
127
126
  description: mockJobContextItem.description.S as string,
@@ -139,22 +138,18 @@ describe('JobContextRepository', (): void => {
139
138
  }),
140
139
  );
141
140
 
142
- const response = await jobContextRepository.findOne(`null`);
141
+ const response = await jobContextRepository.findOne({ contextName: `null` });
143
142
  expect(mockSend).toBeCalledTimes(1);
144
143
  expect(response).toBe(null);
145
144
  });
146
145
  });
147
146
 
148
147
  describe('create', () => {
149
- beforeEach(() => {
150
- jest.resetAllMocks();
151
- });
152
-
153
148
  it(`should create a new job context record and return job context in the response`, async (): Promise<void> => {
154
149
  const newJobContextToCreate = createMockCreateJobContextRequest();
155
150
  const mockResponse = createMockClientPutResponse({
156
151
  Attributes: createMockJobContextDto({
157
- pk: PARTITION_KEYS.context,
152
+ pk: `${ENTITY_TYPES.context}~${tenant}`,
158
153
  sk: newJobContextToCreate.contextName,
159
154
  }),
160
155
  });
@@ -165,16 +160,12 @@ describe('JobContextRepository', (): void => {
165
160
  expect(mockSend).toBeCalledTimes(1);
166
161
  expect(response).toStrictEqual({
167
162
  ...newJobContextToCreate,
168
- type: PARTITION_KEYS.context,
163
+ type: ENTITY_TYPES.context,
169
164
  });
170
165
  });
171
166
  });
172
167
 
173
168
  describe('update', () => {
174
- beforeEach(() => {
175
- jest.resetAllMocks();
176
- });
177
-
178
169
  it(`should update a job context record`, async (): Promise<void> => {
179
170
  const jobContextToUpdate = createMockJobContext();
180
171
  const updatedJobContext: JobContext = {
@@ -197,17 +188,16 @@ describe('JobContextRepository', (): void => {
197
188
 
198
189
  mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
199
190
 
200
- const response = await jobContextRepository.update(jobContextToUpdate.contextName, updatedJobContextRecord);
191
+ const response = await jobContextRepository.update(
192
+ { contextName: jobContextToUpdate.contextName },
193
+ updatedJobContextRecord,
194
+ );
201
195
  expect(response).toStrictEqual(updatedJobContext);
202
196
  expect(mockSend).toBeCalledTimes(1);
203
197
  });
204
198
  });
205
199
 
206
200
  describe('delete', () => {
207
- beforeEach(() => {
208
- jest.resetAllMocks();
209
- });
210
-
211
201
  it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
212
202
  mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
213
203
  return {
@@ -215,7 +205,7 @@ describe('JobContextRepository', (): void => {
215
205
  };
216
206
  });
217
207
 
218
- const response = await jobContextRepository.delete(`name`);
208
+ const response = await jobContextRepository.delete({ contextName: `name` });
219
209
  expect(response).toBe(undefined);
220
210
  expect(mockSend).toBeCalledTimes(1);
221
211
  });
@@ -228,7 +218,7 @@ describe('JobContextRepository', (): void => {
228
218
 
229
219
  beforeEach(() => {
230
220
  mockExclusiveStartKey = {
231
- pk: { S: PARTITION_KEYS.context },
221
+ pk: { S: ENTITY_TYPES.context },
232
222
  sk: { S: 'mockSk' },
233
223
  };
234
224
  mockPaginationInfo = {
@@ -240,11 +230,11 @@ describe('JobContextRepository', (): void => {
240
230
  mockParams = {
241
231
  ExclusiveStartKey: mockExclusiveStartKey,
242
232
  ExpressionAttributeValues: {
243
- ':pk': { S: PARTITION_KEYS.context },
233
+ ':pk': { S: `${ENTITY_TYPES.context}~${tenant}` },
244
234
  },
245
235
  KeyConditionExpression: 'pk = :pk',
246
236
  ScanIndexForward: false,
247
- TableName: process.env.DYNAMODB_TABLE,
237
+ TableName: dynamoTableName,
248
238
  };
249
239
 
250
240
  jest.spyOn(PaginationUtils, 'decodeTokenString').mockReturnValue(mockExclusiveStartKey);
@@ -271,10 +261,10 @@ describe('JobContextRepository', (): void => {
271
261
  expect(result.lastEvaluatedKey).toBe(null);
272
262
  });
273
263
 
274
- it('should return an array of jobs if found', async () => {
264
+ it('should return an array of contexts if found', async () => {
275
265
  const mockJobContextRecords = createMockJobContextDtos();
276
266
  const mockLastEvaluatedKey = {
277
- pk: { S: PARTITION_KEYS.job },
267
+ pk: { S: ENTITY_TYPES.job },
278
268
  sk: { S: 'mockSk' },
279
269
  };
280
270