@squiz/job-runner-lib 2.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
  3. package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
  4. package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
  5. package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
  6. package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
  7. package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
  8. package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
  9. package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
  10. package/lib/domains/job/S3Repository/index.d.ts +2 -1
  11. package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
  12. package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
  13. package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
  14. package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
  15. package/lib/domains/job/index.d.ts +6 -4
  16. package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
  17. package/lib/domains/job/model/Base/Base.d.ts +10 -5
  18. package/lib/domains/job/model/Job/Job.d.ts +9 -6
  19. package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
  20. package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
  21. package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
  22. package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
  23. package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
  24. package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
  25. package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
  26. package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
  27. package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
  28. package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
  29. package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
  30. package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
  31. package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
  32. package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
  33. package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
  34. package/lib/domains/job/repository/index.d.ts +1 -0
  35. package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
  36. package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
  37. package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -3
  38. package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
  39. package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
  40. package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
  41. package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
  42. package/lib/domains/job/services/index.d.ts +2 -1
  43. package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
  44. package/lib/index.d.ts +1 -0
  45. package/lib/index.js +13 -13
  46. package/lib/index.js.map +4 -4
  47. package/lib/jest-global-setup.d.ts +6 -1
  48. package/lib/mocks/mockIoc.d.ts +3 -3
  49. package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
  50. package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
  51. package/lib/utils/index.d.ts +2 -0
  52. package/package.json +12 -11
  53. package/src/domains/component/errors/JobError.spec.ts +30 -0
  54. package/src/domains/component/errors/JobError.ts +8 -1
  55. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
  56. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
  57. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
  58. package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
  59. package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
  60. package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
  61. package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
  62. package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
  63. package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
  64. package/src/domains/job/S3Repository/index.ts +2 -1
  65. package/src/domains/job/constants/InjectTokens.ts +8 -0
  66. package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
  67. package/src/domains/job/constants/Repository.constants.ts +3 -3
  68. package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
  69. package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
  70. package/src/domains/job/index.ts +6 -4
  71. package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
  72. package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
  73. package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
  74. package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
  75. package/src/domains/job/model/Base/Base.ts +11 -5
  76. package/src/domains/job/model/Job/Job.ts +11 -6
  77. package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
  78. package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
  79. package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
  80. package/src/domains/job/model/JobContext/JobContext.ts +7 -5
  81. package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
  82. package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
  83. package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
  84. package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
  85. package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
  86. package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
  87. package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
  88. package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
  89. package/src/domains/job/model/LogPayload/index.ts +1 -0
  90. package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
  91. package/src/domains/job/repository/AbstractRepository.ts +91 -59
  92. package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
  93. package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
  94. package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
  95. package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
  96. package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
  97. package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
  98. package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
  99. package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
  100. package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
  101. package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
  102. package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
  103. package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
  104. package/src/domains/job/repository/index.ts +1 -0
  105. package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
  106. package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
  107. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -3
  108. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +113 -31
  109. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +101 -49
  110. package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
  111. package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
  112. package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
  113. package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
  114. package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
  115. package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
  116. package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
  117. package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
  118. package/src/domains/job/services/index.ts +2 -1
  119. package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
  120. package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
  121. package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
  122. package/src/index.ts +1 -0
  123. package/src/jest-global-setup.ts +8 -4
  124. package/src/mocks/mockIoc.ts +18 -43
  125. package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
  126. package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
  127. package/src/utils/index.ts +2 -0
  128. package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
  129. package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
  130. package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
  131. package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
  132. package/src/custom-types.d.ts +0 -7
  133. package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
  134. package/src/domains/job/S3Repository/S3Repository.ts +0 -27
  135. package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
  136. package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
  137. package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
@@ -3,58 +3,60 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
- import * as dotenv from 'dotenv';
7
- dotenv.config();
8
-
9
6
  import {
10
7
  AttributeValue,
11
8
  DynamoDBClient,
12
- GetItemCommand,
13
- GetItemCommandInput,
14
- PutItemCommand,
15
9
  QueryCommand,
16
10
  QueryCommandInput,
17
- ScanCommand,
18
- ScanCommandOutput,
11
+ QueryCommandOutput,
19
12
  UpdateItemCommand,
20
13
  UpdateItemCommandInput,
21
14
  } from '@aws-sdk/client-dynamodb';
22
15
  import { randomInt } from 'crypto';
23
16
  import { inject } from 'inversify';
24
- import { parsePartitionKey } from '../../ParsePartitionKey';
25
17
  import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
26
- import { JobExecutionPk, PARTITION_KEYS } from '../../constants/Repository.constants';
27
- import { PaginationInfo } from '../../core/pagination/model/PaginationInfo';
28
- import { PaginationQueryResponse } from '../../core/pagination/model/PaginationQueryResponse';
29
- import { PAGINATION_DIRECTION_TYPE_AFTER } from '../../core/pagination/model/PaginationTypes';
30
- import { decodeTokenString } from '../../core/pagination/utils/PaginationUtils';
31
- import { InternalServerError } from '../../errors';
18
+ import { ENTITY_TYPES } from '../../constants/Repository.constants';
19
+ import {
20
+ PaginationInfo,
21
+ PaginationQueryResponse,
22
+ PAGINATION_DIRECTION_TYPE_AFTER,
23
+ decodeTokenString,
24
+ } from '../../core/pagination';
32
25
  import {
33
26
  CreateJobExecutionRequest,
34
27
  JobExecution,
35
28
  JobExecutionDto,
29
+ JobExecutionSk,
36
30
  ListJobExecutionParameters,
37
31
  UpdateJobExecutionParameters,
38
- } from '../../model/JobExecution/JobExecution';
32
+ } from '../../model/JobExecution';
39
33
  import { provideTransient } from '../../transientProvider';
40
34
  import { Repository } from '../AbstractRepository';
35
+ import { InjectTokens } from '../../constants/InjectTokens';
36
+ import { convertDynamoDbItemToJobExecution } from './utils';
41
37
 
42
38
  @provideTransient(JobExecutionRepository)
43
- export class JobExecutionRepository extends Repository<JobExecution, JobExecutionDto> {
44
- public constructor(@inject(DynamoDBClient) protected dynamoDbClient: DynamoDBClient) {
45
- super(dynamoDbClient);
46
- this.tableName = process.env.DYNAMODB_TABLE as string;
39
+ export class JobExecutionRepository extends Repository<JobExecution, JobExecutionDto, JobExecutionSk> {
40
+ public constructor(
41
+ @inject(DynamoDBClient) dynamoDbClient: DynamoDBClient,
42
+ @inject(InjectTokens.DynamoTableName) tableName: string,
43
+ @inject(InjectTokens.Tenant) tenant?: string,
44
+ ) {
45
+ super(dynamoDbClient, tableName, `${ENTITY_TYPES.jobExecution}~${tenant}`);
47
46
  }
48
47
 
49
48
  public async begin(createParams: CreateJobExecutionRequest): Promise<JobExecution> {
50
49
  const newJobQueueTime = new Date().toISOString();
51
50
  const newJobId = `${newJobQueueTime.replace(/[^a-zA-Z0-9 ]/g, '')}-${randomInt(10 ** 9, 10 ** 10 - 1)}`; // append a random 10 digit number on the end of the date string
52
51
  const newJobExecution: JobExecution = {
53
- type: `${PARTITION_KEYS.jobExecution}-${createParams.jobName}`,
52
+ type: `${ENTITY_TYPES.jobExecution}-${createParams.jobName}`,
54
53
  ecsTaskId: null,
55
54
  timeStarted: null,
56
55
  timeFinished: null,
57
56
  input: null,
57
+ retryCount: 0,
58
+ lastFailureReason: null,
59
+ retryAfter: null,
58
60
 
59
61
  id: newJobId,
60
62
  timeQueued: newJobQueueTime,
@@ -63,16 +65,7 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
63
65
  ...createParams,
64
66
  };
65
67
 
66
- const dynamoDbRecord = this.convertToDynamoDbItem(newJobExecution);
67
-
68
- await this.dynamoDbClient.send(
69
- new PutItemCommand({
70
- Item: dynamoDbRecord,
71
- TableName: this.tableName,
72
- }),
73
- );
74
-
75
- return newJobExecution;
68
+ return this.create(newJobExecution);
76
69
  }
77
70
 
78
71
  /**
@@ -81,70 +74,66 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
81
74
  * @returns {Promise<Array<JobExecution>>} The retrieved records.
82
75
  * @memberof JobExecutionRepository
83
76
  */
84
- async list(parameters?: ListJobExecutionParameters): Promise<Array<JobExecution>> {
85
- const expressionAttributeNames: Record<string, string> = {};
86
- const expressionAttributeValues: Record<string, AttributeValue> = {};
87
-
88
- const params = {
89
- ExpressionAttributeNames: expressionAttributeNames,
90
- ExpressionAttributeValues: expressionAttributeValues,
91
- FilterExpression: '',
92
- TableName: this.tableName,
93
- ScanIndexForward: false,
77
+ async list(parameters: ListJobExecutionParameters = {}): Promise<Array<JobExecution>> {
78
+ let indexName: string | undefined = undefined;
79
+ const keyConditionExpressions = ['#pk = :pk'];
80
+ const expressionAttributeNames: Record<string, string> = {
81
+ '#pk': 'pk',
82
+ };
83
+ const expressionAttributeValues: Record<string, AttributeValue> = {
84
+ ':pk': { S: this.pk },
94
85
  };
95
86
 
96
- if (!parameters || !parameters.jobName) {
97
- params.ExpressionAttributeNames['#pk'] = 'pk';
98
- params.ExpressionAttributeValues[':pkPrefix'] = { S: PARTITION_KEYS.jobExecution };
99
- params.FilterExpression += `begins_with(#pk, :pkPrefix)`;
100
- }
101
-
102
- if (parameters) {
103
- const { jobName, status } = parameters;
104
-
105
- if (jobName) {
106
- params.ExpressionAttributeNames['#pk'] = 'pk';
107
- params.ExpressionAttributeValues[':pk'] = { S: `${PARTITION_KEYS.jobExecution}-${parameters.jobName}` };
108
- params.FilterExpression += `#pk = :pk`;
109
- }
110
-
111
- if (status) {
112
- // eslint-disable-next-line dot-notation
113
- params.ExpressionAttributeNames['#status'] = 'status';
114
- params.ExpressionAttributeValues[':status'] = { S: status };
115
- params.FilterExpression += ` and #status = :status`;
87
+ if (parameters.filters) {
88
+ const { jobName, status } = parameters.filters;
89
+
90
+ if (jobName && status) {
91
+ indexName = 'lsi1';
92
+ expressionAttributeNames['#lsi1sk'] = 'lsi1sk';
93
+ expressionAttributeValues[':lsi1sk'] = { S: `${status}~${jobName}` };
94
+ keyConditionExpressions.push('#lsi1sk = :lsi1sk');
95
+ } else if (jobName) {
96
+ expressionAttributeNames['#sk'] = 'sk';
97
+ expressionAttributeValues[':skPrefix'] = { S: `${jobName}~` };
98
+ keyConditionExpressions.push('begins_with(#sk, :skPrefix)');
99
+ } else if (status) {
100
+ indexName = 'lsi1';
101
+ expressionAttributeNames['#lsi1sk'] = 'lsi1sk';
102
+ expressionAttributeValues[':lsi1skPrefix'] = { S: `${status}~` };
103
+ keyConditionExpressions.push('begins_with(#lsi1sk, :lsi1skPrefix)');
116
104
  }
117
105
  }
118
106
 
119
- // Apply pagination for the scan.
120
- // By default, the Scan operation processes data sequentially.
121
- // DynamoDB returns data in increments of 1 MB increments.
122
-
123
- let startKey; /* Indicator of where to paginate from. */
107
+ let startKey: Record<string, AttributeValue> | undefined = undefined; /* Indicator of where to paginate from. */
124
108
  const results: Array<Record<string, AttributeValue>> = [];
125
109
 
126
110
  // Iterate through pages until no more results are found.
127
- while (startKey !== null) {
128
- const res: ScanCommandOutput = await this.dynamoDbClient.send(
129
- new ScanCommand({ ...params, ExclusiveStartKey: startKey }),
111
+ do {
112
+ const res: QueryCommandOutput = await this.dynamoDbClient.send(
113
+ new QueryCommand({
114
+ ExpressionAttributeNames: expressionAttributeNames,
115
+ ExpressionAttributeValues: expressionAttributeValues,
116
+ KeyConditionExpression: keyConditionExpressions.join(' and '),
117
+ IndexName: indexName,
118
+ TableName: this.tableName,
119
+ ScanIndexForward: false,
120
+ ExclusiveStartKey: startKey,
121
+ Limit: parameters.limit,
122
+ ConsistentRead: parameters.consistentRead,
123
+ }),
130
124
  );
131
125
 
132
- if (!res) {
133
- // Stop paginating if result is unexpected.
134
- startKey = null;
135
- } else {
136
- results.push(
137
- ...(res.Items ??
138
- /* istanbul ignore next */
139
- []),
140
- );
141
-
142
- // If no more results, set key to null to terminate loop
143
- startKey = res.LastEvaluatedKey ?? null;
126
+ results.push(...(res?.Items || []));
127
+ startKey = res?.LastEvaluatedKey;
128
+
129
+ if (parameters.limit !== undefined && results.length >= parameters.limit) {
130
+ // Truncate list to max requested items, stop querying.
131
+ results.length = parameters.limit;
132
+ break;
144
133
  }
145
- }
134
+ } while (startKey);
146
135
 
147
- return results.length ? results.map((item) => this.convertToItem(item as JobExecutionDto)) : [];
136
+ return results.map((item) => this.convertToItem(item as JobExecutionDto));
148
137
  }
149
138
 
150
139
  /**
@@ -158,9 +147,10 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
158
147
 
159
148
  const params: QueryCommandInput = {
160
149
  ExpressionAttributeValues: {
161
- ':pk': { S: `${PARTITION_KEYS.jobExecution}-${jobName}` },
150
+ ':pk': { S: this.pk },
151
+ ':skPrefix': { S: `${jobName}~` },
162
152
  },
163
- KeyConditionExpression: 'pk = :pk',
153
+ KeyConditionExpression: 'pk = :pk and begins_with(sk, :skPrefix)',
164
154
  TableName: this.tableName,
165
155
  };
166
156
 
@@ -186,55 +176,28 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
186
176
  };
187
177
  }
188
178
 
189
- /**
190
- * Get one record from database.
191
- * @abstract
192
- * @param {string} sk The sort key used to identify the record.
193
- * @returns {Promise<JobExecution | null>} The retrieved record.
194
- * @memberof Repository
195
- */
196
- async findOne(sk: string, jobName?: string): Promise<JobExecution | null> {
197
- if (!jobName) {
198
- throw new InternalServerError('Job name is required');
199
- }
200
- const params: GetItemCommandInput = {
201
- Key: {
202
- pk: { S: parsePartitionKey(`${PARTITION_KEYS.jobExecution}-${jobName}`) },
203
- sk: { S: sk },
204
- },
205
- TableName: this.tableName,
206
- };
207
- const res = await this.dynamoDbClient.send(new GetItemCommand(params));
208
- return res.Item ? this.convertToItem(res.Item as JobExecutionDto) : null;
209
- }
210
-
211
179
  /**
212
180
  * Update record in database.
213
181
  * @param {string} sk The sort key used to identify the record.
214
182
  * @param {Partial<JobExecution>} input The (partial) record to update.
215
- * @param {string} jobName The name of the job used to identify the record as part of the pk.
216
183
  * @param {UpdateItemCommandInput} options Additional options for the update command.
217
184
  * @returns {Promise<JobExecution | null>} The updated record.
218
185
  * @memberof JobExecutionRepository
219
186
  */
220
187
  async update(
221
- sk: string,
188
+ sk: JobExecutionSk,
222
189
  input: UpdateJobExecutionParameters,
223
- jobName?: string,
224
190
  options?: {
225
191
  ConditionExpression?: string;
226
192
  ExpressionAttributeValues?: Record<string, AttributeValue>;
227
193
  ExpressionAttributeNames?: Record<string, string>;
228
194
  },
229
- ): Promise<JobExecution | null> {
230
- if (!jobName || jobName === '') {
231
- throw new InternalServerError('Job name is required');
232
- }
195
+ ): Promise<JobExecution> {
233
196
  // Initialise update command
234
197
  const updateParams: UpdateItemCommandInput = {
235
198
  Key: {
236
- pk: { S: `${PARTITION_KEYS.jobExecution}-${jobName}` },
237
- sk: { S: sk },
199
+ pk: { S: this.pk },
200
+ sk: { S: this.buildSk(sk) },
238
201
  },
239
202
  ReturnValues: 'ALL_NEW',
240
203
  TableName: this.tableName,
@@ -242,25 +205,36 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
242
205
 
243
206
  const expressionAttributeNames: Record<string, string> = {};
244
207
  const expressionAttributeValues: Record<string, AttributeValue> = {};
208
+ const updateExpressionPairs: Array<string> = [];
245
209
 
246
210
  // To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
247
211
  for (const key of Object.keys(input)) {
248
- if (input[key as keyof UpdateJobExecutionParameters] === null) {
249
- expressionAttributeNames[`#${key}`] = key;
212
+ const val = input[key as keyof UpdateJobExecutionParameters];
213
+ expressionAttributeNames[`#${key}`] = key;
214
+
215
+ if (val === null) {
250
216
  expressionAttributeValues[`:${key}`] = { NULL: true };
217
+ } else if (typeof val === 'number') {
218
+ expressionAttributeValues[`:${key}`] = { N: val.toString() };
251
219
  } else {
252
- const val: string = input[key as keyof UpdateJobExecutionParameters] as string;
253
- expressionAttributeNames[`#${key}`] = key;
254
- expressionAttributeValues[`:${key}`] = { S: val };
220
+ expressionAttributeValues[`:${key}`] = { S: val as string };
255
221
  }
256
- }
257
222
 
258
- const updateExpressionPairs = Object.keys(input).map((key) => `#${key} = :${key}`);
259
- const updateExpression = `SET ${updateExpressionPairs.join(', ')}`;
223
+ updateExpressionPairs.push(`#${key} = :${key}`);
224
+
225
+ if (key === 'status') {
226
+ expressionAttributeNames[`#lsi1sk`] = 'lsi1sk';
227
+ expressionAttributeValues[`:lsi1sk`] = { S: `${val || ''}~${sk.jobName}` };
228
+ expressionAttributeNames[`#gsi1pk`] = 'gsi1pk';
229
+ expressionAttributeValues[`:gsi1pk`] = { S: `${ENTITY_TYPES.jobExecution}~${val}` };
230
+ updateExpressionPairs.push(`#lsi1sk = :lsi1sk`);
231
+ updateExpressionPairs.push(`#gsi1pk = :gsi1pk`);
232
+ }
233
+ }
260
234
 
261
235
  updateParams.ExpressionAttributeNames = expressionAttributeNames;
262
236
  updateParams.ExpressionAttributeValues = expressionAttributeValues;
263
- updateParams.UpdateExpression = updateExpression;
237
+ updateParams.UpdateExpression = `SET ${updateExpressionPairs.join(', ')}`;
264
238
 
265
239
  // Add condition expression if provided
266
240
  if (options?.ConditionExpression) {
@@ -286,7 +260,11 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
286
260
  const updateItemCommand = new UpdateItemCommand(updateParams);
287
261
  const updatedRecord = await this.dynamoDbClient.send(updateItemCommand);
288
262
 
289
- return updatedRecord.Attributes ? this.convertToItem(updatedRecord.Attributes as JobExecutionDto) : null;
263
+ return this.convertToItem(updatedRecord.Attributes as JobExecutionDto);
264
+ }
265
+
266
+ buildSk(sk: JobExecutionSk): string {
267
+ return `${sk.jobName}~${sk.id}`;
290
268
  }
291
269
 
292
270
  /* istanbul ignore next */
@@ -294,32 +272,25 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
294
272
  return {
295
273
  context: { S: item.context },
296
274
  ecsTaskId: item.ecsTaskId ? { S: item.ecsTaskId } : { NULL: true },
275
+ gsi1pk: { S: `${ENTITY_TYPES.jobExecution}~${item.status}` },
297
276
  id: { S: item.id },
298
277
  jobName: { S: item.jobName },
299
- pk: { S: item.type },
300
- sk: { S: item.id },
278
+ lsi1sk: { S: `${item.status}~${item.jobName}` },
279
+ pk: { S: this.pk },
280
+ sk: { S: this.buildSk(item) },
301
281
  status: { S: item.status },
302
282
  timeFinished: item.timeFinished ? { S: item.timeFinished } : { NULL: true },
303
283
  timeQueued: { S: item.timeQueued },
304
284
  timeStarted: item.timeStarted ? { S: item.timeQueued } : { NULL: true },
305
285
  version: item.version ? { S: item.version } : { NULL: true },
306
286
  input: item.input ? { S: JSON.stringify(item.input) } : { NULL: true },
287
+ retryCount: { N: item.retryCount.toString() },
288
+ lastFailureReason: item.lastFailureReason ? { S: item.lastFailureReason } : { NULL: true },
289
+ retryAfter: item.retryAfter ? { S: item.retryAfter } : { NULL: true },
307
290
  };
308
291
  }
309
292
 
310
293
  convertToItem(dbItem: JobExecutionDto): JobExecution {
311
- return {
312
- context: dbItem.context.S as string,
313
- ecsTaskId: dbItem.ecsTaskId.S ? (dbItem.ecsTaskId.S as string) : null,
314
- id: dbItem.id.S as string,
315
- jobName: dbItem.jobName.S as string,
316
- status: dbItem.status.S as JOB_EXECUTION_STATUS,
317
- timeFinished: dbItem.timeFinished.S ? (dbItem.timeFinished.S as string) : null,
318
- timeQueued: dbItem.timeQueued.S as string,
319
- timeStarted: dbItem.timeStarted.S ? (dbItem.timeStarted.S as string) : null,
320
- type: parsePartitionKey(dbItem.pk.S as string) as JobExecutionPk,
321
- version: dbItem.version.S as string,
322
- input: dbItem.input?.S ? JSON.parse(dbItem.input.S) : null,
323
- };
294
+ return convertDynamoDbItemToJobExecution(dbItem);
324
295
  }
325
296
  }
@@ -0,0 +1,22 @@
1
+ import { JOB_EXECUTION_STATUS } from '../../../../constants/JobExecutionStatus.constants';
2
+ import { JobExecution, JobExecutionDto } from '../../../../model/JobExecution';
3
+ import { parsePartitionKey } from '../../../../ParsePartitionKey';
4
+
5
+ export const convertDynamoDbItemToJobExecution = (dbItem: JobExecutionDto): JobExecution => {
6
+ return {
7
+ context: dbItem.context.S as string,
8
+ ecsTaskId: dbItem.ecsTaskId.S ? (dbItem.ecsTaskId.S as string) : null,
9
+ id: dbItem.id.S as string,
10
+ jobName: dbItem.jobName.S as string,
11
+ status: dbItem.status.S as JOB_EXECUTION_STATUS,
12
+ timeFinished: dbItem.timeFinished.S ? (dbItem.timeFinished.S as string) : null,
13
+ timeQueued: dbItem.timeQueued.S as string,
14
+ timeStarted: dbItem.timeStarted.S ? (dbItem.timeStarted.S as string) : null,
15
+ type: `${parsePartitionKey(dbItem.pk.S as string).entityType}-${dbItem.jobName.S}`,
16
+ version: dbItem.version.S as string,
17
+ input: dbItem.input?.S ? JSON.parse(dbItem.input.S) : null,
18
+ retryCount: dbItem.retryCount?.N ? parseInt(dbItem.retryCount.N, 10) : 0,
19
+ lastFailureReason: dbItem.lastFailureReason?.S ? (dbItem.lastFailureReason.S as string) : null,
20
+ retryAfter: dbItem.retryAfter?.S ? (dbItem.retryAfter.S as string) : null,
21
+ };
22
+ };
@@ -0,0 +1 @@
1
+ export * from './convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution';
@@ -12,22 +12,23 @@ import {
12
12
  ScanCommandOutput,
13
13
  } from '@aws-sdk/client-dynamodb';
14
14
  import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
15
- import { testLogger } from '../../../../jest-global-setup';
16
15
  import {
17
16
  createMockClientPutResponse,
18
17
  createMockClientScanResponse,
19
18
  createMockClientGetResponse,
20
19
  } from '../../core/dynamoClient.mock';
21
- import { PARTITION_KEYS } from '../../constants/Repository.constants';
22
- import { JobManifest } from '../../manifest/JobManifestTypes';
23
- import { createMockJobManifest } from '../../manifest/mocks/JobManifest.mock';
24
- import { createMockJobManifestDto, createMockJobManifestDtos } from '../../manifest/mocks/JobManifestDto.mock';
25
- import { createMockJobManifestRecord } from '../../manifest/mocks/JobManifestRecord.mock';
26
- import { createMockCreateJobManifestRequest } from '../../manifest/mocks/JobManifestRequest.mock';
20
+ import { ENTITY_TYPES } from '../../constants/Repository.constants';
21
+ import {
22
+ JobManifest,
23
+ createMockJobManifest,
24
+ createMockJobManifestDto,
25
+ createMockJobManifestDtos,
26
+ createMockJobManifestRecord,
27
+ createMockCreateJobManifestRequest,
28
+ } from '../../manifest';
27
29
  import { JobManifestRepository } from './JobManifestRepository';
28
30
 
29
- // Create a new container tsoa can use
30
- initIocContainer(testLogger);
31
+ initIocContainer();
31
32
 
32
33
  // Mock DynamoDB Client
33
34
  const mockOptions = {} as DynamoDBClientConfig;
@@ -46,10 +47,6 @@ describe('JobManifestRepository', (): void => {
46
47
  });
47
48
 
48
49
  describe('find', () => {
49
- beforeEach(() => {
50
- jest.resetAllMocks();
51
- });
52
-
53
50
  it(`should return an empty list`, async (): Promise<void> => {
54
51
  mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
55
52
 
@@ -101,10 +98,6 @@ describe('JobManifestRepository', (): void => {
101
98
  });
102
99
 
103
100
  describe('findOne', () => {
104
- beforeEach(() => {
105
- jest.resetAllMocks();
106
- });
107
-
108
101
  it(`should return job manifest`, async (): Promise<void> => {
109
102
  const mockJobManifestItem = createMockJobManifestDto();
110
103
  mockSend.mockImplementationOnce(
@@ -114,9 +107,10 @@ describe('JobManifestRepository', (): void => {
114
107
  }),
115
108
  );
116
109
 
117
- const response = await jobManifestRepository.findOne(
118
- `${mockJobManifestItem.name.S}-${mockJobManifestItem.version.S}` as string,
119
- );
110
+ const response = await jobManifestRepository.findOne({
111
+ name: mockJobManifestItem.name.S as string,
112
+ version: mockJobManifestItem.version.S as string,
113
+ });
120
114
  expect(response).toStrictEqual({
121
115
  concurrency: parseInt(mockJobManifestItem.concurrency.N as string),
122
116
  entryFile: mockJobManifestItem.entryFile.S as string,
@@ -137,22 +131,18 @@ describe('JobManifestRepository', (): void => {
137
131
  }),
138
132
  );
139
133
 
140
- const response = await jobManifestRepository.findOne(`null`);
134
+ const response = await jobManifestRepository.findOne({ name: 'null', version: '1.0.0' });
141
135
  expect(mockSend).toBeCalledTimes(1);
142
136
  expect(response).toBe(null);
143
137
  });
144
138
  });
145
139
 
146
140
  describe('create', () => {
147
- beforeEach(() => {
148
- jest.resetAllMocks();
149
- });
150
-
151
141
  it(`should create a new job manifest record and return job manifest in the response`, async (): Promise<void> => {
152
142
  const newJobManifestToCreate = createMockCreateJobManifestRequest();
153
143
  const mockReponse = createMockClientPutResponse({
154
144
  Attributes: createMockJobManifestDto({
155
- pk: PARTITION_KEYS.manifest,
145
+ pk: ENTITY_TYPES.manifest,
156
146
  sk: newJobManifestToCreate.name,
157
147
  }),
158
148
  });
@@ -163,16 +153,12 @@ describe('JobManifestRepository', (): void => {
163
153
  expect(mockSend).toBeCalledTimes(1);
164
154
  expect(response).toStrictEqual({
165
155
  ...newJobManifestToCreate,
166
- type: PARTITION_KEYS.manifest,
156
+ type: ENTITY_TYPES.manifest,
167
157
  });
168
158
  });
169
159
  });
170
160
 
171
161
  describe('update', () => {
172
- beforeEach(() => {
173
- jest.resetAllMocks();
174
- });
175
-
176
162
  it(`should update a job manifest record`, async (): Promise<void> => {
177
163
  const jobManifestToUpdate = createMockJobManifest();
178
164
  const updatedJobManifest: JobManifest = {
@@ -182,7 +168,7 @@ describe('JobManifestRepository', (): void => {
182
168
  };
183
169
  const updatedJobManifestRecord = createMockJobManifestRecord({
184
170
  ...updatedJobManifest,
185
- sk: `${updatedJobManifest.name}-${updatedJobManifest.version}`,
171
+ sk: `${updatedJobManifest.name}~${updatedJobManifest.version}`,
186
172
  });
187
173
  const updatedJobManifestRecordDto = createMockJobManifestDto({
188
174
  ...updatedJobManifestRecord,
@@ -194,7 +180,10 @@ describe('JobManifestRepository', (): void => {
194
180
  mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
195
181
 
196
182
  const response = await jobManifestRepository.update(
197
- `${jobManifestToUpdate.name}-${jobManifestToUpdate.version}`,
183
+ {
184
+ name: jobManifestToUpdate.name,
185
+ version: jobManifestToUpdate.version,
186
+ },
198
187
  updatedJobManifestRecord,
199
188
  );
200
189
  expect(response).toStrictEqual(updatedJobManifest);
@@ -203,10 +192,6 @@ describe('JobManifestRepository', (): void => {
203
192
  });
204
193
 
205
194
  describe('delete', () => {
206
- beforeEach(() => {
207
- jest.resetAllMocks();
208
- });
209
-
210
195
  it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
211
196
  mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
212
197
  return {
@@ -214,7 +199,7 @@ describe('JobManifestRepository', (): void => {
214
199
  };
215
200
  });
216
201
 
217
- const response = await jobManifestRepository.delete(`name`);
202
+ const response = await jobManifestRepository.delete({ name: `name`, version: '1.0.0' });
218
203
  expect(response).toBe(undefined);
219
204
  expect(mockSend).toBeCalledTimes(1);
220
205
  });