@squiz/job-runner-lib 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
- package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
- package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
- package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
- package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
- package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
- package/lib/domains/job/S3Repository/index.d.ts +2 -1
- package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
- package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
- package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
- package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
- package/lib/domains/job/index.d.ts +6 -4
- package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
- package/lib/domains/job/model/Base/Base.d.ts +10 -5
- package/lib/domains/job/model/Job/Job.d.ts +9 -6
- package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
- package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
- package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
- package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
- package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
- package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
- package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
- package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
- package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
- package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
- package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
- package/lib/domains/job/repository/index.d.ts +1 -0
- package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -3
- package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
- package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
- package/lib/domains/job/services/index.d.ts +2 -1
- package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
- package/lib/index.d.ts +1 -0
- package/lib/index.js +13 -13
- package/lib/index.js.map +4 -4
- package/lib/jest-global-setup.d.ts +6 -1
- package/lib/mocks/mockIoc.d.ts +3 -3
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
- package/lib/utils/index.d.ts +2 -0
- package/package.json +12 -11
- package/src/domains/component/errors/JobError.spec.ts +30 -0
- package/src/domains/component/errors/JobError.ts +8 -1
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
- package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
- package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
- package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
- package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
- package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
- package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
- package/src/domains/job/S3Repository/index.ts +2 -1
- package/src/domains/job/constants/InjectTokens.ts +8 -0
- package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
- package/src/domains/job/constants/Repository.constants.ts +3 -3
- package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
- package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
- package/src/domains/job/index.ts +6 -4
- package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
- package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
- package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
- package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
- package/src/domains/job/model/Base/Base.ts +11 -5
- package/src/domains/job/model/Job/Job.ts +11 -6
- package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
- package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
- package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
- package/src/domains/job/model/JobContext/JobContext.ts +7 -5
- package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
- package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
- package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
- package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
- package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
- package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
- package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
- package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
- package/src/domains/job/model/LogPayload/index.ts +1 -0
- package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
- package/src/domains/job/repository/AbstractRepository.ts +91 -59
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
- package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
- package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
- package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
- package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
- package/src/domains/job/repository/index.ts +1 -0
- package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
- package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -3
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +113 -31
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +101 -49
- package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
- package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
- package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
- package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
- package/src/domains/job/services/index.ts +2 -1
- package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
- package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
- package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
- package/src/index.ts +1 -0
- package/src/jest-global-setup.ts +8 -4
- package/src/mocks/mockIoc.ts +18 -43
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
- package/src/utils/index.ts +2 -0
- package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
- package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
- package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
- package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
- package/src/custom-types.d.ts +0 -7
- package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
- package/src/domains/job/S3Repository/S3Repository.ts +0 -27
- package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
- package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
- package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
|
@@ -3,58 +3,60 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import * as dotenv from 'dotenv';
|
|
7
|
-
dotenv.config();
|
|
8
|
-
|
|
9
6
|
import {
|
|
10
7
|
AttributeValue,
|
|
11
8
|
DynamoDBClient,
|
|
12
|
-
GetItemCommand,
|
|
13
|
-
GetItemCommandInput,
|
|
14
|
-
PutItemCommand,
|
|
15
9
|
QueryCommand,
|
|
16
10
|
QueryCommandInput,
|
|
17
|
-
|
|
18
|
-
ScanCommandOutput,
|
|
11
|
+
QueryCommandOutput,
|
|
19
12
|
UpdateItemCommand,
|
|
20
13
|
UpdateItemCommandInput,
|
|
21
14
|
} from '@aws-sdk/client-dynamodb';
|
|
22
15
|
import { randomInt } from 'crypto';
|
|
23
16
|
import { inject } from 'inversify';
|
|
24
|
-
import { parsePartitionKey } from '../../ParsePartitionKey';
|
|
25
17
|
import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
18
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
19
|
+
import {
|
|
20
|
+
PaginationInfo,
|
|
21
|
+
PaginationQueryResponse,
|
|
22
|
+
PAGINATION_DIRECTION_TYPE_AFTER,
|
|
23
|
+
decodeTokenString,
|
|
24
|
+
} from '../../core/pagination';
|
|
32
25
|
import {
|
|
33
26
|
CreateJobExecutionRequest,
|
|
34
27
|
JobExecution,
|
|
35
28
|
JobExecutionDto,
|
|
29
|
+
JobExecutionSk,
|
|
36
30
|
ListJobExecutionParameters,
|
|
37
31
|
UpdateJobExecutionParameters,
|
|
38
|
-
} from '../../model/JobExecution
|
|
32
|
+
} from '../../model/JobExecution';
|
|
39
33
|
import { provideTransient } from '../../transientProvider';
|
|
40
34
|
import { Repository } from '../AbstractRepository';
|
|
35
|
+
import { InjectTokens } from '../../constants/InjectTokens';
|
|
36
|
+
import { convertDynamoDbItemToJobExecution } from './utils';
|
|
41
37
|
|
|
42
38
|
@provideTransient(JobExecutionRepository)
|
|
43
|
-
export class JobExecutionRepository extends Repository<JobExecution, JobExecutionDto> {
|
|
44
|
-
public constructor(
|
|
45
|
-
|
|
46
|
-
|
|
39
|
+
export class JobExecutionRepository extends Repository<JobExecution, JobExecutionDto, JobExecutionSk> {
|
|
40
|
+
public constructor(
|
|
41
|
+
@inject(DynamoDBClient) dynamoDbClient: DynamoDBClient,
|
|
42
|
+
@inject(InjectTokens.DynamoTableName) tableName: string,
|
|
43
|
+
@inject(InjectTokens.Tenant) tenant?: string,
|
|
44
|
+
) {
|
|
45
|
+
super(dynamoDbClient, tableName, `${ENTITY_TYPES.jobExecution}~${tenant}`);
|
|
47
46
|
}
|
|
48
47
|
|
|
49
48
|
public async begin(createParams: CreateJobExecutionRequest): Promise<JobExecution> {
|
|
50
49
|
const newJobQueueTime = new Date().toISOString();
|
|
51
50
|
const newJobId = `${newJobQueueTime.replace(/[^a-zA-Z0-9 ]/g, '')}-${randomInt(10 ** 9, 10 ** 10 - 1)}`; // append a random 10 digit number on the end of the date string
|
|
52
51
|
const newJobExecution: JobExecution = {
|
|
53
|
-
type: `${
|
|
52
|
+
type: `${ENTITY_TYPES.jobExecution}-${createParams.jobName}`,
|
|
54
53
|
ecsTaskId: null,
|
|
55
54
|
timeStarted: null,
|
|
56
55
|
timeFinished: null,
|
|
57
56
|
input: null,
|
|
57
|
+
retryCount: 0,
|
|
58
|
+
lastFailureReason: null,
|
|
59
|
+
retryAfter: null,
|
|
58
60
|
|
|
59
61
|
id: newJobId,
|
|
60
62
|
timeQueued: newJobQueueTime,
|
|
@@ -63,16 +65,7 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
63
65
|
...createParams,
|
|
64
66
|
};
|
|
65
67
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
await this.dynamoDbClient.send(
|
|
69
|
-
new PutItemCommand({
|
|
70
|
-
Item: dynamoDbRecord,
|
|
71
|
-
TableName: this.tableName,
|
|
72
|
-
}),
|
|
73
|
-
);
|
|
74
|
-
|
|
75
|
-
return newJobExecution;
|
|
68
|
+
return this.create(newJobExecution);
|
|
76
69
|
}
|
|
77
70
|
|
|
78
71
|
/**
|
|
@@ -81,70 +74,66 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
81
74
|
* @returns {Promise<Array<JobExecution>>} The retrieved records.
|
|
82
75
|
* @memberof JobExecutionRepository
|
|
83
76
|
*/
|
|
84
|
-
async list(parameters
|
|
85
|
-
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
TableName: this.tableName,
|
|
93
|
-
ScanIndexForward: false,
|
|
77
|
+
async list(parameters: ListJobExecutionParameters = {}): Promise<Array<JobExecution>> {
|
|
78
|
+
let indexName: string | undefined = undefined;
|
|
79
|
+
const keyConditionExpressions = ['#pk = :pk'];
|
|
80
|
+
const expressionAttributeNames: Record<string, string> = {
|
|
81
|
+
'#pk': 'pk',
|
|
82
|
+
};
|
|
83
|
+
const expressionAttributeValues: Record<string, AttributeValue> = {
|
|
84
|
+
':pk': { S: this.pk },
|
|
94
85
|
};
|
|
95
86
|
|
|
96
|
-
if (
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
params.ExpressionAttributeNames['#status'] = 'status';
|
|
114
|
-
params.ExpressionAttributeValues[':status'] = { S: status };
|
|
115
|
-
params.FilterExpression += ` and #status = :status`;
|
|
87
|
+
if (parameters.filters) {
|
|
88
|
+
const { jobName, status } = parameters.filters;
|
|
89
|
+
|
|
90
|
+
if (jobName && status) {
|
|
91
|
+
indexName = 'lsi1';
|
|
92
|
+
expressionAttributeNames['#lsi1sk'] = 'lsi1sk';
|
|
93
|
+
expressionAttributeValues[':lsi1sk'] = { S: `${status}~${jobName}` };
|
|
94
|
+
keyConditionExpressions.push('#lsi1sk = :lsi1sk');
|
|
95
|
+
} else if (jobName) {
|
|
96
|
+
expressionAttributeNames['#sk'] = 'sk';
|
|
97
|
+
expressionAttributeValues[':skPrefix'] = { S: `${jobName}~` };
|
|
98
|
+
keyConditionExpressions.push('begins_with(#sk, :skPrefix)');
|
|
99
|
+
} else if (status) {
|
|
100
|
+
indexName = 'lsi1';
|
|
101
|
+
expressionAttributeNames['#lsi1sk'] = 'lsi1sk';
|
|
102
|
+
expressionAttributeValues[':lsi1skPrefix'] = { S: `${status}~` };
|
|
103
|
+
keyConditionExpressions.push('begins_with(#lsi1sk, :lsi1skPrefix)');
|
|
116
104
|
}
|
|
117
105
|
}
|
|
118
106
|
|
|
119
|
-
|
|
120
|
-
// By default, the Scan operation processes data sequentially.
|
|
121
|
-
// DynamoDB returns data in increments of 1 MB increments.
|
|
122
|
-
|
|
123
|
-
let startKey; /* Indicator of where to paginate from. */
|
|
107
|
+
let startKey: Record<string, AttributeValue> | undefined = undefined; /* Indicator of where to paginate from. */
|
|
124
108
|
const results: Array<Record<string, AttributeValue>> = [];
|
|
125
109
|
|
|
126
110
|
// Iterate through pages until no more results are found.
|
|
127
|
-
|
|
128
|
-
const res:
|
|
129
|
-
new
|
|
111
|
+
do {
|
|
112
|
+
const res: QueryCommandOutput = await this.dynamoDbClient.send(
|
|
113
|
+
new QueryCommand({
|
|
114
|
+
ExpressionAttributeNames: expressionAttributeNames,
|
|
115
|
+
ExpressionAttributeValues: expressionAttributeValues,
|
|
116
|
+
KeyConditionExpression: keyConditionExpressions.join(' and '),
|
|
117
|
+
IndexName: indexName,
|
|
118
|
+
TableName: this.tableName,
|
|
119
|
+
ScanIndexForward: false,
|
|
120
|
+
ExclusiveStartKey: startKey,
|
|
121
|
+
Limit: parameters.limit,
|
|
122
|
+
ConsistentRead: parameters.consistentRead,
|
|
123
|
+
}),
|
|
130
124
|
);
|
|
131
125
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
[]),
|
|
140
|
-
);
|
|
141
|
-
|
|
142
|
-
// If no more results, set key to null to terminate loop
|
|
143
|
-
startKey = res.LastEvaluatedKey ?? null;
|
|
126
|
+
results.push(...(res?.Items || []));
|
|
127
|
+
startKey = res?.LastEvaluatedKey;
|
|
128
|
+
|
|
129
|
+
if (parameters.limit !== undefined && results.length >= parameters.limit) {
|
|
130
|
+
// Truncate list to max requested items, stop querying.
|
|
131
|
+
results.length = parameters.limit;
|
|
132
|
+
break;
|
|
144
133
|
}
|
|
145
|
-
}
|
|
134
|
+
} while (startKey);
|
|
146
135
|
|
|
147
|
-
return results.
|
|
136
|
+
return results.map((item) => this.convertToItem(item as JobExecutionDto));
|
|
148
137
|
}
|
|
149
138
|
|
|
150
139
|
/**
|
|
@@ -158,9 +147,10 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
158
147
|
|
|
159
148
|
const params: QueryCommandInput = {
|
|
160
149
|
ExpressionAttributeValues: {
|
|
161
|
-
':pk': { S:
|
|
150
|
+
':pk': { S: this.pk },
|
|
151
|
+
':skPrefix': { S: `${jobName}~` },
|
|
162
152
|
},
|
|
163
|
-
KeyConditionExpression: 'pk = :pk',
|
|
153
|
+
KeyConditionExpression: 'pk = :pk and begins_with(sk, :skPrefix)',
|
|
164
154
|
TableName: this.tableName,
|
|
165
155
|
};
|
|
166
156
|
|
|
@@ -186,55 +176,28 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
186
176
|
};
|
|
187
177
|
}
|
|
188
178
|
|
|
189
|
-
/**
|
|
190
|
-
* Get one record from database.
|
|
191
|
-
* @abstract
|
|
192
|
-
* @param {string} sk The sort key used to identify the record.
|
|
193
|
-
* @returns {Promise<JobExecution | null>} The retrieved record.
|
|
194
|
-
* @memberof Repository
|
|
195
|
-
*/
|
|
196
|
-
async findOne(sk: string, jobName?: string): Promise<JobExecution | null> {
|
|
197
|
-
if (!jobName) {
|
|
198
|
-
throw new InternalServerError('Job name is required');
|
|
199
|
-
}
|
|
200
|
-
const params: GetItemCommandInput = {
|
|
201
|
-
Key: {
|
|
202
|
-
pk: { S: parsePartitionKey(`${PARTITION_KEYS.jobExecution}-${jobName}`) },
|
|
203
|
-
sk: { S: sk },
|
|
204
|
-
},
|
|
205
|
-
TableName: this.tableName,
|
|
206
|
-
};
|
|
207
|
-
const res = await this.dynamoDbClient.send(new GetItemCommand(params));
|
|
208
|
-
return res.Item ? this.convertToItem(res.Item as JobExecutionDto) : null;
|
|
209
|
-
}
|
|
210
|
-
|
|
211
179
|
/**
|
|
212
180
|
* Update record in database.
|
|
213
181
|
* @param {string} sk The sort key used to identify the record.
|
|
214
182
|
* @param {Partial<JobExecution>} input The (partial) record to update.
|
|
215
|
-
* @param {string} jobName The name of the job used to identify the record as part of the pk.
|
|
216
183
|
* @param {UpdateItemCommandInput} options Additional options for the update command.
|
|
217
184
|
* @returns {Promise<JobExecution | null>} The updated record.
|
|
218
185
|
* @memberof JobExecutionRepository
|
|
219
186
|
*/
|
|
220
187
|
async update(
|
|
221
|
-
sk:
|
|
188
|
+
sk: JobExecutionSk,
|
|
222
189
|
input: UpdateJobExecutionParameters,
|
|
223
|
-
jobName?: string,
|
|
224
190
|
options?: {
|
|
225
191
|
ConditionExpression?: string;
|
|
226
192
|
ExpressionAttributeValues?: Record<string, AttributeValue>;
|
|
227
193
|
ExpressionAttributeNames?: Record<string, string>;
|
|
228
194
|
},
|
|
229
|
-
): Promise<JobExecution
|
|
230
|
-
if (!jobName || jobName === '') {
|
|
231
|
-
throw new InternalServerError('Job name is required');
|
|
232
|
-
}
|
|
195
|
+
): Promise<JobExecution> {
|
|
233
196
|
// Initialise update command
|
|
234
197
|
const updateParams: UpdateItemCommandInput = {
|
|
235
198
|
Key: {
|
|
236
|
-
pk: { S:
|
|
237
|
-
sk: { S: sk },
|
|
199
|
+
pk: { S: this.pk },
|
|
200
|
+
sk: { S: this.buildSk(sk) },
|
|
238
201
|
},
|
|
239
202
|
ReturnValues: 'ALL_NEW',
|
|
240
203
|
TableName: this.tableName,
|
|
@@ -242,25 +205,36 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
242
205
|
|
|
243
206
|
const expressionAttributeNames: Record<string, string> = {};
|
|
244
207
|
const expressionAttributeValues: Record<string, AttributeValue> = {};
|
|
208
|
+
const updateExpressionPairs: Array<string> = [];
|
|
245
209
|
|
|
246
210
|
// To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
|
|
247
211
|
for (const key of Object.keys(input)) {
|
|
248
|
-
|
|
249
|
-
|
|
212
|
+
const val = input[key as keyof UpdateJobExecutionParameters];
|
|
213
|
+
expressionAttributeNames[`#${key}`] = key;
|
|
214
|
+
|
|
215
|
+
if (val === null) {
|
|
250
216
|
expressionAttributeValues[`:${key}`] = { NULL: true };
|
|
217
|
+
} else if (typeof val === 'number') {
|
|
218
|
+
expressionAttributeValues[`:${key}`] = { N: val.toString() };
|
|
251
219
|
} else {
|
|
252
|
-
|
|
253
|
-
expressionAttributeNames[`#${key}`] = key;
|
|
254
|
-
expressionAttributeValues[`:${key}`] = { S: val };
|
|
220
|
+
expressionAttributeValues[`:${key}`] = { S: val as string };
|
|
255
221
|
}
|
|
256
|
-
}
|
|
257
222
|
|
|
258
|
-
|
|
259
|
-
|
|
223
|
+
updateExpressionPairs.push(`#${key} = :${key}`);
|
|
224
|
+
|
|
225
|
+
if (key === 'status') {
|
|
226
|
+
expressionAttributeNames[`#lsi1sk`] = 'lsi1sk';
|
|
227
|
+
expressionAttributeValues[`:lsi1sk`] = { S: `${val || ''}~${sk.jobName}` };
|
|
228
|
+
expressionAttributeNames[`#gsi1pk`] = 'gsi1pk';
|
|
229
|
+
expressionAttributeValues[`:gsi1pk`] = { S: `${ENTITY_TYPES.jobExecution}~${val}` };
|
|
230
|
+
updateExpressionPairs.push(`#lsi1sk = :lsi1sk`);
|
|
231
|
+
updateExpressionPairs.push(`#gsi1pk = :gsi1pk`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
260
234
|
|
|
261
235
|
updateParams.ExpressionAttributeNames = expressionAttributeNames;
|
|
262
236
|
updateParams.ExpressionAttributeValues = expressionAttributeValues;
|
|
263
|
-
updateParams.UpdateExpression =
|
|
237
|
+
updateParams.UpdateExpression = `SET ${updateExpressionPairs.join(', ')}`;
|
|
264
238
|
|
|
265
239
|
// Add condition expression if provided
|
|
266
240
|
if (options?.ConditionExpression) {
|
|
@@ -286,7 +260,11 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
286
260
|
const updateItemCommand = new UpdateItemCommand(updateParams);
|
|
287
261
|
const updatedRecord = await this.dynamoDbClient.send(updateItemCommand);
|
|
288
262
|
|
|
289
|
-
return
|
|
263
|
+
return this.convertToItem(updatedRecord.Attributes as JobExecutionDto);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
buildSk(sk: JobExecutionSk): string {
|
|
267
|
+
return `${sk.jobName}~${sk.id}`;
|
|
290
268
|
}
|
|
291
269
|
|
|
292
270
|
/* istanbul ignore next */
|
|
@@ -294,32 +272,25 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
|
|
|
294
272
|
return {
|
|
295
273
|
context: { S: item.context },
|
|
296
274
|
ecsTaskId: item.ecsTaskId ? { S: item.ecsTaskId } : { NULL: true },
|
|
275
|
+
gsi1pk: { S: `${ENTITY_TYPES.jobExecution}~${item.status}` },
|
|
297
276
|
id: { S: item.id },
|
|
298
277
|
jobName: { S: item.jobName },
|
|
299
|
-
|
|
300
|
-
|
|
278
|
+
lsi1sk: { S: `${item.status}~${item.jobName}` },
|
|
279
|
+
pk: { S: this.pk },
|
|
280
|
+
sk: { S: this.buildSk(item) },
|
|
301
281
|
status: { S: item.status },
|
|
302
282
|
timeFinished: item.timeFinished ? { S: item.timeFinished } : { NULL: true },
|
|
303
283
|
timeQueued: { S: item.timeQueued },
|
|
304
284
|
timeStarted: item.timeStarted ? { S: item.timeQueued } : { NULL: true },
|
|
305
285
|
version: item.version ? { S: item.version } : { NULL: true },
|
|
306
286
|
input: item.input ? { S: JSON.stringify(item.input) } : { NULL: true },
|
|
287
|
+
retryCount: { N: item.retryCount.toString() },
|
|
288
|
+
lastFailureReason: item.lastFailureReason ? { S: item.lastFailureReason } : { NULL: true },
|
|
289
|
+
retryAfter: item.retryAfter ? { S: item.retryAfter } : { NULL: true },
|
|
307
290
|
};
|
|
308
291
|
}
|
|
309
292
|
|
|
310
293
|
convertToItem(dbItem: JobExecutionDto): JobExecution {
|
|
311
|
-
return
|
|
312
|
-
context: dbItem.context.S as string,
|
|
313
|
-
ecsTaskId: dbItem.ecsTaskId.S ? (dbItem.ecsTaskId.S as string) : null,
|
|
314
|
-
id: dbItem.id.S as string,
|
|
315
|
-
jobName: dbItem.jobName.S as string,
|
|
316
|
-
status: dbItem.status.S as JOB_EXECUTION_STATUS,
|
|
317
|
-
timeFinished: dbItem.timeFinished.S ? (dbItem.timeFinished.S as string) : null,
|
|
318
|
-
timeQueued: dbItem.timeQueued.S as string,
|
|
319
|
-
timeStarted: dbItem.timeStarted.S ? (dbItem.timeStarted.S as string) : null,
|
|
320
|
-
type: parsePartitionKey(dbItem.pk.S as string) as JobExecutionPk,
|
|
321
|
-
version: dbItem.version.S as string,
|
|
322
|
-
input: dbItem.input?.S ? JSON.parse(dbItem.input.S) : null,
|
|
323
|
-
};
|
|
294
|
+
return convertDynamoDbItemToJobExecution(dbItem);
|
|
324
295
|
}
|
|
325
296
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { JOB_EXECUTION_STATUS } from '../../../../constants/JobExecutionStatus.constants';
|
|
2
|
+
import { JobExecution, JobExecutionDto } from '../../../../model/JobExecution';
|
|
3
|
+
import { parsePartitionKey } from '../../../../ParsePartitionKey';
|
|
4
|
+
|
|
5
|
+
export const convertDynamoDbItemToJobExecution = (dbItem: JobExecutionDto): JobExecution => {
|
|
6
|
+
return {
|
|
7
|
+
context: dbItem.context.S as string,
|
|
8
|
+
ecsTaskId: dbItem.ecsTaskId.S ? (dbItem.ecsTaskId.S as string) : null,
|
|
9
|
+
id: dbItem.id.S as string,
|
|
10
|
+
jobName: dbItem.jobName.S as string,
|
|
11
|
+
status: dbItem.status.S as JOB_EXECUTION_STATUS,
|
|
12
|
+
timeFinished: dbItem.timeFinished.S ? (dbItem.timeFinished.S as string) : null,
|
|
13
|
+
timeQueued: dbItem.timeQueued.S as string,
|
|
14
|
+
timeStarted: dbItem.timeStarted.S ? (dbItem.timeStarted.S as string) : null,
|
|
15
|
+
type: `${parsePartitionKey(dbItem.pk.S as string).entityType}-${dbItem.jobName.S}`,
|
|
16
|
+
version: dbItem.version.S as string,
|
|
17
|
+
input: dbItem.input?.S ? JSON.parse(dbItem.input.S) : null,
|
|
18
|
+
retryCount: dbItem.retryCount?.N ? parseInt(dbItem.retryCount.N, 10) : 0,
|
|
19
|
+
lastFailureReason: dbItem.lastFailureReason?.S ? (dbItem.lastFailureReason.S as string) : null,
|
|
20
|
+
retryAfter: dbItem.retryAfter?.S ? (dbItem.retryAfter.S as string) : null,
|
|
21
|
+
};
|
|
22
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution';
|
|
@@ -12,22 +12,23 @@ import {
|
|
|
12
12
|
ScanCommandOutput,
|
|
13
13
|
} from '@aws-sdk/client-dynamodb';
|
|
14
14
|
import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
|
|
15
|
-
import { testLogger } from '../../../../jest-global-setup';
|
|
16
15
|
import {
|
|
17
16
|
createMockClientPutResponse,
|
|
18
17
|
createMockClientScanResponse,
|
|
19
18
|
createMockClientGetResponse,
|
|
20
19
|
} from '../../core/dynamoClient.mock';
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
20
|
+
import { ENTITY_TYPES } from '../../constants/Repository.constants';
|
|
21
|
+
import {
|
|
22
|
+
JobManifest,
|
|
23
|
+
createMockJobManifest,
|
|
24
|
+
createMockJobManifestDto,
|
|
25
|
+
createMockJobManifestDtos,
|
|
26
|
+
createMockJobManifestRecord,
|
|
27
|
+
createMockCreateJobManifestRequest,
|
|
28
|
+
} from '../../manifest';
|
|
27
29
|
import { JobManifestRepository } from './JobManifestRepository';
|
|
28
30
|
|
|
29
|
-
|
|
30
|
-
initIocContainer(testLogger);
|
|
31
|
+
initIocContainer();
|
|
31
32
|
|
|
32
33
|
// Mock DynamoDB Client
|
|
33
34
|
const mockOptions = {} as DynamoDBClientConfig;
|
|
@@ -46,10 +47,6 @@ describe('JobManifestRepository', (): void => {
|
|
|
46
47
|
});
|
|
47
48
|
|
|
48
49
|
describe('find', () => {
|
|
49
|
-
beforeEach(() => {
|
|
50
|
-
jest.resetAllMocks();
|
|
51
|
-
});
|
|
52
|
-
|
|
53
50
|
it(`should return an empty list`, async (): Promise<void> => {
|
|
54
51
|
mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
|
|
55
52
|
|
|
@@ -101,10 +98,6 @@ describe('JobManifestRepository', (): void => {
|
|
|
101
98
|
});
|
|
102
99
|
|
|
103
100
|
describe('findOne', () => {
|
|
104
|
-
beforeEach(() => {
|
|
105
|
-
jest.resetAllMocks();
|
|
106
|
-
});
|
|
107
|
-
|
|
108
101
|
it(`should return job manifest`, async (): Promise<void> => {
|
|
109
102
|
const mockJobManifestItem = createMockJobManifestDto();
|
|
110
103
|
mockSend.mockImplementationOnce(
|
|
@@ -114,9 +107,10 @@ describe('JobManifestRepository', (): void => {
|
|
|
114
107
|
}),
|
|
115
108
|
);
|
|
116
109
|
|
|
117
|
-
const response = await jobManifestRepository.findOne(
|
|
118
|
-
|
|
119
|
-
|
|
110
|
+
const response = await jobManifestRepository.findOne({
|
|
111
|
+
name: mockJobManifestItem.name.S as string,
|
|
112
|
+
version: mockJobManifestItem.version.S as string,
|
|
113
|
+
});
|
|
120
114
|
expect(response).toStrictEqual({
|
|
121
115
|
concurrency: parseInt(mockJobManifestItem.concurrency.N as string),
|
|
122
116
|
entryFile: mockJobManifestItem.entryFile.S as string,
|
|
@@ -137,22 +131,18 @@ describe('JobManifestRepository', (): void => {
|
|
|
137
131
|
}),
|
|
138
132
|
);
|
|
139
133
|
|
|
140
|
-
const response = await jobManifestRepository.findOne(
|
|
134
|
+
const response = await jobManifestRepository.findOne({ name: 'null', version: '1.0.0' });
|
|
141
135
|
expect(mockSend).toBeCalledTimes(1);
|
|
142
136
|
expect(response).toBe(null);
|
|
143
137
|
});
|
|
144
138
|
});
|
|
145
139
|
|
|
146
140
|
describe('create', () => {
|
|
147
|
-
beforeEach(() => {
|
|
148
|
-
jest.resetAllMocks();
|
|
149
|
-
});
|
|
150
|
-
|
|
151
141
|
it(`should create a new job manifest record and return job manifest in the response`, async (): Promise<void> => {
|
|
152
142
|
const newJobManifestToCreate = createMockCreateJobManifestRequest();
|
|
153
143
|
const mockReponse = createMockClientPutResponse({
|
|
154
144
|
Attributes: createMockJobManifestDto({
|
|
155
|
-
pk:
|
|
145
|
+
pk: ENTITY_TYPES.manifest,
|
|
156
146
|
sk: newJobManifestToCreate.name,
|
|
157
147
|
}),
|
|
158
148
|
});
|
|
@@ -163,16 +153,12 @@ describe('JobManifestRepository', (): void => {
|
|
|
163
153
|
expect(mockSend).toBeCalledTimes(1);
|
|
164
154
|
expect(response).toStrictEqual({
|
|
165
155
|
...newJobManifestToCreate,
|
|
166
|
-
type:
|
|
156
|
+
type: ENTITY_TYPES.manifest,
|
|
167
157
|
});
|
|
168
158
|
});
|
|
169
159
|
});
|
|
170
160
|
|
|
171
161
|
describe('update', () => {
|
|
172
|
-
beforeEach(() => {
|
|
173
|
-
jest.resetAllMocks();
|
|
174
|
-
});
|
|
175
|
-
|
|
176
162
|
it(`should update a job manifest record`, async (): Promise<void> => {
|
|
177
163
|
const jobManifestToUpdate = createMockJobManifest();
|
|
178
164
|
const updatedJobManifest: JobManifest = {
|
|
@@ -182,7 +168,7 @@ describe('JobManifestRepository', (): void => {
|
|
|
182
168
|
};
|
|
183
169
|
const updatedJobManifestRecord = createMockJobManifestRecord({
|
|
184
170
|
...updatedJobManifest,
|
|
185
|
-
sk: `${updatedJobManifest.name}
|
|
171
|
+
sk: `${updatedJobManifest.name}~${updatedJobManifest.version}`,
|
|
186
172
|
});
|
|
187
173
|
const updatedJobManifestRecordDto = createMockJobManifestDto({
|
|
188
174
|
...updatedJobManifestRecord,
|
|
@@ -194,7 +180,10 @@ describe('JobManifestRepository', (): void => {
|
|
|
194
180
|
mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
|
|
195
181
|
|
|
196
182
|
const response = await jobManifestRepository.update(
|
|
197
|
-
|
|
183
|
+
{
|
|
184
|
+
name: jobManifestToUpdate.name,
|
|
185
|
+
version: jobManifestToUpdate.version,
|
|
186
|
+
},
|
|
198
187
|
updatedJobManifestRecord,
|
|
199
188
|
);
|
|
200
189
|
expect(response).toStrictEqual(updatedJobManifest);
|
|
@@ -203,10 +192,6 @@ describe('JobManifestRepository', (): void => {
|
|
|
203
192
|
});
|
|
204
193
|
|
|
205
194
|
describe('delete', () => {
|
|
206
|
-
beforeEach(() => {
|
|
207
|
-
jest.resetAllMocks();
|
|
208
|
-
});
|
|
209
|
-
|
|
210
195
|
it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
|
|
211
196
|
mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
|
|
212
197
|
return {
|
|
@@ -214,7 +199,7 @@ describe('JobManifestRepository', (): void => {
|
|
|
214
199
|
};
|
|
215
200
|
});
|
|
216
201
|
|
|
217
|
-
const response = await jobManifestRepository.delete(`name
|
|
202
|
+
const response = await jobManifestRepository.delete({ name: `name`, version: '1.0.0' });
|
|
218
203
|
expect(response).toBe(undefined);
|
|
219
204
|
expect(mockSend).toBeCalledTimes(1);
|
|
220
205
|
});
|