@squiz/job-runner-lib 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
- package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
- package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
- package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
- package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
- package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
- package/lib/domains/job/S3Repository/index.d.ts +2 -1
- package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
- package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
- package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
- package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
- package/lib/domains/job/index.d.ts +6 -4
- package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
- package/lib/domains/job/model/Base/Base.d.ts +10 -5
- package/lib/domains/job/model/Job/Job.d.ts +9 -6
- package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
- package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
- package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
- package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
- package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
- package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
- package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
- package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
- package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
- package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
- package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
- package/lib/domains/job/repository/index.d.ts +1 -0
- package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -4
- package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
- package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
- package/lib/domains/job/services/index.d.ts +2 -1
- package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
- package/lib/index.d.ts +1 -0
- package/lib/index.js +13 -13
- package/lib/index.js.map +4 -4
- package/lib/jest-global-setup.d.ts +6 -1
- package/lib/mocks/mockIoc.d.ts +3 -3
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
- package/lib/utils/index.d.ts +2 -0
- package/package.json +12 -11
- package/src/domains/component/errors/JobError.spec.ts +30 -0
- package/src/domains/component/errors/JobError.ts +8 -1
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
- package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
- package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
- package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
- package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
- package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
- package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
- package/src/domains/job/S3Repository/index.ts +2 -1
- package/src/domains/job/constants/InjectTokens.ts +8 -0
- package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
- package/src/domains/job/constants/Repository.constants.ts +3 -3
- package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
- package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
- package/src/domains/job/index.ts +6 -4
- package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
- package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
- package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
- package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
- package/src/domains/job/model/Base/Base.ts +11 -5
- package/src/domains/job/model/Job/Job.ts +11 -6
- package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
- package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
- package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
- package/src/domains/job/model/JobContext/JobContext.ts +7 -5
- package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
- package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
- package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
- package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
- package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
- package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
- package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
- package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
- package/src/domains/job/model/LogPayload/index.ts +1 -0
- package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
- package/src/domains/job/repository/AbstractRepository.ts +91 -59
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
- package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
- package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
- package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
- package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
- package/src/domains/job/repository/index.ts +1 -0
- package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
- package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -4
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +120 -41
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +95 -49
- package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
- package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
- package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
- package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
- package/src/domains/job/services/index.ts +2 -1
- package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
- package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
- package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
- package/src/index.ts +1 -0
- package/src/jest-global-setup.ts +8 -4
- package/src/mocks/mockIoc.ts +18 -43
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
- package/src/utils/index.ts +2 -0
- package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
- package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
- package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
- package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
- package/src/custom-types.d.ts +0 -7
- package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
- package/src/domains/job/S3Repository/S3Repository.ts +0 -27
- package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
- package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
- package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
package/lib/mocks/mockIoc.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { Logger } from '@squiz/optimization-logger';
|
|
1
2
|
import { Container } from 'inversify';
|
|
2
|
-
import { Logger } from '@squiz/dx-logger-lib';
|
|
3
3
|
declare let iocContainer: Container;
|
|
4
|
-
declare function initIocContainer(
|
|
5
|
-
export {
|
|
4
|
+
declare function initIocContainer(logger?: Logger): void;
|
|
5
|
+
export { initIocContainer, iocContainer };
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Logger } from '@squiz/optimization-logger';
|
|
2
|
+
import { ContainerModule } from 'inversify';
|
|
3
|
+
export type CreateJobRunnerModuleProps = {
|
|
4
|
+
dynamoTableName: string;
|
|
5
|
+
jobUploadsBucket: string;
|
|
6
|
+
jobDetailsBucket: string;
|
|
7
|
+
awsRegion: string;
|
|
8
|
+
local: boolean;
|
|
9
|
+
jobStartQueueUrl?: string;
|
|
10
|
+
};
|
|
11
|
+
export type CreateJobRunnerModuleResult = (props: {
|
|
12
|
+
tenant?: string;
|
|
13
|
+
logger: Logger;
|
|
14
|
+
}) => ContainerModule;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@squiz/job-runner-lib",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"private": false,
|
|
@@ -18,32 +18,33 @@
|
|
|
18
18
|
"license": "ISC",
|
|
19
19
|
"devDependencies": {
|
|
20
20
|
"@faker-js/faker": "8.0.2",
|
|
21
|
-
"@
|
|
21
|
+
"@smithy/util-stream": "^4.2.3",
|
|
22
|
+
"@squiz/virus-scanner-lib": "^1.68.0",
|
|
22
23
|
"@types/cli-color": "2.0.6",
|
|
23
24
|
"@types/fs-extra": "11.0.1",
|
|
24
25
|
"@types/jest": "28.1.8",
|
|
25
26
|
"@types/node": "22.9.0",
|
|
26
27
|
"aws-sdk-client-mock": "^2.2.0",
|
|
27
28
|
"aws-sdk-client-mock-jest": "^2.2.0",
|
|
28
|
-
"dotenv": "16.0.3",
|
|
29
29
|
"esbuild": "^0.25.8"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@aws-sdk/client-dynamodb": "3.
|
|
33
|
-
"@aws-sdk/client-ecs": "3.
|
|
34
|
-
"@aws-sdk/client-s3": "3.
|
|
35
|
-
"@aws-sdk/
|
|
32
|
+
"@aws-sdk/client-dynamodb": "3.1003.0",
|
|
33
|
+
"@aws-sdk/client-ecs": "3.1003.0",
|
|
34
|
+
"@aws-sdk/client-s3": "3.1003.0",
|
|
35
|
+
"@aws-sdk/client-sqs": "3.1003.0",
|
|
36
|
+
"@aws-sdk/s3-request-presigner": "3.1003.0",
|
|
36
37
|
"@squiz/dx-common-lib": "^1.66.1",
|
|
37
38
|
"@squiz/dx-json-schema-lib": "1.78.1",
|
|
38
39
|
"@squiz/dx-logger-lib": "^1.64.0",
|
|
39
|
-
"
|
|
40
|
+
"@squiz/optimization-logger": "^1.9.0",
|
|
41
|
+
"axios": "^1.13.6",
|
|
40
42
|
"cli-color": "^2.0.4",
|
|
41
|
-
"dotenv": "16.0.3",
|
|
42
43
|
"fs-extra": "11.1.0",
|
|
44
|
+
"immer": "^10.1.3",
|
|
43
45
|
"inversify": "6.0.1",
|
|
44
46
|
"inversify-binding-decorators": "4.0.0",
|
|
45
47
|
"reflect-metadata": "^0.1.13",
|
|
46
|
-
"
|
|
47
|
-
"winston": "3.8.2"
|
|
48
|
+
"zod": "^4.0.17"
|
|
48
49
|
}
|
|
49
50
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { faker } from '@faker-js/faker';
|
|
2
|
+
import { JobError } from './JobError';
|
|
3
|
+
|
|
4
|
+
describe('JobError', () => {
|
|
5
|
+
const requestId = faker.string.uuid();
|
|
6
|
+
const functionName = faker.lorem.word();
|
|
7
|
+
const jobName = faker.lorem.words();
|
|
8
|
+
const jobVersion = faker.system.semver();
|
|
9
|
+
|
|
10
|
+
it('Inherits the message and stack of the provided Error instance', () => {
|
|
11
|
+
const error = new Error('Job code failed');
|
|
12
|
+
const jobError = new JobError(requestId, error, functionName, jobName, jobVersion);
|
|
13
|
+
|
|
14
|
+
expect(jobError.message).toBe(error.message);
|
|
15
|
+
expect(jobError.stack).toBe(error.stack);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it('Inherits the message of the provided string', () => {
|
|
19
|
+
const error = 'Job code failed';
|
|
20
|
+
const jobError = new JobError(requestId, error, functionName, jobName, jobVersion);
|
|
21
|
+
|
|
22
|
+
expect(jobError.message).toBe(error);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it.each([undefined, null, '', false, true])('Uses a generic message when the error is %s', (error: unknown) => {
|
|
26
|
+
const jobError = new JobError(requestId, error, functionName, jobName, jobVersion);
|
|
27
|
+
|
|
28
|
+
expect(jobError.message).toBe('An error occurred with no additional information available');
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -13,8 +13,15 @@ export class JobError extends ErrorWithHttpStatusCode {
|
|
|
13
13
|
|
|
14
14
|
statusCode: number = 500,
|
|
15
15
|
) {
|
|
16
|
-
|
|
16
|
+
let message = 'An error occurred with no additional information available';
|
|
17
17
|
|
|
18
|
+
if (typeof error === 'string' && error) {
|
|
19
|
+
message = error;
|
|
20
|
+
} else if (error?.message) {
|
|
21
|
+
message = error.message;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
super(statusCode, message);
|
|
18
25
|
this.stack = error?.stack;
|
|
19
26
|
}
|
|
20
27
|
}
|
|
@@ -3,25 +3,34 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
+
import { faker } from '@faker-js/faker';
|
|
6
7
|
import { parsePartitionKey } from './ParsePartitionKey';
|
|
7
8
|
import { InternalServerError } from '../errors';
|
|
9
|
+
import { ENTITY_TYPES } from '../constants/Repository.constants';
|
|
8
10
|
|
|
9
11
|
describe('parsePartitionKey', () => {
|
|
10
|
-
|
|
11
|
-
expect(parsePartitionKey('job')).toBe('job');
|
|
12
|
-
});
|
|
12
|
+
const tenant = faker.string.uuid();
|
|
13
13
|
|
|
14
|
-
it(
|
|
15
|
-
|
|
14
|
+
it.each([
|
|
15
|
+
[ENTITY_TYPES.context, `${ENTITY_TYPES.context}~${tenant}`],
|
|
16
|
+
[ENTITY_TYPES.job, `${ENTITY_TYPES.job}~${tenant}`],
|
|
17
|
+
[ENTITY_TYPES.jobExecution, `${ENTITY_TYPES.jobExecution}~${tenant}`],
|
|
18
|
+
[ENTITY_TYPES.manifest, `${ENTITY_TYPES.manifest}~${tenant}`],
|
|
19
|
+
])('should parse %s partition key', (entityType: string, pk: string) => {
|
|
20
|
+
expect(parsePartitionKey(pk)).toEqual({ entityType, tenant });
|
|
16
21
|
});
|
|
17
22
|
|
|
18
|
-
it('should parse "context"', () => {
|
|
19
|
-
expect(parsePartitionKey('context')).toBe('context');
|
|
20
|
-
});
|
|
21
23
|
it('should throw an error for an invalid status', () => {
|
|
22
24
|
const invalidType = 'invalid';
|
|
23
25
|
expect(() => parsePartitionKey(invalidType)).toThrow(
|
|
24
26
|
new InternalServerError(`'${invalidType}' is an invalid type`),
|
|
25
27
|
);
|
|
26
28
|
});
|
|
29
|
+
|
|
30
|
+
it('should throw if an invalid multi-tenant partition key is provided', () => {
|
|
31
|
+
const invalidType = `invalid~${tenant}`;
|
|
32
|
+
expect(() => parsePartitionKey(invalidType)).toThrow(
|
|
33
|
+
new InternalServerError(`'${invalidType}' is an invalid type`),
|
|
34
|
+
);
|
|
35
|
+
});
|
|
27
36
|
});
|
|
@@ -1,19 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ENTITY_TYPES, EntityType } from '../constants/Repository.constants';
|
|
2
|
+
import { ParsePartitionKeyResult } from './ParsePartitionKey.types';
|
|
3
|
+
import { InternalServerError } from '../errors';
|
|
3
4
|
|
|
4
|
-
export function parsePartitionKey(type: string):
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
return PARTITION_KEYS.manifest;
|
|
13
|
-
case 'context':
|
|
14
|
-
return PARTITION_KEYS.context;
|
|
15
|
-
default:
|
|
16
|
-
throw new InternalServerError(`'${type}' is an invalid type`);
|
|
17
|
-
}
|
|
5
|
+
export function parsePartitionKey(type: string): ParsePartitionKeyResult {
|
|
6
|
+
const entityTypes = Object.values(ENTITY_TYPES).join('|');
|
|
7
|
+
const matches = type.match(new RegExp(`^(?<entityType>${entityTypes})~(?<tenant>.+)$`));
|
|
8
|
+
if (matches?.groups) {
|
|
9
|
+
return {
|
|
10
|
+
entityType: matches.groups.entityType as EntityType,
|
|
11
|
+
tenant: matches.groups.tenant,
|
|
12
|
+
};
|
|
18
13
|
}
|
|
14
|
+
|
|
15
|
+
throw new InternalServerError(`'${type}' is an invalid type`);
|
|
19
16
|
}
|
|
@@ -3,50 +3,66 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import
|
|
6
|
+
import 'aws-sdk-client-mock-jest';
|
|
7
|
+
import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
|
|
8
|
+
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
|
9
|
+
import { sdkStreamMixin } from '@smithy/util-stream';
|
|
10
|
+
import { Readable } from 'node:stream';
|
|
7
11
|
import { JobDetailsRepository } from './JobDetailsRepository';
|
|
8
12
|
import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
|
|
9
|
-
import {
|
|
13
|
+
import { InjectTokens } from '../constants/InjectTokens';
|
|
14
|
+
import { createMockJobContext } from '../model/JobContext';
|
|
15
|
+
import { createMockJobManifest } from '../manifest';
|
|
16
|
+
import { createMockJobExecution } from '../model/JobExecution';
|
|
10
17
|
|
|
11
|
-
|
|
12
|
-
initIocContainer(testLogger);
|
|
18
|
+
initIocContainer();
|
|
13
19
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
20
|
+
describe('JobDetailsRepositoryV2', (): void => {
|
|
21
|
+
let s3Client: AwsClientStub<S3Client>;
|
|
22
|
+
let jobDetailsRepository: JobDetailsRepository;
|
|
23
|
+
let bucketName: string;
|
|
24
|
+
let tenant: string;
|
|
17
25
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
$metadata: {},
|
|
21
|
-
});
|
|
22
|
-
mockClient.send = mockSend;
|
|
23
|
-
|
|
24
|
-
// Inject the S3Repository with a mocked client
|
|
25
|
-
iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
|
|
26
|
-
|
|
27
|
-
let jobDetailsRepository: JobDetailsRepository;
|
|
28
|
-
|
|
29
|
-
describe('JobDetailsRepository', (): void => {
|
|
30
|
-
beforeAll((): void => {
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
s3Client = mockClient(iocContainer.get(S3Client));
|
|
31
28
|
jobDetailsRepository = iocContainer.get(JobDetailsRepository);
|
|
29
|
+
bucketName = iocContainer.get(InjectTokens.JobDetailsBucket);
|
|
30
|
+
tenant = iocContainer.get(InjectTokens.Tenant);
|
|
32
31
|
});
|
|
33
32
|
|
|
34
33
|
describe(`get`, (): void => {
|
|
35
|
-
beforeEach(() => {
|
|
36
|
-
jest.resetAllMocks();
|
|
37
|
-
});
|
|
38
|
-
|
|
39
34
|
it(`should return a response for get`, async (): Promise<void> => {
|
|
40
35
|
const mockKey = `mock`;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
);
|
|
36
|
+
|
|
37
|
+
s3Client.on(GetObjectCommand).resolves({
|
|
38
|
+
Body: sdkStreamMixin(Readable.from(['file'])),
|
|
39
|
+
});
|
|
46
40
|
|
|
47
41
|
const response = await jobDetailsRepository.get(mockKey);
|
|
48
|
-
expect(
|
|
49
|
-
expect(
|
|
42
|
+
expect(response).toBe('file');
|
|
43
|
+
expect(s3Client).toHaveReceivedCommandWith(GetObjectCommand, {
|
|
44
|
+
Bucket: bucketName,
|
|
45
|
+
Key: mockKey,
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe('createJobDetailsFile', (): void => {
|
|
51
|
+
it('should create the job details file', async () => {
|
|
52
|
+
s3Client.on(PutObjectCommand).resolves({});
|
|
53
|
+
|
|
54
|
+
const jobContext = createMockJobContext();
|
|
55
|
+
const jobManifest = createMockJobManifest();
|
|
56
|
+
const jobExecution = createMockJobExecution();
|
|
57
|
+
const expectedKey = `${tenant}/${jobExecution.id}`;
|
|
58
|
+
const response = await jobDetailsRepository.createJobDetailsFile(tenant, jobContext, jobManifest, jobExecution);
|
|
59
|
+
|
|
60
|
+
expect(response).toBe(expectedKey);
|
|
61
|
+
expect(s3Client).toHaveReceivedCommandWith(PutObjectCommand, {
|
|
62
|
+
Bucket: bucketName,
|
|
63
|
+
Key: expectedKey,
|
|
64
|
+
Body: JSON.stringify({ jobContext, jobManifest, jobExecution }),
|
|
65
|
+
});
|
|
50
66
|
});
|
|
51
67
|
});
|
|
52
68
|
});
|
|
@@ -5,12 +5,13 @@ import { provideTransient } from '../transientProvider';
|
|
|
5
5
|
import { JobContext } from '../model/JobContext';
|
|
6
6
|
import { JobManifest } from '../manifest';
|
|
7
7
|
import { JobExecution } from '../model/JobExecution';
|
|
8
|
+
import { InjectTokens } from '../constants/InjectTokens';
|
|
8
9
|
|
|
9
10
|
@provideTransient(JobDetailsRepository)
|
|
10
11
|
export class JobDetailsRepository extends S3RepositoryBase {
|
|
11
12
|
constructor(
|
|
12
13
|
@inject(S3Client) protected s3Client: S3Client,
|
|
13
|
-
@inject(
|
|
14
|
+
@inject(InjectTokens.JobDetailsBucket) bucketName: string,
|
|
14
15
|
) {
|
|
15
16
|
super(s3Client, bucketName);
|
|
16
17
|
}
|
|
@@ -22,15 +23,13 @@ export class JobDetailsRepository extends S3RepositoryBase {
|
|
|
22
23
|
* an error.
|
|
23
24
|
*/
|
|
24
25
|
public async createJobDetailsFile(
|
|
25
|
-
|
|
26
|
+
tenant: string,
|
|
26
27
|
jobContext: JobContext | null,
|
|
27
28
|
jobManifest: JobManifest | null,
|
|
28
29
|
jobExecution: JobExecution,
|
|
29
|
-
) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
public static createJobDetailsFileName(tenantId: string, jobExecutionId: string): string {
|
|
34
|
-
return `${tenantId}/${jobExecutionId}`;
|
|
30
|
+
): Promise<string> {
|
|
31
|
+
const key = `${tenant}/${jobExecution.id}`;
|
|
32
|
+
await this.put(key, JSON.stringify({ jobContext, jobManifest, jobExecution }));
|
|
33
|
+
return key;
|
|
35
34
|
}
|
|
36
35
|
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import 'aws-sdk-client-mock-jest';
|
|
7
|
+
import { faker } from '@faker-js/faker';
|
|
8
|
+
import { GetObjectCommand, ListObjectsV2Command, ListObjectsV2CommandOutput, S3Client } from '@aws-sdk/client-s3';
|
|
9
|
+
import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
|
|
10
|
+
import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
|
|
11
|
+
import { InjectTokens } from '../constants/InjectTokens';
|
|
12
|
+
import { JobUploadsRepository } from './JobUploadsRepository';
|
|
13
|
+
import { sdkStreamMixin } from '@smithy/util-stream';
|
|
14
|
+
import { Readable } from 'node:stream';
|
|
15
|
+
|
|
16
|
+
initIocContainer();
|
|
17
|
+
|
|
18
|
+
describe('JobUploadsRepositoryV2', (): void => {
|
|
19
|
+
let s3Client: AwsClientStub<S3Client>;
|
|
20
|
+
let jobUploadsRepository: JobUploadsRepository;
|
|
21
|
+
let bucketName: string;
|
|
22
|
+
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
s3Client = mockClient(iocContainer.get(S3Client));
|
|
25
|
+
jobUploadsRepository = iocContainer.get(JobUploadsRepository);
|
|
26
|
+
bucketName = iocContainer.get(InjectTokens.JobUploadsBucket);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe(`get`, (): void => {
|
|
30
|
+
it(`should return a response for get`, async (): Promise<void> => {
|
|
31
|
+
const mockKey = `mock`;
|
|
32
|
+
|
|
33
|
+
s3Client.on(GetObjectCommand).resolves({
|
|
34
|
+
Body: sdkStreamMixin(Readable.from(['file'])),
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
const response = await jobUploadsRepository.get(mockKey);
|
|
38
|
+
expect(response).toBe('file');
|
|
39
|
+
expect(s3Client).toHaveReceivedCommandWith(GetObjectCommand, {
|
|
40
|
+
Bucket: bucketName,
|
|
41
|
+
Key: mockKey,
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe('listByTenant', (): void => {
|
|
47
|
+
it('should list objects by prefix with the provided continuation token', async (): Promise<void> => {
|
|
48
|
+
const tenant = faker.string.uuid();
|
|
49
|
+
const continuationToken = faker.string.uuid();
|
|
50
|
+
const expectedResult: ListObjectsV2CommandOutput = {
|
|
51
|
+
Contents: [{ Key: faker.string.uuid() }, { Key: faker.string.uuid() }],
|
|
52
|
+
$metadata: {},
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
s3Client.on(ListObjectsV2Command).resolves(expectedResult);
|
|
56
|
+
|
|
57
|
+
const response = await jobUploadsRepository.listByTenant(tenant, continuationToken);
|
|
58
|
+
expect(response).toBe(expectedResult);
|
|
59
|
+
expect(s3Client).toHaveReceivedCommandWith(ListObjectsV2Command, {
|
|
60
|
+
Bucket: bucketName,
|
|
61
|
+
Prefix: `${tenant}/`,
|
|
62
|
+
ContinuationToken: continuationToken,
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { ListObjectsV2Command, ListObjectsV2CommandOutput, S3Client } from '@aws-sdk/client-s3';
|
|
7
|
+
import { inject } from 'inversify';
|
|
8
|
+
import { S3RepositoryBase } from './S3RepositoryBase';
|
|
9
|
+
import { provideTransient } from '../transientProvider';
|
|
10
|
+
import { InjectTokens } from '../constants/InjectTokens';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Repository class for interacting with the job uploads S3 bucket.
|
|
14
|
+
* @export
|
|
15
|
+
* @class JobUploadsRepository
|
|
16
|
+
*/
|
|
17
|
+
@provideTransient(JobUploadsRepository)
|
|
18
|
+
export class JobUploadsRepository extends S3RepositoryBase {
|
|
19
|
+
public constructor(
|
|
20
|
+
@inject(S3Client) protected s3Client: S3Client,
|
|
21
|
+
@inject(InjectTokens.JobUploadsBucket) bucketName: string,
|
|
22
|
+
) {
|
|
23
|
+
super(s3Client, bucketName);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
public async listByTenant(tenant: string, continuationToken?: string): Promise<ListObjectsV2CommandOutput> {
|
|
27
|
+
return await this.s3Client.send(
|
|
28
|
+
new ListObjectsV2Command({
|
|
29
|
+
Bucket: this.bucketName,
|
|
30
|
+
Prefix: `${tenant}/`,
|
|
31
|
+
ContinuationToken: continuationToken,
|
|
32
|
+
}),
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -4,12 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
|
|
7
|
-
import { S3Repository } from './S3Repository';
|
|
8
7
|
import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
|
|
9
|
-
import {
|
|
8
|
+
import { S3RepositoryBase } from './S3RepositoryBase';
|
|
10
9
|
|
|
11
|
-
|
|
12
|
-
initIocContainer(testLogger);
|
|
10
|
+
initIocContainer();
|
|
13
11
|
|
|
14
12
|
// Mock S3 Client
|
|
15
13
|
const mockOptions = {} as S3ClientConfig;
|
|
@@ -21,21 +19,16 @@ const mockSend = jest.fn().mockResolvedValue({
|
|
|
21
19
|
});
|
|
22
20
|
mockClient.send = mockSend;
|
|
23
21
|
|
|
24
|
-
// Inject the S3Repository with a mocked client
|
|
25
22
|
iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
|
|
26
23
|
|
|
27
|
-
let s3Repository:
|
|
24
|
+
let s3Repository: S3RepositoryBase;
|
|
28
25
|
|
|
29
26
|
describe('S3RepositoryBase', (): void => {
|
|
30
27
|
beforeAll((): void => {
|
|
31
|
-
s3Repository = new
|
|
28
|
+
s3Repository = new S3RepositoryBase(mockClient, 'mock-bucket');
|
|
32
29
|
});
|
|
33
30
|
|
|
34
31
|
describe(`get`, (): void => {
|
|
35
|
-
beforeEach(() => {
|
|
36
|
-
jest.resetAllMocks();
|
|
37
|
-
});
|
|
38
|
-
|
|
39
32
|
it(`should return a response for get`, async (): Promise<void> => {
|
|
40
33
|
const mockKey = `mock`;
|
|
41
34
|
mockSend.mockResolvedValueOnce({
|
|
@@ -61,10 +54,6 @@ describe('S3RepositoryBase', (): void => {
|
|
|
61
54
|
});
|
|
62
55
|
|
|
63
56
|
describe(`put`, (): void => {
|
|
64
|
-
beforeEach(() => {
|
|
65
|
-
jest.resetAllMocks();
|
|
66
|
-
});
|
|
67
|
-
|
|
68
57
|
it(`should return a response for get`, async (): Promise<void> => {
|
|
69
58
|
const mockKey = `mock`;
|
|
70
59
|
const mockObject = ``;
|
|
@@ -79,10 +68,6 @@ describe('S3RepositoryBase', (): void => {
|
|
|
79
68
|
});
|
|
80
69
|
|
|
81
70
|
describe(`check`, (): void => {
|
|
82
|
-
beforeEach(() => {
|
|
83
|
-
jest.resetAllMocks();
|
|
84
|
-
});
|
|
85
|
-
|
|
86
71
|
it(`should return a response if S3 object key exists`, async (): Promise<void> => {
|
|
87
72
|
const mockKey = `mock`;
|
|
88
73
|
mockSend.mockResolvedValueOnce({
|
|
@@ -94,4 +79,13 @@ describe('S3RepositoryBase', (): void => {
|
|
|
94
79
|
expect(response).toBeTruthy();
|
|
95
80
|
});
|
|
96
81
|
});
|
|
82
|
+
|
|
83
|
+
describe(`delete`, (): void => {
|
|
84
|
+
it(`should delete the requested object`, async (): Promise<void> => {
|
|
85
|
+
const mockKey = `mock`;
|
|
86
|
+
const response = await s3Repository.delete(mockKey);
|
|
87
|
+
expect(mockSend).toBeCalledTimes(1);
|
|
88
|
+
expect(response).toBeFalsy();
|
|
89
|
+
});
|
|
90
|
+
});
|
|
97
91
|
});
|
|
@@ -3,10 +3,8 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import * as dotenv from 'dotenv';
|
|
7
|
-
dotenv.config();
|
|
8
|
-
|
|
9
6
|
import {
|
|
7
|
+
DeleteObjectCommand,
|
|
10
8
|
GetObjectCommand,
|
|
11
9
|
HeadObjectCommand,
|
|
12
10
|
HeadObjectCommandOutput,
|
|
@@ -21,10 +19,10 @@ import 'reflect-metadata';
|
|
|
21
19
|
/**
|
|
22
20
|
* Repository class for interacting with S3.
|
|
23
21
|
* @export
|
|
24
|
-
* @class
|
|
22
|
+
* @class S3RepositoryBase
|
|
25
23
|
*/
|
|
26
24
|
@injectable()
|
|
27
|
-
export
|
|
25
|
+
export class S3RepositoryBase {
|
|
28
26
|
protected bucketName: string;
|
|
29
27
|
public constructor(
|
|
30
28
|
@inject(S3Client) protected s3Client: S3Client,
|
|
@@ -38,7 +36,7 @@ export abstract class S3RepositoryBase {
|
|
|
38
36
|
* @param {string} key The S3 object key.
|
|
39
37
|
* @param {PutObjectCommandInputType['Body']} body The body of the S3 object.
|
|
40
38
|
* @return {Promise<void>}
|
|
41
|
-
* @memberof
|
|
39
|
+
* @memberof S3RepositoryBase
|
|
42
40
|
*/
|
|
43
41
|
async put(key: string, body: PutObjectCommandInput['Body']): Promise<void> {
|
|
44
42
|
await this.s3Client.send(
|
|
@@ -50,11 +48,26 @@ export abstract class S3RepositoryBase {
|
|
|
50
48
|
);
|
|
51
49
|
}
|
|
52
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Delete object from S3.
|
|
53
|
+
*
|
|
54
|
+
* @param {string} key The S3 object key.
|
|
55
|
+
* @memberof S3RepositoryBase
|
|
56
|
+
*/
|
|
57
|
+
public async delete(key: string): Promise<void> {
|
|
58
|
+
await this.s3Client.send(
|
|
59
|
+
new DeleteObjectCommand({
|
|
60
|
+
Bucket: this.bucketName,
|
|
61
|
+
Key: key,
|
|
62
|
+
}),
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
|
|
53
66
|
/**
|
|
54
67
|
* Retrieve object from S3.
|
|
55
68
|
* @param {string} key The S3 object key.
|
|
56
69
|
* @return {Promise<string | null>} The S3 object.
|
|
57
|
-
* @memberof
|
|
70
|
+
* @memberof S3RepositoryBase
|
|
58
71
|
*/
|
|
59
72
|
async get(key: string): Promise<string | null> {
|
|
60
73
|
const s3Object = await this.s3Client.send(
|
|
@@ -71,7 +84,7 @@ export abstract class S3RepositoryBase {
|
|
|
71
84
|
* Check if the S3 object key exists.
|
|
72
85
|
* @param {string} key The S3 object key.
|
|
73
86
|
* @return {Promise<HeadObjectCommandOutput>}
|
|
74
|
-
* @memberof
|
|
87
|
+
* @memberof S3RepositoryBase
|
|
75
88
|
*/
|
|
76
89
|
async check(key: string): Promise<HeadObjectCommandOutput> {
|
|
77
90
|
return this.s3Client.send(
|
|
@@ -89,7 +102,9 @@ export abstract class S3RepositoryBase {
|
|
|
89
102
|
});
|
|
90
103
|
|
|
91
104
|
const res = await getSignedUrl(this.s3Client, command, {
|
|
92
|
-
|
|
105
|
+
// In some instances ECS tasks can take 30 minutes from PROVISIONING -> RUNNING.
|
|
106
|
+
// https://squizgroup.atlassian.net/browse/PLATFORM-2402
|
|
107
|
+
expiresIn: 3600, // 1 hour
|
|
93
108
|
});
|
|
94
109
|
return res;
|
|
95
110
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const InjectTokens = {
|
|
2
|
+
Tenant: Symbol('tenant'),
|
|
3
|
+
DynamoTableName: Symbol('dynamoTableName'),
|
|
4
|
+
JobUploadsBucket: Symbol('jobUploadsBucket'),
|
|
5
|
+
JobDetailsBucket: Symbol('jobDetailsBucket'),
|
|
6
|
+
JobStartQueueUrl: Symbol('jobStartQueueUrl'),
|
|
7
|
+
Logger: Symbol('logger'),
|
|
8
|
+
};
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
export type
|
|
6
|
+
export type EntityType = ENTITY_TYPES | EntityTypeRaw;
|
|
7
7
|
|
|
8
8
|
// this needs to be dynamic as it changes based on the jobName,
|
|
9
9
|
// and tsoa can't parse a template literal type
|
|
10
10
|
// so we need to use a string type here
|
|
11
|
-
export type
|
|
11
|
+
export type EntityTypeRaw = string;
|
|
12
12
|
|
|
13
|
-
export enum
|
|
13
|
+
export enum ENTITY_TYPES {
|
|
14
14
|
job = 'job',
|
|
15
15
|
manifest = 'manifest',
|
|
16
16
|
jobExecution = 'jobExecution',
|