@squiz/job-runner-lib 2.3.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
  3. package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
  4. package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
  5. package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
  6. package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
  7. package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
  8. package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
  9. package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
  10. package/lib/domains/job/S3Repository/index.d.ts +2 -1
  11. package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
  12. package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
  13. package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
  14. package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
  15. package/lib/domains/job/index.d.ts +6 -4
  16. package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
  17. package/lib/domains/job/model/Base/Base.d.ts +10 -5
  18. package/lib/domains/job/model/Job/Job.d.ts +9 -6
  19. package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
  20. package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
  21. package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
  22. package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
  23. package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
  24. package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
  25. package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
  26. package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
  27. package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
  28. package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
  29. package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
  30. package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
  31. package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
  32. package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
  33. package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
  34. package/lib/domains/job/repository/index.d.ts +1 -0
  35. package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
  36. package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
  37. package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -4
  38. package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
  39. package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
  40. package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
  41. package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
  42. package/lib/domains/job/services/index.d.ts +2 -1
  43. package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
  44. package/lib/index.d.ts +1 -0
  45. package/lib/index.js +13 -13
  46. package/lib/index.js.map +4 -4
  47. package/lib/jest-global-setup.d.ts +6 -1
  48. package/lib/mocks/mockIoc.d.ts +3 -3
  49. package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
  50. package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
  51. package/lib/utils/index.d.ts +2 -0
  52. package/package.json +12 -11
  53. package/src/domains/component/errors/JobError.spec.ts +30 -0
  54. package/src/domains/component/errors/JobError.ts +8 -1
  55. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
  56. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
  57. package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
  58. package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
  59. package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
  60. package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
  61. package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
  62. package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
  63. package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
  64. package/src/domains/job/S3Repository/index.ts +2 -1
  65. package/src/domains/job/constants/InjectTokens.ts +8 -0
  66. package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
  67. package/src/domains/job/constants/Repository.constants.ts +3 -3
  68. package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
  69. package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
  70. package/src/domains/job/index.ts +6 -4
  71. package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
  72. package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
  73. package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
  74. package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
  75. package/src/domains/job/model/Base/Base.ts +11 -5
  76. package/src/domains/job/model/Job/Job.ts +11 -6
  77. package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
  78. package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
  79. package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
  80. package/src/domains/job/model/JobContext/JobContext.ts +7 -5
  81. package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
  82. package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
  83. package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
  84. package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
  85. package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
  86. package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
  87. package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
  88. package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
  89. package/src/domains/job/model/LogPayload/index.ts +1 -0
  90. package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
  91. package/src/domains/job/repository/AbstractRepository.ts +91 -59
  92. package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
  93. package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
  94. package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
  95. package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
  96. package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
  97. package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
  98. package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
  99. package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
  100. package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
  101. package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
  102. package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
  103. package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
  104. package/src/domains/job/repository/index.ts +1 -0
  105. package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
  106. package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
  107. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -4
  108. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +120 -41
  109. package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +95 -49
  110. package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
  111. package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
  112. package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
  113. package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
  114. package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
  115. package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
  116. package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
  117. package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
  118. package/src/domains/job/services/index.ts +2 -1
  119. package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
  120. package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
  121. package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
  122. package/src/index.ts +1 -0
  123. package/src/jest-global-setup.ts +8 -4
  124. package/src/mocks/mockIoc.ts +18 -43
  125. package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
  126. package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
  127. package/src/utils/index.ts +2 -0
  128. package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
  129. package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
  130. package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
  131. package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
  132. package/src/custom-types.d.ts +0 -7
  133. package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
  134. package/src/domains/job/S3Repository/S3Repository.ts +0 -27
  135. package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
  136. package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
  137. package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
@@ -1 +1,6 @@
1
- export declare const testLogger: import("winston").Logger;
1
+ /*!
2
+ * @license
3
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
+ */
5
+ import { Logger } from '@squiz/optimization-logger';
6
+ export declare const testLogger: Logger;
@@ -1,5 +1,5 @@
1
+ import { Logger } from '@squiz/optimization-logger';
1
2
  import { Container } from 'inversify';
2
- import { Logger } from '@squiz/dx-logger-lib';
3
3
  declare let iocContainer: Container;
4
- declare function initIocContainer(log: Logger): void;
5
- export { iocContainer, initIocContainer };
4
+ declare function initIocContainer(logger?: Logger): void;
5
+ export { initIocContainer, iocContainer };
@@ -0,0 +1,2 @@
1
+ import { CreateJobRunnerModuleProps, CreateJobRunnerModuleResult } from './buildJobRunnerModule.types';
2
+ export declare const buildJobRunnerModule: (props: CreateJobRunnerModuleProps) => CreateJobRunnerModuleResult;
@@ -0,0 +1,14 @@
1
+ import { Logger } from '@squiz/optimization-logger';
2
+ import { ContainerModule } from 'inversify';
3
+ export type CreateJobRunnerModuleProps = {
4
+ dynamoTableName: string;
5
+ jobUploadsBucket: string;
6
+ jobDetailsBucket: string;
7
+ awsRegion: string;
8
+ local: boolean;
9
+ jobStartQueueUrl?: string;
10
+ };
11
+ export type CreateJobRunnerModuleResult = (props: {
12
+ tenant?: string;
13
+ logger: Logger;
14
+ }) => ContainerModule;
@@ -0,0 +1,2 @@
1
+ export * from './buildJobRunnerModule/buildJobRunnerModule';
2
+ export * from './buildJobRunnerModule/buildJobRunnerModule.types';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@squiz/job-runner-lib",
3
- "version": "2.3.0",
3
+ "version": "2.5.0",
4
4
  "description": "",
5
5
  "main": "lib/index.js",
6
6
  "private": false,
@@ -18,32 +18,33 @@
18
18
  "license": "ISC",
19
19
  "devDependencies": {
20
20
  "@faker-js/faker": "8.0.2",
21
- "@squiz/virus-scanner-lib": "^1.67.0",
21
+ "@smithy/util-stream": "^4.2.3",
22
+ "@squiz/virus-scanner-lib": "^1.68.0",
22
23
  "@types/cli-color": "2.0.6",
23
24
  "@types/fs-extra": "11.0.1",
24
25
  "@types/jest": "28.1.8",
25
26
  "@types/node": "22.9.0",
26
27
  "aws-sdk-client-mock": "^2.2.0",
27
28
  "aws-sdk-client-mock-jest": "^2.2.0",
28
- "dotenv": "16.0.3",
29
29
  "esbuild": "^0.25.8"
30
30
  },
31
31
  "dependencies": {
32
- "@aws-sdk/client-dynamodb": "3.848.0",
33
- "@aws-sdk/client-ecs": "3.848.0",
34
- "@aws-sdk/client-s3": "3.850.0",
35
- "@aws-sdk/s3-request-presigner": "3.850.0",
32
+ "@aws-sdk/client-dynamodb": "3.1003.0",
33
+ "@aws-sdk/client-ecs": "3.1003.0",
34
+ "@aws-sdk/client-s3": "3.1003.0",
35
+ "@aws-sdk/client-sqs": "3.1003.0",
36
+ "@aws-sdk/s3-request-presigner": "3.1003.0",
36
37
  "@squiz/dx-common-lib": "^1.66.1",
37
38
  "@squiz/dx-json-schema-lib": "1.78.1",
38
39
  "@squiz/dx-logger-lib": "^1.64.0",
39
- "axios": "1.11.0",
40
+ "@squiz/optimization-logger": "^1.9.0",
41
+ "axios": "^1.13.6",
40
42
  "cli-color": "^2.0.4",
41
- "dotenv": "16.0.3",
42
43
  "fs-extra": "11.1.0",
44
+ "immer": "^10.1.3",
43
45
  "inversify": "6.0.1",
44
46
  "inversify-binding-decorators": "4.0.0",
45
47
  "reflect-metadata": "^0.1.13",
46
- "tsoa": "5.1.1",
47
- "winston": "3.8.2"
48
+ "zod": "^4.0.17"
48
49
  }
49
50
  }
@@ -0,0 +1,30 @@
1
+ import { faker } from '@faker-js/faker';
2
+ import { JobError } from './JobError';
3
+
4
+ describe('JobError', () => {
5
+ const requestId = faker.string.uuid();
6
+ const functionName = faker.lorem.word();
7
+ const jobName = faker.lorem.words();
8
+ const jobVersion = faker.system.semver();
9
+
10
+ it('Inherits the message and stack of the provided Error instance', () => {
11
+ const error = new Error('Job code failed');
12
+ const jobError = new JobError(requestId, error, functionName, jobName, jobVersion);
13
+
14
+ expect(jobError.message).toBe(error.message);
15
+ expect(jobError.stack).toBe(error.stack);
16
+ });
17
+
18
+ it('Inherits the message of the provided string', () => {
19
+ const error = 'Job code failed';
20
+ const jobError = new JobError(requestId, error, functionName, jobName, jobVersion);
21
+
22
+ expect(jobError.message).toBe(error);
23
+ });
24
+
25
+ it.each([undefined, null, '', false, true])('Uses a generic message when the error is %s', (error: unknown) => {
26
+ const jobError = new JobError(requestId, error, functionName, jobName, jobVersion);
27
+
28
+ expect(jobError.message).toBe('An error occurred with no additional information available');
29
+ });
30
+ });
@@ -13,8 +13,15 @@ export class JobError extends ErrorWithHttpStatusCode {
13
13
 
14
14
  statusCode: number = 500,
15
15
  ) {
16
- super(statusCode, error?.message || 'An error occurred with no additional information available');
16
+ let message = 'An error occurred with no additional information available';
17
17
 
18
+ if (typeof error === 'string' && error) {
19
+ message = error;
20
+ } else if (error?.message) {
21
+ message = error.message;
22
+ }
23
+
24
+ super(statusCode, message);
18
25
  this.stack = error?.stack;
19
26
  }
20
27
  }
@@ -3,25 +3,34 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
+ import { faker } from '@faker-js/faker';
6
7
  import { parsePartitionKey } from './ParsePartitionKey';
7
8
  import { InternalServerError } from '../errors';
9
+ import { ENTITY_TYPES } from '../constants/Repository.constants';
8
10
 
9
11
  describe('parsePartitionKey', () => {
10
- it('should parse "job"', () => {
11
- expect(parsePartitionKey('job')).toBe('job');
12
- });
12
+ const tenant = faker.string.uuid();
13
13
 
14
- it('should parse "manifest"', () => {
15
- expect(parsePartitionKey('manifest')).toBe('manifest');
14
+ it.each([
15
+ [ENTITY_TYPES.context, `${ENTITY_TYPES.context}~${tenant}`],
16
+ [ENTITY_TYPES.job, `${ENTITY_TYPES.job}~${tenant}`],
17
+ [ENTITY_TYPES.jobExecution, `${ENTITY_TYPES.jobExecution}~${tenant}`],
18
+ [ENTITY_TYPES.manifest, `${ENTITY_TYPES.manifest}~${tenant}`],
19
+ ])('should parse %s partition key', (entityType: string, pk: string) => {
20
+ expect(parsePartitionKey(pk)).toEqual({ entityType, tenant });
16
21
  });
17
22
 
18
- it('should parse "context"', () => {
19
- expect(parsePartitionKey('context')).toBe('context');
20
- });
21
23
  it('should throw an error for an invalid status', () => {
22
24
  const invalidType = 'invalid';
23
25
  expect(() => parsePartitionKey(invalidType)).toThrow(
24
26
  new InternalServerError(`'${invalidType}' is an invalid type`),
25
27
  );
26
28
  });
29
+
30
+ it('should throw if an invalid multi-tenant partition key is provided', () => {
31
+ const invalidType = `invalid~${tenant}`;
32
+ expect(() => parsePartitionKey(invalidType)).toThrow(
33
+ new InternalServerError(`'${invalidType}' is an invalid type`),
34
+ );
35
+ });
27
36
  });
@@ -1,19 +1,16 @@
1
- import { JobExecutionPk, PARTITION_KEYS, PartitionKey } from '../constants/Repository.constants';
2
- import { InternalServerError } from '../errors/InternalServerError';
1
+ import { ENTITY_TYPES, EntityType } from '../constants/Repository.constants';
2
+ import { ParsePartitionKeyResult } from './ParsePartitionKey.types';
3
+ import { InternalServerError } from '../errors';
3
4
 
4
- export function parsePartitionKey(type: string): PartitionKey | JobExecutionPk {
5
- if (type.startsWith('jobExecution')) {
6
- return type;
7
- } else {
8
- switch (type) {
9
- case 'job':
10
- return PARTITION_KEYS.job;
11
- case 'manifest':
12
- return PARTITION_KEYS.manifest;
13
- case 'context':
14
- return PARTITION_KEYS.context;
15
- default:
16
- throw new InternalServerError(`'${type}' is an invalid type`);
17
- }
5
+ export function parsePartitionKey(type: string): ParsePartitionKeyResult {
6
+ const entityTypes = Object.values(ENTITY_TYPES).join('|');
7
+ const matches = type.match(new RegExp(`^(?<entityType>${entityTypes})~(?<tenant>.+)$`));
8
+ if (matches?.groups) {
9
+ return {
10
+ entityType: matches.groups.entityType as EntityType,
11
+ tenant: matches.groups.tenant,
12
+ };
18
13
  }
14
+
15
+ throw new InternalServerError(`'${type}' is an invalid type`);
19
16
  }
@@ -0,0 +1,6 @@
1
+ import { EntityType } from '../constants/Repository.constants';
2
+
3
+ export type ParsePartitionKeyResult = {
4
+ entityType: EntityType;
5
+ tenant: string;
6
+ };
@@ -3,50 +3,66 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
- import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
6
+ import 'aws-sdk-client-mock-jest';
7
+ import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
8
+ import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
9
+ import { sdkStreamMixin } from '@smithy/util-stream';
10
+ import { Readable } from 'node:stream';
7
11
  import { JobDetailsRepository } from './JobDetailsRepository';
8
12
  import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
9
- import { testLogger } from '../../../jest-global-setup';
13
+ import { InjectTokens } from '../constants/InjectTokens';
14
+ import { createMockJobContext } from '../model/JobContext';
15
+ import { createMockJobManifest } from '../manifest';
16
+ import { createMockJobExecution } from '../model/JobExecution';
10
17
 
11
- // Create a new container tsoa can use
12
- initIocContainer(testLogger);
18
+ initIocContainer();
13
19
 
14
- // Mock S3 Client
15
- const mockOptions = {} as S3ClientConfig;
16
- const mockClient = new S3Client(mockOptions);
20
+ describe('JobDetailsRepositoryV2', (): void => {
21
+ let s3Client: AwsClientStub<S3Client>;
22
+ let jobDetailsRepository: JobDetailsRepository;
23
+ let bucketName: string;
24
+ let tenant: string;
17
25
 
18
- const mockSend = jest.fn().mockResolvedValue({
19
- Body: { transformToString: jest.fn().mockResolvedValue('file') },
20
- $metadata: {},
21
- });
22
- mockClient.send = mockSend;
23
-
24
- // Inject the S3Repository with a mocked client
25
- iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
26
-
27
- let jobDetailsRepository: JobDetailsRepository;
28
-
29
- describe('JobDetailsRepository', (): void => {
30
- beforeAll((): void => {
26
+ beforeEach(() => {
27
+ s3Client = mockClient(iocContainer.get(S3Client));
31
28
  jobDetailsRepository = iocContainer.get(JobDetailsRepository);
29
+ bucketName = iocContainer.get(InjectTokens.JobDetailsBucket);
30
+ tenant = iocContainer.get(InjectTokens.Tenant);
32
31
  });
33
32
 
34
33
  describe(`get`, (): void => {
35
- beforeEach(() => {
36
- jest.resetAllMocks();
37
- });
38
-
39
34
  it(`should return a response for get`, async (): Promise<void> => {
40
35
  const mockKey = `mock`;
41
- mockSend.mockImplementationOnce(() =>
42
- Promise.resolve({
43
- Body: { transformToString: jest.fn().mockResolvedValue('file') },
44
- }),
45
- );
36
+
37
+ s3Client.on(GetObjectCommand).resolves({
38
+ Body: sdkStreamMixin(Readable.from(['file'])),
39
+ });
46
40
 
47
41
  const response = await jobDetailsRepository.get(mockKey);
48
- expect(mockSend).toBeCalledTimes(1);
49
- expect(response).toStrictEqual('file');
42
+ expect(response).toBe('file');
43
+ expect(s3Client).toHaveReceivedCommandWith(GetObjectCommand, {
44
+ Bucket: bucketName,
45
+ Key: mockKey,
46
+ });
47
+ });
48
+ });
49
+
50
+ describe('createJobDetailsFile', (): void => {
51
+ it('should create the job details file', async () => {
52
+ s3Client.on(PutObjectCommand).resolves({});
53
+
54
+ const jobContext = createMockJobContext();
55
+ const jobManifest = createMockJobManifest();
56
+ const jobExecution = createMockJobExecution();
57
+ const expectedKey = `${tenant}/${jobExecution.id}`;
58
+ const response = await jobDetailsRepository.createJobDetailsFile(tenant, jobContext, jobManifest, jobExecution);
59
+
60
+ expect(response).toBe(expectedKey);
61
+ expect(s3Client).toHaveReceivedCommandWith(PutObjectCommand, {
62
+ Bucket: bucketName,
63
+ Key: expectedKey,
64
+ Body: JSON.stringify({ jobContext, jobManifest, jobExecution }),
65
+ });
50
66
  });
51
67
  });
52
68
  });
@@ -5,12 +5,13 @@ import { provideTransient } from '../transientProvider';
5
5
  import { JobContext } from '../model/JobContext';
6
6
  import { JobManifest } from '../manifest';
7
7
  import { JobExecution } from '../model/JobExecution';
8
+ import { InjectTokens } from '../constants/InjectTokens';
8
9
 
9
10
  @provideTransient(JobDetailsRepository)
10
11
  export class JobDetailsRepository extends S3RepositoryBase {
11
12
  constructor(
12
13
  @inject(S3Client) protected s3Client: S3Client,
13
- @inject('JobDetailsBucketName') bucketName: string,
14
+ @inject(InjectTokens.JobDetailsBucket) bucketName: string,
14
15
  ) {
15
16
  super(s3Client, bucketName);
16
17
  }
@@ -22,15 +23,13 @@ export class JobDetailsRepository extends S3RepositoryBase {
22
23
  * an error.
23
24
  */
24
25
  public async createJobDetailsFile(
25
- jobDetailsFileName: string,
26
+ tenant: string,
26
27
  jobContext: JobContext | null,
27
28
  jobManifest: JobManifest | null,
28
29
  jobExecution: JobExecution,
29
- ) {
30
- await this.put(jobDetailsFileName, JSON.stringify({ jobContext, jobManifest, jobExecution }));
31
- }
32
-
33
- public static createJobDetailsFileName(tenantId: string, jobExecutionId: string): string {
34
- return `${tenantId}/${jobExecutionId}`;
30
+ ): Promise<string> {
31
+ const key = `${tenant}/${jobExecution.id}`;
32
+ await this.put(key, JSON.stringify({ jobContext, jobManifest, jobExecution }));
33
+ return key;
35
34
  }
36
35
  }
@@ -0,0 +1,66 @@
1
+ /*!
2
+ * @license
3
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
+ */
5
+
6
+ import 'aws-sdk-client-mock-jest';
7
+ import { faker } from '@faker-js/faker';
8
+ import { GetObjectCommand, ListObjectsV2Command, ListObjectsV2CommandOutput, S3Client } from '@aws-sdk/client-s3';
9
+ import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
10
+ import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
11
+ import { InjectTokens } from '../constants/InjectTokens';
12
+ import { JobUploadsRepository } from './JobUploadsRepository';
13
+ import { sdkStreamMixin } from '@smithy/util-stream';
14
+ import { Readable } from 'node:stream';
15
+
16
+ initIocContainer();
17
+
18
+ describe('JobUploadsRepositoryV2', (): void => {
19
+ let s3Client: AwsClientStub<S3Client>;
20
+ let jobUploadsRepository: JobUploadsRepository;
21
+ let bucketName: string;
22
+
23
+ beforeEach(() => {
24
+ s3Client = mockClient(iocContainer.get(S3Client));
25
+ jobUploadsRepository = iocContainer.get(JobUploadsRepository);
26
+ bucketName = iocContainer.get(InjectTokens.JobUploadsBucket);
27
+ });
28
+
29
+ describe(`get`, (): void => {
30
+ it(`should return a response for get`, async (): Promise<void> => {
31
+ const mockKey = `mock`;
32
+
33
+ s3Client.on(GetObjectCommand).resolves({
34
+ Body: sdkStreamMixin(Readable.from(['file'])),
35
+ });
36
+
37
+ const response = await jobUploadsRepository.get(mockKey);
38
+ expect(response).toBe('file');
39
+ expect(s3Client).toHaveReceivedCommandWith(GetObjectCommand, {
40
+ Bucket: bucketName,
41
+ Key: mockKey,
42
+ });
43
+ });
44
+ });
45
+
46
+ describe('listByTenant', (): void => {
47
+ it('should list objects by prefix with the provided continuation token', async (): Promise<void> => {
48
+ const tenant = faker.string.uuid();
49
+ const continuationToken = faker.string.uuid();
50
+ const expectedResult: ListObjectsV2CommandOutput = {
51
+ Contents: [{ Key: faker.string.uuid() }, { Key: faker.string.uuid() }],
52
+ $metadata: {},
53
+ };
54
+
55
+ s3Client.on(ListObjectsV2Command).resolves(expectedResult);
56
+
57
+ const response = await jobUploadsRepository.listByTenant(tenant, continuationToken);
58
+ expect(response).toBe(expectedResult);
59
+ expect(s3Client).toHaveReceivedCommandWith(ListObjectsV2Command, {
60
+ Bucket: bucketName,
61
+ Prefix: `${tenant}/`,
62
+ ContinuationToken: continuationToken,
63
+ });
64
+ });
65
+ });
66
+ });
@@ -0,0 +1,35 @@
1
+ /*!
2
+ * @license
3
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
+ */
5
+
6
+ import { ListObjectsV2Command, ListObjectsV2CommandOutput, S3Client } from '@aws-sdk/client-s3';
7
+ import { inject } from 'inversify';
8
+ import { S3RepositoryBase } from './S3RepositoryBase';
9
+ import { provideTransient } from '../transientProvider';
10
+ import { InjectTokens } from '../constants/InjectTokens';
11
+
12
+ /**
13
+ * Repository class for interacting with the job uploads S3 bucket.
14
+ * @export
15
+ * @class JobUploadsRepository
16
+ */
17
+ @provideTransient(JobUploadsRepository)
18
+ export class JobUploadsRepository extends S3RepositoryBase {
19
+ public constructor(
20
+ @inject(S3Client) protected s3Client: S3Client,
21
+ @inject(InjectTokens.JobUploadsBucket) bucketName: string,
22
+ ) {
23
+ super(s3Client, bucketName);
24
+ }
25
+
26
+ public async listByTenant(tenant: string, continuationToken?: string): Promise<ListObjectsV2CommandOutput> {
27
+ return await this.s3Client.send(
28
+ new ListObjectsV2Command({
29
+ Bucket: this.bucketName,
30
+ Prefix: `${tenant}/`,
31
+ ContinuationToken: continuationToken,
32
+ }),
33
+ );
34
+ }
35
+ }
@@ -4,12 +4,10 @@
4
4
  */
5
5
 
6
6
  import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
7
- import { S3Repository } from './S3Repository';
8
7
  import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
9
- import { testLogger } from '../../../jest-global-setup';
8
+ import { S3RepositoryBase } from './S3RepositoryBase';
10
9
 
11
- // Create a new container tsoa can use
12
- initIocContainer(testLogger);
10
+ initIocContainer();
13
11
 
14
12
  // Mock S3 Client
15
13
  const mockOptions = {} as S3ClientConfig;
@@ -21,21 +19,16 @@ const mockSend = jest.fn().mockResolvedValue({
21
19
  });
22
20
  mockClient.send = mockSend;
23
21
 
24
- // Inject the S3Repository with a mocked client
25
22
  iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
26
23
 
27
- let s3Repository: S3Repository;
24
+ let s3Repository: S3RepositoryBase;
28
25
 
29
26
  describe('S3RepositoryBase', (): void => {
30
27
  beforeAll((): void => {
31
- s3Repository = new S3Repository(mockClient, 'mock-bucket');
28
+ s3Repository = new S3RepositoryBase(mockClient, 'mock-bucket');
32
29
  });
33
30
 
34
31
  describe(`get`, (): void => {
35
- beforeEach(() => {
36
- jest.resetAllMocks();
37
- });
38
-
39
32
  it(`should return a response for get`, async (): Promise<void> => {
40
33
  const mockKey = `mock`;
41
34
  mockSend.mockResolvedValueOnce({
@@ -61,10 +54,6 @@ describe('S3RepositoryBase', (): void => {
61
54
  });
62
55
 
63
56
  describe(`put`, (): void => {
64
- beforeEach(() => {
65
- jest.resetAllMocks();
66
- });
67
-
68
57
  it(`should return a response for get`, async (): Promise<void> => {
69
58
  const mockKey = `mock`;
70
59
  const mockObject = ``;
@@ -79,10 +68,6 @@ describe('S3RepositoryBase', (): void => {
79
68
  });
80
69
 
81
70
  describe(`check`, (): void => {
82
- beforeEach(() => {
83
- jest.resetAllMocks();
84
- });
85
-
86
71
  it(`should return a response if S3 object key exists`, async (): Promise<void> => {
87
72
  const mockKey = `mock`;
88
73
  mockSend.mockResolvedValueOnce({
@@ -94,4 +79,13 @@ describe('S3RepositoryBase', (): void => {
94
79
  expect(response).toBeTruthy();
95
80
  });
96
81
  });
82
+
83
+ describe(`delete`, (): void => {
84
+ it(`should delete the requested object`, async (): Promise<void> => {
85
+ const mockKey = `mock`;
86
+ const response = await s3Repository.delete(mockKey);
87
+ expect(mockSend).toBeCalledTimes(1);
88
+ expect(response).toBeFalsy();
89
+ });
90
+ });
97
91
  });
@@ -3,10 +3,8 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
- import * as dotenv from 'dotenv';
7
- dotenv.config();
8
-
9
6
  import {
7
+ DeleteObjectCommand,
10
8
  GetObjectCommand,
11
9
  HeadObjectCommand,
12
10
  HeadObjectCommandOutput,
@@ -21,10 +19,10 @@ import 'reflect-metadata';
21
19
  /**
22
20
  * Repository class for interacting with S3.
23
21
  * @export
24
- * @class S3Repository
22
+ * @class S3RepositoryBase
25
23
  */
26
24
  @injectable()
27
- export abstract class S3RepositoryBase {
25
+ export class S3RepositoryBase {
28
26
  protected bucketName: string;
29
27
  public constructor(
30
28
  @inject(S3Client) protected s3Client: S3Client,
@@ -38,7 +36,7 @@ export abstract class S3RepositoryBase {
38
36
  * @param {string} key The S3 object key.
39
37
  * @param {PutObjectCommandInputType['Body']} body The body of the S3 object.
40
38
  * @return {Promise<void>}
41
- * @memberof S3Repository
39
+ * @memberof S3RepositoryBase
42
40
  */
43
41
  async put(key: string, body: PutObjectCommandInput['Body']): Promise<void> {
44
42
  await this.s3Client.send(
@@ -50,11 +48,26 @@ export abstract class S3RepositoryBase {
50
48
  );
51
49
  }
52
50
 
51
+ /**
52
+ * Delete object from S3.
53
+ *
54
+ * @param {string} key The S3 object key.
55
+ * @memberof S3RepositoryBase
56
+ */
57
+ public async delete(key: string): Promise<void> {
58
+ await this.s3Client.send(
59
+ new DeleteObjectCommand({
60
+ Bucket: this.bucketName,
61
+ Key: key,
62
+ }),
63
+ );
64
+ }
65
+
53
66
  /**
54
67
  * Retrieve object from S3.
55
68
  * @param {string} key The S3 object key.
56
69
  * @return {Promise<string | null>} The S3 object.
57
- * @memberof S3Repository
70
+ * @memberof S3RepositoryBase
58
71
  */
59
72
  async get(key: string): Promise<string | null> {
60
73
  const s3Object = await this.s3Client.send(
@@ -71,7 +84,7 @@ export abstract class S3RepositoryBase {
71
84
  * Check if the S3 object key exists.
72
85
  * @param {string} key The S3 object key.
73
86
  * @return {Promise<HeadObjectCommandOutput>}
74
- * @memberof S3Repository
87
+ * @memberof S3RepositoryBase
75
88
  */
76
89
  async check(key: string): Promise<HeadObjectCommandOutput> {
77
90
  return this.s3Client.send(
@@ -89,7 +102,9 @@ export abstract class S3RepositoryBase {
89
102
  });
90
103
 
91
104
  const res = await getSignedUrl(this.s3Client, command, {
92
- expiresIn: 600, // 10 minutes
105
+ // In some instances ECS tasks can take 30 minutes from PROVISIONING -> RUNNING.
106
+ // https://squizgroup.atlassian.net/browse/PLATFORM-2402
107
+ expiresIn: 3600, // 1 hour
93
108
  });
94
109
  return res;
95
110
  }
@@ -1,2 +1,3 @@
1
1
  export * from './JobDetailsRepository';
2
- export * from './S3Repository';
2
+ export * from './JobUploadsRepository';
3
+ export * from './JobUploadsRepository';
@@ -0,0 +1,8 @@
1
+ export const InjectTokens = {
2
+ Tenant: Symbol('tenant'),
3
+ DynamoTableName: Symbol('dynamoTableName'),
4
+ JobUploadsBucket: Symbol('jobUploadsBucket'),
5
+ JobDetailsBucket: Symbol('jobDetailsBucket'),
6
+ JobStartQueueUrl: Symbol('jobStartQueueUrl'),
7
+ Logger: Symbol('logger'),
8
+ };
@@ -6,4 +6,5 @@ export enum JOB_EXECUTION_STATUS {
6
6
  successfullyCompleted = 'successfully-completed',
7
7
  running = 'running',
8
8
  errored = 'errored',
9
+ taskFailedToStart = 'task-failed-to-start',
9
10
  }
@@ -3,14 +3,14 @@
3
3
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
4
4
  */
5
5
 
6
- export type PartitionKey = PARTITION_KEYS | JobExecutionPk;
6
+ export type EntityType = ENTITY_TYPES | EntityTypeRaw;
7
7
 
8
8
  // this needs to be dynamic as it changes based on the jobName,
9
9
  // and tsoa can't parse a template literal type
10
10
  // so we need to use a string type here
11
- export type JobExecutionPk = string;
11
+ export type EntityTypeRaw = string;
12
12
 
13
- export enum PARTITION_KEYS {
13
+ export enum ENTITY_TYPES {
14
14
  job = 'job',
15
15
  manifest = 'manifest',
16
16
  jobExecution = 'jobExecution',