@squiz/job-runner-lib 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/lib/domains/component/errors/JobError.spec.d.ts +1 -0
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.d.ts +2 -2
- package/lib/domains/job/ParsePartitionKey/ParsePartitionKey.types.d.ts +5 -0
- package/lib/domains/job/S3Repository/JobDetailsRepository.d.ts +1 -2
- package/lib/domains/job/S3Repository/JobDetailsRepository.spec.d.ts +1 -1
- package/lib/domains/job/S3Repository/JobUploadsRepository.d.ts +16 -0
- package/lib/domains/job/{services/JobQueueService/JobQueueService.spec.d.ts → S3Repository/JobUploadsRepository.spec.d.ts} +1 -1
- package/lib/domains/job/S3Repository/S3RepositoryBase.d.ts +12 -5
- package/lib/domains/job/S3Repository/index.d.ts +2 -1
- package/lib/domains/job/constants/InjectTokens.d.ts +8 -0
- package/lib/domains/job/constants/JobExecutionStatus.constants.d.ts +2 -1
- package/lib/domains/job/constants/Repository.constants.d.ts +3 -3
- package/lib/domains/job/constants/RetryConfig.constants.d.ts +21 -0
- package/lib/domains/job/index.d.ts +6 -4
- package/lib/domains/job/manifest/JobManifestTypes.d.ts +7 -6
- package/lib/domains/job/model/Base/Base.d.ts +10 -5
- package/lib/domains/job/model/Job/Job.d.ts +9 -6
- package/lib/domains/job/model/JobContext/JobContext.d.ts +6 -5
- package/lib/domains/job/model/JobExecution/JobExecution.d.ts +64 -9
- package/lib/domains/job/model/LogPayload/LogPayload.d.ts +21 -12
- package/lib/domains/job/model/LogPayload/index.d.ts +1 -0
- package/lib/domains/job/model/LogPayload/mocks/LogPayload.mock.d.ts +6 -0
- package/lib/domains/job/repository/AbstractRepository.d.ts +20 -11
- package/lib/domains/job/repository/JobContextRepository/JobContextRepository.d.ts +6 -7
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.d.ts +7 -17
- package/lib/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.d.ts +1 -1
- package/lib/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.d.ts +2 -0
- package/lib/domains/job/repository/JobExecutionRepository/utils/index.d.ts +1 -0
- package/lib/domains/job/repository/JobManifestRepository/JobManifestRepository.d.ts +5 -6
- package/lib/domains/job/repository/JobRepository/JobRepository.d.ts +6 -7
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.d.ts +9 -0
- package/lib/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.d.ts +1 -0
- package/lib/domains/job/repository/index.d.ts +1 -0
- package/lib/domains/job/services/JobContextService/JobContextService.d.ts +6 -3
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.d.ts +10 -6
- package/lib/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.d.ts +1 -3
- package/lib/domains/job/services/JobManifestService/JobManifestService.d.ts +0 -1
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.d.ts +17 -0
- package/lib/domains/job/services/JobStartQueueService/JobStartQueueService.defs.d.ts +25 -0
- package/lib/domains/job/services/JobUploadService/utils/manifestValidator.d.ts +1 -2
- package/lib/domains/job/services/index.d.ts +2 -1
- package/lib/domains/manifest/services/ManifestServiceForDev.d.ts +0 -3
- package/lib/index.d.ts +1 -0
- package/lib/index.js +13 -13
- package/lib/index.js.map +4 -4
- package/lib/jest-global-setup.d.ts +6 -1
- package/lib/mocks/mockIoc.d.ts +3 -3
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.d.ts +2 -0
- package/lib/utils/buildJobRunnerModule/buildJobRunnerModule.types.d.ts +14 -0
- package/lib/utils/index.d.ts +2 -0
- package/package.json +12 -11
- package/src/domains/component/errors/JobError.spec.ts +30 -0
- package/src/domains/component/errors/JobError.ts +8 -1
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.spec.ts +17 -8
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.ts +13 -16
- package/src/domains/job/ParsePartitionKey/ParsePartitionKey.types.ts +6 -0
- package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts +47 -31
- package/src/domains/job/S3Repository/JobDetailsRepository.ts +7 -8
- package/src/domains/job/S3Repository/JobUploadsRepository.spec.ts +66 -0
- package/src/domains/job/S3Repository/JobUploadsRepository.ts +35 -0
- package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts +13 -19
- package/src/domains/job/S3Repository/S3RepositoryBase.ts +24 -9
- package/src/domains/job/S3Repository/index.ts +2 -1
- package/src/domains/job/constants/InjectTokens.ts +8 -0
- package/src/domains/job/constants/JobExecutionStatus.constants.ts +1 -0
- package/src/domains/job/constants/Repository.constants.ts +3 -3
- package/src/domains/job/constants/RetryConfig.constants.ts +29 -0
- package/src/domains/job/core/pagination/services/PaginationService.spec.ts +3 -5
- package/src/domains/job/index.ts +6 -4
- package/src/domains/job/manifest/JobManifestTypes.ts +8 -6
- package/src/domains/job/manifest/mocks/JobManifest.mock.ts +2 -2
- package/src/domains/job/manifest/mocks/JobManifestDto.mock.ts +3 -2
- package/src/domains/job/manifest/mocks/JobManifestRecord.mock.ts +4 -3
- package/src/domains/job/model/Base/Base.ts +11 -5
- package/src/domains/job/model/Job/Job.ts +11 -6
- package/src/domains/job/model/Job/mocks/Job.mock.ts +2 -2
- package/src/domains/job/model/Job/mocks/JobDto.mock.ts +3 -2
- package/src/domains/job/model/Job/mocks/JobRecord.mock.ts +3 -2
- package/src/domains/job/model/JobContext/JobContext.ts +7 -5
- package/src/domains/job/model/JobContext/mocks/JobContext.mock.ts +2 -2
- package/src/domains/job/model/JobContext/mocks/JobContextDto.mock.ts +3 -2
- package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts +3 -2
- package/src/domains/job/model/JobExecution/JobExecution.ts +74 -9
- package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts +7 -4
- package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts +32 -16
- package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts +10 -4
- package/src/domains/job/model/LogPayload/LogPayload.ts +32 -11
- package/src/domains/job/model/LogPayload/index.ts +1 -0
- package/src/domains/job/model/LogPayload/mocks/LogPayload.mock.ts +21 -0
- package/src/domains/job/repository/AbstractRepository.ts +91 -59
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.spec.ts +39 -49
- package/src/domains/job/repository/JobContextRepository/JobContextRepository.ts +33 -21
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts +343 -169
- package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts +112 -141
- package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts +22 -0
- package/src/domains/job/repository/JobExecutionRepository/utils/index.ts +1 -0
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts +23 -38
- package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts +29 -36
- package/src/domains/job/repository/JobRepository/JobRepository.spec.ts +72 -80
- package/src/domains/job/repository/JobRepository/JobRepository.ts +66 -70
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.spec.ts +60 -0
- package/src/domains/job/repository/MultiTenantJobExecutionRepository/MultiTenantJobExecutionRepository.ts +42 -0
- package/src/domains/job/repository/index.ts +1 -0
- package/src/domains/job/services/JobContextService/JobContextService.spec.ts +73 -39
- package/src/domains/job/services/JobContextService/JobContextService.ts +37 -6
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts +1 -3
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts +113 -31
- package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts +101 -49
- package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts +4 -41
- package/src/domains/job/services/JobManifestService/JobManifestService.ts +2 -8
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.defs.ts +27 -0
- package/src/domains/job/services/JobStartQueueService/JobStartQueueService.ts +42 -0
- package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts +1 -1
- package/src/domains/job/services/JobUploadService/JobUploadService.ts +1 -1
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.spec.ts +12 -25
- package/src/domains/job/services/JobUploadService/utils/manifestValidator.ts +2 -7
- package/src/domains/job/services/index.ts +2 -1
- package/src/domains/job/utils/SlackClient/SlackClient.spec.ts +0 -2
- package/src/domains/manifest/services/ManifestServiceForDev.spec.ts +1 -2
- package/src/domains/manifest/services/ManifestServiceForDev.ts +0 -4
- package/src/index.ts +1 -0
- package/src/jest-global-setup.ts +8 -4
- package/src/mocks/mockIoc.ts +18 -43
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts +81 -0
- package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts +13 -0
- package/src/utils/index.ts +2 -0
- package/lib/domains/job/S3Repository/S3Repository.d.ts +0 -15
- package/lib/domains/job/S3Repository/S3Repository.spec.d.ts +0 -5
- package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts +0 -9
- package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts +0 -9
- package/src/custom-types.d.ts +0 -7
- package/src/domains/job/S3Repository/S3Repository.spec.ts +0 -50
- package/src/domains/job/S3Repository/S3Repository.ts +0 -27
- package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts +0 -10
- package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts +0 -190
- package/src/domains/job/services/JobQueueService/JobQueueService.ts +0 -60
package/CHANGELOG.md
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function parsePartitionKey(type: string):
|
|
1
|
+
import { ParsePartitionKeyResult } from './ParsePartitionKey.types';
|
|
2
|
+
export declare function parsePartitionKey(type: string): ParsePartitionKeyResult;
|
|
@@ -12,6 +12,5 @@ export declare class JobDetailsRepository extends S3RepositoryBase {
|
|
|
12
12
|
* We want the file to be created so the job starts/executes rather than remaining in a queued state/causing
|
|
13
13
|
* an error.
|
|
14
14
|
*/
|
|
15
|
-
createJobDetailsFile(
|
|
16
|
-
static createJobDetailsFileName(tenantId: string, jobExecutionId: string): string;
|
|
15
|
+
createJobDetailsFile(tenant: string, jobContext: JobContext | null, jobManifest: JobManifest | null, jobExecution: JobExecution): Promise<string>;
|
|
17
16
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
|
+
*/
|
|
5
|
+
import { ListObjectsV2CommandOutput, S3Client } from '@aws-sdk/client-s3';
|
|
6
|
+
import { S3RepositoryBase } from './S3RepositoryBase';
|
|
7
|
+
/**
|
|
8
|
+
* Repository class for interacting with the job uploads S3 bucket.
|
|
9
|
+
* @export
|
|
10
|
+
* @class JobUploadsRepository
|
|
11
|
+
*/
|
|
12
|
+
export declare class JobUploadsRepository extends S3RepositoryBase {
|
|
13
|
+
protected s3Client: S3Client;
|
|
14
|
+
constructor(s3Client: S3Client, bucketName: string);
|
|
15
|
+
listByTenant(tenant: string, continuationToken?: string): Promise<ListObjectsV2CommandOutput>;
|
|
16
|
+
}
|
|
@@ -7,9 +7,9 @@ import 'reflect-metadata';
|
|
|
7
7
|
/**
|
|
8
8
|
* Repository class for interacting with S3.
|
|
9
9
|
* @export
|
|
10
|
-
* @class
|
|
10
|
+
* @class S3RepositoryBase
|
|
11
11
|
*/
|
|
12
|
-
export declare
|
|
12
|
+
export declare class S3RepositoryBase {
|
|
13
13
|
protected s3Client: S3Client;
|
|
14
14
|
protected bucketName: string;
|
|
15
15
|
constructor(s3Client: S3Client, bucketName: string);
|
|
@@ -18,21 +18,28 @@ export declare abstract class S3RepositoryBase {
|
|
|
18
18
|
* @param {string} key The S3 object key.
|
|
19
19
|
* @param {PutObjectCommandInputType['Body']} body The body of the S3 object.
|
|
20
20
|
* @return {Promise<void>}
|
|
21
|
-
* @memberof
|
|
21
|
+
* @memberof S3RepositoryBase
|
|
22
22
|
*/
|
|
23
23
|
put(key: string, body: PutObjectCommandInput['Body']): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Delete object from S3.
|
|
26
|
+
*
|
|
27
|
+
* @param {string} key The S3 object key.
|
|
28
|
+
* @memberof S3RepositoryBase
|
|
29
|
+
*/
|
|
30
|
+
delete(key: string): Promise<void>;
|
|
24
31
|
/**
|
|
25
32
|
* Retrieve object from S3.
|
|
26
33
|
* @param {string} key The S3 object key.
|
|
27
34
|
* @return {Promise<string | null>} The S3 object.
|
|
28
|
-
* @memberof
|
|
35
|
+
* @memberof S3RepositoryBase
|
|
29
36
|
*/
|
|
30
37
|
get(key: string): Promise<string | null>;
|
|
31
38
|
/**
|
|
32
39
|
* Check if the S3 object key exists.
|
|
33
40
|
* @param {string} key The S3 object key.
|
|
34
41
|
* @return {Promise<HeadObjectCommandOutput>}
|
|
35
|
-
* @memberof
|
|
42
|
+
* @memberof S3RepositoryBase
|
|
36
43
|
*/
|
|
37
44
|
check(key: string): Promise<HeadObjectCommandOutput>;
|
|
38
45
|
getS3SignedUrl(key: string): Promise<string>;
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
* @license
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
|
-
export type
|
|
6
|
-
export type
|
|
7
|
-
export declare enum
|
|
5
|
+
export type EntityType = ENTITY_TYPES | EntityTypeRaw;
|
|
6
|
+
export type EntityTypeRaw = string;
|
|
7
|
+
export declare enum ENTITY_TYPES {
|
|
8
8
|
job = "job",
|
|
9
9
|
manifest = "manifest",
|
|
10
10
|
jobExecution = "jobExecution",
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Retry configuration with exponential backoff delays in seconds.
|
|
7
|
+
* Using SQS message timer limits (max 15 minutes = 900 seconds).
|
|
8
|
+
*
|
|
9
|
+
* Delays: 30s, 1min, 2min, 5min, 10min, 15min
|
|
10
|
+
*/
|
|
11
|
+
export declare const RETRY_DELAYS_SECONDS: ReadonlyArray<number>;
|
|
12
|
+
/**
|
|
13
|
+
* Maximum number of retry attempts for container start failures.
|
|
14
|
+
*/
|
|
15
|
+
export declare const MAX_RETRY_ATTEMPTS: number;
|
|
16
|
+
/**
|
|
17
|
+
* Get the delay in seconds for a given retry attempt.
|
|
18
|
+
* @param retryAttempt - The current retry attempt (0-indexed)
|
|
19
|
+
* @returns The delay in seconds for this retry attempt
|
|
20
|
+
*/
|
|
21
|
+
export declare const getRetryDelaySeconds: (retryAttempt: number) => number;
|
|
@@ -1,18 +1,20 @@
|
|
|
1
|
-
export * from './core/pagination';
|
|
2
1
|
export * from './core/dynamoClient.mock';
|
|
2
|
+
export * from './core/pagination';
|
|
3
|
+
export * from './constants/InjectTokens';
|
|
3
4
|
export * from './constants/JobExecutionStatus.constants';
|
|
4
5
|
export * from './constants/Repository.constants';
|
|
6
|
+
export * from './constants/RetryConfig.constants';
|
|
5
7
|
export * from './services';
|
|
6
8
|
export * from './utils';
|
|
9
|
+
export * from './model/Base';
|
|
7
10
|
export * from './model/Job';
|
|
8
11
|
export * from './model/JobContext';
|
|
9
12
|
export * from './model/JobExecution';
|
|
10
|
-
export * from './model/Base';
|
|
11
13
|
export * from './model/LogPayload';
|
|
12
14
|
export * from './errors';
|
|
13
15
|
export * from './manifest';
|
|
14
16
|
export * from './ParsePartitionKey';
|
|
15
|
-
export * from './transientProvider';
|
|
16
|
-
export * from './singletonProvider';
|
|
17
17
|
export * from './repository';
|
|
18
18
|
export * from './S3Repository';
|
|
19
|
+
export * from './singletonProvider';
|
|
20
|
+
export * from './transientProvider';
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { AttributeValue } from '@aws-sdk/client-dynamodb';
|
|
6
6
|
import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
|
|
7
|
-
import {
|
|
7
|
+
import { EntityType } from '../constants/Repository.constants';
|
|
8
8
|
/**
|
|
9
9
|
* Job manifest interface.
|
|
10
10
|
* @export
|
|
@@ -51,10 +51,10 @@ export interface JobManifest {
|
|
|
51
51
|
concurrency: number;
|
|
52
52
|
/**
|
|
53
53
|
* The DynamoDB record type ('job').
|
|
54
|
-
* @type {
|
|
54
|
+
* @type {EntityType}
|
|
55
55
|
* @memberof JobManifest
|
|
56
56
|
*/
|
|
57
|
-
type:
|
|
57
|
+
type: EntityType;
|
|
58
58
|
}
|
|
59
59
|
/**
|
|
60
60
|
* The Job Manifest record attributes stored in DynamoDB.
|
|
@@ -64,10 +64,10 @@ export interface JobManifest {
|
|
|
64
64
|
export interface JobManifestRecord {
|
|
65
65
|
/**
|
|
66
66
|
* The partition key ('manifest').
|
|
67
|
-
* @type {
|
|
67
|
+
* @type {EntityType}
|
|
68
68
|
* @memberof JobManifestRecord
|
|
69
69
|
*/
|
|
70
|
-
pk:
|
|
70
|
+
pk: EntityType;
|
|
71
71
|
/**
|
|
72
72
|
* The sort key ('name-version').
|
|
73
73
|
* @type {string}
|
|
@@ -123,4 +123,5 @@ export type CreateJobManifestRequest = Omit<JobManifest, 'type'>;
|
|
|
123
123
|
* The request body for updating a job (PUT /job-manifest).
|
|
124
124
|
* @export
|
|
125
125
|
*/
|
|
126
|
-
export type UpdateJobManifestRequest = Partial<Omit<JobManifest, 'name' | 'type'>>;
|
|
126
|
+
export type UpdateJobManifestRequest = Partial<Omit<JobManifest, 'name' | 'version' | 'type'>>;
|
|
127
|
+
export type JobManifestSk = Pick<JobManifest, 'name' | 'version'>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AttributeValue } from '@aws-sdk/client-dynamodb';
|
|
2
|
-
import {
|
|
2
|
+
import { EntityType } from '../../constants/Repository.constants';
|
|
3
3
|
/**
|
|
4
4
|
* The base interface for job runner entities.
|
|
5
5
|
* @export
|
|
@@ -8,10 +8,10 @@ import { PartitionKey } from '../../constants/Repository.constants';
|
|
|
8
8
|
export interface BaseItem {
|
|
9
9
|
/**
|
|
10
10
|
* The property representing the corresponding record's partition key.
|
|
11
|
-
* @type {
|
|
11
|
+
* @type {EntityType}
|
|
12
12
|
* @memberof BaseItem
|
|
13
13
|
*/
|
|
14
|
-
type:
|
|
14
|
+
type: EntityType;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
17
|
* The base interface representing the attributes of a DynamoDB record.
|
|
@@ -21,10 +21,10 @@ export interface BaseItem {
|
|
|
21
21
|
export interface BaseRecord {
|
|
22
22
|
/**
|
|
23
23
|
* The partition key.
|
|
24
|
-
* @type {
|
|
24
|
+
* @type {EntityType}
|
|
25
25
|
* @memberof BaseRecord
|
|
26
26
|
*/
|
|
27
|
-
pk:
|
|
27
|
+
pk: EntityType;
|
|
28
28
|
/**
|
|
29
29
|
* The sort key.
|
|
30
30
|
* @type {string}
|
|
@@ -38,3 +38,8 @@ export interface BaseRecord {
|
|
|
38
38
|
* @interface BaseDto
|
|
39
39
|
*/
|
|
40
40
|
export type BaseDto = Record<keyof BaseRecord, AttributeValue>;
|
|
41
|
+
export type ListParameters<TFilters> = {
|
|
42
|
+
filters?: TFilters;
|
|
43
|
+
consistentRead?: boolean;
|
|
44
|
+
limit?: number;
|
|
45
|
+
};
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
import { AttributeValue } from '@aws-sdk/client-dynamodb';
|
|
6
|
-
import {
|
|
6
|
+
import { EntityType } from '../../constants/Repository.constants';
|
|
7
|
+
import { ListParameters } from '../Base';
|
|
7
8
|
/**
|
|
8
9
|
* Job interface.
|
|
9
10
|
* @export
|
|
@@ -30,10 +31,10 @@ export interface Job {
|
|
|
30
31
|
version: string;
|
|
31
32
|
/**
|
|
32
33
|
* The DynamoDB record type ('job').
|
|
33
|
-
* @type {
|
|
34
|
+
* @type {EntityType}
|
|
34
35
|
* @memberof Job
|
|
35
36
|
*/
|
|
36
|
-
type:
|
|
37
|
+
type: EntityType;
|
|
37
38
|
}
|
|
38
39
|
/**
|
|
39
40
|
* The Job record attributes stored in DynamoDB.
|
|
@@ -43,10 +44,10 @@ export interface Job {
|
|
|
43
44
|
export interface JobRecord {
|
|
44
45
|
/**
|
|
45
46
|
* The partition key ('job').
|
|
46
|
-
* @type {
|
|
47
|
+
* @type {EntityType}
|
|
47
48
|
* @memberof JobRecord
|
|
48
49
|
*/
|
|
49
|
-
pk:
|
|
50
|
+
pk: EntityType;
|
|
50
51
|
/**
|
|
51
52
|
* The sort key ('name-version').
|
|
52
53
|
* @type {string}
|
|
@@ -77,9 +78,11 @@ export type JobDto = Record<keyof JobRecord, AttributeValue>;
|
|
|
77
78
|
* The parameters for listing jobs.
|
|
78
79
|
* @export
|
|
79
80
|
*/
|
|
80
|
-
export type ListJobParameters = Partial<Pick<Job, 'name' | 'version'
|
|
81
|
+
export type ListJobParameters = ListParameters<Partial<Pick<Job, 'name' | 'version'>>>;
|
|
81
82
|
/**
|
|
82
83
|
* The request body for creating a job (POST /job).
|
|
83
84
|
* @export
|
|
84
85
|
*/
|
|
85
86
|
export type CreateJobRequest = Omit<Job, 'type'>;
|
|
87
|
+
export type UpdateJobRequest = Partial<Omit<Job, 'name' | 'version' | 'type'>>;
|
|
88
|
+
export type JobSk = Pick<Job, 'name' | 'version'>;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
5
|
import { AttributeValue } from '@aws-sdk/client-dynamodb';
|
|
6
|
-
import {
|
|
6
|
+
import { EntityType } from '../../constants/Repository.constants';
|
|
7
7
|
/**
|
|
8
8
|
* Environment interface.
|
|
9
9
|
* @interface Environment
|
|
@@ -38,10 +38,10 @@ export interface JobContext {
|
|
|
38
38
|
environment: Environment;
|
|
39
39
|
/**
|
|
40
40
|
* The DynamoDB record type ('context').
|
|
41
|
-
* @type {
|
|
41
|
+
* @type {EntityType}
|
|
42
42
|
* @memberof JobContext
|
|
43
43
|
*/
|
|
44
|
-
type:
|
|
44
|
+
type: EntityType;
|
|
45
45
|
}
|
|
46
46
|
/**
|
|
47
47
|
* The Job Context record attributes stored in DynamoDB.
|
|
@@ -51,10 +51,10 @@ export interface JobContext {
|
|
|
51
51
|
export interface JobContextRecord {
|
|
52
52
|
/**
|
|
53
53
|
* The partition key ('context').
|
|
54
|
-
* @type {
|
|
54
|
+
* @type {EntityType}
|
|
55
55
|
* @memberof JobContextRecord
|
|
56
56
|
*/
|
|
57
|
-
pk:
|
|
57
|
+
pk: EntityType;
|
|
58
58
|
/**
|
|
59
59
|
* The description of the job context.
|
|
60
60
|
* @type {string}
|
|
@@ -85,3 +85,4 @@ export type CreateJobContextRequest = Omit<JobContext, 'type'>;
|
|
|
85
85
|
* @export
|
|
86
86
|
*/
|
|
87
87
|
export type UpdateJobContextRequest = Partial<Omit<JobContext, 'contextName' | 'type'>>;
|
|
88
|
+
export type JobContextSk = Pick<JobContext, 'contextName'>;
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
* @license
|
|
3
3
|
* Copyright Squiz Australia Pty Ltd. All Rights Reserved.
|
|
4
4
|
*/
|
|
5
|
-
import { JobExecutionStatus } from '../../constants/JobExecutionStatus.constants';
|
|
6
|
-
import { JobExecutionPk } from '../../constants/Repository.constants';
|
|
7
5
|
import { AttributeValue } from '@aws-sdk/client-dynamodb';
|
|
6
|
+
import { JobExecutionStatus } from '../../constants/JobExecutionStatus.constants';
|
|
7
|
+
import { ListParameters } from '../Base';
|
|
8
8
|
/**
|
|
9
9
|
* JobExecution interface.
|
|
10
10
|
* @export
|
|
@@ -73,10 +73,29 @@ export interface JobExecution {
|
|
|
73
73
|
input: any;
|
|
74
74
|
/**
|
|
75
75
|
* The DynamoDB record type (dynamic, 'jobExecution-<jobName>').
|
|
76
|
-
* @type {
|
|
76
|
+
* @type {string}
|
|
77
|
+
* @memberof JobExecution
|
|
78
|
+
*/
|
|
79
|
+
type: string;
|
|
80
|
+
/**
|
|
81
|
+
* The number of times the container start has been retried.
|
|
82
|
+
* @type {number}
|
|
83
|
+
* @memberof JobExecution
|
|
84
|
+
*/
|
|
85
|
+
retryCount: number;
|
|
86
|
+
/**
|
|
87
|
+
* The reason for the last container start failure (if any).
|
|
88
|
+
* @type {(string | null)}
|
|
89
|
+
* @memberof JobExecution
|
|
90
|
+
*/
|
|
91
|
+
lastFailureReason: string | null;
|
|
92
|
+
/**
|
|
93
|
+
* ISO timestamp indicating when the execution can be retried.
|
|
94
|
+
* Used to implement exponential backoff - executions with retryAfter > now should be skipped.
|
|
95
|
+
* @type {(string | null)}
|
|
77
96
|
* @memberof JobExecution
|
|
78
97
|
*/
|
|
79
|
-
|
|
98
|
+
retryAfter: string | null;
|
|
80
99
|
}
|
|
81
100
|
/**
|
|
82
101
|
* The JobExecution record attributes stored in DynamoDB.
|
|
@@ -85,17 +104,29 @@ export interface JobExecution {
|
|
|
85
104
|
*/
|
|
86
105
|
export interface JobExecutionRecord {
|
|
87
106
|
/**
|
|
88
|
-
* The partition key (
|
|
89
|
-
* @type {
|
|
107
|
+
* The partition key ('jobExecution~<tenant>').
|
|
108
|
+
* @type {string}
|
|
90
109
|
* @memberof JobExecutionRecord
|
|
91
110
|
*/
|
|
92
|
-
pk:
|
|
111
|
+
pk: string;
|
|
93
112
|
/**
|
|
94
|
-
* The sort key ('
|
|
113
|
+
* The sort key ('<jobName>~<id>').
|
|
95
114
|
* @type {string}
|
|
96
115
|
* @memberof JobExecutionRecord
|
|
97
116
|
*/
|
|
98
117
|
sk: string;
|
|
118
|
+
/**
|
|
119
|
+
* Secondary sort key ('<status>~<jobName>').
|
|
120
|
+
* @type {string}
|
|
121
|
+
* @memberof JobExecutionRecord
|
|
122
|
+
*/
|
|
123
|
+
lsi1sk: string;
|
|
124
|
+
/**
|
|
125
|
+
* Global secondary partition key ('jobExecution~<status>').
|
|
126
|
+
* @type {string}
|
|
127
|
+
* @memberof JobExecutionRecord
|
|
128
|
+
*/
|
|
129
|
+
gsi1pk: string;
|
|
99
130
|
/**
|
|
100
131
|
* The name of the job to execute.
|
|
101
132
|
* @type {string}
|
|
@@ -156,6 +187,25 @@ export interface JobExecutionRecord {
|
|
|
156
187
|
* @memberof JobExecutionRecord
|
|
157
188
|
*/
|
|
158
189
|
input: any;
|
|
190
|
+
/**
|
|
191
|
+
* The number of times the container start has been retried.
|
|
192
|
+
* @type {number}
|
|
193
|
+
* @memberof JobExecutionRecord
|
|
194
|
+
*/
|
|
195
|
+
retryCount: number;
|
|
196
|
+
/**
|
|
197
|
+
* The reason for the last container start failure (if any).
|
|
198
|
+
* @type {(string | null)}
|
|
199
|
+
* @memberof JobExecutionRecord
|
|
200
|
+
*/
|
|
201
|
+
lastFailureReason: string | null;
|
|
202
|
+
/**
|
|
203
|
+
* ISO timestamp indicating when the execution can be retried.
|
|
204
|
+
* Used to implement exponential backoff - executions with retryAfter > now should be skipped.
|
|
205
|
+
* @type {(string | null)}
|
|
206
|
+
* @memberof JobExecutionRecord
|
|
207
|
+
*/
|
|
208
|
+
retryAfter: string | null;
|
|
159
209
|
}
|
|
160
210
|
export type JobExecutionDto = Record<keyof JobExecutionRecord, AttributeValue>;
|
|
161
211
|
/**
|
|
@@ -181,9 +231,14 @@ export interface TerminateJobExecutionRequest {
|
|
|
181
231
|
* The parameters for listing job executions.
|
|
182
232
|
* @export
|
|
183
233
|
*/
|
|
184
|
-
export type ListJobExecutionParameters = Partial<Pick<JobExecution, 'jobName' | 'status'
|
|
234
|
+
export type ListJobExecutionParameters = ListParameters<Partial<Pick<JobExecution, 'jobName' | 'status'>>>;
|
|
185
235
|
/**
|
|
186
236
|
* The parameters for updating job executions.
|
|
187
237
|
* @export
|
|
188
238
|
*/
|
|
189
239
|
export type UpdateJobExecutionParameters = Partial<Omit<JobExecution, 'id' | 'jobName' | 'timeQueued' | 'type'>>;
|
|
240
|
+
export type JobExecutionSk = Pick<JobExecution, 'jobName' | 'id'>;
|
|
241
|
+
/**
|
|
242
|
+
* Array of job executions keys by tenant.
|
|
243
|
+
*/
|
|
244
|
+
export type JobExecutionsByTenant = Record<string, Array<JobExecution>>;
|
|
@@ -1,12 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare enum LogLevel {
|
|
3
|
+
AUDIT = "AUDIT",
|
|
4
|
+
INFO = "INFO",
|
|
5
|
+
WARNING = "WARNING",
|
|
6
|
+
ERROR = "ERROR"
|
|
7
|
+
}
|
|
8
|
+
export declare const LogPayloadSchema: z.ZodObject<{
|
|
9
|
+
message: z.ZodString;
|
|
10
|
+
meta: z.ZodObject<{
|
|
11
|
+
tenant: z.ZodString;
|
|
12
|
+
jobName: z.ZodString;
|
|
13
|
+
level: z.ZodEnum<typeof LogLevel>;
|
|
14
|
+
timestamp: z.ZodISODateTime;
|
|
15
|
+
traceid: z.ZodString;
|
|
16
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
17
|
+
details: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
18
|
+
}, z.core.$strip>;
|
|
19
|
+
}, z.core.$strip>;
|
|
20
|
+
export type LogPayload = z.infer<typeof LogPayloadSchema>;
|
|
21
|
+
export declare const CONSOLE_LOG_METHOD_LEVELS: Partial<Record<keyof typeof console, LogLevel>>;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import 'reflect-metadata';
|
|
2
2
|
import { AttributeValue, DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
|
3
|
-
import {
|
|
4
|
-
import { BaseDto, BaseItem } from '../../job/model/Base/Base';
|
|
3
|
+
import { BaseDto, BaseItem, ListParameters } from '../../job/model/Base/Base';
|
|
5
4
|
/**
|
|
6
5
|
* Generic abstract class for representing repositories interacting with DynamoDB.
|
|
7
6
|
* @export
|
|
@@ -10,11 +9,11 @@ import { BaseDto, BaseItem } from '../../job/model/Base/Base';
|
|
|
10
9
|
* @template T The entity definition.
|
|
11
10
|
* @template U The corresponding DB record attributes for the entity.
|
|
12
11
|
*/
|
|
13
|
-
export declare abstract class Repository<T extends BaseItem, U extends BaseDto
|
|
12
|
+
export declare abstract class Repository<T extends BaseItem, U extends BaseDto, SK extends Partial<T>> {
|
|
14
13
|
protected dynamoDbClient: DynamoDBClient;
|
|
15
|
-
protected pk: PartitionKey;
|
|
16
14
|
protected tableName: string;
|
|
17
|
-
|
|
15
|
+
protected pk: string;
|
|
16
|
+
constructor(dynamoDbClient: DynamoDBClient, tableName: string, pk: string);
|
|
18
17
|
/**
|
|
19
18
|
* Save item into database.
|
|
20
19
|
* @abstract
|
|
@@ -30,22 +29,22 @@ export declare abstract class Repository<T extends BaseItem, U extends BaseDto>
|
|
|
30
29
|
* @returns {Promise<T | null>} The retrieved record.
|
|
31
30
|
* @memberof Repository
|
|
32
31
|
*/
|
|
33
|
-
findOne(sk:
|
|
32
|
+
findOne(sk: SK): Promise<T | null>;
|
|
34
33
|
/**
|
|
35
34
|
* Get list of records from database.
|
|
36
35
|
* @abstract
|
|
37
36
|
* @returns {Promise<Array<T>>} The retrieved records.
|
|
38
37
|
* @memberof Repository
|
|
39
38
|
*/
|
|
40
|
-
list(): Promise<Array<T>>;
|
|
39
|
+
list(parameters?: ListParameters<never>): Promise<Array<T>>;
|
|
41
40
|
/**
|
|
42
41
|
* Update record in database.
|
|
43
42
|
* @param {string} sk The sort key used to identify the record.
|
|
44
|
-
* @param {Partial<T>} input The (partial) record to update.
|
|
43
|
+
* @param {Partial<T>} input The (partial) record to update (properties used in the SK cannot be updated).
|
|
45
44
|
* @returns {Promise<T | null>} The updated record.
|
|
46
45
|
* @memberof Repository
|
|
47
46
|
*/
|
|
48
|
-
update(sk:
|
|
47
|
+
update(sk: SK, input: Partial<Omit<T, keyof SK>>): Promise<T>;
|
|
49
48
|
/**
|
|
50
49
|
* Delete record from database.
|
|
51
50
|
* @abstract
|
|
@@ -53,11 +52,14 @@ export declare abstract class Repository<T extends BaseItem, U extends BaseDto>
|
|
|
53
52
|
* @returns {void}
|
|
54
53
|
* @memberof Repository
|
|
55
54
|
*/
|
|
56
|
-
delete(sk:
|
|
55
|
+
delete(sk: T | SK): Promise<void>;
|
|
56
|
+
batchDelete<TItem extends T | SK>(sk: Array<TItem>): Promise<{
|
|
57
|
+
unprocessedItems: Array<TItem>;
|
|
58
|
+
}>;
|
|
57
59
|
/**
|
|
58
60
|
* Convert the DynanomDB attributes of the entity into a DynamoDB item.
|
|
59
61
|
* @abstract
|
|
60
|
-
* @param {
|
|
62
|
+
* @param {T} item The DynamoDB item attributes to convert.
|
|
61
63
|
* @returns {Record<keyof U, AttributeValue>} The DynamoDB record.
|
|
62
64
|
* @memberof Repository
|
|
63
65
|
*/
|
|
@@ -70,4 +72,11 @@ export declare abstract class Repository<T extends BaseItem, U extends BaseDto>
|
|
|
70
72
|
* @memberof Repository
|
|
71
73
|
*/
|
|
72
74
|
protected abstract convertToItem(record: U): T;
|
|
75
|
+
/**
|
|
76
|
+
* Converts a sort key to a string for querying/putting in Dynamo.
|
|
77
|
+
* @param {SK} sk
|
|
78
|
+
* @return {string}
|
|
79
|
+
* @protected
|
|
80
|
+
*/
|
|
81
|
+
protected abstract buildSk(sk: T | SK): string;
|
|
73
82
|
}
|
|
@@ -4,12 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
|
6
6
|
import { Repository } from '../AbstractRepository';
|
|
7
|
-
import { CreateJobContextRequest, JobContext, JobContextDto } from '../../model/JobContext
|
|
8
|
-
import { PaginationInfo } from '../../core/pagination
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
protected dynamoDbClient: DynamoDBClient;
|
|
12
|
-
constructor(dynamoDbClient: DynamoDBClient);
|
|
7
|
+
import { CreateJobContextRequest, JobContext, JobContextDto, JobContextSk, UpdateJobContextRequest } from '../../model/JobContext';
|
|
8
|
+
import { PaginationInfo, PaginationQueryResponse } from '../../core/pagination';
|
|
9
|
+
export declare class JobContextRepository extends Repository<JobContext, JobContextDto, JobContextSk> {
|
|
10
|
+
constructor(dynamoDbClient: DynamoDBClient, tableName: string, tenant: string);
|
|
13
11
|
create(input: CreateJobContextRequest): Promise<JobContext>;
|
|
14
12
|
/**
|
|
15
13
|
* Get list of records from database.
|
|
@@ -18,7 +16,8 @@ export declare class JobContextRepository extends Repository<JobContext, JobCont
|
|
|
18
16
|
* @memberof Repository
|
|
19
17
|
*/
|
|
20
18
|
getContexts(paginationInfo: PaginationInfo): Promise<PaginationQueryResponse<JobContext>>;
|
|
21
|
-
update(sk:
|
|
19
|
+
update(sk: JobContextSk, input: UpdateJobContextRequest): Promise<JobContext>;
|
|
22
20
|
convertToDynamoDbItem(item: JobContext): JobContextDto;
|
|
23
21
|
convertToItem(dbItem: JobContextDto): JobContext;
|
|
22
|
+
protected buildSk(sk: JobContextSk): string;
|
|
24
23
|
}
|