@squiz/job-runner-lib 2.10.0 → 2.10.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@squiz/job-runner-lib",
-  "version": "2.10.0",
+  "version": "2.10.2",
   "description": "",
   "main": "lib/index.js",
   "private": false,
@@ -19,7 +19,7 @@
   "devDependencies": {
     "@faker-js/faker": "8.0.2",
     "@smithy/util-stream": "4.5.17",
-    "@squiz/virus-scanner-lib": "1.68.0",
+    "@squiz/virus-scanner-lib": "1.68.1",
     "@types/cli-color": "2.0.6",
     "@types/fs-extra": "11.0.1",
     "@types/jest": "28.1.8",
@@ -34,11 +34,12 @@
     "@aws-sdk/client-s3": "3.1003.0",
     "@aws-sdk/client-sqs": "3.1003.0",
     "@aws-sdk/s3-request-presigner": "3.1003.0",
+    "@aws-sdk/util-dynamodb": "3.996.2",
     "@squiz/dx-common-lib": "1.66.1",
     "@squiz/dx-json-schema-lib": "1.86.0",
     "@squiz/dx-logger-lib": "1.64.0",
     "@squiz/optimization-logger": "1.11.2",
-    "axios": "1.16.1",
+    "axios": "1.19.0",
     "cli-color": "2.0.4",
     "fs-extra": "11.1.0",
     "immer": "10.1.3",

package/src/domains/job/errors/ConflictError.ts

@@ -0,0 +1,19 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { ERRORS } from './Error.constants';
+import { ErrorWithHttpStatusCode } from './ErrorWithHttpStatusCode';
+
+/**
+ * Conflict error (409).
+ * @export
+ * @class ConflictError
+ * @extends {ErrorWithHttpStatusCode}
+ */
+export class ConflictError extends ErrorWithHttpStatusCode {
+  constructor(details: string, message = ERRORS.conflict.name) {
+    super(ERRORS.conflict.code, message, details);
+  }
+}

package/src/domains/job/errors/Error.constants.ts

@@ -24,6 +24,10 @@ export const ERRORS = {
     code: 422,
     name: 'Unprocessable Content',
   },
+  conflict: {
+    code: 409,
+    name: 'Conflict',
+  },
 };
 
 export const ERROR_MESSAGES = {

package/src/domains/job/errors/index.ts

@@ -1,4 +1,5 @@
 export * from './BadRequestError';
+export * from './ConflictError';
 export * from './Error.constants';
 export * from './ErrorWithHttpStatusCode';
 export * from './InternalServerError';

package/src/domains/job/model/Base/Base.ts

@@ -48,3 +48,9 @@ export type ListParameters<TFilters> = {
   consistentRead?: boolean;
   limit?: number;
 };
+
+export type UpdateOptions = {
+  conditionExpression?: string;
+  expressionAttributeValues?: Record<string, unknown>;
+  expressionAttributeNames?: Record<string, string>;
+};

package/src/domains/job/model/JobExecution/JobExecution.ts

@@ -62,6 +62,13 @@ export interface JobExecution {
    */
   ecsTaskId: string | null;
 
+  /**
+   * The last version of the ECS task event which was handled.
+   * @type {(number | null)}
+   * @memberof JobExecution
+   */
+  lastEcsTaskEventVersion: number | null;
+
   /**
    * The time the job run was started.
    * @type {(string | null)}
@@ -184,21 +191,28 @@ export interface JobExecutionRecord {
 
   /**
    * The version of the job script to execute.
-   * @type @type {(string)}
+   * @type {(string)}
    * @memberof JobExecutionRecord
    */
   version: string;
 
   /**
    * The ECS task ID of the job run.
-   * @type @type {(string | null)}
+   * @type {(string | null)}
    * @memberof JobExecutionRecord
    */
   ecsTaskId: string | null;
 
+  /**
+   * The last version of the ECS task event which was handled.
+   * @type {(number | null)}
+   * @memberof JobExecution
+   */
+  lastEcsTaskEventVersion: number | null;
+
   /**
    * The time the job run was started.
-   * @type @type {(string | null)}
+   * @type {(string | null)}
    * @memberof JobExecutionRecord
    */
   timeStarted: string | null;

package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts

@@ -24,6 +24,7 @@ export const createMockJobExecution = (partialRecord: Partial<JobExecution> = {}
   return {
     context: faker.string.uuid(),
     ecsTaskId: null,
+    lastEcsTaskEventVersion: null,
     id: `${timeQueued}-${faker.string.uuid()}`,
     jobName,
     status: JOB_EXECUTION_STATUS.queued,

package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts

@@ -34,6 +34,9 @@ export const createMockJobExecutionDto = (partialRecord: Partial<JobExecutionRec
       S: partialRecord.context ?? faker.string.uuid(),
     },
     ecsTaskId: partialRecord.ecsTaskId ? { S: partialRecord.ecsTaskId } : { NULL: true },
+    lastEcsTaskEventVersion: partialRecord.lastEcsTaskEventVersion
+      ? { N: partialRecord.lastEcsTaskEventVersion.toString() }
+      : { NULL: true },
     gsi1pk: {
       S: `${ENTITY_TYPES.jobExecution}~${status}`,
     },

package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts

@@ -28,6 +28,7 @@ export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecution
   return {
     context: faker.string.uuid(),
     ecsTaskId: null,
+    lastEcsTaskEventVersion: null,
     gsi1pk: `${ENTITY_TYPES.jobExecution}~${status}`,
     id: id,
     jobName: jobName,

package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.spec.ts

@@ -207,6 +207,7 @@ describe('JobExecutionRepository', (): void => {
     it(`should return job execution by timeQueued and id and handle any null values`, async (): Promise<void> => {
       const mockJobExecutionRecord = createMockJobExecutionDto({
         ecsTaskId: 'ecs-task-id',
+        lastEcsTaskEventVersion: 1,
         timeFinished: new Date().toISOString(),
         timeStarted: new Date().toISOString(),
         version: '1.0.0',
@@ -225,6 +226,7 @@ describe('JobExecutionRepository', (): void => {
       expect(response).toStrictEqual({
         context: mockJobExecutionRecord.context.S as string,
         ecsTaskId: mockJobExecutionRecord.ecsTaskId.S as string,
+        lastEcsTaskEventVersion: parseInt(mockJobExecutionRecord.lastEcsTaskEventVersion.N as string),
         id: mockJobExecutionRecord.id.S as string,
         jobName: mockJobExecutionRecord.jobName.S as string,
         status: mockJobExecutionRecord.status.S as string,
@@ -452,6 +454,7 @@ describe('JobExecutionRepository', (): void => {
       const updatedAttributes: UpdateJobExecutionParameters = {
         input: faker.lorem.words(),
         ecsTaskId: faker.string.uuid(),
+        lastEcsTaskEventVersion: faker.number.int(),
         version: faker.system.semver(),
         timeStarted: faker.date.recent().toISOString(),
         timeFinished: null,
@@ -471,20 +474,22 @@ describe('JobExecutionRepository', (): void => {
       expect(result).toEqual(updatedJobExecution);
       expect(dynamoClient).toHaveReceivedCommandWith(UpdateItemCommand, {
         ExpressionAttributeNames: {
-          '#context': 'context',
-          '#ecsTaskId': 'ecsTaskId',
-          '#input': 'input',
-          '#timeFinished': 'timeFinished',
-          '#timeStarted': 'timeStarted',
-          '#version': 'version',
+          '#updateExpr0': 'input',
+          '#updateExpr1': 'ecsTaskId',
+          '#updateExpr2': 'lastEcsTaskEventVersion',
+          '#updateExpr3': 'version',
+          '#updateExpr4': 'timeStarted',
+          '#updateExpr5': 'timeFinished',
+          '#updateExpr6': 'context',
         },
         ExpressionAttributeValues: {
-          ':context': { S: updatedAttributes.context! },
-          ':ecsTaskId': { S: updatedAttributes.ecsTaskId! },
-          ':input': { S: updatedAttributes.input },
-          ':timeFinished': { NULL: true },
-          ':timeStarted': { S: updatedAttributes.timeStarted! },
-          ':version': { S: updatedAttributes.version! },
+          ':updateExpr0': { S: updatedAttributes.input },
+          ':updateExpr1': { S: updatedAttributes.ecsTaskId! },
+          ':updateExpr2': { N: `${updatedAttributes.lastEcsTaskEventVersion}` },
+          ':updateExpr3': { S: updatedAttributes.version! },
+          ':updateExpr4': { S: updatedAttributes.timeStarted! },
+          ':updateExpr5': { NULL: true },
+          ':updateExpr6': { S: updatedAttributes.context! },
         },
         Key: {
           pk: { S: `jobExecution~${tenant}` },
@@ -493,7 +498,7 @@ describe('JobExecutionRepository', (): void => {
         ReturnValues: 'ALL_NEW',
         TableName: tableName,
         UpdateExpression:
-          'SET #input = :input, #ecsTaskId = :ecsTaskId, #version = :version, #timeStarted = :timeStarted, #timeFinished = :timeFinished, #context = :context',
+          'SET #updateExpr0 = :updateExpr0, #updateExpr1 = :updateExpr1, #updateExpr2 = :updateExpr2, #updateExpr3 = :updateExpr3, #updateExpr4 = :updateExpr4, #updateExpr5 = :updateExpr5, #updateExpr6 = :updateExpr6',
       });
     });
 
@@ -517,12 +522,68 @@ describe('JobExecutionRepository', (): void => {
         ExpressionAttributeNames: {
           '#gsi1pk': 'gsi1pk',
           '#lsi1sk': 'lsi1sk',
-          '#status': 'status',
+          '#updateExpr0': 'status',
         },
         ExpressionAttributeValues: {
           ':gsi1pk': { S: `jobExecution~${updatedJobExecution.status}` },
           ':lsi1sk': { S: `${updatedJobExecution.status}~${updatedJobExecution.jobName}` },
-          ':status': { S: updatedJobExecution.status },
+          ':updateExpr0': { S: updatedJobExecution.status },
+        },
+        Key: {
+          pk: { S: `jobExecution~${tenant}` },
+          sk: { S: `${jobExecution.jobName}~${jobExecution.id}` },
+        },
+        ReturnValues: 'ALL_NEW',
+        TableName: tableName,
+        UpdateExpression: 'SET #updateExpr0 = :updateExpr0, #lsi1sk = :lsi1sk, #gsi1pk = :gsi1pk',
+      });
+    });
+
+    it('Marshalls expression attribute values provided for conditional updates', async () => {
+      const jobExecution = createMockJobExecution({ status: JOB_EXECUTION_STATUS.queued });
+      const updatedJobExecution = createMockJobExecution({
+        ...jobExecution,
+        status: JOB_EXECUTION_STATUS.running,
+      });
+
+      dynamoClient.on(UpdateItemCommand).resolvesOnce({
+        Attributes: createMockJobExecutionDto(updatedJobExecution),
+      });
+
+      const result = await jobExecutionRepository.update(
+        jobExecution,
+        {
+          status: updatedJobExecution.status,
+        },
+        {
+          conditionExpression: '#ecsTaskId = :ecsTaskId AND #status = :status',
+          expressionAttributeNames: {
+            '#status': 'status',
+            '#ecsTaskId': 'ecsTaskId',
+          },
+          expressionAttributeValues: {
+            ecsTaskId: null,
+            status: JOB_EXECUTION_STATUS.queued,
+          },
+        },
+      );
+
+      expect(result).toEqual(updatedJobExecution);
+      expect(dynamoClient).toHaveReceivedCommandWith(UpdateItemCommand, {
+        ConditionExpression: '#ecsTaskId = :ecsTaskId AND #status = :status',
+        ExpressionAttributeNames: {
+          '#ecsTaskId': 'ecsTaskId',
+          '#gsi1pk': 'gsi1pk',
+          '#lsi1sk': 'lsi1sk',
+          '#status': 'status',
+          '#updateExpr0': 'status',
+        },
+        ExpressionAttributeValues: {
+          ':gsi1pk': { S: 'jobExecution~running' },
+          ':lsi1sk': { S: `running~${jobExecution.jobName}` },
+          ':updateExpr0': { S: 'running' },
+          ecsTaskId: { NULL: true },
+          status: { S: 'queued' },
         },
         Key: {
           pk: { S: `jobExecution~${tenant}` },
@@ -530,7 +591,7 @@ describe('JobExecutionRepository', (): void => {
         },
         ReturnValues: 'ALL_NEW',
         TableName: tableName,
-        UpdateExpression: 'SET #status = :status, #lsi1sk = :lsi1sk, #gsi1pk = :gsi1pk',
+        UpdateExpression: 'SET #updateExpr0 = :updateExpr0, #lsi1sk = :lsi1sk, #gsi1pk = :gsi1pk',
       });
     });
   });

package/src/domains/job/repository/JobExecutionRepository/JobExecutionRepository.ts

@@ -35,6 +35,8 @@ import { provideTransient } from '../../transientProvider';
 import { Repository } from '../AbstractRepository';
 import { InjectTokens } from '../../constants/InjectTokens';
 import { convertDynamoDbItemToJobExecution } from './utils';
+import { UpdateOptions } from '../../model/Base';
+import { marshall } from '@aws-sdk/util-dynamodb';
 
 @provideTransient(JobExecutionRepository)
 export class JobExecutionRepository extends Repository<JobExecution, JobExecutionDto, JobExecutionSk> {
@@ -52,6 +54,7 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
     const newJobExecution: JobExecution = {
       type: `${ENTITY_TYPES.jobExecution}-${createParams.jobName}`,
       ecsTaskId: null,
+      lastEcsTaskEventVersion: null,
       timeStarted: null,
       timeFinished: null,
       input: null,
@@ -240,11 +243,7 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
   async update(
     sk: JobExecutionSk,
     input: UpdateJobExecutionParameters,
-    options?: {
-      ConditionExpression?: string;
-      ExpressionAttributeValues?: Record<string, AttributeValue>;
-      ExpressionAttributeNames?: Record<string, string>;
-    },
+    options?: UpdateOptions,
   ): Promise<JobExecution> {
     // Initialise update command
     const updateParams: UpdateItemCommandInput = {
@@ -259,11 +258,13 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
     const expressionAttributeNames: Record<string, string> = {};
     const expressionAttributeValues: Record<string, AttributeValue> = {};
     const updateExpressionPairs: Array<string> = [];
+    let index = 0;
 
     // To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
-    for (const key of Object.keys(input)) {
-      const val = input[key as keyof UpdateJobExecutionParameters];
-      expressionAttributeNames[`#${key}`] = key;
+    for (const property of Object.keys(input)) {
+      const key = `updateExpr${index++}`;
+      const val = input[property as keyof UpdateJobExecutionParameters];
+      expressionAttributeNames[`#${key}`] = property;
 
       if (val === null) {
         expressionAttributeValues[`:${key}`] = { NULL: true };
@@ -275,7 +276,7 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
 
       updateExpressionPairs.push(`#${key} = :${key}`);
 
-      if (key === 'status') {
+      if (property === 'status') {
         expressionAttributeNames[`#lsi1sk`] = 'lsi1sk';
         expressionAttributeValues[`:lsi1sk`] = { S: `${val || ''}~${sk.jobName}` };
         expressionAttributeNames[`#gsi1pk`] = 'gsi1pk';
@@ -290,23 +291,23 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
     updateParams.UpdateExpression = `SET ${updateExpressionPairs.join(', ')}`;
 
     // Add condition expression if provided
-    if (options?.ConditionExpression) {
-      updateParams.ConditionExpression = options.ConditionExpression;
+    if (options?.conditionExpression) {
+      updateParams.ConditionExpression = options.conditionExpression;
     }
 
-    if (options?.ExpressionAttributeValues) {
+    if (options?.expressionAttributeValues) {
       // Merge existing expression attribute values with the ones from options
       updateParams.ExpressionAttributeValues = {
         ...updateParams.ExpressionAttributeValues,
-        ...options.ExpressionAttributeValues,
+        ...marshall(options.expressionAttributeValues),
       };
     }
 
     // Add expression attribute names if provided
-    if (options?.ExpressionAttributeNames) {
+    if (options?.expressionAttributeNames) {
       updateParams.ExpressionAttributeNames = {
         ...updateParams.ExpressionAttributeNames,
-        ...options.ExpressionAttributeNames,
+        ...options.expressionAttributeNames,
       };
     }
 
@@ -325,6 +326,9 @@ export class JobExecutionRepository extends Repository<JobExecution, JobExecutio
     return {
       context: { S: item.context },
       ecsTaskId: item.ecsTaskId ? { S: item.ecsTaskId } : { NULL: true },
+      lastEcsTaskEventVersion: item.lastEcsTaskEventVersion
+        ? { N: item.lastEcsTaskEventVersion.toString() }
+        : { NULL: true },
       gsi1pk: { S: `${ENTITY_TYPES.jobExecution}~${item.status}` },
       id: { S: item.id },
       jobName: { S: item.jobName },

package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.spec.ts

@@ -0,0 +1,48 @@
+import { faker } from '@faker-js/faker';
+import { JOB_EXECUTION_STATUS } from '../../../../constants/JobExecutionStatus.constants';
+import { JobExecutionDto } from '../../../../model/JobExecution';
+import { convertDynamoDbItemToJobExecution } from './convertDynamoDbItemToJobExecution';
+import { ENTITY_TYPES } from '../../../../constants/Repository.constants';
+
+describe('convertDynamoDbItemToJobExecution', () => {
+  it('Converts minimal execution Dynamo record to model object', () => {
+    const tenant = faker.string.uuid();
+    const dto = {
+      context: { S: faker.lorem.words() },
+      ecsTaskId: { NULL: true },
+      lastEcsTaskEventVersion: undefined,
+      id: { S: faker.string.uuid() },
+      status: { S: JOB_EXECUTION_STATUS.queued },
+      timeFinished: { NULL: true },
+      timeQueued: { S: faker.date.recent().toISOString() },
+      timeStarted: { S: faker.date.recent().toISOString() },
+      pk: { S: `${ENTITY_TYPES.jobExecution}~${tenant}` },
+      jobName: { S: faker.lorem.words() },
+      version: { S: faker.system.semver() },
+      input: undefined,
+      retryCount: undefined,
+      lastFailureReason: undefined,
+      retryAfter: undefined,
+    } as Partial<JobExecutionDto> as JobExecutionDto;
+
+    const execution = convertDynamoDbItemToJobExecution(dto);
+
+    expect(execution).toEqual({
+      context: dto.context.S,
+      ecsTaskId: null,
+      id: dto.id.S,
+      input: null,
+      jobName: dto.jobName.S,
+      lastEcsTaskEventVersion: null,
+      lastFailureReason: null,
+      retryAfter: null,
+      retryCount: 0,
+      status: dto.status.S,
+      timeFinished: null,
+      timeQueued: dto.timeQueued.S,
+      timeStarted: dto.timeStarted.S,
+      type: `${ENTITY_TYPES.jobExecution}-${dto.jobName.S}`,
+      version: dto.version.S,
+    });
+  });
+});

package/src/domains/job/repository/JobExecutionRepository/utils/convertDynamoDbItemToJobExecution/convertDynamoDbItemToJobExecution.ts

@@ -6,6 +6,7 @@ export const convertDynamoDbItemToJobExecution = (dbItem: JobExecutionDto): JobE
   return {
     context: dbItem.context.S as string,
     ecsTaskId: dbItem.ecsTaskId.S ? (dbItem.ecsTaskId.S as string) : null,
+    lastEcsTaskEventVersion: dbItem.lastEcsTaskEventVersion?.N ? parseInt(dbItem.lastEcsTaskEventVersion.N, 10) : null,
     id: dbItem.id.S as string,
     jobName: dbItem.jobName.S as string,
     status: dbItem.status.S as JOB_EXECUTION_STATUS,