@squiz/job-runner-lib 2.7.0 → 2.9.0

package/src/domains/job/repository/JobRepository/JobRepository.ts

@@ -11,18 +11,19 @@ import {
   QueryCommandOutput,
 } from '@aws-sdk/client-dynamodb';
 import { inject } from 'inversify';
-import { provideTransient } from '../../transientProvider';
-import { Repository } from '../AbstractRepository';
+import { InjectTokens } from '../../constants/InjectTokens';
 import { ENTITY_TYPES } from '../../constants/Repository.constants';
-import { parsePartitionKey } from '../../ParsePartitionKey';
-import { CreateJobRequest, Job, JobDto, JobSk, ListJobParameters } from '../../model/Job';
+import { getRuntimeImageOrDefault } from '../../constants/RuntimeImage.constants';
 import {
   decodeTokenString,
   PAGINATION_DIRECTION_TYPE_BEFORE,
   PaginationInfo,
   PaginationQueryResponse,
 } from '../../core/pagination';
-import { InjectTokens } from '../../constants/InjectTokens';
+import { CreateJobRequest, Job, JobDto, JobSk, ListJobParameters } from '../../model/Job';
+import { parsePartitionKey } from '../../ParsePartitionKey';
+import { provideTransient } from '../../transientProvider';
+import { Repository } from '../AbstractRepository';
 
 @provideTransient(JobRepository)
 export class JobRepository extends Repository<Job, JobDto, JobSk> {
@@ -37,6 +38,7 @@ export class JobRepository extends Repository<Job, JobDto, JobSk> {
   public async create(input: CreateJobRequest): Promise<Job> {
     const newJob: Job = {
       ...input,
+      image: getRuntimeImageOrDefault(input.image),
       type: ENTITY_TYPES.job,
     };
 
@@ -157,11 +159,13 @@ export class JobRepository extends Repository<Job, JobDto, JobSk> {
       name: { S: item.name },
       description: item.description ? { S: item.description } : { NULL: true },
       version: { S: item.version },
+      image: { S: item.image },
     };
   }
 
   convertToItem(dbItem: JobDto): Job {
     const { pk, name, version } = dbItem;
+    const image = (dbItem as Partial<JobDto>).image?.S;
     let description;
     // TODO https://jira.squiz.net/browse/CONN-609 remove this check if we tidy up
     // jobs added before this field was implemented
@@ -175,6 +179,7 @@ export class JobRepository extends Repository<Job, JobDto, JobSk> {
       description: description.S ? (description.S as string) : '',
       type: parsePartitionKey(pk.S as string).entityType,
       version: version.S as string,
+      image: getRuntimeImageOrDefault(image),
     };
   }
 

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts

@@ -17,5 +17,6 @@ export interface RunJobTaskParams {
   shortEnvironment: string;
   clusterArn: string;
   taskDefinitionArn: string;
+  taskDefinitionArnsByRuntimeImage: Record<string, string>;
   tenantId: string;
 }

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts

@@ -4,6 +4,7 @@ import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
 import 'aws-sdk-client-mock-jest';
 import { JobDetailsRepository, JobUploadsRepository } from '../../S3Repository';
 import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
+import { RuntimeImageTaskDefinitionError } from '../../errors';
 import { createMockJobManifest } from '../../manifest';
 import { createMockJobContext } from '../../model/JobContext';
 import { createMockJobExecution } from '../../model/JobExecution';
@@ -30,7 +31,11 @@ describe('JobExecutionTaskService', () => {
     shortRegion: 'au',
     shortEnvironment: 'dev',
     clusterArn: `clusterArn`,
-    taskDefinitionArn: `taskDefinitionArn`,
+    taskDefinitionArn: `legacyTaskDefinitionArn`,
+    taskDefinitionArnsByRuntimeImage: {
+      'node:20': 'node20TaskDefinitionArn',
+      'node:24': 'node24TaskDefinitionArn',
+    },
     tenantId: 'some_tenant',
   };
 
@@ -104,7 +109,7 @@ describe('JobExecutionTaskService', () => {
       const mockArn = `mock-arn`;
       const mockTask = { taskArn: mockArn };
       const jobContext = createMockJobContext();
-      const jobManifest = createMockJobManifest();
+      const jobManifest = createMockJobManifest({ image: 'node:24' });
       const jobExecution = createMockJobExecution({
         id: `20250917T005949395Z-9602417782`,
         jobName: 'jobName',
@@ -173,7 +178,7 @@ describe('JobExecutionTaskService', () => {
           { key: 'map-migrated', value: 'migUJ1IX3DUXG' },
           { key: 'cmp:tenant', value: 'some_tenant' },
         ],
-        taskDefinition: defaultRunJobTaskParams.taskDefinitionArn,
+        taskDefinition: defaultRunJobTaskParams.taskDefinitionArnsByRuntimeImage['node:24'],
         // sha1 of execution job name/id + tenant + retryCount.
         clientToken: '4725755e08138edde031a177c36a269ecdbed5a3',
       });
@@ -207,6 +212,51 @@ describe('JobExecutionTaskService', () => {
         null,
         defaultRunJobTaskParams.jobExecution,
       );
+      expect(mockEcsClient).toHaveReceivedCommandWith(RunTaskCommand, {
+        taskDefinition: defaultRunJobTaskParams.taskDefinitionArnsByRuntimeImage['node:20'],
+      });
+    });
+
+    it('should use the legacy task definition ARN when the node:20 map entry is missing', async () => {
+      const mockArn = `mock-arn`;
+
+      mockEcsClient.on(RunTaskCommand).resolves({ tasks: [{ taskArn: mockArn }] });
+      mockJobContextService.findJobContext.mockResolvedValue(createMockJobContext());
+      mockJobManifestService.findJobManifest.mockResolvedValue(createMockJobManifest({ image: 'node:20' }));
+
+      await jobExecutionTaskService.runJobExecutionTask({
+        ...defaultRunJobTaskParams,
+        taskDefinitionArnsByRuntimeImage: {},
+      });
+
+      expect(mockEcsClient).toHaveReceivedCommandWith(RunTaskCommand, {
+        taskDefinition: defaultRunJobTaskParams.taskDefinitionArn,
+      });
+    });
+
+    it('should throw before creating an ECS task when the runtime image is unsupported', async () => {
+      mockJobManifestService.findJobManifest.mockResolvedValue(createMockJobManifest({ image: 'node:26' }));
+
+      await expect(jobExecutionTaskService.runJobExecutionTask(defaultRunJobTaskParams)).rejects.toThrow(
+        RuntimeImageTaskDefinitionError,
+      );
+      expect(mockEcsClient).not.toHaveReceivedCommand(RunTaskCommand);
+      expect(mockJobDetailsRepository.createJobDetailsFile).not.toHaveBeenCalled();
+    });
+
+    it('should throw before creating an ECS task when the runtime image has no task definition ARN', async () => {
+      mockJobManifestService.findJobManifest.mockResolvedValue(createMockJobManifest({ image: 'node:24' }));
+
+      await expect(
+        jobExecutionTaskService.runJobExecutionTask({
+          ...defaultRunJobTaskParams,
+          taskDefinitionArnsByRuntimeImage: {
+            'node:20': 'node20TaskDefinitionArn',
+          },
+        }),
+      ).rejects.toThrow(RuntimeImageTaskDefinitionError);
+      expect(mockEcsClient).not.toHaveReceivedCommand(RunTaskCommand);
+      expect(mockJobDetailsRepository.createJobDetailsFile).not.toHaveBeenCalled();
     });
 
     it('should handle ConflictException with a previously started task arn', async () => {

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts

@@ -7,9 +7,10 @@ import { ConflictException, ECSClient, RunTaskCommand } from '@aws-sdk/client-ec
 import { createHash } from 'node:crypto';
 import { JobDetailsRepository, JobUploadsRepository } from '../../S3Repository';
 import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
+import { JobManifest } from '../../manifest';
 import { JobExecution } from '../../model/JobExecution';
 import { JobExecutionRepository } from '../../repository';
-import { getAwsTags } from '../../utils';
+import { getAwsTags, resolveRuntimeImageTaskDefinitionArn } from '../../utils';
 import { JobContextService } from '../JobContextService/JobContextService';
 import { JobManifestService } from '../JobManifestService/JobManifestService';
 import { HasReachedMaxTenantConcurrencyParams, RunJobTaskParams } from './JobExecutionTaskService.defs';
@@ -56,7 +57,13 @@ export class JobExecutionTaskService {
   }
 
   public async runJobExecutionTask(params: RunJobTaskParams): Promise<string | undefined> {
-    const jobDetailsFileUrl = await this.createJobDetailsFile(params.tenantId, params.jobExecution);
+    const jobManifest = await this.findJobManifest(params.jobExecution);
+    const taskDefinitionArn = resolveRuntimeImageTaskDefinitionArn({
+      image: jobManifest?.image,
+      taskDefinitionArnsByRuntimeImage: params.taskDefinitionArnsByRuntimeImage,
+      legacyTaskDefinitionArn: params.taskDefinitionArn,
+    });
+    const jobDetailsFileUrl = await this.createJobDetailsFile(params.tenantId, params.jobExecution, jobManifest);
     const tags = getAwsTags({
       tenant: params.tenantId,
       region: params.awsRegion,
@@ -104,7 +111,7 @@ export class JobExecutionTaskService {
       const result = await this.ecsClient.send(
         new RunTaskCommand({
           cluster: params.clusterArn,
-          taskDefinition: params.taskDefinitionArn,
+          taskDefinition: taskDefinitionArn,
           overrides: {
             containerOverrides: [
               {
@@ -183,11 +190,11 @@ export class JobExecutionTaskService {
     return createHash('sha1').update(key).digest('hex');
   }
 
-  private async createJobDetailsFile(tenantId: string, executionToStart: JobExecution): Promise<string> {
-    const jobManifest = await this.jobManifestService.findJobManifest(
-      executionToStart.jobName,
-      executionToStart.version,
-    );
+  private async createJobDetailsFile(
+    tenantId: string,
+    executionToStart: JobExecution,
+    jobManifest: JobManifest | null,
+  ): Promise<string> {
     const jobContext = await this.jobContextService.findJobContext(executionToStart.context);
 
     const jobDetailsFileName = await this.jobDetailsRepository.createJobDetailsFile(
@@ -198,4 +205,8 @@ export class JobExecutionTaskService {
     );
     return await this.jobDetailsRepository.getS3SignedUrl(jobDetailsFileName);
   }
+
+  private async findJobManifest(executionToStart: JobExecution): Promise<JobManifest | null> {
+    return await this.jobManifestService.findJobManifest(executionToStart.jobName, executionToStart.version);
+  }
 }

package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts

@@ -2,7 +2,6 @@
  * @license
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
  */
-import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
 import {
   DynamoDBClient,
   DynamoDBClientConfig,
@@ -10,7 +9,13 @@ import {
   PutItemCommandOutput,
   ScanCommandOutput,
 } from '@aws-sdk/client-dynamodb';
-import { JobManifestService } from './JobManifestService';
+import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
+import { ENTITY_TYPES } from '../../constants/Repository.constants';
+import {
+  createMockClientGetResponse,
+  createMockClientPutResponse,
+  createMockClientScanResponse,
+} from '../../core/dynamoClient.mock';
 import {
   createMockCreateJobManifestRequest,
   createMockJobManifest,
@@ -19,12 +24,7 @@ import {
   createMockJobManifestRecord,
   JobManifest,
 } from '../../manifest';
-import {
-  createMockClientGetResponse,
-  createMockClientPutResponse,
-  createMockClientScanResponse,
-} from '../../core/dynamoClient.mock';
-import { ENTITY_TYPES } from '../../constants/Repository.constants';
+import { JobManifestService } from './JobManifestService';
 
 initIocContainer();
 
@@ -88,6 +88,7 @@ describe('JobManifestService', (): void => {
         version: mockJobManifestItem.version.S as string,
         timeoutSeconds: parseInt(mockJobManifestItem.timeoutSeconds.N as string),
         image: mockJobManifestItem.image.S as string,
+        autoRetry: false,
         type: `manifest`,
       } as JobManifest);
       expect(mockSend).toBeCalledTimes(1);

package/src/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.ts

@@ -0,0 +1,39 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import {
+  LEGACY_RUNTIME_IMAGE,
+  SUPPORTED_RUNTIME_IMAGES,
+  getRuntimeImageOrDefault,
+  isSupportedRuntimeImage,
+} from '../../constants/RuntimeImage.constants';
+import { RuntimeImageTaskDefinitionError } from '../../errors';
+import { ResolveRuntimeImageTaskDefinitionArnParams } from './ResolveRuntimeImageTaskDefinitionArn.types';
+
+export const resolveRuntimeImageTaskDefinitionArn = ({
+  image,
+  taskDefinitionArnsByRuntimeImage,
+  legacyTaskDefinitionArn,
+}: ResolveRuntimeImageTaskDefinitionArnParams): string => {
+  const runtimeImage = getRuntimeImageOrDefault(image);
+
+  if (!isSupportedRuntimeImage(runtimeImage)) {
+    throw new RuntimeImageTaskDefinitionError(
+      `Unsupported runtime image "${runtimeImage}". Supported images are: ${SUPPORTED_RUNTIME_IMAGES.join(', ')}`,
+    );
+  }
+
+  const taskDefinitionArn =
+    taskDefinitionArnsByRuntimeImage[runtimeImage] ||
+    (runtimeImage === LEGACY_RUNTIME_IMAGE ? legacyTaskDefinitionArn : undefined);
+
+  if (!taskDefinitionArn) {
+    throw new RuntimeImageTaskDefinitionError(
+      `No ECS task definition ARN configured for runtime image "${runtimeImage}"`,
+    );
+  }
+
+  return taskDefinitionArn;
+};

package/src/domains/job/utils/ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.types.ts

@@ -0,0 +1,10 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+export interface ResolveRuntimeImageTaskDefinitionArnParams {
+  image: string | null | undefined;
+  taskDefinitionArnsByRuntimeImage: Record<string, string>;
+  legacyTaskDefinitionArn: string;
+}

package/src/domains/job/utils/index.ts

@@ -3,3 +3,5 @@ export * from './GetAwsTags/GetAwsTags.types';
 export * from './SlackClient/SlackClient';
 export * from './SlackClient/types/SlackClient.types';
 export * from './SortByTimeQueued/SortByTimeQueued';
+export * from './ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn';
+export * from './ResolveRuntimeImageTaskDefinitionArn/ResolveRuntimeImageTaskDefinitionArn.types';

package/src/domains/manifest/index.ts

@@ -1,10 +1,11 @@
-export * from './services/ManifestServiceForDev';
 export * from './services/JobFunctionService';
+export * from './services/ManifestServiceForDev';
 
+export * from './utils/defaultAutoRetry';
 export * from './utils/getAndAssertJobManifestVersion';
 export * from './utils/isValidManifestVersion';
 export * from './utils/validateRuntimeImage';
 
-export * from './ManifestModel';
 export * from '../job/manifest/JobManifestV1';
 export * from './errors/ManifestValidationError';
+export * from './ManifestModel';

package/src/domains/manifest/utils/defaultAutoRetry.spec.ts

@@ -0,0 +1,16 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { defaultAutoRetry } from './defaultAutoRetry';
+
+describe('defaultAutoRetry', () => {
+  it('returns true when autoRetry is true', () => {
+    expect(defaultAutoRetry(true)).toEqual(true);
+  });
+
+  it.each([false, null, undefined])('returns false when autoRetry is %s', (autoRetry) => {
+    expect(defaultAutoRetry(autoRetry)).toEqual(false);
+  });
+});

package/src/domains/manifest/utils/defaultAutoRetry.ts

@@ -0,0 +1,6 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+export const defaultAutoRetry = (autoRetry: boolean | null | undefined): boolean => autoRetry === true;

package/src/domains/manifest/utils/validateRuntimeImage.spec.ts

@@ -3,16 +3,15 @@
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
  */
 
-import * as runtimeImageConstants from '../../job/constants/RuntimeImage.constants';
-import { RECOMMENDED_RUNTIME_IMAGE, SUPPORTED_RUNTIME_IMAGES } from '../../job/constants/RuntimeImage.constants';
+import {
+  DEPRECATED_RUNTIME_IMAGES,
+  RECOMMENDED_RUNTIME_IMAGE,
+  SUPPORTED_RUNTIME_IMAGES,
+} from '../../job/constants/RuntimeImage.constants';
 import { ManifestValidationError } from '../errors/ManifestValidationError';
 import { validateAndDefaultRuntimeImage } from './validateRuntimeImage';
 
 describe('validateAndDefaultRuntimeImage', () => {
-  afterEach(() => {
-    jest.restoreAllMocks();
-  });
-
   it('returns the recommended runtime image when image is undefined', () => {
     expect(validateAndDefaultRuntimeImage(undefined)).toEqual(RECOMMENDED_RUNTIME_IMAGE);
   });
@@ -21,9 +20,12 @@ describe('validateAndDefaultRuntimeImage', () => {
     expect(validateAndDefaultRuntimeImage(null)).toEqual(RECOMMENDED_RUNTIME_IMAGE);
   });
 
-  it.each(SUPPORTED_RUNTIME_IMAGES)('returns "%s" when it is in the supported list', (supportedImage) => {
-    expect(validateAndDefaultRuntimeImage(supportedImage)).toEqual(supportedImage);
-  });
+  it.each(SUPPORTED_RUNTIME_IMAGES.filter((image) => !DEPRECATED_RUNTIME_IMAGES.includes(image)))(
+    'returns "%s" when it is supported and not deprecated',
+    (supportedImage) => {
+      expect(validateAndDefaultRuntimeImage(supportedImage)).toEqual(supportedImage);
+    },
+  );
 
   it('throws ManifestValidationError when the image is not supported', () => {
     expect(() => validateAndDefaultRuntimeImage('node:18')).toThrow(ManifestValidationError);
@@ -31,9 +33,6 @@ describe('validateAndDefaultRuntimeImage', () => {
   });
 
   it('throws ManifestValidationError when the image is deprecated', () => {
-    jest.spyOn(runtimeImageConstants, 'isSupportedRuntimeImage').mockReturnValue(true);
-    jest.spyOn(runtimeImageConstants, 'isDeprecatedRuntimeImage').mockReturnValue(true);
-
     expect(() => validateAndDefaultRuntimeImage('node:20')).toThrow(ManifestValidationError);
     expect(() => validateAndDefaultRuntimeImage('node:20')).toThrow(/deprecated/);
   });