@squiz/job-runner-lib 1.66.2 → 2.2.0

package/src/domains/job/services/JobManifestService/JobManifestService.spec.ts

@@ -0,0 +1,234 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+import { initIocContainer, iocContainer } from '../../../../mocks/mockIoc';
+import {
+  DeleteItemCommandOutput,
+  DynamoDBClient,
+  DynamoDBClientConfig,
+  GetItemCommandOutput,
+  PutItemCommandOutput,
+  ScanCommandOutput,
+} from '@aws-sdk/client-dynamodb';
+import { testLogger } from '../../../../jest-global-setup';
+import { JobManifestService } from './JobManifestService';
+import {
+  createMockCreateJobManifestRequest,
+  createMockJobManifest,
+  createMockJobManifestDto,
+  createMockJobManifestDtos,
+  createMockJobManifestRecord,
+  JobManifest,
+} from '../../manifest';
+import {
+  createMockClientGetResponse,
+  createMockClientPutResponse,
+  createMockClientScanResponse,
+} from '../../core/dynamoClient.mock';
+import { PARTITION_KEYS } from '../../constants/Repository.constants';
+
+// Create a new container tsoa can use
+initIocContainer(testLogger);
+
+// Mock DynamoDB Client
+const mockOptions = {} as DynamoDBClientConfig;
+const mockClient = new DynamoDBClient(mockOptions);
+
+const mockSend = jest.fn();
+mockClient.send = mockSend;
+
+// Inject the JobManifestRepository property (of JobManifestService) with a mocked client
+iocContainer.rebind<DynamoDBClient>(DynamoDBClient).toConstantValue(mockClient);
+
+let jobManifestService: JobManifestService;
+
+describe('JobManifestService', (): void => {
+  beforeAll((): void => {
+    jobManifestService = iocContainer.get(JobManifestService);
+  });
+
+  describe(`createJobManifest`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return a response for createJob`, async (): Promise<void> => {
+      const newJobManifestToCreate = createMockCreateJobManifestRequest();
+      const mockReponse = createMockClientPutResponse({
+        Attributes: createMockJobManifestDto({
+          pk: PARTITION_KEYS.manifest,
+          sk: newJobManifestToCreate.name,
+        }),
+      });
+
+      mockSend.mockImplementationOnce((): PutItemCommandOutput => mockReponse);
+
+      const response = await jobManifestService.createJobManifest(newJobManifestToCreate);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toStrictEqual({
+        ...newJobManifestToCreate,
+        type: PARTITION_KEYS.manifest,
+      });
+    });
+  });
+
+  describe(`deleteJobManifest`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return an empty response for job manifest deletion`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
+        return {
+          $metadata: {},
+        };
+      });
+
+      const response = await jobManifestService.deleteJobManifest(`name`);
+      expect(response).toBe(undefined);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+  });
+
+  describe(`findJobManifest`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return job manifest`, async (): Promise<void> => {
+      const mockJobManifestItem = createMockJobManifestDto();
+      mockSend.mockImplementationOnce(
+        (): GetItemCommandOutput =>
+          createMockClientGetResponse({
+            Item: mockJobManifestItem,
+          }),
+      );
+
+      const response = await jobManifestService.findJobManifest(
+        mockJobManifestItem.name.S as string,
+        mockJobManifestItem.version.S as string,
+      );
+      expect(response).toStrictEqual({
+        concurrency: parseInt(mockJobManifestItem.concurrency.N as string),
+        entryFile: mockJobManifestItem.entryFile.S as string,
+        entryFunction: JSON.parse(mockJobManifestItem.entryFunction.S as string),
+        name: mockJobManifestItem.name.S as string,
+        version: mockJobManifestItem.version.S as string,
+        timeoutSeconds: parseInt(mockJobManifestItem.timeoutSeconds.N as string),
+        type: `manifest`,
+      } as JobManifest);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+
+    it(`should return null if job manifest does not exist`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: [],
+          }),
+      );
+
+      const response = await jobManifestService.findJobManifest(`null`, `null`);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toBe(null);
+    });
+  });
+
+  describe(`listAllJobManifests`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return an empty list`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
+
+      const response = await jobManifestService.listAllJobManifests();
+      expect(mockSend).toBeCalled();
+      expect(response).toStrictEqual([]);
+    });
+
+    it(`should return a list of existing job manifests`, async (): Promise<void> => {
+      const mockManifestItems = createMockJobManifestDtos();
+
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: mockManifestItems,
+            LastEvaluatedKey: undefined,
+          }),
+      );
+
+      const response = await jobManifestService.listAllJobManifests();
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toHaveLength(mockManifestItems.length);
+    });
+
+    it(`should paginate results to return a list of existing job manifests`, async (): Promise<void> => {
+      const mockManifestItemsPage1 = createMockJobManifestDtos();
+      const mockManifestItemsPage2 = createMockJobManifestDtos();
+
+      mockSend
+        .mockImplementationOnce(
+          (): ScanCommandOutput =>
+            createMockClientScanResponse({
+              Items: mockManifestItemsPage1,
+              LastEvaluatedKey: mockManifestItemsPage1.slice(-1)[0],
+            }),
+        )
+        .mockImplementationOnce(
+          (): ScanCommandOutput =>
+            createMockClientScanResponse({
+              Items: mockManifestItemsPage2,
+              LastEvaluatedKey: undefined,
+            }),
+        );
+
+      const response = await jobManifestService.listAllJobManifests();
+      expect(mockSend).toBeCalledTimes(2);
+      expect(response).toHaveLength(mockManifestItemsPage1.length + mockManifestItemsPage2.length);
+    });
+  });
+
+  describe(`updateJobManifest`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should update a job manifest record`, async (): Promise<void> => {
+      const jobManifestToUpdate = createMockJobManifest({
+        name: `name`,
+      });
+      const updatedJobManifest: JobManifest = {
+        ...jobManifestToUpdate,
+        concurrency: 10,
+        timeoutSeconds: 100,
+      };
+      const updatedJobManifestRecord = createMockJobManifestRecord({
+        sk: `${updatedJobManifest.name}-${updatedJobManifest.version}`,
+        name: updatedJobManifest.name,
+        version: updatedJobManifest.version,
+        entryFile: updatedJobManifest.entryFile,
+        entryFunction: updatedJobManifest.entryFunction,
+        concurrency: updatedJobManifest.concurrency,
+        timeoutSeconds: updatedJobManifest.timeoutSeconds,
+      });
+      const updatedJobManifestRecordDto = createMockJobManifestDto({
+        ...updatedJobManifestRecord,
+      });
+      const putResponse = createMockClientPutResponse({
+        Attributes: { ...updatedJobManifestRecordDto },
+      });
+
+      mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
+
+      const response = await jobManifestService.updateJobManifest(
+        jobManifestToUpdate.name,
+        jobManifestToUpdate.version,
+        updatedJobManifestRecord,
+      );
+      expect(response).toStrictEqual(updatedJobManifest);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+  });
+});

package/src/domains/job/services/JobManifestService/JobManifestService.ts

@@ -0,0 +1,40 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { inject } from 'inversify';
+import { JobManifestRepository } from '../../repository';
+import { provideSingleton } from '../../singletonProvider';
+import { CreateJobManifestRequest, JobManifest, UpdateJobManifestRequest } from '../../manifest';
+
+@provideSingleton(JobManifestService)
+export class JobManifestService {
+  constructor(@inject(JobManifestRepository) protected jobManifestRepository: JobManifestRepository) {}
+
+  public async createJobManifest(input: CreateJobManifestRequest): Promise<JobManifest> {
+    return this.jobManifestRepository.create(input);
+  }
+
+  public async updateJobManifest(
+    name: string,
+    version: string,
+    input: UpdateJobManifestRequest,
+  ): Promise<JobManifest | null> {
+    const sk = `${name}-${version}`;
+    return this.jobManifestRepository.update(sk, input);
+  }
+
+  public async listAllJobManifests(): Promise<Array<JobManifest>> {
+    return this.jobManifestRepository.list();
+  }
+
+  public async findJobManifest(name: string, version: string): Promise<JobManifest | null> {
+    const sk = `${name}-${version}`;
+    return this.jobManifestRepository.findOne(sk);
+  }
+
+  public async deleteJobManifest(name: string): Promise<void> {
+    return await this.jobManifestRepository.delete(name);
+  }
+}

package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts

@@ -0,0 +1,10 @@
+import { RunJobTaskParams } from '../JobExecutionTaskService/JobExecutionTaskService.defs';
+
+/**
+ * The parameters required for running a job as an ECS task.
+ */
+export interface CheckAndStartQueuedJobTaskParams extends Omit<RunJobTaskParams, 'jobExecution'> {
+  jobName: string;
+  runtimeTaskFamily: string;
+  maxTenantConcurrentTasks: number;
+}

package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts

@@ -0,0 +1,190 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { ECSClient } from '@aws-sdk/client-ecs';
+import { faker } from '@faker-js/faker/locale/en';
+import { S3Client } from '@aws-sdk/client-s3';
+import { JobContextRepository, JobExecutionRepository, JobManifestRepository } from '../../repository';
+import { JobContextService } from '../JobContextService/JobContextService';
+import { JobManifestService } from '../JobManifestService/JobManifestService';
+import { JobQueueService } from './JobQueueService';
+import { JobExecutionTaskService } from '../JobExecutionTaskService/JobExecutionTaskService';
+import { PaginationService } from '../../core/pagination';
+import { JobExecution } from '../../model/JobExecution';
+import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
+import { createMockJobManifest } from '../../manifest';
+import { createMockJobContext } from '../../model/JobContext';
+import { CheckAndStartQueuedJobTaskParams } from './JobQueueService.defs';
+import { JobDetailsRepository, S3Repository } from '../../S3Repository';
+
+describe('JobQueueService', () => {
+  const mockJobDetailsBucketName = faker.lorem.word();
+  const mockJobUploadBucketName = faker.lorem.word();
+  const jobExecutionBase = {
+    context: ``,
+    ecsTaskId: ``,
+    input: ``,
+    jobName: `mockJobName`,
+    timeFinished: null,
+    timeStarted: null,
+    type: 'jobExecution',
+    version: '1.2.3',
+  };
+  const mockExecutionOld: JobExecution = {
+    ...jobExecutionBase,
+    id: 'old',
+    status: JOB_EXECUTION_STATUS.queued,
+    timeQueued: faker.date.past().toISOString(),
+  };
+  const mockExecutionNew: JobExecution = {
+    ...jobExecutionBase,
+    id: 'new',
+    status: JOB_EXECUTION_STATUS.queued,
+    timeQueued: faker.date.future().toISOString(),
+  };
+  const mockCheckAndStartQueuedJobParams: CheckAndStartQueuedJobTaskParams = {
+    jobName: faker.lorem.word(),
+    tenantId: faker.string.uuid(),
+    shortEnvironment: faker.lorem.word(),
+    awsRegion: faker.lorem.word(),
+    clusterArn: faker.lorem.word(),
+    subnets: [],
+    maxTenantConcurrentTasks: 20,
+    shortRegion: faker.lorem.word(),
+    runtimeTaskFamily: faker.lorem.word(),
+    taskDefinitionArn: faker.lorem.word(),
+  };
+
+  let mockJobExecutionRepository: jest.Mocked<JobExecutionRepository>;
+  let mockEcsClient: jest.Mocked<ECSClient>;
+  let mockS3Client: jest.Mocked<S3Client>;
+
+  let jobManifestService: JobManifestService;
+  let jobContextService: JobContextService;
+  let jobExecutionTaskService: JobExecutionTaskService;
+  let jobQueueService: JobQueueService;
+  let jobDetailsRepository: JobDetailsRepository;
+  let jobUploadRepository: S3Repository;
+
+  let mockJobContextRepository: JobContextRepository;
+  let mockJobManifestRepository: JobManifestRepository;
+
+  beforeEach(() => {
+    mockJobExecutionRepository = {
+      list: jest.fn(),
+      findOne: jest.fn(),
+      update: jest.fn(),
+    } as unknown as jest.Mocked<JobExecutionRepository>;
+
+    mockEcsClient = {
+      send: jest.fn(),
+    } as unknown as jest.Mocked<ECSClient>;
+
+    mockS3Client = {
+      send: jest.fn().mockResolvedValue({
+        Body: { transformToString: jest.fn().mockResolvedValue('file') },
+        $metadata: {},
+      }),
+    } as unknown as jest.Mocked<S3Client>;
+
+    jobDetailsRepository = new JobDetailsRepository(mockS3Client, mockJobDetailsBucketName);
+    jobUploadRepository = new S3Repository(mockS3Client, mockJobUploadBucketName);
+
+    mockJobContextRepository = {
+      findOne: jest.fn(),
+    } as unknown as jest.Mocked<JobContextRepository>;
+    jobContextService = new JobContextService(mockJobContextRepository, new PaginationService());
+
+    mockJobManifestRepository = {
+      findOne: jest.fn(),
+    } as unknown as jest.Mocked<JobManifestRepository>;
+    jobManifestService = new JobManifestService(mockJobManifestRepository);
+
+    jobExecutionTaskService = new JobExecutionTaskService(
+      mockEcsClient,
+      jobManifestService,
+      jobContextService,
+      jobDetailsRepository,
+      jobUploadRepository,
+    );
+
+    jobQueueService = new JobQueueService(jobExecutionTaskService, mockJobExecutionRepository);
+  });
+
+  describe('checkAndStartQueuedJob', () => {
+    it('should not start any jobs if no pending executions exist', async (): Promise<void> => {
+      mockJobExecutionRepository.list.mockResolvedValue([]);
+      jest.spyOn(jobExecutionTaskService, `hasReachedMaxTenantConcurrency`).mockResolvedValueOnce(false);
+      jest.spyOn(jobExecutionTaskService, `runJobExecutionTask`).mockResolvedValueOnce('');
+
+      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
+      expect(jobExecutionTaskService.runJobExecutionTask).not.toBeCalled();
+    });
+
+    it('should start the oldest queued job', async (): Promise<void> => {
+      mockJobExecutionRepository.list
+        .mockResolvedValueOnce([mockExecutionNew, mockExecutionOld])
+        .mockResolvedValueOnce([mockExecutionOld, mockExecutionNew])
+        .mockResolvedValueOnce([mockExecutionOld, mockExecutionOld]);
+
+      jest
+        .spyOn(jobExecutionTaskService, `hasReachedMaxTenantConcurrency`)
+        .mockResolvedValueOnce(false)
+        .mockResolvedValueOnce(false)
+        .mockResolvedValueOnce(false);
+
+      jest.spyOn(jobExecutionTaskService, `runJobExecutionTask`).mockResolvedValue('');
+
+      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
+      expect(jobExecutionTaskService.runJobExecutionTask).toBeCalledTimes(1);
+      let executionStarted = (jobExecutionTaskService.runJobExecutionTask as jest.Mock).mock.calls[0][0];
+      expect(executionStarted.jobExecution).toBe(mockExecutionOld);
+
+      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
+      expect(jobExecutionTaskService.runJobExecutionTask).toBeCalledTimes(2);
+      executionStarted = (jobExecutionTaskService.runJobExecutionTask as jest.Mock).mock.calls[1][0];
+      expect(executionStarted.jobExecution).toBe(mockExecutionOld);
+
+      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
+      expect(jobExecutionTaskService.runJobExecutionTask).toBeCalledTimes(3);
+      executionStarted = (jobExecutionTaskService.runJobExecutionTask as jest.Mock).mock.calls[2][0];
+      expect(executionStarted.jobExecution).toBe(mockExecutionOld);
+    });
+
+    it('should create job details file when file is not found', async (): Promise<void> => {
+      mockJobExecutionRepository.list.mockResolvedValue([mockExecutionOld]);
+
+      jest.spyOn(jobExecutionTaskService, `hasReachedMaxTenantConcurrency`).mockResolvedValue(false);
+      jest.spyOn(jobExecutionTaskService, `runJobExecutionTask`).mockResolvedValue('mock-task-id');
+
+      // Mock job manifest and context services
+      const mockJobManifest = createMockJobManifest();
+      const mockJobContext = createMockJobContext();
+
+      jest.spyOn(jobManifestService, 'findJobManifest').mockResolvedValue(mockJobManifest);
+      jest.spyOn(jobContextService, 'findJobContext').mockResolvedValue(mockJobContext);
+
+      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
+
+      // Verify that the job execution task was started
+      expect(jobExecutionTaskService.runJobExecutionTask).toHaveBeenCalledWith(
+        expect.objectContaining({
+          jobExecution: mockExecutionOld,
+        }),
+      );
+
+      // Verify that the execution status was updated
+      expect(mockJobExecutionRepository.update).toHaveBeenCalledWith(
+        mockExecutionOld.id,
+        {
+          ecsTaskId: 'mock-task-id',
+          status: JOB_EXECUTION_STATUS.running,
+          timeStarted: expect.any(String),
+        },
+        mockExecutionOld.jobName,
+      );
+    });
+  });
+});

package/src/domains/job/services/JobQueueService/JobQueueService.ts

@@ -0,0 +1,60 @@
+import { JobExecutionTaskService } from '../JobExecutionTaskService/JobExecutionTaskService';
+import { CheckAndStartQueuedJobTaskParams } from './JobQueueService.defs';
+import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
+import { JobExecutionRepository } from '../../repository';
+import { sortByTimeQueued } from '../../utils';
+import { provideSingleton } from '../../singletonProvider';
+import { inject } from 'inversify';
+
+@provideSingleton(JobQueueService)
+export class JobQueueService {
+  public constructor(
+    @inject(JobExecutionTaskService) private readonly jobExecutionTaskService: JobExecutionTaskService,
+    @inject(JobExecutionRepository) private readonly jobExecutionRepository: JobExecutionRepository,
+  ) {}
+
+  public async checkAndStartQueuedJob(params: CheckAndStartQueuedJobTaskParams): Promise<string | null> {
+    const hasReachedMaxConcurrencyForTenant = await this.jobExecutionTaskService.hasReachedMaxTenantConcurrency({
+      clusterArn: params.clusterArn,
+      runtimeTaskFamily: params.runtimeTaskFamily,
+      maxTenantConcurrentTasks: params.maxTenantConcurrentTasks,
+    });
+
+    if (!hasReachedMaxConcurrencyForTenant) {
+      const executions = await this.jobExecutionRepository.list({
+        jobName: params.jobName,
+        status: JOB_EXECUTION_STATUS.queued,
+      });
+
+      if (executions.length > 0) {
+        const executionToStart = executions.sort(sortByTimeQueued)[0];
+        const ecsTaskId = await this.jobExecutionTaskService.runJobExecutionTask({
+          jobExecution: executionToStart,
+
+          awsRegion: params.awsRegion,
+          clusterArn: params.clusterArn,
+          subnets: params.subnets,
+          taskDefinitionArn: params.taskDefinitionArn,
+
+          tenantId: params.tenantId,
+          shortEnvironment: params.shortEnvironment,
+          shortRegion: params.shortRegion,
+        });
+
+        await this.jobExecutionRepository.update(
+          executionToStart.id,
+          {
+            ecsTaskId,
+            status: JOB_EXECUTION_STATUS.running,
+            timeStarted: new Date().toISOString(),
+          },
+          executionToStart.jobName,
+        );
+
+        return executionToStart.id;
+      }
+    }
+
+    return null;
+  }
+}

package/src/domains/job/services/JobUploadService/JobUploadService.mock.ts

@@ -0,0 +1,45 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { JobManifestV1Model } from '../../../manifest';
+
+export const mockManifestJson: JobManifestV1Model = {
+  $schema: 'http://localhost:3040/schemas/v1.json#',
+  name: 'test-job',
+  displayName: 'Test Job',
+  version: '1.0.0',
+  concurrency: 1,
+  timeout: 600,
+  mainFunction: 'main',
+  description: 'some-description',
+  functions: [
+    {
+      name: 'main',
+      entry: 'main.js',
+      input: {
+        type: 'object',
+        properties: {
+          something: {
+            type: 'string',
+          },
+        },
+        required: ['something'],
+      },
+    },
+    {
+      name: 'promise-func',
+      entry: 'promise.js',
+      input: {
+        type: 'object',
+        properties: {
+          something: {
+            type: 'string',
+          },
+        },
+        required: ['something'],
+      },
+    },
+  ],
+};