@squiz/job-runner-lib 2.3.0 → 2.4.0

package/src/mocks/mockIoc.ts

@@ -1,52 +1,27 @@
-import { Container, decorate, injectable } from 'inversify';
-import { Controller } from 'tsoa';
+import { faker } from '@faker-js/faker';
+import { Logger } from '@squiz/optimization-logger';
+import { Container } from 'inversify';
 import { buildProviderModule } from 'inversify-binding-decorators';
-import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
-
-import { Logger } from '@squiz/dx-logger-lib';
-import { S3Client } from '@aws-sdk/client-s3';
-
-// Makes tsoa's Controller injectable
-decorate(injectable(), Controller);
+import { testLogger } from '../jest-global-setup';
+import { buildJobRunnerModule } from '../utils';
 
 let iocContainer: Container;
 
-function initIocContainer(log: Logger): void {
+function initIocContainer(logger?: Logger): void {
+  const tenant = faker.string.uuid();
+  const jobRunnerModule = buildJobRunnerModule({
+    local: true,
+    dynamoTableName: faker.lorem.words(),
+    jobUploadsBucket: faker.lorem.words(),
+    jobDetailsBucket: faker.lorem.words(),
+    awsRegion: faker.lorem.words(),
+  });
+
   // Create a new container tsoa can use
-  iocContainer = new Container();
+  iocContainer = new Container({ skipBaseClassChecks: true });
 
   // make inversify aware of inversify-binding-decorators
   iocContainer.load(buildProviderModule());
-
-  // Init DynamoDB and S3 clients
-  let dynamoDbClient: DynamoDBClient;
-  let s3Client: S3Client;
-
-  if (process.env.ENVIRONMENT !== 'local') {
-    dynamoDbClient = new DynamoDBClient({ region: 'ap-southeast-2' });
-    s3Client = new S3Client({ region: 'ap-southeast-2' });
-  } else {
-    // eslint-disable-next-line no-console
-    console.log(`environment: '${process.env.ENVIRONMENT}'`);
-
-    // eslint-disable-next-line no-console
-    console.log(`table: '${process.env.DYNAMODB_TABLE}'`);
-    dynamoDbClient = new DynamoDBClient({
-      credentials: { accessKeyId: 'key', secretAccessKey: 'key' },
-      endpoint: 'http://localhost:8000',
-    });
-    s3Client = new S3Client({
-      credentials: { accessKeyId: 'key', secretAccessKey: 'key' },
-      endpoint: 'http://localhost:8000',
-    });
-  }
-
-  iocContainer.bind<DynamoDBClient>(DynamoDBClient).toConstantValue(dynamoDbClient);
-  iocContainer.bind<Logger>('Logger').toConstantValue(log);
-  iocContainer.bind<S3Client>(S3Client).toConstantValue(s3Client);
-
-  iocContainer.bind<string>('JobDetailsBucketName').toConstantValue('configObj.jobDetailsBucketName');
-  iocContainer.bind<string>('S3RepositoryBucketName').toConstantValue('configObj.bucketName');
+  iocContainer.load(jobRunnerModule({ tenant, logger: logger || testLogger }));
 }
-
-export { iocContainer, initIocContainer };
+export { initIocContainer, iocContainer };

package/src/utils/buildJobRunnerModule/buildJobRunnerModule.ts

@@ -0,0 +1,81 @@
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { ECSClient } from '@aws-sdk/client-ecs';
+import { S3Client } from '@aws-sdk/client-s3';
+import { SQSClient } from '@aws-sdk/client-sqs';
+import { Logger } from '@squiz/optimization-logger';
+import { ContainerModule, interfaces } from 'inversify';
+import {
+  InjectTokens,
+  JobContextService,
+  JobDetailsRepository,
+  JobExecutionRepository,
+  JobExecutionTaskService,
+  JobManifestService,
+  JobStartQueueService,
+  JobUploadsRepository,
+} from '../../domains/job';
+import { CreateJobRunnerModuleProps, CreateJobRunnerModuleResult } from './buildJobRunnerModule.types';
+
+export const buildJobRunnerModule = (props: CreateJobRunnerModuleProps): CreateJobRunnerModuleResult => {
+  let dynamoDbClient: DynamoDBClient;
+  let s3Client: S3Client;
+  let ecsClient: ECSClient;
+  let sqsClient: SQSClient;
+
+  if (props.local) {
+    dynamoDbClient = new DynamoDBClient({
+      credentials: { accessKeyId: 'key', secretAccessKey: 'key' },
+      endpoint: 'http://localhost:8000',
+    });
+    s3Client = new S3Client({
+      credentials: { accessKeyId: 'key', secretAccessKey: 'key' },
+      endpoint: 'http://localhost:4566', // This is the localstack EDGE_PORT
+      forcePathStyle: true,
+    });
+    ecsClient = new ECSClient({});
+    sqsClient = new SQSClient({
+      credentials: { accessKeyId: 'key', secretAccessKey: 'key' },
+      endpoint: 'http://localhost:4566',
+    });
+  } else {
+    dynamoDbClient = new DynamoDBClient({ region: props.awsRegion });
+    s3Client = new S3Client({ region: props.awsRegion });
+    ecsClient = new ECSClient({ region: props.awsRegion });
+    sqsClient = new SQSClient({ region: props.awsRegion });
+  }
+
+  return ({ tenant, logger }: { tenant?: string; logger: Logger }) => {
+    return new ContainerModule((bind: interfaces.Bind) => {
+      bind(DynamoDBClient).toConstantValue(dynamoDbClient);
+      bind(S3Client).toConstantValue(s3Client);
+      bind(ECSClient).toConstantValue(ecsClient);
+      bind(SQSClient).toConstantValue(sqsClient);
+
+      bind(InjectTokens.Logger).toConstantValue(logger);
+      bind(InjectTokens.Tenant).toConstantValue(tenant);
+      bind(InjectTokens.DynamoTableName).toConstantValue(props.dynamoTableName);
+      bind(InjectTokens.JobUploadsBucket).toConstantValue(props.jobUploadsBucket);
+      bind(InjectTokens.JobDetailsBucket).toConstantValue(props.jobDetailsBucket);
+
+      if (props.jobStartQueueUrl) {
+        bind(InjectTokens.JobStartQueueUrl).toConstantValue(props.jobStartQueueUrl);
+        bind<JobStartQueueService>(JobStartQueueService).toDynamicValue(
+          (context: interfaces.Context) =>
+            new JobStartQueueService(sqsClient, context.container.get(InjectTokens.JobStartQueueUrl)),
+        );
+      }
+
+      bind<JobExecutionTaskService>(JobExecutionTaskService).toDynamicValue(
+        (context: interfaces.Context) =>
+          new JobExecutionTaskService(
+            ecsClient,
+            context.container.get(JobManifestService),
+            context.container.get(JobContextService),
+            context.container.get(JobExecutionRepository),
+            context.container.get(JobDetailsRepository),
+            context.container.get(JobUploadsRepository),
+          ),
+      );
+    });
+  };
+};

package/src/utils/buildJobRunnerModule/buildJobRunnerModule.types.ts

@@ -0,0 +1,13 @@
+import { Logger } from '@squiz/optimization-logger';
+import { ContainerModule } from 'inversify';
+
+export type CreateJobRunnerModuleProps = {
+  dynamoTableName: string;
+  jobUploadsBucket: string;
+  jobDetailsBucket: string;
+  awsRegion: string;
+  local: boolean;
+  jobStartQueueUrl?: string;
+};
+
+export type CreateJobRunnerModuleResult = (props: { tenant?: string; logger: Logger }) => ContainerModule;

package/src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from './buildJobRunnerModule/buildJobRunnerModule';
+export * from './buildJobRunnerModule/buildJobRunnerModule.types';

package/lib/domains/job/S3Repository/S3Repository.d.ts

@@ -1,15 +0,0 @@
-/*!
- * @license
- * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
- */
-import { S3Client } from '@aws-sdk/client-s3';
-import { S3RepositoryBase } from './S3RepositoryBase';
-/**
- * Repository class for interacting with S3.
- * @export
- * @class S3Repository
- */
-export declare class S3Repository extends S3RepositoryBase {
-    protected s3Client: S3Client;
-    constructor(s3Client: S3Client, bucketName: string);
-}

package/lib/domains/job/S3Repository/S3Repository.spec.d.ts

@@ -1,5 +0,0 @@
-/*!
- * @license
- * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
- */
-export {};

package/lib/domains/job/services/JobQueueService/JobQueueService.d.ts

@@ -1,9 +0,0 @@
-import { JobExecutionTaskService } from '../JobExecutionTaskService/JobExecutionTaskService';
-import { CheckAndStartQueuedJobTaskParams } from './JobQueueService.defs';
-import { JobExecutionRepository } from '../../repository';
-export declare class JobQueueService {
-    private readonly jobExecutionTaskService;
-    private readonly jobExecutionRepository;
-    constructor(jobExecutionTaskService: JobExecutionTaskService, jobExecutionRepository: JobExecutionRepository);
-    checkAndStartQueuedJob(params: CheckAndStartQueuedJobTaskParams): Promise<string | null>;
-}

package/lib/domains/job/services/JobQueueService/JobQueueService.defs.d.ts

@@ -1,9 +0,0 @@
-import { RunJobTaskParams } from '../JobExecutionTaskService/JobExecutionTaskService.defs';
-/**
- * The parameters required for running a job as an ECS task.
- */
-export interface CheckAndStartQueuedJobTaskParams extends Omit<RunJobTaskParams, 'jobExecution'> {
-    jobName: string;
-    runtimeTaskFamily: string;
-    maxTenantConcurrentTasks: number;
-}

package/src/custom-types.d.ts

@@ -1,7 +0,0 @@
-import { type Logger } from 'winston';
-
-declare module 'express-serve-static-core' {
-  interface Request {
-    log: Logger;
-  }
-}

package/src/domains/job/S3Repository/S3Repository.spec.ts

@@ -1,50 +0,0 @@
-/*!
- * @license
- * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
- */
-
-import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
-import { S3Repository } from './S3Repository';
-import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
-import { testLogger } from '../../../jest-global-setup';
-
-// Create a new container tsoa can use
-initIocContainer(testLogger);
-
-// Mock S3 Client
-const mockOptions = {} as S3ClientConfig;
-const mockClient = new S3Client(mockOptions);
-
-const mockSend = jest.fn().mockResolvedValue({
-  Body: { transformToString: jest.fn().mockResolvedValue('file') },
-  $metadata: {},
-});
-mockClient.send = mockSend;
-
-// Inject the S3Repository with a mocked client
-iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
-
-let s3Repository: S3Repository;
-
-describe('S3Repository', (): void => {
-  beforeAll((): void => {
-    s3Repository = iocContainer.get(S3Repository);
-  });
-
-  describe(`get`, (): void => {
-    beforeEach(() => {
-      jest.resetAllMocks();
-    });
-
-    it(`should return a response for get`, async (): Promise<void> => {
-      const mockKey = `mock`;
-      mockSend.mockResolvedValueOnce({
-        Body: { transformToString: jest.fn().mockResolvedValue('file') },
-      });
-
-      const response = await s3Repository.get(mockKey);
-      expect(mockSend).toBeCalledTimes(1);
-      expect(response).toStrictEqual('file');
-    });
-  });
-});

package/src/domains/job/S3Repository/S3Repository.ts

@@ -1,27 +0,0 @@
-/*!
- * @license
- * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
- */
-
-import * as dotenv from 'dotenv';
-dotenv.config();
-
-import { S3Client } from '@aws-sdk/client-s3';
-import { inject } from 'inversify';
-import { S3RepositoryBase } from './S3RepositoryBase';
-import { provideTransient } from '../transientProvider';
-
-/**
- * Repository class for interacting with S3.
- * @export
- * @class S3Repository
- */
-@provideTransient(S3Repository)
-export class S3Repository extends S3RepositoryBase {
-  public constructor(
-    @inject(S3Client) protected s3Client: S3Client,
-    @inject('S3RepositoryBucketName') bucketName: string,
-  ) {
-    super(s3Client, bucketName);
-  }
-}

package/src/domains/job/services/JobQueueService/JobQueueService.defs.ts

@@ -1,10 +0,0 @@
-import { RunJobTaskParams } from '../JobExecutionTaskService/JobExecutionTaskService.defs';
-
-/**
- * The parameters required for running a job as an ECS task.
- */
-export interface CheckAndStartQueuedJobTaskParams extends Omit<RunJobTaskParams, 'jobExecution'> {
-  jobName: string;
-  runtimeTaskFamily: string;
-  maxTenantConcurrentTasks: number;
-}

package/src/domains/job/services/JobQueueService/JobQueueService.spec.ts

@@ -1,190 +0,0 @@
-/*!
- * @license
- * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
- */
-
-import { ECSClient } from '@aws-sdk/client-ecs';
-import { faker } from '@faker-js/faker/locale/en';
-import { S3Client } from '@aws-sdk/client-s3';
-import { JobContextRepository, JobExecutionRepository, JobManifestRepository } from '../../repository';
-import { JobContextService } from '../JobContextService/JobContextService';
-import { JobManifestService } from '../JobManifestService/JobManifestService';
-import { JobQueueService } from './JobQueueService';
-import { JobExecutionTaskService } from '../JobExecutionTaskService/JobExecutionTaskService';
-import { PaginationService } from '../../core/pagination';
-import { JobExecution } from '../../model/JobExecution';
-import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
-import { createMockJobManifest } from '../../manifest';
-import { createMockJobContext } from '../../model/JobContext';
-import { CheckAndStartQueuedJobTaskParams } from './JobQueueService.defs';
-import { JobDetailsRepository, S3Repository } from '../../S3Repository';
-
-describe('JobQueueService', () => {
-  const mockJobDetailsBucketName = faker.lorem.word();
-  const mockJobUploadBucketName = faker.lorem.word();
-  const jobExecutionBase = {
-    context: ``,
-    ecsTaskId: ``,
-    input: ``,
-    jobName: `mockJobName`,
-    timeFinished: null,
-    timeStarted: null,
-    type: 'jobExecution',
-    version: '1.2.3',
-  };
-  const mockExecutionOld: JobExecution = {
-    ...jobExecutionBase,
-    id: 'old',
-    status: JOB_EXECUTION_STATUS.queued,
-    timeQueued: faker.date.past().toISOString(),
-  };
-  const mockExecutionNew: JobExecution = {
-    ...jobExecutionBase,
-    id: 'new',
-    status: JOB_EXECUTION_STATUS.queued,
-    timeQueued: faker.date.future().toISOString(),
-  };
-  const mockCheckAndStartQueuedJobParams: CheckAndStartQueuedJobTaskParams = {
-    jobName: faker.lorem.word(),
-    tenantId: faker.string.uuid(),
-    shortEnvironment: faker.lorem.word(),
-    awsRegion: faker.lorem.word(),
-    clusterArn: faker.lorem.word(),
-    subnets: [],
-    maxTenantConcurrentTasks: 20,
-    shortRegion: faker.lorem.word(),
-    runtimeTaskFamily: faker.lorem.word(),
-    taskDefinitionArn: faker.lorem.word(),
-  };
-
-  let mockJobExecutionRepository: jest.Mocked<JobExecutionRepository>;
-  let mockEcsClient: jest.Mocked<ECSClient>;
-  let mockS3Client: jest.Mocked<S3Client>;
-
-  let jobManifestService: JobManifestService;
-  let jobContextService: JobContextService;
-  let jobExecutionTaskService: JobExecutionTaskService;
-  let jobQueueService: JobQueueService;
-  let jobDetailsRepository: JobDetailsRepository;
-  let jobUploadRepository: S3Repository;
-
-  let mockJobContextRepository: JobContextRepository;
-  let mockJobManifestRepository: JobManifestRepository;
-
-  beforeEach(() => {
-    mockJobExecutionRepository = {
-      list: jest.fn(),
-      findOne: jest.fn(),
-      update: jest.fn(),
-    } as unknown as jest.Mocked<JobExecutionRepository>;
-
-    mockEcsClient = {
-      send: jest.fn(),
-    } as unknown as jest.Mocked<ECSClient>;
-
-    mockS3Client = {
-      send: jest.fn().mockResolvedValue({
-        Body: { transformToString: jest.fn().mockResolvedValue('file') },
-        $metadata: {},
-      }),
-    } as unknown as jest.Mocked<S3Client>;
-
-    jobDetailsRepository = new JobDetailsRepository(mockS3Client, mockJobDetailsBucketName);
-    jobUploadRepository = new S3Repository(mockS3Client, mockJobUploadBucketName);
-
-    mockJobContextRepository = {
-      findOne: jest.fn(),
-    } as unknown as jest.Mocked<JobContextRepository>;
-    jobContextService = new JobContextService(mockJobContextRepository, new PaginationService());
-
-    mockJobManifestRepository = {
-      findOne: jest.fn(),
-    } as unknown as jest.Mocked<JobManifestRepository>;
-    jobManifestService = new JobManifestService(mockJobManifestRepository);
-
-    jobExecutionTaskService = new JobExecutionTaskService(
-      mockEcsClient,
-      jobManifestService,
-      jobContextService,
-      jobDetailsRepository,
-      jobUploadRepository,
-    );
-
-    jobQueueService = new JobQueueService(jobExecutionTaskService, mockJobExecutionRepository);
-  });
-
-  describe('checkAndStartQueuedJob', () => {
-    it('should not start any jobs if no pending executions exist', async (): Promise<void> => {
-      mockJobExecutionRepository.list.mockResolvedValue([]);
-      jest.spyOn(jobExecutionTaskService, `hasReachedMaxTenantConcurrency`).mockResolvedValueOnce(false);
-      jest.spyOn(jobExecutionTaskService, `runJobExecutionTask`).mockResolvedValueOnce('');
-
-      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
-      expect(jobExecutionTaskService.runJobExecutionTask).not.toBeCalled();
-    });
-
-    it('should start the oldest queued job', async (): Promise<void> => {
-      mockJobExecutionRepository.list
-        .mockResolvedValueOnce([mockExecutionNew, mockExecutionOld])
-        .mockResolvedValueOnce([mockExecutionOld, mockExecutionNew])
-        .mockResolvedValueOnce([mockExecutionOld, mockExecutionOld]);
-
-      jest
-        .spyOn(jobExecutionTaskService, `hasReachedMaxTenantConcurrency`)
-        .mockResolvedValueOnce(false)
-        .mockResolvedValueOnce(false)
-        .mockResolvedValueOnce(false);
-
-      jest.spyOn(jobExecutionTaskService, `runJobExecutionTask`).mockResolvedValue('');
-
-      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
-      expect(jobExecutionTaskService.runJobExecutionTask).toBeCalledTimes(1);
-      let executionStarted = (jobExecutionTaskService.runJobExecutionTask as jest.Mock).mock.calls[0][0];
-      expect(executionStarted.jobExecution).toBe(mockExecutionOld);
-
-      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
-      expect(jobExecutionTaskService.runJobExecutionTask).toBeCalledTimes(2);
-      executionStarted = (jobExecutionTaskService.runJobExecutionTask as jest.Mock).mock.calls[1][0];
-      expect(executionStarted.jobExecution).toBe(mockExecutionOld);
-
-      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
-      expect(jobExecutionTaskService.runJobExecutionTask).toBeCalledTimes(3);
-      executionStarted = (jobExecutionTaskService.runJobExecutionTask as jest.Mock).mock.calls[2][0];
-      expect(executionStarted.jobExecution).toBe(mockExecutionOld);
-    });
-
-    it('should create job details file when file is not found', async (): Promise<void> => {
-      mockJobExecutionRepository.list.mockResolvedValue([mockExecutionOld]);
-
-      jest.spyOn(jobExecutionTaskService, `hasReachedMaxTenantConcurrency`).mockResolvedValue(false);
-      jest.spyOn(jobExecutionTaskService, `runJobExecutionTask`).mockResolvedValue('mock-task-id');
-
-      // Mock job manifest and context services
-      const mockJobManifest = createMockJobManifest();
-      const mockJobContext = createMockJobContext();
-
-      jest.spyOn(jobManifestService, 'findJobManifest').mockResolvedValue(mockJobManifest);
-      jest.spyOn(jobContextService, 'findJobContext').mockResolvedValue(mockJobContext);
-
-      await jobQueueService.checkAndStartQueuedJob(mockCheckAndStartQueuedJobParams);
-
-      // Verify that the job execution task was started
-      expect(jobExecutionTaskService.runJobExecutionTask).toHaveBeenCalledWith(
-        expect.objectContaining({
-          jobExecution: mockExecutionOld,
-        }),
-      );
-
-      // Verify that the execution status was updated
-      expect(mockJobExecutionRepository.update).toHaveBeenCalledWith(
-        mockExecutionOld.id,
-        {
-          ecsTaskId: 'mock-task-id',
-          status: JOB_EXECUTION_STATUS.running,
-          timeStarted: expect.any(String),
-        },
-        mockExecutionOld.jobName,
-      );
-    });
-  });
-});

package/src/domains/job/services/JobQueueService/JobQueueService.ts

@@ -1,60 +0,0 @@
-import { JobExecutionTaskService } from '../JobExecutionTaskService/JobExecutionTaskService';
-import { CheckAndStartQueuedJobTaskParams } from './JobQueueService.defs';
-import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
-import { JobExecutionRepository } from '../../repository';
-import { sortByTimeQueued } from '../../utils';
-import { provideSingleton } from '../../singletonProvider';
-import { inject } from 'inversify';
-
-@provideSingleton(JobQueueService)
-export class JobQueueService {
-  public constructor(
-    @inject(JobExecutionTaskService) private readonly jobExecutionTaskService: JobExecutionTaskService,
-    @inject(JobExecutionRepository) private readonly jobExecutionRepository: JobExecutionRepository,
-  ) {}
-
-  public async checkAndStartQueuedJob(params: CheckAndStartQueuedJobTaskParams): Promise<string | null> {
-    const hasReachedMaxConcurrencyForTenant = await this.jobExecutionTaskService.hasReachedMaxTenantConcurrency({
-      clusterArn: params.clusterArn,
-      runtimeTaskFamily: params.runtimeTaskFamily,
-      maxTenantConcurrentTasks: params.maxTenantConcurrentTasks,
-    });
-
-    if (!hasReachedMaxConcurrencyForTenant) {
-      const executions = await this.jobExecutionRepository.list({
-        jobName: params.jobName,
-        status: JOB_EXECUTION_STATUS.queued,
-      });
-
-      if (executions.length > 0) {
-        const executionToStart = executions.sort(sortByTimeQueued)[0];
-        const ecsTaskId = await this.jobExecutionTaskService.runJobExecutionTask({
-          jobExecution: executionToStart,
-
-          awsRegion: params.awsRegion,
-          clusterArn: params.clusterArn,
-          subnets: params.subnets,
-          taskDefinitionArn: params.taskDefinitionArn,
-
-          tenantId: params.tenantId,
-          shortEnvironment: params.shortEnvironment,
-          shortRegion: params.shortRegion,
-        });
-
-        await this.jobExecutionRepository.update(
-          executionToStart.id,
-          {
-            ecsTaskId,
-            status: JOB_EXECUTION_STATUS.running,
-            timeStarted: new Date().toISOString(),
-          },
-          executionToStart.jobName,
-        );
-
-        return executionToStart.id;
-      }
-    }
-
-    return null;
-  }
-}