@squiz/job-runner-lib 1.66.2 → 2.2.0

package/src/domains/job/services/JobContextService/JobContextService.ts

@@ -0,0 +1,77 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { inject } from 'inversify';
+import { JobContextRepository } from '../../repository';
+import { provideSingleton } from '../../singletonProvider';
+import {
+  encodeTokenString,
+  getFormattedResponse,
+  PaginationFormattedResponse,
+  PaginationInfo,
+  PaginationKey,
+  PaginationResponse,
+  PaginationService,
+} from '../../core/pagination';
+import { CreateJobContextRequest, JobContext, JobContextDto, UpdateJobContextRequest } from '../../model/JobContext';
+
+@provideSingleton(JobContextService)
+export class JobContextService {
+  constructor(
+    @inject(JobContextRepository) protected jobContextRepository: JobContextRepository,
+    @inject(PaginationService) protected paginationService: PaginationService<JobContext>,
+  ) {}
+
+  public async createJobContext(input: CreateJobContextRequest): Promise<JobContext> {
+    return this.jobContextRepository.create(input);
+  }
+
+  public async updateJobContext(name: string, input: UpdateJobContextRequest): Promise<JobContext | null> {
+    return this.jobContextRepository.update(name, input);
+  }
+
+  public async listAllJobContexts(): Promise<Array<JobContext>> {
+    return this.jobContextRepository.list();
+  }
+
+  public async listJobContexts(paginationInfo: PaginationInfo): Promise<PaginationFormattedResponse<JobContext>> {
+    const { links, data } = await this.listContextsUpToLimit(paginationInfo);
+
+    return getFormattedResponse<JobContext>({ links, data });
+  }
+
+  private async listContextsUpToLimit(paginationInfo: PaginationInfo): Promise<PaginationResponse<JobContext>> {
+    const { limit, token } = paginationInfo;
+
+    let currentToken = token;
+    const contexts: Array<JobContext> = [];
+
+    do {
+      const newRequestData = {
+        ...paginationInfo,
+        token: currentToken,
+      };
+
+      const { items, lastEvaluatedKey } = await this.jobContextRepository.getContexts(newRequestData);
+
+      contexts.push(...items);
+      currentToken = lastEvaluatedKey
+        ? encodeTokenString(lastEvaluatedKey as unknown as PaginationKey<JobContextDto>)
+        : null;
+    } while (currentToken && contexts.length < limit);
+
+    return this.paginationService.addPaginationLinksToResponse(contexts, paginationInfo, currentToken, (context) =>
+      this.jobContextRepository.convertToDynamoDbItem(context),
+    );
+  }
+
+  public async findJobContext(contextName: string): Promise<JobContext | null> {
+    return this.jobContextRepository.findOne(contextName);
+  }
+
+  public async deleteJobContext(contextName: string): Promise<void> {
+    return await this.jobContextRepository.delete(contextName);
+  }
+}

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts

@@ -1,3 +1,5 @@
+import { JobExecution } from '../../model/JobExecution';
+
 /**
  * The required parameters for retrieving the number of running ECS tasks
  */
@@ -7,34 +9,16 @@ export interface ListRunningTasksParams {
   maxTenantConcurrentTasks: number;
 }
 
-/**
- * The parameters required for the logging service configuration.
- */
-interface LoggingServiceConfig {
-  enabled: boolean;
-  tags: string;
-  credentials: string;
-}
-
 /**
  * The parameters required for running a job as an ECS task.
  */
 export interface RunJobTaskParams {
-  jobName: string;
-  jobVersion: string;
-  jobExecutionId: string;
+  jobExecution: JobExecution;
   awsRegion: string;
   shortRegion: string;
   shortEnvironment: string;
   clusterArn: string;
   subnets: Array<string>;
-  maxTenantConcurrentTasks: number;
-  runtimeTaskFamily: string;
   taskDefinitionArn: string;
-  containerName: string;
-  bucketName: string;
-  jobRunnerApiUrl: string;
-  loggingServiceApiConfig: LoggingServiceConfig;
-  dynamoTableName: string;
   tenantId: string;
 }

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts

@@ -1,26 +1,67 @@
-import { ECSClient } from '@aws-sdk/client-ecs';
+import 'aws-sdk-client-mock-jest';
+import { AwsClientStub, mockClient } from 'aws-sdk-client-mock';
+import { ECSClient, ListTasksCommand, RunTaskCommand } from '@aws-sdk/client-ecs';
+import { faker } from '@faker-js/faker';
 import { JobExecutionTaskService } from './JobExecutionTaskService';
+import { JobManifestService } from '../JobManifestService/JobManifestService';
+import { JobContextService } from '../JobContextService/JobContextService';
+import { JobDetailsRepository, S3Repository } from '../../S3Repository';
+import { createMockJobExecution } from '../../model/JobExecution';
+import { createMockJobContext } from '../../model/JobContext';
+import { createMockJobManifest } from '../../manifest';
+import { RunJobTaskParams } from './JobExecutionTaskService.defs';
 
 describe('JobExecutionTaskService', () => {
-  const mockSend = jest.fn();
-
-  let mockEcsClient: ECSClient;
+  let mockEcsClient: AwsClientStub<ECSClient>;
+  let mockJobManifestService: jest.Mocked<JobManifestService>;
+  let mockJobContextService: jest.Mocked<JobContextService>;
+  let mockJobDetailsRepository: jest.Mocked<JobDetailsRepository>;
+  let mockJobUploadRepository: jest.Mocked<S3Repository>;
   let jobExecutionTaskService: JobExecutionTaskService;
 
+  const jobDetailsSignedUrl = faker.internet.url();
+  const jobCodeArtifactSignedUrl = faker.internet.url();
+  const defaultRunJobTaskParams: RunJobTaskParams = {
+    jobExecution: createMockJobExecution(),
+    awsRegion: 'ap-southeast-2',
+    shortRegion: 'au',
+    shortEnvironment: 'dev',
+    clusterArn: `clusterArn`,
+    subnets: ['subnet1'],
+    taskDefinitionArn: `taskDefinitionArn`,
+    tenantId: 'some_tenant',
+  };
+
   beforeEach(() => {
     jest.resetAllMocks();
 
-    mockEcsClient = {
-      send: mockSend,
-    } as unknown as ECSClient;
-
-    jobExecutionTaskService = new JobExecutionTaskService(mockEcsClient);
+    mockEcsClient = mockClient(ECSClient);
+    mockJobManifestService = {
+      findJobManifest: jest.fn(),
+    } as unknown as jest.Mocked<JobManifestService>;
+    mockJobContextService = {
+      findJobContext: jest.fn(),
+    } as unknown as jest.Mocked<JobContextService>;
+    mockJobDetailsRepository = {
+      createJobDetailsFile: jest.fn(),
+      getS3SignedUrl: jest.fn().mockResolvedValue(jobDetailsSignedUrl),
+    } as unknown as jest.Mocked<JobDetailsRepository>;
+    mockJobUploadRepository = {
+      getS3SignedUrl: jest.fn().mockResolvedValue(jobCodeArtifactSignedUrl),
+    } as unknown as jest.Mocked<S3Repository>;
+
+    jobExecutionTaskService = new JobExecutionTaskService(
+      new ECSClient({}),
+      mockJobManifestService,
+      mockJobContextService,
+      mockJobDetailsRepository,
+      mockJobUploadRepository,
+    );
   });
 
   describe(`getTenantRunningTasks`, () => {
     it('should 0 if no tasks were found', async (): Promise<void> => {
-      mockSend.mockImplementationOnce(() => Promise.resolve({}));
-
+      mockEcsClient.on(ListTasksCommand).resolves({});
       const response = await (jobExecutionTaskService as any).getTenantRunningTasks({
         clusterArn: `clusterArn`,
         runtimeTaskFamily: 'some_family',
@@ -33,7 +74,7 @@ describe('JobExecutionTaskService', () => {
 
   describe(`hasReachedMaxTenantConcurrency`, () => {
     it('should return false if number of running jobs is less than max', async (): Promise<void> => {
-      mockSend.mockImplementationOnce(() => Promise.resolve({ taskArns: ['12345', '67890'] }));
+      mockEcsClient.on(ListTasksCommand).resolves({ taskArns: ['12345', '67890'] });
 
       const response = await jobExecutionTaskService.hasReachedMaxTenantConcurrency({
         clusterArn: `clusterArn`,
@@ -45,7 +86,7 @@ describe('JobExecutionTaskService', () => {
     });
 
     it('should return true if number of running jobs is more than max', async (): Promise<void> => {
-      mockSend.mockImplementationOnce(() => Promise.resolve({ taskArns: ['12345', '67890'] }));
+      mockEcsClient.on(ListTasksCommand).resolves({ taskArns: ['12345', '67890'] });
 
       const response = await jobExecutionTaskService.hasReachedMaxTenantConcurrency({
         clusterArn: `clusterArn`,
@@ -61,35 +102,110 @@ describe('JobExecutionTaskService', () => {
     it('should return task arn after starting an ECS task', async (): Promise<void> => {
       const mockArn = `mock-arn`;
       const mockTask = { taskArn: mockArn };
-      mockSend.mockImplementationOnce(() => Promise.resolve({ tasks: [mockTask] }));
-
-      const response = await jobExecutionTaskService.runJobExecutionTask({
-        jobExecutionId: `${new Date().toISOString()}-jobExecutionId`,
-        jobName: `jobName`,
-        jobVersion: `jobVersion`,
-        jobRunnerApiUrl: `jobRunnerApiUrl`,
+      const jobContext = createMockJobContext();
+      const jobManifest = createMockJobManifest();
+      const jobExecution = createMockJobExecution({
+        id: `${new Date().toISOString()}-jobExecutionId`,
+        jobName: 'jobName',
+        version: 'jobVersion',
+      });
 
-        maxTenantConcurrentTasks: 20,
-        runtimeTaskFamily: 'some_family',
-        bucketName: `bucketName`,
-        containerName: `containerName`,
+      mockEcsClient.on(RunTaskCommand).resolves({ tasks: [mockTask] });
+      mockJobContextService.findJobContext.mockResolvedValue(jobContext);
+      mockJobManifestService.findJobManifest.mockResolvedValue(jobManifest);
 
-        awsRegion: 'ap-southeast-2',
-        shortRegion: 'au',
-        shortEnvironment: 'dev',
-        clusterArn: `clusterArn`,
-        subnets: [],
-        taskDefinitionArn: `taskDefinitionArn`,
+      const response = await jobExecutionTaskService.runJobExecutionTask({
+        ...defaultRunJobTaskParams,
+        jobExecution: jobExecution,
+      });
 
-        loggingServiceApiConfig: {
-          credentials: ``,
-          enabled: false,
-          tags: ``,
+      expect(response).toBe(mockArn);
+      expect(mockEcsClient).toHaveReceivedCommandWith(RunTaskCommand, {
+        cluster: 'clusterArn',
+        launchType: 'FARGATE',
+        networkConfiguration: { awsvpcConfiguration: { subnets: ['subnet1'] } },
+        overrides: {
+          containerOverrides: [
+            {
+              name: 'job-runtime-container',
+              environment: [
+                {
+                  name: 'JOB_NAME',
+                  value: jobExecution.jobName,
+                },
+                {
+                  name: 'JOB_EXECUTION_ID',
+                  value: jobExecution.id,
+                },
+                {
+                  name: 'AWS_REGION',
+                  value: defaultRunJobTaskParams.awsRegion,
+                },
+                {
+                  name: 'TENANT_ID',
+                  value: defaultRunJobTaskParams.tenantId,
+                },
+                {
+                  name: 'SHORT_ENVIRONMENT',
+                  value: defaultRunJobTaskParams.shortEnvironment,
+                },
+                {
+                  name: 'SHORT_REGION',
+                  value: defaultRunJobTaskParams.shortRegion,
+                },
+                {
+                  name: 'JOB_CODE_ARTIFACT_URL',
+                  value: jobCodeArtifactSignedUrl,
+                },
+                {
+                  name: 'JOB_DETAILS_FILE_URL',
+                  value: jobDetailsSignedUrl,
+                },
+              ],
+            },
+          ],
         },
-        dynamoTableName: `dynamo`,
-        tenantId: 'some_tenant',
+        tags: [
+          { key: 'Environment', value: defaultRunJobTaskParams.shortEnvironment },
+          { key: 'Region', value: defaultRunJobTaskParams.awsRegion },
+          { key: 'Product', value: 'job-runner' },
+          { key: 'Department', value: 'platform services team' },
+          { key: 'Purpose', value: 'job runner service' },
+          { key: 'map-migrated', value: 'migUJ1IX3DUXG' },
+          { key: 'cmp:tenant', value: 'some_tenant' },
+        ],
+        taskDefinition: defaultRunJobTaskParams.taskDefinitionArn,
       });
-      expect(response).toBe(mockArn);
+      expect(mockJobDetailsRepository.createJobDetailsFile).toHaveBeenCalledWith(
+        `${defaultRunJobTaskParams.tenantId}/${jobExecution.id}`,
+        jobContext,
+        jobManifest,
+        jobExecution,
+      );
+      expect(mockJobUploadRepository.getS3SignedUrl).toHaveBeenCalledWith(
+        `${jobExecution.jobName}/${jobExecution.version}/export.zip`,
+      );
+    });
+
+    it('should run the task if the job manifest and context do not exist', async () => {
+      // We don't allow deleting jobs but contexts can be deleted. Allow the job to be run even if they have been
+      // deleted and error inside of the container so that:
+      //  1. We don't block the queue with potentially broken executions.
+      //  2. Job status/logs more clearly indicate the job is broken rather than potentially remaining queued.
+      const mockArn = `mock-arn`;
+
+      mockEcsClient.on(RunTaskCommand).resolves({ tasks: [{ taskArn: mockArn }] });
+      mockJobContextService.findJobContext.mockResolvedValue(null);
+      mockJobManifestService.findJobManifest.mockResolvedValue(null);
+
+      await jobExecutionTaskService.runJobExecutionTask(defaultRunJobTaskParams);
+
+      expect(mockJobDetailsRepository.createJobDetailsFile).toHaveBeenCalledWith(
+        `${defaultRunJobTaskParams.tenantId}/${defaultRunJobTaskParams.jobExecution.id}`,
+        null,
+        null,
+        defaultRunJobTaskParams.jobExecution,
+      );
     });
   });
 });

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts

@@ -3,21 +3,29 @@
  * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
  */
 
-import { ECSClient, LaunchType, ListTasksCommand, RunTaskCommand } from '@aws-sdk/client-ecs';
-import path from 'path';
-import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
+import { DesiredStatus, ECSClient, LaunchType, ListTasksCommand, RunTaskCommand } from '@aws-sdk/client-ecs';
+import { getAwsTags } from '../../utils';
 import { ListRunningTasksParams, RunJobTaskParams } from './JobExecutionTaskService.defs';
+import { JobDetailsRepository, S3Repository } from '../../S3Repository';
+import { JobExecution } from '../../model/JobExecution';
+import { JobManifestService } from '../JobManifestService/JobManifestService';
+import { JobContextService } from '../JobContextService/JobContextService';
 
 export class JobExecutionTaskService {
-  constructor(protected ecsClient: ECSClient) {}
+  constructor(
+    private readonly ecsClient: ECSClient,
+    private readonly jobManifestService: JobManifestService,
+    private readonly jobContextService: JobContextService,
+    private readonly jobDetailsRepository: JobDetailsRepository,
+    private readonly jobUploadRepository: S3Repository,
+  ) {}
 
   protected async getTenantRunningTasks(params: ListRunningTasksParams): Promise<number> {
     const input = {
-      // ListTasksRequest
       cluster: params.clusterArn,
       family: params.runtimeTaskFamily,
       maxResults: params.maxTenantConcurrentTasks + 1,
-      desiredStatus: JOB_EXECUTION_STATUS.running,
+      desiredStatus: DesiredStatus.RUNNING,
     };
     const command = new ListTasksCommand(input);
     const response = await this.ecsClient.send(command);
@@ -31,6 +39,50 @@ export class JobExecutionTaskService {
   }
 
   public async runJobExecutionTask(params: RunJobTaskParams): Promise<string | undefined> {
+    const jobDetailsFileUrl = await this.createJobDetailsFile(params.tenantId, params.jobExecution);
+    const tags = getAwsTags({
+      tenant: params.tenantId,
+      region: params.awsRegion,
+      environment: params.shortEnvironment,
+    });
+
+    const environmentVariables = [
+      {
+        name: 'JOB_NAME',
+        value: params.jobExecution.jobName,
+      },
+      {
+        name: 'JOB_EXECUTION_ID',
+        value: params.jobExecution.id,
+      },
+      {
+        name: 'AWS_REGION',
+        value: params.awsRegion,
+      },
+      {
+        name: 'TENANT_ID',
+        value: params.tenantId,
+      },
+      {
+        name: 'SHORT_ENVIRONMENT',
+        value: params.shortEnvironment,
+      },
+      {
+        name: 'SHORT_REGION',
+        value: params.shortRegion,
+      },
+      {
+        name: 'JOB_CODE_ARTIFACT_URL',
+        value: await this.jobUploadRepository.getS3SignedUrl(
+          `${params.jobExecution.jobName}/${params.jobExecution.version}/export.zip`,
+        ),
+      },
+      {
+        name: 'JOB_DETAILS_FILE_URL',
+        value: jobDetailsFileUrl,
+      },
+    ];
+
     const result = await this.ecsClient.send(
       new RunTaskCommand({
         cluster: params.clusterArn,
@@ -44,88 +96,17 @@ export class JobExecutionTaskService {
         overrides: {
           containerOverrides: [
             {
-              name: params.containerName,
-              environment: [
-                {
-                  name: 'CONTAINER_NAME',
-                  value: params.containerName,
-                },
-                {
-                  name: 'JOB_NAME',
-                  value: params.jobName,
-                },
-                {
-                  name: 'JOB_EXECUTION_ID',
-                  value: params.jobExecutionId,
-                },
-                {
-                  name: 'JOB_LOCATION_BUCKET',
-                  value: params.bucketName,
-                },
-                {
-                  name: 'JOB_LOCATION_KEY',
-                  value: path.join(params.jobName, params.jobVersion, 'export.zip'),
-                },
-                {
-                  name: 'JOB_RUNNER_API_URL',
-                  value: params.jobRunnerApiUrl,
-                },
-                {
-                  name: 'AWS_REGION',
-                  value: params.awsRegion,
-                },
-                {
-                  name: 'CLUSTER_ARN',
-                  value: params.clusterArn,
-                },
-                {
-                  name: 'SUBNETS_STRING',
-                  value: params.subnets.join('_'),
-                },
-                {
-                  name: 'MAX_TENANT_CONCURRENT_TASKS',
-                  value: params.maxTenantConcurrentTasks.toString(),
-                },
-                {
-                  name: 'RUNTIME_TASK_FAMILY',
-                  value: params.runtimeTaskFamily,
-                },
-                {
-                  name: 'RUNTIME_TASK_DEFINITION_ARN',
-                  value: params.taskDefinitionArn,
-                },
-                {
-                  name: 'LOGGING_SERVICE_API_ENABLED',
-                  value: params.loggingServiceApiConfig.enabled.toString(),
-                },
-                {
-                  name: 'LOGGING_SERVICE_API_TAGS',
-                  value: params.loggingServiceApiConfig.tags,
-                },
-                {
-                  name: 'LOGGING_SERVICE_API_CREDENTIALS',
-                  value: params.loggingServiceApiConfig.credentials,
-                },
-                {
-                  name: 'DYNAMODB_TABLE',
-                  value: params.dynamoTableName,
-                },
-                {
-                  name: 'TENANT_ID',
-                  value: params.tenantId,
-                },
-                {
-                  name: 'SHORT_ENVIRONMENT',
-                  value: params.shortEnvironment,
-                },
-                {
-                  name: 'SHORT_REGION',
-                  value: params.shortRegion,
-                },
-              ],
+              name: 'job-runtime-container',
+              environment: environmentVariables,
             },
           ],
         },
+        tags: [
+          ...Object.entries(tags).map(([key, value]) => ({
+            key,
+            value,
+          })),
+        ],
       }),
     );
 
@@ -135,4 +116,16 @@ export class JobExecutionTaskService {
     const task = result.tasks![0];
     return task.taskArn;
   }
+
+  private async createJobDetailsFile(tenantId: string, executionToStart: JobExecution): Promise<string> {
+    const jobDetailsFileName = JobDetailsRepository.createJobDetailsFileName(tenantId, executionToStart.id);
+    const jobManifest = await this.jobManifestService.findJobManifest(
+      executionToStart.jobName,
+      executionToStart.version,
+    );
+    const jobContext = await this.jobContextService.findJobContext(executionToStart.context);
+
+    await this.jobDetailsRepository.createJobDetailsFile(jobDetailsFileName, jobContext, jobManifest, executionToStart);
+    return await this.jobDetailsRepository.getS3SignedUrl(jobDetailsFileName);
+  }
 }