@squiz/job-runner-lib 1.66.1

package/src/domains/job/repository/JobRepository/JobRepository.ts

@@ -0,0 +1,188 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import {
+  AttributeValue,
+  DynamoDBClient,
+  PutItemCommand,
+  QueryCommand,
+  QueryCommandInput,
+  ScanCommand,
+  ScanCommandOutput,
+} from '@aws-sdk/client-dynamodb';
+import { inject } from 'inversify';
+import { provideTransient } from '../../transientProvider';
+import { Repository } from '../../repository/AbstractRepository';
+import { PARTITION_KEYS } from '../../constants/Repository.constants';
+import { parsePartitionKey } from '../../ParsePartitionKey';
+import { CreateJobRequest, Job, JobDto, ListJobParameters } from '../../model/Job/Job';
+import { PaginationInfo } from '../../core/pagination/model/PaginationInfo';
+import { PaginationQueryResponse } from '../../core/pagination/model/PaginationQueryResponse';
+import { decodeTokenString } from '../../core/pagination/utils/PaginationUtils';
+import { PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/model/PaginationTypes';
+
+@provideTransient(JobRepository)
+export class JobRepository extends Repository<Job, JobDto> {
+  public constructor(@inject(DynamoDBClient) protected dynamoDbClient: DynamoDBClient) {
+    super(dynamoDbClient);
+    this.pk = PARTITION_KEYS.job;
+    this.tableName = process.env.DYNAMODB_TABLE as string;
+  }
+
+  public async create(input: CreateJobRequest): Promise<Job> {
+    const newJob: Job = {
+      ...input,
+      type: this.pk,
+    };
+
+    const dynamoDbRecord = this.convertToDynamoDbItem(newJob);
+
+    await this.dynamoDbClient.send(
+      new PutItemCommand({
+        Item: dynamoDbRecord,
+        TableName: this.tableName,
+      }),
+    );
+
+    return newJob;
+  }
+
+  /**
+   * Get list of records from database.
+   * @abstract
+   * @returns {Promise<Array<T>>} The retrieved records.
+   * @memberof Repository
+   */
+  async list(parameters?: ListJobParameters): Promise<Array<Job>> {
+    const expressionAttributeNames: Record<string, string> = {
+      '#pk': 'pk',
+    };
+    const expressionAttributeValues: Record<string, AttributeValue> = {
+      ':pk': { S: this.pk },
+    };
+
+    const params = {
+      ExpressionAttributeNames: expressionAttributeNames,
+      ExpressionAttributeValues: expressionAttributeValues,
+      FilterExpression: '#pk = :pk',
+      TableName: this.tableName,
+    };
+
+    if (parameters) {
+      const { name, version } = parameters;
+
+      if (name) {
+        params.ExpressionAttributeNames['#name'] = 'name';
+        params.ExpressionAttributeValues[':name'] = { S: name };
+        params.FilterExpression += ` and #name = :name`;
+      }
+
+      if (version) {
+        params.ExpressionAttributeNames['#version'] = 'version';
+        params.ExpressionAttributeValues[':version'] = { S: version };
+        params.FilterExpression += ` and #version = :version`;
+      }
+    }
+
+    // Apply pagination for the scan.
+    // By default, the Scan operation processes data sequentially.
+    // DynamoDB returns data in increments of 1 MB increments.
+
+    let startKey; /* Indicator of where to paginate from. */
+    const results: Array<Record<string, AttributeValue>> = [];
+
+    // Iterate through pages until no more results are found.
+    while (startKey !== null) {
+      const res: ScanCommandOutput = await this.dynamoDbClient.send(
+        new ScanCommand({ ...params, ExclusiveStartKey: startKey }),
+      );
+
+      if (!res) {
+        // Stop paginating if result is unexpected.
+        startKey = null;
+      } else {
+        results.push(
+          ...(res.Items ??
+            /* istanbul ignore next */
+            []),
+        );
+
+        // If no more results, set key to null to terminate loop
+        startKey = res.LastEvaluatedKey ?? null;
+      }
+    }
+
+    return results.length ? results.map((item) => this.convertToItem(item as JobDto)) : [];
+  }
+
+  /**
+   * Get list of records from database.
+   * @abstract
+   * @returns {Promise<Array<T>>} The retrieved records.
+   * @memberof Repository
+   */
+  async getJobs(paginationInfo: PaginationInfo): Promise<PaginationQueryResponse<Job>> {
+    const { token, direction } = paginationInfo;
+
+    const params: QueryCommandInput = {
+      ExpressionAttributeValues: {
+        ':pk': { S: this.pk },
+      },
+      KeyConditionExpression: 'pk = :pk',
+      TableName: this.tableName,
+    };
+
+    if (token) {
+      params.ExclusiveStartKey = decodeTokenString(token);
+    }
+
+    if (direction === PAGINATION_DIRECTION_TYPE_BEFORE) {
+      params.ScanIndexForward = false;
+    }
+
+    const response = await this.dynamoDbClient.send(new QueryCommand(params));
+
+    let items: Array<Job> = [];
+    if (response.Items && response.Items.length) {
+      items = response.Items.map((item) => this.convertToItem(item as JobDto));
+    }
+
+    return {
+      items,
+      lastEvaluatedKey: response.LastEvaluatedKey,
+    };
+  }
+
+  convertToDynamoDbItem(item: Job): JobDto {
+    return {
+      pk: { S: item.type },
+      sk: { S: `${item.name}-${item.version}` },
+      name: { S: item.name },
+      description: item.description ? { S: item.description } : { NULL: true },
+      version: { S: item.version },
+    };
+  }
+
+  convertToItem(dbItem: JobDto): Job {
+    const { pk, name, version } = dbItem;
+    let description;
+    // TODO https://jira.squiz.net/browse/CONN-609 remove this check if we tidy up
+    // jobs added before this field was implemented
+    if (!dbItem.description) {
+      description = { S: null };
+    } else {
+      description = dbItem.description;
+    }
+    return {
+      name: name.S as string,
+      description: description.S ? (description.S as string) : '',
+      type: parsePartitionKey(pk.S as string),
+      version: version.S as string,
+    };
+  }
+}

package/src/domains/job/repository/index.ts

@@ -0,0 +1,5 @@
+export * from './JobContextRepository/JobContextRepository';
+export * from './JobExecutionRepository/JobExecutionRepository';
+export * from './JobManifestRepository/JobManifestRepository';
+export * from './JobRepository/JobRepository';
+export * from './AbstractRepository';

package/src/domains/job/repository/mocks/mockIoc.ts

@@ -0,0 +1,42 @@
+import { Container, decorate, injectable } from 'inversify';
+import { Controller } from 'tsoa';
+import { buildProviderModule } from 'inversify-binding-decorators';
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import color from 'cli-color';
+
+import { Logger } from '@squiz/dx-logger-lib';
+
+// Makes tsoa's Controller injectable
+decorate(injectable(), Controller);
+
+let iocContainer: Container;
+
+function initIocContainer(log: Logger): void {
+  // Create a new container tsoa can use
+  iocContainer = new Container();
+
+  // make inversify aware of inversify-binding-decorators
+  iocContainer.load(buildProviderModule());
+
+  // Init DynamoDB and S3 clients
+  let dynamoDbClient: DynamoDBClient;
+
+  if (process.env.ENVIRONMENT !== 'local') {
+    dynamoDbClient = new DynamoDBClient({ region: 'ap-southeast-2' });
+  } else {
+    // eslint-disable-next-line no-console
+    console.log(`environment: '${color.red(process.env.ENVIRONMENT)}'`);
+
+    // eslint-disable-next-line no-console
+    console.log(`table: '${process.env.DYNAMODB_TABLE}'`);
+    dynamoDbClient = new DynamoDBClient({
+      credentials: { accessKeyId: 'key', secretAccessKey: 'key' },
+      endpoint: 'http://localhost:8000',
+    });
+  }
+
+  iocContainer.bind<DynamoDBClient>(DynamoDBClient).toConstantValue(dynamoDbClient);
+  iocContainer.bind<Logger>('Logger').toConstantValue(log);
+}
+
+export { iocContainer, initIocContainer };

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.defs.ts

@@ -0,0 +1,40 @@
+/**
+ * The required parameters for retrieving the number of running ECS tasks
+ */
+export interface ListRunningTasksParams {
+  clusterArn: string;
+  runtimeTaskFamily: string;
+  maxTenantConcurrentTasks: number;
+}
+
+/**
+ * The parameters required for the logging service configuration.
+ */
+interface LoggingServiceConfig {
+  enabled: boolean;
+  tags: string;
+  credentials: string;
+}
+
+/**
+ * The parameters required for running a job as an ECS task.
+ */
+export interface RunJobTaskParams {
+  jobName: string;
+  jobVersion: string;
+  jobExecutionId: string;
+  awsRegion: string;
+  shortRegion: string;
+  shortEnvironment: string;
+  clusterArn: string;
+  subnets: Array<string>;
+  maxTenantConcurrentTasks: number;
+  runtimeTaskFamily: string;
+  taskDefinitionArn: string;
+  containerName: string;
+  bucketName: string;
+  jobRunnerApiUrl: string;
+  loggingServiceApiConfig: LoggingServiceConfig;
+  dynamoTableName: string;
+  tenantId: string;
+}

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.spec.ts

@@ -0,0 +1,95 @@
+import { ECSClient } from '@aws-sdk/client-ecs';
+import { JobExecutionTaskService } from './JobExecutionTaskService';
+
+describe('JobExecutionTaskService', () => {
+  const mockSend = jest.fn();
+
+  let mockEcsClient: ECSClient;
+  let jobExecutionTaskService: JobExecutionTaskService;
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+
+    mockEcsClient = {
+      send: mockSend,
+    } as unknown as ECSClient;
+
+    jobExecutionTaskService = new JobExecutionTaskService(mockEcsClient);
+  });
+
+  describe(`getTenantRunningTasks`, () => {
+    it('should 0 if no tasks were found', async (): Promise<void> => {
+      mockSend.mockImplementationOnce(() => Promise.resolve({}));
+
+      const response = await (jobExecutionTaskService as any).getTenantRunningTasks({
+        clusterArn: `clusterArn`,
+        runtimeTaskFamily: 'some_family',
+        maxTenantConcurrentTasks: 3,
+      });
+
+      expect(response).toBe(0);
+    });
+  });
+
+  describe(`hasReachedMaxTenantConcurrency`, () => {
+    it('should return false if number of running jobs is less than max', async (): Promise<void> => {
+      mockSend.mockImplementationOnce(() => Promise.resolve({ taskArns: ['12345', '67890'] }));
+
+      const response = await jobExecutionTaskService.hasReachedMaxTenantConcurrency({
+        clusterArn: `clusterArn`,
+        runtimeTaskFamily: 'some_family',
+        maxTenantConcurrentTasks: 3,
+      });
+
+      expect(response).toBe(false);
+    });
+
+    it('should return true if number of running jobs is more than max', async (): Promise<void> => {
+      mockSend.mockImplementationOnce(() => Promise.resolve({ taskArns: ['12345', '67890'] }));
+
+      const response = await jobExecutionTaskService.hasReachedMaxTenantConcurrency({
+        clusterArn: `clusterArn`,
+        runtimeTaskFamily: 'some_family',
+        maxTenantConcurrentTasks: 1,
+      });
+
+      expect(response).toBe(true);
+    });
+  });
+
+  describe(`runJobExecutionTask`, () => {
+    it('should return task arn after starting an ECS task', async (): Promise<void> => {
+      const mockArn = `mock-arn`;
+      const mockTask = { taskArn: mockArn };
+      mockSend.mockImplementationOnce(() => Promise.resolve({ tasks: [mockTask] }));
+
+      const response = await jobExecutionTaskService.runJobExecutionTask({
+        jobExecutionId: `${new Date().toISOString()}-jobExecutionId`,
+        jobName: `jobName`,
+        jobVersion: `jobVersion`,
+        jobRunnerApiUrl: `jobRunnerApiUrl`,
+
+        maxTenantConcurrentTasks: 20,
+        runtimeTaskFamily: 'some_family',
+        bucketName: `bucketName`,
+        containerName: `containerName`,
+
+        awsRegion: 'ap-southeast-2',
+        shortRegion: 'au',
+        shortEnvironment: 'dev',
+        clusterArn: `clusterArn`,
+        subnets: [],
+        taskDefinitionArn: `taskDefinitionArn`,
+
+        loggingServiceApiConfig: {
+          credentials: ``,
+          enabled: false,
+          tags: ``,
+        },
+        dynamoTableName: `dynamo`,
+        tenantId: 'some_tenant',
+      });
+      expect(response).toBe(mockArn);
+    });
+  });
+});

package/src/domains/job/services/JobExecutionTaskService/JobExecutionTaskService.ts

@@ -0,0 +1,138 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { ECSClient, LaunchType, ListTasksCommand, RunTaskCommand } from '@aws-sdk/client-ecs';
+import path from 'path';
+import { JOB_EXECUTION_STATUS } from '../../constants/JobExecutionStatus.constants';
+import { ListRunningTasksParams, RunJobTaskParams } from './JobExecutionTaskService.defs';
+
+export class JobExecutionTaskService {
+  constructor(protected ecsClient: ECSClient) {}
+
+  protected async getTenantRunningTasks(params: ListRunningTasksParams): Promise<number> {
+    const input = {
+      // ListTasksRequest
+      cluster: params.clusterArn,
+      family: params.runtimeTaskFamily,
+      maxResults: params.maxTenantConcurrentTasks + 1,
+      desiredStatus: JOB_EXECUTION_STATUS.running,
+    };
+    const command = new ListTasksCommand(input);
+    const response = await this.ecsClient.send(command);
+    return response.taskArns?.length ?? 0;
+  }
+
+  public async hasReachedMaxTenantConcurrency(params: ListRunningTasksParams): Promise<boolean> {
+    const currentRunningTasks = await this.getTenantRunningTasks(params);
+
+    return currentRunningTasks > params.maxTenantConcurrentTasks;
+  }
+
+  public async runJobExecutionTask(params: RunJobTaskParams): Promise<string | undefined> {
+    const result = await this.ecsClient.send(
+      new RunTaskCommand({
+        cluster: params.clusterArn,
+        networkConfiguration: {
+          awsvpcConfiguration: {
+            subnets: params.subnets,
+          },
+        },
+        launchType: LaunchType.FARGATE,
+        taskDefinition: params.taskDefinitionArn,
+        overrides: {
+          containerOverrides: [
+            {
+              name: params.containerName,
+              environment: [
+                {
+                  name: 'CONTAINER_NAME',
+                  value: params.containerName,
+                },
+                {
+                  name: 'JOB_NAME',
+                  value: params.jobName,
+                },
+                {
+                  name: 'JOB_EXECUTION_ID',
+                  value: params.jobExecutionId,
+                },
+                {
+                  name: 'JOB_LOCATION_BUCKET',
+                  value: params.bucketName,
+                },
+                {
+                  name: 'JOB_LOCATION_KEY',
+                  value: path.join(params.jobName, params.jobVersion, 'export.zip'),
+                },
+                {
+                  name: 'JOB_RUNNER_API_URL',
+                  value: params.jobRunnerApiUrl,
+                },
+                {
+                  name: 'AWS_REGION',
+                  value: params.awsRegion,
+                },
+                {
+                  name: 'CLUSTER_ARN',
+                  value: params.clusterArn,
+                },
+                {
+                  name: 'SUBNETS_STRING',
+                  value: params.subnets.join('_'),
+                },
+                {
+                  name: 'MAX_TENANT_CONCURRENT_TASKS',
+                  value: params.maxTenantConcurrentTasks.toString(),
+                },
+                {
+                  name: 'RUNTIME_TASK_FAMILY',
+                  value: params.runtimeTaskFamily,
+                },
+                {
+                  name: 'RUNTIME_TASK_DEFINITION_ARN',
+                  value: params.taskDefinitionArn,
+                },
+                {
+                  name: 'LOGGING_SERVICE_API_ENABLED',
+                  value: params.loggingServiceApiConfig.enabled.toString(),
+                },
+                {
+                  name: 'LOGGING_SERVICE_API_TAGS',
+                  value: params.loggingServiceApiConfig.tags,
+                },
+                {
+                  name: 'LOGGING_SERVICE_API_CREDENTIALS',
+                  value: params.loggingServiceApiConfig.credentials,
+                },
+                {
+                  name: 'DYNAMODB_TABLE',
+                  value: params.dynamoTableName,
+                },
+                {
+                  name: 'TENANT_ID',
+                  value: params.tenantId,
+                },
+                {
+                  name: 'SHORT_ENVIRONMENT',
+                  value: params.shortEnvironment,
+                },
+                {
+                  name: 'SHORT_REGION',
+                  value: params.shortRegion,
+                },
+              ],
+            },
+          ],
+        },
+      }),
+    );
+
+    // This task ARN can be used to terminate the job task.
+    // If no error is returned, assume task has started.
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const task = result.tasks![0];
+    return task.taskArn;
+  }
+}

package/src/domains/job/services/index.ts

@@ -0,0 +1 @@
+export * from './JobExecutionTaskService/JobExecutionTaskService';

package/src/domains/job/singletonProvider.ts

@@ -0,0 +1,12 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { interfaces } from 'inversify';
+import { fluentProvide } from 'inversify-binding-decorators';
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export const provideSingleton = <T>(identifier: interfaces.ServiceIdentifier<T>): ((target: any) => any) => {
+  return fluentProvide(identifier).inSingletonScope().done();
+};

package/src/domains/job/transientProvider.ts

@@ -0,0 +1,12 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { interfaces } from 'inversify';
+import { fluentProvide } from 'inversify-binding-decorators';
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export const provideTransient = <T>(identifier: interfaces.ServiceIdentifier<T>): ((target: any) => any) => {
+  return fluentProvide(identifier).inTransientScope().done();
+};