@squiz/job-runner-lib 1.66.1

package/src/domains/job/model/JobContext/mocks/JobContextRecord.mock.ts

@@ -0,0 +1,32 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { faker } from '@faker-js/faker/locale/en';
+import { PARTITION_KEYS } from '../../../constants/Repository.constants';
+import { JobContextRecord } from '../JobContext';
+
+export const createMockJobContextRecord = (partialRecord: Partial<JobContextRecord> = {}): JobContextRecord => {
+  const contextName = faker.lorem.word();
+  const numOfEnvVars = faker.number.int({ max: 5, min: 1 });
+  const environment: Record<string, string> = {};
+  const description = faker.lorem.words();
+  for (let i = 0; i < numOfEnvVars; i++) {
+    const key = faker.lorem.word(); // Generate a random key
+    const value = faker.lorem.word(); // Generate a random value
+    environment[key] = value;
+  }
+  return {
+    environment,
+    description,
+    pk: PARTITION_KEYS.context,
+    sk: contextName,
+    ...partialRecord,
+  };
+};
+
+export const createMockJobContextRecords = (size?: number): Array<JobContextRecord> => {
+  const randomSize = size ?? faker.number.int({ max: 3, min: 1 });
+  return Array.from({ length: randomSize }).map(() => createMockJobContextRecord());
+};

package/src/domains/job/model/JobContext/mocks/JobContextRequest.mock.ts

@@ -0,0 +1,26 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { faker } from '@faker-js/faker/locale/en';
+import { CreateJobContextRequest } from '../JobContext';
+
+export const createMockCreateJobContextRequest = (
+  partialRecord: Partial<CreateJobContextRequest> = {},
+): CreateJobContextRequest => {
+  const numOfEnvVars = faker.number.int({ max: 5, min: 1 });
+  const environment: Record<string, string> = {};
+
+  for (let i = 0; i < numOfEnvVars; i++) {
+    const key = faker.lorem.word(); // Generate a random key
+    const value = faker.lorem.word(); // Generate a random value
+    environment[`${key}`] = value;
+  }
+  return {
+    contextName: faker.lorem.word(),
+    environment,
+    description: faker.lorem.words(),
+    ...partialRecord,
+  };
+};

package/src/domains/job/model/JobExecution/JobExecution.ts

@@ -0,0 +1,220 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { JobExecutionStatus } from '../../constants/JobExecutionStatus.constants';
+import { JobExecutionPk } from '../../constants/Repository.constants';
+import { AttributeValue } from '@aws-sdk/client-dynamodb';
+
+/**
+ * JobExecution interface.
+ * @export
+ * @interface JobExecution
+ */
+export interface JobExecution {
+  /**
+   * The id of the job execution.
+   * @type {string}
+   * @memberof JobExecution
+   */
+  id: string;
+
+  /**
+   * The name of the job.
+   * @type {string}
+   * @memberof JobExecution
+   */
+  jobName: string;
+
+  /**
+   * The id of the context to run the job against.
+   * @type {string}
+   * @memberof JobExecution
+   */
+  context: string;
+
+  /**
+   * The status of the job run.
+   * @type {string}
+   * @memberof JobExecution
+   */
+  status: JobExecutionStatus;
+
+  /**
+   * The time the job execution was added to the queue.
+   * @type {string}
+   * @memberof JobExecution
+   */
+  timeQueued: string;
+
+  /**
+   * The version of the job script to execute.
+   * @type {(string)}
+   * @memberof JobExecution
+   */
+  version: string;
+
+  /**
+   * The ECS task ID of the job run.
+   * @type {(string | null)}
+   * @memberof JobExecution
+   */
+  ecsTaskId: string | null;
+
+  /**
+   * The time the job run was started.
+   * @type {(string | null)}
+   * @memberof JobExecution
+   */
+  timeStarted: string | null;
+
+  /**
+   * The time the job run was finished.
+   * @type {(string | null)}
+   * @memberof JobExecution
+   */
+  timeFinished: string | null;
+
+  /**
+   * The input to the job.
+   * @type {any}
+   * @memberof JobExecution
+   */
+  input: any;
+
+  /**
+   * The DynamoDB record type (dynamic, 'jobExecution-<jobName>').
+   * @type {JobExecutionPk}
+   * @memberof JobExecution
+   */
+  type: JobExecutionPk;
+}
+
+/**
+ * The JobExecution record attributes stored in DynamoDB.
+ * @export
+ * @interface JobExecutionRecord
+ */
+export interface JobExecutionRecord {
+  /**
+   * The partition key (dynamic, 'jobExecution-<jobName>').
+   * @type {JobExecutionPk}
+   * @memberof JobExecutionRecord
+   */
+  pk: JobExecutionPk;
+
+  /**
+   * The sort key ('timeQueued-id').
+   * @type {string}
+   * @memberof JobExecutionRecord
+   */
+  sk: string;
+
+  /**
+   * The name of the job to execute.
+   * @type {string}
+   * @memberof JobExecutionRecord
+   */
+  jobName: string;
+
+  /**
+   * The id of the job execution.
+   * @type {string}
+   * @memberof JobExecutionRecord
+   */
+  id: string;
+
+  /**
+   * The time the job execution was added to the queue.
+   * @type {string}
+   * @memberof JobExecutionRecord
+   */
+  timeQueued: string;
+
+  /**
+   * The id of the context to run the job against.
+   * @type {string}
+   * @memberof JobExecutionRecord
+   */
+  context: string;
+
+  /**
+   * The status of the job run.
+   * @type {JobExecutionStatus}
+   * @memberof JobExecutionRecord
+   */
+  status: JobExecutionStatus;
+
+  /**
+   * The version of the job script to execute.
+   * @type @type {(string)}
+   * @memberof JobExecutionRecord
+   */
+  version: string;
+
+  /**
+   * The ECS task ID of the job run.
+   * @type @type {(string | null)}
+   * @memberof JobExecutionRecord
+   */
+  ecsTaskId: string | null;
+
+  /**
+   * The time the job run was started.
+   * @type @type {(string | null)}
+   * @memberof JobExecutionRecord
+   */
+  timeStarted: string | null;
+
+  /**
+   * The time the job run was finished.
+   * @type {(string | null)}
+   * @memberof JobExecutionRecord
+   */
+  timeFinished: string | null;
+
+  /**
+   * The input to the job.
+   * @type {any}
+   * @memberof JobExecutionRecord
+   */
+  input: any;
+}
+
+export type JobExecutionDto = Record<keyof JobExecutionRecord, AttributeValue>;
+
+/**
+ * The request body for adding an item to the job queue (POST /job/begin).
+ * @export
+ */
+export type BeginJobExecutionRequest = Pick<JobExecution, 'jobName' | 'context'> & Partial<Pick<JobExecution, 'input'>>;
+
+/**
+ * The parameters for creating a job execution entity in the DB.
+ * @export
+ */
+export type CreateJobExecutionRequest = Pick<JobExecution, 'jobName' | 'version' | 'context'> &
+  Partial<Pick<JobExecution, 'input'>>;
+
+/**
+ * The request body for terminating a job in the job queue (POST /job/terminate).
+ * @export
+ */
+export interface TerminateJobExecutionRequest {
+  jobName: string;
+  executionId: string;
+  reason: string;
+}
+
+/**
+ * The parameters for listing job executions.
+ * @export
+ */
+export type ListJobExecutionParameters = Partial<Pick<JobExecution, 'jobName' | 'status'>>;
+
+/**
+ * The parameters for updating job executions.
+ * @export
+ */
+export type UpdateJobExecutionParameters = Partial<Omit<JobExecution, 'id' | 'jobName' | 'timeQueued' | 'type'>>;

package/src/domains/job/model/JobExecution/index.ts

@@ -0,0 +1,5 @@
+export * from './JobExecution';
+export * from './mocks/JobExecution.mock';
+export * from './mocks/JobExecutionDto.mock';
+export * from './mocks/JobExecutionRecord.mock';
+export * from './mocks/JobExecutionRequest.mock';

package/src/domains/job/model/JobExecution/mocks/JobExecution.mock.ts

@@ -0,0 +1,43 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
+import { PARTITION_KEYS } from '../../../constants/Repository.constants';
+import { faker } from '@faker-js/faker/locale/en';
+import { JobExecution } from '../JobExecution';
+
+export const createMockJobExecution = (partialRecord: Partial<JobExecution> = {}): JobExecution => {
+  const version = Array.from({ length: 3 })
+    .map(() => faker.number.int())
+    .join('.');
+  const input = faker.helpers.arrayElement([
+    faker.datatype.boolean(),
+    faker.lorem.word(),
+    faker.number.int(),
+    { [faker.lorem.word()]: faker.lorem.words() },
+    null,
+  ]);
+  const jobName = faker.lorem.word();
+  const timeQueued = new Date().toISOString();
+  return {
+    context: faker.string.uuid(),
+    ecsTaskId: null,
+    id: `${timeQueued}-${faker.string.uuid()}`,
+    jobName,
+    status: JOB_EXECUTION_STATUS.queued,
+    timeFinished: null,
+    timeQueued,
+    timeStarted: null,
+    type: `${PARTITION_KEYS.jobExecution}-${jobName}`,
+    version,
+    input,
+    ...partialRecord,
+  };
+};
+
+export const createMockJobExecutions = (size?: number): Array<JobExecution> => {
+  const randomSize = size ?? faker.number.int({ max: 3, min: 1 });
+  return Array.from({ length: randomSize }).map(() => createMockJobExecution());
+};

package/src/domains/job/model/JobExecution/mocks/JobExecutionDto.mock.ts

@@ -0,0 +1,62 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
+import { PARTITION_KEYS } from '../../../constants/Repository.constants';
+import { faker } from '@faker-js/faker/locale/en';
+import { JobExecutionDto, JobExecutionRecord } from '../JobExecution';
+
+export const createMockJobExecutionDto = (partialRecord: Partial<JobExecutionRecord> = {}): JobExecutionDto => {
+  const jobName = faker.lorem.word();
+  const timeQueued = faker.date.recent().toISOString();
+  const id = `${timeQueued}-${faker.string.uuid()}`;
+  const version = Array.from({ length: 3 })
+    .map(() => faker.number.int())
+    .join('.');
+  const input = faker.helpers.arrayElement([
+    faker.datatype.boolean(),
+    faker.lorem.word(),
+    faker.number.int(),
+    { [faker.lorem.word()]: faker.lorem.words },
+    null,
+    undefined,
+  ]);
+
+  return {
+    context: {
+      S: partialRecord.context ?? faker.string.uuid(),
+    },
+    ecsTaskId: partialRecord.ecsTaskId ? { S: partialRecord.ecsTaskId } : { NULL: true },
+    id: {
+      S: partialRecord.id ?? id,
+    },
+    jobName: {
+      S: partialRecord.jobName ?? jobName,
+    },
+    pk: {
+      S: partialRecord.pk ?? `${PARTITION_KEYS.jobExecution}-${partialRecord.jobName ?? jobName}`,
+    },
+    sk: {
+      S: partialRecord.sk ?? `${timeQueued}-${id}`,
+    },
+    status: {
+      S: partialRecord.status ?? faker.helpers.arrayElement(Object.values(JOB_EXECUTION_STATUS)),
+    },
+    timeFinished: partialRecord.timeFinished ? { S: partialRecord.timeFinished } : { NULL: true },
+    timeQueued: {
+      S: partialRecord.timeQueued ?? timeQueued,
+    },
+    timeStarted: partialRecord.timeStarted ? { S: partialRecord.timeStarted } : { NULL: true },
+    version: {
+      S: partialRecord.version ?? version,
+    },
+    input: input ? { S: JSON.stringify(input) } : { NULL: true },
+  };
+};
+
+export const createMockJobExecutionDtos = (size?: number): Array<JobExecutionDto> => {
+  const randomSize = size ?? faker.number.int({ max: 3, min: 1 });
+  return Array.from({ length: randomSize }).map(() => createMockJobExecutionDto());
+};

package/src/domains/job/model/JobExecution/mocks/JobExecutionRecord.mock.ts

@@ -0,0 +1,47 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { JOB_EXECUTION_STATUS } from '../../../constants/JobExecutionStatus.constants';
+import { PARTITION_KEYS } from '../../../constants/Repository.constants';
+import { faker } from '@faker-js/faker/locale/en';
+import { JobExecutionRecord } from '../JobExecution';
+
+export const createMockJobExecutionRecord = (partialRecord: Partial<JobExecutionRecord> = {}): JobExecutionRecord => {
+  const jobName = faker.lorem.word();
+  const id = faker.string.uuid();
+  const timeQueued = faker.date.recent().toISOString();
+  const version = Array.from({ length: 3 })
+    .map(() => faker.number.int())
+    .join('.');
+  const input = faker.helpers.arrayElement([
+    faker.datatype.boolean(),
+    faker.lorem.word(),
+    faker.number.int(),
+    { [faker.lorem.word()]: faker.lorem.words },
+    null,
+    undefined,
+  ]);
+
+  return {
+    context: faker.string.uuid(),
+    ecsTaskId: null,
+    id: id,
+    jobName: jobName,
+    pk: `${PARTITION_KEYS.jobExecution}-${jobName}`,
+    sk: `${timeQueued}-${id}`,
+    status: faker.helpers.objectValue(JOB_EXECUTION_STATUS),
+    timeFinished: null,
+    timeQueued: timeQueued,
+    timeStarted: null,
+    version,
+    input,
+    ...partialRecord,
+  };
+};
+
+export const createMockJobExecutionRecords = (size?: number): Array<JobExecutionRecord> => {
+  const randomSize = size ?? faker.number.int({ max: 3, min: 1 });
+  return Array.from({ length: randomSize }).map(() => createMockJobExecutionRecord());
+};

package/src/domains/job/model/JobExecution/mocks/JobExecutionRequest.mock.ts

@@ -0,0 +1,26 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { faker } from '@faker-js/faker/locale/en';
+import { BeginJobExecutionRequest } from '../JobExecution';
+
+export const createMockBeginJobExecutionRequest = (
+  partialRecord: Partial<BeginJobExecutionRequest> = {},
+): BeginJobExecutionRequest => {
+  const input = faker.helpers.arrayElement([
+    faker.datatype.boolean(),
+    faker.lorem.word(),
+    faker.number.int(),
+    { [faker.lorem.word()]: faker.lorem.words() },
+    null,
+  ]);
+
+  return {
+    context: faker.string.uuid(),
+    jobName: faker.lorem.word(),
+    input,
+    ...partialRecord,
+  };
+};

package/src/domains/job/repository/AbstractRepository.ts

@@ -0,0 +1,219 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+import { injectable } from 'inversify';
+import 'reflect-metadata';
+import {
+  AttributeValue,
+  DeleteItemCommand,
+  DeleteItemCommandInput,
+  DynamoDBClient,
+  GetItemCommand,
+  GetItemCommandInput,
+  PutItemCommand,
+  ScanCommand,
+  ScanCommandInput,
+  ScanCommandOutput,
+  UpdateItemCommand,
+  UpdateItemCommandInput,
+} from '@aws-sdk/client-dynamodb';
+import { parsePartitionKey } from '../ParsePartitionKey';
+import { PartitionKey } from '../constants/Repository.constants';
+import { BaseDto, BaseItem } from '../../job/model/Base/Base';
+
+/**
+ * Generic abstract class for representing repositories interacting with DynamoDB.
+ * @export
+ * @abstract
+ * @class Repository
+ * @template T The entity definition.
+ * @template U The corresponding DB record attributes for the entity.
+ */
+@injectable()
+/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+export abstract class Repository<T extends BaseItem, U extends BaseDto> {
+  protected pk: PartitionKey;
+  protected tableName: string;
+
+  public constructor(protected dynamoDbClient: DynamoDBClient) {
+    // HACK: workaround to address inheritance issue in InversifyJS
+    // Reference: https://github.com/inversify/InversifyJS/blob/master/wiki/inheritance.md
+
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore @ts-2322
+    this.pk = null;
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore @ts-2322
+    this.tableName = null;
+  }
+
+  /**
+   * Save item into database.
+   * @abstract
+   * @param {*} input The item to store.
+   * @returns {Promise<T>} The item stored.
+   * @memberof Repository
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async create(input: T): Promise<T> {
+    const dynamoDbRecord = this.convertToDynamoDbItem(input);
+
+    await this.dynamoDbClient.send(
+      new PutItemCommand({
+        Item: dynamoDbRecord,
+        TableName: this.tableName,
+      }),
+    );
+
+    return input;
+  }
+
+  /**
+   * Get one record from database.
+   * @abstract
+   * @param {string} sk The sort key used to identify the record.
+   * @returns {Promise<T | null>} The retrieved record.
+   * @memberof Repository
+   */
+  async findOne(sk: string): Promise<T | null> {
+    const params: GetItemCommandInput = {
+      Key: {
+        pk: { S: parsePartitionKey(this.pk) },
+        sk: { S: sk },
+      },
+      TableName: this.tableName,
+    };
+    const res = await this.dynamoDbClient.send(new GetItemCommand(params));
+    return res.Item ? this.convertToItem(res.Item as U) : null;
+  }
+
+  /**
+   * Get list of records from database.
+   * @abstract
+   * @returns {Promise<Array<T>>} The retrieved records.
+   * @memberof Repository
+   */
+  async list(): Promise<Array<T>> {
+    const params: ScanCommandInput = {
+      ExpressionAttributeNames: {
+        '#pk': 'pk',
+      },
+      ExpressionAttributeValues: {
+        ':pk': { S: this.pk },
+      },
+      FilterExpression: '#pk = :pk',
+      TableName: this.tableName,
+    };
+
+    // Apply pagination for the scan.
+    // By default, the Scan operation processes data sequentially.
+    // DynamoDB returns data in increments of 1 MB increments.
+
+    let startKey; /* Indicator of where to paginate from. */
+    const results: Array<Record<string, AttributeValue>> = [];
+
+    // Iterate through pages until no more results are found.
+    while (startKey !== null) {
+      const res: ScanCommandOutput = await this.dynamoDbClient.send(
+        new ScanCommand({ ...params, ExclusiveStartKey: startKey }),
+      );
+
+      if (!res) {
+        // Stop paginating if result is unexpected.
+        startKey = null;
+      } else {
+        results.push(...(res.Items ?? []));
+
+        // If no more results, set key to null to terminate loop
+        startKey = res.LastEvaluatedKey ?? null;
+      }
+    }
+
+    return results.length ? results.map((item) => this.convertToItem(item as U)) : [];
+  }
+
+  /**
+   * Update record in database.
+   * @param {string} sk The sort key used to identify the record.
+   * @param {Partial<T>} input The (partial) record to update.
+   * @returns {Promise<T | null>} The updated record.
+   * @memberof Repository
+   */
+  async update(sk: string, input: Partial<T>): Promise<T | null> {
+    // Initialise update command
+    const updateParams: UpdateItemCommandInput = {
+      Key: {
+        pk: { S: this.pk },
+        sk: { S: sk },
+      },
+      ReturnValues: 'ALL_NEW',
+      TableName: this.tableName,
+    };
+
+    const expressionAttributeNames: Record<string, string> = {};
+    const expressionAttributeValues: Record<string, AttributeValue> = {};
+
+    // To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
+    for (const key of Object.keys(input)) {
+      if (input[key as keyof Partial<T>] === null) {
+        expressionAttributeNames[`#${key}`] = key;
+        expressionAttributeValues[`:${key}`] = { NULL: true };
+      } else {
+        const val: string = input[key as keyof Partial<T>] as string;
+        expressionAttributeNames[`#${key}`] = key;
+        expressionAttributeValues[`:${key}`] = { S: val };
+      }
+    }
+
+    const updateExpressionPairs = Object.keys(input).map((key) => `#${key} = :${key}`);
+    const updateExpression = `SET ${updateExpressionPairs.join(', ')}`;
+
+    updateParams.ExpressionAttributeNames = expressionAttributeNames;
+    updateParams.ExpressionAttributeValues = expressionAttributeValues;
+    updateParams.UpdateExpression = updateExpression;
+
+    const updateItemCommand = new UpdateItemCommand(updateParams);
+    const updatedRecord = await this.dynamoDbClient.send(updateItemCommand);
+
+    return updatedRecord.Attributes ? this.convertToItem(updatedRecord.Attributes as U) : null;
+  }
+
+  /**
+   * Delete record from database.
+   * @abstract
+   * @param {string} sk The sort key used to identify the record.
+   * @returns {void}
+   * @memberof Repository
+   */
+  async delete(sk: string): Promise<void> {
+    const deleteCommandParams: DeleteItemCommandInput = {
+      Key: {
+        pk: { S: this.pk },
+        sk: { S: sk },
+      },
+      TableName: this.tableName,
+    };
+
+    await this.dynamoDbClient.send(new DeleteItemCommand(deleteCommandParams));
+    return;
+  }
+
+  /**
+   * Convert the DynanomDB attributes of the entity into a DynamoDB item.
+   * @abstract
+   * @param {U} attributes The DynamoDB item attributes to convert.
+   * @returns {Record<keyof U, AttributeValue>} The DynamoDB record.
+   * @memberof Repository
+   */
+  protected abstract convertToDynamoDbItem(item: T): Record<keyof U, AttributeValue>;
+
+  /**
+   * Convert DynamoDB item to entity.
+   * @abstract
+   * @param {Record<string, AttributeValue>} record The record to convert.
+   * @returns {T} The converted entity.
+   * @memberof Repository
+   */
+  protected abstract convertToItem(record: U): T;
+}