@squiz/job-runner-lib 1.66.1

package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.spec.ts

@@ -0,0 +1,222 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import {
+  DeleteItemCommandOutput,
+  DynamoDBClient,
+  DynamoDBClientConfig,
+  GetItemCommandOutput,
+  PutItemCommandOutput,
+  ScanCommandOutput,
+} from '@aws-sdk/client-dynamodb';
+import { initIocContainer, iocContainer } from '../mocks/mockIoc';
+import { testLogger } from '../../../../jest-global-setup';
+import {
+  createMockClientPutResponse,
+  createMockClientScanResponse,
+  createMockClientGetResponse,
+} from '../../core/dynamoClient.mock';
+import { PARTITION_KEYS } from '../../constants/Repository.constants';
+import { JobManifest } from '../../manifest/JobManifestTypes';
+import { createMockJobManifest } from '../../manifest/mocks/JobManifest.mock';
+import { createMockJobManifestDto, createMockJobManifestDtos } from '../../manifest/mocks/JobManifestDto.mock';
+import { createMockJobManifestRecord } from '../../manifest/mocks/JobManifestRecord.mock';
+import { createMockCreateJobManifestRequest } from '../../manifest/mocks/JobManifestRequest.mock';
+import { JobManifestRepository } from './JobManifestRepository';
+
+// Create a new container tsoa can use
+initIocContainer(testLogger);
+
+// Mock DynamoDB Client
+const mockOptions = {} as DynamoDBClientConfig;
+const mockClient = new DynamoDBClient(mockOptions);
+const mockSend = jest.fn();
+mockClient.send = mockSend;
+
+// Inject JobManifestRepository with a mocked client
+iocContainer.rebind<DynamoDBClient>(DynamoDBClient).toConstantValue(mockClient);
+
+let jobManifestRepository: JobManifestRepository;
+
+describe('JobManifestRepository', (): void => {
+  beforeAll(() => {
+    jobManifestRepository = iocContainer.get(JobManifestRepository);
+  });
+
+  describe('find', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return an empty list`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
+
+      const response = await jobManifestRepository.list();
+      expect(mockSend).toBeCalled();
+      expect(response).toStrictEqual([]);
+    });
+
+    it(`should return a list of existing job manifests`, async (): Promise<void> => {
+      const mockManifestItems = createMockJobManifestDtos();
+
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: mockManifestItems,
+            LastEvaluatedKey: undefined,
+          }),
+      );
+
+      const response = await jobManifestRepository.list();
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toHaveLength(mockManifestItems.length);
+    });
+
+    it(`should paginate results to return a list of existing job manifests`, async (): Promise<void> => {
+      const mockManifestItemsPage1 = createMockJobManifestDtos();
+      const mockManifestItemsPage2 = createMockJobManifestDtos();
+
+      mockSend
+        .mockImplementationOnce(
+          (): ScanCommandOutput =>
+            createMockClientScanResponse({
+              Items: mockManifestItemsPage1,
+              LastEvaluatedKey: mockManifestItemsPage1.slice(-1)[0],
+            }),
+        )
+        .mockImplementationOnce(
+          (): ScanCommandOutput =>
+            createMockClientScanResponse({
+              Items: mockManifestItemsPage2,
+              LastEvaluatedKey: undefined,
+            }),
+        );
+
+      const response = await jobManifestRepository.list();
+      expect(mockSend).toBeCalledTimes(2);
+      expect(response).toHaveLength(mockManifestItemsPage1.length + mockManifestItemsPage2.length);
+    });
+  });
+
+  describe('findOne', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return job manifest`, async (): Promise<void> => {
+      const mockJobManifestItem = createMockJobManifestDto();
+      mockSend.mockImplementationOnce(
+        (): GetItemCommandOutput =>
+          createMockClientGetResponse({
+            Item: mockJobManifestItem,
+          }),
+      );
+
+      const response = await jobManifestRepository.findOne(
+        `${mockJobManifestItem.name.S}-${mockJobManifestItem.version.S}` as string,
+      );
+      expect(response).toStrictEqual({
+        concurrency: parseInt(mockJobManifestItem.concurrency.N as string),
+        entryFile: mockJobManifestItem.entryFile.S as string,
+        name: mockJobManifestItem.name.S as string,
+        version: mockJobManifestItem.version.S as string,
+        entryFunction: JSON.parse(mockJobManifestItem.entryFunction.S as string),
+        timeoutSeconds: parseInt(mockJobManifestItem.timeoutSeconds.N as string),
+        type: `manifest`,
+      } as JobManifest);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+
+    it(`should return null if job manifest does not exist`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: [],
+          }),
+      );
+
+      const response = await jobManifestRepository.findOne(`null`);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toBe(null);
+    });
+  });
+
+  describe('create', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should create a new job manifest record and return job manifest in the response`, async (): Promise<void> => {
+      const newJobManifestToCreate = createMockCreateJobManifestRequest();
+      const mockReponse = createMockClientPutResponse({
+        Attributes: createMockJobManifestDto({
+          pk: PARTITION_KEYS.manifest,
+          sk: newJobManifestToCreate.name,
+        }),
+      });
+
+      mockSend.mockImplementationOnce((): PutItemCommandOutput => mockReponse);
+
+      const response = await jobManifestRepository.create(newJobManifestToCreate);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toStrictEqual({
+        ...newJobManifestToCreate,
+        type: PARTITION_KEYS.manifest,
+      });
+    });
+  });
+
+  describe('update', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should update a job manifest record`, async (): Promise<void> => {
+      const jobManifestToUpdate = createMockJobManifest();
+      const updatedJobManifest: JobManifest = {
+        ...jobManifestToUpdate,
+        concurrency: 10,
+        timeoutSeconds: 100,
+      };
+      const updatedJobManifestRecord = createMockJobManifestRecord({
+        ...updatedJobManifest,
+        sk: `${updatedJobManifest.name}-${updatedJobManifest.version}`,
+      });
+      const updatedJobManifestRecordDto = createMockJobManifestDto({
+        ...updatedJobManifestRecord,
+      });
+      const putResponse = createMockClientPutResponse({
+        Attributes: { ...updatedJobManifestRecordDto },
+      });
+
+      mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
+
+      const response = await jobManifestRepository.update(
+        `${jobManifestToUpdate.name}-${jobManifestToUpdate.version}`,
+        updatedJobManifestRecord,
+      );
+      expect(response).toStrictEqual(updatedJobManifest);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+  });
+
+  describe('delete', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
+        return {
+          $metadata: {},
+        };
+      });
+
+      const response = await jobManifestRepository.delete(`name`);
+      expect(response).toBe(undefined);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+  });
+});

package/src/domains/job/repository/JobManifestRepository/JobManifestRepository.ts

@@ -0,0 +1,120 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import {
+  AttributeValue,
+  DynamoDBClient,
+  PutItemCommand,
+  UpdateItemCommand,
+  UpdateItemCommandInput,
+} from '@aws-sdk/client-dynamodb';
+import { inject } from 'inversify';
+import { provideTransient } from '../../transientProvider';
+import { Repository } from '../AbstractRepository';
+import { PARTITION_KEYS } from '../../constants/Repository.constants';
+import { JobManifest } from '../../manifest/JobManifestTypes';
+import { JobManifestDto, CreateJobManifestRequest } from '../../manifest';
+import { parsePartitionKey } from '../../ParsePartitionKey';
+
+@provideTransient(JobManifestRepository)
+export class JobManifestRepository extends Repository<JobManifest, JobManifestDto> {
+  public constructor(@inject(DynamoDBClient) protected dynamoDbClient: DynamoDBClient) {
+    super(dynamoDbClient);
+    this.pk = PARTITION_KEYS.manifest;
+    this.tableName = process.env.DYNAMODB_TABLE as string;
+  }
+
+  public async create(input: CreateJobManifestRequest): Promise<JobManifest> {
+    const newJobManifest: JobManifest = {
+      ...input,
+      type: this.pk,
+    };
+
+    const dynamoDbRecord = this.convertToDynamoDbItem(newJobManifest);
+
+    await this.dynamoDbClient.send(
+      new PutItemCommand({
+        Item: dynamoDbRecord,
+        TableName: this.tableName,
+      }),
+    );
+
+    return newJobManifest;
+  }
+
+  async update(sk: string, input: Partial<JobManifest>): Promise<JobManifest | null> {
+    // Initialise update command
+    const updateParams: UpdateItemCommandInput = {
+      Key: {
+        pk: { S: this.pk },
+        sk: { S: sk },
+      },
+      ReturnValues: 'ALL_NEW',
+      TableName: this.tableName,
+    };
+
+    const expressionAttributeNames: Record<string, string> = {};
+    const expressionAttributeValues: Record<string, AttributeValue> = {};
+
+    // To avoid conflict with reserved keywords, refer to attributes with an additional '#' prefix
+
+    if (input.concurrency) {
+      expressionAttributeNames['#concurrency'] = 'concurrency';
+      expressionAttributeValues[':concurrency'] = { N: input.concurrency.toString() };
+    }
+
+    if (input.entryFile) {
+      expressionAttributeNames['#entryFile'] = 'entryFile';
+      expressionAttributeValues[':entryFile'] = { S: input.entryFile };
+    }
+
+    if (input.timeoutSeconds) {
+      expressionAttributeNames['#timeoutSeconds'] = 'timeoutSeconds';
+      expressionAttributeValues[':timeoutSeconds'] = { N: input.timeoutSeconds.toString() };
+    }
+
+    const updateExpressionPairs = Object.keys(input).map((key) => `#${key} = :${key}`);
+    const updateExpression = `SET ${updateExpressionPairs.join(', ')}`;
+
+    updateParams.ExpressionAttributeNames = expressionAttributeNames;
+    updateParams.ExpressionAttributeValues = expressionAttributeValues;
+    updateParams.UpdateExpression = updateExpression;
+
+    const updateItemCommand = new UpdateItemCommand(updateParams);
+    const updatedJob = await this.dynamoDbClient.send(updateItemCommand);
+
+    /* istanbul ignore next */
+    return updatedJob.Attributes ? this.convertToItem(updatedJob.Attributes as JobManifestDto) : null;
+  }
+
+  convertToDynamoDbItem(item: JobManifest): JobManifestDto {
+    return {
+      concurrency: { N: item.concurrency.toString() },
+      entryFile: { S: item.entryFile },
+      entryFunction: { S: JSON.stringify(item.entryFunction) },
+      name: { S: item.name },
+      version: { S: item.version },
+      pk: { S: item.type },
+      sk: { S: `${item.name}-${item.version}` },
+      timeoutSeconds: { N: item.timeoutSeconds.toString() },
+    };
+  }
+
+  convertToItem(dbItem: JobManifestDto): JobManifest {
+    const { pk, name, version, concurrency, entryFile, entryFunction, timeoutSeconds } = dbItem;
+    return {
+      concurrency: parseInt(concurrency.N as string),
+      entryFile: entryFile.S as string,
+      entryFunction: JSON.parse(entryFunction.S as string),
+      name: name.S as string,
+      version: version.S as string,
+      timeoutSeconds: parseInt(timeoutSeconds.N as string),
+      type: parsePartitionKey(pk.S as string),
+    };
+  }
+}

package/src/domains/job/repository/JobRepository/JobRepository.spec.ts

@@ -0,0 +1,337 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import {
+  AttributeValue,
+  DynamoDBClient,
+  DynamoDBClientConfig,
+  GetItemCommandOutput,
+  PutItemCommandOutput,
+  QueryCommandInput,
+  ScanCommandOutput,
+} from '@aws-sdk/client-dynamodb';
+import { initIocContainer, iocContainer } from '../mocks/mockIoc';
+import { testLogger } from '../../../../jest-global-setup';
+import {
+  createMockClientPutResponse,
+  createMockClientScanResponse,
+  createMockClientGetResponse,
+} from '../../core/dynamoClient.mock';
+import { PARTITION_KEYS } from '../../constants/Repository.constants';
+import { PaginationInfo } from '../../core/pagination/model/PaginationInfo';
+import { PAGINATION_DIRECTION_TYPE_BEFORE } from '../../core/pagination/model/PaginationTypes';
+import * as PaginationUtils from '../../core/pagination/utils/PaginationUtils';
+import { Job } from '../../model/Job/Job';
+import { createMockJobDto, createMockJobDtos } from '../../model/Job/mocks/JobDto.mock';
+import { createMockCreateJobRequest } from '../../model/Job/mocks/JobRequest.mock';
+import { JobRepository } from './JobRepository';
+// Create a new container tsoa can use
+initIocContainer(testLogger);
+
+// Mock DynamoDB Client
+const mockOptions = {} as DynamoDBClientConfig;
+const mockClient = new DynamoDBClient(mockOptions);
+const mockSend = jest.fn();
+mockClient.send = mockSend;
+
+// Inject JobRepository with a mocked client
+iocContainer.rebind<DynamoDBClient>(DynamoDBClient).toConstantValue(mockClient);
+
+let jobRepository: JobRepository;
+
+describe('JobRepository', (): void => {
+  beforeAll(() => {
+    jobRepository = iocContainer.get(JobRepository);
+  });
+
+  describe('list', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return an empty list`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce((): ScanCommandOutput => createMockClientScanResponse());
+
+      const response = await jobRepository.list();
+      expect(mockSend).toBeCalled();
+      expect(response).toStrictEqual([]);
+    });
+
+    it(`should return a list of existing jobs`, async (): Promise<void> => {
+      const mockRecords = createMockJobDtos();
+
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: mockRecords,
+            LastEvaluatedKey: undefined,
+          }),
+      );
+
+      const response = await jobRepository.list();
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toHaveLength(mockRecords.length);
+    });
+
+    it(`should return a list of existing jobs by name`, async (): Promise<void> => {
+      const mockName = `mock`;
+      const mockRecords = [createMockJobDto({ name: mockName }), createMockJobDto({ name: mockName })];
+
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: mockRecords,
+            LastEvaluatedKey: undefined,
+          }),
+      );
+
+      const response = await jobRepository.list({ name: mockName });
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toHaveLength(mockRecords.length);
+    });
+
+    it(`should return a list of existing jobs by version`, async (): Promise<void> => {
+      const mockVersion = `1.0.1`;
+      const mockRecords = [createMockJobDto({ version: mockVersion }), createMockJobDto({ version: mockVersion })];
+
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: mockRecords,
+            LastEvaluatedKey: undefined,
+          }),
+      );
+
+      const response = await jobRepository.list({ version: mockVersion });
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toHaveLength(mockRecords.length);
+    });
+
+    it(`should paginate results to return a list of existing jobs`, async (): Promise<void> => {
+      const mockRecordsPage1 = createMockJobDtos();
+      const mockRecordsPage2 = createMockJobDtos();
+
+      mockSend
+        .mockImplementationOnce(
+          (): ScanCommandOutput =>
+            createMockClientScanResponse({
+              Items: mockRecordsPage1,
+              LastEvaluatedKey: mockRecordsPage1.slice(-1)[0],
+            }),
+        )
+        .mockImplementationOnce(
+          (): ScanCommandOutput =>
+            createMockClientScanResponse({
+              Items: mockRecordsPage2,
+              LastEvaluatedKey: undefined,
+            }),
+        );
+
+      const response = await jobRepository.list();
+      expect(mockSend).toBeCalledTimes(2);
+      expect(response).toHaveLength(mockRecordsPage1.length + mockRecordsPage2.length);
+    });
+
+    it(`should stop paginating if unexpect result is returned`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce(() => null);
+
+      const response = await jobRepository.list();
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toHaveLength(0);
+    });
+  });
+
+  describe('findOne', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return job by name`, async (): Promise<void> => {
+      const mockJobRecord = createMockJobDto();
+      mockSend.mockImplementationOnce(
+        (): GetItemCommandOutput =>
+          createMockClientGetResponse({
+            Item: mockJobRecord,
+          }),
+      );
+
+      const response = await jobRepository.findOne(mockJobRecord.sk.S as string);
+      expect(response).toStrictEqual({
+        name: mockJobRecord.name.S as string,
+        type: `job`,
+        version: mockJobRecord.version.S as string,
+        description: mockJobRecord.description.S as string,
+      } as Job);
+      expect(mockSend).toBeCalledTimes(1);
+    });
+
+    it(`should return null if job does not exist`, async (): Promise<void> => {
+      mockSend.mockImplementationOnce(
+        (): ScanCommandOutput =>
+          createMockClientScanResponse({
+            Items: [],
+          }),
+      );
+
+      const response = await jobRepository.findOne(`null`);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toBe(null);
+    });
+  });
+
+  describe('create', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should create a new job record and return job in the response`, async (): Promise<void> => {
+      const newJobToCreate = createMockCreateJobRequest();
+      const mockReponse = createMockClientPutResponse({
+        Attributes: {
+          pk: { S: PARTITION_KEYS.job },
+          sk: { S: `${newJobToCreate.name}-${newJobToCreate.version}` },
+          name: { S: newJobToCreate.name },
+          version: { S: newJobToCreate.version },
+        },
+      });
+
+      mockSend.mockImplementationOnce((): PutItemCommandOutput => mockReponse);
+
+      const response = await jobRepository.create(newJobToCreate);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toStrictEqual({
+        ...newJobToCreate,
+        type: PARTITION_KEYS.job,
+      });
+    });
+  });
+
+  describe('getJobs', () => {
+    let mockExclusiveStartKey: Record<string, AttributeValue>;
+    let mockPaginationInfo: PaginationInfo;
+    let mockParams: QueryCommandInput;
+
+    beforeEach(() => {
+      mockExclusiveStartKey = {
+        pk: { S: PARTITION_KEYS.job },
+        sk: { S: 'mockSk' },
+      };
+      mockPaginationInfo = {
+        direction: PAGINATION_DIRECTION_TYPE_BEFORE,
+        limit: 100,
+        requestUrl: 'mockUrl',
+        token: 'mockToken',
+      };
+      mockParams = {
+        ExclusiveStartKey: mockExclusiveStartKey,
+        ExpressionAttributeValues: {
+          ':pk': { S: PARTITION_KEYS.job },
+        },
+        KeyConditionExpression: 'pk = :pk',
+        ScanIndexForward: false,
+        TableName: process.env.DYNAMODB_TABLE,
+      };
+
+      jest.spyOn(PaginationUtils, 'decodeTokenString').mockReturnValue(mockExclusiveStartKey);
+    });
+
+    it('should return an empty array if no jobs are found', async () => {
+      mockSend.mockImplementationOnce(() =>
+        Promise.resolve({
+          Items: [],
+          LastEvaluatedKey: null,
+        }),
+      );
+
+      const mockRequestData: PaginationInfo = mockPaginationInfo;
+
+      const result = await jobRepository.getJobs(mockRequestData);
+
+      expect(mockSend).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { ...mockParams },
+        }),
+      );
+      expect(result.items).toEqual([]);
+      expect(result.lastEvaluatedKey).toBe(null);
+    });
+
+    it('should return an array of jobs if found', async () => {
+      const mockJobRecords = createMockJobDtos();
+      const mockLastEvaluatedKey = {
+        pk: { S: PARTITION_KEYS.job },
+        sk: { S: 'mockSk' },
+      };
+
+      mockSend.mockImplementationOnce(() =>
+        Promise.resolve({
+          Items: mockJobRecords,
+          LastEvaluatedKey: mockLastEvaluatedKey,
+        }),
+      );
+
+      const mockRequestData: PaginationInfo = mockPaginationInfo;
+
+      const result = await jobRepository.getJobs(mockRequestData);
+
+      expect(mockSend).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { ...mockParams },
+        }),
+      );
+      expect(result.items).toHaveLength(mockJobRecords.length);
+      expect(result.lastEvaluatedKey).toStrictEqual(mockLastEvaluatedKey);
+    });
+  });
+
+  // describe('update', () => {
+  //   beforeEach(() => {
+  //     jest.resetAllMocks();
+  //   });
+
+  //   it(`should update a job record`, async (): Promise<void> => {
+  //     const jobToUpdate = createMockJob();
+  //     const updatedJob: Job = {
+  //       ...jobToUpdate,
+  //       name: `new-name`,
+  //       version: `9.9.9`,
+  //     };
+  //     const updatedJobRecord = createMockJobRecord({
+  //       sk: updatedJob.name,
+  //       version: updatedJob.version,
+  //     });
+  //     const updatedJobRecordDto = createMockJobDto({
+  //       ...updatedJobRecord,
+  //     });
+  //     const putResponse = createMockClientPutResponse({
+  //       Attributes: { ...updatedJobRecordDto },
+  //     });
+
+  //     mockSend.mockImplementation((): PutItemCommandOutput => putResponse);
+
+  //     const response = await jobRepository.update(jobToUpdate.name, updatedJobRecord);
+  //     expect(response).toStrictEqual(updatedJob);
+  //     expect(mockSend).toBeCalledTimes(1);
+  //   });
+  // });
+
+  // describe('delete', () => {
+  //   beforeEach(() => {
+  //     jest.resetAllMocks();
+  //   });
+
+  //   it(`should return undefined when calling the delete operation`, async (): Promise<void> => {
+  //     mockSend.mockImplementationOnce((): DeleteItemCommandOutput => {
+  //       return {
+  //         $metadata: {},
+  //       };
+  //     });
+
+  //     const response = await jobRepository.delete(`name`);
+  //     expect(response).toBe(undefined);
+  //     expect(mockSend).toBeCalledTimes(1);
+  //   });
+  // });
+});