@squiz/job-runner-lib 1.66.1 → 2.1.0

package/lib/jest-global-setup.d.ts

@@ -1,3 +1 @@
 export declare const testLogger: import("winston").Logger;
-declare const _default: () => Promise<void>;
-export default _default;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@squiz/job-runner-lib",
-  "version": "1.66.1",
+  "version": "2.1.0",
   "description": "",
   "main": "lib/index.js",
   "private": false,
@@ -8,52 +8,46 @@
     "access": "public"
   },
   "scripts": {
-    "compile": "node build.js && tsc",
+    "compile": "npm run clean && npm run build && tsc",
+    "build": "node build.js",
     "test": "jest -c jest.config.ts",
-    "test:integration": "jest -c jest.integration.config.ts",
     "test:update-snapshots": "jest -c jest.config.ts --updateSnapshot",
     "clean": "rimraf \"tsconfig.tsbuildinfo\" \"./lib\""
   },
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@types/archiver": "5.3.1",
-    "@types/cli-color": "^2.0.6",
-    "@types/express": "4.17.17",
+    "@faker-js/faker": "8.0.2",
+    "@squiz/virus-scanner-lib": "^1.66.1",
+    "@types/cli-color": "2.0.6",
     "@types/fs-extra": "11.0.1",
-    "@types/http-errors": "2.0.1",
     "@types/jest": "28.1.8",
-    "@types/node": "20.12.4",
-    "@types/semver": "7.3.12",
-    "archiver": "5.3.1",
+    "@types/node": "22.9.0",
+    "aws-sdk-client-mock": "^2.2.0",
+    "aws-sdk-client-mock-jest": "^2.2.0",
     "dotenv": "16.0.3",
+    "esbuild": "^0.25.6",
     "jest": "29.4.1",
     "rimraf": "4.1.2",
     "ts-jest": "29.0.5",
-    "ts-loader": "9.3.1",
-    "ts-node": "10.9.1",
-    "typescript": "4.9.4",
-    "unzipper": "0.10.14"
+    "typescript": "4.9.4"
   },
   "dependencies": {
     "@aws-sdk/client-dynamodb": "3.427.0",
     "@aws-sdk/client-ecs": "3.348.0",
-    "@faker-js/faker": "8.0.2",
-    "@squiz/component-logger-lib": "^1.67.0",
-    "@squiz/component-web-api-lib": "^1.65.0",
-    "@squiz/db-lib": "^1.64.0",
+    "@aws-sdk/client-s3": "^3.332.0",
+    "@aws-sdk/s3-request-presigner": "^3.332.0",
     "@squiz/dx-common-lib": "^1.66.1",
-    "@squiz/dx-json-schema-lib": "^1.65.0",
+    "@squiz/dx-json-schema-lib": "1.78.1",
     "@squiz/dx-logger-lib": "^1.64.0",
-    "@types/unzipper": "0.10.5",
-    "axios": "1.6.0",
+    "axios": "1.7.4",
+    "cli-color": "^2.0.2",
     "dotenv": "16.0.3",
     "fs-extra": "11.1.0",
-    "http-errors": "2.0.0",
-    "query-string": "7.1.3",
+    "inversify": "6.0.1",
+    "inversify-binding-decorators": "4.0.0",
     "reflect-metadata": "^0.1.13",
-    "semver": "7.5.2",
+    "tsoa": "5.1.1",
     "winston": "3.8.2"
-  },
-  "gitHead": "ae85975300d01a97f28db9f8831dbd8881dc7b20"
+  }
 }

package/src/domains/component/index.ts

@@ -1,17 +1 @@
-export * from './models/Component';
-export * from './models/ComponentVersion';
-
-export * from './repositories/ComponentRepository';
-export * from './repositories/ComponentVersionRepository';
-
-export * from './services/ComponentService';
-export * from './services/content-item-uri-resolver/ComponentContentUriResolverForPreview';
-
-export * from './web-models/ComponentForWeb';
-export * from './web-models/ComponentsForWeb';
-export * from './web-models/ComponentVersionWebModel';
-
-export * from './component-context/createComponentContextualFunctions';
-export * from './errors/ComponentError';
 export * from './errors/JobError';
-export * from './utils/getComponentFileSystemFolder';

package/src/domains/job/S3Repository/JobDetailsRepository.spec.ts

@@ -0,0 +1,52 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
+import { JobDetailsRepository } from './JobDetailsRepository';
+import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
+import { testLogger } from '../../../jest-global-setup';
+
+// Create a new container tsoa can use
+initIocContainer(testLogger);
+
+// Mock S3 Client
+const mockOptions = {} as S3ClientConfig;
+const mockClient = new S3Client(mockOptions);
+
+const mockSend = jest.fn().mockResolvedValue({
+  Body: { transformToString: jest.fn().mockResolvedValue('file') },
+  $metadata: {},
+});
+mockClient.send = mockSend;
+
+// Inject the S3Repository with a mocked client
+iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
+
+let jobDetailsRepository: JobDetailsRepository;
+
+describe('JobDetailsRepository', (): void => {
+  beforeAll((): void => {
+    jobDetailsRepository = iocContainer.get(JobDetailsRepository);
+  });
+
+  describe(`get`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return a response for get`, async (): Promise<void> => {
+      const mockKey = `mock`;
+      mockSend.mockImplementationOnce(() =>
+        Promise.resolve({
+          Body: { transformToString: jest.fn().mockResolvedValue('file') },
+        }),
+      );
+
+      const response = await jobDetailsRepository.get(mockKey);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toStrictEqual('file');
+    });
+  });
+});

package/src/domains/job/S3Repository/JobDetailsRepository.ts

@@ -0,0 +1,33 @@
+import { S3Client } from '@aws-sdk/client-s3';
+import { inject } from 'inversify';
+import { S3RepositoryBase } from './S3RepositoryBase';
+import { provideTransient } from '../transientProvider';
+import { JobContext } from '../model/JobContext';
+import { JobManifest } from '../manifest';
+import { JobExecution } from '../model/JobExecution';
+
+@provideTransient(JobDetailsRepository)
+export class JobDetailsRepository extends S3RepositoryBase {
+  constructor(@inject(S3Client) protected s3Client: S3Client, @inject('JobDetailsBucketName') bucketName: string) {
+    super(s3Client, bucketName);
+  }
+
+  /**
+   * Create the job details file in S3.
+   * Context is nullable as the user could delete the context between queuing the job and creating the details file.
+   * We want the file to be created so the job starts/executes rather than remaining in a queued state/causing
+   * an error.
+   */
+  public async createJobDetailsFile(
+    jobDetailsFileName: string,
+    jobContext: JobContext | null,
+    jobManifest: JobManifest | null,
+    jobExecution: JobExecution,
+  ) {
+    await this.put(jobDetailsFileName, JSON.stringify({ jobContext, jobManifest, jobExecution }));
+  }
+
+  public static createJobDetailsFileName(tenantId: string, jobExecutionId: string): string {
+    return `${tenantId}/${jobExecutionId}`;
+  }
+}

package/src/domains/job/S3Repository/S3Repository.spec.ts

@@ -0,0 +1,50 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
+import { S3Repository } from './S3Repository';
+import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
+import { testLogger } from '../../../jest-global-setup';
+
+// Create a new container tsoa can use
+initIocContainer(testLogger);
+
+// Mock S3 Client
+const mockOptions = {} as S3ClientConfig;
+const mockClient = new S3Client(mockOptions);
+
+const mockSend = jest.fn().mockResolvedValue({
+  Body: { transformToString: jest.fn().mockResolvedValue('file') },
+  $metadata: {},
+});
+mockClient.send = mockSend;
+
+// Inject the S3Repository with a mocked client
+iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
+
+let s3Repository: S3Repository;
+
+describe('S3Repository', (): void => {
+  beforeAll((): void => {
+    s3Repository = iocContainer.get(S3Repository);
+  });
+
+  describe(`get`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return a response for get`, async (): Promise<void> => {
+      const mockKey = `mock`;
+      mockSend.mockResolvedValueOnce({
+        Body: { transformToString: jest.fn().mockResolvedValue('file') },
+      });
+
+      const response = await s3Repository.get(mockKey);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toStrictEqual('file');
+    });
+  });
+});

package/src/domains/job/S3Repository/S3Repository.ts

@@ -0,0 +1,27 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import { S3Client } from '@aws-sdk/client-s3';
+import { inject } from 'inversify';
+import { S3RepositoryBase } from './S3RepositoryBase';
+import { provideTransient } from '../transientProvider';
+
+/**
+ * Repository class for interacting with S3.
+ * @export
+ * @class S3Repository
+ */
+@provideTransient(S3Repository)
+export class S3Repository extends S3RepositoryBase {
+  public constructor(
+    @inject(S3Client) protected s3Client: S3Client,
+    @inject('S3RepositoryBucketName') bucketName: string,
+  ) {
+    super(s3Client, bucketName);
+  }
+}

package/src/domains/job/S3Repository/S3RepositoryBase.spec.ts

@@ -0,0 +1,97 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
+import { S3Repository } from './S3Repository';
+import { initIocContainer, iocContainer } from '../../../mocks/mockIoc';
+import { testLogger } from '../../../jest-global-setup';
+
+// Create a new container tsoa can use
+initIocContainer(testLogger);
+
+// Mock S3 Client
+const mockOptions = {} as S3ClientConfig;
+const mockClient = new S3Client(mockOptions);
+
+const mockSend = jest.fn().mockResolvedValue({
+  Body: { transformToString: jest.fn().mockResolvedValue('file') },
+  $metadata: {},
+});
+mockClient.send = mockSend;
+
+// Inject the S3Repository with a mocked client
+iocContainer.rebind<S3Client>(S3Client).toConstantValue(mockClient);
+
+let s3Repository: S3Repository;
+
+describe('S3RepositoryBase', (): void => {
+  beforeAll((): void => {
+    s3Repository = new S3Repository(mockClient, 'mock-bucket');
+  });
+
+  describe(`get`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return a response for get`, async (): Promise<void> => {
+      const mockKey = `mock`;
+      mockSend.mockResolvedValueOnce({
+        Body: { transformToString: jest.fn().mockResolvedValue('file') },
+      });
+
+      const response = await s3Repository.get(mockKey);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toStrictEqual('file');
+    });
+
+    it(`should return a null response if undefined`, async (): Promise<void> => {
+      const mockKey = `mock`;
+      const mockResponse = undefined;
+      mockSend.mockResolvedValueOnce({
+        Body: mockResponse,
+      });
+
+      const response = await s3Repository.get(mockKey);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toBeFalsy();
+    });
+  });
+
+  describe(`put`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return a response for get`, async (): Promise<void> => {
+      const mockKey = `mock`;
+      const mockObject = ``;
+      mockSend.mockResolvedValueOnce({
+        Body: { transformToString: jest.fn().mockResolvedValue(mockObject) },
+      });
+
+      const response = await s3Repository.put(mockKey, mockObject);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toBeFalsy();
+    });
+  });
+
+  describe(`check`, (): void => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it(`should return a response if S3 object key exists`, async (): Promise<void> => {
+      const mockKey = `mock`;
+      mockSend.mockResolvedValueOnce({
+        Body: {},
+      });
+
+      const response = await s3Repository.check(mockKey);
+      expect(mockSend).toBeCalledTimes(1);
+      expect(response).toBeTruthy();
+    });
+  });
+});

package/src/domains/job/S3Repository/S3RepositoryBase.ts

@@ -0,0 +1,93 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import {
+  GetObjectCommand,
+  HeadObjectCommand,
+  HeadObjectCommandOutput,
+  PutObjectCommand,
+  PutObjectCommandInputType,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { inject, injectable } from 'inversify';
+import 'reflect-metadata';
+
+/**
+ * Repository class for interacting with S3.
+ * @export
+ * @class S3Repository
+ */
+@injectable()
+export abstract class S3RepositoryBase {
+  protected bucketName: string;
+  public constructor(@inject(S3Client) protected s3Client: S3Client, bucketName: string) {
+    this.bucketName = bucketName;
+  }
+
+  /**
+   * Save object into S3.
+   * @param {string} key The S3 object key.
+   * @param {PutObjectCommandInputType['Body']} body The body of the S3 object.
+   * @return {Promise<void>}
+   * @memberof S3Repository
+   */
+  async put(key: string, body: PutObjectCommandInputType['Body']): Promise<void> {
+    await this.s3Client.send(
+      new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: key,
+        Body: body,
+      }),
+    );
+  }
+
+  /**
+   * Retrieve object from S3.
+   * @param {string} key The S3 object key.
+   * @return {Promise<string | null>} The S3 object.
+   * @memberof S3Repository
+   */
+  async get(key: string): Promise<string | null> {
+    const s3Object = await this.s3Client.send(
+      new GetObjectCommand({
+        Bucket: this.bucketName,
+        Key: key,
+      }),
+    );
+
+    return s3Object.Body?.transformToString() || null;
+  }
+
+  /**
+   * Check if the S3 object key exists.
+   * @param {string} key The S3 object key.
+   * @return {Promise<HeadObjectCommandOutput>}
+   * @memberof S3Repository
+   */
+  async check(key: string): Promise<HeadObjectCommandOutput> {
+    return this.s3Client.send(
+      new HeadObjectCommand({
+        Bucket: this.bucketName,
+        Key: key,
+      }),
+    );
+  }
+
+  async getS3SignedUrl(key: string): Promise<string> {
+    const command = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: key,
+    });
+
+    const res = await getSignedUrl(this.s3Client, command, {
+      expiresIn: 600, // 10 minutes
+    });
+    return res;
+  }
+}

package/src/domains/job/S3Repository/index.ts

@@ -0,0 +1,2 @@
+export * from './JobDetailsRepository';
+export * from './S3Repository';

package/src/domains/job/core/pagination/services/PaginationService.spec.ts

@@ -5,67 +5,18 @@
 
 import { AttributeValue } from '@aws-sdk/client-dynamodb';
 import { faker } from '@faker-js/faker/locale/en';
-import { createMockJob } from '../../../model/Job/mocks/Job.mock';
-import { createMockJobContext } from '../../../model/JobContext/mocks/JobContext.mock';
-import { createMockJobExecution } from '../../../model/JobExecution/mocks/JobExecution.mock';
-import { InternalServerError } from '../../../errors/InternalServerError';
 import { PartitionKey, PARTITION_KEYS } from '../../../constants/Repository.constants';
 import { PaginationInfo } from '../model/PaginationInfo';
 import { PAGINATION_DIRECTION_TYPE_AFTER, PAGINATION_DIRECTION_TYPE_BEFORE } from '../model/PaginationTypes';
 import { encodeTokenString } from '../utils/PaginationUtils';
 import { PaginationService } from './PaginationService';
-import { JobRepository, JobContextRepository, JobExecutionRepository } from '../../../repository';
 
 describe('PaginationService', (): void => {
-  let jobRepository: Record<string, object>;
-  let jobContextRepository: Record<string, object>;
-  let jobExecutionRepository: Record<string, object>;
   let paginationService: PaginationService<any>;
-
-  const mockConvertToDynamoDbItemJob = jest.fn();
-  const mockConvertToDynamoDbItemContext = jest.fn();
-  const mockConvertToDynamoDbItemExecution = jest.fn();
+  const mockConvertToDynamoDbItem = jest.fn();
 
   beforeAll((): void => {
-    jobRepository = {
-      create: jest.fn(),
-      findOne: jest.fn(),
-      list: jest.fn(),
-      update: jest.fn(),
-      delete: jest.fn(),
-
-      getJobs: jest.fn(),
-      convertToDynamoDbItem: mockConvertToDynamoDbItemJob,
-    };
-
-    jobContextRepository = {
-      create: jest.fn(),
-      findOne: jest.fn(),
-      list: jest.fn(),
-      update: jest.fn(),
-      delete: jest.fn(),
-      getContexts: jest.fn(),
-
-      getJobs: jest.fn(),
-      convertToDynamoDbItem: mockConvertToDynamoDbItemContext,
-    };
-
-    jobExecutionRepository = {
-      create: jest.fn(),
-      findOne: jest.fn(),
-      list: jest.fn(),
-      update: jest.fn(),
-      delete: jest.fn(),
-
-      getJobs: jest.fn(),
-      convertToDynamoDbItem: mockConvertToDynamoDbItemExecution,
-    };
-
-    paginationService = new PaginationService<any>(
-      jobRepository as unknown as JobRepository,
-      jobContextRepository as unknown as JobContextRepository,
-      jobExecutionRepository as unknown as JobExecutionRepository,
-    );
+    paginationService = new PaginationService<any>();
   });
 
   describe('addPaginationLinksToResponse', (): void => {
@@ -102,7 +53,7 @@ describe('PaginationService', (): void => {
         }),
       );
 
-      mockConvertToDynamoDbItemJob
+      mockConvertToDynamoDbItem
         .mockImplementationOnce(() => {
           return mockExclusiveStartKey;
         })
@@ -114,9 +65,10 @@ describe('PaginationService', (): void => {
         mockData,
         mockPaginationInfo,
         mockPaginationInfo.token,
+        mockConvertToDynamoDbItem,
       );
 
-      expect(mockConvertToDynamoDbItemJob).toBeCalledTimes(1);
+      expect(mockConvertToDynamoDbItem).toBeCalledTimes(1);
       expect(response.data).toStrictEqual(mockData);
 
       expect(response.links).toBeTruthy();
@@ -132,15 +84,16 @@ describe('PaginationService', (): void => {
         }),
       }).map((_, i) => ({ type: mockType, i }));
 
-      mockConvertToDynamoDbItemJob.mockImplementationOnce(() => mockExclusiveStartKey);
+      mockConvertToDynamoDbItem.mockImplementationOnce(() => mockExclusiveStartKey);
 
       const response = paginationService.addPaginationLinksToResponse(
         mockData,
         { ...mockPaginationInfo, token: null },
         null,
+        mockConvertToDynamoDbItem,
       );
 
-      expect(mockConvertToDynamoDbItemJob).not.toBeCalled();
+      expect(mockConvertToDynamoDbItem).not.toBeCalled();
       expect(response.data).toStrictEqual(mockData);
       expect(response.links).toBeTruthy();
       expect(response.links.prev).toBeNull();
@@ -155,15 +108,16 @@ describe('PaginationService', (): void => {
         }),
       }).map((_, i) => ({ type: mockType, i }));
 
-      mockConvertToDynamoDbItemJob.mockImplementationOnce(() => mockExclusiveStartKey);
+      mockConvertToDynamoDbItem.mockImplementationOnce(() => mockExclusiveStartKey);
 
       const response = paginationService.addPaginationLinksToResponse(
         mockData,
         mockPaginationInfo,
         mockLastEvaluatedKey,
+        mockConvertToDynamoDbItem,
       );
 
-      expect(mockConvertToDynamoDbItemJob).toBeCalledTimes(1);
+      expect(mockConvertToDynamoDbItem).toBeCalledTimes(1);
       expect(response.data).toStrictEqual(mockData);
       expect(response.links).toBeTruthy();
       expect(response.links.prev).toBeTruthy();
@@ -178,15 +132,16 @@ describe('PaginationService', (): void => {
         }),
       }).map((_, i) => ({ type: mockType, i }));
 
-      mockConvertToDynamoDbItemJob.mockImplementationOnce(() => mockExclusiveStartKey);
+      mockConvertToDynamoDbItem.mockImplementationOnce(() => mockExclusiveStartKey);
 
       const response = paginationService.addPaginationLinksToResponse(
         mockData,
         { ...mockPaginationInfo, direction: PAGINATION_DIRECTION_TYPE_BEFORE },
         mockLastEvaluatedKey,
+        mockConvertToDynamoDbItem,
       );
 
-      expect(mockConvertToDynamoDbItemJob).toBeCalledTimes(1);
+      expect(mockConvertToDynamoDbItem).toBeCalledTimes(1);
       expect(response.data).toStrictEqual(mockData.reverse());
       expect(response.links).toBeTruthy();
     });
@@ -199,7 +154,7 @@ describe('PaginationService', (): void => {
         }),
       }).map((_, i) => ({ type: mockType, i }));
 
-      mockConvertToDynamoDbItemJob
+      mockConvertToDynamoDbItem
         .mockImplementationOnce(() => mockExclusiveStartKey)
         .mockImplementationOnce(() => mockExclusiveStartKey);
 
@@ -207,42 +162,12 @@ describe('PaginationService', (): void => {
         mockData,
         mockPaginationInfo,
         mockLastEvaluatedKey,
+        mockConvertToDynamoDbItem,
       );
 
-      expect(mockConvertToDynamoDbItemJob).toBeCalledTimes(2);
+      expect(mockConvertToDynamoDbItem).toBeCalledTimes(2);
       expect(response.data).toHaveLength(mockPaginationInfo.limit);
       expect(response.links).toBeTruthy();
     });
   });
-
-  describe('convertToDynamoDbItem', (): void => {
-    beforeEach(() => {
-      jest.resetAllMocks();
-    });
-    it('should call the jobRepository method for converting to a DynamoDB object', (): void => {
-      const mockJob = createMockJob();
-
-      (paginationService as any).convertToDynamoDbItem(mockJob);
-      expect(mockConvertToDynamoDbItemJob).toBeCalled();
-    });
-
-    it('should call the jobContextRepository method for converting to a DynamoDB object', (): void => {
-      const mockJobContext = createMockJobContext();
-
-      (paginationService as any).convertToDynamoDbItem(mockJobContext);
-      expect(mockConvertToDynamoDbItemContext).toBeCalled();
-    });
-
-    it('should call the jobExecutionRepository method for converting to a DynamoDB object', (): void => {
-      const mockJobExecution = createMockJobExecution();
-
-      (paginationService as any).convertToDynamoDbItem(mockJobExecution);
-      expect(mockConvertToDynamoDbItemExecution).toBeCalled();
-    });
-
-    it('should should throw an error if the partition key is invalid', (): void => {
-      const mockItem = { type: 'invalid' };
-      expect(() => (paginationService as any).convertToDynamoDbItem(mockItem)).toThrowError(InternalServerError);
-    });
-  });
 });