@squiz/job-runner-lib 1.66.1 → 2.1.0

package/src/domains/job/services/JobUploadService/JobUploadService.spec.ts

@@ -0,0 +1,176 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+// Mock external dependencies before any imports
+const mockUploadFile = jest.fn();
+jest.mock('@squiz/virus-scanner-lib', () => {
+  const originalModule = jest.requireActual('@squiz/virus-scanner-lib');
+  return {
+    ...originalModule,
+    uploadFile: mockUploadFile,
+  };
+});
+
+jest.mock('./utils/manifestValidator');
+jest.mock('./utils/uploadInitializer');
+jest.mock('./utils/scanWatcher');
+jest.mock('./utils/resultPoller');
+jest.mock('./utils/cleanup');
+
+const mockMkdtemp = jest.fn();
+const mockRm = jest.fn();
+jest.mock('fs/promises', () => ({
+  mkdtemp: mockMkdtemp,
+  rm: mockRm,
+}));
+
+import { AxiosInstance } from 'axios';
+import { JobUploadService } from './JobUploadService';
+import { cleanupTmpDir } from './utils/cleanup';
+import { getAndValidateManifest } from './utils/manifestValidator';
+import { pollForResult } from './utils/resultPoller';
+import { watchAndWaitForUploadAndScanComplete } from './utils/scanWatcher';
+import { initializeUpload } from './utils/uploadInitializer';
+
+const mockGetAndValidateManifest = getAndValidateManifest as jest.MockedFunction<typeof getAndValidateManifest>;
+const mockInitializeUpload = initializeUpload as jest.MockedFunction<typeof initializeUpload>;
+const mockWatchAndWait = watchAndWaitForUploadAndScanComplete as jest.MockedFunction<
+  typeof watchAndWaitForUploadAndScanComplete
+>;
+const mockPollForResult = pollForResult as jest.MockedFunction<typeof pollForResult>;
+const mockCleanupTmpDir = cleanupTmpDir as jest.MockedFunction<typeof cleanupTmpDir>;
+
+describe('JobUploadService', (): void => {
+  let jobUploadService: JobUploadService;
+
+  const mockAxiosClient = {
+    post: jest.fn(),
+    get: jest.fn(),
+  } as unknown as AxiosInstance;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    jobUploadService = new JobUploadService();
+
+    jest.spyOn(jobUploadService['logger'], 'info').mockImplementation(() => jobUploadService['logger']);
+    jest.spyOn(jobUploadService['logger'], 'error').mockImplementation(() => jobUploadService['logger']);
+    jest.spyOn(jobUploadService['logger'], 'warn').mockImplementation(() => jobUploadService['logger']);
+    jest.spyOn(jobUploadService['logger'], 'debug').mockImplementation(() => jobUploadService['logger']);
+
+    mockMkdtemp.mockResolvedValue('/tmp/job-upload-abc123');
+  });
+
+  describe('uploadJobFolder', (): void => {
+    const mockFolderPath = '/test/job/folder';
+    const mockBaseTempDir = '/tmp';
+
+    it('should successfully upload job folder with all steps', async (): Promise<void> => {
+      const mockManifest = {
+        getName: () => 'test-job',
+        getVersion: () => '1.0.0',
+        getModel: () => ({ name: 'test-job', version: '1.0.0' }),
+      };
+      const mockInitialUpload = {
+        zip: '/tmp/job.zip',
+        upload: { id: 'upload-123' },
+      };
+      const mockResult = { status: 'successful' };
+
+      mockGetAndValidateManifest.mockResolvedValue(mockManifest as any);
+      mockInitializeUpload.mockResolvedValue(mockInitialUpload);
+      mockUploadFile.mockResolvedValue(undefined);
+      mockWatchAndWait.mockResolvedValue(undefined);
+      mockPollForResult.mockResolvedValue(mockResult);
+      mockCleanupTmpDir.mockResolvedValue(undefined);
+
+      await jobUploadService.uploadJobFolder(mockAxiosClient, mockFolderPath, mockBaseTempDir);
+
+      expect(mockMkdtemp).toHaveBeenCalledWith('/tmp/job-upload');
+      expect(mockGetAndValidateManifest).toHaveBeenCalledWith(mockAxiosClient, mockFolderPath, expect.any(Object));
+      expect(mockInitializeUpload).toHaveBeenCalledWith(
+        mockAxiosClient,
+        mockFolderPath,
+        '/tmp/job-upload-abc123',
+        expect.any(Object),
+      );
+      expect(mockUploadFile).toHaveBeenCalledWith(mockInitialUpload.upload, mockInitialUpload.zip);
+      expect(mockWatchAndWait).toHaveBeenCalledWith(mockAxiosClient, '/upload-job/status/', 'upload-123');
+      expect(mockPollForResult).toHaveBeenCalledWith(
+        mockAxiosClient,
+        'upload-123',
+        mockManifest.getModel(),
+        expect.any(Object),
+      );
+      expect(mockCleanupTmpDir).toHaveBeenCalledWith('/tmp/job-upload-abc123', expect.any(Object));
+    });
+
+    it('should cleanup temp directory on success', async (): Promise<void> => {
+      const mockManifest = { getModel: () => ({}) };
+      const mockInitialUpload = { zip: '/tmp/job.zip', upload: { id: 'upload-123' } };
+      const mockResult = { status: 'successful' };
+
+      mockGetAndValidateManifest.mockResolvedValue(mockManifest as any);
+      mockInitializeUpload.mockResolvedValue(mockInitialUpload);
+      mockUploadFile.mockResolvedValue(undefined);
+      mockWatchAndWait.mockResolvedValue(undefined);
+      mockPollForResult.mockResolvedValue(mockResult);
+      mockCleanupTmpDir.mockResolvedValue(undefined);
+
+      await jobUploadService.uploadJobFolder(mockAxiosClient, mockFolderPath, mockBaseTempDir);
+
+      expect(mockCleanupTmpDir).toHaveBeenCalledWith('/tmp/job-upload-abc123', expect.any(Object));
+    });
+
+    it('should cleanup temp directory on error', async (): Promise<void> => {
+      const testError = new Error('Test error');
+      mockGetAndValidateManifest.mockRejectedValue(testError);
+      mockCleanupTmpDir.mockResolvedValue(undefined);
+
+      await expect(jobUploadService.uploadJobFolder(mockAxiosClient, mockFolderPath, mockBaseTempDir)).rejects.toThrow(
+        'Test error',
+      );
+
+      expect(mockCleanupTmpDir).toHaveBeenCalledWith('/tmp/job-upload-abc123', expect.any(Object));
+    });
+
+    it('should handle unsuccessful upload result', async (): Promise<void> => {
+      const mockManifest = { getModel: () => ({}) };
+      const mockInitialUpload = { zip: '/tmp/job.zip', upload: { id: 'upload-123' } };
+      const mockResult = { status: 'failed', message: 'Upload failed' };
+
+      mockGetAndValidateManifest.mockResolvedValue(mockManifest as any);
+      mockInitializeUpload.mockResolvedValue(mockInitialUpload);
+      mockUploadFile.mockResolvedValue(undefined);
+      mockWatchAndWait.mockResolvedValue(undefined);
+      mockPollForResult.mockResolvedValue(mockResult);
+      mockCleanupTmpDir.mockResolvedValue(undefined);
+
+      await jobUploadService.uploadJobFolder(mockAxiosClient, mockFolderPath, mockBaseTempDir);
+
+      // Should call cleanup twice: once for unsuccessful result, once for normal cleanup
+      expect(mockCleanupTmpDir).toHaveBeenCalledTimes(2);
+    });
+
+    it('should handle upload file step failure', async (): Promise<void> => {
+      const mockManifest = { getModel: () => ({}) };
+      const mockInitialUpload = { zip: '/tmp/job.zip', upload: { id: 'upload-123' } };
+      const uploadError = new Error('Upload failed');
+
+      mockGetAndValidateManifest.mockResolvedValue(mockManifest as any);
+      mockInitializeUpload.mockResolvedValue(mockInitialUpload);
+      mockUploadFile.mockRejectedValue(uploadError);
+      mockCleanupTmpDir.mockResolvedValue(undefined);
+
+      await expect(jobUploadService.uploadJobFolder(mockAxiosClient, mockFolderPath, mockBaseTempDir)).rejects.toThrow(
+        'Upload failed',
+      );
+
+      expect(mockCleanupTmpDir).toHaveBeenCalledWith('/tmp/job-upload-abc123', expect.any(Object));
+    });
+  });
+});

package/src/domains/job/services/JobUploadService/JobUploadService.ts

@@ -0,0 +1,66 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { getLogger, Logger } from '@squiz/dx-logger-lib';
+import { uploadFile } from '@squiz/virus-scanner-lib';
+import { AxiosInstance } from 'axios';
+import color from 'cli-color';
+import fsp from 'fs/promises';
+import path from 'path';
+import {
+  cleanupTmpDir,
+  getAndValidateManifest,
+  initializeUpload,
+  pollForResult,
+  watchAndWaitForUploadAndScanComplete,
+} from './utils';
+
+/**
+ * Service for handling job folder uploads
+ */
+export class JobUploadService {
+  private logger: Logger;
+
+  constructor() {
+    this.logger = getLogger({ name: 'upload-job', format: 'human' });
+  }
+
+  /**
+   * Upload a job folder to the job runner service
+   * @param apiClient - Axios client for API calls
+   * @param folderPath - Path to the job folder to upload
+   * @param baseTempDir - Base temporary directory for upload processing
+   */
+  public async uploadJobFolder(apiClient: AxiosInstance, folderPath: string, baseTempDir: string = ''): Promise<void> {
+    const tmpDir = await fsp.mkdtemp(path.resolve(baseTempDir, 'job-upload'));
+
+    try {
+      const manifest = await getAndValidateManifest(apiClient, folderPath, this.logger);
+
+      const initialUpload = await initializeUpload(apiClient, folderPath, tmpDir, this.logger);
+
+      await uploadFile(initialUpload.upload, initialUpload.zip);
+
+      await watchAndWaitForUploadAndScanComplete(apiClient, '/upload-job/status/', initialUpload.upload.id);
+
+      this.logger.info(`deployment id: ${initialUpload.upload.id} status: deploying job folder`);
+
+      const result = await pollForResult(apiClient, initialUpload.upload.id, manifest.getModel(), this.logger);
+
+      await cleanupTmpDir(tmpDir, this.logger);
+
+      if (result.status === 'successful') {
+        this.logger.info(`deployment id: ${initialUpload.upload.id} status: ${color.green('success')}`);
+      } else {
+        const message = result?.message ?? 'unknown';
+        this.logger.error(`failed due an unexpected reason: ${message}`);
+        await cleanupTmpDir(tmpDir, this.logger);
+      }
+    } catch (e) {
+      await cleanupTmpDir(tmpDir, this.logger);
+      throw e;
+    }
+  }
+}

package/src/domains/job/services/JobUploadService/utils/cleanup.spec.ts

@@ -0,0 +1,109 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { Logger } from '@squiz/dx-logger-lib';
+import fsp from 'fs/promises';
+import { cleanupTmpDir } from './cleanup';
+
+jest.mock('fs/promises', () => ({
+  rm: jest.fn(),
+}));
+
+describe('cleanup', () => {
+  let mockLogger: Logger;
+  const mockRm = fsp.rm as jest.MockedFunction<typeof fsp.rm>;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockLogger = {
+      error: jest.fn(),
+      info: jest.fn(),
+      warn: jest.fn(),
+      debug: jest.fn(),
+    } as any;
+  });
+
+  describe('cleanupTmpDir', () => {
+    const testTmpDir = '/tmp/test-directory';
+
+    it('should successfully remove directory', async () => {
+      mockRm.mockResolvedValue(undefined);
+
+      await cleanupTmpDir(testTmpDir, mockLogger);
+
+      expect(mockRm).toHaveBeenCalledWith(testTmpDir, {
+        force: true,
+        recursive: true,
+      });
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+
+    it('should log error when directory removal fails', async () => {
+      const mockError = new Error('Permission denied');
+      mockRm.mockRejectedValue(mockError);
+
+      await cleanupTmpDir(testTmpDir, mockLogger);
+
+      expect(mockRm).toHaveBeenCalledWith(testTmpDir, {
+        force: true,
+        recursive: true,
+      });
+      expect(mockLogger.error).toHaveBeenCalledWith('failed to remove temp directory', mockError);
+    });
+
+    it('should handle ENOENT error gracefully', async () => {
+      const enoentError = new Error('ENOENT: no such file or directory');
+      (enoentError as any).code = 'ENOENT';
+      mockRm.mockRejectedValue(enoentError);
+
+      await cleanupTmpDir(testTmpDir, mockLogger);
+
+      expect(mockRm).toHaveBeenCalledWith(testTmpDir, {
+        force: true,
+        recursive: true,
+      });
+      expect(mockLogger.error).toHaveBeenCalledWith('failed to remove temp directory', enoentError);
+    });
+
+    it('should handle system permission errors', async () => {
+      const permissionError = new Error('EACCES: permission denied');
+      (permissionError as any).code = 'EACCES';
+      mockRm.mockRejectedValue(permissionError);
+
+      await cleanupTmpDir(testTmpDir, mockLogger);
+
+      expect(mockRm).toHaveBeenCalledWith(testTmpDir, {
+        force: true,
+        recursive: true,
+      });
+      expect(mockLogger.error).toHaveBeenCalledWith('failed to remove temp directory', permissionError);
+    });
+
+    it('should handle empty directory path', async () => {
+      mockRm.mockResolvedValue(undefined);
+
+      await cleanupTmpDir('', mockLogger);
+
+      expect(mockRm).toHaveBeenCalledWith('', {
+        force: true,
+        recursive: true,
+      });
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+
+    it('should handle directory path with special characters', async () => {
+      const specialPath = '/tmp/test dir with spaces & symbols!@#';
+      mockRm.mockResolvedValue(undefined);
+
+      await cleanupTmpDir(specialPath, mockLogger);
+
+      expect(mockRm).toHaveBeenCalledWith(specialPath, {
+        force: true,
+        recursive: true,
+      });
+      expect(mockLogger.error).not.toHaveBeenCalled();
+    });
+  });
+});

package/src/domains/job/services/JobUploadService/utils/cleanup.ts

@@ -0,0 +1,18 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { Logger } from '@squiz/dx-logger-lib';
+import fsp from 'fs/promises';
+
+/**
+ * Clean up temporary directory
+ * @param tmpDir - Temporary directory path to clean up
+ * @param logger - Logger instance
+ */
+export async function cleanupTmpDir(tmpDir: string, logger: Logger): Promise<void> {
+  await fsp.rm(tmpDir, { force: true, recursive: true }).catch((error: any) => {
+    logger.error('failed to remove temp directory', error);
+  });
+}

package/src/domains/job/services/JobUploadService/utils/errorUtils.spec.ts

@@ -0,0 +1,76 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+import { AxiosError } from 'axios';
+import { handleError, isAxiosError } from './errorUtils';
+
+describe('errorUtils', () => {
+  describe('isAxiosError', () => {
+    it('should return true for axios errors', () => {
+      const axiosError = new AxiosError('Test error');
+      expect(isAxiosError(axiosError)).toBe(true);
+    });
+
+    it('should return false for non-axios errors', () => {
+      const regularError = new Error('Regular error');
+      expect(isAxiosError(regularError)).toBe(false);
+    });
+
+    it('should return false for null/undefined', () => {
+      expect(isAxiosError(null)).toBe(false);
+      expect(isAxiosError(undefined)).toBe(false);
+    });
+  });
+
+  describe('handleError', () => {
+    it('should handle axios errors with response data message', () => {
+      const axiosError = new AxiosError('Request failed');
+      axiosError.response = {
+        status: 500,
+        data: { message: 'Internal server error' },
+      } as any;
+
+      const result = handleError(axiosError);
+
+      expect(result.message).toBe('Unexpected response code 500. Internal server error');
+    });
+
+    it('should handle axios errors without response data message', () => {
+      const axiosError = new AxiosError('Network error');
+      axiosError.response = {
+        status: 404,
+        data: {},
+      } as any;
+
+      const result = handleError(axiosError);
+
+      expect(result.message).toBe('Network error (code: 404)');
+    });
+
+    it('should handle axios errors without response', () => {
+      const axiosError = new AxiosError('Connection timeout');
+
+      const result = handleError(axiosError);
+
+      expect(result.message).toBe('Connection timeout (code: unknown)');
+    });
+
+    it('should handle non-axios errors', () => {
+      const regularError = new Error('Regular error');
+
+      const result = handleError(regularError);
+
+      expect(result.message).toBe('Regular error');
+    });
+
+    it('should handle errors without message', () => {
+      const errorWithoutMessage = {};
+
+      const result = handleError(errorWithoutMessage);
+
+      expect(result.message).toBe('An error has occurred');
+    });
+  });
+});

package/src/domains/job/services/JobUploadService/utils/errorUtils.ts

@@ -0,0 +1,35 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+/**
+ * Check if error is an AxiosError
+ * @param error - The error to check
+ * @returns true if it's an AxiosError, false otherwise
+ */
+export function isAxiosError(error: any): boolean {
+  return Boolean(error && error.isAxiosError === true);
+}
+
+/**
+ * Handle errors from API calls
+ * @param error - The error to handle
+ * @returns A formatted Error object
+ */
+export function handleError(error: any): Error {
+  const { response } = error;
+  const errorMessage = error.message;
+  const newError = new Error(errorMessage || 'An error has occurred');
+
+  if (isAxiosError(error)) {
+    if (response?.data?.message) {
+      newError.message = `Unexpected response code ${response.status}. ${response.data.message}`;
+    } else {
+      newError.message = `${errorMessage} (code: ${response?.status || 'unknown'})`;
+    }
+  }
+
+  newError.stack = error.stack;
+  return newError;
+}

package/src/domains/job/services/JobUploadService/utils/index.ts

@@ -0,0 +1,15 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+
+/* istanbul ignore file */
+
+export { cleanupTmpDir } from './cleanup';
+export { handleError, isAxiosError } from './errorUtils';
+export { getAndValidateManifest } from './manifestValidator';
+export { handleResponse, isAxiosResponse } from './responseHandler';
+export { pollForResult } from './resultPoller';
+export { watchAndWaitForUploadAndScanComplete } from './scanWatcher';
+export { initializeUpload, type InitialUpload } from './uploadInitializer';
+export { checkIfVersionExists } from './versionChecker';