@squiz/component-cli-lib 1.39.0 → 1.39.1-alpha.1

package/lib/integration-tests/helper.d.ts

@@ -5,6 +5,7 @@ interface Config {
     renderServiceUrl: string;
     contentServiceUrl: string;
     tenantId: string;
+    jobServiceUrl: string;
     ci_buildVersion: string;
     ci_buildBranch: string;
 }
@@ -15,9 +16,11 @@ export declare const managementService: import("axios").AxiosInstance;
 export declare const managementServiceRoot: import("axios").AxiosInstance;
 export declare const renderService: import("axios").AxiosInstance;
 export declare const contentService: import("axios").AxiosInstance;
+export declare const jobService: import("axios").AxiosInstance;
 export declare const ci_buildVersion: string;
 export declare const ci_buildBranch: string;
 export declare function getTestComponents(): Promise<import("@squiz/component-lib").ManifestV1[]>;
+export declare function getTestJobs(): Promise<import("@squiz/component-lib").ManifestV1[]>;
 export declare function createFile(filePath: string, sizeInMB: number): Promise<void>;
 export declare function removeFile(filePath: string): void;
 export declare function deleteComponentSet(webPath: string): Promise<void>;
@@ -26,3 +29,5 @@ export declare function addContentItem(contentItem: ContentApi.ContentItemWebMod
 export declare function deleteContentItem(contentItemId: string): Promise<void>;
 export declare function deleteComponent(manifest: Manifest): Promise<void>;
 export declare function deleteComponents(manifests: Manifest[]): Promise<void>;
+export declare function deleteJob(manifest: Manifest): Promise<void>;
+export declare function deleteJobs(manifests: Manifest[]): Promise<void>;

package/lib/upload-job.d.ts

@@ -0,0 +1,4 @@
+import { AxiosInstance } from 'axios';
+import { Logger } from '@squiz/dx-logger-lib';
+export declare const logger: Logger;
+export declare function uploadJobFolder(apiClient: AxiosInstance, folderPath: string, baseTempDir?: string): Promise<void>;

package/lib/utils.d.ts

@@ -0,0 +1,7 @@
+import { AxiosResponse, AxiosError, AxiosInstance } from 'axios';
+export declare function watchAndWaitForUploadAndScanComplete(apiClient: AxiosInstance, endpoint: string, id: string, managementURL?: string): Promise<void>;
+export declare function checkIfVersionExists(apiClient: AxiosInstance, endpoint: string, managementURL?: string): Promise<boolean>;
+export declare function handleResponse<T>(axiosInstance: Promise<AxiosResponse<T>>): Promise<T>;
+export declare function handleError(error: any): Error;
+export declare function isAxiosError(error: any): error is AxiosError;
+export declare function isAxiosResponse(response: unknown): response is AxiosResponse;

package/lib/utils.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@squiz/component-cli-lib",
-  "version": "1.39.0",
+  "version": "1.39.1-alpha.1",
   "description": "",
   "main": "lib/index.js",
   "scripts": {
@@ -13,12 +13,12 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@squiz/component-lib": "1.39.0",
-    "@squiz/component-web-api-lib": "1.39.0",
-    "@squiz/dx-common-lib": "1.39.0",
-    "@squiz/dx-json-schema-lib": "1.39.0",
-    "@squiz/dx-logger-lib": "1.39.0",
-    "@squiz/virus-scanner-lib": "1.39.0",
+    "@squiz/component-lib": "1.39.1-alpha.1",
+    "@squiz/component-web-api-lib": "1.39.1-alpha.1",
+    "@squiz/dx-common-lib": "1.39.1-alpha.1",
+    "@squiz/dx-json-schema-lib": "1.39.1-alpha.1",
+    "@squiz/dx-logger-lib": "1.39.1-alpha.1",
+    "@squiz/virus-scanner-lib": "1.39.1-alpha.1",
     "@types/cli-color": "2.0.2",
     "@types/express": "4.17.17",
     "@types/jest": "28.1.8",
@@ -32,12 +32,12 @@
     "typescript": "4.9.4"
   },
   "dependencies": {
-    "@squiz/render-runtime-lib": "1.39.0",
+    "@squiz/render-runtime-lib": "1.39.1-alpha.1",
     "archiver": "5.3.1",
     "axios": "1.3.2",
     "cli-color": "^2.0.2",
     "open": "^8.4.0",
     "supertest": "^6.2.3"
   },
-  "gitHead": "ade72ff2cba5a65be6010587a58e8f49153fc806"
+  "gitHead": "16b9f02b751a923a71f96684bd46c781fc97dcbc"
 }

package/src/index.ts

@@ -1,3 +1,4 @@
 export { uploadComponentFolder } from './upload-component-folder';
 export { startDevelopmentRender } from './component-dev';
 export { default as componentInit } from './component-init';
+export { uploadJobFolder } from './upload-job';

package/src/integration-tests/__jobs__/invalid-manifest/main.js

@@ -0,0 +1,3 @@
+module.exports = async function (input, info) {
+  return 'this is a test job';
+};

package/src/integration-tests/__jobs__/invalid-manifest/manifest.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "http://localhost:3040/schemas/JobV1.json#",
+  "name": "invalid manif@st",
+  "version": "1.0.0",
+  "mainFunction": "main",
+  "displayName": "some-displayName",
+  "description": "some-description",
+  "concurrency": 1,
+  "timeout": 10000,
+  "functions": [
+    {
+      "entry": "main.js",
+      "name": "main",
+      "input": {
+        "type": "object",
+        "properties": {
+          "something": {
+            "type": "string"
+          }
+        },
+        "required": ["something"]
+      },
+      "output": {
+        "responseType": "html"
+      }
+    }
+  ]
+}

package/src/integration-tests/__jobs__/invalid-upload/main.js

@@ -0,0 +1,3 @@
+module.exports = async function (input, info) {
+  return 'this is a test job';
+};

package/src/integration-tests/__jobs__/simple-job/main.js

@@ -0,0 +1,3 @@
+module.exports = async function (input, info) {
+  return 'this is a test job';
+};

package/src/integration-tests/__jobs__/simple-job/manifest.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "http://localhost:3040/schemas/JobV1.json#",
+  "name": "simple-job",
+  "displayName": "Simple Job",
+  "version": "1.0.2",
+  "mainFunction": "main",
+  "description": "some-description",
+  "concurrency": 1,
+  "timeout": 10000,
+  "functions": [
+    {
+      "entry": "main.js",
+      "name": "main",
+      "input": {
+        "type": "object",
+        "properties": {
+          "something": {
+            "type": "string"
+          }
+        },
+        "required": ["something"]
+      }
+    }
+  ]
+}

package/src/integration-tests/helper.ts

@@ -17,6 +17,7 @@ interface Config {
   renderServiceUrl: string;
   contentServiceUrl: string;
   tenantId: string;
+  jobServiceUrl: string;
   ci_buildVersion: string;
   ci_buildBranch: string;
 }
@@ -26,6 +27,7 @@ const configObj: Config = {
   renderServiceUrl: parseEnvVarForVar('COMPONENT_RENDER_SERVICE_URL').replace(/\/+$/, ''),
   contentServiceUrl: parseEnvVarForVar('CONTENT_API_URL').replace(/\/+$/, ''),
   tenantId: parseEnvVarForVar('TENANT_ID'),
+  jobServiceUrl: parseEnvVarForVar('JOB_SERVICE_URL').replace(/\/+$/, ''),
   ci_buildVersion: parseEnvVarForVar('CI_COMMIT_SHORT_SHA'),
   ci_buildBranch: parseEnvVarForVar('CI_COMMIT_REF_NAME'),
 };
@@ -109,6 +111,13 @@ export const contentService = axios.create({
   },
 });
 
+export const jobService = axios.create({
+  baseURL: configObj.jobServiceUrl,
+  headers: {
+    authorization: authToken,
+  },
+});
+
 export const ci_buildVersion = configObj.ci_buildVersion;
 export const ci_buildBranch = configObj.ci_buildBranch;
 
@@ -117,6 +126,13 @@ export async function getTestComponents() {
   const manifestService = new ManifestServiceForDev(componentsDir, getLogger({ name: 'getTestComponents' }));
   return await manifestService.listAllComponentManifests();
 }
+
+export async function getTestJobs() {
+  const jobsDir = path.join(__dirname, '/__jobs__/');
+  const manifestService = new ManifestServiceForDev(jobsDir, getLogger({ name: 'getTestJobs' }));
+  return await manifestService.listAllComponentManifests();
+}
+
 export async function createFile(filePath: string, sizeInMB: number) {
   const content = randomBytes(sizeInMB * 1000000);
   await fsp.writeFile(`${filePath}`, content);
@@ -174,3 +190,20 @@ export async function deleteComponent(manifest: Manifest) {
 export async function deleteComponents(manifests: Manifest[]) {
   await Promise.all(manifests.map((manifest) => deleteComponent(manifest)));
 }
+
+export async function deleteJob(manifest: Manifest) {
+  try {
+    await jobService.delete(`/job/${manifest.getName()}/${manifest.getVersion()}`);
+  } catch (error) {
+    if ((error as AxiosError).response?.status === 404) {
+      const logger = getLogger({ name: 'delete job' });
+      logger.info('job not found');
+    } else {
+      throw error;
+    }
+  }
+}
+
+export async function deleteJobs(manifests: Manifest[]) {
+  await Promise.all(manifests.map((manifest) => deleteJob(manifest)));
+}

package/src/upload-component-folder.ts

@@ -1,12 +1,13 @@
-import { ScanStatus, uploadFile } from '@squiz/virus-scanner-lib';
+import { uploadFile } from '@squiz/virus-scanner-lib';
 
 import { zipDirectory } from '@squiz/dx-common-lib';
-import { Manifest, ManifestServiceForDev } from '@squiz/component-lib';
+import { ManifestServiceForDev } from '@squiz/component-lib';
 import fsp from 'fs/promises';
 import path from 'path';
-import { AxiosResponse, AxiosError, AxiosInstance } from 'axios';
+import { AxiosInstance } from 'axios';
 import color from 'cli-color';
 import { getLogger, Logger } from '@squiz/dx-logger-lib';
+import { watchAndWaitForUploadAndScanComplete, checkIfVersionExists, handleResponse, isAxiosResponse } from './utils';
 
 export const logger: Logger = getLogger({ name: 'upload-component', format: 'human' });
 
@@ -24,7 +25,15 @@ export async function uploadComponentFolder(
     const manifest = await manifestService.readManifest(manifestPath);
     await manifestService.assertManifestIsValid(manifestPath, manifest.getModel());
 
-    await preUploadChecks(apiClient, componentServiceManagementUrl, manifest);
+    if (
+      await checkIfVersionExists(
+        apiClient,
+        `/v1/component/${manifest.getName()}/${manifest.getVersion()}`,
+        componentServiceManagementUrl,
+      )
+    ) {
+      throw new Error(`Cannot upload component version, ${manifest.getName()} ${manifest.getVersion()} already exists`);
+    }
     logger.info('Initial scanning');
     const zip = await zipDirectory(folderPath, tmpDir);
 
@@ -35,7 +44,12 @@ export async function uploadComponentFolder(
     logger.info(`deployment id: ${initialUpload.id} status: transferring`);
     await uploadFile(initialUpload, zip);
 
-    await watchAndWaitForUploadAndScanComplete(apiClient, componentServiceManagementUrl, initialUpload.id);
+    await watchAndWaitForUploadAndScanComplete(
+      apiClient,
+      '/v1/upload-component/status/',
+      initialUpload.id,
+      componentServiceManagementUrl,
+    );
 
     logger.info(`deployment id: ${initialUpload.id} status: deploying component folder`);
 
@@ -52,7 +66,13 @@ export async function uploadComponentFolder(
             `unknown, retrying to check the upload status ${retriesAllowed} more times`,
           )}`,
         );
-        if (await checkIfVersionExists(apiClient, componentServiceManagementUrl, manifest)) {
+        if (
+          await checkIfVersionExists(
+            apiClient,
+            `/v1/component/${manifest.getName()}/${manifest.getVersion()}`,
+            componentServiceManagementUrl,
+          )
+        ) {
           result = {
             status: 'successful',
             accessLink: 'unknown',
@@ -89,100 +109,3 @@ export async function uploadComponentFolder(
     throw e;
   }
 }
-
-async function preUploadChecks(apiClient: AxiosInstance, managementURL: string, manifest: Manifest) {
-  if (await checkIfVersionExists(apiClient, managementURL, manifest)) {
-    throw new Error(`Cannot upload component version, ${manifest.getName()} ${manifest.getVersion()} already exists`);
-  }
-}
-
-async function checkIfVersionExists(apiClient: AxiosInstance, managementURL: string, inputManifest: Manifest) {
-  try {
-    const response = await apiClient.get(`/v1/component/${inputManifest.getName()}/${inputManifest.getVersion()}`, {
-      validateStatus: null,
-      baseURL: managementURL,
-    });
-
-    if (response.status === 200) {
-      return true;
-    } else if (response.status === 404) {
-      return false;
-    }
-
-    const error = new AxiosError(
-      response?.data?.message || response.statusText,
-      response.status.toString(),
-      undefined,
-      undefined,
-      response,
-    );
-
-    throw handleError(error);
-  } catch (error) {
-    throw handleError(error);
-  }
-}
-
-async function watchAndWaitForUploadAndScanComplete(
-  apiClient: AxiosInstance,
-  managementURL: string,
-  id: string,
-): Promise<void> {
-  const poll = () =>
-    handleResponse<ScanStatus>(apiClient.get('/v1/upload-component/status/' + id, { baseURL: managementURL }));
-
-  return new Promise((resolve, reject) => {
-    const recurse = () =>
-      poll().then(async (req) => {
-        if (req.status == 'Success') {
-          resolve();
-        } else if (req.status == 'Flagged') {
-          reject(new Error('upload has been flagged as a virus'));
-        } else if (req.status == 'Error') {
-          reject(new Error('there has been an error'));
-        } else {
-          setTimeout(async () => {
-            await recurse();
-          }, 1000);
-        }
-      });
-
-    recurse();
-  });
-}
-
-async function handleResponse<T>(axiosInstance: Promise<AxiosResponse<T>>): Promise<T> {
-  try {
-    const response = await axiosInstance;
-    return response.data;
-  } catch (error) {
-    throw handleError(error);
-  }
-}
-
-function handleError(error: any): Error {
-  const { response } = error;
-  const errorMessage = error.message;
-  const newError = new Error(errorMessage || 'An error has occurred');
-
-  if (isAxiosError(error)) {
-    const errorCode = response.status ?? 'unknown';
-    if (response?.data?.message) {
-      newError.message = `Unexpected response code ${errorCode}. ${response.data.message}`.trim();
-    } else {
-      newError.message = `${errorMessage} (code: ${errorCode})`;
-    }
-  }
-
-  newError.stack = error.stack;
-
-  return newError;
-}
-
-function isAxiosError(error: any): error is AxiosError {
-  return error && error.isAxiosError === true;
-}
-
-function isAxiosResponse(response: unknown): response is AxiosResponse {
-  return response instanceof Object && Object.prototype.hasOwnProperty.call(response, 'status');
-}

package/src/upload-job.ts

@@ -0,0 +1,101 @@
+import { uploadFile } from '@squiz/virus-scanner-lib';
+
+import { zipDirectory } from '@squiz/dx-common-lib';
+import { ManifestServiceForDev, JobManifestV1Model } from '@squiz/component-lib';
+import fsp from 'fs/promises';
+import path from 'path';
+import { AxiosInstance } from 'axios';
+import color from 'cli-color';
+import { getLogger, Logger } from '@squiz/dx-logger-lib';
+import { watchAndWaitForUploadAndScanComplete, handleResponse, checkIfVersionExists, isAxiosResponse } from './utils';
+
+export const logger: Logger = getLogger({ name: 'upload-job', format: 'human' });
+
+export async function uploadJobFolder(
+  apiClient: AxiosInstance,
+  folderPath: string,
+  baseTempDir: string = '',
+): Promise<void> {
+  const tmpDir = await fsp.mkdtemp(path.resolve(baseTempDir, 'job-upload'));
+
+  try {
+    const manifest = await getAndValidateManifest(apiClient, folderPath);
+
+    const initialUpload = await initializeUpload(apiClient, folderPath, tmpDir);
+
+    await uploadFile(initialUpload.upload, initialUpload.zip);
+
+    await watchAndWaitForUploadAndScanComplete(apiClient, '/upload-job/status/', initialUpload.upload.id);
+
+    logger.info(`deployment id: ${initialUpload.upload.id} status: deploying job folder`);
+
+    const result = await pollForResult(apiClient, initialUpload.upload.id, manifest.getModel());
+
+    await cleanupTmpDir(tmpDir);
+
+    if (result.status === 'successful') {
+      logger.info(`deployment id: ${initialUpload.upload.id} status: ${color.green('success')}`);
+    } else {
+      const message = result?.message ?? 'unknown';
+      logger.error(`failed due an unexpected reason: ${message}`);
+      await cleanupTmpDir(tmpDir);
+    }
+  } catch (e) {
+    await cleanupTmpDir(tmpDir);
+    throw e;
+  }
+}
+
+async function initializeUpload(apiClient: AxiosInstance, folderPath: string, tmpDir: string) {
+  logger.info('Initial scanning');
+  const zip = await zipDirectory(folderPath, tmpDir);
+  const initialUpload = { zip: zip, upload: await handleResponse<any>(apiClient.post('/upload-job', {})) };
+  logger.info(`deployment id: ${initialUpload.upload.id} status: transferring`);
+  return initialUpload;
+}
+
+async function getAndValidateManifest(apiClient: AxiosInstance, folderPath: string) {
+  const manifestService = new ManifestServiceForDev(folderPath, logger);
+  const manifestPath = path.join(folderPath, `manifest.json`);
+
+  const manifest = await manifestService.readJobManifest(manifestPath);
+
+  if (await checkIfVersionExists(apiClient, `/job/${manifest.getName()}/${manifest.getVersion()}`)) {
+    throw new Error(`Cannot upload job version, ${manifest.getName()} ${manifest.getVersion()} already exists`);
+  }
+  return manifest;
+}
+
+async function pollForResult(apiClient: AxiosInstance, uploadId: string, manifest: JobManifestV1Model) {
+  let result = await handleResponse<any>(apiClient.post(`/upload-job/next`, { id: uploadId, manifest: manifest }));
+
+  if (!isAxiosResponse(result)) {
+    let retriesAllowed = 12;
+    while (retriesAllowed > 0) {
+      logger.info(
+        `deployment id: ${uploadId} status: ${color.yellow(
+          `unknown, retrying to check the upload status ${retriesAllowed} more times`,
+        )}`,
+      );
+      if (await checkIfVersionExists(apiClient, `/job/${manifest.name}/${manifest.version}`)) {
+        result = {
+          status: 'successful',
+        };
+        break;
+      }
+      await new Promise((r) => setTimeout(r, 10000));
+      retriesAllowed--;
+    }
+    if (retriesAllowed === 0) {
+      logger.error(`deployment id: ${uploadId} status: ${color.red('failed to check the upload status')}`);
+      throw new Error(result.toString());
+    }
+  }
+  return result;
+}
+
+async function cleanupTmpDir(tmpDir: string) {
+  await fsp.rm(tmpDir, { force: true, recursive: true }).catch((error: any) => {
+    logger.error('failed to remove temp directory', error);
+  });
+}