@squiz/job-runner-lib 1.66.1 → 2.1.0

package/src/domains/manifest/ManifestV1.ts

@@ -1,146 +0,0 @@
-import { ManifestV1Model } from './ManifestModel';
-import { AbstractManifest, SchemaVersions } from './AbstractManifest';
-import { BadRequestError, ResourceNotFoundError } from '@squiz/dx-common-lib';
-import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
-import path from 'path';
-import { pathExists } from 'fs-extra';
-import fsp from 'fs/promises';
-
-export class ManifestV1 extends AbstractManifest {
-  constructor(protected manifest: ManifestV1Model, protected manifestPath: string) {
-    super(manifest);
-  }
-
-  public getSchemaVersion(): SchemaVersions {
-    return 'v1';
-  }
-
-  public getModel(): ManifestV1Model {
-    return this.manifest;
-  }
-
-  public getComponentFunctionByName(fnName: string = ''): MANIFEST_MODELS.v1.ComponentFunction {
-    const manifestModel = this.getModel();
-    if (!fnName) {
-      if (manifestModel.mainFunction) {
-        fnName = manifestModel.mainFunction;
-      } else {
-        throw new ResourceNotFoundError('Component main function is not defined');
-      }
-    }
-    const functionDefinition = manifestModel.functions.find((a) => a.name === fnName);
-
-    if (!functionDefinition) {
-      throw new ResourceNotFoundError(
-        `"${fnName}" function for ${this.getName()} ${this.getVersion()} could not be found`,
-      );
-    }
-    return functionDefinition;
-  }
-
-  public async getComponentFunctionEntryFilePath(
-    componentFunction: MANIFEST_MODELS.v1.ComponentFunction,
-  ): Promise<string> {
-    const componentDirectory = this.getComponentDirectory();
-    const inputPath = path.resolve(path.join(componentDirectory, componentFunction.entry));
-    return inputPath;
-  }
-
-  public getComponentDirectory() {
-    return path.resolve(path.join(this.manifestPath, '..'));
-  }
-
-  public async getManifestPath() {
-    return this.manifestPath;
-  }
-
-  public getFunctionPreviewConfiguration(
-    previewKey: string,
-    functionDefinition: MANIFEST_MODELS.v1.ComponentFunction,
-  ): MANIFEST_MODELS.v1.FunctionPreviewConfiguration | undefined {
-    const previewDefinition = this.manifest.previews?.[previewKey];
-
-    if (previewDefinition === undefined) {
-      let message;
-
-      if (this.manifest.previews) {
-        const validPreviewKeys = Object.keys(this.manifest.previews).join(', ');
-        message = `Component has not defined preview with key "${previewKey}", the defined preview keys for this component are "${validPreviewKeys}"`;
-      } else {
-        message = `Component has no previews defined`;
-      }
-
-      throw new ResourceNotFoundError(message);
-    }
-
-    const functionPreview = previewDefinition.functionData[functionDefinition.name];
-
-    return functionPreview;
-  }
-
-  public async getFunctionPreviewData(previewFunction: MANIFEST_MODELS.v1.FunctionPreviewConfiguration) {
-    switch (previewFunction?.inputData?.type) {
-      case 'file': {
-        const dataFilePath = path.resolve(this.getComponentDirectory(), previewFunction.inputData.path);
-
-        if (path.dirname(dataFilePath).startsWith(this.getComponentDirectory()) === false) {
-          throw new BadRequestError(
-            `Preview inputData file "${previewFunction.inputData.path}" is outside the component directory`,
-          );
-        }
-
-        if ((await pathExists(dataFilePath)) === false) {
-          throw new BadRequestError(`Preview inputData file "${previewFunction.inputData.path}" does not exist`);
-        }
-
-        return this.loadDataFromFile(dataFilePath);
-      }
-      case 'inline':
-        return previewFunction.inputData.value;
-      default:
-        throw new Error('Unknown Preview Configuration Type');
-    }
-  }
-
-  public async getMockedUriObject(uri: string) {
-    const data = this.manifest.mockedUris !== undefined ? this.manifest.mockedUris[uri] : undefined;
-
-    if (data === undefined) {
-      throw new BadRequestError(`Mocked URI object "${uri}" not defined in the manifest`);
-    }
-
-    switch (data.type) {
-      case 'file': {
-        const dataFilePath = path.resolve(this.getComponentDirectory(), data.path);
-
-        if (path.dirname(dataFilePath).startsWith(this.getComponentDirectory()) === false) {
-          throw new BadRequestError(`Mocked URI object file "${data.path}" is outside the component directory`);
-        }
-
-        if ((await pathExists(dataFilePath)) === false) {
-          throw new BadRequestError(`Mocked URI object file "${data.path}" does not exist`);
-        }
-
-        return this.loadDataFromFile(dataFilePath);
-      }
-      case 'inline':
-        return data.value;
-      default:
-        throw new Error('Unknown Mocked URL object type');
-    }
-  }
-
-  protected async loadDataFromFile(filePath: string) {
-    switch (path.extname(filePath)) {
-      case '.json': {
-        const contents = await fsp.readFile(filePath, { encoding: 'utf-8' });
-        return JSON.parse(contents);
-      }
-      case '.js':
-        // TODO: DEVX-679 handle cases where export is a function
-        return require(filePath);
-      default:
-        throw new Error('Unknown filetype to load data');
-    }
-  }
-}

package/src/domains/manifest/errors/ManifestLocationError.ts

@@ -1,9 +0,0 @@
-import { ManifestModel } from '../ManifestModel';
-
-export class ManifestLocationError extends Error {
-  constructor(message: string, public manifest: ManifestModel, error?: Error) {
-    super(message);
-    this.stack = error?.stack;
-    this.name = 'ManifestLocationError';
-  }
-}

package/src/domains/manifest/services/AbstractManifestService.spec.ts

@@ -1,427 +0,0 @@
-import { AbstractManifestService } from './AbstractManifestService';
-
-import path from 'path';
-import { ManifestValidationError } from '../errors/ManifestValidationError';
-import { ResourceNotFoundError } from '@squiz/dx-common-lib';
-import { Manifest } from '../ManifestModel';
-import { getLogger } from '@squiz/dx-logger-lib';
-
-class TestAbstractManifestService extends AbstractManifestService {
-  async listAllComponentManifests(): Promise<Manifest[]> {
-    return [];
-  }
-
-  async getManifestPath(namespace: string, componentName: string, version: string): Promise<string> {
-    return path.join(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'test-components',
-      namespace,
-      componentName,
-      version,
-      'manifest.json',
-    );
-  }
-}
-
-describe('AbstractManifestService', () => {
-  const service = new TestAbstractManifestService(getLogger({ name: 'AbstractManifestService' }));
-
-  describe('v1', () => {
-    it('should be able to load a component manifest', async () => {
-      const manifest = await service.loadManifest('unit-test-components', 'test-component', '1.0.0');
-      expect(manifest.getName()).toEqual('unit-test-components/test-component');
-      expect(manifest.getVersion()).toEqual('1.0.0');
-      expect(manifest.getSchemaVersion()).toEqual('v1');
-      expect(manifest.getModel()).toMatchInlineSnapshot(`
-        {
-          "$schema": "http://localhost:3000/schemas/v1.json#",
-          "description": "some-description",
-          "displayName": "some-display-name",
-          "functions": [
-            {
-              "entry": "main.js",
-              "input": {
-                "properties": {
-                  "something": {
-                    "type": "string",
-                  },
-                },
-                "required": [
-                  "something",
-                ],
-                "type": "object",
-              },
-              "name": "main",
-              "output": {
-                "responseType": "html",
-              },
-            },
-            {
-              "entry": "promise.js",
-              "input": {
-                "properties": {
-                  "something": {
-                    "type": "string",
-                  },
-                },
-                "required": [
-                  "something",
-                ],
-                "type": "object",
-              },
-              "name": "promise-func",
-              "output": {
-                "responseType": "html",
-              },
-            },
-          ],
-          "mainFunction": "main",
-          "name": "test-component",
-          "namespace": "unit-test-components",
-          "previews": {
-            "preview1": {
-              "functionData": {
-                "main": {
-                  "inputData": {
-                    "type": "inline",
-                    "value": {
-                      "something": "something",
-                    },
-                  },
-                },
-              },
-            },
-          },
-          "version": "1.0.0",
-        }
-      `);
-    });
-
-    it("should throw a ManifestNotFound error if component doesn't exist", async () => {
-      await expect(service.loadManifest('unit-test-components', 'test-not-real', '1.0.0')).rejects.toThrowError(
-        new ManifestValidationError(`manifest for unit-test-components/test-not-real 1.0.0 could not be found`),
-      );
-    });
-
-    describe('validation rules', () => {
-      it('should ensure the $schema is valid', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-component', '1.0.2')).rejects.toThrowError(
-          new ManifestValidationError(
-            'Could not determine manifest version. Your manifest must have a $schema property defined.',
-          ),
-        );
-      });
-
-      it('should return a validation error when the component has inaccessible entry file', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-component', '1.0.3')).rejects.toThrowError(
-          new ManifestValidationError('"main\'s" entry file "missing-entry-file.js" is inaccessible'),
-        );
-      });
-
-      it('should throw an error if the main functions does not exist', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-component', '1.0.5')).rejects.toThrowError(
-          new ManifestValidationError('"test-component 1.0.5" main function could not be found in function list'),
-        );
-      });
-
-      it('should throw an error if entry file trying to access outside the entry point', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-component', '1.0.6')).rejects.toThrowError(
-          new ManifestValidationError(
-            'access to component "file-accessing-outside-parent-func\'s" entry file "../1.0.4/render-json.js" outside the parent directory in "test-component 1.0.6" is not allowed',
-          ),
-        );
-      });
-
-      it('should throw an error if component has invalid name', async () => {
-        await expect(
-          service.loadManifest('unit-test-components', 'test-component', '1.1.1'),
-        ).rejects.toThrowErrorMatchingInlineSnapshot(
-          `"failed validation: Value in \`#/functions/0/name\` should match \`^[a-z][a-z0-9_\\-]+$\`, but received \`invalid-function-name-!!!!\`"`,
-        );
-      });
-
-      it('should throw an error if component defined static files in function but has no static-root defined', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.6')).rejects.toThrowError(
-          new ManifestValidationError(
-            'function main defines static files but manifest has no staticFiles.locationRoot defined',
-          ),
-        );
-      });
-
-      it('should throw an error if component defined static files in function are not all absolute paths but has no static-root defined', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.5')).rejects.toThrowError(
-          new ManifestValidationError(
-            'function main defines static files but manifest has no staticFiles.locationRoot defined',
-          ),
-        );
-      });
-
-      it('should not throw an error if component defined static files in function are all absolute paths and has no static-root defined', async () => {
-        const manifestPath = path.resolve(
-          `${process.env.DATA_MOUNT_POINT}/unit-test-components/test-static-files/1.0.7/manifest.json`,
-        );
-
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.7')).resolves
-          .toMatchInlineSnapshot(`
-          ManifestV1 {
-            "manifest": {
-              "$schema": "http://localhost:3000/schemas/v1.json#",
-              "description": "some-description",
-              "displayName": "some-display-name",
-              "functions": [
-                {
-                  "entry": "main.js",
-                  "input": {},
-                  "name": "main",
-                  "output": {
-                    "responseType": "html",
-                    "staticFiles": [
-                      {
-                        "file": {
-                          "filepath": "https://cdn.tailwindcss.com",
-                          "type": "js",
-                        },
-                        "location": "header",
-                      },
-                    ],
-                  },
-                },
-              ],
-              "mainFunction": "main",
-              "name": "test-static-files",
-              "namespace": "unit-test-components",
-              "version": "1.0.7",
-            },
-            "manifestPath": "${manifestPath}",
-          }
-        `);
-      });
-
-      it('should throw an error if component has both a static file with absolute path and a relative path but has no static-root defined', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.8')).rejects.toThrowError(
-          new ManifestValidationError(
-            'function main defines static files but manifest has no staticFiles.locationRoot defined',
-          ),
-        );
-      });
-
-      // static files
-      it("should throw an error if fuction's static file is trying to access outside of root", async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.4')).rejects.toThrowError(
-          new ManifestValidationError(
-            'access to a static file "../../static-files-library/static-library-file.js" for "main" outside the parent directory in "test-static-files 1.0.4" is not allowed',
-          ),
-        );
-      });
-
-      it('should throw an error if static root is trying to access outside of root', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.3')).rejects.toThrowError(
-          new ManifestValidationError(
-            'access to a static file locationRoot outside the parent directory in "test-static-files 1.0.3" is not allowed',
-          ),
-        );
-      });
-
-      it("should throw an error if static root is defined but doesn't exist on disk", async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-static-files', '1.0.2')).rejects.toThrowError(
-          new ManifestValidationError(
-            '"main\'s" static file "static-files-library/static-file-does-not-exist.js" is inaccessible',
-          ),
-        );
-      });
-
-      it('should load a manifest containing an icon with valid properties color is enum', async () => {
-        const manifest = await service.loadManifest('unit-test-components', 'test-icon-component', '1.0.0');
-        expect(manifest.getModel().icon).toEqual({ id: 'add_box', color: { type: 'enum', value: 'gray' } });
-      });
-
-      it('should load a manifest containing an icon with valid properties color is hex', async () => {
-        const manifest = await service.loadManifest('unit-test-components', 'test-icon-component', '1.0.3');
-        const expectedResult = {
-          color: {
-            type: 'hex',
-            value: '#2D2D2D',
-          },
-          id: 'zoom_in',
-        };
-        expect(manifest.getModel().icon).toEqual(expectedResult);
-      });
-
-      it('should throw an error when loading manifest containing an icon with invalid properties', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-icon-component', '1.0.1')).rejects
-          .toThrowErrorMatchingInlineSnapshot(`
-          "failed validation: Expected given value \`not_a_valid_id\` in #/icon/id\` to be one of \`[10k, 10mp, 11mp, 123, 12mp, ... 2233 more]\`,
-          Value \`{"type":"hex","value":"(255,255,255)"}\` in \`#/icon/color\` does not match any given oneof schema"
-        `);
-      });
-
-      it('should throw an error when loading manifest containing an icon missing required properties', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-icon-component', '1.0.2')).rejects
-          .toThrowErrorMatchingInlineSnapshot(`
-          "failed validation: Expected given value \`not_a_valid_id\` in #/icon/id\` to be one of \`[10k, 10mp, 11mp, 123, 12mp, ... 2233 more]\`,
-          The required property \`color\` is missing at \`#/icon\`"
-        `);
-      });
-
-      it('should throw error if preview data is not valid', async () => {
-        await expect(service.loadManifest('unit-test-components', 'test-preview-component', '1.0.0')).rejects
-          .toThrowErrorMatchingInlineSnapshot(`
-          "Invalid preview data for preview "invalid-preview-data": failed validation: Expected \`not-an-integer\` (string) in \`#/integerValue\` to be of type \`number\`,
-          The required property \`nested\` is missing at \`#\`"
-        `);
-      });
-
-      it('should throw error if mocked URI data is not valid', async () => {
-        await expect(
-          service.loadManifest('unit-test-components', 'test-preview-component', '1.1.2'),
-        ).rejects.toThrowErrorMatchingInlineSnapshot(
-          `"Mocked URI object file "./non-existent-uri.json" does not exist"`,
-        );
-      });
-
-      it('should not throw error if checking a specific function that is valid in otherwise invalid manifset', async () => {
-        // note that this test scenario cannot happen in prod as one cannot deploy an invalid manifest
-        // this test just ensures that loading manifest with "validateFunction" validation option
-        // will just validate the bits specific to the given function
-        await expect(
-          service.loadManifest('unit-test-components', 'test-component', '1.2.0'),
-        ).rejects.toThrowErrorMatchingInlineSnapshot(
-          `"Invalid preview data for preview "test-preview": failed validation: The required property \`something\` is missing at \`#\`"`,
-        );
-
-        // now try loading again with "validateFunction" validation option
-        // this manifest bits relevant to this function are valid
-        const manifest = await service.loadManifest('unit-test-components', 'test-component', '1.2.0', {
-          validateFunction: 'other',
-        });
-        expect(manifest.getName()).toEqual('unit-test-components/test-component');
-        expect(manifest.getVersion()).toEqual('1.2.0');
-      });
-
-      it('should handle protected namespaces with validation options', async () => {
-        const manifest = await service.loadManifest('unit-test-components', 'test-component', '1.4.0');
-        expect(manifest.getModel().namespace).toEqual('squiz');
-
-        await expect(
-          service.loadManifest('unit-test-components', 'test-component', '1.4.0', { validateNamespace: true }),
-        ).rejects.toThrowErrorMatchingInlineSnapshot(
-          `"Namespace "squiz" is protected. You need to be a member of Squiz organisation and be logged into npm in order to use that namespace."`,
-        );
-      });
-    });
-
-    describe('readManifest', () => {
-      it('should read and validate a manifest from a file path', async () => {
-        const p = path.join(
-          __dirname,
-          '../../../../../..',
-          'test-components/unit-test-components/test-component/1.0.0/manifest.json',
-        );
-
-        await expect(service.readManifest(p)).resolves.toThrowErrorMatchingInlineSnapshot(`undefined`);
-      });
-
-      it("should throw a ResourceNotFoundError error if manifest doesn't exist", async () => {
-        const p = path.join(
-          __dirname,
-          '../../../../../..',
-          'test-components/unit-test-components/not-real/manifest.json',
-        );
-
-        await expect(service.readManifest(p)).rejects.toThrowError(
-          new ResourceNotFoundError(`manifest could not be found`),
-        );
-      });
-    });
-  });
-  describe('JobV1', () => {
-    it('should read and validate a job manifest from a file path', async () => {
-      const p = path.join(__dirname, '../../../../../..', 'test-jobs/unit-test-jobs/test-job/1.0.0/manifest.json');
-      const res = await service.readJobManifest(p);
-
-      expect(res).toMatchInlineSnapshot(`
-      JobManifestV1 {
-        "manifest": {
-          "$schema": "http://localhost:3040/schemas/JobV1.json#",
-          "concurrency": 2,
-          "description": "some-description",
-          "displayName": "Test Job",
-          "functions": [
-            {
-              "entry": "main.js",
-              "input": {
-                "properties": {
-                  "something": {
-                    "type": "string",
-                  },
-                },
-                "required": [
-                  "something",
-                ],
-                "type": "object",
-              },
-              "name": "main",
-            },
-            {
-              "entry": "promise.js",
-              "input": {
-                "properties": {
-                  "something": {
-                    "type": "string",
-                  },
-                },
-                "required": [
-                  "something",
-                ],
-                "type": "object",
-              },
-              "name": "promise-func",
-            },
-          ],
-          "mainFunction": "main",
-          "name": "test-job",
-          "timeout": 600,
-          "version": "1.0.0",
-        },
-        "manifestPath": "${p}",
-      }
-    `);
-    });
-
-    it("should throw a ResourceNotFoundError error if manifest doesn't exist", async () => {
-      const p = path.join(__dirname, '../../../../../..', 'test-jobs/unit-test-jobs/not-real/manifest.json');
-
-      await expect(service.readJobManifest(p)).rejects.toThrowError(
-        new ResourceNotFoundError(`manifest could not be found`),
-      );
-    });
-    it("should error if manifest doesn't meet schema version", async () => {
-      const p = path.join(
-        __dirname,
-        '../../../../../..',
-        'test-jobs/unit-test-jobs/invalid-schema-job/1.0.0/manifest.json',
-      );
-
-      await expect(service.readJobManifest(p)).rejects.toThrowError(
-        new ManifestValidationError(`Could not determine manifest version from $schema value.`),
-      );
-    });
-  });
-  it('should error if manifest is invalid', async () => {
-    const p = path.join(
-      __dirname,
-      '../../../../../..',
-      'test-jobs/unit-test-jobs/invalid-manifest-job/1.0.0/manifest.json',
-    );
-
-    await expect(service.readJobManifest(p)).rejects.toThrowError(
-      new ManifestValidationError('failed validation: The required property `concurrency` is missing at `#`'),
-    );
-  });
-});