@squiz/job-runner-lib 1.66.1

package/src/domains/manifest/services/AbstractManifestService.ts

@@ -0,0 +1,316 @@
+import { isPathTryingToAccessOutsideOfRoot, isReadable, ResourceNotFoundError } from '@squiz/dx-common-lib';
+import { ManifestValidationError } from '../errors/ManifestValidationError';
+import { ManifestModel, Manifest, ManifestV1Model, JobManifestModel } from '../ManifestModel';
+import { getAndAssertManifestVersion } from '../utils/getAndAssertManifestVersion';
+import { getAndAssertJobManifestVersion } from '../utils/getAndAssertJobManifestVersion';
+import { ManifestV1 } from '../ManifestV1';
+import { JobManifestV1 } from '../../job/manifest/JobManifestV1';
+import { JsonValidationService, MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+import fsExtra from 'fs-extra';
+import { assertFuncEntryFilesAreValid } from '../utils/assertFuncEntryFilesAreValid';
+import { assertStaticFilesAreValid } from '../utils/assertStaticFilesAreValid';
+import path from 'path';
+import { assertStaticFilesHaveAbsolutePath } from '../utils/assertStaticFilesHaveAbsolutePath';
+import { Logger } from '@squiz/dx-logger-lib';
+import { ComponentSchemaValidationServiceFactory } from './ComponentSchemaValidationService';
+import {
+  ComponentInputValidationServiceFactory,
+  ComponentInputValidationService,
+} from './ComponentInputValidationService';
+
+export type ManifestValidationOptions = {
+  tempLocation?: boolean;
+  validateFunction?: string;
+  validateNamespace?: boolean;
+};
+
+export interface ManifestService {
+  listAllComponentManifests(): Promise<Manifest[]>;
+  getManifestPath(namespace: string, componentName: string, version: string): Promise<string>;
+  loadManifest(
+    namespace: string,
+    componentName: string,
+    version: string,
+    validationOptions?: ManifestValidationOptions,
+  ): Promise<Manifest>;
+  readManifest(manifestPath: string): Promise<Manifest>;
+  assertManifestIsValid(
+    manifestPath: string,
+    manifest: ManifestModel,
+    options?: ManifestValidationOptions,
+  ): Promise<void>;
+  isComponentDirectory(directory: string): Promise<boolean>;
+}
+
+export abstract class AbstractManifestService implements ManifestService {
+  protected jsonValidationService = ComponentSchemaValidationServiceFactory.create();
+  protected inputValidationService: ComponentInputValidationService = ComponentInputValidationServiceFactory.create();
+  protected jobValidationService: JsonValidationService = new JsonValidationService();
+
+  constructor(protected logger: Logger) {}
+
+  abstract listAllComponentManifests(): Promise<Manifest[]>;
+  abstract getManifestPath(namespace: string, componentName: string, version: string): Promise<string>;
+
+  public async loadManifest(
+    namespace: string,
+    componentName: string,
+    version: string,
+    validationOptions?: ManifestValidationOptions,
+  ): Promise<Manifest> {
+    let manifest: ManifestV1 | undefined = undefined;
+    try {
+      const manifestPath = await this.getManifestPath(namespace, componentName, version);
+      manifest = await this.readManifest(manifestPath);
+      await this.assertManifestIsValid(manifestPath, manifest.getModel(), validationOptions);
+      return manifest;
+    } catch (e) {
+      if (e instanceof ResourceNotFoundError) {
+        this.logger.error(e);
+        let error = e;
+        if (manifest === undefined) {
+          error = new ResourceNotFoundError(`manifest for ${namespace}/${componentName} ${version} could not be found`);
+        }
+        error.stack = e.stack;
+        throw error;
+      }
+
+      throw e;
+    }
+  }
+
+  public async readManifest(manifestPath: string): Promise<Manifest> {
+    if (await fsExtra.pathExists(manifestPath)) {
+      const manifest = JSON.parse(await fsExtra.readFile(manifestPath, { encoding: 'utf-8' })) as ManifestModel;
+      await this.assertManifestMeetsSchemaVersion(manifest);
+      return this.getManifest(manifest, manifestPath);
+    }
+    throw new ResourceNotFoundError(`manifest could not be found`);
+  }
+
+  /**
+   * Manifest factory
+   * @param manifestModel
+   */
+  protected getManifest(manifestModel: ManifestModel, manifestPath: string): Manifest {
+    const schemaVersion = getAndAssertManifestVersion(manifestModel);
+    if (schemaVersion === 'v1') {
+      return new ManifestV1(manifestModel as ManifestV1Model, manifestPath);
+    }
+    throw new ManifestValidationError(`Unsupported manifest schema version ${schemaVersion}`);
+  }
+
+  public async assertManifestIsValid(
+    manifestPath: string,
+    manifestModel: ManifestModel,
+    options?: ManifestValidationOptions,
+  ): Promise<void> {
+    const manifest = this.getManifest(manifestModel, manifestPath);
+
+    let componentFunction: MANIFEST_MODELS.v1.ComponentFunction | undefined = undefined;
+    if (options && 'validateFunction' in options) {
+      componentFunction = manifest.getComponentFunctionByName(options.validateFunction);
+    }
+    if (options && 'validateNamespace' in options && options.validateNamespace === true) {
+      this.assertUserCanUseNamespace(manifestModel);
+    }
+
+    this.assertThatMainFunctionExists(manifestModel);
+    await this.assertStaticFileLocationNotOutsideOfRoot(manifestPath, manifestModel);
+    await this.assertFuncEntryFilesAreValid(manifestPath, manifestModel, componentFunction);
+    await this.assertStaticFilesAreValid(manifestPath, manifestModel, componentFunction);
+    await this.assertPreviewDataIsValid(manifestPath, manifestModel, manifest, componentFunction);
+    await this.assertMockedUriObjectsAreValid(manifestPath, manifestModel, manifest);
+  }
+
+  protected getJobManifest(manifestModel: JobManifestModel, manifestPath: string): JobManifestV1 {
+    const schemaVersion = getAndAssertJobManifestVersion(manifestModel, JobManifestV1.getJobSchemaVersionRegex());
+    if (schemaVersion === 'JobV1') {
+      return new JobManifestV1(manifestModel, manifestPath);
+    }
+    throw new ManifestValidationError(`Unsupported manifest schema version ${schemaVersion}`);
+  }
+
+  public async readJobManifest(manifestPath: string): Promise<JobManifestV1> {
+    if (await fsExtra.pathExists(manifestPath)) {
+      const manifest = JSON.parse(await fsExtra.readFile(manifestPath, { encoding: 'utf-8' })) as JobManifestModel;
+      await this.assertJobManifestMeetsSchemaVersion(manifest, JobManifestV1.getJobSchemaVersionRegex());
+      return this.getJobManifest(manifest, manifestPath);
+    }
+    throw new ResourceNotFoundError(`manifest could not be found`);
+  }
+
+  protected async assertPreviewDataIsValid(
+    manifestPath: string,
+    manifestModel: ManifestModel,
+    manifest: ManifestV1,
+    componentFunction: MANIFEST_MODELS.v1.ComponentFunction | undefined = undefined,
+  ) {
+    if (!manifestModel.previews) {
+      return;
+    }
+    let previewName = '';
+    try {
+      for (const [previewKey, value] of Object.entries(manifestModel.previews)) {
+        previewName = previewKey;
+        for (const [funcName, funcInput] of Object.entries(value.functionData)) {
+          const functionDefinition = manifest.getComponentFunctionByName(funcName);
+          if (componentFunction === undefined || componentFunction.name === functionDefinition.name) {
+            const previewInput = await manifest.getFunctionPreviewData(funcInput);
+            this.inputValidationService.validateInput(previewInput, functionDefinition.input);
+          }
+        }
+      }
+    } catch (e) {
+      if (e instanceof Error) {
+        const message = `Invalid preview data for preview "${previewName}": ${e.message}`;
+        throw new ManifestValidationError(message, e);
+      }
+      throw e;
+    }
+  }
+
+  protected async assertMockedUriObjectsAreValid(
+    manifestPath: string,
+    manifestModel: ManifestModel,
+    manifest: ManifestV1,
+  ) {
+    if (!manifestModel.mockedUris) {
+      return;
+    }
+    try {
+      for (const [uri, _urlData] of Object.entries(manifestModel.mockedUris)) {
+        const _mockedUriObject = await manifest.getMockedUriObject(uri);
+      }
+    } catch (e) {
+      if (e instanceof Error) {
+        throw new ManifestValidationError(e.message, e);
+      }
+      throw e;
+    }
+  }
+
+  protected assertThatMainFunctionExists(manifest: ManifestModel) {
+    if (!manifest.functions.find((f) => f.name === manifest['mainFunction'])) {
+      throw new ManifestValidationError(
+        `"${manifest.name} ${manifest.version}" main function could not be found in function list`,
+      );
+    }
+  }
+
+  protected assertUserCanUseNamespace(manifest: ManifestModel) {
+    if (manifest.namespace.match(/squiz/gim)) {
+      throw new ManifestValidationError(
+        `Namespace "${manifest.namespace}" is protected. You need to be a member of Squiz organisation and be logged into npm in order to use that namespace.`,
+      );
+    }
+  }
+
+  protected async assertStaticFileLocationNotOutsideOfRoot(manifestPath: string, manifest: ManifestModel) {
+    const componentRootPath = path.resolve(manifestPath, '..');
+    const staticFileLocationRoot = manifest['staticFiles']?.['locationRoot'];
+
+    if (staticFileLocationRoot) {
+      if (isPathTryingToAccessOutsideOfRoot(componentRootPath, staticFileLocationRoot)) {
+        throw new Error(
+          `access to a static file locationRoot outside the parent directory in "${manifest.name} ${manifest.version}" is not allowed`,
+        );
+      }
+
+      const rootIsReadable = await isReadable(componentRootPath, staticFileLocationRoot);
+      if (!rootIsReadable) {
+        throw new Error(
+          `static root ${staticFileLocationRoot} from "${manifest.name} ${manifest.version}" is not accessible`,
+        );
+      }
+    }
+  }
+
+  protected async assertStaticFilesAreValid(
+    manifestPath: string,
+    manifest: ManifestModel,
+    componentFunction: MANIFEST_MODELS.v1.ComponentFunction | undefined = undefined,
+  ) {
+    const componentRootPath = path.resolve(manifestPath, '..');
+
+    if (componentFunction !== undefined) {
+      await this.assertFunctionStaticFilesAreValid(componentFunction, manifest, componentRootPath);
+    }
+    for (const func of manifest.functions) {
+      await this.assertFunctionStaticFilesAreValid(func, manifest, componentRootPath);
+    }
+  }
+
+  private async assertFunctionStaticFilesAreValid(
+    componentFunction: MANIFEST_MODELS.v1.ComponentFunction,
+    manifest: ManifestModel,
+    componentRootPath: string,
+  ) {
+    const staticFileLocationRoot = manifest['staticFiles']?.['locationRoot'];
+
+    if (componentFunction.output['responseType'] === 'html') {
+      if (
+        staticFileLocationRoot === undefined &&
+        componentFunction.output['staticFiles'] &&
+        componentFunction.output['staticFiles'].length > 0
+      ) {
+        try {
+          await assertStaticFilesHaveAbsolutePath(componentFunction);
+        } catch (e) {
+          if (e instanceof Error) {
+            throw new ManifestValidationError(e.message, e);
+          }
+          throw e;
+        }
+      } else if (staticFileLocationRoot) {
+        try {
+          await assertStaticFilesAreValid(componentRootPath, manifest, componentFunction, staticFileLocationRoot);
+        } catch (e) {
+          if (e instanceof Error) {
+            throw new ManifestValidationError(e.message, e);
+          }
+          throw e;
+        }
+      }
+    }
+  }
+
+  protected async assertFuncEntryFilesAreValid(
+    manifestPath: string,
+    manifest: ManifestModel,
+    componentFunction: MANIFEST_MODELS.v1.ComponentFunction | undefined = undefined,
+  ) {
+    const componentRootPath = path.resolve(manifestPath, '..');
+
+    try {
+      if (componentFunction !== undefined) {
+        await assertFuncEntryFilesAreValid(componentRootPath, manifest, componentFunction);
+      } else {
+        for (const func of manifest.functions) {
+          await assertFuncEntryFilesAreValid(componentRootPath, manifest, func);
+        }
+      }
+    } catch (e) {
+      if (e instanceof Error) {
+        throw new ManifestValidationError(e.message, e);
+      }
+      throw e;
+    }
+  }
+
+  protected async assertManifestMeetsSchemaVersion(manifest: ManifestModel) {
+    getAndAssertManifestVersion(manifest);
+
+    this.jsonValidationService.validateInput(manifest);
+  }
+
+  protected async assertJobManifestMeetsSchemaVersion(manifest: JobManifestModel, schemaVersionRegex: RegExp) {
+    getAndAssertJobManifestVersion(manifest, schemaVersionRegex);
+    this.jobValidationService.validateManifest(manifest, 'JobV1');
+  }
+
+  public async isComponentDirectory(directory: string): Promise<boolean> {
+    return fsExtra.pathExists(path.join(directory, 'manifest.json'));
+  }
+}