@squiz/job-runner-lib 1.66.2 → 2.1.0

package/src/domains/manifest/services/ComponentFunctionService.ts

@@ -1,178 +0,0 @@
-import path from 'node:path/posix';
-import { ManifestModel, Manifest } from '../ManifestModel';
-import { BadRequestError, never } from '@squiz/dx-common-lib';
-import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
-import {
-  ComponentSchemaValidationService,
-  ComponentSchemaValidationServiceFactory,
-} from './ComponentSchemaValidationService';
-import {
-  ComponentInputValidationService,
-  ComponentInputValidationServiceFactory,
-} from './ComponentInputValidationService';
-
-type FunctionOutputTypes = ManifestModel['functions'][number]['output'];
-type ComponentFunction = MANIFEST_MODELS.v1.ComponentFunction;
-type HtmlStaticFile = MANIFEST_MODELS.v1.HtmlStaticFile;
-
-export class ComponentFunctionService {
-  private schemaValidationService: ComponentSchemaValidationService = ComponentSchemaValidationServiceFactory.create();
-  private inputValidationService: ComponentInputValidationService = ComponentInputValidationServiceFactory.create();
-
-  static buildFileUrl(
-    filepath: string,
-    manifest: Manifest,
-    params: { _componentSet: string } | { _previewKey: string },
-    rootUrl: string,
-  ): string {
-    try {
-      // URL.canParse is added in node 19.9. Instead catch an invalid URL error
-      const url = new URL(filepath);
-      return url.href;
-    } catch (_) {
-      const searchParams = new URLSearchParams(params);
-      const normalizedFilepath = path
-        .normalize(filepath)
-        .replace(/^[/.]+/, '')
-        .replace(/\?.*$/, '');
-      return `${rootUrl.replace(/\/$/, '')}/s/${manifest.id}/${normalizedFilepath}?${searchParams.toString()}`;
-    }
-  }
-
-  public getFunctionDefinitionFromManifest(functionName: string, manifest: Manifest): ComponentFunction | undefined {
-    const componentFunctionDefinition = manifest.getModel().functions.find((f) => f.name === functionName);
-
-    return componentFunctionDefinition;
-  }
-
-  public prepareComponentFunctionForWebResponse(
-    func: ComponentFunction,
-    manifest: Manifest,
-    componentSet: string,
-    rootUrl: string,
-  ): ComponentFunction {
-    if (func.output.responseType === 'html') {
-      func.output.staticFiles = func.output.staticFiles?.map((f: HtmlStaticFile) => ({
-        ...f,
-        file: {
-          ...f.file,
-          filepath: ComponentFunctionService.buildFileUrl(
-            f.file.filepath,
-            manifest,
-            { _componentSet: componentSet },
-            rootUrl,
-          ),
-        },
-      }));
-    }
-
-    return func;
-  }
-
-  public prepareManifestForWebResponse(manifest: Manifest, componentSet: string, rootUrl: string): ManifestModel {
-    const manifestModel = manifest.getModel();
-    const manifestForWeb: ManifestModel = {
-      ...manifestModel,
-      functions: manifestModel.functions.map((f) =>
-        this.prepareComponentFunctionForWebResponse(f, manifest, componentSet, rootUrl),
-      ),
-    };
-    return manifestForWeb;
-  }
-
-  public prepareComponentFunctionForPreview(
-    func: ComponentFunction,
-    manifest: Manifest,
-    previewKey: string,
-    rootUrl: string,
-  ): ComponentFunction {
-    if (func.output.responseType === 'html') {
-      func.output.staticFiles = func.output.staticFiles?.map((f: HtmlStaticFile) => ({
-        ...f,
-        file: {
-          ...f.file,
-          filepath: ComponentFunctionService.buildFileUrl(
-            f.file.filepath,
-            manifest,
-            { _previewKey: previewKey },
-            rootUrl,
-          ),
-        },
-      }));
-    }
-    return func;
-  }
-
-  public prepareManifestForPreview(manifest: Manifest, previewKey: string, rootUrl: string): ManifestModel {
-    const manifestModel = manifest.getModel();
-    const manifestForPreview: ManifestModel = {
-      ...manifestModel,
-      functions: manifestModel.functions.map((f) =>
-        this.prepareComponentFunctionForPreview(f, manifest, previewKey, rootUrl),
-      ),
-    };
-
-    return manifestForPreview;
-  }
-
-  /**
-   * Asserts the provided input matches the input schema of the component function before
-   * any resolution through the render pipeline.
-   * @param input Input value which has not been resolved through render pipeline
-   * @param componentFunctionDefinition
-   */
-  public assertInputValidForComponentFunction(input: unknown, componentFunctionDefinition: ComponentFunction) {
-    if (input === undefined) {
-      throw new BadRequestError(`input value for ${componentFunctionDefinition.name} cannot be undefined`);
-    }
-
-    this.schemaValidationService.validateInput(input, componentFunctionDefinition.input);
-  }
-
-  /**
-   * Asserts the provided input matches the input schema of the component function after
-   * resolution through the render pipeline.
-   * @param input Input value which has been resolved through render pipeline
-   * @param componentFunctionDefinition
-   */
-  public assertRenderInputValidForComponentFunction(input: unknown, componentFunctionDefinition: ComponentFunction) {
-    if (input === undefined) {
-      throw new BadRequestError(`input value for ${componentFunctionDefinition.name} cannot be undefined`);
-    }
-    this.inputValidationService.validateInput(input, componentFunctionDefinition.input);
-  }
-
-  public assertOutputValidForComponentFunction(output: unknown, componentFunctionDefinition: ComponentFunction) {
-    if (componentFunctionDefinition === undefined) {
-      throw new Error(`input value for ${componentFunctionDefinition} cannot be undefined`);
-    }
-    if (output === undefined) {
-      throw new Error(`input value for ${componentFunctionDefinition.name} cannot be undefined`);
-    }
-
-    const outputSchema = this.getOutputValidationSchema(componentFunctionDefinition.output);
-    this.schemaValidationService.validateInput(output, outputSchema);
-  }
-
-  protected getOutputValidationSchema(func: FunctionOutputTypes): object {
-    const responseType = func.responseType;
-
-    switch (responseType) {
-      case 'html':
-        return { type: 'string', minLength: 0 };
-
-      case 'json': {
-        const jsonFunc = func;
-
-        if (jsonFunc.definition) {
-          return jsonFunc.definition;
-        }
-
-        throw new Error('definition is missing for the json');
-      }
-
-      default:
-        never(responseType);
-    }
-  }
-}

package/src/domains/manifest/services/ComponentInputValidationService.ts

@@ -1,22 +0,0 @@
-import {
-  JSONSchemaService,
-  SquizImageType,
-  SquizLinkType,
-  TypeResolver,
-  ComponentInputFormattedTextType,
-  ComponentInputMetaSchema,
-} from '@squiz/dx-json-schema-lib';
-
-export type ComponentInputValidationService = JSONSchemaService<
-  SquizImageType | ComponentInputFormattedTextType | SquizLinkType,
-  never
->;
-
-export class ComponentInputValidationServiceFactory {
-  public static create(): ComponentInputValidationService {
-    return new JSONSchemaService(
-      new TypeResolver([SquizImageType, ComponentInputFormattedTextType, SquizLinkType]),
-      ComponentInputMetaSchema,
-    );
-  }
-}

package/src/domains/manifest/services/ManifestServiceForProduction.spec.ts

@@ -1,90 +0,0 @@
-import { getLogger } from '@squiz/dx-logger-lib';
-import path from 'path';
-import { ManifestServiceForProduction } from './ManifestServiceForProduction';
-
-const COMPONENT_ROOT = process.env.DATA_MOUNT_POINT!;
-
-describe('ManifestServiceForProduction', () => {
-  const service = new ManifestServiceForProduction(COMPONENT_ROOT, getLogger({ name: 'ManifestServiceForProduction' }));
-
-  it('should return a path in the format of <root>/<namespace>/<component name>/<version>', async () => {
-    await expect(service.getManifestPath('foo', 'my-component', '1.0.0')).resolves.toEqual(
-      path.join(COMPONENT_ROOT, 'foo', 'my-component', '1.0.0', 'manifest.json'),
-    );
-  });
-
-  it('finds all components in directory', async () => {
-    const resolvedComponents = await service.listAllComponentManifests();
-
-    const allowedPrefixes = [
-      'test-component',
-      'test-environment-variables',
-      'test-malicious-code',
-      'test-npm-modules',
-      'test-static-files',
-    ];
-
-    expect(
-      resolvedComponents
-        .filter((a) => allowedPrefixes.some((b) => a.getName().startsWith(`unit-test-components/${b}`)))
-        .map((a) => `${a.getName()} ${a.getVersion()}`)
-        .sort(),
-    ).toEqual([
-      'unit-test-components/test-component 1.0.0',
-      'unit-test-components/test-component 1.0.10',
-      'unit-test-components/test-component 1.0.3',
-      'unit-test-components/test-component 1.0.4',
-      'unit-test-components/test-component 1.0.5',
-      'unit-test-components/test-component 1.0.6',
-      'unit-test-components/test-component 1.0.7',
-      'unit-test-components/test-component 1.0.8',
-      'unit-test-components/test-component 1.0.9',
-      'unit-test-components/test-component 1.1.0',
-      'unit-test-components/test-component 1.1.2',
-      'unit-test-components/test-component 1.1.4',
-      'unit-test-components/test-component 1.2.0',
-      'unit-test-components/test-component 1.3.0',
-      'unit-test-components/test-environment-variables 1.0.0',
-      'unit-test-components/test-environment-variables 1.0.1',
-      'unit-test-components/test-environment-variables 1.0.2',
-      'unit-test-components/test-environment-variables 1.0.3',
-      'unit-test-components/test-environment-variables 1.0.4',
-      'unit-test-components/test-malicious-code 1.0.0',
-      'unit-test-components/test-malicious-code 1.1.0',
-      'unit-test-components/test-malicious-code 1.2.0',
-      'unit-test-components/test-malicious-code 1.3.0',
-      'unit-test-components/test-npm-modules 1.0.0',
-      'unit-test-components/test-npm-modules 2.0.0',
-      'unit-test-components/test-npm-modules 3.0.0',
-      'unit-test-components/test-npm-modules 3.1.0',
-      'unit-test-components/test-static-files 1.0.0',
-      'unit-test-components/test-static-files 1.0.1',
-      'unit-test-components/test-static-files 1.0.2',
-      'unit-test-components/test-static-files 1.0.3',
-      'unit-test-components/test-static-files 1.0.4',
-      'unit-test-components/test-static-files 1.0.5',
-      'unit-test-components/test-static-files 1.0.6',
-      'unit-test-components/test-static-files 1.0.7',
-      'unit-test-components/test-static-files 1.0.8',
-    ]);
-  });
-
-  describe('readManifest', () => {
-    it('should ignore manifest location validation if validateLocation parameter is false', async () => {
-      // this package has version mismatch
-      const manifestPath = await service.getManifestPath('unit-test-components', 'test-component', '1.0.1');
-
-      const manifest = await service.readManifest(manifestPath);
-
-      expect(manifest.getVersion()).toEqual('0.0.0');
-    });
-
-    it('should ignore manifest location validation if validateLocation parameter is false  - invalid namespace', async () => {
-      // this package has version mismatch
-      const manifestPath = await service.getManifestPath('unit-test-components', 'test-component', '1.1.3');
-
-      const manifest = await service.readManifest(manifestPath);
-      expect(manifest.getNamespace()).toEqual('foo');
-    });
-  });
-});

package/src/domains/manifest/services/ManifestServiceForProduction.ts

@@ -1,98 +0,0 @@
-import path from 'path';
-import fsp from 'fs/promises';
-
-import { AbstractManifestService, ManifestValidationOptions } from './AbstractManifestService';
-import { Manifest, ManifestModel } from '../ManifestModel';
-import { ManifestLocationError } from '../errors/ManifestLocationError';
-import { Logger } from '@squiz/dx-logger-lib';
-
-export class ManifestServiceForProduction extends AbstractManifestService {
-  constructor(protected componentMountPoint: string, protected logger: Logger) {
-    super(logger);
-  }
-
-  async listAllComponentManifests(): Promise<Manifest[]> {
-    // TODO: replace this with a DB query and not fs reads
-    //
-    const namespaces = await fsp.readdir(this.componentMountPoint, { encoding: 'utf-8' });
-    const manifests: Manifest[] = [];
-
-    for (const namespace of namespaces) {
-      const namespacePath = path.join(this.componentMountPoint, namespace);
-      if ((await fsp.stat(namespacePath)).isDirectory()) {
-        const componentDirs = await fsp.readdir(namespacePath, { encoding: 'utf-8' });
-
-        for (const componentDir of componentDirs) {
-          try {
-            const componentPath = path.join(namespacePath, componentDir);
-
-            if ((await fsp.stat(componentPath)).isDirectory()) {
-              const versionDirs = await fsp.readdir(componentPath, { encoding: 'utf-8' });
-
-              for (const versionDir of versionDirs) {
-                const directory = path.join(componentPath, versionDir);
-
-                if (await this.isComponentDirectory(directory)) {
-                  try {
-                    const manifestPath = path.join(directory, 'manifest.json');
-                    const manifest = await this.readManifest(manifestPath);
-
-                    await this.assertManifestVersionIsTheSameAsFileSystemVersion(manifestPath, manifest.getModel());
-
-                    manifests.push(manifest);
-                  } catch (e) {
-                    // no-op as we're going to replace this function eventually
-                  }
-                }
-              }
-            }
-          } catch (e) {
-            // no-op as we're going to replace this function eventually
-          }
-        }
-      }
-    }
-
-    return manifests;
-  }
-
-  public async assertManifestIsValid(
-    manifestPath: string,
-    manifest: ManifestModel,
-    options?: ManifestValidationOptions,
-  ): Promise<void> {
-    await super.assertManifestIsValid(manifestPath, manifest, options);
-    if (options?.tempLocation !== true) {
-      await this.assertManifestVersionIsTheSameAsFileSystemVersion(manifestPath, manifest);
-    }
-  }
-
-  async getManifestPath(namespace: string, componentName: string, version: string): Promise<string> {
-    return path.join(this.componentMountPoint, namespace, componentName, version, 'manifest.json');
-  }
-
-  protected async assertManifestVersionIsTheSameAsFileSystemVersion(
-    manifestPath: string,
-    manifestModel: ManifestModel,
-  ) {
-    const manifest = this.getManifest(manifestModel, manifestPath);
-    const calculatedManifestPath = path.join(manifest.getName(), manifest.getVersion(), 'manifest.json');
-    if (!manifestPath.endsWith(calculatedManifestPath)) {
-      const manifestFilePathParts = manifestPath.split(path.sep).reverse();
-      const expectedVersion = manifestFilePathParts[1];
-      const expectedNamespace = manifestFilePathParts[3];
-
-      if (expectedVersion !== manifest.getVersion()) {
-        throw new ManifestLocationError(
-          `manifest version miss match, expected version ${expectedVersion} but found ${manifest.getVersion()}`,
-          manifestModel,
-        );
-      } else {
-        throw new ManifestLocationError(
-          `manifest namespace miss match, expected namespace ${expectedNamespace} but found ${manifest.getNamespace()}`,
-          manifestModel,
-        );
-      }
-    }
-  }
-}

package/src/domains/manifest/utils/assertFuncEntryFilesAreValid.ts

@@ -1,19 +0,0 @@
-import { isPathTryingToAccessOutsideOfRoot, isReadable } from '@squiz/dx-common-lib';
-import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
-import { ManifestModel } from '../ManifestModel';
-
-export async function assertFuncEntryFilesAreValid(
-  componentRoot: string,
-  manifest: ManifestModel,
-  func: MANIFEST_MODELS.v1.ComponentFunction,
-) {
-  if (isPathTryingToAccessOutsideOfRoot(componentRoot, func.entry)) {
-    throw new Error(
-      `access to component "${func.name}'s" entry file "${func.entry}" outside the parent directory in "${manifest.name} ${manifest.version}" is not allowed`,
-    );
-  }
-
-  if (!(await isReadable(componentRoot, func.entry))) {
-    throw new Error(`"${func.name}'s" entry file "${func.entry}" is inaccessible`);
-  }
-}

package/src/domains/manifest/utils/assertStaticFilesAreValid.ts

@@ -1,37 +0,0 @@
-import { ManifestModel } from '../ManifestModel';
-import path from 'path';
-import { isPathTryingToAccessOutsideOfRoot, isReadable } from '@squiz/dx-common-lib';
-import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
-
-export async function assertStaticFilesAreValid(
-  componentRoot: string,
-  manifest: ManifestModel,
-  func: MANIFEST_MODELS.v1.ComponentFunction,
-  staticFileLocationRoot: string,
-) {
-  const staticFileRoot = path.join(componentRoot, staticFileLocationRoot);
-  const staticFiles = (func.output as MANIFEST_MODELS.v1.HtmlResponse).staticFiles || [];
-
-  for (const staticFile of staticFiles) {
-    if (staticFile.file.filepath.startsWith('http://') || staticFile.file.filepath.startsWith('https://')) {
-      continue;
-    }
-
-    const staticFileIsTryingToReadOutsSideOfRoot = isPathTryingToAccessOutsideOfRoot(
-      staticFileRoot,
-      staticFile.file.filepath,
-    );
-
-    if (staticFileIsTryingToReadOutsSideOfRoot) {
-      throw new Error(
-        `access to a static file "${staticFile.file.filepath}" for "${func.name}" outside the parent directory in "${manifest.name} ${manifest.version}" is not allowed`,
-      );
-    }
-
-    const staticFileIsReadable = await isReadable(staticFileRoot, staticFile.file.filepath);
-
-    if (!staticFileIsReadable) {
-      throw new Error(`"${func.name}'s" static file "${staticFile.file.filepath}" is inaccessible`);
-    }
-  }
-}

package/src/domains/manifest/utils/assertStaticFilesHaveAbsolutePath.spec.ts

@@ -1,121 +0,0 @@
-import { ManifestModel } from '../ManifestModel';
-import { assertStaticFilesHaveAbsolutePath } from './assertStaticFilesHaveAbsolutePath';
-
-describe('assetStaticFilesHaveAbsolutePath', () => {
-  it('should not throw an error if the filepaths are all absolute', async () => {
-    const manifestModel: ManifestModel = {
-      $schema: './manifest-schemas/v1.json',
-
-      name: 'hello',
-      version: '1.0.1',
-      namespace: 'foo',
-      displayName: 'some display name',
-      description: 'some description',
-
-      mainFunction: 'hello',
-
-      functions: [
-        {
-          entry: 'whatever.js',
-          name: 'hello',
-
-          input: {
-            type: 'object',
-            properties: {
-              hello: { type: 'string' },
-            },
-
-            required: ['hello'],
-          },
-          output: {
-            responseType: 'html',
-            staticFiles: [
-              {
-                location: 'header',
-                file: {
-                  filepath: 'https://cdn.tailwindcss.com',
-                  type: 'js',
-                },
-              },
-              {
-                location: 'header',
-                file: {
-                  filepath: 'https://cdn.jsdelivr.net/npm/bootstrap@4.3.1/dist/css/bootstrap.min.css',
-                  type: 'js',
-                },
-              },
-              {
-                location: 'header',
-                file: {
-                  filepath: 'https://unpkg.com/react@18/umd/react.production.min.js',
-                  type: 'js',
-                },
-              },
-            ],
-          },
-        },
-      ],
-    };
-    const result = manifestModel.functions.map((func) => {
-      assertStaticFilesHaveAbsolutePath(func);
-    });
-    expect(result).toEqual([undefined]);
-  });
-  it('should throw an error if there are relative static file paths', async () => {
-    const manifestModel: ManifestModel = {
-      $schema: './manifest-schemas/v1.json',
-      name: 'hello',
-      version: '1.0.1',
-      namespace: 'foo',
-      displayName: 'some display name',
-      description: 'some description',
-      mainFunction: 'main',
-      functions: [
-        {
-          entry: 'whatever.js',
-          name: 'main',
-
-          input: {
-            type: 'object',
-            properties: {
-              hello: { type: 'string' },
-            },
-
-            required: ['hello'],
-          },
-          output: {
-            responseType: 'html',
-            staticFiles: [
-              {
-                location: 'header',
-                file: {
-                  filepath: 'https://cdn.tailwindcss.com',
-                  type: 'js',
-                },
-              },
-              {
-                location: 'header',
-                file: {
-                  filepath: '../static.test.css',
-                  type: 'css',
-                },
-              },
-              {
-                location: 'header',
-                file: {
-                  filepath: '../static.test.js',
-                  type: 'js',
-                },
-              },
-            ],
-          },
-        },
-      ],
-    };
-    await expect(
-      assertStaticFilesHaveAbsolutePath(manifestModel.functions[0]),
-    ).rejects.toThrowErrorMatchingInlineSnapshot(
-      `"function main defines static files but manifest has no staticFiles.locationRoot defined"`,
-    );
-  });
-});

package/src/domains/manifest/utils/assertStaticFilesHaveAbsolutePath.ts

@@ -1,15 +0,0 @@
-import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
-
-export async function assertStaticFilesHaveAbsolutePath(func: MANIFEST_MODELS.v1.ComponentFunction) {
-  const staticFiles = (func.output as MANIFEST_MODELS.v1.HtmlResponse).staticFiles || [];
-
-  for (const staticFile of staticFiles) {
-    if (staticFile.file.filepath.startsWith('http://') || staticFile.file.filepath.startsWith('https://')) {
-      continue;
-    } else {
-      throw new Error(
-        `function ${func.name} defines static files but manifest has no staticFiles.locationRoot defined`,
-      );
-    }
-  }
-}