@squiz/job-runner-lib 1.66.1

package/src/domains/manifest/services/ManifestServiceForDev.ts

@@ -0,0 +1,96 @@
+import path from 'path';
+import fsp from 'fs/promises';
+import { AbstractManifestService } from './AbstractManifestService';
+import { Logger } from '@squiz/dx-logger-lib';
+import { Manifest } from '../ManifestModel';
+import { ResourceNotFoundError } from '@squiz/dx-common-lib';
+import { getComponentName } from '../../component/utils/getComponentName';
+
+type ManifestAndPath = { directory: string; manifest: Manifest };
+
+export class ManifestServiceForDev extends AbstractManifestService {
+  protected foundManifests: ManifestAndPath[] = [];
+
+  constructor(protected componentMountPoint: string, protected logger: Logger) {
+    super(logger);
+  }
+
+  async listAllComponentManifests(): Promise<Manifest[]> {
+    if (this.foundManifests.length === 0) {
+      this.foundManifests = await this.recursivelyLookForManifests(this.componentMountPoint);
+    }
+
+    return this.foundManifests.map((a) => a.manifest);
+  }
+
+  async getManifestPath(namespace: string, componentName: string, version: string): Promise<string> {
+    if (this.foundManifests.length === 0) {
+      this.foundManifests = await this.recursivelyLookForManifests(this.componentMountPoint);
+    }
+
+    const manifest = this.foundManifests.find(
+      (m) => m.manifest.getName() === getComponentName(namespace, componentName) && m.manifest.getVersion() === version,
+    );
+
+    if (manifest) {
+      return path.join(manifest.directory, 'manifest.json');
+    }
+
+    throw new ResourceNotFoundError('manifest could not be found');
+  }
+
+  protected async recursivelyLookForManifests(directory: string, nesting = 10): Promise<ManifestAndPath[]> {
+    if (await this.isComponentDirectory(directory)) {
+      const manifest = await this.tryToReadManifest(path.join(directory, 'manifest.json'));
+
+      if (manifest) {
+        return [{ directory, manifest }];
+      }
+
+      return [];
+    }
+
+    if (nesting === 0) {
+      return [];
+    }
+
+    const files = await fsp.readdir(directory, { withFileTypes: true, encoding: 'utf-8' });
+    const nestedDirectories = files.filter((file) => file.isDirectory());
+
+    const otherManifests = await Promise.all(
+      nestedDirectories.map((d) => this.recursivelyLookForManifests(path.join(directory, d.name), nesting - 1)),
+    );
+
+    return otherManifests.flat();
+  }
+
+  protected async tryToReadManifest(manifestPath: string): Promise<Manifest | undefined> {
+    let manifest: Manifest | undefined = undefined;
+
+    try {
+      // wrap inside try-catch as manifest reading can fail
+      // if schema version is not specifed in the manifest
+      manifest = await this.readManifest(manifestPath);
+    } catch (e: any) {
+      if (e && e.message) {
+        this.logger.warn(`found manifest ${manifestPath}, but failed schema validation because: ${e.message}`);
+      } else {
+        throw e;
+      }
+    }
+
+    if (manifest) {
+      try {
+        await this.assertManifestIsValid(manifestPath, manifest.getModel());
+      } catch (e: any) {
+        if (e && e.message) {
+          this.logger.warn(`found manifest ${manifestPath}, but failed to load manifest because: ${e.message}`);
+        } else {
+          throw e;
+        }
+      }
+    }
+
+    return manifest;
+  }
+}

package/src/domains/manifest/services/ManifestServiceForProduction.spec.ts

@@ -0,0 +1,90 @@
+import { getLogger } from '@squiz/dx-logger-lib';
+import path from 'path';
+import { ManifestServiceForProduction } from './ManifestServiceForProduction';
+
+const COMPONENT_ROOT = process.env.DATA_MOUNT_POINT!;
+
+describe('ManifestServiceForProduction', () => {
+  const service = new ManifestServiceForProduction(COMPONENT_ROOT, getLogger({ name: 'ManifestServiceForProduction' }));
+
+  it('should return a path in the format of <root>/<namespace>/<component name>/<version>', async () => {
+    await expect(service.getManifestPath('foo', 'my-component', '1.0.0')).resolves.toEqual(
+      path.join(COMPONENT_ROOT, 'foo', 'my-component', '1.0.0', 'manifest.json'),
+    );
+  });
+
+  it('finds all components in directory', async () => {
+    const resolvedComponents = await service.listAllComponentManifests();
+
+    const allowedPrefixes = [
+      'test-component',
+      'test-environment-variables',
+      'test-malicious-code',
+      'test-npm-modules',
+      'test-static-files',
+    ];
+
+    expect(
+      resolvedComponents
+        .filter((a) => allowedPrefixes.some((b) => a.getName().startsWith(`unit-test-components/${b}`)))
+        .map((a) => `${a.getName()} ${a.getVersion()}`)
+        .sort(),
+    ).toEqual([
+      'unit-test-components/test-component 1.0.0',
+      'unit-test-components/test-component 1.0.10',
+      'unit-test-components/test-component 1.0.3',
+      'unit-test-components/test-component 1.0.4',
+      'unit-test-components/test-component 1.0.5',
+      'unit-test-components/test-component 1.0.6',
+      'unit-test-components/test-component 1.0.7',
+      'unit-test-components/test-component 1.0.8',
+      'unit-test-components/test-component 1.0.9',
+      'unit-test-components/test-component 1.1.0',
+      'unit-test-components/test-component 1.1.2',
+      'unit-test-components/test-component 1.1.4',
+      'unit-test-components/test-component 1.2.0',
+      'unit-test-components/test-component 1.3.0',
+      'unit-test-components/test-environment-variables 1.0.0',
+      'unit-test-components/test-environment-variables 1.0.1',
+      'unit-test-components/test-environment-variables 1.0.2',
+      'unit-test-components/test-environment-variables 1.0.3',
+      'unit-test-components/test-environment-variables 1.0.4',
+      'unit-test-components/test-malicious-code 1.0.0',
+      'unit-test-components/test-malicious-code 1.1.0',
+      'unit-test-components/test-malicious-code 1.2.0',
+      'unit-test-components/test-malicious-code 1.3.0',
+      'unit-test-components/test-npm-modules 1.0.0',
+      'unit-test-components/test-npm-modules 2.0.0',
+      'unit-test-components/test-npm-modules 3.0.0',
+      'unit-test-components/test-npm-modules 3.1.0',
+      'unit-test-components/test-static-files 1.0.0',
+      'unit-test-components/test-static-files 1.0.1',
+      'unit-test-components/test-static-files 1.0.2',
+      'unit-test-components/test-static-files 1.0.3',
+      'unit-test-components/test-static-files 1.0.4',
+      'unit-test-components/test-static-files 1.0.5',
+      'unit-test-components/test-static-files 1.0.6',
+      'unit-test-components/test-static-files 1.0.7',
+      'unit-test-components/test-static-files 1.0.8',
+    ]);
+  });
+
+  describe('readManifest', () => {
+    it('should ignore manifest location validation if validateLocation parameter is false', async () => {
+      // this package has version mismatch
+      const manifestPath = await service.getManifestPath('unit-test-components', 'test-component', '1.0.1');
+
+      const manifest = await service.readManifest(manifestPath);
+
+      expect(manifest.getVersion()).toEqual('0.0.0');
+    });
+
+    it('should ignore manifest location validation if validateLocation parameter is false  - invalid namespace', async () => {
+      // this package has version mismatch
+      const manifestPath = await service.getManifestPath('unit-test-components', 'test-component', '1.1.3');
+
+      const manifest = await service.readManifest(manifestPath);
+      expect(manifest.getNamespace()).toEqual('foo');
+    });
+  });
+});

package/src/domains/manifest/services/ManifestServiceForProduction.ts

@@ -0,0 +1,98 @@
+import path from 'path';
+import fsp from 'fs/promises';
+
+import { AbstractManifestService, ManifestValidationOptions } from './AbstractManifestService';
+import { Manifest, ManifestModel } from '../ManifestModel';
+import { ManifestLocationError } from '../errors/ManifestLocationError';
+import { Logger } from '@squiz/dx-logger-lib';
+
+export class ManifestServiceForProduction extends AbstractManifestService {
+  constructor(protected componentMountPoint: string, protected logger: Logger) {
+    super(logger);
+  }
+
+  async listAllComponentManifests(): Promise<Manifest[]> {
+    // TODO: replace this with a DB query and not fs reads
+    //
+    const namespaces = await fsp.readdir(this.componentMountPoint, { encoding: 'utf-8' });
+    const manifests: Manifest[] = [];
+
+    for (const namespace of namespaces) {
+      const namespacePath = path.join(this.componentMountPoint, namespace);
+      if ((await fsp.stat(namespacePath)).isDirectory()) {
+        const componentDirs = await fsp.readdir(namespacePath, { encoding: 'utf-8' });
+
+        for (const componentDir of componentDirs) {
+          try {
+            const componentPath = path.join(namespacePath, componentDir);
+
+            if ((await fsp.stat(componentPath)).isDirectory()) {
+              const versionDirs = await fsp.readdir(componentPath, { encoding: 'utf-8' });
+
+              for (const versionDir of versionDirs) {
+                const directory = path.join(componentPath, versionDir);
+
+                if (await this.isComponentDirectory(directory)) {
+                  try {
+                    const manifestPath = path.join(directory, 'manifest.json');
+                    const manifest = await this.readManifest(manifestPath);
+
+                    await this.assertManifestVersionIsTheSameAsFileSystemVersion(manifestPath, manifest.getModel());
+
+                    manifests.push(manifest);
+                  } catch (e) {
+                    // no-op as we're going to replace this function eventually
+                  }
+                }
+              }
+            }
+          } catch (e) {
+            // no-op as we're going to replace this function eventually
+          }
+        }
+      }
+    }
+
+    return manifests;
+  }
+
+  public async assertManifestIsValid(
+    manifestPath: string,
+    manifest: ManifestModel,
+    options?: ManifestValidationOptions,
+  ): Promise<void> {
+    await super.assertManifestIsValid(manifestPath, manifest, options);
+    if (options?.tempLocation !== true) {
+      await this.assertManifestVersionIsTheSameAsFileSystemVersion(manifestPath, manifest);
+    }
+  }
+
+  async getManifestPath(namespace: string, componentName: string, version: string): Promise<string> {
+    return path.join(this.componentMountPoint, namespace, componentName, version, 'manifest.json');
+  }
+
+  protected async assertManifestVersionIsTheSameAsFileSystemVersion(
+    manifestPath: string,
+    manifestModel: ManifestModel,
+  ) {
+    const manifest = this.getManifest(manifestModel, manifestPath);
+    const calculatedManifestPath = path.join(manifest.getName(), manifest.getVersion(), 'manifest.json');
+    if (!manifestPath.endsWith(calculatedManifestPath)) {
+      const manifestFilePathParts = manifestPath.split(path.sep).reverse();
+      const expectedVersion = manifestFilePathParts[1];
+      const expectedNamespace = manifestFilePathParts[3];
+
+      if (expectedVersion !== manifest.getVersion()) {
+        throw new ManifestLocationError(
+          `manifest version miss match, expected version ${expectedVersion} but found ${manifest.getVersion()}`,
+          manifestModel,
+        );
+      } else {
+        throw new ManifestLocationError(
+          `manifest namespace miss match, expected namespace ${expectedNamespace} but found ${manifest.getNamespace()}`,
+          manifestModel,
+        );
+      }
+    }
+  }
+}

package/src/domains/manifest/utils/assertFuncEntryFilesAreValid.ts

@@ -0,0 +1,19 @@
+import { isPathTryingToAccessOutsideOfRoot, isReadable } from '@squiz/dx-common-lib';
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+import { ManifestModel } from '../ManifestModel';
+
+export async function assertFuncEntryFilesAreValid(
+  componentRoot: string,
+  manifest: ManifestModel,
+  func: MANIFEST_MODELS.v1.ComponentFunction,
+) {
+  if (isPathTryingToAccessOutsideOfRoot(componentRoot, func.entry)) {
+    throw new Error(
+      `access to component "${func.name}'s" entry file "${func.entry}" outside the parent directory in "${manifest.name} ${manifest.version}" is not allowed`,
+    );
+  }
+
+  if (!(await isReadable(componentRoot, func.entry))) {
+    throw new Error(`"${func.name}'s" entry file "${func.entry}" is inaccessible`);
+  }
+}

package/src/domains/manifest/utils/assertStaticFilesAreValid.ts

@@ -0,0 +1,37 @@
+import { ManifestModel } from '../ManifestModel';
+import path from 'path';
+import { isPathTryingToAccessOutsideOfRoot, isReadable } from '@squiz/dx-common-lib';
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+export async function assertStaticFilesAreValid(
+  componentRoot: string,
+  manifest: ManifestModel,
+  func: MANIFEST_MODELS.v1.ComponentFunction,
+  staticFileLocationRoot: string,
+) {
+  const staticFileRoot = path.join(componentRoot, staticFileLocationRoot);
+  const staticFiles = (func.output as MANIFEST_MODELS.v1.HtmlResponse).staticFiles || [];
+
+  for (const staticFile of staticFiles) {
+    if (staticFile.file.filepath.startsWith('http://') || staticFile.file.filepath.startsWith('https://')) {
+      continue;
+    }
+
+    const staticFileIsTryingToReadOutsSideOfRoot = isPathTryingToAccessOutsideOfRoot(
+      staticFileRoot,
+      staticFile.file.filepath,
+    );
+
+    if (staticFileIsTryingToReadOutsSideOfRoot) {
+      throw new Error(
+        `access to a static file "${staticFile.file.filepath}" for "${func.name}" outside the parent directory in "${manifest.name} ${manifest.version}" is not allowed`,
+      );
+    }
+
+    const staticFileIsReadable = await isReadable(staticFileRoot, staticFile.file.filepath);
+
+    if (!staticFileIsReadable) {
+      throw new Error(`"${func.name}'s" static file "${staticFile.file.filepath}" is inaccessible`);
+    }
+  }
+}

package/src/domains/manifest/utils/assertStaticFilesHaveAbsolutePath.spec.ts

@@ -0,0 +1,121 @@
+import { ManifestModel } from '../ManifestModel';
+import { assertStaticFilesHaveAbsolutePath } from './assertStaticFilesHaveAbsolutePath';
+
+describe('assetStaticFilesHaveAbsolutePath', () => {
+  it('should not throw an error if the filepaths are all absolute', async () => {
+    const manifestModel: ManifestModel = {
+      $schema: './manifest-schemas/v1.json',
+
+      name: 'hello',
+      version: '1.0.1',
+      namespace: 'foo',
+      displayName: 'some display name',
+      description: 'some description',
+
+      mainFunction: 'hello',
+
+      functions: [
+        {
+          entry: 'whatever.js',
+          name: 'hello',
+
+          input: {
+            type: 'object',
+            properties: {
+              hello: { type: 'string' },
+            },
+
+            required: ['hello'],
+          },
+          output: {
+            responseType: 'html',
+            staticFiles: [
+              {
+                location: 'header',
+                file: {
+                  filepath: 'https://cdn.tailwindcss.com',
+                  type: 'js',
+                },
+              },
+              {
+                location: 'header',
+                file: {
+                  filepath: 'https://cdn.jsdelivr.net/npm/bootstrap@4.3.1/dist/css/bootstrap.min.css',
+                  type: 'js',
+                },
+              },
+              {
+                location: 'header',
+                file: {
+                  filepath: 'https://unpkg.com/react@18/umd/react.production.min.js',
+                  type: 'js',
+                },
+              },
+            ],
+          },
+        },
+      ],
+    };
+    const result = manifestModel.functions.map((func) => {
+      assertStaticFilesHaveAbsolutePath(func);
+    });
+    expect(result).toEqual([undefined]);
+  });
+  it('should throw an error if there are relative static file paths', async () => {
+    const manifestModel: ManifestModel = {
+      $schema: './manifest-schemas/v1.json',
+      name: 'hello',
+      version: '1.0.1',
+      namespace: 'foo',
+      displayName: 'some display name',
+      description: 'some description',
+      mainFunction: 'main',
+      functions: [
+        {
+          entry: 'whatever.js',
+          name: 'main',
+
+          input: {
+            type: 'object',
+            properties: {
+              hello: { type: 'string' },
+            },
+
+            required: ['hello'],
+          },
+          output: {
+            responseType: 'html',
+            staticFiles: [
+              {
+                location: 'header',
+                file: {
+                  filepath: 'https://cdn.tailwindcss.com',
+                  type: 'js',
+                },
+              },
+              {
+                location: 'header',
+                file: {
+                  filepath: '../static.test.css',
+                  type: 'css',
+                },
+              },
+              {
+                location: 'header',
+                file: {
+                  filepath: '../static.test.js',
+                  type: 'js',
+                },
+              },
+            ],
+          },
+        },
+      ],
+    };
+    await expect(
+      assertStaticFilesHaveAbsolutePath(manifestModel.functions[0]),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"function main defines static files but manifest has no staticFiles.locationRoot defined"`,
+    );
+  });
+});

package/src/domains/manifest/utils/assertStaticFilesHaveAbsolutePath.ts

@@ -0,0 +1,15 @@
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+export async function assertStaticFilesHaveAbsolutePath(func: MANIFEST_MODELS.v1.ComponentFunction) {
+  const staticFiles = (func.output as MANIFEST_MODELS.v1.HtmlResponse).staticFiles || [];
+
+  for (const staticFile of staticFiles) {
+    if (staticFile.file.filepath.startsWith('http://') || staticFile.file.filepath.startsWith('https://')) {
+      continue;
+    } else {
+      throw new Error(
+        `function ${func.name} defines static files but manifest has no staticFiles.locationRoot defined`,
+      );
+    }
+  }
+}

package/src/domains/manifest/utils/getAndAssertJobManifestVersion.spec.ts

@@ -0,0 +1,145 @@
+import { ManifestValidationError } from '../errors/ManifestValidationError';
+import { JobManifestV1 } from '../../job/manifest/JobManifestV1';
+import { JobManifestModel } from '../ManifestModel';
+import { getAndAssertJobManifestVersion } from './getAndAssertJobManifestVersion';
+
+describe('getAndAssertManifestVersion', () => {
+  it("should throw an error if Manifest object hasn't defined a $schema", () => {
+    expect(() =>
+      getAndAssertJobManifestVersion({
+        $schema: '',
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        concurrency: 10,
+        timeout: 10000,
+        displayName: 'something',
+        description: 'some-description',
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      }),
+    ).toThrowError(
+      new ManifestValidationError(
+        'Could not determine manifest version. Your manifest must have a $schema property defined.',
+      ),
+    );
+
+    expect(() =>
+      getAndAssertJobManifestVersion({
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        displayName: 'something',
+        concurrency: 10,
+        timeout: 10000,
+        description: 'some-description',
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      } as unknown as JobManifestModel),
+    ).toThrowError(
+      new ManifestValidationError(
+        'Could not determine manifest version. Your manifest must have a $schema property defined.',
+      ),
+    );
+  });
+
+  it('should throw an error if Manifest object has invalid $schema version', () => {
+    expect(() =>
+      getAndAssertJobManifestVersion({
+        $schema: 'something-invalid',
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        concurrency: 10,
+        timeout: 10000,
+        displayName: 'something',
+        description: 'some-description',
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      }),
+    ).toThrowError(new ManifestValidationError('Could not determine manifest version from $schema value.'));
+  });
+
+  it("should throw an error if Manifest isn't a valid/defined version", () => {
+    expect(() =>
+      getAndAssertJobManifestVersion(
+        {
+          $schema: 'http://../schemas/JobV2.json',
+          mainFunction: 'main',
+          name: 'test',
+          version: '1.0.0',
+          concurrency: 10,
+          timeout: 10000,
+          displayName: 'something',
+          description: 'some-description',
+
+          functions: [
+            {
+              entry: 'entry',
+              input: {},
+              name: 'main',
+              output: {
+                responseType: 'html',
+              },
+            },
+          ],
+        },
+        JobManifestV1.getJobSchemaVersionRegex(),
+      ),
+    ).toThrowError(
+      new ManifestValidationError(`manifest version "JobV2" isn't a valid version. Valid versions are "JobV1"`),
+    );
+  });
+
+  it('should return the schema version number if its valid and exists', () => {
+    expect(
+      getAndAssertJobManifestVersion(
+        {
+          $schema: 'http://../schemas/JobV1.json',
+          mainFunction: 'main',
+          name: 'test',
+          version: '1.0.0',
+          displayName: 'something',
+          description: 'some-description',
+          concurrency: 10,
+          timeout: 10000,
+          functions: [
+            {
+              entry: 'entry',
+              input: {},
+              name: 'main',
+              output: {
+                responseType: 'html',
+              },
+            },
+          ],
+        },
+        JobManifestV1.getJobSchemaVersionRegex(),
+      ),
+    ).toEqual('JobV1');
+  });
+});

package/src/domains/manifest/utils/getAndAssertJobManifestVersion.ts

@@ -0,0 +1,31 @@
+import { SchemaVersions } from '../AbstractManifest';
+import { ManifestValidationError } from '../errors/ManifestValidationError';
+import { JobManifestModel } from '../ManifestModel';
+import { isValidManifestVersion } from './isValidManifestVersion';
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+export function getAndAssertJobManifestVersion(manifest: JobManifestModel, schemaRegexp?: RegExp): SchemaVersions {
+  if (!manifest['$schema']) {
+    throw new ManifestValidationError(
+      'Could not determine manifest version. Your manifest must have a $schema property defined.',
+    );
+  }
+  let match;
+  if (schemaRegexp) {
+    match = schemaRegexp.exec(manifest['$schema']);
+  }
+  const version = match?.groups?.schema;
+
+  if (version === undefined) {
+    throw new ManifestValidationError('Could not determine manifest version from $schema value.');
+  }
+
+  if (isValidManifestVersion(version)) {
+    return version as SchemaVersions;
+  }
+  const allModels = Object.keys(MANIFEST_MODELS);
+  const validJobVersions = allModels.filter((model) => model.includes('JobV'));
+  throw new ManifestValidationError(
+    `manifest version "${version}" isn't a valid version. Valid versions are "${validJobVersions.toString()}"`,
+  );
+}