@squiz/job-runner-lib 1.66.1

package/src/domains/component/services/ComponentService.ts

@@ -0,0 +1,350 @@
+import path from 'path';
+import fsExtra from 'fs-extra';
+import { BadRequestError, getPageInfo, PageQueryResult, ResourceNotFoundError } from '@squiz/dx-common-lib';
+import { Manifest } from '../../manifest/ManifestModel';
+import { ComponentConnectionManager } from '../../db/ComponentConnectionManager';
+import { ManifestService } from '../../manifest';
+import { ManifestValidationError } from '../../manifest/errors/ManifestValidationError';
+import { getComponentFileSystemFolder as getComponentFileSystemFolder } from '../utils/getComponentFileSystemFolder';
+import { ComponentWebModel } from '../web-models/ComponentForWeb';
+import { ComponentsWebModel } from '../web-models/ComponentsForWeb';
+import { ComponentVersionWebModel } from '../web-models/ComponentVersionWebModel';
+import { getComponentName } from '../utils/getComponentName';
+import type { SortDirection } from '@squiz/db-lib';
+import { ComponentVersionInterface } from '../models/ComponentVersion';
+import { SchemaValidationError } from '@squiz/dx-json-schema-lib';
+import { moveFilesAndDirectories } from '../utils/moveFilesAndDirectories';
+
+interface Logger {
+  warn: (...args: any[]) => any;
+  info: (...args: any[]) => any;
+  error: (...args: any[]) => any;
+}
+
+export class ComponentService {
+  constructor(
+    protected db: ComponentConnectionManager,
+    protected logger: Logger,
+    protected dataMountPoint: string,
+    protected manifestService: ManifestService,
+  ) {}
+
+  public async getComponentVersionForWeb(
+    namespace: string,
+    componentName: string,
+    version: string,
+  ): Promise<ComponentVersionWebModel> {
+    await this.assertComponentVersionExists(componentName, namespace, version);
+
+    const componentVersion = await this.db.repositories.componentVersion.findOne({
+      componentName: getComponentName(namespace, componentName),
+      version,
+    });
+
+    if (!componentVersion) {
+      throw new ResourceNotFoundError(`${namespace}/${componentName} ${version} could not be found`);
+    }
+
+    const sets = await this.db.repositories.componentSet.findAllSetsComponentVersionIsIn(
+      namespace,
+      componentName,
+      version,
+    );
+
+    return {
+      namespace,
+      name: componentName,
+      version,
+
+      description: componentVersion.description,
+      displayName: componentVersion.displayName,
+      icon: componentVersion.icon,
+
+      sets: sets.map((a) => ({ displayName: a.displayName, webPath: a.webPath })),
+    };
+  }
+
+  public async getComponentForWeb(namespace: string, componentName: string): Promise<ComponentWebModel> {
+    const componentFullName = getComponentName(namespace, componentName);
+
+    await this.assertComponentExists(componentName, namespace);
+    const componentExists = await this.db.repositories.component.componentExists(componentFullName);
+
+    if (componentExists == false) {
+      throw new ResourceNotFoundError(`${namespace}/${componentName} could not be found`);
+    }
+
+    const componentVersions = await this.db.repositories.componentVersion.find({
+      componentName: componentFullName,
+    });
+
+    return {
+      versions: componentVersions.map((a) => ({
+        description: a.description,
+        displayName: a.displayName,
+        icon: a.icon,
+        version: a.version,
+      })),
+    };
+  }
+
+  public async getPaginatedComponents(
+    pageNumber: number = 1,
+    sortBy: (keyof ComponentVersionInterface)[] = [],
+    sortDirection: SortDirection = 'asc',
+    pageSize: number = 20,
+    searchFields: Partial<Omit<ComponentVersionInterface, 'icon'>> | null = null,
+  ): Promise<PageQueryResult<ComponentsWebModel>> {
+    const pageResult = await this.db.repositories.componentVersion.getPaginatedMaxComponentVersion(
+      pageNumber,
+      sortBy,
+      sortDirection,
+      pageSize,
+      searchFields,
+    );
+
+    const components: ComponentsWebModel[] = [];
+
+    for (const component of pageResult.items) {
+      const componentSets = (
+        await this.db.repositories.componentSet.findAllSetsComponentIsIn(component.componentName)
+      ).map((row) => {
+        return { displayName: row.displayName, webPath: row.webPath };
+      });
+
+      components.push({
+        name: component.componentName,
+        includedInSets: componentSets.length, // TODO: remove this after reviewing its usage in DXP Console
+        componentSets: componentSets,
+        latestVersion: {
+          description: component.description,
+          displayName: component.displayName,
+          icon: component.icon,
+          version: component.version,
+        },
+      });
+    }
+
+    return {
+      items: components,
+      pageInfo: getPageInfo(pageNumber, pageResult.pageSize, pageResult.totalCount),
+    };
+  }
+
+  public async saveComponent(componentLocation: string) {
+    let manifest: Manifest;
+
+    try {
+      const manifestPath = path.join(componentLocation, 'manifest.json');
+      manifest = await this.manifestService.readManifest(manifestPath);
+      await this.manifestService.assertManifestIsValid(manifestPath, manifest.getModel(), {
+        tempLocation: true,
+      });
+    } catch (e: any) {
+      if (e instanceof ManifestValidationError || e instanceof SchemaValidationError) {
+        const error = new BadRequestError(e.message);
+        error.stack = e.stack;
+        throw error;
+      } else {
+        throw e;
+      }
+    }
+
+    const componentName = manifest.getName();
+    const componentVersion = manifest.getVersion();
+
+    if (await this.componentVersionExists(componentName, componentVersion)) {
+      throw new BadRequestError(`Component ${componentName} ${componentVersion} already exists`);
+    }
+
+    const destPath = await this.getValidComponentDestinationPath(componentName, componentVersion);
+
+    await this.moveFilesAndUpdateDb(manifest, componentLocation, destPath);
+
+    return {
+      name: componentName,
+      version: componentVersion,
+      manifest,
+    };
+  }
+
+  protected async componentVersionExists(componentName: string, version: string) {
+    const componentVersion = await this.db.repositories.componentVersion.getComponentByNameAndVersion(
+      componentName,
+      version,
+    );
+
+    return componentVersion !== undefined;
+  }
+
+  public async deleteComponent(namespace: string, name: string): Promise<void> {
+    const componentName = getComponentName(namespace, name);
+    const componentFolder = getComponentFileSystemFolder(namespace, name);
+    this.assertValidComponentName(name, namespace);
+    await this.assertComponentExists(name, namespace);
+
+    const compPath = path.join(this.dataMountPoint, componentFolder);
+
+    try {
+      await this.db.repositories.componentVersion.delete({ componentName });
+    } catch (e) {
+      this.logger.error(e);
+    }
+
+    try {
+      await this.db.repositories.component.delete({ name: componentName });
+    } catch (e) {
+      this.logger.error(e);
+    }
+
+    try {
+      await fsExtra.rm(compPath, { force: true, recursive: true });
+    } catch (e) {
+      this.logger.error(e);
+    }
+
+    const componentExists = await this.db.repositories.component.componentExists(componentName);
+    const filesExist = await fsExtra.pathExists(compPath);
+
+    if (componentExists) {
+      throw new Error(`failed to remove ${componentName} from DB`);
+    }
+
+    if (filesExist) {
+      throw new Error(`failed to remove ${componentName} files from the disk`);
+    }
+  }
+
+  public async deleteComponentVersion(namespace: string, name: string, version: string): Promise<void> {
+    const componentName = getComponentName(namespace, name);
+    const componentFolder = getComponentFileSystemFolder(namespace, name);
+
+    this.assertValidComponentVersion(name, namespace, version);
+    await this.assertComponentVersionExists(name, namespace, version);
+
+    const compVersionPath = path.join(this.dataMountPoint, componentFolder, version);
+
+    try {
+      await this.db.repositories.componentVersion.delete({ componentName, version });
+      await fsExtra.rm(compVersionPath, { force: true, recursive: true });
+    } catch (e) {
+      this.logger.error(e);
+    }
+
+    const componentVersionExists = await this.db.repositories.componentVersion.componentVersionExists(
+      componentName,
+      version,
+    );
+    const filesExist = await fsExtra.pathExists(compVersionPath);
+
+    if (componentVersionExists) {
+      throw new Error(`failed to remove ${componentName} from DB`);
+    }
+
+    if (filesExist) {
+      throw new Error(`failed to remove ${componentName} files from the disk`);
+    }
+  }
+
+  protected assertValidComponentName(name: string, namespace: string) {
+    const componentFolderName = getComponentFileSystemFolder(namespace, name);
+
+    const rootDir = path.resolve(this.dataMountPoint);
+    const compPath = path.join(this.dataMountPoint, componentFolderName);
+
+    const checkPath = path.resolve(this.dataMountPoint) + path.sep + componentFolderName;
+
+    if (checkPath !== compPath || rootDir === compPath) {
+      throw new ResourceNotFoundError(`component name ${componentFolderName} is invalid`);
+    }
+  }
+
+  protected assertValidComponentVersion(name: string, namespace: string, version: string) {
+    const componentFolderName = getComponentFileSystemFolder(namespace, name);
+    const versionPath = path.join(path.resolve(this.dataMountPoint), componentFolderName, version);
+
+    const checkPath = [path.resolve(this.dataMountPoint), componentFolderName, version].join(path.sep);
+
+    if (checkPath !== versionPath) {
+      throw new ResourceNotFoundError(`Component version ${version} is invalid`);
+    }
+  }
+
+  protected async assertComponentExists(name: string, namespace: string) {
+    const componentName = getComponentName(namespace, name);
+    const componentFolderName = getComponentFileSystemFolder(namespace, name);
+    const compPath = path.join(this.dataMountPoint, componentFolderName);
+    if (
+      !(await this.db.repositories.component.componentExists(componentName)) &&
+      !(await fsExtra.pathExists(compPath))
+    ) {
+      throw new ResourceNotFoundError(`Component ${componentName} not found`);
+    }
+  }
+
+  protected async assertComponentVersionExists(name: string, namespace: string, version: string) {
+    const componentName = getComponentName(namespace, name);
+
+    await this.assertComponentExists(name, namespace);
+    const compVersionPath = path.join(this.dataMountPoint, componentName, version);
+    if (
+      !(await this.db.repositories.componentVersion.componentVersionExists(componentName, version)) &&
+      !(await fsExtra.pathExists(compVersionPath))
+    ) {
+      throw new ResourceNotFoundError(`Component version ${version} for component ${componentName} not found`);
+    }
+  }
+
+  protected async moveFilesAndUpdateDb(manifest: Manifest, location: string, destPath: string) {
+    try {
+      await moveFilesAndDirectories(location, destPath);
+    } catch (e) {
+      fsExtra.rm(location, { force: true, recursive: true });
+      throw e;
+    }
+
+    try {
+      // make the new dir readable for all
+      await fsExtra.chmod(destPath, '755');
+    } catch (e) {
+      fsExtra.rm(destPath, { force: true, recursive: true });
+      throw e;
+    }
+
+    try {
+      await this.addComponentVersion(manifest);
+    } catch (e) {
+      fsExtra.rm(destPath, { force: true, recursive: true });
+      throw e;
+    }
+  }
+
+  protected async addComponentVersion(manifest: Manifest) {
+    const version = manifest.getVersion();
+    const componentName = manifest.getName();
+    const { displayName, description, icon } = manifest.getModel();
+
+    if ((await this.db.repositories.component.componentExists(componentName)) === false) {
+      await this.db.repositories.component.addComponent(componentName);
+    }
+
+    await this.db.repositories.componentVersion.addComponentVersion(
+      componentName,
+      version,
+      displayName,
+      description,
+      icon,
+    );
+  }
+
+  protected async getValidComponentDestinationPath(componentName: string, version: string) {
+    const destPath = path.join(this.dataMountPoint, componentName, version);
+    const pathExists = await fsExtra.pathExists(destPath);
+
+    if (pathExists) {
+      throw new BadRequestError(`${componentName} version ${version} already exists and cannot be uploaded`);
+    }
+
+    return destPath;
+  }
+}

package/src/domains/component/services/content-item-uri-resolver/ComponentContentUriResolverForPreview.spec.ts

@@ -0,0 +1,78 @@
+import { ComponentContentUriResolverForPreview, UriResource } from './ComponentContentUriResolverForPreview';
+import { ManifestModel } from '../../../manifest/ManifestModel';
+
+describe('ComponentContentUriResolverForPreview', () => {
+  const mockedUriObj: UriResource = {
+    id: '1',
+    type: 'root_folder',
+    type_name: 'Root Folder',
+    version: '0.0.1',
+    name: 'Root Folder',
+    short_name: '/',
+    status: {
+      id: 2,
+      code: 'under_construction',
+    },
+    created: {
+      date: '2020-02-27T09:47:27+11:00',
+      user_id: null,
+    },
+    updated: {
+      date: '2020-02-27T09:47:27+11:00',
+      user_id: '',
+    },
+    published: {
+      date: '2020-02-27T09:47:27+11:00',
+      user_id: 'null',
+    },
+    status_changed: {
+      date: '2020-02-27T09:47:27+11:00',
+      user_id: 'null',
+    },
+    url: 'xxx',
+    urls: [],
+    attributes: {
+      test: 'test',
+    },
+    additional: {},
+  };
+  const manifest: ManifestModel = {
+    $schema: 'http://localhost:3000/schemas/v1.json#',
+    name: 'some-name',
+    namespace: 'some-ns',
+    displayName: 'some-display-name',
+    description: 'some-description',
+    version: '1.0.0',
+    mainFunction: 'main',
+    functions: [
+      {
+        name: 'main',
+        entry: 'main.js',
+        input: {},
+        output: {
+          responseType: 'html',
+        },
+      },
+    ],
+    mockedUris: {
+      'some-uri-x': {
+        type: 'inline',
+        value: mockedUriObj,
+      },
+    },
+  };
+
+  describe('resolveUri', () => {
+    it('should resolve the mocked uri to a mocked uri object', async () => {
+      const resolver = new ComponentContentUriResolverForPreview(manifest, '');
+      await expect(resolver.resolveUri('some-uri-x')).resolves.toEqual(mockedUriObj);
+    });
+
+    it('should resolve the non-existent mocked uri to undefined', async () => {
+      const resolver = new ComponentContentUriResolverForPreview(manifest, '');
+      await expect(resolver.resolveUri('some-bad-uri-x')).rejects.toThrowError(
+        'Mocked URI object "some-bad-uri-x" not defined in the manifest',
+      );
+    });
+  });
+});

package/src/domains/component/services/content-item-uri-resolver/ComponentContentUriResolverForPreview.ts

@@ -0,0 +1,18 @@
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+import { ManifestModel, ManifestV1 } from '../../../manifest';
+export interface ComponentContentUriResolver {
+  resolveUri(uri: string, manifest: ManifestModel): Promise<UriResource | undefined>;
+}
+
+export type UriResource = MANIFEST_MODELS.v1.MatrixAssetSchema;
+export class ComponentContentUriResolverForPreview implements ComponentContentUriResolver {
+  protected manifestService: ManifestV1;
+
+  constructor(protected manifest: ManifestModel, manifestPath: string | undefined) {
+    this.manifestService = new ManifestV1(manifest, manifestPath ?? '');
+  }
+
+  public async resolveUri(uri: string): Promise<UriResource | undefined> {
+    return await this.manifestService.getMockedUriObject(uri);
+  }
+}

package/src/domains/component/utils/getComponentFileSystemFolder.spec.ts

@@ -0,0 +1,8 @@
+import path from 'path';
+import { getComponentFileSystemFolder } from './getComponentFileSystemFolder';
+
+describe('getComponentFileSystemFolder', () => {
+  it('should return the resulting component name prefixed with the namespace', () => {
+    expect(getComponentFileSystemFolder('foo', 'bar')).toEqual('foo' + path.sep + 'bar');
+  });
+});

package/src/domains/component/utils/getComponentFileSystemFolder.ts

@@ -0,0 +1,5 @@
+import path from 'path';
+
+export function getComponentFileSystemFolder(namespace: string, name: string): string {
+  return path.join(namespace, name);
+}

package/src/domains/component/utils/getComponentName.spec.ts

@@ -0,0 +1,7 @@
+import { getComponentName } from './getComponentName';
+
+describe('getComponentName', () => {
+  it('should return the resulting component name prefixed with the namespace', () => {
+    expect(getComponentName('foo', 'bar')).toEqual('foo/bar');
+  });
+});

package/src/domains/component/utils/getComponentName.ts

@@ -0,0 +1,3 @@
+export function getComponentName(namespace: string, name: string): string {
+  return `${namespace}/${name}`;
+}

package/src/domains/component/utils/moveFilesAndDirectories.spec.ts

@@ -0,0 +1,62 @@
+import fsp from 'fs/promises';
+import fse from 'fs-extra';
+import path from 'path';
+import { moveFilesAndDirectories } from './moveFilesAndDirectories';
+
+const getRandomDirectory = () => `test-${Math.random().toString(36).substring(2, 15)}`;
+
+describe('moveFilesAndDirectories', () => {
+  const src = getRandomDirectory();
+  const dest = getRandomDirectory();
+
+  beforeEach(async () => {
+    await fsp.mkdir(src);
+    await fsp.writeFile(path.join(src, 'file1.txt'), 'file1');
+    await fsp.mkdir(path.join(src, 'subdir'));
+    await fsp.mkdir(path.join(src, 'empty-dir'));
+    await fsp.writeFile(path.join(src, 'subdir/file2.txt'), 'file2');
+  });
+
+  afterEach(async () => {
+    await fsp.rm(src, { recursive: true, force: true });
+
+    if (fse.existsSync(dest)) {
+      await fsp.rm(dest, { recursive: true, force: true });
+    }
+  });
+
+  it('should copy the directory structure from one location to another', async () => {
+    await moveFilesAndDirectories(src, dest);
+
+    const destFiles = await fsp.readdir(dest);
+    expect(destFiles).toEqual(['empty-dir', 'file1.txt', 'subdir']);
+
+    const destSubDirFiles = await fsp.readdir(path.join(dest, 'subdir'));
+    expect(destSubDirFiles).toEqual(['file2.txt']);
+
+    const file1Contents = await fsp.readFile(path.join(dest, 'file1.txt'), 'utf8');
+    expect(file1Contents).toEqual('file1');
+
+    const file2Contents = await fsp.readFile(path.join(dest, 'subdir/file2.txt'), 'utf8');
+    expect(file2Contents).toEqual('file2');
+
+    expect(fse.existsSync(path.join(dest, 'empty-dir'))).toEqual(true);
+
+    expect(fse.existsSync(src)).toEqual(false);
+  });
+
+  it('does should overwrite existing files', async () => {
+    await fsp.mkdir(dest);
+    await fsp.writeFile(path.join(dest, 'file1.txt'), 'existing-file');
+
+    await moveFilesAndDirectories(src, dest);
+
+    const file1Contents = await fsp.readFile(path.join(dest, 'file1.txt'), 'utf8');
+    expect(file1Contents).toEqual('file1');
+  });
+
+  it('throws an error if the source directory does not exist', async () => {
+    const nonExistentSrc = getRandomDirectory();
+    await expect(moveFilesAndDirectories(nonExistentSrc, dest)).rejects.toThrow();
+  });
+});

package/src/domains/component/utils/moveFilesAndDirectories.ts

@@ -0,0 +1,27 @@
+import fsp from 'fs/promises';
+
+export async function moveFilesAndDirectories(src: string, dest: string): Promise<void> {
+  if ((await pathExists(src)) === false) {
+    throw new Error(`Source directory "${src}" does not exist`);
+  }
+
+  await fsp.cp(src, dest, {
+    recursive: true,
+    force: true,
+  });
+
+  await fsp.rm(src, { recursive: true, force: true });
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await fsp.stat(path);
+    return true;
+  } catch (error: any) {
+    if (error?.code === 'ENOENT') {
+      return false;
+    } else {
+      throw error;
+    }
+  }
+}

package/src/domains/component/web-models/ComponentForWeb.ts

@@ -0,0 +1,11 @@
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+type Icon = MANIFEST_MODELS.v1.DxComponentIconsJson;
+export interface ComponentWebModel {
+  versions: Array<{
+    version: string;
+    description: string;
+    displayName: string;
+    icon: Icon | undefined;
+  }>;
+}

package/src/domains/component/web-models/ComponentVersionWebModel.ts

@@ -0,0 +1,16 @@
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+type Icon = MANIFEST_MODELS.v1.DxComponentIconsJson;
+export interface ComponentVersionWebModel {
+  displayName: string;
+  version: string;
+  description: string;
+  name: string;
+  namespace: string;
+  icon: Icon | undefined;
+
+  sets: {
+    displayName: string;
+    webPath: string;
+  }[];
+}

package/src/domains/component/web-models/ComponentsForWeb.ts

@@ -0,0 +1,17 @@
+import { MANIFEST_MODELS } from '@squiz/dx-json-schema-lib';
+
+type Icon = MANIFEST_MODELS.v1.DxComponentIconsJson;
+export interface ComponentsWebModel {
+  name: string;
+  includedInSets: number;
+  componentSets: {
+    displayName: string;
+    webPath: string;
+  }[];
+  latestVersion: {
+    description: string;
+    displayName: string;
+    icon: Icon | undefined;
+    version: string;
+  };
+}

package/src/domains/component-set/errors/InvalidEnvironmentVariablesError.ts

@@ -0,0 +1,11 @@
+import { BadRequestError } from '@squiz/dx-common-lib';
+
+export class InvalidEnvironmentVariablesError extends BadRequestError {
+  constructor(public readonly errors: [string, string][]) {
+    const message = errors.map(([variableName, error]) => `"${variableName}" ${error}`).join('; ');
+
+    super(`Invalid Environment Variables: ${message}`);
+
+    this.name = 'InvalidEnvironmentVariablesError';
+  }
+}

package/src/domains/component-set/index.ts

@@ -0,0 +1,18 @@
+export * from './models/ComponentSet';
+export * from './models/ComponentSetComponentVersion';
+
+export * from './repositories/ComponentSetComponentVersionRepository';
+export * from './repositories/ComponentSetRepository';
+
+export * from './services/ComponentSetService';
+export * from './services/ComponentSetServiceForLocalDev';
+export * from './services/ComponentSetServiceForProduction';
+export * from './services/ComponentSetRuleService';
+
+export * from './web-models/ComponentSetForWeb';
+export * from './web-models/ComponentSetComponentVersionForWeb';
+
+export * from './errors/InvalidEnvironmentVariablesError';
+
+export * from './services/RuntimeComponentSet';
+export * from './services/Headers';