@squiz/job-runner-lib 1.66.2 → 2.1.0

package/src/domains/manifest/utils/getAndAssertManifestVersion.spec.ts

@@ -1,134 +0,0 @@
-import { ManifestValidationError } from '../errors/ManifestValidationError';
-import { ManifestModel } from '../ManifestModel';
-import { getAndAssertManifestVersion } from './getAndAssertManifestVersion';
-
-describe('getAndAssertManifestVersion', () => {
-  it("should throw an error if Manifest object hasn't defined a $schema", () => {
-    expect(() =>
-      getAndAssertManifestVersion({
-        $schema: '',
-        mainFunction: 'main',
-        name: 'test',
-        version: '1.0.0',
-        namespace: 'foo',
-        displayName: 'something',
-        description: 'some-description',
-        functions: [
-          {
-            entry: 'entry',
-            input: {},
-            name: 'main',
-            output: {
-              responseType: 'html',
-            },
-          },
-        ],
-      }),
-    ).toThrowError(
-      new ManifestValidationError(
-        'Could not determine manifest version. Your manifest must have a $schema property defined.',
-      ),
-    );
-
-    expect(() =>
-      getAndAssertManifestVersion({
-        mainFunction: 'main',
-        name: 'test',
-        version: '1.0.0',
-        namespace: 'foo',
-        displayName: 'something',
-        description: 'some-description',
-        functions: [
-          {
-            entry: 'entry',
-            input: {},
-            name: 'main',
-            output: {
-              responseType: 'html',
-            },
-          },
-        ],
-      } as ManifestModel),
-    ).toThrowError(
-      new ManifestValidationError(
-        'Could not determine manifest version. Your manifest must have a $schema property defined.',
-      ),
-    );
-  });
-
-  it('should throw an error if Manifest object has invalid $schema version', () => {
-    expect(() =>
-      getAndAssertManifestVersion({
-        $schema: 'something-invalid',
-        mainFunction: 'main',
-        name: 'test',
-        version: '1.0.0',
-        namespace: 'foo',
-        displayName: 'something',
-        description: 'some-description',
-        functions: [
-          {
-            entry: 'entry',
-            input: {},
-            name: 'main',
-            output: {
-              responseType: 'html',
-            },
-          },
-        ],
-      }),
-    ).toThrowError(new ManifestValidationError('Could not determine manifest version from $schema value.'));
-  });
-
-  it("should throw an error if Manifest isn't a valid/defined version", () => {
-    expect(() =>
-      getAndAssertManifestVersion({
-        $schema: 'http://../schemas/v9999.json',
-        mainFunction: 'main',
-        name: 'test',
-        version: '1.0.0',
-        namespace: 'foo',
-        displayName: 'something',
-        description: 'some-description',
-
-        functions: [
-          {
-            entry: 'entry',
-            input: {},
-            name: 'main',
-            output: {
-              responseType: 'html',
-            },
-          },
-        ],
-      }),
-    ).toThrowError(
-      new ManifestValidationError('manifest version "v9999" isn\'t a valid version. Valid versions are "v1", "JobV1"'),
-    );
-  });
-
-  it('should return the schema version number if its valid and exists', () => {
-    expect(
-      getAndAssertManifestVersion({
-        $schema: 'http://../schemas/v1.json',
-        mainFunction: 'main',
-        name: 'test',
-        version: '1.0.0',
-        namespace: 'foo',
-        displayName: 'something',
-        description: 'some-description',
-
-        functions: [
-          {
-            entry: 'entry',
-            input: {},
-            name: 'main',
-            output: {
-              responseType: 'html',
-            },
-          },
-        ],
-      }),
-    ).toEqual('v1');
-  });
-});

package/src/domains/manifest/utils/getAndAssertManifestVersion.ts

@@ -1,27 +0,0 @@
-import { schemaVersionRegex, SchemaVersions, SCHEMA_VERSIONS } from '../AbstractManifest';
-import { ManifestValidationError } from '../errors/ManifestValidationError';
-import { ManifestModel } from '../ManifestModel';
-import { isValidManifestVersion } from './isValidManifestVersion';
-
-export function getAndAssertManifestVersion(manifest: ManifestModel): SchemaVersions {
-  if (!manifest['$schema']) {
-    throw new ManifestValidationError(
-      'Could not determine manifest version. Your manifest must have a $schema property defined.',
-    );
-  }
-
-  const match = schemaVersionRegex.exec(manifest['$schema']);
-  const version = match?.groups?.schema;
-
-  if (version === undefined) {
-    throw new ManifestValidationError('Could not determine manifest version from $schema value.');
-  }
-
-  if (isValidManifestVersion(version)) {
-    return version as SchemaVersions;
-  }
-
-  throw new ManifestValidationError(
-    `manifest version "${version}" isn't a valid version. Valid versions are "${SCHEMA_VERSIONS.join('", "')}"`,
-  );
-}

package/src/domains/url/convertFunctionStaticFilesToFqdn.spec.ts

@@ -1,24 +0,0 @@
-import { convertFunctionStaticFilesToFqdn } from './convertFunctionStaticFilesToFqdn';
-
-describe('convertFunctionStaticFilesToFqdn', () => {
-  it('should resolve a file path to a component static resource url string', () => {
-    expect(
-      convertFunctionStaticFilesToFqdn('http://localhost:3000/', './hello/file.txt', 'set-a', 'test', '1.0.0'),
-    ).toEqual('http://localhost:3000/s/test/1.0.0/hello/file.txt?_componentSet=set-a');
-  });
-
-  it('should resolve a relative file path', () => {
-    expect(
-      convertFunctionStaticFilesToFqdn('http://localhost:3000/', './hello/../file.txt', 'set-a', 'test', '1.0.0'),
-    ).toEqual('http://localhost:3000/s/test/1.0.0/file.txt?_componentSet=set-a');
-  });
-
-  it('if give a url do nothing but return it', () => {
-    expect(
-      convertFunctionStaticFilesToFqdn('http://localhost:3000/', 'http://hello/world', 'set-a', 'test', '1.0.0'),
-    ).toEqual('http://hello/world');
-    expect(
-      convertFunctionStaticFilesToFqdn('http://localhost:3000/', 'https://hello/world', 'set-a', 'test', '1.0.0'),
-    ).toEqual('https://hello/world');
-  });
-});

package/src/domains/url/convertFunctionStaticFilesToFqdn.ts

@@ -1,25 +0,0 @@
-import { joinAbsoluteUrlPath } from '@squiz/dx-common-lib';
-import queryString from 'query-string';
-
-export function convertFunctionStaticFilesToFqdn(
-  rootUrl: string,
-  file: string,
-  componentSet: string,
-  componentName: string,
-  version: string,
-): string {
-  if (file.startsWith('https://') || file.startsWith('http://')) {
-    return file;
-  }
-
-  const path = joinAbsoluteUrlPath(rootUrl, 's', componentName, version, file);
-  return queryString.stringifyUrl(
-    {
-      url: path,
-      query: {
-        _componentSet: componentSet,
-      },
-    },
-    { skipNull: true, encode: true, arrayFormat: 'bracket' },
-  );
-}

package/src/domains/url/convertFunctionStaticFilesToFqdnForPreview.spec.ts

@@ -1,48 +0,0 @@
-import { convertFunctionStaticFilesToFqdnForPreview } from './convertFunctionStaticFilesToFqdnForPreview';
-
-describe('convertFunctionStaticFilesToFqdnForPreview', () => {
-  it('should resolve a file path to a component static resource url string', () => {
-    expect(
-      convertFunctionStaticFilesToFqdnForPreview(
-        'http://localhost:3000/',
-        './hello/file.txt',
-        'test',
-        '1.0.0',
-        'test-preview',
-      ),
-    ).toEqual('http://localhost:3000/s/test/1.0.0/hello/file.txt?_previewKey=test-preview');
-  });
-
-  it('should resolve a relative file path', () => {
-    expect(
-      convertFunctionStaticFilesToFqdnForPreview(
-        'http://localhost:3000/',
-        './hello/../file.txt',
-        'test',
-        '1.0.0',
-        'test-preview',
-      ),
-    ).toEqual('http://localhost:3000/s/test/1.0.0/file.txt?_previewKey=test-preview');
-  });
-
-  it('if given a url do nothing but return it', () => {
-    expect(
-      convertFunctionStaticFilesToFqdnForPreview(
-        'http://localhost:3000/',
-        'http://hello/world',
-        'test',
-        '1.0.0',
-        'test-preview',
-      ),
-    ).toEqual('http://hello/world');
-    expect(
-      convertFunctionStaticFilesToFqdnForPreview(
-        'http://localhost:3000/',
-        'https://hello/world',
-        'test',
-        '1.0.0',
-        'test-preview',
-      ),
-    ).toEqual('https://hello/world');
-  });
-});

package/src/domains/url/convertFunctionStaticFilesToFqdnForPreview.ts

@@ -1,25 +0,0 @@
-import { joinAbsoluteUrlPath } from '@squiz/dx-common-lib';
-import queryString from 'query-string';
-
-export function convertFunctionStaticFilesToFqdnForPreview(
-  rootUrl: string,
-  file: string,
-  componentName: string,
-  version: string,
-  previewKey: string,
-): string {
-  if (file.startsWith('https://') || file.startsWith('http://')) {
-    return file;
-  }
-
-  const path = joinAbsoluteUrlPath(rootUrl, 's', componentName, version, file);
-  return queryString.stringifyUrl(
-    {
-      url: path,
-      query: {
-        _previewKey: previewKey,
-      },
-    },
-    { skipNull: true, encode: true, arrayFormat: 'bracket' },
-  );
-}

package/src/domains/url/index.ts

@@ -1 +0,0 @@
-export * from './convertFunctionStaticFilesToFqdn';

package/src/migrations/20220704054051_initial.sql

@@ -1,19 +0,0 @@
--- CreateTable
-
-CREATE TABLE "component" ("name" VARCHAR(128) NOT NULL,
-                                              CONSTRAINT "component_pkey" PRIMARY KEY ("name"));
-
--- CreateTable
-
-CREATE TABLE "component_version" ("component_name" VARCHAR(128) NOT NULL,
-                                                                "version" VARCHAR(128) NOT NULL,
-                                                                                       CONSTRAINT "component_version_pkey" PRIMARY KEY ("component_name",
-                                                                                                                                        "version"));
-
--- AddForeignKey
-
-ALTER TABLE "component_version" ADD CONSTRAINT "component_version_component_name_fkey"
-FOREIGN KEY ("component_name") REFERENCES "component"("name") ON
-DELETE CASCADE ON
-UPDATE CASCADE;
-

package/src/migrations/20220718172237_adding_component_sets.sql

@@ -1,23 +0,0 @@
--- CreateTable
-CREATE TABLE "component_set" (
-    "web_path" VARCHAR(128) NOT NULL,
-    "display_name" VARCHAR(128) NOT NULL,
-    "description" VARCHAR,
-    "env_vars" JSONB,
-    "headers" JSONB,
-
-    CONSTRAINT "component_set_pkey" PRIMARY KEY ("web_path")
-);
-
--- CreateTable
-CREATE TABLE "component_set_component_version" (
-    "component_set_web_path" VARCHAR(128) NOT NULL,
-    "component_name" VARCHAR(128) NOT NULL,
-    "component_version" VARCHAR(128) NOT NULL,
-
-    CONSTRAINT "component_set_component_version_pkey" PRIMARY KEY ("component_set_web_path", "component_name", "component_version")
-);
-
--- AddForeignKey
-ALTER TABLE "component_set_component_version" ADD CONSTRAINT "component_set_web_path_fkey" FOREIGN KEY ("component_set_web_path") REFERENCES "component_set"("web_path") ON DELETE CASCADE ON UPDATE CASCADE;
-ALTER TABLE "component_set_component_version" ADD CONSTRAINT "component_version_componet_name_version_fkey" FOREIGN KEY ("component_name", "component_version") REFERENCES "component_version"("component_name", "version") ON DELETE CASCADE ON UPDATE CASCADE;

package/src/migrations/20220728113941_add_env_vars_field.sql

@@ -1 +0,0 @@
-ALTER TABLE "component_set_component_version" ADD COLUMN "env_vars" jsonb;

package/src/migrations/20220817113300_removing_null_props_from_jsonb.sql

@@ -1,41 +0,0 @@
-UPDATE component_set as rc
-set env_vars = (
-
-  -- subtract non string properties, resulting object is all keys have values
-  select COALESCE((o2.env_vars - od.nonStringProps), '{}'::jsonb) as stringProps
-  from component_set as o2 , (
-
-    -- select get an array of all properties whos value is not a string, as an array
-    select web_path, array(
-      select envK
-      from component_set, jsonb_object_keys(env_vars) as envK
-      where  jsonb_typeof(component_set.env_vars->envk) <> 'string' and o.web_path = component_set.web_path
-    ) as nonStringProps
-    from component_set as o
-  ) as od
-
-  -- make sure all the web_paths are the same
-  where o2.web_path = od.web_path and rc.web_path = o2.web_path
-),
-headers = (
-  -- subtract non string properties resulting object is all keys have values
-  select COALESCE((o2.headers - od.nonStringProps), '{}'::jsonb) as stringProps
-  from component_set as o2 , (
-
-    -- select get an array of all properties whos value is not a string, as an array
-    select web_path, array(
-      select headerK
-      from component_set, jsonb_object_keys(headers) as headerK
-      where  jsonb_typeof(component_set.headers->headerK) <> 'string' and o.web_path = component_set.web_path
-    ) as nonStringProps
-    from component_set as o
-  ) as od
-
-  -- make sure all the web_paths are the same
-  where o2.web_path = od.web_path and rc.web_path = o2.web_path
-);
-
--- make columns not nullable
-ALTER TABLE component_set ALTER COLUMN env_vars SET NOT NULL;
-ALTER TABLE component_set ALTER COLUMN headers SET NOT NULL;
-

package/src/migrations/20221027151225_dummy.sql

@@ -1,2 +0,0 @@
--- Dummy migration step for testing https://jira.squiz.net/browse/DEVX-687
-SELECT * FROM "__migrations__";

package/src/migrations/20221117151200_add_detail_columns_to_component_version.js

@@ -1,49 +0,0 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-const path = require('path');
-const fs = require('fs');
-
-exports.default = async function (db, logger) {
-  await db.query(`ALTER TABLE component_version add display_name varchar not null default '';`);
-  await db.query(`ALTER TABLE component_version add description varchar not null default '';`);
-  await db.query(`ALTER TABLE component_version add icon jsonb;`);
-
-  const versions = await db.query(`select * from component_version`);
-  const dataMount = process.env.DATA_MOUNT_POINT;
-
-  if (dataMount === undefined) {
-    throw new Error(`DATA_MOUNT_POINT is undefined`);
-  }
-
-  for (const row of versions.rows) {
-    logger.info(dataMount, row.component_name, row.version, 'manifest.json');
-
-    const manifestPath = path.join(dataMount, row.component_name, row.version, 'manifest.json');
-
-    if (fs.existsSync(manifestPath)) {
-      logger.info(`found manifest ${manifestPath}`);
-
-      const manifestString = fs.readFileSync(manifestPath, {
-        encoding: 'utf-8',
-      });
-      const manifest = JSON.parse(manifestString);
-
-      logger.info(`adding ${manifest.displayName} ${manifest.description} ${manifest.icon} from manifest`);
-      let paramIndex = 1;
-      const params = [];
-      let setClause = `SET display_name=$${paramIndex++}, description=$${paramIndex++}`;
-      params.push(manifest.displayName ?? '');
-      params.push(manifest.description ?? '');
-
-      if (manifest.icon) {
-        setClause = `${setClause}, icon=$${paramIndex++}`;
-        params.push(JSON.stringify(manifest.icon));
-      }
-
-      let sql = `UPDATE component_version ${setClause} WHERE component_name=$${paramIndex++} and version=$${paramIndex++}`;
-      params.push(`${manifest.namespace}/${manifest.name}`);
-      params.push(manifest.version);
-
-      await db.query(sql, params);
-    }
-  }
-};

package/src/migrations/20221220151200_add_required_to_manifests.js

@@ -1,61 +0,0 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-const { JsonValidationService, SchemaValidationError } = require('@squiz/dx-json-schema-lib');
-const assert = require('assert');
-const fsp = require('fs/promises');
-const path = require('path');
-
-async function listDirectories(filePath) {
-  return (await fsp.readdir(filePath, { withFileTypes: true }))
-    .filter((dirent) => dirent.isDirectory())
-    .map((dirent) => path.resolve(filePath, dirent.name));
-}
-
-function addRequiredToNestedObjects(obj) {
-  if ('properties' in obj) {
-    obj.required ??= [];
-    for (const prop of Object.values(obj.properties)) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-}
-
-/**
- * Migrate the manifest files to add required to the input properties
- * when then existing manifest did not provide it. This is to match the
- * change of requirement of having a "required" property specified
- */
-exports.default = async function () {
-  assert.ok(process.env.DATA_MOUNT_POINT, 'DATA_MOUNT_POINT is undefined');
-  const validationService = new JsonValidationService();
-
-  for (const namespaceDir of await listDirectories(process.env.DATA_MOUNT_POINT)) {
-    for (const componentNameDir of await listDirectories(namespaceDir)) {
-      for (const versionDir of await listDirectories(componentNameDir)) {
-        try {
-          const manifestPath = path.resolve(versionDir, 'manifest.json');
-          const manifest = JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
-          if (manifest.$schema?.match(/v1\.json#?$/)) {
-            for (const fn of manifest.functions) {
-              addRequiredToNestedObjects(fn.input);
-            }
-            await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
-            try {
-              validationService.validateManifest(manifest, 'v1');
-            } catch (e) {
-              if (e instanceof SchemaValidationError) {
-                if (e.message.includes("must have required property 'required'")) {
-                  e.message = `Failed to migrate manifest ${manifest.namespace}/${manifest.name}/${manifest.version}: ${e.message}`;
-                  throw e;
-                }
-              }
-            }
-          }
-        } catch (e) {
-          if (e instanceof SchemaValidationError) {
-            throw e;
-          }
-        }
-      }
-    }
-  }
-};

package/src/migrations/20221222151200_add_required_to_manifests.js

@@ -1,138 +0,0 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-const { JsonValidationService, SchemaValidationError } = require('@squiz/dx-json-schema-lib');
-const assert = require('assert');
-const fsp = require('fs/promises');
-const path = require('path');
-
-async function listDirectories(filePath) {
-  return (await fsp.readdir(filePath, { withFileTypes: true }))
-    .filter((dirent) => dirent.isDirectory())
-    .map((dirent) => path.resolve(filePath, dirent.name));
-}
-
-function addRequiredToNestedObjects(obj) {
-  if (typeof obj !== 'object') return;
-
-  if ('properties' in obj) {
-    obj.required ??= [];
-    for (const prop of Object.values(obj.properties)) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-  if ('items' in obj) {
-    addRequiredToNestedObjects(obj.items);
-  }
-
-  if ('additionalProperties' in obj) {
-    addRequiredToNestedObjects(obj.additionalProperties);
-  }
-
-  if ('oneOf' in obj) {
-    for (const prop of obj.oneOf) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-
-  if ('anyOf' in obj) {
-    for (const prop of obj.anyOf) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-
-  if ('allOf' in obj) {
-    for (const prop of obj.allOf) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-
-  if ('not' in obj) {
-    addRequiredToNestedObjects(obj.not);
-  }
-
-  if ('then' in obj) {
-    addRequiredToNestedObjects(obj.then);
-  }
-
-  if ('else' in obj) {
-    addRequiredToNestedObjects(obj.else);
-  }
-
-  if ('if' in obj) {
-    addRequiredToNestedObjects(obj.if);
-  }
-
-  if ('dependencies' in obj) {
-    for (const prop of Object.values(obj.dependencies)) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-
-  if ('propertyNames' in obj) {
-    addRequiredToNestedObjects(obj.propertyNames);
-  }
-
-  if ('contains' in obj) {
-    addRequiredToNestedObjects(obj.contains);
-  }
-
-  if ('patternProperties' in obj) {
-    for (const prop of Object.values(obj.patternProperties)) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-
-  if ('definitions' in obj) {
-    for (const prop of Object.values(obj.definitions)) {
-      addRequiredToNestedObjects(prop);
-    }
-  }
-
-  if ('additionalItems' in obj) {
-    addRequiredToNestedObjects(obj.additionalItems);
-  }
-}
-
-/**
- * Migrate the manifest files to add required to the input properties
- * when then existing manifest did not provide it. This is to match the
- * change of requirement of having a "required" property specified
- */
-exports.default = async function () {
-  assert.ok(process.env.DATA_MOUNT_POINT, 'DATA_MOUNT_POINT is undefined');
-  const validationService = new JsonValidationService();
-
-  for (const namespaceDir of await listDirectories(process.env.DATA_MOUNT_POINT)) {
-    for (const componentNameDir of await listDirectories(namespaceDir)) {
-      for (const versionDir of await listDirectories(componentNameDir)) {
-        const manifestPath = path.resolve(versionDir, 'manifest.json');
-        const manifest = await tryReadManifest(manifestPath);
-        if (!manifest) continue;
-
-        if (manifest.$schema?.match(/v1\.json#?$/)) {
-          for (const fn of manifest.functions) {
-            addRequiredToNestedObjects(fn.input);
-          }
-          await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
-          try {
-            validationService.validateManifest(manifest, 'v1');
-          } catch (e) {
-            if (e instanceof SchemaValidationError) {
-              if (e.message.includes("must have required property 'required'")) {
-                e.message = `Failed to migrate manifest ${manifest.namespace}/${manifest.name}/${manifest.version}: ${e.message}`;
-                throw e;
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-};
-
-async function tryReadManifest(manifestPath) {
-  try {
-    return JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
-  } catch (_e) {
-    return null;
-  }
-}

package/src/migrations/20223525103556_component_set_rule.sql

@@ -1,19 +0,0 @@
-CREATE TABLE "component_set_rule" (
-  "component_name" VARCHAR(128) NOT NULL,
-  "component_set_web_path" VARCHAR(128) NOT NULL REFERENCES "component_set" ("web_path") ON DELETE CASCADE,
-  "version_pattern" VARCHAR(128) NOT NULL,
-
-  CONSTRAINT "component_set_rule_pkey" PRIMARY KEY ("component_name", "component_set_web_path")
-);
-
-CREATE INDEX "component_set_rule_component_name" ON "component_set_rule" ( "component_name" );
-CREATE INDEX "component_set_rule_component_set_web_path" ON "component_set_rule" ( "component_set_web_path" );
-
-
--- NOTE this does not require an update thanks to Postgres smarts:
--- > When a column is added with ADD COLUMN and a non-volatile DEFAULT is specified, 
--- > the default is evaluated at the time of the statement and the result stored in the table's metadata. 
--- > That value will be used for the column for all existing rows. If no DEFAULT is specified, NULL is used. 
--- > In neither case is a rewrite of the table required.
-ALTER TABLE "component_set_component_version"
-  ADD COLUMN "is_visible" BOOLEAN DEFAULT 'true';