@squiz/job-runner-lib 1.66.1

package/src/domains/manifest/utils/getAndAssertManifestVersion.spec.ts

@@ -0,0 +1,134 @@
+import { ManifestValidationError } from '../errors/ManifestValidationError';
+import { ManifestModel } from '../ManifestModel';
+import { getAndAssertManifestVersion } from './getAndAssertManifestVersion';
+
+describe('getAndAssertManifestVersion', () => {
+  it("should throw an error if Manifest object hasn't defined a $schema", () => {
+    expect(() =>
+      getAndAssertManifestVersion({
+        $schema: '',
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        namespace: 'foo',
+        displayName: 'something',
+        description: 'some-description',
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      }),
+    ).toThrowError(
+      new ManifestValidationError(
+        'Could not determine manifest version. Your manifest must have a $schema property defined.',
+      ),
+    );
+
+    expect(() =>
+      getAndAssertManifestVersion({
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        namespace: 'foo',
+        displayName: 'something',
+        description: 'some-description',
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      } as ManifestModel),
+    ).toThrowError(
+      new ManifestValidationError(
+        'Could not determine manifest version. Your manifest must have a $schema property defined.',
+      ),
+    );
+  });
+
+  it('should throw an error if Manifest object has invalid $schema version', () => {
+    expect(() =>
+      getAndAssertManifestVersion({
+        $schema: 'something-invalid',
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        namespace: 'foo',
+        displayName: 'something',
+        description: 'some-description',
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      }),
+    ).toThrowError(new ManifestValidationError('Could not determine manifest version from $schema value.'));
+  });
+
+  it("should throw an error if Manifest isn't a valid/defined version", () => {
+    expect(() =>
+      getAndAssertManifestVersion({
+        $schema: 'http://../schemas/v9999.json',
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        namespace: 'foo',
+        displayName: 'something',
+        description: 'some-description',
+
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      }),
+    ).toThrowError(
+      new ManifestValidationError('manifest version "v9999" isn\'t a valid version. Valid versions are "v1", "JobV1"'),
+    );
+  });
+
+  it('should return the schema version number if its valid and exists', () => {
+    expect(
+      getAndAssertManifestVersion({
+        $schema: 'http://../schemas/v1.json',
+        mainFunction: 'main',
+        name: 'test',
+        version: '1.0.0',
+        namespace: 'foo',
+        displayName: 'something',
+        description: 'some-description',
+
+        functions: [
+          {
+            entry: 'entry',
+            input: {},
+            name: 'main',
+            output: {
+              responseType: 'html',
+            },
+          },
+        ],
+      }),
+    ).toEqual('v1');
+  });
+});

package/src/domains/manifest/utils/getAndAssertManifestVersion.ts

@@ -0,0 +1,27 @@
+import { schemaVersionRegex, SchemaVersions, SCHEMA_VERSIONS } from '../AbstractManifest';
+import { ManifestValidationError } from '../errors/ManifestValidationError';
+import { ManifestModel } from '../ManifestModel';
+import { isValidManifestVersion } from './isValidManifestVersion';
+
+export function getAndAssertManifestVersion(manifest: ManifestModel): SchemaVersions {
+  if (!manifest['$schema']) {
+    throw new ManifestValidationError(
+      'Could not determine manifest version. Your manifest must have a $schema property defined.',
+    );
+  }
+
+  const match = schemaVersionRegex.exec(manifest['$schema']);
+  const version = match?.groups?.schema;
+
+  if (version === undefined) {
+    throw new ManifestValidationError('Could not determine manifest version from $schema value.');
+  }
+
+  if (isValidManifestVersion(version)) {
+    return version as SchemaVersions;
+  }
+
+  throw new ManifestValidationError(
+    `manifest version "${version}" isn't a valid version. Valid versions are "${SCHEMA_VERSIONS.join('", "')}"`,
+  );
+}

package/src/domains/manifest/utils/isValidManifestVersion.spec.ts

@@ -0,0 +1,19 @@
+import { SCHEMA_VERSIONS } from '../AbstractManifest';
+import { isValidManifestVersion } from './isValidManifestVersion';
+
+describe('isValidManifestVersion', () => {
+  it('should return true if the input value is a valid schema version', () => {
+    for (const validVersion of SCHEMA_VERSIONS) {
+      expect(isValidManifestVersion(validVersion)).toEqual(true);
+    }
+  });
+
+  it('should return false if any value other than a valid version is entered', () => {
+    expect(isValidManifestVersion('bad-version')).toEqual(false);
+  });
+
+  it('should not exception if arbitrary objects are added', () => {
+    expect(isValidManifestVersion({ 'bad-version': 123 })).toEqual(false);
+    expect(isValidManifestVersion(undefined)).toEqual(false);
+  });
+});

package/src/domains/manifest/utils/isValidManifestVersion.ts

@@ -0,0 +1,9 @@
+import { SchemaVersions, SCHEMA_VERSIONS } from '../AbstractManifest';
+
+export function isValidManifestVersion(item: unknown): item is SchemaVersions {
+  if (typeof item === 'string') {
+    return SCHEMA_VERSIONS.includes(item);
+  }
+
+  return false;
+}

package/src/domains/url/convertFunctionStaticFilesToFqdn.spec.ts

@@ -0,0 +1,24 @@
+import { convertFunctionStaticFilesToFqdn } from './convertFunctionStaticFilesToFqdn';
+
+describe('convertFunctionStaticFilesToFqdn', () => {
+  it('should resolve a file path to a component static resource url string', () => {
+    expect(
+      convertFunctionStaticFilesToFqdn('http://localhost:3000/', './hello/file.txt', 'set-a', 'test', '1.0.0'),
+    ).toEqual('http://localhost:3000/s/test/1.0.0/hello/file.txt?_componentSet=set-a');
+  });
+
+  it('should resolve a relative file path', () => {
+    expect(
+      convertFunctionStaticFilesToFqdn('http://localhost:3000/', './hello/../file.txt', 'set-a', 'test', '1.0.0'),
+    ).toEqual('http://localhost:3000/s/test/1.0.0/file.txt?_componentSet=set-a');
+  });
+
+  it('if give a url do nothing but return it', () => {
+    expect(
+      convertFunctionStaticFilesToFqdn('http://localhost:3000/', 'http://hello/world', 'set-a', 'test', '1.0.0'),
+    ).toEqual('http://hello/world');
+    expect(
+      convertFunctionStaticFilesToFqdn('http://localhost:3000/', 'https://hello/world', 'set-a', 'test', '1.0.0'),
+    ).toEqual('https://hello/world');
+  });
+});

package/src/domains/url/convertFunctionStaticFilesToFqdn.ts

@@ -0,0 +1,25 @@
+import { joinAbsoluteUrlPath } from '@squiz/dx-common-lib';
+import queryString from 'query-string';
+
+export function convertFunctionStaticFilesToFqdn(
+  rootUrl: string,
+  file: string,
+  componentSet: string,
+  componentName: string,
+  version: string,
+): string {
+  if (file.startsWith('https://') || file.startsWith('http://')) {
+    return file;
+  }
+
+  const path = joinAbsoluteUrlPath(rootUrl, 's', componentName, version, file);
+  return queryString.stringifyUrl(
+    {
+      url: path,
+      query: {
+        _componentSet: componentSet,
+      },
+    },
+    { skipNull: true, encode: true, arrayFormat: 'bracket' },
+  );
+}

package/src/domains/url/convertFunctionStaticFilesToFqdnForPreview.spec.ts

@@ -0,0 +1,48 @@
+import { convertFunctionStaticFilesToFqdnForPreview } from './convertFunctionStaticFilesToFqdnForPreview';
+
+describe('convertFunctionStaticFilesToFqdnForPreview', () => {
+  it('should resolve a file path to a component static resource url string', () => {
+    expect(
+      convertFunctionStaticFilesToFqdnForPreview(
+        'http://localhost:3000/',
+        './hello/file.txt',
+        'test',
+        '1.0.0',
+        'test-preview',
+      ),
+    ).toEqual('http://localhost:3000/s/test/1.0.0/hello/file.txt?_previewKey=test-preview');
+  });
+
+  it('should resolve a relative file path', () => {
+    expect(
+      convertFunctionStaticFilesToFqdnForPreview(
+        'http://localhost:3000/',
+        './hello/../file.txt',
+        'test',
+        '1.0.0',
+        'test-preview',
+      ),
+    ).toEqual('http://localhost:3000/s/test/1.0.0/file.txt?_previewKey=test-preview');
+  });
+
+  it('if given a url do nothing but return it', () => {
+    expect(
+      convertFunctionStaticFilesToFqdnForPreview(
+        'http://localhost:3000/',
+        'http://hello/world',
+        'test',
+        '1.0.0',
+        'test-preview',
+      ),
+    ).toEqual('http://hello/world');
+    expect(
+      convertFunctionStaticFilesToFqdnForPreview(
+        'http://localhost:3000/',
+        'https://hello/world',
+        'test',
+        '1.0.0',
+        'test-preview',
+      ),
+    ).toEqual('https://hello/world');
+  });
+});

package/src/domains/url/convertFunctionStaticFilesToFqdnForPreview.ts

@@ -0,0 +1,25 @@
+import { joinAbsoluteUrlPath } from '@squiz/dx-common-lib';
+import queryString from 'query-string';
+
+export function convertFunctionStaticFilesToFqdnForPreview(
+  rootUrl: string,
+  file: string,
+  componentName: string,
+  version: string,
+  previewKey: string,
+): string {
+  if (file.startsWith('https://') || file.startsWith('http://')) {
+    return file;
+  }
+
+  const path = joinAbsoluteUrlPath(rootUrl, 's', componentName, version, file);
+  return queryString.stringifyUrl(
+    {
+      url: path,
+      query: {
+        _previewKey: previewKey,
+      },
+    },
+    { skipNull: true, encode: true, arrayFormat: 'bracket' },
+  );
+}

package/src/domains/url/index.ts

@@ -0,0 +1 @@
+export * from './convertFunctionStaticFilesToFqdn';

package/src/index.ts

@@ -0,0 +1,10 @@
+export * from './domains/manifest/ManifestModel';
+export * from './domains/db';
+export * from './domains/component-set';
+export * from './domains/component';
+export * from './domains/job';
+export * from './domains/content';
+export * from './domains/url';
+export * from './domains/manifest';
+
+export * as TestHelpers from './TestHelpers';

package/src/jest-global-setup.ts

@@ -0,0 +1,20 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+import { getLogger } from '@squiz/dx-logger-lib';
+
+export const testLogger = getLogger({
+  name: 'component-lib-test',
+});
+
+export default async (): Promise<void> => {
+  process.env.TZ = 'UTC';
+  process.env.DYNAMODB_TABLE = 'jobs-test';
+  process.env.ENVIRONMENT = 'local';
+  process.env.CLUSTER_ARN = '1234';
+  process.env.RUNTIME_TASK_DEFINITION_ARN = '5678';
+  process.env.SUBNETS_STRING = 'subnet1_subnet2';
+  process.env.S3_BUCKET = 'jobs';
+  process.env.JOB_RUNNER_API_URL = 'http://localhost:3040';
+};

package/src/migrations/20220704054051_initial.sql

@@ -0,0 +1,19 @@
+-- CreateTable
+
+CREATE TABLE "component" ("name" VARCHAR(128) NOT NULL,
+                                              CONSTRAINT "component_pkey" PRIMARY KEY ("name"));
+
+-- CreateTable
+
+CREATE TABLE "component_version" ("component_name" VARCHAR(128) NOT NULL,
+                                                                "version" VARCHAR(128) NOT NULL,
+                                                                                       CONSTRAINT "component_version_pkey" PRIMARY KEY ("component_name",
+                                                                                                                                        "version"));
+
+-- AddForeignKey
+
+ALTER TABLE "component_version" ADD CONSTRAINT "component_version_component_name_fkey"
+FOREIGN KEY ("component_name") REFERENCES "component"("name") ON
+DELETE CASCADE ON
+UPDATE CASCADE;
+

package/src/migrations/20220718172237_adding_component_sets.sql

@@ -0,0 +1,23 @@
+-- CreateTable
+CREATE TABLE "component_set" (
+    "web_path" VARCHAR(128) NOT NULL,
+    "display_name" VARCHAR(128) NOT NULL,
+    "description" VARCHAR,
+    "env_vars" JSONB,
+    "headers" JSONB,
+
+    CONSTRAINT "component_set_pkey" PRIMARY KEY ("web_path")
+);
+
+-- CreateTable
+CREATE TABLE "component_set_component_version" (
+    "component_set_web_path" VARCHAR(128) NOT NULL,
+    "component_name" VARCHAR(128) NOT NULL,
+    "component_version" VARCHAR(128) NOT NULL,
+
+    CONSTRAINT "component_set_component_version_pkey" PRIMARY KEY ("component_set_web_path", "component_name", "component_version")
+);
+
+-- AddForeignKey
+ALTER TABLE "component_set_component_version" ADD CONSTRAINT "component_set_web_path_fkey" FOREIGN KEY ("component_set_web_path") REFERENCES "component_set"("web_path") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "component_set_component_version" ADD CONSTRAINT "component_version_componet_name_version_fkey" FOREIGN KEY ("component_name", "component_version") REFERENCES "component_version"("component_name", "version") ON DELETE CASCADE ON UPDATE CASCADE;

package/src/migrations/20220728113941_add_env_vars_field.sql

@@ -0,0 +1 @@
+ALTER TABLE "component_set_component_version" ADD COLUMN "env_vars" jsonb;

package/src/migrations/20220817113300_removing_null_props_from_jsonb.sql

@@ -0,0 +1,41 @@
+UPDATE component_set as rc
+set env_vars = (
+
+  -- subtract non string properties, resulting object is all keys have values
+  select COALESCE((o2.env_vars - od.nonStringProps), '{}'::jsonb) as stringProps
+  from component_set as o2 , (
+
+    -- select get an array of all properties whos value is not a string, as an array
+    select web_path, array(
+      select envK
+      from component_set, jsonb_object_keys(env_vars) as envK
+      where  jsonb_typeof(component_set.env_vars->envk) <> 'string' and o.web_path = component_set.web_path
+    ) as nonStringProps
+    from component_set as o
+  ) as od
+
+  -- make sure all the web_paths are the same
+  where o2.web_path = od.web_path and rc.web_path = o2.web_path
+),
+headers = (
+  -- subtract non string properties resulting object is all keys have values
+  select COALESCE((o2.headers - od.nonStringProps), '{}'::jsonb) as stringProps
+  from component_set as o2 , (
+
+    -- select get an array of all properties whos value is not a string, as an array
+    select web_path, array(
+      select headerK
+      from component_set, jsonb_object_keys(headers) as headerK
+      where  jsonb_typeof(component_set.headers->headerK) <> 'string' and o.web_path = component_set.web_path
+    ) as nonStringProps
+    from component_set as o
+  ) as od
+
+  -- make sure all the web_paths are the same
+  where o2.web_path = od.web_path and rc.web_path = o2.web_path
+);
+
+-- make columns not nullable
+ALTER TABLE component_set ALTER COLUMN env_vars SET NOT NULL;
+ALTER TABLE component_set ALTER COLUMN headers SET NOT NULL;
+

package/src/migrations/20221027151225_dummy.sql

@@ -0,0 +1,2 @@
+-- Dummy migration step for testing https://jira.squiz.net/browse/DEVX-687
+SELECT * FROM "__migrations__";

package/src/migrations/20221117151200_add_detail_columns_to_component_version.js

@@ -0,0 +1,49 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const path = require('path');
+const fs = require('fs');
+
+exports.default = async function (db, logger) {
+  await db.query(`ALTER TABLE component_version add display_name varchar not null default '';`);
+  await db.query(`ALTER TABLE component_version add description varchar not null default '';`);
+  await db.query(`ALTER TABLE component_version add icon jsonb;`);
+
+  const versions = await db.query(`select * from component_version`);
+  const dataMount = process.env.DATA_MOUNT_POINT;
+
+  if (dataMount === undefined) {
+    throw new Error(`DATA_MOUNT_POINT is undefined`);
+  }
+
+  for (const row of versions.rows) {
+    logger.info(dataMount, row.component_name, row.version, 'manifest.json');
+
+    const manifestPath = path.join(dataMount, row.component_name, row.version, 'manifest.json');
+
+    if (fs.existsSync(manifestPath)) {
+      logger.info(`found manifest ${manifestPath}`);
+
+      const manifestString = fs.readFileSync(manifestPath, {
+        encoding: 'utf-8',
+      });
+      const manifest = JSON.parse(manifestString);
+
+      logger.info(`adding ${manifest.displayName} ${manifest.description} ${manifest.icon} from manifest`);
+      let paramIndex = 1;
+      const params = [];
+      let setClause = `SET display_name=$${paramIndex++}, description=$${paramIndex++}`;
+      params.push(manifest.displayName ?? '');
+      params.push(manifest.description ?? '');
+
+      if (manifest.icon) {
+        setClause = `${setClause}, icon=$${paramIndex++}`;
+        params.push(JSON.stringify(manifest.icon));
+      }
+
+      let sql = `UPDATE component_version ${setClause} WHERE component_name=$${paramIndex++} and version=$${paramIndex++}`;
+      params.push(`${manifest.namespace}/${manifest.name}`);
+      params.push(manifest.version);
+
+      await db.query(sql, params);
+    }
+  }
+};

package/src/migrations/20221220151200_add_required_to_manifests.js

@@ -0,0 +1,61 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const { JsonValidationService, SchemaValidationError } = require('@squiz/dx-json-schema-lib');
+const assert = require('assert');
+const fsp = require('fs/promises');
+const path = require('path');
+
+async function listDirectories(filePath) {
+  return (await fsp.readdir(filePath, { withFileTypes: true }))
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => path.resolve(filePath, dirent.name));
+}
+
+function addRequiredToNestedObjects(obj) {
+  if ('properties' in obj) {
+    obj.required ??= [];
+    for (const prop of Object.values(obj.properties)) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+}
+
+/**
+ * Migrate the manifest files to add required to the input properties
+ * when then existing manifest did not provide it. This is to match the
+ * change of requirement of having a "required" property specified
+ */
+exports.default = async function () {
+  assert.ok(process.env.DATA_MOUNT_POINT, 'DATA_MOUNT_POINT is undefined');
+  const validationService = new JsonValidationService();
+
+  for (const namespaceDir of await listDirectories(process.env.DATA_MOUNT_POINT)) {
+    for (const componentNameDir of await listDirectories(namespaceDir)) {
+      for (const versionDir of await listDirectories(componentNameDir)) {
+        try {
+          const manifestPath = path.resolve(versionDir, 'manifest.json');
+          const manifest = JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
+          if (manifest.$schema?.match(/v1\.json#?$/)) {
+            for (const fn of manifest.functions) {
+              addRequiredToNestedObjects(fn.input);
+            }
+            await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
+            try {
+              validationService.validateManifest(manifest, 'v1');
+            } catch (e) {
+              if (e instanceof SchemaValidationError) {
+                if (e.message.includes("must have required property 'required'")) {
+                  e.message = `Failed to migrate manifest ${manifest.namespace}/${manifest.name}/${manifest.version}: ${e.message}`;
+                  throw e;
+                }
+              }
+            }
+          }
+        } catch (e) {
+          if (e instanceof SchemaValidationError) {
+            throw e;
+          }
+        }
+      }
+    }
+  }
+};