@squiz/job-runner-lib 1.66.1

package/src/migrations/20221222151200_add_required_to_manifests.js

@@ -0,0 +1,138 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const { JsonValidationService, SchemaValidationError } = require('@squiz/dx-json-schema-lib');
+const assert = require('assert');
+const fsp = require('fs/promises');
+const path = require('path');
+
+async function listDirectories(filePath) {
+  return (await fsp.readdir(filePath, { withFileTypes: true }))
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => path.resolve(filePath, dirent.name));
+}
+
+function addRequiredToNestedObjects(obj) {
+  if (typeof obj !== 'object') return;
+
+  if ('properties' in obj) {
+    obj.required ??= [];
+    for (const prop of Object.values(obj.properties)) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+  if ('items' in obj) {
+    addRequiredToNestedObjects(obj.items);
+  }
+
+  if ('additionalProperties' in obj) {
+    addRequiredToNestedObjects(obj.additionalProperties);
+  }
+
+  if ('oneOf' in obj) {
+    for (const prop of obj.oneOf) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+
+  if ('anyOf' in obj) {
+    for (const prop of obj.anyOf) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+
+  if ('allOf' in obj) {
+    for (const prop of obj.allOf) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+
+  if ('not' in obj) {
+    addRequiredToNestedObjects(obj.not);
+  }
+
+  if ('then' in obj) {
+    addRequiredToNestedObjects(obj.then);
+  }
+
+  if ('else' in obj) {
+    addRequiredToNestedObjects(obj.else);
+  }
+
+  if ('if' in obj) {
+    addRequiredToNestedObjects(obj.if);
+  }
+
+  if ('dependencies' in obj) {
+    for (const prop of Object.values(obj.dependencies)) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+
+  if ('propertyNames' in obj) {
+    addRequiredToNestedObjects(obj.propertyNames);
+  }
+
+  if ('contains' in obj) {
+    addRequiredToNestedObjects(obj.contains);
+  }
+
+  if ('patternProperties' in obj) {
+    for (const prop of Object.values(obj.patternProperties)) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+
+  if ('definitions' in obj) {
+    for (const prop of Object.values(obj.definitions)) {
+      addRequiredToNestedObjects(prop);
+    }
+  }
+
+  if ('additionalItems' in obj) {
+    addRequiredToNestedObjects(obj.additionalItems);
+  }
+}
+
+/**
+ * Migrate the manifest files to add required to the input properties
+ * when then existing manifest did not provide it. This is to match the
+ * change of requirement of having a "required" property specified
+ */
+exports.default = async function () {
+  assert.ok(process.env.DATA_MOUNT_POINT, 'DATA_MOUNT_POINT is undefined');
+  const validationService = new JsonValidationService();
+
+  for (const namespaceDir of await listDirectories(process.env.DATA_MOUNT_POINT)) {
+    for (const componentNameDir of await listDirectories(namespaceDir)) {
+      for (const versionDir of await listDirectories(componentNameDir)) {
+        const manifestPath = path.resolve(versionDir, 'manifest.json');
+        const manifest = await tryReadManifest(manifestPath);
+        if (!manifest) continue;
+
+        if (manifest.$schema?.match(/v1\.json#?$/)) {
+          for (const fn of manifest.functions) {
+            addRequiredToNestedObjects(fn.input);
+          }
+          await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
+          try {
+            validationService.validateManifest(manifest, 'v1');
+          } catch (e) {
+            if (e instanceof SchemaValidationError) {
+              if (e.message.includes("must have required property 'required'")) {
+                e.message = `Failed to migrate manifest ${manifest.namespace}/${manifest.name}/${manifest.version}: ${e.message}`;
+                throw e;
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+};
+
+async function tryReadManifest(manifestPath) {
+  try {
+    return JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
+  } catch (_e) {
+    return null;
+  }
+}

package/src/migrations/20223525103556_component_set_rule.sql

@@ -0,0 +1,19 @@
+CREATE TABLE "component_set_rule" (
+  "component_name" VARCHAR(128) NOT NULL,
+  "component_set_web_path" VARCHAR(128) NOT NULL REFERENCES "component_set" ("web_path") ON DELETE CASCADE,
+  "version_pattern" VARCHAR(128) NOT NULL,
+
+  CONSTRAINT "component_set_rule_pkey" PRIMARY KEY ("component_name", "component_set_web_path")
+);
+
+CREATE INDEX "component_set_rule_component_name" ON "component_set_rule" ( "component_name" );
+CREATE INDEX "component_set_rule_component_set_web_path" ON "component_set_rule" ( "component_set_web_path" );
+
+
+-- NOTE this does not require an update thanks to Postgres smarts:
+-- > When a column is added with ADD COLUMN and a non-volatile DEFAULT is specified, 
+-- > the default is evaluated at the time of the statement and the result stored in the table's metadata. 
+-- > That value will be used for the column for all existing rows. If no DEFAULT is specified, NULL is used. 
+-- > In neither case is a rewrite of the table required.
+ALTER TABLE "component_set_component_version"
+  ADD COLUMN "is_visible" BOOLEAN DEFAULT 'true';

package/src/migrations/20230111095522_update_environment_property.js

@@ -0,0 +1,72 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const { JsonValidationService, SchemaValidationError } = require('@squiz/dx-json-schema-lib');
+const assert = require('assert');
+const fsp = require('fs/promises');
+const path = require('path');
+
+/**
+ * Migrate the manifest files to update "environment" property from
+ * "array of strings" to "array of objects"
+ * https://jira.squiz.net/browse/DEVX-808
+ */
+exports.default = async function () {
+  assert.ok(process.env.DATA_MOUNT_POINT, 'DATA_MOUNT_POINT is undefined');
+  const validationService = new JsonValidationService();
+
+  for (const namespaceDir of await listDirectories(process.env.DATA_MOUNT_POINT)) {
+    for (const componentNameDir of await listDirectories(namespaceDir)) {
+      for (const versionDir of await listDirectories(componentNameDir)) {
+        const manifestPath = path.resolve(versionDir, 'manifest.json');
+        const manifest = await tryReadManifest(manifestPath);
+        if (!manifest) continue;
+
+        if ('environment' in manifest) {
+          const updatedEnvironment = getUpdatedEnvironmentPropertyValue(manifest.environment);
+
+          if (JSON.stringify(updatedEnvironment) !== JSON.stringify(manifest.environment)) {
+            manifest.environment = updatedEnvironment;
+            try {
+              validationService.validateManifest(manifest, 'v1');
+              await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
+            } catch (e) {
+              if (e instanceof SchemaValidationError) {
+                if (e.message.includes('failed validation: /environment/')) {
+                  e.message = `Failed to migrate manifest ${manifest.namespace}/${manifest.name}/${manifest.version}: ${e.message}`;
+                  throw e;
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+};
+
+function getUpdatedEnvironmentPropertyValue(environment) {
+  const updatedEnvironment = [];
+  for (const envVar of environment) {
+    if (typeof envVar === 'string') {
+      updatedEnvironment.push({
+        name: envVar,
+      });
+    } else {
+      updatedEnvironment.push(envVar);
+    }
+  }
+  return updatedEnvironment;
+}
+
+async function listDirectories(filePath) {
+  return (await fsp.readdir(filePath, { withFileTypes: true }))
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => path.resolve(filePath, dirent.name));
+}
+
+async function tryReadManifest(manifestPath) {
+  try {
+    return JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
+  } catch (_e) {
+    return null;
+  }
+}

package/src/migrations/20230203084900_ensure_static_root_exists.js

@@ -0,0 +1,57 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const assert = require('assert');
+const fsp = require('fs/promises');
+const path = require('path');
+
+/**
+ * Ensure all static root's exist.
+ * https://jira.squiz.net/browse/DEVX-800
+ */
+exports.default = async function () {
+  assert.ok(process.env.DATA_MOUNT_POINT, 'DATA_MOUNT_POINT is undefined');
+
+  for (const namespaceDir of await listDirectories(process.env.DATA_MOUNT_POINT)) {
+    for (const componentNameDir of await listDirectories(namespaceDir)) {
+      for (const versionDir of await listDirectories(componentNameDir)) {
+        const manifestPath = path.resolve(versionDir, 'manifest.json');
+        const manifest = await tryReadManifest(manifestPath);
+        if (!manifest) continue;
+
+        if ('staticFiles' in manifest && 'locationRoot' in manifest.staticFiles) {
+          const staticFiles = path.resolve(path.join(versionDir, manifest.staticFiles.locationRoot));
+
+          if ((await pathExists(staticFiles)) == false) {
+            await fsp.mkdir(staticFiles, { recursive: true });
+          }
+        }
+      }
+    }
+  }
+};
+
+async function listDirectories(filePath) {
+  return (await fsp.readdir(filePath, { withFileTypes: true }))
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => path.resolve(filePath, dirent.name));
+}
+
+async function tryReadManifest(manifestPath) {
+  try {
+    return JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
+  } catch (_e) {
+    return null;
+  }
+}
+
+async function pathExists(path) {
+  try {
+    await fsp.stat(path);
+    return true;
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return false;
+    } else {
+      throw error;
+    }
+  }
+}

package/src/migrations/20230210143300_lowercase_all_uri_names.js

@@ -0,0 +1,106 @@
+const fsp = require('fs/promises');
+const { readdirSync } = require('fs');
+const path = require('path');
+
+/**
+ *
+ * @param {import('pg').PoolClient} db
+ * @param {import('@squiz/dx-logger-lib').Logger}
+ */
+exports.default = async function (db, _logger) {
+  await db.query(`ALTER TABLE component drop constraint IF EXISTS component_name_is_lower`);
+  await db.query(`ALTER TABLE component_version drop constraint IF EXISTS component_version_component_name_is_lower`);
+  await db.query(`ALTER TABLE component_set drop constraint IF EXISTS component_set_web_path_is_lower`);
+
+  await db.query(`UPDATE component SET name = lower(name)`);
+  await db.query(`ALTER TABLE component add constraint component_name_is_lower check (name = lower(name))`);
+  await db.query(`UPDATE component_version SET component_name = lower(component_name)`);
+  await db.query(
+    `ALTER TABLE component_version add constraint component_version_component_name_is_lower check (component_name = lower(component_name))`,
+  );
+
+  await db.query(`UPDATE component_set SET web_path = lower(web_path)`);
+  await db.query(
+    `ALTER TABLE component_set add constraint component_set_web_path_is_lower check (web_path = lower(web_path))`,
+  );
+
+  for await (const { manifest, manifestPath } of getComponentManifests()) {
+    const { namespace, name } = manifest;
+
+    await fsp.writeFile(
+      manifestPath,
+      JSON.stringify(
+        /** @type {import('@squiz/dx-json-schema-lib').MANIFEST_MODELS.v1.ComponentManifest} */
+        {
+          ...manifest,
+          namespace: namespace.toLowerCase(),
+          name: name.toLowerCase(),
+          functions: manifest.functions?.map((fn) => ({
+            ...fn,
+            name: fn.name.toLowerCase(),
+          })),
+          mainFunction: manifest.mainFunction?.toLowerCase(),
+        },
+        null,
+        2,
+      ),
+    );
+    const versionDirectory = path.dirname(manifestPath);
+    const componentNameDirectory = path.dirname(versionDirectory);
+    const namespaceDirectory = path.dirname(componentNameDirectory);
+
+    const lowerName = path.basename(componentNameDirectory).toLowerCase();
+    const lowerNamespace = path.basename(namespaceDirectory).toLowerCase();
+
+    await fsp.rename(componentNameDirectory, path.resolve(namespaceDirectory, lowerName));
+    await fsp.rename(namespaceDirectory, path.resolve(path.dirname(namespaceDirectory), lowerNamespace));
+  }
+};
+
+async function* getComponentManifests() {
+  for (const componentDirectory of listDirectories(process.env.DATA_MOUNT_POINT)) {
+    const manifestPath = path.resolve(componentDirectory, 'manifest.json');
+    const manifest = await tryReadManifest(manifestPath);
+    if (manifest?.$schema?.match(/v1\.json#?$/)) {
+      yield { manifest, manifestPath };
+    }
+  }
+}
+
+/**
+ *
+ * @param {string} filePath
+ * @param {number} nesting
+ * @returns {Generator<string>}
+ */
+function* listDirectories(filePath, nesting = 3) {
+  let files;
+  try {
+    files = readdirSync(filePath, { withFileTypes: true });
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      throw err;
+    }
+    return;
+  }
+
+  for (const file of files) {
+    if (file.name === 'manifest.json') yield filePath;
+    else if (!file.isDirectory()) continue;
+    else if (nesting === 0) yield filePath;
+    else yield* listDirectories(path.resolve(filePath, file.name), nesting - 1);
+  }
+}
+
+/**
+ *
+ * @param {string} manifestPath
+ * @returns {Promise<import('@squiz/dx-json-schema-lib').MANIFEST_MODELS.v1.ComponentManifest | null>}
+ */
+async function tryReadManifest(manifestPath) {
+  try {
+    return JSON.parse(await fsp.readFile(manifestPath, { encoding: 'utf-8' }));
+  } catch (_e) {
+    return null;
+  }
+}

package/src/migrations/20230217151212_component_set_component_version_updates.sql

@@ -0,0 +1,38 @@
+-- Create new tables
+CREATE TABLE "component_set_component_version_excluded" (
+    "component_set_web_path" VARCHAR(128) NOT NULL,
+    "component_name" VARCHAR(128) NOT NULL,
+    "component_version" VARCHAR(128) NOT NULL,
+
+    CONSTRAINT "component_set_component_version_excluded_pkey" PRIMARY KEY ("component_set_web_path", "component_name", "component_version")
+);
+
+CREATE TABLE "component_set_component_version_editable" (
+    "component_set_web_path" VARCHAR(128) NOT NULL,
+    "component_name" VARCHAR(128) NOT NULL,
+    "component_version" VARCHAR(128) NOT NULL,
+
+    CONSTRAINT "component_set_component_version_editable_pkey" PRIMARY KEY ("component_set_web_path", "component_name", "component_version")
+);
+
+-- Add foreign keys
+ALTER TABLE "component_set_component_version_excluded" ADD CONSTRAINT "component_set_web_path_fkey" FOREIGN KEY ("component_set_web_path") REFERENCES "component_set"("web_path") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "component_set_component_version_editable" ADD CONSTRAINT "component_set_web_path_fkey" FOREIGN KEY ("component_set_web_path") REFERENCES "component_set"("web_path") ON DELETE CASCADE ON UPDATE CASCADE;
+
+
+-- Add enum column to component_set_component_version table
+CREATE TYPE "component_set_component_version_status" AS ENUM('active', 'excluded');
+ALTER TABLE "component_set_component_version" ADD COLUMN "status" "component_set_component_version_status" NOT NULL DEFAULT 'active';
+
+-- Migrate the existing "is_visible" data to the new "status" column
+UPDATE "component_set_component_version" 
+SET "status" = (CASE 
+    WHEN "is_visible" = 'true' 
+      THEN 'active'::component_set_component_version_status
+    WHEN "is_visible" = 'false'
+      THEN 'excluded'::component_set_component_version_status
+    END
+);
+    
+-- Delete now un-needed "is_visible" column
+ALTER TABLE "component_set_component_version" DROP COLUMN "is_visible";

package/src/migrations/20230218121212_add_component_set_rules.js

@@ -0,0 +1,31 @@
+/**
+ * Add component set rules for component versions that already exist
+ * in the component set
+ * Do nothing if the rule is already defined for that set
+ * https://jira.squiz.net/browse/DEVX-803
+ */
+exports.default = async function (dbClient) {
+  const result = await dbClient.query("select * from component_set_component_version where status='active'");
+  const rows = result.rows ?? [];
+  const componentSetVersions = {};
+  for (const row of rows) {
+    if (componentSetVersions[row.component_set_web_path] === undefined) {
+      componentSetVersions[row.component_set_web_path] = {};
+    }
+    if (componentSetVersions[row.component_set_web_path][row.component_name] === undefined) {
+      componentSetVersions[row.component_set_web_path][row.component_name] = [];
+    }
+    componentSetVersions[row.component_set_web_path][row.component_name].push(row.component_version);
+  }
+
+  for (const [componentSet, components] of Object.entries(componentSetVersions)) {
+    for (const [componentName, componentVersions] of Object.entries(components)) {
+      await dbClient.query(
+        'insert into ' +
+          'component_set_rule(component_name, component_set_web_path, version_pattern) ' +
+          `values('${componentName}', '${componentSet}', '${componentVersions.join(' || ')}') ` +
+          'ON CONFLICT (component_name, component_set_web_path) DO NOTHING',
+      );
+    }
+  }
+};

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../tsconfig.json",
+
+  "compilerOptions": {
+    "outDir": "lib/",
+    "resolveJsonModule": true,
+    "composite": true,
+    "rootDir": "./src",
+    "sourceMap": true
+  },
+
+  "references": [],
+
+  "include": ["src/**/*.ts", "src/**/*.json"],
+  "exclude": ["jest.config.ts", "src/migrations/**/*"]
+}