@balena/pinejs 14.44.0-linear-runtime-migrator-9d426d29f2b01ef0425c72333c2c691976fdc219 → 15.0.0-delete-state-default-user-permissions-57ce3dc6141cd1e8159f4c34da29d1fb113242c6

package/src/migrator/sync.ts

@@ -1,167 +0,0 @@
-import {
-	modelText,
-	MigrationTuple,
-	Migrations,
-	MigrationError,
-	defaultMigrationCategory,
-	checkModelAlreadyExists,
-	setExecutedMigrations,
-	getExecutedMigrations,
-	lockMigrations,
-} from './utils';
-import type { Tx } from '../database-layer/db';
-import type { Config, Model } from '../config-loader/config-loader';
-
-import * as _ from 'lodash';
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-
-type ApiRootModel = Model & { apiRoot: string };
-
-export const postRun = async (tx: Tx, model: ApiRootModel): Promise<void> => {
-	const { initSql } = model;
-	if (initSql == null) {
-		return;
-	}
-
-	const modelName = model.apiRoot;
-
-	const exists = await checkModelAlreadyExists(tx, modelName);
-	if (!exists) {
-		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-			'First time executing, running init script',
-		);
-
-		await lockMigrations(tx, modelName, async () => {
-			try {
-				await tx.executeSql(initSql);
-			} catch (err) {
-				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-					`postRun locked sql execution error ${err} `,
-				);
-			}
-		});
-	}
-};
-
-export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
-	const { migrations } = model;
-	if (
-		migrations == null ||
-		_.isEmpty(migrations) ||
-		migrations[defaultMigrationCategory] === undefined
-	) {
-		return;
-	}
-	return $run(tx, model, migrations[defaultMigrationCategory]);
-};
-
-const $run = async (
-	tx: Tx,
-	model: ApiRootModel,
-	migrations: Migrations,
-): Promise<void> => {
-	const modelName = model.apiRoot;
-
-	// migrations only run if the model has been executed before,
-	// to make changes that can't be automatically applied
-	const exists = await checkModelAlreadyExists(tx, modelName);
-	if (!exists) {
-		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-			'First time model has executed, skipping migrations',
-		);
-
-		return await setExecutedMigrations(tx, modelName, Object.keys(migrations));
-	}
-	await lockMigrations(tx, modelName, async () => {
-		try {
-			const executedMigrations = await getExecutedMigrations(tx, modelName);
-			const pendingMigrations = filterAndSortPendingMigrations(
-				migrations,
-				executedMigrations,
-			);
-			if (pendingMigrations.length === 0) {
-				return;
-			}
-
-			const newlyExecutedMigrations = await executeMigrations(
-				tx,
-				pendingMigrations,
-			);
-			await setExecutedMigrations(tx, modelName, [
-				...executedMigrations,
-				...newlyExecutedMigrations,
-			]);
-		} catch (err) {
-			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-				`$run executedMigrations error ${err}`,
-			);
-		}
-	});
-};
-
-// turns {"key1": migration, "key3": migration, "key2": migration}
-// into  [["key1", migration], ["key2", migration], ["key3", migration]]
-const filterAndSortPendingMigrations = (
-	migrations: NonNullable<Migrations>,
-	executedMigrations: string[],
-): MigrationTuple[] =>
-	(_(migrations).omit(executedMigrations) as _.Object<typeof migrations>)
-		.toPairs()
-		.sortBy(([migrationKey]) => migrationKey)
-		.value();
-
-const executeMigrations = async (
-	tx: Tx,
-	migrations: MigrationTuple[] = [],
-): Promise<string[]> => {
-	try {
-		for (const migration of migrations) {
-			await executeMigration(tx, migration);
-		}
-	} catch (err) {
-		(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-			'Error while executing migrations, rolled back',
-		);
-		throw new MigrationError(err);
-	}
-	return migrations.map(([migrationKey]) => migrationKey); // return migration keys
-};
-
-const executeMigration = async (
-	tx: Tx,
-	[key, migration]: MigrationTuple,
-): Promise<void> => {
-	(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-		`Running migration ${JSON.stringify(key)}`,
-	);
-
-	if (typeof migration === 'function') {
-		await migration(tx, sbvrUtils);
-	} else if (typeof migration === 'string') {
-		await tx.executeSql(migration);
-	} else {
-		throw new MigrationError(`Invalid migration type: ${typeof migration}`);
-	}
-};
-
-export const config: Config = {
-	models: [
-		{
-			modelName: 'migrations',
-			apiRoot: 'migrations',
-			modelText,
-			migrations: {
-				[defaultMigrationCategory]: {
-					'11.0.0-modified-at': `
-					ALTER TABLE "migration"
-					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-				`,
-					'11.0.1-modified-at': `
-					ALTER TABLE "migration lock"
-					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-				`,
-				},
-			},
-		},
-	],
-};

package/src/migrator/utils.ts

@@ -1,293 +0,0 @@
-import type { Result, Tx } from '../database-layer/db';
-import type { Resolvable } from '../sbvr-api/common-types';
-
-import { Engines } from '@balena/abstract-sql-compiler';
-import * as _ from 'lodash';
-import { TypedError } from 'typed-error';
-import { migrator as migratorEnv } from '../config-loader/env';
-export { migrator as migratorEnv } from '../config-loader/env';
-import { delay } from '../sbvr-api/control-flow';
-
-// tslint:disable-next-line:no-var-requires
-export const modelText = require('./migrations.sbvr');
-
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-
-export const defaultMigrationCategory = 'sync';
-export enum migrationCategories {
-	'sync' = 'sync',
-	'async' = 'async',
-}
-
-type SbvrUtils = typeof sbvrUtils;
-
-export type MigrationTuple = [string, Migration];
-
-export type MigrationFn = (tx: Tx, sbvrUtils: SbvrUtils) => Resolvable<void>;
-export type AsyncMigrationFn = (
-	tx: Tx,
-	sbvrUtils: SbvrUtils,
-) => Resolvable<Result>;
-
-export type AsyncMigration = {
-	fn?: AsyncMigrationFn | undefined;
-	sql?: string | undefined;
-	delayMS?: number | undefined;
-	backoffDelayMS?: number | undefined;
-	errorThreshold?: number | undefined;
-};
-
-export type Migration = string | MigrationFn | AsyncMigration;
-
-export type Migrations = { [key: string]: Migration };
-
-export class MigrationError extends TypedError {}
-
-export type MigrationStatus = {
-	migration_key: string;
-	start_time: Date;
-	last_run_time: Date;
-	run_counter: number;
-	migrated_rows: number;
-	error_counter: number;
-	error_threshold: number;
-	delayMS: number;
-	backoffDelayMS: number;
-	converged_time: Date | undefined;
-	last_error_message: string | undefined;
-	is_backoff: boolean;
-	should_stop: boolean;
-};
-
-// Tagged template to convert binds from `?` format to the necessary output format,
-// eg `$1`/`$2`/etc for postgres
-export const binds = (strings: TemplateStringsArray, ...bindNums: number[]) =>
-	strings
-		.map((str, i) => {
-			if (i === bindNums.length) {
-				return str;
-			}
-			if (i + 1 !== bindNums[i]) {
-				throw new SyntaxError('Migration sql binds must be sequential');
-			}
-			if (sbvrUtils.db.engine === Engines.postgres) {
-				return str + `$${bindNums[i]}`;
-			}
-			return str + `?`;
-		})
-		.join('');
-
-export const lockMigrations = async <T>(
-	tx: Tx,
-	modelName: string,
-	fn: () => Promise<T>,
-): Promise<T> => {
-	try {
-		await tx.executeSql(
-			binds`
-		DELETE FROM "migration lock"
-		WHERE "model name" = ${1}
-		AND "created at" < ${2}`,
-			[modelName, new Date(Date.now() - migratorEnv.lockTimeout)],
-		);
-		await tx.executeSql(
-			binds`
-		INSERT INTO "migration lock" ("model name")
-		VALUES (${1})`,
-			[modelName],
-		);
-	} catch (err) {
-		await delay(migratorEnv.lockFailDelay);
-		throw err;
-	}
-	try {
-		return await fn();
-	} finally {
-		try {
-			await tx.executeSql(
-				binds`
-		DELETE FROM "migration lock"
-		WHERE "model name" = ${1}`,
-				[modelName],
-			);
-		} catch {
-			// We ignore errors here as it's mostly likely caused by the migration failing and
-			// rolling back the transaction, and if we rethrow here we'll overwrite the real error
-			// making it much harder for users to see what went wrong and fix it
-		}
-	}
-};
-
-export const checkModelAlreadyExists = async (
-	tx: Tx,
-	modelName: string,
-): Promise<boolean> => {
-	const result = await tx.tableList("name = 'migration'");
-	if (result.rows.length === 0) {
-		return false;
-	}
-	const { rows } = await tx.executeSql(
-		binds`
-        SELECT 1
-        FROM "model"
-        WHERE "model"."is of-vocabulary" = ${1}
-        LIMIT 1`,
-		[modelName],
-	);
-
-	return rows.length > 0;
-};
-
-export const setExecutedMigrations = async (
-	tx: Tx,
-	modelName: string,
-	executedMigrations: string[],
-): Promise<void> => {
-	const stringifiedMigrations = JSON.stringify(executedMigrations);
-
-	const result = await tx.tableList("name = 'migration'");
-	if (result.rows.length === 0) {
-		return;
-	}
-
-	const { rowsAffected } = await tx.executeSql(
-		binds`
-UPDATE "migration"
-SET "model name" = ${1},
-"executed migrations" = ${2}
-WHERE "migration"."model name" = ${3}`,
-		[modelName, stringifiedMigrations, modelName],
-	);
-
-	if (rowsAffected === 0) {
-		await tx.executeSql(
-			binds`
-INSERT INTO "migration" ("model name", "executed migrations")
-VALUES (${1}, ${2})`,
-			[modelName, stringifiedMigrations],
-		);
-	}
-};
-
-export const getExecutedMigrations = async (
-	tx: Tx,
-	modelName: string,
-): Promise<string[]> => {
-	const { rows } = await tx.executeSql(
-		binds`
-SELECT "migration"."executed migrations" AS "executed_migrations"
-FROM "migration"
-WHERE "migration"."model name" = ${1}`,
-		[modelName],
-	);
-
-	const data = rows[0];
-	if (data == null) {
-		return [];
-	}
-
-	return JSON.parse(data.executed_migrations) as string[];
-};
-
-// Just update the "modified at" field to show liveness
-export const initMigrationStatus = async (
-	tx: Tx,
-	migrationStatus: MigrationStatus,
-): Promise<Result | undefined> => {
-	try {
-		const result = await tx.executeSql(
-			binds`
-INSERT INTO "migration status" ("migration key", "start time", "delayMS", "backoffDelayMS", "is backoff", "error threshold", "should stop")
-VALUES (${1}, ${2}, ${3}, ${4}, ${5}, ${6}, ${7});
-`,
-			[
-				migrationStatus['migration_key'],
-				migrationStatus['start_time'],
-				migrationStatus['delayMS'],
-				migrationStatus['backoffDelayMS'],
-				migrationStatus['is_backoff'] === true ? 1 : 0,
-				migrationStatus['error_threshold'],
-				migrationStatus['should_stop'] === true ? 1 : 0,
-			],
-		);
-		return result;
-	} catch (err) {
-		// it already exists and we can ignore this error.
-	}
-};
-
-// Just update the "modified at" field to show liveness
-export const updateMigrationStatus = async (
-	tx: Tx,
-	migrationStatus: MigrationStatus,
-): Promise<Result | undefined> => {
-	try {
-		return await tx.executeSql(
-			binds`
-UPDATE "migration status"
-SET
-"run counter"	= ${1},
-"last run time" = ${2},
-"migrated rows" = ${3},
-"error counter" = ${4},
-"converged time" = ${5},
-"last error message" = ${6},
-"is backoff" = ${7},
-"should stop" = ${8}
-WHERE "migration status"."migration key" = ${9};`,
-			[
-				migrationStatus['run_counter'],
-				migrationStatus['last_run_time'],
-				migrationStatus['migrated_rows'],
-				migrationStatus['error_counter'],
-				migrationStatus['converged_time'],
-				migrationStatus['last_error_message'],
-				migrationStatus['is_backoff'] === true ? 1 : 0,
-				migrationStatus['should_stop'] === true ? 1 : 0,
-				migrationStatus['migration_key'],
-			],
-		);
-	} catch (err) {
-		// we ignore all errors to not spam the api logs
-	}
-};
-
-export const readMigrationStatus = async (
-	tx: Tx,
-	migrationKey: string,
-): Promise<MigrationStatus | undefined> => {
-	try {
-		const { rows } = await tx.executeSql(
-			binds`
-SELECT *
-FROM "migration status"
-WHERE "migration status"."migration key" = ${1}
-LIMIT 1;`,
-			[migrationKey],
-		);
-
-		const data = rows[0];
-		if (data == null) {
-			return undefined;
-		}
-
-		return {
-			migration_key: data['migration key'],
-			start_time: data['start time'],
-			last_run_time: data['last run time'],
-			run_counter: data['run counter'],
-			migrated_rows: data['migrated rows'],
-			error_counter: data['error counter'],
-			error_threshold: data['error threshold'],
-			delayMS: data['delayMS'],
-			backoffDelayMS: data['backoffDelayMS'],
-			converged_time: data['converged_time'],
-			last_error_message: data['last error message'],
-			is_backoff: data['is backoff'] === 1 ? true : false,
-			should_stop: data['should stop'] === 1 ? true : false,
-		};
-	} catch (err) {
-		// we ignore all errors to not spam the api logs
-		return undefined;
-	}
-};