@balena/pinejs 14.44.0-linear-runtime-migrator-e15bea04e85fb013aeed7d4770052b51747a619c → 15.0.0-delete-state-default-user-permissions-ba0732a0c5d0da9d1d5be818cb08cc898e86ebe3

package/src/migrator/async.ts

@@ -1,279 +0,0 @@
-import type { Tx } from '../database-layer/db';
-import type { Model } from '../config-loader/config-loader';
-
-import * as _ from 'lodash';
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-
-type ApiRootModel = Model & { apiRoot: string };
-
-import {
-	MigrationTuple,
-	AsyncMigrationFn,
-	MigrationCategories,
-	checkModelAlreadyExists,
-	setExecutedMigrations,
-	getExecutedMigrations,
-	migratorEnv,
-	lockMigrations,
-	initMigrationStatus,
-	readMigrationStatus,
-	updateMigrationStatus,
-	RunnableMigrations,
-} from './utils';
-
-export const run = async (model: ApiRootModel): Promise<void> => {
-	const { migrations } = model;
-	if (
-		migrations == null ||
-		_.isEmpty(migrations) ||
-		migrations[MigrationCategories.async] === undefined
-	) {
-		return;
-	}
-	let runMigrations: RunnableMigrations = {};
-	for (const [key, value] of Object.entries(migrations)) {
-		if (key === MigrationCategories.async && value instanceof Object) {
-			runMigrations = { ...runMigrations, ...value };
-		}
-	}
-	await $run(model, runMigrations);
-};
-
-const $run = async (
-	model: ApiRootModel,
-	migrations: RunnableMigrations,
-): Promise<void> => {
-	const modelName = model.apiRoot;
-
-	// init migrations
-	let exists;
-	await sbvrUtils.db.transaction(async (tx) => {
-		exists = await checkModelAlreadyExists(tx, modelName);
-		if (!exists) {
-			(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-				'First time model has executed, skipping migrations',
-			);
-
-			return await setExecutedMigrations(
-				tx,
-				modelName,
-				Object.keys(migrations),
-			);
-		}
-	});
-
-	if (!exists) {
-		return;
-	}
-
-	/**
-	 * preflight check if there are already migrations executed before starting the async scheduler
-	 * this will impplicitly skip async migrations that have been superceeded by synchron migrations.
-	 * e.g.:
-	 *
-	 * sync migrations in repo: 	[001,002,004,005]
-	 * async migrations in repo: 	[003,006]
-	 *
-	 * executed migrations at this point should always contain all sync migrations:
-	 * executed migrations: 		[001,002,004,005]
-	 *
-	 * This will result in only async migration 006 being executed.
-	 *
-	 * The async migrations are ment to be used in seperate deployments to make expensive data migrations
-	 * of multiple million row update queries cheaper and with no downtime / long lasting table lock.
-	 * In the end, after each async migration, the next deploymemnt should follow up the data migration
-	 * with a final sync data migrations.
-	 * A async migration will be executed in iterations until no rows are updated anymore (keep row locks short)
-	 * then it switches into a backoff mode to check with longer delay if data needs to be migrated in the future.
-	 * Example query:
-	 * UPDATE tableA
-	 * SET columnB = columnA
-	 * WHERE id IN (SELECT id
-	 * 	           FROM tableA
-	 *             WHERE (columnA <> columnB) OR (columnA IS NOT NULL AND columnB IS NULL)
-	 *             LIMIT 1000);
-	 *
-	 * The final sync data migration would look like:
-	 * UPDATE tableA
-	 * SET columnB = columnA
-	 * WHERE (columnA <> columnB) OR (columnA IS NOT NULL AND columnB IS NULL);
-	 *
-	 * And will update remaining rows, which ideally are 0 and therefore now rows are locked for the update
-	 *
-	 * In the case of a column rename the columnA could be savely dropped:
-	 * ALTER TABLE tableA
-	 * DROP COLUMN IF EXISTS columnA;
-	 */
-
-	let pendingMigrations: MigrationTuple[] = [];
-	await sbvrUtils.db.transaction(async (tx) => {
-		const executedMigrations = await getExecutedMigrations(tx, modelName);
-		pendingMigrations = filterAndSortPendingAsyncMigrations(
-			migrations,
-			executedMigrations,
-		);
-	});
-
-	// Just schedule the migration workers and don't wait for any return of them
-	// the migration workers run until the next deployment and may synchronise with other
-	// instances via database tables: migration lock and migration status
-	for (const [key, migration] of pendingMigrations) {
-		const initMigrationState = {
-			migration_key: key,
-			start_time: new Date(Date.now()),
-			last_run_time: new Date(Date.now()),
-			run_counter: 0,
-			migrated_rows: 0,
-			error_counter: 0,
-			error_threshold: migratorEnv.asyncMigrationDefaultErrorThreshold,
-			delayMS: migratorEnv.asyncMigrationDefaultDelayMS,
-			backoffDelayMS: migratorEnv.asyncMigrationDefaultBackoffDelayMS,
-			converged_time: undefined,
-			last_error_message: undefined,
-			is_backoff: false,
-			should_stop: false,
-		};
-
-		let asyncRunnerMigratorFn: ((tx: Tx) => Promise<number>) | undefined;
-
-		if (typeof migration === 'object') {
-			if (migration.fn && typeof migration.fn === 'function') {
-				asyncRunnerMigratorFn = async (tx: Tx) =>
-					(await (migration.fn as AsyncMigrationFn)(tx, sbvrUtils))
-						.rowsAffected;
-			} else if (migration.sql && typeof migration.sql === 'string') {
-				asyncRunnerMigratorFn = async (tx: Tx) =>
-					(await tx.executeSql(migration.sql as string)).rowsAffected;
-			} else {
-				// don't break the async migration b/c of one migration fails
-				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-					`Invalid migration object: ${JSON.stringify(migration, null, 2)}`,
-				);
-				continue;
-			}
-
-			initMigrationState.backoffDelayMS =
-				migration.backoffDelayMS ||
-				migratorEnv.asyncMigrationDefaultBackoffDelayMS;
-			initMigrationState.delayMS =
-				migration.delayMS || migratorEnv.asyncMigrationDefaultDelayMS;
-			initMigrationState.error_threshold =
-				migration.errorThreshold ||
-				migratorEnv.asyncMigrationDefaultErrorThreshold;
-		} else if (typeof migration === 'string') {
-			asyncRunnerMigratorFn = async (tx: Tx) =>
-				(await tx.executeSql(migration)).rowsAffected;
-		} else {
-			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-				`Invalid async migration object: ${JSON.stringify(migration, null, 2)}`,
-			);
-			continue;
-		}
-
-		await sbvrUtils.db.transaction(async (tx) =>
-			initMigrationStatus(tx, initMigrationState),
-		);
-
-		const asyncRunner = async () => {
-			await sbvrUtils.db.transaction(async (tx) => {
-				await lockMigrations(tx, modelName, async () => {
-					const migrationState = await readMigrationStatus(tx, key);
-
-					if (!migrationState || migrationState.should_stop === true) {
-						// migration status is unclear stop the migrator
-						// or migration should stop
-						(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-							`stopping async migration: ${key}`,
-						);
-						clearInterval(asyncScheduler);
-						return;
-					}
-					try {
-						// clear the interval to avoid retriggering before finisheing this migraiton query
-						clearInterval(asyncScheduler);
-
-						// sync on the last execution time between instances
-						// precondition: All running instances are running on the same time/block
-						// skip execution
-						if (migrationState.last_run_time) {
-							const durationSinceLastRun =
-								Date.now().valueOf() - migrationState.last_run_time.valueOf();
-							if (
-								(migrationState.is_backoff &&
-									durationSinceLastRun < migrationState.backoffDelayMS) ||
-								(!migrationState.is_backoff &&
-									durationSinceLastRun < migrationState.delayMS)
-							) {
-								// will still execute finally block where the migration lock is released.
-								return;
-							}
-						}
-						// set last run time and run counter only when backoff time sync between
-						// competing instances is in sync
-						migrationState.last_run_time = new Date(Date.now());
-						migrationState.run_counter += 1;
-
-						let migratedRows = 0;
-						await sbvrUtils.db.transaction(async (migrationTx) => {
-							migratedRows = (await asyncRunnerMigratorFn?.(migrationTx)) || 0;
-						});
-
-						migrationState.migrated_rows += migratedRows;
-						if (migratedRows === 0) {
-							// when all rows have been catched up once we only catch up less frequently
-							migrationState.is_backoff = true;
-							if (!migrationState.converged_time) {
-								// only store the first time when migrator converged to all data migrated
-								migrationState.converged_time = new Date(Date.now());
-							}
-						} else {
-							// Only here for the case that after backoff more rows need to be catched up faster
-							// If rows have been updated recently we start the interval again with normal frequency
-							migrationState.is_backoff = false;
-						}
-					} catch (err) {
-						migrationState.error_counter++;
-
-						if (
-							migrationState.error_counter % migrationState.error_threshold ===
-							0
-						) {
-							migrationState.last_error_message = `${err.name} ${err.message}`;
-							(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-								`${key}: ${err.name} ${err.message}`,
-							);
-							migrationState.is_backoff = true;
-						}
-					} finally {
-						if (migrationState.is_backoff) {
-							asyncScheduler = setInterval(
-								asyncRunner,
-								migrationState.backoffDelayMS,
-							);
-						} else {
-							asyncScheduler = setInterval(asyncRunner, migrationState.delayMS);
-						}
-						// using finally as it will also run when return statement is called inside the try block
-						// either success or error release the lock
-						await updateMigrationStatus(tx, migrationState);
-					}
-				});
-			});
-		};
-		let asyncScheduler = setInterval(asyncRunner, initMigrationState.delayMS);
-	}
-};
-
-const filterAndSortPendingAsyncMigrations = (
-	migrations: NonNullable<RunnableMigrations>,
-	executedMigrations: string[],
-): MigrationTuple[] => {
-	// using it for string comparison, any string is greater than ''
-	const latestExecutedMigration = executedMigrations.sort().pop() ?? '';
-
-	return (_(migrations).omit(executedMigrations) as _.Object<typeof migrations>)
-		.toPairs()
-		.filter(([migrationKey]) => migrationKey > latestExecutedMigration)
-		.sortBy(([migrationKey]) => migrationKey)
-		.value();
-};

package/src/migrator/sync.ts

@@ -1,177 +0,0 @@
-import {
-	modelText,
-	MigrationTuple,
-	MigrationError,
-	defaultMigrationCategory,
-	checkModelAlreadyExists,
-	setExecutedMigrations,
-	getExecutedMigrations,
-	lockMigrations,
-	MigrationCategories,
-	RunnableMigrations,
-} from './utils';
-import type { Tx } from '../database-layer/db';
-import type { Config, Model } from '../config-loader/config-loader';
-
-import * as _ from 'lodash';
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-
-type ApiRootModel = Model & { apiRoot: string };
-
-export const postRun = async (tx: Tx, model: ApiRootModel): Promise<void> => {
-	const { initSql } = model;
-	if (initSql == null) {
-		return;
-	}
-
-	const modelName = model.apiRoot;
-
-	const exists = await checkModelAlreadyExists(tx, modelName);
-	if (!exists) {
-		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-			'First time executing, running init script',
-		);
-
-		await lockMigrations(tx, modelName, async () => {
-			try {
-				await tx.executeSql(initSql);
-			} catch (err) {
-				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-					`postRun locked sql execution error ${err} `,
-				);
-				throw new MigrationError(err);
-			}
-		});
-	}
-};
-
-export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
-	const { migrations } = model;
-	if (
-		migrations == null ||
-		_.isEmpty(migrations) ||
-		migrations[defaultMigrationCategory] === undefined
-	) {
-		return;
-	}
-
-	let runMigrations: RunnableMigrations = {};
-	for (const [key, value] of Object.entries(migrations)) {
-		if (key === defaultMigrationCategory && value instanceof Object) {
-			runMigrations = { ...runMigrations, ...value };
-		} else if (!(key in MigrationCategories)) {
-			runMigrations = { ...runMigrations, ...{ [key]: value } };
-		}
-	}
-	return $run(tx, model, runMigrations);
-};
-
-const $run = async (
-	tx: Tx,
-	model: ApiRootModel,
-	migrations: RunnableMigrations,
-): Promise<void> => {
-	const modelName = model.apiRoot;
-
-	// migrations only run if the model has been executed before,
-	// to make changes that can't be automatically applied
-	const exists = await checkModelAlreadyExists(tx, modelName);
-	if (!exists) {
-		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-			'First time model has executed, skipping migrations',
-		);
-
-		return await setExecutedMigrations(tx, modelName, Object.keys(migrations));
-	}
-	await lockMigrations(tx, modelName, async () => {
-		try {
-			const executedMigrations = await getExecutedMigrations(tx, modelName);
-			const pendingMigrations = filterAndSortPendingMigrations(
-				migrations,
-				executedMigrations,
-			);
-			if (pendingMigrations.length === 0) {
-				return;
-			}
-
-			const newlyExecutedMigrations = await executeMigrations(
-				tx,
-				pendingMigrations,
-			);
-			await setExecutedMigrations(tx, modelName, [
-				...executedMigrations,
-				...newlyExecutedMigrations,
-			]);
-		} catch (err) {
-			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-				`$run executedMigrations error ${err}`,
-			);
-			throw new MigrationError(err);
-		}
-	});
-};
-
-// turns {"key1": migration, "key3": migration, "key2": migration}
-// into  [["key1", migration], ["key2", migration], ["key3", migration]]
-const filterAndSortPendingMigrations = (
-	migrations: NonNullable<RunnableMigrations>,
-	executedMigrations: string[],
-): MigrationTuple[] =>
-	(_(migrations).omit(executedMigrations) as _.Object<typeof migrations>)
-		.toPairs()
-		.sortBy(([migrationKey]) => migrationKey)
-		.value();
-
-const executeMigrations = async (
-	tx: Tx,
-	migrations: MigrationTuple[] = [],
-): Promise<string[]> => {
-	try {
-		for (const migration of migrations) {
-			await executeMigration(tx, migration);
-		}
-	} catch (err) {
-		(sbvrUtils.api.migrations?.logger.error ?? console.error)(
-			'Error while executing migrations, rolled back',
-		);
-		throw new MigrationError(err);
-	}
-	return migrations.map(([migrationKey]) => migrationKey); // return migration keys
-};
-
-const executeMigration = async (
-	tx: Tx,
-	[key, migration]: MigrationTuple,
-): Promise<void> => {
-	(sbvrUtils.api.migrations?.logger.info ?? console.info)(
-		`Running migration ${JSON.stringify(key)}`,
-	);
-
-	if (typeof migration === 'function') {
-		await migration(tx, sbvrUtils);
-	} else if (typeof migration === 'string') {
-		await tx.executeSql(migration);
-	} else {
-		throw new MigrationError(`Invalid migration type: ${typeof migration}`);
-	}
-};
-
-export const config: Config = {
-	models: [
-		{
-			modelName: 'migrations',
-			apiRoot: 'migrations',
-			modelText,
-			migrations: {
-				'11.0.0-modified-at': `
-					ALTER TABLE "migration"
-					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-				`,
-				'11.0.1-modified-at': `
-					ALTER TABLE "migration lock"
-					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-				`,
-			},
-		},
-	],
-};