@balena/pinejs 15.0.0-true-boolean-911aca4062d3132ad3c34712014739b6849fa13a → 15.0.0

package/src/migrator/async.ts

@@ -0,0 +1,323 @@
+import type { Tx } from '../database-layer/db';
+import type { Model } from '../config-loader/config-loader';
+
+import * as _ from 'lodash';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+
+type ApiRootModel = Model & { apiRoot: string };
+
+type InitialMigrationStatus = MigrationStatus &
+	Required<
+		Pick<BaseAsyncMigration, 'backoffDelayMS' | 'delayMS' | 'errorThreshold'>
+	>;
+
+import {
+	MigrationTuple,
+	getExecutedMigrations,
+	migratorEnv,
+	lockMigrations,
+	initMigrationStatus,
+	readMigrationStatus,
+	updateMigrationStatus,
+	RunnableAsyncMigrations,
+	getRunnableAsyncMigrations,
+	filterAndSortPendingMigrations,
+	MigrationStatus,
+	BaseAsyncMigration,
+} from './utils';
+import { booleanToEnabledString } from '../config-loader/env';
+
+// log the startup condition of the async migration
+(sbvrUtils.api?.migrations?.logger.info ?? console.info)(
+	`Async migration execution is ${booleanToEnabledString(
+		migratorEnv.asyncMigrationIsEnabled,
+	)}`,
+);
+
+export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { migrations } = model;
+	if (migrations == null || _.isEmpty(migrations)) {
+		return;
+	}
+	const asyncMigrations: RunnableAsyncMigrations | undefined =
+		getRunnableAsyncMigrations(migrations);
+	if (asyncMigrations == null) {
+		return;
+	}
+
+	await $run(tx, model, asyncMigrations);
+};
+
+const $run = async (
+	setupTx: Tx,
+	model: ApiRootModel,
+	migrations: RunnableAsyncMigrations,
+): Promise<void> => {
+	const modelName = model.apiRoot;
+
+	const asyncMigrationSetup: Array<{
+		key: string;
+		initMigrationState: InitialMigrationStatus;
+		asyncRunnerMigratorFn: (tx: Tx) => Promise<number>;
+	}> = [];
+
+	// get a transaction for setting up the async migrator
+
+	const executedMigrations = await getExecutedMigrations(setupTx, modelName);
+
+	// if the model is new, the sync migration parts (marked by finalize=true) are already marked in the sync migration runner.
+
+	/**
+	 * preflight check if there are already migrations executed before starting the async scheduler
+	 * this will implicitly skip async migrations that have been superseded by synchro migrations.
+	 * e.g.:
+	 *
+	 * sync migrations in repo: 	[001,002,004,005]
+	 * async migrations in repo: 	[003,006]
+	 *
+	 * executed migrations at this point should always contain all sync migrations:
+	 * executed migrations: 		[001,002,004,005]
+	 *
+	 * This will result in only async migration 006 being executed.
+	 *
+	 * The async migrations are meant to be used in separate deployments to make expensive data migrations
+	 * of multiple million row update queries cheaper and with no downtime / long lasting table lock.
+	 * In the end, after each async migration, the next deployment should follow up the data migration
+	 * with a final sync data migrations.
+	 * An async migration will be executed in iterations until no rows are updated anymore (keep row locks short)
+	 * then it switches into a backoff mode to check with longer delay if data needs to be migrated in the future.
+	 * Example query:
+	 * UPDATE tableA
+	 * SET columnB = columnA
+	 * WHERE id IN (SELECT id
+	 * 	           FROM tableA
+	 *             WHERE (columnA <> columnB) OR (columnA IS NOT NULL AND columnB IS NULL)
+	 *             LIMIT 1000);
+	 *
+	 * The final sync data migration would look like:
+	 * UPDATE tableA
+	 * SET columnB = columnA
+	 * WHERE (columnA <> columnB) OR (columnA IS NOT NULL AND columnB IS NULL);
+	 *
+	 * And will update remaining rows, which ideally are 0 and therefore no rows are locked for the update
+	 *
+	 * In the case of a column rename the columnA could be safely dropped:
+	 * ALTER TABLE tableA
+	 * DROP COLUMN IF EXISTS columnA;
+	 */
+
+	const pendingMigrations: MigrationTuple[] = filterAndSortPendingMigrations(
+		migrations,
+		executedMigrations,
+	);
+
+	// Just schedule the migration workers and don't wait for any return of them
+	// the migration workers run until the next deployment and may synchronise with other
+	// instances via database tables: migration lock and migration status
+
+	for (const [key, migration] of pendingMigrations) {
+		let asyncRunnerMigratorFn: (tx: Tx) => Promise<number>;
+		let initMigrationState: InitialMigrationStatus = {
+			migration_key: key,
+			start_time: new Date(),
+			last_run_time: new Date(),
+			run_count: 0,
+			migrated_row_count: 0,
+			error_count: 0,
+			errorThreshold: migratorEnv.asyncMigrationDefaultErrorThreshold,
+			delayMS: migratorEnv.asyncMigrationDefaultDelayMS,
+			backoffDelayMS: migratorEnv.asyncMigrationDefaultBackoffDelayMS,
+			converged_time: undefined,
+			is_backing_off: false,
+		};
+
+		if (typeof migration === 'object') {
+			const batchSize =
+				migration.asyncBatchSize || migratorEnv.asyncMigrationDefaultBatchSize;
+			if (migration.asyncFn && typeof migration.asyncFn === 'function') {
+				asyncRunnerMigratorFn = async (tx: Tx) => {
+					return await migration.asyncFn(
+						tx,
+						{
+							batchSize,
+						},
+						sbvrUtils,
+					);
+				};
+			} else if (migration.asyncSql && typeof migration.asyncSql === 'string') {
+				const asyncMigrationSqlStatement = migration.asyncSql?.replace(
+					'%%ASYNC_BATCH_SIZE%%',
+					`${batchSize}`,
+				);
+				asyncRunnerMigratorFn = async (tx: Tx) =>
+					(await tx.executeSql(asyncMigrationSqlStatement)).rowsAffected;
+			} else {
+				// don't break the async migration b/c of one migration fails
+				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+					`Invalid migration object: ${JSON.stringify(migration, null, 2)}`,
+				);
+				continue;
+			}
+
+			initMigrationState = {
+				...initMigrationState,
+				..._.pickBy(
+					_.pick(migration, ['backoffDelayMS', 'delayMS', 'errorThreshold']),
+					(value) => value != null,
+				),
+			};
+		} else if (typeof migration === 'string') {
+			asyncRunnerMigratorFn = async (tx: Tx) =>
+				(await tx.executeSql(migration)).rowsAffected;
+		} else {
+			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+				`Invalid async migration object: ${JSON.stringify(migration, null, 2)}`,
+			);
+			continue;
+		}
+		await initMigrationStatus(setupTx, initMigrationState);
+
+		asyncMigrationSetup.push({
+			key,
+			initMigrationState,
+			asyncRunnerMigratorFn,
+		});
+	}
+
+	// Only after the setupTransaction has successfully finalized the asyncMigration runners will be
+	// created. When the transaction fails, the setup async migration entries in the DB will be
+	// rolled back automatically.
+	setupTx.on('end', () => {
+		for (const {
+			key,
+			initMigrationState,
+			asyncRunnerMigratorFn,
+		} of asyncMigrationSetup) {
+			const asyncRunner = async () => {
+				// don't run the async migration but keep checking
+				if (!migratorEnv.asyncMigrationIsEnabled) {
+					setTimeout(asyncRunner, initMigrationState.backoffDelayMS);
+					return;
+				}
+				try {
+					const $migrationState = await sbvrUtils.db.transaction(
+						async (tx) =>
+							await lockMigrations(
+								{ tx, modelName, blocking: false },
+								async () => {
+									const migrationState = await readMigrationStatus(tx, key);
+
+									if (!migrationState) {
+										// migration status is unclear stop the migrator
+										// or migration should stop
+										(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+											`stopping async migration due to missing migration status: ${key}`,
+										);
+										return false;
+									}
+									// sync on the last execution time between instances
+									// precondition: All running instances are running on the same time/block
+									// skip execution
+									if (migrationState.last_run_time) {
+										const durationSinceLastRun =
+											Date.now() - migrationState.last_run_time.getTime();
+										const delayMs = migrationState.is_backing_off
+											? initMigrationState.backoffDelayMS
+											: initMigrationState.delayMS;
+										if (durationSinceLastRun < delayMs) {
+											// will still execute finally block where the migration lock is released.
+											return;
+										}
+									}
+									try {
+										// here a separate transaction is needed as this migration may fail
+										// when it fails it would break the transaction for managing the migration status
+										const migratedRows = await sbvrUtils.db.transaction(
+											async (migrationTx) => {
+												// disable automatic close on the management transaction as the migration transaction consumes up to max autoClose time
+												// disable first here, to let if fail when it takes to long before coming here to actually migrate.
+												tx.disableAutomaticClose();
+												const rollbackMigrationTx = async () => {
+													// if the parent transaction fails for any reason, the actual running migration transaction has to be rolled back to stop parallel unsafe async migrations.
+													try {
+														if (!migrationTx.isClosed()) {
+															await migrationTx.rollback();
+														}
+													} catch (err) {
+														(
+															sbvrUtils.api.migrations?.logger.error ??
+															console.error
+														)(
+															`error rolling back pending async migration tx on mgmt tx end/rollback: ${key}: ${err}`,
+														);
+													}
+												};
+												tx.on('rollback', rollbackMigrationTx);
+												tx.on('end', rollbackMigrationTx);
+												return (
+													(await asyncRunnerMigratorFn?.(migrationTx)) ?? 0
+												);
+											},
+										);
+										migrationState.migrated_row_count += migratedRows;
+										if (migratedRows === 0) {
+											// when all rows have been catched up once we only catch up less frequently
+											migrationState.is_backing_off = true;
+											// only store the first time when migrator converged to all data migrated
+											migrationState.converged_time ??= new Date();
+										} else {
+											// Only here for the case that after backoff more rows need to be caught up faster
+											// If rows have been updated recently we start the interval again with normal frequency
+											migrationState.is_backing_off = false;
+										}
+									} catch (err: unknown) {
+										migrationState.error_count++;
+										if (err instanceof Error) {
+											if (
+												migrationState.error_count %
+													initMigrationState.errorThreshold ===
+												0
+											) {
+												(
+													sbvrUtils.api.migrations?.logger.error ??
+													console.error
+												)(`${key}: ${err.name} ${err.message}`);
+												migrationState.is_backing_off = true;
+											}
+										} else {
+											(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+												`async migration error unknown: ${key}: ${err}`,
+											);
+										}
+									} finally {
+										// using finally as it will also run when return statement is called inside the try block
+										// either success or error release the lock
+										migrationState.last_run_time = new Date();
+										migrationState.run_count += 1;
+										await updateMigrationStatus(tx, migrationState);
+									}
+									return migrationState;
+								},
+							),
+					);
+					if ($migrationState === false) {
+						// We've stopped the migration intentionally
+						return;
+					}
+					if ($migrationState == null || $migrationState.is_backing_off) {
+						setTimeout(asyncRunner, initMigrationState.backoffDelayMS);
+					} else {
+						setTimeout(asyncRunner, initMigrationState.delayMS);
+					}
+				} catch (err) {
+					(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+						`error running async migration: ${key}: ${err}`,
+					);
+					setTimeout(asyncRunner, initMigrationState.backoffDelayMS);
+				}
+			};
+
+			setTimeout(asyncRunner, initMigrationState.delayMS);
+		}
+	});
+};

package/src/migrator/migrations.sbvr

@@ -21,3 +21,46 @@ Term: migration lock
 
 Fact Type: migration lock has model name
 	Necessity: each migration lock has exactly one model name
+
+Term: migration key
+	Concept Type: Short Text (Type)
+Term: start time
+	Concept Type: Date Time (Type)
+Term: last run time
+	Concept Type: Date Time (Type)
+Term: run count
+	Concept Type: Integer (Type)
+Term: migrated row count
+	Concept Type: Integer (Type)
+Term: error count
+	Concept Type: Integer (Type)
+Term: converged time 
+	Concept Type: Date Time (Type)
+
+
+Term: migration status
+	Reference Scheme: migration key
+	Database ID Field: migration key
+
+Fact Type: migration status has migration key
+	Necessity: each migration status has exactly one migration key
+
+Fact Type: migration status has start time
+	Necessity: each migration status has at most one start time
+
+Fact Type: migration status has last run time
+	Necessity: each migration status has at most one last run time
+
+Fact Type: migration status has run count
+	Necessity: each migration status has exactly one run count
+
+Fact Type: migration status has migrated row count
+	Necessity: each migration status has at most one migrated row count
+
+Fact Type: migration status has error count
+	Necessity: each migration status has at most one error count
+
+Fact Type: migration status is backing off
+
+Fact Type: migration status has converged time 
+	Necessity: each migration status has at most one converged time

package/src/migrator/sync.ts

@@ -0,0 +1,152 @@
+import {
+	modelText,
+	MigrationTuple,
+	MigrationError,
+	setExecutedMigrations,
+	getExecutedMigrations,
+	lockMigrations,
+	RunnableMigrations,
+	filterAndSortPendingMigrations,
+	getRunnableSyncMigrations,
+} from './utils';
+import type { Tx } from '../database-layer/db';
+import type { Config, Model } from '../config-loader/config-loader';
+
+import * as _ from 'lodash';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+
+type ApiRootModel = Model & { apiRoot: string };
+
+export const postRun = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { initSql } = model;
+	if (initSql == null) {
+		return;
+	}
+
+	const modelName = model.apiRoot;
+	const modelIsNew = await sbvrUtils.isModelNew(tx, modelName);
+	if (modelIsNew) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			`First time executing '${modelName}', running init script`,
+		);
+
+		await lockMigrations({ tx, modelName, blocking: true }, async () => {
+			try {
+				await tx.executeSql(initSql);
+			} catch (err: any) {
+				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+					`initSql execution error ${err} `,
+				);
+				throw new MigrationError(err);
+			}
+		});
+	}
+};
+
+export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { migrations } = model;
+	if (migrations == null || _.isEmpty(migrations)) {
+		return;
+	}
+	const runnableMigrations = getRunnableSyncMigrations(migrations);
+	return $run(tx, model, runnableMigrations);
+};
+
+const $run = async (
+	tx: Tx,
+	model: ApiRootModel,
+	migrations: RunnableMigrations,
+): Promise<void> => {
+	const modelName = model.apiRoot;
+
+	// migrations only run if the model has been executed before,
+	// to make changes that can't be automatically applied
+	const modelIsNew = await sbvrUtils.isModelNew(tx, modelName);
+	if (modelIsNew) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			`First time model '${modelName}' has executed, skipping migrations`,
+		);
+
+		return await setExecutedMigrations(tx, modelName, Object.keys(migrations));
+	}
+	await lockMigrations({ tx, modelName, blocking: true }, async () => {
+		try {
+			const executedMigrations = await getExecutedMigrations(tx, modelName);
+			const pendingMigrations = filterAndSortPendingMigrations(
+				migrations,
+				executedMigrations,
+			);
+			if (pendingMigrations.length === 0) {
+				return;
+			}
+
+			const newlyExecutedMigrations = await executeMigrations(
+				tx,
+				pendingMigrations,
+			);
+			await setExecutedMigrations(tx, modelName, [
+				...executedMigrations,
+				...newlyExecutedMigrations,
+			]);
+		} catch (err: any) {
+			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+				`Failed to executed synchronous migrations from api root model ${err}`,
+			);
+			throw new MigrationError(err);
+		}
+	});
+};
+
+const executeMigrations = async (
+	tx: Tx,
+	migrations: MigrationTuple[] = [],
+): Promise<string[]> => {
+	try {
+		for (const migration of migrations) {
+			await executeMigration(tx, migration);
+		}
+	} catch (err: any) {
+		(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+			'Error while executing migrations, rolled back',
+		);
+		throw new MigrationError(err);
+	}
+	return migrations.map(([migrationKey]) => migrationKey); // return migration keys
+};
+
+const executeMigration = async (
+	tx: Tx,
+	[key, migration]: MigrationTuple,
+): Promise<void> => {
+	(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+		`Running migration ${JSON.stringify(key)}`,
+	);
+
+	if (typeof migration === 'function') {
+		await migration(tx, sbvrUtils);
+	} else if (typeof migration === 'string') {
+		await tx.executeSql(migration);
+	} else {
+		throw new MigrationError(`Invalid migration type: ${typeof migration}`);
+	}
+};
+
+export const config: Config = {
+	models: [
+		{
+			modelName: 'migrations',
+			apiRoot: 'migrations',
+			modelText,
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "migration"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'11.0.1-modified-at': `
+					ALTER TABLE "migration lock"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+			},
+		},
+	],
+};