@balena/pinejs 15.0.0-true-boolean-7896b116c446d891d7a0d5e4085c02a13bc9c725 → 15.0.1-build-migrations-clarify-marking-sbvr-optional-d6d0ded8eccc6eadb2492f4697918cf0afd00215-1

package/src/config-loader/config-loader.ts

@@ -1,8 +1,24 @@
 import type * as Express from 'express';
-import type { AbstractSqlModel } from '@balena/abstract-sql-compiler';
+import type {
+	AbstractSqlModel,
+	Definition,
+} from '@balena/abstract-sql-compiler';
 import type { Database } from '../database-layer/db';
-import type { Migration } from '../migrator/migrator';
-import type { AnyObject, Resolvable } from '../sbvr-api/common-types';
+import type {
+	AnyObject,
+	Dictionary,
+	RequiredField,
+	Resolvable,
+} from '../sbvr-api/common-types';
+
+import {
+	Migration,
+	Migrations,
+	defaultMigrationCategory,
+	MigrationCategories,
+	isSyncMigration,
+	isAsyncMigration,
+} from '../migrator/utils';
 
 import * as fs from 'fs';
 import * as _ from 'lodash';
@@ -11,6 +27,7 @@ import * as path from 'path';
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 
 import * as permissions from '../sbvr-api/permissions';
+import { AliasValidNodeType } from '../sbvr-api/translations';
 
 export type SetupFunction = (
 	app: Express.Application,
@@ -25,9 +42,7 @@ export interface Model {
 	modelText?: string;
 	abstractSql?: AbstractSqlModel;
 	migrationsPath?: string;
-	migrations?: {
-		[index: string]: Migration;
-	};
+	migrations?: Migrations;
 	initSqlPath?: string;
 	initSql?: string;
 	customServerCode?:
@@ -36,11 +51,15 @@ export interface Model {
 				setup: SetupFunction;
 		  };
 	logging?: { [key in keyof Console | 'default']?: boolean };
+	translateTo?: Model['apiRoot'];
+	translations?: Dictionary<
+		Definition | Dictionary<string | AliasValidNodeType>
+	>;
 }
 export interface User {
 	username: string;
 	password: string;
-	permissions: string[];
+	permissions?: string[];
 }
 export interface Config {
 	models: Model[];
@@ -77,7 +96,7 @@ const getOrCreatePermission = async (
 ) => {
 	try {
 		return await getOrCreate(authApiTx, 'permission', { name: permissionName });
-	} catch (e) {
+	} catch (e: any) {
 		e.message = `Could not create or find permission "${permissionName}": ${e.message}`;
 		throw e;
 	}
@@ -85,8 +104,8 @@ const getOrCreatePermission = async (
 
 // Setup function
 export const setup = (app: Express.Application) => {
-	const loadConfig = (data: Config): Promise<void> =>
-		sbvrUtils.db.transaction(async (tx) => {
+	const loadConfig = async (data: Config): Promise<void> => {
+		await sbvrUtils.db.transaction(async (tx) => {
 			const authApiTx = sbvrUtils.api.Auth.clone({
 				passthrough: {
 					tx,
@@ -99,20 +118,17 @@ export const setup = (app: Express.Application) => {
 				const permissionsCache: {
 					[index: string]: Promise<number>;
 				} = {};
-				users.forEach((user) => {
+				for (const user of users) {
 					if (user.permissions == null) {
-						return;
+						continue;
 					}
-					user.permissions.forEach((permissionName) => {
-						if (permissionsCache[permissionName] != null) {
-							return;
-						}
-						permissionsCache[permissionName] = getOrCreatePermission(
+					for (const permissionName of user.permissions) {
+						permissionsCache[permissionName] ??= getOrCreatePermission(
 							authApiTx,
 							permissionName,
 						);
-					});
-				});
+					}
+				}
 
 				await Promise.all(
 					users.map(async (user) => {
@@ -138,7 +154,7 @@ export const setup = (app: Express.Application) => {
 									}),
 								);
 							}
-						} catch (e) {
+						} catch (e: any) {
 							e.message = `Could not create or find user "${user.username}": ${e.message}`;
 							throw e;
 						}
@@ -146,44 +162,76 @@ export const setup = (app: Express.Application) => {
 				);
 			}
 
-			await Promise.all(
-				data.models.map(async (model) => {
-					if (
+			const modelPromises: Dictionary<Promise<void>> = _(data.models)
+				.filter(
+					(model): model is RequiredField<typeof model, 'apiRoot'> =>
 						(model.abstractSql != null || model.modelText != null) &&
-						model.apiRoot != null
-					) {
-						try {
-							await sbvrUtils.executeModel(
-								tx,
-								model as sbvrUtils.ExecutableModel,
+						model.apiRoot != null,
+				)
+				.keyBy((model) => model.apiRoot)
+				.mapValues(async (model: Model) => {
+					try {
+						const { translateTo, translations } = model;
+						if (translateTo != null) {
+							// We add an async boundary here so that `mapValues` can complete and assign the
+							// waiting promises to `modelPromises` before we try to access it
+							await null;
+							if (modelPromises[translateTo] == null) {
+								throw new Error(
+									`Cannot translate to non-existent version '${translateTo}'`,
+								);
+							}
+							// Wait on the `translateTo` version since it needs to already be available
+							await modelPromises[translateTo];
+						} else if (translations != null) {
+							throw new Error(
+								'Cannot have translations without a translateTo target',
 							);
+						}
 
-							const apiRoute = `/${model.apiRoot}/*`;
-							app.options(apiRoute, (_req, res) => res.sendStatus(200));
-							app.all(apiRoute, sbvrUtils.handleODataRequest);
+						await sbvrUtils.executeModel(
+							tx,
+							model as sbvrUtils.ExecutableModel,
+						);
 
-							console.info(
-								'Successfully executed ' + model.modelName + ' model.',
-							);
-						} catch (err) {
-							const message = `Failed to execute ${model.modelName} model from ${model.modelFile}`;
-							if (_.isError(err)) {
-								err.message = message;
-								throw err;
-							}
-							throw new Error(message);
+						const apiRoute = `/${model.apiRoot}/*`;
+						app
+							.route(apiRoute)
+							.options((_req, res) => res.status(200).end())
+							.get(sbvrUtils.handleODataRequest)
+							.put(sbvrUtils.handleODataRequest)
+							.post(sbvrUtils.handleODataRequest)
+							.patch(sbvrUtils.handleODataRequest)
+							.merge(sbvrUtils.handleODataRequest)
+							.delete(sbvrUtils.handleODataRequest);
+
+						console.info(
+							'Successfully executed ' + model.modelName + ' model.',
+						);
+					} catch (err: any) {
+						const message = `Failed to execute '${model.modelName}' model from '${model.modelFile}'`;
+						if (_.isError(err)) {
+							err.message = `${message} due to: ${err.message}`;
+							throw err;
 						}
+						throw new Error(message);
 					}
+				})
+				.value();
+			await Promise.all(Object.values(modelPromises));
+
+			await Promise.all(
+				data.models.map(async (model) => {
 					if (model.customServerCode != null) {
 						let customCode: SetupFunction;
 						if (typeof model.customServerCode === 'string') {
 							try {
 								customCode = nodeRequire(model.customServerCode).setup;
-							} catch (e) {
+							} catch (e: any) {
 								e.message = `Error loading custom server code: '${e.message}'`;
 								throw e;
 							}
-						} else if (_.isObject(model.customServerCode)) {
+						} else if (typeof model.customServerCode === 'object') {
 							customCode = model.customServerCode.setup;
 						} else {
 							throw new Error(
@@ -200,7 +248,7 @@ export const setup = (app: Express.Application) => {
 				}),
 			);
 		});
-
+	};
 	const loadConfigFile = async (configPath: string): Promise<Config> => {
 		console.info('Loading config:', configPath);
 		return await import(configPath);
@@ -217,7 +265,7 @@ export const setup = (app: Express.Application) => {
 			} else if (typeof config === 'string') {
 				root = path.dirname(config);
 				configObj = await loadConfigFile(config);
-			} else if (_.isObject(config)) {
+			} else if (typeof config === 'object') {
 				root = process.cwd();
 				configObj = config;
 			} else {
@@ -251,18 +299,84 @@ export const setup = (app: Express.Application) => {
 						await Promise.all(
 							fileNames.map(async (filename) => {
 								const filePath = path.join(migrationsPath, filename);
+								const fileNameParts = filename.split('.', 3);
+								const fileExtension = path.extname(filename);
 								const [migrationKey] = filename.split('-', 1);
+								let migrationCategory = defaultMigrationCategory;
 
-								switch (path.extname(filename)) {
+								if (fileNameParts.length === 3) {
+									if (fileNameParts[1] in MigrationCategories) {
+										migrationCategory = fileNameParts[1] as MigrationCategories;
+									} else {
+										console.error(
+											`Unrecognized migration file category ${
+												fileNameParts[1]
+											}, skipping: ${path.extname(filename)}`,
+										);
+										return;
+									}
+								}
+
+								/**
+								 *  helper to assign migrations with category level to model
+								 *  example migration file names:
+								 *
+								 *  key0-name.ts  				==> defaults startup migration
+								 *  key1-name1.sql 				==> defaults startup migration
+								 *  key2-name2.sync.sql 		==> explicit synchrony migration
+								 *  key3-name3.async.ts			==> async migration (async datafiller)
+								 *  key4-name4.async.sql		==> async migration (async datafiller)
+								 *
+								 */
+								const assignMigrationWithCategory = (
+									newMigrationKey: string,
+									newMigration: Migration,
+								) => {
+									const catMigrations = migrations[migrationCategory] || {};
+									if (
+										typeof newMigration === 'object' &&
+										migrationCategory === MigrationCategories.async
+									) {
+										newMigration.type = MigrationCategories.async;
+									}
+									if (typeof catMigrations === 'object') {
+										migrations[migrationCategory] = {
+											[newMigrationKey]: newMigration,
+											...catMigrations,
+										};
+									}
+								};
+
+								switch (fileExtension) {
 									case '.coffee':
 									case '.ts':
 									case '.js':
-										migrations[migrationKey] = nodeRequire(filePath);
+										const loadeMigration = nodeRequire(filePath);
+										const migration = loadeMigration.default ?? loadeMigration;
+
+										if (
+											!isAsyncMigration(migration) &&
+											!isSyncMigration(migration)
+										) {
+											throw new Error(
+												`loaded migraton file at ${filePath} is neither a synchron nor a asyncron migration definition`,
+											);
+										}
+
+										assignMigrationWithCategory(migrationKey, migration);
 										break;
 									case '.sql':
-										migrations[migrationKey] = await fs.promises.readFile(
-											filePath,
-											'utf8',
+										if (migrationCategory === MigrationCategories.async) {
+											console.error(
+												`Plain async migration sql statement not supported, needs to be an object, skipping: ${path.extname(
+													filename,
+												)}`,
+											);
+											break;
+										}
+										assignMigrationWithCategory(
+											migrationKey,
+											await fs.promises.readFile(filePath, 'utf8'),
 										);
 										break;
 									default:
@@ -282,8 +396,8 @@ export const setup = (app: Express.Application) => {
 				}),
 			);
 			await loadConfig(configObj);
-		} catch (err) {
-			console.error('Error loading application config', err, err.stack);
+		} catch (err: any) {
+			console.error('Error loading application config', err);
 			process.exit(1);
 		}
 	};

package/src/config-loader/env.ts

@@ -1,21 +1,81 @@
-export const { DEBUG } = process.env;
+const { PINEJS_DEBUG } = process.env;
+if (![undefined, '', '0', '1'].includes(PINEJS_DEBUG)) {
+	throw new Error(`Invalid value for PINEJS_DEBUG '${PINEJS_DEBUG}'`);
+}
+export const DEBUG = PINEJS_DEBUG === '1';
+
+type CacheFnOpts<T extends (...args: any[]) => any> =
+	| {
+			primitive?: true;
+			promise?: true;
+			normalizer?: memoizeWeak.MemoizeWeakOptions<T>['normalizer'];
+			weak: true;
+	  }
+	| {
+			primitive?: true;
+			promise?: true;
+			normalizer?: memoize.Options<T>['normalizer'];
+			weak?: undefined;
+	  };
+export type CacheFn = <T extends (...args: any[]) => any>(
+	fn: T,
+	opts?: CacheFnOpts<T>,
+) => T;
+export type CacheOpts =
+	| {
+			max?: number;
+	  }
+	| CacheFn
+	| false;
 
 export const cache = {
 	permissionsLookup: {
 		max: 5000,
-	},
+	} as CacheOpts,
 	parsePermissions: {
 		max: 100000,
-	},
+	} as CacheOpts,
 	parseOData: {
 		max: 100000,
-	},
+	} as CacheOpts,
 	odataToAbstractSql: {
 		max: 10000,
-	},
+	} as CacheOpts,
 	abstractSqlCompiler: {
 		max: 10000,
-	},
+	} as CacheOpts,
+	userPermissions: false as CacheOpts,
+	apiKeyPermissions: false as CacheOpts,
+	apiKeyActorId: false as CacheOpts,
+};
+
+import { boolVar } from '@balena/env-parsing';
+import * as memoize from 'memoizee';
+import memoizeWeak = require('memoizee/weak');
+export const createCache = <T extends (...args: any[]) => any>(
+	cacheName: keyof typeof cache,
+	fn: T,
+	// TODO: Mark this as optional once TS is able to infer the `normalizer` types
+	// when the `weak` differentiating property is not provided.
+	opts: CacheFnOpts<T>,
+) => {
+	const cacheOpts = cache[cacheName];
+	if (cacheOpts === false) {
+		return fn;
+	}
+	if (typeof cacheOpts === 'function') {
+		return cacheOpts(fn, opts);
+	}
+	if (opts?.weak === true) {
+		return memoizeWeak(fn, {
+			...cacheOpts,
+			...opts,
+		});
+	}
+	return memoize(fn, {
+		...cacheOpts,
+		...opts,
+	});
 };
 
 let timeoutMS: number;
@@ -39,10 +99,50 @@ export const db = {
 	keepAlive: true as boolean | undefined,
 	rollbackTimeout: 30000,
 	timeoutMS,
+	maxUses: Infinity,
+	maxLifetimeSeconds: 0,
+	/**
+	 * Check that queries in read-only TXs only contain `SELECT` statements, doing so adds a cost to each query
+	 * in a read-only TX and is unnecessary if it is part of a read-only database transaction. The only time a
+	 * writable transaction should be used with a read-only TX is during a read-only hook within a writable request
+	 * and so should only be able to catch cases of hooks that are incorrectly marked as read-only
+	 *
+	 * Defaults to true when in DEBUG mode, false otherwise
+	 */
+	checkReadOnlyQueries: DEBUG,
 };
 
+export const PINEJS_ADVISORY_LOCK = {
+	namespaceKey: 'pinejs_advisory_lock_namespace',
+	namespaceId: -1,
+};
+// for better readability of logging
+export const booleanToEnabledString = (input: boolean) =>
+	input ? 'enabled' : 'disabled';
+
 export const migrator = {
 	lockTimeout: 5 * 60 * 1000,
 	// Used to delay the failure on lock taking, to avoid spam taking
 	lockFailDelay: 20 * 1000,
+	asyncMigrationDefaultDelayMS: 1000,
+	asyncMigrationDefaultBackoffDelayMS: 60000,
+	asyncMigrationDefaultErrorThreshold: 10,
+	asyncMigrationDefaultBatchSize: 1000,
+
+	/**
+	 * @param asyncMigrationIsEnabled Switch on/off the execution of async migrations
+	 * Example implementation with listening on SIGUSR2 to toggle the AsyncMigrationExecution enable switch
+	 * For runtime switching the async migration execution the SIGUSR2 signal is interpreted as toggle.
+	 * When the process receives a SIGUSR2 signal the async migrations will toggle to be enabled or disabled.
+	 * @example
+	 * process.on('SIGUSR2', () => {
+	 * 	console.info(
+	 * 		`Received SIGUSR2 to toggle async migration execution enabled
+	 * 					from ${migrator.asyncMigrationIsEnabled}
+	 * 					to ${!migrator.asyncMigrationIsEnabled} `,
+	 * 	);
+	 * 	migrator.asyncMigrationIsEnabled = !migrator.asyncMigrationIsEnabled;
+	 * });
+	 */
+	asyncMigrationIsEnabled: boolVar('PINEJS_ASYNC_MIGRATION_ENABLED', true),
 };

package/src/data-server/sbvr-server.js

@@ -47,6 +47,7 @@ const serverIsOnAir = async (_req, _res, next) => {
 	}
 };
 
+/** @type {import('../config-loader/config-loader').Config} */
 export let config = {
 	models: [
 		{
@@ -59,17 +60,22 @@ export let config = {
 ALTER TABLE "textarea"
 ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;\
 `,
-				'15.0.0-true-boolean': {
-					mysql: `\
-ALTER TABLE "textarea"
-MODIFY "is disabled" BOOLEAN NOT NULL;\
-`,
-					postgres: `\
-ALTER TABLE "textarea"
-ALTER COLUMN "is disabled" SET DATA TYPE BOOLEAN USING b::BOOLEAN;\
-`,
-					// No need to migrate for websql
-					websql: '',
+				'15.0.0-data-types': async (tx, sbvrUtils) => {
+					switch (sbvrUtils.db.engine) {
+						case 'mysql':
+							await tx.executeSql(`\
+								ALTER TABLE "textarea"
+								MODIFY "is disabled" BOOLEAN NOT NULL;`);
+							break;
+						case 'postgres':
+							await tx.executeSql(`\
+								ALTER TABLE "textarea"
+								ALTER COLUMN "is disabled" DROP DEFAULT,
+								ALTER COLUMN "is disabled" SET DATA TYPE BOOLEAN USING "is disabled"::BOOLEAN,
+								ALTER COLUMN "is disabled" SET DEFAULT FALSE;`);
+							break;
+						// No need to migrate for websql
+					}
 				},
 			},
 		},
@@ -134,7 +140,7 @@ export async function setup(app, sbvrUtils, db) {
 					const instance = result[0];
 					await sbvrUtils.executeModel(tx, {
 						apiRoot: instance.is_of__vocabulary,
-						modelText: instance.model_value,
+						modelText: instance.model_value.value,
 					});
 				});
 			await isServerOnAir(true);
@@ -153,7 +159,7 @@ export async function setup(app, sbvrUtils, db) {
 		permissions.checkPermissionsMiddleware('all'),
 		serverIsOnAir,
 		(_req, res) => {
-			res.sendStatus(404);
+			res.status(404).end();
 		},
 	);
 
@@ -199,7 +205,7 @@ export async function setup(app, sbvrUtils, db) {
 						});
 					});
 				await isServerOnAir(true);
-				res.sendStatus(200);
+				res.status(200).end();
 			} catch (err) {
 				await isServerOnAir(false);
 				res.status(404).json(err);
@@ -215,7 +221,7 @@ export async function setup(app, sbvrUtils, db) {
 				res.json(results);
 			} catch (err) {
 				console.log('Error validating', err);
-				res.sendStatus(404);
+				res.status(404).end();
 			}
 		},
 	);
@@ -239,10 +245,10 @@ export async function setup(app, sbvrUtils, db) {
 					await sbvrUtils.executeModels(tx, exports.config.models);
 					await setupModels(tx);
 				});
-				res.sendStatus(200);
-			} catch (err) {
-				console.error('Error clearing db', err, err.stack);
-				res.sendStatus(503);
+				res.status(200).end();
+			} catch (/** @type any */ err) {
+				console.error('Error clearing db', err);
+				res.status(503).end();
 			}
 		},
 	);
@@ -266,10 +272,10 @@ export async function setup(app, sbvrUtils, db) {
 						}
 					}
 				});
-				res.sendStatus(200);
-			} catch (err) {
-				console.error('Error importing db', err, err.stack);
-				res.sendStatus(404);
+				res.status(200).end();
+			} catch (/** @type any */ err) {
+				console.error('Error importing db', err);
+				res.status(404).end();
 			}
 		},
 	);
@@ -290,11 +296,11 @@ export async function setup(app, sbvrUtils, db) {
 								'SELECT * FROM "' + tableName + '";',
 							);
 							let insQuery = '';
-							result.rows.forEach((currRow) => {
+							for (const currRow of result.rows) {
 								let notFirst = false;
 								insQuery += 'INSERT INTO "' + tableName + '" (';
 								let valQuery = '';
-								for (let propName of Object.keys(currRow)) {
+								for (const propName of Object.keys(currRow)) {
 									if (notFirst) {
 										insQuery += ',';
 										valQuery += ',';
@@ -305,15 +311,15 @@ export async function setup(app, sbvrUtils, db) {
 									valQuery += "'" + currRow[propName] + "'";
 								}
 								insQuery += ') values (' + valQuery + ');\n';
-							});
+							}
 							exported += insQuery;
 						}),
 					);
 				});
 				res.json(exported);
-			} catch (err) {
-				console.error('Error exporting db', err, err.stack);
-				res.sendStatus(503);
+			} catch (/** @type any */ err) {
+				console.error('Error exporting db', err);
+				res.status(503).end();
 			}
 		},
 	);
@@ -340,10 +346,10 @@ export async function setup(app, sbvrUtils, db) {
 						}),
 					);
 				});
-				res.sendStatus(200);
-			} catch (err) {
-				console.error('Error backing up db', err, err.stack);
-				res.sendStatus(404);
+				res.status(200).end();
+			} catch (/** @type any */ err) {
+				console.error('Error backing up db', err);
+				res.status(404).end();
 			}
 		},
 	);
@@ -369,10 +375,10 @@ export async function setup(app, sbvrUtils, db) {
 						}),
 					);
 				});
-				res.sendStatus(200);
-			} catch (err) {
-				console.error('Error restoring db', err, err.stack);
-				res.sendStatus(404);
+				res.status(200).end();
+			} catch (/** @type any */ err) {
+				console.error('Error restoring db', err);
+				res.status(404).end();
 			}
 		},
 	);
@@ -398,7 +404,7 @@ export async function setup(app, sbvrUtils, db) {
 			sbvrUtils.deleteModel('data'),
 		]);
 		await isServerOnAir(false);
-		res.sendStatus(200);
+		res.status(200).end();
 	});
 
 	await db.transaction(setupModels);