@balena/pinejs 15.0.0-delete-state-default-user-permissions-981931563dc47b2a8b873bc787d7dacfcc6c52e3 → 15.0.0-deprecate-node12-8a99d72ae66d7708293afc56c5d7eb19b39081cd

package/src/migrator/sync.ts

@@ -0,0 +1,169 @@
+import {
+	modelText,
+	MigrationTuple,
+	MigrationError,
+	defaultMigrationCategory,
+	setExecutedMigrations,
+	getExecutedMigrations,
+	lockMigrations,
+	RunnableMigrations,
+} from './utils';
+import type { Tx } from '../database-layer/db';
+import type { Config, Model } from '../config-loader/config-loader';
+
+import * as _ from 'lodash';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+
+type ApiRootModel = Model & { apiRoot: string };
+
+export const postRun = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { initSql } = model;
+	if (initSql == null) {
+		return;
+	}
+
+	const modelName = model.apiRoot;
+	const modelIsNew = await sbvrUtils.isModelNew(tx, modelName);
+	if (modelIsNew) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			`First time executing '${modelName}', running init script`,
+		);
+
+		await lockMigrations(tx, modelName, async () => {
+			try {
+				await tx.executeSql(initSql);
+			} catch (err: any) {
+				(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+					`initSql execution error ${err} `,
+				);
+				throw new MigrationError(err);
+			}
+		});
+	}
+};
+
+export const run = async (tx: Tx, model: ApiRootModel): Promise<void> => {
+	const { migrations } = model;
+	if (migrations == null || _.isEmpty(migrations)) {
+		return;
+	}
+	const defaultMigrations = migrations[defaultMigrationCategory];
+	const runMigrations: RunnableMigrations =
+		defaultMigrationCategory in migrations
+			? typeof defaultMigrations === 'object'
+				? defaultMigrations
+				: {}
+			: migrations;
+
+	return $run(tx, model, runMigrations);
+};
+
+const $run = async (
+	tx: Tx,
+	model: ApiRootModel,
+	migrations: RunnableMigrations,
+): Promise<void> => {
+	const modelName = model.apiRoot;
+
+	// migrations only run if the model has been executed before,
+	// to make changes that can't be automatically applied
+	const modelIsNew = await sbvrUtils.isModelNew(tx, modelName);
+	if (modelIsNew) {
+		(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+			`First time model '${modelName}' has executed, skipping migrations`,
+		);
+
+		return await setExecutedMigrations(tx, modelName, Object.keys(migrations));
+	}
+	await lockMigrations(tx, modelName, async () => {
+		try {
+			const executedMigrations = await getExecutedMigrations(tx, modelName);
+			const pendingMigrations = filterAndSortPendingMigrations(
+				migrations,
+				executedMigrations,
+			);
+			if (pendingMigrations.length === 0) {
+				return;
+			}
+
+			const newlyExecutedMigrations = await executeMigrations(
+				tx,
+				pendingMigrations,
+			);
+			await setExecutedMigrations(tx, modelName, [
+				...executedMigrations,
+				...newlyExecutedMigrations,
+			]);
+		} catch (err: any) {
+			(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+				`Failed to executed synchronous migrations from api root model ${err}`,
+			);
+			throw new MigrationError(err);
+		}
+	});
+};
+
+// turns {"key1": migration, "key3": migration, "key2": migration}
+// into  [["key1", migration], ["key2", migration], ["key3", migration]]
+const filterAndSortPendingMigrations = (
+	migrations: NonNullable<RunnableMigrations>,
+	executedMigrations: string[],
+): MigrationTuple[] =>
+	(_(migrations).omit(executedMigrations) as _.Object<typeof migrations>)
+		.toPairs()
+		.sortBy(([migrationKey]) => migrationKey)
+		.value();
+
+const executeMigrations = async (
+	tx: Tx,
+	migrations: MigrationTuple[] = [],
+): Promise<string[]> => {
+	try {
+		for (const migration of migrations) {
+			await executeMigration(tx, migration);
+		}
+	} catch (err: any) {
+		(sbvrUtils.api.migrations?.logger.error ?? console.error)(
+			'Error while executing migrations, rolled back',
+		);
+		throw new MigrationError(err);
+	}
+	return migrations.map(([migrationKey]) => migrationKey); // return migration keys
+};
+
+const executeMigration = async (
+	tx: Tx,
+	[key, migration]: MigrationTuple,
+): Promise<void> => {
+	(sbvrUtils.api.migrations?.logger.info ?? console.info)(
+		`Running migration ${JSON.stringify(key)}`,
+	);
+
+	if (typeof migration === 'function') {
+		await migration(tx, sbvrUtils);
+	} else if (typeof migration === 'string') {
+		await tx.executeSql(migration);
+	} else {
+		throw new MigrationError(`Invalid migration type: ${typeof migration}`);
+	}
+};
+
+export const config: Config = {
+	models: [
+		{
+			modelName: 'migrations',
+			apiRoot: 'migrations',
+			modelText,
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "migration"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'11.0.1-modified-at': `
+					ALTER TABLE "migration lock"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+			},
+		},
+	],
+};

package/src/migrator/utils.ts

@@ -0,0 +1,154 @@
+import type { Tx } from '../database-layer/db';
+import type { Resolvable } from '../sbvr-api/common-types';
+
+import { Engines } from '@balena/abstract-sql-compiler';
+import * as _ from 'lodash';
+import { TypedError } from 'typed-error';
+import { migrator as migratorEnv } from '../config-loader/env';
+export { migrator as migratorEnv } from '../config-loader/env';
+import { delay } from '../sbvr-api/control-flow';
+
+// tslint:disable-next-line:no-var-requires
+export const modelText = require('./migrations.sbvr');
+
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+export enum MigrationCategories {
+	'sync' = 'sync',
+}
+export const defaultMigrationCategory = MigrationCategories.sync;
+export type CategorizedMigrations = {
+	[key in MigrationCategories]: RunnableMigrations;
+};
+
+type SbvrUtils = typeof sbvrUtils;
+export type MigrationTuple = [string, Migration];
+export type MigrationFn = (tx: Tx, sbvrUtils: SbvrUtils) => Resolvable<void>;
+export type Migration = string | MigrationFn;
+export type RunnableMigrations = { [key: string]: Migration };
+export type Migrations = CategorizedMigrations | RunnableMigrations;
+
+export class MigrationError extends TypedError {}
+
+// Tagged template to convert binds from `?` format to the necessary output format,
+// eg `$1`/`$2`/etc for postgres
+export const binds = (strings: TemplateStringsArray, ...bindNums: number[]) =>
+	strings
+		.map((str, i) => {
+			if (i === bindNums.length) {
+				return str;
+			}
+			if (i + 1 !== bindNums[i]) {
+				throw new SyntaxError('Migration sql binds must be sequential');
+			}
+			if (sbvrUtils.db.engine === Engines.postgres) {
+				return str + `$${bindNums[i]}`;
+			}
+			return str + `?`;
+		})
+		.join('');
+
+export const lockMigrations = async <T>(
+	tx: Tx,
+	modelName: string,
+	fn: () => Promise<T>,
+): Promise<T | undefined> => {
+	if (!(await migrationTablesExist(tx))) {
+		return;
+	}
+
+	try {
+		await tx.executeSql(
+			binds`
+DELETE FROM "migration lock"
+WHERE "model name" = ${1}
+AND "created at" < ${2}`,
+			[modelName, new Date(Date.now() - migratorEnv.lockTimeout)],
+		);
+		await tx.executeSql(
+			binds`
+INSERT INTO "migration lock" ("model name")
+VALUES (${1})`,
+			[modelName],
+		);
+	} catch (err: any) {
+		await delay(migratorEnv.lockFailDelay);
+		throw err;
+	}
+	try {
+		return await fn();
+	} finally {
+		try {
+			await tx.executeSql(
+				binds`
+DELETE FROM "migration lock"
+WHERE "model name" = ${1}`,
+				[modelName],
+			);
+		} catch {
+			// We ignore errors here as it's mostly likely caused by the migration failing and
+			// rolling back the transaction, and if we rethrow here we'll overwrite the real error
+			// making it much harder for users to see what went wrong and fix it
+		}
+	}
+};
+
+export const setExecutedMigrations = async (
+	tx: Tx,
+	modelName: string,
+	executedMigrations: string[],
+): Promise<void> => {
+	const stringifiedMigrations = JSON.stringify(executedMigrations);
+
+	if (!(await migrationTablesExist(tx))) {
+		return;
+	}
+
+	const { rowsAffected } = await tx.executeSql(
+		binds`
+UPDATE "migration"
+SET "model name" = ${1},
+"executed migrations" = ${2}
+WHERE "migration"."model name" = ${3}`,
+		[modelName, stringifiedMigrations, modelName],
+	);
+
+	if (rowsAffected === 0) {
+		await tx.executeSql(
+			binds`
+INSERT INTO "migration" ("model name", "executed migrations")
+VALUES (${1}, ${2})`,
+			[modelName, stringifiedMigrations],
+		);
+	}
+};
+
+export const getExecutedMigrations = async (
+	tx: Tx,
+	modelName: string,
+): Promise<string[]> => {
+	if (!(await migrationTablesExist(tx))) {
+		return [];
+	}
+
+	const { rows } = await tx.executeSql(
+		binds`
+SELECT "migration"."executed migrations" AS "executed_migrations"
+FROM "migration"
+WHERE "migration"."model name" = ${1}`,
+		[modelName],
+	);
+
+	const data = rows[0];
+	if (data == null) {
+		return [];
+	}
+
+	return JSON.parse(data.executed_migrations) as string[];
+};
+
+export const migrationTablesExist = async (tx: Tx) => {
+	const tables = ['migration', 'migration lock'];
+	const where = tables.map((tableName) => `name = '${tableName}'`).join(' OR ');
+	const result = await tx.tableList(where);
+	return result.rows.length === tables.length;
+};

package/src/odata-metadata/odata-metadata-generator.ts

@@ -21,17 +21,14 @@ const forEachUniqueTable = <T>(
 	const usedTableNames: { [tableName: string]: true } = {};
 
 	const result = [];
-	for (const key in model) {
-		if (model.hasOwnProperty(key)) {
-			const table = model[key];
-			if (
-				typeof table !== 'string' &&
-				!table.primitive &&
-				!usedTableNames[table.name]
-			) {
-				usedTableNames[table.name] = true;
-				result.push(callback(key, table));
-			}
+	for (const [key, table] of Object.entries(model)) {
+		if (
+			typeof table !== 'string' &&
+			!table.primitive &&
+			!usedTableNames[table.name]
+		) {
+			usedTableNames[table.name] = true;
+			result.push(callback(key, table));
 		}
 	}
 	return result;

package/src/passport-pinejs/passport-pinejs.ts

@@ -65,8 +65,9 @@ const setup: ConfigLoader.SetupFunction = async (app: Express.Application) => {
 			})(req, res, next);
 
 		logout = (req, _res, next) => {
-			req.logout();
-			next();
+			req.logout((error) => {
+				error ? next(error) : next();
+			});
 		};
 	} else {
 		let loggedIn = false;

package/src/sbvr-api/abstract-sql.ts

@@ -47,7 +47,7 @@ export const compileRequest = (request: ODataRequest) => {
 			);
 			request.sqlQuery = sqlQuery;
 			request.modifiedFields = modifiedFields;
-		} catch (err) {
+		} catch (err: any) {
 			sbvrUtils.api[request.vocabulary].logger.error(
 				'Failed to compile abstract sql: ',
 				request.abstractSqlQuery,
@@ -139,7 +139,7 @@ export const getAndCheckBindValues = async (
 
 			try {
 				return await AbstractSQLCompiler[engine].dataTypeValidate(value, field);
-			} catch (err) {
+			} catch (err: any) {
 				throw new BadRequestError(`"${fieldName}" ${err.message}`);
 			}
 		}),
@@ -173,7 +173,10 @@ const checkModifiedFields = (
 };
 export const isRuleAffected = (
 	rule: AbstractSQLCompiler.SqlRule,
-	request?: ODataRequest,
+	request?: Pick<
+		ODataRequest,
+		'abstractSqlQuery' | 'modifiedFields' | 'method' | 'vocabulary'
+	>,
 ) => {
 	// If there is no abstract sql query then nothing was modified
 	if (request?.abstractSqlQuery == null) {

package/src/sbvr-api/control-flow.ts

@@ -24,7 +24,7 @@ export const settleMapSeries: MappingFunction = async <T, U>(
 	await mapSeries(a, async (p) => {
 		try {
 			return await fn(p);
-		} catch (err) {
+		} catch (err: any) {
 			return ensureError(err);
 		}
 	});
@@ -49,7 +49,7 @@ const mapTill: MappingFunction = async <T, U>(
 		try {
 			const result = await fn(p);
 			results.push(result);
-		} catch (err) {
+		} catch (err: any) {
 			results.push(ensureError(err));
 			break;
 		}

package/src/sbvr-api/hooks.ts

@@ -1,6 +1,6 @@
 import type { OptionalField, Resolvable } from './common-types';
 import type { Tx } from '../database-layer/db';
-import type { ODataRequest } from './uri-parser';
+import type { ODataRequest, ParsedODataRequest } from './uri-parser';
 import type { AnyObject } from 'pinejs-client-core';
 import type { TypedError } from 'typed-error';
 import type { SupportedMethod } from '@balena/odata-to-abstract-sql';
@@ -15,6 +15,7 @@ import {
 	resolveSynonym,
 	getAbstractSqlModel,
 	api,
+	Response,
 } from './sbvr-utils';
 
 export interface HookReq {
@@ -46,6 +47,9 @@ export interface Hooks {
 		options: HookArgs & {
 			tx: Tx;
 			result: any;
+			/** This can be mutated to modify the response sent to the client */
+			response: Response;
+			/** @deprecated Use the response object instead */
 			data?: any;
 		},
 	) => HookResponse;
@@ -127,11 +131,14 @@ export const rollbackRequestHooks = <T extends InstantiatedHooks>(
 	if (hooks == null) {
 		return;
 	}
-	settleMapSeries(_(hooks).flatMap().compact().value(), async (hook) => {
-		if (hook instanceof SideEffectHook) {
-			await hook.rollback();
-		}
-	});
+	settleMapSeries(
+		Object.values(hooks).flatMap((v): Array<Hook<HookFn>> => v),
+		async (hook) => {
+			if (hook instanceof SideEffectHook) {
+				await hook.rollback();
+			}
+		},
+	);
 };
 
 const instantiateHooks = (hooks: HookBlueprints): InstantiatedHooks =>
@@ -187,14 +194,17 @@ const getMethodHooks = memoize(
 );
 export const getHooks = (
 	request: Pick<
-		OptionalField<ODataRequest, 'resourceName'>,
+		OptionalField<ParsedODataRequest, 'resourceName'>,
 		'resourceName' | 'method' | 'vocabulary'
 	>,
 ): InstantiatedHooks => {
 	let { resourceName } = request;
 	if (resourceName != null) {
 		resourceName = resolveSynonym(
-			request as Pick<ODataRequest, 'resourceName' | 'method' | 'vocabulary'>,
+			request as Pick<
+				ParsedODataRequest,
+				'resourceName' | 'method' | 'vocabulary'
+			>,
 		);
 	}
 	return instantiateHooks(

package/src/sbvr-api/odata-response.ts

@@ -148,16 +148,16 @@ export const process = async (
 	);
 
 	const odataIdField = sqlNameToODataName(table.idField);
-	rows.forEach((row) => {
-		processedFields.forEach((fieldName) => {
+	for (const row of rows) {
+		for (const fieldName of processedFields) {
 			row[fieldName] = fetchProcessingFields[fieldName](row[fieldName]);
-		});
+		}
 		if (includeMetadata) {
 			row.__metadata = {
 				uri: resourceURI(vocab, resourceName, row[odataIdField]),
 			};
 		}
-	});
+	}
 
 	if (expandableFields.length > 0) {
 		await Promise.all(