@balena/pinejs 15.0.0-true-boolean-7896b116c446d891d7a0d5e4085c02a13bc9c725 → 15.0.1-build-migrations-clarify-marking-sbvr-optional-d6d0ded8eccc6eadb2492f4697918cf0afd00215-1

package/src/database-layer/db.ts

@@ -2,14 +2,14 @@
 import type * as Mysql from 'mysql';
 import type * as Pg from 'pg';
 import type * as PgConnectionString from 'pg-connection-string';
-import type { Resolvable } from '../sbvr-api/common-types';
+import type { Dictionary, Resolvable } from '../sbvr-api/common-types';
 
 import { Engines } from '@balena/abstract-sql-compiler';
-import * as Bluebird from 'bluebird';
-import * as EventEmitter from 'eventemitter3';
+import { EventEmitter } from 'eventemitter3';
 import * as _ from 'lodash';
 import { TypedError } from 'typed-error';
 import * as env from '../config-loader/env';
+import { fromCallback, timeout } from '../sbvr-api/control-flow';
 
 export const metrics = new EventEmitter();
 
@@ -25,22 +25,18 @@ export interface Row {
 export interface Result {
 	rows: Row[];
 	rowsAffected: number;
-	insertId?: number;
+	insertId?: number | undefined;
 }
 
 export type Sql = string;
 export type Bindings = any[];
 
 const isSqlError = (value: any): value is SQLError => {
-	return (
-		value != null &&
-		value.constructor != null &&
-		value.constructor.name === 'SQLError'
-	);
+	return value?.constructor?.name === 'SQLError';
 };
 
 export class DatabaseError extends TypedError {
-	public code: number | string;
+	public code?: number | string;
 	constructor(message?: string | CodedError | SQLError) {
 		if (isSqlError(message)) {
 			// If this is a SQLError we have to handle it specially (since it's not actually an instance of Error)
@@ -104,9 +100,63 @@ export interface Database extends BaseDatabase {
 	readTransaction: TransactionFn;
 }
 
-export const engines: {
-	[engine: string]: (connectString: string | object) => Database;
-} = {};
+interface EngineParams {
+	[engine: string]: (options: unknown) => Database;
+}
+export const engines = {} as EngineParams;
+
+const types = {
+	integer: {
+		min: -2147483648,
+		max: 2147483647,
+	},
+};
+
+const validateTransactionLockParameter = (
+	value: number,
+	parameterName: string,
+) => {
+	if (
+		!Number.isInteger(value) ||
+		value < types.integer.min ||
+		types.integer.max < value
+	) {
+		throw new TypeError(
+			`Invalid parameter '${parameterName}' provided for transaction lock`,
+		);
+	}
+};
+
+const transactionLockNamespaceMap: Dictionary<number> = {};
+
+/**
+ *
+ * @param namespaceKey 	Any string representing a namespace
+ * @param namespaceId 	For application level locks positive values should be used
+ * 						as internal / low level locks are recommended to use negative values
+ */
+
+export function registerTransactionLockNamespace(
+	namespaceKey: string,
+	namespaceId: number,
+) {
+	validateTransactionLockParameter(namespaceId, 'namespaceId');
+	if (transactionLockNamespaceMap[namespaceKey] != null) {
+		throw new Error(
+			`Error while registering transaction lock namespace '${namespaceKey}'. Namespace key is already registered.`,
+		);
+	}
+	const existingNamespaceEntry = Object.entries(
+		transactionLockNamespaceMap,
+	).find(([, id]) => id === namespaceId);
+	if (existingNamespaceEntry != null) {
+		throw new Error(
+			`Error while registering transaction lock namespace '${namespaceKey}'. Transaction lock namespace id '${namespaceId}' already registered for namespace ${existingNamespaceEntry[0]}.`,
+		);
+	}
+
+	transactionLockNamespaceMap[namespaceKey] = namespaceId;
+}
 
 const atomicExecuteSql: Database['executeSql'] = async function (
 	sql,
@@ -121,9 +171,7 @@ const asyncTryFn = (fn: () => any) => {
 	Promise.resolve().then(fn);
 };
 
-type RejectedFunctions = (
-	message: string,
-) => {
+type RejectedFunctions = (message: string) => {
 	executeSql: Tx['executeSql'];
 	rollback: Tx['rollback'];
 };
@@ -215,7 +263,6 @@ class AutomaticClose {
 export abstract class Tx {
 	private closed = false;
 	protected automaticClose: AutomaticClose;
-	public abstract readonly engine: Engines;
 
 	constructor(
 		protected readOnly: boolean,
@@ -255,7 +302,11 @@ export abstract class Tx {
 		bindings: Bindings = [],
 		...args: any[]
 	): Promise<Result> {
-		if (this.readOnly && !/^\s*SELECT\s(?:[^;]|;\s*SELECT\s)*$/.test(sql)) {
+		if (
+			env.db.checkReadOnlyQueries &&
+			this.readOnly &&
+			!/^\s*SELECT\s(?:[^;]|;\s*SELECT\s)*$/.test(sql)
+		) {
 			throw new ReadOnlyViolationError(
 				`Attempted to run a non-SELECT statement in a read-only tx: ${sql}`,
 			);
@@ -272,7 +323,7 @@ export abstract class Tx {
 		const t0 = Date.now();
 		try {
 			return await this._executeSql(sql, bindings, ...args);
-		} catch (err) {
+		} catch (err: any) {
 			throw wrapDatabaseError(err);
 		} finally {
 			this.automaticClose.decrementPending();
@@ -321,6 +372,9 @@ export abstract class Tx {
 		this.on = onEnd;
 		this.clearListeners();
 	}
+	public disableAutomaticClose(): void {
+		this.automaticClose.cancelPending();
+	}
 
 	private listeners: {
 		end: Array<() => void>;
@@ -346,6 +400,16 @@ export abstract class Tx {
 	protected abstract _rollback(): Promise<void>;
 	protected abstract _commit(): Promise<void>;
 
+	public async getTxLevelLock(
+		_namespaceKey: string,
+		_key: number,
+		_blocking: boolean = true,
+	): Promise<boolean> {
+		throw new Error(
+			'The getTxLevelLock method is not implemented for the current engine.',
+		);
+	}
+
 	public abstract tableList(extraWhereClause?: string): Promise<Result>;
 	public async dropTable(tableName: string, ifExists = true) {
 		if (typeof tableName !== 'string') {
@@ -374,7 +438,7 @@ const createTransaction = (createFunc: CreateTransactionFn): TransactionFn => {
 		let tx;
 		try {
 			tx = await createFunc(stackTraceErr);
-		} catch (err) {
+		} catch (err: any) {
 			throw wrapDatabaseError(err);
 		}
 		if (fn) {
@@ -382,7 +446,7 @@ const createTransaction = (createFunc: CreateTransactionFn): TransactionFn => {
 				const result = await fn(tx);
 				await tx.end();
 				return result;
-			} catch (err) {
+			} catch (err: any) {
 				try {
 					await tx.rollback();
 				} catch {
@@ -403,38 +467,74 @@ try {
 } catch (e) {
 	// Ignore errors
 }
+interface EngineParams {
+	postgres: (
+		options:
+			| string
+			| Pg.PoolConfig
+			| { primary: Pg.PoolConfig; replica?: Pg.PoolConfig },
+	) => Database;
+}
 if (maybePg != null) {
 	const pg = maybePg;
-	engines.postgres = (connectString: string | object): Database => {
+	engines.postgres = (connectString) => {
 		const PG_UNIQUE_VIOLATION = '23505';
 		const PG_FOREIGN_KEY_VIOLATION = '23503';
 		const PG_CHECK_CONSTRAINT_VIOLATION = '23514';
 		const PG_EXCLUSION_CONSTRAINT_VIOLATION = '23P01';
 
-		let config: Pg.PoolConfig;
-		if (typeof connectString === 'string') {
-			const pgConnectionString: typeof PgConnectionString = require('pg-connection-string');
-			// We have to cast because of the use of null vs undefined
-			config = pgConnectionString.parse(connectString) as Pg.PoolConfig;
-		} else {
-			config = connectString;
-		}
-		config.max = env.db.poolSize;
-		config.idleTimeoutMillis = env.db.idleTimeoutMillis;
-		config.statement_timeout = env.db.statementTimeout;
-		config.query_timeout = env.db.queryTimeout;
-		config.connectionTimeoutMillis = env.db.connectionTimeoutMillis;
-		config.keepAlive = env.db.keepAlive;
-		const pool = new pg.Pool(config);
 		const { PG_SCHEMA } = process.env;
-		if (PG_SCHEMA != null) {
-			pool.on('connect', (client) => {
-				client.query({ text: `SET search_path TO "${PG_SCHEMA}"` });
+		const initPool = (config: Pg.PoolConfig) => {
+			config.max ??= env.db.poolSize;
+			config.idleTimeoutMillis ??= env.db.idleTimeoutMillis;
+			config.statement_timeout ??= env.db.statementTimeout;
+			config.query_timeout ??= env.db.queryTimeout;
+			config.connectionTimeoutMillis ??= env.db.connectionTimeoutMillis;
+			config.keepAlive ??= env.db.keepAlive;
+			// @ts-expect-error maxLifetimeSeconds is valid for PgPool but isn't currently in the typings
+			config.maxLifetimeSeconds ??= env.db.maxLifetimeSeconds;
+			config.maxUses ??= env.db.maxUses;
+			const p = new pg.Pool(config);
+			if (PG_SCHEMA != null) {
+				p.on('connect', (client) => {
+					client.query({ text: `SET search_path TO "${PG_SCHEMA}"` });
+				});
+			}
+			p.on('connect', (client) => {
+				client.on('error', (err) => {
+					try {
+						console.error('Releasing client on error:', err);
+						client.release(err);
+					} catch (e) {
+						console.error('Error releasing client on error:', e);
+					}
+				});
 			});
-			pool.on('error', (err) => {
+			p.on('error', (err) => {
 				console.error('Pool error:', err.message);
 			});
+			return p;
+		};
+
+		let pool: Pg.Pool;
+		let replica: Pg.Pool;
+		if (typeof connectString === 'string') {
+			const pgConnectionString: typeof PgConnectionString = require('pg-connection-string');
+			// We have to cast because of the use of null vs undefined
+			const config = pgConnectionString.parse(connectString) as Pg.PoolConfig;
+			pool = initPool(config);
+		} else {
+			const config = connectString;
+			if ('primary' in config) {
+				pool = initPool(config.primary);
+				if (config.replica) {
+					replica = initPool(config.replica);
+				}
+			} else {
+				pool = initPool(config);
+			}
 		}
+		replica ??= pool;
 
 		const createResult = ({
 			rowCount,
@@ -450,8 +550,6 @@ if (maybePg != null) {
 			};
 		};
 		class PostgresTx extends Tx {
-			engine = Engines.postgres;
-
 			constructor(
 				private db: Pg.PoolClient,
 				readOnly: boolean,
@@ -479,7 +577,7 @@ if (maybePg != null) {
 						text: sql,
 						values: bindings,
 					});
-				} catch (err) {
+				} catch (err: any) {
 					if (err.code === PG_UNIQUE_VIOLATION) {
 						throw new UniqueConstraintError(err);
 					}
@@ -504,20 +602,21 @@ if (maybePg != null) {
 					const queryQueue = this.db.queryQueue as Pg.Query[];
 					if (queryQueue.length > 0) {
 						const err = new DatabaseError('Rolling back transaction');
-						queryQueue.forEach((query) => {
+						for (const query of queryQueue) {
 							process.nextTick(() => {
 								// @ts-expect-error typings do not include this function
 								query.handleError(err, this.db.connection);
 							});
-						});
+						}
 						queryQueue.length = 0;
 					}
-					await Bluebird.resolve(this.$executeSql('ROLLBACK;')).timeout(
+					await timeout(
+						this.$executeSql('ROLLBACK;'),
 						env.db.rollbackTimeout,
 						'Rolling back transaction timed out',
 					);
 					this.db.release();
-				} catch (err) {
+				} catch (err: any) {
 					err = wrapDatabaseError(err);
 					this.db.release(err);
 					throw err;
@@ -528,12 +627,45 @@ if (maybePg != null) {
 				try {
 					await this.$executeSql('COMMIT;');
 					this.db.release();
-				} catch (err) {
+				} catch (err: any) {
 					this.db.release(err);
 					throw err;
 				}
 			}
 
+			public override async getTxLevelLock(
+				namespaceKey: string,
+				key: number,
+				blocking: boolean = true,
+			) {
+				validateTransactionLockParameter(key, 'key');
+				const namespaceId = transactionLockNamespaceMap[namespaceKey];
+				if (namespaceId == null) {
+					throw new Error(
+						`Transaction lock namespace ${namespaceKey} not registered.`,
+					);
+				}
+				try {
+					if (blocking) {
+						await this.executeSql(`SELECT pg_advisory_xact_lock($1, $2);`, [
+							namespaceId,
+							key,
+						]);
+						return true;
+					} else {
+						const { rows } = await this.executeSql(
+							`SELECT pg_try_advisory_xact_lock($1, $2);`,
+							[namespaceId, key],
+						);
+						return rows[0].pg_try_advisory_xact_lock === true;
+					}
+				} catch (err) {
+					throw new Error(
+						`getTxLevelLock error during getting lock from postgres db layer ${err}`,
+					);
+				}
+			}
+
 			public async tableList(extraWhereClause: string = '') {
 				if (extraWhereClause !== '') {
 					extraWhereClause = 'WHERE ' + extraWhereClause;
@@ -558,7 +690,7 @@ if (maybePg != null) {
 				return tx;
 			}),
 			readTransaction: createTransaction(async (stackTraceErr) => {
-				const client = await pool.connect();
+				const client = await replica.connect();
 				const tx = new PostgresTx(client, false, stackTraceErr);
 				tx.executeSql('START TRANSACTION READ ONLY;');
 				return tx.asReadOnly();
@@ -575,19 +707,23 @@ try {
 } catch (e) {
 	// Ignore errors
 }
+interface EngineParams {
+	mysql: (options: Mysql.PoolConfig) => Database;
+}
 if (maybeMysql != null) {
 	const mysql = maybeMysql;
-	engines.mysql = (options: Mysql.PoolConfig): Database => {
+	engines.mysql = (options) => {
 		const MYSQL_UNIQUE_VIOLATION = 'ER_DUP_ENTRY';
 		const MYSQL_FOREIGN_KEY_VIOLATION = 'ER_ROW_IS_REFERENCED';
 		const MYSQL_CHECK_CONSTRAINT_VIOLATION = 'ER_CHECK_CONSTRAINT_VIOLATED';
 		const pool = mysql.createPool(options);
 		pool.on('connection', (db) => {
-			db.query("SET sql_mode='ANSI_QUOTES';");
-		});
-		const getConnectionAsync = Bluebird.promisify(pool.getConnection, {
-			context: pool,
+			db.query("SET sql_mode='ANSI';");
 		});
+		const getConnectionAsync = () =>
+			fromCallback<Mysql.PoolConnection>((callback) => {
+				pool.getConnection(callback);
+			});
 
 		interface MysqlRowArray extends Array<Row> {
 			affectedRows: number;
@@ -601,8 +737,6 @@ if (maybeMysql != null) {
 			};
 		};
 		class MySqlTx extends Tx {
-			engine = Engines.mysql;
-
 			constructor(
 				private db: Mysql.Connection,
 				private close: CloseTransactionFn,
@@ -619,10 +753,10 @@ if (maybeMysql != null) {
 			protected async _executeSql(sql: Sql, bindings: Bindings) {
 				let result;
 				try {
-					result = await Bluebird.fromCallback<MysqlRowArray>((callback) => {
+					result = await fromCallback<MysqlRowArray>((callback) => {
 						this.db.query(sql, bindings, callback);
 					});
-				} catch (err) {
+				} catch (err: any) {
 					if (err.code === MYSQL_UNIQUE_VIOLATION) {
 						// We know that the type is an IError for mysql, but typescript doesn't like the catch obj sugar
 						throw new UniqueConstraintError(err as Mysql.MysqlError);
@@ -690,6 +824,9 @@ if (maybeMysql != null) {
 	};
 }
 
+interface EngineParams {
+	websql: (databaseName: string) => Database;
+}
 if (typeof window !== 'undefined' && window.openDatabase != null) {
 	interface WebSqlResult {
 		insertId?: number;
@@ -705,7 +842,7 @@ if (typeof window !== 'undefined' && window.openDatabase != null) {
 		SQLStatementCallback,
 		SQLStatementErrorCallback,
 	];
-	engines.websql = (databaseName: string): Database => {
+	engines.websql = (databaseName) => {
 		const WEBSQL_CONSTRAINT_ERR = 6;
 
 		const db = window.openDatabase(
@@ -736,8 +873,6 @@ if (typeof window !== 'undefined' && window.openDatabase != null) {
 		};
 
 		class WebSqlTx extends Tx {
-			engine = Engines.websql;
-
 			constructor(
 				private tx: WebSqlWrapper,
 				readOnly: boolean,
@@ -754,7 +889,7 @@ if (typeof window !== 'undefined' && window.openDatabase != null) {
 				let result;
 				try {
 					result = await this.tx.executeSql(sql, bindings);
-				} catch (err) {
+				} catch (err: any) {
 					if (err.code === WEBSQL_CONSTRAINT_ERR) {
 						throw new ConstraintError('Constraint failed.');
 					}
@@ -878,7 +1013,13 @@ if (typeof window !== 'undefined' && window.openDatabase != null) {
 	};
 }
 
-export const connect = (databaseOptions: { engine: string; params: {} }) => {
+export type DatabaseOptions<T extends keyof EngineParams> = {
+	engine: T;
+	params: Parameters<EngineParams[T]>[0];
+};
+export const connect = <T extends keyof EngineParams>(
+	databaseOptions: DatabaseOptions<T>,
+) => {
 	if (engines[databaseOptions.engine] == null) {
 		throw new Error('Unsupported database engine: ' + databaseOptions.engine);
 	}

package/src/express-emulator/express.js

@@ -49,16 +49,15 @@ const app = (function () {
 			match,
 			paramName,
 			// Flatten middleware list to handle arrays of middleware in the arg list.
-			middleware: _.flattenDeep(middleware),
+			middleware: middleware.flat(Infinity),
 		});
 	};
 	const process = async function (
 		/** @type string */ method,
 		/** @type string */ uri,
 		/** @type {{[key: string]: any}} */ headers,
-		/** @type any */ body,
+		/** @type any */ body = '',
 	) {
-		body ??= '';
 		if (!handlers[method]) {
 			throw [404, null, null];
 		}
@@ -93,22 +92,22 @@ const app = (function () {
 				json(/** @type any */ obj) {
 					// Stringify and parse to emulate passing over network.
 					obj = JSON.parse(JSON.stringify(obj));
-					if (this.statusCode >= 400) {
-						reject([this.statusCode, obj, null]);
-					} else {
-						resolve([this.statusCode, obj, null]);
-					}
+					this.end(obj);
 				},
 				send(/** @type any */ data) {
 					data = _.cloneDeep(data);
+					this.end(data);
+				},
+				end(/** @type any */ data) {
 					if (this.statusCode >= 400) {
 						reject([this.statusCode, data, null]);
 					} else {
 						resolve([this.statusCode, data, null]);
 					}
 				},
-				sendStatus(/** @type undefined | number */ statusCode) {
-					statusCode ??= this.statusCode;
+				sendStatus(
+					/** @type undefined | number */ statusCode = this.statusCode,
+				) {
 					if (statusCode >= 400) {
 						reject([statusCode, null, null]);
 					} else {
@@ -163,7 +162,7 @@ const app = (function () {
 						checkMethodHandlers();
 					}
 				} else {
-					res.sendStatus(404);
+					res.status(404).end();
 				}
 			};
 			checkMethodHandlers();

package/src/http-transactions/transactions.js

@@ -3,6 +3,7 @@ import { odataNameToSqlName } from '@balena/odata-to-abstract-sql';
 // @ts-ignore
 const transactionModel = require('./transaction.sbvr');
 
+/** @type {import('../config-loader/config-loader').Config} */
 export let config = {
 	models: [
 		{
@@ -24,19 +25,26 @@ ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT N
 ALTER TABLE "transaction"
 ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;\
 `,
-				'15.0.0-true-boolean': {
-					mysql: `\
-ALTER TABLE "lock"
-MODIFY "is exclusive" BOOLEAN NOT NULL,
-MODIFY "is under lock" BOOLEAN NOT NULL;\
-`,
-					postgres: `\
-ALTER TABLE "lock"
-ALTER COLUMN "is exclusive" SET DATA TYPE BOOLEAN USING b::BOOLEAN,
-ALTER COLUMN "is under lock" SET DATA TYPE BOOLEAN USING b::BOOLEAN;\
-`,
-					// No need to migrate for websql
-					websql: '',
+				'15.0.0-data-types': async (tx, sbvrUtils) => {
+					switch (sbvrUtils.db.engine) {
+						case 'mysql':
+							await tx.executeSql(`\
+								ALTER TABLE "lock"
+								MODIFY "is exclusive" BOOLEAN NOT NULL,
+								MODIFY "is under lock" BOOLEAN NOT NULL;`);
+							break;
+						case 'postgres':
+							await tx.executeSql(`\
+								ALTER TABLE "lock"
+								ALTER COLUMN "is exclusive" DROP DEFAULT,
+								ALTER COLUMN "is exclusive" SET DATA TYPE BOOLEAN USING "is exclusive"::BOOLEAN,
+								ALTER COLUMN "is exclusive" SET DEFAULT FALSE,
+								ALTER COLUMN "is under lock" DROP DEFAULT,
+								ALTER COLUMN "is under lock" SET DATA TYPE BOOLEAN USING "is under lock"::BOOLEAN,
+								ALTER COLUMN "is under lock" SET DEFAULT FALSE;`);
+							break;
+						// No need to migrate for websql
+					}
 				},
 			},
 		},
@@ -68,8 +76,8 @@ SELECT NOT EXISTS(
 ) AS result;`,
 						[request.resourceName, id],
 					);
-				} catch (err) {
-					logger.error('Unable to check resource locks', err, err.stack);
+				} catch (/** @type any */ err) {
+					logger.error('Unable to check resource locks', err);
 					throw new Error('Unable to check resource locks');
 				}
 				if ([false, 0, '0'].includes(result.rows[0].result)) {
@@ -151,7 +159,7 @@ WHERE "conditional resource"."transaction" = ?;\
 					[transactionID],
 				);
 
-				conditionalResources.rows.forEach((conditionalResource) => {
+				for (const conditionalResource of conditionalResources.rows) {
 					const { placeholder } = conditionalResource;
 					if (placeholder != null && placeholder.length > 0) {
 						/** @type {Function} */
@@ -165,7 +173,7 @@ WHERE "conditional resource"."transaction" = ?;\
 						// @ts-ignore
 						placeholders[placeholder] = { promise, resolve, reject };
 					}
-				});
+				}
 
 				// get conditional resources (if exist)
 				await Promise.all(
@@ -263,14 +271,14 @@ WHERE "conditional resource"."transaction" = ?;\
 		app.post('/transaction/execute', async (req, res) => {
 			const id = Number(req.body.id);
 			if (Number.isNaN(id)) {
-				res.sendStatus(404);
+				res.status(404).end();
 			} else {
 				try {
 					await endTransaction(id);
 
-					res.sendStatus(200);
-				} catch (err) {
-					console.error('Error ending transaction', err, err.stack);
+					res.status(200).end();
+				} catch (/** @type any */ err) {
+					console.error('Error ending transaction', err);
 					res.status(404).json(err);
 				}
 			}