@balena/pinejs 16.0.0-build--batch-09b8c466600d7df13e6df3eacabaf463d96f652f-1 → 16.0.0-build-fisehara-update-sbvr-types-b58e72aca3193964afac96c955fde178fe39d077-1

package/src/sbvr-api/sbvr-utils.ts

@@ -3,7 +3,9 @@ import type * as Db from '../database-layer/db';
 import type { Model } from '../config-loader/config-loader';
 import type { AnyObject, RequiredField } from './common-types';
 
+// Augment the Express typings
 declare global {
+	// eslint-disable-next-line @typescript-eslint/no-namespace
 	namespace Express {
 		export interface Request {
 			tx?: Db.Tx;
@@ -12,9 +14,9 @@ declare global {
 	}
 }
 
-import * as _ from 'lodash';
+import _ from 'lodash';
 
-import { TypedError } from 'typed-error';
+import type { TypedError } from 'typed-error';
 import { cachedCompile } from './cached-compile';
 
 type LFModel = any[];
@@ -23,14 +25,14 @@ import { version as AbstractSQLCompilerVersion } from '@balena/abstract-sql-comp
 import * as LF2AbstractSQL from '@balena/lf-to-abstract-sql';
 
 import {
-	ODataBinds,
+	type ODataBinds,
 	odataNameToSqlName,
 	sqlNameToODataName,
-	SupportedMethod,
+	type SupportedMethod,
 } from '@balena/odata-to-abstract-sql';
 import sbvrTypes from '@balena/sbvr-types';
 import deepFreeze = require('deep-freeze');
-import { PinejsClientCore, PromiseResultTypes } from 'pinejs-client-core';
+import { PinejsClientCore, type PromiseResultTypes } from 'pinejs-client-core';
 
 import { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser';
 
@@ -38,7 +40,7 @@ import * as asyncMigrator from '../migrator/async';
 import * as syncMigrator from '../migrator/sync';
 import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator';
 
-// tslint:disable-next-line:no-var-requires
+// eslint-disable-next-line @typescript-eslint/no-var-requires
 const devModel = require('./dev.sbvr');
 import * as permissions from './permissions';
 import {
@@ -58,20 +60,20 @@ import {
 	UnauthorizedError,
 } from './errors';
 import * as uriParser from './uri-parser';
-export { ODataRequest } from './uri-parser';
+export type { ODataRequest } from './uri-parser';
 import {
-	HookReq,
-	HookArgs,
+	type HookReq,
+	type HookArgs,
 	rollbackRequestHooks,
 	getHooks,
 	runHooks,
-	InstantiatedHooks,
+	type InstantiatedHooks,
 } from './hooks';
 export {
-	HookReq,
-	HookArgs,
-	HookResponse,
-	Hooks,
+	type HookReq,
+	type HookArgs,
+	type HookResponse,
+	type Hooks,
 	addPureHook,
 	addSideEffectHook,
 } from './hooks';
@@ -94,8 +96,10 @@ export { resolveOdataBind } from './abstract-sql';
 import * as odataResponse from './odata-response';
 import { env } from '../server-glue/module';
 import { translateAbstractSqlModel } from './translations';
-
-const validBatchMethods = new Set(['PUT', 'POST', 'PATCH', 'DELETE', 'GET']);
+import {
+	type MigrationExecutionResult,
+	setExecutedMigrations,
+} from '../migrator/utils';
 
 const LF2AbstractSQLTranslator = LF2AbstractSQL.createTranslator(sbvrTypes);
 const LF2AbstractSQLTranslatorVersion = `${LF2AbstractSQLVersion}+${sbvrTypesVersion}`;
@@ -117,6 +121,7 @@ interface CompiledModel {
 const models: {
 	[vocabulary: string]: CompiledModel & {
 		versions: string[];
+		modelExecutionResult?: ModelExecutionResult;
 	};
 } = {};
 
@@ -135,8 +140,7 @@ export interface ApiKey extends Actor {
 }
 
 export interface Response {
-	id?: string | undefined;
-	status: number;
+	statusCode: number;
 	headers?:
 		| {
 				[headerName: string]: any;
@@ -145,6 +149,12 @@ export interface Response {
 	body?: AnyObject | string;
 }
 
+export type ModelExecutionResult =
+	| undefined
+	| {
+			migrationExecutionResult?: MigrationExecutionResult;
+	  };
+
 const memoizedResolvedSynonym = memoizeWeak(
 	(
 		abstractSqlModel: AbstractSQLCompiler.AbstractSqlModel,
@@ -241,13 +251,14 @@ const prettifyConstraintError = (
 							err.message,
 						);
 					break;
-				case 'postgres':
+				case 'postgres': {
 					const resourceName = resolveSynonym(request);
 					const abstractSqlModel = getFinalAbstractSqlModel(request);
 					matches = new RegExp(
 						'"' + abstractSqlModel.tables[resourceName].name + '_(.*?)_key"',
 					).exec(err.message);
 					break;
+				}
 			}
 			// We know it's the right error type, so if matches exists just throw a generic error message, since we have failed to get the info for a more specific one.
 			if (matches == null) {
@@ -274,7 +285,7 @@ const prettifyConstraintError = (
 							err.message,
 						);
 					break;
-				case 'postgres':
+				case 'postgres': {
 					const resourceName = resolveSynonym(request);
 					const abstractSqlModel = getFinalAbstractSqlModel(request);
 					const tableName = abstractSqlModel.tables[resourceName].name;
@@ -293,6 +304,7 @@ const prettifyConstraintError = (
 						).exec(err.message);
 					}
 					break;
+				}
 			}
 			// We know it's the right error type, so if no matches exists just throw a generic error message,
 			// since we have failed to get the info for a more specific one.
@@ -586,7 +598,7 @@ export const executeModels = async (
 			execModels.map(async (model) => {
 				const { apiRoot } = model;
 
-				await syncMigrator.run(tx, model);
+				const migrationExecutionResult = await syncMigrator.run(tx, model);
 				const compiledModel = generateModels(model, db.engine);
 
 				if (compiledModel.sql) {
@@ -617,6 +629,9 @@ export const executeModels = async (
 				models[apiRoot] = {
 					...compiledModel,
 					versions,
+					modelExecutionResult: {
+						migrationExecutionResult,
+					},
 				};
 
 				// Validate the [empty] model according to the rules.
@@ -723,6 +738,23 @@ export const executeModels = async (
 	}
 };
 
+export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
+	// Executing the `migrations` model takes place after other models have been executed.
+	// Hence, skipped migrations from earlier models are not set as executed as the `migration` table is missing
+	// Here the skipped migrations that haven't been set properly are covered
+	// This is mostly an edge case when running on an empty database schema and migrations model hasn't been executed, yet.
+	// One specifc case are tests to run tests against migrated and unmigrated database states
+
+	for (const modelKey of Object.keys(models)) {
+		const pendingToSetExecutedMigrations =
+			models[modelKey]?.modelExecutionResult?.migrationExecutionResult
+				?.pendingUnsetMigrations;
+		if (pendingToSetExecutedMigrations != null) {
+			await setExecutedMigrations(tx, modelKey, pendingToSetExecutedMigrations);
+		}
+	}
+};
+
 const cleanupModel = (vocab: string) => {
 	delete models[vocab];
 	delete api[vocab];
@@ -955,7 +987,7 @@ export class PinejsClient extends PinejsClientCore<PinejsClient> {
 		req?: permissions.PermissionReq;
 		custom?: AnyObject;
 	}) {
-		return (await runURI(method, url, body, tx, req, custom)) as {};
+		return (await runURI(method, url, body, tx, req, custom)) as object;
 	}
 }
 
@@ -1025,15 +1057,15 @@ export const runURI = async (
 		throw response;
 	}
 
-	const { body: responseBody, status, headers } = response as Response;
+	const { body: responseBody, statusCode, headers } = response as Response;
 
-	if (status != null && status >= 400) {
+	if (statusCode != null && statusCode >= 400) {
 		const ErrorClass =
-			statusCodeToError[status as keyof typeof statusCodeToError];
+			statusCodeToError[statusCode as keyof typeof statusCodeToError];
 		if (ErrorClass != null) {
 			throw new ErrorClass(undefined, responseBody, headers);
 		}
-		throw new HttpError(status, undefined, responseBody, headers);
+		throw new HttpError(statusCode, undefined, responseBody, headers);
 	}
 
 	return responseBody as AnyObject | undefined;
@@ -1072,7 +1104,7 @@ export const getAffectedIds = async (
 	args: HookArgs & {
 		tx: Db.Tx;
 	},
-): Promise<string[]> => {
+): Promise<number[]> => {
 	const { request } = args;
 	if (request.affectedIds) {
 		return request.affectedIds;
@@ -1097,7 +1129,7 @@ const $getAffectedIds = async ({
 	tx,
 }: HookArgs & {
 	tx: Db.Tx;
-}): Promise<string[]> => {
+}): Promise<number[]> => {
 	if (!['PATCH', 'DELETE'].includes(request.method)) {
 		// We can only find the affected ids in advance for requests that modify existing records, if they
 		// can insert new records (POST/PUT) then we're unable to find the ids until the request has actually run
@@ -1110,8 +1142,7 @@ const $getAffectedIds = async ({
 	// And we use the request's url rather than the req for things like batch where the req url is ../$batch
 	const parsedRequest: uriParser.ParsedODataRequest &
 		Partial<Pick<uriParser.ODataRequest, 'engine' | 'translateVersions'>> =
-		await uriParser.parseOData({
-			id: request.id,
+		uriParser.parseOData({
 			method: request.method,
 			url: `/${request.vocabulary}${request.url}`,
 		});
@@ -1157,90 +1188,8 @@ const $getAffectedIds = async ({
 	return result.rows.map((row) => row[idField]);
 };
 
-const validateBatch = (req: Express.Request) => {
-	const { requests } = req.body as { requests: uriParser.UnparsedRequest[] };
-	if (!Array.isArray(requests)) {
-		throw new BadRequestError(
-			'Batch requests must include an array of requests in the body via the "requests" property',
-		);
-	}
-	if (req.headers != null && req.headers['content-type'] == null) {
-		throw new BadRequestError(
-			'Headers in a batch request must include a "content-type" header if they are provided',
-		);
-	}
-	if (
-		requests.find(
-			(request) =>
-				request.headers?.authorization != null ||
-				request.url?.includes('apikey='),
-		) != null
-	) {
-		throw new BadRequestError(
-			'Authorization may only be passed to the main batch request',
-		);
-	}
-	const ids = new Set<string>(
-		requests
-			.map((request) => request.id)
-			.filter((id) => typeof id === 'string') as string[],
-	);
-	if (ids.size !== requests.length) {
-		throw new BadRequestError(
-			'All requests in a batch request must have unique string ids',
-		);
-	}
-
-	for (const request of requests) {
-		if (
-			request.headers != null &&
-			request.headers['content-type'] == null &&
-			(req.headers == null || req.headers['content-type'] == null)
-		) {
-			throw new BadRequestError(
-				'Requests of a batch request that have headers must include a "content-type" header',
-			);
-		}
-		if (request.method == null) {
-			throw new BadRequestError(
-				'Requests of a batch request must have a "method"',
-			);
-		}
-		const upperCaseMethod = request.method.toUpperCase();
-		if (!validBatchMethods.has(upperCaseMethod)) {
-			throw new BadRequestError(
-				`Requests of a batch request must have a method matching one of the following: ${Array.from(
-					validBatchMethods,
-				).join(', ')}`,
-			);
-		}
-		if (
-			request.body !== undefined &&
-			(upperCaseMethod === 'GET' || upperCaseMethod === 'DELETE')
-		) {
-			throw new BadRequestError(
-				'GET and DELETE requests of a batch request must not have a body',
-			);
-		}
-	}
-
-	const urls = new Set<string | undefined>(
-		requests.map((request) => request.url),
-	);
-	if (urls.has(undefined)) {
-		throw new BadRequestError('Requests of a batch request must have a "url"');
-	}
-	if (urls.has('/university/$batch')) {
-		throw new BadRequestError('Batch requests cannot contain batch requests');
-	}
-	const urlModels = new Set(
-		Array.from(urls.values()).map((url: string) => url.split('/')[1]),
-	);
-	if (urlModels.size > 1) {
-		throw new BadRequestError(
-			'Batch requests must consist of requests for only one model',
-		);
-	}
+export const getModel = (vocabulary: string) => {
+	return models[vocabulary];
 };
 
 const runODataRequest = (req: Express.Request, vocabulary: string) => {
@@ -1248,10 +1197,6 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 		api[vocabulary].logger.log('Parsing', req.method, req.url);
 	}
 
-	if (req.url.startsWith(`/${vocabulary}/$batch`)) {
-		validateBatch(req);
-	}
-
 	// Get the hooks for the current method/vocabulary as we know it,
 	// in order to run PREPARSE hooks, before parsing gets us more info
 	const { versions } = models[vocabulary];
@@ -1299,27 +1244,19 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 			await runHooks('PREPARSE', reqHooks, { req, tx: req.tx });
 			let requests: uriParser.UnparsedRequest[];
 			// Check if it is a single request or a batch
-			if (req.url.startsWith(`/${vocabulary}/$batch`)) {
-				await Promise.all(
-					req.body.requests.map(
-						async (request: HookReq) =>
-							await runHooks('PREPARSE', reqHooks, {
-								req: request,
-								tx: req.tx,
-							}),
-					),
-				);
-				requests = req.body.requests;
+			if (req.batch != null && req.batch.length > 0) {
+				requests = req.batch;
 			} else {
 				const { method, url, body } = req;
-				requests = [{ method, url, body }];
+				requests = [{ method, url, data: body }];
 			}
 
 			const prepareRequest = async (
 				parsedRequest: uriParser.ParsedODataRequest &
 					Partial<Pick<uriParser.ODataRequest, 'engine' | 'translateVersions'>>,
 			): Promise<uriParser.ODataRequest> => {
-				if (models[parsedRequest.vocabulary] == null) {
+				const abstractSqlModel = getAbstractSqlModel(parsedRequest);
+				if (abstractSqlModel == null) {
 					throw new BadRequestError(
 						'Unknown vocabulary: ' + parsedRequest.vocabulary,
 					);
@@ -1333,6 +1270,14 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 				>;
 				// Add/check the relevant permissions
 				try {
+					const resolvedResourceName = resolveSynonym($request);
+					if (
+						abstractSqlModel.tables[resolvedResourceName] == null &&
+						!resolvedResourceName.endsWith('#canAccess')
+					) {
+						throw new UnauthorizedError();
+					}
+
 					$request.hooks = [];
 					for (const version of versions) {
 						// We get the hooks list between each `runHooks` so that any resource renames will be used
@@ -1376,13 +1321,7 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 
 			// Parse the OData requests
 			const results = await mappingFn(requests, async (requestPart) => {
-				const parsedRequest = await uriParser.parseOData(
-					requestPart,
-					req.url.startsWith(`/${vocabulary}/$batch`) &&
-						!requestPart.url.includes(`/${vocabulary}/$batch`)
-						? req.headers
-						: undefined,
-				);
+				const parsedRequest = uriParser.parseOData(requestPart);
 
 				let request: uriParser.ODataRequest | uriParser.ODataRequest[];
 				if (Array.isArray(parsedRequest)) {
@@ -1407,7 +1346,7 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 							}
 						});
 						if (Array.isArray(request)) {
-							const changeSetResults = new Map<string, Response>();
+							const changeSetResults = new Map<number, Response>();
 							const changeSetRunner = runChangeSet(req, tx);
 							for (const r of request) {
 								await changeSetRunner(changeSetResults, r);
@@ -1428,7 +1367,7 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 						if (
 							!Array.isArray(result) &&
 							result.body == null &&
-							result.status == null
+							result.statusCode == null
 						) {
 							console.error('No status or body set', req.url, responses);
 							return new InternalRequestError();
@@ -1442,8 +1381,13 @@ const runODataRequest = (req: Express.Request, vocabulary: string) => {
 	};
 };
 
-export const handleODataRequest: Express.Handler = async (req, res, next) => {
+export const getApiRoot = (req: Express.Request): string | undefined => {
 	const [, apiRoot] = req.url.split('/', 2);
+	return apiRoot;
+};
+
+export const handleODataRequest: Express.Handler = async (req, res, next) => {
+	const apiRoot = getApiRoot(req);
 	if (apiRoot == null || models[apiRoot] == null) {
 		return next('route');
 	}
@@ -1457,10 +1401,7 @@ export const handleODataRequest: Express.Handler = async (req, res, next) => {
 
 		res.set('Cache-Control', 'no-cache');
 		// If we are dealing with a single request unpack the response and respond normally
-		if (
-			!req.url.startsWith(`/${apiRoot}/$batch`) ||
-			req.body.requests?.length === 0
-		) {
+		if (req.batch == null || req.batch.length === 0) {
 			let [response] = responses;
 			if (response instanceof HttpError) {
 				response = httpErrorToResponse(response);
@@ -1469,15 +1410,15 @@ export const handleODataRequest: Express.Handler = async (req, res, next) => {
 
 			// Otherwise its a multipart request and we reply with the appropriate multipart response
 		} else {
-			res.status(200).json({
-				responses: responses.map((response) => {
+			(res.status(200) as any).sendMulti(
+				responses.map((response) => {
 					if (response instanceof HttpError) {
 						return httpErrorToResponse(response);
 					} else {
 						return response;
 					}
 				}),
-			});
+			);
 		}
 	} catch (e: any) {
 		if (handleHttpErrors(req, res, e)) {
@@ -1504,16 +1445,16 @@ export const handleHttpErrors = (
 		for (const handleErrorFn of handleErrorFns) {
 			handleErrorFn(req, err);
 		}
-		const response = httpErrorToResponse(err, req);
+		const response = httpErrorToResponse(err);
 		handleResponse(res, response);
 		return true;
 	}
 	return false;
 };
 const handleResponse = (res: Express.Response, response: Response): void => {
-	const { body, headers, status } = response as Response;
+	const { body, headers, statusCode } = response as Response;
 	res.set(headers);
-	res.status(status);
+	res.status(statusCode);
 	if (!body) {
 		res.end();
 	} else {
@@ -1523,12 +1464,10 @@ const handleResponse = (res: Express.Response, response: Response): void => {
 
 const httpErrorToResponse = (
 	err: HttpError,
-	req?: Express.Request,
-): RequiredField<Response, 'status'> => {
-	const message = err.getResponseBody();
+): RequiredField<Response, 'statusCode'> => {
 	return {
-		status: err.status,
-		body: req != null && 'batch' in req ? { responses: [], message } : message,
+		statusCode: err.status,
+		body: err.getResponseBody(),
 		headers: err.headers,
 	};
 };
@@ -1633,7 +1572,7 @@ const runRequest = async (
 const runChangeSet =
 	(req: Express.Request, tx: Db.Tx) =>
 	async (
-		changeSetResults: Map<string, Response>,
+		changeSetResults: Map<number, Response>,
 		request: uriParser.ODataRequest,
 	): Promise<void> => {
 		request = updateBinds(changeSetResults, request);
@@ -1651,7 +1590,7 @@ const runChangeSet =
 // deferred untill the request they reference is run and returns an insert ID.
 // This function compiles the sql query of a request which has been deferred
 const updateBinds = (
-	changeSetResults: Map<string, Response>,
+	changeSetResults: Map<number, Response>,
 	request: uriParser.ODataRequest,
 ) => {
 	if (request._defer) {
@@ -1819,8 +1758,7 @@ const respondGet = async (
 		);
 
 		const response = {
-			id: request.id,
-			status: 200,
+			statusCode: 200,
 			body: { d },
 			headers: { 'content-type': 'application/json' },
 		};
@@ -1835,15 +1773,14 @@ const respondGet = async (
 	} else {
 		if (request.resourceName === '$metadata') {
 			return {
-				id: request.id,
-				status: 200,
+				statusCode: 200,
 				body: models[vocab].odataMetadata,
 				headers: { 'content-type': 'xml' },
 			};
 		} else {
 			// TODO: request.resourceName can be '$serviceroot' or a resource and we should return an odata xml document based on that
 			return {
-				status: 404,
+				statusCode: 404,
 			};
 		}
 	}
@@ -1899,7 +1836,7 @@ const respondPost = async (
 	}
 
 	const response = {
-		status: 201,
+		statusCode: 201,
 		body: result.d[0],
 		headers: {
 			'content-type': 'application/json',
@@ -1947,7 +1884,7 @@ const respondPut = async (
 	tx: Db.Tx,
 ): Promise<Response> => {
 	const response = {
-		status: 200,
+		statusCode: 200,
 	};
 	await runHooks('PRERESPOND', request.hooks, {
 		req,
@@ -2045,3 +1982,18 @@ export const setup = async (
 		// we can't use IF NOT EXISTS on all dbs, so we have to ignore the error raised if this index already exists
 	}
 };
+
+export const postSetup = async (
+	_app: Express.Application,
+	$db: Db.Database,
+): Promise<void> => {
+	exports.db = db = $db;
+	try {
+		await db.transaction(async (tx) => {
+			await postExecuteModels(tx);
+		});
+	} catch (err: any) {
+		console.error('Could not post execute models', err);
+		process.exit(1);
+	}
+};

package/src/sbvr-api/translations.ts

@@ -1,5 +1,5 @@
-import * as _ from 'lodash';
-import {
+import _ from 'lodash';
+import type {
 	AbstractSqlModel,
 	Relationship,
 	ReferencedFieldNode,
@@ -14,7 +14,7 @@ import {
 	UnknownTypeNodes,
 	NullNode,
 } from '@balena/abstract-sql-compiler';
-import { Dictionary } from './common-types';
+import type { Dictionary } from './common-types';
 
 export type AliasValidNodeType =
 	| ReferencedFieldNode
@@ -160,9 +160,8 @@ export const translateAbstractSqlModel = (
 		if (synonym.includes('$')) {
 			fromAbstractSqlModel.synonyms[synonym] = canonicalForm;
 		} else {
-			fromAbstractSqlModel.synonyms[
-				`${synonym}$${toVersion}`
-			] = `${canonicalForm}$${toVersion}`;
+			fromAbstractSqlModel.synonyms[`${synonym}$${toVersion}`] =
+				`${canonicalForm}$${toVersion}`;
 		}
 	}
 	const relationships = _.cloneDeep(toAbstractSqlModel.relationships);
@@ -199,6 +198,10 @@ export const translateAbstractSqlModel = (
 	}
 
 	for (const key of fromResourceKeys) {
+		if (key.includes('$')) {
+			// Skip translated resources, eg `resource$v2`
+			continue;
+		}
 		const translationDefinition = translationDefinitions[key];
 		const table = fromAbstractSqlModel.tables[key];
 		if (translationDefinition) {