@balena/pinejs 17.1.0-build-model-based-typings-437bb06f44567532aec78e550f3d545732466411-1 → 17.1.0-build-joshbwlng-tasks-61ce10e444abec6afea3fec43e9a5c37c7cedea6-1

package/src/sbvr-api/permissions.ts

@@ -1,4 +1,3 @@
-import type AuthModel from './user';
 import type {
 	AbstractSqlModel,
 	AbstractSqlQuery,
@@ -52,7 +51,6 @@ import {
 	type ODataRequest,
 } from './uri-parser';
 import memoizeWeak = require('memoizee/weak');
-import type { Config } from '../config-loader/config-loader';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const userModel: string = require('./user.sbvr');
@@ -1120,7 +1118,7 @@ const memoizedGetConstrainedModel = (
 	getBoundConstrainedMemoizer(abstractSqlModel)(permissionsLookup, vocabulary);
 
 const getCheckPasswordQuery = _.once(() =>
-	sbvrUtils.api.Auth.prepare<{ username: string }, 'user'>({
+	sbvrUtils.api.Auth.prepare<{ username: string }>({
 		resource: 'user',
 		passthrough: {
 			req: rootRead,
@@ -1166,7 +1164,7 @@ export const checkPassword = async (
 
 const $getUserPermissions = (() => {
 	const getUserPermissionsQuery = _.once(() =>
-		sbvrUtils.api.Auth.prepare<{ userId: number }, 'permission'>({
+		sbvrUtils.api.Auth.prepare<{ userId: number }>({
 			resource: 'permission',
 			passthrough: {
 				req: rootRead,
@@ -1277,7 +1275,7 @@ export const getUserPermissions = async (
 
 const $getApiKeyPermissions = (() => {
 	const getApiKeyPermissionsQuery = _.once(() =>
-		sbvrUtils.api.Auth.prepare<{ apiKey: string }, 'permission'>({
+		sbvrUtils.api.Auth.prepare<{ apiKey: string }>({
 			resource: 'permission',
 			passthrough: {
 				req: rootRead,
@@ -1405,7 +1403,7 @@ export const getApiKeyPermissions = async (
 
 const getApiKeyActorId = (() => {
 	const getApiKeyActorIdQuery = _.once(() =>
-		sbvrUtils.api.Auth.prepare<{ apiKey: string }, 'api_key'>({
+		sbvrUtils.api.Auth.prepare<{ apiKey: string }>({
 			resource: 'api_key',
 			passthrough: {
 				req: rootRead,
@@ -1588,7 +1586,7 @@ let guestPermissionsInitialized = false;
 const getGuestPermissions = memoize(
 	async () => {
 		// Get guest user
-		const result = await sbvrUtils.api.Auth.get({
+		const result = (await sbvrUtils.api.Auth.get({
 			resource: 'user',
 			passthrough: {
 				req: rootRead,
@@ -1599,7 +1597,7 @@ const getGuestPermissions = memoize(
 			options: {
 				$select: 'id',
 			},
-		});
+		})) as { id: number } | undefined;
 		if (result == null) {
 			throw new Error('No guest user');
 		}
@@ -1688,53 +1686,49 @@ export const addPermissions = async (
 	}
 };
 
-declare module './sbvr-utils' {
-	export interface API {
-		[authModelConfig.apiRoot]: PinejsClient<AuthModel>;
-	}
-}
-const authModelConfig = {
-	apiRoot: 'Auth',
-	modelText: userModel,
-	customServerCode: exports,
-	migrations: {
-		'11.0.0-modified-at': `
-			ALTER TABLE "actor"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-
-			ALTER TABLE "api key"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-			ALTER TABLE "api key-has-permission"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-			ALTER TABLE "api key-has-role"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-
-			ALTER TABLE "permission"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-
-			ALTER TABLE "role"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-
-			ALTER TABLE "user"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-			ALTER TABLE "user-has-role"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-			ALTER TABLE "user-has-permission"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-		`,
-		'11.0.1-modified-at': `
-			ALTER TABLE "role-has-permission"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-		`,
-		'14.42.0-api-key-expiry-date': `
-			ALTER TABLE "api key"
-			ADD COLUMN IF NOT EXISTS "expiry date" TIMESTAMP NULL;
-		`,
-	},
-} as const satisfies sbvrUtils.ExecutableModel;
 export const config = {
-	models: [authModelConfig],
-} satisfies Config;
+	models: [
+		{
+			apiRoot: 'Auth',
+			modelText: userModel,
+			customServerCode: exports,
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "actor"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+
+					ALTER TABLE "api key"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+					ALTER TABLE "api key-has-permission"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+					ALTER TABLE "api key-has-role"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+
+					ALTER TABLE "permission"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+
+					ALTER TABLE "role"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+
+					ALTER TABLE "user"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+					ALTER TABLE "user-has-role"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+					ALTER TABLE "user-has-permission"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'11.0.1-modified-at': `
+					ALTER TABLE "role-has-permission"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'14.42.0-api-key-expiry-date': `
+					ALTER TABLE "api key"
+					ADD COLUMN IF NOT EXISTS "expiry date" TIMESTAMP NULL;
+				`,
+			},
+		},
+	] as sbvrUtils.ExecutableModel[],
+};
 export const setup = () => {
 	addHook('all', 'all', 'all', {
 		sideEffects: false,

package/src/sbvr-api/sbvr-utils.ts

@@ -2,10 +2,6 @@ import type * as Express from 'express';
 import type * as Db from '../database-layer/db';
 import type { Model } from '../config-loader/config-loader';
 import type { AnyObject, RequiredField } from './common-types';
-import type {
-	PickDeferred,
-	Resource,
-} from '@balena/abstract-sql-to-typescript';
 
 // Augment the Express typings
 declare global {
@@ -36,7 +32,6 @@ import {
 } from '@balena/odata-to-abstract-sql';
 import sbvrTypes from '@balena/sbvr-types';
 import deepFreeze = require('deep-freeze');
-import type { AnyResource, Params } from 'pinejs-client-core';
 import { PinejsClientCore, type PromiseResultTypes } from 'pinejs-client-core';
 
 import { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser';
@@ -45,9 +40,9 @@ import * as asyncMigrator from '../migrator/async';
 import * as syncMigrator from '../migrator/sync';
 import { generateODataMetadata } from '../odata-metadata/odata-metadata-generator';
 
-import type DevModel from './dev';
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const devModel = require('./dev.sbvr');
+import * as tasks from '../tasks';
 import * as permissions from './permissions';
 import {
 	BadRequestError,
@@ -83,6 +78,7 @@ export {
 	addPureHook,
 	addSideEffectHook,
 } from './hooks';
+export { addTaskHandler } from '../tasks';
 
 import memoizeWeak = require('memoizee/weak');
 import * as controlFlow from './control-flow';
@@ -716,7 +712,7 @@ export const executeModels = async (
 							},
 						});
 					}
-					const result = await api.dev.get({
+					const result = (await api.dev.get({
 						resource: 'model',
 						passthrough: {
 							tx,
@@ -729,7 +725,7 @@ export const executeModels = async (
 								model_type: modelType,
 							},
 						},
-					});
+					})) as Array<{ id: number }>;
 
 					let method: SupportedMethod = 'POST';
 					let uri = '/dev/model';
@@ -779,7 +775,7 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
 	// Hence, skipped migrations from earlier models are not set as executed as the `migration` table is missing
 	// Here the skipped migrations that haven't been set properly are covered
 	// This is mostly an edge case when running on an empty database schema and migrations model hasn't been executed, yet.
-	// One specifc case are tests to run tests against migrated and unmigrated database states
+	// One specific case are tests to run tests against migrated and unmigrated database states
 
 	for (const modelKey of Object.keys(models)) {
 		const pendingToSetExecutedMigrations =
@@ -789,6 +785,9 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
 			await setExecutedMigrations(tx, modelKey, pendingToSetExecutedMigrations);
 		}
 	}
+
+	// Initialize task worker and create required hooks
+	await tasks.setup(db);
 };
 
 const cleanupModel = (vocab: string) => {
@@ -1008,16 +1007,7 @@ export type Passthrough = AnyObject & {
 	tx?: Db.Tx;
 };
 
-export class PinejsClient<
-	M extends {
-		[key in keyof M]: Resource;
-	} = {
-		[key in string]: {
-			Read: AnyObject;
-			Write: AnyObject;
-		};
-	},
-> extends PinejsClientCore<unknown, M> {
+export class PinejsClient extends PinejsClientCore {
 	public async _request({
 		method,
 		url,
@@ -1035,37 +1025,11 @@ export class PinejsClient<
 	}) {
 		return (await runURI(method, url, body, tx, req, custom)) as object;
 	}
-
-	public post<TResource extends keyof M & string>(
-		params: {
-			resource: TResource;
-			options?: Params<M[TResource]>['options'] & { returnResource?: true };
-		} & Params<M[TResource]>,
-	): Promise<PickDeferred<M[TResource]['Read']>>;
-	public post<TResource extends keyof M & string>(
-		params: {
-			resource: TResource;
-			options: Params<M[TResource]>['options'] & { returnResource: boolean };
-		} & Params<M[TResource]>,
-	): Promise<Pick<M[TResource]['Read'], 'id'>>; // TODO: This should use the primary key rather than hardcoding `id`
-	/**
-	 * @deprecated POSTing via `url` is deprecated
-	 */
-	public post<T extends Resource = AnyResource>(
-		params: {
-			resource?: undefined;
-			url: NonNullable<Params<T>['url']>;
-		} & Params<T>,
-	): Promise<AnyObject>;
-	public post(params: Params<AnyResource>): Promise<AnyObject> {
-		return super.post(params as Parameters<PinejsClient['post']>[0]);
-	}
 }
 
-export interface API {
+export const api: {
 	[vocab: string]: PinejsClient;
-}
-export const api = {} as API;
+} = {};
 export const logger: {
 	[vocab: string]: Console;
 } = {};
@@ -1172,8 +1136,8 @@ const getIdField = (
 	// TODO: Should resolveSynonym also be using the finalAbstractSqlModel?
 	getFinalAbstractSqlModel(request).tables[resolveSynonym(request)].idField;
 
-export const getAffectedIds = async <Vocab extends string>(
-	args: HookArgs<Vocab> & {
+export const getAffectedIds = async (
+	args: HookArgs & {
 		tx: Db.Tx;
 	},
 ): Promise<number[]> => {
@@ -1195,11 +1159,11 @@ export const getAffectedIds = async <Vocab extends string>(
 	return request.affectedIds;
 };
 
-const $getAffectedIds = async <Vocab extends string>({
+const $getAffectedIds = async ({
 	req,
 	request,
 	tx,
-}: HookArgs<Vocab> & {
+}: HookArgs & {
 	tx: Db.Tx;
 }): Promise<number[]> => {
 	if (!['PATCH', 'DELETE'].includes(request.method)) {
@@ -1582,7 +1546,8 @@ const runRequest = async (
 	if (env.DEBUG) {
 		log.log('Running', req.method, req.url);
 	}
-	let result: Db.Result | number | undefined;
+	let resultGet: Db.Result | undefined;
+	let resultPost: number | undefined;
 
 	try {
 		try {
@@ -1591,18 +1556,18 @@ const runRequest = async (
 
 			switch (request.method) {
 				case 'GET':
-					result = await runGet(req, request, tx);
+					resultGet = await runGet(req, request, tx);
 					break;
 				case 'POST':
-					result = await runPost(req, request, tx);
+					resultPost = await runPost(req, request, tx);
 					break;
 				case 'PUT':
 				case 'PATCH':
 				case 'MERGE':
-					result = await runPut(req, request, tx);
+					await runPut(req, request, tx);
 					break;
 				case 'DELETE':
-					result = await runDelete(req, request, tx);
+					await runDelete(req, request, tx);
 					break;
 			}
 		} catch (err: any) {
@@ -1628,7 +1593,12 @@ const runRequest = async (
 			throw err;
 		}
 
-		await runHooks('POSTRUN', request.hooks, { req, request, result, tx });
+		await runHooks('POSTRUN', request.hooks, {
+			req,
+			request,
+			result: resultGet ?? resultPost,
+			tx,
+		});
 	} catch (err: any) {
 		await runHooks('POSTRUN-ERROR', request.hooks, {
 			req,
@@ -1638,7 +1608,23 @@ const runRequest = async (
 		});
 		throw err;
 	}
-	return await prepareResponse(req, request, result, tx);
+
+	switch (request.method) {
+		case 'GET':
+			return await respondGet(req, request, resultGet, tx);
+		case 'POST':
+			return await respondPost(req, request, resultPost, tx);
+		case 'PUT':
+		case 'PATCH':
+		case 'MERGE':
+			return await respondPut(req, request, tx);
+		case 'DELETE':
+			return await respondDelete(req, request, tx);
+		case 'OPTIONS':
+			return await respondOptions(req, request, tx);
+		default:
+			throw new MethodNotAllowedError();
+	}
 };
 
 const runChangeSet =
@@ -1686,30 +1672,6 @@ const updateBinds = (
 	return request;
 };
 
-const prepareResponse = async (
-	req: Express.Request,
-	request: uriParser.ODataRequest,
-	result: any,
-	tx: Db.Tx,
-): Promise<Response> => {
-	switch (request.method) {
-		case 'GET':
-			return await respondGet(req, request, result, tx);
-		case 'POST':
-			return await respondPost(req, request, result, tx);
-		case 'PUT':
-		case 'PATCH':
-		case 'MERGE':
-			return await respondPut(req, request, result, tx);
-		case 'DELETE':
-			return await respondDelete(req, request, result, tx);
-		case 'OPTIONS':
-			return await respondOptions(req, request, result, tx);
-		default:
-			throw new MethodNotAllowedError();
-	}
-};
-
 const checkReadOnlyRequests = (request: uriParser.ODataRequest) => {
 	if (request.method !== 'GET') {
 		// Only GET requests can be read-only
@@ -1811,11 +1773,17 @@ const runGet = async (
 const respondGet = async (
 	req: Express.Request,
 	request: uriParser.ODataRequest,
-	result: any,
+	result: Db.Result | undefined,
 	tx: Db.Tx,
 ): Promise<Response> => {
 	const vocab = request.vocabulary;
 	if (request.sqlQuery != null) {
+		if (result == null) {
+			// This shouldn't be able to happen because the result should only be null if there's no sqlQuery
+			throw new Error(
+				'Null result passed to respond GET that has a sqlQuery defined',
+			);
+		}
 		const format = request.odataQuery.options?.$format;
 		const metadata =
 			format != null && typeof format === 'object'
@@ -1880,7 +1848,7 @@ const runPost = async (
 const respondPost = async (
 	req: Express.Request,
 	request: uriParser.ODataRequest,
-	id: number,
+	id: number | undefined,
 	tx: Db.Tx,
 ): Promise<Response> => {
 	const vocab = request.vocabulary;
@@ -1930,7 +1898,7 @@ const runPut = async (
 	_req: Express.Request,
 	request: uriParser.ODataRequest,
 	tx: Db.Tx,
-): Promise<undefined> => {
+): Promise<void> => {
 	let rowsAffected: number;
 	// If request.sqlQuery is an array it means it's an UPSERT, ie two queries: [InsertQuery, UpdateQuery]
 	if (Array.isArray(request.sqlQuery)) {
@@ -1946,13 +1914,11 @@ const runPut = async (
 	if (rowsAffected > 0) {
 		await validateModel(tx, _.last(request.translateVersions)!, request);
 	}
-	return undefined;
 };
 
 const respondPut = async (
 	req: Express.Request,
 	request: uriParser.ODataRequest,
-	result: any,
 	tx: Db.Tx,
 ): Promise<Response> => {
 	const response = {
@@ -1961,7 +1927,6 @@ const respondPut = async (
 	await runHooks('PRERESPOND', request.hooks, {
 		req,
 		request,
-		result,
 		response,
 		tx,
 	});
@@ -1974,60 +1939,54 @@ const runDelete = async (
 	_req: Express.Request,
 	request: uriParser.ODataRequest,
 	tx: Db.Tx,
-): Promise<undefined> => {
+): Promise<void> => {
 	const { rowsAffected } = await runQuery(tx, request, undefined, true);
 	if (rowsAffected > 0) {
 		await validateModel(tx, _.last(request.translateVersions)!, request);
 	}
-
-	return undefined;
 };
 
-export interface API {
-	[devModelConfig.apiRoot]: PinejsClient<DevModel>;
-}
-const devModelConfig = {
-	apiRoot: 'dev',
-	modelText: devModel,
-	logging: {
-		log: false,
-	},
-	migrations: {
-		'11.0.0-modified-at': `
-			ALTER TABLE "model"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-		`,
-		'15.0.0-data-types': async ($tx, sbvrUtils) => {
-			switch (sbvrUtils.db.engine) {
-				case 'mysql':
-					await $tx.executeSql(`\
-						ALTER TABLE "model"
-						MODIFY "model value" JSON NOT NULL;
-
-						UPDATE "model"
-						SET "model value" = CAST('{"value":' || CAST("model value" AS CHAR) || '}' AS JSON)
-						WHERE "model type" IN ('se', 'odataMetadata')
-						AND CAST("model value" AS CHAR) LIKE '"%';`);
-					break;
-				case 'postgres':
-					await $tx.executeSql(`\
-						ALTER TABLE "model"
-						ALTER COLUMN "model value" SET DATA TYPE JSONB USING "model value"::JSONB;
-
-						UPDATE "model"
-						SET "model value" = CAST('{"value":' || CAST("model value" AS TEXT) || '}' AS JSON)
-						WHERE "model type" IN ('se', 'odataMetadata')
-						AND CAST("model value" AS TEXT) LIKE '"%';`);
-					break;
-				// No need to migrate for websql
-			}
-		},
-	},
-} as const satisfies ExecutableModel;
 export const executeStandardModels = async (tx: Db.Tx): Promise<void> => {
 	try {
 		// dev model must run first
-		await executeModel(tx, devModelConfig);
+		await executeModel(tx, {
+			apiRoot: 'dev',
+			modelText: devModel,
+			logging: {
+				log: false,
+			},
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "model"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'15.0.0-data-types': async ($tx, sbvrUtils) => {
+					switch (sbvrUtils.db.engine) {
+						case 'mysql':
+							await $tx.executeSql(`\
+								ALTER TABLE "model"
+								MODIFY "model value" JSON NOT NULL;
+
+								UPDATE "model"
+								SET "model value" = CAST('{"value":' || CAST("model value" AS CHAR) || '}' AS JSON)
+								WHERE "model type" IN ('se', 'odataMetadata')
+								AND CAST("model value" AS CHAR) LIKE '"%';`);
+							break;
+						case 'postgres':
+							await $tx.executeSql(`\
+								ALTER TABLE "model"
+								ALTER COLUMN "model value" SET DATA TYPE JSONB USING "model value"::JSONB;
+
+								UPDATE "model"
+								SET "model value" = CAST('{"value":' || CAST("model value" AS TEXT) || '}' AS JSON)
+								WHERE "model type" IN ('se', 'odataMetadata')
+								AND CAST("model value" AS TEXT) LIKE '"%';`);
+							break;
+						// No need to migrate for websql
+					}
+				},
+			},
+		});
 		await executeModels(tx, permissions.config.models);
 		console.info('Successfully executed standard models.');
 	} catch (err: any) {

package/src/server-glue/module.ts

@@ -6,6 +6,7 @@ import * as dbModule from '../database-layer/db';
 import * as configLoader from '../config-loader/config-loader';
 import * as migrator from '../migrator/sync';
 import type * as migratorUtils from '../migrator/utils';
+import * as tasks from '../tasks';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 import { PINEJS_ADVISORY_LOCK } from '../config-loader/env';
@@ -19,6 +20,7 @@ export * as errors from '../sbvr-api/errors';
 export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
+export * as tasks from '../tasks';
 export * as webResourceHandler from '../webresource-handler';
 export type { configLoader as ConfigLoader };
 export type { migratorUtils as Migrator };
@@ -63,6 +65,7 @@ export const init = async <T extends string>(
 		await sbvrUtils.setup(app, db);
 		const cfgLoader = await configLoader.setup(app);
 		await cfgLoader.loadConfig(migrator.config);
+		await cfgLoader.loadConfig(tasks.config);
 
 		const promises: Array<Promise<void>> = [];
 		if (process.env.SBVR_SERVER_ENABLED) {

package/src/tasks/common.ts

@@ -0,0 +1,14 @@
+import Ajv from 'ajv';
+
+// Root path for the tasks API
+export const apiRoot = 'tasks';
+
+// Channel name for task insert notifications
+export const channel = 'pinejs$task_insert';
+
+// Setting inlineRefs=false as without it we run into a
+// "Maximum call stack size exceeded" error apprarently caused
+// by String.prototype._uncountable_words being set in sbvr-parser?
+export const ajv = new Ajv({
+	inlineRefs: false,
+});