@balena/pinejs 20.1.0-build-large-file-uploads-2-1a7d50e8660ff52fe7df8d6ef9ce8b6b707027dc-1 → 21.0.0-build-21-x-aded34a9c5c670522247399b3cc4dc7814430d5e-1

package/src/sbvr-api/hooks.ts

@@ -2,7 +2,6 @@ import type { OptionalField, Resolvable } from './common-types.js';
 import type { Result, Tx } from '../database-layer/db.js';
 import type { ODataRequest, ParsedODataRequest } from './uri-parser.js';
 import type { AnyObject } from 'pinejs-client-core';
-import type { TypedError } from 'typed-error';
 import type { SupportedMethod } from '@balena/odata-to-abstract-sql';
 
 import _ from 'lodash';
@@ -36,6 +35,7 @@ export interface HookArgs<Vocab extends string = string> {
 	api: (typeof api)[Vocab];
 	tx?: Tx | undefined;
 }
+// eslint-disable-next-line @typescript-eslint/no-invalid-void-type -- This makes the type far easier to use when defining functions that typescript defaults as void rather than undefined
 export type HookResponse = PromiseLike<any> | null | void;
 
 export interface Hooks<Vocab extends string = string> {
@@ -59,7 +59,7 @@ export interface Hooks<Vocab extends string = string> {
 	) => HookResponse;
 	/** These are run in reverse translation order from newest to oldest */
 	'POSTRUN-ERROR'?: (
-		options: HookArgs<Vocab> & { tx: Tx; error: TypedError | any },
+		options: HookArgs<Vocab> & { tx: Tx; error: unknown },
 	) => HookResponse;
 }
 export type HookBlueprints = {

package/src/sbvr-api/sbvr-utils.ts

@@ -471,7 +471,7 @@ const bindsForAffectedIds = (
 	return odataBinds;
 };
 
-export const validateModel = async (
+const validateModel = async (
 	tx: Db.Tx,
 	modelName: string,
 	request?: Pick<
@@ -770,6 +770,13 @@ export const executeModels = async (
 				await asyncMigrator.run(tx, model);
 			}),
 		);
+		// Allow the migrations to be GCed after we are done
+		// executing the models & their migrations.
+		for (const model of execModels) {
+			if (model.migrations != null) {
+				delete model.migrations;
+			}
+		}
 	} catch (err) {
 		for (const { apiRoot } of execModels) {
 			cleanupModel(apiRoot);
@@ -800,62 +807,6 @@ const cleanupModel = (vocab: string) => {
 	delete api[vocab];
 };
 
-export const deleteModel = async (vocabulary: string) => {
-	const { sql } = models[vocabulary];
-	if (sql) {
-		await db.transaction(async (tx) => {
-			const dropStatements: Array<Promise<any>> = sql.dropSchema.map(
-				(dropStatement) => tx.executeSql(dropStatement),
-			);
-			await Promise.all(
-				dropStatements.concat([
-					api.dev.delete({
-						resource: 'model',
-						passthrough: {
-							tx,
-							req: permissions.root,
-						},
-						options: {
-							$filter: {
-								is_of__vocabulary: vocabulary,
-							},
-						},
-					}),
-				]),
-			);
-		});
-	}
-	cleanupModel(vocabulary);
-};
-
-const isWhereNode = (
-	x: AbstractSQLCompiler.AbstractSqlType,
-): x is AbstractSQLCompiler.WhereNode => x[0] === 'Where';
-const isEqualsNode = (
-	x: AbstractSQLCompiler.AbstractSqlType,
-): x is AbstractSQLCompiler.EqualsNode => x[0] === 'Equals';
-export const getID = (vocab: string, request: uriParser.ODataRequest) => {
-	if (request.abstractSqlQuery == null) {
-		throw new Error('Can only get the id if an abstractSqlQuery is provided');
-	}
-	const { idField } = models[vocab].abstractSql.tables[request.resourceName];
-	for (const whereClause of request.abstractSqlQuery) {
-		if (isWhereNode(whereClause)) {
-			for (const comparison of whereClause.slice(1)) {
-				if (isEqualsNode(comparison)) {
-					if (comparison[1][2] === idField) {
-						return comparison[2][1];
-					}
-					if (comparison[2][2] === idField) {
-						return comparison[1][1];
-					}
-				}
-			}
-		}
-	}
-	return 0;
-};
-
 export const runRule = (() => {
 	const LF2AbstractSQLPrepHack = LF2AbstractSQL.LF2AbstractSQLPrep._extend({
 		CardinalityOptimisation() {
@@ -2064,7 +2015,7 @@ const devModelConfig = {
 		},
 	},
 } as const satisfies ExecutableModel;
-export const executeStandardModels = async (tx: Db.Tx): Promise<void> => {
+const executeStandardModels = async (tx: Db.Tx): Promise<void> => {
 	try {
 		// dev model must run first
 		await executeModel(tx, devModelConfig);

package/src/sbvr-api/translations.ts

@@ -206,30 +206,17 @@ export const translateAbstractSqlModel = (
 			const { $toResource, ...definition } = translationDefinition;
 			const hasToResource = typeof $toResource === 'string';
 
-			const unaliasedToResource = hasToResource
-				? $toResource.endsWith(toVersionSuffix)
-					? // Ideally we want to rename to the unaliased resource of the next version
-						// so when the alias matches the next version we can just strip it
-						$toResource.slice(0, -toVersionSuffix.length)
-					: // But if we can't safely strip the suffix then we'll use as-is, at least until the next major
-						$toResource
-				: key;
-			let aliasedToResource;
+			const unaliasedToResource = hasToResource ? $toResource : key;
 			if (hasToResource) {
-				resourceRenames[key] = unaliasedToResource;
 				if ($toResource.includes('$')) {
-					// TODO-MAJOR: Change this to an error
-					console.warn(
+					throw new Error(
 						`'$toResource' should be the unaliased name of the resource in the subsequent model and not be targeting a specific model, got '${$toResource}'`,
-						$toResource,
 					);
-					aliasedToResource = $toResource;
-				} else {
-					aliasedToResource = `${$toResource}${toVersionSuffix}`;
 				}
-			} else {
-				aliasedToResource = `${key}${toVersionSuffix}`;
+				resourceRenames[key] = unaliasedToResource;
 			}
+			const aliasedToResource = `${unaliasedToResource}${toVersionSuffix}`;
+
 			const toTable = toAbstractSqlModel.tables[unaliasedToResource];
 			if (!toTable) {
 				if (hasToResource) {

package/src/sbvr-api/uri-parser.ts

@@ -16,7 +16,6 @@ import _ from 'lodash';
 import memoizeWeak from 'memoizee/weak.js';
 
 export { BadRequestError, ParsingError, TranslationError } from './errors.js';
-import deepFreeze from 'deep-freeze';
 import * as env from '../config-loader/env.js';
 import {
 	BadRequestError,
@@ -53,11 +52,19 @@ export interface ParsedODataRequest {
 	id?: number | undefined;
 	_defer?: boolean;
 }
+export interface CachedSqlQuery extends Array<never> {
+	length: 0;
+	engine: AbstractSQLCompiler.Engines;
+	sqlQuery: AbstractSQLCompiler.SqlResult | AbstractSQLCompiler.SqlResult[];
+	modifiedFields?: ReturnType<
+		AbstractSQLCompiler.EngineInstance['getModifiedFields']
+	>;
+}
 export interface ODataRequest extends ParsedODataRequest {
 	translateVersions: string[];
 	abstractSqlModel?: AbstractSQLCompiler.AbstractSqlModel;
 	finalAbstractSqlModel?: AbstractSQLCompiler.AbstractSqlModel;
-	abstractSqlQuery?: AbstractSQLCompiler.AbstractSqlQuery;
+	abstractSqlQuery?: AbstractSQLCompiler.AbstractSqlQuery | CachedSqlQuery;
 	sqlQuery?: AbstractSQLCompiler.SqlResult | AbstractSQLCompiler.SqlResult[];
 	tx?: Tx;
 	modifiedFields?: ReturnType<
@@ -178,8 +185,8 @@ const memoizedOdata2AbstractSQL = (() => {
 					bodyKeys,
 					existingBindVarsLength,
 				);
-				// We deep freeze to prevent mutations, which would pollute the cache
-				deepFreeze(abstractSql);
+				// We do not deep freeze as the sql compilation intentionally mutates this object
+				// to clear the abstractSql tree and replace it with the compile sqlQuery to save memory
 				return abstractSql;
 			} catch (e) {
 				if (e instanceof PermissionError) {

package/src/server-glue/module.ts

@@ -7,7 +7,6 @@ import * as configLoader from '../config-loader/config-loader.js';
 import * as migrator from '../migrator/sync.js';
 import type * as migratorUtils from '../migrator/utils.js';
 import * as tasks from '../tasks/index.js';
-import * as webresource from '../webresource-handler/index.js';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils.js';
 import { PINEJS_ADVISORY_LOCK } from '../config-loader/env.js';
@@ -67,23 +66,10 @@ export const init = async <T extends string>(
 		const cfgLoader = configLoader.setup(app);
 		await cfgLoader.loadConfig(migrator.config);
 		await cfgLoader.loadConfig(tasks.config);
-		await cfgLoader.loadConfig(webresource.config);
 
-		const promises: Array<Promise<void>> = [];
-		if (process.env.SBVR_SERVER_ENABLED) {
-			const sbvrServer = await import('../data-server/sbvr-server.js');
-			const transactions = await import('../http-transactions/transactions.js');
-			promises.push(cfgLoader.loadConfig(sbvrServer.config));
-			promises.push(
-				cfgLoader.loadConfig(transactions.config).then(() => {
-					transactions.addModelHooks('data');
-				}),
-			);
-		}
 		if (!process.env.CONFIG_LOADER_DISABLED) {
-			promises.push(cfgLoader.loadApplicationConfig(config));
+			await cfgLoader.loadApplicationConfig(config);
 		}
-		await Promise.all(promises);
 		// Execute it after all other promises have resolved. Execution of promises is not neccessarily
 		// guaranteed to be sequentially resolving them with Promise.all
 		await sbvrUtils.postSetup(app, db);

package/src/tasks/index.ts

@@ -44,7 +44,7 @@ export const config: ConfigLoader.Config = {
 };
 
 export let worker: Worker | null = null;
-export async function setup(): Promise<void> {
+export function setup(): void {
 	// Async task functionality is only supported on Postgres
 	if (sbvrUtils.db.engine !== 'postgres') {
 		console.warn('Skipping task setup as database not supported');
@@ -56,7 +56,7 @@ export async function setup(): Promise<void> {
 
 	// Add resource hooks
 	addPureHook('POST', apiRoot, 'task', {
-		POSTPARSE: async ({ req, request }) => {
+		POSTPARSE: ({ req, request }) => {
 			// Set the actor
 			request.values.is_created_by__actor =
 				req.user?.actor ?? req.apiKey?.actor;

package/src/tasks/worker.ts

@@ -9,13 +9,14 @@ import { sbvrUtils } from '../server-glue/module.js';
 import { ajv } from './common.js';
 import type { Task } from './tasks.js';
 import type TasksModel from './tasks.js';
+import type { Resolvable } from '../sbvr-api/common-types.js';
 
 interface TaskArgs<T = AnyObject> {
 	api: PinejsClient;
 	params: T;
 }
 
-type TaskResponse = Promise<{
+type TaskResponse = Resolvable<{
 	status: Task['Read']['status'];
 	error?: string;
 }>;
@@ -246,6 +247,9 @@ export class Worker {
 				});
 			} catch (err) {
 				console.error('Failed polling for tasks:', err);
+				// Add some delay after an error to avoid retrying at max speed
+				// TODO: This should be configurable/adaptive
+				await setTimeout(100);
 			} finally {
 				if (!executed) {
 					await setTimeout(this.interval);
@@ -257,7 +261,7 @@ export class Worker {
 		})();
 	}
 
-	public async stop(): Promise<void> {
+	public stop(): void {
 		this.running = false;
 	}
 

package/src/webresource-handler/handlers/NoopHandler.ts

@@ -6,6 +6,7 @@ import type {
 } from '../index.js';
 
 export class NoopHandler implements WebResourceHandler {
+	// eslint-disable-next-line @typescript-eslint/require-await -- We need to return a promise for compatibility reasons.
 	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
 		// handleFile must consume the file stream
 		resource.stream.resume();
@@ -16,9 +17,10 @@ export class NoopHandler implements WebResourceHandler {
 	}
 
 	public async removeFile(): Promise<void> {
-		return;
+		// noop
 	}
 
+	// eslint-disable-next-line @typescript-eslint/require-await -- We need to return a promise for compatibility reasons.
 	public async onPreRespond(webResource: WebResource): Promise<WebResource> {
 		return webResource;
 	}

package/src/webresource-handler/index.ts

@@ -11,20 +11,12 @@ import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
-import type { ConfigLoader } from '../server-glue/module.js';
 import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
-import type WebresourceModel from './webresource.js';
-import { importSBVR } from '../server-glue/sbvr-loader.js';
-import {
-	isMultipartUploadAvailable,
-	multipartUploadHooks,
-} from './multipartUpload.js';
 
 export * from './handlers/index.js';
-export type { BeginUploadResponse } from './multipartUpload.js';
 
 export interface IncomingFile {
 	fieldname: string;
@@ -39,51 +31,10 @@ export interface UploadResponse {
 	filename: string;
 }
 
-export interface BeginMultipartUploadPayload {
-	filename: string;
-	content_type: string;
-	size: number;
-	chunk_size: number;
-}
-
-export interface UploadPart {
-	url: string;
-	chunkSize: number;
-	partNumber: number;
-}
-
-export interface BeginMultipartUploadHandlerResponse {
-	uploadParts: UploadPart[];
-	fileKey: string;
-	uploadId: string;
-}
-
-export interface CommitMultipartUploadPayload {
-	fileKey: string;
-	uploadId: string;
-	filename: string;
-	providerCommitData?: Record<string, any>;
-}
-
-export interface CancelMultipartUploadPayload {
-	fileKey: string;
-	uploadId: string;
-}
-
 export interface WebResourceHandler {
 	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
 	removeFile: (fileReference: string) => Promise<void>;
 	onPreRespond: (webResource: WebResource) => Promise<WebResource>;
-	multipartUpload?: {
-		begin: (
-			fieldName: string,
-			payload: BeginMultipartUploadPayload,
-		) => Promise<BeginMultipartUploadHandlerResponse>;
-		commit: (commitInfo: CommitMultipartUploadPayload) => Promise<WebResource>;
-		cancel: (cancelInfo: CancelMultipartUploadPayload) => Promise<void>;
-		getMinimumPartSize: () => number;
-		getDefaultPartSize: () => number;
-	};
 }
 
 export class WebResourceError extends TypedError {}
@@ -151,7 +102,7 @@ const getRequestUploadValidator = async (
 		return notValidUpload;
 	}
 
-	return async (fieldname: string) => {
+	return (fieldname: string) => {
 		const dbFieldName = odataNameToSqlName(fieldname);
 		return table.fields.some(
 			(field) =>
@@ -337,9 +288,6 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
-		request.custom.isAction !== 'beginUpload' &&
-		request.custom.isAction !== 'commitUpload' &&
-		request.custom.isAction !== 'cancelUpload' &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -536,37 +484,4 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
-
-	if (isMultipartUploadAvailable(handler)) {
-		sbvrUtils.addPureHook(
-			'POST',
-			apiRoot,
-			resourceName,
-			multipartUploadHooks(handler),
-		);
-	}
-};
-
-const initSql = `
-CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
-CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
-`;
-
-const modelText = await importSBVR('./webresource.sbvr', import.meta);
-
-declare module '../sbvr-api/sbvr-utils.js' {
-	export interface API {
-		webresource: PinejsClient<WebresourceModel>;
-	}
-}
-
-export const config: ConfigLoader.Config = {
-	models: [
-		{
-			modelName: 'webresource',
-			apiRoot: 'webresource',
-			modelText,
-			initSql,
-		},
-	],
 };

package/out/data-server/sbvr-server.d.ts

@@ -1,17 +0,0 @@
-import type { SetupFunction } from '../config-loader/config-loader.js';
-import type { Tx } from '../database-layer/db.js';
-export declare const setup: SetupFunction;
-export declare const config: {
-    models: {
-        modelName: string;
-        modelText: string;
-        apiRoot: string;
-        customServerCode: {
-            setup: SetupFunction;
-        };
-        migrations: {
-            '11.0.0-modified-at': string;
-            '15.0.0-data-types': (tx: Tx, sbvrUtils: typeof import("../sbvr-api/sbvr-utils.js")) => Promise<void>;
-        };
-    }[];
-};