@balena/pinejs 20.1.0-build-add-large-file-uploads-interfaces-e645db6b429537d3cb2e40da3167bcfcc6b31b37-1 → 20.1.0-build-large-file-uploads-2-b83400458570d926f7a6abddeb7a149e89f52abe-2

package/src/webresource-handler/index.ts

@@ -11,9 +11,16 @@ import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
+import type { ConfigLoader } from '../server-glue/module.js';
 import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
+import type WebresourceModel from './webresource.js';
+import { importSBVR } from '../server-glue/sbvr-loader.js';
+import {
+	isMultipartUploadAvailable,
+	multipartUploadHooks,
+} from './multipartUpload.js';
 
 export * from './handlers/index.js';
 
@@ -56,17 +63,23 @@ export interface CommitMultipartUploadPayload {
 	providerCommitData?: Record<string, any>;
 }
 
+export interface CancelMultipartUploadPayload {
+	fileKey: string;
+	uploadId: string;
+}
+
 export interface WebResourceHandler {
 	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
 	removeFile: (fileReference: string) => Promise<void>;
 	onPreRespond: (webResource: WebResource) => Promise<WebResource>;
-	beginMultipartUpload?: (
-		fieldName: string,
-		payload: BeginMultipartUploadPayload,
-	) => Promise<BeginMultipartUploadHandlerResponse>;
-	commitMultipartUpload?: (
-		commitInfo: CommitMultipartUploadPayload,
-	) => Promise<WebResource>;
+	multipartUpload?: {
+		begin: (
+			fieldName: string,
+			payload: BeginMultipartUploadPayload,
+		) => Promise<BeginMultipartUploadHandlerResponse>;
+		commit: (commitInfo: CommitMultipartUploadPayload) => Promise<WebResource>;
+		cancel: (cancelInfo: CancelMultipartUploadPayload) => Promise<void>;
+	};
 }
 
 export class WebResourceError extends TypedError {}
@@ -96,30 +109,27 @@ export const setupWebresourceHandler = (handler: WebResourceHandler): void => {
 export const getWebresourceHandler = (): WebResourceHandler | undefined => {
 	return configuredWebResourceHandler;
 };
+const notValidUpload = () => Promise.resolve(false);
 
-const isFileInValidPath = async (
-	fieldname: string,
+const getRequestUploadValidator = async (
 	req: Express.Request,
-): Promise<boolean> => {
+	odataRequest: uriParser.ParsedODataRequest,
+): Promise<(fieldName: string) => Promise<boolean>> => {
 	if (req.method !== 'POST' && req.method !== 'PATCH') {
-		return false;
+		return notValidUpload;
 	}
 
 	const apiRoot = getApiRoot(req);
 	if (apiRoot == null) {
-		return false;
+		return notValidUpload;
 	}
 	const model = getModel(apiRoot);
-	const odataRequest = uriParser.parseOData({
-		url: req.url,
-		method: req.method,
-	});
 	const sqlResourceName = sbvrUtils.resolveSynonym(odataRequest);
 
 	const table = model.abstractSql.tables[sqlResourceName];
 
 	if (table == null) {
-		return false;
+		return notValidUpload;
 	}
 
 	const permission = req.method === 'POST' ? 'create' : 'update';
@@ -134,20 +144,22 @@ const isFileInValidPath = async (
 	);
 
 	if (!hasPermissions) {
-		return false;
+		return notValidUpload;
 	}
 
-	const dbFieldName = odataNameToSqlName(fieldname);
-	return table.fields.some(
-		(field) =>
-			field.fieldName === dbFieldName && field.dataType === 'WebResource',
-	);
+	return async (fieldname: string) => {
+		const dbFieldName = odataNameToSqlName(fieldname);
+		return table.fields.some(
+			(field) =>
+				field.fieldName === dbFieldName && field.dataType === 'WebResource',
+		);
+	};
 };
 
 export const getUploaderMiddlware = (
 	handler: WebResourceHandler,
 ): Express.RequestHandler => {
-	return (req, res, next) => {
+	return async (req, res, next) => {
 		if (!req.is('multipart')) {
 			next();
 			return;
@@ -158,6 +170,20 @@ export const getUploaderMiddlware = (
 		const bb = busboy({ headers: req.headers });
 		let isAborting = false;
 
+		const parsedOdataRequest = uriParser.parseOData({
+			url: req.url,
+			method: req.method,
+		});
+		const webResourcesFieldNames = getWebResourceFields(
+			parsedOdataRequest,
+			false,
+		);
+
+		const isValidUpload = await getRequestUploadValidator(
+			req,
+			parsedOdataRequest,
+		);
+
 		const finishFileUpload = () => {
 			req.unpipe(bb);
 			req.on('readable', req.read.bind(req));
@@ -185,7 +211,7 @@ export const getUploaderMiddlware = (
 			completeUploads.push(
 				(async () => {
 					try {
-						if (!(await isFileInValidPath(fieldname, req))) {
+						if (!(await isValidUpload(fieldname))) {
 							filestream.resume();
 							return;
 						}
@@ -207,7 +233,10 @@ export const getUploaderMiddlware = (
 						uploadedFilePaths.push(result.filename);
 					} catch (err: any) {
 						filestream.resume();
-						throw err;
+						bb.emit(
+							'error',
+							new errors.BadRequestError(err.message ?? 'Error uploading file'),
+						);
 					}
 				})(),
 			);
@@ -217,6 +246,14 @@ export const getUploaderMiddlware = (
 		// This receives the form fields and transforms them into a standard JSON body
 		// This is a similar behavior as previous multer library did
 		bb.on('field', (name, val) => {
+			if (webResourcesFieldNames.includes(name)) {
+				isAborting = true;
+				bb.emit(
+					'error',
+					new errors.BadRequestError('WebResource field must be a blob.'),
+				);
+				return;
+			}
 			req.body[name] = val;
 		});
 
@@ -242,7 +279,7 @@ export const getUploaderMiddlware = (
 			}
 		});
 
-		bb.on('error', async (err) => {
+		bb.on('error', async (err: Error) => {
 			finishFileUpload();
 			await clearFiles();
 
@@ -254,15 +291,17 @@ export const getUploaderMiddlware = (
 				);
 			}
 
-			getLogger(getApiRoot(req)).error('Error uploading file', err);
-			next(err);
+			if (!sbvrUtils.handleHttpErrors(req, res, err)) {
+				getLogger(getApiRoot(req)).error('Error uploading file', err);
+				next(err);
+			}
 		});
 		req.pipe(bb);
 	};
 };
 
-const getWebResourceFields = (
-	request: uriParser.ODataRequest,
+export const getWebResourceFields = (
+	request: uriParser.ParsedODataRequest,
 	useTranslations = true,
 ): string[] => {
 	// Translations will use modifyFields(translated) rather than fields(original) so we need to
@@ -294,6 +333,9 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		request.custom.isAction !== 'beginUpload' &&
+		request.custom.isAction !== 'commitUpload' &&
+		request.custom.isAction !== 'cancelUpload' &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -490,4 +532,37 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+
+	if (isMultipartUploadAvailable(handler)) {
+		sbvrUtils.addPureHook(
+			'POST',
+			apiRoot,
+			resourceName,
+			multipartUploadHooks(handler),
+		);
+	}
+};
+
+const initSql = `
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
+`;
+
+const modelText = await importSBVR('./webresource.sbvr', import.meta);
+
+declare module '../sbvr-api/sbvr-utils.js' {
+	export interface API {
+		webresource: PinejsClient<WebresourceModel>;
+	}
+}
+
+export const config: ConfigLoader.Config = {
+	models: [
+		{
+			modelName: 'webresource',
+			apiRoot: 'webresource',
+			modelText,
+			initSql,
+		},
+	],
 };

package/src/webresource-handler/multipartUpload.ts

@@ -0,0 +1,371 @@
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import { randomUUID } from 'node:crypto';
+import type { AnyObject } from 'pinejs-client-core';
+import type {
+	BeginMultipartUploadPayload,
+	UploadPart,
+	WebResourceHandler,
+} from './index.js';
+import { getWebResourceFields } from './index.js';
+import type { PinejsClient } from '../sbvr-api/sbvr-utils.js';
+import { api } from '../sbvr-api/sbvr-utils.js';
+import type { ODataRequest } from '../sbvr-api/uri-parser.js';
+import { errors, sbvrUtils } from '../server-glue/module.js';
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import * as permissions from '../sbvr-api/permissions.js';
+
+type BeginUploadDbCheck = BeginMultipartUploadPayload & WebResource;
+
+export interface PendingUpload extends BeginMultipartUploadPayload {
+	fieldName: string;
+	fileKey: string;
+	uploadId: string;
+}
+
+export interface BeginUploadResponse {
+	[fieldName: string]: {
+		uuid: string;
+		uploadParts: UploadPart[];
+	};
+}
+
+// Is WebResourceHandler but with beginMultipartUpload and commitMultipartUpload as not optional
+type MultipartUploadHandler = WebResourceHandler &
+	Required<Pick<WebResourceHandler, 'multipartUpload'>>;
+
+const MB = 1024 * 1024;
+
+export const isMultipartUploadAvailable = (
+	webResourceHandler: WebResourceHandler,
+): webResourceHandler is MultipartUploadHandler => {
+	return (
+		webResourceEnv.multipartUploadEnabled &&
+		webResourceHandler.multipartUpload != null
+	);
+};
+
+export const multipartUploadHooks = (
+	webResourceHandler: MultipartUploadHandler,
+): sbvrUtils.Hooks => {
+	return {
+		POSTPARSE: async ({ req, request, tx, api: applicationApi }) => {
+			if (request.odataQuery.property?.resource === 'beginUpload') {
+				const uploadParams = await validateBeginUpload(request, applicationApi);
+
+				// This transaction is necessary because beginUpload requests
+				// will rollback the transaction (in order to first validate)
+				// The metadata requested. If we don't pass any transaction
+				// It will use the default transaction handler which will error out
+				// on any rollback.
+				tx = await sbvrUtils.db.transaction();
+				req.tx = tx;
+				request.tx = tx;
+
+				request.method = 'PATCH';
+				request.values = uploadParams;
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'beginUpload';
+			} else if (request.odataQuery.property?.resource === 'commitUpload') {
+				const commitPayload = await validateCommitUpload(
+					request,
+					applicationApi,
+				);
+
+				const webresource = await webResourceHandler.multipartUpload.commit({
+					fileKey: commitPayload.metadata.fileKey,
+					uploadId: commitPayload.metadata.uploadId,
+					filename: commitPayload.metadata.filename,
+					providerCommitData: commitPayload.providerCommitData,
+				});
+
+				await api.webresource.patch({
+					resource: 'multipart_upload',
+					body: {
+						status: 'completed',
+					},
+					options: {
+						$filter: {
+							uuid: commitPayload.uuid,
+						},
+					},
+					passthrough: {
+						tx: tx,
+						req: permissions.root,
+					},
+				});
+
+				request.method = 'PATCH';
+				request.values = {
+					[commitPayload.metadata.fieldName]: webresource,
+				};
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'commitUpload';
+				request.custom.commitUploadPayload = webresource;
+			} else if (request.odataQuery.property?.resource === 'cancelUpload') {
+				const { uuid, fileKey, uploadId } = await validateCancelPayload(
+					request,
+					applicationApi,
+				);
+
+				await webResourceHandler.multipartUpload.cancel({ fileKey, uploadId });
+
+				await api.webresource.patch({
+					resource: 'multipart_upload',
+					body: {
+						status: 'cancelled',
+					},
+					options: {
+						$filter: { uuid },
+					},
+					passthrough: {
+						tx: tx,
+						req: permissions.root,
+					},
+				});
+
+				request.method = 'GET';
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'cancelUpload';
+			}
+		},
+		PRERESPOND: async ({ req, request, response, tx }) => {
+			if (request.custom.isAction === 'beginUpload') {
+				// In the case where the transaction has failed because it had invalid payload
+				// such as breaking a db constraint, this hook wouldn't have been called
+				// and would rather throw with the rule it failed to validate
+				// We rollback here as the patch was just a way to validate the upload payload
+				await tx.rollback();
+
+				response.statusCode = 200;
+				response.body = await beginUpload({
+					webResourceHandler,
+					odataRequest: request,
+					actorId: req.user?.actor,
+				});
+			} else if (request.custom.isAction === 'commitUpload') {
+				response.body = await webResourceHandler.onPreRespond(
+					request.custom.commitUploadPayload,
+				);
+			} else if (request.custom.isAction === 'cancelUpload') {
+				response.statusCode = 204;
+				delete response.body;
+			}
+		},
+	};
+};
+
+const beginUpload = async ({
+	webResourceHandler,
+	odataRequest,
+	actorId,
+}: {
+	webResourceHandler: MultipartUploadHandler;
+	odataRequest: ODataRequest;
+	actorId?: number;
+}): Promise<BeginUploadResponse> => {
+	const payload = odataRequest.values as {
+		[x: string]: BeginMultipartUploadPayload;
+	};
+	const fieldName = Object.keys(payload)[0];
+	const metadata = payload[fieldName];
+	const { fileKey, uploadId, uploadParts } =
+		await webResourceHandler.multipartUpload.begin(fieldName, metadata);
+	const uuid = randomUUID();
+
+	return await sbvrUtils.db.transaction(async (tx) => {
+		try {
+			await api.webresource.post({
+				resource: 'multipart_upload',
+				body: {
+					uuid,
+					resource_name: odataRequest.resourceName,
+					field_name: fieldName,
+					resource_id: odataRequest.affectedIds?.[0],
+					upload_id: uploadId,
+					file_key: fileKey,
+					status: 'pending',
+					filename: metadata.filename,
+					content_type: metadata.content_type,
+					size: metadata.size,
+					chunk_size: metadata.chunk_size,
+					expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
+					is_created_by__actor: actorId,
+				},
+				passthrough: {
+					req: permissions.root,
+					tx,
+				},
+			});
+			return { [fieldName]: { uuid, uploadParts } };
+		} catch (err) {
+			console.error('failed to start multipart upload', err);
+			throw new errors.BadRequestError('Failed to start multipart upload');
+		}
+	});
+};
+
+const validateBeginUpload = async (
+	request: ODataRequest,
+	applicationApi: PinejsClient,
+) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	await applicationApi.request({
+		method: 'POST',
+		url: request.url.substring(1).replace('beginUpload', 'canAccess'),
+		body: { method: 'PATCH' },
+	});
+
+	const fieldNames = Object.keys(request.values);
+	if (fieldNames.length !== 1) {
+		throw new errors.BadRequestError(
+			'You can only get upload url for one field at a time',
+		);
+	}
+
+	const [fieldName] = fieldNames;
+	const webResourceFields = getWebResourceFields(request, false);
+	if (!webResourceFields.includes(fieldName)) {
+		throw new errors.BadRequestError(
+			`You must provide a valid webresource field from: ${JSON.stringify(webResourceFields)}`,
+		);
+	}
+
+	const beginUploadPayload = parseBeginUploadPayload(request.values[fieldName]);
+	if (beginUploadPayload == null) {
+		throw new errors.BadRequestError('Invalid file metadata');
+	}
+
+	const uploadMetadataCheck: BeginUploadDbCheck = {
+		...beginUploadPayload,
+		// This is "probe" request. We don't actually store anything on the application table yet
+		// We just avoid creating the application record if it would fail anyway for some other db constraint
+		href: 'metadata_check_probe',
+	};
+
+	return { [fieldName]: uploadMetadataCheck };
+};
+
+const parseBeginUploadPayload = (
+	payload: AnyObject,
+): BeginMultipartUploadPayload | null => {
+	if (typeof payload !== 'object') {
+		return null;
+	}
+
+	let { filename, content_type, size, chunk_size } = payload;
+	if (
+		typeof filename !== 'string' ||
+		typeof content_type !== 'string' ||
+		typeof size !== 'number' ||
+		(chunk_size != null && typeof chunk_size !== 'number') ||
+		(chunk_size != null && chunk_size < 5 * MB)
+	) {
+		return null;
+	}
+
+	if (chunk_size == null) {
+		chunk_size = 5 * MB;
+	}
+	return { filename, content_type, size, chunk_size };
+};
+
+const validateCommitUpload = async (
+	request: ODataRequest,
+	applicationApi: PinejsClient,
+) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	await applicationApi.request({
+		method: 'POST',
+		url: request.url.substring(1).replace('commitUpload', 'canAccess'),
+		body: { method: 'PATCH' },
+	});
+
+	const { uuid, providerCommitData } = request.values;
+	if (typeof uuid !== 'string') {
+		throw new errors.BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+			$filter: {
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+			},
+		},
+		passthrough: {
+			tx: request.tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+	}
+
+	const metadata = {
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+		filename: multipartUpload.filename,
+		fieldName: multipartUpload.field_name,
+	};
+
+	return { uuid, providerCommitData, metadata };
+};
+
+const validateCancelPayload = async (
+	request: ODataRequest,
+	applicationApi: PinejsClient,
+) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	await applicationApi.request({
+		method: 'POST',
+		url: request.url.substring(1).replace('cancelUpload', 'canAccess'),
+		body: { method: 'PATCH' },
+	});
+
+	const { uuid } = request.values;
+	if (typeof uuid !== 'string') {
+		throw new errors.BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id'],
+			$filter: {
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+			},
+		},
+		passthrough: {
+			tx: request.tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+	}
+
+	return {
+		uuid,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+	};
+};

package/src/webresource-handler/webresource.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: webresource
+
+Term: actor
+	Concept Type: Integer (Type)
+Term: expiry date
+	Concept Type: Date Time (Type)
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date
+	Necessity: each multipart upload has exactly one expiry date
+Fact type: multipart upload is created by actor
+	Necessity: each multipart upload is created by at most one actor