@balena/pinejs 17.0.0-build-v17-4b8f0faa9617d64057e94cdc0ca92650925c1d39-1 → 17.0.0-build-large-file-uploads-293e65ee371a69130834c50fef2f7f42cc18133f-1

package/src/webresource-handler/index.ts

@@ -13,6 +13,8 @@ import {
 } from '@balena/odata-to-abstract-sql';
 import { errors, permissions } from '../server-glue/module';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import type { AnyObject } from 'pinejs-client-core';
+import { multipartUploadHooks } from './multipartUpload';
 
 export * from './handlers';
 
@@ -29,10 +31,44 @@ export interface UploadResponse {
 	filename: string;
 }
 
+export interface BeginMultipartUploadPayload {
+	filename: string;
+	content_type: string;
+	size: number;
+	chunk_size: number;
+}
+
+export interface UploadPart {
+	url: string;
+	chunkSize: number;
+	partNumber: number;
+}
+
+export interface BeginMultipartUploadHandlerResponse {
+	uploadParts: UploadPart[];
+	fileKey: string;
+	uploadId: string;
+}
+
+export interface CommitMultipartUploadPayload {
+	fileKey: string;
+	uploadId: string;
+	filename: string;
+	providerCommitData?: AnyObject;
+}
+
 export interface WebResourceHandler {
 	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
 	removeFile: (fileReference: string) => Promise<void>;
 	onPreRespond: (webResource: WebResource) => Promise<WebResource>;
+
+	beginMultipartUpload: (
+		fieldName: string,
+		payload: BeginMultipartUploadPayload,
+	) => Promise<BeginMultipartUploadHandlerResponse>;
+	commitMultipartUpload: (
+		commitInfo: CommitMultipartUploadPayload,
+	) => Promise<WebResource>;
 }
 
 type WebResourcesDbResponse = {
@@ -201,7 +237,7 @@ export const getUploaderMiddlware = (
 	};
 };
 
-const getWebResourceFields = (
+export const getWebResourceFields = (
 	request: uriParser.ODataRequest,
 	useTranslations = true,
 ): string[] => {
@@ -234,6 +270,8 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		request.custom.isAction !== 'beginUpload' &&
+		request.custom.isAction !== 'commitUpload' &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -432,4 +470,23 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+
+	sbvrUtils.addPureHook(
+		'POST',
+		apiRoot,
+		resourceName,
+		multipartUploadHooks(handler),
+	);
+};
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const webresourceModel: string = require('./webresource.sbvr');
+export const config = {
+	models: [
+		{
+			apiRoot: 'webresource',
+			modelText: webresourceModel,
+			modelName: 'webresource',
+		},
+	] as sbvrUtils.ExecutableModel[],
 };

package/src/webresource-handler/multipartUpload.ts

@@ -0,0 +1,275 @@
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import { randomUUID } from 'node:crypto';
+import type { AnyObject } from 'pinejs-client-core';
+import type {
+	BeginMultipartUploadPayload,
+	UploadPart,
+	WebResourceHandler,
+} from '.';
+import { getWebResourceFields } from '.';
+import type { PinejsClient } from '../sbvr-api/sbvr-utils';
+import { api } from '../sbvr-api/sbvr-utils';
+import type { ODataRequest } from '../sbvr-api/uri-parser';
+import { errors, sbvrUtils } from '../server-glue/module';
+
+type BeginUploadDbCheck = BeginMultipartUploadPayload & WebResource;
+
+export interface PendingUpload extends BeginMultipartUploadPayload {
+	fieldName: string;
+	fileKey: string;
+	uploadId: string;
+}
+
+export interface BeginUploadResponse {
+	[fieldName: string]: {
+		uuid: string;
+		uploadParts: UploadPart[];
+	};
+}
+
+const MB = 1024 * 1024;
+
+export const multipartUploadHooks = (
+	webResourceHandler: WebResourceHandler,
+): sbvrUtils.Hooks => {
+	return {
+		POSTPARSE: async ({ req, request, tx, api: applicationApi }) => {
+			if (request.odataQuery.property?.resource === 'beginUpload') {
+				const uploadParams = await validateBeginUpload(request, applicationApi);
+
+				// This transaction is necessary because beginUpload requests
+				// will rollback the transaction (in order to first validate)
+				// The metadata requested. If we don't pass any transaction
+				// It will use the default transaction handler which will error out
+				// on any rollback.
+				tx = await sbvrUtils.db.transaction();
+				req.tx = tx;
+				request.tx = tx;
+
+				request.method = 'PATCH';
+				request.values = uploadParams;
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'beginUpload';
+			} else if (request.odataQuery.property?.resource === 'commitUpload') {
+				const commitPayload = await validateCommitUpload(
+					request,
+					applicationApi,
+				);
+
+				const webresource = await webResourceHandler.commitMultipartUpload({
+					fileKey: commitPayload.metadata.fileKey,
+					uploadId: commitPayload.metadata.uploadId,
+					filename: commitPayload.metadata.filename,
+					providerCommitData: commitPayload.providerCommitData,
+				});
+
+				await api.webresource.patch({
+					resource: 'multipart_upload',
+					body: {
+						status: 'completed',
+					},
+					options: {
+						$filter: {
+							uuid: commitPayload.uuid,
+						},
+					},
+					passthrough: {
+						tx: tx,
+					},
+				});
+
+				request.method = 'PATCH';
+				request.values = {
+					[commitPayload.metadata.fieldName]: webresource,
+				};
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'commitUpload';
+				request.custom.commitUploadPayload = webresource;
+			}
+		},
+		PRERESPOND: async ({ req, request, response, tx }) => {
+			if (request.custom.isAction === 'beginUpload') {
+				// In the case where the transaction has failed because it had invalid payload
+				// such as breaking a db constraint, this hook wouldn't have been called
+				// and would rather throw with the rule it failed to validate
+				// We rollback here as the patch was just a way to validate the upload payload
+				await tx.rollback();
+
+				response.statusCode = 200;
+				response.body = await beginUpload(
+					webResourceHandler,
+					request,
+					req.user?.actor,
+				);
+			} else if (request.custom.isAction === 'commitUpload') {
+				response.body = await webResourceHandler.onPreRespond(
+					request.custom.commitUploadPayload,
+				);
+			}
+		},
+	};
+};
+
+export const beginUpload = async (
+	webResourceHandler: WebResourceHandler,
+	odataRequest: ODataRequest,
+	actorId?: number,
+): Promise<BeginUploadResponse> => {
+	const payload = odataRequest.values as {
+		[x: string]: BeginMultipartUploadPayload;
+	};
+	const fieldName = Object.keys(payload)[0];
+	const metadata = payload[fieldName];
+
+	const { fileKey, uploadId, uploadParts } =
+		await webResourceHandler.beginMultipartUpload(fieldName, metadata);
+	const uuid = randomUUID();
+
+	try {
+		await api.webresource.post({
+			resource: 'multipart_upload',
+			body: {
+				uuid,
+				resource_name: odataRequest.resourceName,
+				field_name: fieldName,
+				resource_id: odataRequest.affectedIds?.[0],
+				upload_id: uploadId,
+				file_key: fileKey,
+				status: 'pending',
+				filename: metadata.filename,
+				content_type: metadata.content_type,
+				size: metadata.size,
+				chunk_size: metadata.chunk_size,
+				expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
+				is_created_by__actor: actorId,
+			},
+		});
+	} catch (err) {
+		console.error('failed to start multipart upload', err);
+		throw new errors.BadRequestError('Failed to start multipart upload');
+	}
+
+	return { [fieldName]: { uuid, uploadParts } };
+};
+
+const validateBeginUpload = async (
+	request: ODataRequest,
+	applicationApi: PinejsClient,
+) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	await applicationApi.post({
+		url: request.url.substring(1).replace('beginUpload', 'canAccess'),
+		body: { method: 'PATCH' },
+	});
+
+	const fieldNames = Object.keys(request.values);
+	if (fieldNames.length !== 1) {
+		throw new errors.BadRequestError(
+			'You can only get upload url for one field at a time',
+		);
+	}
+
+	const [fieldName] = fieldNames;
+	const webResourceFields = getWebResourceFields(request, false);
+	if (!webResourceFields.includes(fieldName)) {
+		throw new errors.BadRequestError(
+			`You must provide a valid webresource field from: ${JSON.stringify(webResourceFields)}`,
+		);
+	}
+
+	const beginUploadPayload = parseBeginUploadPayload(request.values[fieldName]);
+	if (beginUploadPayload == null) {
+		throw new errors.BadRequestError('Invalid file metadata');
+	}
+
+	const uploadMetadataCheck: BeginUploadDbCheck = {
+		...beginUploadPayload,
+		href: 'metadata_check',
+	};
+
+	return { [fieldName]: uploadMetadataCheck };
+};
+
+const parseBeginUploadPayload = (
+	payload: AnyObject,
+): BeginMultipartUploadPayload | null => {
+	if (typeof payload !== 'object') {
+		return null;
+	}
+
+	let { filename, content_type, size, chunk_size } = payload;
+	if (
+		typeof filename !== 'string' ||
+		typeof content_type !== 'string' ||
+		typeof size !== 'number' ||
+		(chunk_size != null && typeof chunk_size !== 'number') ||
+		(chunk_size != null && chunk_size < 5 * MB)
+	) {
+		return null;
+	}
+
+	if (chunk_size == null) {
+		chunk_size = 5 * MB;
+	}
+	return { filename, content_type, size, chunk_size };
+};
+
+const validateCommitUpload = async (
+	request: ODataRequest,
+	applicationApi: PinejsClient,
+) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	await applicationApi.post({
+		url: request.url.substring(1).replace('commitUpload', 'canAccess'),
+		body: { method: 'PATCH' },
+	});
+
+	const { uuid, providerCommitData } = request.values;
+	if (typeof uuid !== 'string') {
+		throw new errors.BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = (await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+			$filter: {
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+			},
+		},
+		passthrough: {
+			tx: request.tx,
+		},
+	})) as [
+		{
+			id: number;
+			file_key: string;
+			upload_id: string;
+			field_name: string;
+			filename: string;
+		}?,
+	];
+
+	if (multipartUpload == null) {
+		throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+	}
+
+	const metadata = {
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+		filename: multipartUpload.filename,
+		fieldName: multipartUpload.field_name,
+	};
+
+	return { uuid, providerCommitData, metadata };
+};

package/src/webresource-handler/webresource.sbvr

@@ -0,0 +1,63 @@
+Vocabulary: Auth
+
+Term: actor
+Term: expiry date
+	Concept Type: Date Time (Type)
+
+Vocabulary: webresource
+
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date (Auth)
+	Necessity: each multipart upload has exactly one expiry date (Auth)
+Fact type: multipart upload is created by actor (Auth)
+	Necessity: each multipart upload is created by at most one actor (Auth)
+	Reference Type: informative