@balena/pinejs 21.4.0-build-large-file-uploads-2-56700d672d31db4406ba01ca349d69af5c8611e7-1 → 21.4.0-build-add-actions-example-c20002c2bf051233b333158f3b7a0719f6cf6e4e-1

package/src/webresource-handler/actions/beginUpload.ts

@@ -0,0 +1,160 @@
+import type { AnyObject } from 'pinejs-client-core';
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../actions/index.js';
+import { api, type Response } from '../../sbvr-api/sbvr-utils.js';
+import type { MultipartUploadHandler } from '../multiparUpload.js';
+import { getMultipartUploadHandler } from '../multiparUpload.js';
+import type { BeginMultipartUploadPayload } from '../index.js';
+import { getWebResourceFields } from '../index.js';
+import { BadRequestError, NotImplementedError } from '../../sbvr-api/errors.js';
+import { permissions, sbvrUtils } from '../../server-glue/module.js';
+import { TransactionClosedError } from '../../database-layer/db.js';
+import { randomUUID } from 'crypto';
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+
+type FakeWebResourcePatch = {
+	[key: string]: Omit<WebResource, 'href'> & {
+		href: 'fake_patch';
+	};
+};
+
+export const beginUploadAction = async ({
+	request,
+	tx,
+	affectedIds,
+	req,
+}: ODataActionArgs): Promise<Response> => {
+	const affectedId = affectedIds[0];
+	if (typeof affectedId !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+
+	const handler = getMultipartUploadHandler();
+	const { fieldName, beginUploadPayload } = parseBeginUpload(request, handler);
+
+	await runFakeDbPatch(request, {
+		[fieldName]: { ...beginUploadPayload, href: 'fake_patch' },
+	});
+
+	const { fileKey, uploadId, uploadParts } =
+		await handler.multipartUpload.begin(fieldName, beginUploadPayload);
+	const uuid = randomUUID();
+	await api.webresource.post({
+		resource: 'multipart_upload',
+		body: {
+			uuid,
+			resource_name: request.resourceName,
+			field_name: fieldName,
+			resource_id: affectedId,
+			upload_id: uploadId,
+			file_key: fileKey,
+			status: 'pending',
+			...beginUploadPayload,
+			expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
+			is_created_by__actor: req.user?.actor,
+		},
+		passthrough: {
+			req: permissions.root,
+			tx,
+		},
+	});
+
+	return {
+		body: { [fieldName]: { uuid, uploadParts } },
+		statusCode: 200,
+	};
+};
+
+const parseBeginUpload = (
+	request: ODataActionRequest,
+	webResourceHandler: MultipartUploadHandler,
+) => {
+	const fieldNames = Object.keys(request.values);
+	if (fieldNames.length !== 1) {
+		throw new BadRequestError(
+			'You can only get upload url for one field at a time',
+		);
+	}
+
+	const [fieldName] = fieldNames;
+	const webResourceFields = getWebResourceFields(request, false);
+	if (!webResourceFields.includes(fieldName)) {
+		throw new BadRequestError(
+			`The provided field '${fieldName}' is not a valid webresource`,
+		);
+	}
+
+	const beginUploadPayload = parseBeginUploadPayload(
+		request.values[fieldName],
+		webResourceHandler,
+	);
+	if (beginUploadPayload == null) {
+		throw new BadRequestError('Invalid file metadata');
+	}
+
+	return {
+		beginUploadPayload,
+		fieldName,
+	};
+};
+
+const parseBeginUploadPayload = (
+	payload: AnyObject,
+	webResourceHandler: MultipartUploadHandler,
+): BeginMultipartUploadPayload | null => {
+	if (payload == null || typeof payload !== 'object') {
+		return null;
+	}
+
+	let { filename, content_type, size, chunk_size } = payload;
+	if (
+		typeof filename !== 'string' ||
+		typeof content_type !== 'string' ||
+		typeof size !== 'number' ||
+		(chunk_size != null && typeof chunk_size !== 'number') ||
+		(chunk_size != null &&
+			chunk_size < webResourceHandler.multipartUpload.getMinimumPartSize())
+	) {
+		return null;
+	}
+
+	chunk_size ??= webResourceHandler.multipartUpload.getDefaultPartSize();
+
+	return { filename, content_type, size, chunk_size };
+};
+
+// TODO: comment explaining why run this fake transaction is important
+const runFakeDbPatch = async (
+	request: ODataActionRequest,
+	fakeDbPatch: FakeWebResourcePatch,
+) => {
+	try {
+		await sbvrUtils.db.transaction(async (fakeTx) => {
+			const newUrl = request.url.slice(1).replace(/\/beginUpload$/, '');
+			await api[request.vocabulary].request({
+				method: 'PATCH',
+				url: newUrl,
+				body: fakeDbPatch,
+				// TODO: comment explaining why root is necessary
+				passthrough: { tx: fakeTx, req: permissions.root },
+			});
+
+			await fakeTx.rollback();
+		});
+	} catch (e) {
+		if (
+			// calling rollback() always makes it throw, so we capture and move on in case
+			// the error was just because the tx was (on purpose) rolledback
+			!(
+				e instanceof TransactionClosedError &&
+				e.message === 'Transaction has been rolled back.'
+			)
+		) {
+			throw e;
+		}
+	}
+};

package/src/webresource-handler/actions/canceUpload.ts

@@ -0,0 +1,93 @@
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../actions/index.js';
+import type { Tx } from '../../database-layer/db.js';
+import {
+	BadRequestError,
+	NotImplementedError,
+	UnauthorizedError,
+} from '../../sbvr-api/errors.js';
+import { api, type Response } from '../../sbvr-api/sbvr-utils.js';
+import { permissions } from '../../server-glue/module.js';
+import { getMultipartUploadHandler } from '../multiparUpload.js';
+
+export const cancelUploadAction = async ({
+	request,
+	tx,
+	affectedIds,
+}: ODataActionArgs): Promise<Response> => {
+	const affectedId = affectedIds[0];
+	if (typeof affectedId !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+	const { id, fileKey, uploadId } = await getOngoingUpload(
+		request,
+		affectedId,
+		tx,
+	);
+	const handler = getMultipartUploadHandler();
+
+	await api.webresource.patch({
+		resource: 'multipart_upload',
+		body: {
+			status: 'cancelled',
+		},
+		id,
+		passthrough: {
+			tx: tx,
+			req: permissions.root,
+		},
+	});
+
+	// Note that different then beginUpload/commitUpload where we first do the action on the external service
+	// and then reflect it on the DB, for cancel upload it is the other way around
+	// as the worst case scenario is having a canceled upload which is marked on the DB as something else
+	await handler.multipartUpload.cancel({ fileKey, uploadId });
+
+	return {
+		statusCode: 204,
+	};
+};
+
+const getOngoingUpload = async (
+	request: ODataActionRequest,
+	affectedId: number,
+	tx: Tx,
+) => {
+	const { uuid } = request.values;
+	if (!uuid || typeof uuid !== 'string') {
+		throw new BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id'],
+			$filter: {
+				// TODO: swap this into the ID to see if it works
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+				resource_name: request.resourceName,
+				resource_id: affectedId,
+			},
+		},
+		passthrough: {
+			tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new UnauthorizedError();
+	}
+
+	return {
+		id: multipartUpload.id,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+	};
+};

package/src/webresource-handler/actions/commitUpload.ts

@@ -0,0 +1,116 @@
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../actions/index.js';
+import type { Tx } from '../../database-layer/db.js';
+import {
+	BadRequestError,
+	NotImplementedError,
+	UnauthorizedError,
+} from '../../sbvr-api/errors.js';
+import type { Response } from '../../sbvr-api/sbvr-utils.js';
+import { api } from '../../sbvr-api/sbvr-utils.js';
+import { permissions } from '../../server-glue/module.js';
+import { getMultipartUploadHandler } from '../multiparUpload.js';
+
+export const commitUploadAction = async ({
+	request,
+	tx,
+	affectedIds,
+	api: applicationApi,
+}: ODataActionArgs): Promise<Response> => {
+	const affectedId = affectedIds[0];
+	if (typeof affectedId !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+
+	const multipartUpload = await getOngoingUpload(request, affectedId, tx);
+	const handler = getMultipartUploadHandler();
+
+	const webresource = await handler.multipartUpload.commit({
+		fileKey: multipartUpload.fileKey,
+		uploadId: multipartUpload.uploadId,
+		filename: multipartUpload.filename,
+		providerCommitData: multipartUpload.providerCommitData,
+	});
+
+	await Promise.all([
+		api.webresource.patch({
+			resource: 'multipart_upload',
+			body: {
+				status: 'completed',
+			},
+			options: {
+				$filter: {
+					uuid: multipartUpload.uuid,
+				},
+			},
+			passthrough: {
+				tx: tx,
+				req: permissions.root,
+			},
+		}),
+		applicationApi.patch({
+			resource: request.resourceName,
+			id: affectedId,
+			body: {
+				[multipartUpload.fieldName]: webresource,
+			},
+			passthrough: {
+				tx: tx,
+				// Root is needed as, if you are not root, you are not allowed to directly modify the actual metadata
+				req: permissions.root,
+			},
+		}),
+	]);
+
+	return {
+		body: webresource,
+		statusCode: 200,
+	};
+};
+
+const getOngoingUpload = async (
+	request: ODataActionRequest,
+	affectedId: number,
+	tx: Tx,
+) => {
+	const { uuid, providerCommitData } = request.values;
+	if (!uuid || typeof uuid !== 'string') {
+		throw new BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+			$filter: {
+				// TODO: swap this into the ID to see if it works
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+				resource_name: request.resourceName,
+				resource_id: affectedId,
+			},
+		},
+		passthrough: {
+			tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new UnauthorizedError();
+	}
+
+	return {
+		uuid,
+		providerCommitData,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+		filename: multipartUpload.filename,
+		fieldName: multipartUpload.field_name,
+	};
+};

package/src/webresource-handler/index.ts

@@ -18,13 +18,13 @@ import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
 import type WebresourceModel from './webresource.js';
 import { importSBVR } from '../server-glue/sbvr-loader.js';
-import {
-	isMultipartUploadAvailable,
-	multipartUploadHooks,
-} from './multipartUpload.js';
+import { addAction } from '../actions/index.js';
+import { beginUploadAction } from './actions/beginUpload.js';
+import { isMultipartUploadAvailable } from './multiparUpload.js';
+import { commitUploadAction } from './actions/commitUpload.js';
+import { cancelUploadAction } from './actions/canceUpload.js';
 
 export * from './handlers/index.js';
-export type { BeginUploadResponse } from './multipartUpload.js';
 
 export interface IncomingFile {
 	fieldname: string;
@@ -337,9 +337,6 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
-		request.custom.isAction !== 'beginUpload' &&
-		request.custom.isAction !== 'commitUpload' &&
-		request.custom.isAction !== 'cancelUpload' &&
 		req.user !== permissions.root.user &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
@@ -537,14 +534,19 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+};
 
+export const setupUploadActions = (
+	handler: WebResourceHandler,
+	apiRoot: string,
+	resourceName: string,
+) => {
 	if (isMultipartUploadAvailable(handler)) {
-		sbvrUtils.addPureHook(
-			'POST',
-			apiRoot,
-			resourceName,
-			multipartUploadHooks(handler),
-		);
+		addAction(apiRoot, resourceName, 'beginUpload', beginUploadAction);
+
+		addAction(apiRoot, resourceName, 'commitUpload', commitUploadAction);
+
+		addAction(apiRoot, resourceName, 'cancelUpload', cancelUploadAction);
 	}
 };
 

package/src/webresource-handler/multiparUpload.ts

@@ -0,0 +1,23 @@
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import { NotImplementedError } from '../sbvr-api/errors.js';
+import type { WebResourceHandler } from './index.js';
+import { getWebresourceHandler } from './index.js';
+
+export type MultipartUploadHandler = WebResourceHandler &
+	Required<Pick<WebResourceHandler, 'multipartUpload'>>;
+
+export const isMultipartUploadAvailable = (
+	handler: WebResourceHandler | undefined,
+): handler is MultipartUploadHandler => {
+	return (
+		webResourceEnv.multipartUploadEnabled && handler?.multipartUpload != null
+	);
+};
+
+export const getMultipartUploadHandler = () => {
+	const handler = getWebresourceHandler();
+	if (!isMultipartUploadAvailable(handler)) {
+		throw new NotImplementedError('Multipart uploads not available');
+	}
+	return handler;
+};

package/out/webresource-handler/multipartUpload.d.ts

@@ -1,12 +0,0 @@
-import type { UploadPart, WebResourceHandler } from './index.js';
-import { sbvrUtils } from '../server-glue/module.js';
-export interface BeginUploadResponse {
-    [fieldName: string]: {
-        uuid: string;
-        uploadParts: UploadPart[];
-    };
-}
-type MultipartUploadHandler = WebResourceHandler & Required<Pick<WebResourceHandler, 'multipartUpload'>>;
-export declare const isMultipartUploadAvailable: (webResourceHandler: WebResourceHandler) => webResourceHandler is MultipartUploadHandler;
-export declare const multipartUploadHooks: (webResourceHandler: MultipartUploadHandler) => sbvrUtils.Hooks;
-export {};