@balena/pinejs 17.0.0-build-wip-large-file-uploads-d6522dad962bc0bff6ee7c596df8f43f596b6aaa-1 → 17.0.0-build-wip-large-file-uploads-0c8ef752deac19fa0d6a7dfa9f7173813cab7867-1

package/src/webresource-handler/multipartUpload.ts

@@ -1,10 +1,11 @@
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import { randomUUID } from 'node:crypto';
 import type { AnyObject } from 'pinejs-client-core';
 import type { WebResourceHandler } from '.';
 import { getWebResourceFields } from '.';
+import { api } from '../sbvr-api/sbvr-utils';
 import type { ODataRequest } from '../sbvr-api/uri-parser';
-import { errors, sbvrUtils } from '../server-glue/module';
-import type { WebResource } from '@balena/sbvr-types/out/types/web-resource';
-import { randomUUID } from 'node:crypto';
+import { errors, permissions, sbvrUtils } from '../server-glue/module';
 
 export interface BeginUploadPayload {
 	filename: string;
@@ -76,7 +77,7 @@ export const multipartUploadHooks = (
 				delete request.odataQuery.property;
 				request.custom.isAction = 'beginUpload';
 			} else if (request.odataQuery.property?.resource === 'commitUpload') {
-				const commitPayload = parseCommitUpload(request);
+				const commitPayload = await parseCommitUpload(request);
 
 				await sbvrUtils.api[request.vocabulary].post({
 					url: request.url.substring(1).replace('commitUpload', 'canAccess'),
@@ -89,7 +90,22 @@ export const multipartUploadHooks = (
 					filename: commitPayload.metadata.filename,
 					multipartUploadChecksums: commitPayload.additionalCommitInfo,
 				});
-				delete someKvStoreDbOrRedisIdc[commitPayload.key];
+
+				await api.webresource.patch({
+					resource: 'multipart_upload',
+					body: {
+						status: 'completed',
+					},
+					options: {
+						$filter: {
+							uuid: commitPayload.key,
+						},
+					},
+					passthrough: {
+						req: permissions.root,
+						tx: tx,
+					},
+				});
 
 				request.method = 'PATCH';
 				request.values = {
@@ -101,13 +117,15 @@ export const multipartUploadHooks = (
 				request.custom.commitUploadPayload = webresource;
 			}
 		},
-		PRERESPOND: async ({ request, response, tx }) => {
+		PRERESPOND: async ({ req, request, response, tx }) => {
 			if (request.custom.isAction === 'beginUpload') {
 				await tx.rollback();
+
 				response.statusCode = 200;
 				response.body = await beginUpload(
 					webResourceHandler,
-					request.values as { [x: string]: BeginUploadPayload },
+					request,
+					req.user?.actor,
 				);
 			} else if (request.custom.isAction === 'commitUpload') {
 				response.body = await webResourceHandler.onPreRespond(
@@ -118,25 +136,47 @@ export const multipartUploadHooks = (
 	};
 };
 
-// Really any storage will do.
-// I just honestly believe redis better suits this user case but I am fine if we agree on storing on DB
-// This is not a cache, it is a persistent KV storage.
-const someKvStoreDbOrRedisIdc: Record<string, PendingUpload> = {};
-
 export const beginUpload = async (
 	webResourceHandler: WebResourceHandler,
-	payload: { [fieldName: string]: BeginUploadPayload },
+	odataRequest: ODataRequest,
+	actorId?: number,
 ): Promise<BeginUploadResponse> => {
+	const payload = odataRequest.values as { [x: string]: BeginUploadPayload };
 	const fieldName = Object.keys(payload)[0];
 	const metadata = payload[fieldName];
 
 	const { fileKey, uploadId, uploadUrls } =
 		await webResourceHandler.beginUpload(fieldName, metadata);
-	const key = randomUUID();
-
-	someKvStoreDbOrRedisIdc[key] = { ...metadata, fieldName, fileKey, uploadId };
+	const uuid = randomUUID();
+
+	try {
+		await api.webresource.post({
+			resource: 'multipart_upload',
+			body: {
+				uuid,
+				resource_name: odataRequest.resourceName,
+				field_name: fieldName,
+				resource_id: odataRequest.affectedIds?.[0],
+				upload_id: uploadId,
+				file_key: fileKey,
+				status: 'pending',
+				filename: metadata.filename,
+				content_type: metadata.content_type,
+				size: metadata.size,
+				chunk_size: metadata.chunk_size,
+				expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
+				is_created_by__actor: actorId,
+			},
+			passthrough: {
+				req: permissions.root,
+			},
+		});
+	} catch (err) {
+		console.error('failed to start multipart upload', err);
+		throw new errors.BadRequestError('Failed to start multipart upload');
+	}
 
-	return { [fieldName]: { key, uploadUrls } };
+	return { [fieldName]: { key: uuid, uploadUrls } };
 };
 
 const parseBeginUpload = (request: ODataRequest) => {
@@ -196,7 +236,7 @@ const parseBeginUploadPayload = (
 	return { filename, content_type, size, chunk_size };
 };
 
-const parseCommitUpload = (request: ODataRequest) => {
+const parseCommitUpload = async (request: ODataRequest) => {
 	if (request.odataQuery.key == null) {
 		throw new errors.BadRequestError();
 	}
@@ -206,9 +246,41 @@ const parseCommitUpload = (request: ODataRequest) => {
 		throw new errors.BadRequestError('Invalid key type');
 	}
 
-	if (someKvStoreDbOrRedisIdc[key] == null) {
-		throw new errors.BadRequestError('Invalid key');
+	// TODO: actor permissions
+	const [multipartUpload] = (await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+			$filter: {
+				uuid: key,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+			},
+		},
+		passthrough: {
+			req: permissions.root,
+			tx: request.tx,
+		},
+	})) as [
+		{
+			id: number;
+			file_key: string;
+			upload_id: string;
+			field_name: string;
+			filename: string;
+		}?,
+	];
+
+	if (multipartUpload == null) {
+		throw new errors.BadRequestError(`Invalid upload for key ${key}`);
 	}
 
-	return { key, additionalCommitInfo, metadata: someKvStoreDbOrRedisIdc[key] };
+	const metadata = {
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+		filename: multipartUpload.filename,
+		fieldName: multipartUpload.field_name,
+	};
+
+	return { key, additionalCommitInfo, metadata };
 };

package/src/webresource-handler/webresource.sbvr

@@ -0,0 +1,62 @@
+Vocabulary: Auth
+
+Term: actor
+Term: expiry date
+	Concept Type: Date Time (Type)
+
+Vocabulary: webresource
+
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date (Auth)
+	Necessity: each multipart upload has exactly one expiry date (Auth)
+Fact type: multipart upload is created by actor (Auth)
+	Necessity: each multipart upload is created by at most one actor (Auth)