@balena/pinejs 21.4.0-build-add-odata-actions-8d4944e07924e5cf0a69183c18610e8fd2c41d3b-1 → 21.4.0-build-add-actions-example-f26873fd527dfaaabb15c7bc21476f6b48bfc38a-1

package/src/webresource-handler/index.ts

@@ -11,10 +11,20 @@ import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
+import type { ConfigLoader } from '../server-glue/module.js';
 import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
+import type { Tx } from '../database-layer/db.js';
+import { isPineTasksAvailable } from '../tasks/index.js';
+import type WebresourceModel from './webresource.js';
+import { importSBVR } from '../server-glue/sbvr-loader.js';
+import { beginUploadAction } from './actions/beginUpload.js';
+import { isMultipartUploadAvailable } from './multipartUpload.js';
+import { commitUploadAction } from './actions/commitUpload.js';
+import { cancelUploadAction } from './actions/canceUpload.js';
+import { addAction } from '../sbvr-api/actions.js';
 
 export * from './handlers/index.js';
 
@@ -316,12 +326,34 @@ export const getWebResourceFields = (
 		.map((f) => sqlNameToODataName(f.fieldName));
 };
 
-const deleteFiles = async (
-	keysToDelete: string[],
-	webResourceHandler: WebResourceHandler,
-) => {
-	const promises = keysToDelete.map((r) => webResourceHandler.removeFile(r));
-	await Promise.all(promises);
+const deleteOnStorage = async (keysToDelete: string[], tx?: Tx) => {
+	if (isPineTasksAvailable()) {
+		await Promise.all(
+			keysToDelete.map(async (fileKey) => {
+				return await sbvrUtils.api.tasks.post({
+					resource: 'task',
+					passthrough: {
+						req: permissions.root,
+						tx,
+					},
+					body: {
+						key: crypto.randomUUID(),
+						is_executed_by__handler: 'delete_webresource_file',
+						is_executed_with__parameter_set: {
+							fileKey: fileKey,
+						},
+						attempt_limit: 2 ** 31 - 1,
+					},
+				});
+			}),
+		);
+	} else {
+		await Promise.all(
+			keysToDelete.map(async (fileKey) => {
+				return await configuredWebResourceHandler?.removeFile(fileKey);
+			}),
+		);
+	}
 };
 
 const throwIfWebresourceNotInMultipart = (
@@ -329,6 +361,13 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		// root needs to be able to bypass the multipart check as
+		// it needs to pass the direct payload on multipart uploads (on storage provider)
+		req.user !== permissions.root.user &&
+		// This is checking for HTTP multipart form submission/request (e.g. send the actual file via the API)
+		// Not to confuse with multipart uploads
+		// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Methods/POST#multipart_form_submission
+		// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -338,18 +377,22 @@ const throwIfWebresourceNotInMultipart = (
 	}
 };
 
-const getCreateWebResourceHooks = (
-	webResourceHandler: WebResourceHandler,
-): sbvrUtils.Hooks => {
+const getCreateWebResourceHooks = (): sbvrUtils.Hooks => {
 	return {
 		PRERUN: (hookArgs) => {
 			const webResourceFields = getWebResourceFields(hookArgs.request);
 			throwIfWebresourceNotInMultipart(webResourceFields, hookArgs);
 		},
-		'POSTRUN-ERROR': ({ tx, request }) => {
-			tx.on('rollback', () => {
-				void deleteRollbackPendingFields(request, webResourceHandler);
-			});
+		'POSTRUN-ERROR': async ({ request }) => {
+			const fields = getWebResourceFields(request);
+
+			if (fields.length === 0) {
+				return;
+			}
+
+			const keysToDelete = getWebResourcesKeysFromRequest(fields, request);
+			// Explicitely not passing tx as it will be rolledback as it is an error hook
+			await deleteOnStorage(keysToDelete);
 		},
 	};
 };
@@ -378,21 +421,12 @@ const getWebResourcesKeysFromRequest = (
 		.filter((href) => href != null);
 };
 
-const getRemoveWebResourceHooks = (
-	webResourceHandler: WebResourceHandler,
-): sbvrUtils.Hooks => {
+const getRemoveWebResourceHooks = (): sbvrUtils.Hooks => {
 	return {
 		PRERUN: async (args) => {
 			const { api, request, tx } = args;
 			let webResourceFields = getWebResourceFields(request);
 
-			throwIfWebresourceNotInMultipart(webResourceFields, args);
-
-			// Request failed on DB roundtrip (e.g. DB constraint) and pending files need to be deleted
-			tx.on('rollback', () => {
-				void deleteRollbackPendingFields(request, webResourceHandler);
-			});
-
 			if (request.method === 'PATCH') {
 				webResourceFields = Object.entries(request.values)
 					.filter(
@@ -401,6 +435,8 @@ const getRemoveWebResourceHooks = (
 					)
 					.map(([key]) => key);
 			}
+			request.custom.webResourceFields = webResourceFields;
+			throwIfWebresourceNotInMultipart(webResourceFields, args);
 
 			if (webResourceFields.length === 0) {
 				// No need to delete anything as no file is in the wire
@@ -411,43 +447,32 @@ const getRemoveWebResourceHooks = (
 			// This can only be validated here because we need to first ensure the
 			// request is actually modifying a webresource before erroring out
 			if (request.method === 'PATCH' && request.odataQuery?.key == null) {
-				// When we get here, files have already been uploaded. We need to mark them for deletion.
-				const keysToDelete = getWebResourcesKeysFromRequest(
-					webResourceFields,
-					request,
-				);
-
-				// Set deletion of files on the wire as request will throw
-				tx.on('end', () => {
-					deletePendingFiles(keysToDelete, request, webResourceHandler);
-				});
-
 				throw new errors.BadRequestError(
 					'WebResources can only be updated when providing a resource key.',
 				);
 			}
 
-			// This can be > 1 in both DELETE requests or PATCH requests to not accessible IDs.
 			const ids = await sbvrUtils.getAffectedIds(args);
 			if (ids.length === 0) {
 				// Set deletion of files on the wire as no resource was affected
-				// Note that for DELETE requests it should not find any request on the wire
-				const keysToDelete = getWebResourcesKeysFromRequest(
+				request.custom.onPostRunDelete = getWebResourcesKeysFromRequest(
 					webResourceFields,
 					request,
 				);
-				deletePendingFiles(keysToDelete, request, webResourceHandler);
 				return;
 			}
 
+			// If it reaches here, it means that it will try to patch/delete the webresource
+			// So we need (before postrun) get what are the current keys to be deleted
+			// if post run succeeds
 			const webResources = (await api.get({
 				resource: request.resourceName,
 				passthrough: {
-					tx: args.tx,
+					tx,
 					req: permissions.root,
 				},
 				options: {
-					$select: webResourceFields,
+					$select: request.custom.webResourceFields,
 					$filter: {
 						id: {
 							$in: ids,
@@ -455,60 +480,34 @@ const getRemoveWebResourceHooks = (
 					},
 				},
 			})) as WebResourcesDbResponse[] | undefined | null;
+			request.custom.onPostRunDelete = getWebResourcesHrefs(webResources);
+		},
 
-			// Deletes previous stored resources in case they were patched or the whole entity was deleted
-			tx.on('end', () => {
-				deletePendingFiles(
-					getWebResourcesHrefs(webResources),
-					request,
-					webResourceHandler,
-				);
-			});
+		POSTRUN: async ({ request, tx }) => {
+			// Either patch or delete worked. In either case, schedule the previous existing files to delete
+			await deleteOnStorage(request.custom.onPostRunDelete ?? [], tx);
+		},
+		'POSTRUN-ERROR': async ({ request }) => {
+			const keysToDelete = getWebResourcesKeysFromRequest(
+				request.custom.webResourceFields,
+				request,
+			);
+			// Explicitely not passing tx as it will be rolledback as it is an error hook
+			await deleteOnStorage(keysToDelete);
 		},
 	};
 };
 
-const deleteRollbackPendingFields = async (
-	request: uriParser.ODataRequest,
-	webResourceHandler: WebResourceHandler,
-) => {
-	const fields = getWebResourceFields(request);
-
-	if (fields.length === 0) {
-		return;
-	}
-
-	const keysToDelete = getWebResourcesKeysFromRequest(fields, request);
-	await deleteFiles(keysToDelete, webResourceHandler);
-};
-
-const deletePendingFiles = (
-	keysToDelete: string[],
-	request: uriParser.ODataRequest,
-	webResourceHandler: WebResourceHandler,
-): void => {
-	// on purpose does not await for this promise to resolve
-	try {
-		void deleteFiles(keysToDelete, webResourceHandler);
-	} catch (err) {
-		getLogger(request.vocabulary).error(`Failed to delete pending files`, err);
-	}
-};
-
 export const getDefaultHandler = (): WebResourceHandler => {
 	return new NoopHandler();
 };
 
-export const setupUploadHooks = (
-	handler: WebResourceHandler,
-	apiRoot: string,
-	resourceName: string,
-) => {
+export const setupUploadHooks = (apiRoot: string, resourceName: string) => {
 	sbvrUtils.addPureHook(
 		'DELETE',
 		apiRoot,
 		resourceName,
-		getRemoveWebResourceHooks(handler),
+		getRemoveWebResourceHooks(),
 	);
 
 	sbvrUtils.addPureHook(
@@ -516,13 +515,48 @@ export const setupUploadHooks = (
 		apiRoot,
 		resourceName,
 		// PATCH also needs to remove the old resource in case a webresource was modified
-		getRemoveWebResourceHooks(handler),
+		getRemoveWebResourceHooks(),
 	);
 
 	sbvrUtils.addPureHook(
 		'POST',
 		apiRoot,
 		resourceName,
-		getCreateWebResourceHooks(handler),
+		getCreateWebResourceHooks(),
 	);
 };
+
+export const setupUploadActions = (apiRoot: string, resourceName: string) => {
+	const resource = sqlNameToODataName(resourceName);
+	if (isMultipartUploadAvailable(configuredWebResourceHandler)) {
+		addAction(apiRoot, resource, 'beginUpload', beginUploadAction);
+
+		addAction(apiRoot, resource, 'commitUpload', commitUploadAction);
+
+		addAction(apiRoot, resource, 'cancelUpload', cancelUploadAction);
+	}
+};
+
+const initSql = `
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
+`;
+
+const modelText = await importSBVR('./webresource.sbvr', import.meta);
+
+declare module '../sbvr-api/sbvr-utils.js' {
+	export interface API {
+		webresource: PinejsClient<WebresourceModel>;
+	}
+}
+
+export const config: ConfigLoader.Config = {
+	models: [
+		{
+			modelName: 'webresource',
+			apiRoot: 'webresource',
+			modelText,
+			initSql,
+		},
+	],
+};

package/src/webresource-handler/multipartUpload.ts

@@ -0,0 +1,23 @@
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import { NotImplementedError } from '../sbvr-api/errors.js';
+import type { WebResourceHandler } from './index.js';
+import { getWebresourceHandler } from './index.js';
+
+export type MultipartUploadHandler = WebResourceHandler &
+	Required<Pick<WebResourceHandler, 'multipartUpload'>>;
+
+export const isMultipartUploadAvailable = (
+	handler: WebResourceHandler | undefined,
+): handler is MultipartUploadHandler => {
+	return (
+		webResourceEnv.multipartUploadEnabled && handler?.multipartUpload != null
+	);
+};
+
+export const getMultipartUploadHandler = () => {
+	const handler = getWebresourceHandler();
+	if (!isMultipartUploadAvailable(handler)) {
+		throw new NotImplementedError('Multipart uploads not available');
+	}
+	return handler;
+};

package/src/webresource-handler/webresource.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: webresource
+
+Term: actor
+	Concept Type: Integer (Type)
+Term: expiry date
+	Concept Type: Date Time (Type)
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Big Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date
+	Necessity: each multipart upload has exactly one expiry date
+Fact type: multipart upload is created by actor
+	Necessity: each multipart upload is created by at most one actor

package/src/webresource-handler/webresource.ts

@@ -0,0 +1,48 @@
+// These types were generated by @balena/abstract-sql-to-typescript v5.1.0
+
+import type { Types } from '@balena/abstract-sql-to-typescript';
+
+export interface MultipartUpload {
+	Read: {
+		created_at: Types['Date Time']['Read'];
+		modified_at: Types['Date Time']['Read'];
+		id: Types['Serial']['Read'];
+		uuid: Types['Short Text']['Read'];
+		resource_name: Types['Short Text']['Read'];
+		field_name: Types['Short Text']['Read'];
+		resource_id: Types['Integer']['Read'];
+		upload_id: Types['Short Text']['Read'];
+		file_key: Types['Short Text']['Read'];
+		status: 'pending' | 'completed' | 'cancelled';
+		filename: Types['Short Text']['Read'];
+		content_type: Types['Short Text']['Read'];
+		size: Types['Big Integer']['Read'];
+		chunk_size: Types['Integer']['Read'];
+		expiry_date: Types['Date Time']['Read'];
+		is_created_by__actor: Types['Integer']['Read'] | null;
+	};
+	Write: {
+		created_at: Types['Date Time']['Write'];
+		modified_at: Types['Date Time']['Write'];
+		id: Types['Serial']['Write'];
+		uuid: Types['Short Text']['Write'];
+		resource_name: Types['Short Text']['Write'];
+		field_name: Types['Short Text']['Write'];
+		resource_id: Types['Integer']['Write'];
+		upload_id: Types['Short Text']['Write'];
+		file_key: Types['Short Text']['Write'];
+		status: 'pending' | 'completed' | 'cancelled';
+		filename: Types['Short Text']['Write'];
+		content_type: Types['Short Text']['Write'];
+		size: Types['Big Integer']['Write'];
+		chunk_size: Types['Integer']['Write'];
+		expiry_date: Types['Date Time']['Write'];
+		is_created_by__actor: Types['Integer']['Write'] | null;
+	};
+}
+
+export default interface $Model {
+	multipart_upload: MultipartUpload;
+
+
+}