@balena/pinejs 21.4.0-build-add-actions-example-00cabf6eb1629a826638b8c81dcf6d45a534872b-1 → 21.4.0-build-add-actions-example-6ac2b3e43d47ee50b22cbf3985c736f454ad73bf-1

package/src/webresource-handler/delete-file-task.ts

@@ -0,0 +1,42 @@
+import { addTaskHandler } from '../tasks/index.js';
+import { getWebresourceHandler } from './index.js';
+
+const deleteFileSchema = {
+	type: 'object',
+	properties: {
+		fileKey: {
+			type: 'string',
+		},
+	},
+	required: ['fileKey'],
+	additionalProperties: false,
+} as const;
+
+export const addDeleteFileTaskHandler = () => {
+	addTaskHandler(
+		'delete_webresource_file',
+		async (task) => {
+			const handler = getWebresourceHandler();
+			if (!handler) {
+				return {
+					error: 'Webresource handler not available',
+					status: 'failed',
+				};
+			}
+
+			try {
+				await handler.removeFile(task.params.fileKey);
+				return {
+					status: 'succeeded',
+				};
+			} catch (error) {
+				console.error('Error deleting file:', error);
+				return {
+					error: `${error}`,
+					status: 'failed',
+				};
+			}
+		},
+		deleteFileSchema,
+	);
+};

package/src/webresource-handler/index.ts

@@ -16,6 +16,8 @@ import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
+import type { Tx } from '../database-layer/db.js';
+import { isPineTasksAvailable } from '../tasks/index.js';
 import type WebresourceModel from './webresource.js';
 import { importSBVR } from '../server-glue/sbvr-loader.js';
 import { beginUploadAction } from './actions/beginUpload.js';
@@ -324,12 +326,34 @@ export const getWebResourceFields = (
 		.map((f) => sqlNameToODataName(f.fieldName));
 };
 
-const deleteFiles = async (
-	keysToDelete: string[],
-	webResourceHandler: WebResourceHandler,
-) => {
-	const promises = keysToDelete.map((r) => webResourceHandler.removeFile(r));
-	await Promise.all(promises);
+const deleteOnStorage = async (keysToDelete: string[], tx?: Tx) => {
+	if (isPineTasksAvailable()) {
+		await Promise.all(
+			keysToDelete.map(async (fileKey) => {
+				return await sbvrUtils.api.tasks.post({
+					resource: 'task',
+					passthrough: {
+						req: permissions.root,
+						tx,
+					},
+					body: {
+						key: crypto.randomUUID(),
+						is_executed_by__handler: 'delete_webresource_file',
+						is_executed_with__parameter_set: {
+							fileKey: fileKey,
+						},
+						attempt_limit: 2 ** 31 - 1,
+					},
+				});
+			}),
+		);
+	} else {
+		await Promise.all(
+			keysToDelete.map(async (fileKey) => {
+				return await configuredWebResourceHandler?.removeFile(fileKey);
+			}),
+		);
+	}
 };
 
 const throwIfWebresourceNotInMultipart = (
@@ -353,18 +377,22 @@ const throwIfWebresourceNotInMultipart = (
 	}
 };
 
-const getCreateWebResourceHooks = (
-	webResourceHandler: WebResourceHandler,
-): sbvrUtils.Hooks => {
+const getCreateWebResourceHooks = (): sbvrUtils.Hooks => {
 	return {
 		PRERUN: (hookArgs) => {
 			const webResourceFields = getWebResourceFields(hookArgs.request);
 			throwIfWebresourceNotInMultipart(webResourceFields, hookArgs);
 		},
-		'POSTRUN-ERROR': ({ tx, request }) => {
-			tx.on('rollback', () => {
-				void deleteRollbackPendingFields(request, webResourceHandler);
-			});
+		'POSTRUN-ERROR': async ({ request }) => {
+			const fields = getWebResourceFields(request);
+
+			if (fields.length === 0) {
+				return;
+			}
+
+			const keysToDelete = getWebResourcesKeysFromRequest(fields, request);
+			// Explicitely not passing tx as it will be rolledback as it is an error hook
+			await deleteOnStorage(keysToDelete);
 		},
 	};
 };
@@ -393,21 +421,12 @@ const getWebResourcesKeysFromRequest = (
 		.filter((href) => href != null);
 };
 
-const getRemoveWebResourceHooks = (
-	webResourceHandler: WebResourceHandler,
-): sbvrUtils.Hooks => {
+const getRemoveWebResourceHooks = (): sbvrUtils.Hooks => {
 	return {
 		PRERUN: async (args) => {
 			const { api, request, tx } = args;
 			let webResourceFields = getWebResourceFields(request);
 
-			throwIfWebresourceNotInMultipart(webResourceFields, args);
-
-			// Request failed on DB roundtrip (e.g. DB constraint) and pending files need to be deleted
-			tx.on('rollback', () => {
-				void deleteRollbackPendingFields(request, webResourceHandler);
-			});
-
 			if (request.method === 'PATCH') {
 				webResourceFields = Object.entries(request.values)
 					.filter(
@@ -416,6 +435,8 @@ const getRemoveWebResourceHooks = (
 					)
 					.map(([key]) => key);
 			}
+			request.custom.webResourceFields = webResourceFields;
+			throwIfWebresourceNotInMultipart(webResourceFields, args);
 
 			if (webResourceFields.length === 0) {
 				// No need to delete anything as no file is in the wire
@@ -426,43 +447,32 @@ const getRemoveWebResourceHooks = (
 			// This can only be validated here because we need to first ensure the
 			// request is actually modifying a webresource before erroring out
 			if (request.method === 'PATCH' && request.odataQuery?.key == null) {
-				// When we get here, files have already been uploaded. We need to mark them for deletion.
-				const keysToDelete = getWebResourcesKeysFromRequest(
-					webResourceFields,
-					request,
-				);
-
-				// Set deletion of files on the wire as request will throw
-				tx.on('end', () => {
-					deletePendingFiles(keysToDelete, request, webResourceHandler);
-				});
-
 				throw new errors.BadRequestError(
 					'WebResources can only be updated when providing a resource key.',
 				);
 			}
 
-			// This can be > 1 in both DELETE requests or PATCH requests to not accessible IDs.
 			const ids = await sbvrUtils.getAffectedIds(args);
 			if (ids.length === 0) {
 				// Set deletion of files on the wire as no resource was affected
-				// Note that for DELETE requests it should not find any request on the wire
-				const keysToDelete = getWebResourcesKeysFromRequest(
+				request.custom.onPostRunDelete = getWebResourcesKeysFromRequest(
 					webResourceFields,
 					request,
 				);
-				deletePendingFiles(keysToDelete, request, webResourceHandler);
 				return;
 			}
 
+			// If it reaches here, it means that it will try to patch/delete the webresource
+			// So we need (before postrun) get what are the current keys to be deleted
+			// if post run succeeds
 			const webResources = (await api.get({
 				resource: request.resourceName,
 				passthrough: {
-					tx: args.tx,
+					tx,
 					req: permissions.root,
 				},
 				options: {
-					$select: webResourceFields,
+					$select: request.custom.webResourceFields,
 					$filter: {
 						id: {
 							$in: ids,
@@ -470,60 +480,34 @@ const getRemoveWebResourceHooks = (
 					},
 				},
 			})) as WebResourcesDbResponse[] | undefined | null;
+			request.custom.onPostRunDelete = getWebResourcesHrefs(webResources);
+		},
 
-			// Deletes previous stored resources in case they were patched or the whole entity was deleted
-			tx.on('end', () => {
-				deletePendingFiles(
-					getWebResourcesHrefs(webResources),
-					request,
-					webResourceHandler,
-				);
-			});
+		POSTRUN: async ({ request, tx }) => {
+			// Either patch or delete worked. In either case, schedule the previous existing files to delete
+			await deleteOnStorage(request.custom.onPostRunDelete ?? [], tx);
+		},
+		'POSTRUN-ERROR': async ({ request }) => {
+			const keysToDelete = getWebResourcesKeysFromRequest(
+				request.custom.webResourceFields,
+				request,
+			);
+			// Explicitely not passing tx as it will be rolledback as it is an error hook
+			await deleteOnStorage(keysToDelete);
 		},
 	};
 };
 
-const deleteRollbackPendingFields = async (
-	request: uriParser.ODataRequest,
-	webResourceHandler: WebResourceHandler,
-) => {
-	const fields = getWebResourceFields(request);
-
-	if (fields.length === 0) {
-		return;
-	}
-
-	const keysToDelete = getWebResourcesKeysFromRequest(fields, request);
-	await deleteFiles(keysToDelete, webResourceHandler);
-};
-
-const deletePendingFiles = (
-	keysToDelete: string[],
-	request: uriParser.ODataRequest,
-	webResourceHandler: WebResourceHandler,
-): void => {
-	// on purpose does not await for this promise to resolve
-	try {
-		void deleteFiles(keysToDelete, webResourceHandler);
-	} catch (err) {
-		getLogger(request.vocabulary).error(`Failed to delete pending files`, err);
-	}
-};
-
 export const getDefaultHandler = (): WebResourceHandler => {
 	return new NoopHandler();
 };
 
-export const setupUploadHooks = (
-	handler: WebResourceHandler,
-	apiRoot: string,
-	resourceName: string,
-) => {
+export const setupUploadHooks = (apiRoot: string, resourceName: string) => {
 	sbvrUtils.addPureHook(
 		'DELETE',
 		apiRoot,
 		resourceName,
-		getRemoveWebResourceHooks(handler),
+		getRemoveWebResourceHooks(),
 	);
 
 	sbvrUtils.addPureHook(
@@ -531,24 +515,20 @@ export const setupUploadHooks = (
 		apiRoot,
 		resourceName,
 		// PATCH also needs to remove the old resource in case a webresource was modified
-		getRemoveWebResourceHooks(handler),
+		getRemoveWebResourceHooks(),
 	);
 
 	sbvrUtils.addPureHook(
 		'POST',
 		apiRoot,
 		resourceName,
-		getCreateWebResourceHooks(handler),
+		getCreateWebResourceHooks(),
 	);
 };
 
-export const setupUploadActions = (
-	handler: WebResourceHandler,
-	apiRoot: string,
-	resourceName: string,
-) => {
+export const setupUploadActions = (apiRoot: string, resourceName: string) => {
 	const resource = sqlNameToODataName(resourceName);
-	if (isMultipartUploadAvailable(handler)) {
+	if (isMultipartUploadAvailable(configuredWebResourceHandler)) {
 		addAction(apiRoot, resource, 'beginUpload', beginUploadAction);
 
 		addAction(apiRoot, resource, 'commitUpload', commitUploadAction);

package/src/webresource-handler/webresource.sbvr

@@ -23,7 +23,7 @@ Term: filename
 Term: content type
 	Concept Type: Short Text (Type)
 Term: size
-	Concept Type: Integer (Type)
+	Concept Type: Big Integer (Type)
 Term: chunk size
 	Concept Type: Integer (Type)
 Term: valid until date

package/src/webresource-handler/webresource.ts

@@ -16,7 +16,7 @@ export interface MultipartUpload {
 		status: 'pending' | 'completed' | 'cancelled';
 		filename: Types['Short Text']['Read'];
 		content_type: Types['Short Text']['Read'];
-		size: Types['Integer']['Read'];
+		size: Types['Big Integer']['Read'];
 		chunk_size: Types['Integer']['Read'];
 		expiry_date: Types['Date Time']['Read'];
 		is_created_by__actor: Types['Integer']['Read'] | null;
@@ -34,7 +34,7 @@ export interface MultipartUpload {
 		status: 'pending' | 'completed' | 'cancelled';
 		filename: Types['Short Text']['Write'];
 		content_type: Types['Short Text']['Write'];
-		size: Types['Integer']['Write'];
+		size: Types['Big Integer']['Write'];
 		chunk_size: Types['Integer']['Write'];
 		expiry_date: Types['Date Time']['Write'];
 		is_created_by__actor: Types['Integer']['Write'] | null;