@balena/pinejs 21.4.0-build-add-odata-actions-459d3f3fccd7ac2f79550ad13c8f3f5a818c9ad5-1 → 21.4.0-build-add-actions-example-d3a479f7c677c487d5486ddafa43ff08265b5e83-2

package/src/webresource-handler/actions/canceUpload.ts

@@ -0,0 +1,92 @@
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../sbvr-api/actions.js';
+import type { Tx } from '../../database-layer/db.js';
+import {
+	BadRequestError,
+	NotImplementedError,
+	UnauthorizedError,
+} from '../../sbvr-api/errors.js';
+import { api, type Response } from '../../sbvr-api/sbvr-utils.js';
+import { permissions } from '../../server-glue/module.js';
+import { getMultipartUploadHandler } from '../multipartUpload.js';
+
+export const cancelUploadAction = async ({
+	request,
+	tx,
+	id: resourceId,
+}: ODataActionArgs): Promise<Response> => {
+	if (typeof resourceId !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+	const { id, fileKey, uploadId } = await getOngoingUpload(
+		request,
+		resourceId,
+		tx,
+	);
+	const handler = getMultipartUploadHandler();
+
+	await api.webresource.patch({
+		resource: 'multipart_upload',
+		body: {
+			status: 'cancelled',
+		},
+		id,
+		passthrough: {
+			tx: tx,
+			req: permissions.root,
+		},
+	});
+
+	// Note that different then beginUpload/commitUpload where we first do the action on the external service
+	// and then reflect it on the DB, for cancel upload it is the other way around
+	// as the worst case scenario is having a canceled upload which is marked on the DB as something else
+	await handler.multipartUpload.cancel({ fileKey, uploadId });
+
+	return {
+		statusCode: 204,
+	};
+};
+
+const getOngoingUpload = async (
+	request: ODataActionRequest,
+	affectedId: number,
+	tx: Tx,
+) => {
+	const { uuid } = request.values;
+	if (uuid == null || typeof uuid !== 'string') {
+		throw new BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id'],
+			$filter: {
+				// TODO: swap this into the ID to see if it works
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+				resource_name: request.resourceName,
+				resource_id: affectedId,
+			},
+		},
+		passthrough: {
+			tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new UnauthorizedError();
+	}
+
+	return {
+		id: multipartUpload.id,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+	};
+};

package/src/webresource-handler/actions/commitUpload.ts

@@ -0,0 +1,117 @@
+import type { Tx } from '../../database-layer/db.js';
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../sbvr-api/actions.js';
+import {
+	BadRequestError,
+	NotImplementedError,
+	UnauthorizedError,
+} from '../../sbvr-api/errors.js';
+import type { Response } from '../../sbvr-api/sbvr-utils.js';
+import { api } from '../../sbvr-api/sbvr-utils.js';
+import { permissions } from '../../server-glue/module.js';
+import { getMultipartUploadHandler } from '../multipartUpload.js';
+
+export const commitUploadAction = async ({
+	request,
+	tx,
+	id,
+	api: applicationApi,
+}: ODataActionArgs): Promise<Response> => {
+	if (typeof id !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+
+	const multipartUpload = await getOngoingUpload(request, id, tx);
+	const handler = getMultipartUploadHandler();
+
+	const webresource = await handler.multipartUpload.commit({
+		fileKey: multipartUpload.fileKey,
+		uploadId: multipartUpload.uploadId,
+		filename: multipartUpload.filename,
+		providerCommitData: multipartUpload.providerCommitData,
+	});
+
+	await Promise.all([
+		api.webresource.patch({
+			resource: 'multipart_upload',
+			body: {
+				status: 'completed',
+			},
+			options: {
+				$filter: {
+					uuid: multipartUpload.uuid,
+				},
+			},
+			passthrough: {
+				tx: tx,
+				req: permissions.root,
+			},
+		}),
+		applicationApi.patch({
+			resource: request.resourceName,
+			id,
+			body: {
+				[multipartUpload.fieldName]: webresource,
+			},
+			passthrough: {
+				tx: tx,
+				// Root is needed as, if you are not root, you are not allowed to directly modify the actual metadata
+				req: permissions.root,
+			},
+		}),
+	]);
+
+	const body = await handler.onPreRespond(webresource);
+
+	return {
+		body,
+		statusCode: 200,
+	};
+};
+
+const getOngoingUpload = async (
+	request: ODataActionRequest,
+	affectedId: number,
+	tx: Tx,
+) => {
+	const { uuid, providerCommitData } = request.values;
+	if (uuid == null || typeof uuid !== 'string') {
+		throw new BadRequestError('Invalid uuid type');
+	}
+
+	const [multipartUpload] = await api.webresource.get({
+		resource: 'multipart_upload',
+		options: {
+			$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+			$filter: {
+				// TODO: swap this into the ID to see if it works
+				uuid,
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+				resource_name: request.resourceName,
+				resource_id: affectedId,
+			},
+		},
+		passthrough: {
+			tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new UnauthorizedError();
+	}
+
+	return {
+		uuid,
+		providerCommitData,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+		filename: multipartUpload.filename,
+		fieldName: multipartUpload.field_name,
+	};
+};

package/src/webresource-handler/delete-file-task.ts

@@ -0,0 +1,42 @@
+import { addTaskHandler } from '../tasks/index.js';
+import { getWebresourceHandler } from './index.js';
+
+const deleteFileSchema = {
+	type: 'object',
+	properties: {
+		fileKey: {
+			type: 'string',
+		},
+	},
+	required: ['fileKey'],
+	additionalProperties: false,
+} as const;
+
+export const addDeleteFileTaskHandler = () => {
+	addTaskHandler(
+		'delete_webresource_file',
+		async (task) => {
+			const handler = getWebresourceHandler();
+			if (!handler) {
+				return {
+					error: 'Webresource handler not available',
+					status: 'failed',
+				};
+			}
+
+			try {
+				await handler.removeFile(task.params.fileKey);
+				return {
+					status: 'succeeded',
+				};
+			} catch (error) {
+				console.error('Error deleting file:', error);
+				return {
+					error: `${error}`,
+					status: 'failed',
+				};
+			}
+		},
+		deleteFileSchema,
+	);
+};

package/src/webresource-handler/index.ts

@@ -11,10 +11,19 @@ import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
+import type { ConfigLoader } from '../server-glue/module.js';
 import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
+import { isPineTasksAvailable } from '../tasks/index.js';
+import type WebresourceModel from './webresource.js';
+import { importSBVR } from '../server-glue/sbvr-loader.js';
+import { beginUploadAction } from './actions/beginUpload.js';
+import { isMultipartUploadAvailable } from './multipartUpload.js';
+import { commitUploadAction } from './actions/commitUpload.js';
+import { cancelUploadAction } from './actions/canceUpload.js';
+import { addAction } from '../sbvr-api/actions.js';
 
 export * from './handlers/index.js';
 
@@ -320,7 +329,24 @@ const deleteFiles = async (
 	keysToDelete: string[],
 	webResourceHandler: WebResourceHandler,
 ) => {
-	const promises = keysToDelete.map((r) => webResourceHandler.removeFile(r));
+	const promises = isPineTasksAvailable()
+		? keysToDelete.map((fileKey) =>
+				sbvrUtils.api.tasks.post({
+					resource: 'task',
+					passthrough: {
+						req: permissions.root,
+					},
+					body: {
+						key: crypto.randomUUID(),
+						is_executed_by__handler: 'delete_webresource_file',
+						is_executed_with__parameter_set: {
+							fileKey: fileKey,
+						},
+						attempt_limit: 2 ** 31 - 1,
+					},
+				}),
+			)
+		: keysToDelete.map((r) => webResourceHandler.removeFile(r));
 	await Promise.all(promises);
 };
 
@@ -329,6 +355,13 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		// root needs to be able to bypass the multipart check as
+		// it needs to pass the direct payload on multipart uploads (on storage provider)
+		req.user !== permissions.root.user &&
+		// This is checking for HTTP multipart form submission/request (e.g. send the actual file via the API)
+		// Not to confuse with multipart uploads
+		// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Methods/POST#multipart_form_submission
+		// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -526,3 +559,38 @@ export const setupUploadHooks = (
 		getCreateWebResourceHooks(handler),
 	);
 };
+
+export const setupUploadActions = (apiRoot: string, resourceName: string) => {
+	const resource = sqlNameToODataName(resourceName);
+	if (isMultipartUploadAvailable(configuredWebResourceHandler)) {
+		addAction(apiRoot, resource, 'beginUpload', beginUploadAction);
+
+		addAction(apiRoot, resource, 'commitUpload', commitUploadAction);
+
+		addAction(apiRoot, resource, 'cancelUpload', cancelUploadAction);
+	}
+};
+
+const initSql = `
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
+`;
+
+const modelText = await importSBVR('./webresource.sbvr', import.meta);
+
+declare module '../sbvr-api/sbvr-utils.js' {
+	export interface API {
+		webresource: PinejsClient<WebresourceModel>;
+	}
+}
+
+export const config: ConfigLoader.Config = {
+	models: [
+		{
+			modelName: 'webresource',
+			apiRoot: 'webresource',
+			modelText,
+			initSql,
+		},
+	],
+};

package/src/webresource-handler/multipartUpload.ts

@@ -0,0 +1,23 @@
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import { NotImplementedError } from '../sbvr-api/errors.js';
+import type { WebResourceHandler } from './index.js';
+import { getWebresourceHandler } from './index.js';
+
+export type MultipartUploadHandler = WebResourceHandler &
+	Required<Pick<WebResourceHandler, 'multipartUpload'>>;
+
+export const isMultipartUploadAvailable = (
+	handler: WebResourceHandler | undefined,
+): handler is MultipartUploadHandler => {
+	return (
+		webResourceEnv.multipartUploadEnabled && handler?.multipartUpload != null
+	);
+};
+
+export const getMultipartUploadHandler = () => {
+	const handler = getWebresourceHandler();
+	if (!isMultipartUploadAvailable(handler)) {
+		throw new NotImplementedError('Multipart uploads not available');
+	}
+	return handler;
+};

package/src/webresource-handler/webresource.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: webresource
+
+Term: actor
+	Concept Type: Integer (Type)
+Term: expiry date
+	Concept Type: Date Time (Type)
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Big Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date
+	Necessity: each multipart upload has exactly one expiry date
+Fact type: multipart upload is created by actor
+	Necessity: each multipart upload is created by at most one actor

package/src/webresource-handler/webresource.ts

@@ -0,0 +1,48 @@
+// These types were generated by @balena/abstract-sql-to-typescript v5.1.0
+
+import type { Types } from '@balena/abstract-sql-to-typescript';
+
+export interface MultipartUpload {
+	Read: {
+		created_at: Types['Date Time']['Read'];
+		modified_at: Types['Date Time']['Read'];
+		id: Types['Serial']['Read'];
+		uuid: Types['Short Text']['Read'];
+		resource_name: Types['Short Text']['Read'];
+		field_name: Types['Short Text']['Read'];
+		resource_id: Types['Integer']['Read'];
+		upload_id: Types['Short Text']['Read'];
+		file_key: Types['Short Text']['Read'];
+		status: 'pending' | 'completed' | 'cancelled';
+		filename: Types['Short Text']['Read'];
+		content_type: Types['Short Text']['Read'];
+		size: Types['Big Integer']['Read'];
+		chunk_size: Types['Integer']['Read'];
+		expiry_date: Types['Date Time']['Read'];
+		is_created_by__actor: Types['Integer']['Read'] | null;
+	};
+	Write: {
+		created_at: Types['Date Time']['Write'];
+		modified_at: Types['Date Time']['Write'];
+		id: Types['Serial']['Write'];
+		uuid: Types['Short Text']['Write'];
+		resource_name: Types['Short Text']['Write'];
+		field_name: Types['Short Text']['Write'];
+		resource_id: Types['Integer']['Write'];
+		upload_id: Types['Short Text']['Write'];
+		file_key: Types['Short Text']['Write'];
+		status: 'pending' | 'completed' | 'cancelled';
+		filename: Types['Short Text']['Write'];
+		content_type: Types['Short Text']['Write'];
+		size: Types['Big Integer']['Write'];
+		chunk_size: Types['Integer']['Write'];
+		expiry_date: Types['Date Time']['Write'];
+		is_created_by__actor: Types['Integer']['Write'] | null;
+	};
+}
+
+export default interface $Model {
+	multipart_upload: MultipartUpload;
+
+
+}