@balena/pinejs 21.5.1-build-object-freeze-root-objects-266a461f8ffb383c7c5ae4290263f237d530e42d-1 → 21.6.0-build-add-actions-example-a346f3cb25b768b2b2bbd62bcbc85fbf35eb0908-1

package/src/webresource-handler/actions/beginUpload.ts

@@ -0,0 +1,165 @@
+import { api, type Response } from '../../sbvr-api/sbvr-utils.js';
+import type { MultipartUploadHandler } from '../multipartUpload.js';
+import { getMultipartUploadHandler } from '../multipartUpload.js';
+import type { BeginMultipartUploadPayload } from '../index.js';
+import { getWebResourceFields } from '../index.js';
+import { BadRequestError, NotImplementedError } from '../../sbvr-api/errors.js';
+import { permissions, sbvrUtils } from '../../server-glue/module.js';
+import { randomUUID } from 'crypto';
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../sbvr-api/actions.js';
+import { ajv } from '../../tasks/common.js';
+import type { FromSchema } from 'json-schema-to-ts';
+
+type FakeWebResourcePatch = {
+	[key: string]: Omit<WebResource, 'href'> & {
+		href: 'fake_patch';
+	};
+};
+
+const beginUploadPayloadSchema = {
+	type: 'object',
+	minProperties: 1,
+	maxProperties: 1,
+	additionalProperties: {
+		type: 'object',
+		properties: {
+			filename: { type: 'string' },
+			content_type: { type: 'string' },
+			size: { type: 'number' },
+			chunk_size: { type: 'number' },
+		},
+		required: ['filename', 'content_type', 'size'],
+		additionalProperties: false,
+	},
+} as const;
+
+const validateBeginUpload = ajv.compile<
+	FromSchema<typeof beginUploadPayloadSchema>
+>(beginUploadPayloadSchema);
+
+const beginUploadAction = async ({
+	request,
+	tx,
+	id,
+	req,
+}: ODataActionArgs): Promise<Response> => {
+	if (typeof id !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+
+	const handler = getMultipartUploadHandler();
+	const { fieldName, beginUploadPayload } = parseBeginUpload(request, handler);
+
+	await runFakeDbPatch(request, {
+		[fieldName]: { ...beginUploadPayload, href: 'fake_patch' },
+	});
+
+	const { fileKey, uploadId, uploadParts } =
+		await handler.multipartUpload.begin(fieldName, beginUploadPayload);
+	const uuid = randomUUID();
+	await api.webresource.post({
+		resource: 'multipart_upload',
+		body: {
+			uuid,
+			resource_name: request.resourceName,
+			field_name: fieldName,
+			resource_id: id,
+			upload_id: uploadId,
+			file_key: fileKey,
+			status: 'pending',
+			...beginUploadPayload,
+			expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
+			is_created_by__actor: req.user?.actor,
+		},
+		passthrough: {
+			req: permissions.root,
+			tx,
+		},
+	});
+
+	return {
+		body: { [fieldName]: { uuid, uploadParts } },
+		statusCode: 200,
+	};
+};
+
+const parseBeginUpload = (
+	request: ODataActionRequest,
+	webResourceHandler: MultipartUploadHandler,
+) => {
+	const { values } = request;
+	const isValid = validateBeginUpload(values);
+	if (!isValid) {
+		throw new BadRequestError('Invalid begin upload payload');
+	}
+
+	const fieldName = Object.keys(values)[0];
+
+	const webResourceFields = getWebResourceFields(request, false);
+	if (!webResourceFields.includes(fieldName)) {
+		throw new BadRequestError(
+			`The provided field '${fieldName}' is not a valid webresource`,
+		);
+	}
+	const beginUploadPayload = {
+		...values[fieldName],
+		chunk_size:
+			values[fieldName].chunk_size ??
+			webResourceHandler.multipartUpload.getDefaultPartSize(),
+	} satisfies BeginMultipartUploadPayload;
+
+	if (
+		beginUploadPayload.chunk_size <
+		webResourceHandler.multipartUpload.getMinimumPartSize()
+	) {
+		throw new BadRequestError('Chunk size is too small');
+	}
+
+	return {
+		beginUploadPayload,
+		fieldName,
+	};
+};
+
+// We want beginUpload to fail if the initial payload would already break any constraint/hooks/rule
+// this serves as both an optimization and a better client experience as it avoids succeding a beginUpload
+// (which could allow for potentially very large PartUploads) only for it to fail on commitUpload
+// The current approach to try to achieve this behavior creates this fake database patch:
+// It first tries to patch the file metadata (on a separate tx) to something similar to what it would look like
+// and if it breaks any constraint, then it throws and fail to even start the upload
+// Note that because the href is only generated at commitUpload time, we need to fake it for the time being
+// This could potentially be a problem if an application code has a constraint on the href property
+// However, because this is a internal (pine managed) property we assumed that application code won't do so
+const runFakeDbPatch = async (
+	request: ODataActionRequest,
+	fakeDbPatch: FakeWebResourcePatch,
+) => {
+	const fakeTx = await sbvrUtils.db.transaction();
+	try {
+		const newUrl = request.url
+			.slice(1)
+			.split('?', 1)[0]
+			.replace(/\/beginUpload$/, '');
+
+		await api[request.vocabulary].request({
+			method: 'PATCH',
+			url: newUrl,
+			body: fakeDbPatch,
+			// it needs root as otherwise it would always fail as non-root users
+			// are not allowed to directly patch webresource metadata, onnly upload files
+			passthrough: { tx: fakeTx, req: permissions.root },
+		});
+	} finally {
+		if (!fakeTx.isClosed()) {
+			await fakeTx.rollback();
+		}
+	}
+};
+
+export default beginUploadAction;

package/src/webresource-handler/actions/cancelUpload.ts

@@ -0,0 +1,95 @@
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../sbvr-api/actions.js';
+import type { Tx } from '../../database-layer/db.js';
+import {
+	BadRequestError,
+	NotImplementedError,
+	UnauthorizedError,
+} from '../../sbvr-api/errors.js';
+import { api, type Response } from '../../sbvr-api/sbvr-utils.js';
+import { permissions } from '../../server-glue/module.js';
+import { getMultipartUploadHandler } from '../multipartUpload.js';
+
+const cancelUploadAction = async ({
+	request,
+	tx,
+	id: resourceId,
+}: ODataActionArgs): Promise<Response> => {
+	if (typeof resourceId !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+	const { id, fileKey, uploadId } = await getOngoingUpload(
+		request,
+		resourceId,
+		tx,
+	);
+	const handler = getMultipartUploadHandler();
+
+	await api.webresource.patch({
+		resource: 'multipart_upload',
+		body: {
+			status: 'cancelled',
+		},
+		id,
+		passthrough: {
+			tx: tx,
+			req: permissions.root,
+		},
+	});
+
+	// Note that different then beginUpload/commitUpload where we first do the action on the external service
+	// and then reflect it on the DB, for cancel upload it is the other way around
+	// as the worst case scenario is having a canceled upload which is marked on the DB as something else
+	await handler.multipartUpload.cancel({ fileKey, uploadId });
+
+	return {
+		statusCode: 204,
+	};
+};
+
+const getOngoingUpload = async (
+	request: ODataActionRequest,
+	affectedId: number,
+	tx: Tx,
+) => {
+	const { uuid } = request.values;
+	if (uuid == null || typeof uuid !== 'string') {
+		throw new BadRequestError('Invalid uuid type');
+	}
+
+	const multipartUpload = await api.webresource.get({
+		resource: 'multipart_upload',
+		id: {
+			uuid,
+		},
+		options: {
+			$select: ['id', 'file_key', 'upload_id'],
+			$filter: {
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+				resource_name: request.resourceName,
+				resource_id: affectedId,
+			},
+		},
+		passthrough: {
+			tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new UnauthorizedError();
+	}
+
+	return {
+		id: multipartUpload.id,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+	};
+};
+
+export default cancelUploadAction;

package/src/webresource-handler/actions/commitUpload.ts

@@ -0,0 +1,120 @@
+import type { Tx } from '../../database-layer/db.js';
+import type {
+	ODataActionArgs,
+	ODataActionRequest,
+} from '../../sbvr-api/actions.js';
+import {
+	BadRequestError,
+	NotImplementedError,
+	UnauthorizedError,
+} from '../../sbvr-api/errors.js';
+import type { Response } from '../../sbvr-api/sbvr-utils.js';
+import { api } from '../../sbvr-api/sbvr-utils.js';
+import { permissions } from '../../server-glue/module.js';
+import { getMultipartUploadHandler } from '../multipartUpload.js';
+
+const commitUploadAction = async ({
+	request,
+	tx,
+	id,
+	api: applicationApi,
+}: ODataActionArgs): Promise<Response> => {
+	if (typeof id !== 'number') {
+		throw new NotImplementedError(
+			'multipart upload do not yet support non-numeric ids',
+		);
+	}
+
+	const multipartUpload = await getOngoingUpload(request, id, tx);
+	const handler = getMultipartUploadHandler();
+
+	const webresource = await handler.multipartUpload.commit({
+		fileKey: multipartUpload.fileKey,
+		uploadId: multipartUpload.uploadId,
+		filename: multipartUpload.filename,
+		providerCommitData: multipartUpload.providerCommitData,
+	});
+
+	await Promise.all([
+		api.webresource.patch({
+			resource: 'multipart_upload',
+			body: {
+				status: 'completed',
+			},
+			options: {
+				$filter: {
+					uuid: multipartUpload.uuid,
+				},
+			},
+			passthrough: {
+				tx: tx,
+				req: permissions.root,
+			},
+		}),
+		applicationApi.patch({
+			resource: request.resourceName,
+			id,
+			body: {
+				[multipartUpload.fieldName]: webresource,
+			},
+			passthrough: {
+				tx: tx,
+				// Root is needed as, if you are not root, you are not allowed to directly modify the actual metadata
+				req: permissions.root,
+			},
+		}),
+	]);
+
+	const body = await handler.onPreRespond(webresource);
+
+	return {
+		body,
+		statusCode: 200,
+	};
+};
+
+const getOngoingUpload = async (
+	request: ODataActionRequest,
+	affectedId: number,
+	tx: Tx,
+) => {
+	const { uuid, providerCommitData } = request.values;
+	if (uuid == null || typeof uuid !== 'string') {
+		throw new BadRequestError('Invalid uuid type');
+	}
+
+	const multipartUpload = await api.webresource.get({
+		resource: 'multipart_upload',
+		id: {
+			uuid,
+		},
+		options: {
+			$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+			$filter: {
+				status: 'pending',
+				expiry_date: { $gt: { $now: {} } },
+				resource_name: request.resourceName,
+				resource_id: affectedId,
+			},
+		},
+		passthrough: {
+			tx,
+			req: permissions.rootRead,
+		},
+	});
+
+	if (multipartUpload == null) {
+		throw new UnauthorizedError();
+	}
+
+	return {
+		uuid,
+		providerCommitData,
+		fileKey: multipartUpload.file_key,
+		uploadId: multipartUpload.upload_id,
+		filename: multipartUpload.filename,
+		fieldName: multipartUpload.field_name,
+	};
+};
+
+export default commitUploadAction;

package/src/webresource-handler/actions/index.ts

@@ -0,0 +1,5 @@
+import beginUpload from './beginUpload.js';
+import commitUpload from './commitUpload.js';
+import cancelUpload from './cancelUpload.js';
+
+export { beginUpload, commitUpload, cancelUpload };

package/src/webresource-handler/index.ts

@@ -11,11 +11,17 @@ import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
+import type { ConfigLoader } from '../server-glue/module.js';
 import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
 import { canExecuteTasks } from '../tasks/index.js';
+import { importSBVR } from '../server-glue/sbvr-loader.js';
+import type WebresourceModel from './webresource.js';
+import { isMultipartUploadAvailable } from './multipartUpload.js';
+import { addAction } from '../sbvr-api/actions.js';
+import { beginUpload, commitUpload, cancelUpload } from './actions/index.js';
 
 export * from './handlers/index.js';
 
@@ -352,6 +358,13 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		// root needs to be able to bypass the multipart check as
+		// it needs to pass the direct payload on multipart uploads (on storage provider)
+		req.user !== permissions.root.user &&
+		// This is checking for HTTP multipart form submission/request (e.g. send the actual file via the API)
+		// Not to confuse with multipart uploads
+		// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Methods/POST#multipart_form_submission
+		// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -549,3 +562,36 @@ export const setupUploadHooks = (
 		getCreateWebResourceHooks(handler),
 	);
 };
+
+export const setupUploadActions = (vocab: string, resourceName: string) => {
+	const resource = sqlNameToODataName(resourceName);
+	if (isMultipartUploadAvailable(configuredWebResourceHandler)) {
+		addAction(vocab, resource, 'beginUpload', beginUpload);
+		addAction(vocab, resource, 'commitUpload', commitUpload);
+		addAction(vocab, resource, 'cancelUpload', cancelUpload);
+	}
+};
+
+const initSql = `
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
+`;
+
+const modelText = await importSBVR('./webresource.sbvr', import.meta);
+
+declare module '../sbvr-api/sbvr-utils.js' {
+	export interface API {
+		webresource: PinejsClient<WebresourceModel>;
+	}
+}
+
+export const config: ConfigLoader.Config = {
+	models: [
+		{
+			modelName: 'webresource',
+			apiRoot: 'webresource',
+			modelText,
+			initSql,
+		},
+	],
+};

package/src/webresource-handler/multipartUpload.ts

@@ -0,0 +1,23 @@
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import { NotImplementedError } from '../sbvr-api/errors.js';
+import type { WebResourceHandler } from './index.js';
+import { getWebresourceHandler } from './index.js';
+
+export type MultipartUploadHandler = WebResourceHandler &
+	Required<Pick<WebResourceHandler, 'multipartUpload'>>;
+
+export const isMultipartUploadAvailable = (
+	handler: WebResourceHandler | undefined,
+): handler is MultipartUploadHandler => {
+	return (
+		webResourceEnv.multipartUploadEnabled && handler?.multipartUpload != null
+	);
+};
+
+export const getMultipartUploadHandler = () => {
+	const handler = getWebresourceHandler();
+	if (!isMultipartUploadAvailable(handler)) {
+		throw new NotImplementedError('Multipart uploads not available');
+	}
+	return handler;
+};

package/src/webresource-handler/webresource.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: webresource
+
+Term: actor
+	Concept Type: Integer (Type)
+Term: expiry date
+	Concept Type: Date Time (Type)
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Big Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date
+	Necessity: each multipart upload has exactly one expiry date
+Fact type: multipart upload is created by actor
+	Necessity: each multipart upload is created by at most one actor

package/src/webresource-handler/webresource.ts

@@ -0,0 +1,48 @@
+// These types were generated by @balena/abstract-sql-to-typescript v5.1.0
+
+import type { Types } from '@balena/abstract-sql-to-typescript';
+
+export interface MultipartUpload {
+	Read: {
+		created_at: Types['Date Time']['Read'];
+		modified_at: Types['Date Time']['Read'];
+		id: Types['Serial']['Read'];
+		uuid: Types['Short Text']['Read'];
+		resource_name: Types['Short Text']['Read'];
+		field_name: Types['Short Text']['Read'];
+		resource_id: Types['Integer']['Read'];
+		upload_id: Types['Short Text']['Read'];
+		file_key: Types['Short Text']['Read'];
+		status: 'pending' | 'completed' | 'cancelled';
+		filename: Types['Short Text']['Read'];
+		content_type: Types['Short Text']['Read'];
+		size: Types['Big Integer']['Read'];
+		chunk_size: Types['Integer']['Read'];
+		expiry_date: Types['Date Time']['Read'];
+		is_created_by__actor: Types['Integer']['Read'] | null;
+	};
+	Write: {
+		created_at: Types['Date Time']['Write'];
+		modified_at: Types['Date Time']['Write'];
+		id: Types['Serial']['Write'];
+		uuid: Types['Short Text']['Write'];
+		resource_name: Types['Short Text']['Write'];
+		field_name: Types['Short Text']['Write'];
+		resource_id: Types['Integer']['Write'];
+		upload_id: Types['Short Text']['Write'];
+		file_key: Types['Short Text']['Write'];
+		status: 'pending' | 'completed' | 'cancelled';
+		filename: Types['Short Text']['Write'];
+		content_type: Types['Short Text']['Write'];
+		size: Types['Big Integer']['Write'];
+		chunk_size: Types['Integer']['Write'];
+		expiry_date: Types['Date Time']['Write'];
+		is_created_by__actor: Types['Integer']['Write'] | null;
+	};
+}
+
+export default interface $Model {
+	multipart_upload: MultipartUpload;
+
+
+}