@balena/pinejs 15.0.2-build-renovate-faker-js-faker-8-x-80784e9ad7062c0f45e3d91ce8b9862a22bd8edb-1 → 15.1.0-build-web-resource-4-366a6aee494d7b10ba96d42af0b39b91c57220b3-1

package/src/server-glue/webresource-handler.ts

@@ -0,0 +1,327 @@
+import type * as Express from 'express';
+import * as busboy from 'busboy';
+import * as is from 'type-is';
+import * as stream from 'stream';
+import * as uriParser from '../sbvr-api/uri-parser';
+import { getApiRoot, getModel } from '../sbvr-api/sbvr-utils';
+import { checkPermissions } from '../sbvr-api/permissions';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+import { S3Handler } from './webresource-handlers/S3Handler';
+import { NoopHandler } from './webresource-handlers/NoopHandler';
+import {
+	odataNameToSqlName,
+	sqlNameToODataName,
+} from '@balena/odata-to-abstract-sql';
+import { errors, permissions } from './module';
+
+export interface IncomingFile {
+	fieldname: string;
+	originalname: string;
+	encoding: string;
+	mimetype: string;
+	stream: stream.Readable;
+}
+
+export interface UploadResponse {
+	size: number;
+	filename: string;
+}
+
+export interface WebResourceHandler {
+	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
+	removeFile: (fileReference: string) => Promise<void>;
+}
+
+type WebResourcesDbResponse = {
+	[fieldname: string]: { href: string } | undefined | null;
+};
+
+const ifFileInValidPath = async (
+	fieldname: string,
+	req: Express.Request,
+): Promise<boolean> => {
+	if (req.method !== 'POST' && req.method !== 'PATCH') {
+		return false;
+	}
+
+	const apiRoot = getApiRoot(req);
+	if (apiRoot == null) {
+		return false;
+	}
+	const model = getModel(apiRoot);
+	const { resourceName } = await uriParser.parseOData({
+		url: req.url,
+		method: req.method,
+	});
+
+	const permission = req.method === 'POST' ? 'create' : 'update';
+	const vocab = model.versions[model.versions.length - 1];
+	const hasPermissions = await checkPermissions(
+		req,
+		permission,
+		resourceName,
+		vocab,
+	);
+
+	if (!hasPermissions) {
+		return false;
+	}
+
+	// TODO: This could be cached
+	const fields = model.abstractSql.tables[resourceName].fields;
+	const dbFieldName = odataNameToSqlName(fieldname);
+	for (const field of fields) {
+		if (field.fieldName === dbFieldName && field.dataType === 'WebResource') {
+			return true;
+		}
+	}
+
+	// TODO: We could do a pre-check if there is a SBVR rule specifying file max size
+	// This would avoid needing the roundtrip to DB and uploading a file just to remove it
+
+	return false;
+};
+
+export const getUploaderMiddlware = (
+	handler: WebResourceHandler,
+): Express.RequestHandler => {
+	return async (req, res, next) => {
+		if (!is(req, ['multipart'])) {
+			return next();
+		}
+		const filesUploaded: string[] = [];
+		const completeUploads: Array<Promise<void>> = [];
+
+		const bb = busboy({ headers: req.headers });
+		let isAborting = false;
+
+		const done = () => {
+			req.unpipe(bb);
+			req.on('readable', req.read.bind(req));
+			bb.removeAllListeners();
+		};
+
+		const clearFiles = () => {
+			isAborting = true;
+			const deletions = filesUploaded.map((file) => handler.removeFile(file));
+			// Best effort: We try to remove all uploaded files, but if this fails, there is not much to do
+			return Promise.all(deletions).catch((err) =>
+				console.error('Error deleting file', err),
+			);
+		};
+
+		bb.on('file', async (fieldname, filestream, info) => {
+			if (!isAborting && (await ifFileInValidPath(fieldname, req))) {
+				const file: IncomingFile = {
+					originalname: info.filename,
+					encoding: info.encoding,
+					mimetype: info.mimeType,
+					stream: filestream,
+					fieldname,
+				};
+				const promise = handler.handleFile(file).then((result) => {
+					req.body[fieldname] = {
+						filename: info.filename,
+						contentType: info.mimeType,
+						contentDisposition: undefined,
+						size: result.size,
+						href: result.filename,
+					};
+					filesUploaded.push(result.filename);
+				});
+				completeUploads.push(promise);
+			} else {
+				filestream.resume();
+			}
+		});
+
+		bb.on('field', (name, val, _info) => {
+			req.body[name] = val;
+		});
+
+		bb.on('finish', async () => {
+			try {
+				await Promise.all(completeUploads);
+				done();
+				next();
+			} catch (err: any) {
+				await clearFiles();
+
+				if (err?.message.includes('File size exceeded')) {
+					return sbvrUtils.handleHttpErrors(
+						req,
+						res,
+						new errors.BadRequestError('File size exceeded'),
+					);
+				}
+
+				console.error('Error uploading file', err);
+				next(err);
+			}
+		});
+
+		bb.on('error', async (err) => {
+			await clearFiles();
+			done();
+			next(err);
+		});
+		req.pipe(bb);
+	};
+};
+
+// TODO: this can be cached
+const getWebResourceFields = (request: uriParser.ODataRequest): string[] => {
+	const fields =
+		request.abstractSqlModel?.tables[request.resourceName]?.modifyFields ??
+		request.abstractSqlModel?.tables[request.resourceName]?.fields;
+	if (fields == null) {
+		return [];
+	}
+
+	return fields
+		.filter((f) => f.dataType === 'WebResource')
+		.map((f) => sqlNameToODataName(f.fieldName));
+};
+
+const getModifiedFields = (request: uriParser.ODataRequest): string[] => {
+	return Object.entries(request.values)
+		.filter(([_key, value]) => value !== undefined)
+		.map(([key, _value]) => key);
+};
+
+const deleteFiles = async (
+	keysToDelete: string[],
+	webResourceHandler: WebResourceHandler,
+) => {
+	const promises = keysToDelete.map((r) => webResourceHandler.removeFile(r));
+	await Promise.all(promises);
+};
+
+const getCreateWebResourceHook = (webResourceHandler: WebResourceHandler) => {
+	return {
+		'POSTRUN-ERROR': async ({ tx, request }) => {
+			tx?.on('rollback', async () => {
+				const fields = getWebResourceFields(request);
+
+				if (fields.length === 0) {
+					return;
+				}
+
+				const keysToDelete: string[] = fields
+					.filter((f) => isDefined(request.values[f]))
+					.map((f) => request.values[f].href);
+				await deleteFiles(keysToDelete, webResourceHandler);
+			});
+		},
+	} as sbvrUtils.Hooks;
+};
+
+const isDefined = <T>(x: T | undefined | null): x is T => x != null;
+
+const getWebResourcesHrefs = (webResources?: WebResourcesDbResponse | null) => {
+	const hrefs = Object.values(webResources ?? {})
+		.filter(isDefined)
+		.map((resourceKey) => resourceKey.href);
+	return hrefs;
+};
+
+const getRemoveWebResourceHook = (webResourceHandler: WebResourceHandler) => {
+	return {
+		PRERUN: async (args) => {
+			const { api, request } = args;
+			let fields = getWebResourceFields(request);
+
+			if (fields.length === 0) {
+				return;
+			}
+
+			if (request.method === 'PATCH') {
+				if (request.odataQuery?.key == null) {
+					throw new errors.BadRequestError(
+						'WebResources can only be updated when providing a resource key.',
+					);
+				}
+				const allFields = getModifiedFields(request);
+				fields = fields.filter((f) => allFields.includes(f));
+			}
+
+			if (fields.length === 0) {
+				return;
+			}
+
+			const [id, rest] = await sbvrUtils.getAffectedIds(args);
+			if (id == null) {
+				return;
+			}
+
+			if (rest != null) {
+				throw new errors.BadRequestError(
+					'Resources containing webresources can only be updated/deleted one at a time.',
+				);
+			}
+
+			const webResources = (await api.get({
+				resource: request.resourceName,
+				passthrough: {
+					tx: args.tx,
+					req: permissions.root,
+				},
+				id,
+				options: {
+					$select: fields,
+				},
+			})) as WebResourcesDbResponse | undefined | null;
+
+			request.custom.$pineWebResourcesToDelete =
+				getWebResourcesHrefs(webResources);
+		},
+		POSTRUN: ({ tx, request }) => {
+			tx.on('end', () => {
+				const keysToDelete: string[] =
+					request.custom.$pineWebResourcesToDelete || [];
+				// on purpose does not await for this promise to resolve
+				deleteFiles(keysToDelete, webResourceHandler).catch((err) => {
+					console.error(`Failed to delete pending files`, err);
+				});
+			});
+		},
+	} as sbvrUtils.Hooks;
+};
+
+export const getDefaultHandler = (): WebResourceHandler => {
+	let handler: WebResourceHandler;
+	try {
+		handler = new S3Handler();
+	} catch (e) {
+		console.warn(`Failed to initialize S3 handler, using noop ${e}`);
+		handler = new NoopHandler();
+	}
+	return handler;
+};
+
+export const setupUploadHooks = (
+	handler: WebResourceHandler,
+	apiRoot: string,
+	resourceName: string,
+) => {
+	sbvrUtils.addPureHook(
+		'PATCH',
+		apiRoot,
+		resourceName,
+		getRemoveWebResourceHook(handler),
+	);
+
+	sbvrUtils.addPureHook(
+		'DELETE',
+		apiRoot,
+		resourceName,
+		getRemoveWebResourceHook(handler),
+	);
+
+	sbvrUtils.addPureHook(
+		'POST',
+		apiRoot,
+		resourceName,
+		getCreateWebResourceHook(handler),
+	);
+};

package/src/server-glue/webresource-handlers/NoopHandler.ts

@@ -0,0 +1,20 @@
+import {
+	IncomingFile,
+	UploadResponse,
+	WebResourceHandler,
+} from '../webresource-handler';
+
+export class NoopHandler implements WebResourceHandler {
+	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
+		// handleFile must consume the file stream
+		resource.stream.resume();
+		return {
+			filename: 'noop',
+			size: 0,
+		};
+	}
+
+	public async removeFile(_fileReference: string): Promise<void> {
+		return;
+	}
+}

package/src/server-glue/webresource-handlers/S3Handler.ts

@@ -0,0 +1,99 @@
+import { optionalVar, requiredVar, intVar } from '@balena/env-parsing';
+import {
+	IncomingFile,
+	UploadResponse,
+	WebResourceHandler,
+} from '../webresource-handler';
+import {
+	S3Client,
+	S3ClientConfig,
+	DeleteObjectCommand,
+	PutObjectCommandInput,
+} from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+
+import { randomUUID } from 'crypto';
+
+export class S3Handler implements WebResourceHandler {
+	private readonly config: S3ClientConfig;
+	private readonly bucket: string;
+	private readonly endpoint: string;
+	private readonly maxFileSize = intVar(
+		'PINEJS_WEBRESOURCE_MAXFILESIZE',
+		52428800,
+	);
+	private client: S3Client;
+
+	constructor() {
+		this.endpoint = requiredVar('S3_ENDPOINT');
+		this.config = {
+			region: optionalVar('S3_REGION', 'us-east-1'),
+			credentials: {
+				accessKeyId: requiredVar('S3_ACCESS_KEY'),
+				secretAccessKey: requiredVar('S3_SECRET_KEY'),
+			},
+			endpoint: this.endpoint,
+			forcePathStyle: true,
+		};
+
+		this.bucket = optionalVar(
+			'S3_STORAGE_ADAPTER_BUCKET',
+			'balena-pine-web-resources',
+		);
+		this.client = new S3Client(this.config);
+	}
+
+	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
+		let size = 0;
+		const key = `${resource.fieldname}_${randomUUID()}_${
+			resource.originalname
+		}`;
+		const params: PutObjectCommandInput = {
+			Bucket: this.bucket,
+			ACL: 'public-read',
+			StorageClass: 'STANDARD',
+			Key: key,
+			Body: resource.stream,
+			ContentType: resource.mimetype,
+		};
+		const upload = new Upload({ client: this.client, params });
+
+		upload.on('httpUploadProgress', async (ev) => {
+			size = ev.total ? ev.total : ev.loaded!;
+			if (size > this.maxFileSize) {
+				await upload.abort();
+			}
+		});
+
+		try {
+			await upload.done();
+		} catch (err: any) {
+			resource.stream.resume();
+			if (size > this.maxFileSize) {
+				throw new Error('File size exceeded');
+			}
+			throw err;
+		}
+
+		const filename = this.getS3URL(key);
+		return { size, filename };
+	}
+
+	public async removeFile(fileReference: string): Promise<void> {
+		const fileReferences = fileReference.split('/');
+		const fileKey = fileReferences[fileReferences.length - 1];
+
+		const command = new DeleteObjectCommand({
+			Bucket: this.bucket,
+			Key: fileKey,
+		});
+
+		await this.client.send(command);
+	}
+
+	private getS3URL(key: string) {
+		return this.endpoint.includes(this.bucket)
+			? `${this.endpoint}/${key}`
+			: `${this.endpoint}/${this.bucket}/${key}`;
+	}
+}