@balena/pinejs 15.1.0-build-web-resource-4-60a3313d5465335b98b5a93fdfa92d801e914f5a-4 → 15.1.0-build-upsert-6f9a458cc52c017ddaf48ceb7e4597522cdf4dfc-1

package/src/server-glue/webresource-handler.ts

@@ -1,344 +0,0 @@
-import type * as Express from 'express';
-import * as busboy from 'busboy';
-import * as is from 'type-is';
-import * as stream from 'stream';
-import * as uriParser from '../sbvr-api/uri-parser';
-import { getApiRoot, getModel } from '../sbvr-api/sbvr-utils';
-import { checkPermissions } from '../sbvr-api/permissions';
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-import { S3Handler } from './webresource-handlers/S3Handler';
-import { NoopHandler } from './webresource-handlers/NoopHandler';
-import {
-	odataNameToSqlName,
-	sqlNameToODataName,
-} from '@balena/odata-to-abstract-sql';
-import { errors, permissions } from './module';
-
-export interface IncomingFile {
-	fieldname: string;
-	originalname: string;
-	encoding: string;
-	mimetype: string;
-	stream: stream.Readable;
-}
-
-export interface UploadResponse {
-	size: number;
-	filename: string;
-}
-
-export interface WebResourceHandler {
-	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
-	removeFile: (fileReference: string) => Promise<void>;
-}
-
-export class WebResourceError extends Error {
-	constructor(message: string, originalError?: Error) {
-		super(message);
-		this.name = 'WebResourceError';
-
-		if (originalError) {
-			this.originalError = originalError;
-		}
-	}
-
-	originalError?: Error;
-}
-
-export class FileSizeExceededError extends WebResourceError {
-	constructor(maxSize: number) {
-		super(`File size exceeded the limit of ${maxSize} bytes.`);
-		this.name = 'FileSizeExceededError';
-	}
-}
-
-type WebResourcesDbResponse = {
-	[fieldname: string]: { href: string } | undefined | null;
-};
-
-const ifFileInValidPath = async (
-	fieldname: string,
-	req: Express.Request,
-): Promise<boolean> => {
-	if (req.method !== 'POST' && req.method !== 'PATCH') {
-		return false;
-	}
-
-	const apiRoot = getApiRoot(req);
-	if (apiRoot == null) {
-		return false;
-	}
-	const model = getModel(apiRoot);
-	const { resourceName } = await uriParser.parseOData({
-		url: req.url,
-		method: req.method,
-	});
-
-	const permission = req.method === 'POST' ? 'create' : 'update';
-	const vocab = model.versions[model.versions.length - 1];
-	const hasPermissions = await checkPermissions(
-		req,
-		permission,
-		resourceName,
-		vocab,
-	);
-
-	if (!hasPermissions) {
-		return false;
-	}
-
-	// TODO: This could be cached
-	const fields = model.abstractSql.tables[resourceName].fields;
-	const dbFieldName = odataNameToSqlName(fieldname);
-	for (const field of fields) {
-		if (field.fieldName === dbFieldName && field.dataType === 'WebResource') {
-			return true;
-		}
-	}
-
-	// TODO: We could do a pre-check if there is a SBVR rule specifying file max size
-	// This would avoid needing the roundtrip to DB and uploading a file just to remove it
-
-	return false;
-};
-
-export const getUploaderMiddlware = (
-	handler: WebResourceHandler,
-): Express.RequestHandler => {
-	return async (req, res, next) => {
-		if (!is(req, ['multipart'])) {
-			return next();
-		}
-		const filesUploaded: string[] = [];
-		const completeUploads: Array<Promise<void>> = [];
-
-		const bb = busboy({ headers: req.headers });
-		let isAborting = false;
-
-		const done = () => {
-			req.unpipe(bb);
-			req.on('readable', req.read.bind(req));
-			bb.removeAllListeners();
-		};
-
-		const clearFiles = () => {
-			isAborting = true;
-			const deletions = filesUploaded.map((file) => handler.removeFile(file));
-			// Best effort: We try to remove all uploaded files, but if this fails, there is not much to do
-			return Promise.all(deletions).catch((err) =>
-				console.error('Error deleting file', err),
-			);
-		};
-
-		bb.on('file', async (fieldname, filestream, info) => {
-			if (!isAborting && (await ifFileInValidPath(fieldname, req))) {
-				const file: IncomingFile = {
-					originalname: info.filename,
-					encoding: info.encoding,
-					mimetype: info.mimeType,
-					stream: filestream,
-					fieldname,
-				};
-				const promise = handler.handleFile(file).then((result) => {
-					req.body[fieldname] = {
-						filename: info.filename,
-						contentType: info.mimeType,
-						contentDisposition: undefined,
-						size: result.size,
-						href: result.filename,
-					};
-					filesUploaded.push(result.filename);
-				});
-				completeUploads.push(promise);
-			} else {
-				filestream.resume();
-			}
-		});
-
-		bb.on('field', (name, val, _info) => {
-			req.body[name] = val;
-		});
-
-		bb.on('finish', async () => {
-			try {
-				await Promise.all(completeUploads);
-				done();
-				next();
-			} catch (err: any) {
-				await clearFiles();
-
-				if (err instanceof FileSizeExceededError) {
-					return sbvrUtils.handleHttpErrors(
-						req,
-						res,
-						new errors.BadRequestError(err.message),
-					);
-				}
-
-				console.error('Error uploading file', err);
-				next(err);
-			}
-		});
-
-		bb.on('error', async (err) => {
-			await clearFiles();
-			done();
-			next(err);
-		});
-		req.pipe(bb);
-	};
-};
-
-// TODO: this can be cached
-const getWebResourceFields = (request: uriParser.ODataRequest): string[] => {
-	const fields =
-		request.abstractSqlModel?.tables[request.resourceName]?.modifyFields ??
-		request.abstractSqlModel?.tables[request.resourceName]?.fields;
-	if (fields == null) {
-		return [];
-	}
-
-	return fields
-		.filter((f) => f.dataType === 'WebResource')
-		.map((f) => sqlNameToODataName(f.fieldName));
-};
-
-const getModifiedFields = (request: uriParser.ODataRequest): string[] => {
-	return Object.entries(request.values)
-		.filter(([_key, value]) => value !== undefined)
-		.map(([key, _value]) => key);
-};
-
-const deleteFiles = async (
-	keysToDelete: string[],
-	webResourceHandler: WebResourceHandler,
-) => {
-	const promises = keysToDelete.map((r) => webResourceHandler.removeFile(r));
-	await Promise.all(promises);
-};
-
-const getCreateWebResourceHook = (webResourceHandler: WebResourceHandler) => {
-	return {
-		'POSTRUN-ERROR': async ({ tx, request }) => {
-			tx?.on('rollback', async () => {
-				const fields = getWebResourceFields(request);
-
-				if (fields.length === 0) {
-					return;
-				}
-
-				const keysToDelete: string[] = fields
-					.filter((f) => isDefined(request.values[f]))
-					.map((f) => request.values[f].href);
-				await deleteFiles(keysToDelete, webResourceHandler);
-			});
-		},
-	} as sbvrUtils.Hooks;
-};
-
-const isDefined = <T>(x: T | undefined | null): x is T => x != null;
-
-const getWebResourcesHrefs = (webResources?: WebResourcesDbResponse | null) => {
-	const hrefs = Object.values(webResources ?? {})
-		.filter(isDefined)
-		.map((resourceKey) => resourceKey.href);
-	return hrefs;
-};
-
-const getRemoveWebResourceHook = (webResourceHandler: WebResourceHandler) => {
-	return {
-		PRERUN: async (args) => {
-			const { api, request } = args;
-			let fields = getWebResourceFields(request);
-
-			if (request.method === 'PATCH') {
-				if (request.odataQuery?.key == null) {
-					throw new errors.BadRequestError(
-						'WebResources can only be updated when providing a resource key.',
-					);
-				}
-				const allFields = getModifiedFields(request);
-				fields = fields.filter((f) => allFields.includes(f));
-			}
-
-			if (fields.length === 0) {
-				return;
-			}
-
-			const [id, rest] = await sbvrUtils.getAffectedIds(args);
-			if (id == null) {
-				return;
-			}
-
-			if (rest != null) {
-				throw new errors.BadRequestError(
-					'Resources containing webresources can only be updated/deleted one at a time.',
-				);
-			}
-
-			const webResources = (await api.get({
-				resource: request.resourceName,
-				passthrough: {
-					tx: args.tx,
-					req: permissions.root,
-				},
-				id,
-				options: {
-					$select: fields,
-				},
-			})) as WebResourcesDbResponse | undefined | null;
-
-			request.custom.$pineWebResourcesToDelete =
-				getWebResourcesHrefs(webResources);
-		},
-		POSTRUN: ({ tx, request }) => {
-			tx.on('end', () => {
-				const keysToDelete: string[] =
-					request.custom.$pineWebResourcesToDelete || [];
-				// on purpose does not await for this promise to resolve
-				deleteFiles(keysToDelete, webResourceHandler).catch((err) => {
-					console.error(`Failed to delete pending files`, err);
-				});
-			});
-		},
-	} as sbvrUtils.Hooks;
-};
-
-export const getDefaultHandler = (): WebResourceHandler => {
-	let handler: WebResourceHandler;
-	try {
-		handler = new S3Handler();
-	} catch (e) {
-		console.warn(`Failed to initialize S3 handler, using noop ${e}`);
-		handler = new NoopHandler();
-	}
-	return handler;
-};
-
-export const setupUploadHooks = (
-	handler: WebResourceHandler,
-	apiRoot: string,
-	resourceName: string,
-) => {
-	sbvrUtils.addPureHook(
-		'DELETE',
-		apiRoot,
-		resourceName,
-		getRemoveWebResourceHook(handler),
-	);
-
-	sbvrUtils.addPureHook(
-		'PATCH',
-		apiRoot,
-		resourceName,
-		// PATCH also needs to remove the old resource in case a webresource was modified
-		getRemoveWebResourceHook(handler),
-	);
-
-	sbvrUtils.addPureHook(
-		'POST',
-		apiRoot,
-		resourceName,
-		getCreateWebResourceHook(handler),
-	);
-};

package/src/server-glue/webresource-handlers/NoopHandler.ts

@@ -1,20 +0,0 @@
-import {
-	IncomingFile,
-	UploadResponse,
-	WebResourceHandler,
-} from '../webresource-handler';
-
-export class NoopHandler implements WebResourceHandler {
-	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
-		// handleFile must consume the file stream
-		resource.stream.resume();
-		return {
-			filename: 'noop',
-			size: 0,
-		};
-	}
-
-	public async removeFile(_fileReference: string): Promise<void> {
-		return;
-	}
-}

package/src/server-glue/webresource-handlers/S3Handler.ts

@@ -1,101 +0,0 @@
-import { optionalVar, requiredVar, intVar } from '@balena/env-parsing';
-import {
-	FileSizeExceededError,
-	IncomingFile,
-	UploadResponse,
-	WebResourceError,
-	WebResourceHandler,
-} from '../webresource-handler';
-import {
-	S3Client,
-	S3ClientConfig,
-	DeleteObjectCommand,
-	PutObjectCommandInput,
-} from '@aws-sdk/client-s3';
-import { Upload } from '@aws-sdk/lib-storage';
-
-import { randomUUID } from 'crypto';
-
-export class S3Handler implements WebResourceHandler {
-	private readonly config: S3ClientConfig;
-	private readonly bucket: string;
-	private readonly endpoint: string;
-	private readonly maxFileSize = intVar(
-		'PINEJS_WEBRESOURCE_MAXFILESIZE',
-		52428800,
-	);
-	private client: S3Client;
-
-	constructor() {
-		this.endpoint = requiredVar('S3_ENDPOINT');
-		this.config = {
-			region: optionalVar('S3_REGION', 'us-east-1'),
-			credentials: {
-				accessKeyId: requiredVar('S3_ACCESS_KEY'),
-				secretAccessKey: requiredVar('S3_SECRET_KEY'),
-			},
-			endpoint: this.endpoint,
-			forcePathStyle: true,
-		};
-
-		this.bucket = optionalVar(
-			'S3_STORAGE_ADAPTER_BUCKET',
-			'balena-pine-web-resources',
-		);
-		this.client = new S3Client(this.config);
-	}
-
-	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
-		let size = 0;
-		const key = `${resource.fieldname}_${randomUUID()}_${
-			resource.originalname
-		}`;
-		const params: PutObjectCommandInput = {
-			Bucket: this.bucket,
-			ACL: 'public-read',
-			StorageClass: 'STANDARD',
-			Key: key,
-			Body: resource.stream,
-			ContentType: resource.mimetype,
-		};
-		const upload = new Upload({ client: this.client, params });
-
-		upload.on('httpUploadProgress', async (ev) => {
-			size = ev.total ? ev.total : ev.loaded!;
-			if (size > this.maxFileSize) {
-				await upload.abort();
-			}
-		});
-
-		try {
-			await upload.done();
-		} catch (err: any) {
-			resource.stream.resume();
-			if (size > this.maxFileSize) {
-				throw new FileSizeExceededError(this.maxFileSize);
-			}
-			throw new WebResourceError('Failed uploading resource', err);
-		}
-
-		const filename = this.getS3URL(key);
-		return { size, filename };
-	}
-
-	public async removeFile(fileReference: string): Promise<void> {
-		const fileReferences = fileReference.split('/');
-		const fileKey = fileReferences[fileReferences.length - 1];
-
-		const command = new DeleteObjectCommand({
-			Bucket: this.bucket,
-			Key: fileKey,
-		});
-
-		await this.client.send(command);
-	}
-
-	private getS3URL(key: string) {
-		return this.endpoint.includes(this.bucket)
-			? `${this.endpoint}/${key}`
-			: `${this.endpoint}/${this.bucket}/${key}`;
-	}
-}