@balena/pinejs 16.2.0-build-joshbwlng-tasks-046f828587ae6d1889a3ae3298b3e44e96c74f90-1 → 17.0.0-build-wip-large-file-uploads-d6522dad962bc0bff6ee7c596df8f43f596b6aaa-1

package/src/database-layer/db.ts

@@ -98,13 +98,6 @@ export interface Database extends BaseDatabase {
 	) => Promise<Result>;
 	transaction: TransactionFn;
 	readTransaction: TransactionFn;
-	on?: (
-		name: 'notification',
-		fn: (...args: any[]) => Promise<void>,
-		options?: {
-			channel?: string;
-		},
-	) => void;
 }
 
 interface EngineParams {
@@ -696,23 +689,6 @@ if (maybePg != null) {
 		return {
 			engine: Engines.postgres,
 			executeSql: atomicExecuteSql,
-			on: async (name, fn, options) => {
-				if (name === 'notification' && options?.channel === undefined) {
-					throw new Error('Missing channel option for notification listener');
-				}
-
-				const client = await pool.connect();
-				client.on(name, (msg) => {
-					fn(msg).catch((error) => {
-						console.error('Error handling message:', error);
-					});
-				});
-
-				if (name === 'notification' && options?.channel !== undefined) {
-					await client.query(`LISTEN "${options.channel}";`);
-					// client.release();
-				}
-			},
 			transaction: createTransaction(async (stackTraceErr) => {
 				const client = await pool.connect();
 				const tx = new PostgresTx(client, false, stackTraceErr);

package/src/migrator/utils.ts

@@ -1,7 +1,7 @@
 import type { Result, Tx } from '../database-layer/db';
 import type { Resolvable } from '../sbvr-api/common-types';
 
-import { createHash } from 'crypto';
+import { createHash } from 'node:crypto';
 import { Engines } from '@balena/abstract-sql-compiler';
 import _ from 'lodash';
 import { TypedError } from 'typed-error';

package/src/sbvr-api/sbvr-utils.ts

@@ -42,7 +42,6 @@ import { generateODataMetadata } from '../odata-metadata/odata-metadata-generato
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const devModel = require('./dev.sbvr');
-import * as tasks from '../tasks';
 import * as permissions from './permissions';
 import {
 	BadRequestError,
@@ -78,7 +77,6 @@ export {
 	addPureHook,
 	addSideEffectHook,
 } from './hooks';
-export { addTaskHandler } from '../tasks';
 
 import memoizeWeak = require('memoizee/weak');
 import * as controlFlow from './control-flow';
@@ -775,7 +773,7 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
 	// Hence, skipped migrations from earlier models are not set as executed as the `migration` table is missing
 	// Here the skipped migrations that haven't been set properly are covered
 	// This is mostly an edge case when running on an empty database schema and migrations model hasn't been executed, yet.
-	// One specific case are tests to run tests against migrated and unmigrated database states
+	// One specifc case are tests to run tests against migrated and unmigrated database states
 
 	for (const modelKey of Object.keys(models)) {
 		const pendingToSetExecutedMigrations =
@@ -1985,7 +1983,6 @@ export const executeStandardModels = async (tx: Db.Tx): Promise<void> => {
 			},
 		});
 		await executeModels(tx, permissions.config.models);
-		await executeModels(tx, tasks.config.models);
 		console.info('Successfully executed standard models.');
 	} catch (err: any) {
 		console.error('Failed to execute standard models.', err);
@@ -2002,7 +1999,6 @@ export const setup = async (
 		await db.transaction(async (tx) => {
 			await executeStandardModels(tx);
 			await permissions.setup();
-			await tasks.setup($db, tx);
 		});
 	} catch (err: any) {
 		console.error('Could not execute standard models', err);

package/src/server-glue/module.ts

@@ -19,7 +19,6 @@ export * as errors from '../sbvr-api/errors';
 export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
-export * as tasks from '../tasks';
 export * as webResourceHandler from '../webresource-handler';
 export type { configLoader as ConfigLoader };
 export type { migratorUtils as Migrator };

package/src/webresource-handler/handlers/NoopHandler.ts

@@ -1,5 +1,10 @@
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import type { IncomingFile, UploadResponse, WebResourceHandler } from '..';
+import type {
+	BeginUploadHandlerResponse,
+	BeginUploadPayload,
+	CommitUploadHandlerPayload,
+} from '../multipartUpload';
 
 export class NoopHandler implements WebResourceHandler {
 	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
@@ -18,4 +23,20 @@ export class NoopHandler implements WebResourceHandler {
 	public async onPreRespond(webResource: WebResource): Promise<WebResource> {
 		return webResource;
 	}
+
+	public async beginUpload(
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_fieldName: string,
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_payload: BeginUploadPayload,
+	): Promise<BeginUploadHandlerResponse> {
+		return { fileKey: 'noop', uploadId: 'noop', uploadUrls: [] };
+	}
+
+	public async commitUpload(
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_payload: CommitUploadHandlerPayload,
+	): Promise<WebResource> {
+		return { filename: 'noop', href: 'noop' };
+	}
 }

package/src/webresource-handler/handlers/S3Handler.ts

@@ -6,17 +6,27 @@ import {
 	WebResourceError,
 	type WebResourceHandler,
 } from '..';
+import type {
+	BeginUploadHandlerResponse,
+	BeginUploadPayload,
+	CommitUploadHandlerPayload,
+	UploadUrl,
+} from '../multipartUpload';
 import {
 	S3Client,
 	type S3ClientConfig,
 	DeleteObjectCommand,
 	type PutObjectCommandInput,
 	GetObjectCommand,
+	CreateMultipartUploadCommand,
+	UploadPartCommand,
+	CompleteMultipartUploadCommand,
+	HeadObjectCommand,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
-import { randomUUID } from 'crypto';
+import { randomUUID } from 'node:crypto';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import memoize from 'memoizee';
 
@@ -71,9 +81,7 @@ export class S3Handler implements WebResourceHandler {
 
 	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
 		let size = 0;
-		const key = `${resource.fieldname}_${randomUUID()}_${
-			resource.originalname
-		}`;
+		const key = this.getFileKey(resource.fieldname, resource.originalname);
 		const params: PutObjectCommandInput = {
 			Bucket: this.bucket,
 			Key: key,
@@ -122,6 +130,62 @@ export class S3Handler implements WebResourceHandler {
 		return webResource;
 	}
 
+	public async beginUpload(
+		fieldName: string,
+		payload: BeginUploadPayload,
+	): Promise<BeginUploadHandlerResponse> {
+		const fileKey = this.getFileKey(fieldName, payload.filename);
+
+		const createMultiPartResponse = await this.client.send(
+			new CreateMultipartUploadCommand({
+				Bucket: this.bucket,
+				Key: fileKey,
+				ContentType: payload.content_type,
+			}),
+		);
+
+		if (createMultiPartResponse.UploadId == null) {
+			throw new WebResourceError('Failed to create multipart upload.');
+		}
+
+		const uploadUrls = await this.getPartUploadUrls(
+			fileKey,
+			createMultiPartResponse.UploadId,
+			payload,
+		);
+		return { fileKey, uploadId: createMultiPartResponse.UploadId, uploadUrls };
+	}
+
+	public async commitUpload({
+		fileKey,
+		uploadId,
+		filename,
+		multipartUploadChecksums,
+	}: CommitUploadHandlerPayload): Promise<WebResource> {
+		await this.client.send(
+			new CompleteMultipartUploadCommand({
+				Bucket: this.bucket,
+				Key: fileKey,
+				UploadId: uploadId,
+				MultipartUpload: multipartUploadChecksums,
+			}),
+		);
+
+		const headResult = await this.client.send(
+			new HeadObjectCommand({
+				Bucket: this.bucket,
+				Key: fileKey,
+			}),
+		);
+
+		return {
+			href: this.getS3URL(fileKey),
+			filename: filename,
+			size: headResult.ContentLength,
+			content_type: headResult.ContentType,
+		};
+	}
+
 	private s3SignUrl(fileKey: string): Promise<string> {
 		const command = new GetObjectCommand({
 			Bucket: this.bucket,
@@ -136,8 +200,70 @@ export class S3Handler implements WebResourceHandler {
 		return `${this.config.endpoint}/${this.bucket}/${key}`;
 	}
 
+	private getFileKey(fieldName: string, fileName: string) {
+		return `${fieldName}_${randomUUID()}_${fileName}`;
+	}
+
 	private getKeyFromHref(href: string): string {
 		const hrefWithoutParams = normalizeHref(href);
 		return hrefWithoutParams.substring(hrefWithoutParams.lastIndexOf('/') + 1);
 	}
+
+	private async getPartUploadUrls(
+		fileKey: string,
+		uploadId: string,
+		payload: BeginUploadPayload,
+	): Promise<UploadUrl[]> {
+		const chunkSizesWithParts = await this.getChunkSizesWithParts(
+			payload.size,
+			payload.chunk_size,
+		);
+		return Promise.all(
+			chunkSizesWithParts.map(async ({ chunkSize, partNumber }) => ({
+				chunkSize,
+				partNumber,
+				url: await this.getPartUploadUrl(
+					fileKey,
+					uploadId,
+					partNumber,
+					chunkSize,
+				),
+			})),
+		);
+	}
+
+	private async getPartUploadUrl(
+		fileKey: string,
+		uploadId: string,
+		partNumber: number,
+		partSize: number,
+	): Promise<string> {
+		const command = new UploadPartCommand({
+			Bucket: this.bucket,
+			Key: fileKey,
+			UploadId: uploadId,
+			PartNumber: partNumber,
+			ContentLength: partSize,
+		});
+
+		return getSignedUrl(this.client, command, {
+			expiresIn: this.signedUrlExpireTimeSeconds,
+		});
+	}
+
+	private async getChunkSizesWithParts(
+		size: number,
+		chunkSize: number,
+	): Promise<Array<Pick<UploadUrl, 'chunkSize' | 'partNumber'>>> {
+		const chunkSizesWithParts = [];
+		let partNumber = 1;
+		let remainingSize = size;
+		while (remainingSize > 0) {
+			const currentChunkSize = Math.min(remainingSize, chunkSize);
+			chunkSizesWithParts.push({ chunkSize: currentChunkSize, partNumber });
+			remainingSize -= currentChunkSize;
+			partNumber += 1;
+		}
+		return chunkSizesWithParts;
+	}
 }

package/src/webresource-handler/index.ts

@@ -7,6 +7,12 @@ import type { HookArgs } from '../sbvr-api/hooks';
 import { getApiRoot, getModel } from '../sbvr-api/sbvr-utils';
 import { checkPermissions } from '../sbvr-api/permissions';
 import { NoopHandler } from './handlers/NoopHandler';
+import type {
+	BeginUploadHandlerResponse,
+	BeginUploadPayload,
+	CommitUploadHandlerPayload,
+} from './multipartUpload';
+import { multipartUploadHooks } from './multipartUpload';
 import {
 	odataNameToSqlName,
 	sqlNameToODataName,
@@ -34,6 +40,14 @@ export interface WebResourceHandler {
 	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
 	removeFile: (fileReference: string) => Promise<void>;
 	onPreRespond: (webResource: WebResource) => Promise<WebResource>;
+
+	beginUpload: (
+		fieldName: string,
+		payload: BeginUploadPayload,
+	) => Promise<BeginUploadHandlerResponse>;
+	commitUpload: (
+		commitInfo: CommitUploadHandlerPayload,
+	) => Promise<WebResource>;
 }
 
 export class WebResourceError extends TypedError {}
@@ -216,7 +230,7 @@ export const getUploaderMiddlware = (
 	};
 };
 
-const getWebResourceFields = (
+export const getWebResourceFields = (
 	request: uriParser.ODataRequest,
 	useTranslations = true,
 ): string[] => {
@@ -249,6 +263,8 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		request.custom.isAction !== 'beginUpload' &&
+		request.custom.isAction !== 'commitUpload' &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -447,4 +463,11 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+
+	sbvrUtils.addPureHook(
+		'POST',
+		apiRoot,
+		resourceName,
+		multipartUploadHooks(handler),
+	);
 };

package/src/webresource-handler/multipartUpload.ts

@@ -0,0 +1,214 @@
+import type { AnyObject } from 'pinejs-client-core';
+import type { WebResourceHandler } from '.';
+import { getWebResourceFields } from '.';
+import type { ODataRequest } from '../sbvr-api/uri-parser';
+import { errors, sbvrUtils } from '../server-glue/module';
+import type { WebResource } from '@balena/sbvr-types/out/types/web-resource';
+import { randomUUID } from 'node:crypto';
+
+export interface BeginUploadPayload {
+	filename: string;
+	content_type: string;
+	size: number;
+	chunk_size: number;
+}
+
+type BeginUploadDbCheck = BeginUploadPayload & WebResource;
+
+export interface UploadUrl {
+	url: string;
+	chunkSize: number;
+	partNumber: number;
+}
+
+export interface BeginUploadHandlerResponse {
+	uploadUrls: UploadUrl[];
+	fileKey: string;
+	uploadId: string;
+}
+
+export interface PendingUpload extends BeginUploadPayload {
+	fieldName: string;
+	fileKey: string;
+	uploadId: string;
+}
+
+export interface BeginUploadResponse {
+	[fieldName: string]: {
+		key: string;
+		uploadUrls: UploadUrl[];
+	};
+}
+export interface CommitUploadHandlerPayload {
+	fileKey: string;
+	uploadId: string;
+	filename: string;
+	multipartUploadChecksums?: AnyObject;
+}
+
+const MB = 1024 * 1024;
+
+export const multipartUploadHooks = (
+	webResourceHandler: WebResourceHandler,
+): sbvrUtils.Hooks => {
+	return {
+		POSTPARSE: async ({ req, request, tx }) => {
+			if (request.odataQuery.property?.resource === 'beginUpload') {
+				const uploadParams = parseBeginUpload(request);
+
+				await sbvrUtils.api[request.vocabulary].post({
+					url: request.url.substring(1).replace('beginUpload', 'canAccess'),
+					body: { method: 'PATCH' },
+				});
+
+				// This transaction is necessary because beginUpload requests
+				// will rollback the transaction (in order to first validate)
+				// The metadata requested. If we don't pass any transaction
+				// It will use the default transaction handler which will error out
+				// on any rollback.
+				tx = await sbvrUtils.db.transaction();
+				req.tx = tx;
+				request.tx = tx;
+
+				request.method = 'PATCH';
+				request.values = uploadParams;
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'beginUpload';
+			} else if (request.odataQuery.property?.resource === 'commitUpload') {
+				const commitPayload = parseCommitUpload(request);
+
+				await sbvrUtils.api[request.vocabulary].post({
+					url: request.url.substring(1).replace('commitUpload', 'canAccess'),
+					body: { method: 'PATCH' },
+				});
+
+				const webresource = await webResourceHandler.commitUpload({
+					fileKey: commitPayload.metadata.fileKey,
+					uploadId: commitPayload.metadata.uploadId,
+					filename: commitPayload.metadata.filename,
+					multipartUploadChecksums: commitPayload.additionalCommitInfo,
+				});
+				delete someKvStoreDbOrRedisIdc[commitPayload.key];
+
+				request.method = 'PATCH';
+				request.values = {
+					[commitPayload.metadata.fieldName]: webresource,
+				};
+				request.odataQuery.resource = request.resourceName;
+				delete request.odataQuery.property;
+				request.custom.isAction = 'commitUpload';
+				request.custom.commitUploadPayload = webresource;
+			}
+		},
+		PRERESPOND: async ({ request, response, tx }) => {
+			if (request.custom.isAction === 'beginUpload') {
+				await tx.rollback();
+				response.statusCode = 200;
+				response.body = await beginUpload(
+					webResourceHandler,
+					request.values as { [x: string]: BeginUploadPayload },
+				);
+			} else if (request.custom.isAction === 'commitUpload') {
+				response.body = await webResourceHandler.onPreRespond(
+					request.custom.commitUploadPayload,
+				);
+			}
+		},
+	};
+};
+
+// Really any storage will do.
+// I just honestly believe redis better suits this user case but I am fine if we agree on storing on DB
+// This is not a cache, it is a persistent KV storage.
+const someKvStoreDbOrRedisIdc: Record<string, PendingUpload> = {};
+
+export const beginUpload = async (
+	webResourceHandler: WebResourceHandler,
+	payload: { [fieldName: string]: BeginUploadPayload },
+): Promise<BeginUploadResponse> => {
+	const fieldName = Object.keys(payload)[0];
+	const metadata = payload[fieldName];
+
+	const { fileKey, uploadId, uploadUrls } =
+		await webResourceHandler.beginUpload(fieldName, metadata);
+	const key = randomUUID();
+
+	someKvStoreDbOrRedisIdc[key] = { ...metadata, fieldName, fileKey, uploadId };
+
+	return { [fieldName]: { key, uploadUrls } };
+};
+
+const parseBeginUpload = (request: ODataRequest) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	const fieldNames = Object.keys(request.values);
+	if (fieldNames.length !== 1) {
+		throw new errors.BadRequestError(
+			'You can only get upload url for one field at a time',
+		);
+	}
+
+	const [fieldName] = fieldNames;
+	const webResourceFields = getWebResourceFields(request, false);
+	if (!webResourceFields.includes(fieldName)) {
+		throw new errors.BadRequestError(
+			`You must provide a valid webresource field from: ${JSON.stringify(webResourceFields)}`,
+		);
+	}
+
+	const beginUploadPayload = parseBeginUploadPayload(request.values[fieldName]);
+	if (beginUploadPayload == null) {
+		throw new errors.BadRequestError('Invalid file metadata');
+	}
+
+	const uploadMetadataCheck: BeginUploadDbCheck = {
+		...beginUploadPayload,
+		href: 'metadata_check',
+	};
+
+	return { [fieldName]: uploadMetadataCheck };
+};
+
+const parseBeginUploadPayload = (
+	payload: AnyObject,
+): BeginUploadPayload | null => {
+	if (typeof payload !== 'object') {
+		return null;
+	}
+
+	let { filename, content_type, size, chunk_size } = payload;
+	if (
+		typeof filename !== 'string' ||
+		typeof content_type !== 'string' ||
+		typeof size !== 'number' ||
+		(chunk_size != null && typeof chunk_size !== 'number') ||
+		(chunk_size != null && chunk_size < 5 * MB)
+	) {
+		return null;
+	}
+
+	if (chunk_size == null) {
+		chunk_size = 5 * MB;
+	}
+	return { filename, content_type, size, chunk_size };
+};
+
+const parseCommitUpload = (request: ODataRequest) => {
+	if (request.odataQuery.key == null) {
+		throw new errors.BadRequestError();
+	}
+
+	const { key, additionalCommitInfo } = request.values;
+	if (typeof key !== 'string') {
+		throw new errors.BadRequestError('Invalid key type');
+	}
+
+	if (someKvStoreDbOrRedisIdc[key] == null) {
+		throw new errors.BadRequestError('Invalid key');
+	}
+
+	return { key, additionalCommitInfo, metadata: someKvStoreDbOrRedisIdc[key] };
+};

package/out/tasks/common.d.ts

@@ -1,4 +0,0 @@
-import Ajv from 'ajv';
-export declare const apiRoot = "tasks";
-export declare const channel = "task_insert";
-export declare const ajv: Ajv;

package/out/tasks/common.js

@@ -1,13 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ajv = exports.channel = exports.apiRoot = void 0;
-const ajv_1 = __importDefault(require("ajv"));
-exports.apiRoot = 'tasks';
-exports.channel = 'task_insert';
-exports.ajv = new ajv_1.default({
-    inlineRefs: false,
-});
-//# sourceMappingURL=common.js.map

package/out/tasks/common.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/tasks/common.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAsB;AAGT,QAAA,OAAO,GAAG,OAAO,CAAC;AAGlB,QAAA,OAAO,GAAG,aAAa,CAAC;AAKxB,QAAA,GAAG,GAAG,IAAI,aAAG,CAAC;IAC1B,UAAU,EAAE,KAAK;CACjB,CAAC,CAAC"}

package/out/tasks/index.d.ts

@@ -1,10 +0,0 @@
-import type { Schema } from 'ajv';
-import type * as Db from '../database-layer/db';
-import type { sbvrUtils } from '../server-glue/module';
-import type { TaskHandler } from './types';
-export * from './types';
-export declare const config: {
-    models: sbvrUtils.ExecutableModel[];
-};
-export declare function setup(db: Db.Database, tx: Db.Tx): Promise<void>;
-export declare function addTaskHandler(name: string, fn: TaskHandler['fn'], schema?: Schema): void;