@balena/pinejs 16.2.0-build-joshbwlng-tasks-a83c83b4c78803915d0cb6297cc8cc1862622d08-1 → 17.0.0-build-wip-large-file-uploads-0c8ef752deac19fa0d6a7dfa9f7173813cab7867-1

package/out/webresource-handler/webresource.sbvr

@@ -0,0 +1,62 @@
+Vocabulary: Auth
+
+Term: actor
+Term: expiry date
+	Concept Type: Date Time (Type)
+
+Vocabulary: webresource
+
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date (Auth)
+	Necessity: each multipart upload has exactly one expiry date (Auth)
+Fact type: multipart upload is created by actor (Auth)
+	Necessity: each multipart upload is created by at most one actor (Auth)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@balena/pinejs",
-  "version": "16.2.0-build-joshbwlng-tasks-a83c83b4c78803915d0cb6297cc8cc1862622d08-1",
+  "version": "17.0.0-build-wip-large-file-uploads-0c8ef752deac19fa0d6a7dfa9f7173813cab7867-1",
   "main": "out/server-glue/module",
   "type": "commonjs",
   "repository": "git@github.com:balena-io/pinejs.git",
@@ -20,7 +20,7 @@
     "webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
     "lint": "balena-lint -t tsconfig.dev.json -e js -e ts src build typings Gruntfile.ts && npx tsc --project tsconfig.dev.json --noEmit",
     "test": "npm run lint && npm run build && npm run webpack-build && npm run test:compose",
-    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' SIGINT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 npm run mocha",
+    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' SIGINT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 npm run mocha",
     "mocha": "TS_NODE_FILES=true mocha",
     "prettify": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.ts"
   },
@@ -51,10 +51,8 @@
     "@types/pg": "^8.10.9",
     "@types/randomstring": "^1.1.11",
     "@types/websql": "^0.0.30",
-    "ajv": "^8.12.0",
     "busboy": "^1.6.0",
     "commander": "^11.1.0",
-    "cron-parser": "^4.9.0",
     "deep-freeze": "^0.0.1",
     "eventemitter3": "^5.0.1",
     "express-session": "^1.17.3",
@@ -91,7 +89,6 @@
     "grunt-ts": "^6.0.0-beta.22",
     "grunt-webpack": "^6.0.0",
     "husky": "^9.0.0",
-    "json-schema-to-ts": "^3.0.1",
     "lint-staged": "^15.2.0",
     "load-grunt-tasks": "^5.1.0",
     "mocha": "^10.2.0",
@@ -126,7 +123,7 @@
     "serve-static": "^1.15.0"
   },
   "engines": {
-    "node": ">=16.13.0",
+    "node": ">=18.0.0",
     "npm": ">=8.0.0"
   },
   "lint-staged": {
@@ -147,6 +144,6 @@
     "recursive": true
   },
   "versionist": {
-    "publishedAt": "2024-04-15T01:28:03.598Z"
+    "publishedAt": "2024-04-18T17:52:52.724Z"
   }
 }

package/src/config-loader/env.ts

@@ -49,7 +49,7 @@ export const cache = {
 	apiKeyActorId: false as CacheOpts,
 };
 
-import { boolVar, intVar } from '@balena/env-parsing';
+import { boolVar } from '@balena/env-parsing';
 import memoize from 'memoizee';
 import memoizeWeak = require('memoizee/weak');
 export const createCache = <T extends (...args: any[]) => any>(
@@ -146,8 +146,3 @@ export const migrator = {
 	 */
 	asyncMigrationIsEnabled: boolVar('PINEJS_ASYNC_MIGRATION_ENABLED', true),
 };
-
-export const tasks = {
-	queueConcurrency: intVar('PINEJS_QUEUE_CONCURRENCY', 0),
-	queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
-};

package/src/database-layer/db.ts

@@ -98,13 +98,6 @@ export interface Database extends BaseDatabase {
 	) => Promise<Result>;
 	transaction: TransactionFn;
 	readTransaction: TransactionFn;
-	on?: (
-		name: 'notification',
-		fn: (...args: any[]) => Promise<void>,
-		options?: {
-			channel?: string;
-		},
-	) => void;
 }
 
 interface EngineParams {
@@ -696,23 +689,6 @@ if (maybePg != null) {
 		return {
 			engine: Engines.postgres,
 			executeSql: atomicExecuteSql,
-			on: async (name, fn, options) => {
-				if (name === 'notification' && options?.channel === undefined) {
-					throw new Error('Missing channel option for notification listener');
-				}
-
-				const client = await pool.connect();
-				client.on(name, (msg) => {
-					fn(msg).catch((error) => {
-						console.error('Error handling message:', error);
-					});
-				});
-
-				if (name === 'notification' && options?.channel !== undefined) {
-					await client.query(`LISTEN "${options.channel}";`);
-					// client.release();
-				}
-			},
 			transaction: createTransaction(async (stackTraceErr) => {
 				const client = await pool.connect();
 				const tx = new PostgresTx(client, false, stackTraceErr);

package/src/migrator/utils.ts

@@ -1,7 +1,7 @@
 import type { Result, Tx } from '../database-layer/db';
 import type { Resolvable } from '../sbvr-api/common-types';
 
-import { createHash } from 'crypto';
+import { createHash } from 'node:crypto';
 import { Engines } from '@balena/abstract-sql-compiler';
 import _ from 'lodash';
 import { TypedError } from 'typed-error';

package/src/sbvr-api/sbvr-utils.ts

@@ -42,7 +42,6 @@ import { generateODataMetadata } from '../odata-metadata/odata-metadata-generato
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const devModel = require('./dev.sbvr');
-import * as tasks from '../tasks';
 import * as permissions from './permissions';
 import {
 	BadRequestError,
@@ -78,7 +77,6 @@ export {
 	addPureHook,
 	addSideEffectHook,
 } from './hooks';
-export { addTaskHandler } from '../tasks';
 
 import memoizeWeak = require('memoizee/weak');
 import * as controlFlow from './control-flow';
@@ -775,7 +773,7 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
 	// Hence, skipped migrations from earlier models are not set as executed as the `migration` table is missing
 	// Here the skipped migrations that haven't been set properly are covered
 	// This is mostly an edge case when running on an empty database schema and migrations model hasn't been executed, yet.
-	// One specific case are tests to run tests against migrated and unmigrated database states
+	// One specifc case are tests to run tests against migrated and unmigrated database states
 
 	for (const modelKey of Object.keys(models)) {
 		const pendingToSetExecutedMigrations =
@@ -1985,7 +1983,6 @@ export const executeStandardModels = async (tx: Db.Tx): Promise<void> => {
 			},
 		});
 		await executeModels(tx, permissions.config.models);
-		await executeModels(tx, tasks.config.models);
 		console.info('Successfully executed standard models.');
 	} catch (err: any) {
 		console.error('Failed to execute standard models.', err);
@@ -2002,7 +1999,6 @@ export const setup = async (
 		await db.transaction(async (tx) => {
 			await executeStandardModels(tx);
 			await permissions.setup();
-			await tasks.setup($db, tx);
 		});
 	} catch (err: any) {
 		console.error('Could not execute standard models', err);

package/src/server-glue/module.ts

@@ -5,6 +5,7 @@ import './sbvr-loader';
 import * as dbModule from '../database-layer/db';
 import * as configLoader from '../config-loader/config-loader';
 import * as migrator from '../migrator/sync';
+import * as webResourceHandler from '../webresource-handler';
 import type * as migratorUtils from '../migrator/utils';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
@@ -19,7 +20,6 @@ export * as errors from '../sbvr-api/errors';
 export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
-export * as tasks from '../tasks';
 export * as webResourceHandler from '../webresource-handler';
 export type { configLoader as ConfigLoader };
 export type { migratorUtils as Migrator };
@@ -64,6 +64,7 @@ export const init = async <T extends string>(
 		await sbvrUtils.setup(app, db);
 		const cfgLoader = await configLoader.setup(app);
 		await cfgLoader.loadConfig(migrator.config);
+		await cfgLoader.loadConfig(webResourceHandler.config);
 
 		const promises: Array<Promise<void>> = [];
 		if (process.env.SBVR_SERVER_ENABLED) {

package/src/webresource-handler/handlers/NoopHandler.ts

@@ -1,5 +1,10 @@
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import type { IncomingFile, UploadResponse, WebResourceHandler } from '..';
+import type {
+	BeginUploadHandlerResponse,
+	BeginUploadPayload,
+	CommitUploadHandlerPayload,
+} from '../multipartUpload';
 
 export class NoopHandler implements WebResourceHandler {
 	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
@@ -18,4 +23,20 @@ export class NoopHandler implements WebResourceHandler {
 	public async onPreRespond(webResource: WebResource): Promise<WebResource> {
 		return webResource;
 	}
+
+	public async beginUpload(
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_fieldName: string,
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_payload: BeginUploadPayload,
+	): Promise<BeginUploadHandlerResponse> {
+		return { fileKey: 'noop', uploadId: 'noop', uploadUrls: [] };
+	}
+
+	public async commitUpload(
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_payload: CommitUploadHandlerPayload,
+	): Promise<WebResource> {
+		return { filename: 'noop', href: 'noop' };
+	}
 }

package/src/webresource-handler/handlers/S3Handler.ts

@@ -6,17 +6,27 @@ import {
 	WebResourceError,
 	type WebResourceHandler,
 } from '..';
+import type {
+	BeginUploadHandlerResponse,
+	BeginUploadPayload,
+	CommitUploadHandlerPayload,
+	UploadUrl,
+} from '../multipartUpload';
 import {
 	S3Client,
 	type S3ClientConfig,
 	DeleteObjectCommand,
 	type PutObjectCommandInput,
 	GetObjectCommand,
+	CreateMultipartUploadCommand,
+	UploadPartCommand,
+	CompleteMultipartUploadCommand,
+	HeadObjectCommand,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
-import { randomUUID } from 'crypto';
+import { randomUUID } from 'node:crypto';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import memoize from 'memoizee';
 
@@ -71,9 +81,7 @@ export class S3Handler implements WebResourceHandler {
 
 	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
 		let size = 0;
-		const key = `${resource.fieldname}_${randomUUID()}_${
-			resource.originalname
-		}`;
+		const key = this.getFileKey(resource.fieldname, resource.originalname);
 		const params: PutObjectCommandInput = {
 			Bucket: this.bucket,
 			Key: key,
@@ -122,6 +130,62 @@ export class S3Handler implements WebResourceHandler {
 		return webResource;
 	}
 
+	public async beginUpload(
+		fieldName: string,
+		payload: BeginUploadPayload,
+	): Promise<BeginUploadHandlerResponse> {
+		const fileKey = this.getFileKey(fieldName, payload.filename);
+
+		const createMultiPartResponse = await this.client.send(
+			new CreateMultipartUploadCommand({
+				Bucket: this.bucket,
+				Key: fileKey,
+				ContentType: payload.content_type,
+			}),
+		);
+
+		if (createMultiPartResponse.UploadId == null) {
+			throw new WebResourceError('Failed to create multipart upload.');
+		}
+
+		const uploadUrls = await this.getPartUploadUrls(
+			fileKey,
+			createMultiPartResponse.UploadId,
+			payload,
+		);
+		return { fileKey, uploadId: createMultiPartResponse.UploadId, uploadUrls };
+	}
+
+	public async commitUpload({
+		fileKey,
+		uploadId,
+		filename,
+		multipartUploadChecksums,
+	}: CommitUploadHandlerPayload): Promise<WebResource> {
+		await this.client.send(
+			new CompleteMultipartUploadCommand({
+				Bucket: this.bucket,
+				Key: fileKey,
+				UploadId: uploadId,
+				MultipartUpload: multipartUploadChecksums,
+			}),
+		);
+
+		const headResult = await this.client.send(
+			new HeadObjectCommand({
+				Bucket: this.bucket,
+				Key: fileKey,
+			}),
+		);
+
+		return {
+			href: this.getS3URL(fileKey),
+			filename: filename,
+			size: headResult.ContentLength,
+			content_type: headResult.ContentType,
+		};
+	}
+
 	private s3SignUrl(fileKey: string): Promise<string> {
 		const command = new GetObjectCommand({
 			Bucket: this.bucket,
@@ -136,8 +200,70 @@ export class S3Handler implements WebResourceHandler {
 		return `${this.config.endpoint}/${this.bucket}/${key}`;
 	}
 
+	private getFileKey(fieldName: string, fileName: string) {
+		return `${fieldName}_${randomUUID()}_${fileName}`;
+	}
+
 	private getKeyFromHref(href: string): string {
 		const hrefWithoutParams = normalizeHref(href);
 		return hrefWithoutParams.substring(hrefWithoutParams.lastIndexOf('/') + 1);
 	}
+
+	private async getPartUploadUrls(
+		fileKey: string,
+		uploadId: string,
+		payload: BeginUploadPayload,
+	): Promise<UploadUrl[]> {
+		const chunkSizesWithParts = await this.getChunkSizesWithParts(
+			payload.size,
+			payload.chunk_size,
+		);
+		return Promise.all(
+			chunkSizesWithParts.map(async ({ chunkSize, partNumber }) => ({
+				chunkSize,
+				partNumber,
+				url: await this.getPartUploadUrl(
+					fileKey,
+					uploadId,
+					partNumber,
+					chunkSize,
+				),
+			})),
+		);
+	}
+
+	private async getPartUploadUrl(
+		fileKey: string,
+		uploadId: string,
+		partNumber: number,
+		partSize: number,
+	): Promise<string> {
+		const command = new UploadPartCommand({
+			Bucket: this.bucket,
+			Key: fileKey,
+			UploadId: uploadId,
+			PartNumber: partNumber,
+			ContentLength: partSize,
+		});
+
+		return getSignedUrl(this.client, command, {
+			expiresIn: this.signedUrlExpireTimeSeconds,
+		});
+	}
+
+	private async getChunkSizesWithParts(
+		size: number,
+		chunkSize: number,
+	): Promise<Array<Pick<UploadUrl, 'chunkSize' | 'partNumber'>>> {
+		const chunkSizesWithParts = [];
+		let partNumber = 1;
+		let remainingSize = size;
+		while (remainingSize > 0) {
+			const currentChunkSize = Math.min(remainingSize, chunkSize);
+			chunkSizesWithParts.push({ chunkSize: currentChunkSize, partNumber });
+			remainingSize -= currentChunkSize;
+			partNumber += 1;
+		}
+		return chunkSizesWithParts;
+	}
 }

package/src/webresource-handler/index.ts

@@ -1,19 +1,25 @@
-import type * as Express from 'express';
-import busboy from 'busboy';
-import type * as stream from 'node:stream';
-import * as uriParser from '../sbvr-api/uri-parser';
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-import type { HookArgs } from '../sbvr-api/hooks';
-import { getApiRoot, getModel } from '../sbvr-api/sbvr-utils';
-import { checkPermissions } from '../sbvr-api/permissions';
-import { NoopHandler } from './handlers/NoopHandler';
 import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
-import { errors, permissions } from '../server-glue/module';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import busboy from 'busboy';
+import type * as Express from 'express';
+import type * as stream from 'node:stream';
 import { TypedError } from 'typed-error';
+import type { HookArgs } from '../sbvr-api/hooks';
+import { checkPermissions } from '../sbvr-api/permissions';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+import { getApiRoot, getModel } from '../sbvr-api/sbvr-utils';
+import * as uriParser from '../sbvr-api/uri-parser';
+import { errors, permissions } from '../server-glue/module';
+import { NoopHandler } from './handlers/NoopHandler';
+import type {
+	BeginUploadHandlerResponse,
+	BeginUploadPayload,
+	CommitUploadHandlerPayload,
+} from './multipartUpload';
+import { multipartUploadHooks } from './multipartUpload';
 
 export * from './handlers';
 
@@ -34,6 +40,14 @@ export interface WebResourceHandler {
 	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
 	removeFile: (fileReference: string) => Promise<void>;
 	onPreRespond: (webResource: WebResource) => Promise<WebResource>;
+
+	beginUpload: (
+		fieldName: string,
+		payload: BeginUploadPayload,
+	) => Promise<BeginUploadHandlerResponse>;
+	commitUpload: (
+		commitInfo: CommitUploadHandlerPayload,
+	) => Promise<WebResource>;
 }
 
 export class WebResourceError extends TypedError {}
@@ -216,7 +230,7 @@ export const getUploaderMiddlware = (
 	};
 };
 
-const getWebResourceFields = (
+export const getWebResourceFields = (
 	request: uriParser.ODataRequest,
 	useTranslations = true,
 ): string[] => {
@@ -249,6 +263,8 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		request.custom.isAction !== 'beginUpload' &&
+		request.custom.isAction !== 'commitUpload' &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -447,4 +463,23 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+
+	sbvrUtils.addPureHook(
+		'POST',
+		apiRoot,
+		resourceName,
+		multipartUploadHooks(handler),
+	);
+};
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const webresourceModel: string = require('./webresource.sbvr');
+export const config = {
+	models: [
+		{
+			apiRoot: 'webresource',
+			modelText: webresourceModel,
+			modelName: 'webresource',
+		},
+	] as sbvrUtils.ExecutableModel[],
 };