@balena/pinejs 16.0.0-build--batch-09b8c466600d7df13e6df3eacabaf463d96f652f-1 → 16.0.0-build-fisehara-update-sbvr-types-b58e72aca3193964afac96c955fde178fe39d077-1

package/src/sbvr-api/uri-parser.ts

@@ -12,11 +12,11 @@ import type { AnyObject } from './common-types';
 import * as ODataParser from '@balena/odata-parser';
 export const SyntaxError = ODataParser.SyntaxError;
 import { OData2AbstractSQL } from '@balena/odata-to-abstract-sql';
-import * as _ from 'lodash';
+import _ from 'lodash';
 import memoizeWeak = require('memoizee/weak');
 
 export { BadRequestError, ParsingError, TranslationError } from './errors';
-import * as deepFreeze from 'deep-freeze';
+import deepFreeze from 'deep-freeze';
 import * as env from '../config-loader/env';
 import {
 	BadRequestError,
@@ -25,22 +25,19 @@ import {
 	TranslationError,
 } from './errors';
 import * as sbvrUtils from './sbvr-utils';
-import { IncomingHttpHeaders } from 'http';
 
 export type OdataBinds = ODataBinds;
 
 export interface UnparsedRequest {
-	id?: string;
 	method: string;
 	url: string;
-	body?: any;
-	headers?: IncomingHttpHeaders;
+	data?: any;
+	headers?: { [header: string]: string };
 	changeSet?: UnparsedRequest[];
 	_isChangeSet?: boolean;
 }
 
 export interface ParsedODataRequest {
-	headers?: IncomingHttpHeaders;
 	method: SupportedMethod;
 	url: string;
 	vocabulary: string;
@@ -50,7 +47,7 @@ export interface ParsedODataRequest {
 	odataQuery: ODataQuery;
 	odataBinds: OdataBinds;
 	custom: AnyObject;
-	id?: string | undefined;
+	id?: number | undefined;
 	_defer?: boolean;
 }
 export interface ODataRequest extends ParsedODataRequest {
@@ -63,8 +60,8 @@ export interface ODataRequest extends ParsedODataRequest {
 	modifiedFields?: ReturnType<
 		AbstractSQLCompiler.EngineInstance['getModifiedFields']
 	>;
-	affectedIds?: string[];
-	pendingAffectedIds?: Promise<string[]>;
+	affectedIds?: number[];
+	pendingAffectedIds?: Promise<number[]>;
 	hooks?: Array<[string, InstantiatedHooks]>;
 	engine: AbstractSQLCompiler.Engines;
 }
@@ -238,10 +235,14 @@ const memoizedOdata2AbstractSQL = (() => {
 		>,
 	) => {
 		const { method, odataBinds, values } = request;
-		let { odataQuery } = request;
-		const abstractSqlModel = sbvrUtils.getAbstractSqlModel(request);
 		// Sort the body keys to improve cache hits
 		const sortedBody = Object.keys(values).sort();
+		if (sortedBody.length === 0 && (method === 'PATCH' || method === 'MERGE')) {
+			throw new BadRequestError('No fields to update');
+		}
+
+		let { odataQuery } = request;
+		const abstractSqlModel = sbvrUtils.getAbstractSqlModel(request);
 		// Remove unused options for odata-to-abstract-sql to improve cache hits
 		if (odataQuery.options) {
 			odataQuery = {
@@ -264,22 +265,18 @@ const memoizedOdata2AbstractSQL = (() => {
 
 export const metadataEndpoints = ['$metadata', '$serviceroot'];
 
-export async function parseOData(
+export function parseOData(
 	b: UnparsedRequest & { _isChangeSet?: false },
-	headers?: IncomingHttpHeaders,
-): Promise<ParsedODataRequest>;
-export async function parseOData(
+): ParsedODataRequest;
+export function parseOData(
 	b: UnparsedRequest & { _isChangeSet: true },
-	headers?: IncomingHttpHeaders,
-): Promise<ParsedODataRequest[]>;
-export async function parseOData(
+): ParsedODataRequest[];
+export function parseOData(
 	b: UnparsedRequest,
-	headers?: IncomingHttpHeaders,
-): Promise<ParsedODataRequest | ParsedODataRequest[]>;
-export async function parseOData(
+): ParsedODataRequest | ParsedODataRequest[];
+export function parseOData(
 	b: UnparsedRequest,
-	batchHeaders?: IncomingHttpHeaders,
-): Promise<ParsedODataRequest | ParsedODataRequest[]> {
+): ParsedODataRequest | ParsedODataRequest[] {
 	try {
 		if (b._isChangeSet && b.changeSet != null) {
 			// We sort the CS set once, we must assure that requests which reference
@@ -299,14 +296,12 @@ export async function parseOData(
 			const odata = memoizedParseOdata(url);
 
 			return {
-				id: b.id,
-				headers: { ...batchHeaders, ...b.headers },
 				method: b.method as SupportedMethod,
 				url,
 				vocabulary: apiRoot,
 				resourceName: odata.tree.resource,
 				originalResourceName: odata.tree.resource,
-				values: b.body ?? {},
+				values: b.data ?? {},
 				odataQuery: odata.tree,
 				odataBinds: odata.binds,
 				custom: {},
@@ -371,7 +366,7 @@ const parseODataChangeset = (
 		originalResourceName: odata.tree.resource,
 		odataBinds: odata.binds,
 		odataQuery: odata.tree,
-		values: b.body ?? {},
+		values: b.data ?? {},
 		custom: {},
 		id: contentId,
 		_defer: defer,
@@ -388,7 +383,7 @@ const splitApiRoot = (url: string) => {
 };
 
 const mustExtractHeader = (
-	body: { headers?: IncomingHttpHeaders },
+	body: { headers?: { [header: string]: string } },
 	header: string,
 ) => {
 	const h: any = body.headers?.[header]?.[0];
@@ -425,7 +420,7 @@ export const translateUri = <
 		request = { ...request };
 		request.values = new Proxy(request.values, {
 			set: (obj: ODataRequest['values'], prop: string, value) => {
-				if (!obj.hasOwnProperty(prop)) {
+				if (!Object.prototype.hasOwnProperty.call(obj, prop)) {
 					sbvrUtils.api[request.vocabulary].logger.warn(
 						`Assigning a new request.values property '${prop}' however it will be ignored`,
 					);

package/src/server-glue/global-ext.d.ts

@@ -1,5 +1,6 @@
+// We have to use var when extending the global namespace
+// eslint-disable-next-line no-var
 declare var nodeRequire: NodeRequire;
-/* tslint:disable-next-line:no-namespace */
 declare namespace NodeJS {
 	export interface Process {
 		browser: boolean;

package/src/server-glue/module.ts

@@ -5,19 +5,21 @@ import './sbvr-loader';
 import * as dbModule from '../database-layer/db';
 import * as configLoader from '../config-loader/config-loader';
 import * as migrator from '../migrator/sync';
-import * as migratorUtils from '../migrator/utils';
+import type * as migratorUtils from '../migrator/utils';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 import { PINEJS_ADVISORY_LOCK } from '../config-loader/env';
 
 export * as dbModule from '../database-layer/db';
 export { PinejsSessionStore } from '../pinejs-session-store/pinejs-session-store';
+export { mountLoginRouter } from '../passport-pinejs/mount-login-router';
 export * as sbvrUtils from '../sbvr-api/sbvr-utils';
 export * as permissions from '../sbvr-api/permissions';
 export * as errors from '../sbvr-api/errors';
 export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
+export * as webResourceHandler from '../webresource-handler';
 export type { configLoader as ConfigLoader };
 export type { migratorUtils as Migrator };
 
@@ -64,7 +66,8 @@ export const init = async <T extends string>(
 
 		const promises: Array<Promise<void>> = [];
 		if (process.env.SBVR_SERVER_ENABLED) {
-			const sbvrServer = await import('../data-server/sbvr-server');
+			const sbvrServer = await import('../data-server/sbvr-server.js');
+			// eslint-disable-next-line @typescript-eslint/no-var-requires
 			const transactions = require('../http-transactions/transactions');
 			promises.push(cfgLoader.loadConfig(sbvrServer.config));
 			promises.push(
@@ -77,6 +80,9 @@ export const init = async <T extends string>(
 			promises.push(cfgLoader.loadApplicationConfig(config));
 		}
 		await Promise.all(promises);
+		// Execute it after all other promises have resolved. Execution of promises is not neccessarily
+		// guaranteed to be sequentially resolving them with Promise.all
+		await sbvrUtils.postSetup(app, db);
 
 		return cfgLoader;
 	} catch (err: any) {

package/src/server-glue/sbvr-loader.ts

@@ -15,7 +15,7 @@ if (!process.browser) {
 		global.nodeRequire = require;
 	}
 	// Register a .sbvr loader
-	// tslint:disable-next-line:no-var-requires
+	// eslint-disable-next-line @typescript-eslint/no-var-requires
 	const fs: typeof Fs = require('fs');
 	nodeRequire.extensions['.sbvr'] = (module: NodeModule, filename: string) =>
 		(module.exports = fs.readFileSync(filename, 'utf8'));

package/src/server-glue/server.ts

@@ -1,21 +1,20 @@
-import type * as BodyParser from 'body-parser';
-import type * as Compression from 'compression';
-import type * as CookieParser from 'cookie-parser';
-import type * as ExpressSession from 'express-session';
-import type * as MethodOverride from 'method-override';
-import type * as Multer from 'multer';
+import type BodyParser from 'body-parser';
+import type Compression from 'compression';
+import type CookieParser from 'cookie-parser';
+import type ExpressSession from 'express-session';
+import type MethodOverride from 'method-override';
 import type * as Passport from 'passport';
 import type * as Path from 'path';
-import type * as ServeStatic from 'serve-static';
+import type ServeStatic from 'serve-static';
 
 import * as Pinejs from './module';
 export { sbvrUtils, PinejsSessionStore } from './module';
 
 export { ExtendedSBVRParser } from '../extended-sbvr-parser/extended-sbvr-parser';
 
-import * as passportPinejs from '../passport-pinejs/passport-pinejs';
+import { mountLoginRouter } from '../passport-pinejs/mount-login-router';
 
-import * as express from 'express';
+import express from 'express';
 
 const app = express();
 
@@ -28,17 +27,16 @@ switch (app.get('env')) {
 }
 
 if (!process.browser) {
-	// tslint:disable:no-var-requires
+	/* eslint-disable @typescript-eslint/no-var-requires */
 	const passport: typeof Passport = require('passport');
 	const path: typeof Path = require('path');
 	const compression: typeof Compression = require('compression');
 	const serveStatic: typeof ServeStatic = require('serve-static');
 	const cookieParser: typeof CookieParser = require('cookie-parser');
 	const bodyParser: typeof BodyParser = require('body-parser');
-	const multer: typeof Multer = require('multer');
 	const methodOverride: typeof MethodOverride = require('method-override');
 	const expressSession: typeof ExpressSession = require('express-session');
-	// tslint:enable:no-var-requires
+	/* eslint-enable @typescript-eslint/no-var-requires */
 
 	app.use(compression());
 
@@ -47,7 +45,6 @@ if (!process.browser) {
 
 	app.use(cookieParser());
 	app.use(bodyParser());
-	app.use(multer().any());
 	app.use(methodOverride());
 	app.use(
 		expressSession({
@@ -81,42 +78,7 @@ if (!process.browser) {
 
 export const initialised = Pinejs.init(app)
 	.then(async (configLoader) => {
-		await Promise.all([
-			configLoader.loadConfig(passportPinejs.config),
-			configLoader.loadConfig(Pinejs.PinejsSessionStore.config),
-		]);
-
-		if (
-			typeof process === 'undefined' ||
-			process == null ||
-			!process.env.DISABLE_DEFAULT_AUTH
-		) {
-			app.post(
-				'/login',
-				passportPinejs.login((err, user, req, res) => {
-					if (err) {
-						console.error('Error logging in', err);
-						res.status(500).end();
-					} else if (user === false) {
-						if (req.xhr === true) {
-							res.status(401).end();
-						} else {
-							res.redirect('/login.html');
-						}
-					} else {
-						if (req.xhr === true) {
-							res.status(200).end();
-						} else {
-							res.redirect('/');
-						}
-					}
-				}),
-			);
-
-			app.get('/logout', passportPinejs.logout, (_req, res) => {
-				res.redirect('/');
-			});
-		}
+		await mountLoginRouter(configLoader, app);
 
 		app.listen(process.env.PORT || 1337, () => {
 			console.info('Server started');

package/src/webresource-handler/handlers/NoopHandler.ts

@@ -0,0 +1,21 @@
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import type { IncomingFile, UploadResponse, WebResourceHandler } from '..';
+
+export class NoopHandler implements WebResourceHandler {
+	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
+		// handleFile must consume the file stream
+		resource.stream.resume();
+		return {
+			filename: 'noop',
+			size: 0,
+		};
+	}
+
+	public async removeFile(): Promise<void> {
+		return;
+	}
+
+	public async onPreRespond(webResource: WebResource): Promise<WebResource> {
+		return webResource;
+	}
+}

package/src/webresource-handler/handlers/S3Handler.ts

@@ -0,0 +1,143 @@
+import {
+	FileSizeExceededError,
+	type IncomingFile,
+	normalizeHref,
+	type UploadResponse,
+	WebResourceError,
+	type WebResourceHandler,
+} from '..';
+import {
+	S3Client,
+	type S3ClientConfig,
+	DeleteObjectCommand,
+	type PutObjectCommandInput,
+	GetObjectCommand,
+} from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+import { randomUUID } from 'crypto';
+import type { WebResourceType as WebResource } from '@balena/sbvr-types';
+import memoize from 'memoizee';
+
+export interface S3HandlerProps {
+	region: string;
+	accessKey: string;
+	secretKey: string;
+	endpoint: string;
+	bucket: string;
+	maxSize?: number;
+	signedUrlExpireTimeSeconds?: number;
+	signedUrlCacheExpireTimeSeconds?: number;
+}
+
+export class S3Handler implements WebResourceHandler {
+	private readonly config: S3ClientConfig;
+	private readonly bucket: string;
+	private readonly maxFileSize: number;
+
+	protected readonly signedUrlExpireTimeSeconds: number;
+	protected readonly signedUrlCacheExpireTimeSeconds: number;
+	protected cachedGetSignedUrl: (fileKey: string) => Promise<string>;
+
+	private client: S3Client;
+
+	constructor(config: S3HandlerProps) {
+		this.config = {
+			region: config.region,
+			credentials: {
+				accessKeyId: config.accessKey,
+				secretAccessKey: config.secretKey,
+			},
+			endpoint: config.endpoint,
+			forcePathStyle: true,
+		};
+
+		this.signedUrlExpireTimeSeconds =
+			config.signedUrlExpireTimeSeconds ?? 86400; // 24h
+		this.signedUrlCacheExpireTimeSeconds =
+			config.signedUrlCacheExpireTimeSeconds ?? 82800; // 22h
+
+		this.maxFileSize = config.maxSize ?? 52428800;
+		this.bucket = config.bucket;
+		this.client = new S3Client(this.config);
+
+		// Memoize expects maxAge in MS and s3 signing method in seconds.
+		// Normalization to use only seconds and therefore convert here from seconds to MS
+		this.cachedGetSignedUrl = memoize(this.s3SignUrl, {
+			maxAge: this.signedUrlCacheExpireTimeSeconds * 1000,
+		});
+	}
+
+	public async handleFile(resource: IncomingFile): Promise<UploadResponse> {
+		let size = 0;
+		const key = `${resource.fieldname}_${randomUUID()}_${
+			resource.originalname
+		}`;
+		const params: PutObjectCommandInput = {
+			Bucket: this.bucket,
+			Key: key,
+			Body: resource.stream,
+			ContentType: resource.mimetype,
+		};
+		const upload = new Upload({ client: this.client, params });
+
+		upload.on('httpUploadProgress', async (ev) => {
+			size = ev.total ?? ev.loaded!;
+			if (size > this.maxFileSize) {
+				await upload.abort();
+			}
+		});
+
+		try {
+			await upload.done();
+		} catch (err: any) {
+			resource.stream.resume();
+			if (size > this.maxFileSize) {
+				throw new FileSizeExceededError(this.maxFileSize);
+			}
+			throw new WebResourceError(err);
+		}
+
+		const filename = this.getS3URL(key);
+		return { size, filename };
+	}
+
+	public async removeFile(href: string): Promise<void> {
+		const fileKey = this.getKeyFromHref(href);
+
+		const command = new DeleteObjectCommand({
+			Bucket: this.bucket,
+			Key: fileKey,
+		});
+
+		await this.client.send(command);
+	}
+
+	public async onPreRespond(webResource: WebResource): Promise<WebResource> {
+		if (webResource.href != null) {
+			const fileKey = this.getKeyFromHref(webResource.href);
+			webResource.href = await this.cachedGetSignedUrl(fileKey);
+		}
+		return webResource;
+	}
+
+	private s3SignUrl(fileKey: string): Promise<string> {
+		const command = new GetObjectCommand({
+			Bucket: this.bucket,
+			Key: fileKey,
+		});
+		return getSignedUrl(this.client, command, {
+			expiresIn: this.signedUrlExpireTimeSeconds,
+		});
+	}
+
+	private getS3URL(key: string): string {
+		return `${this.config.endpoint}/${this.bucket}/${key}`;
+	}
+
+	private getKeyFromHref(href: string): string {
+		const hrefWithoutParams = normalizeHref(href);
+		return hrefWithoutParams.substring(hrefWithoutParams.lastIndexOf('/') + 1);
+	}
+}

package/src/webresource-handler/handlers/index.ts

@@ -0,0 +1,2 @@
+export * from './NoopHandler';
+export * from './S3Handler';