@balena/pinejs 20.1.0-build-add-large-file-uploads-interfaces-1aa101d2ee3103f42cf7f7bc29d703a8958011e8-1 → 20.1.0-build-large-file-uploads-2-b83400458570d926f7a6abddeb7a149e89f52abe-2

package/out/webresource-handler/multipartUpload.js

@@ -0,0 +1,259 @@
+import { randomUUID } from 'node:crypto';
+import { getWebResourceFields } from './index.js';
+import { api } from '../sbvr-api/sbvr-utils.js';
+import { errors, sbvrUtils } from '../server-glue/module.js';
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import * as permissions from '../sbvr-api/permissions.js';
+const MB = 1024 * 1024;
+export const isMultipartUploadAvailable = (webResourceHandler) => {
+    return (webResourceEnv.multipartUploadEnabled &&
+        webResourceHandler.multipartUpload != null);
+};
+export const multipartUploadHooks = (webResourceHandler) => {
+    return {
+        POSTPARSE: async ({ req, request, tx, api: applicationApi }) => {
+            if (request.odataQuery.property?.resource === 'beginUpload') {
+                const uploadParams = await validateBeginUpload(request, applicationApi);
+                tx = await sbvrUtils.db.transaction();
+                req.tx = tx;
+                request.tx = tx;
+                request.method = 'PATCH';
+                request.values = uploadParams;
+                request.odataQuery.resource = request.resourceName;
+                delete request.odataQuery.property;
+                request.custom.isAction = 'beginUpload';
+            }
+            else if (request.odataQuery.property?.resource === 'commitUpload') {
+                const commitPayload = await validateCommitUpload(request, applicationApi);
+                const webresource = await webResourceHandler.multipartUpload.commit({
+                    fileKey: commitPayload.metadata.fileKey,
+                    uploadId: commitPayload.metadata.uploadId,
+                    filename: commitPayload.metadata.filename,
+                    providerCommitData: commitPayload.providerCommitData,
+                });
+                await api.webresource.patch({
+                    resource: 'multipart_upload',
+                    body: {
+                        status: 'completed',
+                    },
+                    options: {
+                        $filter: {
+                            uuid: commitPayload.uuid,
+                        },
+                    },
+                    passthrough: {
+                        tx: tx,
+                        req: permissions.root,
+                    },
+                });
+                request.method = 'PATCH';
+                request.values = {
+                    [commitPayload.metadata.fieldName]: webresource,
+                };
+                request.odataQuery.resource = request.resourceName;
+                delete request.odataQuery.property;
+                request.custom.isAction = 'commitUpload';
+                request.custom.commitUploadPayload = webresource;
+            }
+            else if (request.odataQuery.property?.resource === 'cancelUpload') {
+                const { uuid, fileKey, uploadId } = await validateCancelPayload(request, applicationApi);
+                await webResourceHandler.multipartUpload.cancel({ fileKey, uploadId });
+                await api.webresource.patch({
+                    resource: 'multipart_upload',
+                    body: {
+                        status: 'cancelled',
+                    },
+                    options: {
+                        $filter: { uuid },
+                    },
+                    passthrough: {
+                        tx: tx,
+                        req: permissions.root,
+                    },
+                });
+                request.method = 'GET';
+                request.odataQuery.resource = request.resourceName;
+                delete request.odataQuery.property;
+                request.custom.isAction = 'cancelUpload';
+            }
+        },
+        PRERESPOND: async ({ req, request, response, tx }) => {
+            if (request.custom.isAction === 'beginUpload') {
+                await tx.rollback();
+                response.statusCode = 200;
+                response.body = await beginUpload({
+                    webResourceHandler,
+                    odataRequest: request,
+                    actorId: req.user?.actor,
+                });
+            }
+            else if (request.custom.isAction === 'commitUpload') {
+                response.body = await webResourceHandler.onPreRespond(request.custom.commitUploadPayload);
+            }
+            else if (request.custom.isAction === 'cancelUpload') {
+                response.statusCode = 204;
+                delete response.body;
+            }
+        },
+    };
+};
+const beginUpload = async ({ webResourceHandler, odataRequest, actorId, }) => {
+    const payload = odataRequest.values;
+    const fieldName = Object.keys(payload)[0];
+    const metadata = payload[fieldName];
+    const { fileKey, uploadId, uploadParts } = await webResourceHandler.multipartUpload.begin(fieldName, metadata);
+    const uuid = randomUUID();
+    return await sbvrUtils.db.transaction(async (tx) => {
+        try {
+            await api.webresource.post({
+                resource: 'multipart_upload',
+                body: {
+                    uuid,
+                    resource_name: odataRequest.resourceName,
+                    field_name: fieldName,
+                    resource_id: odataRequest.affectedIds?.[0],
+                    upload_id: uploadId,
+                    file_key: fileKey,
+                    status: 'pending',
+                    filename: metadata.filename,
+                    content_type: metadata.content_type,
+                    size: metadata.size,
+                    chunk_size: metadata.chunk_size,
+                    expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000,
+                    is_created_by__actor: actorId,
+                },
+                passthrough: {
+                    req: permissions.root,
+                    tx,
+                },
+            });
+            return { [fieldName]: { uuid, uploadParts } };
+        }
+        catch (err) {
+            console.error('failed to start multipart upload', err);
+            throw new errors.BadRequestError('Failed to start multipart upload');
+        }
+    });
+};
+const validateBeginUpload = async (request, applicationApi) => {
+    if (request.odataQuery.key == null) {
+        throw new errors.BadRequestError();
+    }
+    await applicationApi.request({
+        method: 'POST',
+        url: request.url.substring(1).replace('beginUpload', 'canAccess'),
+        body: { method: 'PATCH' },
+    });
+    const fieldNames = Object.keys(request.values);
+    if (fieldNames.length !== 1) {
+        throw new errors.BadRequestError('You can only get upload url for one field at a time');
+    }
+    const [fieldName] = fieldNames;
+    const webResourceFields = getWebResourceFields(request, false);
+    if (!webResourceFields.includes(fieldName)) {
+        throw new errors.BadRequestError(`You must provide a valid webresource field from: ${JSON.stringify(webResourceFields)}`);
+    }
+    const beginUploadPayload = parseBeginUploadPayload(request.values[fieldName]);
+    if (beginUploadPayload == null) {
+        throw new errors.BadRequestError('Invalid file metadata');
+    }
+    const uploadMetadataCheck = {
+        ...beginUploadPayload,
+        href: 'metadata_check_probe',
+    };
+    return { [fieldName]: uploadMetadataCheck };
+};
+const parseBeginUploadPayload = (payload) => {
+    if (typeof payload !== 'object') {
+        return null;
+    }
+    let { filename, content_type, size, chunk_size } = payload;
+    if (typeof filename !== 'string' ||
+        typeof content_type !== 'string' ||
+        typeof size !== 'number' ||
+        (chunk_size != null && typeof chunk_size !== 'number') ||
+        (chunk_size != null && chunk_size < 5 * MB)) {
+        return null;
+    }
+    if (chunk_size == null) {
+        chunk_size = 5 * MB;
+    }
+    return { filename, content_type, size, chunk_size };
+};
+const validateCommitUpload = async (request, applicationApi) => {
+    if (request.odataQuery.key == null) {
+        throw new errors.BadRequestError();
+    }
+    await applicationApi.request({
+        method: 'POST',
+        url: request.url.substring(1).replace('commitUpload', 'canAccess'),
+        body: { method: 'PATCH' },
+    });
+    const { uuid, providerCommitData } = request.values;
+    if (typeof uuid !== 'string') {
+        throw new errors.BadRequestError('Invalid uuid type');
+    }
+    const [multipartUpload] = await api.webresource.get({
+        resource: 'multipart_upload',
+        options: {
+            $select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+            $filter: {
+                uuid,
+                status: 'pending',
+                expiry_date: { $gt: { $now: {} } },
+            },
+        },
+        passthrough: {
+            tx: request.tx,
+            req: permissions.rootRead,
+        },
+    });
+    if (multipartUpload == null) {
+        throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+    }
+    const metadata = {
+        fileKey: multipartUpload.file_key,
+        uploadId: multipartUpload.upload_id,
+        filename: multipartUpload.filename,
+        fieldName: multipartUpload.field_name,
+    };
+    return { uuid, providerCommitData, metadata };
+};
+const validateCancelPayload = async (request, applicationApi) => {
+    if (request.odataQuery.key == null) {
+        throw new errors.BadRequestError();
+    }
+    await applicationApi.request({
+        method: 'POST',
+        url: request.url.substring(1).replace('cancelUpload', 'canAccess'),
+        body: { method: 'PATCH' },
+    });
+    const { uuid } = request.values;
+    if (typeof uuid !== 'string') {
+        throw new errors.BadRequestError('Invalid uuid type');
+    }
+    const [multipartUpload] = await api.webresource.get({
+        resource: 'multipart_upload',
+        options: {
+            $select: ['id', 'file_key', 'upload_id'],
+            $filter: {
+                uuid,
+                status: 'pending',
+                expiry_date: { $gt: { $now: {} } },
+            },
+        },
+        passthrough: {
+            tx: request.tx,
+            req: permissions.rootRead,
+        },
+    });
+    if (multipartUpload == null) {
+        throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+    }
+    return {
+        uuid,
+        fileKey: multipartUpload.file_key,
+        uploadId: multipartUpload.upload_id,
+    };
+};
+//# sourceMappingURL=multipartUpload.js.map

package/out/webresource-handler/multipartUpload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multipartUpload.js","sourceRoot":"","sources":["../../src/webresource-handler/multipartUpload.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,WAAW,MAAM,4BAA4B,CAAC;AAqB1D,MAAM,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvB,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACzC,kBAAsC,EACS,EAAE;IACjD,OAAO,CACN,cAAc,CAAC,sBAAsB;QACrC,kBAAkB,CAAC,eAAe,IAAI,IAAI,CAC1C,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CACnC,kBAA0C,EACxB,EAAE;IACpB,OAAO;QACN,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE;YAC9D,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAC7D,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;gBAOxE,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;gBACtC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;gBAEhB,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC;gBACzB,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;gBAC9B,OAAO,CAAC,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;gBACnD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,aAAa,CAAC;YACzC,CAAC;iBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACrE,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAC/C,OAAO,EACP,cAAc,CACd,CAAC;gBAEF,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC,eAAe,CAAC,MAAM,CAAC;oBACnE,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,OAAO;oBACvC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,QAAQ;oBACzC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,QAAQ;oBACzC,kBAAkB,EAAE,aAAa,CAAC,kBAAkB;iBACpD,CAAC,CAAC;gBAEH,MAAM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC;oBAC3B,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE;wBACL,MAAM,EAAE,WAAW;qBACnB;oBACD,OAAO,EAAE;wBACR,OAAO,EAAE;4BACR,IAAI,EAAE,aAAa,CAAC,IAAI;yBACxB;qBACD;oBACD,WAAW,EAAE;wBACZ,EAAE,EAAE,EAAE;wBACN,GAAG,EAAE,WAAW,CAAC,IAAI;qBACrB;iBACD,CAAC,CAAC;gBAEH,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC;gBACzB,OAAO,CAAC,MAAM,GAAG;oBAChB,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,WAAW;iBAC/C,CAAC;gBACF,OAAO,CAAC,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;gBACnD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC;gBACzC,OAAO,CAAC,MAAM,CAAC,mBAAmB,GAAG,WAAW,CAAC;YAClD,CAAC;iBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACrE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,qBAAqB,CAC9D,OAAO,EACP,cAAc,CACd,CAAC;gBAEF,MAAM,kBAAkB,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAEvE,MAAM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC;oBAC3B,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE;wBACL,MAAM,EAAE,WAAW;qBACnB;oBACD,OAAO,EAAE;wBACR,OAAO,EAAE,EAAE,IAAI,EAAE;qBACjB;oBACD,WAAW,EAAE;wBACZ,EAAE,EAAE,EAAE;wBACN,GAAG,EAAE,WAAW,CAAC,IAAI;qBACrB;iBACD,CAAC,CAAC;gBAEH,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;gBACvB,OAAO,CAAC,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;gBACnD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC;YAC1C,CAAC;QACF,CAAC;QACD,UAAU,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE;YACpD,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAK/C,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;gBAEpB,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;gBAC1B,QAAQ,CAAC,IAAI,GAAG,MAAM,WAAW,CAAC;oBACjC,kBAAkB;oBAClB,YAAY,EAAE,OAAO;oBACrB,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK;iBACxB,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACvD,QAAQ,CAAC,IAAI,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACpD,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAClC,CAAC;YACH,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACvD,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;gBAC1B,OAAO,QAAQ,CAAC,IAAI,CAAC;YACtB,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,EAC1B,kBAAkB,EAClB,YAAY,EACZ,OAAO,GAKP,EAAgC,EAAE;IAClC,MAAM,OAAO,GAAG,YAAY,CAAC,MAE5B,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,GACvC,MAAM,kBAAkB,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAE1B,OAAO,MAAM,SAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QAClD,IAAI,CAAC;YACJ,MAAM,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;gBAC1B,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE;oBACL,IAAI;oBACJ,aAAa,EAAE,YAAY,CAAC,YAAY;oBACxC,UAAU,EAAE,SAAS;oBACrB,WAAW,EAAE,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;oBAC1C,SAAS,EAAE,QAAQ;oBACnB,QAAQ,EAAE,OAAO;oBACjB,MAAM,EAAE,SAAS;oBACjB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,YAAY,EAAE,QAAQ,CAAC,YAAY;oBACnC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;oBACjD,oBAAoB,EAAE,OAAO;iBAC7B;gBACD,WAAW,EAAE;oBACZ,GAAG,EAAE,WAAW,CAAC,IAAI;oBACrB,EAAE;iBACF;aACD,CAAC,CAAC;YACH,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC;QACtE,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,KAAK,EAChC,OAAqB,EACrB,cAA4B,EAC3B,EAAE;IACH,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,cAAc,CAAC,OAAO,CAAC;QAC5B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC;QACjE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,MAAM,CAAC,eAAe,CAC/B,qDAAqD,CACrD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC;IAC/B,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC/D,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,MAAM,CAAC,eAAe,CAC/B,oDAAoD,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CACvF,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9E,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;QAChC,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,uBAAuB,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,mBAAmB,GAAuB;QAC/C,GAAG,kBAAkB;QAGrB,IAAI,EAAE,sBAAsB;KAC5B,CAAC;IAEF,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,mBAAmB,EAAE,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAC/B,OAAkB,EACmB,EAAE;IACvC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAC3D,IACC,OAAO,QAAQ,KAAK,QAAQ;QAC5B,OAAO,YAAY,KAAK,QAAQ;QAChC,OAAO,IAAI,KAAK,QAAQ;QACxB,CAAC,UAAU,IAAI,IAAI,IAAI,OAAO,UAAU,KAAK,QAAQ,CAAC;QACtD,CAAC,UAAU,IAAI,IAAI,IAAI,UAAU,GAAG,CAAC,GAAG,EAAE,CAAC,EAC1C,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACxB,UAAU,GAAG,CAAC,GAAG,EAAE,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,KAAK,EACjC,OAAqB,EACrB,cAA4B,EAC3B,EAAE;IACH,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,cAAc,CAAC,OAAO,CAAC;QAC5B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC;QAClE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC;QACnD,QAAQ,EAAE,kBAAkB;QAC5B,OAAO,EAAE;YACR,OAAO,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC;YAClE,OAAO,EAAE;gBACR,IAAI;gBACJ,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;aAClC;SACD;QACD,WAAW,EAAE;YACZ,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,GAAG,EAAE,WAAW,CAAC,QAAQ;SACzB;KACD,CAAC,CAAC;IAEH,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,QAAQ,GAAG;QAChB,OAAO,EAAE,eAAe,CAAC,QAAQ;QACjC,QAAQ,EAAE,eAAe,CAAC,SAAS;QACnC,QAAQ,EAAE,eAAe,CAAC,QAAQ;QAClC,SAAS,EAAE,eAAe,CAAC,UAAU;KACrC,CAAC;IAEF,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AAC/C,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EAClC,OAAqB,EACrB,cAA4B,EAC3B,EAAE;IACH,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,cAAc,CAAC,OAAO,CAAC;QAC5B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC;QAClE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC;QACnD,QAAQ,EAAE,kBAAkB;QAC5B,OAAO,EAAE;YACR,OAAO,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,CAAC;YACxC,OAAO,EAAE;gBACR,IAAI;gBACJ,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;aAClC;SACD;QACD,WAAW,EAAE;YACZ,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,GAAG,EAAE,WAAW,CAAC,QAAQ;SACzB;KACD,CAAC,CAAC;IAEH,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO;QACN,IAAI;QACJ,OAAO,EAAE,eAAe,CAAC,QAAQ;QACjC,QAAQ,EAAE,eAAe,CAAC,SAAS;KACnC,CAAC;AACH,CAAC,CAAC"}

package/out/webresource-handler/webresource.d.ts

@@ -0,0 +1,42 @@
+import type { Types } from '@balena/abstract-sql-to-typescript';
+export interface MultipartUpload {
+    Read: {
+        created_at: Types['Date Time']['Read'];
+        modified_at: Types['Date Time']['Read'];
+        id: Types['Serial']['Read'];
+        uuid: Types['Short Text']['Read'];
+        resource_name: Types['Short Text']['Read'];
+        field_name: Types['Short Text']['Read'];
+        resource_id: Types['Integer']['Read'];
+        upload_id: Types['Short Text']['Read'];
+        file_key: Types['Short Text']['Read'];
+        status: 'pending' | 'completed' | 'cancelled';
+        filename: Types['Short Text']['Read'];
+        content_type: Types['Short Text']['Read'];
+        size: Types['Integer']['Read'];
+        chunk_size: Types['Integer']['Read'];
+        expiry_date: Types['Date Time']['Read'];
+        is_created_by__actor: Types['Integer']['Read'] | null;
+    };
+    Write: {
+        created_at: Types['Date Time']['Write'];
+        modified_at: Types['Date Time']['Write'];
+        id: Types['Serial']['Write'];
+        uuid: Types['Short Text']['Write'];
+        resource_name: Types['Short Text']['Write'];
+        field_name: Types['Short Text']['Write'];
+        resource_id: Types['Integer']['Write'];
+        upload_id: Types['Short Text']['Write'];
+        file_key: Types['Short Text']['Write'];
+        status: 'pending' | 'completed' | 'cancelled';
+        filename: Types['Short Text']['Write'];
+        content_type: Types['Short Text']['Write'];
+        size: Types['Integer']['Write'];
+        chunk_size: Types['Integer']['Write'];
+        expiry_date: Types['Date Time']['Write'];
+        is_created_by__actor: Types['Integer']['Write'] | null;
+    };
+}
+export default interface $Model {
+    multipart_upload: MultipartUpload;
+}

package/out/webresource-handler/webresource.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=webresource.js.map

package/out/webresource-handler/webresource.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webresource.js","sourceRoot":"","sources":["../../src/webresource-handler/webresource.ts"],"names":[],"mappings":""}

package/out/webresource-handler/webresource.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: webresource
+
+Term: actor
+	Concept Type: Integer (Type)
+Term: expiry date
+	Concept Type: Date Time (Type)
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date
+	Necessity: each multipart upload has exactly one expiry date
+Fact type: multipart upload is created by actor
+	Necessity: each multipart upload is created by at most one actor

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@balena/pinejs",
-  "version": "20.1.0-build-add-large-file-uploads-interfaces-1aa101d2ee3103f42cf7f7bc29d703a8958011e8-1",
+  "version": "20.1.0-build-large-file-uploads-2-b83400458570d926f7a6abddeb7a149e89f52abe-2",
   "main": "out/server-glue/module.js",
   "type": "module",
   "repository": "git@github.com:balena-io/pinejs.git",
@@ -20,18 +20,18 @@
     "webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
     "lint": "balena-lint -t tsconfig.dev.json -e js -e ts src build typings Gruntfile.cts && npx tsc --project tsconfig.dev.json --noEmit",
     "test": "npm run build && npm run lint && npm run webpack-build && npm run test:compose && npm run test:generated-types",
-    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC npx mocha",
+    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC PINEJS_WEBRESOURCE_MULTIPART_ENABLED=true npx mocha",
     "test:generated-types": "npm run generate-types && git diff --exit-code ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts",
     "lint-fix": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.cts",
-    "generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
+    "generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && node ./bin/sbvr-compiler.js generate-types ./src/webresource-handler/webresource.sbvr ./src/webresource-handler/webresource.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
   },
   "dependencies": {
-    "@balena/abstract-sql-compiler": "^10.1.0",
+    "@balena/abstract-sql-compiler": "^10.2.0",
     "@balena/abstract-sql-to-typescript": "^5.1.0",
     "@balena/env-parsing": "^1.2.0",
     "@balena/lf-to-abstract-sql": "^5.0.3",
-    "@balena/odata-parser": "^3.1.2",
-    "@balena/odata-to-abstract-sql": "^7.0.1",
+    "@balena/odata-parser": "^4.1.0",
+    "@balena/odata-to-abstract-sql": "^7.1.0",
     "@balena/sbvr-parser": "^1.4.6",
     "@balena/sbvr-types": "^9.1.0",
     "@types/body-parser": "^1.19.5",
@@ -69,7 +69,7 @@
   "devDependencies": {
     "@balena/lint": "^8.2.8",
     "@balena/pinejs": "file:./",
-    "@balena/pinejs-webresource-s3": "^1.0.2",
+    "@balena/pinejs-webresource-s3": "2.0.0-build-new-multiparthandle-interface-17681066eade039ed700105913ef65e00fbbfa2a-1",
     "@faker-js/faker": "^9.3.0",
     "@types/busboy": "^1.5.4",
     "@types/chai": "^5.0.1",
@@ -148,6 +148,6 @@
     "recursive": true
   },
   "versionist": {
-    "publishedAt": "2025-02-10T11:41:35.684Z"
+    "publishedAt": "2025-02-13T15:39:03.596Z"
   }
 }

package/src/config-loader/env.ts

@@ -157,6 +157,17 @@ export const tasks = {
 	queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
 };
 
+export const webResource = {
+	multipartUploadEnabled: boolVar(
+		'PINEJS_WEBRESOURCE_MULTIPART_ENABLED',
+		false,
+	),
+	singleUploadMaxFilesize: intVar(
+		'PINEJS_WEBRESOURCE_MAXFILESIZE',
+		299 * 1024 * 1024,
+	),
+};
+
 export const guardTestMockOnly = () => {
 	if (process.env.DEPLOYMENT !== 'TEST') {
 		throw new Error('Attempting to use TEST_MOCK_ONLY outside of tests');

package/src/sbvr-api/abstract-sql.ts

@@ -1,5 +1,6 @@
 import _ from 'lodash';
 
+import type { Engines } from '@balena/abstract-sql-compiler';
 import AbstractSQLCompiler from '@balena/abstract-sql-compiler';
 import type { BindKey } from '@balena/odata-parser';
 import {
@@ -78,7 +79,7 @@ export const getAndCheckBindValues = async (
 	return await Promise.all(
 		bindings.map(async (binding) => {
 			let fieldName = '';
-			let field: { dataType: string };
+			let dataType: string;
 			let value: any;
 			if (binding[0] === 'Bind') {
 				const bindValue = binding[1];
@@ -103,7 +104,7 @@ export const getAndCheckBindValues = async (
 					if (maybeField == null) {
 						throw new Error(`Could not find field '${fieldName}'`);
 					}
-					field = maybeField;
+					dataType = maybeField.dataType;
 				} else if (Number.isInteger(bindValue)) {
 					if (bindValue >= odataBinds.length) {
 						console.error(
@@ -112,9 +113,7 @@ export const getAndCheckBindValues = async (
 						);
 						throw new Error('Invalid binding');
 					}
-					let dataType;
 					[dataType, value] = odataBinds[bindValue];
-					field = { dataType };
 				} else if (typeof bindValue === 'string') {
 					if (!Object.hasOwn(odataBinds, bindValue)) {
 						console.error(
@@ -123,16 +122,12 @@ export const getAndCheckBindValues = async (
 						);
 						throw new Error('Invalid binding');
 					}
-					let dataType;
 					[dataType, value] = odataBinds[bindValue as BindKey];
-					field = { dataType };
 				} else {
 					throw new Error(`Unknown binding: ${binding}`);
 				}
 			} else {
-				let dataType;
 				[dataType, value] = binding;
-				field = { dataType };
 			}
 
 			if (value === undefined) {
@@ -140,7 +135,7 @@ export const getAndCheckBindValues = async (
 			}
 
 			try {
-				return await AbstractSQLCompiler[engine].dataTypeValidate(value, field);
+				return await validateBindingType(engine, value, dataType);
 			} catch (err: any) {
 				throw new BadRequestError(`"${fieldName}" ${err.message}`);
 			}
@@ -148,6 +143,30 @@ export const getAndCheckBindValues = async (
 	);
 };
 
+const validateBindingType = async (
+	engine: Engines,
+	$value: any,
+	$dataType: string,
+): Promise<any> => {
+	if ($dataType === 'List') {
+		if (!Array.isArray($value)) {
+			throw new Error('List value binding must be an array');
+		}
+		return await Promise.all(
+			$value.map(async ([dataType, value]: [string, any]) => {
+				// Null is a special case for list values
+				if (dataType === 'Null') {
+					return null;
+				}
+				return await validateBindingType(engine, value, dataType);
+			}),
+		);
+	}
+	return await AbstractSQLCompiler[engine].dataTypeValidate($value, {
+		dataType: $dataType,
+	});
+};
+
 const checkModifiedFields = (
 	ruleReferencedFields: AbstractSQLCompiler.RuleReferencedFields,
 	modifiedFields: AbstractSQLCompiler.ModifiedFields,

package/src/server-glue/module.ts

@@ -7,6 +7,7 @@ import * as configLoader from '../config-loader/config-loader.js';
 import * as migrator from '../migrator/sync.js';
 import type * as migratorUtils from '../migrator/utils.js';
 import * as tasks from '../tasks/index.js';
+import * as webresource from '../webresource-handler/index.js';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils.js';
 import { PINEJS_ADVISORY_LOCK } from '../config-loader/env.js';
@@ -66,6 +67,7 @@ export const init = async <T extends string>(
 		const cfgLoader = configLoader.setup(app);
 		await cfgLoader.loadConfig(migrator.config);
 		await cfgLoader.loadConfig(tasks.config);
+		await cfgLoader.loadConfig(webresource.config);
 
 		const promises: Array<Promise<void>> = [];
 		if (process.env.SBVR_SERVER_ENABLED) {