@balena/pinejs 21.2.1 → 21.3.0-build-large-file-uploads-2-34511cebbabc0ea17d1d3d43274abd747a7316c6-1

package/out/webresource-handler/multipartUpload.js

@@ -0,0 +1,249 @@
+import { randomUUID } from 'node:crypto';
+import { getWebResourceFields } from './index.js';
+import { api } from '../sbvr-api/sbvr-utils.js';
+import { errors, sbvrUtils } from '../server-glue/module.js';
+import { webResource as webResourceEnv } from '../config-loader/env.js';
+import * as permissions from '../sbvr-api/permissions.js';
+export const isMultipartUploadAvailable = (webResourceHandler) => {
+    return (webResourceEnv.multipartUploadEnabled &&
+        webResourceHandler.multipartUpload != null);
+};
+export const multipartUploadHooks = (webResourceHandler) => {
+    return {
+        POSTPARSE: async ({ req, request, tx, api: applicationApi }) => {
+            if (request.odataQuery.property?.resource === 'beginUpload') {
+                const uploadParams = await validateBeginUpload(request, applicationApi, webResourceHandler);
+                tx = await sbvrUtils.db.transaction();
+                req.tx = tx;
+                request.tx = tx;
+                request.method = 'PATCH';
+                request.values = uploadParams;
+                request.odataQuery.resource = request.resourceName;
+                delete request.odataQuery.property;
+                request.custom.isAction = 'beginUpload';
+            }
+            else if (request.odataQuery.property?.resource === 'commitUpload') {
+                const commitPayload = await validateCommitUpload(request, applicationApi);
+                const webresource = await webResourceHandler.multipartUpload.commit({
+                    fileKey: commitPayload.metadata.fileKey,
+                    uploadId: commitPayload.metadata.uploadId,
+                    filename: commitPayload.metadata.filename,
+                    providerCommitData: commitPayload.providerCommitData,
+                });
+                await api.webresource.patch({
+                    resource: 'multipart_upload',
+                    body: {
+                        status: 'completed',
+                    },
+                    options: {
+                        $filter: {
+                            uuid: commitPayload.uuid,
+                        },
+                    },
+                    passthrough: {
+                        tx: tx,
+                        req: permissions.root,
+                    },
+                });
+                request.method = 'PATCH';
+                request.values = {
+                    [commitPayload.metadata.fieldName]: webresource,
+                };
+                request.odataQuery.resource = request.resourceName;
+                delete request.odataQuery.property;
+                request.custom.isAction = 'commitUpload';
+                request.custom.commitUploadPayload = webresource;
+            }
+            else if (request.odataQuery.property?.resource === 'cancelUpload') {
+                const { uuid, fileKey, uploadId } = await validateCancelPayload(request, applicationApi);
+                await webResourceHandler.multipartUpload.cancel({ fileKey, uploadId });
+                await api.webresource.patch({
+                    resource: 'multipart_upload',
+                    body: {
+                        status: 'cancelled',
+                    },
+                    options: {
+                        $filter: { uuid },
+                    },
+                    passthrough: {
+                        tx: tx,
+                        req: permissions.root,
+                    },
+                });
+                request.method = 'GET';
+                request.odataQuery.resource = request.resourceName;
+                delete request.odataQuery.property;
+                request.custom.isAction = 'cancelUpload';
+            }
+        },
+        PRERESPOND: async ({ req, request, response, tx }) => {
+            if (request.custom.isAction === 'beginUpload') {
+                await tx.rollback();
+                response.statusCode = 200;
+                response.body = await beginUpload({
+                    webResourceHandler,
+                    odataRequest: request,
+                    actorId: req.user?.actor,
+                });
+            }
+            else if (request.custom.isAction === 'commitUpload') {
+                response.body = await webResourceHandler.onPreRespond(request.custom.commitUploadPayload);
+            }
+            else if (request.custom.isAction === 'cancelUpload') {
+                response.statusCode = 204;
+                delete response.body;
+            }
+        },
+    };
+};
+const beginUpload = async ({ webResourceHandler, odataRequest, actorId, }) => {
+    const payload = odataRequest.values;
+    const fieldName = Object.keys(payload)[0];
+    const metadata = payload[fieldName];
+    const { fileKey, uploadId, uploadParts } = await webResourceHandler.multipartUpload.begin(fieldName, metadata);
+    const uuid = randomUUID();
+    return await sbvrUtils.db.transaction(async (tx) => {
+        try {
+            await api.webresource.post({
+                resource: 'multipart_upload',
+                body: {
+                    uuid,
+                    resource_name: odataRequest.resourceName,
+                    field_name: fieldName,
+                    resource_id: odataRequest.affectedIds?.[0],
+                    upload_id: uploadId,
+                    file_key: fileKey,
+                    status: 'pending',
+                    filename: metadata.filename,
+                    content_type: metadata.content_type,
+                    size: metadata.size,
+                    chunk_size: metadata.chunk_size,
+                    expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000,
+                    is_created_by__actor: actorId,
+                },
+                passthrough: {
+                    req: permissions.root,
+                    tx,
+                },
+            });
+            return { [fieldName]: { uuid, uploadParts } };
+        }
+        catch (err) {
+            console.error('failed to start multipart upload', err);
+            throw new errors.BadRequestError('Failed to start multipart upload');
+        }
+    });
+};
+const validateBeginUpload = async (request, applicationApi, webResourceHandler) => {
+    await canAccess(request, applicationApi);
+    const fieldNames = Object.keys(request.values);
+    if (fieldNames.length !== 1) {
+        throw new errors.BadRequestError('You can only get upload url for one field at a time');
+    }
+    const [fieldName] = fieldNames;
+    const webResourceFields = getWebResourceFields(request);
+    if (!webResourceFields.includes(fieldName)) {
+        throw new errors.BadRequestError(`The provided field '${fieldName}' is not a valid webresource`);
+    }
+    const beginUploadPayload = parseBeginUploadPayload(request.values[fieldName], webResourceHandler);
+    if (beginUploadPayload == null) {
+        throw new errors.BadRequestError('Invalid file metadata');
+    }
+    const uploadMetadataCheck = {
+        ...beginUploadPayload,
+        href: 'metadata_check_probe',
+    };
+    return { [fieldName]: uploadMetadataCheck };
+};
+const parseBeginUploadPayload = (payload, webResourceHandler) => {
+    if (payload == null || typeof payload !== 'object') {
+        return null;
+    }
+    let { filename, content_type, size, chunk_size } = payload;
+    if (typeof filename !== 'string' ||
+        typeof content_type !== 'string' ||
+        typeof size !== 'number' ||
+        (chunk_size != null && typeof chunk_size !== 'number') ||
+        (chunk_size != null &&
+            chunk_size < webResourceHandler.multipartUpload.getMinimumPartSize())) {
+        return null;
+    }
+    chunk_size ??= webResourceHandler.multipartUpload.getDefaultPartSize();
+    return { filename, content_type, size, chunk_size };
+};
+const validateCommitUpload = async (request, applicationApi) => {
+    await canAccess(request, applicationApi);
+    const { uuid, providerCommitData } = request.values;
+    if (typeof uuid !== 'string') {
+        throw new errors.BadRequestError('Invalid uuid type');
+    }
+    const [multipartUpload] = await api.webresource.get({
+        resource: 'multipart_upload',
+        options: {
+            $select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
+            $filter: {
+                uuid,
+                status: 'pending',
+                expiry_date: { $gt: { $now: {} } },
+            },
+        },
+        passthrough: {
+            tx: request.tx,
+            req: permissions.rootRead,
+        },
+    });
+    if (multipartUpload == null) {
+        throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+    }
+    const metadata = {
+        fileKey: multipartUpload.file_key,
+        uploadId: multipartUpload.upload_id,
+        filename: multipartUpload.filename,
+        fieldName: multipartUpload.field_name,
+    };
+    return { uuid, providerCommitData, metadata };
+};
+const validateCancelPayload = async (request, applicationApi) => {
+    await canAccess(request, applicationApi);
+    const { uuid } = request.values;
+    if (typeof uuid !== 'string') {
+        throw new errors.BadRequestError('Invalid uuid type');
+    }
+    const [multipartUpload] = await api.webresource.get({
+        resource: 'multipart_upload',
+        options: {
+            $select: ['id', 'file_key', 'upload_id'],
+            $filter: {
+                uuid,
+                status: 'pending',
+                expiry_date: { $gt: { $now: {} } },
+            },
+        },
+        passthrough: {
+            tx: request.tx,
+            req: permissions.rootRead,
+        },
+    });
+    if (multipartUpload == null) {
+        throw new errors.BadRequestError(`Invalid upload for uuid ${uuid}`);
+    }
+    return {
+        uuid,
+        fileKey: multipartUpload.file_key,
+        uploadId: multipartUpload.upload_id,
+    };
+};
+const canAccess = async (request, applicationApi) => {
+    if (request.odataQuery.key == null) {
+        throw new errors.BadRequestError();
+    }
+    const canAccessUrl = request.url
+        .slice(1)
+        .replace(/(beginUpload|commitUpload|cancelUpload)$/, 'canAccess');
+    await applicationApi.request({
+        method: 'POST',
+        url: canAccessUrl,
+        body: { method: 'PATCH' },
+    });
+};
+//# sourceMappingURL=multipartUpload.js.map

package/out/webresource-handler/multipartUpload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multipartUpload.js","sourceRoot":"","sources":["../../src/webresource-handler/multipartUpload.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,WAAW,MAAM,4BAA4B,CAAC;AAe1D,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACzC,kBAAsC,EACS,EAAE;IACjD,OAAO,CACN,cAAc,CAAC,sBAAsB;QACrC,kBAAkB,CAAC,eAAe,IAAI,IAAI,CAC1C,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CACnC,kBAA0C,EACxB,EAAE;IACpB,OAAO;QACN,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,EAAE;YAC9D,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAC7D,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC7C,OAAO,EACP,cAAc,EACd,kBAAkB,CAClB,CAAC;gBAOF,EAAE,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;gBACtC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;gBAEhB,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC;gBACzB,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;gBAC9B,OAAO,CAAC,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;gBACnD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,aAAa,CAAC;YACzC,CAAC;iBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACrE,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAC/C,OAAO,EACP,cAAc,CACd,CAAC;gBAEF,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC,eAAe,CAAC,MAAM,CAAC;oBACnE,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,OAAO;oBACvC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,QAAQ;oBACzC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,QAAQ;oBACzC,kBAAkB,EAAE,aAAa,CAAC,kBAAkB;iBACpD,CAAC,CAAC;gBAEH,MAAM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC;oBAC3B,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE;wBACL,MAAM,EAAE,WAAW;qBACnB;oBACD,OAAO,EAAE;wBACR,OAAO,EAAE;4BACR,IAAI,EAAE,aAAa,CAAC,IAAI;yBACxB;qBACD;oBACD,WAAW,EAAE;wBACZ,EAAE,EAAE,EAAE;wBACN,GAAG,EAAE,WAAW,CAAC,IAAI;qBACrB;iBACD,CAAC,CAAC;gBAEH,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC;gBACzB,OAAO,CAAC,MAAM,GAAG;oBAChB,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,WAAW;iBAC/C,CAAC;gBACF,OAAO,CAAC,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;gBACnD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC;gBACzC,OAAO,CAAC,MAAM,CAAC,mBAAmB,GAAG,WAAW,CAAC;YAClD,CAAC;iBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACrE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,qBAAqB,CAC9D,OAAO,EACP,cAAc,CACd,CAAC;gBAEF,MAAM,kBAAkB,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAEvE,MAAM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC;oBAC3B,QAAQ,EAAE,kBAAkB;oBAC5B,IAAI,EAAE;wBACL,MAAM,EAAE,WAAW;qBACnB;oBACD,OAAO,EAAE;wBACR,OAAO,EAAE,EAAE,IAAI,EAAE;qBACjB;oBACD,WAAW,EAAE;wBACZ,EAAE,EAAE,EAAE;wBACN,GAAG,EAAE,WAAW,CAAC,IAAI;qBACrB;iBACD,CAAC,CAAC;gBAEH,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;gBACvB,OAAO,CAAC,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;gBACnD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC;YAC1C,CAAC;QACF,CAAC;QACD,UAAU,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE;YACpD,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAK/C,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;gBAEpB,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;gBAC1B,QAAQ,CAAC,IAAI,GAAG,MAAM,WAAW,CAAC;oBACjC,kBAAkB;oBAClB,YAAY,EAAE,OAAO;oBACrB,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK;iBACxB,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACvD,QAAQ,CAAC,IAAI,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACpD,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAClC,CAAC;YACH,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;gBACvD,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;gBAC1B,OAAO,QAAQ,CAAC,IAAI,CAAC;YACtB,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,EAC1B,kBAAkB,EAClB,YAAY,EACZ,OAAO,GAKP,EAAgC,EAAE;IAClC,MAAM,OAAO,GAAG,YAAY,CAAC,MAE5B,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,GACvC,MAAM,kBAAkB,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAE1B,OAAO,MAAM,SAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QAClD,IAAI,CAAC;YACJ,MAAM,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;gBAC1B,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE;oBACL,IAAI;oBACJ,aAAa,EAAE,YAAY,CAAC,YAAY;oBACxC,UAAU,EAAE,SAAS;oBACrB,WAAW,EAAE,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;oBAC1C,SAAS,EAAE,QAAQ;oBACnB,QAAQ,EAAE,OAAO;oBACjB,MAAM,EAAE,SAAS;oBACjB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,YAAY,EAAE,QAAQ,CAAC,YAAY;oBACnC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;oBACjD,oBAAoB,EAAE,OAAO;iBAC7B;gBACD,WAAW,EAAE;oBACZ,GAAG,EAAE,WAAW,CAAC,IAAI;oBACrB,EAAE;iBACF;aACD,CAAC,CAAC;YACH,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC;QACtE,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,KAAK,EAChC,OAAqB,EACrB,cAA4B,EAC5B,kBAA0C,EACzC,EAAE;IACH,MAAM,SAAS,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAEzC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,MAAM,CAAC,eAAe,CAC/B,qDAAqD,CACrD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC;IAC/B,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACxD,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,MAAM,CAAC,eAAe,CAC/B,uBAAuB,SAAS,8BAA8B,CAC9D,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,uBAAuB,CACjD,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EACzB,kBAAkB,CAClB,CAAC;IACF,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;QAChC,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,uBAAuB,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,mBAAmB,GAAuB;QAC/C,GAAG,kBAAkB;QAGrB,IAAI,EAAE,sBAAsB;KAC5B,CAAC;IAEF,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,mBAAmB,EAAE,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAC/B,OAAkB,EAClB,kBAA0C,EACL,EAAE;IACvC,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAC3D,IACC,OAAO,QAAQ,KAAK,QAAQ;QAC5B,OAAO,YAAY,KAAK,QAAQ;QAChC,OAAO,IAAI,KAAK,QAAQ;QACxB,CAAC,UAAU,IAAI,IAAI,IAAI,OAAO,UAAU,KAAK,QAAQ,CAAC;QACtD,CAAC,UAAU,IAAI,IAAI;YAClB,UAAU,GAAG,kBAAkB,CAAC,eAAe,CAAC,kBAAkB,EAAE,CAAC,EACrE,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IAED,UAAU,KAAK,kBAAkB,CAAC,eAAe,CAAC,kBAAkB,EAAE,CAAC;IAEvE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,KAAK,EACjC,OAAqB,EACrB,cAA4B,EAC3B,EAAE;IACH,MAAM,SAAS,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAEzC,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC;QACnD,QAAQ,EAAE,kBAAkB;QAC5B,OAAO,EAAE;YACR,OAAO,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC;YAClE,OAAO,EAAE;gBACR,IAAI;gBACJ,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;aAClC;SACD;QACD,WAAW,EAAE;YACZ,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,GAAG,EAAE,WAAW,CAAC,QAAQ;SACzB;KACD,CAAC,CAAC;IAEH,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,QAAQ,GAAG;QAChB,OAAO,EAAE,eAAe,CAAC,QAAQ;QACjC,QAAQ,EAAE,eAAe,CAAC,SAAS;QACnC,QAAQ,EAAE,eAAe,CAAC,QAAQ;QAClC,SAAS,EAAE,eAAe,CAAC,UAAU;KACrC,CAAC;IAEF,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AAC/C,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EAClC,OAAqB,EACrB,cAA4B,EAC3B,EAAE;IACH,MAAM,SAAS,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC;QACnD,QAAQ,EAAE,kBAAkB;QAC5B,OAAO,EAAE;YACR,OAAO,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,CAAC;YACxC,OAAO,EAAE;gBACR,IAAI;gBACJ,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;aAClC;SACD;QACD,WAAW,EAAE;YACZ,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,GAAG,EAAE,WAAW,CAAC,QAAQ;SACzB;KACD,CAAC,CAAC;IAEH,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,MAAM,CAAC,eAAe,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO;QACN,IAAI;QACJ,OAAO,EAAE,eAAe,CAAC,QAAQ;QACjC,QAAQ,EAAE,eAAe,CAAC,SAAS;KACnC,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACtB,OAAqB,EACrB,cAA4B,EAC3B,EAAE;IACH,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG;SAC9B,KAAK,CAAC,CAAC,CAAC;SACR,OAAO,CAAC,0CAA0C,EAAE,WAAW,CAAC,CAAC;IAEnE,MAAM,cAAc,CAAC,OAAO,CAAC;QAC5B,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,YAAY;QACjB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;KACzB,CAAC,CAAC;AACJ,CAAC,CAAC"}

package/out/webresource-handler/webresource.d.ts

@@ -0,0 +1,42 @@
+import type { Types } from '@balena/abstract-sql-to-typescript';
+export interface MultipartUpload {
+    Read: {
+        created_at: Types['Date Time']['Read'];
+        modified_at: Types['Date Time']['Read'];
+        id: Types['Serial']['Read'];
+        uuid: Types['Short Text']['Read'];
+        resource_name: Types['Short Text']['Read'];
+        field_name: Types['Short Text']['Read'];
+        resource_id: Types['Integer']['Read'];
+        upload_id: Types['Short Text']['Read'];
+        file_key: Types['Short Text']['Read'];
+        status: 'pending' | 'completed' | 'cancelled';
+        filename: Types['Short Text']['Read'];
+        content_type: Types['Short Text']['Read'];
+        size: Types['Integer']['Read'];
+        chunk_size: Types['Integer']['Read'];
+        expiry_date: Types['Date Time']['Read'];
+        is_created_by__actor: Types['Integer']['Read'] | null;
+    };
+    Write: {
+        created_at: Types['Date Time']['Write'];
+        modified_at: Types['Date Time']['Write'];
+        id: Types['Serial']['Write'];
+        uuid: Types['Short Text']['Write'];
+        resource_name: Types['Short Text']['Write'];
+        field_name: Types['Short Text']['Write'];
+        resource_id: Types['Integer']['Write'];
+        upload_id: Types['Short Text']['Write'];
+        file_key: Types['Short Text']['Write'];
+        status: 'pending' | 'completed' | 'cancelled';
+        filename: Types['Short Text']['Write'];
+        content_type: Types['Short Text']['Write'];
+        size: Types['Integer']['Write'];
+        chunk_size: Types['Integer']['Write'];
+        expiry_date: Types['Date Time']['Write'];
+        is_created_by__actor: Types['Integer']['Write'] | null;
+    };
+}
+export default interface $Model {
+    multipart_upload: MultipartUpload;
+}

package/out/webresource-handler/webresource.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=webresource.js.map

package/out/webresource-handler/webresource.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webresource.js","sourceRoot":"","sources":["../../src/webresource-handler/webresource.ts"],"names":[],"mappings":""}

package/out/webresource-handler/webresource.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: webresource
+
+Term: actor
+	Concept Type: Integer (Type)
+Term: expiry date
+	Concept Type: Date Time (Type)
+Term: uuid
+	Concept Type: Short Text (Type)
+Term: resource name
+	Concept Type: Short Text (Type)
+Term: field name
+	Concept Type: Short Text (Type)
+Term: resource id
+	Concept Type: Integer (Type)
+Term: upload id
+	Concept Type: Short Text (Type)
+Term: file key
+	Concept Type: Short Text (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: filename
+	Concept Type: Short Text (Type)
+Term: content type
+	Concept Type: Short Text (Type)
+Term: size
+	Concept Type: Integer (Type)
+Term: chunk size
+	Concept Type: Integer (Type)
+Term: valid until date
+	Concept Type: Date Time (Type)
+
+Term: multipart upload
+Fact type: multipart upload has uuid
+	Necessity: each multipart upload has exactly one uuid
+	Necessity: each uuid is of exactly one multipart upload
+Fact type: multipart upload has resource name
+	Necessity: each multipart upload has exactly one resource name
+Fact type: multipart upload has field name
+	Necessity: each multipart upload has exactly one field name
+Fact type: multipart upload has resource id
+	Necessity: each multipart upload has exactly one resource id
+Fact type: multipart upload has upload id
+	Necessity: each multipart upload has exactly one upload id
+Fact type: multipart upload has file key
+	Necessity: each multipart upload has exactly one file key
+Fact type: multipart upload has status
+	Necessity: each multipart upload has exactly one status
+	Definition: "pending" or "completed" or "cancelled"
+Fact type: multipart upload has filename
+	Necessity: each multipart upload has exactly one filename
+Fact type: multipart upload has content type
+	Necessity: each multipart upload has exactly one content type
+Fact type: multipart upload has size
+	Necessity: each multipart upload has exactly one size
+Fact type: multipart upload has chunk size
+	Necessity: each multipart upload has exactly one chunk size
+Fact type: multipart upload has expiry date
+	Necessity: each multipart upload has exactly one expiry date
+Fact type: multipart upload is created by actor
+	Necessity: each multipart upload is created by at most one actor

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@balena/pinejs",
-  "version": "21.2.1",
+  "version": "21.3.0-build-large-file-uploads-2-34511cebbabc0ea17d1d3d43274abd747a7316c6-1",
   "main": "out/server-glue/module.js",
   "type": "module",
   "repository": "git@github.com:balena-io/pinejs.git",
@@ -20,10 +20,10 @@
     "webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
     "lint": "balena-lint -t tsconfig.dev.json -e js -e ts src test build typings Gruntfile.cts && npx tsc --project tsconfig.dev.json --noEmit",
     "test": "npm run build && npm run lint && npm run webpack-build && npm run test:compose && npm run test:generated-types",
-    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC npx mocha",
+    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 TZ=UTC PINEJS_WEBRESOURCE_MULTIPART_ENABLED=true npx mocha",
     "test:generated-types": "npm run generate-types && git diff --exit-code ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts",
     "lint-fix": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.cts",
-    "generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
+    "generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && node ./bin/sbvr-compiler.js generate-types ./src/tasks/tasks.sbvr ./src/tasks/tasks.ts && node ./bin/sbvr-compiler.js generate-types ./src/webresource-handler/webresource.sbvr ./src/webresource-handler/webresource.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
   },
   "dependencies": {
     "@balena/abstract-sql-compiler": "^10.2.3",
@@ -70,7 +70,7 @@
   "devDependencies": {
     "@balena/lint": "^9.1.6",
     "@balena/pinejs": "file:./",
-    "@balena/pinejs-webresource-s3": "^1.0.4",
+    "@balena/pinejs-webresource-s3": "2.0.0-build-new-multiparthandle-interface-66b03581234eaa7ce15c6f389e39b5b7ed3d1bc5-1",
     "@faker-js/faker": "^9.6.0",
     "@types/busboy": "^1.5.4",
     "@types/chai": "^5.2.1",
@@ -149,6 +149,6 @@
     "recursive": true
   },
   "versionist": {
-    "publishedAt": "2025-04-16T13:49:26.328Z"
+    "publishedAt": "2025-04-18T12:23:10.990Z"
   }
 }

package/src/config-loader/env.ts

@@ -159,6 +159,17 @@ export const tasks = {
 	queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
 };
 
+export const webResource = {
+	multipartUploadEnabled: boolVar(
+		'PINEJS_WEBRESOURCE_MULTIPART_ENABLED',
+		false,
+	),
+	singleUploadMaxFilesize: intVar(
+		'PINEJS_WEBRESOURCE_MAXFILESIZE',
+		299 * 1024 * 1024,
+	),
+};
+
 export const guardTestMockOnly = () => {
 	if (process.env.DEPLOYMENT !== 'TEST') {
 		throw new Error('Attempting to use TEST_MOCK_ONLY outside of tests');

package/src/server-glue/module.ts

@@ -7,6 +7,7 @@ import * as configLoader from '../config-loader/config-loader.js';
 import * as migrator from '../migrator/sync.js';
 import type * as migratorUtils from '../migrator/utils.js';
 import * as tasks from '../tasks/index.js';
+import * as webresource from '../webresource-handler/index.js';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils.js';
 import { PINEJS_ADVISORY_LOCK } from '../config-loader/env.js';
@@ -66,6 +67,7 @@ export const init = async <T extends string>(
 		const cfgLoader = configLoader.setup(app);
 		await cfgLoader.loadConfig(migrator.config);
 		await cfgLoader.loadConfig(tasks.config);
+		await cfgLoader.loadConfig(webresource.config);
 
 		if (!process.env.CONFIG_LOADER_DISABLED) {
 			await cfgLoader.loadApplicationConfig(config);

package/src/webresource-handler/index.ts

@@ -11,12 +11,20 @@ import {
 	odataNameToSqlName,
 	sqlNameToODataName,
 } from '@balena/odata-to-abstract-sql';
+import type { ConfigLoader } from '../server-glue/module.js';
 import { errors, permissions } from '../server-glue/module.js';
 import type { WebResourceType as WebResource } from '@balena/sbvr-types';
 import { TypedError } from 'typed-error';
 import type { Resolvable } from '../sbvr-api/common-types.js';
+import type WebresourceModel from './webresource.js';
+import { importSBVR } from '../server-glue/sbvr-loader.js';
+import {
+	isMultipartUploadAvailable,
+	multipartUploadHooks,
+} from './multipartUpload.js';
 
 export * from './handlers/index.js';
+export type { BeginUploadResponse } from './multipartUpload.js';
 
 export interface IncomingFile {
 	fieldname: string;
@@ -31,10 +39,51 @@ export interface UploadResponse {
 	filename: string;
 }
 
+export interface BeginMultipartUploadPayload {
+	filename: string;
+	content_type: string;
+	size: number;
+	chunk_size: number;
+}
+
+export interface UploadPart {
+	url: string;
+	chunkSize: number;
+	partNumber: number;
+}
+
+export interface BeginMultipartUploadHandlerResponse {
+	uploadParts: UploadPart[];
+	fileKey: string;
+	uploadId: string;
+}
+
+export interface CommitMultipartUploadPayload {
+	fileKey: string;
+	uploadId: string;
+	filename: string;
+	providerCommitData?: Record<string, any>;
+}
+
+export interface CancelMultipartUploadPayload {
+	fileKey: string;
+	uploadId: string;
+}
+
 export interface WebResourceHandler {
 	handleFile: (resource: IncomingFile) => Promise<UploadResponse>;
 	removeFile: (fileReference: string) => Promise<void>;
 	onPreRespond: (webResource: WebResource) => Promise<WebResource>;
+	multipartUpload?: {
+		begin: (
+			fieldName: string,
+			payload: BeginMultipartUploadPayload,
+		) => Promise<BeginMultipartUploadHandlerResponse>;
+		commit: (commitInfo: CommitMultipartUploadPayload) => Promise<WebResource>;
+		cancel: (cancelInfo: CancelMultipartUploadPayload) => Promise<void>;
+		getMinimumPartSize: () => number;
+		getDefaultPartSize: () => number;
+	};
 }
 
 export class WebResourceError extends TypedError {}
@@ -288,6 +337,9 @@ const throwIfWebresourceNotInMultipart = (
 	{ req, request }: HookArgs,
 ) => {
 	if (
+		request.custom.isAction !== 'beginUpload' &&
+		request.custom.isAction !== 'commitUpload' &&
+		request.custom.isAction !== 'cancelUpload' &&
 		!req.is?.('multipart') &&
 		webResourceFields.some((field) => request.values[field] != null)
 	) {
@@ -484,4 +536,37 @@ export const setupUploadHooks = (
 		resourceName,
 		getCreateWebResourceHooks(handler),
 	);
+
+	if (isMultipartUploadAvailable(handler)) {
+		sbvrUtils.addPureHook(
+			'POST',
+			apiRoot,
+			resourceName,
+			multipartUploadHooks(handler),
+		);
+	}
+};
+
+const initSql = `
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_uuid ON "multipart upload" (uuid);
+CREATE INDEX IF NOT EXISTS idx_multipart_upload_status ON "multipart upload" (status);
+`;
+
+const modelText = await importSBVR('./webresource.sbvr', import.meta);
+
+declare module '../sbvr-api/sbvr-utils.js' {
+	export interface API {
+		webresource: PinejsClient<WebresourceModel>;
+	}
+}
+
+export const config: ConfigLoader.Config = {
+	models: [
+		{
+			modelName: 'webresource',
+			apiRoot: 'webresource',
+			modelText,
+			initSql,
+		},
+	],
 };