eip-cloud-services 1.6.0 → 1.7.0

package/CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ## Unreleased
 
 ### Added
+- Added browser-safe S3 multipart upload initiation, part signing, completion, and abort helpers for files above the single-PUT limit.
 - Added staging-safe OnDemand Content Ingest, Content Discovery, and Azure Blob upload clients with injectable fetch, timeouts, production-host guards, and signed-query redaction.
 - Added bounded S3 object streams and resumable Azure block-blob uploads for large server-to-server media transfers without temporary disk.
 - Added callback-scoped loopback S3 range reads for seekable media inspection and server-side multipart copy for objects above 5 GiB.

package/README.md

@@ -4,6 +4,8 @@
 
 The EIP Cloud Services Module is a comprehensive Node.js package that provides seamless integration with various cloud services, including Redis, AWS S3, CDN, AWS Lambda, AWS SQS, and MySQL. This module is designed to simplify the complexities of interacting with these cloud services, offering a range of functionalities from data caching and storage to content delivery and serverless computing.
 
+The S3 service supports multipart browser upload orchestration through `createMultipartUpload`, `signMultipartUploadParts`, `completeMultipartUpload`, and `abortMultipartUpload`. Completion lists the uploaded parts from S3 so clients do not need access to response ETags.
+
 ## Installation and Import
 
 To use this module, first install it in your Node.js project. Then, import the required services as follows:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {
@@ -18,6 +18,7 @@
     "@aws-sdk/client-cloudfront": "^3.354.0",
     "@aws-sdk/client-lambda": "^3.356.0",
     "@aws-sdk/client-s3": "^3.354.0",
+    "@aws-sdk/s3-request-presigner": "^3.354.0",
     "@aws-sdk/client-sqs": "^3.356.0",
     "@aws-sdk/client-secrets-manager": "^3.354.0",
     "config": "^3.3.9",

package/src/s3.js

@@ -41,6 +41,7 @@
 const { S3Client, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, CopyObjectCommand, ListObjectsV2Command } = require ( '@aws-sdk/client-s3' );
 const { copyLargeObject } = require ( './s3LargeObject' );
 const { withLocalReadUrl } = require ( './s3LocalReadServer' );
+const multipartUpload = require ( './s3MultipartUpload' );
 const fs = require ( 'fs' );
 const path = require ( 'path' );
 let config = {};
@@ -238,6 +239,40 @@ exports.withLocalReadUrl = async ( key, bucket = config?.s3?.Bucket, callback )
     bucket
 }, callback );
 
+exports.createMultipartUpload = async ( key, bucket = config?.s3?.Bucket, options = {} ) =>
+    multipartUpload.createMultipartUpload ( {
+        client: S3,
+        key,
+        bucket,
+        ...options
+    } );
+
+exports.signMultipartUploadParts = async ( key, uploadId, partNumbers, bucket = config?.s3?.Bucket, options = {} ) =>
+    multipartUpload.signMultipartUploadParts ( {
+        client: S3,
+        key,
+        bucket,
+        uploadId,
+        partNumbers,
+        ...options
+    } );
+
+exports.completeMultipartUpload = async ( key, uploadId, bucket = config?.s3?.Bucket ) =>
+    multipartUpload.completeMultipartUpload ( {
+        client: S3,
+        key,
+        bucket,
+        uploadId
+    } );
+
+exports.abortMultipartUpload = async ( key, uploadId, bucket = config?.s3?.Bucket ) =>
+    multipartUpload.abortMultipartUpload ( {
+        client: S3,
+        key,
+        bucket,
+        uploadId
+    } );
+
 /**
  * Set an object in S3.
  *

package/src/s3MultipartUpload.js

@@ -0,0 +1,154 @@
+const {
+    AbortMultipartUploadCommand,
+    CompleteMultipartUploadCommand,
+    CreateMultipartUploadCommand,
+    ListPartsCommand,
+    UploadPartCommand
+} = require ( '@aws-sdk/client-s3' );
+const { getSignedUrl } = require ( '@aws-sdk/s3-request-presigner' );
+
+const MIB = 1024 * 1024;
+const DEFAULT_PART_BYTES = 128 * MIB;
+const MAX_MULTIPART_PARTS = 10000;
+const MAX_PART_BYTES = 5 * 1024 * 1024 * 1024;
+
+const resolveUploadPartSize = ( contentLength, requestedPartSize = DEFAULT_PART_BYTES ) => {
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( 'S3 multipart upload requires a positive, safe content length' );
+    }
+
+    const minimumForPartLimit = Math.ceil ( contentLength / MAX_MULTIPART_PARTS );
+    const selected = Math.max ( Number ( requestedPartSize ) || DEFAULT_PART_BYTES, minimumForPartLimit );
+    const rounded = Math.ceil ( selected / MIB ) * MIB;
+
+    if ( rounded > MAX_PART_BYTES ) {
+        throw new Error ( `S3 object is too large to upload in ${MAX_MULTIPART_PARTS} parts` );
+    }
+
+    return rounded;
+};
+
+const createMultipartUpload = async ( {
+    client,
+    bucket,
+    key,
+    contentLength,
+    contentType,
+    acl = 'public-read',
+    metadata = {},
+    partSize
+} ) => {
+    const resolvedPartSize = resolveUploadPartSize ( contentLength, partSize );
+    const created = await client.send ( new CreateMultipartUploadCommand ( {
+        Bucket: bucket,
+        Key: key,
+        ContentType: contentType,
+        ACL: acl,
+        Metadata: metadata
+    } ) );
+
+    if ( !created.UploadId ) {
+        throw new Error ( 'S3 did not return an upload id for multipart upload' );
+    }
+
+    return {
+        uploadId: created.UploadId,
+        partSize: resolvedPartSize,
+        partCount: Math.ceil ( contentLength / resolvedPartSize )
+    };
+};
+
+const signMultipartUploadParts = async ( {
+    client,
+    bucket,
+    key,
+    uploadId,
+    partNumbers,
+    expiresIn = 1800
+} ) => {
+    if ( !Array.isArray ( partNumbers ) || partNumbers.length === 0 ) {
+        throw new Error ( 'At least one multipart upload part number is required' );
+    }
+
+    return Promise.all ( partNumbers.map ( async partNumber => {
+        if ( !Number.isInteger ( partNumber ) || partNumber < 1 || partNumber > MAX_MULTIPART_PARTS ) {
+            throw new Error ( `Invalid multipart upload part number: ${partNumber}` );
+        }
+
+        const signedRequest = await getSignedUrl ( client, new UploadPartCommand ( {
+            Bucket: bucket,
+            Key: key,
+            UploadId: uploadId,
+            PartNumber: partNumber
+        } ), { expiresIn } );
+
+        return { partNumber, signedRequest };
+    } ) );
+};
+
+const listAllParts = async ( { client, bucket, key, uploadId } ) => {
+    const parts = [];
+    let partNumberMarker;
+
+    do {
+        const page = await client.send ( new ListPartsCommand ( {
+            Bucket: bucket,
+            Key: key,
+            UploadId: uploadId,
+            ...( partNumberMarker ? { PartNumberMarker: partNumberMarker } : {} )
+        } ) );
+
+        for ( const part of page.Parts || [] ) {
+            if ( !part.ETag || !part.PartNumber ) {
+                throw new Error ( 'S3 returned an incomplete multipart upload part' );
+            }
+            parts.push ( { ETag: part.ETag, PartNumber: part.PartNumber } );
+        }
+
+        partNumberMarker = page.IsTruncated ? page.NextPartNumberMarker : undefined;
+    } while ( partNumberMarker );
+
+    if ( parts.length === 0 ) {
+        throw new Error ( 'S3 multipart upload contains no completed parts' );
+    }
+
+    parts.sort ( ( left, right ) => left.PartNumber - right.PartNumber );
+    parts.forEach ( ( part, index ) => {
+        if ( part.PartNumber !== index + 1 ) {
+            throw new Error ( `S3 multipart upload is missing part ${index + 1}` );
+        }
+    } );
+
+    return parts;
+};
+
+const completeMultipartUpload = async params => {
+    const parts = await listAllParts ( params );
+    await params.client.send ( new CompleteMultipartUploadCommand ( {
+        Bucket: params.bucket,
+        Key: params.key,
+        UploadId: params.uploadId,
+        MultipartUpload: { Parts: parts }
+    } ) );
+
+    return { parts: parts.length };
+};
+
+const abortMultipartUpload = async ( { client, bucket, key, uploadId } ) => {
+    await client.send ( new AbortMultipartUploadCommand ( {
+        Bucket: bucket,
+        Key: key,
+        UploadId: uploadId
+    } ) );
+};
+
+module.exports = {
+    DEFAULT_PART_BYTES,
+    MAX_MULTIPART_PARTS,
+    abortMultipartUpload,
+    completeMultipartUpload,
+    createMultipartUpload,
+    listAllParts,
+    resolveUploadPartSize,
+    signMultipartUploadParts
+};