eip-cloud-services 1.5.1 → 1.7.0

package/CHANGELOG.md

@@ -6,8 +6,10 @@ All notable changes to this project will be documented in this file.
 ## Unreleased
 
 ### Added
+- Added browser-safe S3 multipart upload initiation, part signing, completion, and abort helpers for files above the single-PUT limit.
 - Added staging-safe OnDemand Content Ingest, Content Discovery, and Azure Blob upload clients with injectable fetch, timeouts, production-host guards, and signed-query redaction.
 - Added bounded S3 object streams and resumable Azure block-blob uploads for large server-to-server media transfers without temporary disk.
+- Added callback-scoped loopback S3 range reads for seekable media inspection and server-side multipart copy for objects above 5 GiB.
 
 ## [1.2.5] - 2026-03-02
 

package/README.md

@@ -4,6 +4,8 @@
 
 The EIP Cloud Services Module is a comprehensive Node.js package that provides seamless integration with various cloud services, including Redis, AWS S3, CDN, AWS Lambda, AWS SQS, and MySQL. This module is designed to simplify the complexities of interacting with these cloud services, offering a range of functionalities from data caching and storage to content delivery and serverless computing.
 
+The S3 service supports multipart browser upload orchestration through `createMultipartUpload`, `signMultipartUploadParts`, `completeMultipartUpload`, and `abortMultipartUpload`. Completion lists the uploaded parts from S3 so clients do not need access to response ETags.
+
 ## Installation and Import
 
 To use this module, first install it in your Node.js project. Then, import the required services as follows:
@@ -429,6 +431,35 @@ await s3.del('myObjectKey');
 await s3.copy('sourceObjectKey', 'destinationObjectKey');
 ```
 
+For objects that may exceed S3's 5 GiB atomic-copy limit, use the large-object
+copy helper. It automatically switches to multipart server-side copy and does
+not download the object through the caller:
+
+```javascript
+await s3.copyLargeObject(
+    'temporary/video.mp4',
+    'permanent/video.mp4',
+    'source-bucket',
+    'destination-bucket',
+    { contentType: 'video/mp4' }
+);
+```
+
+### Seekable Local Access to a Private Large Object
+
+Media tools such as FFmpeg often require byte-range seeking. `withLocalReadUrl`
+provides a callback-scoped loopback URL and translates HTTP range requests into
+authenticated S3 reads. The source is never written to local disk:
+
+```javascript
+const metadata = await s3.withLocalReadUrl('temporary/video.mp4', 'source-bucket', async url => {
+    return inspectVideo(url);
+});
+```
+
+The URL is bound to `127.0.0.1`, contains an unguessable path, and is closed as
+soon as the callback settles.
+
 ### Move an Object
 
 ```javascript

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.5.1",
+  "version": "1.7.0",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {
@@ -18,6 +18,7 @@
     "@aws-sdk/client-cloudfront": "^3.354.0",
     "@aws-sdk/client-lambda": "^3.356.0",
     "@aws-sdk/client-s3": "^3.354.0",
+    "@aws-sdk/s3-request-presigner": "^3.354.0",
     "@aws-sdk/client-sqs": "^3.356.0",
     "@aws-sdk/client-secrets-manager": "^3.354.0",
     "config": "^3.3.9",

package/src/s3.js

@@ -39,6 +39,9 @@
  */
 
 const { S3Client, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, CopyObjectCommand, ListObjectsV2Command } = require ( '@aws-sdk/client-s3' );
+const { copyLargeObject } = require ( './s3LargeObject' );
+const { withLocalReadUrl } = require ( './s3LocalReadServer' );
+const multipartUpload = require ( './s3MultipartUpload' );
 const fs = require ( 'fs' );
 const path = require ( 'path' );
 let config = {};
@@ -220,6 +223,56 @@ exports.getStream = async ( key, bucket = config?.s3?.Bucket ) => {
     };
 };
 
+/**
+ * Expose a private S3 object through a short-lived loopback HTTP URL. Range
+ * requests are served from S3 so local media tools can seek without writing
+ * the full object to disk or requiring public S3 network access.
+ *
+ * @param {string} key - The source object key.
+ * @param {string} [bucket=config?.s3?.Bucket] - The source bucket.
+ * @param {(url:string)=>Promise<*>} callback - Work performed while the URL is available.
+ * @returns {Promise<*>} The callback result.
+ */
+exports.withLocalReadUrl = async ( key, bucket = config?.s3?.Bucket, callback ) => withLocalReadUrl ( {
+    client: S3,
+    key,
+    bucket
+}, callback );
+
+exports.createMultipartUpload = async ( key, bucket = config?.s3?.Bucket, options = {} ) =>
+    multipartUpload.createMultipartUpload ( {
+        client: S3,
+        key,
+        bucket,
+        ...options
+    } );
+
+exports.signMultipartUploadParts = async ( key, uploadId, partNumbers, bucket = config?.s3?.Bucket, options = {} ) =>
+    multipartUpload.signMultipartUploadParts ( {
+        client: S3,
+        key,
+        bucket,
+        uploadId,
+        partNumbers,
+        ...options
+    } );
+
+exports.completeMultipartUpload = async ( key, uploadId, bucket = config?.s3?.Bucket ) =>
+    multipartUpload.completeMultipartUpload ( {
+        client: S3,
+        key,
+        bucket,
+        uploadId
+    } );
+
+exports.abortMultipartUpload = async ( key, uploadId, bucket = config?.s3?.Bucket ) =>
+    multipartUpload.abortMultipartUpload ( {
+        client: S3,
+        key,
+        bucket,
+        uploadId
+    } );
+
 /**
  * Set an object in S3.
  *
@@ -361,6 +414,32 @@ exports.copy = async ( sourceKey, destinationKey, sourceBucket = config?.s3?.Buc
     }
 };
 
+/**
+ * Copy an S3 object without downloading it. Objects above S3's 5 GiB atomic
+ * copy limit are copied with multipart upload-part-copy requests.
+ *
+ * @param {string} sourceKey - The source object key.
+ * @param {string} destinationKey - The destination object key.
+ * @param {string} [sourceBucket=config?.s3?.Bucket] - The source bucket.
+ * @param {string} [destinationBucket=config?.s3?.Bucket] - The destination bucket.
+ * @param {object} [options] - Destination and multipart options.
+ * @returns {Promise<{contentLength:number,multipart:boolean,parts:number}>} Copy summary.
+ */
+exports.copyLargeObject = async (
+    sourceKey,
+    destinationKey,
+    sourceBucket = config?.s3?.Bucket,
+    destinationBucket = config?.s3?.Bucket,
+    options = {}
+) => copyLargeObject ( {
+    client: S3,
+    sourceKey,
+    destinationKey,
+    sourceBucket,
+    destinationBucket,
+    ...options
+} );
+
 /**
  * Move an object within S3 to a different location. (This deletes the original like a CUT / PASTE operation)
  *

package/src/s3LargeObject.js

@@ -0,0 +1,142 @@
+const {
+    AbortMultipartUploadCommand,
+    CompleteMultipartUploadCommand,
+    CopyObjectCommand,
+    CreateMultipartUploadCommand,
+    HeadObjectCommand,
+    UploadPartCopyCommand
+} = require ( '@aws-sdk/client-s3' );
+
+const GIB = 1024 * 1024 * 1024;
+const MIB = 1024 * 1024;
+const SINGLE_COPY_LIMIT_BYTES = 5 * GIB;
+const MAX_COPY_PART_BYTES = 5 * GIB;
+const DEFAULT_COPY_PART_BYTES = 512 * MIB;
+const MAX_MULTIPART_PARTS = 10000;
+
+const encodeCopySource = ( bucket, key ) => encodeURIComponent ( `${bucket}/${key}` ).replace ( /%2F/g, '/' );
+
+const resolvePartSize = ( contentLength, requestedPartSize = DEFAULT_COPY_PART_BYTES ) => {
+    const minimumForPartLimit = Math.ceil ( contentLength / MAX_MULTIPART_PARTS );
+    const selected = Math.max ( Number ( requestedPartSize ) || DEFAULT_COPY_PART_BYTES, minimumForPartLimit );
+    const rounded = Math.ceil ( selected / MIB ) * MIB;
+
+    if ( rounded > MAX_COPY_PART_BYTES ) {
+        throw new Error ( `S3 object is too large to copy in ${MAX_MULTIPART_PARTS} parts` );
+    }
+
+    return rounded;
+};
+
+const buildCopyRanges = ( contentLength, requestedPartSize ) => {
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( 'S3 copy source must have a positive, safe content length' );
+    }
+
+    const partSize = resolvePartSize ( contentLength, requestedPartSize );
+    const ranges = [];
+
+    for ( let start = 0, partNumber = 1; start < contentLength; start += partSize, partNumber += 1 ) {
+        ranges.push ( {
+            partNumber,
+            start,
+            end: Math.min ( start + partSize, contentLength ) - 1
+        } );
+    }
+
+    return ranges;
+};
+
+const copyLargeObject = async ( {
+    client,
+    sourceKey,
+    destinationKey,
+    sourceBucket,
+    destinationBucket,
+    acl = 'public-read',
+    contentType,
+    partSize
+} ) => {
+    const source = encodeCopySource ( sourceBucket, sourceKey );
+    const head = await client.send ( new HeadObjectCommand ( {
+        Bucket: sourceBucket,
+        Key: sourceKey
+    } ) );
+    const contentLength = Number ( head.ContentLength );
+
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( `S3 object ${sourceBucket}/${sourceKey} did not return a valid content length` );
+    }
+
+    if ( contentLength <= SINGLE_COPY_LIMIT_BYTES ) {
+        await client.send ( new CopyObjectCommand ( {
+            CopySource: source,
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            ACL: acl,
+            ContentType: contentType || head.ContentType,
+            MetadataDirective: contentType ? 'REPLACE' : 'COPY',
+            ...( contentType ? { Metadata: head.Metadata || {} } : {} )
+        } ) );
+
+        return { contentLength, multipart: false, parts: 1 };
+    }
+
+    const created = await client.send ( new CreateMultipartUploadCommand ( {
+        Bucket: destinationBucket,
+        Key: destinationKey,
+        ACL: acl,
+        ContentType: contentType || head.ContentType,
+        CacheControl: head.CacheControl,
+        ContentDisposition: head.ContentDisposition,
+        ContentEncoding: head.ContentEncoding,
+        ContentLanguage: head.ContentLanguage,
+        Metadata: head.Metadata || {}
+    } ) );
+    const uploadId = created.UploadId;
+
+    if ( !uploadId ) {
+        throw new Error ( 'S3 did not return an upload id for multipart copy' );
+    }
+
+    try {
+        const completedParts = [];
+        for ( const range of buildCopyRanges ( contentLength, partSize ) ) {
+            const copied = await client.send ( new UploadPartCopyCommand ( {
+                Bucket: destinationBucket,
+                Key: destinationKey,
+                CopySource: source,
+                CopySourceRange: `bytes=${range.start}-${range.end}`,
+                PartNumber: range.partNumber,
+                UploadId: uploadId
+            } ) );
+            const etag = copied.CopyPartResult?.ETag;
+            if ( !etag ) throw new Error ( `S3 multipart copy part ${range.partNumber} did not return an ETag` );
+            completedParts.push ( { ETag: etag, PartNumber: range.partNumber } );
+        }
+
+        await client.send ( new CompleteMultipartUploadCommand ( {
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            UploadId: uploadId,
+            MultipartUpload: { Parts: completedParts }
+        } ) );
+
+        return { contentLength, multipart: true, parts: completedParts.length };
+    }
+    catch ( error ) {
+        await client.send ( new AbortMultipartUploadCommand ( {
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            UploadId: uploadId
+        } ) ).catch ( () => {} );
+        throw error;
+    }
+};
+
+module.exports = {
+    DEFAULT_COPY_PART_BYTES,
+    SINGLE_COPY_LIMIT_BYTES,
+    buildCopyRanges,
+    copyLargeObject
+};

package/src/s3LocalReadServer.js

@@ -0,0 +1,111 @@
+const crypto = require ( 'crypto' );
+const http = require ( 'http' );
+const { GetObjectCommand, HeadObjectCommand } = require ( '@aws-sdk/client-s3' );
+
+const parseRange = ( value, contentLength ) => {
+    const match = /^bytes=(\d*)-(\d*)$/i.exec ( value || '' );
+    if ( !match ) return null;
+
+    let start;
+    let end;
+    if ( match[ 1 ] ) {
+        start = Number ( match[ 1 ] );
+        end = match[ 2 ] ? Number ( match[ 2 ] ) : contentLength - 1;
+    }
+    else if ( match[ 2 ] ) {
+        const suffixLength = Number ( match[ 2 ] );
+        start = Math.max ( contentLength - suffixLength, 0 );
+        end = contentLength - 1;
+    }
+
+    if (
+        !Number.isSafeInteger ( start )
+        || !Number.isSafeInteger ( end )
+        || start < 0
+        || end < start
+        || start >= contentLength
+    ) return null;
+
+    return { start, end: Math.min ( end, contentLength - 1 ) };
+};
+
+const listen = server => new Promise ( ( resolve, reject ) => {
+    server.once ( 'error', reject );
+    server.listen ( 0, '127.0.0.1', () => {
+        server.off ( 'error', reject );
+        resolve ();
+    } );
+} );
+
+const close = server => new Promise ( resolve => server.close ( resolve ) );
+
+const withLocalReadUrl = async ( { client, key, bucket }, callback ) => {
+    const head = await client.send ( new HeadObjectCommand ( { Bucket: bucket, Key: key } ) );
+    const contentLength = Number ( head.ContentLength );
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( `S3 object ${bucket}/${key} did not return a valid content length` );
+    }
+
+    const pathname = `/${crypto.randomUUID ()}`;
+    const server = http.createServer ( async ( request, response ) => {
+        if ( request.url !== pathname || ![ 'GET', 'HEAD' ].includes ( request.method ) ) {
+            response.writeHead ( 404 ).end ();
+            return;
+        }
+
+        const range = request.headers.range ? parseRange ( request.headers.range, contentLength ) : null;
+        if ( request.headers.range && !range ) {
+            response.writeHead ( 416, { 'Content-Range': `bytes */${contentLength}` } ).end ();
+            return;
+        }
+
+        const responseLength = range ? range.end - range.start + 1 : contentLength;
+        const headers = {
+            'Accept-Ranges': 'bytes',
+            'Content-Length': responseLength,
+            'Content-Type': head.ContentType || 'application/octet-stream',
+            ...( range ? { 'Content-Range': `bytes ${range.start}-${range.end}/${contentLength}` } : {} )
+        };
+
+        if ( request.method === 'HEAD' ) {
+            response.writeHead ( range ? 206 : 200, headers ).end ();
+            return;
+        }
+
+        try {
+            const object = await client.send ( new GetObjectCommand ( {
+                Bucket: bucket,
+                Key: key,
+                ...( range ? { Range: `bytes=${range.start}-${range.end}` } : {} )
+            } ) );
+            if ( !object.Body || typeof object.Body.pipe !== 'function' ) {
+                throw new Error ( 'S3 did not return a readable object body' );
+            }
+
+            response.writeHead ( range ? 206 : 200, headers );
+            response.once ( 'close', () => object.Body.destroy?. () );
+            object.Body.once ( 'error', () => response.destroy () );
+            object.Body.pipe ( response );
+        }
+        catch {
+            if ( !response.headersSent ) response.writeHead ( 502 ).end ();
+            else response.destroy ();
+        }
+    } );
+
+    await listen ( server );
+    const address = server.address ();
+    const url = `http://127.0.0.1:${address.port}${pathname}`;
+
+    try {
+        return await callback ( url );
+    }
+    finally {
+        await close ( server );
+    }
+};
+
+module.exports = {
+    parseRange,
+    withLocalReadUrl
+};

package/src/s3MultipartUpload.js

@@ -0,0 +1,154 @@
+const {
+    AbortMultipartUploadCommand,
+    CompleteMultipartUploadCommand,
+    CreateMultipartUploadCommand,
+    ListPartsCommand,
+    UploadPartCommand
+} = require ( '@aws-sdk/client-s3' );
+const { getSignedUrl } = require ( '@aws-sdk/s3-request-presigner' );
+
+const MIB = 1024 * 1024;
+const DEFAULT_PART_BYTES = 128 * MIB;
+const MAX_MULTIPART_PARTS = 10000;
+const MAX_PART_BYTES = 5 * 1024 * 1024 * 1024;
+
+const resolveUploadPartSize = ( contentLength, requestedPartSize = DEFAULT_PART_BYTES ) => {
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( 'S3 multipart upload requires a positive, safe content length' );
+    }
+
+    const minimumForPartLimit = Math.ceil ( contentLength / MAX_MULTIPART_PARTS );
+    const selected = Math.max ( Number ( requestedPartSize ) || DEFAULT_PART_BYTES, minimumForPartLimit );
+    const rounded = Math.ceil ( selected / MIB ) * MIB;
+
+    if ( rounded > MAX_PART_BYTES ) {
+        throw new Error ( `S3 object is too large to upload in ${MAX_MULTIPART_PARTS} parts` );
+    }
+
+    return rounded;
+};
+
+const createMultipartUpload = async ( {
+    client,
+    bucket,
+    key,
+    contentLength,
+    contentType,
+    acl = 'public-read',
+    metadata = {},
+    partSize
+} ) => {
+    const resolvedPartSize = resolveUploadPartSize ( contentLength, partSize );
+    const created = await client.send ( new CreateMultipartUploadCommand ( {
+        Bucket: bucket,
+        Key: key,
+        ContentType: contentType,
+        ACL: acl,
+        Metadata: metadata
+    } ) );
+
+    if ( !created.UploadId ) {
+        throw new Error ( 'S3 did not return an upload id for multipart upload' );
+    }
+
+    return {
+        uploadId: created.UploadId,
+        partSize: resolvedPartSize,
+        partCount: Math.ceil ( contentLength / resolvedPartSize )
+    };
+};
+
+const signMultipartUploadParts = async ( {
+    client,
+    bucket,
+    key,
+    uploadId,
+    partNumbers,
+    expiresIn = 1800
+} ) => {
+    if ( !Array.isArray ( partNumbers ) || partNumbers.length === 0 ) {
+        throw new Error ( 'At least one multipart upload part number is required' );
+    }
+
+    return Promise.all ( partNumbers.map ( async partNumber => {
+        if ( !Number.isInteger ( partNumber ) || partNumber < 1 || partNumber > MAX_MULTIPART_PARTS ) {
+            throw new Error ( `Invalid multipart upload part number: ${partNumber}` );
+        }
+
+        const signedRequest = await getSignedUrl ( client, new UploadPartCommand ( {
+            Bucket: bucket,
+            Key: key,
+            UploadId: uploadId,
+            PartNumber: partNumber
+        } ), { expiresIn } );
+
+        return { partNumber, signedRequest };
+    } ) );
+};
+
+const listAllParts = async ( { client, bucket, key, uploadId } ) => {
+    const parts = [];
+    let partNumberMarker;
+
+    do {
+        const page = await client.send ( new ListPartsCommand ( {
+            Bucket: bucket,
+            Key: key,
+            UploadId: uploadId,
+            ...( partNumberMarker ? { PartNumberMarker: partNumberMarker } : {} )
+        } ) );
+
+        for ( const part of page.Parts || [] ) {
+            if ( !part.ETag || !part.PartNumber ) {
+                throw new Error ( 'S3 returned an incomplete multipart upload part' );
+            }
+            parts.push ( { ETag: part.ETag, PartNumber: part.PartNumber } );
+        }
+
+        partNumberMarker = page.IsTruncated ? page.NextPartNumberMarker : undefined;
+    } while ( partNumberMarker );
+
+    if ( parts.length === 0 ) {
+        throw new Error ( 'S3 multipart upload contains no completed parts' );
+    }
+
+    parts.sort ( ( left, right ) => left.PartNumber - right.PartNumber );
+    parts.forEach ( ( part, index ) => {
+        if ( part.PartNumber !== index + 1 ) {
+            throw new Error ( `S3 multipart upload is missing part ${index + 1}` );
+        }
+    } );
+
+    return parts;
+};
+
+const completeMultipartUpload = async params => {
+    const parts = await listAllParts ( params );
+    await params.client.send ( new CompleteMultipartUploadCommand ( {
+        Bucket: params.bucket,
+        Key: params.key,
+        UploadId: params.uploadId,
+        MultipartUpload: { Parts: parts }
+    } ) );
+
+    return { parts: parts.length };
+};
+
+const abortMultipartUpload = async ( { client, bucket, key, uploadId } ) => {
+    await client.send ( new AbortMultipartUploadCommand ( {
+        Bucket: bucket,
+        Key: key,
+        UploadId: uploadId
+    } ) );
+};
+
+module.exports = {
+    DEFAULT_PART_BYTES,
+    MAX_MULTIPART_PARTS,
+    abortMultipartUpload,
+    completeMultipartUpload,
+    createMultipartUpload,
+    listAllParts,
+    resolveUploadPartSize,
+    signMultipartUploadParts
+};