eip-cloud-services 1.5.1 → 1.6.0

package/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 - Added staging-safe OnDemand Content Ingest, Content Discovery, and Azure Blob upload clients with injectable fetch, timeouts, production-host guards, and signed-query redaction.
 - Added bounded S3 object streams and resumable Azure block-blob uploads for large server-to-server media transfers without temporary disk.
+- Added callback-scoped loopback S3 range reads for seekable media inspection and server-side multipart copy for objects above 5 GiB.
 
 ## [1.2.5] - 2026-03-02
 

package/README.md

@@ -429,6 +429,35 @@ await s3.del('myObjectKey');
 await s3.copy('sourceObjectKey', 'destinationObjectKey');
 ```
 
+For objects that may exceed S3's 5 GiB atomic-copy limit, use the large-object
+copy helper. It automatically switches to multipart server-side copy and does
+not download the object through the caller:
+
+```javascript
+await s3.copyLargeObject(
+    'temporary/video.mp4',
+    'permanent/video.mp4',
+    'source-bucket',
+    'destination-bucket',
+    { contentType: 'video/mp4' }
+);
+```
+
+### Seekable Local Access to a Private Large Object
+
+Media tools such as FFmpeg often require byte-range seeking. `withLocalReadUrl`
+provides a callback-scoped loopback URL and translates HTTP range requests into
+authenticated S3 reads. The source is never written to local disk:
+
+```javascript
+const metadata = await s3.withLocalReadUrl('temporary/video.mp4', 'source-bucket', async url => {
+    return inspectVideo(url);
+});
+```
+
+The URL is bound to `127.0.0.1`, contains an unguessable path, and is closed as
+soon as the callback settles.
+
 ### Move an Object
 
 ```javascript

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.5.1",
+  "version": "1.6.0",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {

package/src/s3.js

@@ -39,6 +39,8 @@
  */
 
 const { S3Client, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, CopyObjectCommand, ListObjectsV2Command } = require ( '@aws-sdk/client-s3' );
+const { copyLargeObject } = require ( './s3LargeObject' );
+const { withLocalReadUrl } = require ( './s3LocalReadServer' );
 const fs = require ( 'fs' );
 const path = require ( 'path' );
 let config = {};
@@ -220,6 +222,22 @@ exports.getStream = async ( key, bucket = config?.s3?.Bucket ) => {
     };
 };
 
+/**
+ * Expose a private S3 object through a short-lived loopback HTTP URL. Range
+ * requests are served from S3 so local media tools can seek without writing
+ * the full object to disk or requiring public S3 network access.
+ *
+ * @param {string} key - The source object key.
+ * @param {string} [bucket=config?.s3?.Bucket] - The source bucket.
+ * @param {(url:string)=>Promise<*>} callback - Work performed while the URL is available.
+ * @returns {Promise<*>} The callback result.
+ */
+exports.withLocalReadUrl = async ( key, bucket = config?.s3?.Bucket, callback ) => withLocalReadUrl ( {
+    client: S3,
+    key,
+    bucket
+}, callback );
+
 /**
  * Set an object in S3.
  *
@@ -361,6 +379,32 @@ exports.copy = async ( sourceKey, destinationKey, sourceBucket = config?.s3?.Buc
     }
 };
 
+/**
+ * Copy an S3 object without downloading it. Objects above S3's 5 GiB atomic
+ * copy limit are copied with multipart upload-part-copy requests.
+ *
+ * @param {string} sourceKey - The source object key.
+ * @param {string} destinationKey - The destination object key.
+ * @param {string} [sourceBucket=config?.s3?.Bucket] - The source bucket.
+ * @param {string} [destinationBucket=config?.s3?.Bucket] - The destination bucket.
+ * @param {object} [options] - Destination and multipart options.
+ * @returns {Promise<{contentLength:number,multipart:boolean,parts:number}>} Copy summary.
+ */
+exports.copyLargeObject = async (
+    sourceKey,
+    destinationKey,
+    sourceBucket = config?.s3?.Bucket,
+    destinationBucket = config?.s3?.Bucket,
+    options = {}
+) => copyLargeObject ( {
+    client: S3,
+    sourceKey,
+    destinationKey,
+    sourceBucket,
+    destinationBucket,
+    ...options
+} );
+
 /**
  * Move an object within S3 to a different location. (This deletes the original like a CUT / PASTE operation)
  *

package/src/s3LargeObject.js

@@ -0,0 +1,142 @@
+const {
+    AbortMultipartUploadCommand,
+    CompleteMultipartUploadCommand,
+    CopyObjectCommand,
+    CreateMultipartUploadCommand,
+    HeadObjectCommand,
+    UploadPartCopyCommand
+} = require ( '@aws-sdk/client-s3' );
+
+const GIB = 1024 * 1024 * 1024;
+const MIB = 1024 * 1024;
+const SINGLE_COPY_LIMIT_BYTES = 5 * GIB;
+const MAX_COPY_PART_BYTES = 5 * GIB;
+const DEFAULT_COPY_PART_BYTES = 512 * MIB;
+const MAX_MULTIPART_PARTS = 10000;
+
+const encodeCopySource = ( bucket, key ) => encodeURIComponent ( `${bucket}/${key}` ).replace ( /%2F/g, '/' );
+
+const resolvePartSize = ( contentLength, requestedPartSize = DEFAULT_COPY_PART_BYTES ) => {
+    const minimumForPartLimit = Math.ceil ( contentLength / MAX_MULTIPART_PARTS );
+    const selected = Math.max ( Number ( requestedPartSize ) || DEFAULT_COPY_PART_BYTES, minimumForPartLimit );
+    const rounded = Math.ceil ( selected / MIB ) * MIB;
+
+    if ( rounded > MAX_COPY_PART_BYTES ) {
+        throw new Error ( `S3 object is too large to copy in ${MAX_MULTIPART_PARTS} parts` );
+    }
+
+    return rounded;
+};
+
+const buildCopyRanges = ( contentLength, requestedPartSize ) => {
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( 'S3 copy source must have a positive, safe content length' );
+    }
+
+    const partSize = resolvePartSize ( contentLength, requestedPartSize );
+    const ranges = [];
+
+    for ( let start = 0, partNumber = 1; start < contentLength; start += partSize, partNumber += 1 ) {
+        ranges.push ( {
+            partNumber,
+            start,
+            end: Math.min ( start + partSize, contentLength ) - 1
+        } );
+    }
+
+    return ranges;
+};
+
+const copyLargeObject = async ( {
+    client,
+    sourceKey,
+    destinationKey,
+    sourceBucket,
+    destinationBucket,
+    acl = 'public-read',
+    contentType,
+    partSize
+} ) => {
+    const source = encodeCopySource ( sourceBucket, sourceKey );
+    const head = await client.send ( new HeadObjectCommand ( {
+        Bucket: sourceBucket,
+        Key: sourceKey
+    } ) );
+    const contentLength = Number ( head.ContentLength );
+
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( `S3 object ${sourceBucket}/${sourceKey} did not return a valid content length` );
+    }
+
+    if ( contentLength <= SINGLE_COPY_LIMIT_BYTES ) {
+        await client.send ( new CopyObjectCommand ( {
+            CopySource: source,
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            ACL: acl,
+            ContentType: contentType || head.ContentType,
+            MetadataDirective: contentType ? 'REPLACE' : 'COPY',
+            ...( contentType ? { Metadata: head.Metadata || {} } : {} )
+        } ) );
+
+        return { contentLength, multipart: false, parts: 1 };
+    }
+
+    const created = await client.send ( new CreateMultipartUploadCommand ( {
+        Bucket: destinationBucket,
+        Key: destinationKey,
+        ACL: acl,
+        ContentType: contentType || head.ContentType,
+        CacheControl: head.CacheControl,
+        ContentDisposition: head.ContentDisposition,
+        ContentEncoding: head.ContentEncoding,
+        ContentLanguage: head.ContentLanguage,
+        Metadata: head.Metadata || {}
+    } ) );
+    const uploadId = created.UploadId;
+
+    if ( !uploadId ) {
+        throw new Error ( 'S3 did not return an upload id for multipart copy' );
+    }
+
+    try {
+        const completedParts = [];
+        for ( const range of buildCopyRanges ( contentLength, partSize ) ) {
+            const copied = await client.send ( new UploadPartCopyCommand ( {
+                Bucket: destinationBucket,
+                Key: destinationKey,
+                CopySource: source,
+                CopySourceRange: `bytes=${range.start}-${range.end}`,
+                PartNumber: range.partNumber,
+                UploadId: uploadId
+            } ) );
+            const etag = copied.CopyPartResult?.ETag;
+            if ( !etag ) throw new Error ( `S3 multipart copy part ${range.partNumber} did not return an ETag` );
+            completedParts.push ( { ETag: etag, PartNumber: range.partNumber } );
+        }
+
+        await client.send ( new CompleteMultipartUploadCommand ( {
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            UploadId: uploadId,
+            MultipartUpload: { Parts: completedParts }
+        } ) );
+
+        return { contentLength, multipart: true, parts: completedParts.length };
+    }
+    catch ( error ) {
+        await client.send ( new AbortMultipartUploadCommand ( {
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            UploadId: uploadId
+        } ) ).catch ( () => {} );
+        throw error;
+    }
+};
+
+module.exports = {
+    DEFAULT_COPY_PART_BYTES,
+    SINGLE_COPY_LIMIT_BYTES,
+    buildCopyRanges,
+    copyLargeObject
+};

package/src/s3LocalReadServer.js

@@ -0,0 +1,111 @@
+const crypto = require ( 'crypto' );
+const http = require ( 'http' );
+const { GetObjectCommand, HeadObjectCommand } = require ( '@aws-sdk/client-s3' );
+
+const parseRange = ( value, contentLength ) => {
+    const match = /^bytes=(\d*)-(\d*)$/i.exec ( value || '' );
+    if ( !match ) return null;
+
+    let start;
+    let end;
+    if ( match[ 1 ] ) {
+        start = Number ( match[ 1 ] );
+        end = match[ 2 ] ? Number ( match[ 2 ] ) : contentLength - 1;
+    }
+    else if ( match[ 2 ] ) {
+        const suffixLength = Number ( match[ 2 ] );
+        start = Math.max ( contentLength - suffixLength, 0 );
+        end = contentLength - 1;
+    }
+
+    if (
+        !Number.isSafeInteger ( start )
+        || !Number.isSafeInteger ( end )
+        || start < 0
+        || end < start
+        || start >= contentLength
+    ) return null;
+
+    return { start, end: Math.min ( end, contentLength - 1 ) };
+};
+
+const listen = server => new Promise ( ( resolve, reject ) => {
+    server.once ( 'error', reject );
+    server.listen ( 0, '127.0.0.1', () => {
+        server.off ( 'error', reject );
+        resolve ();
+    } );
+} );
+
+const close = server => new Promise ( resolve => server.close ( resolve ) );
+
+const withLocalReadUrl = async ( { client, key, bucket }, callback ) => {
+    const head = await client.send ( new HeadObjectCommand ( { Bucket: bucket, Key: key } ) );
+    const contentLength = Number ( head.ContentLength );
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( `S3 object ${bucket}/${key} did not return a valid content length` );
+    }
+
+    const pathname = `/${crypto.randomUUID ()}`;
+    const server = http.createServer ( async ( request, response ) => {
+        if ( request.url !== pathname || ![ 'GET', 'HEAD' ].includes ( request.method ) ) {
+            response.writeHead ( 404 ).end ();
+            return;
+        }
+
+        const range = request.headers.range ? parseRange ( request.headers.range, contentLength ) : null;
+        if ( request.headers.range && !range ) {
+            response.writeHead ( 416, { 'Content-Range': `bytes */${contentLength}` } ).end ();
+            return;
+        }
+
+        const responseLength = range ? range.end - range.start + 1 : contentLength;
+        const headers = {
+            'Accept-Ranges': 'bytes',
+            'Content-Length': responseLength,
+            'Content-Type': head.ContentType || 'application/octet-stream',
+            ...( range ? { 'Content-Range': `bytes ${range.start}-${range.end}/${contentLength}` } : {} )
+        };
+
+        if ( request.method === 'HEAD' ) {
+            response.writeHead ( range ? 206 : 200, headers ).end ();
+            return;
+        }
+
+        try {
+            const object = await client.send ( new GetObjectCommand ( {
+                Bucket: bucket,
+                Key: key,
+                ...( range ? { Range: `bytes=${range.start}-${range.end}` } : {} )
+            } ) );
+            if ( !object.Body || typeof object.Body.pipe !== 'function' ) {
+                throw new Error ( 'S3 did not return a readable object body' );
+            }
+
+            response.writeHead ( range ? 206 : 200, headers );
+            response.once ( 'close', () => object.Body.destroy?. () );
+            object.Body.once ( 'error', () => response.destroy () );
+            object.Body.pipe ( response );
+        }
+        catch {
+            if ( !response.headersSent ) response.writeHead ( 502 ).end ();
+            else response.destroy ();
+        }
+    } );
+
+    await listen ( server );
+    const address = server.address ();
+    const url = `http://127.0.0.1:${address.port}${pathname}`;
+
+    try {
+        return await callback ( url );
+    }
+    finally {
+        await close ( server );
+    }
+};
+
+module.exports = {
+    parseRange,
+    withLocalReadUrl
+};