eip-cloud-services 1.5.0 → 1.6.0

package/CHANGELOG.md

@@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 - Added staging-safe OnDemand Content Ingest, Content Discovery, and Azure Blob upload clients with injectable fetch, timeouts, production-host guards, and signed-query redaction.
+- Added bounded S3 object streams and resumable Azure block-blob uploads for large server-to-server media transfers without temporary disk.
+- Added callback-scoped loopback S3 range reads for seekable media inspection and server-side multipart copy for objects above 5 GiB.
 
 ## [1.2.5] - 2026-03-02
 

package/README.md

@@ -128,6 +128,20 @@ await ondemand.uploadAzureBlob ( {
 await ingest.startIngest ( assetId, { url: cleanBlobUrl } );
 ```
 
+For large one-shot sources such as an S3 `GetObject` body, use the block uploader. It buffers only one block at a time, retries replayable blocks independently, and commits the blob only after the exact expected byte length has been received:
+
+```javascript
+const source = await s3.getStream ( objectKey, bucket );
+
+await ondemand.uploadAzureBlockBlob ( {
+    blobUrl: cleanBlobUrl,
+    sasToken: process.env.ONDEMAND_AZURE_SAS_TOKEN,
+    body: source.body,
+    contentLength: source.contentLength,
+    contentType: 'video/mp4'
+} );
+```
+
 # AWS Secrets Manager Module
 
 ## Overview
@@ -415,6 +429,35 @@ await s3.del('myObjectKey');
 await s3.copy('sourceObjectKey', 'destinationObjectKey');
 ```
 
+For objects that may exceed S3's 5 GiB atomic-copy limit, use the large-object
+copy helper. It automatically switches to multipart server-side copy and does
+not download the object through the caller:
+
+```javascript
+await s3.copyLargeObject(
+    'temporary/video.mp4',
+    'permanent/video.mp4',
+    'source-bucket',
+    'destination-bucket',
+    { contentType: 'video/mp4' }
+);
+```
+
+### Seekable Local Access to a Private Large Object
+
+Media tools such as FFmpeg often require byte-range seeking. `withLocalReadUrl`
+provides a callback-scoped loopback URL and translates HTTP range requests into
+authenticated S3 reads. The source is never written to local disk:
+
+```javascript
+const metadata = await s3.withLocalReadUrl('temporary/video.mp4', 'source-bucket', async url => {
+    return inspectVideo(url);
+});
+```
+
+The URL is bound to `127.0.0.1`, contains an unguessable path, and is closed as
+soon as the callback settles.
+
 ### Move an Object
 
 ```javascript

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {

package/src/ondemand.js

@@ -284,6 +284,46 @@ const buildAzureUploadUrl = ( blobUrl, sasToken ) => {
     return url.toString ();
 };
 
+const buildAzureBlockUrl = ( blobUrl, sasToken, parameters ) => {
+    const url = new URL ( buildAzureUploadUrl ( blobUrl, sasToken ) );
+    Object.entries ( parameters ).forEach ( ( [ key, value ] ) => url.searchParams.set ( key, value ) );
+    return url.toString ();
+};
+
+const azureBlockId = index => Buffer.from ( String ( index ).padStart ( 12, '0' ) ).toString ( 'base64' );
+
+const fixedSizeBlocks = async function * ( body, blockSize ) {
+    let pending = Buffer.alloc ( 0 );
+
+    for await ( const sourceChunk of body ) {
+        const chunk = Buffer.isBuffer ( sourceChunk ) ? sourceChunk : Buffer.from ( sourceChunk );
+        pending = pending.length ? Buffer.concat ( [ pending, chunk ] ) : chunk;
+
+        while ( pending.length >= blockSize ) {
+            yield pending.subarray ( 0, blockSize );
+            pending = pending.subarray ( blockSize );
+        }
+    }
+
+    if ( pending.length ) yield pending;
+};
+
+const executeReplayableRequest = async ( request, { maxRetries, retryDelayMs } ) => {
+    let attempt = 0;
+
+    while ( true ) {
+        try {
+            return await executeRequest ( request );
+        }
+        catch ( error ) {
+            const retryableStatus = error?.status === 408 || error?.status === 429 || Number ( error?.status ) >= 500;
+            if ( attempt >= maxRetries || ( error?.status && !retryableStatus ) ) throw error;
+            attempt += 1;
+            await new Promise ( resolve => setTimeout ( resolve, retryDelayMs * attempt ) );
+        }
+    }
+};
+
 /**
  * Stream a source video to an Azure block blob using a SAS token. The returned
  * result never includes the signed request URL.
@@ -336,4 +376,102 @@ exports.uploadAzureBlob = async ( {
     };
 };
 
+/**
+ * Stream a source video into fixed-size Azure blocks and commit the block list.
+ * Only one block is buffered at a time, allowing callers to pipe an S3 GetObject
+ * stream without using local disk. Deterministic block ids make whole-operation
+ * retries safe because an uploaded block is overwritten before the list is
+ * committed.
+ *
+ * @param {object} options
+ * @param {string} options.blobUrl Clean Azure blob URL without a query string.
+ * @param {string} options.sasToken Azure SAS query string.
+ * @param {AsyncIterable<Buffer|Uint8Array|string>} options.body One-shot source stream.
+ * @param {number} options.contentLength Exact expected byte length.
+ * @param {string} [options.contentType='video/mp4'] Final blob MIME type.
+ * @param {number} [options.blockSize=8388608] Bytes buffered per Azure block.
+ * @param {number} [options.maxRetries=2] Per-block retry count.
+ * @param {number} [options.retryDelayMs=250] Linear retry delay base.
+ * @param {Function} [options.fetch] Injectable fetch implementation.
+ * @param {number} [options.timeoutMs=300000] Per-request timeout.
+ * @returns {Promise<{status: number, etag: string|null, blocks: number, bytes: number}>}
+ */
+exports.uploadAzureBlockBlob = async ( {
+    blobUrl,
+    sasToken,
+    body,
+    contentLength,
+    contentType = 'video/mp4',
+    blockSize = 8 * 1024 * 1024,
+    maxRetries = 2,
+    retryDelayMs = 250,
+    fetch: fetchOption,
+    timeoutMs = 300000
+} ) => {
+    if ( !body || typeof body[ Symbol.asyncIterator ] !== 'function' ) {
+        throw new Error ( 'OnDemand Azure block upload body must be an async iterable' );
+    }
+
+    const expectedBytes = Number ( contentLength );
+    if ( !Number.isFinite ( expectedBytes ) || expectedBytes < 0 ) {
+        throw new Error ( 'OnDemand Azure upload content length is required' );
+    }
+    if ( !Number.isInteger ( blockSize ) || blockSize < 1 || blockSize > 100 * 1024 * 1024 ) {
+        throw new Error ( 'OnDemand Azure block size must be between 1 byte and 100 MiB' );
+    }
+
+    const fetchImpl = resolveFetch ( fetchOption );
+    const blockIds = [];
+    let uploadedBytes = 0;
+
+    for await ( const block of fixedSizeBlocks ( body, blockSize ) ) {
+        const blockId = azureBlockId ( blockIds.length );
+        await executeReplayableRequest ( {
+            fetchImpl,
+            url: buildAzureBlockUrl ( blobUrl, sasToken, { comp: 'block', blockid: blockId } ),
+            timeoutMs,
+            options: {
+                method: 'PUT',
+                headers: {
+                    'Content-Type': 'application/octet-stream',
+                    'Content-Length': String ( block.length ),
+                    'x-ms-version': '2018-11-09'
+                },
+                body: block
+            }
+        }, { maxRetries, retryDelayMs } );
+        blockIds.push ( blockId );
+        uploadedBytes += block.length;
+    }
+
+    if ( uploadedBytes !== expectedBytes ) {
+        throw new Error ( `OnDemand Azure upload length mismatch: expected ${expectedBytes}, received ${uploadedBytes}` );
+    }
+
+    const blockList = `<?xml version="1.0" encoding="utf-8"?><BlockList>${blockIds.map ( id => `<Latest>${id}</Latest>` ).join ( '' )}</BlockList>`;
+    const { response } = await executeReplayableRequest ( {
+        fetchImpl,
+        url: buildAzureBlockUrl ( blobUrl, sasToken, { comp: 'blocklist' } ),
+        timeoutMs,
+        includeResponse: true,
+        options: {
+            method: 'PUT',
+            headers: {
+                'Content-Type': 'application/xml',
+                'Content-Length': String ( Buffer.byteLength ( blockList ) ),
+                'x-ms-blob-content-type': contentType,
+                'x-ms-version': '2018-11-09'
+            },
+            body: blockList
+        }
+    }, { maxRetries, retryDelayMs } );
+
+    return {
+        status: response.status,
+        etag: response.headers?.get?.( 'etag' ) || null,
+        blocks: blockIds.length,
+        bytes: uploadedBytes
+    };
+};
+
 exports.OnDemandRequestError = OnDemandRequestError;

package/src/s3.js

@@ -39,6 +39,8 @@
  */
 
 const { S3Client, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, CopyObjectCommand, ListObjectsV2Command } = require ( '@aws-sdk/client-s3' );
+const { copyLargeObject } = require ( './s3LargeObject' );
+const { withLocalReadUrl } = require ( './s3LocalReadServer' );
 const fs = require ( 'fs' );
 const path = require ( 'path' );
 let config = {};
@@ -183,6 +185,59 @@ exports.get = async ( key, bucket = config?.s3?.Bucket, options = {} ) => {
     }
 };
 
+/**
+ * Get an S3 object as a one-shot readable stream without buffering it or writing
+ * it to local disk.
+ *
+ * @param {string} key - The object key.
+ * @param {string} [bucket=config?.s3?.Bucket] - The bucket name.
+ * @returns {Promise<{body: import('stream').Readable, contentLength: number, contentType: string|null, etag: string|null}>}
+ * A stream and the metadata required by downstream streaming uploads.
+ */
+exports.getStream = async ( key, bucket = config?.s3?.Bucket ) => {
+    const response = await S3.send ( new GetObjectCommand ( {
+        Bucket: bucket,
+        Key: key
+    } ) );
+
+    if ( !response.Body || typeof response.Body[ Symbol.asyncIterator ] !== 'function' ) {
+        throw new Error ( `S3 object ${bucket}/${key} did not return a readable stream` );
+    }
+
+    const contentLength = Number ( response.ContentLength );
+    if ( !Number.isFinite ( contentLength ) || contentLength < 0 ) {
+        response.Body.destroy?.();
+        throw new Error ( `S3 object ${bucket}/${key} did not return a valid content length` );
+    }
+
+    if ( config?.s3?.logs === 'verbose' ) {
+        log ( `S3 [GET STREAM]: Streaming ${bucket}/${key}.` );
+    }
+
+    return {
+        body: response.Body,
+        contentLength,
+        contentType: response.ContentType || null,
+        etag: response.ETag || null
+    };
+};
+
+/**
+ * Expose a private S3 object through a short-lived loopback HTTP URL. Range
+ * requests are served from S3 so local media tools can seek without writing
+ * the full object to disk or requiring public S3 network access.
+ *
+ * @param {string} key - The source object key.
+ * @param {string} [bucket=config?.s3?.Bucket] - The source bucket.
+ * @param {(url:string)=>Promise<*>} callback - Work performed while the URL is available.
+ * @returns {Promise<*>} The callback result.
+ */
+exports.withLocalReadUrl = async ( key, bucket = config?.s3?.Bucket, callback ) => withLocalReadUrl ( {
+    client: S3,
+    key,
+    bucket
+}, callback );
+
 /**
  * Set an object in S3.
  *
@@ -324,6 +379,32 @@ exports.copy = async ( sourceKey, destinationKey, sourceBucket = config?.s3?.Buc
     }
 };
 
+/**
+ * Copy an S3 object without downloading it. Objects above S3's 5 GiB atomic
+ * copy limit are copied with multipart upload-part-copy requests.
+ *
+ * @param {string} sourceKey - The source object key.
+ * @param {string} destinationKey - The destination object key.
+ * @param {string} [sourceBucket=config?.s3?.Bucket] - The source bucket.
+ * @param {string} [destinationBucket=config?.s3?.Bucket] - The destination bucket.
+ * @param {object} [options] - Destination and multipart options.
+ * @returns {Promise<{contentLength:number,multipart:boolean,parts:number}>} Copy summary.
+ */
+exports.copyLargeObject = async (
+    sourceKey,
+    destinationKey,
+    sourceBucket = config?.s3?.Bucket,
+    destinationBucket = config?.s3?.Bucket,
+    options = {}
+) => copyLargeObject ( {
+    client: S3,
+    sourceKey,
+    destinationKey,
+    sourceBucket,
+    destinationBucket,
+    ...options
+} );
+
 /**
  * Move an object within S3 to a different location. (This deletes the original like a CUT / PASTE operation)
  *

package/src/s3LargeObject.js

@@ -0,0 +1,142 @@
+const {
+    AbortMultipartUploadCommand,
+    CompleteMultipartUploadCommand,
+    CopyObjectCommand,
+    CreateMultipartUploadCommand,
+    HeadObjectCommand,
+    UploadPartCopyCommand
+} = require ( '@aws-sdk/client-s3' );
+
+const GIB = 1024 * 1024 * 1024;
+const MIB = 1024 * 1024;
+const SINGLE_COPY_LIMIT_BYTES = 5 * GIB;
+const MAX_COPY_PART_BYTES = 5 * GIB;
+const DEFAULT_COPY_PART_BYTES = 512 * MIB;
+const MAX_MULTIPART_PARTS = 10000;
+
+const encodeCopySource = ( bucket, key ) => encodeURIComponent ( `${bucket}/${key}` ).replace ( /%2F/g, '/' );
+
+const resolvePartSize = ( contentLength, requestedPartSize = DEFAULT_COPY_PART_BYTES ) => {
+    const minimumForPartLimit = Math.ceil ( contentLength / MAX_MULTIPART_PARTS );
+    const selected = Math.max ( Number ( requestedPartSize ) || DEFAULT_COPY_PART_BYTES, minimumForPartLimit );
+    const rounded = Math.ceil ( selected / MIB ) * MIB;
+
+    if ( rounded > MAX_COPY_PART_BYTES ) {
+        throw new Error ( `S3 object is too large to copy in ${MAX_MULTIPART_PARTS} parts` );
+    }
+
+    return rounded;
+};
+
+const buildCopyRanges = ( contentLength, requestedPartSize ) => {
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( 'S3 copy source must have a positive, safe content length' );
+    }
+
+    const partSize = resolvePartSize ( contentLength, requestedPartSize );
+    const ranges = [];
+
+    for ( let start = 0, partNumber = 1; start < contentLength; start += partSize, partNumber += 1 ) {
+        ranges.push ( {
+            partNumber,
+            start,
+            end: Math.min ( start + partSize, contentLength ) - 1
+        } );
+    }
+
+    return ranges;
+};
+
+const copyLargeObject = async ( {
+    client,
+    sourceKey,
+    destinationKey,
+    sourceBucket,
+    destinationBucket,
+    acl = 'public-read',
+    contentType,
+    partSize
+} ) => {
+    const source = encodeCopySource ( sourceBucket, sourceKey );
+    const head = await client.send ( new HeadObjectCommand ( {
+        Bucket: sourceBucket,
+        Key: sourceKey
+    } ) );
+    const contentLength = Number ( head.ContentLength );
+
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( `S3 object ${sourceBucket}/${sourceKey} did not return a valid content length` );
+    }
+
+    if ( contentLength <= SINGLE_COPY_LIMIT_BYTES ) {
+        await client.send ( new CopyObjectCommand ( {
+            CopySource: source,
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            ACL: acl,
+            ContentType: contentType || head.ContentType,
+            MetadataDirective: contentType ? 'REPLACE' : 'COPY',
+            ...( contentType ? { Metadata: head.Metadata || {} } : {} )
+        } ) );
+
+        return { contentLength, multipart: false, parts: 1 };
+    }
+
+    const created = await client.send ( new CreateMultipartUploadCommand ( {
+        Bucket: destinationBucket,
+        Key: destinationKey,
+        ACL: acl,
+        ContentType: contentType || head.ContentType,
+        CacheControl: head.CacheControl,
+        ContentDisposition: head.ContentDisposition,
+        ContentEncoding: head.ContentEncoding,
+        ContentLanguage: head.ContentLanguage,
+        Metadata: head.Metadata || {}
+    } ) );
+    const uploadId = created.UploadId;
+
+    if ( !uploadId ) {
+        throw new Error ( 'S3 did not return an upload id for multipart copy' );
+    }
+
+    try {
+        const completedParts = [];
+        for ( const range of buildCopyRanges ( contentLength, partSize ) ) {
+            const copied = await client.send ( new UploadPartCopyCommand ( {
+                Bucket: destinationBucket,
+                Key: destinationKey,
+                CopySource: source,
+                CopySourceRange: `bytes=${range.start}-${range.end}`,
+                PartNumber: range.partNumber,
+                UploadId: uploadId
+            } ) );
+            const etag = copied.CopyPartResult?.ETag;
+            if ( !etag ) throw new Error ( `S3 multipart copy part ${range.partNumber} did not return an ETag` );
+            completedParts.push ( { ETag: etag, PartNumber: range.partNumber } );
+        }
+
+        await client.send ( new CompleteMultipartUploadCommand ( {
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            UploadId: uploadId,
+            MultipartUpload: { Parts: completedParts }
+        } ) );
+
+        return { contentLength, multipart: true, parts: completedParts.length };
+    }
+    catch ( error ) {
+        await client.send ( new AbortMultipartUploadCommand ( {
+            Bucket: destinationBucket,
+            Key: destinationKey,
+            UploadId: uploadId
+        } ) ).catch ( () => {} );
+        throw error;
+    }
+};
+
+module.exports = {
+    DEFAULT_COPY_PART_BYTES,
+    SINGLE_COPY_LIMIT_BYTES,
+    buildCopyRanges,
+    copyLargeObject
+};

package/src/s3LocalReadServer.js

@@ -0,0 +1,111 @@
+const crypto = require ( 'crypto' );
+const http = require ( 'http' );
+const { GetObjectCommand, HeadObjectCommand } = require ( '@aws-sdk/client-s3' );
+
+const parseRange = ( value, contentLength ) => {
+    const match = /^bytes=(\d*)-(\d*)$/i.exec ( value || '' );
+    if ( !match ) return null;
+
+    let start;
+    let end;
+    if ( match[ 1 ] ) {
+        start = Number ( match[ 1 ] );
+        end = match[ 2 ] ? Number ( match[ 2 ] ) : contentLength - 1;
+    }
+    else if ( match[ 2 ] ) {
+        const suffixLength = Number ( match[ 2 ] );
+        start = Math.max ( contentLength - suffixLength, 0 );
+        end = contentLength - 1;
+    }
+
+    if (
+        !Number.isSafeInteger ( start )
+        || !Number.isSafeInteger ( end )
+        || start < 0
+        || end < start
+        || start >= contentLength
+    ) return null;
+
+    return { start, end: Math.min ( end, contentLength - 1 ) };
+};
+
+const listen = server => new Promise ( ( resolve, reject ) => {
+    server.once ( 'error', reject );
+    server.listen ( 0, '127.0.0.1', () => {
+        server.off ( 'error', reject );
+        resolve ();
+    } );
+} );
+
+const close = server => new Promise ( resolve => server.close ( resolve ) );
+
+const withLocalReadUrl = async ( { client, key, bucket }, callback ) => {
+    const head = await client.send ( new HeadObjectCommand ( { Bucket: bucket, Key: key } ) );
+    const contentLength = Number ( head.ContentLength );
+    if ( !Number.isSafeInteger ( contentLength ) || contentLength <= 0 ) {
+        throw new Error ( `S3 object ${bucket}/${key} did not return a valid content length` );
+    }
+
+    const pathname = `/${crypto.randomUUID ()}`;
+    const server = http.createServer ( async ( request, response ) => {
+        if ( request.url !== pathname || ![ 'GET', 'HEAD' ].includes ( request.method ) ) {
+            response.writeHead ( 404 ).end ();
+            return;
+        }
+
+        const range = request.headers.range ? parseRange ( request.headers.range, contentLength ) : null;
+        if ( request.headers.range && !range ) {
+            response.writeHead ( 416, { 'Content-Range': `bytes */${contentLength}` } ).end ();
+            return;
+        }
+
+        const responseLength = range ? range.end - range.start + 1 : contentLength;
+        const headers = {
+            'Accept-Ranges': 'bytes',
+            'Content-Length': responseLength,
+            'Content-Type': head.ContentType || 'application/octet-stream',
+            ...( range ? { 'Content-Range': `bytes ${range.start}-${range.end}/${contentLength}` } : {} )
+        };
+
+        if ( request.method === 'HEAD' ) {
+            response.writeHead ( range ? 206 : 200, headers ).end ();
+            return;
+        }
+
+        try {
+            const object = await client.send ( new GetObjectCommand ( {
+                Bucket: bucket,
+                Key: key,
+                ...( range ? { Range: `bytes=${range.start}-${range.end}` } : {} )
+            } ) );
+            if ( !object.Body || typeof object.Body.pipe !== 'function' ) {
+                throw new Error ( 'S3 did not return a readable object body' );
+            }
+
+            response.writeHead ( range ? 206 : 200, headers );
+            response.once ( 'close', () => object.Body.destroy?. () );
+            object.Body.once ( 'error', () => response.destroy () );
+            object.Body.pipe ( response );
+        }
+        catch {
+            if ( !response.headersSent ) response.writeHead ( 502 ).end ();
+            else response.destroy ();
+        }
+    } );
+
+    await listen ( server );
+    const address = server.address ();
+    const url = `http://127.0.0.1:${address.port}${pathname}`;
+
+    try {
+        return await callback ( url );
+    }
+    finally {
+        await close ( server );
+    }
+};
+
+module.exports = {
+    parseRange,
+    withLocalReadUrl
+};