eip-cloud-services 1.5.0 → 1.5.1

package/CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 - Added staging-safe OnDemand Content Ingest, Content Discovery, and Azure Blob upload clients with injectable fetch, timeouts, production-host guards, and signed-query redaction.
+- Added bounded S3 object streams and resumable Azure block-blob uploads for large server-to-server media transfers without temporary disk.
 
 ## [1.2.5] - 2026-03-02
 

package/README.md

@@ -128,6 +128,20 @@ await ondemand.uploadAzureBlob ( {
 await ingest.startIngest ( assetId, { url: cleanBlobUrl } );
 ```
 
+For large one-shot sources such as an S3 `GetObject` body, use the block uploader. It buffers only one block at a time, retries replayable blocks independently, and commits the blob only after the exact expected byte length has been received:
+
+```javascript
+const source = await s3.getStream ( objectKey, bucket );
+
+await ondemand.uploadAzureBlockBlob ( {
+    blobUrl: cleanBlobUrl,
+    sasToken: process.env.ONDEMAND_AZURE_SAS_TOKEN,
+    body: source.body,
+    contentLength: source.contentLength,
+    contentType: 'video/mp4'
+} );
+```
+
 # AWS Secrets Manager Module
 
 ## Overview

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {

package/src/ondemand.js

@@ -284,6 +284,46 @@ const buildAzureUploadUrl = ( blobUrl, sasToken ) => {
     return url.toString ();
 };
 
+const buildAzureBlockUrl = ( blobUrl, sasToken, parameters ) => {
+    const url = new URL ( buildAzureUploadUrl ( blobUrl, sasToken ) );
+    Object.entries ( parameters ).forEach ( ( [ key, value ] ) => url.searchParams.set ( key, value ) );
+    return url.toString ();
+};
+
+const azureBlockId = index => Buffer.from ( String ( index ).padStart ( 12, '0' ) ).toString ( 'base64' );
+
+const fixedSizeBlocks = async function * ( body, blockSize ) {
+    let pending = Buffer.alloc ( 0 );
+
+    for await ( const sourceChunk of body ) {
+        const chunk = Buffer.isBuffer ( sourceChunk ) ? sourceChunk : Buffer.from ( sourceChunk );
+        pending = pending.length ? Buffer.concat ( [ pending, chunk ] ) : chunk;
+
+        while ( pending.length >= blockSize ) {
+            yield pending.subarray ( 0, blockSize );
+            pending = pending.subarray ( blockSize );
+        }
+    }
+
+    if ( pending.length ) yield pending;
+};
+
+const executeReplayableRequest = async ( request, { maxRetries, retryDelayMs } ) => {
+    let attempt = 0;
+
+    while ( true ) {
+        try {
+            return await executeRequest ( request );
+        }
+        catch ( error ) {
+            const retryableStatus = error?.status === 408 || error?.status === 429 || Number ( error?.status ) >= 500;
+            if ( attempt >= maxRetries || ( error?.status && !retryableStatus ) ) throw error;
+            attempt += 1;
+            await new Promise ( resolve => setTimeout ( resolve, retryDelayMs * attempt ) );
+        }
+    }
+};
+
 /**
  * Stream a source video to an Azure block blob using a SAS token. The returned
  * result never includes the signed request URL.
@@ -336,4 +376,102 @@ exports.uploadAzureBlob = async ( {
     };
 };
 
+/**
+ * Stream a source video into fixed-size Azure blocks and commit the block list.
+ * Only one block is buffered at a time, allowing callers to pipe an S3 GetObject
+ * stream without using local disk. Deterministic block ids make whole-operation
+ * retries safe because an uploaded block is overwritten before the list is
+ * committed.
+ *
+ * @param {object} options
+ * @param {string} options.blobUrl Clean Azure blob URL without a query string.
+ * @param {string} options.sasToken Azure SAS query string.
+ * @param {AsyncIterable<Buffer|Uint8Array|string>} options.body One-shot source stream.
+ * @param {number} options.contentLength Exact expected byte length.
+ * @param {string} [options.contentType='video/mp4'] Final blob MIME type.
+ * @param {number} [options.blockSize=8388608] Bytes buffered per Azure block.
+ * @param {number} [options.maxRetries=2] Per-block retry count.
+ * @param {number} [options.retryDelayMs=250] Linear retry delay base.
+ * @param {Function} [options.fetch] Injectable fetch implementation.
+ * @param {number} [options.timeoutMs=300000] Per-request timeout.
+ * @returns {Promise<{status: number, etag: string|null, blocks: number, bytes: number}>}
+ */
+exports.uploadAzureBlockBlob = async ( {
+    blobUrl,
+    sasToken,
+    body,
+    contentLength,
+    contentType = 'video/mp4',
+    blockSize = 8 * 1024 * 1024,
+    maxRetries = 2,
+    retryDelayMs = 250,
+    fetch: fetchOption,
+    timeoutMs = 300000
+} ) => {
+    if ( !body || typeof body[ Symbol.asyncIterator ] !== 'function' ) {
+        throw new Error ( 'OnDemand Azure block upload body must be an async iterable' );
+    }
+
+    const expectedBytes = Number ( contentLength );
+    if ( !Number.isFinite ( expectedBytes ) || expectedBytes < 0 ) {
+        throw new Error ( 'OnDemand Azure upload content length is required' );
+    }
+    if ( !Number.isInteger ( blockSize ) || blockSize < 1 || blockSize > 100 * 1024 * 1024 ) {
+        throw new Error ( 'OnDemand Azure block size must be between 1 byte and 100 MiB' );
+    }
+
+    const fetchImpl = resolveFetch ( fetchOption );
+    const blockIds = [];
+    let uploadedBytes = 0;
+
+    for await ( const block of fixedSizeBlocks ( body, blockSize ) ) {
+        const blockId = azureBlockId ( blockIds.length );
+        await executeReplayableRequest ( {
+            fetchImpl,
+            url: buildAzureBlockUrl ( blobUrl, sasToken, { comp: 'block', blockid: blockId } ),
+            timeoutMs,
+            options: {
+                method: 'PUT',
+                headers: {
+                    'Content-Type': 'application/octet-stream',
+                    'Content-Length': String ( block.length ),
+                    'x-ms-version': '2018-11-09'
+                },
+                body: block
+            }
+        }, { maxRetries, retryDelayMs } );
+        blockIds.push ( blockId );
+        uploadedBytes += block.length;
+    }
+
+    if ( uploadedBytes !== expectedBytes ) {
+        throw new Error ( `OnDemand Azure upload length mismatch: expected ${expectedBytes}, received ${uploadedBytes}` );
+    }
+
+    const blockList = `<?xml version="1.0" encoding="utf-8"?><BlockList>${blockIds.map ( id => `<Latest>${id}</Latest>` ).join ( '' )}</BlockList>`;
+    const { response } = await executeReplayableRequest ( {
+        fetchImpl,
+        url: buildAzureBlockUrl ( blobUrl, sasToken, { comp: 'blocklist' } ),
+        timeoutMs,
+        includeResponse: true,
+        options: {
+            method: 'PUT',
+            headers: {
+                'Content-Type': 'application/xml',
+                'Content-Length': String ( Buffer.byteLength ( blockList ) ),
+                'x-ms-blob-content-type': contentType,
+                'x-ms-version': '2018-11-09'
+            },
+            body: blockList
+        }
+    }, { maxRetries, retryDelayMs } );
+
+    return {
+        status: response.status,
+        etag: response.headers?.get?.( 'etag' ) || null,
+        blocks: blockIds.length,
+        bytes: uploadedBytes
+    };
+};
+
 exports.OnDemandRequestError = OnDemandRequestError;

package/src/s3.js

@@ -183,6 +183,43 @@ exports.get = async ( key, bucket = config?.s3?.Bucket, options = {} ) => {
     }
 };
 
+/**
+ * Get an S3 object as a one-shot readable stream without buffering it or writing
+ * it to local disk.
+ *
+ * @param {string} key - The object key.
+ * @param {string} [bucket=config?.s3?.Bucket] - The bucket name.
+ * @returns {Promise<{body: import('stream').Readable, contentLength: number, contentType: string|null, etag: string|null}>}
+ * A stream and the metadata required by downstream streaming uploads.
+ */
+exports.getStream = async ( key, bucket = config?.s3?.Bucket ) => {
+    const response = await S3.send ( new GetObjectCommand ( {
+        Bucket: bucket,
+        Key: key
+    } ) );
+
+    if ( !response.Body || typeof response.Body[ Symbol.asyncIterator ] !== 'function' ) {
+        throw new Error ( `S3 object ${bucket}/${key} did not return a readable stream` );
+    }
+
+    const contentLength = Number ( response.ContentLength );
+    if ( !Number.isFinite ( contentLength ) || contentLength < 0 ) {
+        response.Body.destroy?.();
+        throw new Error ( `S3 object ${bucket}/${key} did not return a valid content length` );
+    }
+
+    if ( config?.s3?.logs === 'verbose' ) {
+        log ( `S3 [GET STREAM]: Streaming ${bucket}/${key}.` );
+    }
+
+    return {
+        body: response.Body,
+        contentLength,
+        contentType: response.ContentType || null,
+        etag: response.ETag || null
+    };
+};
+
 /**
  * Set an object in S3.
  *