@karpeleslab/klbfw 0.2.17 → 0.2.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@karpeleslab/klbfw",
-  "version": "0.2.17",
+  "version": "0.2.18",
   "description": "Frontend Framework",
   "main": "index.js",
   "types": "index.d.ts",

package/upload.js

@@ -39,23 +39,22 @@
  * ```js
  * // For Node.js environments, first install dependencies:
  * // npm install node-fetch @xmldom/xmldom
- * 
- * // Create a buffer-based file object for upload
- * const file = {
- *   name: 'test.txt',
- *   size: buffer.length,
- *   type: 'text/plain',
- *   content: buffer, // Buffer or ArrayBuffer with file content
- *   lastModified: Date.now(),
- *   slice: function(start, end) {
- *     return {
- *       content: this.content.slice(start, end)
- *     };
- *   }
- * };
- * 
- * upload.append('Misc/Debug:testUpload', file)
- *   .then(result => console.log('Upload complete', result));
+ *
+ * // Simple upload with a buffer
+ * const { uploadFile } = require('./upload');
+ * const buffer = Buffer.from('Hello, World!');
+ * const result = await uploadFile('Misc/Debug:testUpload', buffer, 'POST', {
+ *   filename: 'hello.txt',
+ *   type: 'text/plain'
+ * });
+ *
+ * // Upload large files using a stream (doesn't load entire file into memory)
+ * const fs = require('fs');
+ * const stream = fs.createReadStream('/path/to/2tb-file.bin');
+ * const result = await uploadFile('Misc/Debug:testUpload', stream, 'POST', {
+ *   filename: 'large-file.bin',
+ *   type: 'application/octet-stream'
+ * });
  * ```
  * 
  * @module upload
@@ -280,11 +279,17 @@ const utils = {
  *   - A Uint8Array or other TypedArray
  *   - A browser File object
  *   - A file-like object with { name, size, type, content, lastModified }
+ *   - A file-like object with { name, size, type, stream } for streaming large files
  *   - A string (will be converted to UTF-8 bytes)
  * @param {string} [method='POST'] - HTTP method for the initial API call
  * @param {Object} [params={}] - Additional parameters to send with the upload.
  *   Can include `filename` and `type` to override defaults.
  * @param {Object} [context=null] - Request context (uses default context if not provided)
+ * @param {Object} [options={}] - Upload options
+ * @param {Function} [options.onProgress] - Progress callback(progress) where progress is 0-1
+ * @param {Function} [options.onError] - Error callback(error, context). Can return a Promise
+ *   that, if resolved, will cause the failed operation to be retried. Context contains
+ *   { phase, blockNum, attempt } for block uploads or { phase, attempt } for other operations.
  * @returns {Promise<Object>} - Resolves with the upload result data
  *
  * @example
@@ -296,17 +301,40 @@ const utils = {
  * });
  *
  * @example
- * // Upload with defaults
- * const result = await uploadFile('Misc/Debug:testUpload', buffer);
+ * // Upload with progress and error handling
+ * const result = await uploadFile('Misc/Debug:testUpload', buffer, 'POST', {
+ *   filename: 'large-file.bin'
+ * }, null, {
+ *   onProgress: (progress) => console.log(`${Math.round(progress * 100)}%`),
+ *   onError: async (error, ctx) => {
+ *     console.log(`Error in ${ctx.phase}, attempt ${ctx.attempt}:`, error.message);
+ *     if (ctx.attempt < 3) {
+ *       await new Promise(r => setTimeout(r, 1000)); // Wait 1s before retry
+ *       return; // Resolve to trigger retry
+ *     }
+ *     throw error; // Give up after 3 attempts
+ *   }
+ * });
  *
  * @example
  * // Upload a File object (browser)
  * const result = await uploadFile('Misc/Debug:testUpload', fileInput.files[0]);
+ *
+ * @example
+ * // Upload a large file using a stream (Node.js) - doesn't load entire file into memory
+ * const fs = require('fs');
+ * const stream = fs.createReadStream('/path/to/large-file.bin');
+ * const result = await uploadFile('Misc/Debug:testUpload', stream, 'POST', {
+ *   filename: 'large-file.bin',
+ *   type: 'application/octet-stream',
+ *   size: 2199023255552  // optional: if known, enables optimal block sizing
+ * });
  */
-async function uploadFile(api, buffer, method, params, context) {
+async function uploadFile(api, buffer, method, params, context, options) {
     // Handle default values
     method = method || 'POST';
     params = params || {};
+    options = options || {};
 
     // Get context from framework if not provided, and add available values
     if (!context) {
@@ -384,8 +412,18 @@ async function uploadFile(api, buffer, method, params, context) {
             content: buffer.content
         };
     }
+    // Handle Node.js readable stream
+    else if (buffer && typeof buffer.read === 'function' && typeof buffer.on === 'function') {
+        fileObj = {
+            name: params.filename || 'file.bin',
+            size: params.size || null,  // null means unknown size
+            type: params.type || 'application/octet-stream',
+            lastModified: Date.now(),
+            stream: buffer
+        };
+    }
     else {
-        throw new Error('Invalid file: must be a Buffer, ArrayBuffer, Uint8Array, File, string, or file-like object with content');
+        throw new Error('Invalid file: must be a Buffer, ArrayBuffer, Uint8Array, File, readable stream, or file-like object with content');
     }
 
     // Merge params with file metadata (file metadata takes precedence for these fields)
@@ -401,12 +439,12 @@ async function uploadFile(api, buffer, method, params, context) {
 
     // Method 1: AWS signed multipart upload
     if (data.Cloud_Aws_Bucket_Upload__) {
-        return doAwsUpload(fileObj, data, context);
+        return doAwsUpload(fileObj, data, context, options);
     }
 
     // Method 2: Direct PUT upload
     if (data.PUT) {
-        return doPutUpload(fileObj, data, context);
+        return doPutUpload(fileObj, data, context, options);
     }
 
     throw new Error('Invalid upload response format: no upload method available');
@@ -416,26 +454,169 @@ async function uploadFile(api, buffer, method, params, context) {
  * Perform a direct PUT upload (simple upload method)
  * @private
  */
-async function doPutUpload(file, uploadInfo, context) {
-    const blockSize = uploadInfo.Blocksize || file.size;
-    const blocks = Math.ceil(file.size / blockSize);
+async function doPutUpload(file, uploadInfo, context, options) {
+    const { onProgress, onError } = options;
+
+    // Calculate block size
+    // - If size known: use server's Blocksize or file size
+    // - If size unknown (streaming): use 526MB default
+    let blockSize;
+    let blocks = null;
+
+    if (file.size) {
+        blockSize = uploadInfo.Blocksize || file.size;
+        blocks = Math.ceil(file.size / blockSize);
+    } else {
+        blockSize = 551550976;  // 526MB
+    }
 
-    // Upload blocks with concurrency limit
     const maxConcurrent = 3;
+    let completedBlocks = 0;
+
+    // Stream-based upload: read sequentially, upload in parallel
+    if (file.stream) {
+        let blockNum = 0;
+        let streamEnded = false;
+        let byteOffset = 0;
+        const pendingUploads = [];
+
+        while (!streamEnded || pendingUploads.length > 0) {
+            // Read and start uploads up to maxConcurrent
+            while (!streamEnded && pendingUploads.length < maxConcurrent) {
+                const chunkData = await readChunkFromStream(file.stream, blockSize);
+                if (chunkData === null) {
+                    streamEnded = true;
+                    break;
+                }
+
+                const currentBlock = blockNum++;
+                const startByte = byteOffset;
+                byteOffset += chunkData.byteLength;
+
+                const uploadPromise = uploadPutBlockWithDataAndRetry(
+                    uploadInfo, currentBlock, startByte, chunkData, file.type, onError
+                ).then(() => {
+                    completedBlocks++;
+                    if (onProgress && blocks) {
+                        onProgress(completedBlocks / blocks);
+                    }
+                });
+
+                pendingUploads.push(uploadPromise);
+            }
+
+            // Wait for at least one upload to complete before reading more
+            if (pendingUploads.length > 0) {
+                await Promise.race(pendingUploads);
+                // Remove completed promises
+                for (let i = pendingUploads.length - 1; i >= 0; i--) {
+                    const status = await Promise.race([
+                        pendingUploads[i].then(() => 'done'),
+                        Promise.resolve('pending')
+                    ]);
+                    if (status === 'done') {
+                        pendingUploads.splice(i, 1);
+                    }
+                }
+            }
+        }
+
+        blocks = blockNum;
+    } else {
+        // Buffer-based upload: original logic
+        for (let i = 0; i < blocks; i += maxConcurrent) {
+            const batch = [];
+            for (let j = i; j < Math.min(i + maxConcurrent, blocks); j++) {
+                batch.push(
+                    uploadPutBlockWithRetry(file, uploadInfo, j, blockSize, onError)
+                        .then(() => {
+                            completedBlocks++;
+                            if (onProgress) {
+                                onProgress(completedBlocks / blocks);
+                            }
+                        })
+                );
+            }
+
+            await Promise.all(batch);
+        }
+    }
 
-    // Process blocks in batches
-    for (let i = 0; i < blocks; i += maxConcurrent) {
-        const batch = [];
-        for (let j = i; j < Math.min(i + maxConcurrent, blocks); j++) {
-            batch.push(uploadPutBlock(file, uploadInfo, j, blockSize));
+    // All blocks done, call completion with retry support
+    let attempt = 0;
+    while (true) {
+        attempt++;
+        try {
+            const completeResponse = await rest.rest(uploadInfo.Complete, 'POST', {}, context);
+            return completeResponse.data;
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'complete', attempt });
+                // If onError resolves, retry
+                continue;
+            }
+            throw error;
         }
+    }
+}
+
+/**
+ * Upload a single block via PUT with pre-read data and retry support
+ * @private
+ */
+async function uploadPutBlockWithDataAndRetry(uploadInfo, blockNum, startByte, data, contentType, onError) {
+    let attempt = 0;
+    while (true) {
+        attempt++;
+        try {
+            const headers = {
+                'Content-Type': contentType || 'application/octet-stream'
+            };
+
+            // Add Content-Range for multipart PUT
+            headers['Content-Range'] = `bytes ${startByte}-${startByte + data.byteLength - 1}/*`;
+
+            const response = await utils.fetch(uploadInfo.PUT, {
+                method: 'PUT',
+                body: data,
+                headers: headers
+            });
+
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
 
-        await Promise.all(batch);
+            await response.text();
+            return;
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'upload', blockNum, attempt });
+                continue;
+            }
+            throw error;
+        }
     }
+}
 
-    // All blocks done, call completion
-    const completeResponse = await rest.rest(uploadInfo.Complete, 'POST', {}, context);
-    return completeResponse.data;
+/**
+ * Upload a single block via PUT with retry support
+ * @private
+ */
+async function uploadPutBlockWithRetry(file, uploadInfo, blockNum, blockSize, onError) {
+    let attempt = 0;
+    while (true) {
+        attempt++;
+        try {
+            return await uploadPutBlock(file, uploadInfo, blockNum, blockSize);
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'upload', blockNum, attempt });
+                // If onError resolves, retry
+                continue;
+            }
+            throw error;
+        }
+    }
 }
 
 /**
@@ -475,61 +656,220 @@ async function uploadPutBlock(file, uploadInfo, blockNum, blockSize) {
  * Perform an AWS multipart upload
  * @private
  */
-async function doAwsUpload(file, uploadInfo, context) {
-    // Calculate optimal block size (min 5MB for AWS, target ~10k parts)
-    let blockSize = Math.ceil(file.size / 10000);
-    if (blockSize < 5242880) blockSize = 5242880;
-
-    const blocks = Math.ceil(file.size / blockSize);
+async function doAwsUpload(file, uploadInfo, context, options) {
+    const { onProgress, onError } = options;
+
+    // Calculate block size
+    // - If size known: target ~10k parts, min 5MB
+    // - If size unknown: use 526MB (allows up to ~5TB with 10k parts)
+    let blockSize;
+    let blocks = null;  // null means unknown (streaming)
+
+    if (file.size) {
+        blockSize = Math.ceil(file.size / 10000);
+        if (blockSize < 5242880) blockSize = 5242880;
+        blocks = Math.ceil(file.size / blockSize);
+    } else {
+        blockSize = 551550976;  // 526MB
+    }
 
-    // Initialize multipart upload
-    const initResponse = await awsReq(
-        uploadInfo,
-        'POST',
-        'uploads=',
-        '',
-        { 'Content-Type': file.type || 'application/octet-stream', 'X-Amz-Acl': 'private' },
-        context
-    );
-    const initXml = await initResponse.text();
-    const dom = utils.parseXML(initXml);
-    const uploadId = dom.querySelector('UploadId').innerHTML;
+    // Initialize multipart upload with retry support
+    let uploadId;
+    let initAttempt = 0;
+    while (true) {
+        initAttempt++;
+        try {
+            const initResponse = await awsReq(
+                uploadInfo,
+                'POST',
+                'uploads=',
+                '',
+                { 'Content-Type': file.type || 'application/octet-stream', 'X-Amz-Acl': 'private' },
+                context
+            );
+            const initXml = await initResponse.text();
+            const dom = utils.parseXML(initXml);
+            uploadId = dom.querySelector('UploadId').innerHTML;
+            break;
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'init', attempt: initAttempt });
+                continue;
+            }
+            throw error;
+        }
+    }
 
-    // Upload all parts with concurrency limit
     const etags = {};
     const maxConcurrent = 3;
+    let completedBlocks = 0;
+
+    // Stream-based upload: read sequentially, upload in parallel
+    if (file.stream) {
+        let blockNum = 0;
+        let streamEnded = false;
+        const pendingUploads = [];
+
+        while (!streamEnded || pendingUploads.length > 0) {
+            // Read and start uploads up to maxConcurrent
+            while (!streamEnded && pendingUploads.length < maxConcurrent) {
+                const chunkData = await readChunkFromStream(file.stream, blockSize);
+                if (chunkData === null) {
+                    streamEnded = true;
+                    break;
+                }
 
-    for (let i = 0; i < blocks; i += maxConcurrent) {
-        const batch = [];
-        for (let j = i; j < Math.min(i + maxConcurrent, blocks); j++) {
-            batch.push(
-                uploadAwsBlock(file, uploadInfo, uploadId, j, blockSize, context)
-                    .then(etag => { etags[j] = etag; })
-            );
+                const currentBlock = blockNum++;
+                const uploadPromise = uploadAwsBlockWithDataAndRetry(
+                    uploadInfo, uploadId, currentBlock, chunkData, context, onError
+                ).then(etag => {
+                    etags[currentBlock] = etag;
+                    completedBlocks++;
+                    if (onProgress && blocks) {
+                        onProgress(completedBlocks / blocks);
+                    }
+                });
+
+                pendingUploads.push(uploadPromise);
+            }
+
+            // Wait for at least one upload to complete before reading more
+            if (pendingUploads.length > 0) {
+                await Promise.race(pendingUploads);
+                // Remove completed promises
+                for (let i = pendingUploads.length - 1; i >= 0; i--) {
+                    const status = await Promise.race([
+                        pendingUploads[i].then(() => 'done'),
+                        Promise.resolve('pending')
+                    ]);
+                    if (status === 'done') {
+                        pendingUploads.splice(i, 1);
+                    }
+                }
+            }
         }
 
-        await Promise.all(batch);
+        blocks = blockNum;  // Now we know the total
+    } else {
+        // Buffer-based upload: original logic
+        for (let i = 0; i < blocks; i += maxConcurrent) {
+            const batch = [];
+            for (let j = i; j < Math.min(i + maxConcurrent, blocks); j++) {
+                batch.push(
+                    uploadAwsBlockWithRetry(file, uploadInfo, uploadId, j, blockSize, context, onError)
+                        .then(etag => {
+                            etags[j] = etag;
+                            completedBlocks++;
+                            if (onProgress) {
+                                onProgress(completedBlocks / blocks);
+                            }
+                        })
+                );
+            }
+
+            await Promise.all(batch);
+        }
     }
 
-    // Complete multipart upload
+    // Complete multipart upload with retry support
     let xml = '<CompleteMultipartUpload>';
     for (let i = 0; i < blocks; i++) {
         xml += `<Part><PartNumber>${i + 1}</PartNumber><ETag>${etags[i]}</ETag></Part>`;
     }
     xml += '</CompleteMultipartUpload>';
 
-    const completeResponse = await awsReq(uploadInfo, 'POST', `uploadId=${uploadId}`, xml, null, context);
-    await completeResponse.text();
+    let completeAttempt = 0;
+    while (true) {
+        completeAttempt++;
+        try {
+            const completeResponse = await awsReq(uploadInfo, 'POST', `uploadId=${uploadId}`, xml, null, context);
+            await completeResponse.text();
+            break;
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'complete', attempt: completeAttempt });
+                continue;
+            }
+            throw error;
+        }
+    }
 
-    // Call server-side completion handler
-    const finalResponse = await rest.rest(
-        `Cloud/Aws/Bucket/Upload/${uploadInfo.Cloud_Aws_Bucket_Upload__}:handleComplete`,
-        'POST',
-        {},
-        context
-    );
+    // Call server-side completion handler with retry support
+    let handleAttempt = 0;
+    while (true) {
+        handleAttempt++;
+        try {
+            const finalResponse = await rest.rest(
+                `Cloud/Aws/Bucket/Upload/${uploadInfo.Cloud_Aws_Bucket_Upload__}:handleComplete`,
+                'POST',
+                {},
+                context
+            );
+            return finalResponse.data;
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'handleComplete', attempt: handleAttempt });
+                continue;
+            }
+            throw error;
+        }
+    }
+}
+
+/**
+ * Upload a block to AWS S3 with pre-read data and retry support
+ * @private
+ */
+async function uploadAwsBlockWithDataAndRetry(uploadInfo, uploadId, blockNum, data, context, onError) {
+    let attempt = 0;
+    while (true) {
+        attempt++;
+        try {
+            const awsPartNumber = blockNum + 1;
+            const response = await awsReq(
+                uploadInfo,
+                'PUT',
+                `partNumber=${awsPartNumber}&uploadId=${uploadId}`,
+                data,
+                null,
+                context
+            );
 
-    return finalResponse.data;
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+
+            const etag = response.headers.get('ETag');
+            await response.text();
+            return etag;
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'upload', blockNum, attempt });
+                continue;
+            }
+            throw error;
+        }
+    }
+}
+
+/**
+ * Upload a single block to AWS S3 with retry support
+ * @private
+ */
+async function uploadAwsBlockWithRetry(file, uploadInfo, uploadId, blockNum, blockSize, context, onError) {
+    let attempt = 0;
+    while (true) {
+        attempt++;
+        try {
+            return await uploadAwsBlock(file, uploadInfo, uploadId, blockNum, blockSize, context);
+        } catch (error) {
+            if (onError) {
+                await onError(error, { phase: 'upload', blockNum, attempt });
+                continue;
+            }
+            throw error;
+        }
+    }
 }
 
 /**
@@ -561,6 +901,82 @@ async function uploadAwsBlock(file, uploadInfo, uploadId, blockNum, blockSize, c
     return etag;
 }
 
+/**
+ * Read a chunk of specified size from a stream
+ * @private
+ * @param {ReadableStream} stream - Node.js readable stream
+ * @param {number} size - Number of bytes to read
+ * @returns {Promise<ArrayBuffer|null>} - ArrayBuffer with data, or null if stream ended
+ */
+function readChunkFromStream(stream, size) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        let bytesRead = 0;
+
+        const onReadable = () => {
+            let chunk;
+            while (bytesRead < size && (chunk = stream.read(Math.min(size - bytesRead, 65536))) !== null) {
+                chunks.push(chunk);
+                bytesRead += chunk.length;
+            }
+
+            if (bytesRead >= size) {
+                cleanup();
+                resolve(combineChunks(chunks));
+            }
+        };
+
+        const onEnd = () => {
+            cleanup();
+            if (bytesRead === 0) {
+                resolve(null);  // Stream ended, no more data
+            } else {
+                resolve(combineChunks(chunks));
+            }
+        };
+
+        const onError = (err) => {
+            cleanup();
+            reject(err);
+        };
+
+        const cleanup = () => {
+            stream.removeListener('readable', onReadable);
+            stream.removeListener('end', onEnd);
+            stream.removeListener('error', onError);
+        };
+
+        stream.on('readable', onReadable);
+        stream.on('end', onEnd);
+        stream.on('error', onError);
+
+        // Try reading immediately in case data is already buffered
+        onReadable();
+    });
+}
+
+/**
+ * Combine chunks into a single ArrayBuffer
+ * @private
+ */
+function combineChunks(chunks) {
+    if (chunks.length === 0) {
+        return new ArrayBuffer(0);
+    }
+    if (chunks.length === 1) {
+        const chunk = chunks[0];
+        return chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.length);
+    }
+    const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.length), offset);
+        offset += chunk.length;
+    }
+    return result.buffer;
+}
+
 /**
  * Read a slice of a file as ArrayBuffer
  * @private
@@ -762,25 +1178,30 @@ module.exports.upload = (function () {
     function handleFailure(up, error) {
         // Skip if upload is no longer running
         if (!(up.up_id in state.running)) return;
-        
+
         // Check if already in failed list
         for (const failedItem of state.failed) {
             if (failedItem.up_id === up.up_id) {
                 return; // Already recorded as failed
             }
         }
-        
+
         // Record failure
         up.failure = error;
         state.failed.push(up);
         delete state.running[up.up_id];
-        
+
+        // Reject the promise so callers know the upload failed
+        if (up.reject) {
+            up.reject(error);
+        }
+
         // Continue processing queue
         upload.run();
-        
+
         // Notify progress
         sendProgress();
-        
+
         // Dispatch failure event
         utils.dispatchEvent("upload:failed", {
             item: up,