files.com 1.0.413 → 1.0.415

package/_VERSION

@@ -1 +1 @@
-1.0.413
+1.0.415

package/lib/Api.js

@@ -28,11 +28,14 @@ var _fetchWithTimeout = function _fetchWithTimeout(url) {
   var _ref = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {},
     timeoutSecs = _ref.timeoutSecs,
     options = (0, _objectWithoutProperties2.default)(_ref, _excluded);
+  var timeoutId;
   return timeoutSecs <= 0 ? (0, _crossFetch.default)(url, options) : Promise.race([(0, _crossFetch.default)(url, options), new Promise(function (_, reject) {
-    setTimeout(function () {
+    timeoutId = setTimeout(function () {
       return reject(new errors.FilesError('Request timed out'));
     }, timeoutSecs * 1000);
-  })]);
+  })]).finally(function () {
+    return clearTimeout(timeoutId);
+  });
 };
 var fetchWithRetry = /*#__PURE__*/function () {
   var _ref2 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(url, options) {

package/lib/Files.js

@@ -11,7 +11,7 @@ var endpointPrefix = '/api/rest/v1';
 var apiKey;
 var baseUrl = 'https://app.files.com';
 var sessionId = null;
-var version = "1.0.413";
+var version = "1.0.415";
 var userAgent = "Files.com JavaScript SDK v".concat(version);
 var logLevel = _Logger.LogLevel.INFO;
 var debugRequest = false;

package/lib/models/File.js

@@ -936,118 +936,165 @@ _class = File;
             var chunks = [];
             var length = 0;
             var concurrentUploads = [];
+            var chunkBuffer = null;
+            var streamEnded = false;
+            var handleStreamEnd = /*#__PURE__*/function () {
+              var _ref19 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee15() {
+                var _options$getAgentForU, buffer, nextFileUploadPart, upload_uri, agent, response, createdFile;
+                return _regenerator.default.wrap(function _callee15$(_context15) {
+                  while (1) switch (_context15.prev = _context15.next) {
+                    case 0:
+                      if (!(chunkBuffer !== null || !streamEnded)) {
+                        _context15.next = 2;
+                        break;
+                      }
+                      return _context15.abrupt("return");
+                    case 2:
+                      _context15.prev = 2;
+                      if (!(chunks.length > 0)) {
+                        _context15.next = 11;
+                        break;
+                      }
+                      buffer = _safeBuffer.Buffer.concat(chunks);
+                      _context15.next = 7;
+                      return _class._continueUpload(destinationPath, ++part, firstFileUploadPart, options);
+                    case 7:
+                      nextFileUploadPart = _context15.sent;
+                      upload_uri = determinePartUploadUri(nextFileUploadPart); // instantiate an httpsAgent dynamically if needed
+                      agent = ((_options$getAgentForU = options.getAgentForUrl) === null || _options$getAgentForU === void 0 ? void 0 : _options$getAgentForU.call(options, upload_uri)) || (options === null || options === void 0 ? void 0 : options.agent);
+                      concurrentUploads.push(_Api.default.sendFilePart(upload_uri, 'PUT', buffer, {
+                        agent: agent
+                      }));
+                    case 11:
+                      _context15.next = 13;
+                      return Promise.all(concurrentUploads);
+                    case 13:
+                      _context15.next = 15;
+                      return _class._completeUpload(firstFileUploadPart, options);
+                    case 15:
+                      response = _context15.sent;
+                      createdFile = new _class(response.data, options);
+                      resolve(createdFile);
+                      _context15.next = 23;
+                      break;
+                    case 20:
+                      _context15.prev = 20;
+                      _context15.t0 = _context15["catch"](2);
+                      reject(_context15.t0);
+                    case 23:
+                    case "end":
+                      return _context15.stop();
+                  }
+                }, _callee15, null, [[2, 20]]);
+              }));
+              return function handleStreamEnd() {
+                return _ref19.apply(this, arguments);
+              };
+            }();
             readableStream.on('error', function (error) {
               reject(error);
             });
+
+            // note that for a network stream, each chunk is typically less than partsize * 2, but
+            // if a stream has been created based on very large data, it's possible for a chunk to
+            // contain the entire file and we could get a single chunk with length >= partsize * 3
             readableStream.on('data', /*#__PURE__*/function () {
-              var _ref19 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee15(chunk) {
-                var nextLength, excessLength, chunkBuffer, _options$getAgentForU, tailLength, lastChunkForPart, firstChunkForNextPart, buffer, nextFileUploadPart, upload_uri, agent, uploadPromise;
-                return _regenerator.default.wrap(function _callee15$(_context15) {
-                  while (1) switch (_context15.prev = _context15.next) {
+              var _ref20 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee16(chunk) {
+                var excessLength, _options$getAgentForU2, lengthForEndOfCurrentPart, lastChunkForCurrentPart, chunkBufferAfterCurrentPart, buffer, nextFileUploadPart, upload_uri, agent, uploadPromise, isNextChunkAtLeastOnePart;
+                return _regenerator.default.wrap(function _callee16$(_context16) {
+                  while (1) switch (_context16.prev = _context16.next) {
                     case 0:
-                      _context15.prev = 0;
-                      nextLength = length + chunk.length;
-                      excessLength = nextLength - firstFileUploadPart.partsize;
+                      _context16.prev = 0;
+                      excessLength = length + chunk.length - firstFileUploadPart.partsize;
                       chunkBuffer = _safeBuffer.Buffer.from(chunk);
                       if (!(excessLength > 0)) {
-                        _context15.next = 28;
+                        _context16.next = 30;
                         break;
                       }
                       readableStream.pause();
-
+                    case 5:
+                      if (!chunkBuffer) {
+                        _context16.next = 27;
+                        break;
+                      }
                       // the amount to append this last part with to make it exactly the full partsize
-                      tailLength = chunkBuffer.length - excessLength;
-                      lastChunkForPart = chunkBuffer.subarray(0, tailLength);
-                      firstChunkForNextPart = chunkBuffer.subarray(tailLength);
-                      chunks.push(lastChunkForPart);
+                      lengthForEndOfCurrentPart = chunkBuffer.length - excessLength;
+                      lastChunkForCurrentPart = chunkBuffer.subarray(0, lengthForEndOfCurrentPart);
+                      chunkBufferAfterCurrentPart = chunkBuffer.subarray(lengthForEndOfCurrentPart);
+                      chunks.push(lastChunkForCurrentPart);
                       buffer = _safeBuffer.Buffer.concat(chunks);
-                      _context15.next = 13;
+                      _context16.next = 13;
                       return _class._continueUpload(destinationPath, ++part, firstFileUploadPart, options);
                     case 13:
-                      nextFileUploadPart = _context15.sent;
+                      nextFileUploadPart = _context16.sent;
                       upload_uri = determinePartUploadUri(nextFileUploadPart); // instantiate an httpsAgent dynamically if needed
-                      agent = ((_options$getAgentForU = options.getAgentForUrl) === null || _options$getAgentForU === void 0 ? void 0 : _options$getAgentForU.call(options, upload_uri)) || (options === null || options === void 0 ? void 0 : options.agent);
+                      agent = ((_options$getAgentForU2 = options.getAgentForUrl) === null || _options$getAgentForU2 === void 0 ? void 0 : _options$getAgentForU2.call(options, upload_uri)) || (options === null || options === void 0 ? void 0 : options.agent);
                       uploadPromise = _Api.default.sendFilePart(upload_uri, 'PUT', buffer, {
                         agent: agent
                       });
                       if (!firstFileUploadPart.parallel_parts) {
-                        _context15.next = 21;
+                        _context16.next = 21;
                         break;
                       }
                       concurrentUploads.push(uploadPromise);
-                      _context15.next = 23;
+                      _context16.next = 23;
                       break;
                     case 21:
-                      _context15.next = 23;
+                      _context16.next = 23;
                       return uploadPromise;
                     case 23:
-                      chunks = [firstChunkForNextPart];
-                      length = firstChunkForNextPart.length;
+                      // determine if the remainder of the excess chunk data is too large to be a single part
+                      isNextChunkAtLeastOnePart = chunkBufferAfterCurrentPart.length >= firstFileUploadPart.partsize; // the excess data contains >= 1 full part, so we'll loop again to enqueue
+                      // the next part for upload and continue processing any excess beyond that
+                      if (isNextChunkAtLeastOnePart) {
+                        chunks = [];
+                        length = 0;
+                        chunkBuffer = chunkBufferAfterCurrentPart;
+                        excessLength = chunkBuffer.length - firstFileUploadPart.partsize;
+                        // the excess data is less than a full part, so we'll enqueue it
+                      } else if (chunkBufferAfterCurrentPart.length > 0) {
+                        chunks = [chunkBufferAfterCurrentPart];
+                        length = chunkBufferAfterCurrentPart.length;
+                        chunkBuffer = null;
+                      } else {
+                        chunkBuffer = null;
+                      }
+                      _context16.next = 5;
+                      break;
+                    case 27:
                       readableStream.resume();
-                      _context15.next = 30;
+                      _context16.next = 33;
                       break;
-                    case 28:
+                    case 30:
                       chunks.push(chunkBuffer);
                       length += chunk.length;
-                    case 30:
-                      _context15.next = 35;
+                      chunkBuffer = null;
+                    case 33:
+                      if (streamEnded) {
+                        handleStreamEnd();
+                      }
+                      _context16.next = 39;
                       break;
-                    case 32:
-                      _context15.prev = 32;
-                      _context15.t0 = _context15["catch"](0);
-                      reject(_context15.t0);
-                    case 35:
+                    case 36:
+                      _context16.prev = 36;
+                      _context16.t0 = _context16["catch"](0);
+                      reject(_context16.t0);
+                    case 39:
                     case "end":
-                      return _context15.stop();
+                      return _context16.stop();
                   }
-                }, _callee15, null, [[0, 32]]);
+                }, _callee16, null, [[0, 36]]);
               }));
               return function (_x20) {
-                return _ref19.apply(this, arguments);
+                return _ref20.apply(this, arguments);
               };
             }());
-            readableStream.on('end', /*#__PURE__*/(0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee16() {
-              var _options$getAgentForU2, buffer, nextFileUploadPart, upload_uri, agent, response, createdFile;
-              return _regenerator.default.wrap(function _callee16$(_context16) {
-                while (1) switch (_context16.prev = _context16.next) {
-                  case 0:
-                    _context16.prev = 0;
-                    if (!(chunks.length > 0)) {
-                      _context16.next = 9;
-                      break;
-                    }
-                    buffer = _safeBuffer.Buffer.concat(chunks);
-                    _context16.next = 5;
-                    return _class._continueUpload(destinationPath, ++part, firstFileUploadPart, options);
-                  case 5:
-                    nextFileUploadPart = _context16.sent;
-                    upload_uri = determinePartUploadUri(nextFileUploadPart); // instantiate an httpsAgent dynamically if needed
-                    agent = ((_options$getAgentForU2 = options.getAgentForUrl) === null || _options$getAgentForU2 === void 0 ? void 0 : _options$getAgentForU2.call(options, upload_uri)) || (options === null || options === void 0 ? void 0 : options.agent);
-                    concurrentUploads.push(_Api.default.sendFilePart(upload_uri, 'PUT', buffer, {
-                      agent: agent
-                    }));
-                  case 9:
-                    _context16.next = 11;
-                    return Promise.all(concurrentUploads);
-                  case 11:
-                    _context16.next = 13;
-                    return _class._completeUpload(firstFileUploadPart, options);
-                  case 13:
-                    response = _context16.sent;
-                    createdFile = new _class(response.data, options);
-                    resolve(createdFile);
-                    _context16.next = 21;
-                    break;
-                  case 18:
-                    _context16.prev = 18;
-                    _context16.t0 = _context16["catch"](0);
-                    reject(_context16.t0);
-                  case 21:
-                  case "end":
-                    return _context16.stop();
-                }
-              }, _callee16, null, [[0, 18]]);
-            })));
+
+            // note that this event may occur while there is still data being processed above
+            readableStream.on('end', function () {
+              streamEnded = true;
+              handleStreamEnd();
+            });
           });
         case 10:
           file = _context17.sent;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "files.com",
-  "version": "1.0.413",
+  "version": "1.0.415",
   "description": "Files.com SDK for JavaScript",
   "keywords": [
     "files.com",

package/src/Api.js

@@ -5,15 +5,17 @@ import * as errors from './Errors'
 import Logger from './Logger'
 import { isEmpty, isObject } from './utils'
 
-const _fetchWithTimeout = (url, { timeoutSecs, ...options } = {}) =>
-  timeoutSecs <= 0
+const _fetchWithTimeout = (url, { timeoutSecs, ...options } = {}) => {
+  let timeoutId
+  return timeoutSecs <= 0
     ? fetch(url, options)
     : Promise.race([
         fetch(url, options),
         new Promise((_, reject) => {
-          setTimeout(() => reject(new errors.FilesError('Request timed out')), timeoutSecs * 1000)
+          timeoutId = setTimeout(() => reject(new errors.FilesError('Request timed out')), timeoutSecs * 1000)
         })
-      ])
+      ]).finally(() => clearTimeout(timeoutId))
+}
 
 const fetchWithRetry = async (url, options, retries = 0) => {
   const maxRetries = Files.getMaxNetworkRetries()

package/src/Files.js

@@ -5,7 +5,7 @@ const endpointPrefix = '/api/rest/v1'
 let apiKey
 let baseUrl = 'https://app.files.com'
 let sessionId = null
-let version = "1.0.413"
+let version = "1.0.415"
 let userAgent = `Files.com JavaScript SDK v${version}`
 
 let logLevel = LogLevel.INFO

package/src/models/File.js

@@ -97,79 +97,121 @@ class File {
         let length = 0
         const concurrentUploads = []
 
+        let chunkBuffer = null
+        let streamEnded = false
+
+        const handleStreamEnd = async () => {
+          if (chunkBuffer !== null || !streamEnded) {
+            return
+          }
+    
+          try {
+            if (chunks.length > 0) {
+              const buffer = Buffer.concat(chunks)
+              const nextFileUploadPart = await File._continueUpload(destinationPath, ++part, firstFileUploadPart, options)
+    
+              const upload_uri = determinePartUploadUri(nextFileUploadPart)
+    
+              // instantiate an httpsAgent dynamically if needed
+              const agent = options.getAgentForUrl?.(upload_uri) || options?.agent
+    
+              concurrentUploads.push(Api.sendFilePart(upload_uri, 'PUT', buffer, { agent }))
+            }
+    
+            await Promise.all(concurrentUploads)
+    
+            const response = await File._completeUpload(firstFileUploadPart, options)
+            const createdFile = new File(response.data, options)
+    
+            resolve(createdFile)
+          } catch (error) {
+            reject(error)
+          }
+        }
+
         readableStream.on('error', error => { reject(error) })
 
+        // note that for a network stream, each chunk is typically less than partsize * 2, but
+        // if a stream has been created based on very large data, it's possible for a chunk to
+        // contain the entire file and we could get a single chunk with length >= partsize * 3
         readableStream.on('data', async chunk => {
           try {
-            const nextLength = length + chunk.length
-            const excessLength = nextLength - firstFileUploadPart.partsize
+            let excessLength = (length + chunk.length) - firstFileUploadPart.partsize
 
-            const chunkBuffer = Buffer.from(chunk)
+            chunkBuffer = Buffer.from(chunk)
 
             if (excessLength > 0) {
               readableStream.pause()
 
-              // the amount to append this last part with to make it exactly the full partsize
-              const tailLength = chunkBuffer.length - excessLength
+              while (chunkBuffer) {
+                // the amount to append this last part with to make it exactly the full partsize
+                const lengthForEndOfCurrentPart = chunkBuffer.length - excessLength
 
-              const lastChunkForPart = chunkBuffer.subarray(0, tailLength)
-              const firstChunkForNextPart = chunkBuffer.subarray(tailLength)
+                const lastChunkForCurrentPart = chunkBuffer.subarray(0, lengthForEndOfCurrentPart)
+                const chunkBufferAfterCurrentPart = chunkBuffer.subarray(lengthForEndOfCurrentPart)
 
-              chunks.push(lastChunkForPart)
+                chunks.push(lastChunkForCurrentPart)
 
-              const buffer = Buffer.concat(chunks)
-              const nextFileUploadPart = await File._continueUpload(destinationPath, ++part, firstFileUploadPart, options)
+                const buffer = Buffer.concat(chunks)
+                const nextFileUploadPart = await File._continueUpload(destinationPath, ++part, firstFileUploadPart, options)
 
-              const upload_uri = determinePartUploadUri(nextFileUploadPart)
+                const upload_uri = determinePartUploadUri(nextFileUploadPart)
 
-              // instantiate an httpsAgent dynamically if needed
-              const agent = options.getAgentForUrl?.(upload_uri) || options?.agent
+                // instantiate an httpsAgent dynamically if needed
+                const agent = options.getAgentForUrl?.(upload_uri) || options?.agent
 
-              const uploadPromise = Api.sendFilePart(upload_uri, 'PUT', buffer, { agent })
+                const uploadPromise = Api.sendFilePart(upload_uri, 'PUT', buffer, { agent })
 
-              if (firstFileUploadPart.parallel_parts) {
-                concurrentUploads.push(uploadPromise)
-              } else {
-                await uploadPromise
-              }
+                if (firstFileUploadPart.parallel_parts) {
+                  concurrentUploads.push(uploadPromise)
+                } else {
+                  await uploadPromise
+                }
+
+                // determine if the remainder of the excess chunk data is too large to be a single part
+                const isNextChunkAtLeastOnePart = chunkBufferAfterCurrentPart.length >= firstFileUploadPart.partsize
+
+                // the excess data contains >= 1 full part, so we'll loop again to enqueue
+                // the next part for upload and continue processing any excess beyond that
+                if (isNextChunkAtLeastOnePart) {
+                  chunks = []
+                  length = 0
 
-              chunks = [firstChunkForNextPart]
-              length = firstChunkForNextPart.length
+                  chunkBuffer = chunkBufferAfterCurrentPart
+                  excessLength = chunkBuffer.length - firstFileUploadPart.partsize
+                // the excess data is less than a full part, so we'll enqueue it
+                } else if (chunkBufferAfterCurrentPart.length > 0) {
+                  chunks = [chunkBufferAfterCurrentPart]
+                  length = chunkBufferAfterCurrentPart.length
+
+                  chunkBuffer = null
+                } else {
+                  chunkBuffer = null
+                }
+              }
 
               readableStream.resume()
             } else {
               chunks.push(chunkBuffer)
               length += chunk.length
-            }
-          } catch (error) {
-            reject(error)
-          }
-        })
-
-        readableStream.on('end', async () => {
-          try {
-            if (chunks.length > 0) {
-              const buffer = Buffer.concat(chunks)
-              const nextFileUploadPart = await File._continueUpload(destinationPath, ++part, firstFileUploadPart, options)
-
-              const upload_uri = determinePartUploadUri(nextFileUploadPart)
 
-              // instantiate an httpsAgent dynamically if needed
-              const agent = options.getAgentForUrl?.(upload_uri) || options?.agent
-
-              concurrentUploads.push(Api.sendFilePart(upload_uri, 'PUT', buffer, { agent }))
+              chunkBuffer = null
             }
 
-            await Promise.all(concurrentUploads)
-
-            const response = await File._completeUpload(firstFileUploadPart, options)
-            const createdFile = new File(response.data, options)
-
-            resolve(createdFile)
+            if (streamEnded) {
+              handleStreamEnd()
+            }
           } catch (error) {
             reject(error)
           }
         })
+
+        // note that this event may occur while there is still data being processed above
+        readableStream.on('end', () => {
+          streamEnded = true
+  
+          handleStreamEnd()
+        })
       })
 
       return file

package/test/src/index.js

@@ -142,7 +142,7 @@ const testSuite = async () => {
   }
 
   /* to run this test, put a file (or symlink) at huge-file.ext * /
-  const testUploadHugeFile = async () => {
+  const testUploadFileForHugeFile = async () => {
     const sourceFilePath = '../huge-file.ext'
 
     const displayName = `huge-file__${nonce}.ext`
@@ -161,7 +161,33 @@ const testSuite = async () => {
 
     await file.delete()
 
-    Logger.info('***** testUploadHugeFile() succeeded! *****')
+    Logger.info('***** testUploadFileForHugeFile() succeeded! *****')
+  }
+
+  /* to run this test, put a file (or symlink) at huge-file.ext * /
+  const testUploadDataForHugeFile = async () => {
+    const sourceFilePath = '../huge-file.ext'
+
+    const displayName = `huge-file__${nonce}.ext`
+    const destinationPath = `${SDK_TEST_ROOT_FOLDER}/${displayName}`
+
+    const fs = require('fs/promises')
+    const data = await fs.readFile(sourceFilePath, { encoding: "utf8" })
+
+    const file = await File.uploadData(destinationPath, data)
+
+    invariant(!!file.path, 'Uploaded file response object should have a path')
+    invariant(file.display_name === displayName, 'Uploaded file response object should have the same display_name as the file we uploaded')
+
+    const foundFile = await File.find(destinationPath)
+
+    invariant(foundFile.path === destinationPath, 'Found file should have the same path as the file we uploaded')
+    invariant(foundFile.display_name === displayName, 'Found file should have the same display_name as the file we uploaded')
+    invariant(typeof foundFile.getDownloadUri() === 'undefined', 'Found file should not have a download uri yet')
+
+    await file.delete()
+
+    Logger.info('***** testUploadDataForHugeFile() succeeded! *****')
   }
   /**/
 
@@ -242,7 +268,8 @@ const testSuite = async () => {
     await testFolderListAutoPagination()
     await testUploadAndDownloadToFile()
     await testUploadAndDownloadToString()
-    // await testUploadHugeFile() // to run this test, put a file (or symlink) at huge-file.ext
+    // await testUploadDataForHugeFile() // to run this test, put a file (or symlink) at huge-file.ext
+    // await testUploadFileForHugeFile() // to run this test, put a file (or symlink) at huge-file.ext
     await testSession()
     await testFailure()
     await testUserListAndUpdate()