box-ui-elements 23.4.0-beta.11 → 23.4.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/explorer.js +1 -1
- package/dist/picker.js +1 -1
- package/dist/uploader.js +1 -1
- package/es/api/uploads/FolderUploadNode.js +41 -6
- package/es/api/uploads/FolderUploadNode.js.flow +45 -5
- package/es/api/uploads/FolderUploadNode.js.map +1 -1
- package/es/utils/sleep.js +3 -0
- package/es/utils/sleep.js.flow +2 -0
- package/es/utils/sleep.js.map +1 -0
- package/package.json +1 -1
- package/src/api/uploads/FolderUploadNode.js +45 -5
- package/src/api/uploads/__tests__/FolderUploadNode.test.js +49 -0
- package/src/utils/sleep.js +2 -0
|
@@ -11,7 +11,10 @@ function _toPrimitive(t, r) { if ("object" != typeof t || !t) return t; var e =
|
|
|
11
11
|
import noop from 'lodash/noop';
|
|
12
12
|
import { getFileFromEntry } from '../../utils/uploads';
|
|
13
13
|
import FolderAPI from '../Folder';
|
|
14
|
-
import { STATUS_COMPLETE, STATUS_ERROR, ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED, ERROR_CODE_ITEM_NAME_IN_USE } from '../../constants';
|
|
14
|
+
import { STATUS_COMPLETE, STATUS_ERROR, ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED, ERROR_CODE_ITEM_NAME_IN_USE, DEFAULT_RETRY_DELAY_MS, MS_IN_S } from '../../constants';
|
|
15
|
+
import sleep from '../../utils/sleep';
|
|
16
|
+
const CHILD_FOLDER_UPLOAD_CONCURRENCY = 3;
|
|
17
|
+
const MAX_RETRIES = 3;
|
|
15
18
|
class FolderUploadNode {
|
|
16
19
|
/**
|
|
17
20
|
* [constructor]
|
|
@@ -32,10 +35,30 @@ class FolderUploadNode {
|
|
|
32
35
|
* @returns {Promise}
|
|
33
36
|
*/
|
|
34
37
|
_defineProperty(this, "uploadChildFolders", async errorCallback => {
|
|
38
|
+
// Gets FolderUploadNode values from this.folders key value pairs object
|
|
35
39
|
// $FlowFixMe
|
|
36
40
|
const folders = Object.values(this.folders);
|
|
37
|
-
|
|
38
|
-
|
|
41
|
+
|
|
42
|
+
// Worker function: picks the next folder from the array and uploads until no more folders are available
|
|
43
|
+
const worker = async () => {
|
|
44
|
+
while (folders.length > 0) {
|
|
45
|
+
const folder = folders.pop();
|
|
46
|
+
if (folder) {
|
|
47
|
+
// Await is needed to help ensure rate limit is respected
|
|
48
|
+
// eslint-disable-next-line no-await-in-loop
|
|
49
|
+
await folder.upload(this.folderId, errorCallback);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// Spawns up to CHILD_FOLDER_UPLOAD_CONCURRENCY workers that upload folders in parallel until folders array is empty
|
|
55
|
+
const workers = [];
|
|
56
|
+
for (let i = 0; i < CHILD_FOLDER_UPLOAD_CONCURRENCY && i < folders.length; i += 1) {
|
|
57
|
+
workers.push(worker());
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Waits for all workers to finish
|
|
61
|
+
await Promise.all(workers);
|
|
39
62
|
});
|
|
40
63
|
/**
|
|
41
64
|
* Create folder and add it to the upload queue
|
|
@@ -45,7 +68,7 @@ class FolderUploadNode {
|
|
|
45
68
|
* @param {boolean} isRoot
|
|
46
69
|
* @returns {Promise}
|
|
47
70
|
*/
|
|
48
|
-
_defineProperty(this, "createAndUploadFolder", async (errorCallback, isRoot) => {
|
|
71
|
+
_defineProperty(this, "createAndUploadFolder", async (errorCallback, isRoot, retryCount = 0) => {
|
|
49
72
|
await this.buildCurrentFolderFromEntry();
|
|
50
73
|
let errorEncountered = false;
|
|
51
74
|
let errorCode = '';
|
|
@@ -53,9 +76,20 @@ class FolderUploadNode {
|
|
|
53
76
|
const data = await this.createFolder();
|
|
54
77
|
this.folderId = data.id;
|
|
55
78
|
} catch (error) {
|
|
56
|
-
// @TODO: Handle 429
|
|
57
79
|
if (error.code === ERROR_CODE_ITEM_NAME_IN_USE) {
|
|
58
80
|
this.folderId = error.context_info.conflicts[0].id;
|
|
81
|
+
} else if (error.status === 429 && retryCount < MAX_RETRIES) {
|
|
82
|
+
// Set a default exponential backoff delay with a random jitter(0–999 ms) to avoid all requests being sent at once
|
|
83
|
+
// This will be overridden if the Retry-After header is present in the response
|
|
84
|
+
let retryAfterMs = DEFAULT_RETRY_DELAY_MS * 2 ** retryCount + Math.floor(Math.random() * 1000);
|
|
85
|
+
if (error.headers) {
|
|
86
|
+
const retryAfterHeaderSec = parseInt(error.headers['retry-after'] || error.headers.get('Retry-After'), 10);
|
|
87
|
+
if (!Number.isNaN(retryAfterHeaderSec)) {
|
|
88
|
+
retryAfterMs = retryAfterHeaderSec * MS_IN_S;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
await sleep(retryAfterMs);
|
|
92
|
+
return this.createAndUploadFolder(errorCallback, isRoot, retryCount + 1);
|
|
59
93
|
} else if (isRoot) {
|
|
60
94
|
errorCallback(error);
|
|
61
95
|
} else {
|
|
@@ -73,7 +107,7 @@ class FolderUploadNode {
|
|
|
73
107
|
|
|
74
108
|
// The root folder has already been added to the upload queue in ContentUploader
|
|
75
109
|
if (isRoot) {
|
|
76
|
-
return;
|
|
110
|
+
return undefined;
|
|
77
111
|
}
|
|
78
112
|
const folderObject = {
|
|
79
113
|
extension: '',
|
|
@@ -90,6 +124,7 @@ class FolderUploadNode {
|
|
|
90
124
|
};
|
|
91
125
|
}
|
|
92
126
|
this.addFolderToUploadQueue(folderObject);
|
|
127
|
+
return undefined;
|
|
93
128
|
});
|
|
94
129
|
/**
|
|
95
130
|
* Format files to Array<UploadFileWithAPIOptions> for upload
|
|
@@ -11,6 +11,8 @@ import {
|
|
|
11
11
|
STATUS_ERROR,
|
|
12
12
|
ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED,
|
|
13
13
|
ERROR_CODE_ITEM_NAME_IN_USE,
|
|
14
|
+
DEFAULT_RETRY_DELAY_MS,
|
|
15
|
+
MS_IN_S,
|
|
14
16
|
} from '../../constants';
|
|
15
17
|
import type {
|
|
16
18
|
UploadFileWithAPIOptions,
|
|
@@ -18,6 +20,10 @@ import type {
|
|
|
18
20
|
FolderUploadItem,
|
|
19
21
|
DirectoryReader,
|
|
20
22
|
} from '../../common/types/upload';
|
|
23
|
+
import sleep from '../../utils/sleep';
|
|
24
|
+
|
|
25
|
+
const CHILD_FOLDER_UPLOAD_CONCURRENCY = 3;
|
|
26
|
+
const MAX_RETRIES = 3;
|
|
21
27
|
|
|
22
28
|
class FolderUploadNode {
|
|
23
29
|
addFolderToUploadQueue: Function;
|
|
@@ -93,11 +99,30 @@ class FolderUploadNode {
|
|
|
93
99
|
* @returns {Promise}
|
|
94
100
|
*/
|
|
95
101
|
uploadChildFolders = async (errorCallback: Function) => {
|
|
102
|
+
// Gets FolderUploadNode values from this.folders key value pairs object
|
|
96
103
|
// $FlowFixMe
|
|
97
104
|
const folders: Array<FolderUploadNode> = Object.values(this.folders);
|
|
98
|
-
const promises = folders.map(folder => folder.upload(this.folderId, errorCallback));
|
|
99
105
|
|
|
100
|
-
|
|
106
|
+
// Worker function: picks the next folder from the array and uploads until no more folders are available
|
|
107
|
+
const worker = async () => {
|
|
108
|
+
while (folders.length > 0) {
|
|
109
|
+
const folder = folders.pop();
|
|
110
|
+
if (folder) {
|
|
111
|
+
// Await is needed to help ensure rate limit is respected
|
|
112
|
+
// eslint-disable-next-line no-await-in-loop
|
|
113
|
+
await folder.upload(this.folderId, errorCallback);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// Spawns up to CHILD_FOLDER_UPLOAD_CONCURRENCY workers that upload folders in parallel until folders array is empty
|
|
119
|
+
const workers = [];
|
|
120
|
+
for (let i = 0; i < CHILD_FOLDER_UPLOAD_CONCURRENCY && i < folders.length; i += 1) {
|
|
121
|
+
workers.push(worker());
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Waits for all workers to finish
|
|
125
|
+
await Promise.all(workers);
|
|
101
126
|
};
|
|
102
127
|
|
|
103
128
|
/**
|
|
@@ -108,7 +133,7 @@ class FolderUploadNode {
|
|
|
108
133
|
* @param {boolean} isRoot
|
|
109
134
|
* @returns {Promise}
|
|
110
135
|
*/
|
|
111
|
-
createAndUploadFolder = async (errorCallback: Function, isRoot: boolean) => {
|
|
136
|
+
createAndUploadFolder = async (errorCallback: Function, isRoot: boolean, retryCount: number = 0) => {
|
|
112
137
|
await this.buildCurrentFolderFromEntry();
|
|
113
138
|
|
|
114
139
|
let errorEncountered = false;
|
|
@@ -117,9 +142,23 @@ class FolderUploadNode {
|
|
|
117
142
|
const data = await this.createFolder();
|
|
118
143
|
this.folderId = data.id;
|
|
119
144
|
} catch (error) {
|
|
120
|
-
// @TODO: Handle 429
|
|
121
145
|
if (error.code === ERROR_CODE_ITEM_NAME_IN_USE) {
|
|
122
146
|
this.folderId = error.context_info.conflicts[0].id;
|
|
147
|
+
} else if (error.status === 429 && retryCount < MAX_RETRIES) {
|
|
148
|
+
// Set a default exponential backoff delay with a random jitter(0–999 ms) to avoid all requests being sent at once
|
|
149
|
+
// This will be overridden if the Retry-After header is present in the response
|
|
150
|
+
let retryAfterMs = DEFAULT_RETRY_DELAY_MS * 2 ** retryCount + Math.floor(Math.random() * 1000);
|
|
151
|
+
if (error.headers) {
|
|
152
|
+
const retryAfterHeaderSec = parseInt(
|
|
153
|
+
error.headers['retry-after'] || error.headers.get('Retry-After'),
|
|
154
|
+
10,
|
|
155
|
+
);
|
|
156
|
+
if (!Number.isNaN(retryAfterHeaderSec)) {
|
|
157
|
+
retryAfterMs = retryAfterHeaderSec * MS_IN_S;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
await sleep(retryAfterMs);
|
|
161
|
+
return this.createAndUploadFolder(errorCallback, isRoot, retryCount + 1);
|
|
123
162
|
} else if (isRoot) {
|
|
124
163
|
errorCallback(error);
|
|
125
164
|
} else {
|
|
@@ -135,7 +174,7 @@ class FolderUploadNode {
|
|
|
135
174
|
|
|
136
175
|
// The root folder has already been added to the upload queue in ContentUploader
|
|
137
176
|
if (isRoot) {
|
|
138
|
-
return;
|
|
177
|
+
return undefined;
|
|
139
178
|
}
|
|
140
179
|
|
|
141
180
|
const folderObject: FolderUploadItem = {
|
|
@@ -153,6 +192,7 @@ class FolderUploadNode {
|
|
|
153
192
|
}
|
|
154
193
|
|
|
155
194
|
this.addFolderToUploadQueue(folderObject);
|
|
195
|
+
return undefined;
|
|
156
196
|
};
|
|
157
197
|
|
|
158
198
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FolderUploadNode.js","names":["noop","getFileFromEntry","FolderAPI","STATUS_COMPLETE","STATUS_ERROR","ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED","ERROR_CODE_ITEM_NAME_IN_USE","FolderUploadNode","constructor","name","addFilesToUploadQueue","addFolderToUploadQueue","fileAPIOptions","baseAPIOptions","entry","_defineProperty","errorCallback","folders","Object","values","promises","map","folder","upload","folderId","Promise","all","isRoot","buildCurrentFolderFromEntry","errorEncountered","errorCode","data","createFolder","id","error","code","context_info","conflicts","folderObject","extension","status","isFolder","size","progress","files","file","options","_objectSpread","uploadInitTimestamp","Date","now","entries","isFile","push","reader","resolve","readEntries","length","createFolderUploadNodesFromEntries","readEntry","createReader","parentFolderId","createAndUploadFolder","getFolderId","getFormattedFiles","uploadChildFolders","folderAPI","reject","create"],"sources":["../../../src/api/uploads/FolderUploadNode.js"],"sourcesContent":["/**\n * @flow\n * @file Recursively create folder and upload files\n * @author Box\n */\nimport noop from 'lodash/noop';\nimport { getFileFromEntry } from '../../utils/uploads';\nimport FolderAPI from '../Folder';\nimport {\n STATUS_COMPLETE,\n STATUS_ERROR,\n ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED,\n ERROR_CODE_ITEM_NAME_IN_USE,\n} from '../../constants';\nimport type {\n UploadFileWithAPIOptions,\n FileSystemFileEntry,\n FolderUploadItem,\n DirectoryReader,\n} from '../../common/types/upload';\n\nclass FolderUploadNode {\n addFolderToUploadQueue: Function;\n\n files: Array<File> = [];\n\n folderId: string;\n\n folders: Object = {};\n\n name: string;\n\n parentFolderId: string;\n\n addFilesToUploadQueue: Function;\n\n fileAPIOptions: Object;\n\n baseAPIOptions: Object;\n\n entry: ?FileSystemFileEntry;\n\n /**\n * [constructor]\n *\n * @param {string} name\n * @param {Function} addFilesToUploadQueue\n * @param {Function} addFolderToUploadQueue\n * @returns {void}\n */\n constructor(\n name: string,\n addFilesToUploadQueue: Function,\n addFolderToUploadQueue: Function,\n fileAPIOptions: Object,\n baseAPIOptions: Object,\n entry?: FileSystemFileEntry,\n ) {\n this.name = name;\n this.addFilesToUploadQueue = addFilesToUploadQueue;\n this.addFolderToUploadQueue = addFolderToUploadQueue;\n this.fileAPIOptions = fileAPIOptions;\n this.baseAPIOptions = baseAPIOptions;\n this.entry = entry;\n }\n\n /**\n * Upload a folder\n *\n * @public\n * @param {string} parentFolderId\n * @param {Function} errorCallback\n * @param {boolean} isRoot\n * @returns {Promise}\n */\n async upload(parentFolderId: string, errorCallback: Function, isRoot: boolean = false) {\n this.parentFolderId = parentFolderId;\n\n await this.createAndUploadFolder(errorCallback, isRoot);\n\n // Check if folder was successfully created before we attempt to upload its contents.\n if (this.getFolderId()) {\n this.addFilesToUploadQueue(this.getFormattedFiles(), noop, true);\n await this.uploadChildFolders(errorCallback);\n }\n }\n\n /**\n * Upload all child folders\n *\n * @private\n * @param {Function} errorCallback\n * @returns {Promise}\n */\n uploadChildFolders = async (errorCallback: Function) => {\n // $FlowFixMe\n const folders: Array<FolderUploadNode> = Object.values(this.folders);\n const promises = folders.map(folder => folder.upload(this.folderId, errorCallback));\n\n await Promise.all(promises);\n };\n\n /**\n * Create folder and add it to the upload queue\n *\n * @private\n * @param {Function} errorCallback\n * @param {boolean} isRoot\n * @returns {Promise}\n */\n createAndUploadFolder = async (errorCallback: Function, isRoot: boolean) => {\n await this.buildCurrentFolderFromEntry();\n\n let errorEncountered = false;\n let errorCode = '';\n try {\n const data = await this.createFolder();\n this.folderId = data.id;\n } catch (error) {\n // @TODO: Handle 429\n if (error.code === ERROR_CODE_ITEM_NAME_IN_USE) {\n this.folderId = error.context_info.conflicts[0].id;\n } else if (isRoot) {\n errorCallback(error);\n } else {\n // If this is a child folder of the folder being uploaded, this errorCallback will set\n // an error message on the root folder being uploaded. Set a generic messages saying that a\n // child has caused the error. The child folder will be tagged with the error message in\n // the call to this.addFolderToUploadQueue below\n errorEncountered = true;\n errorCode = error.code;\n errorCallback({ code: ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED });\n }\n }\n\n // The root folder has already been added to the upload queue in ContentUploader\n if (isRoot) {\n return;\n }\n\n const folderObject: FolderUploadItem = {\n extension: '',\n name: this.name,\n status: STATUS_COMPLETE,\n isFolder: true,\n size: 1,\n progress: 100,\n };\n\n if (errorEncountered) {\n folderObject.status = STATUS_ERROR;\n folderObject.error = { code: errorCode };\n }\n\n this.addFolderToUploadQueue(folderObject);\n };\n\n /**\n * Format files to Array<UploadFileWithAPIOptions> for upload\n *\n * @private\n * @returns {Array<UploadFileWithAPIOptions>}\n */\n getFormattedFiles = (): Array<UploadFileWithAPIOptions> =>\n this.files.map((file: File) => ({\n file,\n options: {\n ...this.fileAPIOptions,\n folderId: this.folderId,\n uploadInitTimestamp: Date.now(),\n },\n }));\n\n /**\n * Promisify create folder\n *\n * @private\n * @returns {Promise}\n */\n createFolder(): Promise<any> {\n const folderAPI = new FolderAPI({\n ...this.baseAPIOptions,\n id: `folder_${this.parentFolderId}`,\n });\n return new Promise((resolve, reject) => {\n folderAPI.create(this.parentFolderId, this.name, resolve, reject);\n });\n }\n\n /**\n * Create FolderUploadNode instances from entries\n *\n * @private\n * @param {Array<FileSystemFileEntry>} entries\n * @returns {Promise<any>}\n */\n createFolderUploadNodesFromEntries = async (entries: Array<FileSystemFileEntry>): Promise<any> => {\n await Promise.all(\n entries.map(async entry => {\n const { isFile, name } = entry;\n\n if (isFile) {\n const file = await getFileFromEntry(entry);\n this.files.push(file);\n return;\n }\n\n this.folders[name] = new FolderUploadNode(\n name,\n this.addFilesToUploadQueue,\n this.addFolderToUploadQueue,\n this.fileAPIOptions,\n {\n ...this.baseAPIOptions,\n ...this.fileAPIOptions,\n },\n entry,\n );\n }),\n );\n };\n\n /**\n * Recursively read an entry\n *\n * @private\n * @param {DirectoryReader} reader\n * @param {Function} resolve\n * @returns {void}\n */\n readEntry = (reader: DirectoryReader, resolve: Function) => {\n reader.readEntries(async entries => {\n // Quit recursing when there are no remaining entries.\n if (!entries.length) {\n resolve();\n return;\n }\n\n await this.createFolderUploadNodesFromEntries(entries);\n\n this.readEntry(reader, resolve);\n }, noop);\n };\n\n /**\n * Build current folder from entry\n *\n * @private\n * @returns {Promise<any>}\n */\n buildCurrentFolderFromEntry = (): Promise<any> => {\n if (!this.entry) {\n return Promise.resolve();\n }\n\n return new Promise(resolve => {\n // $FlowFixMe entry is not empty\n const reader = this.entry.createReader();\n\n this.readEntry(reader, resolve);\n });\n };\n\n /**\n * Returns the folderId\n * @returns {string}\n */\n getFolderId = (): string => {\n return this.folderId;\n };\n}\n\nexport default FolderUploadNode;\n"],"mappings":";;;;;AAAA;AACA;AACA;AACA;AACA;AACA,OAAOA,IAAI,MAAM,aAAa;AAC9B,SAASC,gBAAgB,QAAQ,qBAAqB;AACtD,OAAOC,SAAS,MAAM,WAAW;AACjC,SACIC,eAAe,EACfC,YAAY,EACZC,qCAAqC,EACrCC,2BAA2B,QACxB,iBAAiB;AAQxB,MAAMC,gBAAgB,CAAC;EAqBnB;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;EACIC,WAAWA,CACPC,KAAY,EACZC,qBAA+B,EAC/BC,sBAAgC,EAChCC,cAAsB,EACtBC,cAAsB,EACtBC,MAA2B,EAC7B;IAAAC,eAAA,gBAjCmB,EAAE;IAAAA,eAAA,kBAIL,CAAC,CAAC;IA2DpB;AACJ;AACA;AACA;AACA;AACA;AACA;IANIA,eAAA,6BAOqB,MAAOC,aAAuB,IAAK;MACpD;MACA,MAAMC,OAAgC,GAAGC,MAAM,CAACC,MAAM,CAAC,IAAI,CAACF,OAAO,CAAC;MACpE,MAAMG,QAAQ,GAAGH,OAAO,CAACI,GAAG,CAACC,MAAM,IAAIA,MAAM,CAACC,MAAM,CAAC,IAAI,CAACC,QAAQ,EAAER,aAAa,CAAC,CAAC;MAEnF,MAAMS,OAAO,CAACC,GAAG,CAACN,QAAQ,CAAC;IAC/B,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;IAPIL,eAAA,gCAQwB,OAAOC,aAAuB,EAAEW,MAAe,KAAK;MACxE,MAAM,IAAI,CAACC,2BAA2B,CAAC,CAAC;MAExC,IAAIC,gBAAgB,GAAG,KAAK;MAC5B,IAAIC,SAAS,GAAG,EAAE;MAClB,IAAI;QACA,MAAMC,IAAI,GAAG,MAAM,IAAI,CAACC,YAAY,CAAC,CAAC;QACtC,IAAI,CAACR,QAAQ,GAAGO,IAAI,CAACE,EAAE;MAC3B,CAAC,CAAC,OAAOC,KAAK,EAAE;QACZ;QACA,IAAIA,KAAK,CAACC,IAAI,KAAK7B,2BAA2B,EAAE;UAC5C,IAAI,CAACkB,QAAQ,GAAGU,KAAK,CAACE,YAAY,CAACC,SAAS,CAAC,CAAC,CAAC,CAACJ,EAAE;QACtD,CAAC,MAAM,IAAIN,MAAM,EAAE;UACfX,aAAa,CAACkB,KAAK,CAAC;QACxB,CAAC,MAAM;UACH;UACA;UACA;UACA;UACAL,gBAAgB,GAAG,IAAI;UACvBC,SAAS,GAAGI,KAAK,CAACC,IAAI;UACtBnB,aAAa,CAAC;YAAEmB,IAAI,EAAE9B;UAAsC,CAAC,CAAC;QAClE;MACJ;;MAEA;MACA,IAAIsB,MAAM,EAAE;QACR;MACJ;MAEA,MAAMW,YAA8B,GAAG;QACnCC,SAAS,EAAE,EAAE;QACb9B,IAAI,EAAE,IAAI,CAACA,IAAI;QACf+B,MAAM,EAAErC,eAAe;QACvBsC,QAAQ,EAAE,IAAI;QACdC,IAAI,EAAE,CAAC;QACPC,QAAQ,EAAE;MACd,CAAC;MAED,IAAId,gBAAgB,EAAE;QAClBS,YAAY,CAACE,MAAM,GAAGpC,YAAY;QAClCkC,YAAY,CAACJ,KAAK,GAAG;UAAEC,IAAI,EAAEL;QAAU,CAAC;MAC5C;MAEA,IAAI,CAACnB,sBAAsB,CAAC2B,YAAY,CAAC;IAC7C,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;IALIvB,eAAA,4BAMoB,MAChB,IAAI,CAAC6B,KAAK,CAACvB,GAAG,CAAEwB,IAAU,KAAM;MAC5BA,IAAI;MACJC,OAAO,EAAAC,aAAA,CAAAA,aAAA,KACA,IAAI,CAACnC,cAAc;QACtBY,QAAQ,EAAE,IAAI,CAACA,QAAQ;QACvBwB,mBAAmB,EAAEC,IAAI,CAACC,GAAG,CAAC;MAAC;IAEvC,CAAC,CAAC,CAAC;IAkBP;AACJ;AACA;AACA;AACA;AACA;AACA;IANInC,eAAA,6CAOqC,MAAOoC,OAAmC,IAAmB;MAC9F,MAAM1B,OAAO,CAACC,GAAG,CACbyB,OAAO,CAAC9B,GAAG,CAAC,MAAMP,KAAK,IAAI;QACvB,MAAM;UAAEsC,MAAM;UAAE3C;QAAK,CAAC,GAAGK,KAAK;QAE9B,IAAIsC,MAAM,EAAE;UACR,MAAMP,IAAI,GAAG,MAAM5C,gBAAgB,CAACa,KAAK,CAAC;UAC1C,IAAI,CAAC8B,KAAK,CAACS,IAAI,CAACR,IAAI,CAAC;UACrB;QACJ;QAEA,IAAI,CAAC5B,OAAO,CAACR,IAAI,CAAC,GAAG,IAAIF,gBAAgB,CACrCE,IAAI,EACJ,IAAI,CAACC,qBAAqB,EAC1B,IAAI,CAACC,sBAAsB,EAC3B,IAAI,CAACC,cAAc,EAAAmC,aAAA,CAAAA,aAAA,KAEZ,IAAI,CAAClC,cAAc,GACnB,IAAI,CAACD,cAAc,GAE1BE,KACJ,CAAC;MACL,CAAC,CACL,CAAC;IACL,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;IAPIC,eAAA,oBAQY,CAACuC,MAAuB,EAAEC,OAAiB,KAAK;MACxDD,MAAM,CAACE,WAAW,CAAC,MAAML,OAAO,IAAI;QAChC;QACA,IAAI,CAACA,OAAO,CAACM,MAAM,EAAE;UACjBF,OAAO,CAAC,CAAC;UACT;QACJ;QAEA,MAAM,IAAI,CAACG,kCAAkC,CAACP,OAAO,CAAC;QAEtD,IAAI,CAACQ,SAAS,CAACL,MAAM,EAAEC,OAAO,CAAC;MACnC,CAAC,EAAEvD,IAAI,CAAC;IACZ,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;IALIe,eAAA,sCAM8B,MAAoB;MAC9C,IAAI,CAAC,IAAI,CAACD,KAAK,EAAE;QACb,OAAOW,OAAO,CAAC8B,OAAO,CAAC,CAAC;MAC5B;MAEA,OAAO,IAAI9B,OAAO,CAAC8B,OAAO,IAAI;QAC1B;QACA,MAAMD,MAAM,GAAG,IAAI,CAACxC,KAAK,CAAC8C,YAAY,CAAC,CAAC;QAExC,IAAI,CAACD,SAAS,CAACL,MAAM,EAAEC,OAAO,CAAC;MACnC,CAAC,CAAC;IACN,CAAC;IAED;AACJ;AACA;AACA;IAHIxC,eAAA,sBAIc,MAAc;MACxB,OAAO,IAAI,CAACS,QAAQ;IACxB,CAAC;IAnNG,IAAI,CAACf,IAAI,GAAGA,KAAI;IAChB,IAAI,CAACC,qBAAqB,GAAGA,qBAAqB;IAClD,IAAI,CAACC,sBAAsB,GAAGA,sBAAsB;IACpD,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAI,CAACC,KAAK,GAAGA,MAAK;EACtB;;EAEA;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACI,MAAMS,MAAMA,CAACsC,cAAsB,EAAE7C,aAAuB,EAAEW,MAAe,GAAG,KAAK,EAAE;IACnF,IAAI,CAACkC,cAAc,GAAGA,cAAc;IAEpC,MAAM,IAAI,CAACC,qBAAqB,CAAC9C,aAAa,EAAEW,MAAM,CAAC;;IAEvD;IACA,IAAI,IAAI,CAACoC,WAAW,CAAC,CAAC,EAAE;MACpB,IAAI,CAACrD,qBAAqB,CAAC,IAAI,CAACsD,iBAAiB,CAAC,CAAC,EAAEhE,IAAI,EAAE,IAAI,CAAC;MAChE,MAAM,IAAI,CAACiE,kBAAkB,CAACjD,aAAa,CAAC;IAChD;EACJ;EAwFA;AACJ;AACA;AACA;AACA;AACA;EACIgB,YAAYA,CAAA,EAAiB;IACzB,MAAMkC,SAAS,GAAG,IAAIhE,SAAS,CAAA6C,aAAA,CAAAA,aAAA,KACxB,IAAI,CAAClC,cAAc;MACtBoB,EAAE,EAAE,UAAU,IAAI,CAAC4B,cAAc;IAAE,EACtC,CAAC;IACF,OAAO,IAAIpC,OAAO,CAAC,CAAC8B,OAAO,EAAEY,MAAM,KAAK;MACpCD,SAAS,CAACE,MAAM,CAAC,IAAI,CAACP,cAAc,EAAE,IAAI,CAACpD,IAAI,EAAE8C,OAAO,EAAEY,MAAM,CAAC;IACrE,CAAC,CAAC;EACN;AAmFJ;AAEA,eAAe5D,gBAAgB","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"FolderUploadNode.js","names":["noop","getFileFromEntry","FolderAPI","STATUS_COMPLETE","STATUS_ERROR","ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED","ERROR_CODE_ITEM_NAME_IN_USE","DEFAULT_RETRY_DELAY_MS","MS_IN_S","sleep","CHILD_FOLDER_UPLOAD_CONCURRENCY","MAX_RETRIES","FolderUploadNode","constructor","name","addFilesToUploadQueue","addFolderToUploadQueue","fileAPIOptions","baseAPIOptions","entry","_defineProperty","errorCallback","folders","Object","values","worker","length","folder","pop","upload","folderId","workers","i","push","Promise","all","isRoot","retryCount","buildCurrentFolderFromEntry","errorEncountered","errorCode","data","createFolder","id","error","code","context_info","conflicts","status","retryAfterMs","Math","floor","random","headers","retryAfterHeaderSec","parseInt","get","Number","isNaN","createAndUploadFolder","undefined","folderObject","extension","isFolder","size","progress","files","map","file","options","_objectSpread","uploadInitTimestamp","Date","now","entries","isFile","reader","resolve","readEntries","createFolderUploadNodesFromEntries","readEntry","createReader","parentFolderId","getFolderId","getFormattedFiles","uploadChildFolders","folderAPI","reject","create"],"sources":["../../../src/api/uploads/FolderUploadNode.js"],"sourcesContent":["/**\n * @flow\n * @file Recursively create folder and upload files\n * @author Box\n */\nimport noop from 'lodash/noop';\nimport { getFileFromEntry } from '../../utils/uploads';\nimport FolderAPI from '../Folder';\nimport {\n STATUS_COMPLETE,\n STATUS_ERROR,\n ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED,\n ERROR_CODE_ITEM_NAME_IN_USE,\n DEFAULT_RETRY_DELAY_MS,\n MS_IN_S,\n} from '../../constants';\nimport type {\n UploadFileWithAPIOptions,\n FileSystemFileEntry,\n FolderUploadItem,\n DirectoryReader,\n} from '../../common/types/upload';\nimport sleep from '../../utils/sleep';\n\nconst CHILD_FOLDER_UPLOAD_CONCURRENCY = 3;\nconst MAX_RETRIES = 3;\n\nclass FolderUploadNode {\n addFolderToUploadQueue: Function;\n\n files: Array<File> = [];\n\n folderId: string;\n\n folders: Object = {};\n\n name: string;\n\n parentFolderId: string;\n\n addFilesToUploadQueue: Function;\n\n fileAPIOptions: Object;\n\n baseAPIOptions: Object;\n\n entry: ?FileSystemFileEntry;\n\n /**\n * [constructor]\n *\n * @param {string} name\n * @param {Function} addFilesToUploadQueue\n * @param {Function} addFolderToUploadQueue\n * @returns {void}\n */\n constructor(\n name: string,\n addFilesToUploadQueue: Function,\n addFolderToUploadQueue: Function,\n fileAPIOptions: Object,\n baseAPIOptions: Object,\n entry?: FileSystemFileEntry,\n ) {\n this.name = name;\n this.addFilesToUploadQueue = addFilesToUploadQueue;\n this.addFolderToUploadQueue = addFolderToUploadQueue;\n this.fileAPIOptions = fileAPIOptions;\n this.baseAPIOptions = baseAPIOptions;\n this.entry = entry;\n }\n\n /**\n * Upload a folder\n *\n * @public\n * @param {string} parentFolderId\n * @param {Function} errorCallback\n * @param {boolean} isRoot\n * @returns {Promise}\n */\n async upload(parentFolderId: string, errorCallback: Function, isRoot: boolean = false) {\n this.parentFolderId = parentFolderId;\n\n await this.createAndUploadFolder(errorCallback, isRoot);\n\n // Check if folder was successfully created before we attempt to upload its contents.\n if (this.getFolderId()) {\n this.addFilesToUploadQueue(this.getFormattedFiles(), noop, true);\n await this.uploadChildFolders(errorCallback);\n }\n }\n\n /**\n * Upload all child folders\n *\n * @private\n * @param {Function} errorCallback\n * @returns {Promise}\n */\n uploadChildFolders = async (errorCallback: Function) => {\n // Gets FolderUploadNode values from this.folders key value pairs object\n // $FlowFixMe\n const folders: Array<FolderUploadNode> = Object.values(this.folders);\n\n // Worker function: picks the next folder from the array and uploads until no more folders are available\n const worker = async () => {\n while (folders.length > 0) {\n const folder = folders.pop();\n if (folder) {\n // Await is needed to help ensure rate limit is respected\n // eslint-disable-next-line no-await-in-loop\n await folder.upload(this.folderId, errorCallback);\n }\n }\n };\n\n // Spawns up to CHILD_FOLDER_UPLOAD_CONCURRENCY workers that upload folders in parallel until folders array is empty\n const workers = [];\n for (let i = 0; i < CHILD_FOLDER_UPLOAD_CONCURRENCY && i < folders.length; i += 1) {\n workers.push(worker());\n }\n\n // Waits for all workers to finish\n await Promise.all(workers);\n };\n\n /**\n * Create folder and add it to the upload queue\n *\n * @private\n * @param {Function} errorCallback\n * @param {boolean} isRoot\n * @returns {Promise}\n */\n createAndUploadFolder = async (errorCallback: Function, isRoot: boolean, retryCount: number = 0) => {\n await this.buildCurrentFolderFromEntry();\n\n let errorEncountered = false;\n let errorCode = '';\n try {\n const data = await this.createFolder();\n this.folderId = data.id;\n } catch (error) {\n if (error.code === ERROR_CODE_ITEM_NAME_IN_USE) {\n this.folderId = error.context_info.conflicts[0].id;\n } else if (error.status === 429 && retryCount < MAX_RETRIES) {\n // Set a default exponential backoff delay with a random jitter(0–999 ms) to avoid all requests being sent at once\n // This will be overridden if the Retry-After header is present in the response\n let retryAfterMs = DEFAULT_RETRY_DELAY_MS * 2 ** retryCount + Math.floor(Math.random() * 1000);\n if (error.headers) {\n const retryAfterHeaderSec = parseInt(\n error.headers['retry-after'] || error.headers.get('Retry-After'),\n 10,\n );\n if (!Number.isNaN(retryAfterHeaderSec)) {\n retryAfterMs = retryAfterHeaderSec * MS_IN_S;\n }\n }\n await sleep(retryAfterMs);\n return this.createAndUploadFolder(errorCallback, isRoot, retryCount + 1);\n } else if (isRoot) {\n errorCallback(error);\n } else {\n // If this is a child folder of the folder being uploaded, this errorCallback will set\n // an error message on the root folder being uploaded. Set a generic messages saying that a\n // child has caused the error. The child folder will be tagged with the error message in\n // the call to this.addFolderToUploadQueue below\n errorEncountered = true;\n errorCode = error.code;\n errorCallback({ code: ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED });\n }\n }\n\n // The root folder has already been added to the upload queue in ContentUploader\n if (isRoot) {\n return undefined;\n }\n\n const folderObject: FolderUploadItem = {\n extension: '',\n name: this.name,\n status: STATUS_COMPLETE,\n isFolder: true,\n size: 1,\n progress: 100,\n };\n\n if (errorEncountered) {\n folderObject.status = STATUS_ERROR;\n folderObject.error = { code: errorCode };\n }\n\n this.addFolderToUploadQueue(folderObject);\n return undefined;\n };\n\n /**\n * Format files to Array<UploadFileWithAPIOptions> for upload\n *\n * @private\n * @returns {Array<UploadFileWithAPIOptions>}\n */\n getFormattedFiles = (): Array<UploadFileWithAPIOptions> =>\n this.files.map((file: File) => ({\n file,\n options: {\n ...this.fileAPIOptions,\n folderId: this.folderId,\n uploadInitTimestamp: Date.now(),\n },\n }));\n\n /**\n * Promisify create folder\n *\n * @private\n * @returns {Promise}\n */\n createFolder(): Promise<any> {\n const folderAPI = new FolderAPI({\n ...this.baseAPIOptions,\n id: `folder_${this.parentFolderId}`,\n });\n return new Promise((resolve, reject) => {\n folderAPI.create(this.parentFolderId, this.name, resolve, reject);\n });\n }\n\n /**\n * Create FolderUploadNode instances from entries\n *\n * @private\n * @param {Array<FileSystemFileEntry>} entries\n * @returns {Promise<any>}\n */\n createFolderUploadNodesFromEntries = async (entries: Array<FileSystemFileEntry>): Promise<any> => {\n await Promise.all(\n entries.map(async entry => {\n const { isFile, name } = entry;\n\n if (isFile) {\n const file = await getFileFromEntry(entry);\n this.files.push(file);\n return;\n }\n\n this.folders[name] = new FolderUploadNode(\n name,\n this.addFilesToUploadQueue,\n this.addFolderToUploadQueue,\n this.fileAPIOptions,\n {\n ...this.baseAPIOptions,\n ...this.fileAPIOptions,\n },\n entry,\n );\n }),\n );\n };\n\n /**\n * Recursively read an entry\n *\n * @private\n * @param {DirectoryReader} reader\n * @param {Function} resolve\n * @returns {void}\n */\n readEntry = (reader: DirectoryReader, resolve: Function) => {\n reader.readEntries(async entries => {\n // Quit recursing when there are no remaining entries.\n if (!entries.length) {\n resolve();\n return;\n }\n\n await this.createFolderUploadNodesFromEntries(entries);\n\n this.readEntry(reader, resolve);\n }, noop);\n };\n\n /**\n * Build current folder from entry\n *\n * @private\n * @returns {Promise<any>}\n */\n buildCurrentFolderFromEntry = (): Promise<any> => {\n if (!this.entry) {\n return Promise.resolve();\n }\n\n return new Promise(resolve => {\n // $FlowFixMe entry is not empty\n const reader = this.entry.createReader();\n\n this.readEntry(reader, resolve);\n });\n };\n\n /**\n * Returns the folderId\n * @returns {string}\n */\n getFolderId = (): string => {\n return this.folderId;\n };\n}\n\nexport default FolderUploadNode;\n"],"mappings":";;;;;AAAA;AACA;AACA;AACA;AACA;AACA,OAAOA,IAAI,MAAM,aAAa;AAC9B,SAASC,gBAAgB,QAAQ,qBAAqB;AACtD,OAAOC,SAAS,MAAM,WAAW;AACjC,SACIC,eAAe,EACfC,YAAY,EACZC,qCAAqC,EACrCC,2BAA2B,EAC3BC,sBAAsB,EACtBC,OAAO,QACJ,iBAAiB;AAOxB,OAAOC,KAAK,MAAM,mBAAmB;AAErC,MAAMC,+BAA+B,GAAG,CAAC;AACzC,MAAMC,WAAW,GAAG,CAAC;AAErB,MAAMC,gBAAgB,CAAC;EAqBnB;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;EACIC,WAAWA,CACPC,KAAY,EACZC,qBAA+B,EAC/BC,sBAAgC,EAChCC,cAAsB,EACtBC,cAAsB,EACtBC,MAA2B,EAC7B;IAAAC,eAAA,gBAjCmB,EAAE;IAAAA,eAAA,kBAIL,CAAC,CAAC;IA2DpB;AACJ;AACA;AACA;AACA;AACA;AACA;IANIA,eAAA,6BAOqB,MAAOC,aAAuB,IAAK;MACpD;MACA;MACA,MAAMC,OAAgC,GAAGC,MAAM,CAACC,MAAM,CAAC,IAAI,CAACF,OAAO,CAAC;;MAEpE;MACA,MAAMG,MAAM,GAAG,MAAAA,CAAA,KAAY;QACvB,OAAOH,OAAO,CAACI,MAAM,GAAG,CAAC,EAAE;UACvB,MAAMC,MAAM,GAAGL,OAAO,CAACM,GAAG,CAAC,CAAC;UAC5B,IAAID,MAAM,EAAE;YACR;YACA;YACA,MAAMA,MAAM,CAACE,MAAM,CAAC,IAAI,CAACC,QAAQ,EAAET,aAAa,CAAC;UACrD;QACJ;MACJ,CAAC;;MAED;MACA,MAAMU,OAAO,GAAG,EAAE;MAClB,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,+BAA+B,IAAIsB,CAAC,GAAGV,OAAO,CAACI,MAAM,EAAEM,CAAC,IAAI,CAAC,EAAE;QAC/ED,OAAO,CAACE,IAAI,CAACR,MAAM,CAAC,CAAC,CAAC;MAC1B;;MAEA;MACA,MAAMS,OAAO,CAACC,GAAG,CAACJ,OAAO,CAAC;IAC9B,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;IAPIX,eAAA,gCAQwB,OAAOC,aAAuB,EAAEe,MAAe,EAAEC,UAAkB,GAAG,CAAC,KAAK;MAChG,MAAM,IAAI,CAACC,2BAA2B,CAAC,CAAC;MAExC,IAAIC,gBAAgB,GAAG,KAAK;MAC5B,IAAIC,SAAS,GAAG,EAAE;MAClB,IAAI;QACA,MAAMC,IAAI,GAAG,MAAM,IAAI,CAACC,YAAY,CAAC,CAAC;QACtC,IAAI,CAACZ,QAAQ,GAAGW,IAAI,CAACE,EAAE;MAC3B,CAAC,CAAC,OAAOC,KAAK,EAAE;QACZ,IAAIA,KAAK,CAACC,IAAI,KAAKvC,2BAA2B,EAAE;UAC5C,IAAI,CAACwB,QAAQ,GAAGc,KAAK,CAACE,YAAY,CAACC,SAAS,CAAC,CAAC,CAAC,CAACJ,EAAE;QACtD,CAAC,MAAM,IAAIC,KAAK,CAACI,MAAM,KAAK,GAAG,IAAIX,UAAU,GAAG1B,WAAW,EAAE;UACzD;UACA;UACA,IAAIsC,YAAY,GAAG1C,sBAAsB,GAAG,CAAC,IAAI8B,UAAU,GAAGa,IAAI,CAACC,KAAK,CAACD,IAAI,CAACE,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC;UAC9F,IAAIR,KAAK,CAACS,OAAO,EAAE;YACf,MAAMC,mBAAmB,GAAGC,QAAQ,CAChCX,KAAK,CAACS,OAAO,CAAC,aAAa,CAAC,IAAIT,KAAK,CAACS,OAAO,CAACG,GAAG,CAAC,aAAa,CAAC,EAChE,EACJ,CAAC;YACD,IAAI,CAACC,MAAM,CAACC,KAAK,CAACJ,mBAAmB,CAAC,EAAE;cACpCL,YAAY,GAAGK,mBAAmB,GAAG9C,OAAO;YAChD;UACJ;UACA,MAAMC,KAAK,CAACwC,YAAY,CAAC;UACzB,OAAO,IAAI,CAACU,qBAAqB,CAACtC,aAAa,EAAEe,MAAM,EAAEC,UAAU,GAAG,CAAC,CAAC;QAC5E,CAAC,MAAM,IAAID,MAAM,EAAE;UACff,aAAa,CAACuB,KAAK,CAAC;QACxB,CAAC,MAAM;UACH;UACA;UACA;UACA;UACAL,gBAAgB,GAAG,IAAI;UACvBC,SAAS,GAAGI,KAAK,CAACC,IAAI;UACtBxB,aAAa,CAAC;YAAEwB,IAAI,EAAExC;UAAsC,CAAC,CAAC;QAClE;MACJ;;MAEA;MACA,IAAI+B,MAAM,EAAE;QACR,OAAOwB,SAAS;MACpB;MAEA,MAAMC,YAA8B,GAAG;QACnCC,SAAS,EAAE,EAAE;QACbhD,IAAI,EAAE,IAAI,CAACA,IAAI;QACfkC,MAAM,EAAE7C,eAAe;QACvB4D,QAAQ,EAAE,IAAI;QACdC,IAAI,EAAE,CAAC;QACPC,QAAQ,EAAE;MACd,CAAC;MAED,IAAI1B,gBAAgB,EAAE;QAClBsB,YAAY,CAACb,MAAM,GAAG5C,YAAY;QAClCyD,YAAY,CAACjB,KAAK,GAAG;UAAEC,IAAI,EAAEL;QAAU,CAAC;MAC5C;MAEA,IAAI,CAACxB,sBAAsB,CAAC6C,YAAY,CAAC;MACzC,OAAOD,SAAS;IACpB,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;IALIxC,eAAA,4BAMoB,MAChB,IAAI,CAAC8C,KAAK,CAACC,GAAG,CAAEC,IAAU,KAAM;MAC5BA,IAAI;MACJC,OAAO,EAAAC,aAAA,CAAAA,aAAA,KACA,IAAI,CAACrD,cAAc;QACtBa,QAAQ,EAAE,IAAI,CAACA,QAAQ;QACvByC,mBAAmB,EAAEC,IAAI,CAACC,GAAG,CAAC;MAAC;IAEvC,CAAC,CAAC,CAAC;IAkBP;AACJ;AACA;AACA;AACA;AACA;AACA;IANIrD,eAAA,6CAOqC,MAAOsD,OAAmC,IAAmB;MAC9F,MAAMxC,OAAO,CAACC,GAAG,CACbuC,OAAO,CAACP,GAAG,CAAC,MAAMhD,KAAK,IAAI;QACvB,MAAM;UAAEwD,MAAM;UAAE7D;QAAK,CAAC,GAAGK,KAAK;QAE9B,IAAIwD,MAAM,EAAE;UACR,MAAMP,IAAI,GAAG,MAAMnE,gBAAgB,CAACkB,KAAK,CAAC;UAC1C,IAAI,CAAC+C,KAAK,CAACjC,IAAI,CAACmC,IAAI,CAAC;UACrB;QACJ;QAEA,IAAI,CAAC9C,OAAO,CAACR,IAAI,CAAC,GAAG,IAAIF,gBAAgB,CACrCE,IAAI,EACJ,IAAI,CAACC,qBAAqB,EAC1B,IAAI,CAACC,sBAAsB,EAC3B,IAAI,CAACC,cAAc,EAAAqD,aAAA,CAAAA,aAAA,KAEZ,IAAI,CAACpD,cAAc,GACnB,IAAI,CAACD,cAAc,GAE1BE,KACJ,CAAC;MACL,CAAC,CACL,CAAC;IACL,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;IAPIC,eAAA,oBAQY,CAACwD,MAAuB,EAAEC,OAAiB,KAAK;MACxDD,MAAM,CAACE,WAAW,CAAC,MAAMJ,OAAO,IAAI;QAChC;QACA,IAAI,CAACA,OAAO,CAAChD,MAAM,EAAE;UACjBmD,OAAO,CAAC,CAAC;UACT;QACJ;QAEA,MAAM,IAAI,CAACE,kCAAkC,CAACL,OAAO,CAAC;QAEtD,IAAI,CAACM,SAAS,CAACJ,MAAM,EAAEC,OAAO,CAAC;MACnC,CAAC,EAAE7E,IAAI,CAAC;IACZ,CAAC;IAED;AACJ;AACA;AACA;AACA;AACA;IALIoB,eAAA,sCAM8B,MAAoB;MAC9C,IAAI,CAAC,IAAI,CAACD,KAAK,EAAE;QACb,OAAOe,OAAO,CAAC2C,OAAO,CAAC,CAAC;MAC5B;MAEA,OAAO,IAAI3C,OAAO,CAAC2C,OAAO,IAAI;QAC1B;QACA,MAAMD,MAAM,GAAG,IAAI,CAACzD,KAAK,CAAC8D,YAAY,CAAC,CAAC;QAExC,IAAI,CAACD,SAAS,CAACJ,MAAM,EAAEC,OAAO,CAAC;MACnC,CAAC,CAAC;IACN,CAAC;IAED;AACJ;AACA;AACA;IAHIzD,eAAA,sBAIc,MAAc;MACxB,OAAO,IAAI,CAACU,QAAQ;IACxB,CAAC;IArPG,IAAI,CAAChB,IAAI,GAAGA,KAAI;IAChB,IAAI,CAACC,qBAAqB,GAAGA,qBAAqB;IAClD,IAAI,CAACC,sBAAsB,GAAGA,sBAAsB;IACpD,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAI,CAACC,KAAK,GAAGA,MAAK;EACtB;;EAEA;AACJ;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACI,MAAMU,MAAMA,CAACqD,cAAsB,EAAE7D,aAAuB,EAAEe,MAAe,GAAG,KAAK,EAAE;IACnF,IAAI,CAAC8C,cAAc,GAAGA,cAAc;IAEpC,MAAM,IAAI,CAACvB,qBAAqB,CAACtC,aAAa,EAAEe,MAAM,CAAC;;IAEvD;IACA,IAAI,IAAI,CAAC+C,WAAW,CAAC,CAAC,EAAE;MACpB,IAAI,CAACpE,qBAAqB,CAAC,IAAI,CAACqE,iBAAiB,CAAC,CAAC,EAAEpF,IAAI,EAAE,IAAI,CAAC;MAChE,MAAM,IAAI,CAACqF,kBAAkB,CAAChE,aAAa,CAAC;IAChD;EACJ;EA0HA;AACJ;AACA;AACA;AACA;AACA;EACIqB,YAAYA,CAAA,EAAiB;IACzB,MAAM4C,SAAS,GAAG,IAAIpF,SAAS,CAAAoE,aAAA,CAAAA,aAAA,KACxB,IAAI,CAACpD,cAAc;MACtByB,EAAE,EAAE,UAAU,IAAI,CAACuC,cAAc;IAAE,EACtC,CAAC;IACF,OAAO,IAAIhD,OAAO,CAAC,CAAC2C,OAAO,EAAEU,MAAM,KAAK;MACpCD,SAAS,CAACE,MAAM,CAAC,IAAI,CAACN,cAAc,EAAE,IAAI,CAACpE,IAAI,EAAE+D,OAAO,EAAEU,MAAM,CAAC;IACrE,CAAC,CAAC;EACN;AAmFJ;AAEA,eAAe3E,gBAAgB","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sleep.js","names":["sleep","ms","Promise","resolve","setTimeout"],"sources":["../../src/utils/sleep.js"],"sourcesContent":["const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));\nexport default sleep;\n"],"mappings":"AAAA,MAAMA,KAAK,GAAGC,EAAE,IAAI,IAAIC,OAAO,CAACC,OAAO,IAAIC,UAAU,CAACD,OAAO,EAAEF,EAAE,CAAC,CAAC;AACnE,eAAeD,KAAK","ignoreList":[]}
|
package/package.json
CHANGED
|
@@ -11,6 +11,8 @@ import {
|
|
|
11
11
|
STATUS_ERROR,
|
|
12
12
|
ERROR_CODE_UPLOAD_CHILD_FOLDER_FAILED,
|
|
13
13
|
ERROR_CODE_ITEM_NAME_IN_USE,
|
|
14
|
+
DEFAULT_RETRY_DELAY_MS,
|
|
15
|
+
MS_IN_S,
|
|
14
16
|
} from '../../constants';
|
|
15
17
|
import type {
|
|
16
18
|
UploadFileWithAPIOptions,
|
|
@@ -18,6 +20,10 @@ import type {
|
|
|
18
20
|
FolderUploadItem,
|
|
19
21
|
DirectoryReader,
|
|
20
22
|
} from '../../common/types/upload';
|
|
23
|
+
import sleep from '../../utils/sleep';
|
|
24
|
+
|
|
25
|
+
const CHILD_FOLDER_UPLOAD_CONCURRENCY = 3;
|
|
26
|
+
const MAX_RETRIES = 3;
|
|
21
27
|
|
|
22
28
|
class FolderUploadNode {
|
|
23
29
|
addFolderToUploadQueue: Function;
|
|
@@ -93,11 +99,30 @@ class FolderUploadNode {
|
|
|
93
99
|
* @returns {Promise}
|
|
94
100
|
*/
|
|
95
101
|
uploadChildFolders = async (errorCallback: Function) => {
|
|
102
|
+
// Gets FolderUploadNode values from this.folders key value pairs object
|
|
96
103
|
// $FlowFixMe
|
|
97
104
|
const folders: Array<FolderUploadNode> = Object.values(this.folders);
|
|
98
|
-
const promises = folders.map(folder => folder.upload(this.folderId, errorCallback));
|
|
99
105
|
|
|
100
|
-
|
|
106
|
+
// Worker function: picks the next folder from the array and uploads until no more folders are available
|
|
107
|
+
const worker = async () => {
|
|
108
|
+
while (folders.length > 0) {
|
|
109
|
+
const folder = folders.pop();
|
|
110
|
+
if (folder) {
|
|
111
|
+
// Await is needed to help ensure rate limit is respected
|
|
112
|
+
// eslint-disable-next-line no-await-in-loop
|
|
113
|
+
await folder.upload(this.folderId, errorCallback);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// Spawns up to CHILD_FOLDER_UPLOAD_CONCURRENCY workers that upload folders in parallel until folders array is empty
|
|
119
|
+
const workers = [];
|
|
120
|
+
for (let i = 0; i < CHILD_FOLDER_UPLOAD_CONCURRENCY && i < folders.length; i += 1) {
|
|
121
|
+
workers.push(worker());
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Waits for all workers to finish
|
|
125
|
+
await Promise.all(workers);
|
|
101
126
|
};
|
|
102
127
|
|
|
103
128
|
/**
|
|
@@ -108,7 +133,7 @@ class FolderUploadNode {
|
|
|
108
133
|
* @param {boolean} isRoot
|
|
109
134
|
* @returns {Promise}
|
|
110
135
|
*/
|
|
111
|
-
createAndUploadFolder = async (errorCallback: Function, isRoot: boolean) => {
|
|
136
|
+
createAndUploadFolder = async (errorCallback: Function, isRoot: boolean, retryCount: number = 0) => {
|
|
112
137
|
await this.buildCurrentFolderFromEntry();
|
|
113
138
|
|
|
114
139
|
let errorEncountered = false;
|
|
@@ -117,9 +142,23 @@ class FolderUploadNode {
|
|
|
117
142
|
const data = await this.createFolder();
|
|
118
143
|
this.folderId = data.id;
|
|
119
144
|
} catch (error) {
|
|
120
|
-
// @TODO: Handle 429
|
|
121
145
|
if (error.code === ERROR_CODE_ITEM_NAME_IN_USE) {
|
|
122
146
|
this.folderId = error.context_info.conflicts[0].id;
|
|
147
|
+
} else if (error.status === 429 && retryCount < MAX_RETRIES) {
|
|
148
|
+
// Set a default exponential backoff delay with a random jitter(0–999 ms) to avoid all requests being sent at once
|
|
149
|
+
// This will be overridden if the Retry-After header is present in the response
|
|
150
|
+
let retryAfterMs = DEFAULT_RETRY_DELAY_MS * 2 ** retryCount + Math.floor(Math.random() * 1000);
|
|
151
|
+
if (error.headers) {
|
|
152
|
+
const retryAfterHeaderSec = parseInt(
|
|
153
|
+
error.headers['retry-after'] || error.headers.get('Retry-After'),
|
|
154
|
+
10,
|
|
155
|
+
);
|
|
156
|
+
if (!Number.isNaN(retryAfterHeaderSec)) {
|
|
157
|
+
retryAfterMs = retryAfterHeaderSec * MS_IN_S;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
await sleep(retryAfterMs);
|
|
161
|
+
return this.createAndUploadFolder(errorCallback, isRoot, retryCount + 1);
|
|
123
162
|
} else if (isRoot) {
|
|
124
163
|
errorCallback(error);
|
|
125
164
|
} else {
|
|
@@ -135,7 +174,7 @@ class FolderUploadNode {
|
|
|
135
174
|
|
|
136
175
|
// The root folder has already been added to the upload queue in ContentUploader
|
|
137
176
|
if (isRoot) {
|
|
138
|
-
return;
|
|
177
|
+
return undefined;
|
|
139
178
|
}
|
|
140
179
|
|
|
141
180
|
const folderObject: FolderUploadItem = {
|
|
@@ -153,6 +192,7 @@ class FolderUploadNode {
|
|
|
153
192
|
}
|
|
154
193
|
|
|
155
194
|
this.addFolderToUploadQueue(folderObject);
|
|
195
|
+
return undefined;
|
|
156
196
|
};
|
|
157
197
|
|
|
158
198
|
/**
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import noop from 'lodash/noop';
|
|
2
2
|
import FolderUploadNode from '../FolderUploadNode';
|
|
3
3
|
import FolderAPI from '../../Folder';
|
|
4
|
+
import sleep from '../../../utils/sleep';
|
|
4
5
|
import {
|
|
5
6
|
ERROR_CODE_ITEM_NAME_IN_USE,
|
|
6
7
|
STATUS_COMPLETE,
|
|
@@ -12,6 +13,7 @@ jest.mock('../../../utils/uploads', () => ({
|
|
|
12
13
|
...jest.requireActual('../../../utils/uploads'),
|
|
13
14
|
getFileFromEntry: jest.fn(entry => entry),
|
|
14
15
|
}));
|
|
16
|
+
jest.mock('../../../utils/sleep', () => jest.fn(() => Promise.resolve()));
|
|
15
17
|
|
|
16
18
|
let folderUploadNodeInstance;
|
|
17
19
|
let folderCreateMock;
|
|
@@ -30,6 +32,10 @@ describe('api/uploads/FolderUploadNode', () => {
|
|
|
30
32
|
}));
|
|
31
33
|
});
|
|
32
34
|
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
jest.restoreAllMocks();
|
|
37
|
+
});
|
|
38
|
+
|
|
33
39
|
describe('upload()', () => {
|
|
34
40
|
test('should call createAndUploadFolder(), addFilesToUploadQueue() and uploadChildFolders()', async () => {
|
|
35
41
|
const errorCallback = () => 'errorCallback';
|
|
@@ -182,6 +188,49 @@ describe('api/uploads/FolderUploadNode', () => {
|
|
|
182
188
|
|
|
183
189
|
expect(folderUploadNodeInstance.addFolderToUploadQueue).not.toHaveBeenCalled();
|
|
184
190
|
});
|
|
191
|
+
|
|
192
|
+
test('should retry on 429 with default delay', async () => {
|
|
193
|
+
const error = { status: 429 };
|
|
194
|
+
const success = { id: '123' };
|
|
195
|
+
const createFolder = jest.fn().mockRejectedValueOnce(error).mockResolvedValueOnce(success);
|
|
196
|
+
folderUploadNodeInstance.createFolder = createFolder;
|
|
197
|
+
folderUploadNodeInstance.addFolderToUploadQueue = jest.fn();
|
|
198
|
+
|
|
199
|
+
await folderUploadNodeInstance.createAndUploadFolder(jest.fn(), false, 0);
|
|
200
|
+
|
|
201
|
+
expect(createFolder).toHaveBeenCalledTimes(2);
|
|
202
|
+
expect(folderUploadNodeInstance.folderId).toBe('123');
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test.each`
|
|
206
|
+
headersMock | description
|
|
207
|
+
${{ 'retry-after': '2' }} | ${'lower case plain object'}
|
|
208
|
+
${{ get: key => (key === 'Retry-After' ? '2' : undefined) }} | ${'capitalized map object'}
|
|
209
|
+
`('should handle $description headers', async ({ headersMock }) => {
|
|
210
|
+
const error = { status: 429, headers: headersMock };
|
|
211
|
+
const success = { id: '123' };
|
|
212
|
+
const createFolder = jest.fn().mockRejectedValueOnce(error).mockResolvedValueOnce(success);
|
|
213
|
+
folderUploadNodeInstance.createFolder = createFolder;
|
|
214
|
+
folderUploadNodeInstance.addFolderToUploadQueue = jest.fn();
|
|
215
|
+
|
|
216
|
+
await folderUploadNodeInstance.createAndUploadFolder(jest.fn(), false, 0);
|
|
217
|
+
|
|
218
|
+
expect(sleep).toHaveBeenCalledWith(2000);
|
|
219
|
+
expect(createFolder).toHaveBeenCalledTimes(2);
|
|
220
|
+
expect(folderUploadNodeInstance.folderId).toBe('123');
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
test('should stop retrying after max retries', async () => {
|
|
224
|
+
const error = { status: 429 };
|
|
225
|
+
const createFolder = jest.fn().mockRejectedValue(error);
|
|
226
|
+
folderUploadNodeInstance.createFolder = createFolder;
|
|
227
|
+
folderUploadNodeInstance.addFolderToUploadQueue = jest.fn();
|
|
228
|
+
|
|
229
|
+
await folderUploadNodeInstance.createAndUploadFolder(jest.fn(), false, 3);
|
|
230
|
+
|
|
231
|
+
expect(createFolder).toHaveBeenCalledTimes(1);
|
|
232
|
+
expect(folderUploadNodeInstance.folderId).toBeUndefined();
|
|
233
|
+
});
|
|
185
234
|
});
|
|
186
235
|
|
|
187
236
|
describe('getFormattedFiles()', () => {
|