@ardrive/turbo-sdk 1.31.0-alpha.1 → 1.31.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -2
- package/bundles/web.bundle.min.js +342 -475
- package/lib/cjs/cli/commands/uploadFolder.js +2 -1
- package/lib/cjs/cli/options.js +5 -0
- package/lib/cjs/cli/utils.js +1 -0
- package/lib/cjs/common/chunked.js +88 -27
- package/lib/cjs/common/http.js +39 -4
- package/lib/cjs/common/payment.js +2 -1
- package/lib/cjs/common/upload.js +5 -6
- package/lib/cjs/types.js +13 -1
- package/lib/cjs/utils/axiosClient.js +3 -37
- package/lib/cjs/version.js +1 -1
- package/lib/esm/cli/commands/uploadFolder.js +2 -1
- package/lib/esm/cli/options.js +5 -0
- package/lib/esm/cli/utils.js +1 -0
- package/lib/esm/common/chunked.js +89 -28
- package/lib/esm/common/http.js +40 -5
- package/lib/esm/common/payment.js +2 -1
- package/lib/esm/common/upload.js +5 -6
- package/lib/esm/types.js +12 -0
- package/lib/esm/utils/axiosClient.js +3 -14
- package/lib/esm/version.js +1 -1
- package/lib/types/cli/commands/uploadFolder.d.ts.map +1 -1
- package/lib/types/cli/options.d.ts +13 -0
- package/lib/types/cli/options.d.ts.map +1 -1
- package/lib/types/cli/types.d.ts +1 -0
- package/lib/types/cli/types.d.ts.map +1 -1
- package/lib/types/cli/utils.d.ts +3 -10
- package/lib/types/cli/utils.d.ts.map +1 -1
- package/lib/types/common/chunked.d.ts +5 -1
- package/lib/types/common/chunked.d.ts.map +1 -1
- package/lib/types/common/http.d.ts +5 -3
- package/lib/types/common/http.d.ts.map +1 -1
- package/lib/types/common/payment.d.ts +1 -1
- package/lib/types/common/payment.d.ts.map +1 -1
- package/lib/types/common/upload.d.ts +2 -2
- package/lib/types/common/upload.d.ts.map +1 -1
- package/lib/types/types.d.ts +34 -2
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/utils/axiosClient.d.ts +8 -4
- package/lib/types/utils/axiosClient.d.ts.map +1 -1
- package/lib/types/version.d.ts +1 -1
- package/lib/types/version.d.ts.map +1 -1
- package/package.json +9 -10
|
@@ -21,7 +21,7 @@ const utils_js_1 = require("../utils.js");
|
|
|
21
21
|
async function uploadFolder(options) {
|
|
22
22
|
const turbo = await (0, utils_js_1.turboFromOptions)(options);
|
|
23
23
|
const paidBy = await (0, utils_js_1.paidByFromOptions)(options, turbo);
|
|
24
|
-
const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, } = (0, utils_js_1.getUploadFolderOptions)(options);
|
|
24
|
+
const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, } = (0, utils_js_1.getUploadFolderOptions)(options);
|
|
25
25
|
const customTags = (0, utils_js_1.getTagsFromOptions)(options);
|
|
26
26
|
const result = await turbo.uploadFolder({
|
|
27
27
|
folderPath: folderPath,
|
|
@@ -35,6 +35,7 @@ async function uploadFolder(options) {
|
|
|
35
35
|
chunkByteCount,
|
|
36
36
|
chunkingMode,
|
|
37
37
|
maxChunkConcurrency,
|
|
38
|
+
maxFinalizeMs,
|
|
38
39
|
});
|
|
39
40
|
console.log('Uploaded folder:', JSON.stringify(result, null, 2));
|
|
40
41
|
}
|
package/lib/cjs/cli/options.js
CHANGED
|
@@ -163,6 +163,10 @@ exports.optionMap = {
|
|
|
163
163
|
alias: '--max-chunk-concurrency <maxChunkConcurrency>',
|
|
164
164
|
description: 'Maximum number of concurrent chunks to upload per file',
|
|
165
165
|
},
|
|
166
|
+
maxFinalizeMs: {
|
|
167
|
+
alias: '--max-finalize-ms <maxFinalizeMs>',
|
|
168
|
+
description: 'Maximum time in milliseconds to wait for the finalization of all chunks after the last chunk is uploaded. Defaults to 1 minute per GiB of the total file size.',
|
|
169
|
+
},
|
|
166
170
|
chunkByteCount: {
|
|
167
171
|
alias: '--chunk-byte-count <chunkByteCount>',
|
|
168
172
|
description: 'Size of each chunk in bytes',
|
|
@@ -196,6 +200,7 @@ exports.uploadOptions = [
|
|
|
196
200
|
exports.optionMap.useSignerBalanceFirst,
|
|
197
201
|
exports.optionMap.tags,
|
|
198
202
|
exports.optionMap.maxChunkConcurrency,
|
|
203
|
+
exports.optionMap.maxFinalizeMs,
|
|
199
204
|
exports.optionMap.chunkByteCount,
|
|
200
205
|
exports.optionMap.chunkingMode,
|
|
201
206
|
];
|
package/lib/cjs/cli/utils.js
CHANGED
|
@@ -294,5 +294,6 @@ function getChunkingOptions(options) {
|
|
|
294
294
|
maxChunkConcurrency: options.maxChunkConcurrency !== undefined
|
|
295
295
|
? +options.maxChunkConcurrency
|
|
296
296
|
: undefined,
|
|
297
|
+
maxFinalizeMs: options.maxFinalizeMs !== undefined ? +options.maxFinalizeMs : undefined,
|
|
297
298
|
};
|
|
298
299
|
}
|
|
@@ -22,11 +22,15 @@ exports.splitReadableStreamIntoChunks = splitReadableStreamIntoChunks;
|
|
|
22
22
|
const axios_1 = require("axios");
|
|
23
23
|
const plimit_lit_1 = require("plimit-lit");
|
|
24
24
|
const types_js_1 = require("../types.js");
|
|
25
|
+
const common_js_1 = require("../utils/common.js");
|
|
26
|
+
const errors_js_1 = require("../utils/errors.js");
|
|
25
27
|
const events_js_1 = require("./events.js");
|
|
26
28
|
const logger_js_1 = require("./logger.js");
|
|
27
29
|
const fiveMiB = 5 * 1024 * 1024; // 5 MiB
|
|
28
30
|
const fiveHundredMiB = fiveMiB * 100; // 500 MiB
|
|
29
31
|
exports.defaultMaxChunkConcurrency = 5;
|
|
32
|
+
// Limit uploaders to protect server
|
|
33
|
+
const absoluteMaxChunkConcurrency = 256;
|
|
30
34
|
exports.maxChunkByteCount = fiveHundredMiB;
|
|
31
35
|
exports.minChunkByteCount = fiveMiB;
|
|
32
36
|
exports.defaultChunkByteCount = exports.minChunkByteCount;
|
|
@@ -37,17 +41,19 @@ const chunkingHeader = { 'x-chunking-version': '2' };
|
|
|
37
41
|
* uploading them in parallel, and emitting progress/error events.
|
|
38
42
|
*/
|
|
39
43
|
class ChunkedUploader {
|
|
40
|
-
constructor({ http, token, maxChunkConcurrency = exports.defaultMaxChunkConcurrency, chunkByteCount = exports.defaultChunkByteCount, logger = logger_js_1.TurboWinstonLogger.default, chunkingMode = 'auto', dataItemByteCount, }) {
|
|
41
|
-
this.chunkByteCount = chunkByteCount;
|
|
42
|
-
this.maxChunkConcurrency = maxChunkConcurrency;
|
|
43
|
-
this.http = http;
|
|
44
|
-
this.token = token;
|
|
45
|
-
this.logger = logger;
|
|
44
|
+
constructor({ http, token, maxChunkConcurrency = exports.defaultMaxChunkConcurrency, maxFinalizeMs, chunkByteCount = exports.defaultChunkByteCount, logger = logger_js_1.TurboWinstonLogger.default, chunkingMode = 'auto', dataItemByteCount, }) {
|
|
46
45
|
this.assertChunkParams({
|
|
47
46
|
chunkByteCount,
|
|
48
47
|
chunkingMode,
|
|
49
48
|
maxChunkConcurrency,
|
|
49
|
+
maxFinalizeMs,
|
|
50
50
|
});
|
|
51
|
+
this.chunkByteCount = chunkByteCount;
|
|
52
|
+
this.maxChunkConcurrency = maxChunkConcurrency;
|
|
53
|
+
this.maxFinalizeMs = maxFinalizeMs;
|
|
54
|
+
this.http = http;
|
|
55
|
+
this.token = token;
|
|
56
|
+
this.logger = logger;
|
|
51
57
|
this.shouldUseChunkUploader = this.shouldChunkUpload({
|
|
52
58
|
chunkByteCount,
|
|
53
59
|
chunkingMode,
|
|
@@ -65,11 +71,18 @@ class ChunkedUploader {
|
|
|
65
71
|
const isMoreThanTwoChunksOfData = dataItemByteCount > chunkByteCount * 2;
|
|
66
72
|
return isMoreThanTwoChunksOfData;
|
|
67
73
|
}
|
|
68
|
-
assertChunkParams({ chunkByteCount, chunkingMode, maxChunkConcurrency, }) {
|
|
74
|
+
assertChunkParams({ chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, }) {
|
|
75
|
+
if (maxFinalizeMs !== undefined &&
|
|
76
|
+
(Number.isNaN(maxFinalizeMs) ||
|
|
77
|
+
!Number.isInteger(maxFinalizeMs) ||
|
|
78
|
+
maxFinalizeMs < 0)) {
|
|
79
|
+
throw new Error('Invalid max finalization wait time. Must be a non-negative integer.');
|
|
80
|
+
}
|
|
69
81
|
if (Number.isNaN(maxChunkConcurrency) ||
|
|
70
82
|
!Number.isInteger(maxChunkConcurrency) ||
|
|
71
|
-
maxChunkConcurrency < 1
|
|
72
|
-
|
|
83
|
+
maxChunkConcurrency < 1 ||
|
|
84
|
+
maxChunkConcurrency > absoluteMaxChunkConcurrency) {
|
|
85
|
+
throw new Error('Invalid max chunk concurrency. Must be an integer of at least 1 and at most 256.');
|
|
73
86
|
}
|
|
74
87
|
if (Number.isNaN(chunkByteCount) ||
|
|
75
88
|
!Number.isInteger(chunkByteCount) ||
|
|
@@ -92,8 +105,8 @@ class ChunkedUploader {
|
|
|
92
105
|
});
|
|
93
106
|
if (res.chunkSize !== this.chunkByteCount) {
|
|
94
107
|
this.logger.warn('Chunk size mismatch! Overriding with server value.', {
|
|
95
|
-
|
|
96
|
-
|
|
108
|
+
clientExpected: this.chunkByteCount,
|
|
109
|
+
serverReturned: res.chunkSize,
|
|
97
110
|
});
|
|
98
111
|
this.chunkByteCount = res.chunkSize;
|
|
99
112
|
}
|
|
@@ -137,11 +150,6 @@ class ChunkedUploader {
|
|
|
137
150
|
const chunkByteCount = chunk.length;
|
|
138
151
|
const chunkOffset = currentOffset;
|
|
139
152
|
currentOffset += chunkByteCount;
|
|
140
|
-
this.logger.debug('Queueing chunk', {
|
|
141
|
-
chunkPartNumber,
|
|
142
|
-
chunkOffset,
|
|
143
|
-
chunkByteCount,
|
|
144
|
-
});
|
|
145
153
|
const promise = limit(async () => {
|
|
146
154
|
if (firstError !== undefined) {
|
|
147
155
|
return;
|
|
@@ -197,26 +205,79 @@ class ChunkedUploader {
|
|
|
197
205
|
if (firstError !== undefined) {
|
|
198
206
|
throw firstError;
|
|
199
207
|
}
|
|
208
|
+
const finalizeResponse = await this.finalizeUpload(uploadId, dataItemByteCount, dataItemOpts?.paidBy, combinedSignal);
|
|
209
|
+
emitter.emit('upload-success');
|
|
210
|
+
return finalizeResponse;
|
|
211
|
+
}
|
|
212
|
+
toGiB(bytes) {
|
|
213
|
+
return bytes / 1024 ** 3;
|
|
214
|
+
}
|
|
215
|
+
async finalizeUpload(uploadId, dataItemByteCount, paidBy, signal) {
|
|
216
|
+
// Wait up to 1 minute per GiB of data for the upload to finalize
|
|
217
|
+
const fileSizeInGiB = Math.ceil(this.toGiB(dataItemByteCount));
|
|
218
|
+
const defaultMaxWaitTimeMins = fileSizeInGiB;
|
|
219
|
+
const maxWaitTimeMs = this.maxFinalizeMs ?? defaultMaxWaitTimeMins * 60 * 1000;
|
|
220
|
+
const minimumWaitPerStepMs =
|
|
221
|
+
// Per step, files smaller than 100MB will wait 2 second,
|
|
222
|
+
dataItemByteCount < 1024 * 1024 * 100
|
|
223
|
+
? 2000
|
|
224
|
+
: // files smaller than 3 GiB will wait 3 seconds,
|
|
225
|
+
dataItemByteCount < 1024 * 1024 * 1024 * 3
|
|
226
|
+
? 3000
|
|
227
|
+
: // and larger files will wait 1 second per GiB with max of 10 seconds
|
|
228
|
+
Math.max(1000 * fileSizeInGiB, 10000);
|
|
200
229
|
const paidByHeader = {};
|
|
201
|
-
if (
|
|
202
|
-
paidByHeader['x-paid-by'] = Array.isArray(
|
|
203
|
-
?
|
|
204
|
-
:
|
|
230
|
+
if (paidBy !== undefined) {
|
|
231
|
+
paidByHeader['x-paid-by'] = Array.isArray(paidBy)
|
|
232
|
+
? paidBy.join(',')
|
|
233
|
+
: paidBy;
|
|
205
234
|
}
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
const finalizeResponse = await this.http.post({
|
|
209
|
-
endpoint: `/chunks/${this.token}/${uploadId}/-1`,
|
|
235
|
+
await this.http.post({
|
|
236
|
+
endpoint: `/chunks/${this.token}/${uploadId}/finalize`,
|
|
210
237
|
data: Buffer.alloc(0),
|
|
211
238
|
headers: {
|
|
212
239
|
'Content-Type': 'application/octet-stream',
|
|
213
240
|
...paidByHeader,
|
|
214
241
|
...chunkingHeader,
|
|
215
242
|
},
|
|
216
|
-
signal
|
|
243
|
+
signal,
|
|
217
244
|
});
|
|
218
|
-
|
|
219
|
-
|
|
245
|
+
this.logger.debug(`Confirming upload to Turbo with uploadId ${uploadId} for up to ${defaultMaxWaitTimeMins} minutes.`);
|
|
246
|
+
const startTime = Date.now();
|
|
247
|
+
const cutoffTime = startTime + maxWaitTimeMs;
|
|
248
|
+
let attempts = 0;
|
|
249
|
+
while (Date.now() < cutoffTime) {
|
|
250
|
+
// Wait for 3/4 of the time remaining per attempt or minimum step
|
|
251
|
+
const waitTimeMs = Math.min(Math.floor((cutoffTime - Date.now()) * (3 / 4)), minimumWaitPerStepMs);
|
|
252
|
+
await (0, common_js_1.sleep)(waitTimeMs);
|
|
253
|
+
if (signal?.aborted) {
|
|
254
|
+
this.logger.warn(`Upload finalization aborted by signal.`);
|
|
255
|
+
throw new axios_1.CanceledError();
|
|
256
|
+
}
|
|
257
|
+
const response = await this.http.get({
|
|
258
|
+
endpoint: `/chunks/${this.token}/${uploadId}/status`,
|
|
259
|
+
signal,
|
|
260
|
+
});
|
|
261
|
+
this.logger.debug(`Upload status found: ${response.status}`, {
|
|
262
|
+
status: response.status,
|
|
263
|
+
attempts: attempts++,
|
|
264
|
+
maxWaitTimeMs,
|
|
265
|
+
minimumWaitPerStepMs,
|
|
266
|
+
waitTimeMs,
|
|
267
|
+
elapsedMs: Date.now() - startTime,
|
|
268
|
+
});
|
|
269
|
+
if (response.status === 'FINALIZED') {
|
|
270
|
+
this.logger.debug(`Upload finalized successfully.`);
|
|
271
|
+
return response.receipt;
|
|
272
|
+
}
|
|
273
|
+
if (response.status === 'UNDERFUNDED') {
|
|
274
|
+
throw new errors_js_1.FailedRequestError(`Insufficient balance`, 402);
|
|
275
|
+
}
|
|
276
|
+
if (types_js_1.multipartFailedStatus.includes(response.status)) {
|
|
277
|
+
throw new errors_js_1.FailedRequestError(`Upload failed with multi-part status ${response.status}`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
throw new Error(`Upload multi-part finalization has timed out for Upload ID ${uploadId}`);
|
|
220
281
|
}
|
|
221
282
|
}
|
|
222
283
|
exports.ChunkedUploader = ChunkedUploader;
|
package/lib/cjs/common/http.js
CHANGED
|
@@ -17,27 +17,34 @@ exports.TurboHTTPService = void 0;
|
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
19
|
const axios_1 = require("axios");
|
|
20
|
+
const node_stream_1 = require("node:stream");
|
|
20
21
|
const axiosClient_js_1 = require("../utils/axiosClient.js");
|
|
22
|
+
const common_js_1 = require("../utils/common.js");
|
|
21
23
|
const errors_js_1 = require("../utils/errors.js");
|
|
22
24
|
class TurboHTTPService {
|
|
23
|
-
constructor({ url, retryConfig, logger, }) {
|
|
25
|
+
constructor({ url, logger, retryConfig = (0, axiosClient_js_1.defaultRetryConfig)(logger), }) {
|
|
24
26
|
this.logger = logger;
|
|
25
27
|
this.axios = (0, axiosClient_js_1.createAxiosInstance)({
|
|
26
28
|
axiosConfig: {
|
|
27
29
|
baseURL: url,
|
|
28
30
|
maxRedirects: 0, // prevents backpressure issues when uploading larger streams via https
|
|
29
31
|
},
|
|
30
|
-
retryConfig,
|
|
31
32
|
logger: this.logger,
|
|
32
33
|
});
|
|
34
|
+
this.retryConfig = retryConfig;
|
|
33
35
|
}
|
|
34
36
|
async get({ endpoint, signal, allowedStatuses = [200, 202], headers, }) {
|
|
35
|
-
return this.
|
|
37
|
+
return this.retryRequest(() => this.axios.get(endpoint, { headers, signal }), allowedStatuses);
|
|
36
38
|
}
|
|
37
39
|
async post({ endpoint, signal, allowedStatuses = [200, 202], headers, data, }) {
|
|
38
40
|
// Buffer and Readable → keep Axios (streams work fine there)
|
|
39
41
|
if (!(data instanceof ReadableStream)) {
|
|
40
|
-
|
|
42
|
+
if (data instanceof node_stream_1.Readable) {
|
|
43
|
+
return this.tryRequest(
|
|
44
|
+
// Can't retry a Readable stream that has already been partially consumed
|
|
45
|
+
() => this.axios.post(endpoint, data, { headers, signal }), allowedStatuses);
|
|
46
|
+
}
|
|
47
|
+
return this.retryRequest(() => this.axios.post(endpoint, data, { headers, signal }), allowedStatuses);
|
|
41
48
|
}
|
|
42
49
|
// Browser ReadableStream → use fetch with progressive enhancement of duplex
|
|
43
50
|
// Note: fetch does not support streams in Safari and Firefox, so we convert to Blob
|
|
@@ -95,6 +102,34 @@ class TurboHTTPService {
|
|
|
95
102
|
throw error;
|
|
96
103
|
}
|
|
97
104
|
}
|
|
105
|
+
async retryRequest(request, allowedStatuses) {
|
|
106
|
+
let attempt = 0;
|
|
107
|
+
let lastError;
|
|
108
|
+
while (attempt < this.retryConfig.retries) {
|
|
109
|
+
try {
|
|
110
|
+
const resp = await this.tryRequest(request, allowedStatuses);
|
|
111
|
+
return resp;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
if (error instanceof errors_js_1.FailedRequestError) {
|
|
115
|
+
lastError = error;
|
|
116
|
+
this.retryConfig.onRetry(attempt + 1, error);
|
|
117
|
+
if (error.status !== undefined &&
|
|
118
|
+
error.status >= 400 &&
|
|
119
|
+
error.status < 500) {
|
|
120
|
+
// If it's a client error, we can stop retrying
|
|
121
|
+
throw error;
|
|
122
|
+
}
|
|
123
|
+
await (0, common_js_1.sleep)(this.retryConfig.retryDelay(attempt + 1));
|
|
124
|
+
attempt++;
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
throw new errors_js_1.FailedRequestError('Max retries reached - ' + lastError?.message, lastError?.status);
|
|
132
|
+
}
|
|
98
133
|
}
|
|
99
134
|
exports.TurboHTTPService = TurboHTTPService;
|
|
100
135
|
async function toFetchBody(data) {
|
|
@@ -17,13 +17,14 @@ exports.TurboAuthenticatedPaymentService = exports.TurboUnauthenticatedPaymentSe
|
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
19
|
const bignumber_js_1 = require("bignumber.js");
|
|
20
|
+
const axiosClient_js_1 = require("../utils/axiosClient.js");
|
|
20
21
|
const http_js_1 = require("./http.js");
|
|
21
22
|
const logger_js_1 = require("./logger.js");
|
|
22
23
|
const index_js_1 = require("./token/index.js");
|
|
23
24
|
exports.developmentPaymentServiceURL = 'https://payment.ardrive.dev';
|
|
24
25
|
exports.defaultPaymentServiceURL = 'https://payment.ardrive.io';
|
|
25
26
|
class TurboUnauthenticatedPaymentService {
|
|
26
|
-
constructor({ url = exports.defaultPaymentServiceURL,
|
|
27
|
+
constructor({ url = exports.defaultPaymentServiceURL, logger = logger_js_1.TurboWinstonLogger.default, retryConfig = (0, axiosClient_js_1.defaultRetryConfig)(logger), token = 'arweave', }) {
|
|
27
28
|
this.logger = logger;
|
|
28
29
|
this.httpService = new http_js_1.TurboHTTPService({
|
|
29
30
|
url: `${url}/v1`,
|
package/lib/cjs/common/upload.js
CHANGED
|
@@ -192,6 +192,7 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
|
|
|
192
192
|
logger: this.logger,
|
|
193
193
|
dataItemByteCount: dataItemSizeFactory(),
|
|
194
194
|
chunkingMode: params.chunkingMode,
|
|
195
|
+
maxFinalizeMs: params.maxFinalizeMs,
|
|
195
196
|
});
|
|
196
197
|
if (chunkedUploader.shouldUseChunkUploader) {
|
|
197
198
|
const response = await chunkedUploader.upload({
|
|
@@ -234,13 +235,10 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
|
|
|
234
235
|
resolve();
|
|
235
236
|
});
|
|
236
237
|
});
|
|
237
|
-
await Promise.race([
|
|
238
|
-
(0, common_js_1.sleep)(retryDelay(retries, error)),
|
|
239
|
-
abortEventPromise,
|
|
240
|
-
]);
|
|
238
|
+
await Promise.race([(0, common_js_1.sleep)(retryDelay(retries)), abortEventPromise]);
|
|
241
239
|
}
|
|
242
240
|
}
|
|
243
|
-
const msg = `Failed to upload file after ${
|
|
241
|
+
const msg = `Failed to upload file after ${retries + 1} attempts\n${lastError instanceof Error ? lastError.message : lastError}`;
|
|
244
242
|
// After all retries, throw the last error for catching
|
|
245
243
|
if (lastError instanceof errors_js_1.FailedRequestError) {
|
|
246
244
|
lastError.message = msg;
|
|
@@ -291,7 +289,7 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
|
|
|
291
289
|
*/
|
|
292
290
|
async uploadFolder(params) {
|
|
293
291
|
this.logger.debug('Uploading folder...', { params });
|
|
294
|
-
const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, } = params;
|
|
292
|
+
const { dataItemOpts, signal, manifestOptions = {}, maxConcurrentUploads = 1, throwOnFailure = true, maxChunkConcurrency, chunkByteCount, chunkingMode, maxFinalizeMs, } = params;
|
|
295
293
|
const { disableManifest, indexFile, fallbackFile } = manifestOptions;
|
|
296
294
|
const paths = {};
|
|
297
295
|
const response = {
|
|
@@ -365,6 +363,7 @@ class TurboAuthenticatedBaseUploadService extends TurboUnauthenticatedUploadServ
|
|
|
365
363
|
dataItemOpts: { ...dataItemOpts, tags: tagsWithManifestContentType },
|
|
366
364
|
chunkByteCount,
|
|
367
365
|
maxChunkConcurrency,
|
|
366
|
+
maxFinalizeMs,
|
|
368
367
|
chunkingMode,
|
|
369
368
|
});
|
|
370
369
|
return {
|
package/lib/cjs/types.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.validChunkingModes = exports.isJWK = exports.isWebUploadFolderParams = exports.isNodeUploadFolderParams = exports.tokenTypes = exports.fiatCurrencyTypes = void 0;
|
|
3
|
+
exports.validChunkingModes = exports.isJWK = exports.isWebUploadFolderParams = exports.isNodeUploadFolderParams = exports.multipartFinalizedStatus = exports.multipartFailedStatus = exports.multipartPendingStatus = exports.tokenTypes = exports.fiatCurrencyTypes = void 0;
|
|
4
4
|
exports.isCurrency = isCurrency;
|
|
5
5
|
exports.isKyvePrivateKey = isKyvePrivateKey;
|
|
6
6
|
exports.isEthPrivateKey = isEthPrivateKey;
|
|
@@ -31,6 +31,18 @@ exports.tokenTypes = [
|
|
|
31
31
|
'pol',
|
|
32
32
|
'base-eth',
|
|
33
33
|
];
|
|
34
|
+
exports.multipartPendingStatus = [
|
|
35
|
+
'ASSEMBLING',
|
|
36
|
+
'VALIDATING',
|
|
37
|
+
'FINALIZING',
|
|
38
|
+
];
|
|
39
|
+
exports.multipartFailedStatus = [
|
|
40
|
+
'UNDERFUNDED',
|
|
41
|
+
'INVALID',
|
|
42
|
+
'APPROVAL_FAILED',
|
|
43
|
+
'REVOKE_FAILED',
|
|
44
|
+
];
|
|
45
|
+
exports.multipartFinalizedStatus = ['FINALIZED'];
|
|
34
46
|
const isNodeUploadFolderParams = (p) => p.folderPath !== undefined;
|
|
35
47
|
exports.isNodeUploadFolderParams = isNodeUploadFolderParams;
|
|
36
48
|
const isWebUploadFolderParams = (p) => p.files !== undefined;
|
|
@@ -1,27 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
4
|
};
|
|
@@ -42,8 +19,7 @@ exports.createAxiosInstance = exports.defaultRetryConfig = exports.defaultReques
|
|
|
42
19
|
* See the License for the specific language governing permissions and
|
|
43
20
|
* limitations under the License.
|
|
44
21
|
*/
|
|
45
|
-
const axios_1 =
|
|
46
|
-
const axios_retry_1 = __importDefault(require("axios-retry"));
|
|
22
|
+
const axios_1 = __importDefault(require("axios"));
|
|
47
23
|
const logger_js_1 = require("../common/logger.js");
|
|
48
24
|
const version_js_1 = require("../version.js");
|
|
49
25
|
exports.defaultRequestHeaders = {
|
|
@@ -51,19 +27,14 @@ exports.defaultRequestHeaders = {
|
|
|
51
27
|
'x-turbo-source-identifier': 'turbo-sdk',
|
|
52
28
|
};
|
|
53
29
|
const defaultRetryConfig = (logger = logger_js_1.TurboWinstonLogger.default) => ({
|
|
54
|
-
retryDelay:
|
|
30
|
+
retryDelay: (retryCount) => Math.min(1000 * 2 ** (retryCount - 1), 30 * 1000), // exponential backoff up to 30s
|
|
55
31
|
retries: 5,
|
|
56
|
-
retryCondition: (error) => {
|
|
57
|
-
return (!(error instanceof axios_1.CanceledError) &&
|
|
58
|
-
axios_retry_1.default.isIdempotentRequestError(error) &&
|
|
59
|
-
axios_retry_1.default.isNetworkError(error));
|
|
60
|
-
},
|
|
61
32
|
onRetry: (retryCount, error) => {
|
|
62
33
|
logger.debug(`Request failed, ${error}. Retry attempt #${retryCount}...`);
|
|
63
34
|
},
|
|
64
35
|
});
|
|
65
36
|
exports.defaultRetryConfig = defaultRetryConfig;
|
|
66
|
-
const createAxiosInstance = ({
|
|
37
|
+
const createAxiosInstance = ({ axiosConfig = {}, } = {}) => {
|
|
67
38
|
const axiosInstance = axios_1.default.create({
|
|
68
39
|
...axiosConfig,
|
|
69
40
|
headers: {
|
|
@@ -73,11 +44,6 @@ const createAxiosInstance = ({ logger = logger_js_1.TurboWinstonLogger.default,
|
|
|
73
44
|
adapter: 'fetch',
|
|
74
45
|
validateStatus: () => true, // don't throw on non-200 status codes
|
|
75
46
|
});
|
|
76
|
-
if (retryConfig.retries !== undefined && retryConfig.retries > 0) {
|
|
77
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
78
|
-
// @ts-ignore
|
|
79
|
-
(0, axios_retry_1.default)(axiosInstance, retryConfig);
|
|
80
|
-
}
|
|
81
47
|
return axiosInstance;
|
|
82
48
|
};
|
|
83
49
|
exports.createAxiosInstance = createAxiosInstance;
|
package/lib/cjs/version.js
CHANGED
|
@@ -18,7 +18,7 @@ import { getTagsFromOptions, getUploadFolderOptions, paidByFromOptions, turboFro
|
|
|
18
18
|
export async function uploadFolder(options) {
|
|
19
19
|
const turbo = await turboFromOptions(options);
|
|
20
20
|
const paidBy = await paidByFromOptions(options, turbo);
|
|
21
|
-
const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, } = getUploadFolderOptions(options);
|
|
21
|
+
const { disableManifest, fallbackFile, folderPath, indexFile, maxConcurrentUploads, chunkByteCount, chunkingMode, maxChunkConcurrency, maxFinalizeMs, } = getUploadFolderOptions(options);
|
|
22
22
|
const customTags = getTagsFromOptions(options);
|
|
23
23
|
const result = await turbo.uploadFolder({
|
|
24
24
|
folderPath: folderPath,
|
|
@@ -32,6 +32,7 @@ export async function uploadFolder(options) {
|
|
|
32
32
|
chunkByteCount,
|
|
33
33
|
chunkingMode,
|
|
34
34
|
maxChunkConcurrency,
|
|
35
|
+
maxFinalizeMs,
|
|
35
36
|
});
|
|
36
37
|
console.log('Uploaded folder:', JSON.stringify(result, null, 2));
|
|
37
38
|
}
|
package/lib/esm/cli/options.js
CHANGED
|
@@ -160,6 +160,10 @@ export const optionMap = {
|
|
|
160
160
|
alias: '--max-chunk-concurrency <maxChunkConcurrency>',
|
|
161
161
|
description: 'Maximum number of concurrent chunks to upload per file',
|
|
162
162
|
},
|
|
163
|
+
maxFinalizeMs: {
|
|
164
|
+
alias: '--max-finalize-ms <maxFinalizeMs>',
|
|
165
|
+
description: 'Maximum time in milliseconds to wait for the finalization of all chunks after the last chunk is uploaded. Defaults to 1 minute per GiB of the total file size.',
|
|
166
|
+
},
|
|
163
167
|
chunkByteCount: {
|
|
164
168
|
alias: '--chunk-byte-count <chunkByteCount>',
|
|
165
169
|
description: 'Size of each chunk in bytes',
|
|
@@ -193,6 +197,7 @@ export const uploadOptions = [
|
|
|
193
197
|
optionMap.useSignerBalanceFirst,
|
|
194
198
|
optionMap.tags,
|
|
195
199
|
optionMap.maxChunkConcurrency,
|
|
200
|
+
optionMap.maxFinalizeMs,
|
|
196
201
|
optionMap.chunkByteCount,
|
|
197
202
|
optionMap.chunkingMode,
|
|
198
203
|
];
|
package/lib/esm/cli/utils.js
CHANGED
|
@@ -271,5 +271,6 @@ export function getChunkingOptions(options) {
|
|
|
271
271
|
maxChunkConcurrency: options.maxChunkConcurrency !== undefined
|
|
272
272
|
? +options.maxChunkConcurrency
|
|
273
273
|
: undefined,
|
|
274
|
+
maxFinalizeMs: options.maxFinalizeMs !== undefined ? +options.maxFinalizeMs : undefined,
|
|
274
275
|
};
|
|
275
276
|
}
|