bytifi 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/bytifi.js +7 -2
- package/lib/upload.js +16 -90
- package/package.json +1 -1
package/bin/bytifi.js
CHANGED
|
@@ -291,6 +291,10 @@ function writeProgress(label, percent) {
|
|
|
291
291
|
process.stderr.write(`\r${label}: ${percent}%`)
|
|
292
292
|
}
|
|
293
293
|
|
|
294
|
+
function finishProgressLine() {
|
|
295
|
+
process.stderr.write('\n')
|
|
296
|
+
}
|
|
297
|
+
|
|
294
298
|
function clearProgressLine() {
|
|
295
299
|
process.stderr.write('\r\x1b[K')
|
|
296
300
|
}
|
|
@@ -342,7 +346,7 @@ async function runUpload(filePath, options) {
|
|
|
342
346
|
})
|
|
343
347
|
|
|
344
348
|
if (showProgress) {
|
|
345
|
-
|
|
349
|
+
finishProgressLine()
|
|
346
350
|
}
|
|
347
351
|
|
|
348
352
|
if (options.json) {
|
|
@@ -399,7 +403,7 @@ async function runDecrypt(input, options) {
|
|
|
399
403
|
})
|
|
400
404
|
|
|
401
405
|
if (showProgress) {
|
|
402
|
-
|
|
406
|
+
finishProgressLine()
|
|
403
407
|
}
|
|
404
408
|
|
|
405
409
|
if (options.json) {
|
|
@@ -531,6 +535,7 @@ async function main() {
|
|
|
531
535
|
|
|
532
536
|
main().catch((error) => {
|
|
533
537
|
clearProgressLine()
|
|
538
|
+
finishProgressLine()
|
|
534
539
|
const verbose = process.argv.includes('--verbose')
|
|
535
540
|
printError(error, verbose)
|
|
536
541
|
process.exit(exitCodeForError(error))
|
package/lib/upload.js
CHANGED
|
@@ -136,69 +136,6 @@ function buildResult(payload, context, shareUrl) {
|
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
-
function createBoundedQueue(maxSize) {
|
|
140
|
-
const items = []
|
|
141
|
-
const waiters = []
|
|
142
|
-
let closed = false
|
|
143
|
-
|
|
144
|
-
function notifyWaiters() {
|
|
145
|
-
while (waiters.length > 0) {
|
|
146
|
-
const resolve = waiters.shift()
|
|
147
|
-
resolve()
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
return {
|
|
152
|
-
async push(item, signal) {
|
|
153
|
-
while (items.length >= maxSize) {
|
|
154
|
-
if (signal?.aborted) {
|
|
155
|
-
throw new Error('Upload aborted.')
|
|
156
|
-
}
|
|
157
|
-
await new Promise((resolve, reject) => {
|
|
158
|
-
const onAbort = () => {
|
|
159
|
-
const index = waiters.indexOf(resolve)
|
|
160
|
-
if (index >= 0) waiters.splice(index, 1)
|
|
161
|
-
reject(new Error('Upload aborted.'))
|
|
162
|
-
}
|
|
163
|
-
waiters.push(resolve)
|
|
164
|
-
signal?.addEventListener('abort', onAbort, { once: true })
|
|
165
|
-
})
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
items.push(item)
|
|
169
|
-
notifyWaiters()
|
|
170
|
-
},
|
|
171
|
-
async take(signal) {
|
|
172
|
-
while (items.length === 0) {
|
|
173
|
-
if (closed) {
|
|
174
|
-
return null
|
|
175
|
-
}
|
|
176
|
-
if (signal?.aborted) {
|
|
177
|
-
throw new Error('Upload aborted.')
|
|
178
|
-
}
|
|
179
|
-
await new Promise((resolve, reject) => {
|
|
180
|
-
const onAbort = () => {
|
|
181
|
-
const index = waiters.indexOf(resolve)
|
|
182
|
-
if (index >= 0) waiters.splice(index, 1)
|
|
183
|
-
reject(new Error('Upload aborted.'))
|
|
184
|
-
}
|
|
185
|
-
waiters.push(resolve)
|
|
186
|
-
signal?.addEventListener('abort', onAbort, { once: true })
|
|
187
|
-
})
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
return items.shift()
|
|
191
|
-
},
|
|
192
|
-
close() {
|
|
193
|
-
closed = true
|
|
194
|
-
notifyWaiters()
|
|
195
|
-
},
|
|
196
|
-
get pending() {
|
|
197
|
-
return items.length
|
|
198
|
-
},
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
139
|
async function collectEncryptedBuffer(filePath, context, { onProgress, signal } = {}) {
|
|
203
140
|
const fileHandle = await fs.open(filePath, 'r')
|
|
204
141
|
const encryptedParts = []
|
|
@@ -276,6 +213,17 @@ async function pollUploadStatus(sessionToken, { apiKey, baseUrl, signal }) {
|
|
|
276
213
|
}
|
|
277
214
|
}
|
|
278
215
|
|
|
216
|
+
function resolveUploadConcurrency(originalSize, requested) {
|
|
217
|
+
const requestedValue = Number(requested)
|
|
218
|
+
if (Number.isFinite(requestedValue) && requestedValue > 0) {
|
|
219
|
+
return Math.min(16, Math.max(1, Math.floor(requestedValue)))
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (originalSize >= 3 * 1024 * 1024 * 1024) return 2
|
|
223
|
+
if (originalSize >= 1024 * 1024 * 1024) return 3
|
|
224
|
+
return DEFAULT_CONCURRENCY
|
|
225
|
+
}
|
|
226
|
+
|
|
279
227
|
async function uploadMultipartStreaming(filePath, context, {
|
|
280
228
|
apiKey,
|
|
281
229
|
baseUrl,
|
|
@@ -306,10 +254,9 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
306
254
|
|
|
307
255
|
const sessionToken = initPayload.sessionToken
|
|
308
256
|
const workerCount = Math.min(
|
|
309
|
-
|
|
257
|
+
resolveUploadConcurrency(context.originalSize, concurrency || initPayload.concurrency),
|
|
310
258
|
context.chunkCount,
|
|
311
259
|
)
|
|
312
|
-
const queue = createBoundedQueue(Math.max(2, workerCount * 2))
|
|
313
260
|
const fileHandle = await fs.open(filePath, 'r')
|
|
314
261
|
let nextChunkIndex = 0
|
|
315
262
|
let completedParts = 0
|
|
@@ -320,7 +267,7 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
320
267
|
return chunkIndex
|
|
321
268
|
}
|
|
322
269
|
|
|
323
|
-
async function
|
|
270
|
+
async function pipelineWorker() {
|
|
324
271
|
while (true) {
|
|
325
272
|
if (signal?.aborted) {
|
|
326
273
|
throw new Error('Upload aborted.')
|
|
@@ -332,32 +279,15 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
332
279
|
}
|
|
333
280
|
|
|
334
281
|
const encryptedPart = await encryptChunkFromFile(fileHandle, chunkIndex, context)
|
|
335
|
-
await queue.push({
|
|
336
|
-
partNumber: chunkIndex + 1,
|
|
337
|
-
body: encryptedPart,
|
|
338
|
-
}, signal)
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
async function uploadWorker() {
|
|
343
|
-
while (true) {
|
|
344
|
-
if (signal?.aborted) {
|
|
345
|
-
throw new Error('Upload aborted.')
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
const part = await queue.take(signal)
|
|
349
|
-
if (!part) {
|
|
350
|
-
return
|
|
351
|
-
}
|
|
352
282
|
|
|
353
283
|
await apiFetchWithRetry(
|
|
354
284
|
baseUrl,
|
|
355
|
-
`/api/public/upload/part?sessionToken=${encodeURIComponent(sessionToken)}&partNumber=${
|
|
285
|
+
`/api/public/upload/part?sessionToken=${encodeURIComponent(sessionToken)}&partNumber=${chunkIndex + 1}`,
|
|
356
286
|
{
|
|
357
287
|
apiKey,
|
|
358
288
|
method: 'PUT',
|
|
359
289
|
headers: { 'Content-Type': 'application/octet-stream' },
|
|
360
|
-
body:
|
|
290
|
+
body: encryptedPart,
|
|
361
291
|
signal,
|
|
362
292
|
},
|
|
363
293
|
)
|
|
@@ -368,11 +298,7 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
368
298
|
}
|
|
369
299
|
|
|
370
300
|
try {
|
|
371
|
-
|
|
372
|
-
const encryptWorkers = Array.from({ length: workerCount }, () => encryptWorker())
|
|
373
|
-
await Promise.all(encryptWorkers)
|
|
374
|
-
queue.close()
|
|
375
|
-
await Promise.all(uploadWorkers)
|
|
301
|
+
await Promise.all(Array.from({ length: workerCount }, () => pipelineWorker()))
|
|
376
302
|
} catch (error) {
|
|
377
303
|
await apiFetchWithRetry(baseUrl, '/api/public/upload/abort', {
|
|
378
304
|
apiKey,
|