bytifi 0.1.5 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -4
- package/bin/bytifi.js +21 -6
- package/lib/crypto.js +59 -54
- package/lib/decrypt.js +87 -7
- package/lib/upload.js +102 -13
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -55,10 +55,12 @@ Prefer the environment variable — keys on the command line can appear in shell
|
|
|
55
55
|
bytifi upload ./photo.png
|
|
56
56
|
bytifi upload "./my video (1).mp4"
|
|
57
57
|
bytifi upload ./report.pdf --expires 60 --delete-on-download
|
|
58
|
-
bytifi upload ./
|
|
58
|
+
bytifi upload ./logs.txt --concurrency 8 --json > upload.json
|
|
59
59
|
bytifi upload ./photo.png -q
|
|
60
60
|
```
|
|
61
61
|
|
|
62
|
+
All uploads are gzip-compressed per chunk before encryption. Files over **10 MB** use multipart upload automatically.
|
|
63
|
+
|
|
62
64
|
Upload accepts **one file at a time**. Quote paths that contain spaces. Avoid shell globs like `**` — your shell may expand them into dozens of paths.
|
|
63
65
|
|
|
64
66
|
### Decrypt from a link
|
|
@@ -126,6 +128,7 @@ Note: with `npm exec`, put `--` before the file path so npm does not swallow `--
|
|
|
126
128
|
| `-q, --quiet` | Print only the share URL |
|
|
127
129
|
| `--verbose` | Print API error details to stderr |
|
|
128
130
|
| `--mime-type` | Override detected MIME type |
|
|
131
|
+
| `--concurrency` | Parallel encrypt/upload workers, 1–16 (default: `4`) |
|
|
129
132
|
| `--base-url` | API base URL (default: `https://bytifi.com`) |
|
|
130
133
|
|
|
131
134
|
### Decrypt options
|
|
@@ -146,7 +149,7 @@ Note: with `npm exec`, put `--` before the file path so npm does not swallow `--
|
|
|
146
149
|
|
|
147
150
|
Exit codes: `0` success, `1` usage error, `2` API error, `3` network error.
|
|
148
151
|
|
|
149
|
-
JSON output (`--json`) for upload includes `shareUrl`, `encryptedFile`, `link`, `encryptionToken`, `clientEncryptionMeta`, and `expiresAt`.
|
|
152
|
+
JSON output (`--json`) for upload includes `shareUrl`, `encryptedFile`, `link`, `encryptionToken`, `clientEncryptionMeta`, `compression`, and `expiresAt`.
|
|
150
153
|
|
|
151
154
|
JSON output for decrypt includes `outputPath`, `originalName`, `size`, `mimeType`, `expiresAt`, `link`, `storageMode`, and `sourcePath` (for local decrypt).
|
|
152
155
|
|
|
@@ -156,10 +159,23 @@ Files over ~100 MB encrypted use multipart upload automatically. Progress prints
|
|
|
156
159
|
|
|
157
160
|
**Upload**
|
|
158
161
|
|
|
159
|
-
1.
|
|
160
|
-
2. Uploads encrypted bytes via
|
|
162
|
+
1. Gzip-compresses each chunk, then encrypts locally with AES-GCM (meta `version: 2`)
|
|
163
|
+
2. Uploads encrypted bytes via multipart pipeline for files >10 MB
|
|
161
164
|
3. Prints a share URL including `#token=...`
|
|
162
165
|
|
|
166
|
+
## Compatibility
|
|
167
|
+
|
|
168
|
+
| Scenario | Works? |
|
|
169
|
+
|----------|--------|
|
|
170
|
+
| Old links (no compression) | Yes — everywhere |
|
|
171
|
+
| New CLI upload + new CLI decrypt | Yes |
|
|
172
|
+
| New CLI upload + **updated website** decrypt | Yes — **deploy Bytifi-Website** after upgrading CLI |
|
|
173
|
+
| New CLI upload + old website (not deployed) | **Broken in browser** — garbled download |
|
|
174
|
+
| New CLI upload + old CLI (≤0.1.5) decrypt | **Broken** — upgrade CLI |
|
|
175
|
+
| Website upload (no compression) | Yes — unchanged |
|
|
176
|
+
|
|
177
|
+
Compressed files larger than one chunk use **part-based storage**; decrypt via share link or CLI part download.
|
|
178
|
+
|
|
163
179
|
**Decrypt**
|
|
164
180
|
|
|
165
181
|
1. Reads link metadata from `/api/link/:token`, from `--upload-json`, or from `--meta`
|
package/bin/bytifi.js
CHANGED
|
@@ -26,6 +26,7 @@ Upload options:
|
|
|
26
26
|
-q, --quiet Print only the share URL
|
|
27
27
|
--verbose Show API error details on stderr
|
|
28
28
|
--mime-type <type> Override detected MIME type
|
|
29
|
+
--concurrency <n> Parallel part workers (default: 4)
|
|
29
30
|
--base-url <url> API base URL (default: https://bytifi.com)
|
|
30
31
|
|
|
31
32
|
Decrypt options:
|
|
@@ -52,15 +53,16 @@ Exit codes:
|
|
|
52
53
|
3 network error
|
|
53
54
|
|
|
54
55
|
Examples:
|
|
56
|
+
bytifi --version
|
|
57
|
+
export BYTIFI_API_KEY=usk_your_key
|
|
58
|
+
|
|
55
59
|
bytifi upload ./photo.png
|
|
56
|
-
bytifi upload ./
|
|
60
|
+
bytifi upload "./my report.pdf" --expires 60 --delete-on-download
|
|
61
|
+
bytifi upload ./logs.txt --concurrency 8 --json > upload.json
|
|
57
62
|
bytifi upload ./large.iso -q
|
|
58
63
|
|
|
59
|
-
bytifi decrypt 'https://bytifi.com/link?link=
|
|
60
|
-
bytifi decrypt
|
|
61
|
-
|
|
62
|
-
bytifi decrypt ./downloaded.encrypted --upload-json upload.json
|
|
63
|
-
bytifi decrypt "./my file(1).mp4" --link abc --token 'ENCRYPTION_TOKEN'
|
|
64
|
+
bytifi decrypt 'https://bytifi.com/link?link=LINK#token=KEY'
|
|
65
|
+
bytifi decrypt ./downloaded.bin --upload-json upload.json -o ./restored.bin
|
|
64
66
|
`)
|
|
65
67
|
}
|
|
66
68
|
|
|
@@ -82,6 +84,7 @@ function parseUploadArgs(argv) {
|
|
|
82
84
|
quiet: false,
|
|
83
85
|
verbose: false,
|
|
84
86
|
mimeType: '',
|
|
87
|
+
concurrency: 4,
|
|
85
88
|
baseUrl: 'https://bytifi.com',
|
|
86
89
|
help: false,
|
|
87
90
|
version: false,
|
|
@@ -143,6 +146,17 @@ function parseUploadArgs(argv) {
|
|
|
143
146
|
continue
|
|
144
147
|
}
|
|
145
148
|
|
|
149
|
+
if (arg === '--concurrency') {
|
|
150
|
+
const raw = readFlagValue(argv, index, arg)
|
|
151
|
+
const concurrency = Number(raw)
|
|
152
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16) {
|
|
153
|
+
throw new Error('concurrency must be an integer between 1 and 16.')
|
|
154
|
+
}
|
|
155
|
+
options.concurrency = concurrency
|
|
156
|
+
index += 1
|
|
157
|
+
continue
|
|
158
|
+
}
|
|
159
|
+
|
|
146
160
|
if (arg === '--base-url') {
|
|
147
161
|
options.baseUrl = readFlagValue(argv, index, arg)
|
|
148
162
|
index += 1
|
|
@@ -312,6 +326,7 @@ async function runUpload(filePath, options) {
|
|
|
312
326
|
expiresInMinutes: options.expiresInMinutes,
|
|
313
327
|
deleteOnDownload: options.deleteOnDownload,
|
|
314
328
|
mimeType: options.mimeType || undefined,
|
|
329
|
+
concurrency: options.concurrency,
|
|
315
330
|
signal: abortController.signal,
|
|
316
331
|
onProgress: showProgress
|
|
317
332
|
? (percent) => {
|
package/lib/crypto.js
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
import crypto from 'node:crypto'
|
|
2
2
|
import fs from 'node:fs/promises'
|
|
3
3
|
import path from 'node:path'
|
|
4
|
+
import zlib from 'node:zlib'
|
|
5
|
+
import { promisify } from 'node:util'
|
|
4
6
|
import { fromBase64Url, toBase64Url } from './base64url.js'
|
|
5
7
|
|
|
8
|
+
const gzipAsync = promisify(zlib.gzip)
|
|
9
|
+
const gunzipAsync = promisify(zlib.gunzip)
|
|
10
|
+
|
|
6
11
|
export const ENCRYPTED_PART_SIZE = 32 * 1024 * 1024
|
|
7
12
|
export const PLAIN_CHUNK_SIZE = ENCRYPTED_PART_SIZE - 16
|
|
8
13
|
export const DIRECT_UPLOAD_LIMIT_BYTES = 100 * 1024 * 1024
|
|
14
|
+
export const MULTIPART_THRESHOLD_BYTES = 10 * 1024 * 1024
|
|
9
15
|
|
|
10
16
|
export function buildChunkIv(noncePrefix, chunkIndex) {
|
|
11
17
|
const iv = Buffer.alloc(12)
|
|
@@ -14,10 +20,39 @@ export function buildChunkIv(noncePrefix, chunkIndex) {
|
|
|
14
20
|
return iv
|
|
15
21
|
}
|
|
16
22
|
|
|
17
|
-
export function
|
|
23
|
+
export function resolveCompressionMode() {
|
|
24
|
+
return 'gzip'
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function usesChunkCompression(meta) {
|
|
28
|
+
return meta?.compression?.algorithm === 'gzip'
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function normalizeCompressionMeta(rawCompression) {
|
|
32
|
+
if (!rawCompression || typeof rawCompression !== 'object') {
|
|
33
|
+
return { algorithm: 'none', scope: 'chunk' }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const algorithm = String(rawCompression.algorithm || 'none').toLowerCase()
|
|
37
|
+
if (algorithm === 'gzip') {
|
|
38
|
+
return { algorithm: 'gzip', scope: 'chunk' }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return { algorithm: 'none', scope: 'chunk' }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function compressPlainChunk(plainChunk) {
|
|
45
|
+
return gzipAsync(plainChunk)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function decompressPlainChunk(compressedChunk) {
|
|
49
|
+
return gunzipAsync(compressedChunk)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function encryptChunk(payloadChunk, key, noncePrefix, chunkIndex) {
|
|
18
53
|
const iv = buildChunkIv(noncePrefix, chunkIndex)
|
|
19
54
|
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv)
|
|
20
|
-
const encrypted = Buffer.concat([cipher.update(
|
|
55
|
+
const encrypted = Buffer.concat([cipher.update(payloadChunk), cipher.final()])
|
|
21
56
|
const tag = cipher.getAuthTag()
|
|
22
57
|
return Buffer.concat([encrypted, tag])
|
|
23
58
|
}
|
|
@@ -30,8 +65,9 @@ export function buildClientEncryptionMeta({
|
|
|
30
65
|
mimeType,
|
|
31
66
|
}) {
|
|
32
67
|
return {
|
|
33
|
-
version:
|
|
68
|
+
version: 2,
|
|
34
69
|
algorithm: 'AES-GCM',
|
|
70
|
+
compression: { algorithm: 'gzip', scope: 'chunk' },
|
|
35
71
|
chunkSize: plainChunkSize,
|
|
36
72
|
chunkCount,
|
|
37
73
|
noncePrefix: toBase64Url(noncePrefix),
|
|
@@ -84,6 +120,7 @@ export function createEncryptionContext({
|
|
|
84
120
|
mimeType,
|
|
85
121
|
originalSize,
|
|
86
122
|
plainChunkSize,
|
|
123
|
+
compression: 'gzip',
|
|
87
124
|
}
|
|
88
125
|
}
|
|
89
126
|
|
|
@@ -129,54 +166,9 @@ export async function encryptChunkFromFile(fileHandle, chunkIndex, context) {
|
|
|
129
166
|
context.plainChunkSize,
|
|
130
167
|
)
|
|
131
168
|
|
|
132
|
-
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export async function collectEncryptedParts(filePath, context) {
|
|
136
|
-
const fileHandle = await fs.open(filePath, 'r')
|
|
137
|
-
const encryptedParts = []
|
|
138
|
-
|
|
139
|
-
try {
|
|
140
|
-
for (let chunkIndex = 0; chunkIndex < context.chunkCount; chunkIndex += 1) {
|
|
141
|
-
encryptedParts.push(await encryptChunkFromFile(fileHandle, chunkIndex, context))
|
|
142
|
-
}
|
|
143
|
-
} finally {
|
|
144
|
-
await fileHandle.close()
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
return encryptedParts
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
export async function encryptFileBuffer(fileBuffer, {
|
|
151
|
-
originalName = 'upload',
|
|
152
|
-
mimeType = 'application/octet-stream',
|
|
153
|
-
tokenBytes = crypto.randomBytes(32),
|
|
154
|
-
noncePrefix = crypto.randomBytes(8),
|
|
155
|
-
plainChunkSize = PLAIN_CHUNK_SIZE,
|
|
156
|
-
} = {}) {
|
|
157
|
-
const context = createEncryptionContext({
|
|
158
|
-
originalSize: fileBuffer.length,
|
|
159
|
-
originalName,
|
|
160
|
-
mimeType,
|
|
161
|
-
tokenBytes,
|
|
162
|
-
noncePrefix,
|
|
163
|
-
plainChunkSize,
|
|
164
|
-
})
|
|
169
|
+
let payload = await compressPlainChunk(plainChunk)
|
|
165
170
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
for (let chunkIndex = 0; chunkIndex < context.chunkCount; chunkIndex += 1) {
|
|
169
|
-
const start = chunkIndex * plainChunkSize
|
|
170
|
-
const end = Math.min(fileBuffer.length, start + plainChunkSize)
|
|
171
|
-
const plainChunk = fileBuffer.subarray(start, end)
|
|
172
|
-
encryptedParts.push(encryptChunk(plainChunk, context.tokenBytes, context.noncePrefix, chunkIndex))
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
return {
|
|
176
|
-
...context,
|
|
177
|
-
encryptedParts,
|
|
178
|
-
encryptedBuffer: Buffer.concat(encryptedParts),
|
|
179
|
-
}
|
|
171
|
+
return encryptChunk(payload, context.tokenBytes, context.noncePrefix, chunkIndex)
|
|
180
172
|
}
|
|
181
173
|
|
|
182
174
|
export function decryptChunk(encryptedChunk, tokenBytes, noncePrefix, chunkIndex) {
|
|
@@ -192,6 +184,16 @@ export function decryptChunk(encryptedChunk, tokenBytes, noncePrefix, chunkIndex
|
|
|
192
184
|
return Buffer.concat([decipher.update(ciphertext), decipher.final()])
|
|
193
185
|
}
|
|
194
186
|
|
|
187
|
+
export async function decryptPlainChunkFromEncrypted(encryptedChunk, tokenBytes, noncePrefix, chunkIndex, meta) {
|
|
188
|
+
const payload = decryptChunk(encryptedChunk, tokenBytes, noncePrefix, chunkIndex)
|
|
189
|
+
|
|
190
|
+
if (!usesChunkCompression(meta)) {
|
|
191
|
+
return payload
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return decompressPlainChunk(payload)
|
|
195
|
+
}
|
|
196
|
+
|
|
195
197
|
export function importToken(token) {
|
|
196
198
|
const tokenBytes = fromBase64Url(token)
|
|
197
199
|
if (tokenBytes.length !== 32) {
|
|
@@ -215,6 +217,7 @@ export function normalizeClientEncryptionMeta(rawMeta) {
|
|
|
215
217
|
return {
|
|
216
218
|
version: Number(rawMeta.version) || 1,
|
|
217
219
|
algorithm: String(rawMeta.algorithm || 'AES-GCM'),
|
|
220
|
+
compression: normalizeCompressionMeta(rawMeta.compression),
|
|
218
221
|
chunkSize,
|
|
219
222
|
chunkCount,
|
|
220
223
|
noncePrefix: rawMeta.noncePrefix,
|
|
@@ -226,20 +229,22 @@ export function normalizeClientEncryptionMeta(rawMeta) {
|
|
|
226
229
|
export function buildEncryptedChunkPlan(meta) {
|
|
227
230
|
const chunks = []
|
|
228
231
|
let offset = 0
|
|
232
|
+
const variableEncryptedSizes = usesChunkCompression(meta)
|
|
229
233
|
|
|
230
234
|
for (let chunkIndex = 0; chunkIndex < meta.chunkCount; chunkIndex += 1) {
|
|
231
235
|
const start = chunkIndex * meta.chunkSize
|
|
232
236
|
const plainSize = Math.min(meta.originalSize - start, meta.chunkSize)
|
|
233
|
-
const encryptedSize = plainSize + 16
|
|
237
|
+
const encryptedSize = variableEncryptedSizes ? null : plainSize + 16
|
|
234
238
|
|
|
235
|
-
chunks.push({ chunkIndex, encryptedSize, plainSize })
|
|
236
|
-
offset += encryptedSize
|
|
239
|
+
chunks.push({ chunkIndex, encryptedSize, plainSize, variableEncryptedSize: variableEncryptedSizes })
|
|
240
|
+
offset += encryptedSize || 0
|
|
237
241
|
}
|
|
238
242
|
|
|
239
243
|
return {
|
|
240
244
|
chunks,
|
|
241
|
-
totalEncryptedSize: offset,
|
|
245
|
+
totalEncryptedSize: variableEncryptedSizes ? null : offset,
|
|
242
246
|
totalPlainSize: meta.originalSize,
|
|
247
|
+
variableEncryptedSizes,
|
|
243
248
|
}
|
|
244
249
|
}
|
|
245
250
|
|
package/lib/decrypt.js
CHANGED
|
@@ -3,9 +3,10 @@ import path from 'node:path'
|
|
|
3
3
|
import process from 'node:process'
|
|
4
4
|
import {
|
|
5
5
|
buildEncryptedChunkPlan,
|
|
6
|
-
|
|
6
|
+
decryptPlainChunkFromEncrypted,
|
|
7
7
|
importToken,
|
|
8
8
|
normalizeClientEncryptionMeta,
|
|
9
|
+
usesChunkCompression,
|
|
9
10
|
} from './crypto.js'
|
|
10
11
|
import { fromBase64Url } from './base64url.js'
|
|
11
12
|
import { BytifiApiError, BytifiNetworkError } from './upload.js'
|
|
@@ -249,7 +250,13 @@ async function decryptFromParts({
|
|
|
249
250
|
|
|
250
251
|
const chunk = chunks[index]
|
|
251
252
|
const encryptedPart = await fetchEncryptedPart(baseUrl, linkToken, chunk.chunkIndex + 1, signal)
|
|
252
|
-
const plainPart =
|
|
253
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
254
|
+
encryptedPart,
|
|
255
|
+
tokenBytes,
|
|
256
|
+
noncePrefix,
|
|
257
|
+
chunk.chunkIndex,
|
|
258
|
+
meta,
|
|
259
|
+
)
|
|
253
260
|
|
|
254
261
|
await fileHandle.write(plainPart)
|
|
255
262
|
decryptedBytes += plainPart.length
|
|
@@ -269,7 +276,6 @@ async function decryptFromSingleFile({
|
|
|
269
276
|
onProgress,
|
|
270
277
|
signal,
|
|
271
278
|
}) {
|
|
272
|
-
const { chunks, totalEncryptedSize } = buildEncryptedChunkPlan(meta)
|
|
273
279
|
const response = await fetch(encryptedFileUrl, { signal })
|
|
274
280
|
|
|
275
281
|
if (!response.ok) {
|
|
@@ -280,6 +286,28 @@ async function decryptFromSingleFile({
|
|
|
280
286
|
})
|
|
281
287
|
}
|
|
282
288
|
|
|
289
|
+
if (usesChunkCompression(meta)) {
|
|
290
|
+
if (meta.chunkCount !== 1) {
|
|
291
|
+
throw new Error(
|
|
292
|
+
'Compressed files with multiple chunks require part-based download. Open the share link in a browser or use part files.',
|
|
293
|
+
)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const encryptedBuffer = Buffer.from(await response.arrayBuffer())
|
|
297
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
298
|
+
encryptedBuffer,
|
|
299
|
+
tokenBytes,
|
|
300
|
+
noncePrefix,
|
|
301
|
+
0,
|
|
302
|
+
meta,
|
|
303
|
+
)
|
|
304
|
+
await fs.writeFile(outputPath, plainPart)
|
|
305
|
+
onProgress?.(100)
|
|
306
|
+
return
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const { chunks, totalEncryptedSize } = buildEncryptedChunkPlan(meta)
|
|
310
|
+
|
|
283
311
|
if (!response.body) {
|
|
284
312
|
const encryptedBuffer = Buffer.from(await response.arrayBuffer())
|
|
285
313
|
await writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, noncePrefix, outputPath, onProgress, signal)
|
|
@@ -342,7 +370,13 @@ async function decryptFromSingleFile({
|
|
|
342
370
|
throw new Error('Encrypted file ended before all parts were downloaded.')
|
|
343
371
|
}
|
|
344
372
|
|
|
345
|
-
const plainPart =
|
|
373
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
374
|
+
encryptedChunk,
|
|
375
|
+
tokenBytes,
|
|
376
|
+
noncePrefix,
|
|
377
|
+
chunk.chunkIndex,
|
|
378
|
+
meta,
|
|
379
|
+
)
|
|
346
380
|
await fileHandle.write(plainPart)
|
|
347
381
|
decryptedBytes += plainPart.length
|
|
348
382
|
nextChunkIndex += 1
|
|
@@ -357,6 +391,23 @@ async function decryptFromSingleFile({
|
|
|
357
391
|
}
|
|
358
392
|
|
|
359
393
|
async function writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, noncePrefix, outputPath, onProgress, signal) {
|
|
394
|
+
if (usesChunkCompression(meta)) {
|
|
395
|
+
if (chunks.length !== 1) {
|
|
396
|
+
throw new Error('Compressed files with multiple chunks require part-based storage.')
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
400
|
+
encryptedBuffer,
|
|
401
|
+
tokenBytes,
|
|
402
|
+
noncePrefix,
|
|
403
|
+
0,
|
|
404
|
+
meta,
|
|
405
|
+
)
|
|
406
|
+
await fs.writeFile(outputPath, plainPart)
|
|
407
|
+
onProgress?.(100)
|
|
408
|
+
return
|
|
409
|
+
}
|
|
410
|
+
|
|
360
411
|
const fileHandle = await fs.open(outputPath, 'w')
|
|
361
412
|
let offset = 0
|
|
362
413
|
let decryptedBytes = 0
|
|
@@ -372,7 +423,13 @@ async function writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, n
|
|
|
372
423
|
throw new Error('Encrypted file ended before all parts were downloaded.')
|
|
373
424
|
}
|
|
374
425
|
|
|
375
|
-
const plainPart =
|
|
426
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
427
|
+
encryptedChunk,
|
|
428
|
+
tokenBytes,
|
|
429
|
+
noncePrefix,
|
|
430
|
+
chunk.chunkIndex,
|
|
431
|
+
meta,
|
|
432
|
+
)
|
|
376
433
|
await fileHandle.write(plainPart)
|
|
377
434
|
offset += chunk.encryptedSize
|
|
378
435
|
decryptedBytes += plainPart.length
|
|
@@ -395,6 +452,17 @@ async function decryptFromLocalSingleFile({
|
|
|
395
452
|
signal,
|
|
396
453
|
}) {
|
|
397
454
|
const { chunks } = buildEncryptedChunkPlan(meta)
|
|
455
|
+
|
|
456
|
+
if (usesChunkCompression(meta)) {
|
|
457
|
+
if (meta.chunkCount !== 1) {
|
|
458
|
+
throw new Error('Compressed multi-chunk files must be decrypted from a parts directory.')
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const encryptedBuffer = await fs.readFile(encryptedFilePath)
|
|
462
|
+
await writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, noncePrefix, outputPath, onProgress, signal)
|
|
463
|
+
return
|
|
464
|
+
}
|
|
465
|
+
|
|
398
466
|
const stat = await fs.stat(encryptedFilePath)
|
|
399
467
|
|
|
400
468
|
if (stat.size <= 64 * 1024 * 1024) {
|
|
@@ -421,7 +489,13 @@ async function decryptFromLocalSingleFile({
|
|
|
421
489
|
throw new Error('Encrypted file ended before all parts were read.')
|
|
422
490
|
}
|
|
423
491
|
|
|
424
|
-
const plainPart =
|
|
492
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
493
|
+
encryptedChunk,
|
|
494
|
+
tokenBytes,
|
|
495
|
+
noncePrefix,
|
|
496
|
+
chunk.chunkIndex,
|
|
497
|
+
meta,
|
|
498
|
+
)
|
|
425
499
|
await outputHandle.write(plainPart)
|
|
426
500
|
offset += chunk.encryptedSize
|
|
427
501
|
decryptedBytes += plainPart.length
|
|
@@ -485,7 +559,13 @@ async function decryptFromLocalParts({
|
|
|
485
559
|
}
|
|
486
560
|
|
|
487
561
|
const encryptedPart = await fs.readFile(path.join(partsDirectory, partEntry.name))
|
|
488
|
-
const plainPart =
|
|
562
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
563
|
+
encryptedPart,
|
|
564
|
+
tokenBytes,
|
|
565
|
+
noncePrefix,
|
|
566
|
+
chunk.chunkIndex,
|
|
567
|
+
meta,
|
|
568
|
+
)
|
|
489
569
|
|
|
490
570
|
await fileHandle.write(plainPart)
|
|
491
571
|
decryptedBytes += plainPart.length
|
package/lib/upload.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
MULTIPART_THRESHOLD_BYTES,
|
|
3
3
|
encryptChunkFromFile,
|
|
4
4
|
resolveUploadFile,
|
|
5
5
|
} from './crypto.js'
|
|
@@ -132,6 +132,70 @@ function buildResult(payload, context, shareUrl) {
|
|
|
132
132
|
expiresAt: payload.expiresAt,
|
|
133
133
|
deleteOnDownload: payload.deleteOnDownload,
|
|
134
134
|
clientEncrypted: payload.clientEncrypted,
|
|
135
|
+
compression: context.compression,
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function createBoundedQueue(maxSize) {
|
|
140
|
+
const items = []
|
|
141
|
+
const waiters = []
|
|
142
|
+
let closed = false
|
|
143
|
+
|
|
144
|
+
function notifyWaiters() {
|
|
145
|
+
while (waiters.length > 0) {
|
|
146
|
+
const resolve = waiters.shift()
|
|
147
|
+
resolve()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
async push(item, signal) {
|
|
153
|
+
while (items.length >= maxSize) {
|
|
154
|
+
if (signal?.aborted) {
|
|
155
|
+
throw new Error('Upload aborted.')
|
|
156
|
+
}
|
|
157
|
+
await new Promise((resolve, reject) => {
|
|
158
|
+
const onAbort = () => {
|
|
159
|
+
const index = waiters.indexOf(resolve)
|
|
160
|
+
if (index >= 0) waiters.splice(index, 1)
|
|
161
|
+
reject(new Error('Upload aborted.'))
|
|
162
|
+
}
|
|
163
|
+
waiters.push(resolve)
|
|
164
|
+
signal?.addEventListener('abort', onAbort, { once: true })
|
|
165
|
+
})
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
items.push(item)
|
|
169
|
+
notifyWaiters()
|
|
170
|
+
},
|
|
171
|
+
async take(signal) {
|
|
172
|
+
while (items.length === 0) {
|
|
173
|
+
if (closed) {
|
|
174
|
+
return null
|
|
175
|
+
}
|
|
176
|
+
if (signal?.aborted) {
|
|
177
|
+
throw new Error('Upload aborted.')
|
|
178
|
+
}
|
|
179
|
+
await new Promise((resolve, reject) => {
|
|
180
|
+
const onAbort = () => {
|
|
181
|
+
const index = waiters.indexOf(resolve)
|
|
182
|
+
if (index >= 0) waiters.splice(index, 1)
|
|
183
|
+
reject(new Error('Upload aborted.'))
|
|
184
|
+
}
|
|
185
|
+
waiters.push(resolve)
|
|
186
|
+
signal?.addEventListener('abort', onAbort, { once: true })
|
|
187
|
+
})
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return items.shift()
|
|
191
|
+
},
|
|
192
|
+
close() {
|
|
193
|
+
closed = true
|
|
194
|
+
notifyWaiters()
|
|
195
|
+
},
|
|
196
|
+
get pending() {
|
|
197
|
+
return items.length
|
|
198
|
+
},
|
|
135
199
|
}
|
|
136
200
|
}
|
|
137
201
|
|
|
@@ -221,7 +285,7 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
221
285
|
signal,
|
|
222
286
|
concurrency = DEFAULT_CONCURRENCY,
|
|
223
287
|
}) {
|
|
224
|
-
const
|
|
288
|
+
const maxPartSize = context.meta.chunkSize + 16
|
|
225
289
|
const initPayload = await apiFetchWithRetry(baseUrl, '/api/public/upload/init', {
|
|
226
290
|
apiKey,
|
|
227
291
|
method: 'POST',
|
|
@@ -233,7 +297,7 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
233
297
|
originalSize: context.originalSize,
|
|
234
298
|
clientEncrypted: true,
|
|
235
299
|
clientEncryptionMeta: context.meta,
|
|
236
|
-
partSize,
|
|
300
|
+
partSize: maxPartSize,
|
|
237
301
|
expiresInMinutes,
|
|
238
302
|
deleteOnDownload,
|
|
239
303
|
}),
|
|
@@ -242,37 +306,58 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
242
306
|
|
|
243
307
|
const sessionToken = initPayload.sessionToken
|
|
244
308
|
const workerCount = Math.min(
|
|
245
|
-
Number(initPayload.concurrency) ||
|
|
309
|
+
Number(concurrency) || Number(initPayload.concurrency) || DEFAULT_CONCURRENCY,
|
|
246
310
|
context.chunkCount,
|
|
247
311
|
)
|
|
312
|
+
const queue = createBoundedQueue(Math.max(2, workerCount * 2))
|
|
248
313
|
const fileHandle = await fs.open(filePath, 'r')
|
|
249
314
|
let nextChunkIndex = 0
|
|
250
315
|
let completedParts = 0
|
|
251
316
|
|
|
252
|
-
|
|
317
|
+
function claimChunkIndex() {
|
|
318
|
+
const chunkIndex = nextChunkIndex
|
|
319
|
+
nextChunkIndex += 1
|
|
320
|
+
return chunkIndex
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async function encryptWorker() {
|
|
253
324
|
while (true) {
|
|
254
325
|
if (signal?.aborted) {
|
|
255
326
|
throw new Error('Upload aborted.')
|
|
256
327
|
}
|
|
257
328
|
|
|
258
|
-
const chunkIndex =
|
|
259
|
-
nextChunkIndex += 1
|
|
260
|
-
|
|
329
|
+
const chunkIndex = claimChunkIndex()
|
|
261
330
|
if (chunkIndex >= context.chunkCount) {
|
|
262
331
|
return
|
|
263
332
|
}
|
|
264
333
|
|
|
265
|
-
const partNumber = chunkIndex + 1
|
|
266
334
|
const encryptedPart = await encryptChunkFromFile(fileHandle, chunkIndex, context)
|
|
335
|
+
await queue.push({
|
|
336
|
+
partNumber: chunkIndex + 1,
|
|
337
|
+
body: encryptedPart,
|
|
338
|
+
}, signal)
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async function uploadWorker() {
|
|
343
|
+
while (true) {
|
|
344
|
+
if (signal?.aborted) {
|
|
345
|
+
throw new Error('Upload aborted.')
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const part = await queue.take(signal)
|
|
349
|
+
if (!part) {
|
|
350
|
+
return
|
|
351
|
+
}
|
|
267
352
|
|
|
268
353
|
await apiFetchWithRetry(
|
|
269
354
|
baseUrl,
|
|
270
|
-
`/api/public/upload/part?sessionToken=${encodeURIComponent(sessionToken)}&partNumber=${partNumber}`,
|
|
355
|
+
`/api/public/upload/part?sessionToken=${encodeURIComponent(sessionToken)}&partNumber=${part.partNumber}`,
|
|
271
356
|
{
|
|
272
357
|
apiKey,
|
|
273
358
|
method: 'PUT',
|
|
274
359
|
headers: { 'Content-Type': 'application/octet-stream' },
|
|
275
|
-
body:
|
|
360
|
+
body: part.body,
|
|
276
361
|
signal,
|
|
277
362
|
},
|
|
278
363
|
)
|
|
@@ -283,7 +368,11 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
283
368
|
}
|
|
284
369
|
|
|
285
370
|
try {
|
|
286
|
-
|
|
371
|
+
const uploadWorkers = Array.from({ length: workerCount }, () => uploadWorker())
|
|
372
|
+
const encryptWorkers = Array.from({ length: workerCount }, () => encryptWorker())
|
|
373
|
+
await Promise.all(encryptWorkers)
|
|
374
|
+
queue.close()
|
|
375
|
+
await Promise.all(uploadWorkers)
|
|
287
376
|
} catch (error) {
|
|
288
377
|
await apiFetchWithRetry(baseUrl, '/api/public/upload/abort', {
|
|
289
378
|
apiKey,
|
|
@@ -323,7 +412,7 @@ export async function uploadFile(filePath, options) {
|
|
|
323
412
|
mimeType: options.mimeType,
|
|
324
413
|
})
|
|
325
414
|
|
|
326
|
-
if (context.
|
|
415
|
+
if (context.originalSize <= MULTIPART_THRESHOLD_BYTES) {
|
|
327
416
|
const encryptedBuffer = await collectEncryptedBuffer(absolutePath, context, {
|
|
328
417
|
onProgress: options.onProgress,
|
|
329
418
|
signal: options.signal,
|