bytifi 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -8
- package/bin/bytifi.js +25 -1
- package/lib/crypto.js +120 -53
- package/lib/decrypt.js +91 -11
- package/lib/upload.js +104 -14
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -55,10 +55,12 @@ Prefer the environment variable — keys on the command line can appear in shell
|
|
|
55
55
|
bytifi upload ./photo.png
|
|
56
56
|
bytifi upload "./my video (1).mp4"
|
|
57
57
|
bytifi upload ./report.pdf --expires 60 --delete-on-download
|
|
58
|
-
bytifi upload ./
|
|
58
|
+
bytifi upload ./logs.txt --compress auto --concurrency 8 --json > upload.json
|
|
59
59
|
bytifi upload ./photo.png -q
|
|
60
60
|
```
|
|
61
61
|
|
|
62
|
+
Files over **10 MB** use multipart upload automatically (lower memory use). Compression defaults to **`auto`**: gzip for text-like files, skipped for video/images/zip.
|
|
63
|
+
|
|
62
64
|
Upload accepts **one file at a time**. Quote paths that contain spaces. Avoid shell globs like `**` — your shell may expand them into dozens of paths.
|
|
63
65
|
|
|
64
66
|
### Decrypt from a link
|
|
@@ -90,7 +92,7 @@ Bytifi uses two different values — don't swap them:
|
|
|
90
92
|
|
|
91
93
|
| Name | Upload JSON field | Example location |
|
|
92
94
|
|------|-------------------|------------------|
|
|
93
|
-
| **Link ID** | `
|
|
95
|
+
| **Link ID** | `link` | `/f/QeVuslvdaP-okMxG`, `link?link=QeVuslvdaP-okMxG` |
|
|
94
96
|
| **Encryption token** | `encryptionToken` | `#token=2LTlmBrDkO4GJg0...` in `shareUrl` |
|
|
95
97
|
|
|
96
98
|
- `--link` = link ID (short, ~16 chars)
|
|
@@ -126,6 +128,8 @@ Note: with `npm exec`, put `--` before the file path so npm does not swallow `--
|
|
|
126
128
|
| `-q, --quiet` | Print only the share URL |
|
|
127
129
|
| `--verbose` | Print API error details to stderr |
|
|
128
130
|
| `--mime-type` | Override detected MIME type |
|
|
131
|
+
| `--compress` | Compression mode: `auto`, `gzip`, or `off` (default: `auto`) |
|
|
132
|
+
| `--concurrency` | Parallel encrypt/upload workers, 1–16 (default: `4`) |
|
|
129
133
|
| `--base-url` | API base URL (default: `https://bytifi.com`) |
|
|
130
134
|
|
|
131
135
|
### Decrypt options
|
|
@@ -133,7 +137,7 @@ Note: with `npm exec`, put `--` before the file path so npm does not swallow `--
|
|
|
133
137
|
| Flag | Description |
|
|
134
138
|
|------|-------------|
|
|
135
139
|
| `--token` | Encryption key from `#token=...` (`encryptionToken` in upload JSON) |
|
|
136
|
-
| `--link` | Link ID from upload JSON `
|
|
140
|
+
| `--link` | Link ID from upload JSON `link` field (`/f/LINK_ID`) |
|
|
137
141
|
| `--upload-json` | Upload `--json` output file (recommended for downloaded files) |
|
|
138
142
|
| `--meta` | Saved `clientEncryptionMeta` JSON for offline decrypt |
|
|
139
143
|
| `--share-url` | Share URL to read token/metadata while decrypting a local file |
|
|
@@ -146,9 +150,9 @@ Note: with `npm exec`, put `--` before the file path so npm does not swallow `--
|
|
|
146
150
|
|
|
147
151
|
Exit codes: `0` success, `1` usage error, `2` API error, `3` network error.
|
|
148
152
|
|
|
149
|
-
JSON output (`--json`) for upload includes `shareUrl`, `encryptedFile`, `encryptionToken`, `clientEncryptionMeta`, `
|
|
153
|
+
JSON output (`--json`) for upload includes `shareUrl`, `encryptedFile`, `link`, `encryptionToken`, `clientEncryptionMeta`, `compression`, and `expiresAt`.
|
|
150
154
|
|
|
151
|
-
JSON output for decrypt includes `outputPath`, `originalName`, `size`, `mimeType`, `expiresAt`, `
|
|
155
|
+
JSON output for decrypt includes `outputPath`, `originalName`, `size`, `mimeType`, `expiresAt`, `link`, `storageMode`, and `sourcePath` (for local decrypt).
|
|
152
156
|
|
|
153
157
|
Files over ~100 MB encrypted use multipart upload automatically. Progress prints to stderr unless `--json` or `--quiet` is set.
|
|
154
158
|
|
|
@@ -156,9 +160,23 @@ Files over ~100 MB encrypted use multipart upload automatically. Progress prints
|
|
|
156
160
|
|
|
157
161
|
**Upload**
|
|
158
162
|
|
|
159
|
-
1.
|
|
160
|
-
2.
|
|
161
|
-
3.
|
|
163
|
+
1. Optionally gzip-compress each chunk (`--compress auto|gzip|off`)
|
|
164
|
+
2. Encrypts locally with AES-GCM (same format as the website, meta `version: 2` when compressed)
|
|
165
|
+
3. Uploads encrypted bytes via multipart pipeline for files >10 MB
|
|
166
|
+
4. Prints a share URL including `#token=...`
|
|
167
|
+
|
|
168
|
+
## Compatibility
|
|
169
|
+
|
|
170
|
+
| Scenario | Works? |
|
|
171
|
+
|----------|--------|
|
|
172
|
+
| Old links (no compression) | Yes — everywhere |
|
|
173
|
+
| New CLI upload + new CLI decrypt | Yes |
|
|
174
|
+
| New CLI upload + **updated website** decrypt | Yes — **deploy Bytifi-Website** after upgrading CLI |
|
|
175
|
+
| New CLI upload + old website (not deployed) | **Broken in browser** — garbled download |
|
|
176
|
+
| New CLI upload + old CLI (≤0.1.5) decrypt | **Broken** — upgrade CLI |
|
|
177
|
+
| Website upload (no compression) | Yes — unchanged |
|
|
178
|
+
|
|
179
|
+
Compressed files larger than one chunk use **part-based storage**; decrypt via share link or CLI part download.
|
|
162
180
|
|
|
163
181
|
**Decrypt**
|
|
164
182
|
|
package/bin/bytifi.js
CHANGED
|
@@ -26,11 +26,13 @@ Upload options:
|
|
|
26
26
|
-q, --quiet Print only the share URL
|
|
27
27
|
--verbose Show API error details on stderr
|
|
28
28
|
--mime-type <type> Override detected MIME type
|
|
29
|
+
--compress <mode> Compression: auto|gzip|off (default: auto)
|
|
30
|
+
--concurrency <n> Parallel part workers (default: 4)
|
|
29
31
|
--base-url <url> API base URL (default: https://bytifi.com)
|
|
30
32
|
|
|
31
33
|
Decrypt options:
|
|
32
34
|
--token <token> Encryption key from #token=... (not the link ID)
|
|
33
|
-
--link <
|
|
35
|
+
--link <id> Link ID from upload JSON "link" field (/f/LINK_ID)
|
|
34
36
|
--upload-json <path> Upload --json output (easiest for downloaded files)
|
|
35
37
|
--meta <path> Saved clientEncryptionMeta JSON (offline decrypt)
|
|
36
38
|
--share-url <url> Share URL for token/metadata when decrypting a local file
|
|
@@ -54,6 +56,7 @@ Exit codes:
|
|
|
54
56
|
Examples:
|
|
55
57
|
bytifi upload ./photo.png
|
|
56
58
|
bytifi upload ./video.mp4 --expires 60 --json > upload.json
|
|
59
|
+
bytifi upload ./logs.tar --compress gzip --concurrency 8
|
|
57
60
|
bytifi upload ./large.iso -q
|
|
58
61
|
|
|
59
62
|
bytifi decrypt 'https://bytifi.com/link?link=abc#token=...'
|
|
@@ -82,6 +85,8 @@ function parseUploadArgs(argv) {
|
|
|
82
85
|
quiet: false,
|
|
83
86
|
verbose: false,
|
|
84
87
|
mimeType: '',
|
|
88
|
+
compressionMode: 'auto',
|
|
89
|
+
concurrency: 4,
|
|
85
90
|
baseUrl: 'https://bytifi.com',
|
|
86
91
|
help: false,
|
|
87
92
|
version: false,
|
|
@@ -143,6 +148,23 @@ function parseUploadArgs(argv) {
|
|
|
143
148
|
continue
|
|
144
149
|
}
|
|
145
150
|
|
|
151
|
+
if (arg === '--compress') {
|
|
152
|
+
options.compressionMode = readFlagValue(argv, index, arg)
|
|
153
|
+
index += 1
|
|
154
|
+
continue
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (arg === '--concurrency') {
|
|
158
|
+
const raw = readFlagValue(argv, index, arg)
|
|
159
|
+
const concurrency = Number(raw)
|
|
160
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16) {
|
|
161
|
+
throw new Error('concurrency must be an integer between 1 and 16.')
|
|
162
|
+
}
|
|
163
|
+
options.concurrency = concurrency
|
|
164
|
+
index += 1
|
|
165
|
+
continue
|
|
166
|
+
}
|
|
167
|
+
|
|
146
168
|
if (arg === '--base-url') {
|
|
147
169
|
options.baseUrl = readFlagValue(argv, index, arg)
|
|
148
170
|
index += 1
|
|
@@ -312,6 +334,8 @@ async function runUpload(filePath, options) {
|
|
|
312
334
|
expiresInMinutes: options.expiresInMinutes,
|
|
313
335
|
deleteOnDownload: options.deleteOnDownload,
|
|
314
336
|
mimeType: options.mimeType || undefined,
|
|
337
|
+
compressionMode: options.compressionMode,
|
|
338
|
+
concurrency: options.concurrency,
|
|
315
339
|
signal: abortController.signal,
|
|
316
340
|
onProgress: showProgress
|
|
317
341
|
? (percent) => {
|
package/lib/crypto.js
CHANGED
|
@@ -1,11 +1,30 @@
|
|
|
1
1
|
import crypto from 'node:crypto'
|
|
2
2
|
import fs from 'node:fs/promises'
|
|
3
3
|
import path from 'node:path'
|
|
4
|
+
import zlib from 'node:zlib'
|
|
5
|
+
import { promisify } from 'node:util'
|
|
4
6
|
import { fromBase64Url, toBase64Url } from './base64url.js'
|
|
5
7
|
|
|
8
|
+
const gzipAsync = promisify(zlib.gzip)
|
|
9
|
+
const gunzipAsync = promisify(zlib.gunzip)
|
|
10
|
+
|
|
6
11
|
export const ENCRYPTED_PART_SIZE = 32 * 1024 * 1024
|
|
7
12
|
export const PLAIN_CHUNK_SIZE = ENCRYPTED_PART_SIZE - 16
|
|
8
13
|
export const DIRECT_UPLOAD_LIMIT_BYTES = 100 * 1024 * 1024
|
|
14
|
+
export const MULTIPART_THRESHOLD_BYTES = 10 * 1024 * 1024
|
|
15
|
+
|
|
16
|
+
const SKIP_COMPRESS_EXTENSIONS = new Set([
|
|
17
|
+
'.7z', '.avi', '.br', '.bz2', '.gif', '.gz', '.jpeg', '.jpg', '.mkv', '.mov',
|
|
18
|
+
'.mp3', '.mp4', '.png', '.rar', '.webp', '.woff', '.woff2', '.xz', '.zip', '.zst', '.iso',
|
|
19
|
+
])
|
|
20
|
+
|
|
21
|
+
const SKIP_COMPRESS_MIME_EXACT = new Set([
|
|
22
|
+
'application/gzip',
|
|
23
|
+
'application/pdf',
|
|
24
|
+
'application/x-7z-compressed',
|
|
25
|
+
'application/x-rar-compressed',
|
|
26
|
+
'application/zip',
|
|
27
|
+
])
|
|
9
28
|
|
|
10
29
|
export function buildChunkIv(noncePrefix, chunkIndex) {
|
|
11
30
|
const iv = Buffer.alloc(12)
|
|
@@ -14,10 +33,75 @@ export function buildChunkIv(noncePrefix, chunkIndex) {
|
|
|
14
33
|
return iv
|
|
15
34
|
}
|
|
16
35
|
|
|
17
|
-
export function
|
|
36
|
+
export function resolveCompressionMode(requested, mimeType, originalName) {
|
|
37
|
+
const mode = String(requested || 'auto').toLowerCase()
|
|
38
|
+
|
|
39
|
+
if (mode === 'off' || mode === 'none') {
|
|
40
|
+
return 'none'
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (mode === 'gzip') {
|
|
44
|
+
return 'gzip'
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (mode === 'auto') {
|
|
48
|
+
return shouldSkipAutoCompression(mimeType, originalName) ? 'none' : 'gzip'
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
throw new Error('compress must be one of: auto, gzip, off')
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function shouldSkipAutoCompression(mimeType, originalName) {
|
|
55
|
+
const extension = path.extname(String(originalName || '')).toLowerCase()
|
|
56
|
+
if (SKIP_COMPRESS_EXTENSIONS.has(extension)) {
|
|
57
|
+
return true
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const normalizedMime = String(mimeType || '').toLowerCase()
|
|
61
|
+
if (SKIP_COMPRESS_MIME_EXACT.has(normalizedMime)) {
|
|
62
|
+
return true
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (normalizedMime.startsWith('video/') || normalizedMime.startsWith('audio/')) {
|
|
66
|
+
return true
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (normalizedMime.startsWith('image/')) {
|
|
70
|
+
return true
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return false
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function usesChunkCompression(meta) {
|
|
77
|
+
return meta?.compression?.algorithm === 'gzip'
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function normalizeCompressionMeta(rawCompression) {
|
|
81
|
+
if (!rawCompression || typeof rawCompression !== 'object') {
|
|
82
|
+
return { algorithm: 'none', scope: 'chunk' }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const algorithm = String(rawCompression.algorithm || 'none').toLowerCase()
|
|
86
|
+
if (algorithm === 'gzip') {
|
|
87
|
+
return { algorithm: 'gzip', scope: 'chunk' }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return { algorithm: 'none', scope: 'chunk' }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export async function compressPlainChunk(plainChunk) {
|
|
94
|
+
return gzipAsync(plainChunk)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export async function decompressPlainChunk(compressedChunk) {
|
|
98
|
+
return gunzipAsync(compressedChunk)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export function encryptChunk(payloadChunk, key, noncePrefix, chunkIndex) {
|
|
18
102
|
const iv = buildChunkIv(noncePrefix, chunkIndex)
|
|
19
103
|
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv)
|
|
20
|
-
const encrypted = Buffer.concat([cipher.update(
|
|
104
|
+
const encrypted = Buffer.concat([cipher.update(payloadChunk), cipher.final()])
|
|
21
105
|
const tag = cipher.getAuthTag()
|
|
22
106
|
return Buffer.concat([encrypted, tag])
|
|
23
107
|
}
|
|
@@ -28,10 +112,17 @@ export function buildClientEncryptionMeta({
|
|
|
28
112
|
noncePrefix,
|
|
29
113
|
originalSize,
|
|
30
114
|
mimeType,
|
|
115
|
+
compression = 'none',
|
|
31
116
|
}) {
|
|
117
|
+
const compressionMeta = normalizeCompressionMeta({
|
|
118
|
+
algorithm: compression === 'gzip' ? 'gzip' : 'none',
|
|
119
|
+
scope: 'chunk',
|
|
120
|
+
})
|
|
121
|
+
|
|
32
122
|
return {
|
|
33
|
-
version: 1,
|
|
123
|
+
version: compressionMeta.algorithm === 'gzip' ? 2 : 1,
|
|
34
124
|
algorithm: 'AES-GCM',
|
|
125
|
+
compression: compressionMeta,
|
|
35
126
|
chunkSize: plainChunkSize,
|
|
36
127
|
chunkCount,
|
|
37
128
|
noncePrefix: toBase64Url(noncePrefix),
|
|
@@ -60,10 +151,12 @@ export function createEncryptionContext({
|
|
|
60
151
|
tokenBytes = crypto.randomBytes(32),
|
|
61
152
|
noncePrefix = crypto.randomBytes(8),
|
|
62
153
|
plainChunkSize = PLAIN_CHUNK_SIZE,
|
|
154
|
+
compressionMode = 'auto',
|
|
63
155
|
}) {
|
|
64
156
|
if (tokenBytes.length !== 32) throw new Error('Encryption token must be 32 bytes.')
|
|
65
157
|
if (noncePrefix.length !== 8) throw new Error('Nonce prefix must be 8 bytes.')
|
|
66
158
|
|
|
159
|
+
const compression = resolveCompressionMode(compressionMode, mimeType, originalName)
|
|
67
160
|
const { chunkCount, encryptedSize } = calculateEncryptedSize(originalSize, plainChunkSize)
|
|
68
161
|
const meta = buildClientEncryptionMeta({
|
|
69
162
|
plainChunkSize,
|
|
@@ -71,6 +164,7 @@ export function createEncryptionContext({
|
|
|
71
164
|
noncePrefix,
|
|
72
165
|
originalSize,
|
|
73
166
|
mimeType,
|
|
167
|
+
compression,
|
|
74
168
|
})
|
|
75
169
|
|
|
76
170
|
return {
|
|
@@ -84,6 +178,7 @@ export function createEncryptionContext({
|
|
|
84
178
|
mimeType,
|
|
85
179
|
originalSize,
|
|
86
180
|
plainChunkSize,
|
|
181
|
+
compression,
|
|
87
182
|
}
|
|
88
183
|
}
|
|
89
184
|
|
|
@@ -104,6 +199,7 @@ export async function resolveUploadFile(filePath, options = {}) {
|
|
|
104
199
|
originalSize: stat.size,
|
|
105
200
|
originalName,
|
|
106
201
|
mimeType,
|
|
202
|
+
compressionMode: options.compressionMode,
|
|
107
203
|
}),
|
|
108
204
|
}
|
|
109
205
|
}
|
|
@@ -129,54 +225,12 @@ export async function encryptChunkFromFile(fileHandle, chunkIndex, context) {
|
|
|
129
225
|
context.plainChunkSize,
|
|
130
226
|
)
|
|
131
227
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
export async function collectEncryptedParts(filePath, context) {
|
|
136
|
-
const fileHandle = await fs.open(filePath, 'r')
|
|
137
|
-
const encryptedParts = []
|
|
138
|
-
|
|
139
|
-
try {
|
|
140
|
-
for (let chunkIndex = 0; chunkIndex < context.chunkCount; chunkIndex += 1) {
|
|
141
|
-
encryptedParts.push(await encryptChunkFromFile(fileHandle, chunkIndex, context))
|
|
142
|
-
}
|
|
143
|
-
} finally {
|
|
144
|
-
await fileHandle.close()
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
return encryptedParts
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
export async function encryptFileBuffer(fileBuffer, {
|
|
151
|
-
originalName = 'upload',
|
|
152
|
-
mimeType = 'application/octet-stream',
|
|
153
|
-
tokenBytes = crypto.randomBytes(32),
|
|
154
|
-
noncePrefix = crypto.randomBytes(8),
|
|
155
|
-
plainChunkSize = PLAIN_CHUNK_SIZE,
|
|
156
|
-
} = {}) {
|
|
157
|
-
const context = createEncryptionContext({
|
|
158
|
-
originalSize: fileBuffer.length,
|
|
159
|
-
originalName,
|
|
160
|
-
mimeType,
|
|
161
|
-
tokenBytes,
|
|
162
|
-
noncePrefix,
|
|
163
|
-
plainChunkSize,
|
|
164
|
-
})
|
|
165
|
-
|
|
166
|
-
const encryptedParts = []
|
|
167
|
-
|
|
168
|
-
for (let chunkIndex = 0; chunkIndex < context.chunkCount; chunkIndex += 1) {
|
|
169
|
-
const start = chunkIndex * plainChunkSize
|
|
170
|
-
const end = Math.min(fileBuffer.length, start + plainChunkSize)
|
|
171
|
-
const plainChunk = fileBuffer.subarray(start, end)
|
|
172
|
-
encryptedParts.push(encryptChunk(plainChunk, context.tokenBytes, context.noncePrefix, chunkIndex))
|
|
228
|
+
let payload = plainChunk
|
|
229
|
+
if (context.compression === 'gzip') {
|
|
230
|
+
payload = await compressPlainChunk(plainChunk)
|
|
173
231
|
}
|
|
174
232
|
|
|
175
|
-
return
|
|
176
|
-
...context,
|
|
177
|
-
encryptedParts,
|
|
178
|
-
encryptedBuffer: Buffer.concat(encryptedParts),
|
|
179
|
-
}
|
|
233
|
+
return encryptChunk(payload, context.tokenBytes, context.noncePrefix, chunkIndex)
|
|
180
234
|
}
|
|
181
235
|
|
|
182
236
|
export function decryptChunk(encryptedChunk, tokenBytes, noncePrefix, chunkIndex) {
|
|
@@ -192,6 +246,16 @@ export function decryptChunk(encryptedChunk, tokenBytes, noncePrefix, chunkIndex
|
|
|
192
246
|
return Buffer.concat([decipher.update(ciphertext), decipher.final()])
|
|
193
247
|
}
|
|
194
248
|
|
|
249
|
+
export async function decryptPlainChunkFromEncrypted(encryptedChunk, tokenBytes, noncePrefix, chunkIndex, meta) {
|
|
250
|
+
const payload = decryptChunk(encryptedChunk, tokenBytes, noncePrefix, chunkIndex)
|
|
251
|
+
|
|
252
|
+
if (!usesChunkCompression(meta)) {
|
|
253
|
+
return payload
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return decompressPlainChunk(payload)
|
|
257
|
+
}
|
|
258
|
+
|
|
195
259
|
export function importToken(token) {
|
|
196
260
|
const tokenBytes = fromBase64Url(token)
|
|
197
261
|
if (tokenBytes.length !== 32) {
|
|
@@ -215,6 +279,7 @@ export function normalizeClientEncryptionMeta(rawMeta) {
|
|
|
215
279
|
return {
|
|
216
280
|
version: Number(rawMeta.version) || 1,
|
|
217
281
|
algorithm: String(rawMeta.algorithm || 'AES-GCM'),
|
|
282
|
+
compression: normalizeCompressionMeta(rawMeta.compression),
|
|
218
283
|
chunkSize,
|
|
219
284
|
chunkCount,
|
|
220
285
|
noncePrefix: rawMeta.noncePrefix,
|
|
@@ -226,20 +291,22 @@ export function normalizeClientEncryptionMeta(rawMeta) {
|
|
|
226
291
|
export function buildEncryptedChunkPlan(meta) {
|
|
227
292
|
const chunks = []
|
|
228
293
|
let offset = 0
|
|
294
|
+
const variableEncryptedSizes = usesChunkCompression(meta)
|
|
229
295
|
|
|
230
296
|
for (let chunkIndex = 0; chunkIndex < meta.chunkCount; chunkIndex += 1) {
|
|
231
297
|
const start = chunkIndex * meta.chunkSize
|
|
232
298
|
const plainSize = Math.min(meta.originalSize - start, meta.chunkSize)
|
|
233
|
-
const encryptedSize = plainSize + 16
|
|
299
|
+
const encryptedSize = variableEncryptedSizes ? null : plainSize + 16
|
|
234
300
|
|
|
235
|
-
chunks.push({ chunkIndex, encryptedSize, plainSize })
|
|
236
|
-
offset += encryptedSize
|
|
301
|
+
chunks.push({ chunkIndex, encryptedSize, plainSize, variableEncryptedSize: variableEncryptedSizes })
|
|
302
|
+
offset += encryptedSize || 0
|
|
237
303
|
}
|
|
238
304
|
|
|
239
305
|
return {
|
|
240
306
|
chunks,
|
|
241
|
-
totalEncryptedSize: offset,
|
|
307
|
+
totalEncryptedSize: variableEncryptedSizes ? null : offset,
|
|
242
308
|
totalPlainSize: meta.originalSize,
|
|
309
|
+
variableEncryptedSizes,
|
|
243
310
|
}
|
|
244
311
|
}
|
|
245
312
|
|
package/lib/decrypt.js
CHANGED
|
@@ -3,9 +3,10 @@ import path from 'node:path'
|
|
|
3
3
|
import process from 'node:process'
|
|
4
4
|
import {
|
|
5
5
|
buildEncryptedChunkPlan,
|
|
6
|
-
|
|
6
|
+
decryptPlainChunkFromEncrypted,
|
|
7
7
|
importToken,
|
|
8
8
|
normalizeClientEncryptionMeta,
|
|
9
|
+
usesChunkCompression,
|
|
9
10
|
} from './crypto.js'
|
|
10
11
|
import { fromBase64Url } from './base64url.js'
|
|
11
12
|
import { BytifiApiError, BytifiNetworkError } from './upload.js'
|
|
@@ -179,7 +180,7 @@ async function resolveEncryptionMeta({
|
|
|
179
180
|
if (!linkToken) {
|
|
180
181
|
throw new Error(
|
|
181
182
|
'Missing link metadata for a downloaded file. Pass --upload-json upload.json, --link LINK_ID, or --meta meta.json.\n'
|
|
182
|
-
+ 'The link ID is the `
|
|
183
|
+
+ 'The link ID is the `link` field in upload JSON (also appears as /f/LINK_ID and link?link=LINK_ID).',
|
|
183
184
|
)
|
|
184
185
|
}
|
|
185
186
|
|
|
@@ -249,7 +250,13 @@ async function decryptFromParts({
|
|
|
249
250
|
|
|
250
251
|
const chunk = chunks[index]
|
|
251
252
|
const encryptedPart = await fetchEncryptedPart(baseUrl, linkToken, chunk.chunkIndex + 1, signal)
|
|
252
|
-
const plainPart =
|
|
253
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
254
|
+
encryptedPart,
|
|
255
|
+
tokenBytes,
|
|
256
|
+
noncePrefix,
|
|
257
|
+
chunk.chunkIndex,
|
|
258
|
+
meta,
|
|
259
|
+
)
|
|
253
260
|
|
|
254
261
|
await fileHandle.write(plainPart)
|
|
255
262
|
decryptedBytes += plainPart.length
|
|
@@ -269,7 +276,6 @@ async function decryptFromSingleFile({
|
|
|
269
276
|
onProgress,
|
|
270
277
|
signal,
|
|
271
278
|
}) {
|
|
272
|
-
const { chunks, totalEncryptedSize } = buildEncryptedChunkPlan(meta)
|
|
273
279
|
const response = await fetch(encryptedFileUrl, { signal })
|
|
274
280
|
|
|
275
281
|
if (!response.ok) {
|
|
@@ -280,6 +286,28 @@ async function decryptFromSingleFile({
|
|
|
280
286
|
})
|
|
281
287
|
}
|
|
282
288
|
|
|
289
|
+
if (usesChunkCompression(meta)) {
|
|
290
|
+
if (meta.chunkCount !== 1) {
|
|
291
|
+
throw new Error(
|
|
292
|
+
'Compressed files with multiple chunks require part-based download. Open the share link in a browser or use part files.',
|
|
293
|
+
)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const encryptedBuffer = Buffer.from(await response.arrayBuffer())
|
|
297
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
298
|
+
encryptedBuffer,
|
|
299
|
+
tokenBytes,
|
|
300
|
+
noncePrefix,
|
|
301
|
+
0,
|
|
302
|
+
meta,
|
|
303
|
+
)
|
|
304
|
+
await fs.writeFile(outputPath, plainPart)
|
|
305
|
+
onProgress?.(100)
|
|
306
|
+
return
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const { chunks, totalEncryptedSize } = buildEncryptedChunkPlan(meta)
|
|
310
|
+
|
|
283
311
|
if (!response.body) {
|
|
284
312
|
const encryptedBuffer = Buffer.from(await response.arrayBuffer())
|
|
285
313
|
await writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, noncePrefix, outputPath, onProgress, signal)
|
|
@@ -342,7 +370,13 @@ async function decryptFromSingleFile({
|
|
|
342
370
|
throw new Error('Encrypted file ended before all parts were downloaded.')
|
|
343
371
|
}
|
|
344
372
|
|
|
345
|
-
const plainPart =
|
|
373
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
374
|
+
encryptedChunk,
|
|
375
|
+
tokenBytes,
|
|
376
|
+
noncePrefix,
|
|
377
|
+
chunk.chunkIndex,
|
|
378
|
+
meta,
|
|
379
|
+
)
|
|
346
380
|
await fileHandle.write(plainPart)
|
|
347
381
|
decryptedBytes += plainPart.length
|
|
348
382
|
nextChunkIndex += 1
|
|
@@ -357,6 +391,23 @@ async function decryptFromSingleFile({
|
|
|
357
391
|
}
|
|
358
392
|
|
|
359
393
|
async function writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, noncePrefix, outputPath, onProgress, signal) {
|
|
394
|
+
if (usesChunkCompression(meta)) {
|
|
395
|
+
if (chunks.length !== 1) {
|
|
396
|
+
throw new Error('Compressed files with multiple chunks require part-based storage.')
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
400
|
+
encryptedBuffer,
|
|
401
|
+
tokenBytes,
|
|
402
|
+
noncePrefix,
|
|
403
|
+
0,
|
|
404
|
+
meta,
|
|
405
|
+
)
|
|
406
|
+
await fs.writeFile(outputPath, plainPart)
|
|
407
|
+
onProgress?.(100)
|
|
408
|
+
return
|
|
409
|
+
}
|
|
410
|
+
|
|
360
411
|
const fileHandle = await fs.open(outputPath, 'w')
|
|
361
412
|
let offset = 0
|
|
362
413
|
let decryptedBytes = 0
|
|
@@ -372,7 +423,13 @@ async function writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, n
|
|
|
372
423
|
throw new Error('Encrypted file ended before all parts were downloaded.')
|
|
373
424
|
}
|
|
374
425
|
|
|
375
|
-
const plainPart =
|
|
426
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
427
|
+
encryptedChunk,
|
|
428
|
+
tokenBytes,
|
|
429
|
+
noncePrefix,
|
|
430
|
+
chunk.chunkIndex,
|
|
431
|
+
meta,
|
|
432
|
+
)
|
|
376
433
|
await fileHandle.write(plainPart)
|
|
377
434
|
offset += chunk.encryptedSize
|
|
378
435
|
decryptedBytes += plainPart.length
|
|
@@ -395,6 +452,17 @@ async function decryptFromLocalSingleFile({
|
|
|
395
452
|
signal,
|
|
396
453
|
}) {
|
|
397
454
|
const { chunks } = buildEncryptedChunkPlan(meta)
|
|
455
|
+
|
|
456
|
+
if (usesChunkCompression(meta)) {
|
|
457
|
+
if (meta.chunkCount !== 1) {
|
|
458
|
+
throw new Error('Compressed multi-chunk files must be decrypted from a parts directory.')
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const encryptedBuffer = await fs.readFile(encryptedFilePath)
|
|
462
|
+
await writeDecryptedBuffer(encryptedBuffer, chunks, meta, tokenBytes, noncePrefix, outputPath, onProgress, signal)
|
|
463
|
+
return
|
|
464
|
+
}
|
|
465
|
+
|
|
398
466
|
const stat = await fs.stat(encryptedFilePath)
|
|
399
467
|
|
|
400
468
|
if (stat.size <= 64 * 1024 * 1024) {
|
|
@@ -421,7 +489,13 @@ async function decryptFromLocalSingleFile({
|
|
|
421
489
|
throw new Error('Encrypted file ended before all parts were read.')
|
|
422
490
|
}
|
|
423
491
|
|
|
424
|
-
const plainPart =
|
|
492
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
493
|
+
encryptedChunk,
|
|
494
|
+
tokenBytes,
|
|
495
|
+
noncePrefix,
|
|
496
|
+
chunk.chunkIndex,
|
|
497
|
+
meta,
|
|
498
|
+
)
|
|
425
499
|
await outputHandle.write(plainPart)
|
|
426
500
|
offset += chunk.encryptedSize
|
|
427
501
|
decryptedBytes += plainPart.length
|
|
@@ -485,7 +559,13 @@ async function decryptFromLocalParts({
|
|
|
485
559
|
}
|
|
486
560
|
|
|
487
561
|
const encryptedPart = await fs.readFile(path.join(partsDirectory, partEntry.name))
|
|
488
|
-
const plainPart =
|
|
562
|
+
const plainPart = await decryptPlainChunkFromEncrypted(
|
|
563
|
+
encryptedPart,
|
|
564
|
+
tokenBytes,
|
|
565
|
+
noncePrefix,
|
|
566
|
+
chunk.chunkIndex,
|
|
567
|
+
meta,
|
|
568
|
+
)
|
|
489
569
|
|
|
490
570
|
await fileHandle.write(plainPart)
|
|
491
571
|
decryptedBytes += plainPart.length
|
|
@@ -506,7 +586,7 @@ function hintTokenConfusion(encryptionToken, linkToken) {
|
|
|
506
586
|
if (token.length <= 20) {
|
|
507
587
|
throw new Error(
|
|
508
588
|
`"${token}" looks like a link ID, not an encryption token.\n`
|
|
509
|
-
+ 'Use --link for the link ID (`
|
|
589
|
+
+ 'Use --link for the link ID (`link` in upload JSON) and --token for the encryption key (`encryptionToken`).\n'
|
|
510
590
|
+ 'Easiest: bytifi decrypt ./file.encrypted --upload-json upload.json',
|
|
511
591
|
)
|
|
512
592
|
}
|
|
@@ -533,7 +613,7 @@ export async function loadUploadJson(uploadJsonPath) {
|
|
|
533
613
|
}
|
|
534
614
|
|
|
535
615
|
return {
|
|
536
|
-
linkToken: String(parsed.token || '').trim(),
|
|
616
|
+
linkToken: String(parsed.link || parsed.token || '').trim(),
|
|
537
617
|
encryptionToken,
|
|
538
618
|
meta,
|
|
539
619
|
originalName: parsed.originalName || 'download',
|
|
@@ -591,7 +671,7 @@ function buildDecryptResult({
|
|
|
591
671
|
size,
|
|
592
672
|
mimeType,
|
|
593
673
|
expiresAt,
|
|
594
|
-
linkToken,
|
|
674
|
+
link: linkToken,
|
|
595
675
|
storageMode,
|
|
596
676
|
sourcePath,
|
|
597
677
|
}
|
package/lib/upload.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
MULTIPART_THRESHOLD_BYTES,
|
|
3
3
|
encryptChunkFromFile,
|
|
4
4
|
resolveUploadFile,
|
|
5
5
|
} from './crypto.js'
|
|
@@ -124,7 +124,7 @@ function buildResult(payload, context, shareUrl) {
|
|
|
124
124
|
shareUrl,
|
|
125
125
|
url: payload.url,
|
|
126
126
|
encryptedFile: payload.downloadUrl,
|
|
127
|
-
|
|
127
|
+
link: payload.token,
|
|
128
128
|
encryptionToken: context.token,
|
|
129
129
|
clientEncryptionMeta: context.meta,
|
|
130
130
|
originalName: payload.originalName || context.originalName,
|
|
@@ -132,6 +132,70 @@ function buildResult(payload, context, shareUrl) {
|
|
|
132
132
|
expiresAt: payload.expiresAt,
|
|
133
133
|
deleteOnDownload: payload.deleteOnDownload,
|
|
134
134
|
clientEncrypted: payload.clientEncrypted,
|
|
135
|
+
compression: context.compression,
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function createBoundedQueue(maxSize) {
|
|
140
|
+
const items = []
|
|
141
|
+
const waiters = []
|
|
142
|
+
let closed = false
|
|
143
|
+
|
|
144
|
+
function notifyWaiters() {
|
|
145
|
+
while (waiters.length > 0) {
|
|
146
|
+
const resolve = waiters.shift()
|
|
147
|
+
resolve()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
async push(item, signal) {
|
|
153
|
+
while (items.length >= maxSize) {
|
|
154
|
+
if (signal?.aborted) {
|
|
155
|
+
throw new Error('Upload aborted.')
|
|
156
|
+
}
|
|
157
|
+
await new Promise((resolve, reject) => {
|
|
158
|
+
const onAbort = () => {
|
|
159
|
+
const index = waiters.indexOf(resolve)
|
|
160
|
+
if (index >= 0) waiters.splice(index, 1)
|
|
161
|
+
reject(new Error('Upload aborted.'))
|
|
162
|
+
}
|
|
163
|
+
waiters.push(resolve)
|
|
164
|
+
signal?.addEventListener('abort', onAbort, { once: true })
|
|
165
|
+
})
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
items.push(item)
|
|
169
|
+
notifyWaiters()
|
|
170
|
+
},
|
|
171
|
+
async take(signal) {
|
|
172
|
+
while (items.length === 0) {
|
|
173
|
+
if (closed) {
|
|
174
|
+
return null
|
|
175
|
+
}
|
|
176
|
+
if (signal?.aborted) {
|
|
177
|
+
throw new Error('Upload aborted.')
|
|
178
|
+
}
|
|
179
|
+
await new Promise((resolve, reject) => {
|
|
180
|
+
const onAbort = () => {
|
|
181
|
+
const index = waiters.indexOf(resolve)
|
|
182
|
+
if (index >= 0) waiters.splice(index, 1)
|
|
183
|
+
reject(new Error('Upload aborted.'))
|
|
184
|
+
}
|
|
185
|
+
waiters.push(resolve)
|
|
186
|
+
signal?.addEventListener('abort', onAbort, { once: true })
|
|
187
|
+
})
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return items.shift()
|
|
191
|
+
},
|
|
192
|
+
close() {
|
|
193
|
+
closed = true
|
|
194
|
+
notifyWaiters()
|
|
195
|
+
},
|
|
196
|
+
get pending() {
|
|
197
|
+
return items.length
|
|
198
|
+
},
|
|
135
199
|
}
|
|
136
200
|
}
|
|
137
201
|
|
|
@@ -221,7 +285,7 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
221
285
|
signal,
|
|
222
286
|
concurrency = DEFAULT_CONCURRENCY,
|
|
223
287
|
}) {
|
|
224
|
-
const
|
|
288
|
+
const maxPartSize = context.meta.chunkSize + 16
|
|
225
289
|
const initPayload = await apiFetchWithRetry(baseUrl, '/api/public/upload/init', {
|
|
226
290
|
apiKey,
|
|
227
291
|
method: 'POST',
|
|
@@ -233,7 +297,7 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
233
297
|
originalSize: context.originalSize,
|
|
234
298
|
clientEncrypted: true,
|
|
235
299
|
clientEncryptionMeta: context.meta,
|
|
236
|
-
partSize,
|
|
300
|
+
partSize: maxPartSize,
|
|
237
301
|
expiresInMinutes,
|
|
238
302
|
deleteOnDownload,
|
|
239
303
|
}),
|
|
@@ -242,37 +306,58 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
242
306
|
|
|
243
307
|
const sessionToken = initPayload.sessionToken
|
|
244
308
|
const workerCount = Math.min(
|
|
245
|
-
Number(initPayload.concurrency) ||
|
|
309
|
+
Number(concurrency) || Number(initPayload.concurrency) || DEFAULT_CONCURRENCY,
|
|
246
310
|
context.chunkCount,
|
|
247
311
|
)
|
|
312
|
+
const queue = createBoundedQueue(Math.max(2, workerCount * 2))
|
|
248
313
|
const fileHandle = await fs.open(filePath, 'r')
|
|
249
314
|
let nextChunkIndex = 0
|
|
250
315
|
let completedParts = 0
|
|
251
316
|
|
|
252
|
-
|
|
317
|
+
function claimChunkIndex() {
|
|
318
|
+
const chunkIndex = nextChunkIndex
|
|
319
|
+
nextChunkIndex += 1
|
|
320
|
+
return chunkIndex
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async function encryptWorker() {
|
|
253
324
|
while (true) {
|
|
254
325
|
if (signal?.aborted) {
|
|
255
326
|
throw new Error('Upload aborted.')
|
|
256
327
|
}
|
|
257
328
|
|
|
258
|
-
const chunkIndex =
|
|
259
|
-
nextChunkIndex += 1
|
|
260
|
-
|
|
329
|
+
const chunkIndex = claimChunkIndex()
|
|
261
330
|
if (chunkIndex >= context.chunkCount) {
|
|
262
331
|
return
|
|
263
332
|
}
|
|
264
333
|
|
|
265
|
-
const partNumber = chunkIndex + 1
|
|
266
334
|
const encryptedPart = await encryptChunkFromFile(fileHandle, chunkIndex, context)
|
|
335
|
+
await queue.push({
|
|
336
|
+
partNumber: chunkIndex + 1,
|
|
337
|
+
body: encryptedPart,
|
|
338
|
+
}, signal)
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async function uploadWorker() {
|
|
343
|
+
while (true) {
|
|
344
|
+
if (signal?.aborted) {
|
|
345
|
+
throw new Error('Upload aborted.')
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const part = await queue.take(signal)
|
|
349
|
+
if (!part) {
|
|
350
|
+
return
|
|
351
|
+
}
|
|
267
352
|
|
|
268
353
|
await apiFetchWithRetry(
|
|
269
354
|
baseUrl,
|
|
270
|
-
`/api/public/upload/part?sessionToken=${encodeURIComponent(sessionToken)}&partNumber=${partNumber}`,
|
|
355
|
+
`/api/public/upload/part?sessionToken=${encodeURIComponent(sessionToken)}&partNumber=${part.partNumber}`,
|
|
271
356
|
{
|
|
272
357
|
apiKey,
|
|
273
358
|
method: 'PUT',
|
|
274
359
|
headers: { 'Content-Type': 'application/octet-stream' },
|
|
275
|
-
body:
|
|
360
|
+
body: part.body,
|
|
276
361
|
signal,
|
|
277
362
|
},
|
|
278
363
|
)
|
|
@@ -283,7 +368,11 @@ async function uploadMultipartStreaming(filePath, context, {
|
|
|
283
368
|
}
|
|
284
369
|
|
|
285
370
|
try {
|
|
286
|
-
|
|
371
|
+
const uploadWorkers = Array.from({ length: workerCount }, () => uploadWorker())
|
|
372
|
+
const encryptWorkers = Array.from({ length: workerCount }, () => encryptWorker())
|
|
373
|
+
await Promise.all(encryptWorkers)
|
|
374
|
+
queue.close()
|
|
375
|
+
await Promise.all(uploadWorkers)
|
|
287
376
|
} catch (error) {
|
|
288
377
|
await apiFetchWithRetry(baseUrl, '/api/public/upload/abort', {
|
|
289
378
|
apiKey,
|
|
@@ -321,9 +410,10 @@ export async function uploadFile(filePath, options) {
|
|
|
321
410
|
|
|
322
411
|
const { absolutePath, context } = await resolveUploadFile(filePath, {
|
|
323
412
|
mimeType: options.mimeType,
|
|
413
|
+
compressionMode: options.compressionMode,
|
|
324
414
|
})
|
|
325
415
|
|
|
326
|
-
if (context.
|
|
416
|
+
if (context.originalSize <= MULTIPART_THRESHOLD_BYTES) {
|
|
327
417
|
const encryptedBuffer = await collectEncryptedBuffer(absolutePath, context, {
|
|
328
418
|
onProgress: options.onProgress,
|
|
329
419
|
signal: options.signal,
|