@shieldfive/cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +117 -4
- package/README.md +130 -12
- package/SECURITY.md +9 -5
- package/package.json +1 -1
- package/src/agent/client.mjs +102 -0
- package/src/agent/ledger.mjs +254 -0
- package/src/agent/main.mjs +72 -0
- package/src/agent/paths.mjs +132 -0
- package/src/agent/protocol.mjs +125 -0
- package/src/agent/server.mjs +525 -0
- package/src/auth.mjs +79 -1
- package/src/cli.mjs +365 -36
- package/src/prompt.mjs +79 -0
- package/src/upload.mjs +207 -73
- package/src/uploadCrypto.mjs +62 -5
package/src/upload.mjs
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
// ShieldFive CLI — upload: create-session -> chunk-encrypt ->
|
|
2
|
-
//
|
|
1
|
+
// ShieldFive CLI — upload: create-session -> chunk-encrypt -> upload -> finalize.
|
|
2
|
+
// Ported from the apps (mobile cloudApi.ts + syncService.ts).
|
|
3
3
|
//
|
|
4
|
-
// Two paths, chosen by the server:
|
|
5
|
-
// direct — a single-chunk file (<= chunkSize). One
|
|
6
|
-
//
|
|
7
|
-
//
|
|
4
|
+
// Two paths, chosen by the server, and they no longer share a wire protocol:
|
|
5
|
+
// direct — a single-chunk file (<= chunkSize). One presigned S3 PUT, no
|
|
6
|
+
// Authorization header and no storage id at finalize. The server
|
|
7
|
+
// moved this path off b2_upload_file in web b52b4c8 (PR #726).
|
|
8
|
+
// large — a multipart file (> chunkSize). b2_upload_part per chunk, with a
|
|
9
|
+
// Backblaze token and X-Bz-* headers, then b2_finish_large_file
|
|
10
|
+
// (server-side) at finalize. Unchanged.
|
|
8
11
|
//
|
|
9
12
|
// Chunks are read from disk one at a time (never the whole file into memory),
|
|
10
13
|
// so a multi-gigabyte upload holds at most one chunk (~5 MiB) resident.
|
|
@@ -27,6 +30,17 @@ const COMPLETE_PATH = '/api/files/complete-upload'
|
|
|
27
30
|
const PART_URL_PATH = '/api/files/upload-part-url'
|
|
28
31
|
const CHUNK_SIZE = 5 * 1024 * 1024
|
|
29
32
|
|
|
33
|
+
// Backblaze documents retrying b2_upload_part on 5xx / network failures, and a
|
|
34
|
+
// part upload is idempotent by (part number, SHA-1) — retrying overwrites the
|
|
35
|
+
// same part. Bound the retries and back off so one transient blip does not
|
|
36
|
+
// abort a whole multi-gigabyte upload.
|
|
37
|
+
const MAX_PART_ATTEMPTS = 3
|
|
38
|
+
const PART_RETRY_BASE_DELAY_MS = 250
|
|
39
|
+
|
|
40
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
41
|
+
const partRetryDelayMs = (attempt) =>
|
|
42
|
+
PART_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1) // 250ms, 500ms, ...
|
|
43
|
+
|
|
30
44
|
// Read a file in fixed-size chunks without holding it all in memory. Each yield
|
|
31
45
|
// is an independent copy (the read buffer is reused), so the caller may retain
|
|
32
46
|
// or transform a chunk across the next read safely.
|
|
@@ -67,7 +81,7 @@ export async function createUploadSession({
|
|
|
67
81
|
size,
|
|
68
82
|
}) {
|
|
69
83
|
const rootKeyB64 = bytesToBase64(rootKey)
|
|
70
|
-
const nameEnvelope = encryptMetadataV4(name, rootKeyB64)
|
|
84
|
+
const nameEnvelope = await encryptMetadataV4(name, rootKeyB64)
|
|
71
85
|
const csk = generateRandomKeyB64()
|
|
72
86
|
const cskEnvelope = wrapKeyB64({ wrappingKeyB64: rootKeyB64, keyToWrapB64: csk })
|
|
73
87
|
const noncePrefix = generateNoncePrefixB64()
|
|
@@ -101,29 +115,33 @@ export async function createUploadSession({
|
|
|
101
115
|
return { session: body, csk, noncePrefix }
|
|
102
116
|
}
|
|
103
117
|
|
|
104
|
-
// Direct upload:
|
|
105
|
-
//
|
|
106
|
-
|
|
118
|
+
// Direct upload: a presigned S3 PUT (web b52b4c8 / PR #726), not b2_upload_file.
|
|
119
|
+
// The SigV4 signature in the URL IS the credential and it authorises exactly one
|
|
120
|
+
// object key, so Authorization and X-Bz-* are neither required nor signed --
|
|
121
|
+
// sending them would be noise, and an Authorization header would mislead the
|
|
122
|
+
// next reader into thinking the token still matters here.
|
|
123
|
+
//
|
|
124
|
+
// Content-Type is sent so the stored object carries a sensible type. It is
|
|
125
|
+
// currently unsigned, but only because the vault call site presigns without
|
|
126
|
+
// ContentLength; if that ever changes, this header becomes load-bearing and a
|
|
127
|
+
// mismatch 403s every direct upload. See the web repo's
|
|
128
|
+
// tests/storage/presignedPutContract.test.ts, which pins the signed-header set.
|
|
129
|
+
//
|
|
130
|
+
// Returns nothing. A presigned PUT answers with an ETag, not a Backblaze file
|
|
131
|
+
// id, so there is no storage id to hand to finalize.
|
|
132
|
+
async function putDirectPresigned({ session, ciphertext }) {
|
|
107
133
|
const res = await fetch(session.uploadUrl, {
|
|
108
|
-
method: '
|
|
134
|
+
method: 'PUT',
|
|
109
135
|
headers: {
|
|
110
|
-
|
|
111
|
-
'X-Bz-File-Name': encodeURIComponent(session.storagePath),
|
|
112
|
-
'X-Bz-Content-Sha1': sha1,
|
|
113
|
-
'Content-Type': 'application/octet-stream',
|
|
136
|
+
'Content-Type': session.contentType || 'application/octet-stream',
|
|
114
137
|
},
|
|
115
138
|
body: ciphertext,
|
|
116
139
|
})
|
|
117
140
|
if (!res.ok) {
|
|
118
141
|
throw new Error(
|
|
119
|
-
`
|
|
142
|
+
`Direct upload failed (HTTP ${res.status}): ${await res.text().catch(() => '')}`,
|
|
120
143
|
)
|
|
121
144
|
}
|
|
122
|
-
const body = await res.json().catch(() => ({}))
|
|
123
|
-
if (!body.fileId) {
|
|
124
|
-
throw new Error(`Backblaze response missing fileId: ${JSON.stringify(body)}`)
|
|
125
|
-
}
|
|
126
|
-
return body.fileId
|
|
127
145
|
}
|
|
128
146
|
|
|
129
147
|
// Ask the server for a fresh b2_upload_part URL + token when the current one
|
|
@@ -147,8 +165,11 @@ async function refreshUploadPartUrl({ apiBaseUrl, accessToken, fileId }) {
|
|
|
147
165
|
}
|
|
148
166
|
|
|
149
167
|
// One b2_upload_part POST. On an expired token (401/403) refresh the part URL
|
|
150
|
-
//
|
|
151
|
-
//
|
|
168
|
+
// and retry immediately, mirroring uploadMultipartCiphertextPart in the apps.
|
|
169
|
+
// On a transient failure (5xx / 408 / 429 / network error) refresh the part URL
|
|
170
|
+
// and retry with exponential backoff, up to MAX_PART_ATTEMPTS, so a single blip
|
|
171
|
+
// does not abort the whole upload. Returns the (possibly refreshed) url/token so
|
|
172
|
+
// the caller reuses them for the next part.
|
|
152
173
|
async function putPartToBackblaze({
|
|
153
174
|
apiBaseUrl,
|
|
154
175
|
accessToken,
|
|
@@ -173,22 +194,57 @@ async function putPartToBackblaze({
|
|
|
173
194
|
|
|
174
195
|
let url = uploadUrl
|
|
175
196
|
let token = authToken
|
|
176
|
-
let
|
|
197
|
+
let lastError
|
|
177
198
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
199
|
+
for (let tries = 1; tries <= MAX_PART_ATTEMPTS; tries += 1) {
|
|
200
|
+
let res
|
|
201
|
+
try {
|
|
202
|
+
res = await attempt(url, token)
|
|
203
|
+
} catch (err) {
|
|
204
|
+
// Network/transport failure (reset, timeout, DNS). Retry with a fresh
|
|
205
|
+
// part URL after a backoff.
|
|
206
|
+
lastError = err
|
|
207
|
+
if (tries === MAX_PART_ATTEMPTS) break
|
|
208
|
+
await sleep(partRetryDelayMs(tries))
|
|
209
|
+
;({ uploadUrl: url, authToken: token } = await refreshUploadPartUrl({
|
|
210
|
+
apiBaseUrl,
|
|
211
|
+
accessToken,
|
|
212
|
+
fileId,
|
|
213
|
+
}))
|
|
214
|
+
continue
|
|
215
|
+
}
|
|
184
216
|
|
|
185
|
-
|
|
186
|
-
|
|
217
|
+
if (res.status === 401 || res.status === 403) {
|
|
218
|
+
// Expected token rotation, not a transient failure: refresh and retry
|
|
219
|
+
// immediately without consuming a backoff slot.
|
|
220
|
+
const refreshed = await refreshUploadPartUrl({ apiBaseUrl, accessToken, fileId })
|
|
221
|
+
url = refreshed.uploadUrl
|
|
222
|
+
token = refreshed.authToken
|
|
223
|
+
res = await attempt(url, token)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (res.ok) return { uploadUrl: url, authToken: token }
|
|
227
|
+
|
|
228
|
+
lastError = new Error(
|
|
187
229
|
`Backblaze part ${partNumber} upload failed (HTTP ${res.status}): ` +
|
|
188
230
|
`${await res.text().catch(() => '')}`,
|
|
189
231
|
)
|
|
232
|
+
|
|
233
|
+
// 5xx / 408 / 429 are worth retrying; other 4xx (after the auth refresh
|
|
234
|
+
// above) are terminal.
|
|
235
|
+
const retryable =
|
|
236
|
+
res.status >= 500 || res.status === 429 || res.status === 408
|
|
237
|
+
if (!retryable || tries === MAX_PART_ATTEMPTS) throw lastError
|
|
238
|
+
|
|
239
|
+
await sleep(partRetryDelayMs(tries))
|
|
240
|
+
;({ uploadUrl: url, authToken: token } = await refreshUploadPartUrl({
|
|
241
|
+
apiBaseUrl,
|
|
242
|
+
accessToken,
|
|
243
|
+
fileId,
|
|
244
|
+
}))
|
|
190
245
|
}
|
|
191
|
-
|
|
246
|
+
|
|
247
|
+
throw lastError ?? new Error(`Backblaze part ${partNumber} upload failed`)
|
|
192
248
|
}
|
|
193
249
|
|
|
194
250
|
async function finalizeUpload({
|
|
@@ -218,8 +274,9 @@ async function finalizeUpload({
|
|
|
218
274
|
}
|
|
219
275
|
|
|
220
276
|
// Single-chunk file: encrypt the one chunk (AES-GCM, suite 0x01) + SHA-1 +
|
|
221
|
-
// proof, PUT it, finalize. b2FileId is
|
|
222
|
-
|
|
277
|
+
// proof, PUT it at the presigned URL, finalize. No b2FileId is sent -- the
|
|
278
|
+
// server HEADs the stored object and derives the storage id itself.
|
|
279
|
+
async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk }) {
|
|
223
280
|
const chunkSize = session.chunkSize ?? CHUNK_SIZE
|
|
224
281
|
let plaintext = null
|
|
225
282
|
for await (const chunk of readFileChunks(path, chunkSize)) {
|
|
@@ -227,6 +284,7 @@ async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix
|
|
|
227
284
|
throw new Error('Direct upload session is larger than a single chunk.')
|
|
228
285
|
}
|
|
229
286
|
plaintext = chunk
|
|
287
|
+
onPlaintextChunk?.(chunk)
|
|
230
288
|
}
|
|
231
289
|
if (plaintext === null) plaintext = new Uint8Array(0)
|
|
232
290
|
|
|
@@ -239,13 +297,17 @@ async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix
|
|
|
239
297
|
chunkSize,
|
|
240
298
|
})
|
|
241
299
|
|
|
242
|
-
|
|
300
|
+
await putDirectPresigned({ session, ciphertext })
|
|
243
301
|
|
|
302
|
+
// b2FileId is deliberately omitted. JSON.stringify drops the undefined key,
|
|
303
|
+
// so complete-upload/finalize.ts takes its "no client-supplied id" branch and
|
|
304
|
+
// HEADs the object for the real version id. Forwarding the PUT's ETag instead
|
|
305
|
+
// would be persisted verbatim into files.b2_file_id with no format check --
|
|
306
|
+
// a bogus delete handle, permanently, which is worse than failing.
|
|
244
307
|
return finalizeUpload({
|
|
245
308
|
apiBaseUrl,
|
|
246
309
|
accessToken,
|
|
247
310
|
fileId: session.fileId,
|
|
248
|
-
b2FileId,
|
|
249
311
|
partSha1Array: [sha1Hex],
|
|
250
312
|
proof: proofHex,
|
|
251
313
|
ciphertextHash: sha1Hex,
|
|
@@ -258,49 +320,112 @@ async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix
|
|
|
258
320
|
// bytes of the stored object, which is exactly part 1. b2FileId is the
|
|
259
321
|
// large-file id from the session; the server calls b2_finish_large_file with
|
|
260
322
|
// the ordered part hashes at finalize.
|
|
261
|
-
|
|
323
|
+
// Bounded concurrency for multipart part uploads. The serial version left the
|
|
324
|
+
// network idle during encrypt and the CPU idle during each PUT; uploading a few
|
|
325
|
+
// parts at once fills the pipe. Backblaze needs one upload URL per concurrent
|
|
326
|
+
// connection, so K concurrent parts draw K URLs from a small pool. Default 3;
|
|
327
|
+
// override with SF_UPLOAD_CONCURRENCY (clamped 1..6). SF_UPLOAD_CONCURRENCY=1
|
|
328
|
+
// reproduces the exact serial behaviour. Memory ceiling ≈ 2 * K * chunkSize
|
|
329
|
+
// (K plaintext chunks + K ciphertexts resident at once).
|
|
330
|
+
const DEFAULT_UPLOAD_CONCURRENCY = 3
|
|
331
|
+
const MAX_UPLOAD_CONCURRENCY = 6
|
|
332
|
+
function resolveUploadConcurrency() {
|
|
333
|
+
const raw = Number.parseInt(process.env.SF_UPLOAD_CONCURRENCY ?? '', 10)
|
|
334
|
+
if (Number.isInteger(raw) && raw >= 1) {
|
|
335
|
+
return Math.min(raw, MAX_UPLOAD_CONCURRENCY)
|
|
336
|
+
}
|
|
337
|
+
return DEFAULT_UPLOAD_CONCURRENCY
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk }) {
|
|
262
341
|
if (!session.b2FileId) {
|
|
263
342
|
throw new Error('Multipart upload session is missing storage id (b2FileId).')
|
|
264
343
|
}
|
|
265
344
|
const chunkSize = session.chunkSize ?? CHUNK_SIZE
|
|
345
|
+
const concurrency = resolveUploadConcurrency()
|
|
266
346
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
const
|
|
270
|
-
let proof
|
|
271
|
-
let
|
|
347
|
+
// Part SHA-1s keyed by chunkIndex, NOT push order, so parts finishing out of
|
|
348
|
+
// order under concurrency never scramble the manifest the server checks.
|
|
349
|
+
const partSha1ByIndex = []
|
|
350
|
+
let proof // computed from chunk 0 only (the server verifies the proof over part 1)
|
|
351
|
+
let firstError = null
|
|
272
352
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
chunkSize: chunkIndex === 0 ? chunkSize : undefined,
|
|
282
|
-
})
|
|
283
|
-
if (chunkIndex === 0) proof = encrypted.proofHex
|
|
353
|
+
// Pool of b2_upload_part URLs — one per concurrent connection. Seed with the
|
|
354
|
+
// session URL, fetch more on demand up to `concurrency`, and return each after
|
|
355
|
+
// a successful part so URLs are reused rather than re-fetched per part.
|
|
356
|
+
// putPartToBackblaze already self-heals expired tokens and retries 5xx.
|
|
357
|
+
const urlPool = [{ uploadUrl: session.uploadUrl, authToken: session.authToken }]
|
|
358
|
+
const acquireUrl = async () =>
|
|
359
|
+
urlPool.pop() ??
|
|
360
|
+
(await refreshUploadPartUrl({ apiBaseUrl, accessToken, fileId: session.fileId }))
|
|
284
361
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
362
|
+
const inFlight = new Set()
|
|
363
|
+
const dispatch = (chunkIndex, plaintext) => {
|
|
364
|
+
const run = (async () => {
|
|
365
|
+
try {
|
|
366
|
+
const encrypted = await encryptChunkWithDigest({
|
|
367
|
+
cskB64: csk,
|
|
368
|
+
noncePrefixB64: noncePrefix,
|
|
369
|
+
chunkIndex,
|
|
370
|
+
plaintext,
|
|
371
|
+
proofKeyHex: chunkIndex === 0 ? session.proofKey : undefined,
|
|
372
|
+
chunkSize: chunkIndex === 0 ? chunkSize : undefined,
|
|
373
|
+
})
|
|
374
|
+
if (chunkIndex === 0) proof = encrypted.proofHex
|
|
375
|
+
|
|
376
|
+
const slot = await acquireUrl()
|
|
377
|
+
const updated = await putPartToBackblaze({
|
|
378
|
+
apiBaseUrl,
|
|
379
|
+
accessToken,
|
|
380
|
+
fileId: session.fileId,
|
|
381
|
+
uploadUrl: slot.uploadUrl,
|
|
382
|
+
authToken: slot.authToken,
|
|
383
|
+
partNumber: chunkIndex + 1, // Backblaze parts are 1-based
|
|
384
|
+
ciphertext: encrypted.ciphertext,
|
|
385
|
+
sha1: encrypted.sha1Hex,
|
|
386
|
+
})
|
|
387
|
+
// Return the (possibly refreshed) URL for the next part to reuse. A URL
|
|
388
|
+
// whose part failed is intentionally NOT returned (it may be bad).
|
|
389
|
+
urlPool.push({ uploadUrl: updated.uploadUrl, authToken: updated.authToken })
|
|
390
|
+
partSha1ByIndex[chunkIndex] = encrypted.sha1Hex
|
|
391
|
+
} catch (err) {
|
|
392
|
+
if (!firstError) firstError = err
|
|
393
|
+
}
|
|
394
|
+
})()
|
|
395
|
+
inFlight.add(run)
|
|
396
|
+
run.finally(() => inFlight.delete(run))
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
let chunkIndex = 0
|
|
400
|
+
for await (const plaintext of readFileChunks(path, chunkSize)) {
|
|
401
|
+
onPlaintextChunk?.(plaintext)
|
|
402
|
+
if (firstError) break
|
|
403
|
+
// Backpressure: hold at most `concurrency` parts in flight. inFlight is
|
|
404
|
+
// non-empty when this runs (size >= concurrency >= 1), so race never hangs.
|
|
405
|
+
while (inFlight.size >= concurrency) {
|
|
406
|
+
await Promise.race(inFlight)
|
|
407
|
+
if (firstError) break
|
|
408
|
+
}
|
|
409
|
+
if (firstError) break
|
|
410
|
+
dispatch(chunkIndex, plaintext)
|
|
298
411
|
chunkIndex += 1
|
|
299
412
|
}
|
|
300
413
|
|
|
301
|
-
|
|
414
|
+
// Let every in-flight part settle before finalizing or surfacing the error.
|
|
415
|
+
await Promise.allSettled(inFlight)
|
|
416
|
+
if (firstError) throw firstError
|
|
417
|
+
|
|
418
|
+
if (!chunkIndex || !proof) {
|
|
302
419
|
throw new Error('Multipart upload produced no parts.')
|
|
303
420
|
}
|
|
421
|
+
// Dense, in chunk order; a gap means a part silently went missing.
|
|
422
|
+
const partSha1Array = Array.from(
|
|
423
|
+
{ length: chunkIndex },
|
|
424
|
+
(_value, index) => partSha1ByIndex[index],
|
|
425
|
+
)
|
|
426
|
+
if (partSha1Array.some((sha1) => typeof sha1 !== 'string')) {
|
|
427
|
+
throw new Error('Multipart upload is missing a part.')
|
|
428
|
+
}
|
|
304
429
|
|
|
305
430
|
const ciphertextHash = await getCiphertextHashFromParts(partSha1Array)
|
|
306
431
|
|
|
@@ -315,7 +440,11 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
|
|
|
315
440
|
})
|
|
316
441
|
}
|
|
317
442
|
|
|
318
|
-
|
|
443
|
+
// `onPlaintextChunk`, when given, is called with every plaintext chunk in file
|
|
444
|
+
// order, from the same read that feeds encryption. The agent hashes those
|
|
445
|
+
// chunks to record what was actually uploaded; hashing the file separately,
|
|
446
|
+
// before or after, cannot see a change made and undone while it was being read.
|
|
447
|
+
export async function uploadFile({ apiBaseUrl, accessToken, rootKey, name, path, size, onPlaintextChunk }) {
|
|
319
448
|
const { session, csk, noncePrefix } = await createUploadSession({
|
|
320
449
|
apiBaseUrl,
|
|
321
450
|
accessToken,
|
|
@@ -325,15 +454,20 @@ export async function uploadFile({ apiBaseUrl, accessToken, rootKey, name, path,
|
|
|
325
454
|
})
|
|
326
455
|
|
|
327
456
|
if (session.uploadKind === 'direct') {
|
|
328
|
-
|
|
329
|
-
|
|
457
|
+
// No authToken and no storagePath check: since web b52b4c8 (PR #726) the
|
|
458
|
+
// direct path is a presigned S3 PUT, so the signature inside uploadUrl is
|
|
459
|
+
// the only credential and it already binds the object key. Requiring the
|
|
460
|
+
// token the server stopped issuing is what made every <= 5 MiB upload fail
|
|
461
|
+
// from 2026-09-03. The 'large' branch below still needs both.
|
|
462
|
+
if (!session.uploadUrl) {
|
|
463
|
+
throw new Error('Direct upload session is missing an upload URL.')
|
|
330
464
|
}
|
|
331
|
-
await uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix, path })
|
|
465
|
+
await uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk })
|
|
332
466
|
} else if (session.uploadKind === 'large') {
|
|
333
467
|
if (!session.uploadUrl || !session.authToken) {
|
|
334
468
|
throw new Error('Multipart upload session is missing credentials.')
|
|
335
469
|
}
|
|
336
|
-
await uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePrefix, path })
|
|
470
|
+
await uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk })
|
|
337
471
|
} else {
|
|
338
472
|
throw new Error(`Unknown uploadKind "${session.uploadKind}" for "${name}".`)
|
|
339
473
|
}
|
package/src/uploadCrypto.mjs
CHANGED
|
@@ -10,6 +10,25 @@ import { gcm } from '@noble/ciphers/aes.js'
|
|
|
10
10
|
import { argon2id } from '@noble/hashes/argon2.js'
|
|
11
11
|
import { hmac } from '@noble/hashes/hmac.js'
|
|
12
12
|
import { sha256 } from '@noble/hashes/sha2.js'
|
|
13
|
+
import sodium from 'libsodium-wrappers-sumo'
|
|
14
|
+
|
|
15
|
+
// libsodium's Argon2id (crypto_pwhash) is byte-identical to @noble's at the same
|
|
16
|
+
// params but ~15x faster (native/WASM vs pure JS). It is used for the ENCRYPT
|
|
17
|
+
// side of the v4 filename metadata — the once-per-file KDF that makes `sf sync`
|
|
18
|
+
// over many files KDF-bound. decryptMetadataV4 stays on @noble (off the hot
|
|
19
|
+
// path, and a second independent implementation guards reads against a
|
|
20
|
+
// libsodium regression). libsodium is already a dependency and initialised in
|
|
21
|
+
// the unlock path; this memoised gate awaits sodium.ready once.
|
|
22
|
+
let sodiumReadyPromise
|
|
23
|
+
const getSodium = async () => {
|
|
24
|
+
if (!sodiumReadyPromise) {
|
|
25
|
+
sodiumReadyPromise = (async () => {
|
|
26
|
+
await sodium.ready
|
|
27
|
+
return sodium
|
|
28
|
+
})()
|
|
29
|
+
}
|
|
30
|
+
return sodiumReadyPromise
|
|
31
|
+
}
|
|
13
32
|
|
|
14
33
|
const AES_GCM_TAG_BYTES = 16
|
|
15
34
|
|
|
@@ -24,6 +43,30 @@ export function aesGcmEncrypt(key, iv, plaintext) {
|
|
|
24
43
|
return gcm(key, iv).encrypt(plaintext)
|
|
25
44
|
}
|
|
26
45
|
|
|
46
|
+
// WebCrypto AES-GCM. Its output is ciphertext||tag with a 16-byte (128-bit)
|
|
47
|
+
// tag — byte-identical to @noble's gcm().encrypt for the same key, 12-byte
|
|
48
|
+
// nonce and no additional data — so ciphertexts, the SHA-1 digests Backblaze
|
|
49
|
+
// checks, and the HMAC upload proof are all unchanged. Native WebCrypto runs
|
|
50
|
+
// at ~5 GiB/s versus @noble's ~70 MiB/s, so it is used for the per-chunk file
|
|
51
|
+
// encryption (the throughput-bound path). @noble stays for the once-per-file
|
|
52
|
+
// key wrap and filename metadata, and for all decryption. It is async because
|
|
53
|
+
// subtle.encrypt is; the chunk path (encryptChunkWithDigest) is already async.
|
|
54
|
+
async function aesGcmEncryptSubtle(key, iv, plaintext) {
|
|
55
|
+
const cryptoKey = await webcrypto.subtle.importKey(
|
|
56
|
+
'raw',
|
|
57
|
+
key,
|
|
58
|
+
{ name: 'AES-GCM' },
|
|
59
|
+
false,
|
|
60
|
+
['encrypt'],
|
|
61
|
+
)
|
|
62
|
+
const ciphertext = await webcrypto.subtle.encrypt(
|
|
63
|
+
{ name: 'AES-GCM', iv },
|
|
64
|
+
cryptoKey,
|
|
65
|
+
plaintext,
|
|
66
|
+
)
|
|
67
|
+
return new Uint8Array(ciphertext)
|
|
68
|
+
}
|
|
69
|
+
|
|
27
70
|
export function generateRandomKeyB64(bytes = 32) {
|
|
28
71
|
return bytesToBase64(randomBytes(bytes))
|
|
29
72
|
}
|
|
@@ -63,10 +106,24 @@ const ARGON2_PARAMS = {
|
|
|
63
106
|
|
|
64
107
|
// v4 filename metadata (Argon2id "interactive" + AES-GCM) — the format the web
|
|
65
108
|
// reads. rootKeySecret is the base64 string of the vault root key.
|
|
66
|
-
|
|
109
|
+
//
|
|
110
|
+
// Derives the metadata key with libsodium crypto_pwhash (ARGON2ID13, opslimit=2,
|
|
111
|
+
// memlimit=64 MiB) — byte-identical to ARGON2_PARAMS.interactive but far faster.
|
|
112
|
+
// The envelope (v/ct/iv/tag/salt/kdf) is unchanged, so it decrypts on the web
|
|
113
|
+
// and mobile clients exactly as before. Async because of the libsodium ready
|
|
114
|
+
// gate; the sole caller (createUploadSession) is already async.
|
|
115
|
+
export async function encryptMetadataV4(plaintext, rootKeySecret) {
|
|
116
|
+
const s = await getSodium()
|
|
67
117
|
const salt = randomBytes(16)
|
|
68
118
|
const iv = randomBytes(12)
|
|
69
|
-
const metadataKey =
|
|
119
|
+
const metadataKey = s.crypto_pwhash(
|
|
120
|
+
32,
|
|
121
|
+
rootKeySecret,
|
|
122
|
+
salt,
|
|
123
|
+
2,
|
|
124
|
+
64 * 1024 * 1024,
|
|
125
|
+
s.crypto_pwhash_ALG_ARGON2ID13,
|
|
126
|
+
)
|
|
70
127
|
const ciphertextWithTag = aesGcmEncrypt(metadataKey, iv, utf8(plaintext))
|
|
71
128
|
const tagStart = ciphertextWithTag.length - AES_GCM_TAG_BYTES
|
|
72
129
|
return {
|
|
@@ -104,7 +161,7 @@ function hexToBytes(hex) {
|
|
|
104
161
|
}
|
|
105
162
|
|
|
106
163
|
// nonce = 4-byte random prefix || 8-byte big-endian chunk counter.
|
|
107
|
-
export function encryptFileChunk({ cskB64, noncePrefixB64, chunkIndex, plaintext }) {
|
|
164
|
+
export async function encryptFileChunk({ cskB64, noncePrefixB64, chunkIndex, plaintext }) {
|
|
108
165
|
const noncePrefix = base64ToBytes(noncePrefixB64)
|
|
109
166
|
if (noncePrefix.length !== 4) throw new Error('Invalid nonce prefix')
|
|
110
167
|
const nonce = new Uint8Array(12)
|
|
@@ -114,7 +171,7 @@ export function encryptFileChunk({ cskB64, noncePrefixB64, chunkIndex, plaintext
|
|
|
114
171
|
nonce[11 - i] = Number(counter & 0xffn)
|
|
115
172
|
counter >>= 8n
|
|
116
173
|
}
|
|
117
|
-
return
|
|
174
|
+
return aesGcmEncryptSubtle(base64ToBytes(cskB64), nonce, plaintext)
|
|
118
175
|
}
|
|
119
176
|
|
|
120
177
|
function buildUploadProofPrefix({ cipherVersion, chunkSize, noncePrefixBytes }) {
|
|
@@ -166,7 +223,7 @@ export async function encryptChunkWithDigest({
|
|
|
166
223
|
proofKeyHex,
|
|
167
224
|
chunkSize,
|
|
168
225
|
}) {
|
|
169
|
-
const ciphertext = encryptFileChunk({
|
|
226
|
+
const ciphertext = await encryptFileChunk({
|
|
170
227
|
cskB64,
|
|
171
228
|
noncePrefixB64,
|
|
172
229
|
chunkIndex,
|