@shieldfive/cli 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +109 -0
- package/README.md +166 -34
- package/SECURITY.md +9 -5
- package/package.json +6 -3
- package/src/agent/client.mjs +102 -0
- package/src/agent/ledger.mjs +254 -0
- package/src/agent/main.mjs +72 -0
- package/src/agent/paths.mjs +132 -0
- package/src/agent/protocol.mjs +125 -0
- package/src/agent/server.mjs +525 -0
- package/src/argon2.mjs +69 -0
- package/src/auth.mjs +111 -7
- package/src/cli.mjs +381 -38
- package/src/loginSecret.mjs +63 -0
- package/src/migrationInventory.mjs +45 -0
- package/src/prompt.mjs +79 -0
- package/src/sync.mjs +68 -7
- package/src/unlock.mjs +32 -16
- package/src/upload.mjs +108 -67
- package/src/uploadCrypto.mjs +153 -113
package/src/upload.mjs
CHANGED
|
@@ -10,25 +10,37 @@
|
|
|
10
10
|
// (server-side) at finalize. Unchanged.
|
|
11
11
|
//
|
|
12
12
|
// Chunks are read from disk one at a time (never the whole file into memory),
|
|
13
|
-
// so a multi-gigabyte upload holds at most
|
|
13
|
+
// so a multi-gigabyte upload holds at most a few chunks (~5 MiB each) resident.
|
|
14
|
+
//
|
|
15
|
+
// Format: cipher_version 3, the web's default (suite 0x03, v1 wire format; see
|
|
16
|
+
// uploadCrypto.mjs). Every storage part is one encrypted chunk frame; part 1
|
|
17
|
+
// also carries the header. The header's file_id is the row id the server
|
|
18
|
+
// assigned at create-upload-session, so the stored object is bound to its row.
|
|
14
19
|
|
|
15
20
|
import { open } from 'node:fs/promises'
|
|
16
21
|
|
|
22
|
+
import { encryptNameV6 } from '@shieldfive/crypto/vault'
|
|
23
|
+
|
|
17
24
|
import {
|
|
25
|
+
base64ToBytes,
|
|
18
26
|
bytesToBase64,
|
|
19
|
-
|
|
27
|
+
CIPHER_VERSION_PQ,
|
|
28
|
+
computeV3UploadProof,
|
|
20
29
|
encryptMetadataV4,
|
|
21
|
-
|
|
30
|
+
encryptPartsV3,
|
|
22
31
|
generateRandomKeyB64,
|
|
23
32
|
getCiphertextHashFromParts,
|
|
24
33
|
hashMetadataV4,
|
|
34
|
+
sha1Hex,
|
|
35
|
+
uuidToBytes,
|
|
36
|
+
V1_CHUNK_SIZE,
|
|
37
|
+
vaultMlKemPublicKey,
|
|
25
38
|
wrapKeyB64,
|
|
26
39
|
} from './uploadCrypto.mjs'
|
|
27
40
|
|
|
28
41
|
const CREATE_SESSION_PATH = '/api/files/create-upload-session'
|
|
29
42
|
const COMPLETE_PATH = '/api/files/complete-upload'
|
|
30
43
|
const PART_URL_PATH = '/api/files/upload-part-url'
|
|
31
|
-
const CHUNK_SIZE = 5 * 1024 * 1024
|
|
32
44
|
|
|
33
45
|
// Backblaze documents retrying b2_upload_part on 5xx / network failures, and a
|
|
34
46
|
// part upload is idempotent by (part number, SHA-1) — retrying overwrites the
|
|
@@ -81,10 +93,14 @@ export async function createUploadSession({
|
|
|
81
93
|
size,
|
|
82
94
|
}) {
|
|
83
95
|
const rootKeyB64 = bytesToBase64(rootKey)
|
|
96
|
+
// The row id does not exist yet, so the first name is the row-unbound v4
|
|
97
|
+
// envelope (as on the web). finalizeUpload replaces it with a v6 envelope
|
|
98
|
+
// bound to the row id.
|
|
84
99
|
const nameEnvelope = await encryptMetadataV4(name, rootKeyB64)
|
|
100
|
+
// For suite 0x03 this is the classical envelope key; csk_wrapped holds it
|
|
101
|
+
// under the parent key (the root key: sf uploads to the vault root).
|
|
85
102
|
const csk = generateRandomKeyB64()
|
|
86
103
|
const cskEnvelope = wrapKeyB64({ wrappingKeyB64: rootKeyB64, keyToWrapB64: csk })
|
|
87
|
-
const noncePrefix = generateNoncePrefixB64()
|
|
88
104
|
|
|
89
105
|
const res = await fetch(new URL(CREATE_SESSION_PATH, apiBaseUrl), {
|
|
90
106
|
method: 'POST',
|
|
@@ -92,14 +108,15 @@ export async function createUploadSession({
|
|
|
92
108
|
Authorization: `Bearer ${accessToken}`,
|
|
93
109
|
'Content-Type': 'application/json',
|
|
94
110
|
},
|
|
111
|
+
// v1 wire format: no cipher_nonce_prefix and no chunkSize (the server
|
|
112
|
+
// rejects both; they live in the header).
|
|
95
113
|
body: JSON.stringify({
|
|
96
114
|
name_encrypted: JSON.stringify(nameEnvelope),
|
|
97
115
|
nameHash: hashMetadataV4(name.toLowerCase(), rootKeyB64),
|
|
98
116
|
csk_wrapped: cskEnvelope.wrapped,
|
|
99
117
|
csk_iv: cskEnvelope.iv,
|
|
100
|
-
|
|
118
|
+
cipherVersion: CIPHER_VERSION_PQ,
|
|
101
119
|
sizeBytes: size,
|
|
102
|
-
chunkSize: CHUNK_SIZE,
|
|
103
120
|
folderId: null,
|
|
104
121
|
contentType: 'application/octet-stream',
|
|
105
122
|
thumbnail_ciphertext_b64: null,
|
|
@@ -112,7 +129,7 @@ export async function createUploadSession({
|
|
|
112
129
|
`create-upload-session failed (HTTP ${res.status}): ${JSON.stringify(body)}`,
|
|
113
130
|
)
|
|
114
131
|
}
|
|
115
|
-
return { session: body, csk
|
|
132
|
+
return { session: body, csk }
|
|
116
133
|
}
|
|
117
134
|
|
|
118
135
|
// Direct upload: a presigned S3 PUT (web b52b4c8 / PR #726), not b2_upload_file.
|
|
@@ -255,6 +272,7 @@ async function finalizeUpload({
|
|
|
255
272
|
partSha1Array,
|
|
256
273
|
proof,
|
|
257
274
|
ciphertextHash,
|
|
275
|
+
nameEncryptedV6,
|
|
258
276
|
}) {
|
|
259
277
|
const res = await fetch(new URL(COMPLETE_PATH, apiBaseUrl), {
|
|
260
278
|
method: 'POST',
|
|
@@ -262,7 +280,7 @@ async function finalizeUpload({
|
|
|
262
280
|
Authorization: `Bearer ${accessToken}`,
|
|
263
281
|
'Content-Type': 'application/json',
|
|
264
282
|
},
|
|
265
|
-
body: JSON.stringify({ fileId, b2FileId, partSha1Array, proof, ciphertextHash }),
|
|
283
|
+
body: JSON.stringify({ fileId, b2FileId, partSha1Array, proof, ciphertextHash, nameEncryptedV6 }),
|
|
266
284
|
})
|
|
267
285
|
const body = await res.json().catch(() => ({}))
|
|
268
286
|
if (!res.ok) {
|
|
@@ -273,28 +291,20 @@ async function finalizeUpload({
|
|
|
273
291
|
return body
|
|
274
292
|
}
|
|
275
293
|
|
|
276
|
-
// Single-chunk file:
|
|
277
|
-
//
|
|
278
|
-
//
|
|
279
|
-
async function uploadDirect({ apiBaseUrl, accessToken, session,
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
if (plaintext !== null) {
|
|
294
|
+
// Single-chunk file: one part (header || frame_0), SHA-1 + proof, PUT it at
|
|
295
|
+
// the presigned URL, finalize. No b2FileId is sent -- the server HEADs the
|
|
296
|
+
// stored object and derives the storage id itself.
|
|
297
|
+
async function uploadDirect({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6 }) {
|
|
298
|
+
let ciphertext = null
|
|
299
|
+
for await (const part of parts) {
|
|
300
|
+
if (ciphertext !== null) {
|
|
284
301
|
throw new Error('Direct upload session is larger than a single chunk.')
|
|
285
302
|
}
|
|
286
|
-
|
|
303
|
+
ciphertext = part
|
|
287
304
|
}
|
|
288
|
-
if (
|
|
289
|
-
|
|
290
|
-
const {
|
|
291
|
-
cskB64: csk,
|
|
292
|
-
noncePrefixB64: noncePrefix,
|
|
293
|
-
chunkIndex: 0,
|
|
294
|
-
plaintext,
|
|
295
|
-
proofKeyHex: session.proofKey,
|
|
296
|
-
chunkSize,
|
|
297
|
-
})
|
|
305
|
+
if (ciphertext === null) throw new Error('Direct upload produced no ciphertext.')
|
|
306
|
+
const sha1 = await sha1Hex(ciphertext)
|
|
307
|
+
const proof = await computeV3UploadProof({ proofKeyHex: session.proofKey, part0: ciphertext })
|
|
298
308
|
|
|
299
309
|
await putDirectPresigned({ session, ciphertext })
|
|
300
310
|
|
|
@@ -307,16 +317,17 @@ async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix
|
|
|
307
317
|
apiBaseUrl,
|
|
308
318
|
accessToken,
|
|
309
319
|
fileId: session.fileId,
|
|
310
|
-
partSha1Array: [
|
|
311
|
-
proof
|
|
312
|
-
ciphertextHash:
|
|
320
|
+
partSha1Array: [sha1],
|
|
321
|
+
proof,
|
|
322
|
+
ciphertextHash: sha1,
|
|
323
|
+
nameEncryptedV6,
|
|
313
324
|
})
|
|
314
325
|
}
|
|
315
326
|
|
|
316
|
-
// Multipart file: stream each
|
|
317
|
-
// b2_upload_part, collect part SHA-1s. The upload
|
|
318
|
-
//
|
|
319
|
-
//
|
|
327
|
+
// Multipart file: stream each encrypted part (one chunk frame each; part 1
|
|
328
|
+
// also carries the header), b2_upload_part, collect part SHA-1s. The upload
|
|
329
|
+
// proof covers the header and the first frame only — the server re-derives it
|
|
330
|
+
// from the start of the stored object, which is exactly part 1. b2FileId is the
|
|
320
331
|
// large-file id from the session; the server calls b2_finish_large_file with
|
|
321
332
|
// the ordered part hashes at finalize.
|
|
322
333
|
// Bounded concurrency for multipart part uploads. The serial version left the
|
|
@@ -336,11 +347,10 @@ function resolveUploadConcurrency() {
|
|
|
336
347
|
return DEFAULT_UPLOAD_CONCURRENCY
|
|
337
348
|
}
|
|
338
349
|
|
|
339
|
-
async function uploadMultipart({ apiBaseUrl, accessToken, session,
|
|
350
|
+
async function uploadMultipart({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6 }) {
|
|
340
351
|
if (!session.b2FileId) {
|
|
341
352
|
throw new Error('Multipart upload session is missing storage id (b2FileId).')
|
|
342
353
|
}
|
|
343
|
-
const chunkSize = session.chunkSize ?? CHUNK_SIZE
|
|
344
354
|
const concurrency = resolveUploadConcurrency()
|
|
345
355
|
|
|
346
356
|
// Part SHA-1s keyed by chunkIndex, NOT push order, so parts finishing out of
|
|
@@ -359,18 +369,13 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
|
|
|
359
369
|
(await refreshUploadPartUrl({ apiBaseUrl, accessToken, fileId: session.fileId }))
|
|
360
370
|
|
|
361
371
|
const inFlight = new Set()
|
|
362
|
-
const dispatch = (chunkIndex,
|
|
372
|
+
const dispatch = (chunkIndex, ciphertext) => {
|
|
363
373
|
const run = (async () => {
|
|
364
374
|
try {
|
|
365
|
-
const encrypted = await
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
plaintext,
|
|
370
|
-
proofKeyHex: chunkIndex === 0 ? session.proofKey : undefined,
|
|
371
|
-
chunkSize: chunkIndex === 0 ? chunkSize : undefined,
|
|
372
|
-
})
|
|
373
|
-
if (chunkIndex === 0) proof = encrypted.proofHex
|
|
375
|
+
const encrypted = { ciphertext, sha1Hex: await sha1Hex(ciphertext) }
|
|
376
|
+
if (chunkIndex === 0) {
|
|
377
|
+
proof = await computeV3UploadProof({ proofKeyHex: session.proofKey, part0: ciphertext })
|
|
378
|
+
}
|
|
374
379
|
|
|
375
380
|
const slot = await acquireUrl()
|
|
376
381
|
const updated = await putPartToBackblaze({
|
|
@@ -396,7 +401,7 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
|
|
|
396
401
|
}
|
|
397
402
|
|
|
398
403
|
let chunkIndex = 0
|
|
399
|
-
for await (const
|
|
404
|
+
for await (const ciphertext of parts) {
|
|
400
405
|
if (firstError) break
|
|
401
406
|
// Backpressure: hold at most `concurrency` parts in flight. inFlight is
|
|
402
407
|
// non-empty when this runs (size >= concurrency >= 1), so race never hangs.
|
|
@@ -405,7 +410,7 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
|
|
|
405
410
|
if (firstError) break
|
|
406
411
|
}
|
|
407
412
|
if (firstError) break
|
|
408
|
-
dispatch(chunkIndex,
|
|
413
|
+
dispatch(chunkIndex, ciphertext)
|
|
409
414
|
chunkIndex += 1
|
|
410
415
|
}
|
|
411
416
|
|
|
@@ -435,35 +440,71 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
|
|
|
435
440
|
partSha1Array,
|
|
436
441
|
proof,
|
|
437
442
|
ciphertextHash,
|
|
443
|
+
nameEncryptedV6,
|
|
438
444
|
})
|
|
439
445
|
}
|
|
440
446
|
|
|
441
|
-
|
|
442
|
-
|
|
447
|
+
// `onPlaintextChunk`, when given, is called with every plaintext chunk in file
|
|
448
|
+
// order, from the same read that feeds encryption. The agent hashes those
|
|
449
|
+
// chunks to record what was actually uploaded; hashing the file separately,
|
|
450
|
+
// before or after, cannot see a change made and undone while it was being read.
|
|
451
|
+
export async function uploadFile({ apiBaseUrl, accessToken, rootKey, name, path, size, onPlaintextChunk }) {
|
|
452
|
+
// Derived before anything is sent, so a vault whose identity cannot be
|
|
453
|
+
// derived fails before a session (and a pending row) exists.
|
|
454
|
+
const recipientPublicKey = await vaultMlKemPublicKey(rootKey)
|
|
455
|
+
const { session, csk } = await createUploadSession({
|
|
443
456
|
apiBaseUrl,
|
|
444
457
|
accessToken,
|
|
445
458
|
rootKey,
|
|
446
459
|
name,
|
|
447
460
|
size,
|
|
448
461
|
})
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
throw new Error('Direct upload session is missing an upload URL.')
|
|
462
|
+
if (!session.proofKey) throw new Error('Upload session is missing its proof key.')
|
|
463
|
+
|
|
464
|
+
// The server's v1 chunk size (5 MiB): each encrypted chunk is one storage part.
|
|
465
|
+
const chunkSize = session.chunkSize ?? V1_CHUNK_SIZE
|
|
466
|
+
async function* plaintextChunks() {
|
|
467
|
+
for await (const chunk of readFileChunks(path, chunkSize)) {
|
|
468
|
+
onPlaintextChunk?.(chunk)
|
|
469
|
+
yield chunk
|
|
458
470
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
471
|
+
}
|
|
472
|
+
const envelopeKey = base64ToBytes(csk)
|
|
473
|
+
const parts = encryptPartsV3({
|
|
474
|
+
plaintextChunks: plaintextChunks(),
|
|
475
|
+
plaintextSize: size,
|
|
476
|
+
chunkSize,
|
|
477
|
+
envelopeKey,
|
|
478
|
+
fileId: uuidToBytes(session.fileId),
|
|
479
|
+
recipientPublicKey,
|
|
480
|
+
})
|
|
481
|
+
// The name, re-sealed with the row id as AAD (v6), replaces the v4 one at
|
|
482
|
+
// finalize, so the backend cannot relabel this row with another file's name.
|
|
483
|
+
const nameEncryptedV6 = JSON.stringify(
|
|
484
|
+
await encryptNameV6({ name, folderKey: rootKey, rowId: session.fileId }),
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
try {
|
|
488
|
+
if (session.uploadKind === 'direct') {
|
|
489
|
+
// No authToken and no storagePath check: since web b52b4c8 (PR #726) the
|
|
490
|
+
// direct path is a presigned S3 PUT, so the signature inside uploadUrl is
|
|
491
|
+
// the only credential and it already binds the object key. Requiring the
|
|
492
|
+
// token the server stopped issuing is what made every <= 5 MiB upload fail
|
|
493
|
+
// from 2026-09-03. The 'large' branch below still needs both.
|
|
494
|
+
if (!session.uploadUrl) {
|
|
495
|
+
throw new Error('Direct upload session is missing an upload URL.')
|
|
496
|
+
}
|
|
497
|
+
await uploadDirect({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6 })
|
|
498
|
+
} else if (session.uploadKind === 'large') {
|
|
499
|
+
if (!session.uploadUrl || !session.authToken) {
|
|
500
|
+
throw new Error('Multipart upload session is missing credentials.')
|
|
501
|
+
}
|
|
502
|
+
await uploadMultipart({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6 })
|
|
503
|
+
} else {
|
|
504
|
+
throw new Error(`Unknown uploadKind "${session.uploadKind}" for "${name}".`)
|
|
463
505
|
}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
throw new Error(`Unknown uploadKind "${session.uploadKind}" for "${name}".`)
|
|
506
|
+
} finally {
|
|
507
|
+
envelopeKey.fill(0)
|
|
467
508
|
}
|
|
468
509
|
|
|
469
510
|
return { fileId: session.fileId }
|
package/src/uploadCrypto.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
// ShieldFive CLI — upload field crypto,
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// ShieldFive CLI — upload field crypto. The key wrap, v4 filename envelope and
|
|
2
|
+
// keyed name hash are ported byte-for-byte from the apps
|
|
3
|
+
// (mobile/src/features/crypto/encryptionCompat.ts); the file body is the web's
|
|
4
|
+
// default format, written by @shieldfive/crypto (see the end of this file). Same @noble versions as the apps
|
|
5
5
|
// (@noble/ciphers@2, @noble/hashes@2) so the bytes match.
|
|
6
6
|
|
|
7
7
|
import { webcrypto } from 'node:crypto'
|
|
@@ -12,6 +12,11 @@ import { hmac } from '@noble/hashes/hmac.js'
|
|
|
12
12
|
import { sha256 } from '@noble/hashes/sha2.js'
|
|
13
13
|
import sodium from 'libsodium-wrappers-sumo'
|
|
14
14
|
|
|
15
|
+
import { parseHeader } from '@shieldfive/crypto/format'
|
|
16
|
+
import { createIdentity } from '@shieldfive/crypto/identity'
|
|
17
|
+
import { encryptStreamPqHybridV1 } from '@shieldfive/crypto/streams/pq-hybrid-v1'
|
|
18
|
+
import { buildUploadProofV3 } from '@shieldfive/crypto/vault'
|
|
19
|
+
|
|
15
20
|
// libsodium's Argon2id (crypto_pwhash) is byte-identical to @noble's at the same
|
|
16
21
|
// params but ~15x faster (native/WASM vs pure JS). It is used for the ENCRYPT
|
|
17
22
|
// side of the v4 filename metadata — the once-per-file KDF that makes `sf sync`
|
|
@@ -43,38 +48,10 @@ export function aesGcmEncrypt(key, iv, plaintext) {
|
|
|
43
48
|
return gcm(key, iv).encrypt(plaintext)
|
|
44
49
|
}
|
|
45
50
|
|
|
46
|
-
// WebCrypto AES-GCM. Its output is ciphertext||tag with a 16-byte (128-bit)
|
|
47
|
-
// tag — byte-identical to @noble's gcm().encrypt for the same key, 12-byte
|
|
48
|
-
// nonce and no additional data — so ciphertexts, the SHA-1 digests Backblaze
|
|
49
|
-
// checks, and the HMAC upload proof are all unchanged. Native WebCrypto runs
|
|
50
|
-
// at ~5 GiB/s versus @noble's ~70 MiB/s, so it is used for the per-chunk file
|
|
51
|
-
// encryption (the throughput-bound path). @noble stays for the once-per-file
|
|
52
|
-
// key wrap and filename metadata, and for all decryption. It is async because
|
|
53
|
-
// subtle.encrypt is; the chunk path (encryptChunkWithDigest) is already async.
|
|
54
|
-
async function aesGcmEncryptSubtle(key, iv, plaintext) {
|
|
55
|
-
const cryptoKey = await webcrypto.subtle.importKey(
|
|
56
|
-
'raw',
|
|
57
|
-
key,
|
|
58
|
-
{ name: 'AES-GCM' },
|
|
59
|
-
false,
|
|
60
|
-
['encrypt'],
|
|
61
|
-
)
|
|
62
|
-
const ciphertext = await webcrypto.subtle.encrypt(
|
|
63
|
-
{ name: 'AES-GCM', iv },
|
|
64
|
-
cryptoKey,
|
|
65
|
-
plaintext,
|
|
66
|
-
)
|
|
67
|
-
return new Uint8Array(ciphertext)
|
|
68
|
-
}
|
|
69
|
-
|
|
70
51
|
export function generateRandomKeyB64(bytes = 32) {
|
|
71
52
|
return bytesToBase64(randomBytes(bytes))
|
|
72
53
|
}
|
|
73
54
|
|
|
74
|
-
export function generateNoncePrefixB64() {
|
|
75
|
-
return bytesToBase64(randomBytes(4))
|
|
76
|
-
}
|
|
77
|
-
|
|
78
55
|
export function wrapKeyB64({ wrappingKeyB64, keyToWrapB64 }) {
|
|
79
56
|
const iv = randomBytes(12)
|
|
80
57
|
const ciphertext = aesGcmEncrypt(
|
|
@@ -148,9 +125,7 @@ export function decryptMetadataV4(payload, rootKeySecret) {
|
|
|
148
125
|
return new TextDecoder().decode(plaintext)
|
|
149
126
|
}
|
|
150
127
|
|
|
151
|
-
// ──
|
|
152
|
-
|
|
153
|
-
const UPLOAD_PROOF_VERSION = 1
|
|
128
|
+
// ── Storage-part digests ────────────────────────────────────────────────────
|
|
154
129
|
|
|
155
130
|
function hexToBytes(hex) {
|
|
156
131
|
const out = new Uint8Array(hex.length / 2)
|
|
@@ -160,89 +135,11 @@ function hexToBytes(hex) {
|
|
|
160
135
|
return out
|
|
161
136
|
}
|
|
162
137
|
|
|
163
|
-
// nonce = 4-byte random prefix || 8-byte big-endian chunk counter.
|
|
164
|
-
export async function encryptFileChunk({ cskB64, noncePrefixB64, chunkIndex, plaintext }) {
|
|
165
|
-
const noncePrefix = base64ToBytes(noncePrefixB64)
|
|
166
|
-
if (noncePrefix.length !== 4) throw new Error('Invalid nonce prefix')
|
|
167
|
-
const nonce = new Uint8Array(12)
|
|
168
|
-
nonce.set(noncePrefix, 0)
|
|
169
|
-
let counter = BigInt(chunkIndex)
|
|
170
|
-
for (let i = 0; i < 8; i += 1) {
|
|
171
|
-
nonce[11 - i] = Number(counter & 0xffn)
|
|
172
|
-
counter >>= 8n
|
|
173
|
-
}
|
|
174
|
-
return aesGcmEncryptSubtle(base64ToBytes(cskB64), nonce, plaintext)
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
function buildUploadProofPrefix({ cipherVersion, chunkSize, noncePrefixBytes }) {
|
|
178
|
-
const size = Math.floor(chunkSize)
|
|
179
|
-
if (!Number.isFinite(size) || size <= 0) {
|
|
180
|
-
throw new Error('Invalid chunk size for upload proof')
|
|
181
|
-
}
|
|
182
|
-
if (noncePrefixBytes.length !== 4) {
|
|
183
|
-
throw new Error('Invalid nonce prefix length for upload proof')
|
|
184
|
-
}
|
|
185
|
-
const prefix = new Uint8Array(1 + 1 + 4 + 4)
|
|
186
|
-
prefix[0] = UPLOAD_PROOF_VERSION
|
|
187
|
-
prefix[1] = cipherVersion
|
|
188
|
-
new DataView(prefix.buffer).setUint32(2, size, false)
|
|
189
|
-
prefix.set(noncePrefixBytes, 6)
|
|
190
|
-
return prefix
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
export function computeAesGcmUploadProof({
|
|
194
|
-
proofKeyHex,
|
|
195
|
-
cipherVersion,
|
|
196
|
-
chunkSize,
|
|
197
|
-
noncePrefixB64,
|
|
198
|
-
ciphertextChunk,
|
|
199
|
-
}) {
|
|
200
|
-
const proofKey = hexToBytes(proofKeyHex)
|
|
201
|
-
const prefix = buildUploadProofPrefix({
|
|
202
|
-
cipherVersion,
|
|
203
|
-
chunkSize,
|
|
204
|
-
noncePrefixBytes: base64ToBytes(noncePrefixB64),
|
|
205
|
-
})
|
|
206
|
-
const payload = new Uint8Array(prefix.length + ciphertextChunk.length)
|
|
207
|
-
payload.set(prefix, 0)
|
|
208
|
-
payload.set(ciphertextChunk, prefix.length)
|
|
209
|
-
return bytesToHex(hmac(sha256, proofKey, payload))
|
|
210
|
-
}
|
|
211
|
-
|
|
212
138
|
export async function sha1Hex(bytes) {
|
|
213
139
|
const digest = await webcrypto.subtle.digest('SHA-1', bytes)
|
|
214
140
|
return bytesToHex(new Uint8Array(digest))
|
|
215
141
|
}
|
|
216
142
|
|
|
217
|
-
// Encrypt one chunk and produce its SHA-1 (for Backblaze) and the upload proof.
|
|
218
|
-
export async function encryptChunkWithDigest({
|
|
219
|
-
cskB64,
|
|
220
|
-
noncePrefixB64,
|
|
221
|
-
chunkIndex,
|
|
222
|
-
plaintext,
|
|
223
|
-
proofKeyHex,
|
|
224
|
-
chunkSize,
|
|
225
|
-
}) {
|
|
226
|
-
const ciphertext = await encryptFileChunk({
|
|
227
|
-
cskB64,
|
|
228
|
-
noncePrefixB64,
|
|
229
|
-
chunkIndex,
|
|
230
|
-
plaintext,
|
|
231
|
-
})
|
|
232
|
-
const digest = await sha1Hex(ciphertext)
|
|
233
|
-
const proofHex =
|
|
234
|
-
proofKeyHex && chunkSize
|
|
235
|
-
? computeAesGcmUploadProof({
|
|
236
|
-
proofKeyHex,
|
|
237
|
-
cipherVersion: 1,
|
|
238
|
-
chunkSize,
|
|
239
|
-
noncePrefixB64,
|
|
240
|
-
ciphertextChunk: ciphertext,
|
|
241
|
-
})
|
|
242
|
-
: undefined
|
|
243
|
-
return { ciphertext, sha1Hex: digest, proofHex }
|
|
244
|
-
}
|
|
245
|
-
|
|
246
143
|
// Multipart ciphertextHash — the value `complete-upload` checks against the
|
|
247
144
|
// server's own computeLargeFileSha1. One part: the part's SHA-1, lowercased.
|
|
248
145
|
// Many parts: SHA-1 over the concatenated raw 20-byte part digests (NOT the
|
|
@@ -262,3 +159,146 @@ export async function getCiphertextHashFromParts(partSha1Array) {
|
|
|
262
159
|
}
|
|
263
160
|
return sha1Hex(bytes)
|
|
264
161
|
}
|
|
162
|
+
|
|
163
|
+
// ── File encryption, v1 wire format, suite 0x03 (cipher_version 3) ───────────
|
|
164
|
+
//
|
|
165
|
+
// What the web app writes by default (encryptModal.tsx, `v1-pq-hybrid`), using
|
|
166
|
+
// the same @shieldfive/crypto stream encoder: a MAC'd header carrying
|
|
167
|
+
// plaintext_size and total_chunks, chunk AAD binding index/total/is_final (so a
|
|
168
|
+
// dropped or reordered chunk fails to decrypt), file_id = the server row id (so
|
|
169
|
+
// the blob cannot be moved onto another row), and ML-KEM-1024 + X25519 hybrid
|
|
170
|
+
// key encapsulation to the owner's vault identity.
|
|
171
|
+
//
|
|
172
|
+
// This replaces the legacy v0 writer (suite 0x01 chunks under a random 4-byte
|
|
173
|
+
// nonce prefix, no header): no truncation detection, no file binding, no PQ.
|
|
174
|
+
// The spec (crypto spec/format-v0.md) forbids new v0 writes.
|
|
175
|
+
|
|
176
|
+
export const V1_CHUNK_SIZE = 5 * 1024 * 1024
|
|
177
|
+
export const CIPHER_VERSION_PQ = 3
|
|
178
|
+
const FRAME_LENGTH_BYTES = 4
|
|
179
|
+
|
|
180
|
+
export function uuidToBytes(id) {
|
|
181
|
+
const hex = String(id).replace(/-/g, '')
|
|
182
|
+
if (!/^[0-9a-f]{32}$/i.test(hex)) throw new Error(`Not a UUID: ${id}`)
|
|
183
|
+
return hexToBytes(hex.toLowerCase())
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// The owner's ML-KEM public key, derived from the root key exactly as the web
|
|
187
|
+
// keyring does (utils/pqIdentity.ts: createIdentity with the root key as the
|
|
188
|
+
// master secret; userId does not affect the key bytes). Derived locally, never
|
|
189
|
+
// taken from the server, so a server cannot substitute a recipient. Cached by a
|
|
190
|
+
// hash of the root key: the public key is not secret, the derivation is not free.
|
|
191
|
+
const identityCache = new Map()
|
|
192
|
+
export async function vaultMlKemPublicKey(rootKey) {
|
|
193
|
+
const id = bytesToHex(sha256(rootKey))
|
|
194
|
+
let pk = identityCache.get(id)
|
|
195
|
+
if (!pk) {
|
|
196
|
+
const identity = await createIdentity({ userId: 'shieldfive-cli', masterSecret: rootKey })
|
|
197
|
+
identity.mlKemSecretKey?.fill?.(0)
|
|
198
|
+
pk = identity.publicBundle.mlKemPublicKey
|
|
199
|
+
identityCache.set(id, pk)
|
|
200
|
+
}
|
|
201
|
+
return pk
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function toReadableStream(iterable) {
|
|
205
|
+
const it = iterable[Symbol.asyncIterator]()
|
|
206
|
+
return new ReadableStream({
|
|
207
|
+
async pull(controller) {
|
|
208
|
+
const { value, done } = await it.next()
|
|
209
|
+
if (done) controller.close()
|
|
210
|
+
else controller.enqueue(value)
|
|
211
|
+
},
|
|
212
|
+
async cancel() {
|
|
213
|
+
await it.return?.()
|
|
214
|
+
},
|
|
215
|
+
})
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Encrypt a plaintext chunk stream as one suite-0x03 object and yield it as
|
|
220
|
+
* storage parts: part 0 is `header || frame_0`, part i is `frame_i`. That is the
|
|
221
|
+
* layout the server's proof verifier and the web uploader use (one encrypted
|
|
222
|
+
* chunk per storage part). The split is made from the bytes themselves (header
|
|
223
|
+
* length, then each frame's length prefix), not from how the encoder happens
|
|
224
|
+
* to enqueue its output.
|
|
225
|
+
*/
|
|
226
|
+
export async function* encryptPartsV3({
|
|
227
|
+
plaintextChunks,
|
|
228
|
+
plaintextSize,
|
|
229
|
+
chunkSize,
|
|
230
|
+
envelopeKey,
|
|
231
|
+
fileId,
|
|
232
|
+
recipientPublicKey,
|
|
233
|
+
}) {
|
|
234
|
+
const { ciphertext, combinedKey } = await encryptStreamPqHybridV1(toReadableStream(plaintextChunks), {
|
|
235
|
+
recipientPublicKey,
|
|
236
|
+
envelopeKey,
|
|
237
|
+
plaintextSize,
|
|
238
|
+
chunkSize,
|
|
239
|
+
fileId,
|
|
240
|
+
})
|
|
241
|
+
// combinedKey is used by the stream while it runs; wiped in the finally below.
|
|
242
|
+
|
|
243
|
+
const reader = ciphertext.getReader()
|
|
244
|
+
let buf = new Uint8Array(0)
|
|
245
|
+
let done = false
|
|
246
|
+
const pull = async () => {
|
|
247
|
+
const r = await reader.read()
|
|
248
|
+
if (r.done) {
|
|
249
|
+
done = true
|
|
250
|
+
return
|
|
251
|
+
}
|
|
252
|
+
const next = new Uint8Array(buf.length + r.value.length)
|
|
253
|
+
next.set(buf, 0)
|
|
254
|
+
next.set(r.value, buf.length)
|
|
255
|
+
buf = next
|
|
256
|
+
}
|
|
257
|
+
const take = (n) => {
|
|
258
|
+
const out = buf.slice(0, n)
|
|
259
|
+
buf = buf.slice(n)
|
|
260
|
+
return out
|
|
261
|
+
}
|
|
262
|
+
const readFrame = async () => {
|
|
263
|
+
while (buf.length < FRAME_LENGTH_BYTES && !done) await pull()
|
|
264
|
+
if (buf.length === 0 && done) return null
|
|
265
|
+
if (buf.length < FRAME_LENGTH_BYTES) throw new Error('Encrypted stream ended inside a chunk frame.')
|
|
266
|
+
const len = new DataView(buf.buffer, buf.byteOffset, buf.byteLength).getUint32(0, false)
|
|
267
|
+
while (buf.length < FRAME_LENGTH_BYTES + len && !done) await pull()
|
|
268
|
+
if (buf.length < FRAME_LENGTH_BYTES + len) throw new Error('Encrypted stream ended inside a chunk frame.')
|
|
269
|
+
return take(FRAME_LENGTH_BYTES + len)
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
try {
|
|
273
|
+
// Header: parse as soon as enough bytes are buffered.
|
|
274
|
+
let headerLength = null
|
|
275
|
+
while (headerLength === null) {
|
|
276
|
+
try {
|
|
277
|
+
headerLength = parseHeader(buf).headerLength
|
|
278
|
+
} catch (err) {
|
|
279
|
+
if (done) throw err
|
|
280
|
+
await pull()
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const header = take(headerLength)
|
|
284
|
+
const first = await readFrame()
|
|
285
|
+
if (!first) throw new Error('Encrypted stream has no chunks.')
|
|
286
|
+
const part0 = new Uint8Array(header.length + first.length)
|
|
287
|
+
part0.set(header, 0)
|
|
288
|
+
part0.set(first, header.length)
|
|
289
|
+
yield part0
|
|
290
|
+
for (;;) {
|
|
291
|
+
const frame = await readFrame()
|
|
292
|
+
if (!frame) break
|
|
293
|
+
yield frame
|
|
294
|
+
}
|
|
295
|
+
} finally {
|
|
296
|
+
reader.releaseLock()
|
|
297
|
+
combinedKey.fill(0)
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** The suite-0x03 upload proof, from the library both other writers use. */
|
|
302
|
+
export function computeV3UploadProof({ proofKeyHex, part0 }) {
|
|
303
|
+
return buildUploadProofV3({ proofKeyHex, ciphertext: part0 })
|
|
304
|
+
}
|