@shieldfive/cli 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/upload.mjs CHANGED
@@ -10,25 +10,41 @@
10
10
  // (server-side) at finalize. Unchanged.
11
11
  //
12
12
  // Chunks are read from disk one at a time (never the whole file into memory),
13
- // so a multi-gigabyte upload holds at most one chunk (~5 MiB) resident.
13
+ // so a multi-gigabyte upload holds at most a few chunks (~5 MiB each) resident.
14
+ //
15
+ // Format: cipher_version 3, the web's default (suite 0x03, v1 wire format; see
16
+ // uploadCrypto.mjs). Every storage part is one encrypted chunk frame; part 1
17
+ // also carries the header. The header's file_id is the row id the server
18
+ // assigned at create-upload-session, so the stored object is bound to its row.
14
19
 
20
+ import { constants as fsConstants } from 'node:fs'
15
21
  import { open } from 'node:fs/promises'
22
+ import { webcrypto } from 'node:crypto'
23
+
24
+ import { encryptNameV6 } from '@shieldfive/crypto/vault'
16
25
 
17
26
  import {
27
+ base64ToBytes,
18
28
  bytesToBase64,
19
- encryptChunkWithDigest,
20
- encryptMetadataV4,
21
- generateNoncePrefixB64,
29
+ CIPHER_VERSION_PQ,
30
+ computeV3EmptyUploadProof,
31
+ computeV3UploadProof,
32
+ encryptPartsV3,
22
33
  generateRandomKeyB64,
23
34
  getCiphertextHashFromParts,
24
35
  hashMetadataV4,
36
+ sha1Hex,
37
+ uuidToBytes,
38
+ V1_CHUNK_SIZE,
39
+ vaultMlKemPublicKey,
25
40
  wrapKeyB64,
26
41
  } from './uploadCrypto.mjs'
27
42
 
28
43
  const CREATE_SESSION_PATH = '/api/files/create-upload-session'
29
44
  const COMPLETE_PATH = '/api/files/complete-upload'
30
45
  const PART_URL_PATH = '/api/files/upload-part-url'
31
- const CHUNK_SIZE = 5 * 1024 * 1024
46
+ const DELETE_PATH = '/api/files/delete'
47
+ const CAPABILITIES_PATH = '/api/files/upload-capabilities'
32
48
 
33
49
  // Backblaze documents retrying b2_upload_part on 5xx / network failures, and a
34
50
  // part upload is idempotent by (part number, SHA-1) — retrying overwrites the
@@ -44,8 +60,16 @@ const partRetryDelayMs = (attempt) =>
44
60
  // Read a file in fixed-size chunks without holding it all in memory. Each yield
45
61
  // is an independent copy (the read buffer is reused), so the caller may retain
46
62
  // or transform a chunk across the next read safely.
63
+ //
64
+ // O_NOFOLLOW: callers list files with lstat and skip symlinks, but a regular
65
+ // file could be swapped for a symlink between that check and this open. With
66
+ // O_NOFOLLOW the open fails (ELOOP) instead of uploading whatever the link
67
+ // points at, possibly outside the folder being pushed. (Undefined on Windows,
68
+ // where it is simply not applied.)
69
+ const OPEN_READ_NOFOLLOW = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0)
70
+
47
71
  export async function* readFileChunks(path, chunkSize) {
48
- const handle = await open(path, 'r')
72
+ const handle = await open(path, OPEN_READ_NOFOLLOW)
49
73
  try {
50
74
  const buffer = Buffer.allocUnsafe(chunkSize)
51
75
  for (;;) {
@@ -73,18 +97,40 @@ export async function* readFileChunks(path, chunkSize) {
73
97
  }
74
98
  }
75
99
 
100
+ // A fresh UUIDv7 (RFC 9562): 48-bit ms timestamp, version 7, variant 10.
101
+ // The web app's newRowBoundId (utils/fileRowBinding.ts) builds the same shape;
102
+ // the server's create-upload-session takes it as clientUploadId.
103
+ export function newRowBoundId(now = Date.now()) {
104
+ const bytes = webcrypto.getRandomValues(new Uint8Array(16))
105
+ let ts = BigInt(now)
106
+ for (let i = 5; i >= 0; i -= 1) {
107
+ bytes[i] = Number(ts & 0xffn)
108
+ ts >>= 8n
109
+ }
110
+ bytes[6] = (bytes[6] & 0x0f) | 0x70
111
+ bytes[8] = (bytes[8] & 0x3f) | 0x80
112
+ const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('')
113
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
114
+ }
115
+
76
116
  export async function createUploadSession({
77
117
  apiBaseUrl,
78
118
  accessToken,
79
119
  rootKey,
80
120
  name,
81
121
  size,
122
+ rowId,
82
123
  }) {
83
124
  const rootKeyB64 = bytesToBase64(rootKey)
84
- const nameEnvelope = await encryptMetadataV4(name, rootKeyB64)
125
+ // The client picks the row id (UUIDv7, sent as clientUploadId), so the very
126
+ // first stored name is already the v6 envelope bound to that row. No v4
127
+ // (row-unbound) name is ever written, even for an upload that fails before
128
+ // finalize.
129
+ const nameEnvelopeV6 = await encryptNameV6({ name, folderKey: rootKey, rowId })
130
+ // For suite 0x03 this is the classical envelope key; csk_wrapped holds it
131
+ // under the parent key (the root key: sf uploads to the vault root).
85
132
  const csk = generateRandomKeyB64()
86
133
  const cskEnvelope = wrapKeyB64({ wrappingKeyB64: rootKeyB64, keyToWrapB64: csk })
87
- const noncePrefix = generateNoncePrefixB64()
88
134
 
89
135
  const res = await fetch(new URL(CREATE_SESSION_PATH, apiBaseUrl), {
90
136
  method: 'POST',
@@ -92,14 +138,16 @@ export async function createUploadSession({
92
138
  Authorization: `Bearer ${accessToken}`,
93
139
  'Content-Type': 'application/json',
94
140
  },
141
+ // v1 wire format: no cipher_nonce_prefix and no chunkSize (the server
142
+ // rejects both; they live in the header).
95
143
  body: JSON.stringify({
96
- name_encrypted: JSON.stringify(nameEnvelope),
144
+ clientUploadId: rowId,
145
+ name_encrypted: JSON.stringify(nameEnvelopeV6),
97
146
  nameHash: hashMetadataV4(name.toLowerCase(), rootKeyB64),
98
147
  csk_wrapped: cskEnvelope.wrapped,
99
148
  csk_iv: cskEnvelope.iv,
100
- cipher_nonce_prefix: noncePrefix,
149
+ cipherVersion: CIPHER_VERSION_PQ,
101
150
  sizeBytes: size,
102
- chunkSize: CHUNK_SIZE,
103
151
  folderId: null,
104
152
  contentType: 'application/octet-stream',
105
153
  thumbnail_ciphertext_b64: null,
@@ -112,7 +160,7 @@ export async function createUploadSession({
112
160
  `create-upload-session failed (HTTP ${res.status}): ${JSON.stringify(body)}`,
113
161
  )
114
162
  }
115
- return { session: body, csk, noncePrefix }
163
+ return { session: body, csk, nameEncryptedV6: JSON.stringify(nameEnvelopeV6) }
116
164
  }
117
165
 
118
166
  // Direct upload: a presigned S3 PUT (web b52b4c8 / PR #726), not b2_upload_file.
@@ -255,15 +303,24 @@ async function finalizeUpload({
255
303
  partSha1Array,
256
304
  proof,
257
305
  ciphertextHash,
306
+ nameEncryptedV6,
258
307
  }) {
259
- const res = await fetch(new URL(COMPLETE_PATH, apiBaseUrl), {
260
- method: 'POST',
261
- headers: {
262
- Authorization: `Bearer ${accessToken}`,
263
- 'Content-Type': 'application/json',
264
- },
265
- body: JSON.stringify({ fileId, b2FileId, partSha1Array, proof, ciphertextHash }),
266
- })
308
+ let res
309
+ try {
310
+ res = await fetch(new URL(COMPLETE_PATH, apiBaseUrl), {
311
+ method: 'POST',
312
+ headers: {
313
+ Authorization: `Bearer ${accessToken}`,
314
+ 'Content-Type': 'application/json',
315
+ },
316
+ body: JSON.stringify({ fileId, b2FileId, partSha1Array, proof, ciphertextHash, nameEncryptedV6 }),
317
+ })
318
+ } catch (err) {
319
+ // The request may have reached the server and finalized the file; only the
320
+ // answer was lost. Cancelling now could bin a file that is in the vault.
321
+ err.finalizeOutcomeUnknown = true
322
+ throw err
323
+ }
267
324
  const body = await res.json().catch(() => ({}))
268
325
  if (!res.ok) {
269
326
  throw new Error(
@@ -273,29 +330,24 @@ async function finalizeUpload({
273
330
  return body
274
331
  }
275
332
 
276
- // Single-chunk file: encrypt the one chunk (AES-GCM, suite 0x01) + SHA-1 +
277
- // proof, PUT it at the presigned URL, finalize. No b2FileId is sent -- the
278
- // server HEADs the stored object and derives the storage id itself.
279
- async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk }) {
280
- const chunkSize = session.chunkSize ?? CHUNK_SIZE
281
- let plaintext = null
282
- for await (const chunk of readFileChunks(path, chunkSize)) {
283
- if (plaintext !== null) {
333
+ // Single-chunk file: one part (header || frame_0), SHA-1 + proof, PUT it at
334
+ // the presigned URL, finalize. No b2FileId is sent -- the server HEADs the
335
+ // stored object and derives the storage id itself.
336
+ async function uploadDirect({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6, empty = false }) {
337
+ let ciphertext = null
338
+ for await (const part of parts) {
339
+ if (ciphertext !== null) {
284
340
  throw new Error('Direct upload session is larger than a single chunk.')
285
341
  }
286
- plaintext = chunk
287
- onPlaintextChunk?.(chunk)
342
+ ciphertext = part
288
343
  }
289
- if (plaintext === null) plaintext = new Uint8Array(0)
290
-
291
- const { ciphertext, sha1Hex, proofHex } = await encryptChunkWithDigest({
292
- cskB64: csk,
293
- noncePrefixB64: noncePrefix,
294
- chunkIndex: 0,
295
- plaintext,
296
- proofKeyHex: session.proofKey,
297
- chunkSize,
298
- })
344
+ if (ciphertext === null) throw new Error('Direct upload produced no ciphertext.')
345
+ // For a 0-byte file part 1 is the header alone, and the proof covers it:
346
+ // the server's empty-file contract is HMAC(proofKey, [3][3] || header).
347
+ const sha1 = await sha1Hex(ciphertext)
348
+ const proof = empty
349
+ ? computeV3EmptyUploadProof({ proofKeyHex: session.proofKey, header: ciphertext })
350
+ : await computeV3UploadProof({ proofKeyHex: session.proofKey, part0: ciphertext })
299
351
 
300
352
  await putDirectPresigned({ session, ciphertext })
301
353
 
@@ -308,16 +360,17 @@ async function uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix
308
360
  apiBaseUrl,
309
361
  accessToken,
310
362
  fileId: session.fileId,
311
- partSha1Array: [sha1Hex],
312
- proof: proofHex,
313
- ciphertextHash: sha1Hex,
363
+ partSha1Array: [sha1],
364
+ proof,
365
+ ciphertextHash: sha1,
366
+ nameEncryptedV6,
314
367
  })
315
368
  }
316
369
 
317
- // Multipart file: stream each chunk, encrypt (per-chunk nonce via chunkIndex),
318
- // b2_upload_part, collect part SHA-1s. The upload proof is computed from the
319
- // FIRST chunk only — the server re-derives it from the first chunkSize+tag
320
- // bytes of the stored object, which is exactly part 1. b2FileId is the
370
+ // Multipart file: stream each encrypted part (one chunk frame each; part 1
371
+ // also carries the header), b2_upload_part, collect part SHA-1s. The upload
372
+ // proof covers the header and the first frame only — the server re-derives it
373
+ // from the start of the stored object, which is exactly part 1. b2FileId is the
321
374
  // large-file id from the session; the server calls b2_finish_large_file with
322
375
  // the ordered part hashes at finalize.
323
376
  // Bounded concurrency for multipart part uploads. The serial version left the
@@ -337,11 +390,10 @@ function resolveUploadConcurrency() {
337
390
  return DEFAULT_UPLOAD_CONCURRENCY
338
391
  }
339
392
 
340
- async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk }) {
393
+ async function uploadMultipart({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6 }) {
341
394
  if (!session.b2FileId) {
342
395
  throw new Error('Multipart upload session is missing storage id (b2FileId).')
343
396
  }
344
- const chunkSize = session.chunkSize ?? CHUNK_SIZE
345
397
  const concurrency = resolveUploadConcurrency()
346
398
 
347
399
  // Part SHA-1s keyed by chunkIndex, NOT push order, so parts finishing out of
@@ -360,18 +412,13 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
360
412
  (await refreshUploadPartUrl({ apiBaseUrl, accessToken, fileId: session.fileId }))
361
413
 
362
414
  const inFlight = new Set()
363
- const dispatch = (chunkIndex, plaintext) => {
415
+ const dispatch = (chunkIndex, ciphertext) => {
364
416
  const run = (async () => {
365
417
  try {
366
- const encrypted = await encryptChunkWithDigest({
367
- cskB64: csk,
368
- noncePrefixB64: noncePrefix,
369
- chunkIndex,
370
- plaintext,
371
- proofKeyHex: chunkIndex === 0 ? session.proofKey : undefined,
372
- chunkSize: chunkIndex === 0 ? chunkSize : undefined,
373
- })
374
- if (chunkIndex === 0) proof = encrypted.proofHex
418
+ const encrypted = { ciphertext, sha1Hex: await sha1Hex(ciphertext) }
419
+ if (chunkIndex === 0) {
420
+ proof = await computeV3UploadProof({ proofKeyHex: session.proofKey, part0: ciphertext })
421
+ }
375
422
 
376
423
  const slot = await acquireUrl()
377
424
  const updated = await putPartToBackblaze({
@@ -397,8 +444,7 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
397
444
  }
398
445
 
399
446
  let chunkIndex = 0
400
- for await (const plaintext of readFileChunks(path, chunkSize)) {
401
- onPlaintextChunk?.(plaintext)
447
+ for await (const ciphertext of parts) {
402
448
  if (firstError) break
403
449
  // Backpressure: hold at most `concurrency` parts in flight. inFlight is
404
450
  // non-empty when this runs (size >= concurrency >= 1), so race never hangs.
@@ -407,7 +453,7 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
407
453
  if (firstError) break
408
454
  }
409
455
  if (firstError) break
410
- dispatch(chunkIndex, plaintext)
456
+ dispatch(chunkIndex, ciphertext)
411
457
  chunkIndex += 1
412
458
  }
413
459
 
@@ -437,22 +483,126 @@ async function uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePre
437
483
  partSha1Array,
438
484
  proof,
439
485
  ciphertextHash,
486
+ nameEncryptedV6,
440
487
  })
441
488
  }
442
489
 
490
+ // Best-effort cancel of a session that will never finalize. The pending row
491
+ // otherwise holds quota (and, for multipart, an unfinished Backblaze large
492
+ // file) until the server's stale-upload reaper gets to it. POST
493
+ // /api/files/delete with the row id is what the web app's cancel path uses:
494
+ // for a non-ready row it cancels the large file and drops the metadata. Never
495
+ // throws; a failure here must not hide the error that caused it.
496
+ export async function cancelUploadSession({ apiBaseUrl, accessToken, fileId }) {
497
+ try {
498
+ const res = await fetch(new URL(DELETE_PATH, apiBaseUrl), {
499
+ method: 'POST',
500
+ headers: {
501
+ Authorization: `Bearer ${accessToken}`,
502
+ 'Content-Type': 'application/json',
503
+ },
504
+ body: JSON.stringify({ ids: [fileId] }),
505
+ })
506
+ return res.ok
507
+ } catch {
508
+ return false
509
+ }
510
+ }
511
+
512
+ export class EmptyFileUnsupportedError extends Error {
513
+ constructor(name) {
514
+ super(
515
+ `"${name}" is empty (0 bytes) and this ShieldFive server does not accept ` +
516
+ 'empty files yet. Nothing was uploaded for it.',
517
+ )
518
+ this.name = 'EmptyFileUnsupportedError'
519
+ this.code = 'empty_file_unsupported'
520
+ }
521
+ }
522
+
523
+ // Zero-byte cipher-v3 uploads are behind a server capability (web
524
+ // docs/desktop-empty-file-contract.md). Ask before creating a session, so an
525
+ // older or flag-off server gets no pending row at all.
526
+ async function serverAcceptsEmptyFiles({ apiBaseUrl, accessToken }) {
527
+ try {
528
+ const res = await fetch(new URL(CAPABILITIES_PATH, apiBaseUrl), {
529
+ headers: { Authorization: `Bearer ${accessToken}` },
530
+ })
531
+ if (!res.ok) return false
532
+ const body = await res.json().catch(() => ({}))
533
+ return body?.emptyCipherV3 === true
534
+ } catch {
535
+ return false
536
+ }
537
+ }
538
+
443
539
  // `onPlaintextChunk`, when given, is called with every plaintext chunk in file
444
540
  // order, from the same read that feeds encryption. The agent hashes those
445
541
  // chunks to record what was actually uploaded; hashing the file separately,
446
542
  // before or after, cannot see a change made and undone while it was being read.
447
- export async function uploadFile({ apiBaseUrl, accessToken, rootKey, name, path, size, onPlaintextChunk }) {
448
- const { session, csk, noncePrefix } = await createUploadSession({
543
+ // `rowId` is for tests only; callers let it default to a fresh UUIDv7.
544
+ export async function uploadFile({ apiBaseUrl, accessToken, rootKey, name, path, size, onPlaintextChunk, rowId = newRowBoundId() }) {
545
+ // Derived before anything is sent, so a vault whose identity cannot be
546
+ // derived fails before a session (and a pending row) exists.
547
+ const recipientPublicKey = await vaultMlKemPublicKey(rootKey)
548
+ if (size === 0 && !(await serverAcceptsEmptyFiles({ apiBaseUrl, accessToken }))) {
549
+ throw new EmptyFileUnsupportedError(name)
550
+ }
551
+ const { session, csk, nameEncryptedV6 } = await createUploadSession({
449
552
  apiBaseUrl,
450
553
  accessToken,
451
554
  rootKey,
452
555
  name,
453
556
  size,
557
+ rowId,
454
558
  })
559
+ // From here on a pending row exists. Any failure before finalize answers
560
+ // cancels it (see cancelUploadSession).
561
+ const envelopeKey = base64ToBytes(csk)
562
+ try {
563
+ // The name and the file header are bound to rowId. A server that ignored
564
+ // clientUploadId would hand back another id; refuse it (and cancel the
565
+ // row it made) rather than upload under a mismatched binding.
566
+ if (session?.fileId !== rowId) {
567
+ throw new Error(
568
+ `create-upload-session returned file id ${session?.fileId} for requested ${rowId}; ` +
569
+ 'this ShieldFive server is too old for this client.',
570
+ )
571
+ }
572
+ await sendUpload({ apiBaseUrl, accessToken, name, path, size, onPlaintextChunk, session, envelopeKey, recipientPublicKey, nameEncryptedV6 })
573
+ } catch (err) {
574
+ if (!err?.finalizeOutcomeUnknown && session?.fileId) {
575
+ await cancelUploadSession({ apiBaseUrl, accessToken, fileId: session.fileId })
576
+ }
577
+ throw err
578
+ } finally {
579
+ envelopeKey.fill(0)
580
+ }
581
+
582
+ return { fileId: session.fileId }
583
+ }
584
+
585
+ async function sendUpload({ apiBaseUrl, accessToken, name, path, size, onPlaintextChunk, session, envelopeKey, recipientPublicKey, nameEncryptedV6 }) {
586
+ if (!session.proofKey) throw new Error('Upload session is missing its proof key.')
455
587
 
588
+ // The server's v1 chunk size (5 MiB): each encrypted chunk is one storage part.
589
+ const chunkSize = session.chunkSize ?? V1_CHUNK_SIZE
590
+ async function* plaintextChunks() {
591
+ for await (const chunk of readFileChunks(path, chunkSize)) {
592
+ onPlaintextChunk?.(chunk)
593
+ yield chunk
594
+ }
595
+ }
596
+ const parts = encryptPartsV3({
597
+ plaintextChunks: plaintextChunks(),
598
+ plaintextSize: size,
599
+ chunkSize,
600
+ envelopeKey,
601
+ fileId: uuidToBytes(session.fileId),
602
+ recipientPublicKey,
603
+ })
604
+ // The same v6 name sent at create (bound to the row id) is sent again at
605
+ // finalize, which still expects it.
456
606
  if (session.uploadKind === 'direct') {
457
607
  // No authToken and no storagePath check: since web b52b4c8 (PR #726) the
458
608
  // direct path is a presigned S3 PUT, so the signature inside uploadUrl is
@@ -462,15 +612,13 @@ export async function uploadFile({ apiBaseUrl, accessToken, rootKey, name, path,
462
612
  if (!session.uploadUrl) {
463
613
  throw new Error('Direct upload session is missing an upload URL.')
464
614
  }
465
- await uploadDirect({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk })
615
+ await uploadDirect({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6, empty: size === 0 })
466
616
  } else if (session.uploadKind === 'large') {
467
617
  if (!session.uploadUrl || !session.authToken) {
468
618
  throw new Error('Multipart upload session is missing credentials.')
469
619
  }
470
- await uploadMultipart({ apiBaseUrl, accessToken, session, csk, noncePrefix, path, onPlaintextChunk })
620
+ await uploadMultipart({ apiBaseUrl, accessToken, session, parts, nameEncryptedV6 })
471
621
  } else {
472
622
  throw new Error(`Unknown uploadKind "${session.uploadKind}" for "${name}".`)
473
623
  }
474
-
475
- return { fileId: session.fileId }
476
624
  }