@protontech/drive-sdk 0.0.12 → 0.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/errors.d.ts +7 -3
  2. package/dist/errors.js +9 -4
  3. package/dist/errors.js.map +1 -1
  4. package/dist/interface/index.d.ts +1 -1
  5. package/dist/interface/nodes.d.ts +12 -1
  6. package/dist/interface/nodes.js +11 -0
  7. package/dist/interface/nodes.js.map +1 -1
  8. package/dist/interface/upload.d.ts +51 -3
  9. package/dist/internal/apiService/driveTypes.d.ts +1341 -465
  10. package/dist/internal/apiService/errors.js +2 -2
  11. package/dist/internal/apiService/errors.js.map +1 -1
  12. package/dist/internal/apiService/transformers.js +2 -0
  13. package/dist/internal/apiService/transformers.js.map +1 -1
  14. package/dist/internal/asyncIteratorMap.d.ts +15 -0
  15. package/dist/internal/asyncIteratorMap.js +59 -0
  16. package/dist/internal/asyncIteratorMap.js.map +1 -0
  17. package/dist/internal/asyncIteratorMap.test.d.ts +1 -0
  18. package/dist/internal/asyncIteratorMap.test.js +120 -0
  19. package/dist/internal/asyncIteratorMap.test.js.map +1 -0
  20. package/dist/internal/nodes/apiService.d.ts +2 -2
  21. package/dist/internal/nodes/apiService.js +16 -6
  22. package/dist/internal/nodes/apiService.js.map +1 -1
  23. package/dist/internal/nodes/apiService.test.js +30 -8
  24. package/dist/internal/nodes/apiService.test.js.map +1 -1
  25. package/dist/internal/nodes/cache.js +1 -0
  26. package/dist/internal/nodes/cache.js.map +1 -1
  27. package/dist/internal/nodes/cache.test.js +1 -0
  28. package/dist/internal/nodes/cache.test.js.map +1 -1
  29. package/dist/internal/nodes/cryptoService.test.js +34 -0
  30. package/dist/internal/nodes/cryptoService.test.js.map +1 -1
  31. package/dist/internal/nodes/index.test.js +3 -1
  32. package/dist/internal/nodes/index.test.js.map +1 -1
  33. package/dist/internal/nodes/interface.d.ts +3 -1
  34. package/dist/internal/nodes/nodesAccess.js +28 -7
  35. package/dist/internal/nodes/nodesAccess.js.map +1 -1
  36. package/dist/internal/nodes/nodesAccess.test.js +7 -6
  37. package/dist/internal/nodes/nodesAccess.test.js.map +1 -1
  38. package/dist/internal/sharing/apiService.js +19 -2
  39. package/dist/internal/sharing/apiService.js.map +1 -1
  40. package/dist/internal/upload/fileUploader.d.ts +49 -53
  41. package/dist/internal/upload/fileUploader.js +91 -395
  42. package/dist/internal/upload/fileUploader.js.map +1 -1
  43. package/dist/internal/upload/fileUploader.test.js +38 -292
  44. package/dist/internal/upload/fileUploader.test.js.map +1 -1
  45. package/dist/internal/upload/index.d.ts +3 -3
  46. package/dist/internal/upload/index.js +20 -41
  47. package/dist/internal/upload/index.js.map +1 -1
  48. package/dist/internal/upload/manager.d.ts +1 -1
  49. package/dist/internal/upload/manager.js +16 -19
  50. package/dist/internal/upload/manager.js.map +1 -1
  51. package/dist/internal/upload/manager.test.js +42 -83
  52. package/dist/internal/upload/manager.test.js.map +1 -1
  53. package/dist/internal/upload/streamUploader.d.ts +62 -0
  54. package/dist/internal/upload/streamUploader.js +441 -0
  55. package/dist/internal/upload/streamUploader.js.map +1 -0
  56. package/dist/internal/upload/streamUploader.test.d.ts +1 -0
  57. package/dist/internal/upload/streamUploader.test.js +358 -0
  58. package/dist/internal/upload/streamUploader.test.js.map +1 -0
  59. package/dist/protonDriveClient.d.ts +4 -4
  60. package/dist/protonDriveClient.js +1 -1
  61. package/dist/protonDriveClient.js.map +1 -1
  62. package/package.json +2 -2
  63. package/src/errors.ts +10 -4
  64. package/src/interface/index.ts +1 -1
  65. package/src/interface/nodes.ts +11 -0
  66. package/src/interface/upload.ts +53 -3
  67. package/src/internal/apiService/driveTypes.ts +1341 -465
  68. package/src/internal/apiService/errors.ts +3 -2
  69. package/src/internal/apiService/transformers.ts +2 -0
  70. package/src/internal/asyncIteratorMap.test.ts +150 -0
  71. package/src/internal/asyncIteratorMap.ts +64 -0
  72. package/src/internal/nodes/apiService.test.ts +36 -7
  73. package/src/internal/nodes/apiService.ts +19 -7
  74. package/src/internal/nodes/cache.test.ts +1 -0
  75. package/src/internal/nodes/cache.ts +1 -0
  76. package/src/internal/nodes/cryptoService.test.ts +38 -0
  77. package/src/internal/nodes/index.test.ts +3 -1
  78. package/src/internal/nodes/interface.ts +4 -1
  79. package/src/internal/nodes/nodesAccess.test.ts +7 -6
  80. package/src/internal/nodes/nodesAccess.ts +30 -7
  81. package/src/internal/sharing/apiService.ts +24 -2
  82. package/src/internal/upload/fileUploader.test.ts +46 -376
  83. package/src/internal/upload/fileUploader.ts +114 -494
  84. package/src/internal/upload/index.ts +26 -50
  85. package/src/internal/upload/manager.test.ts +45 -92
  86. package/src/internal/upload/manager.ts +30 -32
  87. package/src/internal/upload/streamUploader.test.ts +469 -0
  88. package/src/internal/upload/streamUploader.ts +552 -0
  89. package/src/protonDriveClient.ts +5 -4
@@ -1,94 +1,37 @@
1
- import { c } from "ttag";
2
-
3
- import { Thumbnail, Logger, ThumbnailType, UploadMetadata } from "../../interface";
4
- import { IntegrityError } from "../../errors";
5
- import { LoggerWithPrefix } from "../../telemetry";
6
- import { APIHTTPError, HTTPErrorCode, NotFoundAPIError } from "../apiService";
7
- import { getErrorMessage } from "../errors";
8
- import { mergeUint8Arrays } from "../utils";
9
- import { waitForCondition } from '../wait';
1
+ import { Thumbnail, UploadMetadata } from "../../interface";
10
2
  import { UploadAPIService } from "./apiService";
11
3
  import { BlockVerifier } from "./blockVerifier";
12
4
  import { UploadController } from './controller';
13
5
  import { UploadCryptoService } from "./cryptoService";
14
- import { UploadDigests } from "./digests";
15
- import { NodeRevisionDraft, EncryptedBlock, EncryptedThumbnail, EncryptedBlockMetadata } from "./interface";
16
- import { UploadTelemetry } from './telemetry';
17
- import { ChunkStreamReader } from './chunkStreamReader';
6
+ import { NodeRevisionDraft } from "./interface";
18
7
  import { UploadManager } from "./manager";
8
+ import { StreamUploader } from './streamUploader';
9
+ import { UploadTelemetry } from './telemetry';
19
10
 
20
11
  /**
21
- * File chunk size in bytes representing the size of each block.
22
- */
23
- export const FILE_CHUNK_SIZE = 4 * 1024 * 1024;
24
-
25
- /**
26
- * Maximum number of blocks that can be buffered before upload.
27
- * This is to prevent using too much memory.
28
- */
29
- const MAX_BUFFERED_BLOCKS = 15;
30
-
31
- /**
32
- * Maximum number of blocks that can be uploaded at the same time.
33
- * This is to prevent overloading the server with too many requests.
34
- */
35
- const MAX_UPLOADING_BLOCKS = 5;
36
-
37
- /**
38
- * Maximum number of retries for block encryption.
39
- * This is to automatically retry random errors that can happen
40
- * during encryption, for example bitflips.
41
- */
42
- const MAX_BLOCK_ENCRYPTION_RETRIES = 1;
43
-
44
- /**
45
- * Maximum number of retries for block upload.
46
- * This is to ensure we don't end up in an infinite loop.
47
- */
48
- const MAX_BLOCK_UPLOAD_RETRIES = 3;
49
-
50
- /**
51
- * Fileuploader is responsible for uploading file content to the server.
52
- *
53
- * It handles the encryption of file blocks and thumbnails, as well as
54
- * the upload process itself. It manages the upload queue and ensures
55
- * that the upload process is efficient and does not overload the server.
12
+ * Uploader is generic class responsible for creating a revision draft
13
+ * and initiate the upload process for a file object or a stream.
14
+ *
15
+ * This class is not meant to be used directly, but rather to be extended
16
+ * by `FileUploader` and `FileRevisionUploader`.
56
17
  */
57
- export class Fileuploader {
58
- private logger: Logger;
59
-
60
- private digests: UploadDigests;
61
- private controller: UploadController;
62
- private abortController: AbortController;
63
-
64
- private encryptedThumbnails = new Map<ThumbnailType, EncryptedThumbnail>();
65
- private encryptedBlocks = new Map<number, EncryptedBlock>();
66
- private encryptionFinished = false;
67
-
68
- private ongoingUploads = new Map<string, {
69
- uploadPromise: Promise<void>,
70
- encryptedBlock: EncryptedBlock | EncryptedThumbnail,
71
- }>();
72
- private uploadedThumbnails: ({ type: ThumbnailType } & EncryptedBlockMetadata)[] = [];
73
- private uploadedBlocks: ({ index: number } & EncryptedBlockMetadata)[] = [];
18
+ class Uploader {
19
+ protected controller: UploadController;
20
+ protected abortController: AbortController;
74
21
 
75
22
  constructor(
76
- private telemetry: UploadTelemetry,
77
- private apiService: UploadAPIService,
78
- private cryptoService: UploadCryptoService,
79
- private uploadManager: UploadManager,
80
- private blockVerifier: BlockVerifier,
81
- private revisionDraft: NodeRevisionDraft,
82
- private metadata: UploadMetadata,
83
- private onFinish: (failure: boolean) => Promise<void>,
84
- private signal?: AbortSignal,
23
+ protected telemetry: UploadTelemetry,
24
+ protected apiService: UploadAPIService,
25
+ protected cryptoService: UploadCryptoService,
26
+ protected manager: UploadManager,
27
+ protected metadata: UploadMetadata,
28
+ protected onFinish: () => void,
29
+ protected signal?: AbortSignal,
85
30
  ) {
86
31
  this.telemetry = telemetry;
87
- this.logger = telemetry.getLoggerForRevision(revisionDraft.nodeRevisionUid);
88
32
  this.apiService = apiService;
89
33
  this.cryptoService = cryptoService;
90
- this.blockVerifier = blockVerifier;
91
- this.revisionDraft = revisionDraft;
34
+ this.manager = manager;
92
35
  this.metadata = metadata;
93
36
  this.onFinish = onFinish;
94
37
 
@@ -100,11 +43,10 @@ export class Fileuploader {
100
43
  });
101
44
  }
102
45
 
103
- this.digests = new UploadDigests();
104
46
  this.controller = new UploadController();
105
47
  }
106
48
 
107
- writeFile(fileObject: File, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): UploadController {
49
+ async writeFile(fileObject: File, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<UploadController> {
108
50
  if (this.controller.promise) {
109
51
  throw new Error(`Upload already started`);
110
52
  }
@@ -117,460 +59,138 @@ export class Fileuploader {
117
59
  if (!this.metadata.modificationTime) {
118
60
  this.metadata.modificationTime = new Date(fileObject.lastModified);
119
61
  }
120
- return this.writeStream(fileObject.stream(), thumbnails, onProgress);
62
+ this.controller.promise = this.startUpload(fileObject.stream(), thumbnails, onProgress);
63
+ return this.controller;
121
64
  }
122
65
 
123
- writeStream(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): UploadController {
66
+ async writeStream(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<UploadController> {
124
67
  if (this.controller.promise) {
125
68
  throw new Error(`Upload already started`);
126
69
  }
127
- this.controller.promise = this.uploadStream(stream, thumbnails, onProgress);
70
+ this.controller.promise = this.startUpload(stream, thumbnails, onProgress);
128
71
  return this.controller;
129
72
  }
130
73
 
131
- private async uploadStream(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<string> {
132
- let failure = false;
133
-
134
- // File progress is tracked for telemetry - to track at what
135
- // point the download failed.
136
- let fileProgress = 0;
137
-
138
- try {
139
- this.logger.info(`Starting upload`);
140
- await this.encryptAndUploadBlocks(stream, thumbnails, (uploadedBytes) => {
141
- fileProgress += uploadedBytes;
142
- onProgress?.(uploadedBytes);
143
- })
144
-
145
- this.logger.debug(`All blocks uploaded, committing`);
146
- await this.commitFile(thumbnails);
147
-
148
- void this.telemetry.uploadFinished(this.revisionDraft.nodeRevisionUid, fileProgress);
149
- this.logger.info(`Upload succeeded`);
150
- } catch (error: unknown) {
151
- failure = true;
152
- this.logger.error(`Upload failed`, error);
153
- void this.telemetry.uploadFailed(this.revisionDraft.nodeRevisionUid, error, fileProgress, this.metadata.expectedSize);
154
- throw error;
155
- } finally {
156
- this.logger.debug(`Upload cleanup`);
157
-
158
- // Help the garbage collector to clean up the memory.
159
- this.encryptedBlocks.clear();
160
- this.encryptedThumbnails.clear();
161
- this.ongoingUploads.clear();
162
- this.uploadedBlocks = [];
163
- this.uploadedThumbnails = [];
164
- this.encryptionFinished = false;
165
-
166
- await this.onFinish(failure);
167
- }
168
-
169
- return this.revisionDraft.nodeRevisionUid;
74
+ protected async startUpload(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<string> {
75
+ const uploader = await this.initStreamUploader();
76
+ return uploader.start(stream, thumbnails, onProgress);
170
77
  }
171
78
 
172
- private async encryptAndUploadBlocks(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void) {
173
- // We await for the encryption of thumbnails to finish before
174
- // starting the upload. This is because we need to request the
175
- // upload tokens for the thumbnails with the first blocks.
176
- await this.encryptThumbnails(thumbnails);
177
-
178
- // Encrypting blocks and uploading them is done in parallel.
179
- // For that reason, we want to await for the encryption later.
180
- // However, jest complains if encryptBlock rejects asynchronously.
181
- // For that reason we handle manually to save error to the variable
182
- // and throw if set after we await for the encryption.
183
- let encryptionError;
184
- const encryptBlocksPromise = this.encryptBlocks(stream).catch((error) => {
185
- encryptionError = error;
186
- void this.abortUpload(error);
187
- });
188
-
189
- while (!encryptionError) {
190
- await this.controller.waitIfPaused();
191
- await this.waitForUploadCapacityAndBufferedBlocks();
192
-
193
- if (this.isEncryptionFullyFinished) {
194
- break;
195
- }
196
-
197
- await this.requestAndInitiateUpload(onProgress);
79
+ protected async initStreamUploader(): Promise<StreamUploader> {
80
+ const { revisionDraft, blockVerifier } = await this.createRevisionDraft();
198
81
 
199
- if (this.isEncryptionFullyFinished) {
200
- break;
82
+ const onFinish = async (failure: boolean) => {
83
+ this.onFinish();
84
+ if (failure) {
85
+ await this.manager.deleteDraftNode(revisionDraft.nodeUid);
201
86
  }
202
87
  }
203
88
 
204
- this.logger.debug(`All blocks uploading, waiting for them to finish`);
205
- // Technically this is finished as while-block above will break
206
- // when encryption is finished. But in case of error there could
207
- // be a race condition that would cause the encryptionError to
208
- // not be set yet.
209
- await encryptBlocksPromise;
210
- if (encryptionError) {
211
- throw encryptionError;
212
- }
213
- await Promise.all(this.ongoingUploads.values().map(({ uploadPromise }) => uploadPromise));
214
- }
215
-
216
- private async commitFile(thumbnails: Thumbnail[]) {
217
- this.verifyIntegrity(thumbnails);
218
-
219
- const uploadedBlocks = Array.from(this.uploadedBlocks.values());
220
- uploadedBlocks.sort((a, b) => a.index - b.index);
221
-
222
- const extendedAttributes = {
223
- modificationTime: this.metadata.modificationTime,
224
- size: this.metadata.expectedSize,
225
- blockSizes: uploadedBlocks.map(block => block.originalSize),
226
- digests: this.digests.digests(),
227
- };
228
- const encryptedSize = uploadedBlocks.reduce((sum, block) => sum + block.encryptedSize, 0);
229
- await this.uploadManager.commitDraft(
230
- this.revisionDraft,
231
- this.manifest,
89
+ return new StreamUploader(
90
+ this.telemetry,
91
+ this.apiService,
92
+ this.cryptoService,
93
+ this.manager,
94
+ blockVerifier,
95
+ revisionDraft,
232
96
  this.metadata,
233
- extendedAttributes,
234
- encryptedSize,
97
+ onFinish,
98
+ this.signal,
235
99
  );
236
100
  }
237
101
 
238
- private async encryptThumbnails(thumbnails: Thumbnail[]) {
239
- if (new Set(thumbnails.map(({ type }) => type)).size !== thumbnails.length) {
240
- throw new Error(`Duplicate thumbnail types`);
241
- }
242
-
243
- for (const thumbnail of thumbnails) {
244
- this.logger.debug(`Encrypting thumbnail ${thumbnail.type}`);
245
- const encryptedThumbnail = await this.cryptoService.encryptThumbnail(this.revisionDraft.nodeKeys, thumbnail);
246
- this.encryptedThumbnails.set(thumbnail.type, encryptedThumbnail);
247
- }
102
+ protected async createRevisionDraft(): Promise<{ revisionDraft: NodeRevisionDraft, blockVerifier: BlockVerifier }> {
103
+ throw new Error('Not implemented');
248
104
  }
105
+ }
249
106
 
250
- private async encryptBlocks(stream: ReadableStream) {
251
- try {
252
- let index = 0;
253
- const reader = new ChunkStreamReader(stream, FILE_CHUNK_SIZE);
254
- for await (const block of reader.iterateChunks()) {
255
- index++;
256
-
257
- this.digests.update(block);
258
-
259
- await this.controller.waitIfPaused();
260
- await this.waitForBufferCapacity();
261
-
262
- this.logger.debug(`Encrypting block ${index}`);
263
- let attempt = 0;
264
- let integrityError = false;
265
- let encryptedBlock;
266
- while (!encryptedBlock) {
267
- attempt++;
268
-
269
- try {
270
- encryptedBlock = await this.cryptoService.encryptBlock(
271
- (encryptedBlock) => this.blockVerifier.verifyBlock(encryptedBlock),
272
- this.revisionDraft.nodeKeys,
273
- block,
274
- index,
275
- );
276
- if (integrityError) {
277
- void this.telemetry.logBlockVerificationError(true);
278
- }
279
- } catch (error: unknown) {
280
- if (error instanceof IntegrityError) {
281
- integrityError = true;
282
- }
283
-
284
- if (attempt <= MAX_BLOCK_ENCRYPTION_RETRIES) {
285
- this.logger.warn(`Block encryption failed #${attempt}, retrying: ${getErrorMessage(error)}`);
286
- continue;
287
- }
107
+ /**
108
+ * Uploader implementation for a new file.
109
+ */
110
+ export class FileUploader extends Uploader {
111
+ constructor(
112
+ telemetry: UploadTelemetry,
113
+ apiService: UploadAPIService,
114
+ cryptoService: UploadCryptoService,
115
+ manager: UploadManager,
116
+ private parentFolderUid: string,
117
+ private name: string,
118
+ metadata: UploadMetadata,
119
+ onFinish: () => void,
120
+ signal?: AbortSignal,
121
+ ) {
122
+ super(telemetry, apiService, cryptoService, manager, metadata, onFinish, signal);
288
123
 
289
- this.logger.error(`Failed to encrypt block ${index}`, error);
290
- if (integrityError) {
291
- void this.telemetry.logBlockVerificationError(false);
292
- }
293
- throw error;
294
- }
295
- }
296
- this.encryptedBlocks.set(index, encryptedBlock);
297
- }
298
- } finally {
299
- this.encryptionFinished = true;
300
- }
124
+ this.parentFolderUid = parentFolderUid;
125
+ this.name = name;
301
126
  }
302
127
 
303
- private async requestAndInitiateUpload(onProgress?: (uploadedBytes: number) => void): Promise<void> {
304
- this.logger.info(`Requesting upload tokens for ${this.encryptedBlocks.size} blocks`);
305
- const uploadTokens = await this.apiService.requestBlockUpload(
306
- this.revisionDraft.nodeRevisionUid,
307
- this.revisionDraft.nodeKeys.signatureAddress.addressId,
308
- {
309
- contentBlocks: Array.from(this.encryptedBlocks.values().map(block => ({
310
- index: block.index,
311
- encryptedSize: block.encryptedSize,
312
- hash: block.hash,
313
- armoredSignature: block.armoredSignature,
314
- verificationToken: block.verificationToken,
315
- }))),
316
- thumbnails: Array.from(this.encryptedThumbnails.values().map(block => ({
317
- type: block.type,
318
- encryptedSize: block.encryptedSize,
319
- hash: block.hash,
320
- }))),
321
- },
322
- );
128
+ protected async createRevisionDraft(): Promise<{ revisionDraft: NodeRevisionDraft, blockVerifier: BlockVerifier }> {
129
+ let revisionDraft, blockVerifier;
130
+ try {
131
+ revisionDraft = await this.manager.createDraftNode(this.parentFolderUid, this.name, this.metadata);
323
132
 
324
- for (const thumbnailToken of uploadTokens.thumbnailTokens) {
325
- let encryptedThumbnail = this.encryptedThumbnails.get(thumbnailToken.type);
326
- if (!encryptedThumbnail) {
327
- throw new Error(`Thumbnail ${thumbnailToken.type} not found`);
133
+ blockVerifier = new BlockVerifier(this.apiService, this.cryptoService, revisionDraft.nodeKeys.key, revisionDraft.nodeRevisionUid);
134
+ await blockVerifier.loadVerificationData();
135
+ } catch (error: unknown) {
136
+ this.onFinish();
137
+ if (revisionDraft) {
138
+ await this.manager.deleteDraftNode(revisionDraft.nodeUid);
328
139
  }
329
-
330
- this.encryptedThumbnails.delete(thumbnailToken.type);
331
-
332
- const uploadKey = `thumbnail:${thumbnailToken.type}`;
333
- this.ongoingUploads.set(uploadKey, {
334
- uploadPromise: this.uploadThumbnail(
335
- thumbnailToken,
336
- encryptedThumbnail,
337
- onProgress,
338
- ).finally(() => {
339
- this.ongoingUploads.delete(uploadKey);
340
-
341
- // Help the garbage collector to clean up the memory.
342
- encryptedThumbnail = undefined;
343
- }),
344
- encryptedBlock: encryptedThumbnail,
345
- });
140
+ void this.telemetry.uploadInitFailed(this.parentFolderUid, error, this.metadata.expectedSize);
141
+ throw error;
346
142
  }
347
143
 
348
- for (const blockToken of uploadTokens.blockTokens) {
349
- let encryptedBlock = this.encryptedBlocks.get(blockToken.index);
350
- if (!encryptedBlock) {
351
- throw new Error(`Block ${blockToken.index} not found`);
352
- }
353
-
354
- this.encryptedBlocks.delete(blockToken.index);
355
-
356
- const uploadKey = `block:${blockToken.index}`;
357
- this.ongoingUploads.set(uploadKey, {
358
- uploadPromise: this.uploadBlock(
359
- blockToken,
360
- encryptedBlock,
361
- onProgress,
362
- ).finally(() => {
363
- this.ongoingUploads.delete(uploadKey);
364
-
365
- // Help the garbage collector to clean up the memory.
366
- encryptedBlock = undefined;
367
- }),
368
- encryptedBlock,
369
- });
144
+ return {
145
+ revisionDraft,
146
+ blockVerifier,
370
147
  }
371
148
  }
372
149
 
373
- private async uploadThumbnail(
374
- uploadToken: { bareUrl: string, token: string },
375
- encryptedThumbnail: EncryptedThumbnail,
376
- onProgress?: (uploadedBytes: number) => void,
377
- ) {
378
- const logger = new LoggerWithPrefix(this.logger, `thubmnail ${uploadToken.token}`);
379
- logger.info(`Upload started`);
380
-
381
- let blockProgress = 0;
382
- let attempt = 0;
383
-
384
- while (true) {
385
- attempt++;
386
- try {
387
- logger.debug(`Uploading`);
388
- await this.apiService.uploadBlock(
389
- uploadToken.bareUrl,
390
- uploadToken.token,
391
- encryptedThumbnail.encryptedData,
392
- (uploadedBytes) => {
393
- blockProgress += uploadedBytes;
394
- onProgress?.(uploadedBytes);
395
- },
396
- this.abortController.signal,
397
- )
398
- this.uploadedThumbnails.push({
399
- type: encryptedThumbnail.type,
400
- hash: encryptedThumbnail.hash,
401
- encryptedSize: encryptedThumbnail.encryptedSize,
402
- originalSize: encryptedThumbnail.originalSize,
403
- })
404
- break;
405
- } catch (error: unknown) {
406
- if (blockProgress !== 0) {
407
- onProgress?.(-blockProgress);
408
- blockProgress = 0;
409
- }
410
-
411
- // Note: We don't handle token expiration for thumbnails, because
412
- // the API requires the thumbnails to be requested with the first
413
- // upload block request. Thumbnails are tiny, so this edge case
414
- // should be very rare and considering it is the beginning of the
415
- // upload, the whole retry is cheap.
416
-
417
- // Upload can fail for various reasons, for example integrity
418
- // can fail due to bitflips. We want to retry and solve the issue
419
- // seamlessly for the user. We retry only once, because we don't
420
- // want to get stuck in a loop.
421
- if (attempt <= MAX_BLOCK_UPLOAD_RETRIES) {
422
- logger.warn(`Upload failed #${attempt}, retrying: ${getErrorMessage(error)}`);
423
- continue;
424
- }
425
-
426
- logger.error(`Upload failed`, error);
427
- await this.abortUpload(error);
428
- throw error;
429
- }
430
- }
431
-
432
- logger.info(`Uploaded`);
150
+ async getAvailableName(): Promise<string> {
151
+ const availableName = await this.manager.findAvailableName(this.parentFolderUid, this.name);
152
+ return availableName;
433
153
  }
154
+ }
434
155
 
435
- private async uploadBlock(
436
- uploadToken: { index: number, bareUrl: string, token: string },
437
- encryptedBlock: EncryptedBlock,
438
- onProgress?: (uploadedBytes: number) => void,
156
+ /**
157
+ * Uploader implementation for a new file revision.
158
+ */
159
+ export class FileRevisionUploader extends Uploader {
160
+ constructor(
161
+ telemetry: UploadTelemetry,
162
+ apiService: UploadAPIService,
163
+ cryptoService: UploadCryptoService,
164
+ manager: UploadManager,
165
+ private nodeUid: string,
166
+ metadata: UploadMetadata,
167
+ onFinish: () => void,
168
+ signal?: AbortSignal,
439
169
  ) {
440
- const logger = new LoggerWithPrefix(this.logger, `block ${uploadToken.index}:${uploadToken.token}`);
441
- logger.info(`Upload started`);
170
+ super(telemetry, apiService, cryptoService, manager, metadata, onFinish, signal);
442
171
 
443
- let blockProgress = 0;
444
- let attempt = 0;
445
-
446
- while (true) {
447
- attempt++;
448
- try {
449
- logger.debug(`Uploading`);
450
- await this.apiService.uploadBlock(
451
- uploadToken.bareUrl,
452
- uploadToken.token,
453
- encryptedBlock.encryptedData,
454
- (uploadedBytes) => {
455
- blockProgress += uploadedBytes;
456
- onProgress?.(uploadedBytes);
457
- },
458
- this.abortController.signal,
459
- )
460
- this.uploadedBlocks.push({
461
- index: encryptedBlock.index,
462
- hash: encryptedBlock.hash,
463
- encryptedSize: encryptedBlock.encryptedSize,
464
- originalSize: encryptedBlock.originalSize,
465
- })
466
- break;
467
- } catch (error: unknown) {
468
- if (blockProgress !== 0) {
469
- onProgress?.(-blockProgress);
470
- blockProgress = 0;
471
- }
472
-
473
- if (
474
- (error instanceof APIHTTPError && error.statusCode === HTTPErrorCode.NOT_FOUND) ||
475
- (error instanceof NotFoundAPIError)
476
- ) {
477
- logger.warn(`Token expired, fetching new token and retrying`);
478
- const uploadTokens = await this.apiService.requestBlockUpload(
479
- this.revisionDraft.nodeRevisionUid,
480
- this.revisionDraft.nodeKeys.signatureAddress.addressId,
481
- {
482
- contentBlocks: [{
483
- index: encryptedBlock.index,
484
- encryptedSize: encryptedBlock.encryptedSize,
485
- hash: encryptedBlock.hash,
486
- armoredSignature: encryptedBlock.armoredSignature,
487
- verificationToken: encryptedBlock.verificationToken,
488
- }],
489
- },
490
- );
491
- uploadToken = uploadTokens.blockTokens[0];
492
- continue;
493
- }
494
-
495
- // Upload can fail for various reasons, for example integrity
496
- // can fail due to bitflips. We want to retry and solve the issue
497
- // seamlessly for the user. We retry only once, because we don't
498
- // want to get stuck in a loop.
499
- if (attempt <= MAX_BLOCK_UPLOAD_RETRIES) {
500
- logger.warn(`Upload failed #${attempt}, retrying: ${getErrorMessage(error)}`);
501
- continue;
502
- }
503
-
504
- logger.error(`Upload failed`, error);
505
- await this.abortUpload(error);
506
- throw error;
507
- }
508
- }
509
-
510
- logger.info(`Uploaded`);
172
+ this.nodeUid = nodeUid;
511
173
  }
512
174
 
513
- private async waitForBufferCapacity() {
514
- if (this.encryptedBlocks.size >= MAX_BUFFERED_BLOCKS) {
515
- await waitForCondition(() => this.encryptedBlocks.size < MAX_BUFFERED_BLOCKS);
516
- }
517
- }
518
-
519
- private async waitForUploadCapacityAndBufferedBlocks() {
520
- while (this.ongoingUploads.size >= MAX_UPLOADING_BLOCKS) {
521
- await Promise.race(this.ongoingUploads.values().map(({ uploadPromise }) => uploadPromise));
522
- }
523
- await waitForCondition(() => this.encryptedBlocks.size > 0 || this.encryptionFinished);
524
- }
175
+ protected async createRevisionDraft(): Promise<{ revisionDraft: NodeRevisionDraft, blockVerifier: BlockVerifier }> {
176
+ let revisionDraft, blockVerifier;
177
+ try {
178
+ revisionDraft = await this.manager.createDraftRevision(this.nodeUid, this.metadata);
525
179
 
526
- private verifyIntegrity(thumbnails: Thumbnail[]) {
527
- const expectedBlockCount = Math.ceil(this.metadata.expectedSize / FILE_CHUNK_SIZE) + (thumbnails ? thumbnails?.length : 0);
528
- if (this.uploadedBlockCount !== expectedBlockCount) {
529
- throw new IntegrityError(c('Error').t`Some file parts failed to upload`, {
530
- uploadedBlockCount: this.uploadedBlockCount,
531
- expectedBlockCount,
532
- });
533
- }
534
- if (this.uploadedOriginalFileSize !== this.metadata.expectedSize) {
535
- throw new IntegrityError(c('Error').t`Some file bytes failed to upload`, {
536
- uploadedOriginalFileSize: this.uploadedOriginalFileSize,
537
- expectedFileSize: this.metadata.expectedSize,
538
- });
180
+ blockVerifier = new BlockVerifier(this.apiService, this.cryptoService, revisionDraft.nodeKeys.key, revisionDraft.nodeRevisionUid);
181
+ await blockVerifier.loadVerificationData();
182
+ } catch (error: unknown) {
183
+ this.onFinish();
184
+ if (revisionDraft) {
185
+ await this.manager.deleteDraftRevision(revisionDraft.nodeRevisionUid);
186
+ }
187
+ void this.telemetry.uploadInitFailed(this.nodeUid, error, this.metadata.expectedSize);
188
+ throw error;
539
189
  }
540
- }
541
-
542
- /**
543
- * Check if the encryption is fully finished.
544
- * This means that all blocks and thumbnails have been encrypted and
545
- * requested to be uploaded, and there are no more blocks or thumbnails
546
- * to encrypt and upload.
547
- */
548
- private get isEncryptionFullyFinished(): boolean {
549
- return this.encryptionFinished && this.encryptedBlocks.size === 0 && this.encryptedThumbnails.size === 0;
550
- }
551
-
552
- private get uploadedBlockCount(): number {
553
- return this.uploadedBlocks.length + this.uploadedThumbnails.length;
554
- }
555
-
556
- private get uploadedOriginalFileSize(): number {
557
- return this.uploadedBlocks.reduce((sum, { originalSize }) => sum + originalSize, 0);
558
- }
559
-
560
- private get manifest(): Uint8Array {
561
- this.uploadedThumbnails.sort((a, b) => a.type - b.type);
562
- this.uploadedBlocks.sort((a, b) => a.index - b.index);
563
- const hashes = [
564
- ...this.uploadedThumbnails.map(({ hash }) => hash),
565
- ...this.uploadedBlocks.map(({ hash }) => hash),
566
- ];
567
- return mergeUint8Arrays(hashes);
568
- }
569
190
 
570
- private async abortUpload(error: unknown) {
571
- if (this.abortController.signal.aborted || this.signal?.aborted) {
572
- return;
191
+ return {
192
+ revisionDraft,
193
+ blockVerifier,
573
194
  }
574
- this.abortController.abort(error);
575
195
  }
576
196
  }