@protontech/drive-sdk 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/errors.d.ts +7 -3
- package/dist/errors.js +9 -4
- package/dist/errors.js.map +1 -1
- package/dist/interface/index.d.ts +1 -1
- package/dist/interface/nodes.d.ts +12 -1
- package/dist/interface/nodes.js +11 -0
- package/dist/interface/nodes.js.map +1 -1
- package/dist/interface/upload.d.ts +51 -3
- package/dist/internal/apiService/driveTypes.d.ts +1341 -465
- package/dist/internal/apiService/errors.js +2 -2
- package/dist/internal/apiService/errors.js.map +1 -1
- package/dist/internal/apiService/transformers.js +2 -0
- package/dist/internal/apiService/transformers.js.map +1 -1
- package/dist/internal/asyncIteratorMap.d.ts +15 -0
- package/dist/internal/asyncIteratorMap.js +59 -0
- package/dist/internal/asyncIteratorMap.js.map +1 -0
- package/dist/internal/asyncIteratorMap.test.d.ts +1 -0
- package/dist/internal/asyncIteratorMap.test.js +120 -0
- package/dist/internal/asyncIteratorMap.test.js.map +1 -0
- package/dist/internal/nodes/apiService.d.ts +2 -2
- package/dist/internal/nodes/apiService.js +16 -6
- package/dist/internal/nodes/apiService.js.map +1 -1
- package/dist/internal/nodes/apiService.test.js +30 -8
- package/dist/internal/nodes/apiService.test.js.map +1 -1
- package/dist/internal/nodes/cache.js +1 -0
- package/dist/internal/nodes/cache.js.map +1 -1
- package/dist/internal/nodes/cache.test.js +1 -0
- package/dist/internal/nodes/cache.test.js.map +1 -1
- package/dist/internal/nodes/cryptoService.test.js +34 -0
- package/dist/internal/nodes/cryptoService.test.js.map +1 -1
- package/dist/internal/nodes/index.test.js +3 -1
- package/dist/internal/nodes/index.test.js.map +1 -1
- package/dist/internal/nodes/interface.d.ts +3 -1
- package/dist/internal/nodes/nodesAccess.js +28 -7
- package/dist/internal/nodes/nodesAccess.js.map +1 -1
- package/dist/internal/nodes/nodesAccess.test.js +7 -6
- package/dist/internal/nodes/nodesAccess.test.js.map +1 -1
- package/dist/internal/sharing/apiService.js +19 -2
- package/dist/internal/sharing/apiService.js.map +1 -1
- package/dist/internal/upload/fileUploader.d.ts +49 -53
- package/dist/internal/upload/fileUploader.js +91 -395
- package/dist/internal/upload/fileUploader.js.map +1 -1
- package/dist/internal/upload/fileUploader.test.js +38 -292
- package/dist/internal/upload/fileUploader.test.js.map +1 -1
- package/dist/internal/upload/index.d.ts +3 -3
- package/dist/internal/upload/index.js +20 -41
- package/dist/internal/upload/index.js.map +1 -1
- package/dist/internal/upload/manager.d.ts +1 -1
- package/dist/internal/upload/manager.js +16 -19
- package/dist/internal/upload/manager.js.map +1 -1
- package/dist/internal/upload/manager.test.js +42 -83
- package/dist/internal/upload/manager.test.js.map +1 -1
- package/dist/internal/upload/streamUploader.d.ts +62 -0
- package/dist/internal/upload/streamUploader.js +441 -0
- package/dist/internal/upload/streamUploader.js.map +1 -0
- package/dist/internal/upload/streamUploader.test.d.ts +1 -0
- package/dist/internal/upload/streamUploader.test.js +358 -0
- package/dist/internal/upload/streamUploader.test.js.map +1 -0
- package/dist/protonDriveClient.d.ts +4 -4
- package/dist/protonDriveClient.js +1 -1
- package/dist/protonDriveClient.js.map +1 -1
- package/package.json +2 -2
- package/src/errors.ts +10 -4
- package/src/interface/index.ts +1 -1
- package/src/interface/nodes.ts +11 -0
- package/src/interface/upload.ts +53 -3
- package/src/internal/apiService/driveTypes.ts +1341 -465
- package/src/internal/apiService/errors.ts +3 -2
- package/src/internal/apiService/transformers.ts +2 -0
- package/src/internal/asyncIteratorMap.test.ts +150 -0
- package/src/internal/asyncIteratorMap.ts +64 -0
- package/src/internal/nodes/apiService.test.ts +36 -7
- package/src/internal/nodes/apiService.ts +19 -7
- package/src/internal/nodes/cache.test.ts +1 -0
- package/src/internal/nodes/cache.ts +1 -0
- package/src/internal/nodes/cryptoService.test.ts +38 -0
- package/src/internal/nodes/index.test.ts +3 -1
- package/src/internal/nodes/interface.ts +4 -1
- package/src/internal/nodes/nodesAccess.test.ts +7 -6
- package/src/internal/nodes/nodesAccess.ts +30 -7
- package/src/internal/sharing/apiService.ts +24 -2
- package/src/internal/upload/fileUploader.test.ts +46 -376
- package/src/internal/upload/fileUploader.ts +114 -494
- package/src/internal/upload/index.ts +26 -50
- package/src/internal/upload/manager.test.ts +45 -92
- package/src/internal/upload/manager.ts +30 -32
- package/src/internal/upload/streamUploader.test.ts +469 -0
- package/src/internal/upload/streamUploader.ts +552 -0
- package/src/protonDriveClient.ts +5 -4
|
@@ -1,94 +1,37 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import { Thumbnail, Logger, ThumbnailType, UploadMetadata } from "../../interface";
|
|
4
|
-
import { IntegrityError } from "../../errors";
|
|
5
|
-
import { LoggerWithPrefix } from "../../telemetry";
|
|
6
|
-
import { APIHTTPError, HTTPErrorCode, NotFoundAPIError } from "../apiService";
|
|
7
|
-
import { getErrorMessage } from "../errors";
|
|
8
|
-
import { mergeUint8Arrays } from "../utils";
|
|
9
|
-
import { waitForCondition } from '../wait';
|
|
1
|
+
import { Thumbnail, UploadMetadata } from "../../interface";
|
|
10
2
|
import { UploadAPIService } from "./apiService";
|
|
11
3
|
import { BlockVerifier } from "./blockVerifier";
|
|
12
4
|
import { UploadController } from './controller';
|
|
13
5
|
import { UploadCryptoService } from "./cryptoService";
|
|
14
|
-
import {
|
|
15
|
-
import { NodeRevisionDraft, EncryptedBlock, EncryptedThumbnail, EncryptedBlockMetadata } from "./interface";
|
|
16
|
-
import { UploadTelemetry } from './telemetry';
|
|
17
|
-
import { ChunkStreamReader } from './chunkStreamReader';
|
|
6
|
+
import { NodeRevisionDraft } from "./interface";
|
|
18
7
|
import { UploadManager } from "./manager";
|
|
8
|
+
import { StreamUploader } from './streamUploader';
|
|
9
|
+
import { UploadTelemetry } from './telemetry';
|
|
19
10
|
|
|
20
11
|
/**
|
|
21
|
-
*
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
* Maximum number of blocks that can be buffered before upload.
|
|
27
|
-
* This is to prevent using too much memory.
|
|
28
|
-
*/
|
|
29
|
-
const MAX_BUFFERED_BLOCKS = 15;
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* Maximum number of blocks that can be uploaded at the same time.
|
|
33
|
-
* This is to prevent overloading the server with too many requests.
|
|
34
|
-
*/
|
|
35
|
-
const MAX_UPLOADING_BLOCKS = 5;
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Maximum number of retries for block encryption.
|
|
39
|
-
* This is to automatically retry random errors that can happen
|
|
40
|
-
* during encryption, for example bitflips.
|
|
41
|
-
*/
|
|
42
|
-
const MAX_BLOCK_ENCRYPTION_RETRIES = 1;
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Maximum number of retries for block upload.
|
|
46
|
-
* This is to ensure we don't end up in an infinite loop.
|
|
47
|
-
*/
|
|
48
|
-
const MAX_BLOCK_UPLOAD_RETRIES = 3;
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Fileuploader is responsible for uploading file content to the server.
|
|
52
|
-
*
|
|
53
|
-
* It handles the encryption of file blocks and thumbnails, as well as
|
|
54
|
-
* the upload process itself. It manages the upload queue and ensures
|
|
55
|
-
* that the upload process is efficient and does not overload the server.
|
|
12
|
+
* Uploader is generic class responsible for creating a revision draft
|
|
13
|
+
* and initiate the upload process for a file object or a stream.
|
|
14
|
+
*
|
|
15
|
+
* This class is not meant to be used directly, but rather to be extended
|
|
16
|
+
* by `FileUploader` and `FileRevisionUploader`.
|
|
56
17
|
*/
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
private digests: UploadDigests;
|
|
61
|
-
private controller: UploadController;
|
|
62
|
-
private abortController: AbortController;
|
|
63
|
-
|
|
64
|
-
private encryptedThumbnails = new Map<ThumbnailType, EncryptedThumbnail>();
|
|
65
|
-
private encryptedBlocks = new Map<number, EncryptedBlock>();
|
|
66
|
-
private encryptionFinished = false;
|
|
67
|
-
|
|
68
|
-
private ongoingUploads = new Map<string, {
|
|
69
|
-
uploadPromise: Promise<void>,
|
|
70
|
-
encryptedBlock: EncryptedBlock | EncryptedThumbnail,
|
|
71
|
-
}>();
|
|
72
|
-
private uploadedThumbnails: ({ type: ThumbnailType } & EncryptedBlockMetadata)[] = [];
|
|
73
|
-
private uploadedBlocks: ({ index: number } & EncryptedBlockMetadata)[] = [];
|
|
18
|
+
class Uploader {
|
|
19
|
+
protected controller: UploadController;
|
|
20
|
+
protected abortController: AbortController;
|
|
74
21
|
|
|
75
22
|
constructor(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
private onFinish: (failure: boolean) => Promise<void>,
|
|
84
|
-
private signal?: AbortSignal,
|
|
23
|
+
protected telemetry: UploadTelemetry,
|
|
24
|
+
protected apiService: UploadAPIService,
|
|
25
|
+
protected cryptoService: UploadCryptoService,
|
|
26
|
+
protected manager: UploadManager,
|
|
27
|
+
protected metadata: UploadMetadata,
|
|
28
|
+
protected onFinish: () => void,
|
|
29
|
+
protected signal?: AbortSignal,
|
|
85
30
|
) {
|
|
86
31
|
this.telemetry = telemetry;
|
|
87
|
-
this.logger = telemetry.getLoggerForRevision(revisionDraft.nodeRevisionUid);
|
|
88
32
|
this.apiService = apiService;
|
|
89
33
|
this.cryptoService = cryptoService;
|
|
90
|
-
this.
|
|
91
|
-
this.revisionDraft = revisionDraft;
|
|
34
|
+
this.manager = manager;
|
|
92
35
|
this.metadata = metadata;
|
|
93
36
|
this.onFinish = onFinish;
|
|
94
37
|
|
|
@@ -100,11 +43,10 @@ export class Fileuploader {
|
|
|
100
43
|
});
|
|
101
44
|
}
|
|
102
45
|
|
|
103
|
-
this.digests = new UploadDigests();
|
|
104
46
|
this.controller = new UploadController();
|
|
105
47
|
}
|
|
106
48
|
|
|
107
|
-
writeFile(fileObject: File, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): UploadController {
|
|
49
|
+
async writeFile(fileObject: File, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<UploadController> {
|
|
108
50
|
if (this.controller.promise) {
|
|
109
51
|
throw new Error(`Upload already started`);
|
|
110
52
|
}
|
|
@@ -117,460 +59,138 @@ export class Fileuploader {
|
|
|
117
59
|
if (!this.metadata.modificationTime) {
|
|
118
60
|
this.metadata.modificationTime = new Date(fileObject.lastModified);
|
|
119
61
|
}
|
|
120
|
-
|
|
62
|
+
this.controller.promise = this.startUpload(fileObject.stream(), thumbnails, onProgress);
|
|
63
|
+
return this.controller;
|
|
121
64
|
}
|
|
122
65
|
|
|
123
|
-
writeStream(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): UploadController {
|
|
66
|
+
async writeStream(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<UploadController> {
|
|
124
67
|
if (this.controller.promise) {
|
|
125
68
|
throw new Error(`Upload already started`);
|
|
126
69
|
}
|
|
127
|
-
this.controller.promise = this.
|
|
70
|
+
this.controller.promise = this.startUpload(stream, thumbnails, onProgress);
|
|
128
71
|
return this.controller;
|
|
129
72
|
}
|
|
130
73
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
// File progress is tracked for telemetry - to track at what
|
|
135
|
-
// point the download failed.
|
|
136
|
-
let fileProgress = 0;
|
|
137
|
-
|
|
138
|
-
try {
|
|
139
|
-
this.logger.info(`Starting upload`);
|
|
140
|
-
await this.encryptAndUploadBlocks(stream, thumbnails, (uploadedBytes) => {
|
|
141
|
-
fileProgress += uploadedBytes;
|
|
142
|
-
onProgress?.(uploadedBytes);
|
|
143
|
-
})
|
|
144
|
-
|
|
145
|
-
this.logger.debug(`All blocks uploaded, committing`);
|
|
146
|
-
await this.commitFile(thumbnails);
|
|
147
|
-
|
|
148
|
-
void this.telemetry.uploadFinished(this.revisionDraft.nodeRevisionUid, fileProgress);
|
|
149
|
-
this.logger.info(`Upload succeeded`);
|
|
150
|
-
} catch (error: unknown) {
|
|
151
|
-
failure = true;
|
|
152
|
-
this.logger.error(`Upload failed`, error);
|
|
153
|
-
void this.telemetry.uploadFailed(this.revisionDraft.nodeRevisionUid, error, fileProgress, this.metadata.expectedSize);
|
|
154
|
-
throw error;
|
|
155
|
-
} finally {
|
|
156
|
-
this.logger.debug(`Upload cleanup`);
|
|
157
|
-
|
|
158
|
-
// Help the garbage collector to clean up the memory.
|
|
159
|
-
this.encryptedBlocks.clear();
|
|
160
|
-
this.encryptedThumbnails.clear();
|
|
161
|
-
this.ongoingUploads.clear();
|
|
162
|
-
this.uploadedBlocks = [];
|
|
163
|
-
this.uploadedThumbnails = [];
|
|
164
|
-
this.encryptionFinished = false;
|
|
165
|
-
|
|
166
|
-
await this.onFinish(failure);
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
return this.revisionDraft.nodeRevisionUid;
|
|
74
|
+
protected async startUpload(stream: ReadableStream, thumbnails: Thumbnail[], onProgress?: (uploadedBytes: number) => void): Promise<string> {
|
|
75
|
+
const uploader = await this.initStreamUploader();
|
|
76
|
+
return uploader.start(stream, thumbnails, onProgress);
|
|
170
77
|
}
|
|
171
78
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
// starting the upload. This is because we need to request the
|
|
175
|
-
// upload tokens for the thumbnails with the first blocks.
|
|
176
|
-
await this.encryptThumbnails(thumbnails);
|
|
177
|
-
|
|
178
|
-
// Encrypting blocks and uploading them is done in parallel.
|
|
179
|
-
// For that reason, we want to await for the encryption later.
|
|
180
|
-
// However, jest complains if encryptBlock rejects asynchronously.
|
|
181
|
-
// For that reason we handle manually to save error to the variable
|
|
182
|
-
// and throw if set after we await for the encryption.
|
|
183
|
-
let encryptionError;
|
|
184
|
-
const encryptBlocksPromise = this.encryptBlocks(stream).catch((error) => {
|
|
185
|
-
encryptionError = error;
|
|
186
|
-
void this.abortUpload(error);
|
|
187
|
-
});
|
|
188
|
-
|
|
189
|
-
while (!encryptionError) {
|
|
190
|
-
await this.controller.waitIfPaused();
|
|
191
|
-
await this.waitForUploadCapacityAndBufferedBlocks();
|
|
192
|
-
|
|
193
|
-
if (this.isEncryptionFullyFinished) {
|
|
194
|
-
break;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
await this.requestAndInitiateUpload(onProgress);
|
|
79
|
+
protected async initStreamUploader(): Promise<StreamUploader> {
|
|
80
|
+
const { revisionDraft, blockVerifier } = await this.createRevisionDraft();
|
|
198
81
|
|
|
199
|
-
|
|
200
|
-
|
|
82
|
+
const onFinish = async (failure: boolean) => {
|
|
83
|
+
this.onFinish();
|
|
84
|
+
if (failure) {
|
|
85
|
+
await this.manager.deleteDraftNode(revisionDraft.nodeUid);
|
|
201
86
|
}
|
|
202
87
|
}
|
|
203
88
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
throw encryptionError;
|
|
212
|
-
}
|
|
213
|
-
await Promise.all(this.ongoingUploads.values().map(({ uploadPromise }) => uploadPromise));
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
private async commitFile(thumbnails: Thumbnail[]) {
|
|
217
|
-
this.verifyIntegrity(thumbnails);
|
|
218
|
-
|
|
219
|
-
const uploadedBlocks = Array.from(this.uploadedBlocks.values());
|
|
220
|
-
uploadedBlocks.sort((a, b) => a.index - b.index);
|
|
221
|
-
|
|
222
|
-
const extendedAttributes = {
|
|
223
|
-
modificationTime: this.metadata.modificationTime,
|
|
224
|
-
size: this.metadata.expectedSize,
|
|
225
|
-
blockSizes: uploadedBlocks.map(block => block.originalSize),
|
|
226
|
-
digests: this.digests.digests(),
|
|
227
|
-
};
|
|
228
|
-
const encryptedSize = uploadedBlocks.reduce((sum, block) => sum + block.encryptedSize, 0);
|
|
229
|
-
await this.uploadManager.commitDraft(
|
|
230
|
-
this.revisionDraft,
|
|
231
|
-
this.manifest,
|
|
89
|
+
return new StreamUploader(
|
|
90
|
+
this.telemetry,
|
|
91
|
+
this.apiService,
|
|
92
|
+
this.cryptoService,
|
|
93
|
+
this.manager,
|
|
94
|
+
blockVerifier,
|
|
95
|
+
revisionDraft,
|
|
232
96
|
this.metadata,
|
|
233
|
-
|
|
234
|
-
|
|
97
|
+
onFinish,
|
|
98
|
+
this.signal,
|
|
235
99
|
);
|
|
236
100
|
}
|
|
237
101
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
throw new Error(`Duplicate thumbnail types`);
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
for (const thumbnail of thumbnails) {
|
|
244
|
-
this.logger.debug(`Encrypting thumbnail ${thumbnail.type}`);
|
|
245
|
-
const encryptedThumbnail = await this.cryptoService.encryptThumbnail(this.revisionDraft.nodeKeys, thumbnail);
|
|
246
|
-
this.encryptedThumbnails.set(thumbnail.type, encryptedThumbnail);
|
|
247
|
-
}
|
|
102
|
+
protected async createRevisionDraft(): Promise<{ revisionDraft: NodeRevisionDraft, blockVerifier: BlockVerifier }> {
|
|
103
|
+
throw new Error('Not implemented');
|
|
248
104
|
}
|
|
105
|
+
}
|
|
249
106
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
while (!encryptedBlock) {
|
|
267
|
-
attempt++;
|
|
268
|
-
|
|
269
|
-
try {
|
|
270
|
-
encryptedBlock = await this.cryptoService.encryptBlock(
|
|
271
|
-
(encryptedBlock) => this.blockVerifier.verifyBlock(encryptedBlock),
|
|
272
|
-
this.revisionDraft.nodeKeys,
|
|
273
|
-
block,
|
|
274
|
-
index,
|
|
275
|
-
);
|
|
276
|
-
if (integrityError) {
|
|
277
|
-
void this.telemetry.logBlockVerificationError(true);
|
|
278
|
-
}
|
|
279
|
-
} catch (error: unknown) {
|
|
280
|
-
if (error instanceof IntegrityError) {
|
|
281
|
-
integrityError = true;
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
if (attempt <= MAX_BLOCK_ENCRYPTION_RETRIES) {
|
|
285
|
-
this.logger.warn(`Block encryption failed #${attempt}, retrying: ${getErrorMessage(error)}`);
|
|
286
|
-
continue;
|
|
287
|
-
}
|
|
107
|
+
/**
|
|
108
|
+
* Uploader implementation for a new file.
|
|
109
|
+
*/
|
|
110
|
+
export class FileUploader extends Uploader {
|
|
111
|
+
constructor(
|
|
112
|
+
telemetry: UploadTelemetry,
|
|
113
|
+
apiService: UploadAPIService,
|
|
114
|
+
cryptoService: UploadCryptoService,
|
|
115
|
+
manager: UploadManager,
|
|
116
|
+
private parentFolderUid: string,
|
|
117
|
+
private name: string,
|
|
118
|
+
metadata: UploadMetadata,
|
|
119
|
+
onFinish: () => void,
|
|
120
|
+
signal?: AbortSignal,
|
|
121
|
+
) {
|
|
122
|
+
super(telemetry, apiService, cryptoService, manager, metadata, onFinish, signal);
|
|
288
123
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
void this.telemetry.logBlockVerificationError(false);
|
|
292
|
-
}
|
|
293
|
-
throw error;
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
this.encryptedBlocks.set(index, encryptedBlock);
|
|
297
|
-
}
|
|
298
|
-
} finally {
|
|
299
|
-
this.encryptionFinished = true;
|
|
300
|
-
}
|
|
124
|
+
this.parentFolderUid = parentFolderUid;
|
|
125
|
+
this.name = name;
|
|
301
126
|
}
|
|
302
127
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
this.
|
|
307
|
-
this.revisionDraft.nodeKeys.signatureAddress.addressId,
|
|
308
|
-
{
|
|
309
|
-
contentBlocks: Array.from(this.encryptedBlocks.values().map(block => ({
|
|
310
|
-
index: block.index,
|
|
311
|
-
encryptedSize: block.encryptedSize,
|
|
312
|
-
hash: block.hash,
|
|
313
|
-
armoredSignature: block.armoredSignature,
|
|
314
|
-
verificationToken: block.verificationToken,
|
|
315
|
-
}))),
|
|
316
|
-
thumbnails: Array.from(this.encryptedThumbnails.values().map(block => ({
|
|
317
|
-
type: block.type,
|
|
318
|
-
encryptedSize: block.encryptedSize,
|
|
319
|
-
hash: block.hash,
|
|
320
|
-
}))),
|
|
321
|
-
},
|
|
322
|
-
);
|
|
128
|
+
protected async createRevisionDraft(): Promise<{ revisionDraft: NodeRevisionDraft, blockVerifier: BlockVerifier }> {
|
|
129
|
+
let revisionDraft, blockVerifier;
|
|
130
|
+
try {
|
|
131
|
+
revisionDraft = await this.manager.createDraftNode(this.parentFolderUid, this.name, this.metadata);
|
|
323
132
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
133
|
+
blockVerifier = new BlockVerifier(this.apiService, this.cryptoService, revisionDraft.nodeKeys.key, revisionDraft.nodeRevisionUid);
|
|
134
|
+
await blockVerifier.loadVerificationData();
|
|
135
|
+
} catch (error: unknown) {
|
|
136
|
+
this.onFinish();
|
|
137
|
+
if (revisionDraft) {
|
|
138
|
+
await this.manager.deleteDraftNode(revisionDraft.nodeUid);
|
|
328
139
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
const uploadKey = `thumbnail:${thumbnailToken.type}`;
|
|
333
|
-
this.ongoingUploads.set(uploadKey, {
|
|
334
|
-
uploadPromise: this.uploadThumbnail(
|
|
335
|
-
thumbnailToken,
|
|
336
|
-
encryptedThumbnail,
|
|
337
|
-
onProgress,
|
|
338
|
-
).finally(() => {
|
|
339
|
-
this.ongoingUploads.delete(uploadKey);
|
|
340
|
-
|
|
341
|
-
// Help the garbage collector to clean up the memory.
|
|
342
|
-
encryptedThumbnail = undefined;
|
|
343
|
-
}),
|
|
344
|
-
encryptedBlock: encryptedThumbnail,
|
|
345
|
-
});
|
|
140
|
+
void this.telemetry.uploadInitFailed(this.parentFolderUid, error, this.metadata.expectedSize);
|
|
141
|
+
throw error;
|
|
346
142
|
}
|
|
347
143
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
throw new Error(`Block ${blockToken.index} not found`);
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
this.encryptedBlocks.delete(blockToken.index);
|
|
355
|
-
|
|
356
|
-
const uploadKey = `block:${blockToken.index}`;
|
|
357
|
-
this.ongoingUploads.set(uploadKey, {
|
|
358
|
-
uploadPromise: this.uploadBlock(
|
|
359
|
-
blockToken,
|
|
360
|
-
encryptedBlock,
|
|
361
|
-
onProgress,
|
|
362
|
-
).finally(() => {
|
|
363
|
-
this.ongoingUploads.delete(uploadKey);
|
|
364
|
-
|
|
365
|
-
// Help the garbage collector to clean up the memory.
|
|
366
|
-
encryptedBlock = undefined;
|
|
367
|
-
}),
|
|
368
|
-
encryptedBlock,
|
|
369
|
-
});
|
|
144
|
+
return {
|
|
145
|
+
revisionDraft,
|
|
146
|
+
blockVerifier,
|
|
370
147
|
}
|
|
371
148
|
}
|
|
372
149
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
onProgress?: (uploadedBytes: number) => void,
|
|
377
|
-
) {
|
|
378
|
-
const logger = new LoggerWithPrefix(this.logger, `thubmnail ${uploadToken.token}`);
|
|
379
|
-
logger.info(`Upload started`);
|
|
380
|
-
|
|
381
|
-
let blockProgress = 0;
|
|
382
|
-
let attempt = 0;
|
|
383
|
-
|
|
384
|
-
while (true) {
|
|
385
|
-
attempt++;
|
|
386
|
-
try {
|
|
387
|
-
logger.debug(`Uploading`);
|
|
388
|
-
await this.apiService.uploadBlock(
|
|
389
|
-
uploadToken.bareUrl,
|
|
390
|
-
uploadToken.token,
|
|
391
|
-
encryptedThumbnail.encryptedData,
|
|
392
|
-
(uploadedBytes) => {
|
|
393
|
-
blockProgress += uploadedBytes;
|
|
394
|
-
onProgress?.(uploadedBytes);
|
|
395
|
-
},
|
|
396
|
-
this.abortController.signal,
|
|
397
|
-
)
|
|
398
|
-
this.uploadedThumbnails.push({
|
|
399
|
-
type: encryptedThumbnail.type,
|
|
400
|
-
hash: encryptedThumbnail.hash,
|
|
401
|
-
encryptedSize: encryptedThumbnail.encryptedSize,
|
|
402
|
-
originalSize: encryptedThumbnail.originalSize,
|
|
403
|
-
})
|
|
404
|
-
break;
|
|
405
|
-
} catch (error: unknown) {
|
|
406
|
-
if (blockProgress !== 0) {
|
|
407
|
-
onProgress?.(-blockProgress);
|
|
408
|
-
blockProgress = 0;
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
// Note: We don't handle token expiration for thumbnails, because
|
|
412
|
-
// the API requires the thumbnails to be requested with the first
|
|
413
|
-
// upload block request. Thumbnails are tiny, so this edge case
|
|
414
|
-
// should be very rare and considering it is the beginning of the
|
|
415
|
-
// upload, the whole retry is cheap.
|
|
416
|
-
|
|
417
|
-
// Upload can fail for various reasons, for example integrity
|
|
418
|
-
// can fail due to bitflips. We want to retry and solve the issue
|
|
419
|
-
// seamlessly for the user. We retry only once, because we don't
|
|
420
|
-
// want to get stuck in a loop.
|
|
421
|
-
if (attempt <= MAX_BLOCK_UPLOAD_RETRIES) {
|
|
422
|
-
logger.warn(`Upload failed #${attempt}, retrying: ${getErrorMessage(error)}`);
|
|
423
|
-
continue;
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
logger.error(`Upload failed`, error);
|
|
427
|
-
await this.abortUpload(error);
|
|
428
|
-
throw error;
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
logger.info(`Uploaded`);
|
|
150
|
+
async getAvailableName(): Promise<string> {
|
|
151
|
+
const availableName = await this.manager.findAvailableName(this.parentFolderUid, this.name);
|
|
152
|
+
return availableName;
|
|
433
153
|
}
|
|
154
|
+
}
|
|
434
155
|
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
156
|
+
/**
|
|
157
|
+
* Uploader implementation for a new file revision.
|
|
158
|
+
*/
|
|
159
|
+
export class FileRevisionUploader extends Uploader {
|
|
160
|
+
constructor(
|
|
161
|
+
telemetry: UploadTelemetry,
|
|
162
|
+
apiService: UploadAPIService,
|
|
163
|
+
cryptoService: UploadCryptoService,
|
|
164
|
+
manager: UploadManager,
|
|
165
|
+
private nodeUid: string,
|
|
166
|
+
metadata: UploadMetadata,
|
|
167
|
+
onFinish: () => void,
|
|
168
|
+
signal?: AbortSignal,
|
|
439
169
|
) {
|
|
440
|
-
|
|
441
|
-
logger.info(`Upload started`);
|
|
170
|
+
super(telemetry, apiService, cryptoService, manager, metadata, onFinish, signal);
|
|
442
171
|
|
|
443
|
-
|
|
444
|
-
let attempt = 0;
|
|
445
|
-
|
|
446
|
-
while (true) {
|
|
447
|
-
attempt++;
|
|
448
|
-
try {
|
|
449
|
-
logger.debug(`Uploading`);
|
|
450
|
-
await this.apiService.uploadBlock(
|
|
451
|
-
uploadToken.bareUrl,
|
|
452
|
-
uploadToken.token,
|
|
453
|
-
encryptedBlock.encryptedData,
|
|
454
|
-
(uploadedBytes) => {
|
|
455
|
-
blockProgress += uploadedBytes;
|
|
456
|
-
onProgress?.(uploadedBytes);
|
|
457
|
-
},
|
|
458
|
-
this.abortController.signal,
|
|
459
|
-
)
|
|
460
|
-
this.uploadedBlocks.push({
|
|
461
|
-
index: encryptedBlock.index,
|
|
462
|
-
hash: encryptedBlock.hash,
|
|
463
|
-
encryptedSize: encryptedBlock.encryptedSize,
|
|
464
|
-
originalSize: encryptedBlock.originalSize,
|
|
465
|
-
})
|
|
466
|
-
break;
|
|
467
|
-
} catch (error: unknown) {
|
|
468
|
-
if (blockProgress !== 0) {
|
|
469
|
-
onProgress?.(-blockProgress);
|
|
470
|
-
blockProgress = 0;
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
if (
|
|
474
|
-
(error instanceof APIHTTPError && error.statusCode === HTTPErrorCode.NOT_FOUND) ||
|
|
475
|
-
(error instanceof NotFoundAPIError)
|
|
476
|
-
) {
|
|
477
|
-
logger.warn(`Token expired, fetching new token and retrying`);
|
|
478
|
-
const uploadTokens = await this.apiService.requestBlockUpload(
|
|
479
|
-
this.revisionDraft.nodeRevisionUid,
|
|
480
|
-
this.revisionDraft.nodeKeys.signatureAddress.addressId,
|
|
481
|
-
{
|
|
482
|
-
contentBlocks: [{
|
|
483
|
-
index: encryptedBlock.index,
|
|
484
|
-
encryptedSize: encryptedBlock.encryptedSize,
|
|
485
|
-
hash: encryptedBlock.hash,
|
|
486
|
-
armoredSignature: encryptedBlock.armoredSignature,
|
|
487
|
-
verificationToken: encryptedBlock.verificationToken,
|
|
488
|
-
}],
|
|
489
|
-
},
|
|
490
|
-
);
|
|
491
|
-
uploadToken = uploadTokens.blockTokens[0];
|
|
492
|
-
continue;
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
// Upload can fail for various reasons, for example integrity
|
|
496
|
-
// can fail due to bitflips. We want to retry and solve the issue
|
|
497
|
-
// seamlessly for the user. We retry only once, because we don't
|
|
498
|
-
// want to get stuck in a loop.
|
|
499
|
-
if (attempt <= MAX_BLOCK_UPLOAD_RETRIES) {
|
|
500
|
-
logger.warn(`Upload failed #${attempt}, retrying: ${getErrorMessage(error)}`);
|
|
501
|
-
continue;
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
logger.error(`Upload failed`, error);
|
|
505
|
-
await this.abortUpload(error);
|
|
506
|
-
throw error;
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
logger.info(`Uploaded`);
|
|
172
|
+
this.nodeUid = nodeUid;
|
|
511
173
|
}
|
|
512
174
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
private async waitForUploadCapacityAndBufferedBlocks() {
|
|
520
|
-
while (this.ongoingUploads.size >= MAX_UPLOADING_BLOCKS) {
|
|
521
|
-
await Promise.race(this.ongoingUploads.values().map(({ uploadPromise }) => uploadPromise));
|
|
522
|
-
}
|
|
523
|
-
await waitForCondition(() => this.encryptedBlocks.size > 0 || this.encryptionFinished);
|
|
524
|
-
}
|
|
175
|
+
protected async createRevisionDraft(): Promise<{ revisionDraft: NodeRevisionDraft, blockVerifier: BlockVerifier }> {
|
|
176
|
+
let revisionDraft, blockVerifier;
|
|
177
|
+
try {
|
|
178
|
+
revisionDraft = await this.manager.createDraftRevision(this.nodeUid, this.metadata);
|
|
525
179
|
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
throw new IntegrityError(c('Error').t`Some file bytes failed to upload`, {
|
|
536
|
-
uploadedOriginalFileSize: this.uploadedOriginalFileSize,
|
|
537
|
-
expectedFileSize: this.metadata.expectedSize,
|
|
538
|
-
});
|
|
180
|
+
blockVerifier = new BlockVerifier(this.apiService, this.cryptoService, revisionDraft.nodeKeys.key, revisionDraft.nodeRevisionUid);
|
|
181
|
+
await blockVerifier.loadVerificationData();
|
|
182
|
+
} catch (error: unknown) {
|
|
183
|
+
this.onFinish();
|
|
184
|
+
if (revisionDraft) {
|
|
185
|
+
await this.manager.deleteDraftRevision(revisionDraft.nodeRevisionUid);
|
|
186
|
+
}
|
|
187
|
+
void this.telemetry.uploadInitFailed(this.nodeUid, error, this.metadata.expectedSize);
|
|
188
|
+
throw error;
|
|
539
189
|
}
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
/**
|
|
543
|
-
* Check if the encryption is fully finished.
|
|
544
|
-
* This means that all blocks and thumbnails have been encrypted and
|
|
545
|
-
* requested to be uploaded, and there are no more blocks or thumbnails
|
|
546
|
-
* to encrypt and upload.
|
|
547
|
-
*/
|
|
548
|
-
private get isEncryptionFullyFinished(): boolean {
|
|
549
|
-
return this.encryptionFinished && this.encryptedBlocks.size === 0 && this.encryptedThumbnails.size === 0;
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
private get uploadedBlockCount(): number {
|
|
553
|
-
return this.uploadedBlocks.length + this.uploadedThumbnails.length;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
private get uploadedOriginalFileSize(): number {
|
|
557
|
-
return this.uploadedBlocks.reduce((sum, { originalSize }) => sum + originalSize, 0);
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
private get manifest(): Uint8Array {
|
|
561
|
-
this.uploadedThumbnails.sort((a, b) => a.type - b.type);
|
|
562
|
-
this.uploadedBlocks.sort((a, b) => a.index - b.index);
|
|
563
|
-
const hashes = [
|
|
564
|
-
...this.uploadedThumbnails.map(({ hash }) => hash),
|
|
565
|
-
...this.uploadedBlocks.map(({ hash }) => hash),
|
|
566
|
-
];
|
|
567
|
-
return mergeUint8Arrays(hashes);
|
|
568
|
-
}
|
|
569
190
|
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
191
|
+
return {
|
|
192
|
+
revisionDraft,
|
|
193
|
+
blockVerifier,
|
|
573
194
|
}
|
|
574
|
-
this.abortController.abort(error);
|
|
575
195
|
}
|
|
576
196
|
}
|