@protontech/drive-sdk 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache/index.d.ts +2 -0
- package/dist/cache/index.js +6 -0
- package/dist/cache/index.js.map +1 -0
- package/dist/cache/interface.d.ts +105 -0
- package/dist/cache/interface.js +3 -0
- package/dist/cache/interface.js.map +1 -0
- package/dist/cache/memoryCache.d.ts +18 -0
- package/dist/cache/memoryCache.js +78 -0
- package/dist/cache/memoryCache.js.map +1 -0
- package/dist/cache/memoryCache.test.d.ts +1 -0
- package/dist/cache/memoryCache.test.js +121 -0
- package/dist/cache/memoryCache.test.js.map +1 -0
- package/dist/crypto/hmac.d.ts +22 -0
- package/dist/crypto/hmac.js +44 -0
- package/dist/crypto/hmac.js.map +1 -0
- package/dist/crypto/utils.d.ts +2 -0
- package/dist/crypto/utils.js +35 -0
- package/dist/crypto/utils.js.map +1 -0
- package/dist/errors.d.ts +142 -0
- package/dist/errors.js +168 -0
- package/dist/errors.js.map +1 -0
- package/dist/interface/account.js +3 -0
- package/dist/interface/account.js.map +1 -0
- package/dist/interface/author.d.ts +26 -0
- package/dist/interface/author.js +3 -0
- package/dist/interface/author.js.map +1 -0
- package/dist/interface/download.d.ts +29 -0
- package/dist/interface/download.js +3 -0
- package/dist/interface/download.js.map +1 -0
- package/dist/interface/httpClient.d.ts +38 -0
- package/dist/interface/httpClient.js +3 -0
- package/dist/interface/httpClient.js.map +1 -0
- package/dist/interface/index.d.ts +1 -1
- package/dist/interface/nodes.d.ts +12 -1
- package/dist/interface/nodes.js +11 -0
- package/dist/interface/nodes.js.map +1 -1
- package/dist/interface/result.d.ts +9 -0
- package/dist/interface/result.js +11 -0
- package/dist/interface/result.js.map +1 -0
- package/dist/interface/thumbnail.d.ts +17 -0
- package/dist/interface/thumbnail.js +9 -0
- package/dist/interface/thumbnail.js.map +1 -0
- package/dist/interface/upload.d.ts +64 -0
- package/dist/interface/upload.js +3 -0
- package/dist/interface/upload.js.map +1 -0
- package/dist/internal/apiService/driveTypes.d.ts +1341 -465
- package/dist/internal/apiService/errorCodes.d.ts +30 -0
- package/dist/internal/apiService/errorCodes.js +11 -0
- package/dist/internal/apiService/errorCodes.js.map +1 -0
- package/dist/internal/apiService/errors.js +2 -2
- package/dist/internal/apiService/errors.js.map +1 -1
- package/dist/internal/apiService/observerStream.d.ts +3 -0
- package/dist/internal/apiService/observerStream.js +15 -0
- package/dist/internal/apiService/observerStream.js.map +1 -0
- package/dist/internal/apiService/transformers.js +2 -0
- package/dist/internal/apiService/transformers.js.map +1 -1
- package/dist/internal/asyncIteratorMap.d.ts +15 -0
- package/dist/internal/asyncIteratorMap.js +59 -0
- package/dist/internal/asyncIteratorMap.js.map +1 -0
- package/dist/internal/asyncIteratorMap.test.d.ts +1 -0
- package/dist/internal/asyncIteratorMap.test.js +120 -0
- package/dist/internal/asyncIteratorMap.test.js.map +1 -0
- package/dist/internal/batchLoading.d.ts +34 -0
- package/dist/internal/batchLoading.js +68 -0
- package/dist/internal/batchLoading.js.map +1 -0
- package/dist/internal/batchLoading.test.d.ts +1 -0
- package/dist/internal/batchLoading.test.js +50 -0
- package/dist/internal/batchLoading.test.js.map +1 -0
- package/dist/internal/download/controller.d.ts +8 -0
- package/dist/internal/download/controller.js +22 -0
- package/dist/internal/download/controller.js.map +1 -0
- package/dist/internal/download/queue.d.ts +5 -0
- package/dist/internal/download/queue.js +31 -0
- package/dist/internal/download/queue.js.map +1 -0
- package/dist/internal/errors.js +28 -0
- package/dist/internal/errors.js.map +1 -0
- package/dist/internal/errors.test.js +22 -0
- package/dist/internal/errors.test.js.map +1 -0
- package/dist/internal/events/interface.d.ts +47 -0
- package/dist/internal/events/interface.js +12 -0
- package/dist/internal/events/interface.js.map +1 -0
- package/dist/internal/nodes/apiService.d.ts +2 -2
- package/dist/internal/nodes/apiService.js +16 -6
- package/dist/internal/nodes/apiService.js.map +1 -1
- package/dist/internal/nodes/apiService.test.js +30 -8
- package/dist/internal/nodes/apiService.test.js.map +1 -1
- package/dist/internal/nodes/cache.js +1 -0
- package/dist/internal/nodes/cache.js.map +1 -1
- package/dist/internal/nodes/cache.test.js +1 -0
- package/dist/internal/nodes/cache.test.js.map +1 -1
- package/dist/internal/nodes/cryptoService.test.js +34 -0
- package/dist/internal/nodes/cryptoService.test.js.map +1 -1
- package/dist/internal/nodes/index.test.js +3 -1
- package/dist/internal/nodes/index.test.js.map +1 -1
- package/dist/internal/nodes/interface.d.ts +3 -1
- package/dist/internal/nodes/mediaTypes.d.ts +2 -0
- package/dist/internal/nodes/mediaTypes.js +13 -0
- package/dist/internal/nodes/mediaTypes.js.map +1 -0
- package/dist/internal/nodes/nodesAccess.js +28 -7
- package/dist/internal/nodes/nodesAccess.js.map +1 -1
- package/dist/internal/nodes/nodesAccess.test.js +7 -6
- package/dist/internal/nodes/nodesAccess.test.js.map +1 -1
- package/dist/internal/nodes/validations.d.ts +4 -0
- package/dist/internal/nodes/validations.js +21 -0
- package/dist/internal/nodes/validations.js.map +1 -0
- package/dist/internal/sharing/apiService.js +19 -2
- package/dist/internal/sharing/apiService.js.map +1 -1
- package/dist/internal/uids.d.ts +38 -0
- package/dist/internal/uids.js +85 -0
- package/dist/internal/uids.js.map +1 -0
- package/dist/internal/upload/chunkStreamReader.d.ts +13 -0
- package/dist/internal/upload/chunkStreamReader.js +46 -0
- package/dist/internal/upload/chunkStreamReader.js.map +1 -0
- package/dist/internal/upload/chunkStreamReader.test.d.ts +1 -0
- package/dist/internal/upload/chunkStreamReader.test.js +75 -0
- package/dist/internal/upload/chunkStreamReader.test.js.map +1 -0
- package/dist/internal/upload/controller.d.ts +8 -0
- package/dist/internal/upload/controller.js +25 -0
- package/dist/internal/upload/controller.js.map +1 -0
- package/dist/internal/upload/digests.d.ts +8 -0
- package/dist/internal/upload/digests.js +22 -0
- package/dist/internal/upload/digests.js.map +1 -0
- package/dist/internal/upload/fileUploader.d.ts +49 -53
- package/dist/internal/upload/fileUploader.js +91 -395
- package/dist/internal/upload/fileUploader.js.map +1 -1
- package/dist/internal/upload/fileUploader.test.js +38 -292
- package/dist/internal/upload/fileUploader.test.js.map +1 -1
- package/dist/internal/upload/index.d.ts +3 -3
- package/dist/internal/upload/index.js +20 -41
- package/dist/internal/upload/index.js.map +1 -1
- package/dist/internal/upload/manager.d.ts +1 -1
- package/dist/internal/upload/manager.js +16 -19
- package/dist/internal/upload/manager.js.map +1 -1
- package/dist/internal/upload/manager.test.js +42 -83
- package/dist/internal/upload/manager.test.js.map +1 -1
- package/dist/internal/upload/queue.d.ts +5 -0
- package/dist/internal/upload/queue.js +32 -0
- package/dist/internal/upload/queue.js.map +1 -0
- package/dist/internal/upload/streamUploader.d.ts +62 -0
- package/dist/internal/upload/streamUploader.js +441 -0
- package/dist/internal/upload/streamUploader.js.map +1 -0
- package/dist/internal/upload/streamUploader.test.d.ts +1 -0
- package/dist/internal/upload/streamUploader.test.js +358 -0
- package/dist/internal/upload/streamUploader.test.js.map +1 -0
- package/dist/internal/utils.d.ts +1 -0
- package/dist/internal/utils.js +13 -0
- package/dist/internal/utils.js.map +1 -0
- package/dist/internal/wait.d.ts +3 -0
- package/dist/internal/wait.js +28 -0
- package/dist/internal/wait.js.map +1 -0
- package/dist/internal/wait.test.d.ts +1 -0
- package/dist/internal/wait.test.js +21 -0
- package/dist/internal/wait.test.js.map +1 -0
- package/dist/protonDriveClient.d.ts +4 -4
- package/dist/protonDriveClient.js +1 -1
- package/dist/protonDriveClient.js.map +1 -1
- package/package.json +2 -2
- package/src/errors.ts +10 -4
- package/src/interface/index.ts +1 -1
- package/src/interface/nodes.ts +11 -0
- package/src/interface/upload.ts +53 -3
- package/src/internal/apiService/driveTypes.ts +1341 -465
- package/src/internal/apiService/errors.ts +3 -2
- package/src/internal/apiService/transformers.ts +2 -0
- package/src/internal/asyncIteratorMap.test.ts +150 -0
- package/src/internal/asyncIteratorMap.ts +64 -0
- package/src/internal/nodes/apiService.test.ts +36 -7
- package/src/internal/nodes/apiService.ts +19 -7
- package/src/internal/nodes/cache.test.ts +1 -0
- package/src/internal/nodes/cache.ts +1 -0
- package/src/internal/nodes/cryptoService.test.ts +38 -0
- package/src/internal/nodes/index.test.ts +3 -1
- package/src/internal/nodes/interface.ts +4 -1
- package/src/internal/nodes/nodesAccess.test.ts +7 -6
- package/src/internal/nodes/nodesAccess.ts +30 -7
- package/src/internal/sharing/apiService.ts +24 -2
- package/src/internal/upload/fileUploader.test.ts +46 -376
- package/src/internal/upload/fileUploader.ts +114 -494
- package/src/internal/upload/index.ts +26 -50
- package/src/internal/upload/manager.test.ts +45 -92
- package/src/internal/upload/manager.ts +30 -32
- package/src/internal/upload/streamUploader.test.ts +469 -0
- package/src/internal/upload/streamUploader.ts +552 -0
- package/src/protonDriveClient.ts +5 -4
|
@@ -3,6 +3,7 @@ import { c } from 'ttag';
|
|
|
3
3
|
import { PrivateKey, SessionKey } from "../../crypto";
|
|
4
4
|
import { InvalidNameError, Logger, MissingNode, NodeType, Result, resultError, resultOk } from "../../interface";
|
|
5
5
|
import { DecryptionError, ProtonDriveError } from "../../errors";
|
|
6
|
+
import { asyncIteratorMap } from '../asyncIteratorMap';
|
|
6
7
|
import { getErrorMessage } from '../errors';
|
|
7
8
|
import { BatchLoading } from "../batchLoading";
|
|
8
9
|
import { makeNodeUid, splitNodeUid } from "../uids";
|
|
@@ -15,6 +16,16 @@ import { SharesService, EncryptedNode, DecryptedUnparsedNode, DecryptedNode, Dec
|
|
|
15
16
|
import { validateNodeName } from "./validations";
|
|
16
17
|
import { isProtonDocument, isProtonSheet } from './mediaTypes';
|
|
17
18
|
|
|
19
|
+
// This is the number of nodes that are loaded in parallel.
|
|
20
|
+
// It is a trade-off between initial wait time and overhead of API calls.
|
|
21
|
+
const BATCH_LOADING_SIZE = 30;
|
|
22
|
+
|
|
23
|
+
// This is the number of nodes that are decrypted in parallel.
|
|
24
|
+
// It is a trade-off between performance and memory usage.
|
|
25
|
+
// Higher number means more memory usage, but faster decryption.
|
|
26
|
+
// Lower number means less memory usage, but slower decryption.
|
|
27
|
+
const DECRYPTION_CONCURRENCY = 15;
|
|
28
|
+
|
|
18
29
|
/**
|
|
19
30
|
* Provides access to node metadata.
|
|
20
31
|
*
|
|
@@ -64,7 +75,7 @@ export class NodesAccess {
|
|
|
64
75
|
// Ensure the parent is loaded and up-to-date.
|
|
65
76
|
const parentNode = await this.getNode(parentNodeUid);
|
|
66
77
|
|
|
67
|
-
const batchLoading = new BatchLoading<string, DecryptedNode>({ iterateItems: (nodeUids) => this.loadNodes(nodeUids, signal) });
|
|
78
|
+
const batchLoading = new BatchLoading<string, DecryptedNode>({ iterateItems: (nodeUids) => this.loadNodes(nodeUids, signal), batchSize: BATCH_LOADING_SIZE });
|
|
68
79
|
|
|
69
80
|
const areChildrenCached = await this.cache.isFolderChildrenLoaded(parentNodeUid);
|
|
70
81
|
if (areChildrenCached) {
|
|
@@ -100,7 +111,7 @@ export class NodesAccess {
|
|
|
100
111
|
// Improvement requested: keep status of loaded trash and leverage cache.
|
|
101
112
|
async *iterateTrashedNodes(signal?: AbortSignal): AsyncGenerator<DecryptedNode> {
|
|
102
113
|
const { volumeId } = await this.shareService.getMyFilesIDs();
|
|
103
|
-
const batchLoading = new BatchLoading<string, DecryptedNode>({ iterateItems: (nodeUids) => this.loadNodes(nodeUids, signal) });
|
|
114
|
+
const batchLoading = new BatchLoading<string, DecryptedNode>({ iterateItems: (nodeUids) => this.loadNodes(nodeUids, signal), batchSize: BATCH_LOADING_SIZE });
|
|
104
115
|
for await (const nodeUid of this.apiService.iterateTrashedNodeUids(volumeId, signal)) {
|
|
105
116
|
let node;
|
|
106
117
|
try {
|
|
@@ -118,7 +129,7 @@ export class NodesAccess {
|
|
|
118
129
|
}
|
|
119
130
|
|
|
120
131
|
async *iterateNodes(nodeUids: string[], signal?: AbortSignal): AsyncGenerator<DecryptedNode | MissingNode> {
|
|
121
|
-
const batchLoading = new BatchLoading<string, DecryptedNode | MissingNode>({ iterateItems: (nodeUids) => this.loadNodesWithMissingReport(nodeUids, signal) });
|
|
132
|
+
const batchLoading = new BatchLoading<string, DecryptedNode | MissingNode>({ iterateItems: (nodeUids) => this.loadNodesWithMissingReport(nodeUids, signal), batchSize: BATCH_LOADING_SIZE });
|
|
122
133
|
for await (const result of this.cache.iterateNodes(nodeUids)) {
|
|
123
134
|
if (result.ok && !result.node.isStale) {
|
|
124
135
|
yield result.node;
|
|
@@ -130,7 +141,8 @@ export class NodesAccess {
|
|
|
130
141
|
}
|
|
131
142
|
|
|
132
143
|
private async loadNode(nodeUid: string): Promise<{ node: DecryptedNode, keys?: DecryptedNodeKeys }> {
|
|
133
|
-
const
|
|
144
|
+
const { volumeId: ownVolumeId } = await this.shareService.getMyFilesIDs();
|
|
145
|
+
const encryptedNode = await this.apiService.getNode(nodeUid, ownVolumeId);
|
|
134
146
|
return this.decryptNode(encryptedNode);
|
|
135
147
|
}
|
|
136
148
|
|
|
@@ -147,13 +159,24 @@ export class NodesAccess {
|
|
|
147
159
|
const returnedNodeUids: string[] = [];
|
|
148
160
|
const errors = [];
|
|
149
161
|
|
|
150
|
-
|
|
162
|
+
const { volumeId: ownVolumeId } = await this.shareService.getMyFilesIDs();
|
|
163
|
+
|
|
164
|
+
const encryptedNodesIterator = this.apiService.iterateNodes(nodeUids, ownVolumeId, signal);
|
|
165
|
+
const decryptNodeMapper = async (encryptedNode: EncryptedNode): Promise<Result<DecryptedNode, unknown>> => {
|
|
151
166
|
returnedNodeUids.push(encryptedNode.uid);
|
|
152
167
|
try {
|
|
153
168
|
const { node } = await this.decryptNode(encryptedNode);
|
|
154
|
-
|
|
169
|
+
return resultOk(node);
|
|
155
170
|
} catch (error: unknown) {
|
|
156
|
-
|
|
171
|
+
return resultError(error);
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
const decryptedNodesIterator = asyncIteratorMap(encryptedNodesIterator, decryptNodeMapper, DECRYPTION_CONCURRENCY);
|
|
175
|
+
for await (const node of decryptedNodesIterator) {
|
|
176
|
+
if (node.ok) {
|
|
177
|
+
yield node.value;
|
|
178
|
+
} else {
|
|
179
|
+
errors.push(node.error);
|
|
157
180
|
}
|
|
158
181
|
}
|
|
159
182
|
|
|
@@ -49,6 +49,14 @@ type PostShareUrlResponse = drivePaths['/drive/shares/{shareID}/urls']['post']['
|
|
|
49
49
|
type PutShareUrlRequest = Extract<drivePaths['/drive/shares/{shareID}/urls/{urlID}']['put']['requestBody'], { 'content': object }>['content']['application/json'];
|
|
50
50
|
type PutShareUrlResponse = drivePaths['/drive/shares/{shareID}/urls/{urlID}']['put']['responses']['200']['content']['application/json'];
|
|
51
51
|
|
|
52
|
+
// We do not support photos and albums yet.
|
|
53
|
+
const SUPPORTED_SHARE_TARGET_TYPES = [
|
|
54
|
+
0, // Root
|
|
55
|
+
1, // Folder
|
|
56
|
+
2, // File
|
|
57
|
+
5, // Proton vendor (documents and sheets)
|
|
58
|
+
];
|
|
59
|
+
|
|
52
60
|
/**
|
|
53
61
|
* Provides API communication for fetching and managing sharing.
|
|
54
62
|
*
|
|
@@ -81,7 +89,14 @@ export class SharingAPIService {
|
|
|
81
89
|
while (true) {
|
|
82
90
|
const response = await this.apiService.get<GetSharedWithMeNodesResponse>(`drive/v2/sharedwithme?${anchor ? `AnchorID=${anchor}` : ''}`, signal);
|
|
83
91
|
for (const link of response.Links) {
|
|
84
|
-
|
|
92
|
+
const nodeUid = makeNodeUid(link.VolumeID, link.LinkID);
|
|
93
|
+
|
|
94
|
+
if (!SUPPORTED_SHARE_TARGET_TYPES.includes(link.ShareTargetType)) {
|
|
95
|
+
this.logger.warn(`Unsupported share target type ${link.ShareTargetType} for node ${nodeUid}`);
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
yield nodeUid;
|
|
85
100
|
}
|
|
86
101
|
|
|
87
102
|
if (!response.More || !response.AnchorID) {
|
|
@@ -96,7 +111,14 @@ export class SharingAPIService {
|
|
|
96
111
|
while (true) {
|
|
97
112
|
const response = await this.apiService.get<GetInvitationsResponse>(`drive/v2/shares/invitations?${anchor ? `AnchorID=${anchor}` : ''}`, signal);
|
|
98
113
|
for (const invitation of response.Invitations) {
|
|
99
|
-
|
|
114
|
+
const invitationUid = makeInvitationUid(invitation.ShareID, invitation.InvitationID);
|
|
115
|
+
|
|
116
|
+
if (!SUPPORTED_SHARE_TARGET_TYPES.includes(invitation.ShareTargetType)) {
|
|
117
|
+
this.logger.warn(`Unsupported share target type ${invitation.ShareTargetType} for invitation ${invitationUid}`);
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
yield invitationUid;
|
|
100
122
|
}
|
|
101
123
|
|
|
102
124
|
if (!response.More || !response.AnchorID) {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { Thumbnail,
|
|
2
|
-
import {
|
|
3
|
-
import { FILE_CHUNK_SIZE, Fileuploader } from './fileUploader';
|
|
1
|
+
import { Thumbnail, UploadMetadata } from '../../interface';
|
|
2
|
+
import { FileUploader } from './fileUploader';
|
|
4
3
|
import { UploadTelemetry } from './telemetry';
|
|
5
4
|
import { UploadAPIService } from './apiService';
|
|
6
5
|
import { UploadCryptoService } from './cryptoService';
|
|
@@ -8,7 +7,6 @@ import { UploadController } from './controller';
|
|
|
8
7
|
import { BlockVerifier } from './blockVerifier';
|
|
9
8
|
import { NodeRevisionDraft } from './interface';
|
|
10
9
|
import { UploadManager } from './manager';
|
|
11
|
-
import { IntegrityError } from '../../errors';
|
|
12
10
|
|
|
13
11
|
const BLOCK_ENCRYPTION_OVERHEAD = 10000;
|
|
14
12
|
|
|
@@ -41,7 +39,9 @@ describe('FileUploader', () => {
|
|
|
41
39
|
let onFinish: () => Promise<void>;
|
|
42
40
|
let abortController: AbortController;
|
|
43
41
|
|
|
44
|
-
let uploader:
|
|
42
|
+
let uploader: FileUploader;
|
|
43
|
+
|
|
44
|
+
let startUploadSpy: jest.SpyInstance;
|
|
45
45
|
|
|
46
46
|
beforeEach(() => {
|
|
47
47
|
// @ts-expect-error No need to implement all methods for mocking
|
|
@@ -103,411 +103,81 @@ describe('FileUploader', () => {
|
|
|
103
103
|
},
|
|
104
104
|
} as NodeRevisionDraft;
|
|
105
105
|
|
|
106
|
-
metadata = {
|
|
107
|
-
// 3 blocks: 4 + 4 + 2 MB
|
|
108
|
-
expectedSize: 10 * 1024 * 1024,
|
|
109
|
-
} as UploadMetadata;
|
|
106
|
+
metadata = {} as UploadMetadata;
|
|
110
107
|
|
|
111
108
|
controller = new UploadController();
|
|
112
109
|
onFinish = jest.fn();
|
|
113
110
|
abortController = new AbortController();
|
|
114
111
|
|
|
115
|
-
uploader = new
|
|
112
|
+
uploader = new FileUploader(
|
|
116
113
|
telemetry,
|
|
117
114
|
apiService,
|
|
118
115
|
cryptoService,
|
|
119
116
|
uploadManager,
|
|
120
|
-
|
|
121
|
-
|
|
117
|
+
'parentFolderUid',
|
|
118
|
+
'name',
|
|
122
119
|
metadata,
|
|
123
120
|
onFinish,
|
|
124
121
|
abortController.signal,
|
|
125
122
|
);
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
describe('writeFile', () => {
|
|
129
|
-
it('should set modification time if not set', () => {
|
|
130
|
-
// @ts-expect-error Ignore mocking File
|
|
131
|
-
const file = {
|
|
132
|
-
lastModified: 123456789,
|
|
133
|
-
stream: jest.fn().mockReturnValue('stream'),
|
|
134
|
-
} as File;
|
|
135
|
-
const thumbnails: Thumbnail[] = [];
|
|
136
|
-
const onProgress = jest.fn();
|
|
137
|
-
|
|
138
|
-
const writeStreamSpy = jest.spyOn(uploader, 'writeStream').mockReturnValue(controller);
|
|
139
|
-
|
|
140
|
-
uploader.writeFile(file, thumbnails, onProgress);
|
|
141
|
-
|
|
142
|
-
expect(metadata.modificationTime).toEqual(new Date(123456789));
|
|
143
|
-
expect(writeStreamSpy).toHaveBeenCalledWith('stream', thumbnails, onProgress);
|
|
144
|
-
});
|
|
145
|
-
});
|
|
146
|
-
|
|
147
|
-
describe('writeStream', () => {
|
|
148
|
-
let uploadStreamSpy: jest.SpyInstance;
|
|
149
|
-
beforeEach(() => {
|
|
150
|
-
uploadStreamSpy = jest.spyOn(uploader as any, 'uploadStream').mockResolvedValue('revisionUid');
|
|
151
|
-
});
|
|
152
|
-
|
|
153
|
-
it('should throw an error if upload already started', () => {
|
|
154
|
-
uploader.writeStream(new ReadableStream(), [], jest.fn());
|
|
155
|
-
|
|
156
|
-
expect(() => {
|
|
157
|
-
uploader.writeStream(new ReadableStream(), [], jest.fn());
|
|
158
|
-
}).toThrow('Upload already started');
|
|
159
|
-
});
|
|
160
|
-
|
|
161
|
-
it('should start the upload process', async () => {
|
|
162
|
-
const stream = new ReadableStream();
|
|
163
|
-
const thumbnails: Thumbnail[] = [];
|
|
164
|
-
const onProgress = jest.fn();
|
|
165
123
|
|
|
166
|
-
|
|
167
|
-
expect(uploadStreamSpy).toHaveBeenCalledWith(stream, thumbnails, onProgress);
|
|
168
|
-
});
|
|
124
|
+
startUploadSpy = jest.spyOn(uploader as any, 'startUpload').mockReturnValue(Promise.resolve('revisionUid'));
|
|
169
125
|
});
|
|
170
126
|
|
|
171
|
-
describe('
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
const numberOfExpectedBlocks = Math.ceil(metadata.expectedSize / FILE_CHUNK_SIZE);
|
|
183
|
-
expect(uploadManager.commitDraft).toHaveBeenCalledTimes(1);
|
|
184
|
-
expect(uploadManager.commitDraft).toHaveBeenCalledWith(
|
|
185
|
-
revisionDraft,
|
|
186
|
-
expect.anything(),
|
|
187
|
-
metadata,
|
|
188
|
-
{
|
|
189
|
-
size: metadata.expectedSize,
|
|
190
|
-
blockSizes: metadata.expectedSize ? [
|
|
191
|
-
...Array(numberOfExpectedBlocks - 1).fill(FILE_CHUNK_SIZE),
|
|
192
|
-
metadata.expectedSize % FILE_CHUNK_SIZE
|
|
193
|
-
] : [],
|
|
194
|
-
modificationTime: undefined,
|
|
195
|
-
digests: {
|
|
196
|
-
sha1: expect.anything(),
|
|
197
|
-
}
|
|
198
|
-
},
|
|
199
|
-
metadata.expectedSize + numberOfExpectedBlocks * BLOCK_ENCRYPTION_OVERHEAD,
|
|
200
|
-
);
|
|
201
|
-
expect(telemetry.uploadFinished).toHaveBeenCalledTimes(1);
|
|
202
|
-
expect(telemetry.uploadFinished).toHaveBeenCalledWith('revisionUid', metadata.expectedSize + thumbnailSize);
|
|
203
|
-
expect(telemetry.uploadFailed).not.toHaveBeenCalled();
|
|
204
|
-
expect(onFinish).toHaveBeenCalledTimes(1);
|
|
205
|
-
expect(onFinish).toHaveBeenCalledWith(false);
|
|
206
|
-
};
|
|
207
|
-
|
|
208
|
-
const verifyFailure = async (error: string, uploadedBytes: number | undefined, expectedSize = metadata.expectedSize) => {
|
|
209
|
-
const controller = uploader.writeStream(stream, thumbnails, onProgress);
|
|
210
|
-
await expect(controller.completion()).rejects.toThrow(error);
|
|
211
|
-
|
|
212
|
-
expect(telemetry.uploadFinished).not.toHaveBeenCalled();
|
|
213
|
-
expect(telemetry.uploadFailed).toHaveBeenCalledTimes(1);
|
|
214
|
-
expect(telemetry.uploadFailed).toHaveBeenCalledWith(
|
|
215
|
-
'revisionUid',
|
|
216
|
-
new Error(error),
|
|
217
|
-
uploadedBytes === undefined ? expect.anything() : uploadedBytes,
|
|
218
|
-
expectedSize,
|
|
219
|
-
);
|
|
220
|
-
expect(onFinish).toHaveBeenCalledTimes(1);
|
|
221
|
-
expect(onFinish).toHaveBeenCalledWith(true);
|
|
222
|
-
};
|
|
223
|
-
|
|
224
|
-
const verifyOnProgress = async (uploadedBytes: number[]) => {
|
|
225
|
-
expect(onProgress).toHaveBeenCalledTimes(uploadedBytes.length);
|
|
226
|
-
for (let i = 0; i < uploadedBytes.length; i++) {
|
|
227
|
-
expect(onProgress).toHaveBeenNthCalledWith(i + 1, uploadedBytes[i]);
|
|
228
|
-
}
|
|
229
|
-
};
|
|
230
|
-
|
|
231
|
-
beforeEach(() => {
|
|
232
|
-
onProgress = jest.fn();
|
|
233
|
-
thumbnails = [
|
|
234
|
-
{
|
|
235
|
-
type: ThumbnailType.Type1,
|
|
236
|
-
thumbnail: new Uint8Array(1024),
|
|
237
|
-
}
|
|
238
|
-
];
|
|
239
|
-
thumbnailSize = thumbnails.reduce((acc, thumbnail) => acc + thumbnail.thumbnail.length, 0);
|
|
240
|
-
stream = new ReadableStream({
|
|
241
|
-
start(controller) {
|
|
242
|
-
const chunkSize = 1024;
|
|
243
|
-
const chunkCount = metadata.expectedSize / chunkSize;
|
|
244
|
-
for (let i = 1; i <= chunkCount; i++) {
|
|
245
|
-
controller.enqueue(new Uint8Array(chunkSize));
|
|
246
|
-
}
|
|
247
|
-
controller.close();
|
|
248
|
-
},
|
|
249
|
-
});
|
|
250
|
-
});
|
|
251
|
-
|
|
252
|
-
it("should upload successfully", async () => {
|
|
253
|
-
await verifySuccess();
|
|
254
|
-
expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(1);
|
|
255
|
-
expect(apiService.uploadBlock).toHaveBeenCalledTimes(4); // 3 blocks + 1 thumbnail
|
|
256
|
-
expect(blockVerifier.verifyBlock).toHaveBeenCalledTimes(3); // 3 blocks
|
|
257
|
-
expect(telemetry.logBlockVerificationError).not.toHaveBeenCalled();
|
|
258
|
-
await verifyOnProgress([thumbnailSize, 4 * 1024 * 1024, 4 * 1024 * 1024, 2 * 1024 * 1024]);
|
|
259
|
-
});
|
|
260
|
-
|
|
261
|
-
it("should upload successfully empty file without thumbnail", async () => {
|
|
262
|
-
metadata = {
|
|
263
|
-
expectedSize: 0,
|
|
264
|
-
} as UploadMetadata;
|
|
265
|
-
stream = new ReadableStream({
|
|
266
|
-
start(controller) {
|
|
267
|
-
controller.close();
|
|
268
|
-
},
|
|
269
|
-
});
|
|
270
|
-
thumbnails = [];
|
|
271
|
-
thumbnailSize = 0;
|
|
272
|
-
uploader = new Fileuploader(
|
|
273
|
-
telemetry,
|
|
274
|
-
apiService,
|
|
275
|
-
cryptoService,
|
|
276
|
-
uploadManager,
|
|
277
|
-
blockVerifier,
|
|
278
|
-
revisionDraft,
|
|
279
|
-
metadata,
|
|
280
|
-
onFinish,
|
|
281
|
-
);
|
|
282
|
-
|
|
283
|
-
await verifySuccess();
|
|
284
|
-
expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(0);
|
|
285
|
-
expect(apiService.uploadBlock).toHaveBeenCalledTimes(0);
|
|
286
|
-
expect(blockVerifier.verifyBlock).toHaveBeenCalledTimes(0);
|
|
287
|
-
await verifyOnProgress([]);
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
it("should upload successfully empty file with thumbnail", async () => {
|
|
291
|
-
metadata = {
|
|
292
|
-
expectedSize: 0,
|
|
293
|
-
} as UploadMetadata;
|
|
294
|
-
stream = new ReadableStream({
|
|
295
|
-
start(controller) {
|
|
296
|
-
controller.close();
|
|
297
|
-
},
|
|
298
|
-
});
|
|
299
|
-
uploader = new Fileuploader(
|
|
300
|
-
telemetry,
|
|
301
|
-
apiService,
|
|
302
|
-
cryptoService,
|
|
303
|
-
uploadManager,
|
|
304
|
-
blockVerifier,
|
|
305
|
-
revisionDraft,
|
|
306
|
-
metadata,
|
|
307
|
-
onFinish,
|
|
308
|
-
);
|
|
309
|
-
|
|
310
|
-
await verifySuccess();
|
|
311
|
-
expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(1);
|
|
312
|
-
expect(apiService.uploadBlock).toHaveBeenCalledTimes(1);
|
|
313
|
-
expect(blockVerifier.verifyBlock).toHaveBeenCalledTimes(0);
|
|
314
|
-
await verifyOnProgress([thumbnailSize]);
|
|
315
|
-
});
|
|
316
|
-
|
|
317
|
-
it('should handle failure when encrypting thumbnails', async () => {
|
|
318
|
-
cryptoService.encryptThumbnail = jest.fn().mockImplementation(async function () {
|
|
319
|
-
throw new Error('Failed to encrypt thumbnail');
|
|
320
|
-
});
|
|
321
|
-
|
|
322
|
-
await verifyFailure('Failed to encrypt thumbnail', 0);
|
|
323
|
-
expect(cryptoService.encryptThumbnail).toHaveBeenCalledTimes(1);
|
|
324
|
-
});
|
|
325
|
-
|
|
326
|
-
it('should handle failure when encrypting block', async () => {
|
|
327
|
-
cryptoService.encryptBlock = jest.fn().mockImplementation(async function () {
|
|
328
|
-
throw new Error('Failed to encrypt block');
|
|
329
|
-
});
|
|
330
|
-
|
|
331
|
-
// Encrypting thumbnails is before blocks, thus it can be uploaded before failure.
|
|
332
|
-
await verifyFailure('Failed to encrypt block', 1024);
|
|
333
|
-
// 1 block + 1 retry, others are skipped
|
|
334
|
-
expect(cryptoService.encryptBlock).toHaveBeenCalledTimes(2);
|
|
335
|
-
});
|
|
336
|
-
|
|
337
|
-
it('should handle one time-off failure when encrypting block', async () => {
|
|
338
|
-
let count = 0;
|
|
339
|
-
cryptoService.encryptBlock = jest.fn().mockImplementation(async function (verifyBlock, keys, block, index) {
|
|
340
|
-
if (count === 0) {
|
|
341
|
-
count++;
|
|
342
|
-
throw new Error('Failed to encrypt block');
|
|
343
|
-
}
|
|
344
|
-
return mockEncryptBlock(verifyBlock, keys, block, index);
|
|
345
|
-
});
|
|
346
|
-
|
|
347
|
-
await verifySuccess();
|
|
348
|
-
// 1 block + 1 retry + 2 other blocks without retry
|
|
349
|
-
expect(cryptoService.encryptBlock).toHaveBeenCalledTimes(4);
|
|
350
|
-
await verifyOnProgress([thumbnailSize, 4 * 1024 * 1024, 4 * 1024 * 1024, 2 * 1024 * 1024]);
|
|
351
|
-
});
|
|
352
|
-
|
|
353
|
-
it('should handle failure when requesting tokens', async () => {
|
|
354
|
-
apiService.requestBlockUpload = jest.fn().mockImplementation(async function () {
|
|
355
|
-
throw new Error('Failed to request tokens');
|
|
356
|
-
});
|
|
357
|
-
|
|
358
|
-
await verifyFailure('Failed to request tokens', 0);
|
|
359
|
-
});
|
|
127
|
+
describe('writeFile', () => {
|
|
128
|
+
// @ts-expect-error Ignore mocking File
|
|
129
|
+
const file = {
|
|
130
|
+
type: 'image/png',
|
|
131
|
+
size: 1000,
|
|
132
|
+
lastModified: 123456789,
|
|
133
|
+
stream: jest.fn().mockReturnValue('stream'),
|
|
134
|
+
} as File;
|
|
135
|
+
const thumbnails: Thumbnail[] = [];
|
|
136
|
+
const onProgress = jest.fn();
|
|
360
137
|
|
|
361
|
-
it('should
|
|
362
|
-
|
|
363
|
-
if (token === 'token/thumbnail:1') {
|
|
364
|
-
throw new Error('Failed to upload thumbnail');
|
|
365
|
-
}
|
|
366
|
-
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
367
|
-
});
|
|
138
|
+
it('should set media type if not set', async () => {
|
|
139
|
+
await uploader.writeFile(file, thumbnails, onProgress);
|
|
368
140
|
|
|
369
|
-
|
|
370
|
-
|
|
141
|
+
expect(metadata.mediaType).toEqual('image/png');
|
|
142
|
+
expect(startUploadSpy).toHaveBeenCalledWith('stream', thumbnails, onProgress);
|
|
371
143
|
});
|
|
372
144
|
|
|
373
|
-
it('should
|
|
374
|
-
|
|
375
|
-
apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
|
|
376
|
-
if (token === 'token/thumbnail:1' && count === 0) {
|
|
377
|
-
count++;
|
|
378
|
-
throw new Error('Failed to upload thumbnail');
|
|
379
|
-
}
|
|
380
|
-
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
381
|
-
});
|
|
145
|
+
it('should set expected size if not set', async () => {
|
|
146
|
+
await uploader.writeFile(file, thumbnails, onProgress);
|
|
382
147
|
|
|
383
|
-
|
|
384
|
-
expect(
|
|
385
|
-
// 3 blocks + 1 retry + 1 thumbnail
|
|
386
|
-
expect(apiService.uploadBlock).toHaveBeenCalledTimes(5);
|
|
387
|
-
await verifyOnProgress([4 * 1024 * 1024, 4 * 1024 * 1024, 2 * 1024 * 1024, 1024]);
|
|
148
|
+
expect(metadata.expectedSize).toEqual(file.size);
|
|
149
|
+
expect(startUploadSpy).toHaveBeenCalledWith('stream', thumbnails, onProgress);
|
|
388
150
|
});
|
|
389
151
|
|
|
390
|
-
it('should
|
|
391
|
-
|
|
392
|
-
if (token === 'token/block:3') {
|
|
393
|
-
throw new Error('Failed to upload block');
|
|
394
|
-
}
|
|
395
|
-
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
396
|
-
});
|
|
152
|
+
it('should set modification time if not set', async () => {
|
|
153
|
+
await uploader.writeFile(file, thumbnails, onProgress);
|
|
397
154
|
|
|
398
|
-
|
|
399
|
-
|
|
155
|
+
expect(metadata.modificationTime).toEqual(new Date(123456789));
|
|
156
|
+
expect(startUploadSpy).toHaveBeenCalledWith('stream', thumbnails, onProgress);
|
|
400
157
|
});
|
|
401
158
|
|
|
402
|
-
it('should
|
|
403
|
-
|
|
404
|
-
apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
|
|
405
|
-
if (token === 'token/block:2' && count === 0) {
|
|
406
|
-
count++;
|
|
407
|
-
throw new Error('Failed to upload block');
|
|
408
|
-
}
|
|
409
|
-
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
410
|
-
});
|
|
159
|
+
it('should throw an error if upload already started', async () => {
|
|
160
|
+
await uploader.writeFile(file, thumbnails, onProgress);
|
|
411
161
|
|
|
412
|
-
await
|
|
413
|
-
expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(1);
|
|
414
|
-
// 3 blocks + 1 retry + 1 thumbnail
|
|
415
|
-
expect(apiService.uploadBlock).toHaveBeenCalledTimes(5);
|
|
416
|
-
await verifyOnProgress([1024, 4 * 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024]);
|
|
162
|
+
await expect(uploader.writeFile(file, thumbnails, onProgress)).rejects.toThrow('Upload already started');
|
|
417
163
|
});
|
|
164
|
+
});
|
|
418
165
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
count++;
|
|
424
|
-
throw new APIHTTPError('Expired token', HTTPErrorCode.NOT_FOUND);
|
|
425
|
-
}
|
|
426
|
-
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
427
|
-
});
|
|
166
|
+
describe('writeStream', () => {
|
|
167
|
+
const stream = new ReadableStream();
|
|
168
|
+
const thumbnails: Thumbnail[] = [];
|
|
169
|
+
const onProgress = jest.fn();
|
|
428
170
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(2);
|
|
432
|
-
expect(apiService.requestBlockUpload).toHaveBeenCalledWith(
|
|
433
|
-
revisionDraft.nodeRevisionUid,
|
|
434
|
-
revisionDraft.nodeKeys.signatureAddress.addressId,
|
|
435
|
-
{
|
|
436
|
-
contentBlocks: [
|
|
437
|
-
{
|
|
438
|
-
index: 2,
|
|
439
|
-
encryptedSize: 4 * 1024 * 1024 + 10000,
|
|
440
|
-
hash: 'blockHash',
|
|
441
|
-
armoredSignature: 'signature',
|
|
442
|
-
verificationToken: 'verificationToken',
|
|
443
|
-
}
|
|
444
|
-
],
|
|
445
|
-
},
|
|
446
|
-
);
|
|
447
|
-
// 3 blocks + 1 retry + 1 thumbnail
|
|
448
|
-
expect(apiService.uploadBlock).toHaveBeenCalledTimes(5);
|
|
449
|
-
await verifyOnProgress([1024, 4 * 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024]);
|
|
450
|
-
});
|
|
171
|
+
it('should start the upload process', async () => {
|
|
172
|
+
await uploader.writeStream(stream, thumbnails, onProgress);
|
|
451
173
|
|
|
452
|
-
|
|
453
|
-
const error = new Error('Aborted');
|
|
454
|
-
const controller = uploader.writeStream(stream, thumbnails, onProgress);
|
|
455
|
-
abortController.abort(error);
|
|
456
|
-
await controller.completion();
|
|
457
|
-
expect(apiService.uploadBlock.mock.calls[0][4]?.aborted).toBe(true);
|
|
174
|
+
expect(startUploadSpy).toHaveBeenCalledWith(stream, thumbnails, onProgress);
|
|
458
175
|
});
|
|
459
176
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
blockVerifier.verifyBlock = jest.fn().mockRejectedValue(new IntegrityError('Block verification error'));
|
|
463
|
-
await verifyFailure('Block verification error', 1024);
|
|
464
|
-
expect(telemetry.logBlockVerificationError).toHaveBeenCalledWith(false);
|
|
465
|
-
});
|
|
466
|
-
|
|
467
|
-
it('should report block verification error when retry helped', async () => {
|
|
468
|
-
blockVerifier.verifyBlock = jest.fn().mockRejectedValueOnce(new IntegrityError('Block verification error')).mockResolvedValue({
|
|
469
|
-
verificationToken: new Uint8Array(),
|
|
470
|
-
});
|
|
471
|
-
await verifySuccess();
|
|
472
|
-
expect(telemetry.logBlockVerificationError).toHaveBeenCalledWith(true);
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
it('should throw an error if block count does not match', async () => {
|
|
476
|
-
uploader = new Fileuploader(
|
|
477
|
-
telemetry,
|
|
478
|
-
apiService,
|
|
479
|
-
cryptoService,
|
|
480
|
-
uploadManager,
|
|
481
|
-
blockVerifier,
|
|
482
|
-
revisionDraft,
|
|
483
|
-
{
|
|
484
|
-
// Fake expected size to break verification
|
|
485
|
-
expectedSize: 1 * 1024 * 1024 + 1024,
|
|
486
|
-
mediaType: '',
|
|
487
|
-
},
|
|
488
|
-
onFinish,
|
|
489
|
-
);
|
|
490
|
-
|
|
491
|
-
await verifyFailure(
|
|
492
|
-
'Some file parts failed to upload',
|
|
493
|
-
10 * 1024 * 1024 + 1024,
|
|
494
|
-
1 * 1024 * 1024 + 1024,
|
|
495
|
-
);
|
|
496
|
-
});
|
|
497
|
-
|
|
498
|
-
it('should throw an error if file size does not match', async () => {
|
|
499
|
-
cryptoService.encryptBlock = jest.fn().mockImplementation(async (_, __, block, index) => ({
|
|
500
|
-
index,
|
|
501
|
-
encryptedData: block,
|
|
502
|
-
armoredSignature: 'signature',
|
|
503
|
-
verificationToken: 'verificationToken',
|
|
504
|
-
originalSize: 0, // Fake original size to break verification
|
|
505
|
-
encryptedSize: block.length + 10000,
|
|
506
|
-
hash: 'blockHash',
|
|
507
|
-
}));
|
|
177
|
+
it('should throw an error if upload already started', async () => {
|
|
178
|
+
await uploader.writeStream(stream, thumbnails, onProgress);
|
|
508
179
|
|
|
509
|
-
|
|
510
|
-
});
|
|
180
|
+
await expect(uploader.writeStream(stream, thumbnails, onProgress)).rejects.toThrow('Upload already started');
|
|
511
181
|
});
|
|
512
182
|
});
|
|
513
183
|
});
|