@protontech/drive-sdk 0.14.1 → 0.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/internal/batchLoading.d.ts +2 -0
- package/dist/internal/batchLoading.js +18 -5
- package/dist/internal/batchLoading.js.map +1 -1
- package/dist/internal/batchLoading.test.js +92 -0
- package/dist/internal/batchLoading.test.js.map +1 -1
- package/dist/internal/nodes/nodesAccess.js +1 -1
- package/dist/internal/nodes/nodesAccess.js.map +1 -1
- package/dist/internal/nodes/nodesAccess.test.js +3 -2
- package/dist/internal/nodes/nodesAccess.test.js.map +1 -1
- package/dist/internal/photos/index.js +3 -1
- package/dist/internal/photos/index.js.map +1 -1
- package/dist/internal/photos/upload.d.ts +1 -1
- package/dist/internal/photos/upload.js +2 -2
- package/dist/internal/photos/upload.js.map +1 -1
- package/dist/internal/sharing/events.d.ts +4 -2
- package/dist/internal/sharing/events.js +40 -9
- package/dist/internal/sharing/events.js.map +1 -1
- package/dist/internal/sharing/events.test.js +30 -2
- package/dist/internal/sharing/events.test.js.map +1 -1
- package/dist/internal/sharing/index.js +1 -1
- package/dist/internal/sharing/index.js.map +1 -1
- package/dist/internal/upload/fileUploader.d.ts +19 -3
- package/dist/internal/upload/fileUploader.js +31 -5
- package/dist/internal/upload/fileUploader.js.map +1 -1
- package/dist/internal/upload/fileUploader.test.js +1 -1
- package/dist/internal/upload/fileUploader.test.js.map +1 -1
- package/dist/internal/upload/index.js +4 -11
- package/dist/internal/upload/index.js.map +1 -1
- package/dist/internal/upload/index.test.js +104 -45
- package/dist/internal/upload/index.test.js.map +1 -1
- package/dist/internal/upload/smallFileUploader.d.ts +14 -14
- package/dist/internal/upload/smallFileUploader.js +38 -20
- package/dist/internal/upload/smallFileUploader.js.map +1 -1
- package/dist/internal/upload/smallFileUploader.test.js +35 -36
- package/dist/internal/upload/smallFileUploader.test.js.map +1 -1
- package/package.json +1 -1
- package/src/internal/batchLoading.test.ts +104 -0
- package/src/internal/batchLoading.ts +21 -5
- package/src/internal/nodes/nodesAccess.test.ts +4 -3
- package/src/internal/nodes/nodesAccess.ts +1 -1
- package/src/internal/photos/index.ts +2 -0
- package/src/internal/photos/upload.ts +13 -1
- package/src/internal/sharing/events.test.ts +35 -2
- package/src/internal/sharing/events.ts +47 -10
- package/src/internal/sharing/index.ts +1 -0
- package/src/internal/upload/fileUploader.test.ts +1 -0
- package/src/internal/upload/fileUploader.ts +60 -2
- package/src/internal/upload/index.test.ts +121 -63
- package/src/internal/upload/index.ts +4 -30
- package/src/internal/upload/smallFileUploader.test.ts +33 -40
- package/src/internal/upload/smallFileUploader.ts +47 -36
|
@@ -1,18 +1,23 @@
|
|
|
1
|
-
import { FeatureFlagProvider,
|
|
1
|
+
import { FeatureFlagProvider, ThumbnailType, UploadMetadata } from '../../interface';
|
|
2
2
|
import { getMockTelemetry } from '../../tests/telemetry';
|
|
3
|
-
import { FileRevisionUploader, FileUploader } from './fileUploader';
|
|
3
|
+
import { FileRevisionUploader, FileUploader, Uploader } from './fileUploader';
|
|
4
4
|
import { initUploadModule } from './index';
|
|
5
|
-
import { SmallFileRevisionUploader, SmallFileUploader } from './smallFileUploader';
|
|
6
5
|
|
|
7
|
-
const
|
|
6
|
+
const RAW_SMALL_FILE_SIZE_LIMIT = (128 * 1024) / 1.1; // 128 KiB, must match index.ts
|
|
8
7
|
|
|
9
|
-
describe('initUploadModule
|
|
8
|
+
describe('initUploadModule', () => {
|
|
10
9
|
const parentFolderUid = 'parent-folder-uid';
|
|
11
10
|
const name = 'test-file.txt';
|
|
12
11
|
const nodeUid = 'node-uid';
|
|
13
12
|
|
|
14
13
|
let featureFlagProvider: jest.Mocked<FeatureFlagProvider>;
|
|
15
14
|
let uploadModule: ReturnType<typeof initUploadModule>;
|
|
15
|
+
let initSmallFileSpy: jest.SpyInstance;
|
|
16
|
+
let initSmallRevisionSpy: jest.SpyInstance;
|
|
17
|
+
let initStreamSpy: jest.SpyInstance;
|
|
18
|
+
|
|
19
|
+
let stream: ReadableStream;
|
|
20
|
+
const thumbnail100k = { type: ThumbnailType.Type1, thumbnail: new Uint8Array(100_000) };
|
|
16
21
|
|
|
17
22
|
beforeEach(() => {
|
|
18
23
|
const apiService = {};
|
|
@@ -31,69 +36,122 @@ describe('initUploadModule - uploader selection', () => {
|
|
|
31
36
|
nodesService as any,
|
|
32
37
|
featureFlagProvider as any,
|
|
33
38
|
);
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
describe('getFileUploader', () => {
|
|
37
|
-
it('returns SmallFileUploader when feature flag is enabled and file size is below limit', async () => {
|
|
38
|
-
featureFlagProvider.isEnabled.mockResolvedValue(true);
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
expect(uploader).toBeInstanceOf(SmallFileUploader);
|
|
40
|
+
initSmallFileSpy = jest.spyOn(FileUploader.prototype as any, 'initSmallFileUploader').mockResolvedValue({
|
|
41
|
+
nodeRevisionUid: 'revision-uid',
|
|
42
|
+
nodeUid: 'node-uid',
|
|
44
43
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const uploader = await uploadModule.getFileUploader(parentFolderUid, name, metadata);
|
|
63
|
-
|
|
64
|
-
expect(uploader).toBeInstanceOf(FileUploader);
|
|
44
|
+
initSmallRevisionSpy = jest
|
|
45
|
+
.spyOn(FileRevisionUploader.prototype as any, 'initSmallFileUploader')
|
|
46
|
+
.mockResolvedValue({
|
|
47
|
+
nodeRevisionUid: 'revision-uid',
|
|
48
|
+
nodeUid: 'node-uid',
|
|
49
|
+
});
|
|
50
|
+
initStreamSpy = jest.spyOn(Uploader.prototype as any, 'initStreamUploader').mockResolvedValue({
|
|
51
|
+
start: jest.fn().mockResolvedValue({
|
|
52
|
+
nodeRevisionUid: 'revision-uid',
|
|
53
|
+
nodeUid: 'node-uid',
|
|
54
|
+
}),
|
|
55
|
+
} as any);
|
|
56
|
+
|
|
57
|
+
stream = new ReadableStream({
|
|
58
|
+
start(controller) {
|
|
59
|
+
controller.close();
|
|
60
|
+
},
|
|
65
61
|
});
|
|
66
62
|
});
|
|
67
63
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const metadata: UploadMetadata = { expectedSize: 1, mediaType: 'text/plain' };
|
|
73
|
-
const uploader = await uploadModule.getFileRevisionUploader(nodeUid, metadata);
|
|
74
|
-
|
|
75
|
-
expect(uploader).toBeInstanceOf(SmallFileRevisionUploader);
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
it('returns FileRevisionUploader when feature flag is enabled but file size exceeds limit', async () => {
|
|
79
|
-
featureFlagProvider.isEnabled.mockResolvedValue(true);
|
|
80
|
-
|
|
81
|
-
const metadata: UploadMetadata = {
|
|
82
|
-
expectedSize: SMALL_FILE_SIZE_LIMIT + 1,
|
|
83
|
-
mediaType: 'text/plain',
|
|
84
|
-
};
|
|
85
|
-
const uploader = await uploadModule.getFileRevisionUploader(nodeUid, metadata);
|
|
86
|
-
|
|
87
|
-
expect(uploader).toBeInstanceOf(FileRevisionUploader);
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
it('returns FileRevisionUploader when feature flag is disabled even for small file', async () => {
|
|
91
|
-
featureFlagProvider.isEnabled.mockResolvedValue(false);
|
|
92
|
-
|
|
93
|
-
const metadata: UploadMetadata = { expectedSize: 1, mediaType: 'text/plain' };
|
|
94
|
-
const uploader = await uploadModule.getFileRevisionUploader(nodeUid, metadata);
|
|
64
|
+
afterEach(() => {
|
|
65
|
+
jest.restoreAllMocks();
|
|
66
|
+
});
|
|
95
67
|
|
|
96
|
-
|
|
68
|
+
async function drainUpload(controller: { completion(): Promise<unknown> }) {
|
|
69
|
+
await controller.completion();
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const suites = [
|
|
73
|
+
{
|
|
74
|
+
method: 'getFileUploader',
|
|
75
|
+
getUploader: (metadata: UploadMetadata) => uploadModule.getFileUploader(parentFolderUid, name, metadata),
|
|
76
|
+
expect: (option: 'small' | 'stream') => {
|
|
77
|
+
if (option === 'stream') {
|
|
78
|
+
expect(initStreamSpy).toHaveBeenCalled();
|
|
79
|
+
expect(initSmallFileSpy).not.toHaveBeenCalled();
|
|
80
|
+
expect(initSmallRevisionSpy).not.toHaveBeenCalled();
|
|
81
|
+
} else {
|
|
82
|
+
expect(initSmallFileSpy).toHaveBeenCalled();
|
|
83
|
+
expect(initStreamSpy).not.toHaveBeenCalled();
|
|
84
|
+
expect(initSmallRevisionSpy).not.toHaveBeenCalled();
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
method: 'getFileRevisionUploader',
|
|
90
|
+
getUploader: (metadata: UploadMetadata) => uploadModule.getFileRevisionUploader(nodeUid, metadata),
|
|
91
|
+
expect: (option: 'small' | 'stream') => {
|
|
92
|
+
if (option === 'stream') {
|
|
93
|
+
expect(initStreamSpy).toHaveBeenCalled();
|
|
94
|
+
expect(initSmallFileSpy).not.toHaveBeenCalled();
|
|
95
|
+
expect(initSmallRevisionSpy).not.toHaveBeenCalled();
|
|
96
|
+
} else {
|
|
97
|
+
expect(initSmallRevisionSpy).toHaveBeenCalled();
|
|
98
|
+
expect(initSmallFileSpy).not.toHaveBeenCalled();
|
|
99
|
+
expect(initStreamSpy).not.toHaveBeenCalled();
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
];
|
|
104
|
+
for (const suite of suites) {
|
|
105
|
+
describe(suite.method, () => {
|
|
106
|
+
it('uses stream path when feature flag is disabled even for small file', async () => {
|
|
107
|
+
featureFlagProvider.isEnabled.mockResolvedValue(false);
|
|
108
|
+
|
|
109
|
+
const metadata: UploadMetadata = { expectedSize: 1, mediaType: 'text/plain' };
|
|
110
|
+
const uploader = await suite.getUploader(metadata);
|
|
111
|
+
await drainUpload(await uploader.uploadFromStream(stream, []));
|
|
112
|
+
|
|
113
|
+
suite.expect('stream');
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('uses small-file path when flag is on and encrypted total size is below cap', async () => {
|
|
117
|
+
featureFlagProvider.isEnabled.mockResolvedValue(true);
|
|
118
|
+
|
|
119
|
+
const metadata: UploadMetadata = { expectedSize: 100, mediaType: 'text/plain' };
|
|
120
|
+
const uploader = await suite.getUploader(metadata);
|
|
121
|
+
await drainUpload(await uploader.uploadFromStream(stream, []));
|
|
122
|
+
|
|
123
|
+
suite.expect('small');
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('uses small-file path when flag is on and encrypted total size with thumbnails is below cap', async () => {
|
|
127
|
+
featureFlagProvider.isEnabled.mockResolvedValue(true);
|
|
128
|
+
|
|
129
|
+
const metadata: UploadMetadata = { expectedSize: 100, mediaType: 'image/jpeg' };
|
|
130
|
+
const uploader = await suite.getUploader(metadata);
|
|
131
|
+
await drainUpload(await uploader.uploadFromStream(stream, [thumbnail100k]));
|
|
132
|
+
|
|
133
|
+
suite.expect('small');
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('uses stream path when feature flag is enabled but raw file size exceeds limit', async () => {
|
|
137
|
+
featureFlagProvider.isEnabled.mockResolvedValue(true);
|
|
138
|
+
|
|
139
|
+
const metadata: UploadMetadata = { expectedSize: RAW_SMALL_FILE_SIZE_LIMIT, mediaType: 'text/plain' };
|
|
140
|
+
const uploader = await suite.getUploader(metadata);
|
|
141
|
+
await drainUpload(await uploader.uploadFromStream(stream, []));
|
|
142
|
+
|
|
143
|
+
suite.expect('stream');
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it('uses stream path when thumbnail bytes push encrypted total size with thumbnail exceeds limit', async () => {
|
|
147
|
+
featureFlagProvider.isEnabled.mockResolvedValue(true);
|
|
148
|
+
|
|
149
|
+
const metadata: UploadMetadata = { expectedSize: 100_000, mediaType: 'image/jpeg' };
|
|
150
|
+
const uploader = await suite.getUploader(metadata);
|
|
151
|
+
await drainUpload(await uploader.uploadFromStream(stream, [thumbnail100k]));
|
|
152
|
+
|
|
153
|
+
suite.expect('stream');
|
|
154
|
+
});
|
|
97
155
|
});
|
|
98
|
-
}
|
|
156
|
+
}
|
|
99
157
|
});
|
|
@@ -8,7 +8,6 @@ import { FileUploader as FileUploaderClass, FileRevisionUploader } from './fileU
|
|
|
8
8
|
import { NodesService, SharesService } from './interface';
|
|
9
9
|
import { UploadManager } from './manager';
|
|
10
10
|
import { UploadQueue } from './queue';
|
|
11
|
-
import { SmallFileRevisionUploader, SmallFileUploader } from './smallFileUploader';
|
|
12
11
|
import { UploadTelemetry } from './telemetry';
|
|
13
12
|
|
|
14
13
|
const SMALL_FILE_SIZE_LIMIT = 128 * 1024; // 128 KiB
|
|
@@ -38,13 +37,13 @@ export function initUploadModule(
|
|
|
38
37
|
|
|
39
38
|
const queue = new UploadQueue();
|
|
40
39
|
|
|
41
|
-
async function
|
|
40
|
+
async function shouldUseSmallFileUpload(expectedSize: number): Promise<boolean> {
|
|
42
41
|
const isEnabled =
|
|
43
42
|
allowSmallFileUpload && (await featureFlagProvider.isEnabled(FeatureFlags.DriveSmallFileUpload));
|
|
44
43
|
if (!isEnabled) {
|
|
45
44
|
return false;
|
|
46
45
|
}
|
|
47
|
-
return
|
|
46
|
+
return expectedSize < SMALL_FILE_SIZE_LIMIT;
|
|
48
47
|
}
|
|
49
48
|
|
|
50
49
|
/**
|
|
@@ -66,20 +65,6 @@ export function initUploadModule(
|
|
|
66
65
|
queue.releaseCapacity(metadata.expectedSize);
|
|
67
66
|
};
|
|
68
67
|
|
|
69
|
-
if (await useSmallFileUpload(metadata)) {
|
|
70
|
-
return new SmallFileUploader(
|
|
71
|
-
uploadTelemetry,
|
|
72
|
-
api,
|
|
73
|
-
cryptoService,
|
|
74
|
-
manager,
|
|
75
|
-
metadata,
|
|
76
|
-
onFinish,
|
|
77
|
-
signal,
|
|
78
|
-
parentFolderUid,
|
|
79
|
-
name,
|
|
80
|
-
);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
68
|
return new FileUploaderClass(
|
|
84
69
|
uploadTelemetry,
|
|
85
70
|
api,
|
|
@@ -89,6 +74,7 @@ export function initUploadModule(
|
|
|
89
74
|
name,
|
|
90
75
|
metadata,
|
|
91
76
|
onFinish,
|
|
77
|
+
shouldUseSmallFileUpload,
|
|
92
78
|
signal,
|
|
93
79
|
);
|
|
94
80
|
}
|
|
@@ -111,19 +97,6 @@ export function initUploadModule(
|
|
|
111
97
|
queue.releaseCapacity(metadata.expectedSize);
|
|
112
98
|
};
|
|
113
99
|
|
|
114
|
-
if (await useSmallFileUpload(metadata)) {
|
|
115
|
-
return new SmallFileRevisionUploader(
|
|
116
|
-
uploadTelemetry,
|
|
117
|
-
api,
|
|
118
|
-
cryptoService,
|
|
119
|
-
manager,
|
|
120
|
-
metadata,
|
|
121
|
-
onFinish,
|
|
122
|
-
signal,
|
|
123
|
-
nodeUid,
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
100
|
return new FileRevisionUploader(
|
|
128
101
|
uploadTelemetry,
|
|
129
102
|
api,
|
|
@@ -132,6 +105,7 @@ export function initUploadModule(
|
|
|
132
105
|
nodeUid,
|
|
133
106
|
metadata,
|
|
134
107
|
onFinish,
|
|
108
|
+
shouldUseSmallFileUpload,
|
|
135
109
|
signal,
|
|
136
110
|
);
|
|
137
111
|
}
|
|
@@ -6,9 +6,10 @@ import { UploadAPIService } from './apiService';
|
|
|
6
6
|
import { UploadCryptoService } from './cryptoService';
|
|
7
7
|
import { UploadManager } from './manager';
|
|
8
8
|
import { NodeCrypto } from './interface';
|
|
9
|
+
import { mergeUint8Arrays } from '../utils';
|
|
9
10
|
|
|
10
|
-
const MOCK_BLOCK_HASH = new Uint8Array(32).fill(
|
|
11
|
-
const MOCK_VERIFICATION_TOKEN = new Uint8Array(16).fill(
|
|
11
|
+
const MOCK_BLOCK_HASH = new Uint8Array(32).fill(4);
|
|
12
|
+
const MOCK_VERIFICATION_TOKEN = new Uint8Array(16).fill(5);
|
|
12
13
|
|
|
13
14
|
function createStream(bytes: number[]): ReadableStream<Uint8Array> {
|
|
14
15
|
return new ReadableStream({
|
|
@@ -108,7 +109,7 @@ describe('SmallFileUploader', () => {
|
|
|
108
109
|
encryptedData: new Uint8Array(thumbnail.thumbnail),
|
|
109
110
|
originalSize: thumbnail.thumbnail.length,
|
|
110
111
|
encryptedSize: thumbnail.thumbnail.length + 100,
|
|
111
|
-
|
|
112
|
+
hashPromise: Promise.resolve(new Uint8Array(32).fill(thumbnail.type)),
|
|
112
113
|
})),
|
|
113
114
|
encryptBlock: jest.fn().mockImplementation(mockEncryptBlock),
|
|
114
115
|
verifyBlock: jest.fn().mockResolvedValue({ verificationToken: MOCK_VERIFICATION_TOKEN }),
|
|
@@ -138,7 +139,6 @@ describe('SmallFileUploader', () => {
|
|
|
138
139
|
function createUploader() {
|
|
139
140
|
return new SmallFileUploader(
|
|
140
141
|
telemetry,
|
|
141
|
-
apiService,
|
|
142
142
|
cryptoService,
|
|
143
143
|
uploadManager,
|
|
144
144
|
metadata,
|
|
@@ -157,24 +157,13 @@ describe('SmallFileUploader', () => {
|
|
|
157
157
|
const uploader = createUploader();
|
|
158
158
|
const stream = createStream([1, 2, 3]);
|
|
159
159
|
|
|
160
|
-
const
|
|
161
|
-
const result = await controller.completion();
|
|
160
|
+
const result = await uploader.upload(stream, thumbnails, onProgress);
|
|
162
161
|
|
|
163
162
|
expect(uploadManager.generateNewFileCrypto).toHaveBeenCalledWith(parentFolderUid, name);
|
|
164
163
|
expect(uploadManager.uploadFile).toHaveBeenCalledTimes(1);
|
|
165
164
|
expect(result).toEqual({ nodeUid: 'nodeUid', nodeRevisionUid: 'nodeRevisionUid' });
|
|
166
165
|
expect(onProgress).toHaveBeenCalledWith(metadata.expectedSize);
|
|
167
166
|
});
|
|
168
|
-
|
|
169
|
-
it('should throw if upload already started', async () => {
|
|
170
|
-
const uploader = createUploader();
|
|
171
|
-
const stream = createStream([1, 2, 3]);
|
|
172
|
-
|
|
173
|
-
await uploader.uploadFromStream(stream, thumbnails, onProgress);
|
|
174
|
-
await expect(uploader.uploadFromStream(stream, thumbnails, onProgress)).rejects.toThrow(
|
|
175
|
-
'Upload already started',
|
|
176
|
-
);
|
|
177
|
-
});
|
|
178
167
|
});
|
|
179
168
|
|
|
180
169
|
describe('buildPayloads (via upload flow)', () => {
|
|
@@ -186,8 +175,7 @@ describe('SmallFileUploader', () => {
|
|
|
186
175
|
{ type: ThumbnailType.Type2, thumbnail: new Uint8Array([30, 40, 50]) },
|
|
187
176
|
];
|
|
188
177
|
|
|
189
|
-
await uploader.
|
|
190
|
-
await (uploader as any).controller.completion();
|
|
178
|
+
await uploader.upload(stream, thumbnails, undefined);
|
|
191
179
|
|
|
192
180
|
expect(uploadManager.uploadFile).toHaveBeenCalledWith(
|
|
193
181
|
parentFolderUid,
|
|
@@ -203,8 +191,16 @@ describe('SmallFileUploader', () => {
|
|
|
203
191
|
verificationToken: MOCK_VERIFICATION_TOKEN,
|
|
204
192
|
}),
|
|
205
193
|
[
|
|
206
|
-
{
|
|
207
|
-
|
|
194
|
+
{
|
|
195
|
+
type: ThumbnailType.Type1,
|
|
196
|
+
encryptedData: expect.any(Uint8Array),
|
|
197
|
+
blockHash: new Uint8Array(32).fill(ThumbnailType.Type1),
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
type: ThumbnailType.Type2,
|
|
201
|
+
encryptedData: expect.any(Uint8Array),
|
|
202
|
+
blockHash: new Uint8Array(32).fill(ThumbnailType.Type2),
|
|
203
|
+
},
|
|
208
204
|
],
|
|
209
205
|
);
|
|
210
206
|
|
|
@@ -212,7 +208,11 @@ describe('SmallFileUploader', () => {
|
|
|
212
208
|
expect(cryptoService.encryptThumbnail).toHaveBeenCalledTimes(2);
|
|
213
209
|
expect(cryptoService.commitFile).toHaveBeenCalledWith(
|
|
214
210
|
expect.anything(),
|
|
215
|
-
|
|
211
|
+
mergeUint8Arrays([
|
|
212
|
+
new Uint8Array(32).fill(ThumbnailType.Type1),
|
|
213
|
+
new Uint8Array(32).fill(ThumbnailType.Type2),
|
|
214
|
+
MOCK_BLOCK_HASH,
|
|
215
|
+
]),
|
|
216
216
|
expect.any(String),
|
|
217
217
|
);
|
|
218
218
|
});
|
|
@@ -223,8 +223,7 @@ describe('SmallFileUploader', () => {
|
|
|
223
223
|
metadata.expectedSize = content.length;
|
|
224
224
|
const stream = createStream(content);
|
|
225
225
|
|
|
226
|
-
await uploader.
|
|
227
|
-
await (uploader as any).controller.completion();
|
|
226
|
+
await uploader.upload(stream, [], undefined);
|
|
228
227
|
|
|
229
228
|
expect(cryptoService.encryptBlock).toHaveBeenCalledWith(
|
|
230
229
|
expect.any(Function),
|
|
@@ -241,8 +240,7 @@ describe('SmallFileUploader', () => {
|
|
|
241
240
|
];
|
|
242
241
|
const stream = createStream([1, 2, 3]);
|
|
243
242
|
|
|
244
|
-
await uploader.
|
|
245
|
-
await (uploader as any).controller.completion();
|
|
243
|
+
await uploader.upload(stream, thumbnails, undefined);
|
|
246
244
|
|
|
247
245
|
expect(cryptoService.encryptThumbnail).toHaveBeenCalledWith(
|
|
248
246
|
expect.objectContaining({
|
|
@@ -259,8 +257,7 @@ describe('SmallFileUploader', () => {
|
|
|
259
257
|
const uploader = createUploader();
|
|
260
258
|
const stream = createStream([1, 2, 3]);
|
|
261
259
|
|
|
262
|
-
await uploader.
|
|
263
|
-
await (uploader as any).controller.completion();
|
|
260
|
+
await uploader.upload(stream, [], undefined);
|
|
264
261
|
|
|
265
262
|
const [nodeKeys, manifest, extendedAttributes] = (cryptoService.commitFile as jest.Mock).mock.calls[0];
|
|
266
263
|
expect(manifest).toEqual(MOCK_BLOCK_HASH);
|
|
@@ -275,10 +272,10 @@ describe('SmallFileUploader', () => {
|
|
|
275
272
|
metadata.expectedSize = 5;
|
|
276
273
|
const stream = createStream([1, 2, 3]); // only 3 bytes
|
|
277
274
|
|
|
278
|
-
const
|
|
275
|
+
const promise = uploader.upload(stream, [], undefined);
|
|
279
276
|
|
|
280
|
-
await expect(
|
|
281
|
-
await expect(
|
|
277
|
+
await expect(promise).rejects.toThrow(IntegrityError);
|
|
278
|
+
await expect(promise).rejects.toMatchObject({
|
|
282
279
|
debug: { actual: 3, expected: 5 },
|
|
283
280
|
});
|
|
284
281
|
});
|
|
@@ -288,10 +285,10 @@ describe('SmallFileUploader', () => {
|
|
|
288
285
|
metadata.expectedSha1 = 'a'.repeat(40); // wrong sha1
|
|
289
286
|
const stream = createStream([1, 2, 3]);
|
|
290
287
|
|
|
291
|
-
const
|
|
288
|
+
const promise = uploader.upload(stream, [], undefined);
|
|
292
289
|
|
|
293
|
-
await expect(
|
|
294
|
-
await expect(
|
|
290
|
+
await expect(promise).rejects.toThrow(IntegrityError);
|
|
291
|
+
await expect(promise).rejects.toMatchObject({
|
|
295
292
|
debug: expect.objectContaining({
|
|
296
293
|
expectedSha1: 'a'.repeat(40),
|
|
297
294
|
}),
|
|
@@ -306,8 +303,7 @@ describe('SmallFileUploader', () => {
|
|
|
306
303
|
const stream = createStream([]);
|
|
307
304
|
const onProgress = jest.fn();
|
|
308
305
|
|
|
309
|
-
const
|
|
310
|
-
const result = await controller.completion();
|
|
306
|
+
const result = await uploader.upload(stream, [], onProgress);
|
|
311
307
|
|
|
312
308
|
expect(result).toEqual({ nodeUid: 'nodeUid', nodeRevisionUid: 'nodeRevisionUid' });
|
|
313
309
|
expect(cryptoService.encryptBlock).not.toHaveBeenCalled();
|
|
@@ -430,7 +426,6 @@ describe('SmallFileRevisionUploader', () => {
|
|
|
430
426
|
function createUploader() {
|
|
431
427
|
return new SmallFileRevisionUploader(
|
|
432
428
|
telemetry,
|
|
433
|
-
apiService,
|
|
434
429
|
cryptoService,
|
|
435
430
|
uploadManager,
|
|
436
431
|
metadata,
|
|
@@ -444,8 +439,7 @@ describe('SmallFileRevisionUploader', () => {
|
|
|
444
439
|
const uploader = createUploader();
|
|
445
440
|
const stream = createStream([1, 2, 3]);
|
|
446
441
|
|
|
447
|
-
const
|
|
448
|
-
const result = await controller.completion();
|
|
442
|
+
const result = await uploader.upload(stream, [], undefined);
|
|
449
443
|
|
|
450
444
|
expect(result).toEqual({ nodeUid: 'nodeUid', nodeRevisionUid: 'nodeRevisionUid' });
|
|
451
445
|
expect(cryptoService.encryptBlock).toHaveBeenCalledWith(expect.any(Function), expect.anything(), Uint8Array.from([1, 2, 3]), 0);
|
|
@@ -471,8 +465,7 @@ describe('SmallFileRevisionUploader', () => {
|
|
|
471
465
|
const uploader = createUploader();
|
|
472
466
|
const stream = createStream([]);
|
|
473
467
|
|
|
474
|
-
const
|
|
475
|
-
const result = await controller.completion();
|
|
468
|
+
const result = await uploader.upload(stream, [], undefined);
|
|
476
469
|
|
|
477
470
|
expect(result).toEqual({ nodeUid: 'nodeUid', nodeRevisionUid: 'nodeRevisionUid' });
|
|
478
471
|
expect(cryptoService.encryptBlock).not.toHaveBeenCalled();
|
|
@@ -3,12 +3,11 @@ import { AbortError, IntegrityError } from '../../errors';
|
|
|
3
3
|
import { Logger, Thumbnail, ThumbnailType, UploadMetadata } from '../../interface';
|
|
4
4
|
import { getErrorMessage } from '../errors';
|
|
5
5
|
import { generateFileExtendedAttributes } from '../nodes';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { mergeUint8Arrays } from '../utils';
|
|
7
|
+
import { verifyBlockWithContentKey } from './blockVerifier';
|
|
8
8
|
import { UploadCryptoService } from './cryptoService';
|
|
9
9
|
import { UploadDigests } from './digests';
|
|
10
|
-
import {
|
|
11
|
-
import { NodeRevisionDraft, NodeCrypto } from './interface';
|
|
10
|
+
import { NodeCrypto } from './interface';
|
|
12
11
|
import { UploadManager } from './manager';
|
|
13
12
|
import { readStreamToUint8Array } from './streamReader';
|
|
14
13
|
import { MAX_BLOCK_ENCRYPTION_RETRIES } from './streamUploader';
|
|
@@ -25,34 +24,23 @@ export type NodeKeys = {
|
|
|
25
24
|
* Base uploader for small file and small revision uploads.
|
|
26
25
|
* Shares the single-request flow: read content, get node crypto, encrypt, then call API.
|
|
27
26
|
*/
|
|
28
|
-
abstract class SmallUploader
|
|
27
|
+
abstract class SmallUploader {
|
|
29
28
|
protected logger: Logger;
|
|
29
|
+
protected abortController: AbortController;
|
|
30
30
|
|
|
31
31
|
constructor(
|
|
32
|
-
telemetry: UploadTelemetry,
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
signal: AbortSignal | undefined,
|
|
32
|
+
protected telemetry: UploadTelemetry,
|
|
33
|
+
protected cryptoService: UploadCryptoService,
|
|
34
|
+
protected manager: UploadManager,
|
|
35
|
+
protected metadata: UploadMetadata,
|
|
36
|
+
protected onFinish: () => void,
|
|
37
|
+
protected signal: AbortSignal | undefined,
|
|
39
38
|
) {
|
|
40
|
-
super(telemetry, apiService, cryptoService, manager, metadata, onFinish, signal);
|
|
41
39
|
this.logger = telemetry.getLoggerForSmallUpload();
|
|
42
|
-
|
|
43
|
-
protected async createRevisionDraft(): Promise<{
|
|
44
|
-
revisionDraft: NodeRevisionDraft;
|
|
45
|
-
blockVerifier: BlockVerifier;
|
|
46
|
-
}> {
|
|
47
|
-
throw new Error('Small upload does not use revision draft');
|
|
40
|
+
this.abortController = new AbortController();
|
|
48
41
|
}
|
|
49
42
|
|
|
50
|
-
|
|
51
|
-
protected async deleteRevisionDraft(revisionDraft: NodeRevisionDraft): Promise<void> {
|
|
52
|
-
throw new Error('Small upload does not use revision draft');
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
protected async startUpload(
|
|
43
|
+
async upload(
|
|
56
44
|
stream: ReadableStream,
|
|
57
45
|
thumbnails: Thumbnail[],
|
|
58
46
|
onProgress?: (uploadedBytes: number) => void,
|
|
@@ -105,7 +93,8 @@ abstract class SmallUploader extends Uploader {
|
|
|
105
93
|
this.encryptThumbnails(nodeKeys, thumbnails),
|
|
106
94
|
this.encryptContentBlock(nodeKeys, content.data),
|
|
107
95
|
]);
|
|
108
|
-
const
|
|
96
|
+
const manifest = await this.getManifest(encryptedBlock, encryptedThumbnails);
|
|
97
|
+
const commitPayload = await this.encryptCommitPayload(nodeKeys, content.sha1, manifest);
|
|
109
98
|
|
|
110
99
|
return {
|
|
111
100
|
commitPayload,
|
|
@@ -147,12 +136,22 @@ abstract class SmallUploader extends Uploader {
|
|
|
147
136
|
private async encryptThumbnails(
|
|
148
137
|
nodeKeys: NodeKeys,
|
|
149
138
|
thumbnails: Thumbnail[],
|
|
150
|
-
): Promise<
|
|
139
|
+
): Promise<
|
|
140
|
+
{
|
|
141
|
+
type: ThumbnailType;
|
|
142
|
+
encryptedData: Uint8Array<ArrayBuffer>;
|
|
143
|
+
blockHash: Uint8Array<ArrayBuffer>;
|
|
144
|
+
}[]
|
|
145
|
+
> {
|
|
151
146
|
const result = [];
|
|
152
147
|
for (const thumbnail of thumbnails) {
|
|
153
148
|
this.logger.debug(`Encrypting thumbnail ${thumbnail.type}`);
|
|
154
149
|
const enc = await this.cryptoService.encryptThumbnail(nodeKeys, thumbnail);
|
|
155
|
-
result.push({
|
|
150
|
+
result.push({
|
|
151
|
+
type: thumbnail.type,
|
|
152
|
+
encryptedData: enc.encryptedData,
|
|
153
|
+
blockHash: await enc.hashPromise,
|
|
154
|
+
});
|
|
156
155
|
}
|
|
157
156
|
return result;
|
|
158
157
|
}
|
|
@@ -228,21 +227,35 @@ abstract class SmallUploader extends Uploader {
|
|
|
228
227
|
};
|
|
229
228
|
}
|
|
230
229
|
|
|
231
|
-
private async
|
|
232
|
-
nodeKeys: NodeKeys,
|
|
233
|
-
contentSha1: string,
|
|
230
|
+
private async getManifest(
|
|
234
231
|
encryptedBlock:
|
|
235
232
|
| {
|
|
236
233
|
blockHash: Uint8Array<ArrayBuffer>;
|
|
237
234
|
}
|
|
238
235
|
| undefined,
|
|
236
|
+
encryptedThumbnails: {
|
|
237
|
+
type: ThumbnailType;
|
|
238
|
+
blockHash: Uint8Array<ArrayBuffer>;
|
|
239
|
+
}[],
|
|
240
|
+
): Promise<Uint8Array<ArrayBuffer>> {
|
|
241
|
+
encryptedThumbnails.sort((a, b) => a.type - b.type);
|
|
242
|
+
const hashes = [
|
|
243
|
+
...(await Promise.all(encryptedThumbnails.map(({ blockHash }) => blockHash))),
|
|
244
|
+
...(encryptedBlock ? [encryptedBlock.blockHash] : []),
|
|
245
|
+
];
|
|
246
|
+
return mergeUint8Arrays(hashes);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private async encryptCommitPayload(
|
|
250
|
+
nodeKeys: NodeKeys,
|
|
251
|
+
contentSha1: string,
|
|
252
|
+
manifest: Uint8Array<ArrayBuffer>,
|
|
239
253
|
): Promise<{
|
|
240
254
|
armoredManifestSignature: string;
|
|
241
255
|
armoredExtendedAttributes: string;
|
|
242
256
|
}> {
|
|
243
257
|
this.logger.debug(`Preparing commit payload`);
|
|
244
258
|
|
|
245
|
-
const manifest = encryptedBlock ? encryptedBlock.blockHash : new Uint8Array(0);
|
|
246
259
|
const extendedAttributes = generateFileExtendedAttributes(
|
|
247
260
|
{
|
|
248
261
|
modificationTime: this.metadata.modificationTime,
|
|
@@ -266,7 +279,6 @@ abstract class SmallUploader extends Uploader {
|
|
|
266
279
|
export class SmallFileUploader extends SmallUploader {
|
|
267
280
|
constructor(
|
|
268
281
|
telemetry: UploadTelemetry,
|
|
269
|
-
apiService: UploadAPIService,
|
|
270
282
|
cryptoService: UploadCryptoService,
|
|
271
283
|
manager: UploadManager,
|
|
272
284
|
metadata: UploadMetadata,
|
|
@@ -275,7 +287,7 @@ export class SmallFileUploader extends SmallUploader {
|
|
|
275
287
|
private parentFolderUid: string,
|
|
276
288
|
private name: string,
|
|
277
289
|
) {
|
|
278
|
-
super(telemetry,
|
|
290
|
+
super(telemetry, cryptoService, manager, metadata, onFinish, signal);
|
|
279
291
|
this.parentFolderUid = parentFolderUid;
|
|
280
292
|
this.name = name;
|
|
281
293
|
}
|
|
@@ -317,7 +329,6 @@ export class SmallFileUploader extends SmallUploader {
|
|
|
317
329
|
export class SmallFileRevisionUploader extends SmallUploader {
|
|
318
330
|
constructor(
|
|
319
331
|
telemetry: UploadTelemetry,
|
|
320
|
-
apiService: UploadAPIService,
|
|
321
332
|
cryptoService: UploadCryptoService,
|
|
322
333
|
manager: UploadManager,
|
|
323
334
|
metadata: UploadMetadata,
|
|
@@ -325,7 +336,7 @@ export class SmallFileRevisionUploader extends SmallUploader {
|
|
|
325
336
|
signal: AbortSignal | undefined,
|
|
326
337
|
private nodeUid: string,
|
|
327
338
|
) {
|
|
328
|
-
super(telemetry,
|
|
339
|
+
super(telemetry, cryptoService, manager, metadata, onFinish, signal);
|
|
329
340
|
this.nodeUid = nodeUid;
|
|
330
341
|
}
|
|
331
342
|
|