@protontech/drive-sdk 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/internal/apiService/apiService.js +6 -2
- package/dist/internal/apiService/apiService.js.map +1 -1
- package/dist/internal/apiService/apiService.test.js +9 -1
- package/dist/internal/apiService/apiService.test.js.map +1 -1
- package/dist/internal/download/seekableStream.d.ts +1 -2
- package/dist/internal/download/seekableStream.js +28 -11
- package/dist/internal/download/seekableStream.js.map +1 -1
- package/dist/internal/download/seekableStream.test.js +100 -0
- package/dist/internal/download/seekableStream.test.js.map +1 -1
- package/dist/internal/photos/index.js +2 -2
- package/dist/internal/photos/index.js.map +1 -1
- package/dist/internal/upload/index.js +4 -4
- package/dist/internal/upload/index.js.map +1 -1
- package/dist/internal/upload/queue.d.ts +18 -3
- package/dist/internal/upload/queue.js +27 -8
- package/dist/internal/upload/queue.js.map +1 -1
- package/dist/internal/upload/queue.test.d.ts +1 -0
- package/dist/internal/upload/queue.test.js +103 -0
- package/dist/internal/upload/queue.test.js.map +1 -0
- package/dist/internal/upload/streamUploader.d.ts +3 -0
- package/dist/internal/upload/streamUploader.js +29 -1
- package/dist/internal/upload/streamUploader.js.map +1 -1
- package/dist/internal/upload/streamUploader.test.js +65 -6
- package/dist/internal/upload/streamUploader.test.js.map +1 -1
- package/dist/internal/upload/telemetry.js +2 -3
- package/dist/internal/upload/telemetry.js.map +1 -1
- package/package.json +1 -1
- package/src/internal/apiService/apiService.test.ts +12 -2
- package/src/internal/apiService/apiService.ts +7 -2
- package/src/internal/download/seekableStream.test.ts +126 -1
- package/src/internal/download/seekableStream.ts +34 -13
- package/src/internal/photos/index.ts +2 -2
- package/src/internal/upload/index.ts +4 -4
- package/src/internal/upload/queue.test.ts +130 -0
- package/src/internal/upload/queue.ts +34 -9
- package/src/internal/upload/streamUploader.test.ts +80 -7
- package/src/internal/upload/streamUploader.ts +33 -1
- package/src/internal/upload/telemetry.ts +2 -3
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SeekableReadableStream, BufferedSeekableStream } from './seekableStream';
|
|
1
|
+
import { SeekableReadableStream, BufferedSeekableStream, UnderlyingSeekableSource } from './seekableStream';
|
|
2
2
|
|
|
3
3
|
describe('SeekableReadableStream', () => {
|
|
4
4
|
it('should call the seek callback when seek is called', async () => {
|
|
@@ -32,6 +32,7 @@ describe('SeekableReadableStream', () => {
|
|
|
32
32
|
describe('BufferedSeekableStream', () => {
|
|
33
33
|
let startWithCloseMock: jest.Mock;
|
|
34
34
|
let pullMock: jest.Mock;
|
|
35
|
+
let seekableSource: UnderlyingSeekableSource;
|
|
35
36
|
|
|
36
37
|
const data1 = new Uint8Array([1, 2, 3, 4, 5]);
|
|
37
38
|
const data2 = new Uint8Array([6, 7, 8, 9, 10]);
|
|
@@ -53,6 +54,26 @@ describe('BufferedSeekableStream', () => {
|
|
|
53
54
|
}
|
|
54
55
|
readIndex++;
|
|
55
56
|
});
|
|
57
|
+
|
|
58
|
+
// Simulates a real seekable source where seek repositions the read pointer
|
|
59
|
+
const fileData = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
|
|
60
|
+
let readPosition = 0;
|
|
61
|
+
const chunkSize = 5;
|
|
62
|
+
|
|
63
|
+
seekableSource = {
|
|
64
|
+
pull: jest.fn().mockImplementation((controller) => {
|
|
65
|
+
if (readPosition >= fileData.length) {
|
|
66
|
+
controller.close();
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const chunk = fileData.slice(readPosition, readPosition + chunkSize);
|
|
70
|
+
readPosition += chunk.length;
|
|
71
|
+
controller.enqueue(chunk);
|
|
72
|
+
}),
|
|
73
|
+
seek: jest.fn().mockImplementation((position: number) => {
|
|
74
|
+
readPosition = position;
|
|
75
|
+
}),
|
|
76
|
+
};
|
|
56
77
|
});
|
|
57
78
|
|
|
58
79
|
it('should throw error if highWaterMark is not 0', () => {
|
|
@@ -184,4 +205,108 @@ describe('BufferedSeekableStream', () => {
|
|
|
184
205
|
const result4 = await stream.read(2);
|
|
185
206
|
expect(result4).toEqual({ value: new Uint8Array([10]), done: true });
|
|
186
207
|
});
|
|
208
|
+
|
|
209
|
+
it('should catch and ignore TypeError from releaseLock during seek', async () => {
|
|
210
|
+
const stream = new BufferedSeekableStream({
|
|
211
|
+
pull: pullMock,
|
|
212
|
+
seek: jest.fn(),
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
await stream.read(2);
|
|
216
|
+
|
|
217
|
+
const reader = (stream as any).reader;
|
|
218
|
+
const originalReleaseLock = reader.releaseLock.bind(reader);
|
|
219
|
+
jest.spyOn(reader, 'releaseLock').mockImplementation(() => {
|
|
220
|
+
originalReleaseLock();
|
|
221
|
+
throw new TypeError('Reader has pending read requests');
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
await expect(stream.seek(0)).resolves.not.toThrow();
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
it('should re-throw non-TypeError errors from releaseLock during seek', async () => {
|
|
228
|
+
const stream = new BufferedSeekableStream({
|
|
229
|
+
pull: pullMock,
|
|
230
|
+
seek: jest.fn(),
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
await stream.read(2);
|
|
234
|
+
|
|
235
|
+
const reader = (stream as any).reader;
|
|
236
|
+
const customError = new Error('Custom error');
|
|
237
|
+
jest.spyOn(reader, 'releaseLock').mockImplementation(() => {
|
|
238
|
+
throw customError;
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
await expect(stream.seek(0)).rejects.toThrow(customError);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it('should not call underlying seek when seeking within buffer range', async () => {
|
|
245
|
+
const seekMock = jest.fn();
|
|
246
|
+
const stream = new BufferedSeekableStream({
|
|
247
|
+
pull: pullMock,
|
|
248
|
+
seek: seekMock,
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
await stream.read(2);
|
|
252
|
+
expect(seekMock).not.toHaveBeenCalled();
|
|
253
|
+
|
|
254
|
+
// Seek within buffer range (buffer has bytes for positions 2-4)
|
|
255
|
+
await stream.seek(3);
|
|
256
|
+
expect(seekMock).not.toHaveBeenCalled();
|
|
257
|
+
|
|
258
|
+
// Seek to current position (still within buffer)
|
|
259
|
+
await stream.seek(3);
|
|
260
|
+
expect(seekMock).not.toHaveBeenCalled();
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
it('should not corrupt data when seeking to current position with seekable underlying source', async () => {
|
|
264
|
+
const stream = new BufferedSeekableStream(seekableSource);
|
|
265
|
+
|
|
266
|
+
// Read first 3 bytes [0, 1, 2], buffer will have [0, 1, 2, 3, 4]
|
|
267
|
+
const result1 = await stream.read(3);
|
|
268
|
+
expect(result1.value).toEqual(new Uint8Array([0, 1, 2]));
|
|
269
|
+
expect(seekableSource.seek).not.toHaveBeenCalled();
|
|
270
|
+
|
|
271
|
+
// Seek to position 3 (current position), should use buffer without seeking underlying source
|
|
272
|
+
await stream.seek(3);
|
|
273
|
+
expect(seekableSource.seek).not.toHaveBeenCalled();
|
|
274
|
+
|
|
275
|
+
// Buffer has [3, 4], needs 2 more from underlying source
|
|
276
|
+
// Underlying source stays at position 5, giving [5, 6, 7, 8, 9]
|
|
277
|
+
// Buffer becomes [3, 4, 5, 6, 7, 8, 9]
|
|
278
|
+
const result2 = await stream.read(4);
|
|
279
|
+
expect(result2.value).toEqual(new Uint8Array([3, 4, 5, 6]));
|
|
280
|
+
expect(seekableSource.seek).not.toHaveBeenCalled();
|
|
281
|
+
|
|
282
|
+
// Continue reading to verify stream integrity
|
|
283
|
+
const result3 = await stream.read(3);
|
|
284
|
+
expect(result3.value).toEqual(new Uint8Array([7, 8, 9]));
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
it('should call underlying seek only when seeking outside buffer range', async () => {
|
|
288
|
+
const stream = new BufferedSeekableStream(seekableSource);
|
|
289
|
+
|
|
290
|
+
// Read first 3 bytes [0, 1, 2], buffer will have [0, 1, 2, 3, 4]
|
|
291
|
+
await stream.read(3);
|
|
292
|
+
expect(seekableSource.seek).not.toHaveBeenCalled();
|
|
293
|
+
|
|
294
|
+
// Seek backward (outside buffer range) - should call underlying seek
|
|
295
|
+
await stream.seek(0);
|
|
296
|
+
expect(seekableSource.seek).toHaveBeenCalledWith(0);
|
|
297
|
+
expect(seekableSource.seek).toHaveBeenCalledTimes(1);
|
|
298
|
+
|
|
299
|
+
// Read and verify data is correct after backward seek
|
|
300
|
+
const result1 = await stream.read(3);
|
|
301
|
+
expect(result1.value).toEqual(new Uint8Array([0, 1, 2]));
|
|
302
|
+
|
|
303
|
+
// Seek forward past buffer end - should call underlying seek
|
|
304
|
+
await stream.seek(10);
|
|
305
|
+
expect(seekableSource.seek).toHaveBeenCalledWith(10);
|
|
306
|
+
expect(seekableSource.seek).toHaveBeenCalledTimes(2);
|
|
307
|
+
|
|
308
|
+
// Read and verify data is correct after forward seek
|
|
309
|
+
const result2 = await stream.read(3);
|
|
310
|
+
expect(result2.value).toEqual(new Uint8Array([10, 11, 12]));
|
|
311
|
+
});
|
|
187
312
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
interface UnderlyingSeekableSource extends UnderlyingDefaultSource<Uint8Array> {
|
|
1
|
+
export interface UnderlyingSeekableSource extends UnderlyingDefaultSource<Uint8Array> {
|
|
2
2
|
seek: (position: number) => void | Promise<void>;
|
|
3
3
|
}
|
|
4
4
|
|
|
@@ -22,7 +22,10 @@ interface UnderlyingSeekableSource extends UnderlyingDefaultSource<Uint8Array> {
|
|
|
22
22
|
export class SeekableReadableStream extends ReadableStream<Uint8Array> {
|
|
23
23
|
private seekCallback: (position: number) => void | Promise<void>;
|
|
24
24
|
|
|
25
|
-
constructor(
|
|
25
|
+
constructor(
|
|
26
|
+
{ seek, ...underlyingSource }: UnderlyingSeekableSource,
|
|
27
|
+
queuingStrategy?: QueuingStrategy<Uint8Array>,
|
|
28
|
+
) {
|
|
26
29
|
super(underlyingSource, queuingStrategy);
|
|
27
30
|
this.seekCallback = seek;
|
|
28
31
|
}
|
|
@@ -160,23 +163,41 @@ export class BufferedSeekableStream extends SeekableReadableStream {
|
|
|
160
163
|
async seek(position: number): Promise<void> {
|
|
161
164
|
const endOfBufferPosition = this.currentPosition + (this.buffer.length - this.bufferPosition);
|
|
162
165
|
|
|
163
|
-
if (position > endOfBufferPosition) {
|
|
164
|
-
this.buffer = new Uint8Array(0);
|
|
165
|
-
this.bufferPosition = 0;
|
|
166
|
-
} else if (position < this.currentPosition) {
|
|
166
|
+
if (position > endOfBufferPosition || position < this.currentPosition) {
|
|
167
167
|
this.buffer = new Uint8Array(0);
|
|
168
168
|
this.bufferPosition = 0;
|
|
169
|
+
|
|
170
|
+
await super.seek(position);
|
|
171
|
+
|
|
172
|
+
if (this.reader) {
|
|
173
|
+
try {
|
|
174
|
+
this.reader.releaseLock();
|
|
175
|
+
} catch (error) {
|
|
176
|
+
// Streams API spec-compliant behavior: releaseLock() only throws TypeError when
|
|
177
|
+
// there are pending read requests. This can occur due to timing differences between
|
|
178
|
+
// when read() promises resolve on the client side vs when the browser's internal
|
|
179
|
+
// stream mechanism fully completes.
|
|
180
|
+
//
|
|
181
|
+
// This manifests more frequently in Firefox than Chrome due to implementation
|
|
182
|
+
// timing differences, but both are following the spec correctly.
|
|
183
|
+
//
|
|
184
|
+
// References:
|
|
185
|
+
// - https://github.com/whatwg/streams/issues/1000
|
|
186
|
+
// - https://developer.mozilla.org/en-US/docs/Web/API/ReadableStreamDefaultReader/releaseLock
|
|
187
|
+
//
|
|
188
|
+
// Safe to ignore since we're acquiring a new reader immediately after.
|
|
189
|
+
if (!(error instanceof TypeError)) {
|
|
190
|
+
throw error;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
this.reader = super.getReader();
|
|
195
|
+
this.streamClosed = false;
|
|
169
196
|
} else {
|
|
197
|
+
// Position is within buffer range, just update buffer position.
|
|
170
198
|
this.bufferPosition += position - this.currentPosition;
|
|
171
199
|
}
|
|
172
200
|
|
|
173
|
-
await super.seek(position);
|
|
174
|
-
|
|
175
|
-
if (this.reader) {
|
|
176
|
-
this.reader.releaseLock();
|
|
177
|
-
}
|
|
178
|
-
this.reader = super.getReader();
|
|
179
|
-
this.streamClosed = false;
|
|
180
201
|
this.currentPosition = position;
|
|
181
202
|
}
|
|
182
203
|
}
|
|
@@ -159,10 +159,10 @@ export function initPhotoUploadModule(
|
|
|
159
159
|
metadata: PhotoUploadMetadata,
|
|
160
160
|
signal?: AbortSignal,
|
|
161
161
|
): Promise<PhotoFileUploader> {
|
|
162
|
-
await queue.waitForCapacity(signal);
|
|
162
|
+
await queue.waitForCapacity(metadata.expectedSize, signal);
|
|
163
163
|
|
|
164
164
|
const onFinish = () => {
|
|
165
|
-
queue.releaseCapacity();
|
|
165
|
+
queue.releaseCapacity(metadata.expectedSize);
|
|
166
166
|
};
|
|
167
167
|
|
|
168
168
|
return new PhotoFileUploader(
|
|
@@ -45,10 +45,10 @@ export function initUploadModule(
|
|
|
45
45
|
metadata: UploadMetadata,
|
|
46
46
|
signal?: AbortSignal,
|
|
47
47
|
): Promise<FileUploader> {
|
|
48
|
-
await queue.waitForCapacity(signal);
|
|
48
|
+
await queue.waitForCapacity(metadata.expectedSize, signal);
|
|
49
49
|
|
|
50
50
|
const onFinish = () => {
|
|
51
|
-
queue.releaseCapacity();
|
|
51
|
+
queue.releaseCapacity(metadata.expectedSize);
|
|
52
52
|
};
|
|
53
53
|
|
|
54
54
|
return new FileUploader(
|
|
@@ -76,10 +76,10 @@ export function initUploadModule(
|
|
|
76
76
|
metadata: UploadMetadata,
|
|
77
77
|
signal?: AbortSignal,
|
|
78
78
|
): Promise<FileRevisionUploader> {
|
|
79
|
-
await queue.waitForCapacity(signal);
|
|
79
|
+
await queue.waitForCapacity(metadata.expectedSize, signal);
|
|
80
80
|
|
|
81
81
|
const onFinish = () => {
|
|
82
|
-
queue.releaseCapacity();
|
|
82
|
+
queue.releaseCapacity(metadata.expectedSize);
|
|
83
83
|
};
|
|
84
84
|
|
|
85
85
|
return new FileRevisionUploader(
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { AbortError } from '../../errors';
|
|
2
|
+
import { UploadQueue } from './queue';
|
|
3
|
+
import { FILE_CHUNK_SIZE } from './streamUploader';
|
|
4
|
+
|
|
5
|
+
describe('UploadQueue', () => {
|
|
6
|
+
let queue: UploadQueue;
|
|
7
|
+
|
|
8
|
+
beforeEach(() => {
|
|
9
|
+
queue = new UploadQueue();
|
|
10
|
+
jest.useFakeTimers();
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
afterEach(() => {
|
|
14
|
+
jest.useRealTimers();
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('should resolve immediately when queue is empty', async () => {
|
|
18
|
+
const promise = queue.waitForCapacity(0);
|
|
19
|
+
await promise;
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('should resolve immediately when under file upload limit', async () => {
|
|
23
|
+
// Fill queue with 4 uploads (limit is 5)
|
|
24
|
+
for (let i = 0; i < 4; i++) {
|
|
25
|
+
await queue.waitForCapacity(0);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const promise = queue.waitForCapacity(0);
|
|
29
|
+
await promise;
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it('should wait when max concurrent file uploads is reached', async () => {
|
|
33
|
+
// Fill queue to max (5 uploads)
|
|
34
|
+
for (let i = 0; i < 5; i++) {
|
|
35
|
+
await queue.waitForCapacity(0);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
let resolved = false;
|
|
39
|
+
const promise = queue.waitForCapacity(0).then(() => {
|
|
40
|
+
resolved = true;
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
44
|
+
expect(resolved).toBe(false);
|
|
45
|
+
|
|
46
|
+
queue.releaseCapacity(0);
|
|
47
|
+
|
|
48
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
49
|
+
await promise;
|
|
50
|
+
expect(resolved).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('should wait when max concurrent upload size is reached', async () => {
|
|
54
|
+
// Fill queue with one large file that exceeds size limit
|
|
55
|
+
const largeSize = 10 * FILE_CHUNK_SIZE;
|
|
56
|
+
await queue.waitForCapacity(largeSize);
|
|
57
|
+
|
|
58
|
+
let resolved = false;
|
|
59
|
+
const promise = queue.waitForCapacity(0).then(() => {
|
|
60
|
+
resolved = true;
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
64
|
+
expect(resolved).toBe(false);
|
|
65
|
+
|
|
66
|
+
queue.releaseCapacity(largeSize);
|
|
67
|
+
|
|
68
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
69
|
+
await promise;
|
|
70
|
+
expect(resolved).toBe(true);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('should track expected size correctly', async () => {
|
|
74
|
+
const size1 = 5 * FILE_CHUNK_SIZE;
|
|
75
|
+
const size2 = 4 * FILE_CHUNK_SIZE;
|
|
76
|
+
|
|
77
|
+
await queue.waitForCapacity(size1);
|
|
78
|
+
await queue.waitForCapacity(size2);
|
|
79
|
+
|
|
80
|
+
// Total is 9 * FILE_CHUNK_SIZE, limit is 10 * FILE_CHUNK_SIZE
|
|
81
|
+
// So next upload should still be allowed immediately
|
|
82
|
+
const promise = queue.waitForCapacity(3 * FILE_CHUNK_SIZE);
|
|
83
|
+
await promise;
|
|
84
|
+
|
|
85
|
+
// But now we're at limit, next one should wait
|
|
86
|
+
let resolved = false;
|
|
87
|
+
const waitingPromise = queue.waitForCapacity(0).then(() => {
|
|
88
|
+
resolved = true;
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
92
|
+
expect(resolved).toBe(false);
|
|
93
|
+
|
|
94
|
+
queue.releaseCapacity(size1);
|
|
95
|
+
await jest.advanceTimersByTimeAsync(100);
|
|
96
|
+
await waitingPromise;
|
|
97
|
+
expect(resolved).toBe(true);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('should reject when signal is aborted', async () => {
|
|
101
|
+
// Fill queue to max
|
|
102
|
+
for (let i = 0; i < 5; i++) {
|
|
103
|
+
await queue.waitForCapacity(0);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const controller = new AbortController();
|
|
107
|
+
const promise = queue.waitForCapacity(0, controller.signal);
|
|
108
|
+
|
|
109
|
+
controller.abort();
|
|
110
|
+
|
|
111
|
+
// Attach rejection handler BEFORE advancing timers to avoid unhandled rejection
|
|
112
|
+
const expectation = expect(promise).rejects.toThrow(AbortError);
|
|
113
|
+
await jest.advanceTimersByTimeAsync(50);
|
|
114
|
+
await expectation;
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('should reject immediately if signal is already aborted', async () => {
|
|
118
|
+
// Fill queue to max
|
|
119
|
+
for (let i = 0; i < 5; i++) {
|
|
120
|
+
await queue.waitForCapacity(0);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const controller = new AbortController();
|
|
124
|
+
controller.abort();
|
|
125
|
+
|
|
126
|
+
const promise = queue.waitForCapacity(0, controller.signal);
|
|
127
|
+
await expect(promise).rejects.toThrow(AbortError);
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
|
|
@@ -1,4 +1,23 @@
|
|
|
1
1
|
import { waitForCondition } from '../wait';
|
|
2
|
+
import { FILE_CHUNK_SIZE } from './streamUploader';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Maximum number of concurrent file uploads.
|
|
6
|
+
*
|
|
7
|
+
* It avoids uploading too many files at the same time. The total file size
|
|
8
|
+
* below also limits that, but if the file is empty, we still need to make
|
|
9
|
+
* a reasonable number of requests.
|
|
10
|
+
*/
|
|
11
|
+
const MAX_CONCURRENT_FILE_UPLOADS = 5;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Maximum total file size that can be uploaded concurrently.
|
|
15
|
+
*
|
|
16
|
+
* It avoids uploading too many blocks at the same time, ensuring that on poor
|
|
17
|
+
* connection we don't do too many things at the same time that all fail due
|
|
18
|
+
* to network issues.
|
|
19
|
+
*/
|
|
20
|
+
const MAX_CONCURRENT_UPLOAD_SIZE = 10 * FILE_CHUNK_SIZE;
|
|
2
21
|
|
|
3
22
|
/**
|
|
4
23
|
* A queue that limits the number of concurrent uploads.
|
|
@@ -14,18 +33,24 @@ import { waitForCondition } from '../wait';
|
|
|
14
33
|
* uploaded. That is something we want to add in the future to be
|
|
15
34
|
* more performant for many small file uploads.
|
|
16
35
|
*/
|
|
17
|
-
const MAX_CONCURRENT_UPLOADS = 5;
|
|
18
|
-
|
|
19
36
|
export class UploadQueue {
|
|
20
|
-
private
|
|
37
|
+
private totalFileUploads = 0;
|
|
38
|
+
|
|
39
|
+
private totalExpectedSize = 0;
|
|
21
40
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
41
|
+
async waitForCapacity(expectedSize: number, signal?: AbortSignal) {
|
|
42
|
+
await waitForCondition(
|
|
43
|
+
() =>
|
|
44
|
+
this.totalFileUploads < MAX_CONCURRENT_FILE_UPLOADS &&
|
|
45
|
+
this.totalExpectedSize < MAX_CONCURRENT_UPLOAD_SIZE,
|
|
46
|
+
signal,
|
|
47
|
+
);
|
|
48
|
+
this.totalFileUploads++;
|
|
49
|
+
this.totalExpectedSize += expectedSize;
|
|
26
50
|
}
|
|
27
51
|
|
|
28
|
-
releaseCapacity() {
|
|
29
|
-
this.
|
|
52
|
+
releaseCapacity(expectedSize: number) {
|
|
53
|
+
this.totalFileUploads--;
|
|
54
|
+
this.totalExpectedSize -= expectedSize;
|
|
30
55
|
}
|
|
31
56
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { Thumbnail, ThumbnailType, UploadMetadata } from '../../interface';
|
|
1
|
+
import { Logger, Thumbnail, ThumbnailType, UploadMetadata } from '../../interface';
|
|
2
2
|
import { IntegrityError } from '../../errors';
|
|
3
|
+
import { getMockLogger } from '../../tests/logger';
|
|
3
4
|
import { APIHTTPError, HTTPErrorCode } from '../apiService';
|
|
4
5
|
import { FILE_CHUNK_SIZE, StreamUploader } from './streamUploader';
|
|
5
6
|
import { UploadTelemetry } from './telemetry';
|
|
@@ -40,6 +41,7 @@ function mockUploadBlock(
|
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
describe('StreamUploader', () => {
|
|
44
|
+
let logger: Logger;
|
|
43
45
|
let telemetry: UploadTelemetry;
|
|
44
46
|
let apiService: jest.Mocked<UploadAPIService>;
|
|
45
47
|
let cryptoService: UploadCryptoService;
|
|
@@ -54,14 +56,11 @@ describe('StreamUploader', () => {
|
|
|
54
56
|
let uploader: StreamUploader;
|
|
55
57
|
|
|
56
58
|
beforeEach(() => {
|
|
59
|
+
logger = getMockLogger();
|
|
60
|
+
|
|
57
61
|
// @ts-expect-error No need to implement all methods for mocking
|
|
58
62
|
telemetry = {
|
|
59
|
-
getLoggerForRevision: jest.fn().mockReturnValue(
|
|
60
|
-
debug: jest.fn(),
|
|
61
|
-
info: jest.fn(),
|
|
62
|
-
warn: jest.fn(),
|
|
63
|
-
error: jest.fn(),
|
|
64
|
-
}),
|
|
63
|
+
getLoggerForRevision: jest.fn().mockReturnValue(logger),
|
|
65
64
|
logBlockVerificationError: jest.fn(),
|
|
66
65
|
uploadFailed: jest.fn(),
|
|
67
66
|
uploadFinished: jest.fn(),
|
|
@@ -403,6 +402,80 @@ describe('StreamUploader', () => {
|
|
|
403
402
|
await verifyOnProgress([1024, 4 * 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024]);
|
|
404
403
|
});
|
|
405
404
|
|
|
405
|
+
it('should handle timeout when uploading block', async () => {
|
|
406
|
+
const error = new Error('TimeoutError');
|
|
407
|
+
error.name = 'TimeoutError';
|
|
408
|
+
|
|
409
|
+
let count = 0;
|
|
410
|
+
apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
|
|
411
|
+
if (token === 'token/block:1' && count === 0) {
|
|
412
|
+
count++;
|
|
413
|
+
throw error;
|
|
414
|
+
}
|
|
415
|
+
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
expect((uploader as any).maxUploadingBlocks).toEqual(5);
|
|
419
|
+
await verifySuccess();
|
|
420
|
+
expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(1);
|
|
421
|
+
// 3 blocks + 1 timeout retry + 1 thumbnail
|
|
422
|
+
expect(apiService.uploadBlock).toHaveBeenCalledTimes(5);
|
|
423
|
+
expect(logger.warn).toHaveBeenCalledTimes(2);
|
|
424
|
+
expect(logger.warn).toHaveBeenCalledWith(
|
|
425
|
+
'block 1:token/block:1: Upload timeout, limiting upload capacity to 1 block',
|
|
426
|
+
);
|
|
427
|
+
expect(logger.warn).toHaveBeenCalledWith('block 1:token/block:1: Upload timeout, retrying');
|
|
428
|
+
expect((uploader as any).maxUploadingBlocks).toEqual(1);
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
it('limitUploadCapacity should wait for the previous blocks to finish', async () => {
|
|
432
|
+
const error = new Error('TimeoutError');
|
|
433
|
+
error.name = 'TimeoutError';
|
|
434
|
+
|
|
435
|
+
const events: string[] = [];
|
|
436
|
+
let block1Resolver: (() => void) | undefined;
|
|
437
|
+
let block2FirstAttempt = true;
|
|
438
|
+
|
|
439
|
+
apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
|
|
440
|
+
if (token === 'token/block:1') {
|
|
441
|
+
events.push('block1:upload:start');
|
|
442
|
+
await new Promise<void>((resolve) => {
|
|
443
|
+
block1Resolver = resolve;
|
|
444
|
+
});
|
|
445
|
+
events.push('block1:upload:end');
|
|
446
|
+
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
447
|
+
}
|
|
448
|
+
if (token === 'token/block:2') {
|
|
449
|
+
if (block2FirstAttempt) {
|
|
450
|
+
block2FirstAttempt = false;
|
|
451
|
+
events.push('block2:timeout');
|
|
452
|
+
// Resolve block 1 after a small delay to simulate real-world conditions
|
|
453
|
+
setTimeout(() => block1Resolver?.(), 100);
|
|
454
|
+
throw error;
|
|
455
|
+
}
|
|
456
|
+
events.push('block2:retry');
|
|
457
|
+
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
458
|
+
}
|
|
459
|
+
// Block 3 and thumbnails proceed normally
|
|
460
|
+
return mockUploadBlock(bareUrl, token, block, onProgress);
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
await verifySuccess();
|
|
464
|
+
|
|
465
|
+
expect(events).toMatchObject([
|
|
466
|
+
'block1:upload:start',
|
|
467
|
+
'block2:timeout',
|
|
468
|
+
'block1:upload:end',
|
|
469
|
+
'block2:retry',
|
|
470
|
+
]);
|
|
471
|
+
|
|
472
|
+
// Also verify the warning messages were logged
|
|
473
|
+
expect(logger.warn).toHaveBeenCalledWith(
|
|
474
|
+
'block 2:token/block:2: Upload timeout, limiting upload capacity to 1 block',
|
|
475
|
+
);
|
|
476
|
+
expect(logger.warn).toHaveBeenCalledWith('block 2:token/block:2: Upload timeout, retrying');
|
|
477
|
+
});
|
|
478
|
+
|
|
406
479
|
it('should handle expired token when uploading block', async () => {
|
|
407
480
|
let count = 0;
|
|
408
481
|
apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
|
|
@@ -55,6 +55,8 @@ const MAX_BLOCK_UPLOAD_RETRIES = 3;
|
|
|
55
55
|
* that the upload process is efficient and does not overload the server.
|
|
56
56
|
*/
|
|
57
57
|
export class StreamUploader {
|
|
58
|
+
protected maxUploadingBlocks = MAX_UPLOADING_BLOCKS;
|
|
59
|
+
|
|
58
60
|
protected logger: Logger;
|
|
59
61
|
|
|
60
62
|
protected digests: UploadDigests;
|
|
@@ -67,6 +69,7 @@ export class StreamUploader {
|
|
|
67
69
|
protected ongoingUploads = new Map<
|
|
68
70
|
string,
|
|
69
71
|
{
|
|
72
|
+
index?: number;
|
|
70
73
|
uploadPromise: Promise<void>;
|
|
71
74
|
encryptedBlock: EncryptedBlock | EncryptedThumbnail;
|
|
72
75
|
}
|
|
@@ -376,6 +379,7 @@ export class StreamUploader {
|
|
|
376
379
|
|
|
377
380
|
const uploadKey = `block:${blockToken.index}`;
|
|
378
381
|
this.ongoingUploads.set(uploadKey, {
|
|
382
|
+
index: blockToken.index,
|
|
379
383
|
uploadPromise: this.uploadBlock(blockToken, encryptedBlock, onProgress).finally(() => {
|
|
380
384
|
this.ongoingUploads.delete(uploadKey);
|
|
381
385
|
|
|
@@ -500,6 +504,13 @@ export class StreamUploader {
|
|
|
500
504
|
blockProgress = 0;
|
|
501
505
|
}
|
|
502
506
|
|
|
507
|
+
if (error instanceof Error && error.name === 'TimeoutError') {
|
|
508
|
+
logger.warn(`Upload timeout, limiting upload capacity to 1 block`);
|
|
509
|
+
await this.limitUploadCapacity(uploadToken.index);
|
|
510
|
+
logger.warn(`Upload timeout, retrying`);
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
|
|
503
514
|
if (
|
|
504
515
|
(error instanceof APIHTTPError && error.statusCode === HTTPErrorCode.NOT_FOUND) ||
|
|
505
516
|
error instanceof NotFoundAPIError
|
|
@@ -542,6 +553,27 @@ export class StreamUploader {
|
|
|
542
553
|
logger.info(`Uploaded`);
|
|
543
554
|
}
|
|
544
555
|
|
|
556
|
+
private async limitUploadCapacity(index: number) {
|
|
557
|
+
this.maxUploadingBlocks = 1;
|
|
558
|
+
|
|
559
|
+
// This ensures that when the upload is downscaled, all ongoing block
|
|
560
|
+
// uploads are waiting for their turn one by one.
|
|
561
|
+
try {
|
|
562
|
+
await waitForCondition(() => {
|
|
563
|
+
const ongoingIndexes = Array.from(this.ongoingUploads.values())
|
|
564
|
+
.map(({ index: ongoingIndex }) => ongoingIndex)
|
|
565
|
+
.filter((ongoingIndex) => ongoingIndex !== undefined);
|
|
566
|
+
ongoingIndexes.sort((a, b) => a - b);
|
|
567
|
+
return ongoingIndexes[0] === index;
|
|
568
|
+
}, this.abortController.signal);
|
|
569
|
+
} catch (error: unknown) {
|
|
570
|
+
if (error instanceof AbortError) {
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
throw error;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
545
577
|
private async waitForBufferCapacity() {
|
|
546
578
|
if (this.encryptedBlocks.size >= MAX_BUFFERED_BLOCKS) {
|
|
547
579
|
try {
|
|
@@ -559,7 +591,7 @@ export class StreamUploader {
|
|
|
559
591
|
}
|
|
560
592
|
|
|
561
593
|
private async waitForUploadCapacityAndBufferedBlocks() {
|
|
562
|
-
while (this.ongoingUploads.size >=
|
|
594
|
+
while (this.ongoingUploads.size >= this.maxUploadingBlocks) {
|
|
563
595
|
await Promise.race(this.ongoingUploads.values().map(({ uploadPromise }) => uploadPromise));
|
|
564
596
|
}
|
|
565
597
|
try {
|
|
@@ -13,13 +13,12 @@ export class UploadTelemetry {
|
|
|
13
13
|
private sharesService: SharesService,
|
|
14
14
|
) {
|
|
15
15
|
this.telemetry = telemetry;
|
|
16
|
-
this.logger = this.telemetry.getLogger('
|
|
16
|
+
this.logger = this.telemetry.getLogger('upload');
|
|
17
17
|
this.sharesService = sharesService;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
getLoggerForRevision(revisionUid: string) {
|
|
21
|
-
|
|
22
|
-
return new LoggerWithPrefix(logger, `revision ${revisionUid}`);
|
|
21
|
+
return new LoggerWithPrefix(this.logger, `revision ${revisionUid}`);
|
|
23
22
|
}
|
|
24
23
|
|
|
25
24
|
logBlockVerificationError(retryHelped: boolean) {
|