@protontech/drive-sdk 0.9.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/internal/apiService/apiService.js +6 -2
  2. package/dist/internal/apiService/apiService.js.map +1 -1
  3. package/dist/internal/apiService/apiService.test.js +9 -1
  4. package/dist/internal/apiService/apiService.test.js.map +1 -1
  5. package/dist/internal/download/seekableStream.d.ts +1 -2
  6. package/dist/internal/download/seekableStream.js +28 -11
  7. package/dist/internal/download/seekableStream.js.map +1 -1
  8. package/dist/internal/download/seekableStream.test.js +100 -0
  9. package/dist/internal/download/seekableStream.test.js.map +1 -1
  10. package/dist/internal/photos/index.js +2 -2
  11. package/dist/internal/photos/index.js.map +1 -1
  12. package/dist/internal/upload/index.js +4 -4
  13. package/dist/internal/upload/index.js.map +1 -1
  14. package/dist/internal/upload/queue.d.ts +18 -3
  15. package/dist/internal/upload/queue.js +27 -8
  16. package/dist/internal/upload/queue.js.map +1 -1
  17. package/dist/internal/upload/queue.test.d.ts +1 -0
  18. package/dist/internal/upload/queue.test.js +103 -0
  19. package/dist/internal/upload/queue.test.js.map +1 -0
  20. package/dist/internal/upload/streamUploader.d.ts +3 -0
  21. package/dist/internal/upload/streamUploader.js +29 -1
  22. package/dist/internal/upload/streamUploader.js.map +1 -1
  23. package/dist/internal/upload/streamUploader.test.js +65 -6
  24. package/dist/internal/upload/streamUploader.test.js.map +1 -1
  25. package/dist/internal/upload/telemetry.js +2 -3
  26. package/dist/internal/upload/telemetry.js.map +1 -1
  27. package/package.json +1 -1
  28. package/src/internal/apiService/apiService.test.ts +12 -2
  29. package/src/internal/apiService/apiService.ts +7 -2
  30. package/src/internal/download/seekableStream.test.ts +126 -1
  31. package/src/internal/download/seekableStream.ts +34 -13
  32. package/src/internal/photos/index.ts +2 -2
  33. package/src/internal/upload/index.ts +4 -4
  34. package/src/internal/upload/queue.test.ts +130 -0
  35. package/src/internal/upload/queue.ts +34 -9
  36. package/src/internal/upload/streamUploader.test.ts +80 -7
  37. package/src/internal/upload/streamUploader.ts +33 -1
  38. package/src/internal/upload/telemetry.ts +2 -3
@@ -1,4 +1,4 @@
1
- import { SeekableReadableStream, BufferedSeekableStream } from './seekableStream';
1
+ import { SeekableReadableStream, BufferedSeekableStream, UnderlyingSeekableSource } from './seekableStream';
2
2
 
3
3
  describe('SeekableReadableStream', () => {
4
4
  it('should call the seek callback when seek is called', async () => {
@@ -32,6 +32,7 @@ describe('SeekableReadableStream', () => {
32
32
  describe('BufferedSeekableStream', () => {
33
33
  let startWithCloseMock: jest.Mock;
34
34
  let pullMock: jest.Mock;
35
+ let seekableSource: UnderlyingSeekableSource;
35
36
 
36
37
  const data1 = new Uint8Array([1, 2, 3, 4, 5]);
37
38
  const data2 = new Uint8Array([6, 7, 8, 9, 10]);
@@ -53,6 +54,26 @@ describe('BufferedSeekableStream', () => {
53
54
  }
54
55
  readIndex++;
55
56
  });
57
+
58
+ // Simulates a real seekable source where seek repositions the read pointer
59
+ const fileData = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
60
+ let readPosition = 0;
61
+ const chunkSize = 5;
62
+
63
+ seekableSource = {
64
+ pull: jest.fn().mockImplementation((controller) => {
65
+ if (readPosition >= fileData.length) {
66
+ controller.close();
67
+ return;
68
+ }
69
+ const chunk = fileData.slice(readPosition, readPosition + chunkSize);
70
+ readPosition += chunk.length;
71
+ controller.enqueue(chunk);
72
+ }),
73
+ seek: jest.fn().mockImplementation((position: number) => {
74
+ readPosition = position;
75
+ }),
76
+ };
56
77
  });
57
78
 
58
79
  it('should throw error if highWaterMark is not 0', () => {
@@ -184,4 +205,108 @@ describe('BufferedSeekableStream', () => {
184
205
  const result4 = await stream.read(2);
185
206
  expect(result4).toEqual({ value: new Uint8Array([10]), done: true });
186
207
  });
208
+
209
+ it('should catch and ignore TypeError from releaseLock during seek', async () => {
210
+ const stream = new BufferedSeekableStream({
211
+ pull: pullMock,
212
+ seek: jest.fn(),
213
+ });
214
+
215
+ await stream.read(2);
216
+
217
+ const reader = (stream as any).reader;
218
+ const originalReleaseLock = reader.releaseLock.bind(reader);
219
+ jest.spyOn(reader, 'releaseLock').mockImplementation(() => {
220
+ originalReleaseLock();
221
+ throw new TypeError('Reader has pending read requests');
222
+ });
223
+
224
+ await expect(stream.seek(0)).resolves.not.toThrow();
225
+ });
226
+
227
+ it('should re-throw non-TypeError errors from releaseLock during seek', async () => {
228
+ const stream = new BufferedSeekableStream({
229
+ pull: pullMock,
230
+ seek: jest.fn(),
231
+ });
232
+
233
+ await stream.read(2);
234
+
235
+ const reader = (stream as any).reader;
236
+ const customError = new Error('Custom error');
237
+ jest.spyOn(reader, 'releaseLock').mockImplementation(() => {
238
+ throw customError;
239
+ });
240
+
241
+ await expect(stream.seek(0)).rejects.toThrow(customError);
242
+ });
243
+
244
+ it('should not call underlying seek when seeking within buffer range', async () => {
245
+ const seekMock = jest.fn();
246
+ const stream = new BufferedSeekableStream({
247
+ pull: pullMock,
248
+ seek: seekMock,
249
+ });
250
+
251
+ await stream.read(2);
252
+ expect(seekMock).not.toHaveBeenCalled();
253
+
254
+ // Seek within buffer range (buffer has bytes for positions 2-4)
255
+ await stream.seek(3);
256
+ expect(seekMock).not.toHaveBeenCalled();
257
+
258
+ // Seek to current position (still within buffer)
259
+ await stream.seek(3);
260
+ expect(seekMock).not.toHaveBeenCalled();
261
+ });
262
+
263
+ it('should not corrupt data when seeking to current position with seekable underlying source', async () => {
264
+ const stream = new BufferedSeekableStream(seekableSource);
265
+
266
+ // Read first 3 bytes [0, 1, 2], buffer will have [0, 1, 2, 3, 4]
267
+ const result1 = await stream.read(3);
268
+ expect(result1.value).toEqual(new Uint8Array([0, 1, 2]));
269
+ expect(seekableSource.seek).not.toHaveBeenCalled();
270
+
271
+ // Seek to position 3 (current position), should use buffer without seeking underlying source
272
+ await stream.seek(3);
273
+ expect(seekableSource.seek).not.toHaveBeenCalled();
274
+
275
+ // Buffer has [3, 4], needs 2 more from underlying source
276
+ // Underlying source stays at position 5, giving [5, 6, 7, 8, 9]
277
+ // Buffer becomes [3, 4, 5, 6, 7, 8, 9]
278
+ const result2 = await stream.read(4);
279
+ expect(result2.value).toEqual(new Uint8Array([3, 4, 5, 6]));
280
+ expect(seekableSource.seek).not.toHaveBeenCalled();
281
+
282
+ // Continue reading to verify stream integrity
283
+ const result3 = await stream.read(3);
284
+ expect(result3.value).toEqual(new Uint8Array([7, 8, 9]));
285
+ });
286
+
287
+ it('should call underlying seek only when seeking outside buffer range', async () => {
288
+ const stream = new BufferedSeekableStream(seekableSource);
289
+
290
+ // Read first 3 bytes [0, 1, 2], buffer will have [0, 1, 2, 3, 4]
291
+ await stream.read(3);
292
+ expect(seekableSource.seek).not.toHaveBeenCalled();
293
+
294
+ // Seek backward (outside buffer range) - should call underlying seek
295
+ await stream.seek(0);
296
+ expect(seekableSource.seek).toHaveBeenCalledWith(0);
297
+ expect(seekableSource.seek).toHaveBeenCalledTimes(1);
298
+
299
+ // Read and verify data is correct after backward seek
300
+ const result1 = await stream.read(3);
301
+ expect(result1.value).toEqual(new Uint8Array([0, 1, 2]));
302
+
303
+ // Seek forward past buffer end - should call underlying seek
304
+ await stream.seek(10);
305
+ expect(seekableSource.seek).toHaveBeenCalledWith(10);
306
+ expect(seekableSource.seek).toHaveBeenCalledTimes(2);
307
+
308
+ // Read and verify data is correct after forward seek
309
+ const result2 = await stream.read(3);
310
+ expect(result2.value).toEqual(new Uint8Array([10, 11, 12]));
311
+ });
187
312
  });
@@ -1,4 +1,4 @@
1
- interface UnderlyingSeekableSource extends UnderlyingDefaultSource<Uint8Array> {
1
+ export interface UnderlyingSeekableSource extends UnderlyingDefaultSource<Uint8Array> {
2
2
  seek: (position: number) => void | Promise<void>;
3
3
  }
4
4
 
@@ -22,7 +22,10 @@ interface UnderlyingSeekableSource extends UnderlyingDefaultSource<Uint8Array> {
22
22
  export class SeekableReadableStream extends ReadableStream<Uint8Array> {
23
23
  private seekCallback: (position: number) => void | Promise<void>;
24
24
 
25
- constructor({ seek, ...underlyingSource }: UnderlyingSeekableSource, queuingStrategy?: QueuingStrategy<Uint8Array>) {
25
+ constructor(
26
+ { seek, ...underlyingSource }: UnderlyingSeekableSource,
27
+ queuingStrategy?: QueuingStrategy<Uint8Array>,
28
+ ) {
26
29
  super(underlyingSource, queuingStrategy);
27
30
  this.seekCallback = seek;
28
31
  }
@@ -160,23 +163,41 @@ export class BufferedSeekableStream extends SeekableReadableStream {
160
163
  async seek(position: number): Promise<void> {
161
164
  const endOfBufferPosition = this.currentPosition + (this.buffer.length - this.bufferPosition);
162
165
 
163
- if (position > endOfBufferPosition) {
164
- this.buffer = new Uint8Array(0);
165
- this.bufferPosition = 0;
166
- } else if (position < this.currentPosition) {
166
+ if (position > endOfBufferPosition || position < this.currentPosition) {
167
167
  this.buffer = new Uint8Array(0);
168
168
  this.bufferPosition = 0;
169
+
170
+ await super.seek(position);
171
+
172
+ if (this.reader) {
173
+ try {
174
+ this.reader.releaseLock();
175
+ } catch (error) {
176
+ // Streams API spec-compliant behavior: releaseLock() only throws TypeError when
177
+ // there are pending read requests. This can occur due to timing differences between
178
+ // when read() promises resolve on the client side vs when the browser's internal
179
+ // stream mechanism fully completes.
180
+ //
181
+ // This manifests more frequently in Firefox than Chrome due to implementation
182
+ // timing differences, but both are following the spec correctly.
183
+ //
184
+ // References:
185
+ // - https://github.com/whatwg/streams/issues/1000
186
+ // - https://developer.mozilla.org/en-US/docs/Web/API/ReadableStreamDefaultReader/releaseLock
187
+ //
188
+ // Safe to ignore since we're acquiring a new reader immediately after.
189
+ if (!(error instanceof TypeError)) {
190
+ throw error;
191
+ }
192
+ }
193
+ }
194
+ this.reader = super.getReader();
195
+ this.streamClosed = false;
169
196
  } else {
197
+ // Position is within buffer range, just update buffer position.
170
198
  this.bufferPosition += position - this.currentPosition;
171
199
  }
172
200
 
173
- await super.seek(position);
174
-
175
- if (this.reader) {
176
- this.reader.releaseLock();
177
- }
178
- this.reader = super.getReader();
179
- this.streamClosed = false;
180
201
  this.currentPosition = position;
181
202
  }
182
203
  }
@@ -159,10 +159,10 @@ export function initPhotoUploadModule(
159
159
  metadata: PhotoUploadMetadata,
160
160
  signal?: AbortSignal,
161
161
  ): Promise<PhotoFileUploader> {
162
- await queue.waitForCapacity(signal);
162
+ await queue.waitForCapacity(metadata.expectedSize, signal);
163
163
 
164
164
  const onFinish = () => {
165
- queue.releaseCapacity();
165
+ queue.releaseCapacity(metadata.expectedSize);
166
166
  };
167
167
 
168
168
  return new PhotoFileUploader(
@@ -45,10 +45,10 @@ export function initUploadModule(
45
45
  metadata: UploadMetadata,
46
46
  signal?: AbortSignal,
47
47
  ): Promise<FileUploader> {
48
- await queue.waitForCapacity(signal);
48
+ await queue.waitForCapacity(metadata.expectedSize, signal);
49
49
 
50
50
  const onFinish = () => {
51
- queue.releaseCapacity();
51
+ queue.releaseCapacity(metadata.expectedSize);
52
52
  };
53
53
 
54
54
  return new FileUploader(
@@ -76,10 +76,10 @@ export function initUploadModule(
76
76
  metadata: UploadMetadata,
77
77
  signal?: AbortSignal,
78
78
  ): Promise<FileRevisionUploader> {
79
- await queue.waitForCapacity(signal);
79
+ await queue.waitForCapacity(metadata.expectedSize, signal);
80
80
 
81
81
  const onFinish = () => {
82
- queue.releaseCapacity();
82
+ queue.releaseCapacity(metadata.expectedSize);
83
83
  };
84
84
 
85
85
  return new FileRevisionUploader(
@@ -0,0 +1,130 @@
1
+ import { AbortError } from '../../errors';
2
+ import { UploadQueue } from './queue';
3
+ import { FILE_CHUNK_SIZE } from './streamUploader';
4
+
5
+ describe('UploadQueue', () => {
6
+ let queue: UploadQueue;
7
+
8
+ beforeEach(() => {
9
+ queue = new UploadQueue();
10
+ jest.useFakeTimers();
11
+ });
12
+
13
+ afterEach(() => {
14
+ jest.useRealTimers();
15
+ });
16
+
17
+ it('should resolve immediately when queue is empty', async () => {
18
+ const promise = queue.waitForCapacity(0);
19
+ await promise;
20
+ });
21
+
22
+ it('should resolve immediately when under file upload limit', async () => {
23
+ // Fill queue with 4 uploads (limit is 5)
24
+ for (let i = 0; i < 4; i++) {
25
+ await queue.waitForCapacity(0);
26
+ }
27
+
28
+ const promise = queue.waitForCapacity(0);
29
+ await promise;
30
+ });
31
+
32
+ it('should wait when max concurrent file uploads is reached', async () => {
33
+ // Fill queue to max (5 uploads)
34
+ for (let i = 0; i < 5; i++) {
35
+ await queue.waitForCapacity(0);
36
+ }
37
+
38
+ let resolved = false;
39
+ const promise = queue.waitForCapacity(0).then(() => {
40
+ resolved = true;
41
+ });
42
+
43
+ await jest.advanceTimersByTimeAsync(100);
44
+ expect(resolved).toBe(false);
45
+
46
+ queue.releaseCapacity(0);
47
+
48
+ await jest.advanceTimersByTimeAsync(100);
49
+ await promise;
50
+ expect(resolved).toBe(true);
51
+ });
52
+
53
+ it('should wait when max concurrent upload size is reached', async () => {
54
+ // Fill queue with one large file that exceeds size limit
55
+ const largeSize = 10 * FILE_CHUNK_SIZE;
56
+ await queue.waitForCapacity(largeSize);
57
+
58
+ let resolved = false;
59
+ const promise = queue.waitForCapacity(0).then(() => {
60
+ resolved = true;
61
+ });
62
+
63
+ await jest.advanceTimersByTimeAsync(100);
64
+ expect(resolved).toBe(false);
65
+
66
+ queue.releaseCapacity(largeSize);
67
+
68
+ await jest.advanceTimersByTimeAsync(100);
69
+ await promise;
70
+ expect(resolved).toBe(true);
71
+ });
72
+
73
+ it('should track expected size correctly', async () => {
74
+ const size1 = 5 * FILE_CHUNK_SIZE;
75
+ const size2 = 4 * FILE_CHUNK_SIZE;
76
+
77
+ await queue.waitForCapacity(size1);
78
+ await queue.waitForCapacity(size2);
79
+
80
+ // Total is 9 * FILE_CHUNK_SIZE, limit is 10 * FILE_CHUNK_SIZE
81
+ // So next upload should still be allowed immediately
82
+ const promise = queue.waitForCapacity(3 * FILE_CHUNK_SIZE);
83
+ await promise;
84
+
85
+ // But now we're at limit, next one should wait
86
+ let resolved = false;
87
+ const waitingPromise = queue.waitForCapacity(0).then(() => {
88
+ resolved = true;
89
+ });
90
+
91
+ await jest.advanceTimersByTimeAsync(100);
92
+ expect(resolved).toBe(false);
93
+
94
+ queue.releaseCapacity(size1);
95
+ await jest.advanceTimersByTimeAsync(100);
96
+ await waitingPromise;
97
+ expect(resolved).toBe(true);
98
+ });
99
+
100
+ it('should reject when signal is aborted', async () => {
101
+ // Fill queue to max
102
+ for (let i = 0; i < 5; i++) {
103
+ await queue.waitForCapacity(0);
104
+ }
105
+
106
+ const controller = new AbortController();
107
+ const promise = queue.waitForCapacity(0, controller.signal);
108
+
109
+ controller.abort();
110
+
111
+ // Attach rejection handler BEFORE advancing timers to avoid unhandled rejection
112
+ const expectation = expect(promise).rejects.toThrow(AbortError);
113
+ await jest.advanceTimersByTimeAsync(50);
114
+ await expectation;
115
+ });
116
+
117
+ it('should reject immediately if signal is already aborted', async () => {
118
+ // Fill queue to max
119
+ for (let i = 0; i < 5; i++) {
120
+ await queue.waitForCapacity(0);
121
+ }
122
+
123
+ const controller = new AbortController();
124
+ controller.abort();
125
+
126
+ const promise = queue.waitForCapacity(0, controller.signal);
127
+ await expect(promise).rejects.toThrow(AbortError);
128
+ });
129
+ });
130
+
@@ -1,4 +1,23 @@
1
1
  import { waitForCondition } from '../wait';
2
+ import { FILE_CHUNK_SIZE } from './streamUploader';
3
+
4
+ /**
5
+ * Maximum number of concurrent file uploads.
6
+ *
7
+ * It avoids uploading too many files at the same time. The total file size
8
+ * below also limits that, but if the file is empty, we still need to make
9
+ * a reasonable number of requests.
10
+ */
11
+ const MAX_CONCURRENT_FILE_UPLOADS = 5;
12
+
13
+ /**
14
+ * Maximum total file size that can be uploaded concurrently.
15
+ *
16
+ * It avoids uploading too many blocks at the same time, ensuring that on poor
17
+ * connection we don't do too many things at the same time that all fail due
18
+ * to network issues.
19
+ */
20
+ const MAX_CONCURRENT_UPLOAD_SIZE = 10 * FILE_CHUNK_SIZE;
2
21
 
3
22
  /**
4
23
  * A queue that limits the number of concurrent uploads.
@@ -14,18 +33,24 @@ import { waitForCondition } from '../wait';
14
33
  * uploaded. That is something we want to add in the future to be
15
34
  * more performant for many small file uploads.
16
35
  */
17
- const MAX_CONCURRENT_UPLOADS = 5;
18
-
19
36
  export class UploadQueue {
20
- private capacity = 0;
37
+ private totalFileUploads = 0;
38
+
39
+ private totalExpectedSize = 0;
21
40
 
22
- // TODO: use expected size to control the size of the queue
23
- async waitForCapacity(signal?: AbortSignal) {
24
- await waitForCondition(() => this.capacity < MAX_CONCURRENT_UPLOADS, signal);
25
- this.capacity++;
41
+ async waitForCapacity(expectedSize: number, signal?: AbortSignal) {
42
+ await waitForCondition(
43
+ () =>
44
+ this.totalFileUploads < MAX_CONCURRENT_FILE_UPLOADS &&
45
+ this.totalExpectedSize < MAX_CONCURRENT_UPLOAD_SIZE,
46
+ signal,
47
+ );
48
+ this.totalFileUploads++;
49
+ this.totalExpectedSize += expectedSize;
26
50
  }
27
51
 
28
- releaseCapacity() {
29
- this.capacity--;
52
+ releaseCapacity(expectedSize: number) {
53
+ this.totalFileUploads--;
54
+ this.totalExpectedSize -= expectedSize;
30
55
  }
31
56
  }
@@ -1,5 +1,6 @@
1
- import { Thumbnail, ThumbnailType, UploadMetadata } from '../../interface';
1
+ import { Logger, Thumbnail, ThumbnailType, UploadMetadata } from '../../interface';
2
2
  import { IntegrityError } from '../../errors';
3
+ import { getMockLogger } from '../../tests/logger';
3
4
  import { APIHTTPError, HTTPErrorCode } from '../apiService';
4
5
  import { FILE_CHUNK_SIZE, StreamUploader } from './streamUploader';
5
6
  import { UploadTelemetry } from './telemetry';
@@ -40,6 +41,7 @@ function mockUploadBlock(
40
41
  }
41
42
 
42
43
  describe('StreamUploader', () => {
44
+ let logger: Logger;
43
45
  let telemetry: UploadTelemetry;
44
46
  let apiService: jest.Mocked<UploadAPIService>;
45
47
  let cryptoService: UploadCryptoService;
@@ -54,14 +56,11 @@ describe('StreamUploader', () => {
54
56
  let uploader: StreamUploader;
55
57
 
56
58
  beforeEach(() => {
59
+ logger = getMockLogger();
60
+
57
61
  // @ts-expect-error No need to implement all methods for mocking
58
62
  telemetry = {
59
- getLoggerForRevision: jest.fn().mockReturnValue({
60
- debug: jest.fn(),
61
- info: jest.fn(),
62
- warn: jest.fn(),
63
- error: jest.fn(),
64
- }),
63
+ getLoggerForRevision: jest.fn().mockReturnValue(logger),
65
64
  logBlockVerificationError: jest.fn(),
66
65
  uploadFailed: jest.fn(),
67
66
  uploadFinished: jest.fn(),
@@ -403,6 +402,80 @@ describe('StreamUploader', () => {
403
402
  await verifyOnProgress([1024, 4 * 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024]);
404
403
  });
405
404
 
405
+ it('should handle timeout when uploading block', async () => {
406
+ const error = new Error('TimeoutError');
407
+ error.name = 'TimeoutError';
408
+
409
+ let count = 0;
410
+ apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
411
+ if (token === 'token/block:1' && count === 0) {
412
+ count++;
413
+ throw error;
414
+ }
415
+ return mockUploadBlock(bareUrl, token, block, onProgress);
416
+ });
417
+
418
+ expect((uploader as any).maxUploadingBlocks).toEqual(5);
419
+ await verifySuccess();
420
+ expect(apiService.requestBlockUpload).toHaveBeenCalledTimes(1);
421
+ // 3 blocks + 1 timeout retry + 1 thumbnail
422
+ expect(apiService.uploadBlock).toHaveBeenCalledTimes(5);
423
+ expect(logger.warn).toHaveBeenCalledTimes(2);
424
+ expect(logger.warn).toHaveBeenCalledWith(
425
+ 'block 1:token/block:1: Upload timeout, limiting upload capacity to 1 block',
426
+ );
427
+ expect(logger.warn).toHaveBeenCalledWith('block 1:token/block:1: Upload timeout, retrying');
428
+ expect((uploader as any).maxUploadingBlocks).toEqual(1);
429
+ });
430
+
431
+ it('limitUploadCapacity should wait for the previous blocks to finish', async () => {
432
+ const error = new Error('TimeoutError');
433
+ error.name = 'TimeoutError';
434
+
435
+ const events: string[] = [];
436
+ let block1Resolver: (() => void) | undefined;
437
+ let block2FirstAttempt = true;
438
+
439
+ apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
440
+ if (token === 'token/block:1') {
441
+ events.push('block1:upload:start');
442
+ await new Promise<void>((resolve) => {
443
+ block1Resolver = resolve;
444
+ });
445
+ events.push('block1:upload:end');
446
+ return mockUploadBlock(bareUrl, token, block, onProgress);
447
+ }
448
+ if (token === 'token/block:2') {
449
+ if (block2FirstAttempt) {
450
+ block2FirstAttempt = false;
451
+ events.push('block2:timeout');
452
+ // Resolve block 1 after a small delay to simulate real-world conditions
453
+ setTimeout(() => block1Resolver?.(), 100);
454
+ throw error;
455
+ }
456
+ events.push('block2:retry');
457
+ return mockUploadBlock(bareUrl, token, block, onProgress);
458
+ }
459
+ // Block 3 and thumbnails proceed normally
460
+ return mockUploadBlock(bareUrl, token, block, onProgress);
461
+ });
462
+
463
+ await verifySuccess();
464
+
465
+ expect(events).toMatchObject([
466
+ 'block1:upload:start',
467
+ 'block2:timeout',
468
+ 'block1:upload:end',
469
+ 'block2:retry',
470
+ ]);
471
+
472
+ // Also verify the warning messages were logged
473
+ expect(logger.warn).toHaveBeenCalledWith(
474
+ 'block 2:token/block:2: Upload timeout, limiting upload capacity to 1 block',
475
+ );
476
+ expect(logger.warn).toHaveBeenCalledWith('block 2:token/block:2: Upload timeout, retrying');
477
+ });
478
+
406
479
  it('should handle expired token when uploading block', async () => {
407
480
  let count = 0;
408
481
  apiService.uploadBlock = jest.fn().mockImplementation(async function (bareUrl, token, block, onProgress) {
@@ -55,6 +55,8 @@ const MAX_BLOCK_UPLOAD_RETRIES = 3;
55
55
  * that the upload process is efficient and does not overload the server.
56
56
  */
57
57
  export class StreamUploader {
58
+ protected maxUploadingBlocks = MAX_UPLOADING_BLOCKS;
59
+
58
60
  protected logger: Logger;
59
61
 
60
62
  protected digests: UploadDigests;
@@ -67,6 +69,7 @@ export class StreamUploader {
67
69
  protected ongoingUploads = new Map<
68
70
  string,
69
71
  {
72
+ index?: number;
70
73
  uploadPromise: Promise<void>;
71
74
  encryptedBlock: EncryptedBlock | EncryptedThumbnail;
72
75
  }
@@ -376,6 +379,7 @@ export class StreamUploader {
376
379
 
377
380
  const uploadKey = `block:${blockToken.index}`;
378
381
  this.ongoingUploads.set(uploadKey, {
382
+ index: blockToken.index,
379
383
  uploadPromise: this.uploadBlock(blockToken, encryptedBlock, onProgress).finally(() => {
380
384
  this.ongoingUploads.delete(uploadKey);
381
385
 
@@ -500,6 +504,13 @@ export class StreamUploader {
500
504
  blockProgress = 0;
501
505
  }
502
506
 
507
+ if (error instanceof Error && error.name === 'TimeoutError') {
508
+ logger.warn(`Upload timeout, limiting upload capacity to 1 block`);
509
+ await this.limitUploadCapacity(uploadToken.index);
510
+ logger.warn(`Upload timeout, retrying`);
511
+ continue;
512
+ }
513
+
503
514
  if (
504
515
  (error instanceof APIHTTPError && error.statusCode === HTTPErrorCode.NOT_FOUND) ||
505
516
  error instanceof NotFoundAPIError
@@ -542,6 +553,27 @@ export class StreamUploader {
542
553
  logger.info(`Uploaded`);
543
554
  }
544
555
 
556
+ private async limitUploadCapacity(index: number) {
557
+ this.maxUploadingBlocks = 1;
558
+
559
+ // This ensures that when the upload is downscaled, all ongoing block
560
+ // uploads are waiting for their turn one by one.
561
+ try {
562
+ await waitForCondition(() => {
563
+ const ongoingIndexes = Array.from(this.ongoingUploads.values())
564
+ .map(({ index: ongoingIndex }) => ongoingIndex)
565
+ .filter((ongoingIndex) => ongoingIndex !== undefined);
566
+ ongoingIndexes.sort((a, b) => a - b);
567
+ return ongoingIndexes[0] === index;
568
+ }, this.abortController.signal);
569
+ } catch (error: unknown) {
570
+ if (error instanceof AbortError) {
571
+ return;
572
+ }
573
+ throw error;
574
+ }
575
+ }
576
+
545
577
  private async waitForBufferCapacity() {
546
578
  if (this.encryptedBlocks.size >= MAX_BUFFERED_BLOCKS) {
547
579
  try {
@@ -559,7 +591,7 @@ export class StreamUploader {
559
591
  }
560
592
 
561
593
  private async waitForUploadCapacityAndBufferedBlocks() {
562
- while (this.ongoingUploads.size >= MAX_UPLOADING_BLOCKS) {
594
+ while (this.ongoingUploads.size >= this.maxUploadingBlocks) {
563
595
  await Promise.race(this.ongoingUploads.values().map(({ uploadPromise }) => uploadPromise));
564
596
  }
565
597
  try {
@@ -13,13 +13,12 @@ export class UploadTelemetry {
13
13
  private sharesService: SharesService,
14
14
  ) {
15
15
  this.telemetry = telemetry;
16
- this.logger = this.telemetry.getLogger('download');
16
+ this.logger = this.telemetry.getLogger('upload');
17
17
  this.sharesService = sharesService;
18
18
  }
19
19
 
20
20
  getLoggerForRevision(revisionUid: string) {
21
- const logger = this.telemetry.getLogger('upload');
22
- return new LoggerWithPrefix(logger, `revision ${revisionUid}`);
21
+ return new LoggerWithPrefix(this.logger, `revision ${revisionUid}`);
23
22
  }
24
23
 
25
24
  logBlockVerificationError(retryHelped: boolean) {