@knime/utils 1.0.16 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,22 @@
1
1
  # @knime/utils
2
2
 
3
+ ## 1.1.0
4
+
5
+ ### Minor Changes
6
+
7
+ - a8eebec: - Add `promise` utils
8
+ - `retryPromise` util function to retry async calls
9
+ - `createAbortablePromise` util function to leverage AbortControllers and
10
+ add abort behavior to an async call
11
+ - Add uploadManager util. Multi-file and multi-part upload handler
12
+
13
+ ## 1.0.17
14
+
15
+ ### Patch Changes
16
+
17
+ - Updated dependencies [df2caa5]
18
+ - @knime/styles@1.1.0
19
+
3
20
  ## 1.0.16
4
21
 
5
22
  ### Patch Changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@knime/utils",
3
- "version": "1.0.16",
3
+ "version": "1.1.0",
4
4
  "description": "Internal utility functions and constants shared across @knime packages.",
5
5
  "homepage": "https://knime.github.io/webapps-common/",
6
6
  "license": "GPL 3 and Additional Permissions according to Sec. 7 (SEE the file LICENSE)",
@@ -10,7 +10,7 @@
10
10
  "consola": "3.2.3",
11
11
  "date-fns": "2.30.0",
12
12
  "date-fns-tz": "2.0.0",
13
- "@knime/styles": "1.0.11"
13
+ "@knime/styles": "1.1.0"
14
14
  },
15
15
  "main": "src/index.ts",
16
16
  "module": "src/index.ts",
@@ -33,7 +33,8 @@
33
33
  }
34
34
  },
35
35
  "devDependencies": {
36
- "flush-promises": "^1.0.2"
36
+ "flush-promises": "^1.0.2",
37
+ "mock-xmlhttprequest": "^8.3.0"
37
38
  },
38
39
  "scripts": {
39
40
  "test:unit": "TZ='Europe/Berlin' vitest"
@@ -1,6 +1,7 @@
1
1
  import { isAfter, isBefore } from "date-fns";
2
- import updateTime from "./updateTime";
2
+
3
3
  import updateDate from "./updateDate";
4
+ import updateTime from "./updateTime";
4
5
 
5
6
  /**
6
7
  * Check if date is before min.
@@ -8,13 +8,13 @@ import odtIcon from "@knime/styles/img/icons/file-odt.svg";
8
8
  import pdfIcon from "@knime/styles/img/icons/file-pdf.svg";
9
9
  import pptxIcon from "@knime/styles/img/icons/file-pptx.svg";
10
10
  import psIcon from "@knime/styles/img/icons/file-ps.svg";
11
+ import fileIcon from "@knime/styles/img/icons/file-question.svg";
12
+ import txtIcon from "@knime/styles/img/icons/file-text.svg";
11
13
  import xlsIcon from "@knime/styles/img/icons/file-xls.svg";
12
14
  import xlsxIcon from "@knime/styles/img/icons/file-xlsx.svg";
13
15
  import xmlIcon from "@knime/styles/img/icons/file-xml.svg";
14
- import zipIcon from "@knime/styles/img/icons/file-zip.svg";
15
16
  import exeIcon from "@knime/styles/img/icons/file-zip-exe.svg";
16
- import txtIcon from "@knime/styles/img/icons/file-text.svg";
17
- import fileIcon from "@knime/styles/img/icons/file-question.svg";
17
+ import zipIcon from "@knime/styles/img/icons/file-zip.svg";
18
18
 
19
19
  export const icons = {
20
20
  csvIcon,
package/src/index.ts CHANGED
@@ -1,36 +1,37 @@
1
1
  import { capitalize, caseFormatter } from "./capitalize";
2
+ import { isAfterMaxDate, isBeforeMinDate } from "./dateMinMaxCheck";
2
3
  import debounce from "./debounce";
3
- import { filters } from "./filters";
4
- import truncateString from "./truncateString";
5
- import getWrappedAroundIndex from "./getWrappedAroundIndex";
6
- import sleep from "./sleep";
7
4
  import { icons, isIconExisting } from "./fileTypeIcons";
8
-
9
- import updateDate from "./updateDate";
10
- import updateTime from "./updateTime";
11
- import { isAfterMaxDate, isBeforeMinDate } from "./dateMinMaxCheck";
5
+ import { filters } from "./filters";
12
6
  import {
13
7
  formatDateString,
14
8
  formatDateTimeString,
15
9
  formatLocalDateTimeString,
16
10
  formatTimeString,
17
11
  } from "./format";
18
-
19
- import getLocalTimeZone from "./localTimezone";
20
- import numIntegerDigits from "./numIntegerDigits";
21
-
22
12
  import {
23
- formatHotkeys,
24
13
  formatHotkey,
14
+ formatHotkeys,
25
15
  getDefaultSeparator,
26
16
  } from "./formatHotkeys";
27
-
28
17
  import type {
29
18
  Hotkey as _Hotkey,
30
19
  KnownHotkey as _KnownHotkey,
31
20
  } from "./formatHotkeys";
32
-
21
+ import getWrappedAroundIndex from "./getWrappedAroundIndex";
22
+ import getLocalTimeZone from "./localTimezone";
33
23
  import * as navigatorUtils from "./navigator";
24
+ import numIntegerDigits from "./numIntegerDigits";
25
+ import * as promise from "./promise";
26
+ import sleep from "./sleep";
27
+ import truncateString from "./truncateString";
28
+ import updateDate from "./updateDate";
29
+ import updateTime from "./updateTime";
30
+ import type {
31
+ UploaderManagerConfig as _UploadManagerConfig,
32
+ UploadStateName as _UploadStateName,
33
+ } from "./uploadManager";
34
+ import * as uploadManager from "./uploadManager";
34
35
 
35
36
  const hotkeys = { formatHotkeys, formatHotkey, getDefaultSeparator };
36
37
 
@@ -39,7 +40,14 @@ export namespace HotkeysNS {
39
40
  export type KnownHotkey = _KnownHotkey;
40
41
  }
41
42
 
43
+ export namespace UploadManagerNS {
44
+ export type UploaderConfig = _UploadManagerConfig;
45
+ export type UploadStateName = _UploadStateName;
46
+ }
47
+
42
48
  export {
49
+ uploadManager,
50
+ promise,
43
51
  capitalize,
44
52
  caseFormatter,
45
53
  debounce,
@@ -0,0 +1,31 @@
1
+ import { describe, expect, it, vi } from "vitest";
2
+ import flushPromises from "flush-promises";
3
+
4
+ import { AbortError, createAbortablePromise } from "../createAbortablePromise";
5
+
6
+ describe("createAbortablePromise", () => {
7
+ it("should abort promise", async () => {
8
+ const mockFetchCall = () =>
9
+ new Promise((r) => setTimeout(() => r("resolved-value"), 300));
10
+
11
+ const { abortController, runAbortablePromise } = createAbortablePromise();
12
+
13
+ const catchSpy = vi.fn((e) => {
14
+ if (e instanceof AbortError) {
15
+ return "aborted";
16
+ }
17
+
18
+ throw e;
19
+ });
20
+
21
+ const promise = runAbortablePromise(() => mockFetchCall()).catch(catchSpy);
22
+
23
+ const abortError = new AbortError("aborting");
24
+ abortController.abort(abortError);
25
+
26
+ await flushPromises();
27
+
28
+ expect(catchSpy).toHaveBeenCalledWith(abortError);
29
+ expect(promise).resolves.toBe("aborted");
30
+ });
31
+ });
@@ -0,0 +1,73 @@
1
+ import { describe, expect, it, vi } from "vitest";
2
+
3
+ import { retryPromise } from "../retryPromise";
4
+
5
+ describe("retryPromise", () => {
6
+ it("should retry", async () => {
7
+ let callCount = 0;
8
+ const mockFetchCall = vi.fn(
9
+ () =>
10
+ new Promise((resolve, reject) => {
11
+ callCount++;
12
+
13
+ if (callCount < 3) {
14
+ reject(new Error("failure"));
15
+ return;
16
+ }
17
+
18
+ if (callCount >= 3) {
19
+ resolve("success");
20
+ }
21
+ }),
22
+ );
23
+
24
+ const doneSpy = vi.fn();
25
+ const catchSpy = vi.fn();
26
+
27
+ const promise = retryPromise(() => mockFetchCall())
28
+ .then(doneSpy)
29
+ .catch(catchSpy);
30
+
31
+ await promise;
32
+
33
+ expect(doneSpy).toHaveBeenCalledOnce();
34
+ expect(catchSpy).not.toHaveBeenCalled();
35
+ expect(mockFetchCall).toHaveBeenCalledTimes(3);
36
+ });
37
+
38
+ it("should throw if retry count is exceeded", async () => {
39
+ const error = new Error("failure");
40
+ const mockFetchCall = vi.fn(() => Promise.reject(error));
41
+
42
+ const doneSpy = vi.fn();
43
+
44
+ const promise = retryPromise(() => mockFetchCall()).then(doneSpy);
45
+
46
+ await promise.catch(() => {}); // ignore errors caught by test
47
+ expect(() => promise).rejects.toThrowError(error);
48
+
49
+ expect(doneSpy).not.toHaveBeenCalled();
50
+ expect(mockFetchCall).toHaveBeenCalledTimes(6);
51
+ });
52
+
53
+ it("should not retry for specific errors", async () => {
54
+ class MyError extends Error {}
55
+ const myError = new MyError("failure");
56
+ const mockFetchCall = vi.fn(() => Promise.reject(myError));
57
+
58
+ const doneSpy = vi.fn();
59
+
60
+ const promise = retryPromise(
61
+ () => mockFetchCall(),
62
+ 2,
63
+ 0,
64
+ (e) => e instanceof MyError,
65
+ ).then(doneSpy);
66
+
67
+ await promise.catch(() => {}); // ignore errors caught by test
68
+ expect(() => promise).rejects.toThrowError(myError);
69
+
70
+ expect(doneSpy).not.toHaveBeenCalled();
71
+ expect(mockFetchCall).toHaveBeenCalledOnce();
72
+ });
73
+ });
@@ -0,0 +1,56 @@
1
+ export class AbortError extends Error {}
2
+
3
+ /**
4
+ * Helper to create an abortable promise. Returns an AbortController and a Function
5
+ * that you can pass your async handler to. Whenever the AbortController signal is
6
+ * aborted then the async handler's promise will reject with an AbortError
7
+ *
8
+ * Example:
9
+ * ```
10
+ * import { abortUtils } from "@knime/utils"
11
+ * const myFetchCall = async (params...) => {...};
12
+ *
13
+ * try {
14
+ * const { runAbortablePromise, abortController } = createAbortablePromise(() =>
15
+ * myFetchCall(params...),
16
+ * );
17
+ * } catch (e) {
18
+ * if (e instanceof abortUtils.AbortError) {
19
+ * console.info('ignore due to abort');
20
+ * return;
21
+ * }
22
+ * }
23
+ *
24
+ * // somewhere else (while request is in-flight)
25
+ * abortController.abort(new abortUtils.AbortError('reason here'))
26
+ * ```
27
+ */
28
+ export const createAbortablePromise = () => {
29
+ const abortController: AbortController = new AbortController();
30
+
31
+ const runAbortablePromise = <T = unknown>(request: () => Promise<T>) => {
32
+ return new Promise<T>((resolve, reject) => {
33
+ // abort logic
34
+ const abortListener = ({ target }: Event) => {
35
+ abortController.signal.removeEventListener("abort", abortListener);
36
+ const reason = (target as AbortSignal).reason;
37
+ reject(reason instanceof AbortError ? reason : new AbortError(reason));
38
+ };
39
+
40
+ // the actual call
41
+ request()
42
+ .then(resolve)
43
+ .catch((error) => {
44
+ if (error instanceof AbortError) {
45
+ return;
46
+ }
47
+
48
+ reject(error instanceof Error ? error : new Error(error));
49
+ });
50
+
51
+ abortController.signal.addEventListener("abort", abortListener);
52
+ });
53
+ };
54
+
55
+ return { abortController, runAbortablePromise };
56
+ };
@@ -0,0 +1,2 @@
1
+ export * from "./createAbortablePromise";
2
+ export * from "./retryPromise";
@@ -0,0 +1,36 @@
1
+ /* eslint-disable func-style */
2
+ import sleep from "../sleep";
3
+
4
+ const DEFAULT_RETRY_DELAY_MS = 0;
5
+ const DEFAULT_RETRY_COUNT = 5;
6
+
7
+ /**
8
+ * Helper to add retry capability to an async function
9
+ * @param fn async function
10
+ * @param retryCount max number of retries. defaults to 5
11
+ * @param retryDelayMS delay in-between retries in millisecs. defaults to 0
12
+ * @param excludeError predicate fn to exclude specific errors from causing a retry
13
+ */
14
+ export async function retryPromise<T>(
15
+ fn: () => Promise<T>,
16
+ retryCount = DEFAULT_RETRY_COUNT,
17
+ retryDelayMS = DEFAULT_RETRY_DELAY_MS,
18
+ excludeError?: (error: Error) => boolean,
19
+ ) {
20
+ try {
21
+ return await fn();
22
+ } catch (error) {
23
+ if (excludeError?.(error as Error)) {
24
+ throw error;
25
+ }
26
+
27
+ if (retryCount > 0) {
28
+ await sleep(retryDelayMS);
29
+
30
+ // eslint-disable-next-line no-return-await
31
+ return await retryPromise(fn, retryCount - 1, retryDelayMS);
32
+ }
33
+
34
+ throw error;
35
+ }
36
+ }
package/src/updateDate.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { isValid, getYear, getMonth, getDate, set } from "date-fns";
1
+ import { getDate, getMonth, getYear, isValid, set } from "date-fns";
2
2
  /**
3
3
  * Updates the date part (year, day and month) of a date object (base)
4
4
  * time, offset (timezone) and so on will be left untouched.
@@ -0,0 +1,302 @@
1
+ /* eslint-disable vitest/no-disabled-tests */
2
+ import { beforeEach, describe, expect, it, vi } from "vitest";
3
+ import { newServer } from "mock-xmlhttprequest";
4
+
5
+ import { createUploadManager } from "..";
6
+
7
+ describe("uploadManager", () => {
8
+ const fiftyBytes = new Array(50).fill("A");
9
+
10
+ const file = new File(fiftyBytes, "mock-file.txt", {
11
+ type: "text/plain",
12
+ });
13
+
14
+ type MockXHRHandler = NonNullable<Parameters<typeof newServer>[0]>[string][1];
15
+
16
+ const defaultUploadPartResponse: MockXHRHandler = {
17
+ // status: 200 is the default
18
+ headers: { "Content-Type": "application/json", Etag: '"partId"' },
19
+ body: "{}",
20
+ };
21
+
22
+ type Options = {
23
+ uploadPartResponse?: MockXHRHandler;
24
+ };
25
+
26
+ const setup = ({ uploadPartResponse }: Options = {}) => {
27
+ const server = newServer({
28
+ put: ["/success", uploadPartResponse ?? defaultUploadPartResponse],
29
+ });
30
+
31
+ server.install();
32
+
33
+ return { server };
34
+ };
35
+
36
+ const mockUploadURLMetadata = { method: "PUT", url: "/success" };
37
+ const resolveFilePartUploadURL = vi.fn();
38
+
39
+ beforeEach(() => {
40
+ resolveFilePartUploadURL.mockImplementation(() =>
41
+ Promise.resolve(mockUploadURLMetadata),
42
+ );
43
+ });
44
+
45
+ const createIdGenerator = () => {
46
+ const generator = (function* () {
47
+ let id = 0;
48
+ while (true) {
49
+ yield id++;
50
+ }
51
+ })();
52
+
53
+ return () => `mock-upload--${generator.next().value}`;
54
+ };
55
+
56
+ const getUploadId = createIdGenerator();
57
+
58
+ describe("success", () => {
59
+ it("should handle `uploadFiles`", async () => {
60
+ setup();
61
+ const { uploadFiles } = createUploadManager({
62
+ resolveFilePartUploadURL,
63
+ });
64
+
65
+ const uploadId = getUploadId();
66
+ const { completed, failed, cancelled } = await uploadFiles([
67
+ { uploadId, file },
68
+ ]);
69
+
70
+ expect(completed).toEqual([uploadId]);
71
+ expect(failed.length).toBe(0);
72
+ expect(cancelled.length).toBe(0);
73
+ expect(resolveFilePartUploadURL).toHaveBeenCalled();
74
+ });
75
+
76
+ it("should succeed if resolveFilePartUrl is retried successfully", async () => {
77
+ let callCount = 0;
78
+ resolveFilePartUploadURL.mockImplementation(() => {
79
+ callCount++;
80
+ if (callCount < 2) {
81
+ return Promise.reject(new Error("error that will be retried"));
82
+ }
83
+
84
+ return Promise.resolve(mockUploadURLMetadata);
85
+ });
86
+
87
+ setup();
88
+ const { uploadFiles } = createUploadManager({
89
+ resolveFilePartUploadURL,
90
+ });
91
+
92
+ const uploadId = getUploadId();
93
+ const { completed, failed, cancelled } = await uploadFiles([
94
+ { uploadId, file },
95
+ ]);
96
+
97
+ expect(completed).toEqual([uploadId]);
98
+ expect(failed.length).toBe(0);
99
+ expect(cancelled.length).toBe(0);
100
+ expect(resolveFilePartUploadURL).toHaveBeenCalled();
101
+ });
102
+
103
+ it("should succeed if uploadPart request is retried successfully", async () => {
104
+ let callCount = 0;
105
+ setup({
106
+ uploadPartResponse: (request) => {
107
+ callCount++;
108
+ if (callCount <= 3) {
109
+ request.respond(400, {});
110
+ }
111
+
112
+ request.respond(200, { Etag: '"partId"' });
113
+ },
114
+ });
115
+
116
+ const uploadId = getUploadId();
117
+ const { uploadFiles } = createUploadManager({
118
+ resolveFilePartUploadURL,
119
+ });
120
+
121
+ const { completed, failed, cancelled } = await uploadFiles([
122
+ { uploadId, file },
123
+ ]);
124
+
125
+ expect(completed).toEqual([uploadId]);
126
+ expect(failed.length).toBe(0);
127
+ expect(cancelled.length).toBe(0);
128
+ expect(resolveFilePartUploadURL).toHaveBeenCalled();
129
+ });
130
+ });
131
+
132
+ describe("callbacks", () => {
133
+ it("should call onFileUploadComplete", async () => {
134
+ setup();
135
+ const onComplete = vi.fn();
136
+ const onFailed = vi.fn();
137
+ const { uploadFiles } = createUploadManager({
138
+ resolveFilePartUploadURL,
139
+ onFileUploadComplete: onComplete,
140
+ onFileUploadFailed: onFailed,
141
+ });
142
+
143
+ const uploadId1 = getUploadId();
144
+ const uploadId2 = getUploadId();
145
+
146
+ const { completed, failed, cancelled } = await uploadFiles([
147
+ { uploadId: uploadId1, file },
148
+ { uploadId: uploadId2, file },
149
+ ]);
150
+
151
+ expect(completed).toEqual([uploadId1, uploadId2]);
152
+ expect(failed.length).toBe(0);
153
+ expect(cancelled.length).toBe(0);
154
+ expect(onComplete).toHaveBeenCalledWith({
155
+ uploadId: uploadId1,
156
+ filePartIds: { 1: "partId" },
157
+ });
158
+ expect(onComplete).toHaveBeenCalledWith({
159
+ uploadId: uploadId2,
160
+ filePartIds: { 1: "partId" },
161
+ });
162
+ expect(onFailed).not.toHaveBeenCalled();
163
+ });
164
+
165
+ it("should call onFileUploadFailed", async () => {
166
+ setup({
167
+ uploadPartResponse: (request) => {
168
+ request.respond(400, {});
169
+ },
170
+ });
171
+ const onComplete = vi.fn();
172
+ const onFailed = vi.fn();
173
+ const { uploadFiles } = createUploadManager({
174
+ resolveFilePartUploadURL,
175
+ onFileUploadComplete: onComplete,
176
+ onFileUploadFailed: onFailed,
177
+ });
178
+
179
+ const uploadId1 = getUploadId();
180
+ const uploadId2 = getUploadId();
181
+ const { completed, failed, cancelled } = await uploadFiles([
182
+ { uploadId: uploadId1, file },
183
+ { uploadId: uploadId2, file },
184
+ ]);
185
+
186
+ expect(completed.length).toBe(0);
187
+ expect(failed).toEqual([uploadId1, uploadId2]);
188
+ expect(cancelled.length).toBe(0);
189
+ expect(onFailed).toHaveBeenCalledWith(
190
+ uploadId1,
191
+ new Error("Failed to upload chunk 0"),
192
+ );
193
+ expect(onFailed).toHaveBeenCalledWith(
194
+ uploadId2,
195
+ new Error("Failed to upload chunk 0"),
196
+ );
197
+ expect(onComplete).not.toHaveBeenCalled();
198
+ });
199
+
200
+ it("should call onProgress callback", async () => {
201
+ setup();
202
+ const onProgress = vi.fn();
203
+
204
+ const { uploadFiles } = createUploadManager({
205
+ resolveFilePartUploadURL,
206
+ onProgress,
207
+ });
208
+
209
+ await uploadFiles([{ uploadId: getUploadId(), file }]);
210
+
211
+ expect(onProgress).toHaveBeenCalled();
212
+ });
213
+ });
214
+
215
+ describe("cancellation", () => {
216
+ it("should handle cancellation", async () => {
217
+ setup();
218
+ const { uploadFiles, cancel } = createUploadManager({
219
+ resolveFilePartUploadURL,
220
+ });
221
+
222
+ const uploadId = getUploadId();
223
+ const upload = uploadFiles([{ uploadId, file }]);
224
+
225
+ cancel(uploadId);
226
+
227
+ const { completed, cancelled, failed } = await upload;
228
+ expect(completed.length).toBe(0);
229
+ expect(cancelled).toEqual([uploadId]);
230
+ expect(failed.length).toBe(0);
231
+ });
232
+ });
233
+
234
+ describe("failures", () => {
235
+ it("should fail when resolving upload url fails", async () => {
236
+ resolveFilePartUploadURL.mockImplementation(() =>
237
+ Promise.reject(new Error("error")),
238
+ );
239
+
240
+ setup();
241
+ const { uploadFiles } = createUploadManager({
242
+ resolveFilePartUploadURL,
243
+ });
244
+
245
+ const uploadId = getUploadId();
246
+ const { completed, failed, cancelled } = await uploadFiles([
247
+ { uploadId, file },
248
+ ]);
249
+
250
+ expect(completed.length).toBe(0);
251
+ expect(failed).toEqual([uploadId]);
252
+ expect(cancelled.length).toBe(0);
253
+ expect(resolveFilePartUploadURL).toHaveBeenCalled();
254
+ });
255
+
256
+ it("should fail when uploading part request fails", async () => {
257
+ setup({
258
+ uploadPartResponse: (request) => {
259
+ request.respond(400, {});
260
+ },
261
+ });
262
+
263
+ const { uploadFiles } = createUploadManager({
264
+ resolveFilePartUploadURL,
265
+ });
266
+
267
+ const uploadId = getUploadId();
268
+
269
+ const { completed, failed, cancelled } = await uploadFiles([
270
+ { uploadId, file },
271
+ ]);
272
+
273
+ expect(completed.length).toBe(0);
274
+ expect(failed).toEqual([uploadId]);
275
+ expect(cancelled.length).toBe(0);
276
+ expect(resolveFilePartUploadURL).toHaveBeenCalled();
277
+ });
278
+
279
+ it("should fail when etag is missing from upload part response", async () => {
280
+ setup({
281
+ uploadPartResponse: (request) => {
282
+ request.respond(200, {});
283
+ },
284
+ });
285
+
286
+ const { uploadFiles } = createUploadManager({
287
+ resolveFilePartUploadURL,
288
+ });
289
+
290
+ const uploadId = getUploadId();
291
+
292
+ const { completed, failed, cancelled } = await uploadFiles([
293
+ { uploadId, file },
294
+ ]);
295
+
296
+ expect(completed.length).toBe(0);
297
+ expect(failed).toEqual([uploadId]);
298
+ expect(cancelled.length).toBe(0);
299
+ expect(resolveFilePartUploadURL).toHaveBeenCalled();
300
+ });
301
+ });
302
+ });
@@ -0,0 +1,363 @@
1
+ import { AbortError, createAbortablePromise, retryPromise } from "../promise";
2
+
3
+ // eslint-disable-next-line no-magic-numbers
4
+ const CHUNK_SIZE = 50 * 1024 * 1024; // 50 MB per chunk
5
+
6
+ const DEFAULT_RETRY_COUNT = 5;
7
+ const DEFAULT_RETRY_DELAY_MS = 50;
8
+
9
+ export type UploadStateName =
10
+ | "idle"
11
+ | "inprogress"
12
+ | "cancelled"
13
+ | "failed"
14
+ | "complete";
15
+
16
+ export type PartUploadResponse = {
17
+ partId: string;
18
+ };
19
+
20
+ const uploadChunkWithProgress = (params: {
21
+ method: string;
22
+ url: string;
23
+ chunk: Blob;
24
+ chunkIndex: number;
25
+ onProgress: (progress: number) => void;
26
+ abortSignal: AbortSignal;
27
+ }) => {
28
+ const { method, url, chunk, chunkIndex, onProgress, abortSignal } = params;
29
+
30
+ const OK = 200;
31
+
32
+ return new Promise<{ partId: string }>((resolve, reject) => {
33
+ const xhr = new XMLHttpRequest();
34
+
35
+ xhr.open(method, url, true);
36
+
37
+ abortSignal.addEventListener("abort", () => xhr.abort());
38
+
39
+ // Track the upload progress of the current chunk
40
+ xhr.upload.onprogress = (event) => {
41
+ if (event.lengthComputable) {
42
+ onProgress(event.loaded);
43
+ }
44
+ };
45
+
46
+ xhr.onload = () => {
47
+ if (xhr.status === OK) {
48
+ const etag = xhr.getResponseHeader("ETag");
49
+
50
+ if (!etag) {
51
+ reject(new Error("Invalid part upload response"));
52
+ return;
53
+ }
54
+
55
+ // remove possible doublequotes (") around the etag and/or weak identifier (if either is present)
56
+ // https://datatracker.ietf.org/doc/html/rfc7232#section-2.3
57
+ resolve({ partId: etag.replace(/^(?:W\/)?"|"$/g, "") });
58
+ } else {
59
+ reject(new Error(`Failed to upload chunk ${chunkIndex}`));
60
+ }
61
+ };
62
+
63
+ xhr.onerror = () =>
64
+ reject(new Error(`Failed to upload chunk ${chunkIndex}`));
65
+
66
+ xhr.send(chunk);
67
+ });
68
+ };
69
+
70
+ export type UploaderManagerConfig = {
71
+ /**
72
+ * Function to resolve the metadata for a request to upload a file part
73
+ * @param uploadId
74
+ * @param partNumber
75
+ * @returns
76
+ */
77
+ resolveFilePartUploadURL: (
78
+ uploadId: string,
79
+ partNumber: number,
80
+ ) => Promise<{ method: string; url: string }>;
81
+
82
+ /**
83
+ * Callback that gets triggered when the whole file is uploaded.
84
+ * @param
85
+ * @returns
86
+ */
87
+ onFileUploadComplete?: (payload: {
88
+ /**
89
+ * Identifier of the upload for this file (must be unique)
90
+ */
91
+ uploadId: string;
92
+ /**
93
+ * Ids of all the uploaded file parts
94
+ */
95
+ filePartIds: Record<number, string>;
96
+ }) => void;
97
+
98
+ /**
99
+ * Callback that gets triggered when an upload has failed.
100
+ * @param uploadId
101
+ * @param error the error that cause the failure if it exists
102
+ * @returns
103
+ */
104
+ onFileUploadFailed?: (uploadId: string, error: Error) => void;
105
+
106
+ /**
107
+ * Callback that gets triggered when the progress of a file part upload
108
+ * for a specific upload id is updated
109
+ * @param uploadId
110
+ * @param progress
111
+ * @returns
112
+ */
113
+ onProgress?: (uploadId: string, progress: number) => void;
114
+ };
115
+
116
+ type UploadState = {
117
+ uploadId: string;
118
+ file: File;
119
+ completedChunkCount: number;
120
+ uploadedByteCount: number;
121
+ state: UploadStateName;
122
+ abortController: AbortController;
123
+ };
124
+
125
+ /**
126
+ * Creates and returns an `uploadManager`, a comprehensive multi-file, multi-part upload handler.
127
+ *
128
+ * The `uploadManager` offers functionality for handling file uploads in parts and provides
129
+ * several built-in mechanisms to ensure the process is reliable and adaptable.
130
+ *
131
+ * Key features include:
132
+ *
133
+ * - **Chunked File Uploads**: Splits provided files into manageable chunks and initiates the upload
134
+ * for each part using the `resolveFilePartUploadURL` function provided in the configuration.
135
+ *
136
+ * - **Retry Mechanism**: Automatically retries failed upload attempts up to a maximum of 5 times
137
+ * per failed request.
138
+ *
139
+ * - **Upload Cancellation**: Allows cancellation of ongoing uploads, giving users control over the process.
140
+ *
141
+ * - **Progress Tracking**: Monitors the progress of each file part upload and triggers the `onProgress`
142
+ * callback to provide updates.
143
+ *
144
+ * - **Error Handling**: The returned `uploadFiles` function does not throw errors directly. Instead,
145
+ * it manages retries and stops any failed uploads after the retry limit is reached, marking them as failed.
146
+ * This ensures that the promise it returns completes gracefully without uncaught exceptions.
147
+ *
148
+ */
149
+ export const createUploadManager = (config: UploaderManagerConfig) => {
150
+ const {
151
+ resolveFilePartUploadURL: getUploadFilePartUrl,
152
+ onFileUploadComplete,
153
+ onFileUploadFailed,
154
+ } = config;
155
+ const uploadStateMap = new Map<string, UploadState>();
156
+
157
+ const completedUploads = new Set<string>();
158
+ const cancelledUploads = new Set<string>();
159
+ const failedUploads = new Set<string>();
160
+
161
+ const assertState = (uploadId: string, state: UploadStateName) =>
162
+ uploadStateMap.has(uploadId) &&
163
+ uploadStateMap.get(uploadId)!.state === state;
164
+
165
+ const getUploadState = (uploadId: string) => uploadStateMap.get(uploadId);
166
+ const setUploadState = (uploadId: string, state: Partial<UploadState>) => {
167
+ uploadStateMap.set(uploadId, {
168
+ ...uploadStateMap.get(uploadId)!,
169
+ ...state,
170
+ });
171
+ };
172
+
173
+ const isFailed = (uploadId: string) => assertState(uploadId, "failed");
174
+ const isCancelled = (uploadId: string) => assertState(uploadId, "cancelled");
175
+
176
+ const getTotalFileParts = (file: File) => Math.ceil(file.size / CHUNK_SIZE);
177
+
178
+ const abortInFlightUpload = (uploadId: string, reason: string) => {
179
+ const currentState = uploadStateMap.get(uploadId);
180
+
181
+ if (!currentState) {
182
+ return;
183
+ }
184
+
185
+ currentState.abortController.abort(
186
+ new AbortError(
187
+ `Upload (${currentState.uploadId}) for file "${currentState.file.name}" aborted. Reason: "${reason}"`,
188
+ ),
189
+ );
190
+ };
191
+
192
+ const cancel = (uploadId: string) => {
193
+ if (!uploadStateMap.has(uploadId)) {
194
+ consola.error("cancel:: Invalid upload id");
195
+ return;
196
+ }
197
+
198
+ setUploadState(uploadId, { state: "cancelled" });
199
+
200
+ abortInFlightUpload(uploadId, "upload cancelled");
201
+ cancelledUploads.add(uploadId);
202
+ };
203
+
204
+ const setFailed = (uploadId: string, error: Error) => {
205
+ if (!uploadStateMap.has(uploadId)) {
206
+ consola.error("failed:: Invalid upload id");
207
+ return;
208
+ }
209
+
210
+ setUploadState(uploadId, { state: "failed" });
211
+ onFileUploadFailed?.(uploadId, error);
212
+ failedUploads.add(uploadId);
213
+
214
+ if (completedUploads.has(uploadId)) {
215
+ completedUploads.delete(uploadId);
216
+ }
217
+ };
218
+
219
+ const uploadFileFromChunkIndex = async (
220
+ uploadId: string,
221
+ file: File,
222
+ initialChunkIndex = 0,
223
+ ) => {
224
+ const totalChunks = getTotalFileParts(file);
225
+ const totalBytes = file.size;
226
+ const completedPartIds: Record<number, string> = {};
227
+
228
+ const { abortController, runAbortablePromise } = createAbortablePromise();
229
+
230
+ uploadStateMap.set(uploadId, {
231
+ uploadId,
232
+ file,
233
+ state: "inprogress",
234
+ uploadedByteCount: uploadStateMap.get(uploadId)?.uploadedByteCount ?? 0,
235
+ completedChunkCount:
236
+ uploadStateMap.get(uploadId)?.completedChunkCount ?? 0,
237
+ abortController,
238
+ });
239
+
240
+ let chunkIndex = initialChunkIndex;
241
+ let uploadedBytes = uploadStateMap.get(uploadId)?.uploadedByteCount;
242
+
243
+ while (chunkIndex < totalChunks) {
244
+ if (isCancelled(uploadId) || isFailed(uploadId)) {
245
+ break;
246
+ }
247
+
248
+ const start = chunkIndex * CHUNK_SIZE;
249
+ const end = Math.min(start + CHUNK_SIZE, file.size);
250
+
251
+ const chunk = file.slice(start, end);
252
+
253
+ try {
254
+ const { method, url } = await retryPromise(
255
+ () => getUploadFilePartUrl(uploadId, chunkIndex + 1),
256
+ DEFAULT_RETRY_COUNT,
257
+ DEFAULT_RETRY_DELAY_MS,
258
+ );
259
+
260
+ consola.trace("File part upload url resolved", { method, url });
261
+
262
+ const onChunkUploadProgress = (uploadedChunkBytes: number) => {
263
+ uploadedBytes = uploadedChunkBytes + start;
264
+ setUploadState(uploadId, { uploadedByteCount: uploadedBytes });
265
+
266
+ const progress = Math.round((uploadedBytes / totalBytes) * 100);
267
+
268
+ config.onProgress?.(uploadId, progress);
269
+ };
270
+
271
+ const filePartResponse = await runAbortablePromise(() =>
272
+ retryPromise(
273
+ () =>
274
+ uploadChunkWithProgress({
275
+ method,
276
+ url,
277
+ chunk,
278
+ chunkIndex,
279
+ onProgress: onChunkUploadProgress,
280
+ abortSignal: abortController.signal,
281
+ }),
282
+ DEFAULT_RETRY_COUNT,
283
+ DEFAULT_RETRY_DELAY_MS,
284
+ ),
285
+ );
286
+
287
+ consola.trace(
288
+ `${file.name}: Chunk ${chunkIndex + 1} of ${totalChunks} uploaded successfully`,
289
+ );
290
+
291
+ completedPartIds[chunkIndex + 1] = filePartResponse.partId;
292
+
293
+ setUploadState(uploadId, {
294
+ completedChunkCount:
295
+ uploadStateMap.get(uploadId)!.completedChunkCount + 1,
296
+ });
297
+ } catch (error) {
298
+ if (error instanceof AbortError) {
299
+ consola.info("Aborted::", error);
300
+ return;
301
+ }
302
+
303
+ setFailed(
304
+ uploadId,
305
+ error instanceof Error ? error : new Error(error as any),
306
+ );
307
+ return;
308
+ }
309
+
310
+ chunkIndex++;
311
+ }
312
+
313
+ if (isCancelled(uploadId) || isFailed(uploadId)) {
314
+ return;
315
+ }
316
+
317
+ setUploadState(uploadId, {
318
+ completedChunkCount: totalChunks,
319
+ state: "complete",
320
+ });
321
+ completedUploads.add(uploadId);
322
+ onFileUploadComplete?.({ uploadId, filePartIds: completedPartIds });
323
+ };
324
+
325
+ const uploadFiles = (files: Array<{ uploadId: string; file: File }>) => {
326
+ const result: Array<Promise<any>> = [];
327
+
328
+ for (const { uploadId, file } of files) {
329
+ result.push(uploadFileFromChunkIndex(uploadId, file));
330
+ }
331
+
332
+ return Promise.allSettled(result).then(() => {
333
+ // state cleanup
334
+ const completed = [...completedUploads.values()];
335
+ const cancelled = [...cancelledUploads.values()];
336
+ const failed = [...failedUploads.values()];
337
+
338
+ [...completed, ...cancelled, ...failed].forEach((uploadId) =>
339
+ uploadStateMap.delete(uploadId),
340
+ );
341
+
342
+ completedUploads.clear();
343
+ cancelledUploads.clear();
344
+ failedUploads.clear();
345
+
346
+ return {
347
+ completed,
348
+ cancelled,
349
+ failed,
350
+ };
351
+ });
352
+ };
353
+
354
+ return {
355
+ uploadFiles,
356
+
357
+ getUploadState,
358
+ isFailed,
359
+ isCancelled,
360
+ cancel,
361
+ setFailed,
362
+ };
363
+ };