@gentleduck/upload 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +297 -0
- package/dist/core/index.d.ts +2 -3
- package/dist/core/index.js +1 -1
- package/dist/core-BVgloTiP.js +10 -0
- package/dist/index-DCqyBVcx.d.ts +1441 -0
- package/dist/index-DsZHPmJm.d.ts +225 -0
- package/dist/index-KROXtLQh.d.ts +281 -0
- package/dist/index.d.ts +4 -5
- package/dist/index.js +1 -1
- package/dist/react/index.d.ts +2 -90
- package/dist/react/index.js +1 -1
- package/dist/react-euPrDv24.js +1 -0
- package/dist/strategies/index.d.ts +2 -56
- package/dist/strategies/index.js +1 -1
- package/dist/strategies-CUF0s68j.js +1 -0
- package/package.json +3 -2
- package/src/__tests__/emitter.test.ts +85 -0
- package/src/__tests__/fingerprint.test.ts +49 -0
- package/src/__tests__/guards.test.ts +40 -0
- package/src/__tests__/id.test.ts +67 -0
- package/src/__tests__/mime-sniff.test.ts +142 -0
- package/src/__tests__/persistence.test.ts +191 -0
- package/src/__tests__/sanitize-filename.test.ts +109 -0
- package/src/__tests__/sleep.test.ts +34 -0
- package/src/__tests__/static-analysis.test.ts +145 -0
- package/src/__tests__/store-libs.test.ts +192 -0
- package/src/__tests__/transport-fetch.test.ts +108 -0
- package/src/__tests__/transport-libs.test.ts +42 -0
- package/src/__tests__/transport.test.ts +182 -0
- package/src/core/__tests__/errors.test.ts +152 -0
- package/src/core/client.ts +3 -0
- package/src/core/contracts/contracts.types.ts +373 -0
- package/src/core/contracts/index.ts +3 -8
- package/src/core/contracts/transport/index.ts +3 -2
- package/src/core/contracts/transport/transport.fetch.ts +203 -0
- package/src/core/contracts/transport/transport.libs.ts +13 -39
- package/src/core/contracts/transport/transport.ts +96 -90
- package/src/core/contracts/transport/transport.types.ts +71 -35
- package/src/core/engine/__tests__/config-nan-bypass.test.ts +40 -0
- package/src/core/engine/engine.libs.ts +35 -0
- package/src/core/engine/engine.types.ts +490 -0
- package/src/core/engine/index.ts +3 -8
- package/src/core/engine/reducer.ts +93 -53
- package/src/core/engine/store/handlers/add-file.ts +43 -48
- package/src/core/engine/store/handlers/cancel.ts +13 -8
- package/src/core/engine/store/handlers/clean-up.ts +11 -8
- package/src/core/engine/store/handlers/create-intent.ts +82 -27
- package/src/core/engine/store/handlers/finalize.ts +30 -13
- package/src/core/engine/store/handlers/index.ts +7 -7
- package/src/core/engine/store/handlers/pause.ts +8 -6
- package/src/core/engine/store/handlers/run-upload.ts +54 -39
- package/src/core/engine/store/index.ts +27 -21
- package/src/core/engine/store/store.dispatch.ts +10 -8
- package/src/core/engine/store/store.libs.ts +197 -94
- package/src/core/engine/store/store.runtime.ts +52 -53
- package/src/core/engine/store/store.schedule.ts +27 -21
- package/src/core/engine/store/store.types.ts +177 -227
- package/src/core/engine/validation/__tests__/intent.test.ts +70 -0
- package/src/core/engine/validation/file.ts +8 -4
- package/src/core/engine/validation/index.ts +1 -0
- package/src/core/engine/validation/intent.ts +49 -27
- package/src/core/engine/validation/list.ts +14 -15
- package/src/core/engine/validation/mime.ts +39 -0
- package/src/core/errors.ts +271 -0
- package/src/core/index.ts +28 -5
- package/src/core/persistence/__tests__/deserialize-nan-bypass.test.ts +100 -0
- package/src/core/persistence/__tests__/persistence.test.ts +85 -0
- package/src/core/persistence/adapters.indexeddb.ts +9 -3
- package/src/core/persistence/adapters.local.ts +9 -3
- package/src/core/persistence/adapters.memory.ts +35 -13
- package/src/core/persistence/index.ts +5 -5
- package/src/core/persistence/persistence.ts +73 -60
- package/src/core/persistence/persistence.types.ts +140 -60
- package/src/core/utils/__tests__/guards.test.ts +93 -0
- package/src/core/utils/__tests__/url-safety.test.ts +112 -0
- package/src/core/utils/async.ts +7 -2
- package/src/core/utils/constants.ts +3 -9
- package/src/core/utils/emitter/emitter.ts +126 -0
- package/src/core/utils/emitter/emitter.types.ts +33 -16
- package/src/core/utils/emitter/index.ts +2 -74
- package/src/core/utils/fingerprint.ts +16 -5
- package/src/core/utils/guards.ts +91 -1
- package/src/core/utils/hash/__tests__/sha256.test.ts +61 -0
- package/src/core/utils/hash/index.ts +126 -0
- package/src/core/utils/hash/sha256.ts +179 -0
- package/src/core/utils/id.ts +33 -1
- package/src/core/utils/index.ts +15 -6
- package/src/core/utils/mime-sniff.ts +119 -0
- package/src/core/utils/sanitize-filename.ts +85 -0
- package/src/core/utils/url-safety.ts +156 -0
- package/src/react/__tests__/use-dropzone.test.ts +64 -0
- package/src/react/index.ts +5 -2
- package/src/react/upload-provider.tsx +59 -95
- package/src/react/uploader.types.ts +80 -0
- package/src/react/use-dropzone.tsx +226 -0
- package/src/react/use-uploader.tsx +97 -93
- package/src/strategies/_shared/__tests__/retry.test.ts +120 -0
- package/src/strategies/_shared/retry.ts +79 -0
- package/src/strategies/index.ts +9 -15
- package/src/strategies/multipart/__tests__/multipart.test.ts +229 -0
- package/src/strategies/multipart/index.ts +211 -116
- package/src/strategies/post/__tests__/post-url-safety.test.ts +119 -0
- package/src/strategies/post/index.ts +113 -71
- package/src/strategies/put/__tests__/put.test.ts +132 -0
- package/src/strategies/put/index.ts +139 -0
- package/src/strategies/registry/index.ts +12 -9
- package/src/strategies/tus/__tests__/tus.test.ts +159 -0
- package/src/strategies/tus/index.ts +211 -0
- package/dist/async-DxRWu4le.js +0 -1
- package/dist/core-BNxznRif.js +0 -2
- package/dist/index-BXg9WIce.d.ts +0 -987
- package/dist/strategy.types-gg9ZStRe.d.ts +0 -225
- package/src/core/client/config.types.ts +0 -77
- package/src/core/client/index.ts +0 -4
- package/src/core/client/plugins.types.ts +0 -46
- package/src/core/client/retry.types.ts +0 -6
- package/src/core/client/uploadClient.ts +0 -19
- package/src/core/contracts/backendApi.ts +0 -62
- package/src/core/contracts/cursor.types.ts +0 -23
- package/src/core/contracts/errors.types.ts +0 -67
- package/src/core/contracts/fingerprint.types.ts +0 -18
- package/src/core/contracts/intent.types.ts +0 -39
- package/src/core/contracts/result.types.ts +0 -11
- package/src/core/contracts/strategy.types.ts +0 -45
- package/src/core/engine/commands.types.ts +0 -17
- package/src/core/engine/event-map.types.ts +0 -45
- package/src/core/engine/internal-events.types.ts +0 -222
- package/src/core/engine/outcome.types.ts +0 -9
- package/src/core/engine/progress.types.ts +0 -15
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'
|
|
2
|
+
import type { Transport } from '../../../core'
|
|
3
|
+
import { __resetMultipartWarningsForTests, multipartStrategy, validatePartUrl } from '../index'
|
|
4
|
+
|
|
5
|
+
describe('validatePartUrl', () => {
|
|
6
|
+
beforeEach(() => {
|
|
7
|
+
__resetMultipartWarningsForTests()
|
|
8
|
+
})
|
|
9
|
+
|
|
10
|
+
test('rejects file:// URL', () => {
|
|
11
|
+
expect(() => validatePartUrl('file:///etc/passwd')).toThrow(/forbidden protocol/)
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
test('rejects javascript: URL', () => {
|
|
15
|
+
expect(() => validatePartUrl('javascript:alert(1)')).toThrow(/forbidden protocol/)
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
test('rejects malformed URL', () => {
|
|
19
|
+
expect(() => validatePartUrl('not a url')).toThrow(/malformed/)
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
test('rejects empty URL', () => {
|
|
23
|
+
expect(() => validatePartUrl('')).toThrow(/invalid URL/)
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
test('rejects URL containing ".." segment', () => {
|
|
27
|
+
expect(() => validatePartUrl('https://upload.example.com/a/../b')).toThrow(/\.\./)
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
test('rejects http://127.0.0.1/upload when allowPrivateHosts is false', () => {
|
|
31
|
+
expect(() => validatePartUrl('http://127.0.0.1/upload')).toThrow(/private\/loopback/)
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
test('rejects RFC1918 hosts (10/8, 172.16/12, 192.168/16)', () => {
|
|
35
|
+
expect(() => validatePartUrl('http://10.0.0.5/x')).toThrow(/private\/loopback/)
|
|
36
|
+
expect(() => validatePartUrl('http://172.16.1.1/x')).toThrow(/private\/loopback/)
|
|
37
|
+
expect(() => validatePartUrl('http://192.168.1.1/x')).toThrow(/private\/loopback/)
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
test('rejects IPv6 loopback ::1 and link-local fe80::', () => {
|
|
41
|
+
expect(() => validatePartUrl('http://[::1]/x')).toThrow(/private\/loopback/)
|
|
42
|
+
expect(() => validatePartUrl('http://[fe80::1]/x')).toThrow(/private\/loopback/)
|
|
43
|
+
expect(() => validatePartUrl('http://[fc00::1]/x')).toThrow(/private\/loopback/)
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
test('rejects IPv6 unspecified :: and multicast ff02::1', () => {
|
|
47
|
+
expect(() => validatePartUrl('http://[::]/x')).toThrow(/private\/loopback/)
|
|
48
|
+
expect(() => validatePartUrl('http://[ff02::1]/x')).toThrow(/private\/loopback/)
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
test('rejects IPv4-mapped IPv6 (dotted-quad, canonical hex, fully expanded)', () => {
|
|
52
|
+
expect(() => validatePartUrl('http://[::ffff:127.0.0.1]/')).toThrow(/private\/loopback/)
|
|
53
|
+
expect(() => validatePartUrl('http://[::ffff:7f00:1]/')).toThrow(/private\/loopback/)
|
|
54
|
+
expect(() => validatePartUrl('http://[0:0:0:0:0:ffff:7f00:1]/')).toThrow(/private\/loopback/)
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
test('rejects 6to4 IPv6 carrying a private IPv4 (2002:7f00:1::)', () => {
|
|
58
|
+
expect(() => validatePartUrl('http://[2002:7f00:1::]/')).toThrow(/private\/loopback/)
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('rejects NAT64 IPv6 carrying a private IPv4 (64:ff9b::7f00:1)', () => {
|
|
62
|
+
expect(() => validatePartUrl('http://[64:ff9b::7f00:1]/')).toThrow(/private\/loopback/)
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
test('rejects AWS cloud-metadata 169.254.169.254', () => {
|
|
66
|
+
expect(() => validatePartUrl('http://169.254.169.254/latest/meta-data/')).toThrow(/private\/loopback/)
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
test('rejects CGNAT 100.64.0.0/10', () => {
|
|
70
|
+
expect(() => validatePartUrl('http://100.64.5.5/')).toThrow(/private\/loopback/)
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
test('rejects multicast 224.0.0.0/4 and broadcast 255.255.255.255', () => {
|
|
74
|
+
expect(() => validatePartUrl('http://224.0.0.1/')).toThrow(/private\/loopback/)
|
|
75
|
+
expect(() => validatePartUrl('http://255.255.255.255/')).toThrow(/private\/loopback/)
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
test('accepts http://127.0.0.1/upload when allowPrivateHosts is true', () => {
|
|
79
|
+
expect(() => validatePartUrl('http://127.0.0.1/upload', { allowPrivateHosts: true })).not.toThrow()
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
test('accepts hardened private/embedded forms when allowPrivateHosts is true', () => {
|
|
83
|
+
const cases = [
|
|
84
|
+
'http://[::1]/',
|
|
85
|
+
'http://[::ffff:127.0.0.1]/',
|
|
86
|
+
'http://[::ffff:7f00:1]/',
|
|
87
|
+
'http://[2002:7f00:1::]/',
|
|
88
|
+
'http://[64:ff9b::7f00:1]/',
|
|
89
|
+
'http://169.254.169.254/latest/meta-data/',
|
|
90
|
+
'http://100.64.5.5/',
|
|
91
|
+
'http://224.0.0.1/',
|
|
92
|
+
'http://255.255.255.255/',
|
|
93
|
+
'http://[::]/',
|
|
94
|
+
'http://[ff02::1]/',
|
|
95
|
+
]
|
|
96
|
+
for (const u of cases) {
|
|
97
|
+
expect(() => validatePartUrl(u, { allowPrivateHosts: true })).not.toThrow()
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
test('public URL still flows through (happy path)', () => {
|
|
102
|
+
expect(() =>
|
|
103
|
+
validatePartUrl('https://upload.example.com/parts/3', {
|
|
104
|
+
allowedHosts: ['upload.example.com'],
|
|
105
|
+
}),
|
|
106
|
+
).not.toThrow()
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
test('allowedHosts: accepts listed host, rejects others', () => {
|
|
110
|
+
expect(() =>
|
|
111
|
+
validatePartUrl('https://upload.example.com/parts/3', {
|
|
112
|
+
allowedHosts: ['upload.example.com'],
|
|
113
|
+
}),
|
|
114
|
+
).not.toThrow()
|
|
115
|
+
|
|
116
|
+
expect(() =>
|
|
117
|
+
validatePartUrl('https://evil.example.com/parts/3', {
|
|
118
|
+
allowedHosts: ['upload.example.com'],
|
|
119
|
+
}),
|
|
120
|
+
).toThrow(/allow-list/)
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
test('allowedHosts matches case-insensitively and is port-aware', () => {
|
|
124
|
+
expect(() =>
|
|
125
|
+
validatePartUrl('https://Upload.Example.COM:8443/p', {
|
|
126
|
+
allowedHosts: ['upload.example.com:8443'],
|
|
127
|
+
}),
|
|
128
|
+
).not.toThrow()
|
|
129
|
+
|
|
130
|
+
expect(() =>
|
|
131
|
+
validatePartUrl('https://upload.example.com:9000/p', {
|
|
132
|
+
allowedHosts: ['upload.example.com:8443'],
|
|
133
|
+
}),
|
|
134
|
+
).toThrow(/allow-list/)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test('warns once on omitted allowedHosts, suppresses on subsequent calls', () => {
|
|
138
|
+
const spy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
|
139
|
+
try {
|
|
140
|
+
validatePartUrl('https://upload.example.com/a')
|
|
141
|
+
validatePartUrl('https://upload.example.com/b')
|
|
142
|
+
validatePartUrl('https://upload.example.com/c')
|
|
143
|
+
expect(spy).toHaveBeenCalledTimes(1)
|
|
144
|
+
expect(spy.mock.calls[0]?.[0]).toMatch(/allowedHosts/)
|
|
145
|
+
} finally {
|
|
146
|
+
spy.mockRestore()
|
|
147
|
+
}
|
|
148
|
+
})
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
describe('multipartStrategy happy path', () => {
|
|
152
|
+
beforeEach(() => {
|
|
153
|
+
__resetMultipartWarningsForTests()
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
afterEach(() => {
|
|
157
|
+
vi.restoreAllMocks()
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
function makeCtx(opts: { signPartUrl: string; fileSize?: number }) {
|
|
161
|
+
const size = opts.fileSize ?? 8
|
|
162
|
+
const file = new File([new Uint8Array(size)], 'a.bin')
|
|
163
|
+
const transport: Pick<Transport.Options, 'put' | 'postForm' | 'patch'> = {
|
|
164
|
+
put: vi.fn(async (_args: Parameters<Transport.Options['put']>[0]) => ({ etag: 'etag-1' })),
|
|
165
|
+
postForm: vi.fn(async () => ({})),
|
|
166
|
+
patch: vi.fn(async () => ({})),
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const cursors: Array<unknown> = []
|
|
170
|
+
const ctx = {
|
|
171
|
+
file,
|
|
172
|
+
intent: {
|
|
173
|
+
strategy: 'multipart' as const,
|
|
174
|
+
fileId: 'f1',
|
|
175
|
+
uploadId: 'u1',
|
|
176
|
+
partSize: size, // one part
|
|
177
|
+
partCount: 1,
|
|
178
|
+
},
|
|
179
|
+
signal: new AbortController().signal,
|
|
180
|
+
transport: transport,
|
|
181
|
+
api: {
|
|
182
|
+
createIntent: vi.fn(),
|
|
183
|
+
complete: vi.fn(),
|
|
184
|
+
multipart: {
|
|
185
|
+
signPart: vi.fn(async () => ({ url: opts.signPartUrl, headers: undefined })),
|
|
186
|
+
completeMultipart: vi.fn(async () => undefined),
|
|
187
|
+
},
|
|
188
|
+
} as unknown as Parameters<ReturnType<typeof multipartStrategy>['start']>[0]['api'],
|
|
189
|
+
readCursor: () => undefined,
|
|
190
|
+
persistCursor: (c: unknown) => {
|
|
191
|
+
cursors.push(c)
|
|
192
|
+
},
|
|
193
|
+
reportProgress: () => {},
|
|
194
|
+
}
|
|
195
|
+
return { ctx, transport, cursors }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
test('signed https://upload.example.com URL flows through to transport.put', async () => {
|
|
199
|
+
const { ctx, transport } = makeCtx({ signPartUrl: 'https://upload.example.com/parts/3' })
|
|
200
|
+
const start = multipartStrategy({ allowedHosts: ['upload.example.com'] })
|
|
201
|
+
await start.start(ctx as unknown as Parameters<typeof start.start>[0])
|
|
202
|
+
expect(transport.put).toHaveBeenCalledTimes(1)
|
|
203
|
+
const callArgs = (transport.put as ReturnType<typeof vi.fn>).mock.calls[0]?.[0] as { url: string }
|
|
204
|
+
expect(callArgs.url).toBe('https://upload.example.com/parts/3')
|
|
205
|
+
})
|
|
206
|
+
|
|
207
|
+
test('file:// signed URL is rejected before reaching transport.put', async () => {
|
|
208
|
+
const { ctx, transport } = makeCtx({ signPartUrl: 'file:///etc/passwd' })
|
|
209
|
+
const start = multipartStrategy({ allowedHosts: ['upload.example.com'] })
|
|
210
|
+
await expect(start.start(ctx as unknown as Parameters<typeof start.start>[0])).rejects.toThrow(/forbidden protocol/)
|
|
211
|
+
expect(transport.put).not.toHaveBeenCalled()
|
|
212
|
+
})
|
|
213
|
+
|
|
214
|
+
test('private-host signed URL is rejected when allowPrivateHosts is false', async () => {
|
|
215
|
+
const { ctx, transport } = makeCtx({ signPartUrl: 'http://127.0.0.1/upload' })
|
|
216
|
+
const start = multipartStrategy({})
|
|
217
|
+
// suppress the one-time warn
|
|
218
|
+
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
|
219
|
+
await expect(start.start(ctx as unknown as Parameters<typeof start.start>[0])).rejects.toThrow(/private\/loopback/)
|
|
220
|
+
expect(transport.put).not.toHaveBeenCalled()
|
|
221
|
+
})
|
|
222
|
+
|
|
223
|
+
test('private-host signed URL is accepted when allowPrivateHosts is true', async () => {
|
|
224
|
+
const { ctx, transport } = makeCtx({ signPartUrl: 'http://127.0.0.1/upload' })
|
|
225
|
+
const start = multipartStrategy({ allowPrivateHosts: true, allowedHosts: ['127.0.0.1'] })
|
|
226
|
+
await start.start(ctx as unknown as Parameters<typeof start.start>[0])
|
|
227
|
+
expect(transport.put).toHaveBeenCalledTimes(1)
|
|
228
|
+
})
|
|
229
|
+
})
|
|
@@ -1,53 +1,138 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* S3/MinIO-style multipart strategy: sign each part on demand via `signPart`,
|
|
3
|
+
* PUT, then finalize with `completeMultipart` using collected ETags. Resumable
|
|
4
|
+
* persists ETags in the cursor and skips already-completed sessions.
|
|
4
5
|
*/
|
|
5
6
|
|
|
6
|
-
import type {
|
|
7
|
-
import {
|
|
7
|
+
import type { Contracts } from '../../core'
|
|
8
|
+
import { UploadEngineError } from '../../core'
|
|
9
|
+
import { validateUploadUrl } from '../../core/utils/url-safety'
|
|
10
|
+
import { withRetry } from '../_shared/retry'
|
|
8
11
|
|
|
9
12
|
const DEFAULT_MAX_PART_CONCURRENCY = 4
|
|
10
13
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Namespace containing types and interfaces specific to the S3/MinIO multipart upload strategy.
|
|
16
|
+
* @author wildduck2 <https://github.com/wildduck2>
|
|
17
|
+
*/
|
|
18
|
+
export namespace MultipartStrategy {
|
|
19
|
+
/**
|
|
20
|
+
* Configuration options for the multipart upload strategy.
|
|
21
|
+
*/
|
|
22
|
+
export type Config = {
|
|
23
|
+
/** Maximum concurrent HTTP PUT part uploads. Defaults to 4. */
|
|
24
|
+
maxPartConcurrency?: number
|
|
25
|
+
/**
|
|
26
|
+
* Optional case-insensitive list of trusted hosts (with optional port, e.g. `upload.example.com`).
|
|
27
|
+
* When specified, every signed part URL must match one of the listed hosts.
|
|
28
|
+
*/
|
|
29
|
+
allowedHosts?: string[]
|
|
30
|
+
/**
|
|
31
|
+
* When `true`, allows private network IP addresses (loopback, link-local, RFC1918) in signed part URLs.
|
|
32
|
+
* Defaults to `false`.
|
|
33
|
+
*/
|
|
34
|
+
allowPrivateHosts?: boolean
|
|
35
|
+
}
|
|
25
36
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
37
|
+
/**
|
|
38
|
+
* Intent payload detailing part sizing and active upload session IDs returned by backends.
|
|
39
|
+
*/
|
|
40
|
+
export type Intent = {
|
|
41
|
+
/** Discriminant matching strategy configuration registry keys. */
|
|
42
|
+
strategy: 'multipart'
|
|
43
|
+
/** Unique database identifier for the file resource. */
|
|
44
|
+
fileId: string
|
|
45
|
+
/** Active multipart session ID returned by S3/GCS. */
|
|
46
|
+
uploadId: string
|
|
47
|
+
/** Chunk size in bytes for each part slice (typically 5MB minimum). */
|
|
48
|
+
partSize: number
|
|
49
|
+
/** Total count of parts predicted for this file size. */
|
|
50
|
+
partCount: number
|
|
51
|
+
|
|
52
|
+
/** Optional array of pre-generated URLs and headers for each part (if signed up front). */
|
|
53
|
+
parts?: Array<{
|
|
54
|
+
partNumber: number
|
|
55
|
+
url: string
|
|
56
|
+
headers?: Record<string, string>
|
|
57
|
+
}>
|
|
58
|
+
}
|
|
32
59
|
|
|
33
60
|
/**
|
|
34
|
-
*
|
|
35
|
-
* This prevents re-sending completeMultipart on resume.
|
|
61
|
+
* Persisted resume state for multipart strategy uploads.
|
|
36
62
|
*/
|
|
37
|
-
|
|
63
|
+
export type Cursor = {
|
|
64
|
+
/** List of completed parts. */
|
|
65
|
+
done: Array<{
|
|
66
|
+
partNumber: number
|
|
67
|
+
etag: string
|
|
68
|
+
size: number
|
|
69
|
+
}>
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* True if the backend assembled the parts.
|
|
73
|
+
* Prevents repeating completeMultipart calls on subsequent resume attempts.
|
|
74
|
+
*/
|
|
75
|
+
completed?: true
|
|
76
|
+
}
|
|
38
77
|
}
|
|
39
78
|
|
|
40
|
-
|
|
41
|
-
|
|
79
|
+
let warnedMissingAllowedHosts = false
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Reset internal warn-once latches. Test-only.
|
|
83
|
+
*
|
|
84
|
+
* @internal
|
|
85
|
+
*/
|
|
86
|
+
export function __resetMultipartWarningsForTests(): void {
|
|
87
|
+
warnedMissingAllowedHosts = false
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Validate the per-part URL returned by `signPart` before it reaches the
|
|
92
|
+
* transport. Delegates to the shared {@link validateUploadUrl} SSRF guard and
|
|
93
|
+
* warns once when no `allowedHosts` are configured. Throws on rejection.
|
|
94
|
+
*
|
|
95
|
+
* @internal
|
|
96
|
+
*/
|
|
97
|
+
export function validatePartUrl(
|
|
98
|
+
rawUrl: string,
|
|
99
|
+
opts: { allowedHosts?: string[]; allowPrivateHosts?: boolean } = {},
|
|
100
|
+
): void {
|
|
101
|
+
if ((!opts.allowedHosts || opts.allowedHosts.length === 0) && !warnedMissingAllowedHosts) {
|
|
102
|
+
warnedMissingAllowedHosts = true
|
|
103
|
+
console.warn(
|
|
104
|
+
'[duck-upload] multipartStrategy: no `allowedHosts` configured. Signed part URLs will be host-unrestricted. ' +
|
|
105
|
+
'Set MultipartStrategy.Config.allowedHosts to lock the upload host.',
|
|
106
|
+
)
|
|
107
|
+
}
|
|
108
|
+
validateUploadUrl(rawUrl, 'multipart.signPart', opts)
|
|
42
109
|
}
|
|
43
110
|
|
|
111
|
+
// types moved to MultipartStrategy namespace above
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* S3/GCS-compatible Resumable Multipart Upload Strategy.
|
|
115
|
+
*
|
|
116
|
+
* Slices files into chunks, uploads them concurrently using S3 PUT requests,
|
|
117
|
+
* and completes the upload on the backend once all chunks are transferred.
|
|
118
|
+
*
|
|
119
|
+
* @example
|
|
120
|
+
* ```ts
|
|
121
|
+
* const registry = createStrategyRegistry([
|
|
122
|
+
* multipartStrategy({ maxPartConcurrency: 4 })
|
|
123
|
+
* ]);
|
|
124
|
+
* ```
|
|
125
|
+
* @author wildduck2 <https://github.com/wildduck2>
|
|
126
|
+
*/
|
|
44
127
|
export function multipartStrategy<
|
|
45
|
-
M extends
|
|
46
|
-
C extends
|
|
128
|
+
M extends Contracts.Intent.Map & { multipart: MultipartStrategy.Intent },
|
|
129
|
+
C extends Contracts.Cursor.Map<M> & { multipart?: MultipartStrategy.Cursor },
|
|
47
130
|
P extends string = string,
|
|
48
|
-
R extends
|
|
49
|
-
>(opts?:
|
|
131
|
+
R extends Contracts.Result.Base = Contracts.Result.Base,
|
|
132
|
+
>(opts?: MultipartStrategy.Config): Contracts.Strategy.Me<M, C, P, R, 'multipart'> {
|
|
50
133
|
const maxPartConcurrency = Math.max(1, opts?.maxPartConcurrency ?? DEFAULT_MAX_PART_CONCURRENCY)
|
|
134
|
+
const allowedHosts = opts?.allowedHosts ?? []
|
|
135
|
+
const allowPrivateHosts = opts?.allowPrivateHosts === true
|
|
51
136
|
|
|
52
137
|
return {
|
|
53
138
|
id: 'multipart',
|
|
@@ -58,21 +143,19 @@ export function multipartStrategy<
|
|
|
58
143
|
const totalBytes = ctx.file.size
|
|
59
144
|
const partSize = Math.max(1, intent.partSize)
|
|
60
145
|
|
|
61
|
-
// Trust backend partCount
|
|
146
|
+
// Trust backend `partCount` when provided S3-style backends enforce a maxParts rule.
|
|
62
147
|
const totalParts = Math.max(1, intent.partCount ?? Math.ceil(totalBytes / partSize))
|
|
63
148
|
|
|
64
|
-
// Restore cursor
|
|
65
149
|
const cursor = ctx.readCursor()
|
|
66
150
|
const done = new Map<number, { etag: string; size: number }>()
|
|
67
151
|
if (cursor?.done) {
|
|
68
152
|
for (const p of cursor.done) done.set(p.partNumber, { etag: p.etag, size: p.size })
|
|
69
153
|
}
|
|
70
154
|
|
|
71
|
-
//
|
|
72
|
-
//
|
|
155
|
+
// Skip re-completing if a previous run already finalized the multipart
|
|
156
|
+
// session. The store still runs its generic finalization step.
|
|
73
157
|
const alreadyCompleted = cursor?.completed === true
|
|
74
158
|
|
|
75
|
-
// Progress
|
|
76
159
|
const inflightBytes = new Map<number, number>()
|
|
77
160
|
let finishedBytes = 0
|
|
78
161
|
for (const v of done.values()) finishedBytes += v.size
|
|
@@ -83,7 +166,6 @@ export function multipartStrategy<
|
|
|
83
166
|
ctx.reportProgress({ uploadedBytes: finishedBytes + inflight, totalBytes })
|
|
84
167
|
}
|
|
85
168
|
|
|
86
|
-
// Build queue
|
|
87
169
|
const partsToUpload: Array<{ partNumber: number; start: number; end: number }> = []
|
|
88
170
|
for (let partNumber = 1; partNumber <= totalParts; partNumber++) {
|
|
89
171
|
if (done.has(partNumber)) continue
|
|
@@ -102,90 +184,88 @@ export function multipartStrategy<
|
|
|
102
184
|
const running = new Set<Promise<void>>()
|
|
103
185
|
|
|
104
186
|
const getSignedPart = async (partNumber: number) => {
|
|
105
|
-
//
|
|
187
|
+
// Fast path: backend pre-signed all parts in `intent.parts`.
|
|
106
188
|
const fromIntent = intent.parts?.find((x) => x.partNumber === partNumber)
|
|
107
189
|
if (fromIntent) return fromIntent
|
|
108
190
|
|
|
109
|
-
// Normal path: ask backend for this part url
|
|
110
191
|
if (!ctx.api.multipart?.signPart) {
|
|
111
|
-
throw new
|
|
112
|
-
|
|
113
|
-
|
|
192
|
+
throw new UploadEngineError('validation_failed', {
|
|
193
|
+
message:
|
|
194
|
+
'multipart.signPart is missing in UploadApi. Implement it to call your backend sign-part endpoint.',
|
|
195
|
+
})
|
|
114
196
|
}
|
|
115
197
|
|
|
116
198
|
const out = await ctx.api.multipart.signPart(
|
|
117
|
-
{ fileId: intent.fileId, uploadId: intent.uploadId, partNumber },
|
|
118
|
-
{
|
|
199
|
+
{ fileId: intent.fileId, uploadId: intent.uploadId, partNumber, attempt: ctx.attempt },
|
|
200
|
+
{
|
|
201
|
+
localId: ctx.localId,
|
|
202
|
+
purpose: ctx.purpose,
|
|
203
|
+
file: ctx.file,
|
|
204
|
+
intent: ctx.intent,
|
|
205
|
+
signal: ctx.signal,
|
|
206
|
+
fingerprint: ctx.fingerprint,
|
|
207
|
+
attempt: ctx.attempt,
|
|
208
|
+
createdAt: ctx.createdAt,
|
|
209
|
+
steps: ctx.steps,
|
|
210
|
+
meta: ctx.meta,
|
|
211
|
+
lastError: ctx.lastError,
|
|
212
|
+
},
|
|
119
213
|
)
|
|
120
214
|
|
|
121
215
|
return { partNumber, url: out.url, headers: out.headers }
|
|
122
216
|
}
|
|
123
217
|
|
|
124
|
-
const uploadOne =
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
218
|
+
const uploadOne = (p: { partNumber: number; start: number; end: number }): Promise<void> =>
|
|
219
|
+
withRetry(
|
|
220
|
+
async () => {
|
|
221
|
+
try {
|
|
222
|
+
const blob = ctx.file.slice(p.start, p.end)
|
|
223
|
+
const size = blob.size
|
|
224
|
+
|
|
225
|
+
const signed = await getSignedPart(p.partNumber)
|
|
226
|
+
|
|
227
|
+
validatePartUrl(signed.url, { allowedHosts, allowPrivateHosts })
|
|
228
|
+
|
|
229
|
+
const res = await ctx.transport.put({
|
|
230
|
+
url: signed.url,
|
|
231
|
+
body: blob,
|
|
232
|
+
headers: signed.headers ?? {},
|
|
233
|
+
signal: ctx.signal,
|
|
234
|
+
onProgress: (loaded) => {
|
|
235
|
+
inflightBytes.set(p.partNumber, loaded)
|
|
236
|
+
report()
|
|
237
|
+
},
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
inflightBytes.delete(p.partNumber)
|
|
241
|
+
|
|
242
|
+
const etag = res.etag
|
|
243
|
+
if (!etag) {
|
|
244
|
+
throw new UploadEngineError('upload_failed', {
|
|
245
|
+
message:
|
|
246
|
+
'Missing ETag from upload part response. Ensure MinIO/S3 CORS exposes ETag (Access-Control-Expose-Headers: ETag).',
|
|
247
|
+
})
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
finishedBytes += size
|
|
251
|
+
done.set(p.partNumber, { etag, size })
|
|
252
|
+
|
|
253
|
+
const snapshot: MultipartStrategy.Cursor = {
|
|
254
|
+
done: Array.from(done.entries())
|
|
255
|
+
.map(([partNumber, v]) => ({ partNumber, etag: v.etag, size: v.size }))
|
|
256
|
+
.sort((a, b) => a.partNumber - b.partNumber),
|
|
257
|
+
}
|
|
258
|
+
ctx.persistCursor(snapshot as C['multipart'])
|
|
142
259
|
report()
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
const etag = res.etag
|
|
149
|
-
if (!etag) {
|
|
150
|
-
throw new Error(
|
|
151
|
-
'Missing ETag from upload part response. Ensure MinIO/S3 CORS exposes ETag (Access-Control-Expose-Headers: ETag).',
|
|
152
|
-
)
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
finishedBytes += size
|
|
156
|
-
done.set(p.partNumber, { etag, size })
|
|
157
|
-
|
|
158
|
-
const snapshot: MultipartCursor = {
|
|
159
|
-
done: Array.from(done.entries())
|
|
160
|
-
.map(([partNumber, v]) => ({ partNumber, etag: v.etag, size: v.size }))
|
|
161
|
-
.sort((a, b) => a.partNumber - b.partNumber),
|
|
162
|
-
}
|
|
163
|
-
ctx.persistCursor(snapshot as C['multipart'])
|
|
164
|
-
report()
|
|
165
|
-
} catch (err) {
|
|
166
|
-
inflightBytes.delete(p.partNumber)
|
|
167
|
-
|
|
168
|
-
if (ctx.signal?.aborted || isAbort(err)) throw err
|
|
169
|
-
|
|
170
|
-
// basic retry for network-ish failures
|
|
171
|
-
if (retryCount < maxRetries) {
|
|
172
|
-
const msg = err instanceof Error ? err.message : String(err)
|
|
173
|
-
const retryable =
|
|
174
|
-
/network/i.test(msg) ||
|
|
175
|
-
/timeout/i.test(msg) ||
|
|
176
|
-
/5\d\d/.test(msg) ||
|
|
177
|
-
/ECONNRESET/i.test(msg) ||
|
|
178
|
-
/EHOSTUNREACH/i.test(msg)
|
|
179
|
-
|
|
180
|
-
if (retryable) {
|
|
181
|
-
await sleep(2 ** retryCount * 500)
|
|
182
|
-
return uploadOne(p, retryCount + 1)
|
|
260
|
+
} catch (err) {
|
|
261
|
+
// Clear this part's in-flight bytes before retry/rethrow so the
|
|
262
|
+
// progress total does not double-count a re-attempted part.
|
|
263
|
+
inflightBytes.delete(p.partNumber)
|
|
264
|
+
throw err
|
|
183
265
|
}
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
}
|
|
188
|
-
}
|
|
266
|
+
},
|
|
267
|
+
{ signal: ctx.signal },
|
|
268
|
+
)
|
|
189
269
|
|
|
190
270
|
while (queue.length > 0 || running.size > 0) {
|
|
191
271
|
while (queue.length > 0 && running.size < maxPartConcurrency) {
|
|
@@ -211,9 +291,10 @@ export function multipartStrategy<
|
|
|
211
291
|
async function completeMultipart() {
|
|
212
292
|
if (alreadyCompleted) return
|
|
213
293
|
if (!ctx.api.multipart?.completeMultipart) {
|
|
214
|
-
throw new
|
|
215
|
-
|
|
216
|
-
|
|
294
|
+
throw new UploadEngineError('validation_failed', {
|
|
295
|
+
message:
|
|
296
|
+
'multipart.completeMultipart is missing in UploadApi. Implement it to call your backend complete endpoint.',
|
|
297
|
+
})
|
|
217
298
|
}
|
|
218
299
|
|
|
219
300
|
const parts = Array.from(done.entries())
|
|
@@ -221,16 +302,30 @@ export function multipartStrategy<
|
|
|
221
302
|
.sort((a, b) => a.partNumber - b.partNumber)
|
|
222
303
|
|
|
223
304
|
if (parts.length !== totalParts) {
|
|
224
|
-
throw new
|
|
305
|
+
throw new UploadEngineError('complete_failed', {
|
|
306
|
+
message: `Cannot complete multipart: expected ${totalParts} parts, got ${parts.length}`,
|
|
307
|
+
})
|
|
225
308
|
}
|
|
226
309
|
|
|
227
310
|
await ctx.api.multipart.completeMultipart(
|
|
228
311
|
{ fileId: intent.fileId, uploadId: intent.uploadId, parts },
|
|
229
|
-
{
|
|
312
|
+
{
|
|
313
|
+
localId: ctx.localId,
|
|
314
|
+
purpose: ctx.purpose,
|
|
315
|
+
file: ctx.file,
|
|
316
|
+
intent: ctx.intent,
|
|
317
|
+
signal: ctx.signal,
|
|
318
|
+
fingerprint: ctx.fingerprint,
|
|
319
|
+
attempt: ctx.attempt,
|
|
320
|
+
createdAt: ctx.createdAt,
|
|
321
|
+
steps: ctx.steps,
|
|
322
|
+
meta: ctx.meta,
|
|
323
|
+
lastError: ctx.lastError,
|
|
324
|
+
},
|
|
230
325
|
)
|
|
231
326
|
|
|
232
|
-
// Persist
|
|
233
|
-
const snapshot:
|
|
327
|
+
// Persist `completed: true` so resume after this point does not re-call `completeMultipart`.
|
|
328
|
+
const snapshot: MultipartStrategy.Cursor = {
|
|
234
329
|
done: Array.from(done.entries())
|
|
235
330
|
.map(([partNumber, v]) => ({ partNumber, etag: v.etag, size: v.size }))
|
|
236
331
|
.sort((a, b) => a.partNumber - b.partNumber),
|