@gentleduck/upload 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +170 -193
- package/dist/core/index.d.ts +2 -3
- package/dist/core/index.js +1 -1
- package/dist/core-BVgloTiP.js +10 -0
- package/dist/index-DCqyBVcx.d.ts +1441 -0
- package/dist/index-DsZHPmJm.d.ts +225 -0
- package/dist/index-KROXtLQh.d.ts +281 -0
- package/dist/index.d.ts +4 -5
- package/dist/index.js +1 -1
- package/dist/react/index.d.ts +2 -51
- package/dist/react/index.js +1 -1
- package/dist/react-euPrDv24.js +1 -0
- package/dist/strategies/index.d.ts +2 -56
- package/dist/strategies/index.js +1 -1
- package/dist/strategies-CUF0s68j.js +1 -0
- package/package.json +3 -2
- package/src/{__test__ → __tests__}/emitter.test.ts +4 -4
- package/src/{__test__ → __tests__}/fingerprint.test.ts +1 -1
- package/src/{__test__ → __tests__}/guards.test.ts +15 -5
- package/src/__tests__/id.test.ts +67 -0
- package/src/__tests__/mime-sniff.test.ts +142 -0
- package/src/{__test__ → __tests__}/persistence.test.ts +43 -8
- package/src/__tests__/sanitize-filename.test.ts +109 -0
- package/src/__tests__/sleep.test.ts +34 -0
- package/src/__tests__/static-analysis.test.ts +145 -0
- package/src/{__test__ → __tests__}/store-libs.test.ts +57 -3
- package/src/__tests__/transport-fetch.test.ts +108 -0
- package/src/{__test__ → __tests__}/transport.test.ts +5 -5
- package/src/core/__tests__/errors.test.ts +152 -0
- package/src/core/client.ts +3 -0
- package/src/core/contracts/contracts.types.ts +335 -141
- package/src/core/contracts/index.ts +3 -8
- package/src/core/contracts/transport/index.ts +3 -2
- package/src/core/contracts/transport/transport.fetch.ts +203 -0
- package/src/core/contracts/transport/transport.libs.ts +7 -8
- package/src/core/contracts/transport/transport.ts +91 -83
- package/src/core/contracts/transport/transport.types.ts +71 -35
- package/src/core/engine/__tests__/config-nan-bypass.test.ts +40 -0
- package/src/core/engine/engine.libs.ts +35 -0
- package/src/core/engine/engine.types.ts +367 -241
- package/src/core/engine/index.ts +3 -8
- package/src/core/engine/reducer.ts +89 -43
- package/src/core/engine/store/handlers/add-file.ts +39 -29
- package/src/core/engine/store/handlers/cancel.ts +13 -8
- package/src/core/engine/store/handlers/clean-up.ts +11 -8
- package/src/core/engine/store/handlers/create-intent.ts +82 -27
- package/src/core/engine/store/handlers/finalize.ts +30 -13
- package/src/core/engine/store/handlers/index.ts +7 -7
- package/src/core/engine/store/handlers/pause.ts +8 -6
- package/src/core/engine/store/handlers/run-upload.ts +49 -34
- package/src/core/engine/store/index.ts +27 -21
- package/src/core/engine/store/store.dispatch.ts +10 -8
- package/src/core/engine/store/store.libs.ts +184 -41
- package/src/core/engine/store/store.runtime.ts +45 -36
- package/src/core/engine/store/store.schedule.ts +27 -21
- package/src/core/engine/store/store.types.ts +177 -227
- package/src/core/engine/validation/__tests__/intent.test.ts +70 -0
- package/src/core/engine/validation/file.ts +7 -3
- package/src/core/engine/validation/index.ts +1 -0
- package/src/core/engine/validation/intent.ts +49 -27
- package/src/core/engine/validation/list.ts +14 -15
- package/src/core/engine/validation/mime.ts +39 -0
- package/src/core/errors.ts +271 -0
- package/src/core/index.ts +28 -5
- package/src/core/persistence/__tests__/deserialize-nan-bypass.test.ts +100 -0
- package/src/core/persistence/__tests__/persistence.test.ts +85 -0
- package/src/core/persistence/adapters.indexeddb.ts +9 -3
- package/src/core/persistence/adapters.local.ts +9 -3
- package/src/core/persistence/adapters.memory.ts +35 -13
- package/src/core/persistence/index.ts +5 -5
- package/src/core/persistence/persistence.ts +73 -60
- package/src/core/persistence/persistence.types.ts +140 -60
- package/src/core/utils/__tests__/guards.test.ts +93 -0
- package/src/core/utils/__tests__/url-safety.test.ts +112 -0
- package/src/core/utils/async.ts +7 -2
- package/src/core/utils/constants.ts +3 -9
- package/src/core/utils/emitter/emitter.ts +7 -24
- package/src/core/utils/emitter/emitter.types.ts +33 -16
- package/src/core/utils/emitter/index.ts +2 -74
- package/src/core/utils/fingerprint.ts +16 -5
- package/src/core/utils/guards.ts +91 -1
- package/src/core/utils/hash/__tests__/sha256.test.ts +61 -0
- package/src/core/utils/hash/index.ts +126 -0
- package/src/core/utils/hash/sha256.ts +179 -0
- package/src/core/utils/id.ts +33 -1
- package/src/core/utils/index.ts +15 -6
- package/src/core/utils/mime-sniff.ts +119 -0
- package/src/core/utils/sanitize-filename.ts +85 -0
- package/src/core/utils/url-safety.ts +156 -0
- package/src/react/__tests__/use-dropzone.test.ts +64 -0
- package/src/react/index.ts +5 -2
- package/src/react/upload-provider.tsx +68 -53
- package/src/react/uploader.types.ts +58 -61
- package/src/react/use-dropzone.tsx +226 -0
- package/src/react/use-uploader.tsx +98 -91
- package/src/strategies/_shared/__tests__/retry.test.ts +120 -0
- package/src/strategies/_shared/retry.ts +79 -0
- package/src/strategies/index.ts +9 -15
- package/src/strategies/multipart/__tests__/multipart.test.ts +229 -0
- package/src/strategies/multipart/index.ts +206 -108
- package/src/strategies/post/__tests__/post-url-safety.test.ts +119 -0
- package/src/strategies/post/index.ts +115 -33
- package/src/strategies/put/__tests__/put.test.ts +132 -0
- package/src/strategies/put/index.ts +139 -0
- package/src/strategies/registry/index.ts +12 -9
- package/src/strategies/tus/__tests__/tus.test.ts +159 -0
- package/src/strategies/tus/index.ts +211 -0
- package/dist/async-DxRWu4le.js +0 -1
- package/dist/core-CMkrP9DX.js +0 -2
- package/dist/index-C38Jmizu.d.ts +0 -1005
- package/dist/strategy.types-gg9ZStRe.d.ts +0 -225
- package/src/__test__/effect-pool.test.ts +0 -5
- package/src/__test__/id.test.ts +0 -14
- package/src/__test__/store-integration.test.ts +0 -5
- package/src/core/client/client.ts +0 -50
- package/src/core/client/client.types.ts +0 -184
- package/src/core/client/config.types.ts +0 -77
- package/src/core/client/index.ts +0 -4
- package/src/core/client/plugins.types.ts +0 -46
- package/src/core/client/retry.types.ts +0 -6
- package/src/core/client/uploadClient.ts +0 -19
- package/src/core/contracts/backendApi.ts +0 -62
- package/src/core/contracts/cursor.types.ts +0 -23
- package/src/core/contracts/errors.types.ts +0 -67
- package/src/core/contracts/fingerprint.types.ts +0 -18
- package/src/core/contracts/intent.types.ts +0 -39
- package/src/core/contracts/result.types.ts +0 -11
- package/src/core/contracts/strategy.types.ts +0 -45
- package/src/core/engine/commands.types.ts +0 -17
- package/src/core/engine/event-map.types.ts +0 -45
- package/src/core/engine/internal-events.types.ts +0 -202
- package/src/core/engine/outcome.types.ts +0 -9
- package/src/core/engine/progress.types.ts +0 -15
- package/src/core/engine/store/handlers.ts +0 -819
- package/src/core/engine/validation.ts +0 -216
- /package/src/{__test__ → __tests__}/transport-libs.test.ts +0 -0
|
@@ -1,54 +1,138 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* S3/MinIO-style multipart strategy: sign each part on demand via `signPart`,
|
|
3
|
-
* PUT, then finalize with `completeMultipart` using collected ETags. Resumable
|
|
3
|
+
* PUT, then finalize with `completeMultipart` using collected ETags. Resumable
|
|
4
4
|
* persists ETags in the cursor and skips already-completed sessions.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type {
|
|
8
|
-
import {
|
|
7
|
+
import type { Contracts } from '../../core'
|
|
8
|
+
import { UploadEngineError } from '../../core'
|
|
9
|
+
import { validateUploadUrl } from '../../core/utils/url-safety'
|
|
10
|
+
import { withRetry } from '../_shared/retry'
|
|
9
11
|
|
|
10
12
|
const DEFAULT_MAX_PART_CONCURRENCY = 4
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Namespace containing types and interfaces specific to the S3/MinIO multipart upload strategy.
|
|
16
|
+
* @author wildduck2 <https://github.com/wildduck2>
|
|
17
|
+
*/
|
|
18
|
+
export namespace MultipartStrategy {
|
|
19
|
+
/**
|
|
20
|
+
* Configuration options for the multipart upload strategy.
|
|
21
|
+
*/
|
|
22
|
+
export type Config = {
|
|
23
|
+
/** Maximum concurrent HTTP PUT part uploads. Defaults to 4. */
|
|
24
|
+
maxPartConcurrency?: number
|
|
25
|
+
/**
|
|
26
|
+
* Optional case-insensitive list of trusted hosts (with optional port, e.g. `upload.example.com`).
|
|
27
|
+
* When specified, every signed part URL must match one of the listed hosts.
|
|
28
|
+
*/
|
|
29
|
+
allowedHosts?: string[]
|
|
30
|
+
/**
|
|
31
|
+
* When `true`, allows private network IP addresses (loopback, link-local, RFC1918) in signed part URLs.
|
|
32
|
+
* Defaults to `false`.
|
|
33
|
+
*/
|
|
34
|
+
allowPrivateHosts?: boolean
|
|
35
|
+
}
|
|
26
36
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
37
|
+
/**
|
|
38
|
+
* Intent payload detailing part sizing and active upload session IDs returned by backends.
|
|
39
|
+
*/
|
|
40
|
+
export type Intent = {
|
|
41
|
+
/** Discriminant matching strategy configuration registry keys. */
|
|
42
|
+
strategy: 'multipart'
|
|
43
|
+
/** Unique database identifier for the file resource. */
|
|
44
|
+
fileId: string
|
|
45
|
+
/** Active multipart session ID returned by S3/GCS. */
|
|
46
|
+
uploadId: string
|
|
47
|
+
/** Chunk size in bytes for each part slice (typically 5MB minimum). */
|
|
48
|
+
partSize: number
|
|
49
|
+
/** Total count of parts predicted for this file size. */
|
|
50
|
+
partCount: number
|
|
51
|
+
|
|
52
|
+
/** Optional array of pre-generated URLs and headers for each part (if signed up front). */
|
|
53
|
+
parts?: Array<{
|
|
54
|
+
partNumber: number
|
|
55
|
+
url: string
|
|
56
|
+
headers?: Record<string, string>
|
|
57
|
+
}>
|
|
58
|
+
}
|
|
33
59
|
|
|
34
60
|
/**
|
|
35
|
-
*
|
|
36
|
-
* This prevents re-sending completeMultipart on resume.
|
|
61
|
+
* Persisted resume state for multipart strategy uploads.
|
|
37
62
|
*/
|
|
38
|
-
|
|
63
|
+
export type Cursor = {
|
|
64
|
+
/** List of completed parts. */
|
|
65
|
+
done: Array<{
|
|
66
|
+
partNumber: number
|
|
67
|
+
etag: string
|
|
68
|
+
size: number
|
|
69
|
+
}>
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* True if the backend assembled the parts.
|
|
73
|
+
* Prevents repeating completeMultipart calls on subsequent resume attempts.
|
|
74
|
+
*/
|
|
75
|
+
completed?: true
|
|
76
|
+
}
|
|
39
77
|
}
|
|
40
78
|
|
|
41
|
-
|
|
42
|
-
|
|
79
|
+
let warnedMissingAllowedHosts = false
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Reset internal warn-once latches. Test-only.
|
|
83
|
+
*
|
|
84
|
+
* @internal
|
|
85
|
+
*/
|
|
86
|
+
export function __resetMultipartWarningsForTests(): void {
|
|
87
|
+
warnedMissingAllowedHosts = false
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Validate the per-part URL returned by `signPart` before it reaches the
|
|
92
|
+
* transport. Delegates to the shared {@link validateUploadUrl} SSRF guard and
|
|
93
|
+
* warns once when no `allowedHosts` are configured. Throws on rejection.
|
|
94
|
+
*
|
|
95
|
+
* @internal
|
|
96
|
+
*/
|
|
97
|
+
export function validatePartUrl(
|
|
98
|
+
rawUrl: string,
|
|
99
|
+
opts: { allowedHosts?: string[]; allowPrivateHosts?: boolean } = {},
|
|
100
|
+
): void {
|
|
101
|
+
if ((!opts.allowedHosts || opts.allowedHosts.length === 0) && !warnedMissingAllowedHosts) {
|
|
102
|
+
warnedMissingAllowedHosts = true
|
|
103
|
+
console.warn(
|
|
104
|
+
'[duck-upload] multipartStrategy: no `allowedHosts` configured. Signed part URLs will be host-unrestricted. ' +
|
|
105
|
+
'Set MultipartStrategy.Config.allowedHosts to lock the upload host.',
|
|
106
|
+
)
|
|
107
|
+
}
|
|
108
|
+
validateUploadUrl(rawUrl, 'multipart.signPart', opts)
|
|
43
109
|
}
|
|
44
110
|
|
|
111
|
+
// types moved to MultipartStrategy namespace above
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* S3/GCS-compatible Resumable Multipart Upload Strategy.
|
|
115
|
+
*
|
|
116
|
+
* Slices files into chunks, uploads them concurrently using S3 PUT requests,
|
|
117
|
+
* and completes the upload on the backend once all chunks are transferred.
|
|
118
|
+
*
|
|
119
|
+
* @example
|
|
120
|
+
* ```ts
|
|
121
|
+
* const registry = createStrategyRegistry([
|
|
122
|
+
* multipartStrategy({ maxPartConcurrency: 4 })
|
|
123
|
+
* ]);
|
|
124
|
+
* ```
|
|
125
|
+
* @author wildduck2 <https://github.com/wildduck2>
|
|
126
|
+
*/
|
|
45
127
|
export function multipartStrategy<
|
|
46
|
-
M extends
|
|
47
|
-
C extends
|
|
128
|
+
M extends Contracts.Intent.Map & { multipart: MultipartStrategy.Intent },
|
|
129
|
+
C extends Contracts.Cursor.Map<M> & { multipart?: MultipartStrategy.Cursor },
|
|
48
130
|
P extends string = string,
|
|
49
|
-
R extends
|
|
50
|
-
>(opts?:
|
|
131
|
+
R extends Contracts.Result.Base = Contracts.Result.Base,
|
|
132
|
+
>(opts?: MultipartStrategy.Config): Contracts.Strategy.Me<M, C, P, R, 'multipart'> {
|
|
51
133
|
const maxPartConcurrency = Math.max(1, opts?.maxPartConcurrency ?? DEFAULT_MAX_PART_CONCURRENCY)
|
|
134
|
+
const allowedHosts = opts?.allowedHosts ?? []
|
|
135
|
+
const allowPrivateHosts = opts?.allowPrivateHosts === true
|
|
52
136
|
|
|
53
137
|
return {
|
|
54
138
|
id: 'multipart',
|
|
@@ -59,7 +143,7 @@ export function multipartStrategy<
|
|
|
59
143
|
const totalBytes = ctx.file.size
|
|
60
144
|
const partSize = Math.max(1, intent.partSize)
|
|
61
145
|
|
|
62
|
-
// Trust backend `partCount` when provided
|
|
146
|
+
// Trust backend `partCount` when provided S3-style backends enforce a maxParts rule.
|
|
63
147
|
const totalParts = Math.max(1, intent.partCount ?? Math.ceil(totalBytes / partSize))
|
|
64
148
|
|
|
65
149
|
const cursor = ctx.readCursor()
|
|
@@ -100,89 +184,88 @@ export function multipartStrategy<
|
|
|
100
184
|
const running = new Set<Promise<void>>()
|
|
101
185
|
|
|
102
186
|
const getSignedPart = async (partNumber: number) => {
|
|
103
|
-
//
|
|
187
|
+
// Fast path: backend pre-signed all parts in `intent.parts`.
|
|
104
188
|
const fromIntent = intent.parts?.find((x) => x.partNumber === partNumber)
|
|
105
189
|
if (fromIntent) return fromIntent
|
|
106
190
|
|
|
107
191
|
if (!ctx.api.multipart?.signPart) {
|
|
108
|
-
throw new
|
|
109
|
-
|
|
110
|
-
|
|
192
|
+
throw new UploadEngineError('validation_failed', {
|
|
193
|
+
message:
|
|
194
|
+
'multipart.signPart is missing in UploadApi. Implement it to call your backend sign-part endpoint.',
|
|
195
|
+
})
|
|
111
196
|
}
|
|
112
197
|
|
|
113
198
|
const out = await ctx.api.multipart.signPart(
|
|
114
|
-
{ fileId: intent.fileId, uploadId: intent.uploadId, partNumber },
|
|
115
|
-
{
|
|
199
|
+
{ fileId: intent.fileId, uploadId: intent.uploadId, partNumber, attempt: ctx.attempt },
|
|
200
|
+
{
|
|
201
|
+
localId: ctx.localId,
|
|
202
|
+
purpose: ctx.purpose,
|
|
203
|
+
file: ctx.file,
|
|
204
|
+
intent: ctx.intent,
|
|
205
|
+
signal: ctx.signal,
|
|
206
|
+
fingerprint: ctx.fingerprint,
|
|
207
|
+
attempt: ctx.attempt,
|
|
208
|
+
createdAt: ctx.createdAt,
|
|
209
|
+
steps: ctx.steps,
|
|
210
|
+
meta: ctx.meta,
|
|
211
|
+
lastError: ctx.lastError,
|
|
212
|
+
},
|
|
116
213
|
)
|
|
117
214
|
|
|
118
215
|
return { partNumber, url: out.url, headers: out.headers }
|
|
119
216
|
}
|
|
120
217
|
|
|
121
|
-
const uploadOne =
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
218
|
+
const uploadOne = (p: { partNumber: number; start: number; end: number }): Promise<void> =>
|
|
219
|
+
withRetry(
|
|
220
|
+
async () => {
|
|
221
|
+
try {
|
|
222
|
+
const blob = ctx.file.slice(p.start, p.end)
|
|
223
|
+
const size = blob.size
|
|
224
|
+
|
|
225
|
+
const signed = await getSignedPart(p.partNumber)
|
|
226
|
+
|
|
227
|
+
validatePartUrl(signed.url, { allowedHosts, allowPrivateHosts })
|
|
228
|
+
|
|
229
|
+
const res = await ctx.transport.put({
|
|
230
|
+
url: signed.url,
|
|
231
|
+
body: blob,
|
|
232
|
+
headers: signed.headers ?? {},
|
|
233
|
+
signal: ctx.signal,
|
|
234
|
+
onProgress: (loaded) => {
|
|
235
|
+
inflightBytes.set(p.partNumber, loaded)
|
|
236
|
+
report()
|
|
237
|
+
},
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
inflightBytes.delete(p.partNumber)
|
|
241
|
+
|
|
242
|
+
const etag = res.etag
|
|
243
|
+
if (!etag) {
|
|
244
|
+
throw new UploadEngineError('upload_failed', {
|
|
245
|
+
message:
|
|
246
|
+
'Missing ETag from upload part response. Ensure MinIO/S3 CORS exposes ETag (Access-Control-Expose-Headers: ETag).',
|
|
247
|
+
})
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
finishedBytes += size
|
|
251
|
+
done.set(p.partNumber, { etag, size })
|
|
252
|
+
|
|
253
|
+
const snapshot: MultipartStrategy.Cursor = {
|
|
254
|
+
done: Array.from(done.entries())
|
|
255
|
+
.map(([partNumber, v]) => ({ partNumber, etag: v.etag, size: v.size }))
|
|
256
|
+
.sort((a, b) => a.partNumber - b.partNumber),
|
|
257
|
+
}
|
|
258
|
+
ctx.persistCursor(snapshot as C['multipart'])
|
|
139
259
|
report()
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const etag = res.etag
|
|
146
|
-
if (!etag) {
|
|
147
|
-
throw new Error(
|
|
148
|
-
'Missing ETag from upload part response. Ensure MinIO/S3 CORS exposes ETag (Access-Control-Expose-Headers: ETag).',
|
|
149
|
-
)
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
finishedBytes += size
|
|
153
|
-
done.set(p.partNumber, { etag, size })
|
|
154
|
-
|
|
155
|
-
const snapshot: MultipartCursor = {
|
|
156
|
-
done: Array.from(done.entries())
|
|
157
|
-
.map(([partNumber, v]) => ({ partNumber, etag: v.etag, size: v.size }))
|
|
158
|
-
.sort((a, b) => a.partNumber - b.partNumber),
|
|
159
|
-
}
|
|
160
|
-
ctx.persistCursor(snapshot as C['multipart'])
|
|
161
|
-
report()
|
|
162
|
-
} catch (err) {
|
|
163
|
-
inflightBytes.delete(p.partNumber)
|
|
164
|
-
|
|
165
|
-
if (ctx.signal?.aborted || isAbort(err)) throw err
|
|
166
|
-
|
|
167
|
-
// Retry transient network errors with exponential backoff.
|
|
168
|
-
if (retryCount < maxRetries) {
|
|
169
|
-
const msg = err instanceof Error ? err.message : String(err)
|
|
170
|
-
const retryable =
|
|
171
|
-
/network/i.test(msg) ||
|
|
172
|
-
/timeout/i.test(msg) ||
|
|
173
|
-
/5\d\d/.test(msg) ||
|
|
174
|
-
/ECONNRESET/i.test(msg) ||
|
|
175
|
-
/EHOSTUNREACH/i.test(msg)
|
|
176
|
-
|
|
177
|
-
if (retryable) {
|
|
178
|
-
await sleep(2 ** retryCount * 500)
|
|
179
|
-
return uploadOne(p, retryCount + 1)
|
|
260
|
+
} catch (err) {
|
|
261
|
+
// Clear this part's in-flight bytes before retry/rethrow so the
|
|
262
|
+
// progress total does not double-count a re-attempted part.
|
|
263
|
+
inflightBytes.delete(p.partNumber)
|
|
264
|
+
throw err
|
|
180
265
|
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
}
|
|
185
|
-
}
|
|
266
|
+
},
|
|
267
|
+
{ signal: ctx.signal },
|
|
268
|
+
)
|
|
186
269
|
|
|
187
270
|
while (queue.length > 0 || running.size > 0) {
|
|
188
271
|
while (queue.length > 0 && running.size < maxPartConcurrency) {
|
|
@@ -208,9 +291,10 @@ export function multipartStrategy<
|
|
|
208
291
|
async function completeMultipart() {
|
|
209
292
|
if (alreadyCompleted) return
|
|
210
293
|
if (!ctx.api.multipart?.completeMultipart) {
|
|
211
|
-
throw new
|
|
212
|
-
|
|
213
|
-
|
|
294
|
+
throw new UploadEngineError('validation_failed', {
|
|
295
|
+
message:
|
|
296
|
+
'multipart.completeMultipart is missing in UploadApi. Implement it to call your backend complete endpoint.',
|
|
297
|
+
})
|
|
214
298
|
}
|
|
215
299
|
|
|
216
300
|
const parts = Array.from(done.entries())
|
|
@@ -218,16 +302,30 @@ export function multipartStrategy<
|
|
|
218
302
|
.sort((a, b) => a.partNumber - b.partNumber)
|
|
219
303
|
|
|
220
304
|
if (parts.length !== totalParts) {
|
|
221
|
-
throw new
|
|
305
|
+
throw new UploadEngineError('complete_failed', {
|
|
306
|
+
message: `Cannot complete multipart: expected ${totalParts} parts, got ${parts.length}`,
|
|
307
|
+
})
|
|
222
308
|
}
|
|
223
309
|
|
|
224
310
|
await ctx.api.multipart.completeMultipart(
|
|
225
311
|
{ fileId: intent.fileId, uploadId: intent.uploadId, parts },
|
|
226
|
-
{
|
|
312
|
+
{
|
|
313
|
+
localId: ctx.localId,
|
|
314
|
+
purpose: ctx.purpose,
|
|
315
|
+
file: ctx.file,
|
|
316
|
+
intent: ctx.intent,
|
|
317
|
+
signal: ctx.signal,
|
|
318
|
+
fingerprint: ctx.fingerprint,
|
|
319
|
+
attempt: ctx.attempt,
|
|
320
|
+
createdAt: ctx.createdAt,
|
|
321
|
+
steps: ctx.steps,
|
|
322
|
+
meta: ctx.meta,
|
|
323
|
+
lastError: ctx.lastError,
|
|
324
|
+
},
|
|
227
325
|
)
|
|
228
326
|
|
|
229
327
|
// Persist `completed: true` so resume after this point does not re-call `completeMultipart`.
|
|
230
|
-
const snapshot:
|
|
328
|
+
const snapshot: MultipartStrategy.Cursor = {
|
|
231
329
|
done: Array.from(done.entries())
|
|
232
330
|
.map(([partNumber, v]) => ({ partNumber, etag: v.etag, size: v.size }))
|
|
233
331
|
.sort((a, b) => a.partNumber - b.partNumber),
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, test, vi } from 'vitest'
|
|
2
|
+
import { __resetPostWarningsForTests, type PostIntent, PostStrategy } from '../index'
|
|
3
|
+
|
|
4
|
+
function makeCtx(url: string) {
|
|
5
|
+
const file = new File(['hello'], 'a.txt', { type: 'text/plain' })
|
|
6
|
+
return {
|
|
7
|
+
intent: { strategy: 'post', fileId: 'f', url, fields: {} } satisfies PostIntent,
|
|
8
|
+
file,
|
|
9
|
+
signal: new AbortController().signal,
|
|
10
|
+
reportProgress: vi.fn(),
|
|
11
|
+
transport: {
|
|
12
|
+
postForm: vi.fn(async () => ({ headers: {} })),
|
|
13
|
+
},
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
describe('PostStrategy SSRF guard', () => {
|
|
18
|
+
beforeEach(() => {
|
|
19
|
+
__resetPostWarningsForTests()
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
test('rejects file://', async () => {
|
|
23
|
+
const s = PostStrategy()
|
|
24
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
25
|
+
await expect(s.start(makeCtx('file:///etc/passwd') as any)).rejects.toThrow(/forbidden protocol/)
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
test('rejects javascript:', async () => {
|
|
29
|
+
const s = PostStrategy()
|
|
30
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
31
|
+
await expect(s.start(makeCtx('javascript:alert(1)') as any)).rejects.toThrow(/forbidden protocol/)
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
test('rejects loopback', async () => {
|
|
35
|
+
const s = PostStrategy()
|
|
36
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
37
|
+
await expect(s.start(makeCtx('https://127.0.0.1/up') as any)).rejects.toThrow(/private/)
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
test('rejects AWS metadata IPv4', async () => {
|
|
41
|
+
const s = PostStrategy()
|
|
42
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
43
|
+
await expect(s.start(makeCtx('https://169.254.169.254/up') as any)).rejects.toThrow(/private/)
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
test('rejects host outside allowedHosts', async () => {
|
|
47
|
+
const s = PostStrategy({ allowedHosts: ['up.example.com'] })
|
|
48
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
49
|
+
await expect(s.start(makeCtx('https://evil.example.com/up') as any)).rejects.toThrow(/allow-list/)
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
test('accepts allowlisted host', async () => {
|
|
53
|
+
const s = PostStrategy({ allowedHosts: ['up.example.com'] })
|
|
54
|
+
const ctx = makeCtx('https://up.example.com/up')
|
|
55
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
56
|
+
await expect(s.start(ctx as any)).resolves.toBeUndefined()
|
|
57
|
+
expect(ctx.transport.postForm).toHaveBeenCalled()
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
test('allowPrivateHosts lets loopback through (opt-in)', async () => {
|
|
61
|
+
const s = PostStrategy({ allowPrivateHosts: true })
|
|
62
|
+
const ctx = makeCtx('https://127.0.0.1/up')
|
|
63
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
64
|
+
await expect(s.start(ctx as any)).resolves.toBeUndefined()
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
test('warns once when allowedHosts not set', async () => {
|
|
68
|
+
const s = PostStrategy()
|
|
69
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
|
70
|
+
try {
|
|
71
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
72
|
+
await s.start(makeCtx('https://up.example.com/up') as any).catch(() => {})
|
|
73
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
74
|
+
await s.start(makeCtx('https://up.example.com/up') as any).catch(() => {})
|
|
75
|
+
expect(warn).toHaveBeenCalledTimes(1)
|
|
76
|
+
} finally {
|
|
77
|
+
warn.mockRestore()
|
|
78
|
+
}
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
test('caller label is "post.intent"', async () => {
|
|
82
|
+
const s = PostStrategy()
|
|
83
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
84
|
+
await expect(s.start(makeCtx('file:///etc/passwd') as any)).rejects.toThrow(/post\.intent/)
|
|
85
|
+
})
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
describe('PostStrategy retry', () => {
|
|
89
|
+
beforeEach(() => {
|
|
90
|
+
__resetPostWarningsForTests()
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
test('retries a transient failure then succeeds', async () => {
|
|
94
|
+
const s = PostStrategy({ allowedHosts: ['up.example.com'], maxRetries: 2 })
|
|
95
|
+
const ctx = makeCtx('https://up.example.com/up')
|
|
96
|
+
let calls = 0
|
|
97
|
+
ctx.transport.postForm = vi.fn(async () => {
|
|
98
|
+
calls++
|
|
99
|
+
if (calls < 2) throw new Error('network error')
|
|
100
|
+
return { headers: {} }
|
|
101
|
+
})
|
|
102
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
103
|
+
await expect(s.start(ctx as any)).resolves.toBeUndefined()
|
|
104
|
+
expect(calls).toBe(2)
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
test('does not retry a non-transient failure', async () => {
|
|
108
|
+
const s = PostStrategy({ allowedHosts: ['up.example.com'] })
|
|
109
|
+
const ctx = makeCtx('https://up.example.com/up')
|
|
110
|
+
let calls = 0
|
|
111
|
+
ctx.transport.postForm = vi.fn(async () => {
|
|
112
|
+
calls++
|
|
113
|
+
throw new Error('Upload failed with status 403')
|
|
114
|
+
})
|
|
115
|
+
// biome-ignore lint/suspicious/noExplicitAny: test ctx shape is a subset of the full strategy ctx
|
|
116
|
+
await expect(s.start(ctx as any)).rejects.toThrow(/403/)
|
|
117
|
+
expect(calls).toBe(1)
|
|
118
|
+
})
|
|
119
|
+
})
|
|
@@ -1,53 +1,135 @@
|
|
|
1
|
+
import type { Contracts } from '../../core'
|
|
2
|
+
import { UploadEngineError } from '../../core'
|
|
3
|
+
import { validateUploadUrl } from '../../core/utils/url-safety'
|
|
4
|
+
import { withRetry } from '../_shared/retry'
|
|
5
|
+
|
|
1
6
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
7
|
+
* Types for the presigned HTTP POST upload strategy.
|
|
8
|
+
* @author wildduck2 <https://github.com/wildduck2>
|
|
4
9
|
*/
|
|
10
|
+
export namespace PostStrategy {
|
|
11
|
+
/**
|
|
12
|
+
* Presigned HTTP POST strategy configuration options.
|
|
13
|
+
*/
|
|
14
|
+
export type Config = {
|
|
15
|
+
/**
|
|
16
|
+
* Case-insensitive allow-list of host names (with optional port). When
|
|
17
|
+
* set, the presigned POST URL must match a listed host or it is rejected.
|
|
18
|
+
*/
|
|
19
|
+
allowedHosts?: string[]
|
|
20
|
+
/**
|
|
21
|
+
* When `true`, allow private-network IP literals in the POST URL.
|
|
22
|
+
* Defaults to `false`.
|
|
23
|
+
*/
|
|
24
|
+
allowPrivateHosts?: boolean
|
|
25
|
+
/**
|
|
26
|
+
* Retries after the first attempt for transient network failures.
|
|
27
|
+
* Defaults to the shared retry default (3).
|
|
28
|
+
*/
|
|
29
|
+
maxRetries?: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Presigned HTTP POST intent payload mapping parameters returned by backend endpoints.
|
|
34
|
+
*/
|
|
35
|
+
export type Intent = {
|
|
36
|
+
/** Discriminant identifier matching strategy configuration registry keys. */
|
|
37
|
+
strategy: 'post'
|
|
38
|
+
/** Unique database identifier for the file resource. */
|
|
39
|
+
fileId: string
|
|
40
|
+
/** Presigned POST action destination URL (e.g. presigned S3/GCS bucket location). */
|
|
41
|
+
url: string
|
|
42
|
+
/** Key/value pair list representing form properties (e.g. AWS credential fields, policies). */
|
|
43
|
+
fields: Record<string, string>
|
|
44
|
+
/** Optional epoch string indicating when the pre-signed credentials lapse. */
|
|
45
|
+
expiresAt?: string
|
|
46
|
+
}
|
|
5
47
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
fileId: string
|
|
11
|
-
/** Presigned POST URL (form action). */
|
|
12
|
-
url: string
|
|
13
|
-
/** Form fields to include before the file. */
|
|
14
|
-
fields: Record<string, string>
|
|
15
|
-
/** Optional expiry of the presigned URL. */
|
|
16
|
-
expiresAt?: string
|
|
48
|
+
/**
|
|
49
|
+
* Direct strategy cursor state representing that no resumable bytes exist.
|
|
50
|
+
*/
|
|
51
|
+
export type Cursor = Record<string, never>
|
|
17
52
|
}
|
|
18
53
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
54
|
+
/** Convenience alias for {@link PostStrategy.Intent}. */
|
|
55
|
+
export type PostIntent = PostStrategy.Intent
|
|
56
|
+
|
|
57
|
+
let warnedMissingAllowedHosts = false
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Reset the warn-once latch. Test-only.
|
|
61
|
+
* @internal
|
|
62
|
+
*/
|
|
63
|
+
export function __resetPostWarningsForTests(): void {
|
|
64
|
+
warnedMissingAllowedHosts = false
|
|
22
65
|
}
|
|
23
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Presigned HTTP POST form strategy.
|
|
69
|
+
*
|
|
70
|
+
* Sends the entire file along with backend policy fields as a single
|
|
71
|
+
* `multipart/form-data` request body. Does not support resumability.
|
|
72
|
+
*
|
|
73
|
+
* Security: the presigned URL is backend-supplied and flows straight to the
|
|
74
|
+
* transport, so a compromised backend or MITM could return a `file:`,
|
|
75
|
+
* `javascript:`, or private-network URL. Every URL is checked with
|
|
76
|
+
* {@link validateUploadUrl}; set {@link PostStrategy.Config.allowedHosts} to
|
|
77
|
+
* lock it to known hosts, or leave `allowPrivateHosts` at its default `false`
|
|
78
|
+
* to block loopback/private addresses.
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* ```ts
|
|
82
|
+
* const strategies = createStrategyRegistry([
|
|
83
|
+
* PostStrategy({ allowedHosts: ['uploads.example.com'] }),
|
|
84
|
+
* MultipartStrategy()
|
|
85
|
+
* ]);
|
|
86
|
+
* ```
|
|
87
|
+
* @author wildduck2 <https://github.com/wildduck2>
|
|
88
|
+
*/
|
|
24
89
|
export function PostStrategy<
|
|
25
|
-
M extends
|
|
26
|
-
C extends
|
|
27
|
-
P extends string
|
|
28
|
-
R extends
|
|
29
|
-
>():
|
|
90
|
+
M extends Contracts.Intent.Map,
|
|
91
|
+
C extends Contracts.Cursor.Map<M>,
|
|
92
|
+
P extends string,
|
|
93
|
+
R extends Contracts.Result.Base = Contracts.Result.Base,
|
|
94
|
+
>(opts: PostStrategy.Config = {}): Contracts.Strategy.Me<M, C, P, R, 'post'> {
|
|
95
|
+
const allowedHosts = opts.allowedHosts ?? []
|
|
96
|
+
const allowPrivateHosts = opts.allowPrivateHosts === true
|
|
97
|
+
|
|
30
98
|
return {
|
|
31
99
|
id: 'post',
|
|
32
100
|
resumable: false,
|
|
33
101
|
|
|
34
102
|
async start(ctx) {
|
|
35
|
-
const intent = ctx.intent
|
|
103
|
+
const intent = ctx.intent as unknown as PostStrategy.Intent
|
|
36
104
|
|
|
37
105
|
if (!intent.url) {
|
|
38
|
-
throw new
|
|
106
|
+
throw new UploadEngineError('validation_failed', { message: 'post.start: intent missing url' })
|
|
39
107
|
}
|
|
40
108
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
109
|
+
if (allowedHosts.length === 0 && !warnedMissingAllowedHosts) {
|
|
110
|
+
warnedMissingAllowedHosts = true
|
|
111
|
+
console.warn(
|
|
112
|
+
'[duck-upload] PostStrategy: no `allowedHosts` configured. Presigned POST URLs will be host-unrestricted. ' +
|
|
113
|
+
'Set PostStrategy.Config.allowedHosts to lock the upload host.',
|
|
114
|
+
)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
validateUploadUrl(intent.url, 'post.intent', { allowedHosts, allowPrivateHosts })
|
|
118
|
+
|
|
119
|
+
await withRetry(
|
|
120
|
+
() =>
|
|
121
|
+
ctx.transport.postForm({
|
|
122
|
+
url: intent.url,
|
|
123
|
+
file: ctx.file,
|
|
124
|
+
fields: intent.fields,
|
|
125
|
+
filename: ctx.file.name,
|
|
126
|
+
signal: ctx.signal,
|
|
127
|
+
onProgress(uploadedBytes, totalBytes) {
|
|
128
|
+
ctx.reportProgress({ uploadedBytes, totalBytes })
|
|
129
|
+
},
|
|
130
|
+
}),
|
|
131
|
+
{ signal: ctx.signal, ...(opts.maxRetries !== undefined ? { maxRetries: opts.maxRetries } : {}) },
|
|
132
|
+
)
|
|
51
133
|
},
|
|
52
134
|
}
|
|
53
135
|
}
|