@uploadista/core 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +231 -0
- package/.turbo/turbo-format.log +5 -0
- package/LICENSE +21 -0
- package/README.md +1120 -0
- package/dist/chunk-CUT6urMc.cjs +1 -0
- package/dist/debounce-C2SeqcxD.js +2 -0
- package/dist/debounce-C2SeqcxD.js.map +1 -0
- package/dist/debounce-LZK7yS7Z.cjs +1 -0
- package/dist/errors/index.cjs +1 -0
- package/dist/errors/index.d.cts +3 -0
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.d.ts.map +1 -0
- package/dist/errors/index.js +2 -0
- package/dist/errors/uploadista-error.d.ts +209 -0
- package/dist/errors/uploadista-error.d.ts.map +1 -0
- package/dist/errors/uploadista-error.js +322 -0
- package/dist/errors-8i_aMxOE.js +1 -0
- package/dist/errors-CRm1FHHT.cjs +0 -0
- package/dist/flow/edge.d.ts +47 -0
- package/dist/flow/edge.d.ts.map +1 -0
- package/dist/flow/edge.js +40 -0
- package/dist/flow/event.d.ts +206 -0
- package/dist/flow/event.d.ts.map +1 -0
- package/dist/flow/event.js +53 -0
- package/dist/flow/flow-server.d.ts +223 -0
- package/dist/flow/flow-server.d.ts.map +1 -0
- package/dist/flow/flow-server.js +614 -0
- package/dist/flow/flow.d.ts +238 -0
- package/dist/flow/flow.d.ts.map +1 -0
- package/dist/flow/flow.js +629 -0
- package/dist/flow/index.cjs +1 -0
- package/dist/flow/index.d.cts +6 -0
- package/dist/flow/index.d.ts +24 -0
- package/dist/flow/index.d.ts.map +1 -0
- package/dist/flow/index.js +24 -0
- package/dist/flow/node.d.ts +136 -0
- package/dist/flow/node.d.ts.map +1 -0
- package/dist/flow/node.js +153 -0
- package/dist/flow/nodes/index.d.ts +8 -0
- package/dist/flow/nodes/index.d.ts.map +1 -0
- package/dist/flow/nodes/index.js +7 -0
- package/dist/flow/nodes/input-node.d.ts +78 -0
- package/dist/flow/nodes/input-node.d.ts.map +1 -0
- package/dist/flow/nodes/input-node.js +233 -0
- package/dist/flow/nodes/storage-node.d.ts +67 -0
- package/dist/flow/nodes/storage-node.d.ts.map +1 -0
- package/dist/flow/nodes/storage-node.js +94 -0
- package/dist/flow/nodes/streaming-input-node.d.ts +69 -0
- package/dist/flow/nodes/streaming-input-node.d.ts.map +1 -0
- package/dist/flow/nodes/streaming-input-node.js +156 -0
- package/dist/flow/nodes/transform-node.d.ts +85 -0
- package/dist/flow/nodes/transform-node.d.ts.map +1 -0
- package/dist/flow/nodes/transform-node.js +107 -0
- package/dist/flow/parallel-scheduler.d.ts +175 -0
- package/dist/flow/parallel-scheduler.d.ts.map +1 -0
- package/dist/flow/parallel-scheduler.js +193 -0
- package/dist/flow/plugins/credential-provider.d.ts +47 -0
- package/dist/flow/plugins/credential-provider.d.ts.map +1 -0
- package/dist/flow/plugins/credential-provider.js +24 -0
- package/dist/flow/plugins/image-ai-plugin.d.ts +61 -0
- package/dist/flow/plugins/image-ai-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/image-ai-plugin.js +21 -0
- package/dist/flow/plugins/image-plugin.d.ts +52 -0
- package/dist/flow/plugins/image-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/image-plugin.js +22 -0
- package/dist/flow/plugins/types/describe-image-node.d.ts +16 -0
- package/dist/flow/plugins/types/describe-image-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/describe-image-node.js +9 -0
- package/dist/flow/plugins/types/index.d.ts +9 -0
- package/dist/flow/plugins/types/index.d.ts.map +1 -0
- package/dist/flow/plugins/types/index.js +8 -0
- package/dist/flow/plugins/types/optimize-node.d.ts +20 -0
- package/dist/flow/plugins/types/optimize-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/optimize-node.js +11 -0
- package/dist/flow/plugins/types/remove-background-node.d.ts +16 -0
- package/dist/flow/plugins/types/remove-background-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/remove-background-node.js +9 -0
- package/dist/flow/plugins/types/resize-node.d.ts +21 -0
- package/dist/flow/plugins/types/resize-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/resize-node.js +16 -0
- package/dist/flow/plugins/zip-plugin.d.ts +62 -0
- package/dist/flow/plugins/zip-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/zip-plugin.js +21 -0
- package/dist/flow/typed-flow.d.ts +90 -0
- package/dist/flow/typed-flow.d.ts.map +1 -0
- package/dist/flow/typed-flow.js +59 -0
- package/dist/flow/types/flow-file.d.ts +45 -0
- package/dist/flow/types/flow-file.d.ts.map +1 -0
- package/dist/flow/types/flow-file.js +27 -0
- package/dist/flow/types/flow-job.d.ts +118 -0
- package/dist/flow/types/flow-job.d.ts.map +1 -0
- package/dist/flow/types/flow-job.js +11 -0
- package/dist/flow/types/flow-types.d.ts +321 -0
- package/dist/flow/types/flow-types.d.ts.map +1 -0
- package/dist/flow/types/flow-types.js +52 -0
- package/dist/flow/types/index.d.ts +4 -0
- package/dist/flow/types/index.d.ts.map +1 -0
- package/dist/flow/types/index.js +3 -0
- package/dist/flow/types/run-args.d.ts +38 -0
- package/dist/flow/types/run-args.d.ts.map +1 -0
- package/dist/flow/types/run-args.js +30 -0
- package/dist/flow/types/type-validator.d.ts +26 -0
- package/dist/flow/types/type-validator.d.ts.map +1 -0
- package/dist/flow/types/type-validator.js +134 -0
- package/dist/flow/utils/resolve-upload-metadata.d.ts +11 -0
- package/dist/flow/utils/resolve-upload-metadata.d.ts.map +1 -0
- package/dist/flow/utils/resolve-upload-metadata.js +28 -0
- package/dist/flow-2zXnEiWL.cjs +1 -0
- package/dist/flow-CRaKy7Vj.js +2 -0
- package/dist/flow-CRaKy7Vj.js.map +1 -0
- package/dist/generate-id-Dm-Vboxq.d.ts +34 -0
- package/dist/generate-id-Dm-Vboxq.d.ts.map +1 -0
- package/dist/generate-id-LjJRLD6N.d.cts +34 -0
- package/dist/generate-id-LjJRLD6N.d.cts.map +1 -0
- package/dist/generate-id-xHp_Z7Cl.cjs +1 -0
- package/dist/generate-id-yohS1ZDk.js +2 -0
- package/dist/generate-id-yohS1ZDk.js.map +1 -0
- package/dist/index-BO8GZlbD.d.cts +1040 -0
- package/dist/index-BO8GZlbD.d.cts.map +1 -0
- package/dist/index-BoGG5KAY.d.ts +1 -0
- package/dist/index-BtBZHVmz.d.cts +1 -0
- package/dist/index-D-CoVpkZ.d.ts +1004 -0
- package/dist/index-D-CoVpkZ.d.ts.map +1 -0
- package/dist/index.cjs +1 -0
- package/dist/index.d.cts +6 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/logger/logger.cjs +1 -0
- package/dist/logger/logger.d.cts +8 -0
- package/dist/logger/logger.d.cts.map +1 -0
- package/dist/logger/logger.d.ts +5 -0
- package/dist/logger/logger.d.ts.map +1 -0
- package/dist/logger/logger.js +10 -0
- package/dist/logger/logger.js.map +1 -0
- package/dist/semaphore-0ZwjVpyF.js +2 -0
- package/dist/semaphore-0ZwjVpyF.js.map +1 -0
- package/dist/semaphore-BHprIjFI.d.cts +37 -0
- package/dist/semaphore-BHprIjFI.d.cts.map +1 -0
- package/dist/semaphore-DThupBkc.d.ts +37 -0
- package/dist/semaphore-DThupBkc.d.ts.map +1 -0
- package/dist/semaphore-DVrONiAV.cjs +1 -0
- package/dist/stream-limiter-CoWKv39w.js +2 -0
- package/dist/stream-limiter-CoWKv39w.js.map +1 -0
- package/dist/stream-limiter-JgOwmkMa.cjs +1 -0
- package/dist/streams/multi-stream.cjs +1 -0
- package/dist/streams/multi-stream.d.cts +91 -0
- package/dist/streams/multi-stream.d.cts.map +1 -0
- package/dist/streams/multi-stream.d.ts +86 -0
- package/dist/streams/multi-stream.d.ts.map +1 -0
- package/dist/streams/multi-stream.js +149 -0
- package/dist/streams/multi-stream.js.map +1 -0
- package/dist/streams/stream-limiter.cjs +1 -0
- package/dist/streams/stream-limiter.d.cts +36 -0
- package/dist/streams/stream-limiter.d.cts.map +1 -0
- package/dist/streams/stream-limiter.d.ts +27 -0
- package/dist/streams/stream-limiter.d.ts.map +1 -0
- package/dist/streams/stream-limiter.js +49 -0
- package/dist/streams/stream-splitter.cjs +1 -0
- package/dist/streams/stream-splitter.d.cts +68 -0
- package/dist/streams/stream-splitter.d.cts.map +1 -0
- package/dist/streams/stream-splitter.d.ts +51 -0
- package/dist/streams/stream-splitter.d.ts.map +1 -0
- package/dist/streams/stream-splitter.js +175 -0
- package/dist/streams/stream-splitter.js.map +1 -0
- package/dist/types/data-store-registry.d.ts +13 -0
- package/dist/types/data-store-registry.d.ts.map +1 -0
- package/dist/types/data-store-registry.js +4 -0
- package/dist/types/data-store.d.ts +316 -0
- package/dist/types/data-store.d.ts.map +1 -0
- package/dist/types/data-store.js +157 -0
- package/dist/types/event-broadcaster.d.ts +28 -0
- package/dist/types/event-broadcaster.d.ts.map +1 -0
- package/dist/types/event-broadcaster.js +6 -0
- package/dist/types/event-emitter.d.ts +378 -0
- package/dist/types/event-emitter.d.ts.map +1 -0
- package/dist/types/event-emitter.js +223 -0
- package/dist/types/index.cjs +1 -0
- package/dist/types/index.d.cts +6 -0
- package/dist/types/index.d.ts +10 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +9 -0
- package/dist/types/input-file.d.ts +104 -0
- package/dist/types/input-file.d.ts.map +1 -0
- package/dist/types/input-file.js +27 -0
- package/dist/types/kv-store.d.ts +281 -0
- package/dist/types/kv-store.d.ts.map +1 -0
- package/dist/types/kv-store.js +234 -0
- package/dist/types/middleware.d.ts +17 -0
- package/dist/types/middleware.d.ts.map +1 -0
- package/dist/types/middleware.js +21 -0
- package/dist/types/upload-event.d.ts +105 -0
- package/dist/types/upload-event.d.ts.map +1 -0
- package/dist/types/upload-event.js +71 -0
- package/dist/types/upload-file.d.ts +136 -0
- package/dist/types/upload-file.d.ts.map +1 -0
- package/dist/types/upload-file.js +34 -0
- package/dist/types/websocket.d.ts +144 -0
- package/dist/types/websocket.d.ts.map +1 -0
- package/dist/types/websocket.js +40 -0
- package/dist/types-BT-cvi7T.cjs +1 -0
- package/dist/types-DhU2j-XF.js +2 -0
- package/dist/types-DhU2j-XF.js.map +1 -0
- package/dist/upload/convert-to-stream.d.ts +38 -0
- package/dist/upload/convert-to-stream.d.ts.map +1 -0
- package/dist/upload/convert-to-stream.js +43 -0
- package/dist/upload/convert-upload-to-flow-file.d.ts +14 -0
- package/dist/upload/convert-upload-to-flow-file.d.ts.map +1 -0
- package/dist/upload/convert-upload-to-flow-file.js +21 -0
- package/dist/upload/create-upload.d.ts +68 -0
- package/dist/upload/create-upload.d.ts.map +1 -0
- package/dist/upload/create-upload.js +157 -0
- package/dist/upload/index.cjs +1 -0
- package/dist/upload/index.d.cts +6 -0
- package/dist/upload/index.d.ts +4 -0
- package/dist/upload/index.d.ts.map +1 -0
- package/dist/upload/index.js +3 -0
- package/dist/upload/mime.d.ts +24 -0
- package/dist/upload/mime.d.ts.map +1 -0
- package/dist/upload/mime.js +351 -0
- package/dist/upload/upload-chunk.d.ts +58 -0
- package/dist/upload/upload-chunk.d.ts.map +1 -0
- package/dist/upload/upload-chunk.js +277 -0
- package/dist/upload/upload-server.d.ts +221 -0
- package/dist/upload/upload-server.d.ts.map +1 -0
- package/dist/upload/upload-server.js +181 -0
- package/dist/upload/upload-strategy-negotiator.d.ts +148 -0
- package/dist/upload/upload-strategy-negotiator.d.ts.map +1 -0
- package/dist/upload/upload-strategy-negotiator.js +217 -0
- package/dist/upload/upload-url.d.ts +68 -0
- package/dist/upload/upload-url.d.ts.map +1 -0
- package/dist/upload/upload-url.js +142 -0
- package/dist/upload/write-to-store.d.ts +77 -0
- package/dist/upload/write-to-store.d.ts.map +1 -0
- package/dist/upload/write-to-store.js +147 -0
- package/dist/upload-DLuICjpP.cjs +1 -0
- package/dist/upload-DaXO34dE.js +2 -0
- package/dist/upload-DaXO34dE.js.map +1 -0
- package/dist/uploadista-error-BB-Wdiz9.cjs +22 -0
- package/dist/uploadista-error-BVsVxqvz.js +23 -0
- package/dist/uploadista-error-BVsVxqvz.js.map +1 -0
- package/dist/uploadista-error-CwxYs4EB.d.ts +52 -0
- package/dist/uploadista-error-CwxYs4EB.d.ts.map +1 -0
- package/dist/uploadista-error-kKlhLRhY.d.cts +52 -0
- package/dist/uploadista-error-kKlhLRhY.d.cts.map +1 -0
- package/dist/utils/checksum.d.ts +22 -0
- package/dist/utils/checksum.d.ts.map +1 -0
- package/dist/utils/checksum.js +49 -0
- package/dist/utils/debounce.cjs +1 -0
- package/dist/utils/debounce.d.cts +38 -0
- package/dist/utils/debounce.d.cts.map +1 -0
- package/dist/utils/debounce.d.ts +36 -0
- package/dist/utils/debounce.d.ts.map +1 -0
- package/dist/utils/debounce.js +73 -0
- package/dist/utils/generate-id.cjs +1 -0
- package/dist/utils/generate-id.d.cts +2 -0
- package/dist/utils/generate-id.d.ts +32 -0
- package/dist/utils/generate-id.d.ts.map +1 -0
- package/dist/utils/generate-id.js +23 -0
- package/dist/utils/md5.cjs +1 -0
- package/dist/utils/md5.d.cts +73 -0
- package/dist/utils/md5.d.cts.map +1 -0
- package/dist/utils/md5.d.ts +71 -0
- package/dist/utils/md5.d.ts.map +1 -0
- package/dist/utils/md5.js +417 -0
- package/dist/utils/md5.js.map +1 -0
- package/dist/utils/once.cjs +1 -0
- package/dist/utils/once.d.cts +25 -0
- package/dist/utils/once.d.cts.map +1 -0
- package/dist/utils/once.d.ts +21 -0
- package/dist/utils/once.d.ts.map +1 -0
- package/dist/utils/once.js +54 -0
- package/dist/utils/once.js.map +1 -0
- package/dist/utils/semaphore.cjs +1 -0
- package/dist/utils/semaphore.d.cts +3 -0
- package/dist/utils/semaphore.d.ts +78 -0
- package/dist/utils/semaphore.d.ts.map +1 -0
- package/dist/utils/semaphore.js +134 -0
- package/dist/utils/throttle.cjs +1 -0
- package/dist/utils/throttle.d.cts +24 -0
- package/dist/utils/throttle.d.cts.map +1 -0
- package/dist/utils/throttle.d.ts +18 -0
- package/dist/utils/throttle.d.ts.map +1 -0
- package/dist/utils/throttle.js +20 -0
- package/dist/utils/throttle.js.map +1 -0
- package/docs/PARALLEL_EXECUTION.md +206 -0
- package/docs/PARALLEL_EXECUTION_QUICKSTART.md +142 -0
- package/docs/PARALLEL_EXECUTION_REFACTOR.md +184 -0
- package/package.json +80 -0
- package/src/errors/__tests__/uploadista-error.test.ts +251 -0
- package/src/errors/index.ts +2 -0
- package/src/errors/uploadista-error.ts +394 -0
- package/src/flow/README.md +352 -0
- package/src/flow/edge.test.ts +146 -0
- package/src/flow/edge.ts +60 -0
- package/src/flow/event.ts +229 -0
- package/src/flow/flow-server.ts +1089 -0
- package/src/flow/flow.ts +1050 -0
- package/src/flow/index.ts +28 -0
- package/src/flow/node.ts +249 -0
- package/src/flow/nodes/index.ts +8 -0
- package/src/flow/nodes/input-node.ts +296 -0
- package/src/flow/nodes/storage-node.ts +128 -0
- package/src/flow/nodes/transform-node.ts +154 -0
- package/src/flow/parallel-scheduler.ts +259 -0
- package/src/flow/plugins/credential-provider.ts +48 -0
- package/src/flow/plugins/image-ai-plugin.ts +66 -0
- package/src/flow/plugins/image-plugin.ts +60 -0
- package/src/flow/plugins/types/describe-image-node.ts +16 -0
- package/src/flow/plugins/types/index.ts +9 -0
- package/src/flow/plugins/types/optimize-node.ts +18 -0
- package/src/flow/plugins/types/remove-background-node.ts +18 -0
- package/src/flow/plugins/types/resize-node.ts +26 -0
- package/src/flow/plugins/zip-plugin.ts +69 -0
- package/src/flow/typed-flow.ts +279 -0
- package/src/flow/types/flow-file.ts +51 -0
- package/src/flow/types/flow-job.ts +138 -0
- package/src/flow/types/flow-types.ts +353 -0
- package/src/flow/types/index.ts +6 -0
- package/src/flow/types/run-args.ts +40 -0
- package/src/flow/types/type-validator.ts +204 -0
- package/src/flow/utils/resolve-upload-metadata.ts +48 -0
- package/src/index.ts +5 -0
- package/src/logger/logger.ts +14 -0
- package/src/streams/stream-limiter.test.ts +150 -0
- package/src/streams/stream-limiter.ts +75 -0
- package/src/types/data-store.ts +427 -0
- package/src/types/event-broadcaster.ts +39 -0
- package/src/types/event-emitter.ts +349 -0
- package/src/types/index.ts +9 -0
- package/src/types/input-file.ts +107 -0
- package/src/types/kv-store.ts +375 -0
- package/src/types/middleware.ts +54 -0
- package/src/types/upload-event.ts +75 -0
- package/src/types/upload-file.ts +139 -0
- package/src/types/websocket.ts +65 -0
- package/src/upload/convert-to-stream.ts +48 -0
- package/src/upload/create-upload.ts +214 -0
- package/src/upload/index.ts +3 -0
- package/src/upload/mime.ts +436 -0
- package/src/upload/upload-chunk.ts +364 -0
- package/src/upload/upload-server.ts +390 -0
- package/src/upload/upload-strategy-negotiator.ts +316 -0
- package/src/upload/upload-url.ts +173 -0
- package/src/upload/write-to-store.ts +211 -0
- package/src/utils/checksum.ts +61 -0
- package/src/utils/debounce.test.ts +126 -0
- package/src/utils/debounce.ts +89 -0
- package/src/utils/generate-id.ts +35 -0
- package/src/utils/md5.ts +475 -0
- package/src/utils/once.test.ts +83 -0
- package/src/utils/once.ts +63 -0
- package/src/utils/throttle.test.ts +101 -0
- package/src/utils/throttle.ts +29 -0
- package/tsconfig.json +20 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/tsdown.config.ts +25 -0
- package/vitest.config.ts +15 -0
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helper to check if buffer matches a byte pattern at given offset
|
|
3
|
+
*/
|
|
4
|
+
function checkBytes(buffer, pattern, offset = 0) {
|
|
5
|
+
if (buffer.length < offset + pattern.length)
|
|
6
|
+
return false;
|
|
7
|
+
return pattern.every((byte, i) => buffer[offset + i] === byte);
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Helper to check if buffer matches a string pattern at given offset
|
|
11
|
+
*/
|
|
12
|
+
function checkString(buffer, str, offset = 0) {
|
|
13
|
+
if (buffer.length < offset + str.length)
|
|
14
|
+
return false;
|
|
15
|
+
for (let i = 0; i < str.length; i++) {
|
|
16
|
+
if (buffer[offset + i] !== str.charCodeAt(i))
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Detect MIME type from buffer using magic bytes (file signatures).
|
|
23
|
+
* Supports a wide range of common file types including images, videos, audio, documents, and archives.
|
|
24
|
+
*
|
|
25
|
+
* @param buffer - File content as Uint8Array
|
|
26
|
+
* @param filename - Optional filename for extension-based fallback
|
|
27
|
+
* @returns Detected MIME type or "application/octet-stream" if unknown
|
|
28
|
+
*/
|
|
29
|
+
export const detectMimeType = (buffer, filename) => {
|
|
30
|
+
if (buffer.length === 0) {
|
|
31
|
+
return "application/octet-stream";
|
|
32
|
+
}
|
|
33
|
+
// ===== IMAGES =====
|
|
34
|
+
// PNG: 89 50 4E 47 0D 0A 1A 0A
|
|
35
|
+
if (checkBytes(buffer, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) {
|
|
36
|
+
return "image/png";
|
|
37
|
+
}
|
|
38
|
+
// JPEG: FF D8 FF
|
|
39
|
+
if (checkBytes(buffer, [0xff, 0xd8, 0xff])) {
|
|
40
|
+
return "image/jpeg";
|
|
41
|
+
}
|
|
42
|
+
// GIF87a or GIF89a
|
|
43
|
+
if (checkString(buffer, "GIF87a") || checkString(buffer, "GIF89a")) {
|
|
44
|
+
return "image/gif";
|
|
45
|
+
}
|
|
46
|
+
// WebP: RIFF....WEBP
|
|
47
|
+
if (checkBytes(buffer, [0x52, 0x49, 0x46, 0x46]) &&
|
|
48
|
+
buffer.length >= 12 &&
|
|
49
|
+
checkString(buffer, "WEBP", 8)) {
|
|
50
|
+
return "image/webp";
|
|
51
|
+
}
|
|
52
|
+
// AVIF: ....ftypavif or ....ftypavis
|
|
53
|
+
if (buffer.length >= 12 &&
|
|
54
|
+
checkBytes(buffer, [0x00, 0x00, 0x00], 0) &&
|
|
55
|
+
checkString(buffer, "ftyp", 4) &&
|
|
56
|
+
(checkString(buffer, "avif", 8) || checkString(buffer, "avis", 8))) {
|
|
57
|
+
return "image/avif";
|
|
58
|
+
}
|
|
59
|
+
// HEIC/HEIF: ....ftypheic or ....ftypheif or ....ftypmif1
|
|
60
|
+
if (buffer.length >= 12 &&
|
|
61
|
+
checkString(buffer, "ftyp", 4) &&
|
|
62
|
+
(checkString(buffer, "heic", 8) ||
|
|
63
|
+
checkString(buffer, "heif", 8) ||
|
|
64
|
+
checkString(buffer, "mif1", 8))) {
|
|
65
|
+
return "image/heic";
|
|
66
|
+
}
|
|
67
|
+
// BMP: 42 4D
|
|
68
|
+
if (checkBytes(buffer, [0x42, 0x4d])) {
|
|
69
|
+
return "image/bmp";
|
|
70
|
+
}
|
|
71
|
+
// TIFF (little-endian): 49 49 2A 00
|
|
72
|
+
if (checkBytes(buffer, [0x49, 0x49, 0x2a, 0x00])) {
|
|
73
|
+
return "image/tiff";
|
|
74
|
+
}
|
|
75
|
+
// TIFF (big-endian): 4D 4D 00 2A
|
|
76
|
+
if (checkBytes(buffer, [0x4d, 0x4d, 0x00, 0x2a])) {
|
|
77
|
+
return "image/tiff";
|
|
78
|
+
}
|
|
79
|
+
// ICO: 00 00 01 00
|
|
80
|
+
if (checkBytes(buffer, [0x00, 0x00, 0x01, 0x00])) {
|
|
81
|
+
return "image/x-icon";
|
|
82
|
+
}
|
|
83
|
+
// SVG (XML-based, check for <svg or <?xml)
|
|
84
|
+
if (buffer.length >= 5) {
|
|
85
|
+
const text = new TextDecoder("utf-8", { fatal: false }).decode(buffer.slice(0, Math.min(1024, buffer.length)));
|
|
86
|
+
if (text.includes("<svg") || (text.includes("<?xml") && text.includes("<svg"))) {
|
|
87
|
+
return "image/svg+xml";
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// ===== VIDEOS =====
|
|
91
|
+
// MP4/M4V/M4A: ....ftyp
|
|
92
|
+
if (buffer.length >= 12 && checkString(buffer, "ftyp", 4)) {
|
|
93
|
+
const subtype = new TextDecoder().decode(buffer.slice(8, 12));
|
|
94
|
+
if (subtype.startsWith("mp4") ||
|
|
95
|
+
subtype.startsWith("M4") ||
|
|
96
|
+
subtype.startsWith("isom")) {
|
|
97
|
+
return "video/mp4";
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
// WebM: 1A 45 DF A3
|
|
101
|
+
if (checkBytes(buffer, [0x1a, 0x45, 0xdf, 0xa3])) {
|
|
102
|
+
return "video/webm";
|
|
103
|
+
}
|
|
104
|
+
// AVI: RIFF....AVI
|
|
105
|
+
if (checkBytes(buffer, [0x52, 0x49, 0x46, 0x46]) &&
|
|
106
|
+
buffer.length >= 12 &&
|
|
107
|
+
checkString(buffer, "AVI ", 8)) {
|
|
108
|
+
return "video/x-msvideo";
|
|
109
|
+
}
|
|
110
|
+
// MOV (QuickTime): ....moov or ....mdat or ....free
|
|
111
|
+
if (buffer.length >= 8 &&
|
|
112
|
+
(checkString(buffer, "moov", 4) ||
|
|
113
|
+
checkString(buffer, "mdat", 4) ||
|
|
114
|
+
checkString(buffer, "free", 4))) {
|
|
115
|
+
return "video/quicktime";
|
|
116
|
+
}
|
|
117
|
+
// MKV: 1A 45 DF A3 (same as WebM but check for Matroska)
|
|
118
|
+
if (checkBytes(buffer, [0x1a, 0x45, 0xdf, 0xa3]) &&
|
|
119
|
+
buffer.length >= 100) {
|
|
120
|
+
const text = new TextDecoder("utf-8", { fatal: false }).decode(buffer.slice(0, 100));
|
|
121
|
+
if (text.includes("matroska")) {
|
|
122
|
+
return "video/x-matroska";
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// ===== AUDIO =====
|
|
126
|
+
// MP3: FF FB or FF F3 or FF F2 or ID3
|
|
127
|
+
if (checkBytes(buffer, [0xff, 0xfb]) ||
|
|
128
|
+
checkBytes(buffer, [0xff, 0xf3]) ||
|
|
129
|
+
checkBytes(buffer, [0xff, 0xf2]) ||
|
|
130
|
+
checkString(buffer, "ID3")) {
|
|
131
|
+
return "audio/mpeg";
|
|
132
|
+
}
|
|
133
|
+
// WAV: RIFF....WAVE
|
|
134
|
+
if (checkBytes(buffer, [0x52, 0x49, 0x46, 0x46]) &&
|
|
135
|
+
buffer.length >= 12 &&
|
|
136
|
+
checkString(buffer, "WAVE", 8)) {
|
|
137
|
+
return "audio/wav";
|
|
138
|
+
}
|
|
139
|
+
// FLAC: 66 4C 61 43 (fLaC)
|
|
140
|
+
if (checkString(buffer, "fLaC")) {
|
|
141
|
+
return "audio/flac";
|
|
142
|
+
}
|
|
143
|
+
// OGG: 4F 67 67 53 (OggS)
|
|
144
|
+
if (checkString(buffer, "OggS")) {
|
|
145
|
+
return "audio/ogg";
|
|
146
|
+
}
|
|
147
|
+
// M4A: ....ftypM4A
|
|
148
|
+
if (buffer.length >= 12 &&
|
|
149
|
+
checkString(buffer, "ftyp", 4) &&
|
|
150
|
+
checkString(buffer, "M4A", 8)) {
|
|
151
|
+
return "audio/mp4";
|
|
152
|
+
}
|
|
153
|
+
// ===== DOCUMENTS =====
|
|
154
|
+
// PDF: 25 50 44 46 (%PDF)
|
|
155
|
+
if (checkString(buffer, "%PDF")) {
|
|
156
|
+
return "application/pdf";
|
|
157
|
+
}
|
|
158
|
+
// ===== ARCHIVES =====
|
|
159
|
+
// ZIP: 50 4B 03 04 or 50 4B 05 06 (empty archive) or 50 4B 07 08 (spanned archive)
|
|
160
|
+
if (checkBytes(buffer, [0x50, 0x4b, 0x03, 0x04]) ||
|
|
161
|
+
checkBytes(buffer, [0x50, 0x4b, 0x05, 0x06]) ||
|
|
162
|
+
checkBytes(buffer, [0x50, 0x4b, 0x07, 0x08])) {
|
|
163
|
+
// Could be ZIP, DOCX, XLSX, PPTX, JAR, APK, etc.
|
|
164
|
+
// Check for Office formats
|
|
165
|
+
if (buffer.length >= 1024) {
|
|
166
|
+
const text = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
|
|
167
|
+
if (text.includes("word/"))
|
|
168
|
+
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
169
|
+
if (text.includes("xl/"))
|
|
170
|
+
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
|
171
|
+
if (text.includes("ppt/"))
|
|
172
|
+
return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
|
|
173
|
+
}
|
|
174
|
+
return "application/zip";
|
|
175
|
+
}
|
|
176
|
+
// RAR: 52 61 72 21 1A 07 (Rar!)
|
|
177
|
+
if (checkBytes(buffer, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07])) {
|
|
178
|
+
return "application/x-rar-compressed";
|
|
179
|
+
}
|
|
180
|
+
// 7Z: 37 7A BC AF 27 1C
|
|
181
|
+
if (checkBytes(buffer, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])) {
|
|
182
|
+
return "application/x-7z-compressed";
|
|
183
|
+
}
|
|
184
|
+
// GZIP: 1F 8B
|
|
185
|
+
if (checkBytes(buffer, [0x1f, 0x8b])) {
|
|
186
|
+
return "application/gzip";
|
|
187
|
+
}
|
|
188
|
+
// TAR (ustar): "ustar" at offset 257
|
|
189
|
+
if (buffer.length >= 262 && checkString(buffer, "ustar", 257)) {
|
|
190
|
+
return "application/x-tar";
|
|
191
|
+
}
|
|
192
|
+
// ===== FONTS =====
|
|
193
|
+
// WOFF: 77 4F 46 46 (wOFF)
|
|
194
|
+
if (checkString(buffer, "wOFF")) {
|
|
195
|
+
return "font/woff";
|
|
196
|
+
}
|
|
197
|
+
// WOFF2: 77 4F 46 32 (wOF2)
|
|
198
|
+
if (checkString(buffer, "wOF2")) {
|
|
199
|
+
return "font/woff2";
|
|
200
|
+
}
|
|
201
|
+
// TTF: 00 01 00 00 00
|
|
202
|
+
if (checkBytes(buffer, [0x00, 0x01, 0x00, 0x00, 0x00])) {
|
|
203
|
+
return "font/ttf";
|
|
204
|
+
}
|
|
205
|
+
// OTF: 4F 54 54 4F (OTTO)
|
|
206
|
+
if (checkString(buffer, "OTTO")) {
|
|
207
|
+
return "font/otf";
|
|
208
|
+
}
|
|
209
|
+
// ===== TEXT =====
|
|
210
|
+
// JSON (basic check for { or [)
|
|
211
|
+
if (buffer.length >= 1) {
|
|
212
|
+
const firstByte = buffer[0];
|
|
213
|
+
if (firstByte === 0x7b || firstByte === 0x5b) {
|
|
214
|
+
// { or [
|
|
215
|
+
try {
|
|
216
|
+
const text = new TextDecoder("utf-8").decode(buffer.slice(0, Math.min(1024, buffer.length)));
|
|
217
|
+
JSON.parse(text.trim());
|
|
218
|
+
return "application/json";
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
// Not valid JSON
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
// Fallback to extension-based detection
|
|
226
|
+
if (filename) {
|
|
227
|
+
const ext = filename.split(".").pop()?.toLowerCase();
|
|
228
|
+
switch (ext) {
|
|
229
|
+
// Images
|
|
230
|
+
case "jpg":
|
|
231
|
+
case "jpeg":
|
|
232
|
+
return "image/jpeg";
|
|
233
|
+
case "png":
|
|
234
|
+
return "image/png";
|
|
235
|
+
case "gif":
|
|
236
|
+
return "image/gif";
|
|
237
|
+
case "webp":
|
|
238
|
+
return "image/webp";
|
|
239
|
+
case "avif":
|
|
240
|
+
return "image/avif";
|
|
241
|
+
case "heic":
|
|
242
|
+
case "heif":
|
|
243
|
+
return "image/heic";
|
|
244
|
+
case "bmp":
|
|
245
|
+
return "image/bmp";
|
|
246
|
+
case "tiff":
|
|
247
|
+
case "tif":
|
|
248
|
+
return "image/tiff";
|
|
249
|
+
case "ico":
|
|
250
|
+
return "image/x-icon";
|
|
251
|
+
case "svg":
|
|
252
|
+
return "image/svg+xml";
|
|
253
|
+
// Videos
|
|
254
|
+
case "mp4":
|
|
255
|
+
case "m4v":
|
|
256
|
+
return "video/mp4";
|
|
257
|
+
case "webm":
|
|
258
|
+
return "video/webm";
|
|
259
|
+
case "avi":
|
|
260
|
+
return "video/x-msvideo";
|
|
261
|
+
case "mov":
|
|
262
|
+
return "video/quicktime";
|
|
263
|
+
case "mkv":
|
|
264
|
+
return "video/x-matroska";
|
|
265
|
+
// Audio
|
|
266
|
+
case "mp3":
|
|
267
|
+
return "audio/mpeg";
|
|
268
|
+
case "wav":
|
|
269
|
+
return "audio/wav";
|
|
270
|
+
case "flac":
|
|
271
|
+
return "audio/flac";
|
|
272
|
+
case "ogg":
|
|
273
|
+
return "audio/ogg";
|
|
274
|
+
case "m4a":
|
|
275
|
+
return "audio/mp4";
|
|
276
|
+
// Documents
|
|
277
|
+
case "pdf":
|
|
278
|
+
return "application/pdf";
|
|
279
|
+
case "docx":
|
|
280
|
+
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
281
|
+
case "xlsx":
|
|
282
|
+
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
|
283
|
+
case "pptx":
|
|
284
|
+
return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
|
|
285
|
+
// Archives
|
|
286
|
+
case "zip":
|
|
287
|
+
return "application/zip";
|
|
288
|
+
case "rar":
|
|
289
|
+
return "application/x-rar-compressed";
|
|
290
|
+
case "7z":
|
|
291
|
+
return "application/x-7z-compressed";
|
|
292
|
+
case "gz":
|
|
293
|
+
case "gzip":
|
|
294
|
+
return "application/gzip";
|
|
295
|
+
case "tar":
|
|
296
|
+
return "application/x-tar";
|
|
297
|
+
// Fonts
|
|
298
|
+
case "woff":
|
|
299
|
+
return "font/woff";
|
|
300
|
+
case "woff2":
|
|
301
|
+
return "font/woff2";
|
|
302
|
+
case "ttf":
|
|
303
|
+
return "font/ttf";
|
|
304
|
+
case "otf":
|
|
305
|
+
return "font/otf";
|
|
306
|
+
// Text
|
|
307
|
+
case "txt":
|
|
308
|
+
return "text/plain";
|
|
309
|
+
case "json":
|
|
310
|
+
return "application/json";
|
|
311
|
+
case "xml":
|
|
312
|
+
return "application/xml";
|
|
313
|
+
case "html":
|
|
314
|
+
case "htm":
|
|
315
|
+
return "text/html";
|
|
316
|
+
case "css":
|
|
317
|
+
return "text/css";
|
|
318
|
+
case "js":
|
|
319
|
+
return "application/javascript";
|
|
320
|
+
case "csv":
|
|
321
|
+
return "text/csv";
|
|
322
|
+
default:
|
|
323
|
+
return "application/octet-stream";
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return "application/octet-stream";
|
|
327
|
+
};
|
|
328
|
+
/**
|
|
329
|
+
* Compare two MIME types with lenient matching.
|
|
330
|
+
* Matches on major type (e.g., "image/*") to allow for minor variations.
|
|
331
|
+
*
|
|
332
|
+
* @param declared - MIME type provided by client
|
|
333
|
+
* @param detected - MIME type detected from file content
|
|
334
|
+
* @returns true if MIME types are compatible
|
|
335
|
+
*
|
|
336
|
+
* @example
|
|
337
|
+
* compareMimeTypes("image/png", "image/apng") // true
|
|
338
|
+
* compareMimeTypes("image/jpeg", "image/png") // true (both images)
|
|
339
|
+
* compareMimeTypes("image/png", "application/pdf") // false
|
|
340
|
+
*/
|
|
341
|
+
export function compareMimeTypes(declared, detected) {
|
|
342
|
+
// Exact match
|
|
343
|
+
if (declared === detected) {
|
|
344
|
+
return true;
|
|
345
|
+
}
|
|
346
|
+
// Extract major types (e.g., "image" from "image/png")
|
|
347
|
+
const declaredMajor = declared.split("/")[0];
|
|
348
|
+
const detectedMajor = detected.split("/")[0];
|
|
349
|
+
// Compare major types for lenient matching
|
|
350
|
+
return declaredMajor === detectedMajor;
|
|
351
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { UploadistaError } from "../errors/uploadista-error";
|
|
3
|
+
import { type EventEmitter, type KvStore, type UploadEvent, type UploadFile, type UploadFileDataStoresShape } from "../types";
|
|
4
|
+
/**
|
|
5
|
+
* Uploads a chunk of data for an existing upload.
|
|
6
|
+
*
|
|
7
|
+
* This function handles the core chunk upload logic including:
|
|
8
|
+
* - Retrieving upload metadata from KV store
|
|
9
|
+
* - Routing to appropriate data store based on storage ID
|
|
10
|
+
* - Writing chunk data to storage with progress tracking
|
|
11
|
+
* - Updating upload offset and metadata
|
|
12
|
+
* - Emitting progress events
|
|
13
|
+
* - Validating upload completion (checksum, MIME type)
|
|
14
|
+
*
|
|
15
|
+
* The function includes comprehensive observability with:
|
|
16
|
+
* - Effect tracing spans for performance monitoring
|
|
17
|
+
* - Metrics tracking for chunk size, throughput, and success rates
|
|
18
|
+
* - Structured logging for debugging and monitoring
|
|
19
|
+
* - Error handling with proper UploadistaError types
|
|
20
|
+
*
|
|
21
|
+
* @param uploadId - Unique identifier for the upload
|
|
22
|
+
* @param clientId - Client identifier (null for anonymous uploads)
|
|
23
|
+
* @param chunk - ReadableStream containing the chunk data to upload
|
|
24
|
+
* @param dataStoreService - Service for routing to appropriate data stores
|
|
25
|
+
* @param kvStore - KV store for upload metadata persistence
|
|
26
|
+
* @param eventEmitter - Event emitter for progress and validation events
|
|
27
|
+
* @returns Effect that yields the updated UploadFile with new offset
|
|
28
|
+
*
|
|
29
|
+
* @example
|
|
30
|
+
* ```typescript
|
|
31
|
+
* // Upload a chunk for an existing upload
|
|
32
|
+
* const uploadChunkEffect = uploadChunk(
|
|
33
|
+
* "upload-123",
|
|
34
|
+
* "client-456",
|
|
35
|
+
* chunkStream,
|
|
36
|
+
* {
|
|
37
|
+
* dataStoreService,
|
|
38
|
+
* kvStore,
|
|
39
|
+
* eventEmitter
|
|
40
|
+
* }
|
|
41
|
+
* );
|
|
42
|
+
*
|
|
43
|
+
* // Run with dependencies
|
|
44
|
+
* const result = await Effect.runPromise(
|
|
45
|
+
* uploadChunkEffect.pipe(
|
|
46
|
+
* Effect.provide(dataStoreLayer),
|
|
47
|
+
* Effect.provide(kvStoreLayer),
|
|
48
|
+
* Effect.provide(eventEmitterLayer)
|
|
49
|
+
* )
|
|
50
|
+
* );
|
|
51
|
+
* ```
|
|
52
|
+
*/
|
|
53
|
+
export declare const uploadChunk: (uploadId: string, clientId: string | null, chunk: ReadableStream, { dataStoreService, kvStore, eventEmitter, }: {
|
|
54
|
+
dataStoreService: UploadFileDataStoresShape;
|
|
55
|
+
kvStore: KvStore<UploadFile>;
|
|
56
|
+
eventEmitter: EventEmitter<UploadEvent>;
|
|
57
|
+
}) => Effect.Effect<UploadFile, UploadistaError, never>;
|
|
58
|
+
//# sourceMappingURL=upload-chunk.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upload-chunk.d.ts","sourceRoot":"","sources":["../../src/upload/upload-chunk.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA4B,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,OAAO,EACZ,KAAK,WAAW,EAEhB,KAAK,UAAU,EACf,KAAK,yBAAyB,EAC/B,MAAM,UAAU,CAAC;AAKlB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,eAAO,MAAM,WAAW,GACtB,UAAU,MAAM,EAChB,UAAU,MAAM,GAAG,IAAI,EACvB,OAAO,cAAc,EACrB,8CAIG;IACD,gBAAgB,EAAE,yBAAyB,CAAC;IAC5C,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7B,YAAY,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;CACzC,sDAoHA,CAAC"}
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import { Effect, Metric, MetricBoundaries } from "effect";
|
|
2
|
+
import { UploadistaError } from "../errors/uploadista-error";
|
|
3
|
+
import { UploadEventType, } from "../types";
|
|
4
|
+
import { computeChecksum } from "../utils/checksum";
|
|
5
|
+
import { compareMimeTypes, detectMimeType } from "./mime";
|
|
6
|
+
import { writeToStore } from "./write-to-store";
|
|
7
|
+
/**
|
|
8
|
+
* Uploads a chunk of data for an existing upload.
|
|
9
|
+
*
|
|
10
|
+
* This function handles the core chunk upload logic including:
|
|
11
|
+
* - Retrieving upload metadata from KV store
|
|
12
|
+
* - Routing to appropriate data store based on storage ID
|
|
13
|
+
* - Writing chunk data to storage with progress tracking
|
|
14
|
+
* - Updating upload offset and metadata
|
|
15
|
+
* - Emitting progress events
|
|
16
|
+
* - Validating upload completion (checksum, MIME type)
|
|
17
|
+
*
|
|
18
|
+
* The function includes comprehensive observability with:
|
|
19
|
+
* - Effect tracing spans for performance monitoring
|
|
20
|
+
* - Metrics tracking for chunk size, throughput, and success rates
|
|
21
|
+
* - Structured logging for debugging and monitoring
|
|
22
|
+
* - Error handling with proper UploadistaError types
|
|
23
|
+
*
|
|
24
|
+
* @param uploadId - Unique identifier for the upload
|
|
25
|
+
* @param clientId - Client identifier (null for anonymous uploads)
|
|
26
|
+
* @param chunk - ReadableStream containing the chunk data to upload
|
|
27
|
+
* @param dataStoreService - Service for routing to appropriate data stores
|
|
28
|
+
* @param kvStore - KV store for upload metadata persistence
|
|
29
|
+
* @param eventEmitter - Event emitter for progress and validation events
|
|
30
|
+
* @returns Effect that yields the updated UploadFile with new offset
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```typescript
|
|
34
|
+
* // Upload a chunk for an existing upload
|
|
35
|
+
* const uploadChunkEffect = uploadChunk(
|
|
36
|
+
* "upload-123",
|
|
37
|
+
* "client-456",
|
|
38
|
+
* chunkStream,
|
|
39
|
+
* {
|
|
40
|
+
* dataStoreService,
|
|
41
|
+
* kvStore,
|
|
42
|
+
* eventEmitter
|
|
43
|
+
* }
|
|
44
|
+
* );
|
|
45
|
+
*
|
|
46
|
+
* // Run with dependencies
|
|
47
|
+
* const result = await Effect.runPromise(
|
|
48
|
+
* uploadChunkEffect.pipe(
|
|
49
|
+
* Effect.provide(dataStoreLayer),
|
|
50
|
+
* Effect.provide(kvStoreLayer),
|
|
51
|
+
* Effect.provide(eventEmitterLayer)
|
|
52
|
+
* )
|
|
53
|
+
* );
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
export const uploadChunk = (uploadId, clientId, chunk, { dataStoreService, kvStore, eventEmitter, }) => Effect.gen(function* () {
|
|
57
|
+
// Get file from KV store
|
|
58
|
+
const file = yield* kvStore.get(uploadId);
|
|
59
|
+
// Get datastore
|
|
60
|
+
const dataStore = yield* dataStoreService.getDataStore(file.storage.id, clientId);
|
|
61
|
+
// Note: AbortController could be used for cancellation if needed
|
|
62
|
+
// Write to store using writeToStore Effect
|
|
63
|
+
const controller = new AbortController();
|
|
64
|
+
const chunkSize = yield* writeToStore({
|
|
65
|
+
dataStore,
|
|
66
|
+
data: chunk,
|
|
67
|
+
upload: file,
|
|
68
|
+
maxFileSize: 100_000_000,
|
|
69
|
+
controller,
|
|
70
|
+
uploadProgressInterval: 200,
|
|
71
|
+
eventEmitter,
|
|
72
|
+
});
|
|
73
|
+
file.offset = chunkSize;
|
|
74
|
+
// Update KV store
|
|
75
|
+
yield* kvStore.set(uploadId, file);
|
|
76
|
+
// Emit progress event
|
|
77
|
+
yield* eventEmitter.emit(file.id, {
|
|
78
|
+
type: UploadEventType.UPLOAD_PROGRESS,
|
|
79
|
+
data: {
|
|
80
|
+
id: file.id,
|
|
81
|
+
progress: file.offset,
|
|
82
|
+
total: file.size ?? 0,
|
|
83
|
+
},
|
|
84
|
+
flow: file.flow,
|
|
85
|
+
});
|
|
86
|
+
// Check if upload is complete and run validation
|
|
87
|
+
if (file.size && file.offset === file.size) {
|
|
88
|
+
yield* validateUpload({
|
|
89
|
+
file,
|
|
90
|
+
dataStore,
|
|
91
|
+
eventEmitter,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
return file;
|
|
95
|
+
}).pipe(
|
|
96
|
+
// Add tracing span for chunk upload
|
|
97
|
+
Effect.withSpan("upload-chunk", {
|
|
98
|
+
attributes: {
|
|
99
|
+
"upload.id": uploadId,
|
|
100
|
+
"chunk.upload_id": uploadId,
|
|
101
|
+
},
|
|
102
|
+
}),
|
|
103
|
+
// Track chunk upload metrics
|
|
104
|
+
Effect.tap((file) => Effect.gen(function* () {
|
|
105
|
+
// Increment chunk uploaded counter
|
|
106
|
+
yield* Metric.increment(Metric.counter("chunk_uploaded_total", {
|
|
107
|
+
description: "Total number of chunks uploaded",
|
|
108
|
+
}));
|
|
109
|
+
// Record chunk size
|
|
110
|
+
const chunkSize = file.offset;
|
|
111
|
+
const chunkSizeHistogram = Metric.histogram("chunk_size_bytes", MetricBoundaries.linear({
|
|
112
|
+
start: 262_144,
|
|
113
|
+
width: 262_144,
|
|
114
|
+
count: 20,
|
|
115
|
+
}));
|
|
116
|
+
yield* Metric.update(chunkSizeHistogram, chunkSize);
|
|
117
|
+
// Update throughput gauge
|
|
118
|
+
if (file.size && file.size > 0) {
|
|
119
|
+
const throughput = chunkSize; // bytes processed
|
|
120
|
+
const throughputGauge = Metric.gauge("upload_throughput_bytes_per_second");
|
|
121
|
+
yield* Metric.set(throughputGauge, throughput);
|
|
122
|
+
}
|
|
123
|
+
})),
|
|
124
|
+
// Add structured logging for chunk progress
|
|
125
|
+
Effect.tap((file) => Effect.logDebug("Chunk uploaded").pipe(Effect.annotateLogs({
|
|
126
|
+
"upload.id": file.id,
|
|
127
|
+
"chunk.size": file.offset.toString(),
|
|
128
|
+
"chunk.progress": file.size && file.size > 0
|
|
129
|
+
? ((file.offset / file.size) * 100).toFixed(2)
|
|
130
|
+
: "0",
|
|
131
|
+
"upload.total_size": file.size?.toString() ?? "0",
|
|
132
|
+
}))),
|
|
133
|
+
// Handle errors with logging
|
|
134
|
+
Effect.tapError((error) => Effect.logError("Chunk upload failed").pipe(Effect.annotateLogs({
|
|
135
|
+
"upload.id": uploadId,
|
|
136
|
+
error: String(error),
|
|
137
|
+
}))));
|
|
138
|
+
/**
|
|
139
|
+
* Validates an upload after completion.
|
|
140
|
+
*
|
|
141
|
+
* Performs comprehensive validation including:
|
|
142
|
+
* - Checksum validation (if provided) using the specified algorithm
|
|
143
|
+
* - MIME type validation (if required by data store capabilities)
|
|
144
|
+
* - File size validation against data store limits
|
|
145
|
+
*
|
|
146
|
+
* Validation results are emitted as events and failures result in:
|
|
147
|
+
* - Cleanup of uploaded data from storage
|
|
148
|
+
* - Removal of metadata from KV store
|
|
149
|
+
* - Appropriate error responses
|
|
150
|
+
*
|
|
151
|
+
* The function respects data store capabilities for validation limits
|
|
152
|
+
* and provides detailed error information for debugging.
|
|
153
|
+
*
|
|
154
|
+
* @param file - The upload file to validate
|
|
155
|
+
* @param dataStore - Data store containing the uploaded file
|
|
156
|
+
* @param eventEmitter - Event emitter for validation events
|
|
157
|
+
* @returns Effect that completes validation or fails with UploadistaError
|
|
158
|
+
*
|
|
159
|
+
* @example
|
|
160
|
+
* ```typescript
|
|
161
|
+
* // Validate upload after completion
|
|
162
|
+
* const validationEffect = validateUpload({
|
|
163
|
+
* file: completedUpload,
|
|
164
|
+
* dataStore: s3DataStore,
|
|
165
|
+
* eventEmitter: progressEmitter
|
|
166
|
+
* });
|
|
167
|
+
*
|
|
168
|
+
* // Run validation
|
|
169
|
+
* await Effect.runPromise(validationEffect);
|
|
170
|
+
* ```
|
|
171
|
+
*/
|
|
172
|
+
const validateUpload = ({ file, dataStore, eventEmitter, }) => Effect.gen(function* () {
|
|
173
|
+
const capabilities = dataStore.getCapabilities();
|
|
174
|
+
// Check if file exceeds max validation size
|
|
175
|
+
if (capabilities.maxValidationSize &&
|
|
176
|
+
file.size &&
|
|
177
|
+
file.size > capabilities.maxValidationSize) {
|
|
178
|
+
yield* eventEmitter.emit(file.id, {
|
|
179
|
+
type: UploadEventType.UPLOAD_VALIDATION_WARNING,
|
|
180
|
+
data: {
|
|
181
|
+
id: file.id,
|
|
182
|
+
message: `File size (${file.size} bytes) exceeds max validation size (${capabilities.maxValidationSize} bytes). Validation skipped.`,
|
|
183
|
+
},
|
|
184
|
+
flow: file.flow,
|
|
185
|
+
});
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
// Read file from datastore for validation
|
|
189
|
+
const fileBytes = yield* dataStore.read(file.id);
|
|
190
|
+
// Validate checksum if provided
|
|
191
|
+
if (file.checksum && file.checksumAlgorithm) {
|
|
192
|
+
const computedChecksum = yield* computeChecksum(fileBytes, file.checksumAlgorithm);
|
|
193
|
+
if (computedChecksum !== file.checksum) {
|
|
194
|
+
// Emit validation failure event
|
|
195
|
+
yield* eventEmitter.emit(file.id, {
|
|
196
|
+
type: UploadEventType.UPLOAD_VALIDATION_FAILED,
|
|
197
|
+
data: {
|
|
198
|
+
id: file.id,
|
|
199
|
+
reason: "checksum_mismatch",
|
|
200
|
+
expected: file.checksum,
|
|
201
|
+
actual: computedChecksum,
|
|
202
|
+
},
|
|
203
|
+
flow: file.flow,
|
|
204
|
+
});
|
|
205
|
+
// Clean up file and remove from KV store
|
|
206
|
+
yield* dataStore.remove(file.id);
|
|
207
|
+
// Fail with checksum mismatch error
|
|
208
|
+
return yield* UploadistaError.fromCode("CHECKSUM_MISMATCH", {
|
|
209
|
+
body: `Checksum validation failed. Expected: ${file.checksum}, Got: ${computedChecksum}`,
|
|
210
|
+
details: {
|
|
211
|
+
uploadId: file.id,
|
|
212
|
+
expected: file.checksum,
|
|
213
|
+
actual: computedChecksum,
|
|
214
|
+
algorithm: file.checksumAlgorithm,
|
|
215
|
+
},
|
|
216
|
+
}).toEffect();
|
|
217
|
+
}
|
|
218
|
+
// Emit checksum validation success
|
|
219
|
+
yield* eventEmitter.emit(file.id, {
|
|
220
|
+
type: UploadEventType.UPLOAD_VALIDATION_SUCCESS,
|
|
221
|
+
data: {
|
|
222
|
+
id: file.id,
|
|
223
|
+
validationType: "checksum",
|
|
224
|
+
algorithm: file.checksumAlgorithm,
|
|
225
|
+
},
|
|
226
|
+
flow: file.flow,
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
// Validate MIME type if required by capabilities
|
|
230
|
+
if (capabilities.requiresMimeTypeValidation) {
|
|
231
|
+
const detectedMimeType = detectMimeType(fileBytes);
|
|
232
|
+
const declaredMimeType = file.metadata?.type;
|
|
233
|
+
if (declaredMimeType &&
|
|
234
|
+
!compareMimeTypes(declaredMimeType, detectedMimeType)) {
|
|
235
|
+
// Emit validation failure event
|
|
236
|
+
yield* eventEmitter.emit(file.id, {
|
|
237
|
+
type: UploadEventType.UPLOAD_VALIDATION_FAILED,
|
|
238
|
+
data: {
|
|
239
|
+
id: file.id,
|
|
240
|
+
reason: "mimetype_mismatch",
|
|
241
|
+
expected: declaredMimeType,
|
|
242
|
+
actual: detectedMimeType,
|
|
243
|
+
},
|
|
244
|
+
flow: file.flow,
|
|
245
|
+
});
|
|
246
|
+
// Clean up file and remove from KV store
|
|
247
|
+
yield* dataStore.remove(file.id);
|
|
248
|
+
// Fail with MIME type mismatch error
|
|
249
|
+
return yield* UploadistaError.fromCode("MIMETYPE_MISMATCH", {
|
|
250
|
+
body: `MIME type validation failed. Expected: ${declaredMimeType}, Detected: ${detectedMimeType}`,
|
|
251
|
+
details: {
|
|
252
|
+
uploadId: file.id,
|
|
253
|
+
expected: declaredMimeType,
|
|
254
|
+
actual: detectedMimeType,
|
|
255
|
+
},
|
|
256
|
+
}).toEffect();
|
|
257
|
+
}
|
|
258
|
+
// Emit MIME type validation success
|
|
259
|
+
yield* eventEmitter.emit(file.id, {
|
|
260
|
+
type: UploadEventType.UPLOAD_VALIDATION_SUCCESS,
|
|
261
|
+
data: {
|
|
262
|
+
id: file.id,
|
|
263
|
+
validationType: "mimetype",
|
|
264
|
+
},
|
|
265
|
+
flow: file.flow,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
}).pipe(Effect.withSpan("validate-upload", {
|
|
269
|
+
attributes: {
|
|
270
|
+
"upload.id": file.id,
|
|
271
|
+
"validation.checksum_provided": file.checksum ? "true" : "false",
|
|
272
|
+
"validation.mime_required": dataStore.getCapabilities()
|
|
273
|
+
.requiresMimeTypeValidation
|
|
274
|
+
? "true"
|
|
275
|
+
: "false",
|
|
276
|
+
},
|
|
277
|
+
}));
|