@uploadista/core 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +231 -0
- package/.turbo/turbo-format.log +5 -0
- package/LICENSE +21 -0
- package/README.md +1120 -0
- package/dist/chunk-CUT6urMc.cjs +1 -0
- package/dist/debounce-C2SeqcxD.js +2 -0
- package/dist/debounce-C2SeqcxD.js.map +1 -0
- package/dist/debounce-LZK7yS7Z.cjs +1 -0
- package/dist/errors/index.cjs +1 -0
- package/dist/errors/index.d.cts +3 -0
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.d.ts.map +1 -0
- package/dist/errors/index.js +2 -0
- package/dist/errors/uploadista-error.d.ts +209 -0
- package/dist/errors/uploadista-error.d.ts.map +1 -0
- package/dist/errors/uploadista-error.js +322 -0
- package/dist/errors-8i_aMxOE.js +1 -0
- package/dist/errors-CRm1FHHT.cjs +0 -0
- package/dist/flow/edge.d.ts +47 -0
- package/dist/flow/edge.d.ts.map +1 -0
- package/dist/flow/edge.js +40 -0
- package/dist/flow/event.d.ts +206 -0
- package/dist/flow/event.d.ts.map +1 -0
- package/dist/flow/event.js +53 -0
- package/dist/flow/flow-server.d.ts +223 -0
- package/dist/flow/flow-server.d.ts.map +1 -0
- package/dist/flow/flow-server.js +614 -0
- package/dist/flow/flow.d.ts +238 -0
- package/dist/flow/flow.d.ts.map +1 -0
- package/dist/flow/flow.js +629 -0
- package/dist/flow/index.cjs +1 -0
- package/dist/flow/index.d.cts +6 -0
- package/dist/flow/index.d.ts +24 -0
- package/dist/flow/index.d.ts.map +1 -0
- package/dist/flow/index.js +24 -0
- package/dist/flow/node.d.ts +136 -0
- package/dist/flow/node.d.ts.map +1 -0
- package/dist/flow/node.js +153 -0
- package/dist/flow/nodes/index.d.ts +8 -0
- package/dist/flow/nodes/index.d.ts.map +1 -0
- package/dist/flow/nodes/index.js +7 -0
- package/dist/flow/nodes/input-node.d.ts +78 -0
- package/dist/flow/nodes/input-node.d.ts.map +1 -0
- package/dist/flow/nodes/input-node.js +233 -0
- package/dist/flow/nodes/storage-node.d.ts +67 -0
- package/dist/flow/nodes/storage-node.d.ts.map +1 -0
- package/dist/flow/nodes/storage-node.js +94 -0
- package/dist/flow/nodes/streaming-input-node.d.ts +69 -0
- package/dist/flow/nodes/streaming-input-node.d.ts.map +1 -0
- package/dist/flow/nodes/streaming-input-node.js +156 -0
- package/dist/flow/nodes/transform-node.d.ts +85 -0
- package/dist/flow/nodes/transform-node.d.ts.map +1 -0
- package/dist/flow/nodes/transform-node.js +107 -0
- package/dist/flow/parallel-scheduler.d.ts +175 -0
- package/dist/flow/parallel-scheduler.d.ts.map +1 -0
- package/dist/flow/parallel-scheduler.js +193 -0
- package/dist/flow/plugins/credential-provider.d.ts +47 -0
- package/dist/flow/plugins/credential-provider.d.ts.map +1 -0
- package/dist/flow/plugins/credential-provider.js +24 -0
- package/dist/flow/plugins/image-ai-plugin.d.ts +61 -0
- package/dist/flow/plugins/image-ai-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/image-ai-plugin.js +21 -0
- package/dist/flow/plugins/image-plugin.d.ts +52 -0
- package/dist/flow/plugins/image-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/image-plugin.js +22 -0
- package/dist/flow/plugins/types/describe-image-node.d.ts +16 -0
- package/dist/flow/plugins/types/describe-image-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/describe-image-node.js +9 -0
- package/dist/flow/plugins/types/index.d.ts +9 -0
- package/dist/flow/plugins/types/index.d.ts.map +1 -0
- package/dist/flow/plugins/types/index.js +8 -0
- package/dist/flow/plugins/types/optimize-node.d.ts +20 -0
- package/dist/flow/plugins/types/optimize-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/optimize-node.js +11 -0
- package/dist/flow/plugins/types/remove-background-node.d.ts +16 -0
- package/dist/flow/plugins/types/remove-background-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/remove-background-node.js +9 -0
- package/dist/flow/plugins/types/resize-node.d.ts +21 -0
- package/dist/flow/plugins/types/resize-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/resize-node.js +16 -0
- package/dist/flow/plugins/zip-plugin.d.ts +62 -0
- package/dist/flow/plugins/zip-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/zip-plugin.js +21 -0
- package/dist/flow/typed-flow.d.ts +90 -0
- package/dist/flow/typed-flow.d.ts.map +1 -0
- package/dist/flow/typed-flow.js +59 -0
- package/dist/flow/types/flow-file.d.ts +45 -0
- package/dist/flow/types/flow-file.d.ts.map +1 -0
- package/dist/flow/types/flow-file.js +27 -0
- package/dist/flow/types/flow-job.d.ts +118 -0
- package/dist/flow/types/flow-job.d.ts.map +1 -0
- package/dist/flow/types/flow-job.js +11 -0
- package/dist/flow/types/flow-types.d.ts +321 -0
- package/dist/flow/types/flow-types.d.ts.map +1 -0
- package/dist/flow/types/flow-types.js +52 -0
- package/dist/flow/types/index.d.ts +4 -0
- package/dist/flow/types/index.d.ts.map +1 -0
- package/dist/flow/types/index.js +3 -0
- package/dist/flow/types/run-args.d.ts +38 -0
- package/dist/flow/types/run-args.d.ts.map +1 -0
- package/dist/flow/types/run-args.js +30 -0
- package/dist/flow/types/type-validator.d.ts +26 -0
- package/dist/flow/types/type-validator.d.ts.map +1 -0
- package/dist/flow/types/type-validator.js +134 -0
- package/dist/flow/utils/resolve-upload-metadata.d.ts +11 -0
- package/dist/flow/utils/resolve-upload-metadata.d.ts.map +1 -0
- package/dist/flow/utils/resolve-upload-metadata.js +28 -0
- package/dist/flow-2zXnEiWL.cjs +1 -0
- package/dist/flow-CRaKy7Vj.js +2 -0
- package/dist/flow-CRaKy7Vj.js.map +1 -0
- package/dist/generate-id-Dm-Vboxq.d.ts +34 -0
- package/dist/generate-id-Dm-Vboxq.d.ts.map +1 -0
- package/dist/generate-id-LjJRLD6N.d.cts +34 -0
- package/dist/generate-id-LjJRLD6N.d.cts.map +1 -0
- package/dist/generate-id-xHp_Z7Cl.cjs +1 -0
- package/dist/generate-id-yohS1ZDk.js +2 -0
- package/dist/generate-id-yohS1ZDk.js.map +1 -0
- package/dist/index-BO8GZlbD.d.cts +1040 -0
- package/dist/index-BO8GZlbD.d.cts.map +1 -0
- package/dist/index-BoGG5KAY.d.ts +1 -0
- package/dist/index-BtBZHVmz.d.cts +1 -0
- package/dist/index-D-CoVpkZ.d.ts +1004 -0
- package/dist/index-D-CoVpkZ.d.ts.map +1 -0
- package/dist/index.cjs +1 -0
- package/dist/index.d.cts +6 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/logger/logger.cjs +1 -0
- package/dist/logger/logger.d.cts +8 -0
- package/dist/logger/logger.d.cts.map +1 -0
- package/dist/logger/logger.d.ts +5 -0
- package/dist/logger/logger.d.ts.map +1 -0
- package/dist/logger/logger.js +10 -0
- package/dist/logger/logger.js.map +1 -0
- package/dist/semaphore-0ZwjVpyF.js +2 -0
- package/dist/semaphore-0ZwjVpyF.js.map +1 -0
- package/dist/semaphore-BHprIjFI.d.cts +37 -0
- package/dist/semaphore-BHprIjFI.d.cts.map +1 -0
- package/dist/semaphore-DThupBkc.d.ts +37 -0
- package/dist/semaphore-DThupBkc.d.ts.map +1 -0
- package/dist/semaphore-DVrONiAV.cjs +1 -0
- package/dist/stream-limiter-CoWKv39w.js +2 -0
- package/dist/stream-limiter-CoWKv39w.js.map +1 -0
- package/dist/stream-limiter-JgOwmkMa.cjs +1 -0
- package/dist/streams/multi-stream.cjs +1 -0
- package/dist/streams/multi-stream.d.cts +91 -0
- package/dist/streams/multi-stream.d.cts.map +1 -0
- package/dist/streams/multi-stream.d.ts +86 -0
- package/dist/streams/multi-stream.d.ts.map +1 -0
- package/dist/streams/multi-stream.js +149 -0
- package/dist/streams/multi-stream.js.map +1 -0
- package/dist/streams/stream-limiter.cjs +1 -0
- package/dist/streams/stream-limiter.d.cts +36 -0
- package/dist/streams/stream-limiter.d.cts.map +1 -0
- package/dist/streams/stream-limiter.d.ts +27 -0
- package/dist/streams/stream-limiter.d.ts.map +1 -0
- package/dist/streams/stream-limiter.js +49 -0
- package/dist/streams/stream-splitter.cjs +1 -0
- package/dist/streams/stream-splitter.d.cts +68 -0
- package/dist/streams/stream-splitter.d.cts.map +1 -0
- package/dist/streams/stream-splitter.d.ts +51 -0
- package/dist/streams/stream-splitter.d.ts.map +1 -0
- package/dist/streams/stream-splitter.js +175 -0
- package/dist/streams/stream-splitter.js.map +1 -0
- package/dist/types/data-store-registry.d.ts +13 -0
- package/dist/types/data-store-registry.d.ts.map +1 -0
- package/dist/types/data-store-registry.js +4 -0
- package/dist/types/data-store.d.ts +316 -0
- package/dist/types/data-store.d.ts.map +1 -0
- package/dist/types/data-store.js +157 -0
- package/dist/types/event-broadcaster.d.ts +28 -0
- package/dist/types/event-broadcaster.d.ts.map +1 -0
- package/dist/types/event-broadcaster.js +6 -0
- package/dist/types/event-emitter.d.ts +378 -0
- package/dist/types/event-emitter.d.ts.map +1 -0
- package/dist/types/event-emitter.js +223 -0
- package/dist/types/index.cjs +1 -0
- package/dist/types/index.d.cts +6 -0
- package/dist/types/index.d.ts +10 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +9 -0
- package/dist/types/input-file.d.ts +104 -0
- package/dist/types/input-file.d.ts.map +1 -0
- package/dist/types/input-file.js +27 -0
- package/dist/types/kv-store.d.ts +281 -0
- package/dist/types/kv-store.d.ts.map +1 -0
- package/dist/types/kv-store.js +234 -0
- package/dist/types/middleware.d.ts +17 -0
- package/dist/types/middleware.d.ts.map +1 -0
- package/dist/types/middleware.js +21 -0
- package/dist/types/upload-event.d.ts +105 -0
- package/dist/types/upload-event.d.ts.map +1 -0
- package/dist/types/upload-event.js +71 -0
- package/dist/types/upload-file.d.ts +136 -0
- package/dist/types/upload-file.d.ts.map +1 -0
- package/dist/types/upload-file.js +34 -0
- package/dist/types/websocket.d.ts +144 -0
- package/dist/types/websocket.d.ts.map +1 -0
- package/dist/types/websocket.js +40 -0
- package/dist/types-BT-cvi7T.cjs +1 -0
- package/dist/types-DhU2j-XF.js +2 -0
- package/dist/types-DhU2j-XF.js.map +1 -0
- package/dist/upload/convert-to-stream.d.ts +38 -0
- package/dist/upload/convert-to-stream.d.ts.map +1 -0
- package/dist/upload/convert-to-stream.js +43 -0
- package/dist/upload/convert-upload-to-flow-file.d.ts +14 -0
- package/dist/upload/convert-upload-to-flow-file.d.ts.map +1 -0
- package/dist/upload/convert-upload-to-flow-file.js +21 -0
- package/dist/upload/create-upload.d.ts +68 -0
- package/dist/upload/create-upload.d.ts.map +1 -0
- package/dist/upload/create-upload.js +157 -0
- package/dist/upload/index.cjs +1 -0
- package/dist/upload/index.d.cts +6 -0
- package/dist/upload/index.d.ts +4 -0
- package/dist/upload/index.d.ts.map +1 -0
- package/dist/upload/index.js +3 -0
- package/dist/upload/mime.d.ts +24 -0
- package/dist/upload/mime.d.ts.map +1 -0
- package/dist/upload/mime.js +351 -0
- package/dist/upload/upload-chunk.d.ts +58 -0
- package/dist/upload/upload-chunk.d.ts.map +1 -0
- package/dist/upload/upload-chunk.js +277 -0
- package/dist/upload/upload-server.d.ts +221 -0
- package/dist/upload/upload-server.d.ts.map +1 -0
- package/dist/upload/upload-server.js +181 -0
- package/dist/upload/upload-strategy-negotiator.d.ts +148 -0
- package/dist/upload/upload-strategy-negotiator.d.ts.map +1 -0
- package/dist/upload/upload-strategy-negotiator.js +217 -0
- package/dist/upload/upload-url.d.ts +68 -0
- package/dist/upload/upload-url.d.ts.map +1 -0
- package/dist/upload/upload-url.js +142 -0
- package/dist/upload/write-to-store.d.ts +77 -0
- package/dist/upload/write-to-store.d.ts.map +1 -0
- package/dist/upload/write-to-store.js +147 -0
- package/dist/upload-DLuICjpP.cjs +1 -0
- package/dist/upload-DaXO34dE.js +2 -0
- package/dist/upload-DaXO34dE.js.map +1 -0
- package/dist/uploadista-error-BB-Wdiz9.cjs +22 -0
- package/dist/uploadista-error-BVsVxqvz.js +23 -0
- package/dist/uploadista-error-BVsVxqvz.js.map +1 -0
- package/dist/uploadista-error-CwxYs4EB.d.ts +52 -0
- package/dist/uploadista-error-CwxYs4EB.d.ts.map +1 -0
- package/dist/uploadista-error-kKlhLRhY.d.cts +52 -0
- package/dist/uploadista-error-kKlhLRhY.d.cts.map +1 -0
- package/dist/utils/checksum.d.ts +22 -0
- package/dist/utils/checksum.d.ts.map +1 -0
- package/dist/utils/checksum.js +49 -0
- package/dist/utils/debounce.cjs +1 -0
- package/dist/utils/debounce.d.cts +38 -0
- package/dist/utils/debounce.d.cts.map +1 -0
- package/dist/utils/debounce.d.ts +36 -0
- package/dist/utils/debounce.d.ts.map +1 -0
- package/dist/utils/debounce.js +73 -0
- package/dist/utils/generate-id.cjs +1 -0
- package/dist/utils/generate-id.d.cts +2 -0
- package/dist/utils/generate-id.d.ts +32 -0
- package/dist/utils/generate-id.d.ts.map +1 -0
- package/dist/utils/generate-id.js +23 -0
- package/dist/utils/md5.cjs +1 -0
- package/dist/utils/md5.d.cts +73 -0
- package/dist/utils/md5.d.cts.map +1 -0
- package/dist/utils/md5.d.ts +71 -0
- package/dist/utils/md5.d.ts.map +1 -0
- package/dist/utils/md5.js +417 -0
- package/dist/utils/md5.js.map +1 -0
- package/dist/utils/once.cjs +1 -0
- package/dist/utils/once.d.cts +25 -0
- package/dist/utils/once.d.cts.map +1 -0
- package/dist/utils/once.d.ts +21 -0
- package/dist/utils/once.d.ts.map +1 -0
- package/dist/utils/once.js +54 -0
- package/dist/utils/once.js.map +1 -0
- package/dist/utils/semaphore.cjs +1 -0
- package/dist/utils/semaphore.d.cts +3 -0
- package/dist/utils/semaphore.d.ts +78 -0
- package/dist/utils/semaphore.d.ts.map +1 -0
- package/dist/utils/semaphore.js +134 -0
- package/dist/utils/throttle.cjs +1 -0
- package/dist/utils/throttle.d.cts +24 -0
- package/dist/utils/throttle.d.cts.map +1 -0
- package/dist/utils/throttle.d.ts +18 -0
- package/dist/utils/throttle.d.ts.map +1 -0
- package/dist/utils/throttle.js +20 -0
- package/dist/utils/throttle.js.map +1 -0
- package/docs/PARALLEL_EXECUTION.md +206 -0
- package/docs/PARALLEL_EXECUTION_QUICKSTART.md +142 -0
- package/docs/PARALLEL_EXECUTION_REFACTOR.md +184 -0
- package/package.json +80 -0
- package/src/errors/__tests__/uploadista-error.test.ts +251 -0
- package/src/errors/index.ts +2 -0
- package/src/errors/uploadista-error.ts +394 -0
- package/src/flow/README.md +352 -0
- package/src/flow/edge.test.ts +146 -0
- package/src/flow/edge.ts +60 -0
- package/src/flow/event.ts +229 -0
- package/src/flow/flow-server.ts +1089 -0
- package/src/flow/flow.ts +1050 -0
- package/src/flow/index.ts +28 -0
- package/src/flow/node.ts +249 -0
- package/src/flow/nodes/index.ts +8 -0
- package/src/flow/nodes/input-node.ts +296 -0
- package/src/flow/nodes/storage-node.ts +128 -0
- package/src/flow/nodes/transform-node.ts +154 -0
- package/src/flow/parallel-scheduler.ts +259 -0
- package/src/flow/plugins/credential-provider.ts +48 -0
- package/src/flow/plugins/image-ai-plugin.ts +66 -0
- package/src/flow/plugins/image-plugin.ts +60 -0
- package/src/flow/plugins/types/describe-image-node.ts +16 -0
- package/src/flow/plugins/types/index.ts +9 -0
- package/src/flow/plugins/types/optimize-node.ts +18 -0
- package/src/flow/plugins/types/remove-background-node.ts +18 -0
- package/src/flow/plugins/types/resize-node.ts +26 -0
- package/src/flow/plugins/zip-plugin.ts +69 -0
- package/src/flow/typed-flow.ts +279 -0
- package/src/flow/types/flow-file.ts +51 -0
- package/src/flow/types/flow-job.ts +138 -0
- package/src/flow/types/flow-types.ts +353 -0
- package/src/flow/types/index.ts +6 -0
- package/src/flow/types/run-args.ts +40 -0
- package/src/flow/types/type-validator.ts +204 -0
- package/src/flow/utils/resolve-upload-metadata.ts +48 -0
- package/src/index.ts +5 -0
- package/src/logger/logger.ts +14 -0
- package/src/streams/stream-limiter.test.ts +150 -0
- package/src/streams/stream-limiter.ts +75 -0
- package/src/types/data-store.ts +427 -0
- package/src/types/event-broadcaster.ts +39 -0
- package/src/types/event-emitter.ts +349 -0
- package/src/types/index.ts +9 -0
- package/src/types/input-file.ts +107 -0
- package/src/types/kv-store.ts +375 -0
- package/src/types/middleware.ts +54 -0
- package/src/types/upload-event.ts +75 -0
- package/src/types/upload-file.ts +139 -0
- package/src/types/websocket.ts +65 -0
- package/src/upload/convert-to-stream.ts +48 -0
- package/src/upload/create-upload.ts +214 -0
- package/src/upload/index.ts +3 -0
- package/src/upload/mime.ts +436 -0
- package/src/upload/upload-chunk.ts +364 -0
- package/src/upload/upload-server.ts +390 -0
- package/src/upload/upload-strategy-negotiator.ts +316 -0
- package/src/upload/upload-url.ts +173 -0
- package/src/upload/write-to-store.ts +211 -0
- package/src/utils/checksum.ts +61 -0
- package/src/utils/debounce.test.ts +126 -0
- package/src/utils/debounce.ts +89 -0
- package/src/utils/generate-id.ts +35 -0
- package/src/utils/md5.ts +475 -0
- package/src/utils/once.test.ts +83 -0
- package/src/utils/once.ts +63 -0
- package/src/utils/throttle.test.ts +101 -0
- package/src/utils/throttle.ts +29 -0
- package/tsconfig.json +20 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/tsdown.config.ts +25 -0
- package/vitest.config.ts +15 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { UploadistaError } from "../errors/uploadista-error";
|
|
3
|
+
export declare function once<T, A extends unknown[], Return>(fn: (this: T, ...args: A) => Return): (this: T, ...args: A) => Return;
|
|
4
|
+
/**
|
|
5
|
+
* Effect-based once utilities
|
|
6
|
+
*/
|
|
7
|
+
export declare const OnceEffect: {
|
|
8
|
+
/**
|
|
9
|
+
* Creates an Effect-based once function that only executes once
|
|
10
|
+
* @param effect - The effect to execute only once
|
|
11
|
+
* @returns Effect that caches the result after first execution
|
|
12
|
+
*/
|
|
13
|
+
make: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | UploadistaError, R>;
|
|
14
|
+
/**
|
|
15
|
+
* Creates a legacy once function wrapper
|
|
16
|
+
* @param fn - Function to wrap
|
|
17
|
+
* @returns Once-wrapped function
|
|
18
|
+
*/
|
|
19
|
+
legacy: typeof once;
|
|
20
|
+
};
|
|
21
|
+
//# sourceMappingURL=once.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"once.d.ts","sourceRoot":"","sources":["../../src/utils/once.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAG7D,wBAAgB,IAAI,CAAC,CAAC,EAAE,CAAC,SAAS,OAAO,EAAE,EAAE,MAAM,EACjD,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,KAAK,MAAM,UAIT,CAAC,WAAW,CAAC,KAAG,MAAM,CAYjD;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;IACrB;;;;OAIG;WACI,CAAC,EAAE,CAAC,EAAE,CAAC,UACJ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KAC7B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,CAAC;IAsB3C;;;;OAIG;;CAEJ,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { UploadistaError } from "../errors/uploadista-error";
|
|
3
|
+
// only call a function once
|
|
4
|
+
export function once(fn) {
|
|
5
|
+
let called = false;
|
|
6
|
+
let value;
|
|
7
|
+
const f = function (...args) {
|
|
8
|
+
if (called) {
|
|
9
|
+
if (value) {
|
|
10
|
+
return value;
|
|
11
|
+
}
|
|
12
|
+
throw new Error("Function called more than once");
|
|
13
|
+
}
|
|
14
|
+
called = true;
|
|
15
|
+
value = fn.apply(this, args);
|
|
16
|
+
return value;
|
|
17
|
+
};
|
|
18
|
+
return f;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Effect-based once utilities
|
|
22
|
+
*/
|
|
23
|
+
export const OnceEffect = {
|
|
24
|
+
/**
|
|
25
|
+
* Creates an Effect-based once function that only executes once
|
|
26
|
+
* @param effect - The effect to execute only once
|
|
27
|
+
* @returns Effect that caches the result after first execution
|
|
28
|
+
*/
|
|
29
|
+
make: (effect) => {
|
|
30
|
+
let cached;
|
|
31
|
+
let called = false;
|
|
32
|
+
return Effect.gen(function* () {
|
|
33
|
+
if (called) {
|
|
34
|
+
if (cached !== undefined) {
|
|
35
|
+
return cached;
|
|
36
|
+
}
|
|
37
|
+
yield* new UploadistaError({
|
|
38
|
+
code: "UNKNOWN_ERROR",
|
|
39
|
+
status: 500,
|
|
40
|
+
body: "Effect called more than once with undefined result",
|
|
41
|
+
}).toEffect();
|
|
42
|
+
}
|
|
43
|
+
called = true;
|
|
44
|
+
cached = yield* effect;
|
|
45
|
+
return cached;
|
|
46
|
+
});
|
|
47
|
+
},
|
|
48
|
+
/**
|
|
49
|
+
* Creates a legacy once function wrapper
|
|
50
|
+
* @param fn - Function to wrap
|
|
51
|
+
* @returns Once-wrapped function
|
|
52
|
+
*/
|
|
53
|
+
legacy: once,
|
|
54
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"once.js","names":["value: Return | undefined","cached: A | undefined"],"sources":["../../src/utils/once.ts"],"sourcesContent":["import { Effect } from \"effect\";\nimport { UploadistaError } from \"../errors/uploadista-error\";\n\n// only call a function once\nexport function once<T, A extends unknown[], Return>(\n fn: (this: T, ...args: A) => Return,\n) {\n let called = false;\n let value: Return | undefined;\n const f = function (this: T, ...args: A): Return {\n if (called) {\n if (value) {\n return value;\n }\n throw new Error(\"Function called more than once\");\n }\n called = true;\n value = fn.apply(this, args);\n return value;\n };\n return f;\n}\n\n/**\n * Effect-based once utilities\n */\nexport const OnceEffect = {\n /**\n * Creates an Effect-based once function that only executes once\n * @param effect - The effect to execute only once\n * @returns Effect that caches the result after first execution\n */\n make: <A, E, R>(\n effect: Effect.Effect<A, E, R>,\n ): Effect.Effect<A, E | UploadistaError, R> => {\n let cached: A | undefined;\n let called = false;\n\n return Effect.gen(function* () {\n if (called) {\n if (cached !== undefined) {\n return cached;\n }\n yield* new UploadistaError({\n code: \"UNKNOWN_ERROR\",\n status: 500,\n body: \"Effect called more than once with undefined result\",\n }).toEffect();\n }\n\n called = true;\n cached = yield* effect;\n return cached;\n });\n },\n\n /**\n * Creates a legacy once function wrapper\n * @param fn - Function to wrap\n * @returns Once-wrapped function\n */\n legacy: once,\n};\n"],"mappings":"kGAIA,SAAgB,EACd,EACA,CACA,IAAI,EAAS,GACTA,EAYJ,OAXU,SAAmB,GAAG,EAAiB,CAC/C,GAAI,EAAQ,CACV,GAAI,EACF,OAAO,EAET,MAAU,MAAM,iCAAiC,CAInD,MAFA,GAAS,GACT,EAAQ,EAAG,MAAM,KAAM,EAAK,CACrB,GAQX,MAAa,EAAa,CAMxB,KACE,GAC6C,CAC7C,IAAIC,EACA,EAAS,GAEb,OAAO,EAAO,IAAI,WAAa,CAC7B,GAAI,EAAQ,CACV,GAAI,IAAW,IAAA,GACb,OAAO,EAET,MAAO,IAAI,EAAgB,CACzB,KAAM,gBACN,OAAQ,IACR,KAAM,qDACP,CAAC,CAAC,UAAU,CAKf,MAFA,GAAS,GACT,EAAS,MAAO,EACT,GACP,EAQJ,OAAQ,EACT"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const e=require(`../semaphore-DVrONiAV.cjs`);exports.SemaphoreEffect=e.SemaphoreEffect,exports.permit=e.permit,exports.semaphore=e.semaphore;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { Effect, type Scope } from "effect";
|
|
2
|
+
import type { UploadistaError } from "../errors/uploadista-error";
|
|
3
|
+
type ReleaseCallback = () => Promise<void>;
|
|
4
|
+
/**
|
|
5
|
+
* Creates a permit that must be explicitly released to free up a semaphore slot.
|
|
6
|
+
*
|
|
7
|
+
* Permits are used internally by semaphores to track resource acquisition and release.
|
|
8
|
+
* Each permit represents one available slot in the semaphore's concurrency limit.
|
|
9
|
+
*
|
|
10
|
+
* @param onRelease - Callback function to execute when the permit is released
|
|
11
|
+
* @returns A permit object with a release method
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
14
|
+
export declare function permit(onRelease: ReleaseCallback): {
|
|
15
|
+
release: () => Promise<void>;
|
|
16
|
+
};
|
|
17
|
+
export type Permit = ReturnType<typeof permit>;
|
|
18
|
+
/**
|
|
19
|
+
* Creates a semaphore for controlling concurrent access to resources.
|
|
20
|
+
*
|
|
21
|
+
* A semaphore limits the number of concurrent operations that can access a shared
|
|
22
|
+
* resource. This is useful for:
|
|
23
|
+
* - Limiting parallel file uploads
|
|
24
|
+
* - Controlling database connection pool size
|
|
25
|
+
* - Throttling API requests
|
|
26
|
+
* - Managing concurrent flow node execution
|
|
27
|
+
*
|
|
28
|
+
* When all permits are in use, new acquire() calls will wait until a permit
|
|
29
|
+
* becomes available.
|
|
30
|
+
*
|
|
31
|
+
* @param count - Maximum number of concurrent operations allowed
|
|
32
|
+
* @returns A semaphore object with an acquire method
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```typescript
|
|
36
|
+
* const uploadSemaphore = semaphore(3); // Max 3 concurrent uploads
|
|
37
|
+
*
|
|
38
|
+
* async function uploadFile(file: File) {
|
|
39
|
+
* const permit = await uploadSemaphore.acquire();
|
|
40
|
+
* try {
|
|
41
|
+
* // Upload file (max 3 concurrent)
|
|
42
|
+
* await uploadToServer(file);
|
|
43
|
+
* } finally {
|
|
44
|
+
* await permit.release();
|
|
45
|
+
* }
|
|
46
|
+
* }
|
|
47
|
+
*
|
|
48
|
+
* // Process files with concurrency limit
|
|
49
|
+
* await Promise.all(files.map(uploadFile));
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
export declare function semaphore(count: number): {
|
|
53
|
+
acquire: () => Promise<Permit>;
|
|
54
|
+
};
|
|
55
|
+
export type Semaphore = ReturnType<typeof semaphore>;
|
|
56
|
+
/**
|
|
57
|
+
* Effect-based semaphore utilities
|
|
58
|
+
*/
|
|
59
|
+
export declare const SemaphoreEffect: {
|
|
60
|
+
/**
|
|
61
|
+
* Creates an Effect-based semaphore with resource management
|
|
62
|
+
* @param count - Number of permits available
|
|
63
|
+
* @returns Effect semaphore with automatic resource cleanup
|
|
64
|
+
*/
|
|
65
|
+
make: (count: number) => Effect.Effect<{
|
|
66
|
+
acquire: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | UploadistaError, R>;
|
|
67
|
+
}, never, Scope.Scope>;
|
|
68
|
+
/**
|
|
69
|
+
* Creates a legacy semaphore wrapper
|
|
70
|
+
* @param count - Number of permits
|
|
71
|
+
* @returns Legacy semaphore instance
|
|
72
|
+
*/
|
|
73
|
+
legacy: (count: number) => {
|
|
74
|
+
acquire: () => Promise<Permit>;
|
|
75
|
+
};
|
|
76
|
+
};
|
|
77
|
+
export {};
|
|
78
|
+
//# sourceMappingURL=semaphore.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semaphore.d.ts","sourceRoot":"","sources":["../../src/utils/semaphore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAElE,KAAK,eAAe,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAE3C;;;;;;;;;GASG;AACH,wBAAgB,MAAM,CAAC,SAAS,EAAE,eAAe;;EAahD;AAED,MAAM,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAO/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM;mBAIjB,OAAO,CAAC,MAAM,CAAC;EAgCpC;AAED,MAAM,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,SAAS,CAAC,CAAC;AAErD;;GAEG;AACH,eAAO,MAAM,eAAe;IAC1B;;;;OAIG;kBAEM,MAAM,KACZ,MAAM,CAAC,MAAM,CACd;QACE,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EACf,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KAC3B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC;KAC/C,EACD,KAAK,EACL,KAAK,CAAC,KAAK,CACZ;IAyCD;;;;OAIG;oBACa,MAAM;uBArGF,OAAO,CAAC,MAAM,CAAC;;CAsGpC,CAAC"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Creates a permit that must be explicitly released to free up a semaphore slot.
|
|
4
|
+
*
|
|
5
|
+
* Permits are used internally by semaphores to track resource acquisition and release.
|
|
6
|
+
* Each permit represents one available slot in the semaphore's concurrency limit.
|
|
7
|
+
*
|
|
8
|
+
* @param onRelease - Callback function to execute when the permit is released
|
|
9
|
+
* @returns A permit object with a release method
|
|
10
|
+
* @internal
|
|
11
|
+
*/
|
|
12
|
+
export function permit(onRelease) {
|
|
13
|
+
let isReleased = false;
|
|
14
|
+
async function release() {
|
|
15
|
+
if (!isReleased) {
|
|
16
|
+
isReleased = true;
|
|
17
|
+
await onRelease();
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
release,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Creates a semaphore for controlling concurrent access to resources.
|
|
26
|
+
*
|
|
27
|
+
* A semaphore limits the number of concurrent operations that can access a shared
|
|
28
|
+
* resource. This is useful for:
|
|
29
|
+
* - Limiting parallel file uploads
|
|
30
|
+
* - Controlling database connection pool size
|
|
31
|
+
* - Throttling API requests
|
|
32
|
+
* - Managing concurrent flow node execution
|
|
33
|
+
*
|
|
34
|
+
* When all permits are in use, new acquire() calls will wait until a permit
|
|
35
|
+
* becomes available.
|
|
36
|
+
*
|
|
37
|
+
* @param count - Maximum number of concurrent operations allowed
|
|
38
|
+
* @returns A semaphore object with an acquire method
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const uploadSemaphore = semaphore(3); // Max 3 concurrent uploads
|
|
43
|
+
*
|
|
44
|
+
* async function uploadFile(file: File) {
|
|
45
|
+
* const permit = await uploadSemaphore.acquire();
|
|
46
|
+
* try {
|
|
47
|
+
* // Upload file (max 3 concurrent)
|
|
48
|
+
* await uploadToServer(file);
|
|
49
|
+
* } finally {
|
|
50
|
+
* await permit.release();
|
|
51
|
+
* }
|
|
52
|
+
* }
|
|
53
|
+
*
|
|
54
|
+
* // Process files with concurrency limit
|
|
55
|
+
* await Promise.all(files.map(uploadFile));
|
|
56
|
+
* ```
|
|
57
|
+
*/
|
|
58
|
+
export function semaphore(count) {
|
|
59
|
+
let availablePermits = count;
|
|
60
|
+
const deferreds = [];
|
|
61
|
+
function acquire() {
|
|
62
|
+
if (availablePermits > 0) {
|
|
63
|
+
return Promise.resolve(createPermit());
|
|
64
|
+
}
|
|
65
|
+
const deferred = {};
|
|
66
|
+
deferred.promise = new Promise((resolve) => {
|
|
67
|
+
deferred.resolve = resolve;
|
|
68
|
+
});
|
|
69
|
+
deferreds.push(deferred);
|
|
70
|
+
return deferred.promise;
|
|
71
|
+
}
|
|
72
|
+
function createPermit() {
|
|
73
|
+
availablePermits--;
|
|
74
|
+
return permit(async () => {
|
|
75
|
+
availablePermits++;
|
|
76
|
+
if (deferreds.length > 0) {
|
|
77
|
+
const deferred = deferreds.shift();
|
|
78
|
+
if (deferred) {
|
|
79
|
+
deferred.resolve?.(createPermit());
|
|
80
|
+
await deferred.promise;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
acquire,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Effect-based semaphore utilities
|
|
91
|
+
*/
|
|
92
|
+
export const SemaphoreEffect = {
|
|
93
|
+
/**
|
|
94
|
+
* Creates an Effect-based semaphore with resource management
|
|
95
|
+
* @param count - Number of permits available
|
|
96
|
+
* @returns Effect semaphore with automatic resource cleanup
|
|
97
|
+
*/
|
|
98
|
+
make: (count) => Effect.acquireRelease(Effect.sync(() => {
|
|
99
|
+
let availablePermits = count;
|
|
100
|
+
const queue = [];
|
|
101
|
+
const acquire = (effect) => Effect.gen(function* () {
|
|
102
|
+
// Wait for a permit
|
|
103
|
+
yield* Effect.async((resume) => {
|
|
104
|
+
if (availablePermits > 0) {
|
|
105
|
+
availablePermits--;
|
|
106
|
+
resume(Effect.void);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
queue.push((releasePermit) => {
|
|
110
|
+
availablePermits--;
|
|
111
|
+
resume(Effect.void);
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
// Execute the effect with cleanup
|
|
116
|
+
return yield* Effect.ensuring(effect, Effect.sync(() => {
|
|
117
|
+
availablePermits++;
|
|
118
|
+
if (queue.length > 0) {
|
|
119
|
+
const nextWaiting = queue.shift();
|
|
120
|
+
if (nextWaiting) {
|
|
121
|
+
nextWaiting(() => { });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}));
|
|
125
|
+
});
|
|
126
|
+
return { acquire };
|
|
127
|
+
}), () => Effect.void),
|
|
128
|
+
/**
|
|
129
|
+
* Creates a legacy semaphore wrapper
|
|
130
|
+
* @param count - Number of permits
|
|
131
|
+
* @returns Legacy semaphore instance
|
|
132
|
+
*/
|
|
133
|
+
legacy: (count) => semaphore(count),
|
|
134
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const e=require(`../debounce-LZK7yS7Z.cjs`);function t(t,n,{leading:r=!0,trailing:i=!0}={}){return e.debounce(t,n,{leading:r,trailing:i})}const n={legacy:t};exports.ThrottleEffect=n,exports.throttle=t;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//#region src/utils/throttle.d.ts
|
|
2
|
+
declare function throttle<T, A extends unknown[], Return>(fn: (this: T, ...args: A) => Return, wait: number, {
|
|
3
|
+
leading,
|
|
4
|
+
trailing
|
|
5
|
+
}?: {
|
|
6
|
+
leading?: boolean;
|
|
7
|
+
trailing?: boolean;
|
|
8
|
+
}): (this: T, ...args: A) => void;
|
|
9
|
+
/**
|
|
10
|
+
* Effect-based throttle utilities
|
|
11
|
+
*/
|
|
12
|
+
declare const ThrottleEffect: {
|
|
13
|
+
/**
|
|
14
|
+
* Creates a legacy throttle function wrapper
|
|
15
|
+
* @param fn - Function to throttle
|
|
16
|
+
* @param wait - Wait time in milliseconds
|
|
17
|
+
* @param options - Throttle options
|
|
18
|
+
* @returns Throttled function
|
|
19
|
+
*/
|
|
20
|
+
legacy: typeof throttle;
|
|
21
|
+
};
|
|
22
|
+
//#endregion
|
|
23
|
+
export { ThrottleEffect, throttle };
|
|
24
|
+
//# sourceMappingURL=throttle.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle.d.cts","names":[],"sources":["../../src/utils/throttle.ts"],"sourcesContent":[],"mappings":";iBAEgB,oDACH,YAAY,MAAM;;;;EADf,OAAA,CAAA,EAAA,OAAQ;EAAA,QAAA,CAAA,EAAA,OAAA;SACX,EAKsC,CALtC,EAAA,GAAA,IAAA,EAKsC,CALtC,EAAA,GAAA,IAAA;;;;AAIT,cAYS,cAZT,EAAA;;;;AAYJ;;;;iBASC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export declare function throttle<T, A extends unknown[], Return>(fn: (this: T, ...args: A) => Return, wait: number, { leading, trailing, }?: {
|
|
2
|
+
leading?: boolean;
|
|
3
|
+
trailing?: boolean;
|
|
4
|
+
}): (this: T, ...args: A) => void;
|
|
5
|
+
/**
|
|
6
|
+
* Effect-based throttle utilities
|
|
7
|
+
*/
|
|
8
|
+
export declare const ThrottleEffect: {
|
|
9
|
+
/**
|
|
10
|
+
* Creates a legacy throttle function wrapper
|
|
11
|
+
* @param fn - Function to throttle
|
|
12
|
+
* @param wait - Wait time in milliseconds
|
|
13
|
+
* @param options - Throttle options
|
|
14
|
+
* @returns Throttled function
|
|
15
|
+
*/
|
|
16
|
+
legacy: typeof throttle;
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=throttle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle.d.ts","sourceRoot":"","sources":["../../src/utils/throttle.ts"],"names":[],"mappings":"AAEA,wBAAgB,QAAQ,CAAC,CAAC,EAAE,CAAC,SAAS,OAAO,EAAE,EAAE,MAAM,EACrD,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,KAAK,MAAM,EACnC,IAAI,EAAE,MAAM,EACZ,EACE,OAAc,EACd,QAAe,GAChB,GAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,iCAMlD;AAED;;GAEG;AACH,eAAO,MAAM,cAAc;IACzB;;;;;;OAMG;;CAEJ,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { debounce } from "./debounce.js";
|
|
2
|
+
export function throttle(fn, wait, { leading = true, trailing = true, } = {}) {
|
|
3
|
+
return debounce(fn, wait, {
|
|
4
|
+
leading,
|
|
5
|
+
trailing,
|
|
6
|
+
});
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Effect-based throttle utilities
|
|
10
|
+
*/
|
|
11
|
+
export const ThrottleEffect = {
|
|
12
|
+
/**
|
|
13
|
+
* Creates a legacy throttle function wrapper
|
|
14
|
+
* @param fn - Function to throttle
|
|
15
|
+
* @param wait - Wait time in milliseconds
|
|
16
|
+
* @param options - Throttle options
|
|
17
|
+
* @returns Throttled function
|
|
18
|
+
*/
|
|
19
|
+
legacy: throttle,
|
|
20
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle.js","names":[],"sources":["../../src/utils/throttle.ts"],"sourcesContent":["import { debounce } from \"./debounce.js\";\n\nexport function throttle<T, A extends unknown[], Return>(\n fn: (this: T, ...args: A) => Return,\n wait: number,\n {\n leading = true,\n trailing = true,\n }: { leading?: boolean; trailing?: boolean } = {},\n) {\n return debounce(fn, wait, {\n leading,\n trailing,\n });\n}\n\n/**\n * Effect-based throttle utilities\n */\nexport const ThrottleEffect = {\n /**\n * Creates a legacy throttle function wrapper\n * @param fn - Function to throttle\n * @param wait - Wait time in milliseconds\n * @param options - Throttle options\n * @returns Throttled function\n */\n legacy: throttle,\n};\n"],"mappings":"mDAEA,SAAgB,EACd,EACA,EACA,CACE,UAAU,GACV,WAAW,IACkC,EAAE,CACjD,CACA,OAAO,EAAS,EAAI,EAAM,CACxB,UACA,WACD,CAAC,CAMJ,MAAa,EAAiB,CAQ5B,OAAQ,EACT"}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# Parallel Flow Execution
|
|
2
|
+
|
|
3
|
+
The uploadista flow engine now supports parallel execution of independent nodes using Effect's native concurrency primitives, significantly improving performance for complex flows with multiple processing branches.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
The parallel execution feature automatically identifies nodes that can run concurrently (nodes that don't depend on each other) and groups them into execution levels. Nodes within the same level are executed in parallel with controlled concurrency, while different levels are executed sequentially to maintain dependencies.
|
|
8
|
+
|
|
9
|
+
## Key Features
|
|
10
|
+
|
|
11
|
+
- **Automatic Dependency Analysis**: The scheduler analyzes the flow graph and groups independent nodes into parallel execution levels using Kahn's algorithm
|
|
12
|
+
- **Effect-Based Concurrency**: Uses Effect's `Effect.all()` with concurrency limits to manage parallel execution safely
|
|
13
|
+
- **Backward Compatibility**: Falls back to sequential execution when parallel execution is disabled (default behavior)
|
|
14
|
+
- **Error Handling**: Proper error propagation and failure handling across parallel branches
|
|
15
|
+
- **Type-Safe**: Full TypeScript support with proper Effect typing
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
To enable parallel execution, set the `parallelExecution` option when creating a flow:
|
|
20
|
+
|
|
21
|
+
```typescript
|
|
22
|
+
import { createFlowWithSchema } from '@uploadista/core/flow';
|
|
23
|
+
|
|
24
|
+
const flow = yield* createFlowWithSchema({
|
|
25
|
+
flowId: 'my-parallel-flow',
|
|
26
|
+
name: 'Parallel Processing Flow',
|
|
27
|
+
nodes: [inputNode, processA, processB, processC, mergeNode, outputNode],
|
|
28
|
+
edges: [
|
|
29
|
+
{ source: 'input', target: 'processA' },
|
|
30
|
+
{ source: 'input', target: 'processB' },
|
|
31
|
+
{ source: 'input', target: 'processC' },
|
|
32
|
+
{ source: 'processA', target: 'merge' },
|
|
33
|
+
{ source: 'processB', target: 'merge' },
|
|
34
|
+
{ source: 'processC', target: 'merge' },
|
|
35
|
+
{ source: 'merge', target: 'output' }
|
|
36
|
+
],
|
|
37
|
+
inputSchema: myInputSchema,
|
|
38
|
+
outputSchema: myOutputSchema,
|
|
39
|
+
parallelExecution: {
|
|
40
|
+
enabled: true,
|
|
41
|
+
maxConcurrency: 4, // Maximum number of concurrent nodes per level
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Configuration Options
|
|
47
|
+
|
|
48
|
+
- `enabled`: Boolean flag to enable/disable parallel execution (default: `false`)
|
|
49
|
+
- `maxConcurrency`: Maximum number of nodes to execute concurrently within a level (default: `4`)
|
|
50
|
+
|
|
51
|
+
## How It Works
|
|
52
|
+
|
|
53
|
+
### 1. Dependency Analysis
|
|
54
|
+
The scheduler analyzes the flow graph using Kahn's algorithm to build a dependency graph and calculate the in-degree (number of dependencies) for each node.
|
|
55
|
+
|
|
56
|
+
### 2. Level Grouping
|
|
57
|
+
Nodes are grouped into execution levels based on their dependencies:
|
|
58
|
+
- **Level 0**: Input nodes (no dependencies)
|
|
59
|
+
- **Level 1**: Nodes that depend only on Level 0 nodes
|
|
60
|
+
- **Level N**: Nodes that depend on Level N-1 nodes
|
|
61
|
+
|
|
62
|
+
### 3. Parallel Execution with Effect
|
|
63
|
+
Within each level, nodes are executed in parallel using Effect's `Effect.all()` with concurrency control. This ensures:
|
|
64
|
+
- No more than `maxConcurrency` nodes execute simultaneously
|
|
65
|
+
- Resource exhaustion is prevented
|
|
66
|
+
- Proper error handling and cleanup
|
|
67
|
+
|
|
68
|
+
### 4. Sequential Level Processing
|
|
69
|
+
Levels are processed sequentially in a for loop. Each level completes before the next level begins, ensuring all dependencies are satisfied.
|
|
70
|
+
|
|
71
|
+
## Performance Benefits
|
|
72
|
+
|
|
73
|
+
Parallel execution can significantly reduce flow execution time, especially for:
|
|
74
|
+
- Image processing pipelines with multiple transformations
|
|
75
|
+
- Multi-branch conditional flows
|
|
76
|
+
- Independent data processing tasks
|
|
77
|
+
- Complex validation and transformation chains
|
|
78
|
+
|
|
79
|
+
Expected performance improvements depend on:
|
|
80
|
+
- Number of independent processing branches
|
|
81
|
+
- Individual node execution time
|
|
82
|
+
- Available system resources
|
|
83
|
+
|
|
84
|
+
## Example Flow Structure
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
Input Node
|
|
88
|
+
↓
|
|
89
|
+
┌─────────┼─────────┐
|
|
90
|
+
↓ ↓ ↓
|
|
91
|
+
ProcessA ProcessB ProcessC ← Level 1 (parallel)
|
|
92
|
+
└─────────┼─────────┘
|
|
93
|
+
↓
|
|
94
|
+
Merge Node ← Level 2 (sequential)
|
|
95
|
+
↓
|
|
96
|
+
Output Node ← Level 3 (sequential)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
In this example:
|
|
100
|
+
- ProcessA, ProcessB, and ProcessC run in parallel
|
|
101
|
+
- Merge Node waits for all parallel processes to complete
|
|
102
|
+
- Output Node processes the merged result
|
|
103
|
+
|
|
104
|
+
## Error Handling
|
|
105
|
+
|
|
106
|
+
If any node in a parallel level fails:
|
|
107
|
+
- The error is propagated immediately (Effect's `Promise.all` behavior)
|
|
108
|
+
- Remaining nodes in the level may continue, but the flow halts
|
|
109
|
+
- Error details are available for debugging
|
|
110
|
+
- Resources are properly cleaned up through Effect's resource management
|
|
111
|
+
|
|
112
|
+
## Monitoring and Debugging
|
|
113
|
+
|
|
114
|
+
The parallel scheduler provides execution level information and statistics:
|
|
115
|
+
|
|
116
|
+
```typescript
|
|
117
|
+
const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
|
|
118
|
+
|
|
119
|
+
// Get execution levels
|
|
120
|
+
const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
|
|
121
|
+
console.log(`Flow has ${levels.length} execution levels`);
|
|
122
|
+
levels.forEach(level => {
|
|
123
|
+
console.log(`Level ${level.level}: [${level.nodes.join(', ')}]`);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
// Get scheduler stats
|
|
127
|
+
const stats = scheduler.getStats();
|
|
128
|
+
console.log(`Max concurrency: ${stats.maxConcurrency}`);
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Debug Logging
|
|
132
|
+
|
|
133
|
+
When parallel execution is enabled, the flow engine emits debug logs:
|
|
134
|
+
```
|
|
135
|
+
Flow my-flow: Executing in parallel mode (maxConcurrency: 4)
|
|
136
|
+
Flow my-flow: Grouped nodes into 3 execution levels
|
|
137
|
+
Flow my-flow: Executing level 0 with nodes: input_1
|
|
138
|
+
Flow my-flow: Executing level 1 with nodes: resize_1, optimize_1
|
|
139
|
+
Flow my-flow: Executing level 2 with nodes: output_1
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Limitations & Considerations
|
|
143
|
+
|
|
144
|
+
- **Pausable nodes**: If a node pauses in a level, the entire level pauses
|
|
145
|
+
- **Conditional nodes**: Conditions are evaluated before parallel execution begins; the node is either included or skipped for the level
|
|
146
|
+
- **Multi-input nodes**: Must have all dependencies completed before execution (enforced by level grouping)
|
|
147
|
+
- **Resource-intensive nodes**: May benefit from lower `maxConcurrency` settings to prevent resource contention
|
|
148
|
+
|
|
149
|
+
## Migration from Sequential Flows
|
|
150
|
+
|
|
151
|
+
Existing flows work without any changes:
|
|
152
|
+
- Default behavior remains sequential execution (`enabled: false`)
|
|
153
|
+
- Enable parallel execution by adding the `parallelExecution` configuration
|
|
154
|
+
- No changes required to existing node implementations
|
|
155
|
+
- Flows automatically respect dependency constraints
|
|
156
|
+
|
|
157
|
+
## Implementation Details
|
|
158
|
+
|
|
159
|
+
The parallel execution system consists of three main components:
|
|
160
|
+
|
|
161
|
+
### 1. ParallelScheduler (`parallel-scheduler.ts`)
|
|
162
|
+
- Analyzes flow dependencies using Kahn's algorithm
|
|
163
|
+
- Groups nodes into execution levels
|
|
164
|
+
- Provides `executeNodesInParallel()` which uses `Effect.all()` with concurrency control
|
|
165
|
+
- Type-safe Effect-based API with full TypeScript support
|
|
166
|
+
|
|
167
|
+
```typescript
|
|
168
|
+
const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
|
|
169
|
+
const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
|
|
170
|
+
const results = yield* scheduler.executeNodesInParallel(executors);
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### 2. Flow Engine Integration (`flow.ts`)
|
|
174
|
+
- Detects when `parallelExecution.enabled` is true
|
|
175
|
+
- Switches between sequential and parallel execution paths
|
|
176
|
+
- Maintains backward compatibility (sequential is default)
|
|
177
|
+
- Properly handles paused nodes in parallel execution
|
|
178
|
+
|
|
179
|
+
### 3. Effect-Based Concurrency
|
|
180
|
+
Uses Effect's `Effect.all()` with `concurrency` option:
|
|
181
|
+
```typescript
|
|
182
|
+
Effect.all(nodeExecutors, { concurrency: maxConcurrency })
|
|
183
|
+
```
|
|
184
|
+
This ensures resource-safe parallel execution without manual semaphore management.
|
|
185
|
+
|
|
186
|
+
## Performance Characteristics
|
|
187
|
+
|
|
188
|
+
### Best Case Scenarios
|
|
189
|
+
- **Independent processing branches**: 3+ parallel nodes can reduce execution time proportionally
|
|
190
|
+
- **I/O-bound operations**: Network calls or disk I/O can run concurrently
|
|
191
|
+
- **CPU-bound with other operations**: Can interleave with other workloads
|
|
192
|
+
|
|
193
|
+
### Example Performance Gain
|
|
194
|
+
```
|
|
195
|
+
Sequential: Input (1s) → Resize (2s) → Optimize (2s) → Output (1s) = 6 seconds
|
|
196
|
+
Parallel: Input (1s) → [Resize (2s) + Optimize (2s) in parallel] → Output (1s) = 4 seconds
|
|
197
|
+
Speedup: 33% faster with just 2 parallel operations
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## Future Enhancements
|
|
201
|
+
|
|
202
|
+
Potential improvements for future releases:
|
|
203
|
+
- Dynamic concurrency adjustment based on resource availability
|
|
204
|
+
- Per-node concurrency limits for resource-intensive operations
|
|
205
|
+
- Parallel level visualization in flow builder
|
|
206
|
+
- Performance metrics collection and reporting
|